Skip to content

ios: bring the t3/ios-mvf UI test flake fixes to beta #169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AidenJohnston merged 4 commits into from
Sep 27, 2026
Merged

ios: bring the t3/ios-mvf UI test flake fixes to beta #169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f45fef0
Give the instruction card's reply button a solid fill
SamGu-NRX Sep 27, 2026
3011831
ios tests: audit the walk once the autopilot holds it
AidenJohnston Sep 27, 2026
029050e
Replay: an upper walk that sees the wall above 6.5 ft, and a gap the …
SamGu-NRX Sep 27, 2026
9fc6019
ios tests: say what the held walk shows
AidenJohnston Sep 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions ios/HouseScan.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,66 @@
"Fixtures/synthetic-wall-lidar/keyframes/k00041.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00041.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00041.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00042.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00042.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00042.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00043.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00043.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00043.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00044.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00044.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00044.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00045.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00045.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00045.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00046.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00046.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00046.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00047.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00047.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00047.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00048.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00048.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00048.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00049.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00049.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00049.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00050.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00050.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00050.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00051.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00051.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00051.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00052.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00052.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00052.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00053.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00053.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00053.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00054.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00054.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00054.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00055.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00055.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00055.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00056.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00056.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00056.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00057.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00057.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00057.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00058.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00058.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00058.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00059.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00059.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00059.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00060.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00060.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00060.jpg",
"Fixtures/synthetic-wall-lidar/keyframes/k00061.confidence.u8",
"Fixtures/synthetic-wall-lidar/keyframes/k00061.depth.f32",
"Fixtures/synthetic-wall-lidar/keyframes/k00061.jpg",
"Fixtures/synthetic-wall-lidar/session.json",
"Fixtures/synthetic-wall/keyframes",
"Fixtures/synthetic-wall/keyframes/k00001.jpg",
Expand Down Expand Up @@ -206,6 +266,26 @@
"Fixtures/synthetic-wall/keyframes/k00039.jpg",
"Fixtures/synthetic-wall/keyframes/k00040.jpg",
"Fixtures/synthetic-wall/keyframes/k00041.jpg",
"Fixtures/synthetic-wall/keyframes/k00042.jpg",
"Fixtures/synthetic-wall/keyframes/k00043.jpg",
"Fixtures/synthetic-wall/keyframes/k00044.jpg",
"Fixtures/synthetic-wall/keyframes/k00045.jpg",
"Fixtures/synthetic-wall/keyframes/k00046.jpg",
"Fixtures/synthetic-wall/keyframes/k00047.jpg",
"Fixtures/synthetic-wall/keyframes/k00048.jpg",
"Fixtures/synthetic-wall/keyframes/k00049.jpg",
"Fixtures/synthetic-wall/keyframes/k00050.jpg",
"Fixtures/synthetic-wall/keyframes/k00051.jpg",
"Fixtures/synthetic-wall/keyframes/k00052.jpg",
"Fixtures/synthetic-wall/keyframes/k00053.jpg",
"Fixtures/synthetic-wall/keyframes/k00054.jpg",
"Fixtures/synthetic-wall/keyframes/k00055.jpg",
"Fixtures/synthetic-wall/keyframes/k00056.jpg",
"Fixtures/synthetic-wall/keyframes/k00057.jpg",
"Fixtures/synthetic-wall/keyframes/k00058.jpg",
"Fixtures/synthetic-wall/keyframes/k00059.jpg",
"Fixtures/synthetic-wall/keyframes/k00060.jpg",
"Fixtures/synthetic-wall/keyframes/k00061.jpg",
"Fixtures/synthetic-wall/session.json",
);
target = EF941BB37D13A698579DF2A6 /* HouseScanUITests */;
Expand Down
6 changes: 5 additions & 1 deletion ios/HouseScan/Runtime/Autopilot.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ final class Autopilot {
guard await waitUntil(timeout: 60, { self.engine.replay != nil || self.engine.state.failure != nil }), let replay = engine.replay else {
return fail("replay did not load")
}
async let prepared: Void = replay.prepareHeldBack()
async let prepared: Void = replay.prepareHeldBack(endsWherePhoneStood: engine.options.autopilotCantGetThere)
await pause(hold)
await engine.waitForGate(.onboarding)
engine.finishOnboarding()
Expand Down Expand Up @@ -341,6 +341,10 @@ final class Autopilot {
await pause(hold)
await answerOpenSky()
log("answered the overhead question: open sky; \(overheadSummary)")
// The rest of the tilt-up run (the synthetic replay's upper walk) plays out, so its
// views are kept before the walk ends.
guard let replay = engine.replay else { return }
_ = await waitUntil(timeout: 30) { !replay.isPlaying }
}

/// An overhead gap request: the engine plays the recording's tilt-up frames for it, and the
Expand Down
12 changes: 9 additions & 3 deletions ios/HouseScan/Runtime/ReplayPlayer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,17 @@ final class ReplayPlayer {

/// Finds the frames to hold back for the autopilot's gap loop. Heavy, so it runs off the
/// main actor; call once before the walk.
func prepareHeldBack() async {
let planned = planned
/// Planned over the walk's frames only: the recording's closing tilt-up run is played for the
/// tilt-up step and for requests above the walk (`ScanEngine.tiltUpFrames`), never in the walk.
/// With `endsWherePhoneStood`, against the ends a walk ended by "Can't get there" sets,
/// 1 m inside them (the cells skipped there are within 0.5 m of the phone).
func prepareHeldBack(endsWherePhoneStood: Bool = false) async {
let walkEnd = ScanEngine.tiltUpFrames(in: self, map: CoverageMap(wall: wall)).lowerBound
let planned = Array(planned[..<walkEnd])
let wall = wall
let policy: ReplayPlanning.EndPolicy = endsWherePhoneStood ? .walked(margin: 1) : .coveredExtremes
heldBack = await Task.detached(priority: .userInitiated) {
ReplayPlanning.heldBackWindow(frames: planned, wall: wall)
ReplayPlanning.heldBackWindow(frames: planned, wall: wall, ends: policy)
}.value
}

Expand Down
14 changes: 10 additions & 4 deletions ios/HouseScan/Runtime/ScanEngine.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,9 +320,10 @@ final class ScanEngine {
live?.setMode(.walk)
if let replay {
autoCapture.reset()
// An overhead request is answered by tilting up, which the tilt-up frames show;
// other requests by the frames held back from the walk.
if case .overhead = gapPlan?.need, let map = coverage, !Self.tiltUpFrames(in: replay, map: map).isEmpty {
// An overhead request, or one for the wall above what the walk saw, is answered
// by tilting up, which the tilt-up frames show; other requests by the frames held
// back from the walk.
if gapPlan?.asksAboveTheWalk == true, let map = coverage, !Self.tiltUpFrames(in: replay, map: map).isEmpty {
replay.play(range: Self.tiltUpFrames(in: replay, map: map), speed: replaySpeed)
} else {
let range = replay.heldBack.map { ReplayPlanning.gapReplayRange($0.frames) } ?? 0..<replay.frames.count
Expand All @@ -349,10 +350,15 @@ final class ScanEngine {
private var autoAdvanceDelay: Double { options.autopilot ? max(1.2, options.autopilotHold) : 1.2 }

/// With `-autopilotGate`, waits until the UI test has finished with `phase` (its file exists),
/// for at most two minutes so a lost gate file can't hang the app.
/// for at most two minutes so a lost gate file can't hang the app. While it waits, the file
/// `<phase>.held` says the autopilot has finished with the screen and holds it still, for a UI
/// test that must audit it settled rather than while the autopilot drives it.
func waitForGate(_ phase: ScanPhase) async {
guard options.autopilot, let gate = options.autopilotGate else { return }
let file = gate.appending(path: phase.rawValue)
if !FileManager.default.fileExists(atPath: file.path) {
try? Data().write(to: gate.appending(path: "\(phase.rawValue).held"))
}
let deadline = ContinuousClock.now + .seconds(120)
while ContinuousClock.now < deadline, !FileManager.default.fileExists(atPath: file.path) {
try? await Task.sleep(for: .milliseconds(100))
Expand Down
2 changes: 1 addition & 1 deletion ios/HouseScan/UI/Components/InstructionCard.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ struct InstructionCard: View {
.foregroundStyle(Palette.chalk)
.padding(.horizontal, 16)
.frame(minHeight: Metrics.minTarget)
.background(.white.opacity(0.14), in: .capsule)
.background(Palette.replyFill, in: .capsule)
.contentShape(.capsule)
.buttonStyle(PressableStyle())
// The colours are set above and PressableStyle doesn't read isEnabled, so the
Expand Down
5 changes: 5 additions & 0 deletions ios/HouseScan/UI/Theme/Theme.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ enum Palette {
/// is about 4.9:1 on paper, but the audit rated "See it on your wall" as only nearly
/// passing; white on this is about 6.2:1.
static let signalFill = Color(red: 26 / 255.0, green: 88 / 255.0, blue: 214 / 255.0)
/// A reply button on the dark instruction card: what 14% white over `ink` (#0B1220) looks like,
/// as a solid colour (#2D333F), so the button's contrast with `chalk` text (about 11:1) doesn't
/// depend on how the translucent layer is composited over the camera behind the card. The
/// accessibility audit rejected the translucent version on CI's LiDAR replay.
static let replyFill = Color(red: 45 / 255.0, green: 51 / 255.0, blue: 63 / 255.0)
static let unseen = Color("CoverageUnseen")
static let seen = Color("CoverageSeen")
static let covered = Color("CoverageCovered")
Expand Down
9 changes: 9 additions & 0 deletions ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,15 @@ public struct GapPlan: Sendable, Equatable {
/// Nil for a request built in meters, whose span is then read at the export's four decimals.
public var requestedSpanFt: ClosedRange<Double>?

/// A request a view tilted up at the wall answers: something overhead, or the wall seen
/// higher than the walk's band.
public var asksAboveTheWalk: Bool {
switch need {
case .overhead, .wallUp: true
case .cells, .groundOut, .walkOut: false
}
}

public init(
band: SurfaceBand, span: ClosedRange<Float>, reason: Reason, need: Need = .cells,
requestedOutFt: Double? = nil, requestedSpanFt: ClosedRange<Double>? = nil
Expand Down
70 changes: 56 additions & 14 deletions ios/HouseScanKit/Sources/HouseScanKit/Replay/ReplayPlanning.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,16 +133,32 @@ public enum ReplayPlanning {
public var gap: GapPlan
}

/// Where the walk that leaves a window out will set its ends.
public enum EndPolicy: Sendable, Equatable {
/// Marked at the covered extremes, as the autopilot's `markEnds` does.
case coveredExtremes
/// Set where the phone stood farthest each way ("Can't get there", `WalkedEnd`), after
/// "Can't get there" on the ground by the meter. The gap must then lie `margin` meters
/// inside those ends: "Can't get there" at an end also skips the cells there.
case walked(margin: Float)
}

/// Chooses frames to hold back from the autopilot's walk so the gap loop has something real
/// to do: without them the planner finds a gap, and replaying them alone closes it.
///
/// Candidates are contiguous windows of 3 to 10 frames, tried from the middle of the walk
/// outward. The gap must lie within where the rest of the walk went (`isWithinWalk`): a gap
/// past the farthest frame the phone stood at falls outside ends set where the phone is
/// ("Can't get there"), and then nothing asks for it. Each is screened on coverage rebuilt from what every frame sees (computed once),
/// then confirmed with the full auto-capture simulation the app runs. Nil when no window works,
/// for example when the planner's nearest gap is one no frame of the replay can close.
public static func heldBackWindow(frames: [PlannedFrame], wall: WallFrame, planner: GapPlanner = GapPlanner(), config: CoverageConfig = CoverageConfig()) -> HeldBackWindow? {
/// ("Can't get there"), and then nothing asks for it. With `ends` `.walked`, the window is
/// planned against the ends that flow sets and the cells it skips, so the gap it leaves is
/// one that flow asks for. Each is screened on coverage rebuilt from what every frame sees
/// (computed once), then confirmed with the full auto-capture simulation the app runs. Nil
/// when no window works, for example when the planner's nearest gap is one no frame of the
/// replay can close.
public static func heldBackWindow(
frames: [PlannedFrame], wall: WallFrame, ends policy: EndPolicy = .coveredExtremes,
planner: GapPlanner = GapPlanner(), config: CoverageConfig = CoverageConfig()
) -> HeldBackWindow? {
guard frames.count >= 6 else { return nil }
let empty = CoverageMap(wall: wall, config: config)
let sightings = frames.map { $0.trackingNormal ? empty.visibleCells(from: $0.camera) : [] }
Expand All @@ -157,17 +173,18 @@ public enum ReplayPlanning {
for index in kept.sorted() where !window.contains(index) {
rest.record(sightings[index], from: frames[index].camera.position)
}
guard let ends = coveredExtremes(rest) else { continue }
rest.setEnd(.left, at: ends.lowerBound)
rest.setEnd(.right, at: ends.upperBound)
let stood = frames.indices.filter { !window.contains($0) }.map { frames[$0] }
guard let ends = self.ends(policy, rest, stood: stood, wall: wall) else { continue }
prepare(&rest, ends: ends, policy: policy)
guard let gap = planner.plan(rest),
isWithinWalk(gap, kept.sorted().filter { !window.contains($0) }.map { frames[$0] }, wall: wall) else { continue }
isWithinWalk(gap, kept.sorted().filter { !window.contains($0) }.map { frames[$0] }, wall: wall),
isInside(gap, ends: ends, policy: policy) else { continue }
var restored = rest
for index in window { restored.record(sightings[index], from: frames[index].camera.position) }
guard planner.isSatisfied(gap, restored) else { continue }
// Screened: confirm with the exact auto-capture path, a few times at most.
confirmations += 1
if let confirmed = confirm(frames, window: window, wall: wall, planner: planner, config: config) {
if let confirmed = confirm(frames, window: window, wall: wall, policy: policy, planner: planner, config: config) {
return confirmed
}
if confirmations >= 12 { return nil }
Expand Down Expand Up @@ -199,18 +216,43 @@ public enum ReplayPlanning {
return low...high
}

private static func confirm(_ frames: [PlannedFrame], window: Range<Int>, wall: WallFrame, planner: GapPlanner, config: CoverageConfig) -> HeldBackWindow? {
private static func confirm(
_ frames: [PlannedFrame], window: Range<Int>, wall: WallFrame, policy: EndPolicy, planner: GapPlanner, config: CoverageConfig
) -> HeldBackWindow? {
var rest = frames
rest.removeSubrange(window)
var walked = simulateWalk(rest, wall: wall, config: config)
guard let ends = coveredExtremes(walked) else { return nil }
walked.setEnd(.left, at: ends.lowerBound)
walked.setEnd(.right, at: ends.upperBound)
guard let gap = planner.plan(walked), isWithinWalk(gap, rest, wall: wall),
guard let ends = self.ends(policy, walked, stood: rest, wall: wall) else { return nil }
prepare(&walked, ends: ends, policy: policy)
guard let gap = planner.plan(walked), isWithinWalk(gap, rest, wall: wall), isInside(gap, ends: ends, policy: policy),
satisfiedWithLeadIn(frames, window: window, walked: walked, gap: gap, planner: planner) else { return nil }
return HeldBackWindow(frames: window, ends: ends, gap: gap)
}

private static func ends(_ policy: EndPolicy, _ map: CoverageMap, stood: [PlannedFrame], wall: WallFrame) -> ClosedRange<Float>? {
switch policy {
case .coveredExtremes:
return coveredExtremes(map)
case .walked:
let s = stood.map { wall.wallPoint($0.camera.position).s }
guard let low = s.min(), let high = s.max(), low < 0, high > 0 else { return nil }
return low...high
Comment thread
SamGu-NRX marked this conversation as resolved.
}
}

/// Sets the ends, and for a walked flow skips the ground by the meter, as "Can't get there"
/// on the walk's first request does (`GuidancePlanner.groundByMeterDone`, ±0.5 m).
private static func prepare(_ map: inout CoverageMap, ends: ClosedRange<Float>, policy: EndPolicy) {
map.setEnd(.left, at: ends.lowerBound)
map.setEnd(.right, at: ends.upperBound)
if case .walked = policy { map.markSkipped(.ground, -0.5...0.5) }
}

private static func isInside(_ gap: GapPlan, ends: ClosedRange<Float>, policy: EndPolicy) -> Bool {
guard case .walked(let margin) = policy else { return true }
return ends.lowerBound + margin <= gap.span.lowerBound && gap.span.upperBound <= ends.upperBound - margin
}

/// Whether `gap` lies between the farthest positions along the wall `frames` stood at.
private static func isWithinWalk(_ gap: GapPlan, _ frames: [PlannedFrame], wall: WallFrame) -> Bool {
let s = frames.map { wall.wallPoint($0.camera.position).s }
Expand Down
Loading
Loading