Skip to content
13 changes: 11 additions & 2 deletions ios/HouseScan/Contract/ScanContract.swift
Original file line number Diff line number Diff line change
Expand Up @@ -759,6 +759,10 @@ enum SpotCheckAnswer: Equatable, Sendable {
case clear
/// Something stands there: the scan stops claiming that area and is checked again.
case somethingThere
/// The homeowner can't see or reach the area to say: it was not observed, which is neither
/// an obstruction nor clear. The scan stops claiming that area and is checked again, and the
/// result says nobody checked it.
case cannotCheck
}

/// The one question asked before an answer's spot is shown as the result: is anything standing
Expand Down Expand Up @@ -891,6 +895,11 @@ final class ScanViewState {
var result: ResultPresentation?
/// The homeowner's check of the proposed spot, retained beside the result.
var spotCheck: SpotCheck?
/// True when the homeowner answered "I can't check this area" about an area other than the
/// spot the result names, or while it names none. That area stays out of the scan, so the
/// result says so beside whatever it says about its own spot. Cleared with the scan's spot
/// checks.
var uncheckedAreaElsewhere = false
/// True while the engine sees the AR scene drawing the result in the live camera
/// (`ResultOverlayPolicy`). The AR screen then draws no overlay of its own; otherwise it
/// draws `BatteryOverlay`.
Expand Down Expand Up @@ -995,8 +1004,8 @@ protocol ScanActions: AnyObject {
func retryUpload()
/// After a rejected upload: back to the feature review, keeping the scan.
func backToReview()
/// The answer to `ScanViewState.spotCheck`: true when nothing stands in the area.
func answerSpotCheck(clear: Bool)
/// The answer to `ScanViewState.spotCheck`.
func answerSpotCheck(_ answer: SpotCheckAnswer)
/// Start a capture for a server-listed missing item.
func captureMissing(_ id: String)
func showAR()
Expand Down
32 changes: 20 additions & 12 deletions ios/HouseScan/Runtime/Autopilot.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ final class Autopilot {
/// back to pixels use the same size.
private let viewSize = CGSize(width: 393, height: 852)
private var hold: Double { engine.options.autopilotHold }
/// Set once a spot check was answered "Something's there" (`-autopilotSomethingThere`): the
/// checks after it are answered "It's clear", so a server that keeps moving the spot can't
/// keep the run going.
private var answeredSomethingThere = false
/// Set once a spot check was answered "Something's there" (`-autopilotSomethingThere`) or "I
/// can't check this area" (`-autopilotCannotCheck`): the checks after it are answered "It's
/// clear", so a server that keeps moving the spot can't keep the run going.
private var withheldClear = false

init(engine: ScanEngine) {
self.engine = engine
Expand Down Expand Up @@ -399,18 +399,26 @@ final class Autopilot {
return false
}

/// "It's clear", or with `-autopilotSomethingThere` "Something's there" the first time. After
/// a refusal the stretch it withdrew goes to the gate folder as `spot-refusal.json` (the
/// stretch's s in feet, as scene.json's spans), for the UI test to check the scene against.
/// "It's clear", or the first time "Something's there" with `-autopilotSomethingThere` and "I
/// can't check this area" with `-autopilotCannotCheck`. After either, the stretch the answer
/// withdrew goes to the gate folder as `spot-refusal.json` (the stretch's s in feet, as
/// scene.json's spans), for the UI test to check the scene against.
private func answerSpotCheck() async {
guard let check = engine.state.spotCheck, check.answer == nil else { return }
await pause(hold)
await engine.waitForGate(.spotConfirm)
let refuse = engine.options.autopilotSomethingThere && !answeredSomethingThere
engine.answerSpotCheck(clear: !refuse)
log("spot check \(check.id) over \(format(check.area)): \(refuse ? "something's there" : "it's clear")")
if refuse {
answeredSomethingThere = true
let options = engine.options
let first: SpotCheckAnswer = options.autopilotCannotCheck ? .cannotCheck : options.autopilotSomethingThere ? .somethingThere : .clear
let answer: SpotCheckAnswer = withheldClear ? .clear : first
engine.answerSpotCheck(answer)
let said = switch answer {
case .clear: "it's clear"
case .somethingThere: "something's there"
case .cannotCheck: "I can't check this area"
}
log("spot check \(check.id) over \(format(check.area)): \(said)")
if answer != .clear {
withheldClear = true
writeRefusalForTest()
}
_ = await waitUntil(timeout: 20) { self.engine.state.phase != .spotConfirm }
Expand Down
13 changes: 13 additions & 0 deletions ios/HouseScan/Runtime/LaunchOptions.swift
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ import OSLog
/// marking the ends (`Autopilot.endWalkByCantGetThere`).
/// - `-autopilotSomethingThere`: the autopilot answers the first spot check "Something's there"
/// instead of "It's clear", so the scan is checked again without that area.
/// - `-autopilotCannotCheck`: the autopilot answers the first spot check "I can't check this
/// area" instead of "It's clear". It can't be combined with `-autopilotSomethingThere`.
/// - `-sampleResultAfterSpotAnswer <path>` (debug builds only): with the bundled sample, the
/// upload sent after a spot check answered "Something's there" or "I can't check this area",
/// and every later upload of that scan, is answered with the server answer in this JSON file,
/// such as one without a spot. Start over or a new wall goes back to the bundled sample.
/// - `-injectGroundRise <meters>`: with `-replay`, `-autopilot` and `-autopilotGate`, once the
/// first upload starts, each time a file named `inject-ground` appears in the gate folder the
/// app deletes it and hands the engine a detected floor that many meters above its current
Expand All @@ -40,6 +46,8 @@ struct LaunchOptions: Equatable {
var autopilotGate: URL?
var autopilotCantGetThere = false
var autopilotSomethingThere = false
var autopilotCannotCheck = false
var sampleResultAfterSpotAnswer: URL?
var simulateAppStore = false
var injectGroundRise: Float?

Expand All @@ -57,6 +65,11 @@ struct LaunchOptions: Equatable {
autopilot = arguments.contains("-autopilot")
autopilotCantGetThere = arguments.contains("-autopilotCantGetThere")
autopilotSomethingThere = arguments.contains("-autopilotSomethingThere")
autopilotCannotCheck = arguments.contains("-autopilotCannotCheck")
precondition(!(autopilotSomethingThere && autopilotCannotCheck), "-autopilotSomethingThere and -autopilotCannotCheck each choose the first spot answer; pass one")
#if DEBUG
sampleResultAfterSpotAnswer = value(after: "-sampleResultAfterSpotAnswer").map { URL(fileURLWithPath: $0) }
#endif
simulateAppStore = arguments.contains("-simulateAppStore")
serverURL = (value(after: "-serverURL") ?? defaultServerURL).flatMap(Self.serverURL)
sampleResult = arguments.contains("-sampleResult")
Expand Down
5 changes: 5 additions & 0 deletions ios/HouseScan/Runtime/ResultClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ final class SampleResultClient: ResultClient {
let isSample = true
/// Seconds the fake upload takes, so each upload state is visible in demos and UI tests.
let pace: Double
/// A server answer in a JSON file that replaces the bundled sample while it is set. Only the
/// engine sets it, from `-sampleResultAfterSpotAnswer` and the scan's spot answers
/// (`ScanEngine.spotConfirm`), for UI tests.
var answerFile: URL?

init(pace: Double) {
self.pace = pace
Expand All @@ -159,6 +163,7 @@ final class SampleResultClient: ResultClient {
progress(Double(step) / 4)
}
try await Task.sleep(for: .seconds(pace))
if let answerFile { return try Data(contentsOf: answerFile) }
guard let url = Bundle.main.url(forResource: "SampleResult", withExtension: "json") else { throw UploadError.missingSample }
return try Data(contentsOf: url)
}
Expand Down
57 changes: 41 additions & 16 deletions ios/HouseScan/Runtime/ScanEngine+Confirm.swift
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ struct SpotConfirmState {
var request: GuidanceLog.Request?
/// Checks asked in this scan, for their ids.
var asked = 0
/// The area of the spot the answer being shown names, nil while it names none: the result
/// speaks of an unchecked area apart from it (`ScanViewState.uncheckedAreaElsewhere`).
var shownArea: SpotArea?
}

/// The spot check: before an answer's spot is shown as the result, the homeowner is shown the
Expand All @@ -30,11 +33,13 @@ struct SpotConfirmState {
///
/// Camera-only coverage and the walked path claim wall, ground and clear space they never saw
/// (HouseScanKit `CoverageMap`, "Bounded exceptions"). "It's clear" backs those claims for this
/// spot and shows the result. "Something's there" withdraws them over the area
/// (`CoverageMap.withdrawClaims`), so the scene reports it unseen, and uploads again, as a closed
/// gap does; the new answer's spot is asked about again unless an earlier answer already covers
/// it (`SpotConfirmations.settling`). Nothing new goes into scene.json: the answer shows only as
/// the stretch the scene no longer claims, and in the packet's guidance log.
/// spot and shows the result. "Something's there" and "I can't check this area" withdraw them
/// over the area (`CoverageMap.withdrawClaims`), so the scene reports it unseen, and upload
/// again, as a closed gap does; the new answer's spot is asked about again unless an earlier
/// answer already covers it (`SpotConfirmations.settling`), and then the result says which of
/// the two the homeowner gave. Nothing new goes into scene.json: the answer shows only as the
/// stretch the scene no longer claims, and in the packet's guidance log, where both close the
/// check as skipped.
///
/// The bundled sample is checked like a server's answer, and the screen labels it a sample: the
/// UI tests and the offline demo only ever see the sample, so skipping it would leave the step
Expand Down Expand Up @@ -66,16 +71,19 @@ extension ScanEngine {
guard let result = state.result, let spot = result.spot, let wall = coverage?.wall,
let sceneSHA256 = spotConfirm.sceneSHA256, let answerSHA256 = spotConfirm.answerSHA256 else {
state.spotCheck = nil
spotConfirm.shownArea = nil
go(.result)
return
}
let area = SpotArea(
spot: spot.span, spotOut: spot.offsetFromWall...(spot.offsetFromWall + spot.depth),
zones: result.clearances.map { (span: $0.span, depth: $0.depth) })
spotConfirm.shownArea = area
spotConfirm.asked += 1
let id = spotConfirm.asked
if let settled = spotConfirm.confirmations.settling(area) {
// Kept for the result, which says when the homeowner said something stands there.
// Kept for the result, which says when the homeowner said something stands there or
// couldn't check the area.
state.spotCheck = Self.spotCheck(id: id, area: area, spot: spot, photo: nil, answer: Self.answer(settled.answer), isSample: result.isSample)
RuntimeLog.engine.info("spot check: s=\(area.spot.lowerBound)...\(area.spot.upperBound) was answered before (\(settled.answer.rawValue, privacy: .public)); showing the result")
go(.result)
Expand Down Expand Up @@ -116,23 +124,30 @@ extension ScanEngine {
}

/// The homeowner's answer to the spot check on screen.
func answerSpotCheck(clear: Bool) {
func answerSpotCheck(_ answer: SpotCheckAnswer) {
guard state.phase == .spotConfirm, var check = state.spotCheck, check.answer == nil, var confirmation = spotConfirm.pending else { return }
confirmation.answer = clear ? .clear : .somethingThere
confirmation.answer = Self.confirmationAnswer(answer)
spotConfirm.confirmations.record(confirmation)
spotConfirm.pending = nil
check.answer = Self.answer(confirmation.answer)
check.answer = answer
state.spotCheck = check
// A refusal sends the area to review without the claims, like an overhead answer.
resolveGuidance(clear ? .met : .skipped)
// An answer other than "It's clear" sends the area to review without the claims, like an
// overhead answer.
resolveGuidance(confirmation.answer.guidanceOutcome)
let area = confirmation.area
if !clear { updateCoverage { $0.withdrawClaims(over: area.span) } }
RuntimeLog.engine.info("spot check \(check.id): \(clear ? "clear" : "something there, claims withdrawn", privacy: .public) over s=\(area.span.lowerBound)...\(area.span.upperBound), photo \(confirmation.photoID ?? "none", privacy: .public), answer \(confirmation.answerSHA256.prefix(12), privacy: .public), scene \(confirmation.sceneSHA256.prefix(12), privacy: .public)")
let keepsClaims = confirmation.answer.keepsClaims
if !keepsClaims { updateCoverage { $0.withdrawClaims(over: area.span) } }
let said = switch confirmation.answer {
case .clear: "clear"
case .somethingThere: "something there, claims withdrawn"
case .cannotCheck: "can't check, claims withdrawn"
}
RuntimeLog.engine.info("spot check \(check.id): \(said, privacy: .public) over s=\(area.span.lowerBound)...\(area.span.upperBound), photo \(confirmation.photoID ?? "none", privacy: .public), answer \(confirmation.answerSHA256.prefix(12), privacy: .public), scene \(confirmation.sceneSHA256.prefix(12), privacy: .public)")
let id = check.id
Task {
try? await Task.sleep(for: .seconds(spotAnsweredHold))
guard state.phase == .spotConfirm, state.spotCheck?.id == id else { return }
guard clear else {
guard keepsClaims else {
// The scan without those claims goes to the server again.
startUpload()
return
Expand All @@ -147,8 +162,9 @@ extension ScanEngine {
}

/// Whether a capture can settle a gap request: not over a stretch whose claims the homeowner
/// withdrew, where a new view would claim the same thing past the same obstruction. Overhead
/// requests are not about that stretch's wall or ground.
/// withdrew, where a new view would claim the same thing past the same obstruction, or past
/// whatever kept the homeowner from checking it. Overhead requests are not about that
/// stretch's wall or ground.
func captureCanSettle(_ plan: GapPlan) -> Bool {
if case .overhead = plan.need { return true }
return !(coverage?.hasWithdrawnClaims(overlapping: plan.span) ?? false)
Expand Down Expand Up @@ -199,6 +215,15 @@ extension ScanEngine {
switch answer {
case .clear: .clear
case .somethingThere: .somethingThere
case .cannotCheck: .cannotCheck
}
}

private static func confirmationAnswer(_ answer: SpotCheckAnswer) -> SpotConfirmationAnswer {
switch answer {
case .clear: .clear
case .somethingThere: .somethingThere
case .cannotCheck: .cannotCheck
}
}
}
22 changes: 18 additions & 4 deletions ios/HouseScan/Runtime/ScanEngine.swift
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,20 @@ final class ScanEngine {
}
/// Every request the homeowner was shown, for the packet.
var guidanceLog = GuidanceLog()
/// The spot check (`ScanEngine+Confirm.swift`).
var spotConfirm = SpotConfirmState()
/// The spot check (`ScanEngine+Confirm.swift`). What follows from its records is read from
/// them on every change, so recording an answer sets it and a reset clears it:
/// `ScanViewState.uncheckedAreaElsewhere`, and the UI tests' answer file
/// (`-sampleResultAfterSpotAnswer`), which applies only once this scan has an answer other
/// than "It's clear", so a scan after Start over or a new wall starts on the bundled sample.
var spotConfirm = SpotConfirmState() {
didSet {
let elsewhere = spotConfirm.confirmations.leftUnchecked(besides: spotConfirm.shownArea)
if state.uncheckedAreaElsewhere != elsewhere { state.uncheckedAreaElsewhere = elsewhere }
if let file = options.sampleResultAfterSpotAnswer, let sample = resultClient as? SampleResultClient {
sample.answerFile = spotConfirm.confirmations.records.contains { !$0.answer.keepsClaims } ? file : nil
}
}
}
/// When each mark was made, on the capture clock (`MarkKey`).
var markTimes: [String: Double] = [:]
/// The packet's clock for guidance and marks: the latest frame's time, ARFrame.timestamp
Expand Down Expand Up @@ -1717,6 +1729,7 @@ final class ScanEngine {
state.spotCheck = nil
spotConfirm.pending = nil
spotConfirm.request = nil
spotConfirm.shownArea = nil
// A spot photo still loading for this answer (`presentAnswer`) must not open its check
// once a retry brings back an equal answer.
spotConfirm.asked += 1
Expand Down Expand Up @@ -2125,8 +2138,9 @@ final class ScanEngine {
// A request raised while the phone has lost its place could only time out: show the result.
guard !automaticGapsStopped, !state.tracking.hasLostItsPlace, let map = coverage else { return [] }
let asked = automaticGaps + (asking.map { [$0] } ?? [])
// A new view cannot settle an area the homeowner has already said is obstructed.
// Filter before the request limit so refused areas do not consume the remaining slots.
// A new view cannot settle an area the homeowner has already said is obstructed, or
// couldn't check. Filter before the request limit so those areas do not consume the
// remaining slots.
let capturable = result.missingEvidence.filter { item in
gapPlanner.plan(for: item, leftEnd: map.leftEnd, rightEnd: map.rightEnd, limitEnds: map.limitEnds)
.map { captureCanSettle($0) } ?? false
Expand Down
Loading
Loading