Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ios/HouseScan.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
"Fixtures/results/no-clean-spot.json",
Fixtures/results/pass.json,
"Fixtures/results/reject-nearest.json",
"Fixtures/results/spot-left.json",
"Fixtures/results/spot-straddle.json",
"Fixtures/results/unsure-view.json",
"Fixtures/synthetic-wall",
"Fixtures/synthetic-wall-lidar",
Expand Down
25 changes: 17 additions & 8 deletions ios/HouseScan/Runtime/ScanEngine+Confirm.swift
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,6 @@ extension ScanEngine {
let candidates = store.keyframes.map { SpotPhotoCandidate(id: $0.id, camera: $0.camera, trackingNormal: $0.tracking == .normal) }
let choice = SpotPhoto.best(candidates, area: area, wall: wall)
let stored = choice.flatMap { choice in store.keyframes.first { $0.id == choice.id } }
// `answer` is replaced by the homeowner's (`answerSpotCheck`) before it is recorded.
spotConfirm.pending = SpotConfirmation(area: area, answerSHA256: answerSHA256, sceneSHA256: sceneSHA256, photoID: stored?.id, answer: .clear)
spotConfirm.request = spotCheckRequest(area: area, photoID: stored?.id, answerSHA256: answerSHA256, sceneSHA256: sceneSHA256, id: id)
let file = stored.map { store.directory.appending(path: $0.fileName) }
if let choice {
RuntimeLog.engine.info("spot check: photo \(choice.id, privacy: .public) shows \(Int(choice.footprintInView * 100))% of the spot and \(Int(choice.areaInView * 100))% of s=\(area.span.lowerBound)...\(area.span.upperBound)")
Expand All @@ -100,8 +97,20 @@ extension ScanEngine {
if let image, let camera = stored?.camera {
photo = SpotCheck.Photo(image: image, projection: CameraProjection(
cameraToWorld: camera.cameraToWorld, intrinsics: camera.intrinsics, imageSize: camera.imageSize))
} else if let stored {
RuntimeLog.engine.error("spot check: keyframe \(stored.id, privacy: .public) didn't load; asking without a photo")
}
state.spotCheck = Self.spotCheck(id: id, area: area, spot: spot, photo: photo, answer: nil, isSample: result.isSample)
let check = Self.spotCheck(id: id, area: area, spot: spot, photo: photo, answer: nil, isSample: result.isSample)
// Recorded once the photo has loaded, from what the screen shows: the photo only when
// it is outlined on screen, and the question asked with or without the outline.
let outlined = check.outline(on: state.wall) != nil
let shownID = outlined ? stored?.id : nil
// `answer` is replaced by the homeowner's (`answerSpotCheck`) before it is recorded.
spotConfirm.pending = SpotConfirmation(area: area, answerSHA256: answerSHA256, sceneSHA256: sceneSHA256, photoID: shownID, answer: .clear)
spotConfirm.request = spotCheckRequest(
area: area, question: ScanCopy.spotQuestionShown(outlined: outlined), photoID: shownID,
answerSHA256: answerSHA256, sceneSHA256: sceneSHA256, id: id)
state.spotCheck = check
go(.spotConfirm)
}
}
Expand Down Expand Up @@ -155,15 +164,15 @@ extension ScanEngine {
var spotCheckGuidance: GuidanceLog.Request? { spotConfirm.request }

/// A gap_band request on the ground over the area. scene.json and the packet have no field
/// for the check, so the message carries what binds it: the photo shown and the sha256 of the
/// answer and scene it was about.
private func spotCheckRequest(area: SpotArea, photoID: String?, answerSHA256: String, sceneSHA256: String, id: Int) -> GuidanceLog.Request {
/// for the check, so the message carries what binds it: the question and photo shown and the
/// sha256 of the answer and scene it was about.
private func spotCheckRequest(area: SpotArea, question: Instruction, photoID: String?, answerSHA256: String, sceneSHA256: String, id: Int) -> GuidanceLog.Request {
let format = { (value: Float) in String(format: "%.2f", value) }
let binding = "Spot check \(id): keyframe \(photoID ?? "none") shown; spot s \(format(area.spot.lowerBound)) to \(format(area.spot.upperBound)) m, "
+ "area s \(format(area.span.lowerBound)) to \(format(area.span.upperBound)) m out to \(format(area.depth)) m; "
+ "answer sha256 \(answerSHA256), scene sha256 \(sceneSHA256)."
return GuidanceLog.Request(
topic: .spotCheck(id: id), kind: .gapBand, origin: .phone, message: Self.text(ScanCopy.spotQuestion) + " " + binding,
topic: .spotCheck(id: id), kind: .gapBand, origin: .phone, message: Self.text(question) + " " + binding,
band: .ground, span: area.span)
}

Expand Down
57 changes: 51 additions & 6 deletions ios/HouseScan/UI/Copy/SpotCheckCopy.swift
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,26 @@ import Foundation

/// The spot check's words (`SpotConfirmScreen`, and the guidance log's record of the question).
extension ScanCopy {
/// One question about the wall and the ground, with the reason for asking.
/// One question about the wall and the ground, with the reason for asking, when the area is
/// outlined on a photo.
static let spotQuestion = Instruction(
title: "Is anything standing in the marked area?",
detail: "Check the wall and the ground inside the blue outline. Photos can miss a low bush, a bin or a step."
)

/// The same question when no outline is drawn: no kept photo shows the area, or there is no
/// wall to draw it with. The area is then given in words (`spotSpace`), so nothing points
/// at an outline that isn't on screen.
static let spotQuestionInPerson = Instruction(
title: "Is anything standing in this space?",
detail: "We couldn't mark this space on a photo, so take a look yourself. Check the wall and the ground for a low bush, a bin, a step or anything else."
)
static let spotSpaceTitle = "Where to look"

/// The question as shown (`SpotCheck.outline(on:)`), on screen and in the guidance log.
static func spotQuestionShown(outlined: Bool) -> Instruction {
outlined ? spotQuestion : spotQuestionInPerson
}
static let spotClear = "It's clear"
static let spotSomethingThere = "Something's there"
static let spotClearHint = "Shows your result."
Expand All @@ -22,19 +37,49 @@ extension ScanCopy {
/// Said on the result when the homeowner said something stands where its spot is.
static let spotRefused = "You said something stands where this spot is. Your scan leaves that area out, and an installer would need to check it."

/// "From 3 ft to 8 ft 6 in right of your meter": where the area runs. `spoken` spells the
/// units out for VoiceOver.
/// "From 3 ft to 8 ft 6 in right of your meter": where the area runs along the wall, nearer
/// end first. An end less than half an inch from the meter rounds to 0 in and is read as the
/// meter itself, so -0.3...24 in reads "From your meter to 2 ft right of it", not "From 0 in
/// left to 2 ft right". `spoken` spells the units out for VoiceOver.
static func spotArea(_ span: ClosedRange<Float>, spoken: Bool = false) -> String {
let length = { (meters: Float) in spoken ? Distance.spoken(meters) : Distance.feetAndInches(meters) }
let side = { (s: Float) in s < 0 ? "left" : "right" }
let atMeter = { (s: Float) in Int((abs(s) / Distance.metersPerInch).rounded()) == 0 }
let low = span.lowerBound, high = span.upperBound
switch (atMeter(low), atMeter(high)) {
case (true, true): return "At your meter"
case (true, false): return "From your meter to \(length(high)) \(side(high)) of it"
case (false, true): return "From your meter to \(length(low)) \(side(low)) of it"
case (false, false): break
}
if low >= 0 { return "From \(length(low)) to \(length(high)) right of your meter" }
if high <= 0 { return "From \(length(high)) to \(length(low)) left of your meter" }
return "From \(length(low)) left to \(length(high)) right of your meter"
}

/// What VoiceOver reads for the photo.
/// The asked-about area's three extents, each readable on its own: along the wall from the
/// meter (`SpotCheck.area`), out from the wall over the ground (`SpotCheck.areaDepth`), and up
/// the wall face to the battery's height (`SpotCheck.spotHeight`). These are the edges
/// `SpotOutline` draws on a photo, and no more: the words describe the area the question is
/// about, not every clearance the server checks.
static func spotSpace(_ check: SpotCheck, spoken: Bool = false) -> (along: String, out: String, up: String) {
let length = { (meters: Float) in spoken ? Distance.spoken(meters) : Distance.feetAndInches(meters) }
return (
along: spotArea(check.area, spoken: spoken),
out: "From the wall out to \(length(check.areaDepth))",
up: "From the ground up to \(length(check.spotHeight))"
)
}

/// `spotSpace` as one spoken sentence, for VoiceOver.
static func spotSpaceSpoken(_ check: SpotCheck) -> String {
let space = spotSpace(check, spoken: true)
return "\(space.along), \(space.out.lowercasedFirst), and \(space.up.lowercasedFirst)."
}

/// What VoiceOver reads for the outlined photo: what the outline marks, and where, since a
/// VoiceOver user can't see the outline's edges.
static func spotPhotoDescription(_ check: SpotCheck) -> String {
let place = spotArea(check.area, spoken: true)
return "A blue outline marks where the battery would stand and the space around it. \(place)."
"A blue outline marks where the battery would stand and the space around it. \(spotSpaceSpoken(check))"
}
}
8 changes: 8 additions & 0 deletions ios/HouseScan/UI/Preview/DemoEngine.swift
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,14 @@ final class DemoEngine: ScanActions {
if let raw = value("-uiDemoSpotAnswered"), state.phase == .spotConfirm {
state.spotCheck?.answer = raw == "somethingThere" ? .somethingThere : .clear
}
// The spot check without an outline: no kept photo shows the area (`-uiDemoSpotNoPhoto`),
// or a photo but no wall to draw the area on it with (`-uiDemoSpotNoWall`).
if arguments.contains("-uiDemoSpotNoPhoto"), state.phase == .spotConfirm {
state.spotCheck?.photo = nil
}
if arguments.contains("-uiDemoSpotNoWall"), state.phase == .spotConfirm {
state.wall = nil
}
if arguments.contains("-uiDemoFollowUp") {
enterFollowUp(at: state.phase)
}
Expand Down
143 changes: 96 additions & 47 deletions ios/HouseScan/UI/Screens/SpotConfirmScreen.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ import SwiftUI
/// outlined, and one question with two equal answers. A photo can claim wall and ground behind a
/// bush, so the homeowner, who is standing there, says whether anything is in the way.
///
/// Without an outline (no kept photo shows the area, or there is no wall to draw it with) the
/// area is given in words instead: where it runs beside the meter, how far out from the wall and
/// how high up it. The question then names no outline, and no photo is shown, since a photo with
/// nothing marked on it would leave the homeowner guessing which part of it is meant.
///
/// The answer replaces the two buttons and stays up a moment, saying what happens next, before
/// the engine moves on (to the result, or to checking the wall again).
struct SpotConfirmScreen: View {
Expand All @@ -23,15 +28,17 @@ struct SpotConfirmScreen: View {
}

private func content(_ check: SpotCheck) -> some View {
ScrollView {
let outline = check.outline(on: state.wall)
let question = ScanCopy.spotQuestionShown(outlined: outline != nil)
return ScrollView {
VStack(alignment: .leading, spacing: 20) {
badges(check)
VStack(alignment: .leading, spacing: 8) {
Text(ScanCopy.spotQuestion.title)
Text(question.title)
.font(Typeface.screenTitle)
.fixedSize(horizontal: false, vertical: true)
.accessibilityAddTraits(.isHeader)
if let detail = ScanCopy.spotQuestion.detail {
if let detail = question.detail {
Text(detail)
.font(Typeface.hint)
.foregroundStyle(Palette.muted)
Expand All @@ -40,13 +47,17 @@ struct SpotConfirmScreen: View {
}
.accessibilityElement(children: .combine)
.accessibilityIdentifier("spot.question")
VStack(alignment: .leading, spacing: 10) {
photo(check)
Text(ScanCopy.spotArea(check.area))
.font(.subheadline.weight(.medium))
.foregroundStyle(Palette.muted)
.fixedSize(horizontal: false, vertical: true)
.accessibilityHidden(true)
if let outline {
VStack(alignment: .leading, spacing: 10) {
outlinedPhoto(outline.photo, wall: outline.wall, check: check)
Text(ScanCopy.spotArea(check.area))
.font(.subheadline.weight(.medium))
.foregroundStyle(Palette.muted)
.fixedSize(horizontal: false, vertical: true)
.accessibilityHidden(true)
}
} else {
space(check)
}
if typeSize.isAccessibilitySize {
answerArea(check)
Expand Down Expand Up @@ -104,45 +115,73 @@ struct SpotConfirmScreen: View {
}
}

@ViewBuilder
private func photo(_ check: SpotCheck) -> some View {
Group {
if let photo = check.photo, let wall = state.wall {
let size = photo.projection.imageSize
// The sensor image shown upright (rotated 90° clockwise) at its own aspect, so
// filling the frame crops nothing and the projection's fill mapping holds.
Color.clear
.aspectRatio(CGFloat(size.y / size.x), contentMode: .fit)
.overlay {
Image(decorative: photo.image, scale: 1, orientation: .right)
.resizable()
.scaledToFill()
}
.overlay {
SpotOutline(photo: photo, wall: wall, check: check)
.opacity(outlineShown ? 1 : 0)
}
.clipShape(.rect(cornerRadius: Metrics.cardRadius, style: .continuous))
// Tall enough to judge, short enough that the question and the photo share
// the screen with the answers on a 6.1 in phone.
.frame(maxHeight: 400)
.frame(maxWidth: .infinity)
} else {
// No kept photo shows the spot from the front: the homeowner looks at the wall.
Label("No photo shows this spot well. Take a look at the wall itself.", systemImage: "photo")
.font(Typeface.hint)
.foregroundStyle(Palette.ink)
.fixedSize(horizontal: false, vertical: true)
.padding(18)
.frame(maxWidth: .infinity, alignment: .leading)
.background(Palette.surface, in: .rect(cornerRadius: Metrics.cardRadius, style: .continuous))
private func outlinedPhoto(_ photo: SpotCheck.Photo, wall: WallGeometry, check: SpotCheck) -> some View {
let size = photo.projection.imageSize
// The sensor image shown upright (rotated 90° clockwise) at its own aspect, so filling
// the frame crops nothing and the projection's fill mapping holds.
return Color.clear
.aspectRatio(CGFloat(size.y / size.x), contentMode: .fit)
.overlay {
Image(decorative: photo.image, scale: 1, orientation: .right)
.resizable()
.scaledToFill()
}
.overlay {
SpotOutline(photo: photo, wall: wall, check: check)
.opacity(outlineShown ? 1 : 0)
}
.clipShape(.rect(cornerRadius: Metrics.cardRadius, style: .continuous))
// Tall enough to judge, short enough that the question and the photo share the
// screen with the answers on a 6.1 in phone.
.frame(maxHeight: 400)
.frame(maxWidth: .infinity)
.accessibilityElement(children: .ignore)
.accessibilityLabel("Photo of your wall")
.accessibilityValue(ScanCopy.spotPhotoDescription(check))
.accessibilityAddTraits(.isImage)
.accessibilityIdentifier("spot.photo")
}

/// The area in words, in the photo's place: the three edges the outline would draw. Each
/// line is its own VoiceOver element, read with the units spelled out, so the rendered lines
/// stay visible to the accessibility audit and the UI tests.
private func space(_ check: SpotCheck) -> some View {
let shown = ScanCopy.spotSpace(check)
let spoken = ScanCopy.spotSpace(check, spoken: true)
return VStack(alignment: .leading, spacing: 10) {
// One element read as the title alone: VoiceOver names the "eye" symbol "Show", and
// the identifier landed on it (CI run 37108821815 read the title as "Show").
Label {
Text(ScanCopy.spotSpaceTitle)
} icon: {
Image(systemName: "eye").accessibilityHidden(true)
}
.font(Typeface.caption)
.foregroundStyle(Palette.muted)
.accessibilityElement(children: .combine)
.accessibilityAddTraits(.isHeader)
.accessibilityIdentifier("spot.area.title")
VStack(alignment: .leading, spacing: 6) {
spaceLine(shown.along, spoken: spoken.along, id: "along")
spaceLine(shown.out, spoken: spoken.out, id: "out")
spaceLine(shown.up, spoken: spoken.up, id: "up")
}
}
.accessibilityElement(children: .ignore)
.accessibilityLabel(check.photo == nil ? "No photo of this spot" : "Photo of your wall")
.accessibilityValue(ScanCopy.spotPhotoDescription(check))
.accessibilityAddTraits(check.photo == nil ? [] : .isImage)
.accessibilityIdentifier("spot.photo")
.padding(18)
.frame(maxWidth: .infinity, alignment: .leading)
.background(Palette.surface, in: .rect(cornerRadius: Metrics.cardRadius, style: .continuous))
.accessibilityElement(children: .contain)
.accessibilityIdentifier("spot.area")
}

/// In the primary color, as the question: Ink stays near-black in dark mode, where Surface
/// turns dark, and this screen follows the phone's appearance.
private func spaceLine(_ text: String, spoken: String, id: String) -> some View {
Text(text)
.font(Typeface.hint)
.fixedSize(horizontal: false, vertical: true)
.accessibilityLabel(spoken)
.accessibilityIdentifier("spot.area.\(id)")
}

/// The two answers, or once answered, what happens next.
Expand Down Expand Up @@ -172,6 +211,16 @@ struct SpotConfirmScreen: View {
}
}

extension SpotCheck {
/// The photo and the wall to outline the area on it with, or nil when either is missing.
/// The screen's question, picture and VoiceOver text follow this one answer, and so does
/// the guidance log's record of what was shown (`ScanEngine.presentAnswer`).
func outline(on wall: WallGeometry?) -> (photo: Photo, wall: WallGeometry)? {
guard let photo = self.photo, let wall else { return nil }
return (photo, wall)
}
}

/// The answer given, and what happens next.
private struct Answered: View {
var answer: SpotCheckAnswer
Expand Down
Loading
Loading