diff --git a/experiments/live-dots/.gitignore b/experiments/live-dots/.gitignore new file mode 100644 index 000000000..05dff4040 --- /dev/null +++ b/experiments/live-dots/.gitignore @@ -0,0 +1,3 @@ +Fixtures/ +.build/ +.swiftpm/ diff --git a/experiments/live-dots/Package.swift b/experiments/live-dots/Package.swift new file mode 100644 index 000000000..5824148b4 --- /dev/null +++ b/experiments/live-dots/Package.swift @@ -0,0 +1,24 @@ +// swift-tools-version: 6.2 +import PackageDescription + +let package = Package( + name: "LiveDots", + platforms: [.macOS(.v15)], + targets: [ + // Fixture decoding and the dot field's rules. No UI, no Metal, no main actor: it runs off + // the main thread while the app precomputes the replay, and the tests drive it directly. + .target( + name: "LiveDotsCore", + // Unoptimised, fusing the 41 keyframes takes about 40 s instead of under 1 s, which + // made plain `swift run LiveDots` sit on its loading bar. + swiftSettings: [.unsafeFlags(["-O"], .when(configuration: .debug))] + ), + .executableTarget( + name: "LiveDots", + dependencies: ["LiveDotsCore"], + swiftSettings: [.defaultIsolation(MainActor.self)] + ), + .testTarget(name: "LiveDotsCoreTests", dependencies: ["LiveDotsCore"]), + ], + swiftLanguageModes: [.v6] +) diff --git a/experiments/live-dots/README.md b/experiments/live-dots/README.md new file mode 100644 index 000000000..2560810da --- /dev/null +++ b/experiments/live-dots/README.md @@ -0,0 +1,20 @@ +# Live dots + +**Question.** Do dots built from recorded depth read as sitting on the wall's edges and surfaces, denser on edges and absent where unseen, at 60 fps with 6,000 dots? It passes if the painted outlines and the bin's silhouette carry edge dots while under 5% of plain-brick voxels do, no dot lands on unseen wall, births ease in without a pop, and 6,000 dots render well inside 16.7 ms. The rules, sizes and colours are in the header of `Sources/LiveDotsCore/DotField.swift`; the fog's borrowings are in the header of its section in `Shaders.swift`. + +**Run.** Needs Xcode 27 and, for video, ffmpeg. + +``` +./fetch-fixtures.sh +swift run LiveDots +swift test +./make-video.sh [output folder] +``` + +Flags: `--scheme hologram|constellation|ember` (three looks), `--fog on|off|veil` (fog that lifts a beat after dots arrive, or the old flat veil), `--still --at ` (a still after playing to that moment). `make-video.sh` and `--still` produced `results/`, downscaled to 800 px. + +The fixture is the synthetic wall from #10, and the meter, gas meter, door and window boxes come from its scene, not a detector. The no-LiDAR mode is simulated from its depth: feature points use a 0.1 gradient, since ARKit finds them on brick texture too, and live while seen in 2 of the last 6 keyframes, where the real app would use 3 of 10 at frame rate. + +**Result.** It passes. At 0.2, 100% of outline voxels and 1% of brick voxels become edges, and nothing draws behind the bin. On an M4 Pro, 6,000 dots take 0.54 ms of GPU per 1170 × 2532 frame, 1.0 ms with fog. + +**What it changes.** Building this into `Map3DOverlay` (#21) needs a surface-sample iterator with per-frame deltas and a 5 cm point set beside the 10 cm map. diff --git a/experiments/live-dots/Sources/LiveDots/Benchmark.swift b/experiments/live-dots/Sources/LiveDots/Benchmark.swift new file mode 100644 index 000000000..286c2d6b6 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/Benchmark.swift @@ -0,0 +1,48 @@ +import Foundation +import LiveDotsCore +import QuartzCore + +/// Frame cost with exactly 6,000 dots, every one mid-birth and mid-evidence-rise so the shader +/// runs its full easing path, at the export size and at the app's Retina size. +enum Benchmark { + static func run(fixture: String?) throws { + let data = try ReplayData.loadNow(folder: FixtureLocator.folder(fixture)) + let renderer = try DotRenderer(data: data) + guard let fullest = data.lidar.states.max(by: { $0.sprites.count < $1.sprites.count }) else { return } + let time = fullest.time + 0.1 + var sprites: [DotSprite] = [] + var copy = 0 + while sprites.count < Tuning.dotCap { + for s in fullest.sprites where sprites.count < Tuning.dotCap { + // Later copies shift 1 cm per pass so they don't stack on the same pixel. + sprites.append(DotSprite( + id: s.id &+ UInt64(copy) << 56, position: s.position + SIMD3(0.01 * Float(copy), 0, 0), kind: s.kind, + onOccluder: s.onOccluder, birthTime: time - 0.1, fromOpacity: 0.45, toOpacity: 0.9, + opacityTime: time - 0.1, edgeSince: s.edgeSince, deathTime: .infinity)) + } + copy += 1 + } + renderer.spriteOverride = sprites + let edges = sprites.count { $0.edgeSince < .infinity } + print("keyframe \(fullest.index + 1): \(sprites.count) dots (\(edges) edges), each with a halo: \(2 * sprites.count) point sprites") + + for (label, scale) in [("1170 x 2532 (export, 3x)", Float(3)), ("780 x 1688 (app window, 2x)", Float(2))] { + let target = try OffscreenTarget(renderer: renderer, scale: scale) + let request = FrameRequest(mode: .lidar, keyframe: fullest.index, time: time, reduceMotion: false, fog: .on) + for _ in 0..<30 { try target.render(request, readBack: false) } + var gpu: [Double] = [], wall: [Double] = [] + for _ in 0..<300 { + let start = CACurrentMediaTime() + gpu.append(try target.render(request, readBack: false) * 1000) + wall.append((CACurrentMediaTime() - start) * 1000) + } + gpu.sort() + wall.sort() + func line(_ name: String, _ values: [Double]) -> String { + let mean = values.reduce(0, +) / Double(values.count) + return "\(name) mean \(String(format: "%.2f", mean)) ms, p95 \(String(format: "%.2f", values[values.count * 95 / 100])) ms" + } + print("\(label): \(line("GPU", gpu)); \(line("encode+submit+wait", wall))") + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/BoxOverlay.swift b/experiments/live-dots/Sources/LiveDots/BoxOverlay.swift new file mode 100644 index 000000000..e602a09f3 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/BoxOverlay.swift @@ -0,0 +1,103 @@ +import LiveDotsCore +import SwiftUI + +/// Brackets around the recognised things on the wall (simulated from the fixture's scene), in +/// the manner of RoomPlan's outlines. The electric meter is solid Signal blue from the first +/// keyframe it is in view; the others appear dashed at 60% Hologram (not settled) and turn solid +/// at 90% once seen face-on. Appearing fades in over 200 ms, settling crossfades over 150 ms, and +/// the meter flashes once when the hold completes. All of it is opacity, so Reduce Motion needs +/// no variant. +struct BoxOverlay: View { + var keyframe: Keyframe + var boxes: [BoxState] + var time: Float + + static let signalBlue = Color(red: 0x1F / 255, green: 0x66 / 255, blue: 0xF2 / 255) + static let viewSize = SIMD2(390, 844) + + var body: some View { + Canvas(rendersAsynchronously: false) { context, _ in + let projection = ScreenProjection(keyframe: keyframe, viewSize: Self.viewSize) + for (box, state) in zip(RecognisedBox.fixture, boxes) { + guard let appeared = state.appearedAt, time >= appeared else { continue } + let points = box.corners.compactMap(projection.screenPoint) + guard points.count == 4 else { continue } + let corners = Self.outset(points.map { CGPoint(x: CGFloat($0.x), y: CGFloat($0.y)) }) + let shown = Double(CubicBezier.strongEaseOut((time - appeared) / 0.2)) + let settled = state.solidAt.map { Double(CubicBezier.strongEaseOut((time - $0) / 0.15)) } ?? 0 + let path = Self.bracket(corners) + if box.isMeter { + context.stroke(path, with: .color(Self.signalBlue.opacity(shown)), style: Self.solid) + let flash = flashStrength + if flash > 0 { + context.stroke(path, with: .color(.white.opacity(0.8 * flash)), style: StrokeStyle(lineWidth: 3, lineCap: .round, lineJoin: .round)) + } + } else { + context.stroke(path, with: .color(Palette.hologram.opacity(0.6 * (1 - settled) * shown)), style: Self.dashed) + context.stroke(path, with: .color(Palette.hologram.opacity(0.9 * settled * shown)), style: Self.solid) + } + drawCaption(box.caption, above: corners, opacity: shown, in: &context) + } + } + .accessibilityElement() + .accessibilityLabel(accessibilitySummary) + } + + static let solid = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round) + static let dashed = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round, dash: [5, 4]) + + /// The 150 ms brighten after the hold's ring completes, a sine bump. + private var flashStrength: Double { + let start = Schedule.start(of: Schedule.holdIndex) + Schedule.holdDuration + let p = (time - start) / Schedule.flashDuration + guard p > 0, p < 1 else { return 0 } + return Double(sin(p * Float.pi)) + } + + /// Moves each corner 6 pt away from the box's centre. On the edge itself, a light bracket + /// vanished against the window's white frame. + static func outset(_ corners: [CGPoint]) -> [CGPoint] { + let cx = corners.map(\.x).reduce(0, +) / 4, cy = corners.map(\.y).reduce(0, +) / 4 + return corners.map { p in + let dx = p.x - cx, dy = p.y - cy + let length = max((dx * dx + dy * dy).squareRoot(), 0.001) + return CGPoint(x: p.x + dx / length * 6, y: p.y + dy / length * 6) + } + } + + /// Four L-shaped corners, each arm 18 pt or 40% of its edge, with a 4 pt rounded bend. + static func bracket(_ corners: [CGPoint]) -> Path { + var path = Path() + for i in corners.indices { + let corner = corners[i], previous = corners[(i + 3) % 4], next = corners[(i + 1) % 4] + func toward(_ other: CGPoint) -> CGPoint { + let dx = other.x - corner.x, dy = other.y - corner.y + let length = max((dx * dx + dy * dy).squareRoot(), 0.001) + let arm = min(18, 0.4 * length) + return CGPoint(x: corner.x + dx / length * arm, y: corner.y + dy / length * arm) + } + path.move(to: toward(previous)) + path.addArc(tangent1End: corner, tangent2End: toward(next), radius: 4) + path.addLine(to: toward(next)) + } + return path + } + + private func drawCaption(_ caption: String, above corners: [CGPoint], opacity: Double, in context: inout GraphicsContext) { + let top = CGPoint(x: (corners[0].x + corners[1].x) / 2, y: min(corners[0].y, corners[1].y) - 13) + var layer = context + layer.opacity = opacity + layer.addFilter(.shadow(color: .black.opacity(0.4), radius: 6, y: 1)) + let text = Text(caption) + .font(.system(size: 13, weight: .semibold, design: .rounded)) + .foregroundStyle(.white) + layer.draw(text, at: top, anchor: .center) + } + + private var accessibilitySummary: String { + let names = zip(RecognisedBox.fixture, boxes).compactMap { box, state in + state.appearedAt.map { time >= $0 ? box.caption : nil } ?? nil + } + return names.isEmpty ? "Nothing recognised yet" : "Recognised: " + names.joined(separator: ", ") + } +} diff --git a/experiments/live-dots/Sources/LiveDots/Command.swift b/experiments/live-dots/Sources/LiveDots/Command.swift new file mode 100644 index 000000000..7cb03c185 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/Command.swift @@ -0,0 +1,100 @@ +import LiveDotsCore + +/// What the executable was asked to do. With no flags it opens the window. +enum Command { + case app(fixture: String?) + /// Every frame of the replay at 30 fps to PNG files in a folder, or raw BGRA to stdout for "-". + case export(destination: String, options: RenderOptions) + /// One settled keyframe (1-based, as in the fixture's ids) to a PNG file. + case still(keyframe: Int, at: Float?, output: String, options: RenderOptions) + case gradientReport(fixture: String?) + case benchmark(fixture: String?) + case fieldReport(fixture: String?) + + struct RenderOptions { + var fixture: String? + var mode: CaptureMode = .lidar + var reduceMotion = false + var fog: FogStyle = .on + var scheme: DotScheme = .hologram + } + + struct UsageError: Error, CustomStringConvertible { + let problem: String + var description: String { + """ + \(problem) + usage: LiveDots [--fixture ] + LiveDots --export --mode lidar|nolidar [--scheme hologram|constellation|ember] [--fog on|off|veil] [--reduce-motion] [--fixture ] + LiveDots --still [--at ] --out --mode lidar|nolidar [--scheme ...] [--fog on|off|veil] [--reduce-motion] [--fixture ] + LiveDots --gradient-report [--fixture ] + LiveDots --benchmark [--fixture ] + LiveDots --field-report [--fixture ] + """ + } + } + + static func parse(_ arguments: [String]) throws(UsageError) -> Command { + var options = RenderOptions() + var export: String?, still: Int?, output: String?, mode: String?, at: Float? + var report = false, benchmark = false, fieldReport = false + var rest = arguments[...] + func value(for flag: String) throws(UsageError) -> String { + guard let next = rest.popFirst() else { throw UsageError(problem: "\(flag) needs a value") } + return next + } + while let flag = rest.popFirst() { + switch flag { + case "--fixture": options.fixture = try value(for: flag) + case "--export": export = try value(for: flag) + case "--still": + let text = try value(for: flag) + guard let number = Int(text), number >= 1 else { throw UsageError(problem: "--still needs a keyframe number from 1, got \(text)") } + still = number + case "--out": output = try value(for: flag) + case "--at": + let text = try value(for: flag) + guard let seconds = Float(text), seconds >= 0 else { throw UsageError(problem: "--at needs seconds from 0, got \(text)") } + at = seconds + case "--mode": mode = try value(for: flag) + case "--scheme": + let name = try value(for: flag) + guard let scheme = DotScheme(rawValue: name) else { + throw UsageError(problem: "--scheme must be hologram, constellation or ember, got \(name)") + } + options.scheme = scheme + case "--reduce-motion": options.reduceMotion = true + case "--fog": + let name = try value(for: flag) + guard let fog = FogStyle(rawValue: name) else { throw UsageError(problem: "--fog must be on, off or veil, got \(name)") } + options.fog = fog + case "--gradient-report": report = true + case "--benchmark": benchmark = true + case "--field-report": fieldReport = true + // Xcode and LaunchServices pass these to GUI launches. + case let other where other.hasPrefix("-NS") || other.hasPrefix("-Apple"): _ = rest.popFirst() + default: throw UsageError(problem: "unknown argument \(flag)") + } + } + if let mode { + switch mode { + case "lidar": options.mode = .lidar + case "nolidar": options.mode = .noLidar + default: throw UsageError(problem: "--mode must be lidar or nolidar, got \(mode)") + } + } + if report { return .gradientReport(fixture: options.fixture) } + if benchmark { return .benchmark(fixture: options.fixture) } + if fieldReport { return .fieldReport(fixture: options.fixture) } + if let export { + guard mode != nil else { throw UsageError(problem: "--export needs --mode") } + return .export(destination: export, options: options) + } + if let still { + guard mode != nil else { throw UsageError(problem: "--still needs --mode") } + guard let output else { throw UsageError(problem: "--still needs --out ") } + return .still(keyframe: still, at: at, output: output, options: options) + } + return .app(fixture: options.fixture) + } +} diff --git a/experiments/live-dots/Sources/LiveDots/ContentView.swift b/experiments/live-dots/Sources/LiveDots/ContentView.swift new file mode 100644 index 000000000..be5b0d67b --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/ContentView.swift @@ -0,0 +1,79 @@ +import LiveDotsCore +import SwiftUI + +/// The phone on the left, the controls on the right. +struct ContentView: View { + let fixtureArgument: String? + + @State private var stage: Stage = .loading + @State private var progress = LoadProgress() + @Environment(\.accessibilityReduceMotion) private var systemReduceMotion + + enum Stage { + case loading + case failed(String) + case ready(DotRenderer, ReplayPlayer, FrameTimer) + } + + var body: some View { + HStack(alignment: .top, spacing: 32) { + PhoneFrame { + switch stage { + case .loading: + ProgressView(value: progress.fraction) { + Text("Fusing keyframes") + } + .progressViewStyle(.linear) + .tint(Palette.hologram) + .foregroundStyle(.white) + .padding(48) + case let .failed(message): + Text(message) + .font(.callout) + .foregroundStyle(.white) + .padding(32) + case let .ready(renderer, player, timer): + ZStack { + DotsMetalView(renderer: renderer, player: player, timer: timer) + .accessibilityHidden(true) + TimelineView(.animation) { _ in + let data = renderer.data, request = player.request + PhoneChrome( + state: data.timeline(request.mode).states[request.keyframe], + keyframe: data.replay.keyframes[request.keyframe], + boxes: data.boxes[request.keyframe], time: request.time) + } + } + } + } + Group { + if case let .ready(renderer, player, timer) = stage { + ControlsPanel(player: player, data: renderer.data, timer: timer) + } else { + Color.clear + } + } + .frame(width: 280) + } + .padding(28) + .background(Palette.window) + .preferredColorScheme(.dark) + .task { await load() } + } + + private func load() async { + let progress = progress + do { + let folder = try FixtureLocator.folder(fixtureArgument) + let data = try await ReplayData.load(folder: folder) { done, total in + Task { @MainActor in progress.fraction = Double(done) / Double(total) } + } + let renderer = try DotRenderer(data: data) + let player = ReplayPlayer(keyframeCount: data.keyframeCount, reduceMotion: systemReduceMotion) + stage = .ready(renderer, player, FrameTimer()) + player.togglePlayback() + } catch { + stage = .failed(String(describing: error)) + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/ControlsPanel.swift b/experiments/live-dots/Sources/LiveDots/ControlsPanel.swift new file mode 100644 index 000000000..5eb59f322 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/ControlsPanel.swift @@ -0,0 +1,153 @@ +import LiveDotsCore +import SwiftUI + +/// Playback and comparison controls, outside the phone so nothing but the product sits on it. +struct ControlsPanel: View { + @Bindable var player: ReplayPlayer + let data: ReplayData + let timer: FrameTimer + + var body: some View { + VStack(alignment: .leading, spacing: 22) { + VStack(alignment: .leading, spacing: 4) { + Text("Live dots") + .font(.title2.weight(.semibold)) + Text("Synthetic wall fixture, \(data.keyframeCount) keyframes played at 4 per second.") + .font(.callout) + .foregroundStyle(.secondary) + } + + VStack(alignment: .leading, spacing: 10) { + HStack { + Button(player.isPlaying ? "Pause" : "Play", systemImage: player.isPlaying ? "pause.fill" : "play.fill") { + player.togglePlayback() + } + .keyboardShortcut(.space, modifiers: []) + .controlSize(.large) + Spacer() + Text("Keyframe \(player.keyframe + 1) of \(data.keyframeCount)") + .monospacedDigit() + .foregroundStyle(.secondary) + } + Slider(value: keyframeBinding, in: 0...Double(data.keyframeCount - 1), step: 1) { + Text("Keyframe") + } + .labelsHidden() + } + + VStack(alignment: .leading, spacing: 8) { + Picker("Depth", selection: $player.mode) { + Text("LiDAR").tag(CaptureMode.lidar) + Text("No LiDAR").tag(CaptureMode.noLidar) + } + .pickerStyle(.segmented) + .labelsHidden() + if player.mode == .noLidar { + Text("Simulated. The feature points and wall plane come from the fixture's depth, which a phone without LiDAR does not have.") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + + VStack(alignment: .leading, spacing: 8) { + Picker("Look", selection: $player.scheme) { + Text("Hologram").tag(DotScheme.hologram) + Text("Constellation").tag(DotScheme.constellation) + Text("Ember").tag(DotScheme.ember) + } + .pickerStyle(.segmented) + .labelsHidden() + Text(schemeCaption) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + VStack(alignment: .leading, spacing: 10) { + Toggle("Reduce Motion", isOn: $player.reduceMotion) + Picker("Fog", selection: $player.fog) { + Text("Fog").tag(FogStyle.on) + Text("Flat veil").tag(FogStyle.veil) + Text("Off").tag(FogStyle.off) + } + .pickerStyle(.segmented) + Text("Fog covers everything not yet measured and lifts where dots arrive. Flat veil is the old approach, 35% over wall cells no dot has reached.") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + .toggleStyle(.switch) + + Divider() + + stats + legend + Spacer(minLength: 0) + } + .frame(maxHeight: 844) + } + + private var schemeCaption: String { + switch player.scheme { + case .hologram: "Edge and surface dots; opacity rises with each new view." + case .constellation: "Edges only, linked into outlines. The wall's surface shows nothing." + case .ember: "New dots are born amber and cool to white over 6 s, so the newest part of the map glows." + } + } + + private var keyframeBinding: Binding { + Binding(get: { Double(player.keyframe) }, set: { player.scrub(to: Int($0.rounded())) }) + } + + private var stats: some View { + let state = data.timeline(player.mode).states[player.keyframe] + return TimelineView(.periodic(from: .now, by: 0.5)) { _ in + let averages = timer.averages + Grid(alignment: .leading, horizontalSpacing: 16, verticalSpacing: 6) { + GridRow { + Text("Dots in field") + Text(state.fieldCount, format: .number) + } + GridRow { + Text("Edge dots") + Text(state.edgeCount, format: .number) + } + GridRow { + Text("Coverage") + Text(Double(state.coverage), format: .percent.precision(.fractionLength(0))) + } + GridRow { + Text("Frame") + Text("GPU \(averages.gpuMilliseconds, format: .number.precision(.fractionLength(2))) ms, CPU \(averages.cpuMilliseconds, format: .number.precision(.fractionLength(2))) ms") + } + } + .font(.callout.monospacedDigit()) + .foregroundStyle(.secondary) + } + } + + private var legend: some View { + VStack(alignment: .leading, spacing: 6) { + Label { + Text("Surface and edges") + } icon: { + swatch(core: Palette.hologram, glow: Palette.glow) + } + Label { + Text("In front of the wall") + } icon: { + swatch(core: Palette.hiddenViolet, glow: Palette.hiddenViolet) + } + } + .font(.callout) + .foregroundStyle(.secondary) + } + + private func swatch(core: Color, glow: Color) -> some View { + Circle() + .fill(core) + .frame(width: 6, height: 6) + .background(Circle().fill(glow.opacity(0.45)).frame(width: 14, height: 14)) + } +} diff --git a/experiments/live-dots/Sources/LiveDots/DotRenderer.swift b/experiments/live-dots/Sources/LiveDots/DotRenderer.swift new file mode 100644 index 000000000..bb7ca2926 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/DotRenderer.swift @@ -0,0 +1,274 @@ +import LiveDotsCore +import Metal +import MetalKit +import simd + +/// What one frame shows: which keyframe, when on the playback clock, and the toggles. +struct FrameRequest: Equatable { + var mode: CaptureMode + var keyframe: Int + /// Playback seconds; animations are evaluated at this time. + var time: Float + var reduceMotion: Bool + var fog: FogStyle + var scheme: DotScheme = .hologram + /// Seconds on a clock that keeps running while playback is paused, for the fog's drift. + var ambientTime: Float = 0 + /// Seconds since the previous frame, for the fog's lag. + var frameInterval: Float = 1 / 60 + + /// A scrub or a still shows its keyframe settled; the renderer passes a far-future time. + var isSettled: Bool { time > 1e4 } +} + +/// Draws a keyframe: the camera image, the optional fog veil, then the dots as additive point +/// sprites from one vertex buffer. The buffer is rebuilt only when the keyframe or mode changes; +/// between keyframes the shader animates births and evidence from the playback time. +final class DotRenderer { + enum SetupError: Error, CustomStringConvertible { + case noMetalDevice + case noCommandQueue + case buffer(String) + + var description: String { + switch self { + case .noMetalDevice: "no Metal device on this Mac" + case .noCommandQueue: "Metal would not make a command queue" + case let .buffer(what): "Metal would not allocate the \(what) buffer" + } + } + } + + static let pixelFormat = MTLPixelFormat.bgra8Unorm + + let device: MTLDevice + let queue: MTLCommandQueue + let data: ReplayData + private let cameraPipeline: MTLRenderPipelineState + private let veilPipeline: MTLRenderPipelineState + private let dotPipeline: MTLRenderPipelineState + private let linkPipeline: MTLRenderPipelineState + private let cameraImages: [MTLTexture] + private let fog: FogLayer + private var splatBuffer: MTLBuffer? + private var splatCount = 0 + + private var spriteBuffer: MTLBuffer? + private var spriteCount = 0 + private var veilBuffer: MTLBuffer? + private var veilVertexCount = 0 + private var linkBuffer: MTLBuffer? + private var linkVertexCount = 0 + private var cachedKey: (mode: CaptureMode, keyframe: Int, scheme: DotScheme)? + /// Replaces the timeline's sprites, for the 6,000-dot benchmark. + var spriteOverride: [DotSprite]? { + didSet { cachedKey = nil } + } + + init(data: ReplayData, device: MTLDevice? = MTLCreateSystemDefaultDevice()) throws { + guard let device else { throw SetupError.noMetalDevice } + guard let queue = device.makeCommandQueue() else { throw SetupError.noCommandQueue } + self.device = device + self.queue = queue + self.data = data + + let library = try device.makeLibrary(source: Shaders.source, options: nil) + func pipeline(_ vertex: String, _ fragment: String, blend: (MTLRenderPipelineColorAttachmentDescriptor) -> Void) throws -> MTLRenderPipelineState { + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: vertex) + descriptor.fragmentFunction = library.makeFunction(name: fragment) + descriptor.colorAttachments[0].pixelFormat = Self.pixelFormat + blend(descriptor.colorAttachments[0]) + return try device.makeRenderPipelineState(descriptor: descriptor) + } + cameraPipeline = try pipeline("cameraVertex", "cameraFragment") { _ in } + veilPipeline = try pipeline("veilVertex", "veilFragment") { a in + a.isBlendingEnabled = true + a.sourceRGBBlendFactor = .sourceAlpha + a.destinationRGBBlendFactor = .oneMinusSourceAlpha + a.sourceAlphaBlendFactor = .zero + a.destinationAlphaBlendFactor = .one + } + // Additive: the dots are light on the camera image, never paint over it. + let additive: (MTLRenderPipelineColorAttachmentDescriptor) -> Void = { a in + a.isBlendingEnabled = true + a.sourceRGBBlendFactor = .one + a.destinationRGBBlendFactor = .one + a.sourceAlphaBlendFactor = .zero + a.destinationAlphaBlendFactor = .one + } + dotPipeline = try pipeline("dotVertex", "dotFragment", blend: additive) + linkPipeline = try pipeline("linkVertex", "linkFragment", blend: additive) + + fog = try FogLayer(device: device, library: library, replay: data.replay) + + let loader = MTKTextureLoader(device: device) + cameraImages = try data.replay.keyframes.map { keyframe in + try loader.newTexture( + URL: data.replay.folder.appendingPathComponent(keyframe.imagePath), + options: [.SRGB: false, .textureStorageMode: MTLStorageMode.private.rawValue]) + } + } + + /// Encodes one frame into `pass`, whose target is `pixelSize` pixels at `pointScale` pixels + /// per point. + func encode(_ request: FrameRequest, into commandBuffer: MTLCommandBuffer, pass: MTLRenderPassDescriptor, + pixelSize: SIMD2, pointScale: Float) throws { + let state = data.timeline(request.mode).states[request.keyframe] + let keyframe = data.replay.keyframes[request.keyframe] + try prepareBuffers(request.mode, state, scheme: request.scheme) + let projection = ScreenProjection(keyframe: keyframe, viewSize: pixelSize) + + var clip = projection.clipMatrix + if request.fog == .on { + fog.encodeMask( + into: commandBuffer, splats: splatBuffer, splatCount: splatCount, clip: clip, + pixelsPerMetre: keyframe.intrinsics.x * projection.scale * Float(FogLayer.maskWidth) / pixelSize.x, + time: request.time, dt: request.frameInterval, reduceMotion: request.reduceMotion, reset: request.isSettled) + } + + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) else { return } + defer { encoder.endEncoding() } + + var camera = CameraUniforms( + offset: projection.offset, scale: projection.scale, pad: 0, + imageSize: SIMD2(Float(keyframe.width), Float(keyframe.height))) + encoder.setRenderPipelineState(cameraPipeline) + encoder.setFragmentBytes(&camera, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(cameraImages[request.keyframe], index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + + if request.fog == .on { + fog.encodeComposite(into: encoder, uniforms: FogUniforms( + viewSize: pixelSize, offset: projection.offset, scale: projection.scale, time: request.ambientTime, + imageSize: camera.imageSize, motion: request.reduceMotion ? 0 : 1, pad0: 0, pad1: .zero), + keyframe: request.keyframe) + } + if request.fog == .veil, let veilBuffer, veilVertexCount > 0 { + encoder.setRenderPipelineState(veilPipeline) + encoder.setVertexBuffer(veilBuffer, offset: 0, index: 0) + encoder.setVertexBytes(&clip, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: veilVertexCount) + } + + var dots = DotUniforms( + clip: clip, cameraAndTime: SIMD4(keyframe.cameraPosition, request.time), + pointScale: pointScale, reduceMotion: request.reduceMotion ? 1 : 0, + scheme: Float(DotScheme.allCases.firstIndex(of: request.scheme) ?? 0), pad: 0) + if let linkBuffer, linkVertexCount > 0 { + encoder.setRenderPipelineState(linkPipeline) + encoder.setVertexBuffer(linkBuffer, offset: 0, index: 0) + encoder.setVertexBytes(&dots, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .line, vertexStart: 0, vertexCount: linkVertexCount) + } + if let spriteBuffer, spriteCount > 0 { + encoder.setRenderPipelineState(dotPipeline) + encoder.setVertexBuffer(spriteBuffer, offset: 0, index: 0) + encoder.setVertexBytes(&dots, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: spriteCount) + } + } + + private func prepareBuffers(_ mode: CaptureMode, _ state: KeyframeState, scheme: DotScheme) throws { + if let cachedKey, cachedKey.mode == mode, cachedKey.keyframe == state.index, cachedKey.scheme == scheme { return } + cachedKey = (mode, state.index, scheme) + + let all = spriteOverride ?? state.sprites + // Every live dot clears fog, whatever the scheme draws. + let splats = all.map(SpriteVertex.init) + splatCount = splats.count + splatBuffer = splats.isEmpty ? nil : try makeBuffer(splats, "splat") + + let sprites = all.filter { scheme.draws($0.kind) } + let vertices = Self.vertices(for: sprites) + spriteCount = vertices.count + spriteBuffer = vertices.isEmpty ? nil : try makeBuffer(vertices, "dot") + + let links = scheme == .constellation ? Self.linkVertices(for: sprites) : [] + linkVertexCount = links.count + linkBuffer = links.isEmpty ? nil : try makeBuffer(links, "link") + + var veil: [SIMD4] = [] + for cell in state.unseenCells { + let (x0, y0, s) = (cell.minX, cell.minY, WallCell.size) + let a = SIMD4(x0, y0, 0, 1), b = SIMD4(x0 + s, y0, 0, 1) + let c = SIMD4(x0 + s, y0 + s, 0, 1), d = SIMD4(x0, y0 + s, 0, 1) + veil += [a, b, c, a, c, d] + } + veilVertexCount = veil.count + veilBuffer = veil.isEmpty ? nil : try makeBuffer(veil, "veil") + } + + private func makeBuffer(_ items: [T], _ what: String) throws -> MTLBuffer { + let buffer = items.withUnsafeBytes { raw in + raw.baseAddress.flatMap { device.makeBuffer(bytes: $0, length: raw.count, options: .storageModeShared) } + } + guard let buffer else { throw SetupError.buffer(what) } + return buffer + } + + /// Two vertices per dot: its halo (size and opacity set in the shader by kind), then the + /// core. Halos come first so the sharp cores sit on top of them. + static func vertices(for sprites: [DotSprite]) -> [SpriteVertex] { + let cores = sprites.map(SpriteVertex.init) + return cores.map(\.asHalo) + cores + } +} + +extension DotRenderer { + /// Two vertices per link. Both carry the link's own timing: it is born with the later of its + /// two dots (counting a dot as born when it became an edge), dies with the earlier, and takes + /// the lower opacity. + static func linkVertices(for sprites: [DotSprite]) -> [SpriteVertex] { + EdgeLinks.links(among: sprites).flatMap { pair -> [SpriteVertex] in + let a = sprites[Int(pair.x)], b = sprites[Int(pair.y)] + let timing = DotSprite( + id: 0, position: .zero, kind: .edge, onOccluder: a.onOccluder && b.onOccluder, + birthTime: max(max(a.birthTime, a.edgeSince), max(b.birthTime, b.edgeSince)), + fromOpacity: min(a.fromOpacity, b.fromOpacity), toOpacity: min(a.toOpacity, b.toOpacity), + opacityTime: max(a.opacityTime, b.opacityTime), edgeSince: -.infinity, + deathTime: min(a.deathTime, b.deathTime)) + var start = SpriteVertex(timing), end = start + start.a = SIMD4(a.position, 0) + end.a = SIMD4(b.position, 0) + return [start, end] + } + } +} + +/// Matches `Sprite` in the shader. Infinite times are clamped to plus or minus 1e6 s, because +/// Metal compiles with fast math, which assumes no infinities. +struct SpriteVertex { + var a: SIMD4 + var b: SIMD4 + var c: SIMD4 + + init(_ s: DotSprite) { + func finite(_ t: Float) -> Float { min(max(t, -1e6), 1e6) } + a = SIMD4(s.position, s.kind == .feature ? 1 : 0) + b = SIMD4(finite(s.birthTime), s.fromOpacity, s.toOpacity, finite(s.opacityTime)) + c = SIMD4(finite(s.edgeSince), finite(s.deathTime), s.onOccluder ? 1 : 0, 0) + } + + var asHalo: SpriteVertex { + var halo = self + halo.c.w = 1 + return halo + } +} + +struct CameraUniforms { + var offset: SIMD2 + var scale: Float + var pad: Float + var imageSize: SIMD2 +} + +struct DotUniforms { + var clip: simd_float4x4 + var cameraAndTime: SIMD4 + var pointScale: Float + var reduceMotion: Float + var scheme: Float + var pad: Float +} diff --git a/experiments/live-dots/Sources/LiveDots/DotsMetalView.swift b/experiments/live-dots/Sources/LiveDots/DotsMetalView.swift new file mode 100644 index 000000000..eb417528c --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/DotsMetalView.swift @@ -0,0 +1,65 @@ +import AppKit +import MetalKit +import SwiftUI + +/// The camera image and dots, drawn by `DotRenderer` into an MTKView at the display's rate. +struct DotsMetalView: NSViewRepresentable { + let renderer: DotRenderer + let player: ReplayPlayer + let timer: FrameTimer + + func makeCoordinator() -> Coordinator { + Coordinator(renderer: renderer, player: player, timer: timer) + } + + func makeNSView(context: Context) -> MTKView { + let view = MTKView(frame: .zero, device: renderer.device) + view.colorPixelFormat = DotRenderer.pixelFormat + view.clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1) + view.preferredFramesPerSecond = 60 + view.delegate = context.coordinator + view.setAccessibilityElement(false) + return view + } + + func updateNSView(_ view: MTKView, context: Context) {} + + final class Coordinator: NSObject, MTKViewDelegate { + let renderer: DotRenderer + let player: ReplayPlayer + let timer: FrameTimer + + init(renderer: DotRenderer, player: ReplayPlayer, timer: FrameTimer) { + self.renderer = renderer + self.player = player + self.timer = timer + } + + func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {} + + func draw(in view: MTKView) { + player.tick(now: CACurrentMediaTime()) + guard let pass = view.currentRenderPassDescriptor, let drawable = view.currentDrawable, + let commandBuffer = renderer.queue.makeCommandBuffer() + else { return } + let start = CACurrentMediaTime() + let request = player.request + do { + try renderer.encode( + request, into: commandBuffer, pass: pass, + pixelSize: SIMD2(Float(view.drawableSize.width), Float(view.drawableSize.height)), + pointScale: Float(view.window?.backingScaleFactor ?? 2)) + } catch { + assertionFailure("frame encode failed: \(error)") + } + commandBuffer.present(drawable) + let cpu = CACurrentMediaTime() - start + let sprites = renderer.data.timeline(request.mode).states[request.keyframe].sprites.count + let timer = timer + commandBuffer.addCompletedHandler { buffer in + timer.record(cpuSeconds: cpu, gpuSeconds: buffer.gpuEndTime - buffer.gpuStartTime, sprites: sprites) + } + commandBuffer.commit() + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/Exporter.swift b/experiments/live-dots/Sources/LiveDots/Exporter.swift new file mode 100644 index 000000000..2e50429d8 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/Exporter.swift @@ -0,0 +1,111 @@ +import CoreGraphics +import Foundation +import ImageIO +import LiveDotsCore +import SwiftUI +import UniformTypeIdentifiers + +/// Renders the replay offscreen at 1170 x 2532 with the same renderer and chrome as the app. +enum Exporter { + static let framesPerSecond: Float = 30 + + struct Failure: Error, CustomStringConvertible { + let description: String + } + + /// Every frame from keyframe 1 until the last keyframe's births finish. `destination` is a + /// folder for frame_00000.png onward, or "-" for raw BGRA frames on stdout (what + /// make-video.sh pipes into ffmpeg, so no frame folder ever touches the disk). + static func exportReplay(to destination: String, options: Command.RenderOptions) throws { + let data = try ReplayData.loadNow(folder: FixtureLocator.folder(options.fixture)) + let renderer = try DotRenderer(data: data) + let target = try OffscreenTarget(renderer: renderer) + let seconds = Schedule.end(count: data.keyframeCount) + Tuning.birthDuration + 0.15 + let frameCount = Int((seconds * framesPerSecond).rounded(.up)) + let toStdout = destination == "-" + let folder = URL(fileURLWithPath: destination, isDirectory: true) + if !toStdout { try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) } + + for frame in 0.. FrameRequest { + FrameRequest( + mode: options.mode, keyframe: Schedule.keyframe(at: time, count: data.keyframeCount), time: time, + reduceMotion: options.reduceMotion, fog: options.fog, scheme: options.scheme, + ambientTime: time, frameInterval: 1 / framesPerSecond) + } + + /// One keyframe as a PNG. Without `at`, the keyframe settled: every animation finished and + /// the fog lag caught up. With `at`, the moment `at` seconds after the keyframe appears, + /// reached by playing the replay from the start at 30 fps, so the fog's lag is what the + /// video shows. + static func exportStill(keyframe number: Int, at offset: Float?, to path: String, options: Command.RenderOptions) throws { + let data = try ReplayData.loadNow(folder: FixtureLocator.folder(options.fixture)) + guard number <= data.keyframeCount else { + throw Failure(description: "--still \(number) is past the last keyframe, \(data.keyframeCount)") + } + let renderer = try DotRenderer(data: data) + let target = try OffscreenTarget(renderer: renderer) + let state = data.timeline(options.mode).states[number - 1] + let request: FrameRequest + if let offset { + let end = Schedule.start(of: number - 1) + offset + var frame = 0 + while Float(frame + 1) / framesPerSecond < end { + try target.render(playbackRequest(at: Float(frame) / framesPerSecond, data: data, options: options), readBack: false) + frame += 1 + } + request = playbackRequest(at: end, data: data, options: options) + } else { + request = FrameRequest( + mode: options.mode, keyframe: number - 1, time: 1e5, + reduceMotion: options.reduceMotion, fog: options.fog, scheme: options.scheme) + } + try target.render(request) + try composite(chrome(for: request, data: data), onto: target) + try writePNG(target, to: URL(fileURLWithPath: path)) + FileHandle.standardError.write(Data( + "\(path): keyframe \(number), \(options.mode.rawValue), \(options.scheme.rawValue), \(state.sprites.count) sprites drawn, \(state.fieldCount) dots in field (\(state.edgeCount) edges), coverage \(Int((state.coverage * 100).rounded()))%\n".utf8)) + } + + private static func composite(_ chrome: CGImage?, onto target: OffscreenTarget) throws { + guard let chrome else { return } + let context = try target.context() + context.draw(chrome, in: CGRect(x: 0, y: 0, width: target.width, height: target.height)) + } + + private static func writePNG(_ target: OffscreenTarget, to url: URL) throws { + guard let image = try target.context().makeImage(), + let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil) + else { throw Failure(description: "can't write \(url.path)") } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { throw Failure(description: "can't write \(url.path)") } + } + + /// The SwiftUI chrome for one frame. Rendered every frame: the boxes fade and the hold ring + /// fills between keyframes. + private static func chrome(for request: FrameRequest, data: ReplayData) -> CGImage? { + let renderer = ImageRenderer(content: PhoneChrome( + state: data.timeline(request.mode).states[request.keyframe], + keyframe: data.replay.keyframes[request.keyframe], + boxes: data.boxes[request.keyframe], time: request.time) + .frame(width: CGFloat(OffscreenTarget.pointSize.x), height: CGFloat(OffscreenTarget.pointSize.y))) + renderer.scale = CGFloat(OffscreenTarget.scale) + return renderer.cgImage + } +} diff --git a/experiments/live-dots/Sources/LiveDots/FieldReport.swift b/experiments/live-dots/Sources/LiveDots/FieldReport.swift new file mode 100644 index 000000000..e22b7d3ed --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/FieldReport.swift @@ -0,0 +1,25 @@ +import Foundation +import LiveDotsCore + +/// Per-keyframe counts for both modes: field size, edges (and how many are still under the +/// oblique cap), dots drawn in view, coverage, unseen fog cells, and simulated feature +/// candidates before thinning. +enum FieldReport { + static func run(fixture: String?) throws { + let data = try ReplayData.loadNow(folder: FixtureLocator.folder(fixture)) + var field = VoxelField() + for i in data.lidar.states.indices { + let l = data.lidar.states[i], n = data.noLidar.states[i] + let features = n.sprites.count { $0.kind == .feature && $0.deathTime == .infinity } + let planes = n.sprites.count { $0.kind == .plane } + let planeRows = Set(n.sprites.filter { $0.kind == .plane }.map { Int(($0.position.y / Tuning.planeCell).rounded(.down)) }).sorted() + let planeColumns = Set(n.sprites.filter { $0.kind == .plane }.map { Int(($0.position.x / Tuning.planeCell).rounded(.down)) }).sorted() + let keyframe = data.replay.keyframes[i] + let frame = FrameInput(index: i, keyframe: keyframe, depth: try data.replay.depth(for: keyframe), gradient: GradientPyramid(image: try data.replay.image(for: keyframe))) + let candidates = FeatureField.candidates(frame, threshold: Tuning.featureGradientThreshold).count + field.integrate(frame) + let oblique = field.dots().count { $0.kind == .edge && !$0.faceOn } + print("k\(i + 1) x=\(String(format: "%.1f", data.replay.keyframes[i].cameraPosition.x)) lidar field \(l.fieldCount) edges \(l.edgeCount) (\(oblique) not yet face-on) drawn \(l.sprites.count) cov \(Int(l.coverage * 100))% unseen \(l.unseenCells.count) | nolidar field \(n.fieldCount) candidates \(candidates) features drawn \(features) planes drawn \(planes) over rows \(planeRows.first ?? 0)...\(planeRows.last ?? 0) columns \(planeColumns.first ?? 0)...\(planeColumns.last ?? 0) cov \(Int(n.coverage * 100))%") + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/FogLayer.swift b/experiments/live-dots/Sources/LiveDots/FogLayer.swift new file mode 100644 index 000000000..974a49111 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/FogLayer.swift @@ -0,0 +1,171 @@ +import LiveDotsCore +import Metal +import simd + +/// How the unmeasured world is shown. +enum FogStyle: String, CaseIterable { + /// The fog layer: one continuous haze that dissolves where dots arrive. + case on + /// The old comparison: a flat 35% veil over 30 cm wall cells no dot has reached. + case veil + case off +} + +/// The fog layer's GPU state: the 96 x 208 reveal mask, its blur, the lagged copy that carries +/// from frame to frame, and each keyframe's depth for the depth term. The shaders and what they +/// borrow are described at the fog section of Shaders.swift. +final class FogLayer { + static let maskWidth = 96, maskHeight = 208 + + private let splatPipeline: MTLRenderPipelineState + private let blurPipeline: MTLRenderPipelineState + private let lagPipeline: MTLRenderPipelineState + private let compositePipeline: MTLRenderPipelineState + private let accumulation: MTLTexture + private let blurred: MTLTexture + private let reveal: MTLTexture + private let lag: [MTLTexture] + private var current = 0 + private var primed = false + private let depths: [MTLTexture] + + init(device: MTLDevice, library: MTLLibrary, replay: Replay) throws { + func pipeline(_ vertex: String, _ fragment: String, format: MTLPixelFormat, additive: Bool = false, + premultiplied: Bool = false) throws -> MTLRenderPipelineState { + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: vertex) + descriptor.fragmentFunction = library.makeFunction(name: fragment) + guard let a = descriptor.colorAttachments[0] else { throw DotRenderer.SetupError.buffer("fog pipeline") } + a.pixelFormat = format + if additive || premultiplied { + a.isBlendingEnabled = true + a.sourceRGBBlendFactor = .one + a.destinationRGBBlendFactor = additive ? .one : .oneMinusSourceAlpha + a.sourceAlphaBlendFactor = .zero + a.destinationAlphaBlendFactor = .one + } + return try device.makeRenderPipelineState(descriptor: descriptor) + } + // Half floats, not the 8 bits the design named: splats add up past 1 before the blur, + // and an 8-bit target would clamp each one as it lands. + let maskFormat = MTLPixelFormat.r16Float + splatPipeline = try pipeline("splatVertex", "splatFragment", format: maskFormat, additive: true) + blurPipeline = try pipeline("cameraVertex", "blurFragment", format: maskFormat) + lagPipeline = try pipeline("cameraVertex", "lagFragment", format: maskFormat) + compositePipeline = try pipeline("cameraVertex", "fogFragment", format: DotRenderer.pixelFormat, premultiplied: true) + + func mask() throws -> MTLTexture { + let d = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: maskFormat, width: Self.maskWidth, height: Self.maskHeight, mipmapped: false) + d.usage = [.renderTarget, .shaderRead] + d.storageMode = .private + guard let texture = device.makeTexture(descriptor: d) else { throw DotRenderer.SetupError.buffer("fog mask") } + return texture + } + accumulation = try mask() + blurred = try mask() + reveal = try mask() + lag = [try mask(), try mask()] + + depths = try replay.keyframes.map { keyframe in + let map = try replay.depth(for: keyframe) + let d = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: .r32Float, width: map.width, height: map.height, mipmapped: false) + d.usage = [.shaderRead] + d.storageMode = .shared + guard let texture = device.makeTexture(descriptor: d) else { throw DotRenderer.SetupError.buffer("depth") } + map.meters.withUnsafeBytes { raw in + if let base = raw.baseAddress { + texture.replace( + region: MTLRegionMake2D(0, 0, map.width, map.height), mipmapLevel: 0, + withBytes: base, bytesPerRow: map.width * 4) + } + } + return texture + } + } + + /// Splats, blurs and eases the reveal mask. `reset` snaps the lag to the new mask (a scrub, a + /// still, the first frame). + func encodeMask( + into commandBuffer: MTLCommandBuffer, splats: MTLBuffer?, splatCount: Int, clip: simd_float4x4, + pixelsPerMetre: Float, time: Float, dt: Float, reduceMotion: Bool, reset: Bool + ) { + func pass(_ target: MTLTexture, clear: Bool, _ body: (MTLRenderCommandEncoder) -> Void) { + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = target + descriptor.colorAttachments[0].loadAction = clear ? .clear : .dontCare + descriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) + descriptor.colorAttachments[0].storeAction = .store + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { return } + body(encoder) + encoder.endEncoding() + } + pass(accumulation, clear: true) { encoder in + guard let splats, splatCount > 0 else { return } + var uniforms = SplatUniforms(clip: clip, pixelsPerMetre: pixelsPerMetre, time: time, pad: .zero) + encoder.setRenderPipelineState(splatPipeline) + encoder.setVertexBuffer(splats, offset: 0, index: 0) + encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: splatCount) + } + for (source, target, direction, finish) in [ + (accumulation, blurred, SIMD2(1, 0), Int32(0)), (blurred, reveal, SIMD2(0, 1), Int32(1)), + ] { + pass(target, clear: false) { encoder in + var direction = direction, finish = finish + encoder.setRenderPipelineState(blurPipeline) + encoder.setFragmentTexture(source, index: 0) + encoder.setFragmentBytes(&direction, length: MemoryLayout>.stride, index: 0) + encoder.setFragmentBytes(&finish, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + } + } + let previous = lag[current] + current = 1 - current + var uniforms = LagUniforms(dt: dt, reduceMotion: reduceMotion ? 1 : 0, reset: reset || !primed ? 1 : 0, pad: 0) + primed = true + pass(lag[current], clear: false) { encoder in + encoder.setRenderPipelineState(lagPipeline) + encoder.setFragmentTexture(reveal, index: 0) + encoder.setFragmentTexture(previous, index: 1) + encoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + } + } + + /// Draws the fog over the camera image, before the dots. + func encodeComposite(into encoder: MTLRenderCommandEncoder, uniforms: FogUniforms, keyframe: Int) { + var uniforms = uniforms + encoder.setRenderPipelineState(compositePipeline) + encoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(lag[current], index: 0) + encoder.setFragmentTexture(depths[keyframe], index: 1) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + } +} + +struct SplatUniforms { + var clip: simd_float4x4 + var pixelsPerMetre: Float + var time: Float + var pad: SIMD2 +} + +struct LagUniforms { + var dt: Float + var reduceMotion: Int32 + var reset: Int32 + var pad: Int32 +} + +struct FogUniforms { + var viewSize: SIMD2 + var offset: SIMD2 + var scale: Float + var time: Float + var imageSize: SIMD2 + var motion: Float + var pad0: Float + var pad1: SIMD2 +} diff --git a/experiments/live-dots/Sources/LiveDots/FrameTimer.swift b/experiments/live-dots/Sources/LiveDots/FrameTimer.swift new file mode 100644 index 000000000..905794844 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/FrameTimer.swift @@ -0,0 +1,23 @@ +import Synchronization + +/// Running averages of the live view's frame cost. Command buffers report GPU time on a Metal +/// thread, so the numbers sit behind a lock. +nonisolated final class FrameTimer: Sendable { + struct Averages { + var cpuMilliseconds: Double = 0 + var gpuMilliseconds: Double = 0 + var sprites = 0 + } + + private let state = Mutex(Averages()) + + func record(cpuSeconds: Double, gpuSeconds: Double, sprites: Int) { + state.withLock { + $0.cpuMilliseconds = $0.cpuMilliseconds * 0.95 + cpuSeconds * 1000 * 0.05 + $0.gpuMilliseconds = $0.gpuMilliseconds * 0.95 + gpuSeconds * 1000 * 0.05 + $0.sprites = sprites + } + } + + var averages: Averages { state.withLock { $0 } } +} diff --git a/experiments/live-dots/Sources/LiveDots/GradientReport.swift b/experiments/live-dots/Sources/LiveDots/GradientReport.swift new file mode 100644 index 000000000..6a3dbf5ab --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/GradientReport.swift @@ -0,0 +1,89 @@ +import Foundation +import LiveDotsCore +import simd + +/// Evidence for `Tuning.gradientThreshold`. Wall voxels of the synthetic fixture are split by +/// what the scene generator painted there: on an outline of the meter, gas meter, door or window +/// (within 2.5 cm), or plain brick at least 10 cm from any outline. For each, the per-voxel +/// gradient is computed two ways, a Sobel on the full-size JPEG and the voxel-scale pyramid the +/// field uses, and the report prints percentiles and the share above each candidate threshold. +enum GradientReport { + /// Painted rectangles from ios/Tools/make-synthetic-replay.swift on t3/ios-mvf, as + /// (minX, minY, maxX, maxY). The door's bottom is the ground, so it is left out. + static let outlines: [SIMD4] = [ + SIMD4(-0.15, 1.3, 0.15, 1.7), SIMD4(-1.35, 0.3, -1.05, 0.8), + SIMD4(-3.4, 0, -2.5, 2.05), SIMD4(2.0, 0.9, 3.0, 2.0), + ] + + static func distanceToOutline(_ x: Float, _ y: Float) -> Float { + var best = Float.infinity + for r in outlines { + let insideX = x >= r.x && x <= r.z, insideY = y >= r.y && y <= r.w + if insideY { best = min(best, abs(x - r.x), abs(x - r.z)) } + if insideX { best = min(best, abs(y - r.w)) } + if insideX, r.y > 0 { best = min(best, abs(y - r.y)) } + if !insideX, !insideY { + let dx = min(abs(x - r.x), abs(x - r.z)), dy = min(abs(y - r.y), abs(y - r.w)) + best = min(best, (dx * dx + dy * dy).squareRoot()) + } + } + return best + } + + static func insidePainted(_ x: Float, _ y: Float) -> Bool { + outlines.contains { x > $0.x && x < $0.z && y > $0.y && y < $0.w } + } + + static func run(fixture: String?) throws { + let replay = try Replay.load(folder: FixtureLocator.folder(fixture)) + // key -> (max over keyframes of the keyframe mean) for both methods + var full: [VoxelKey: Float] = [:], scaled: [VoxelKey: Float] = [:] + let size = Tuning.voxelSize + for keyframe in replay.keyframes { + let depth = try replay.depth(for: keyframe) + let pyramid = GradientPyramid(image: try replay.image(for: keyframe)) + let scale = Float(keyframe.width) / Float(depth.width) + var sums: [VoxelKey: SIMD3] = [:] + for j in 0..= Tuning.minimumConfidence, Tuning.depthRange.contains(d) else { continue } + let u = Float(i) + 0.5, v = Float(j) + 0.5 + let p = CameraMath.transform(keyframe.cameraToWorld, CameraMath.unproject(u: u, v: v, depth: d, intrinsics: depth.intrinsics)) + guard abs(p.z) < size / 2 else { continue } + let key = VoxelKey((p / size).rounded(.toNearestOrAwayFromZero)) + let level = GradientPyramid.level(depth: d, focal: keyframe.intrinsics.x) + sums[key, default: .zero] += SIMD3( + pyramid.magnitude(u: u * scale, v: v * scale, level: 0), + pyramid.magnitude(u: u * scale, v: v * scale, level: level), 1) + } + } + for (key, s) in sums { + full[key] = max(full[key] ?? 0, s.x / s.z) + scaled[key] = max(scaled[key] ?? 0, s.y / s.z) + } + } + var outline: [VoxelKey] = [], brick: [VoxelKey] = [] + for key in full.keys { + let x = Float(key.x) * size, y = Float(key.y) * size + let behindBin = x > 1.1 && x < 1.9 && y < 1.2 + guard y > 0.12, y < 2.9, abs(x) < 5.9, !behindBin else { continue } + let distance = distanceToOutline(x, y) + if distance <= size / 2 { outline.append(key) } + else if distance >= 0.1, !insidePainted(x, y) { brick.append(key) } + } + print("wall voxels: \(outline.count) on painted outlines, \(brick.count) plain brick") + for (name, values) in [("full-size Sobel", full), ("voxel-scale pyramid", scaled)] { + print("\n\(name)") + for (label, keys) in [("outline", outline), ("brick", brick)] { + let sorted = keys.compactMap { values[$0] }.sorted() + guard !sorted.isEmpty else { continue } + func p(_ q: Double) -> String { String(format: "%.3f", sorted[min(Int(Double(sorted.count) * q), sorted.count - 1)]) } + let shares = [0.1, 0.15, 0.2, 0.25, 0.3].map { t in + String(format: "%.2f: %3.0f%%", t, 100 * Double(sorted.count { $0 >= Float(t) }) / Double(sorted.count)) + } + print(" \(label.padding(toLength: 8, withPad: " ", startingAt: 0)) p10 \(p(0.1)) p50 \(p(0.5)) p90 \(p(0.9)) p99 \(p(0.99)) | above " + shares.joined(separator: ", ")) + } + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/LiveDotsApp.swift b/experiments/live-dots/Sources/LiveDots/LiveDotsApp.swift new file mode 100644 index 000000000..aa260c569 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/LiveDotsApp.swift @@ -0,0 +1,27 @@ +import AppKit +import SwiftUI + +struct LiveDotsApp: App { + /// Set by main.swift before launch. + static var fixtureArgument: String? + + @NSApplicationDelegateAdaptor private var delegate: AppDelegate + + var body: some Scene { + WindowGroup("Live dots") { + ContentView(fixtureArgument: Self.fixtureArgument) + } + .windowResizability(.contentSize) + } +} + +/// A SwiftPM executable has no bundle, so macOS would start it as a background process with no +/// Dock icon and no key window. +final class AppDelegate: NSObject, NSApplicationDelegate { + func applicationDidFinishLaunching(_ notification: Notification) { + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate() + } + + func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true } +} diff --git a/experiments/live-dots/Sources/LiveDots/LoadProgress.swift b/experiments/live-dots/Sources/LiveDots/LoadProgress.swift new file mode 100644 index 000000000..c0430d266 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/LoadProgress.swift @@ -0,0 +1,7 @@ +import Observation + +/// Share of keyframes fused so far, for the loading bar. +@Observable +final class LoadProgress { + var fraction: Double = 0 +} diff --git a/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift b/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift new file mode 100644 index 000000000..ded73f408 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift @@ -0,0 +1,74 @@ +import CoreGraphics +import Metal + +/// A render target the size of an iPhone 15 Pro screen (1170 x 2532 pixels at 3x) whose pixels +/// come back to the CPU, for the export and the benchmark. +final class OffscreenTarget { + struct Failure: Error, CustomStringConvertible { + let description: String + } + + static let pointSize = SIMD2(390, 844) + static let scale: Float = 3 + + let width: Int + let height: Int + private let renderer: DotRenderer + private let texture: MTLTexture + private let readback: MTLBuffer + + init(renderer: DotRenderer, pointSize: SIMD2 = pointSize, scale: Float = scale) throws { + self.renderer = renderer + width = Int(pointSize.x * scale) + height = Int(pointSize.y * scale) + let descriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: DotRenderer.pixelFormat, width: width, height: height, mipmapped: false) + descriptor.usage = [.renderTarget] + descriptor.storageMode = .private + guard let texture = renderer.device.makeTexture(descriptor: descriptor), + let readback = renderer.device.makeBuffer(length: width * height * 4, options: .storageModeShared) + else { throw Failure(description: "Metal would not allocate a \(width) x \(height) target") } + self.texture = texture + self.readback = readback + } + + /// Renders and waits. Returns GPU seconds. With `readBack`, the pixels land in `pixels`. + @discardableResult + func render(_ request: FrameRequest, readBack: Bool = true) throws -> Double { + guard let commandBuffer = renderer.queue.makeCommandBuffer() else { throw Failure(description: "no command buffer") } + let pass = MTLRenderPassDescriptor() + pass.colorAttachments[0].texture = texture + pass.colorAttachments[0].loadAction = .clear + pass.colorAttachments[0].storeAction = .store + pass.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1) + try renderer.encode( + request, into: commandBuffer, pass: pass, + pixelSize: SIMD2(Float(width), Float(height)), pointScale: Float(width) / OffscreenTarget.pointSize.x) + if readBack, let blit = commandBuffer.makeBlitCommandEncoder() { + blit.copy( + from: texture, sourceSlice: 0, sourceLevel: 0, sourceOrigin: MTLOrigin(x: 0, y: 0, z: 0), + sourceSize: MTLSize(width: width, height: height, depth: 1), + to: readback, destinationOffset: 0, destinationBytesPerRow: width * 4, destinationBytesPerImage: width * height * 4) + blit.endEncoding() + } + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + if let error = commandBuffer.error { throw Failure(description: "GPU error: \(error)") } + return commandBuffer.gpuEndTime - commandBuffer.gpuStartTime + } + + /// A bitmap context over the last rendered pixels (BGRA, premultiplied), for compositing. + func context() throws -> CGContext { + guard let space = CGColorSpace(name: CGColorSpace.sRGB), + let context = CGContext( + data: readback.contents(), width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, + space: space, + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) + else { throw Failure(description: "no bitmap context for the readback") } + return context + } + + var bytes: UnsafeRawBufferPointer { + UnsafeRawBufferPointer(start: readback.contents(), count: width * height * 4) + } +} diff --git a/experiments/live-dots/Sources/LiveDots/Palette.swift b/experiments/live-dots/Sources/LiveDots/Palette.swift new file mode 100644 index 000000000..8ac7c5154 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/Palette.swift @@ -0,0 +1,11 @@ +import SwiftUI + +enum Palette { + /// The dots' one colour, #E6ECF4. + static let hologram = Color(red: 0xE6 / 255, green: 0xEC / 255, blue: 0xF4 / 255) + /// The glow around every Hologram dot, #9CC8FF. + static let glow = Color(red: 0x9C / 255, green: 0xC8 / 255, blue: 0xFF / 255) + /// Dots on the occluder, core and glow, #B49CFF. + static let hiddenViolet = Color(red: 0xB4 / 255, green: 0x9C / 255, blue: 0xFF / 255) + static let window = Color(red: 0.055, green: 0.059, blue: 0.071) +} diff --git a/experiments/live-dots/Sources/LiveDots/PhoneChrome.swift b/experiments/live-dots/Sources/LiveDots/PhoneChrome.swift new file mode 100644 index 000000000..a9d33a1eb --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/PhoneChrome.swift @@ -0,0 +1,110 @@ +import LiveDotsCore +import SwiftUI + +/// Everything drawn over the camera besides the dots and fog: the recognised boxes, the ring at +/// the top and one line of guidance at the bottom. The export renders this same view per frame, +/// so stills and videos match the app. +struct PhoneChrome: View { + var state: KeyframeState + var keyframe: Keyframe + var boxes: [BoxState] + /// Playback seconds; the fades and the hold ring are evaluated here. + var time: Float + + var body: some View { + ZStack { + BoxOverlay(keyframe: keyframe, boxes: boxes, time: time) + VStack(spacing: 0) { + CoverageRing(fraction: ringFraction) + .padding(.top, 62) + Spacer(minLength: 0) + Text(state.instruction.text) + // iOS's .title3 is 20 pt; macOS's is 15, so the size is spelled out to match the phone. + .font(.system(size: 20, weight: .semibold, design: .rounded)) + .foregroundStyle(.white) + .shadow(color: .black.opacity(0.4), radius: 6, y: 1) + .multilineTextAlignment(.center) + .padding(.horizontal, 32) + .padding(.bottom, state.instruction == .tiltDown ? 14 : 58) + if state.instruction == .tiltDown { + Image(systemName: "chevron.down") + .font(.system(size: 22, weight: .semibold)) + .foregroundStyle(.white) + .shadow(color: .black.opacity(0.4), radius: 6, y: 1) + .padding(.bottom, 20) + .accessibilityHidden(true) + } + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .overlay(alignment: .bottomTrailing) { + if state.instruction == .walkLeft || state.instruction == .walkRight { + DistanceBadge(offset: keyframe.cameraPosition.x) + .padding(16) + } + } + .allowsHitTesting(false) + } + + /// While the meter locks, the ring fills over the 1 s hold; otherwise it shows walk coverage. + private var ringFraction: Float { + guard state.instruction == .holdStill else { return state.coverage } + return min(max((time - Schedule.start(of: Schedule.holdIndex)) / Schedule.holdDuration, 0), 1) + } +} + +/// A 28 pt hairline ring that fills clockwise from the top with the covered fraction of the zone +/// around the meter. It jumps to each new value: coverage changes only at keyframes, and a sweep +/// would suggest progress between them that did not happen. +struct CoverageRing: View { + var fraction: Float + + var body: some View { + ZStack { + Circle() + .stroke(Palette.hologram.opacity(0.28), lineWidth: 2) + Circle() + .trim(from: 0, to: CGFloat(min(max(fraction, 0), 1))) + .stroke(Palette.hologram, style: StrokeStyle(lineWidth: 2, lineCap: .round)) + .rotationEffect(.degrees(-90)) + } + .padding(1) + .frame(width: 28, height: 28) + .shadow(color: .black.opacity(0.35), radius: 3) + .transaction { $0.animation = nil } + .accessibilityElement() + .accessibilityLabel("Wall coverage around the meter") + .accessibilityValue(Text(Double(fraction), format: .percent.precision(.fractionLength(0)))) + } +} + +/// How far along the wall the camera is from the meter, bottom right during the walk, after +/// Starlink's compass badge. 44 pt round; "At meter" is wider than 44 pt at 13 pt, so the badge +/// is a capsule at least 44 pt across rather than a circle that clips it. +struct DistanceBadge: View { + /// Camera x minus the meter's x, in metres; negative is left of the meter. + var offset: Float + + var body: some View { + Text(label) + .font(.system(size: 13, weight: .semibold, design: .rounded)) + .monospacedDigit() + .foregroundStyle(.white) + .padding(.horizontal, 8) + .frame(minWidth: 44, minHeight: 44) + .background(.black.opacity(0.45), in: .capsule) + .accessibilityLabel(accessibilityText) + } + + private var feet: Int { Int((abs(offset) * 3.28084).rounded()) } + + private var label: String { + if abs(offset) < 0.3048 { return "At meter" } + return "\(feet) ft \(offset < 0 ? "←" : "→")" + } + + private var accessibilityText: String { + if abs(offset) < 0.3048 { return "At the meter" } + return "\(feet) feet \(offset < 0 ? "left" : "right") of the meter" + } +} diff --git a/experiments/live-dots/Sources/LiveDots/PhoneFrame.swift b/experiments/live-dots/Sources/LiveDots/PhoneFrame.swift new file mode 100644 index 000000000..069f6ad3b --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/PhoneFrame.swift @@ -0,0 +1,21 @@ +import SwiftUI + +/// A 390 x 844 pt screen with the rounded corners and bezel of a current iPhone. +struct PhoneFrame: View { + @ViewBuilder var content: Content + + var body: some View { + ZStack { + Color.black + content + } + .frame(width: 390, height: 844) + .clipShape(.rect(cornerRadius: 55, style: .continuous)) + .padding(10) + .background(.black, in: .rect(cornerRadius: 65, style: .continuous)) + .overlay { + RoundedRectangle(cornerRadius: 65, style: .continuous) + .strokeBorder(.white.opacity(0.14), lineWidth: 1) + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/ReplayData.swift b/experiments/live-dots/Sources/LiveDots/ReplayData.swift new file mode 100644 index 000000000..9791b5072 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/ReplayData.swift @@ -0,0 +1,58 @@ +import Foundation +import LiveDotsCore + +/// The fixture and both modes' precomputed timelines. +struct ReplayData: Sendable { + let replay: Replay + let lidar: DotTimeline + let noLidar: DotTimeline + /// Recognised-box states per keyframe. + let boxes: [[BoxState]] + + var keyframeCount: Int { replay.keyframes.count } + + func timeline(_ mode: CaptureMode) -> DotTimeline { + switch mode { + case .lidar: lidar + case .noLidar: noLidar + } + } + + /// Reads the fixture and runs both fields over every keyframe, off the main actor. + @concurrent + static func load(folder: URL, progress: (@Sendable (Int, Int) -> Void)? = nil) async throws -> ReplayData { + let replay = try Replay.load(folder: folder) + let (lidar, noLidar) = try DotTimeline.build(replay: replay, progress: progress) + return ReplayData(replay: replay, lidar: lidar, noLidar: noLidar, boxes: BoxState.timeline(for: replay.keyframes)) + } + + /// The same, blocking, for the command-line paths. + nonisolated static func loadNow(folder: URL) throws -> ReplayData { + let replay = try Replay.load(folder: folder) + let (lidar, noLidar) = try DotTimeline.build(replay: replay) + return ReplayData(replay: replay, lidar: lidar, noLidar: noLidar, boxes: BoxState.timeline(for: replay.keyframes)) + } +} + +enum FixtureLocator { + struct Missing: Error, CustomStringConvertible { + let tried: [String] + var description: String { + "no fixture at \(tried.joined(separator: " or ")); run ./fetch-fixtures.sh from experiments/live-dots first" + } + } + + /// `--fixture` if given, else Fixtures/synthetic-wall-lidar under the working directory, else + /// under this package's folder (so `swift run` works from anywhere). + static func folder(_ explicit: String?) throws -> URL { + let packageRoot = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + let candidates = explicit.map { [URL(fileURLWithPath: $0)] } ?? [ + URL(fileURLWithPath: FileManager.default.currentDirectoryPath).appendingPathComponent("Fixtures/synthetic-wall-lidar"), + packageRoot.appendingPathComponent("Fixtures/synthetic-wall-lidar"), + ] + for url in candidates where FileManager.default.fileExists(atPath: url.appendingPathComponent("session.json").path) { + return url + } + throw Missing(tried: candidates.map(\.path)) + } +} diff --git a/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift b/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift new file mode 100644 index 000000000..0b606dbd1 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift @@ -0,0 +1,77 @@ +import LiveDotsCore +import Observation +import QuartzCore + +/// The playback state the controls edit and the renderer reads. Keyframes play at 4 per second +/// and cut; the dots animate on the same clock. +@Observable +final class ReplayPlayer { + let keyframeCount: Int + var mode: CaptureMode = .lidar + var scheme: DotScheme = .hologram + var reduceMotion: Bool + var fog: FogStyle = .on + private(set) var isPlaying = false + private(set) var keyframe = 0 + /// Playback seconds. Keyframe k is on screen from k / 4 to (k + 1) / 4. + @ObservationIgnored private(set) var playhead: Float = 0 + /// After a scrub the frame shows its settled state instead of replaying births. + @ObservationIgnored private var settled = false + @ObservationIgnored private var lastTick: CFTimeInterval? + @ObservationIgnored private var lastFrame: CFTimeInterval? + @ObservationIgnored private var ambient: Float = 0 + @ObservationIgnored private var frameInterval: Float = 1 / 60 + + /// Playback runs this long past the last keyframe so its births finish. + private var end: Float { Schedule.end(count: keyframeCount) + Tuning.birthDuration } + + init(keyframeCount: Int, reduceMotion: Bool) { + self.keyframeCount = keyframeCount + self.reduceMotion = reduceMotion + } + + var request: FrameRequest { + FrameRequest( + mode: mode, keyframe: keyframe, time: settled ? 1e5 : playhead, + reduceMotion: reduceMotion, fog: fog, scheme: scheme, ambientTime: ambient, frameInterval: frameInterval) + } + + func togglePlayback() { + if isPlaying { + isPlaying = false + return + } + if playhead >= end - 1e-3 || (keyframe == keyframeCount - 1 && settled) { seek(to: 0, settle: false) } + settled = false + lastTick = nil + isPlaying = true + } + + /// Jumps to a keyframe and shows it settled. Scrubbing while playing keeps playing from there. + func scrub(to index: Int) { + seek(to: index, settle: !isPlaying) + } + + private func seek(to index: Int, settle: Bool) { + keyframe = min(max(index, 0), keyframeCount - 1) + playhead = Schedule.start(of: keyframe) + settled = settle + lastTick = nil + } + + /// Advances the clock; the Metal view calls this once per display frame. + func tick(now: CFTimeInterval) { + if let lastFrame { frameInterval = min(Float(now - lastFrame), 0.1) } + lastFrame = now + ambient += frameInterval + guard isPlaying else { return } + if let lastTick { playhead += Float(now - lastTick) } + lastTick = now + let index = Schedule.keyframe(at: playhead, count: keyframeCount) + if index != keyframe { keyframe = index } + if playhead >= end { + playhead = end + isPlaying = false + } + } +} diff --git a/experiments/live-dots/Sources/LiveDots/Shaders.swift b/experiments/live-dots/Sources/LiveDots/Shaders.swift new file mode 100644 index 000000000..528c732f1 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/Shaders.swift @@ -0,0 +1,342 @@ +/// Metal source, compiled at launch with `makeLibrary(source:)` so `swift build` needs no +/// Metal toolchain step. Every value the shader bakes in matches `Tuning`. +enum Shaders { + static let source = """ + #include + using namespace metal; + + // cubic-bezier(0.23, 1, 0.32, 1): solve x(t) = p by Newton, return y(t). Same as CubicBezier.swift. + static float strongEaseOut(float p) { + p = clamp(p, 0.0f, 1.0f); + if (p <= 0.0f || p >= 1.0f) { return p; } + const float x1 = 0.23f, x2 = 0.32f, y1 = 1.0f, y2 = 1.0f; + float t = p; + for (int i = 0; i < 8; i++) { + float u = 1.0f - t; + float x = 3.0f * u * u * t * x1 + 3.0f * u * t * t * x2 + t * t * t - p; + float dx = 3.0f * u * u * x1 + 6.0f * u * t * (x2 - x1) + 3.0f * t * t * (1.0f - x2); + if (fabs(dx) < 1e-6f) { break; } + t = clamp(t - x / dx, 0.0f, 1.0f); + } + float u = 1.0f - t; + return 3.0f * u * u * t * y1 + 3.0f * u * t * t * y2 + t * t * t; + } + + // MARK: Camera image + + struct CameraUniforms { + float2 offset; // pixels: top-left of the portrait image on screen + float scale; // screen pixels per portrait image pixel + float pad; + float2 imageSize; // landscape image size + }; + + struct CameraOut { float4 position [[position]]; }; + + vertex CameraOut cameraVertex(uint vid [[vertex_id]]) { + float2 p = float2((vid << 1) & 2, vid & 2); + CameraOut out; + out.position = float4(p * 2.0f - 1.0f, 0.5f, 1.0f); + return out; + } + + // Screen pixel -> portrait pixel -> landscape pixel (turned back 90 degrees counterclockwise). + fragment float4 cameraFragment(CameraOut in [[stage_in]], + constant CameraUniforms& u [[buffer(0)]], + texture2d image [[texture(0)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + float2 portrait = (in.position.xy - u.offset) / u.scale; + float2 landscape = float2(portrait.y, u.imageSize.y - portrait.x); + return float4(image.sample(s, landscape / u.imageSize).rgb, 1.0f); + } + + // MARK: Fog comparison veil + + vertex float4 veilVertex(uint vid [[vertex_id]], + const device float4* positions [[buffer(0)]], + constant float4x4& clip [[buffer(1)]]) { + return clip * positions[vid]; + } + + fragment float4 veilFragment() { + return float4(0.0f, 0.0f, 0.0f, 0.35f); + } + + // MARK: Dots + + struct Sprite { + float4 a; // world xyz, 1 for a simulated feature point (fixed size), else 0 + float4 b; // birth time, from opacity, to opacity, opacity change time + float4 c; // edge since, death time, violet (0 or 1), 1 for the halo sprite + }; + + struct DotUniforms { + float4x4 clip; + float4 cameraAndTime; // camera xyz, playback time + float pointScale; // pixels per point + float reduceMotion; // 1 drops the birth scale + float scheme; // 0 hologram, 1 constellation, 2 ember + float pad; + }; + + constant float3 hologram = float3(0xE6, 0xEC, 0xF4) / 255.0f; + constant float3 glow = float3(0x9C, 0xC8, 0xFF) / 255.0f; + constant float3 violet = float3(0xB4, 0x9C, 0xFF) / 255.0f; + constant float3 emberCore = float3(0xFF, 0xB4, 0x54) / 255.0f; + constant float3 emberHalo = float3(0xFF, 0xC9, 0x78) / 255.0f; + + // Constellation hides flat dots, so a dot that turns edge is born then, not at its voxel's birth. + static float visibleBirth(Sprite s, int scheme) { + return scheme == 1 ? max(s.b.x, s.c.x) : s.b.x; + } + + struct DotOut { + float4 position [[position]]; + float size [[point_size]]; + float4 color; + float inner; // radius fraction where the soft edge starts + }; + + vertex DotOut dotVertex(uint vid [[vertex_id]], + const device Sprite* sprites [[buffer(0)]], + constant DotUniforms& u [[buffer(1)]]) { + Sprite s = sprites[vid]; + float t = u.cameraAndTime.w; + int scheme = int(u.scheme + 0.5f); + float birth = strongEaseOut((t - visibleBirth(s, scheme)) / 0.35f); + float evidence = mix(s.b.y, s.b.z, strongEaseOut((t - s.b.w) / 0.25f)); + // Ember: amber at 90% at birth, cooling linearly to the evidence opacity over 6 s. + float warmth = scheme == 2 ? clamp(1.0f - (t - s.b.x) / 6.0f, 0.0f, 1.0f) : 0.0f; + evidence = mix(evidence, 0.9f, warmth); + float fade = 1.0f - strongEaseOut((t - s.c.y) / 0.25f); + float edge = strongEaseOut((t - s.c.x) / 0.25f); + float alpha = evidence * birth * fade; + bool feature = s.a.w > 0.5f; + bool halo = s.c.w > 0.5f; + + // Flat 2.5 pt and edge 3.5 pt from 2.5 m out, growing linearly to 4 and 6 pt at 1 m; + // constellation edges 4.5 pt growing to 7. + float size = 4.5f; + if (!feature) { + float far = clamp((length(s.a.xyz - u.cameraAndTime.xyz) - 1.0f) / 1.5f, 0.0f, 1.0f); + size = scheme == 1 ? mix(7.0f, 4.5f, far) : mix(mix(4.0f, 2.5f, far), mix(6.0f, 3.5f, far), edge); + } + float inner = feature ? 0.3f : 0.7f; + float3 color = hologram; + float3 warm = emberCore; + if (halo) { + // Edge 4x at 22%, flat 2x at 8%, feature 3x at 18%; a flat dot turning edge grows into it. + size *= feature ? 3.0f : mix(2.0f, 4.0f, edge); + alpha *= feature ? 0.18f : mix(0.08f, 0.22f, edge); + inner = 0.0f; + color = glow; + warm = emberHalo; + } + color = mix(mix(color, violet, s.c.z), warm, warmth); + float scale = u.reduceMotion > 0.5f ? 1.0f : mix(0.6f, 1.0f, birth); + + DotOut out; + out.position = u.clip * float4(s.a.xyz, 1.0f); + if (alpha < 1.0f / 512.0f) { out.position = float4(0.0f, 0.0f, -1.0f, 1.0f); } + out.size = size * scale * u.pointScale; + out.color = float4(color, 1.0f) * alpha; + out.inner = inner; + return out; + } + + // MARK: Constellation links + + struct LinkOut { + float4 position [[position]]; + float4 color; + }; + + // Each link vertex carries the link's timing (latest birth, earliest death, lower opacity of + // its two dots), so the line appears and fades with the dots it joins. + vertex LinkOut linkVertex(uint vid [[vertex_id]], + const device Sprite* sprites [[buffer(0)]], + constant DotUniforms& u [[buffer(1)]]) { + Sprite s = sprites[vid]; + float t = u.cameraAndTime.w; + float birth = strongEaseOut((t - s.b.x) / 0.35f); + float evidence = mix(s.b.y, s.b.z, strongEaseOut((t - s.b.w) / 0.25f)); + float fade = 1.0f - strongEaseOut((t - s.c.y) / 0.25f); + LinkOut out; + out.position = u.clip * float4(s.a.xyz, 1.0f); + out.color = float4(mix(glow, violet, s.c.z), 1.0f) * (0.35f * evidence * birth * fade); + return out; + } + + fragment float4 linkFragment(LinkOut in [[stage_in]]) { + return in.color; + } + + // Round sprite, fully opaque inside `inner`, fading to zero at the rim. Halos use 0: a glow. + fragment float4 dotFragment(DotOut in [[stage_in]], float2 pc [[point_coord]]) { + float r = length(pc * 2.0f - 1.0f); + return in.color * (1.0f - smoothstep(in.inner, 1.0f, r)); + } + + // MARK: Fog + // + // The unmeasured world sits under one continuous fog that dissolves where dots arrive. Each + // frame every live dot splats a soft disc (0.35 m, 6 to 30 mask pixels) into a 96 x 208 + // reveal mask; a radius-5 Gaussian blur and smoothstep(0.12, 0.55) feather its edge; a lagged + // copy eases toward it so the fog lifts a beat after the dots appear (600 ms time constant, + // a linear 400 ms fade under Reduce Motion) and returns quickly (120 ms) where a new view is + // unmeasured. The fog itself is Chalk mixed toward a cool + // grey, slightly thinner up close, textured by slow domain-warped fbm. + // + // Borrowed: the noise-edged fog-of-war mask from Civilization VI (lexdev.net case study); the + // low-resolution mask sampled bilinearly after a blur (Andrew Hung, "Implementing Attractive + // Fog of War in Unity"); reveal-by-measurement from Polycam's blue grid turning into white + // mesh and Scaniverse's hatching over unscanned areas; the live point cloud and capture dial + // of Apple's Object Capture; RoomPlan's coaching line and animated outlines (the boxes and + // guidance, drawn in SwiftUI). + + struct SplatUniforms { + float4x4 clip; + float pixelsPerMetre; // mask pixels per metre at 1 m + float time; // playback seconds + float2 pad; + }; + + struct SplatOut { + float4 position [[position]]; + float size [[point_size]]; + float weight; + }; + + vertex SplatOut splatVertex(uint vid [[vertex_id]], + const device Sprite* sprites [[buffer(0)]], + constant SplatUniforms& u [[buffer(1)]]) { + Sprite s = sprites[vid]; + float t = u.time; + float birth = strongEaseOut((t - s.b.x) / 0.35f); + float evidence = mix(s.b.y, s.b.z, strongEaseOut((t - s.b.w) / 0.25f)); + float fade = 1.0f - strongEaseOut((t - s.c.y) / 0.25f); + float factor = clamp((evidence - 0.45f) / 0.45f, 0.0f, 1.0f); + SplatOut out; + out.weight = birth * fade * (0.6f + 0.4f * factor); + out.position = u.clip * float4(s.a.xyz, 1.0f); + float radius = clamp(0.35f * u.pixelsPerMetre / max(out.position.w, 0.05f), 6.0f, 30.0f); + out.size = 2.0f * radius; + if (out.weight < 1.0f / 512.0f || out.position.w < 0.05f) { out.position = float4(0.0f, 0.0f, -1.0f, 1.0f); } + return out; + } + + fragment float4 splatFragment(SplatOut in [[stage_in]], float2 pc [[point_coord]]) { + float2 d = pc * 2.0f - 1.0f; + float k = saturate(1.0f - dot(d, d)); + return float4(in.weight * k * k, 0.0f, 0.0f, 0.0f); + } + + // Separable Gaussian, radius 5 texels (sigma 2.5). The second pass clamps to 1 and applies + // smoothstep(0.12, 0.55), the same as FogMask.shape. + fragment float4 blurFragment(CameraOut in [[stage_in]], + texture2d source [[texture(0)]], + constant int2& direction [[buffer(0)]], + constant int& finish [[buffer(1)]]) { + int2 p = int2(in.position.xy); + int2 limit = int2(source.get_width() - 1, source.get_height() - 1); + float sum = 0.0f, total = 0.0f; + for (int k = -5; k <= 5; k++) { + float w = exp(-float(k * k) / (2.0f * 2.5f * 2.5f)); + sum += w * source.read(uint2(clamp(p + k * direction, int2(0), limit))).r; + total += w; + } + float v = sum / total; + if (finish != 0) { v = smoothstep(0.12f, 0.55f, min(v, 1.0f)); } + return float4(v, 0.0f, 0.0f, 0.0f); + } + + struct LagUniforms { + float dt; + int reduceMotion; + int reset; + int pad; + }; + + // Same rule as FogMask.lagStep. + fragment float4 lagFragment(CameraOut in [[stage_in]], + texture2d target [[texture(0)]], + texture2d previous [[texture(1)]], + constant LagUniforms& u [[buffer(0)]]) { + uint2 p = uint2(in.position.xy); + float goal = target.read(p).r; + if (u.reset != 0) { return float4(goal, 0.0f, 0.0f, 0.0f); } + float last = previous.read(p).r; + bool lifting = goal > last; + float next; + if (u.reduceMotion != 0) { + float step = u.dt / (lifting ? 0.4f : 0.12f); + next = last + clamp(goal - last, -step, step); + } else { + next = last + (goal - last) * (1.0f - exp(-u.dt / (lifting ? 0.6f : 0.12f))); + } + return float4(next, 0.0f, 0.0f, 0.0f); + } + + struct FogUniforms { + float2 viewSize; // pixels + float2 offset; // camera image placement, as CameraUniforms + float scale; + float time; // ambient seconds, runs while paused + float2 imageSize; + float motion; // 0 under Reduce Motion: no drift, no breathing + float pad0; + float2 pad1; + }; + + static float hash21(float2 p) { + p = fract(p * float2(123.34f, 456.21f)); + p += dot(p, p + 45.32f); + return fract(p.x * p.y); + } + + static float valueNoise(float2 p) { + float2 i = floor(p), f = fract(p); + float2 w = f * f * (3.0f - 2.0f * f); + float a = hash21(i), b = hash21(i + float2(1, 0)), c = hash21(i + float2(0, 1)), d = hash21(i + float2(1, 1)); + return mix(mix(a, b, w.x), mix(c, d, w.x), w.y); + } + + static float fbm(float2 p) { + float sum = 0.0f, amplitude = 0.5f; + for (int i = 0; i < 3; i++) { + sum += amplitude * valueNoise(p); + p = p * 2.03f + float2(17.1f, 9.2f); + amplitude *= 0.5f; + } + return sum / 0.875f; + } + + fragment float4 fogFragment(CameraOut in [[stage_in]], + constant FogUniforms& u [[buffer(0)]], + texture2d reveal [[texture(0)]], + texture2d depth [[texture(1)]]) { + constexpr sampler linear(filter::linear, address::clamp_to_edge); + constexpr sampler nearest(filter::nearest, address::clamp_to_edge); + float2 screen = in.position.xy; + float revealed = reveal.sample(linear, screen / u.viewSize).r; + + float2 portrait = (screen - u.offset) / u.scale; + float2 landscape = float2(portrait.y, u.imageSize.y - portrait.x); + float d = depth.sample(nearest, landscape / u.imageSize).r; + if (d <= 0.0f) { d = 100.0f; } // no hit: far + float depthTerm = 0.75f + 0.25f * smoothstep(1.0f, 4.5f, d); + + // uv in screen widths, so the noise is round, not stretched down the tall screen. + float2 uv = screen / u.viewSize.x; + float2 q = uv * 2.2f + u.motion * u.time * 0.02f * float2(0.94f, 0.34f); + float2 warp = float2(fbm(q + float2(3.1f, 1.7f)), fbm(q + float2(-2.3f, 5.2f))); + float noiseTerm = 0.78f + 0.22f * fbm(q + 0.8f * warp); + float breathing = 1.0f + u.motion * 0.04f * sin(2.0f * M_PI_F * 0.08f * u.time); + + float alpha = saturate(0.62f * depthTerm * noiseTerm * (1.0f - revealed) * breathing); + float3 chalk = float3(0xF7, 0xF5, 0xEF) / 255.0f; + float3 cool = float3(0xD9, 0xE2, 0xEC) / 255.0f; + return float4(mix(chalk, cool, 0.35f) * alpha, alpha); + } + """ +} diff --git a/experiments/live-dots/Sources/LiveDots/main.swift b/experiments/live-dots/Sources/LiveDots/main.swift new file mode 100644 index 000000000..c2ddf4b40 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDots/main.swift @@ -0,0 +1,22 @@ +import Foundation + +do { + switch try Command.parse(Array(CommandLine.arguments.dropFirst())) { + case let .app(fixture): + LiveDotsApp.fixtureArgument = fixture + LiveDotsApp.main() + case let .export(destination, options): + try Exporter.exportReplay(to: destination, options: options) + case let .still(keyframe, at, output, options): + try Exporter.exportStill(keyframe: keyframe, at: at, to: output, options: options) + case let .gradientReport(fixture): + try GradientReport.run(fixture: fixture) + case let .benchmark(fixture): + try Benchmark.run(fixture: fixture) + case let .fieldReport(fixture): + try FieldReport.run(fixture: fixture) + } +} catch { + FileHandle.standardError.write(Data("LiveDots: \(error)\n".utf8)) + exit(1) +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Coverage.swift b/experiments/live-dots/Sources/LiveDotsCore/Coverage.swift new file mode 100644 index 000000000..73e842909 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/Coverage.swift @@ -0,0 +1,19 @@ +/// The ring's fraction: how much of the requirement zone around the meter (x from -3 to 3 m on +/// the wall) has 30 cm columns holding at least one dot seen from two or more directions. +public enum Coverage { + public static var columnCount: Int { + Int(((Tuning.coverageZone.upperBound - Tuning.coverageZone.lowerBound) / Tuning.coverageColumn).rounded()) + } + + /// Dots count when they sit on the wall plane (within 10 cm of z = 0) and are not on the bin. + public static func fraction(of dots: [FieldDot]) -> Float { + var covered = Set() + for dot in dots where dot.views >= Tuning.coverageMinViews && abs(dot.position.z) <= 0.1 && !dot.onOccluder { + let offset = dot.position.x - Tuning.coverageZone.lowerBound + guard offset >= 0 else { continue } + let column = Int((offset / Tuning.coverageColumn).rounded(.down)) + if column < columnCount { covered.insert(column) } + } + return Float(covered.count) / Float(columnCount) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/CubicBezier.swift b/experiments/live-dots/Sources/LiveDotsCore/CubicBezier.swift new file mode 100644 index 000000000..55f26cc94 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/CubicBezier.swift @@ -0,0 +1,44 @@ +/// A CSS-style cubic-bezier timing curve from (0, 0) to (1, 1). The Metal shader carries the same +/// solver; this copy is for tests and CPU-side opacity bookkeeping. +public struct CubicBezier: Sendable, Equatable { + public let x1: Float, y1: Float, x2: Float, y2: Float + + public init(x1: Float, y1: Float, x2: Float, y2: Float) { + self.x1 = x1; self.y1 = y1; self.x2 = x2; self.y2 = y2 + } + + /// `cubic-bezier(0.23, 1, 0.32, 1)`, the strong ease-out every dot transition uses. + public static let strongEaseOut = CubicBezier(x1: 0.23, y1: 1, x2: 0.32, y2: 1) + + /// The curve's y for x = `progress`, clamped to [0, 1]. + public func callAsFunction(_ progress: Float) -> Float { + if progress <= 0 { return 0 } + if progress >= 1 { return 1 } + var t = progress + for _ in 0..<8 { + let error = sample(t, x1, x2) - progress + if abs(error) < 1e-6 { return sample(t, y1, y2) } + let slope = derivative(t, x1, x2) + if abs(slope) < 1e-6 { break } + t = min(max(t - error / slope, 0), 1) + } + var low: Float = 0, high: Float = 1 + for _ in 0..<40 { + let x = sample(t, x1, x2) + if abs(x - progress) < 1e-6 { break } + if x < progress { low = t } else { high = t } + t = (low + high) / 2 + } + return sample(t, y1, y2) + } + + private func sample(_ t: Float, _ a: Float, _ b: Float) -> Float { + let u = 1 - t + return 3 * u * u * t * a + 3 * u * t * t * b + t * t * t + } + + private func derivative(_ t: Float, _ a: Float, _ b: Float) -> Float { + let u = 1 - t + return 3 * u * u * a + 6 * u * t * (b - a) + 3 * t * t * (1 - b) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift b/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift new file mode 100644 index 000000000..6fbb8ce9d --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift @@ -0,0 +1,76 @@ +import Foundation +import simd + +/// One keyframe's LiDAR depth: metres along camera -z (not along the ray), 0 where nothing was hit. +public struct DepthMap: Sendable { + public let width: Int + public let height: Int + public let meters: [Float] + /// 0 low, 1 medium, 2 high. All high when the recording had no confidence file. + public let confidence: [UInt8] + /// fx, fy, cx, cy in depth-map pixels. + public let intrinsics: SIMD4 + + public init(meters: [Float], confidence: [UInt8], width: Int, height: Int, intrinsics: SIMD4) { + precondition(meters.count == width * height && confidence.count == width * height, "depth arrays must match the map size") + self.meters = meters + self.confidence = confidence + self.width = width + self.height = height + self.intrinsics = intrinsics + } + + /// Decodes the raw files. Byte counts are checked against the declared size before anything + /// is read, and a mismatch names the file and both counts. + public init( + depth: Data, depthPath: String, confidence: Data?, confidencePath: String?, + width: Int, height: Int, intrinsics: SIMD4 + ) throws(FixtureError) { + let count = width * height + guard depth.count == count * 4 else { + throw .depthByteCount(path: depthPath, found: depth.count, expected: count * 4, width: width, height: height) + } + if let confidence, confidence.count != count { + throw .confidenceByteCount(path: confidencePath ?? "confidence", found: confidence.count, expected: count, width: width, height: height) + } + let meters: [Float] = depth.withUnsafeBytes { raw in + (0.. Float? { + guard u >= 0, v >= 0 else { return nil } + let i = Int(u), j = Int(v) + guard i < width, j < height else { return nil } + return meters[j * width + i] + } +} + +public enum CameraMath { + /// Camera-space point for continuous pixel (u, v) at depth `depth` along -z. A pixel's centre + /// is (i + 0.5, j + 0.5), which is how the fixture generator cast its rays. + public static func unproject(u: Float, v: Float, depth: Float, intrinsics k: SIMD4) -> SIMD3 { + SIMD3((u - k.z) * depth / k.x, -(v - k.w) * depth / k.y, -depth) + } + + /// Continuous pixel and depth of a camera-space point, or nil when it is not in front. + public static func project(_ p: SIMD3, intrinsics k: SIMD4) -> (u: Float, v: Float, depth: Float)? { + let depth = -p.z + guard depth > 1e-4 else { return nil } + return (k.x * p.x / depth + k.z, -k.y * p.y / depth + k.w, depth) + } + + public static func transform(_ m: simd_float4x4, _ p: SIMD3) -> SIMD3 { + let r = m * SIMD4(p, 1) + return SIMD3(r.x, r.y, r.z) + } + + public static func rotate(_ m: simd_float4x4, _ v: SIMD3) -> SIMD3 { + let r = m * SIMD4(v, 0) + return SIMD3(r.x, r.y, r.z) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/DotField.swift b/experiments/live-dots/Sources/LiveDotsCore/DotField.swift new file mode 100644 index 000000000..33836f984 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/DotField.swift @@ -0,0 +1,361 @@ +// The dot field: what the live capture view draws over the camera, and why. No-LiDAR mode is +// simulated in SimulatedFeatures.swift; this file is LiDAR mode. +// +// Each keyframe's depth samples with medium or high confidence and depth from 0.3 to 6 m are +// unprojected and fused into 5 cm voxels. A voxel keeps the keyframe it was first seen in, its +// hit count, its distinct view directions (a new one must differ from every stored one by more +// than 15 degrees), a normal from central differences on the depth map (set by the first +// keyframe, averaged after) and the image gradient its samples saw. Its dot sits at the centre +// plus a fixed jitter of up to 30% of the voxel, in the plane of its first normal, so the field +// never reads as a grid. +// +// A voxel becomes an edge, and stays one, when +// (a) a face neighbour's normal differs by more than 35 degrees (the wall-to-ground corner, +// the bin's corners); +// (b) it has surface along one side and seen-empty space along another (the wall's top against +// the sky, the bin's silhouette). Seen-empty means a depth ray passed the neighbour with +// 10 cm to spare, so the edge of what has been scanned so far is never an edge; or +// (c) the colour gradient at its samples reaches 0.2 (painted outlines). The gradient is read +// from a pyramid level whose pixel spans about one voxel, and ignored near a depth +// silhouette, where the colour step is occlusion rather than paint. On the fixture +// (`LiveDots --gradient-report`) 100% of voxels on painted outlines and 1% of plain-brick +// voxels reach 0.2. A Sobel on the full-size JPEG gives 31% and 19%, because mortar joints +// are as sharp as paint. +// The simulated no-LiDAR feature points use a lower gradient, 0.1, on purpose: ARKit's +// rawFeaturePoints land on any high-contrast texture, brick joints included, while an edge dot +// should mark only an outline. +// Every edge voxel gets a dot. Flat voxels get one per 2 x 2 x 2 group (the lowest hash, then +// kept), a quarter of the edge density on a wall. At most 6,000 dots draw in view; past that, +// flat dots drop in hash order. +// +// Opacity is 45% at one view and rises 15 points a view to 90% at four. An edge dot on a +// surface within 45 degrees of vertical stays at or below 60% until one view saw it within 30 +// degrees of face-on. +// +// Drawing (the shader in Sources/LiveDots/Shaders.swift): flat dots are 2.5 pt and edge dots +// 3.5 pt at 2.5 m and beyond, growing linearly to 4 and 6 pt at 1 m and no larger closer in, for +// a view used 1.5 to 3 m from the wall. Every dot carries an additive halo: 4x size at 22% for +// edges, 2x at 8% for flat dots, 3x at 18% for the 4.5 pt simulated feature points. Cores are +// Hologram #E6ECF4 and halos #9CC8FF; dots more than 0.25 m in front of the wall and above the +// ground are on the occluder and draw violet #B49CFF, halo included. +// That is the hologram scheme; DotScheme.swift describes the constellation and ember looks. + +import simd + +public typealias VoxelKey = SIMD3 + +/// What one 5 cm voxel has accumulated. +public struct Voxel: Sendable { + public let firstSeenFrame: Int + public internal(set) var hitCount: Int + public internal(set) var views: ViewDirections + /// Sum of per-keyframe unit normals: set by the first hit, averaged by later ones. + public internal(set) var normalSum: SIMD3 + /// Fixed at the first hit, in the plane of the first normal. + public let jitter: SIMD3 + /// Highest per-keyframe mean colour gradient of the samples that landed here. + public internal(set) var gradient: Float + /// Highest cosine between a view direction and the voxel's normal at that view. Kept apart + /// from `views`, which drops a face-on view that falls within 15 degrees of an oblique one. + public internal(set) var faceOnCosine: Float + /// The keyframe the voxel first classified as an edge. Edges stay edges, so a dot never + /// shrinks back or loses its glow. + public internal(set) var edgeSinceFrame: Int? + + public var normal: SIMD3? { + let length = simd_length(normalSum) + return length > 1e-3 ? normalSum / length : nil + } + + public var isEdge: Bool { edgeSinceFrame != nil } +} + +public enum EdgeReason: Sendable, Equatable { + /// A face neighbour's normal differs by more than 35 degrees. + case crease + /// Seen-empty space on one side along the surface, surface on another. + case boundary + /// The image gradient at the voxel crosses the threshold (painted outlines). + case imageEdge +} + +/// The LiDAR-mode dot field: depth samples fused into 5 cm voxels, each voxel classified as edge +/// or flat, and the dots drawn from them. See the comment at the top of this file. +public struct VoxelField: Sendable { + public let size: Float + public private(set) var voxels: [VoxelKey: Voxel] = [:] + /// Voxels a depth ray has passed through with room to spare. Only in-plane neighbours of + /// occupied voxels are tested, which is all the boundary rule needs. + public private(set) var free: Set = [] + /// One flat voxel per 2 x 2 x 2 group, chosen once by lowest hash and then kept, so a flat + /// dot never hops to a sibling as the group fills in. + private var representatives: [VoxelKey: VoxelKey] = [:] + + static let faceAxes: [VoxelKey] = [ + VoxelKey(1, 0, 0), VoxelKey(-1, 0, 0), VoxelKey(0, 1, 0), + VoxelKey(0, -1, 0), VoxelKey(0, 0, 1), VoxelKey(0, 0, -1), + ] + + public init(size: Float = Tuning.voxelSize) { + self.size = size + } + + /// Voxel centres sit on multiples of `size`, so the fixture's wall (z = 0) and ground (y = 0) + /// run through voxel centres instead of splitting between two layers. + public func key(for p: SIMD3) -> VoxelKey { + VoxelKey((p / size).rounded(.toNearestOrAwayFromZero)) + } + + public func centre(of key: VoxelKey) -> SIMD3 { + SIMD3(key) * size + } + + // MARK: Building blocks, public so the tests can build a field by hand + + /// Records one keyframe's samples in a voxel: `normal` is their mean (nil if none had one), + /// `gradient` their mean image gradient, `camera` the camera position. + public mutating func observe( + _ key: VoxelKey, normal: SIMD3?, gradient: Float, samples: Int, camera: SIMD3, frame: Int + ) { + let centre = centre(of: key) + var voxel = voxels[key] ?? Voxel( + firstSeenFrame: frame, hitCount: 0, views: ViewDirections(), normalSum: .zero, + jitter: Jitter.offset(for: key, normal: normal ?? SIMD3(0, 0, 1), cellSize: size), + gradient: 0, faceOnCosine: -1, edgeSinceFrame: nil) + voxel.hitCount += samples + if let normal, simd_length(normal) > 1e-3 { voxel.normalSum += simd_normalize(normal) } + voxel.gradient = max(voxel.gradient, gradient) + voxel.views.insert(camera - centre) + if let current = voxel.normal, simd_length(camera - centre) > 1e-6 { + voxel.faceOnCosine = max(voxel.faceOnCosine, simd_dot(simd_normalize(camera - centre), current)) + } + voxels[key] = voxel + free.remove(key) + } + + public mutating func markFree(_ key: VoxelKey) { + if voxels[key] == nil { free.insert(key) } + } + + public func edgeReason(for key: VoxelKey) -> EdgeReason? { + guard let voxel = voxels[key] else { return nil } + let normal = voxel.normal + var surfaceNeighbour = false, emptyNeighbour = false + for axis in Self.faceAxes { + let along = normal.map { abs(simd_dot(SIMD3(axis), $0)) < Tuning.alongSurfaceCosine } ?? false + if let other = voxels[key &+ axis] { + if let normal, let otherNormal = other.normal, Self.normalsDisagree(normal, otherNormal) { + return .crease + } + if along { surfaceNeighbour = true } + } else if along, free.contains(key &+ axis) { + emptyNeighbour = true + } + } + if surfaceNeighbour, emptyNeighbour { return .boundary } + if voxel.gradient >= Tuning.gradientThreshold { return .imageEdge } + return nil + } + + public static func normalsDisagree(_ a: SIMD3, _ b: SIMD3) -> Bool { + simd_dot(simd_normalize(a), simd_normalize(b)) < cos(Tuning.creaseDegrees * Float.pi / 180) + } + + /// Marks new edges (sticky), then gives each group without one a flat representative. + public mutating func classify(frame: Int) { + let newEdges = voxels.keys.filter { voxels[$0]?.isEdge == false && edgeReason(for: $0) != nil } + for key in newEdges { voxels[key]?.edgeSinceFrame = frame } + + var best: [VoxelKey: (key: VoxelKey, hash: UInt64)] = [:] + for (key, voxel) in voxels where !voxel.isEdge { + let group = key &>> 1 + guard representatives[group] == nil else { continue } + let hash = StableHash.hash(key, seed: 0xF1A7) + if let current = best[group], current.hash <= hash { continue } + best[group] = (key, hash) + } + for (group, choice) in best { representatives[group] = choice.key } + } + + // MARK: Keyframes + + public mutating func integrate(_ frame: FrameInput) { + let depth = frame.depth + let pose = frame.keyframe.cameraToWorld + let camera = frame.keyframe.cameraPosition + let imageScale = Float(frame.keyframe.width) / Float(depth.width) + let focal = frame.keyframe.intrinsics.x + + let discontinuity = Self.discontinuityDistance(depth) + + struct Accumulator { var count = 0; var normals = 0; var normalSum = SIMD3.zero; var gradientSum: Float = 0 } + var accumulators: [VoxelKey: Accumulator] = [:] + for j in 0..= Tuning.minimumConfidence, Tuning.depthRange.contains(d) else { continue } + let u = Float(i) + 0.5, v = Float(j) + 0.5 + let world = CameraMath.transform(pose, CameraMath.unproject(u: u, v: v, depth: d, intrinsics: depth.intrinsics)) + let level = GradientPyramid.level(depth: d, focal: focal) + // Near a silhouette the colour step is one surface hiding another, not paint. + // The boundary rule handles silhouettes from geometry; left in, this made wall + // voxels beside the bin, and the whole foreshortened bin lid, into edges. + let reach = 1.5 * Float(1 << level) / imageScale + 1 + let gradient = discontinuity[index] <= reach + ? 0 : frame.gradient.magnitude(u: u * imageScale, v: v * imageScale, level: level) + var accumulator = accumulators[key(for: world)] ?? Accumulator() + accumulator.count += 1 + accumulator.gradientSum += gradient + if let normal = Self.cameraNormal(depth: depth, i: i, j: j) { + accumulator.normals += 1 + accumulator.normalSum += CameraMath.rotate(pose, normal) + } + accumulators[key(for: world)] = accumulator + } + } + for (key, a) in accumulators { + observe( + key, normal: a.normals > 0 ? a.normalSum / Float(a.normals) : nil, + gradient: a.gradientSum / Float(a.count), samples: a.count, camera: camera, frame: frame.index) + } + carveFreeSpace(frame) + classify(frame: frame.index) + } + + /// Tests the empty in-plane neighbours of occupied voxels against this keyframe's depth: a + /// neighbour is seen-empty when the ray through it hit nothing or hit something farther by + /// `Tuning.freeSpaceMargin`. A neighbour outside the view or behind something stays unknown, + /// which is why the edge of what has been scanned so far never counts as a boundary. + mutating func carveFreeSpace(_ frame: FrameInput) { + let depth = frame.depth + let worldToCamera = frame.keyframe.cameraToWorld.inverse + var found = Set() + for (key, voxel) in voxels { + guard let normal = voxel.normal else { continue } + for axis in Self.faceAxes where abs(simd_dot(SIMD3(axis), normal)) < Tuning.alongSurfaceCosine { + let neighbour = key &+ axis + guard voxels[neighbour] == nil, !free.contains(neighbour) else { continue } + let camera = CameraMath.transform(worldToCamera, centre(of: neighbour)) + guard let pixel = CameraMath.project(camera, intrinsics: depth.intrinsics), + pixel.depth >= Tuning.depthRange.lowerBound, + let measured = depth.depth(atU: pixel.u, v: pixel.v) + else { continue } + if measured == 0 || measured > pixel.depth + Tuning.freeSpaceMargin { found.insert(neighbour) } + } + } + free.formUnion(found) + } + + /// Per depth pixel, the distance in pixels (chamfer, 1 and 1.414 steps) to the nearest + /// silhouette: a pixel whose 4-neighbour jumps by more than 10% of its depth or hit nothing. + static func discontinuityDistance(_ depth: DepthMap) -> [Float] { + let w = depth.width, h = depth.height, m = depth.meters + var distance = [Float](repeating: .greatestFiniteMagnitude, count: w * h) + for j in 0.. 0 else { continue } + for (di, dj) in [(1, 0), (-1, 0), (0, 1), (0, -1)] { + let x = i + di, y = j + dj + guard x >= 0, y >= 0, x < w, y < h else { continue } + let n = m[y * w + x] + if n == 0 || abs(n - d) > 0.1 * d { distance[j * w + i] = 0; break } + } + } + } + let diagonal = Float(2).squareRoot() + func relax(_ index: Int, _ x: Int, _ y: Int, _ step: Float) { + guard x >= 0, y >= 0, x < w, y < h else { return } + distance[index] = min(distance[index], distance[y * w + x] + step) + } + for j in 0.. SIMD3? { + let d0 = depth.meters[j * depth.width + i] + func point(_ i: Int, _ j: Int) -> SIMD3? { + guard i >= 0, j >= 0, i < depth.width, j < depth.height else { return nil } + let d = depth.meters[j * depth.width + i] + guard d > 0, abs(d - d0) < 0.1 * d0 else { return nil } + return CameraMath.unproject(u: Float(i) + 0.5, v: Float(j) + 0.5, depth: d, intrinsics: depth.intrinsics) + } + guard let centre = point(i, j) else { return nil } + func tangent(_ plus: SIMD3?, _ minus: SIMD3?) -> SIMD3? { + switch (plus, minus) { + case let (p?, m?): p - m + case let (p?, nil): p - centre + case let (nil, m?): centre - m + case (nil, nil): nil + } + } + guard let tx = tangent(point(i + 1, j), point(i - 1, j)), let ty = tangent(point(i, j + 1), point(i, j - 1)) else { return nil } + let n = simd_cross(tx, ty) + let length = simd_length(n) + guard length > 1e-9 else { return nil } + let unit = n / length + return simd_dot(unit, -centre) < 0 ? -unit : unit + } + + // MARK: Dots + + /// Every edge voxel, plus each group's flat representative. The on-screen cap is applied + /// per keyframe by the timeline, which knows what is in view. + public func dots() -> [FieldDot] { + var dots: [FieldDot] = [] + for (key, voxel) in voxels where voxel.isEdge { + dots.append(dot(key, voxel, kind: .edge)) + } + for (_, key) in representatives { + guard let voxel = voxels[key], !voxel.isEdge else { continue } + dots.append(dot(key, voxel, kind: .flat)) + } + return dots + } + + /// Wall cells holding at least one occupied voxel on the wall plane, for the fog comparison. + public func seenWallCells() -> Set { + var cells = Set() + for key in voxels.keys where key.z == 0 { + let c = centre(of: key) + if let cell = WallCell.containing(x: c.x, y: c.y) { cells.insert(cell) } + } + return cells + } + + private func dot(_ key: VoxelKey, _ voxel: Voxel, kind: DotKind) -> FieldDot { + let centre = centre(of: key) + let wallLike = voxel.normal.map { abs($0.y) < Tuning.cappedNormalMaxY } ?? false + let faceOn = kind != .edge || !wallLike || voxel.faceOnCosine >= cos(Tuning.faceOnDegrees * Float.pi / 180) + return FieldDot( + id: Self.id(for: key), position: centre + voxel.jitter, kind: kind, + views: voxel.views.count, onOccluder: FieldDot.isOnOccluder(centre), faceOn: faceOn, + normal: voxel.normal ?? .zero) + } + + /// Packs the key into 21 bits per axis (plus or minus 52 km at 5 cm), so ids never collide. + static func id(for key: VoxelKey) -> UInt64 { + let bias: Int64 = 1 << 20 + let x = UInt64(Int64(key.x) + bias) & 0x1F_FFFF + let y = UInt64(Int64(key.y) + bias) & 0x1F_FFFF + let z = UInt64(Int64(key.z) + bias) & 0x1F_FFFF + return x | (y << 21) | (z << 42) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/DotScheme.swift b/experiments/live-dots/Sources/LiveDotsCore/DotScheme.swift new file mode 100644 index 000000000..9b9b9960d --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/DotScheme.swift @@ -0,0 +1,33 @@ +import simd + +/// Three looks for the same dot field, to compare on one replay before one goes into the app. +public enum DotScheme: String, Sendable, CaseIterable { + /// White cores, blue halos, edge and flat dots, opacity from evidence. + case hologram + /// Edges only, joined by hairline links into a line drawing; the wall's surface shows nothing. + case constellation + /// As hologram, but a dot is born amber and cools to white over the 6 s after its birth, so + /// the newest part of the map glows. Seeing a dot again never re-warms it: LiDAR re-measures + /// everything in view every keyframe, which kept the whole screen amber. + case ember + + /// Whether a dot of `kind` draws at all. + public func draws(_ kind: DotKind) -> Bool { + switch self { + case .hologram, .ember: true + case .constellation: kind == .edge || kind == .feature + } + } + + /// Links join each edge dot to its two nearest edge neighbours within 8 cm on the same + /// surface (normals within 35 degrees). + public static let linkRadius: Float = 0.08 + public static let linksPerDot = 2 + /// Seconds of playback for an ember dot to cool from amber to white. + public static let emberCooling: Float = 6 + + /// 1 at birth, falling linearly to 0 `emberCooling` seconds later. + public static func warmth(at t: Float, birth: Float) -> Float { + min(max(1 - (t - birth) / emberCooling, 0), 1) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/DotTimeline.swift b/experiments/live-dots/Sources/LiveDotsCore/DotTimeline.swift new file mode 100644 index 000000000..ce66943a2 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/DotTimeline.swift @@ -0,0 +1,189 @@ +import simd + +public enum CaptureMode: String, Sendable, CaseIterable { + case lidar + /// Simulated from the same depth; see `FeatureField`. + case noLidar +} + +/// One dot as the renderer draws it at one keyframe: where it is and the times its animations +/// start, all on the playback clock (seconds, keyframe k shown at k / 4). The shader evaluates +/// the curves, so the vertex buffer changes only at keyframes. +public struct DotSprite: Sendable, Equatable { + public let id: UInt64 + public let position: SIMD3 + public let kind: DotKind + public let onOccluder: Bool + public let birthTime: Float + /// Opacity eases from `fromOpacity` to `toOpacity` over 250 ms starting at `opacityTime`. + public let fromOpacity: Float + public let toOpacity: Float + public let opacityTime: Float + /// When the dot became an edge: -infinity if it was born one, +infinity if it is flat. + public let edgeSince: Float + /// When it stopped being in the field (a feature point dying, a flat dot capped out), + /// +infinity while it is alive. A dying dot fades out over 250 ms. + public let deathTime: Float + /// Unit surface normal, zero when unknown. + public let normal: SIMD3 + + public init( + id: UInt64, position: SIMD3, kind: DotKind, onOccluder: Bool, birthTime: Float, + fromOpacity: Float, toOpacity: Float, opacityTime: Float, edgeSince: Float, deathTime: Float, + normal: SIMD3 = .zero + ) { + self.normal = normal + self.id = id + self.position = position + self.kind = kind + self.onOccluder = onOccluder + self.birthTime = birthTime + self.fromOpacity = fromOpacity + self.toOpacity = toOpacity + self.opacityTime = opacityTime + self.edgeSince = edgeSince + self.deathTime = deathTime + } + + /// The opacity the CPU expects at playback time `t`, ignoring the birth ramp. + public func evidenceOpacity(at t: Float) -> Float { + let progress = (t - opacityTime) / Tuning.evidenceDuration + return fromOpacity + (toOpacity - fromOpacity) * CubicBezier.strongEaseOut(progress) + } +} + +public struct KeyframeState: Sendable { + public let index: Int + /// Playback time at which this keyframe appears. + public let time: Float + /// Dots to draw: alive, in the phone's view and not hidden behind nearer depth, capped at + /// `Tuning.dotCap`, plus this keyframe's deaths. + public let sprites: [DotSprite] + /// Dots in the field, in view or not. + public let fieldCount: Int + public let edgeCount: Int + public let coverage: Float + public let unseenCells: [WallCell] + public let instruction: Instruction +} + +public struct DotTimeline: Sendable { + public let mode: CaptureMode + public let states: [KeyframeState] + + /// Runs both fields over every keyframe once. `progress` gets (done, total) after each. + public static func build( + replay: Replay, progress: (@Sendable (Int, Int) -> Void)? = nil + ) throws(FixtureError) -> (lidar: DotTimeline, noLidar: DotTimeline) { + let instructions = Instruction.sequence(for: replay.keyframes) + var voxels = VoxelField() + var features = FeatureField() + var lidar = Builder(), noLidar = Builder() + for (index, keyframe) in replay.keyframes.enumerated() { + let frame = FrameInput( + index: index, keyframe: keyframe, depth: try replay.depth(for: keyframe), + gradient: GradientPyramid(image: try replay.image(for: keyframe))) + voxels.integrate(frame) + features.integrate(frame) + lidar.append(frame, dots: voxels.dots(), seen: voxels.seenWallCells(), instruction: instructions[index]) + noLidar.append(frame, dots: features.dots(), seen: features.seenWallCells(), instruction: instructions[index]) + progress?(index + 1, replay.keyframes.count) + } + return (DotTimeline(mode: .lidar, states: lidar.states), DotTimeline(mode: .noLidar, states: noLidar.states)) + } + + /// Turns each keyframe's field into sprites with birth, evidence and death times. + struct Builder { + struct Track { + var birthTime: Float + var from: Float + var to: Float + var opacityTime: Float + var edgeSince: Float + var last: FieldDot + } + + private(set) var states: [KeyframeState] = [] + private var tracks: [UInt64: Track] = [:] + + mutating func append(_ frame: FrameInput, dots: [FieldDot], seen: Set, instruction: Instruction) { + let t = Schedule.start(of: frame.index) + let view = View(frame) + var inView: [(sprite: DotSprite, kind: DotKind)] = [] + var alive = Set() + for dot in dots { + alive.insert(dot.id) + let level = dot.opacity + var track: Track + if var existing = tracks[dot.id] { + if level != existing.to { + existing.from = sprite(existing, deathTime: .infinity).evidenceOpacity(at: t) + existing.to = level + existing.opacityTime = t + } + if dot.kind == .edge, existing.edgeSince == .infinity { existing.edgeSince = t } + existing.last = dot + track = existing + } else { + track = Track( + birthTime: t, from: level, to: level, opacityTime: -.infinity, + edgeSince: dot.kind == .edge ? -.infinity : .infinity, last: dot) + } + tracks[dot.id] = track + if view.shows(dot.position) { inView.append((sprite(track, deathTime: .infinity), dot.kind)) } + } + var sprites = Self.capped(inView) + for (id, track) in tracks where !alive.contains(id) { + if view.shows(track.last.position) { sprites.append(sprite(track, deathTime: t)) } + tracks[id] = nil + } + // Additive blending into an 8-bit target rounds after every sprite, so a fixed order + // keeps exports byte-identical between runs. + sprites.sort { $0.id < $1.id } + states.append(KeyframeState( + index: frame.index, time: t, sprites: sprites, fieldCount: dots.count, + edgeCount: dots.count { $0.kind == .edge }, coverage: Coverage.fraction(of: dots), + unseenCells: WallCell.all.filter { !seen.contains($0) }, instruction: instruction)) + } + + /// Past `Tuning.dotCap` dots in view, drops flat dots in hash order; never edge dots. + static func capped(_ inView: [(sprite: DotSprite, kind: DotKind)]) -> [DotSprite] { + guard inView.count > Tuning.dotCap else { return inView.map(\.sprite) } + let edges = inView.filter { $0.kind == .edge }.map(\.sprite) + let others = inView.filter { $0.kind != .edge }.map(\.sprite) + .sorted { StableHash.mix($0.id) < StableHash.mix($1.id) } + return edges + others.prefix(max(Tuning.dotCap - edges.count, 0)) + } + + private func sprite(_ track: Track, deathTime: Float) -> DotSprite { + DotSprite( + id: track.last.id, position: track.last.position, kind: track.last.kind, onOccluder: track.last.onOccluder, + birthTime: track.birthTime, fromOpacity: track.from, toOpacity: track.to, opacityTime: track.opacityTime, + edgeSince: track.edgeSince, deathTime: deathTime, normal: track.last.normal) + } + + /// What the phone shows at a keyframe: the 390 x 844 pt portrait crop of the camera, minus + /// anything the keyframe's depth shows something nearer in front of by + /// `Tuning.occlusionMargin`, so a wall dot never draws on top of the bin. + struct View { + let projection: ScreenProjection + let depth: DepthMap + + init(_ frame: FrameInput) { + projection = ScreenProjection(keyframe: frame.keyframe, viewSize: SIMD2(390, 844)) + depth = frame.depth + } + + func shows(_ p: SIMD3) -> Bool { + guard let screen = projection.screenPoint(p), + screen.x >= 0, screen.y >= 0, screen.x <= projection.viewSize.x, screen.y <= projection.viewSize.y + else { return false } + let camera = CameraMath.transform(projection.worldToCamera, p) + guard let pixel = CameraMath.project(camera, intrinsics: depth.intrinsics), + let measured = depth.depth(atU: pixel.u, v: pixel.v) + else { return false } + return !(measured > 0 && measured < pixel.depth - Tuning.occlusionMargin) + } + } + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/EdgeLinks.swift b/experiments/live-dots/Sources/LiveDotsCore/EdgeLinks.swift new file mode 100644 index 000000000..36169b2eb --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/EdgeLinks.swift @@ -0,0 +1,39 @@ +import simd + +/// The constellation scheme's hairlines. Each edge dot links to its `DotScheme.linksPerDot` +/// nearest edge dots within `DotScheme.linkRadius` whose normals agree within 35 degrees, so a +/// link follows an outline and never jumps from the wall to the bin in front of it. +public enum EdgeLinks { + /// Index pairs into `sprites`, lower index first, each pair once. + public static func links(among sprites: [DotSprite]) -> [SIMD2] { + let radius = DotScheme.linkRadius + func cell(_ p: SIMD3) -> SIMD3 { SIMD3((p / radius).rounded(.down)) } + let edges = sprites.indices.filter { sprites[$0].kind == .edge && simd_length(sprites[$0].normal) > 0.5 } + var grid: [SIMD3: [Int]] = [:] + for i in edges { grid[cell(sprites[i].position), default: []].append(i) } + + var pairs = Set>() + for i in edges { + let a = sprites[i] + var nearest: [(index: Int, distance: Float)] = [] + let c = cell(a.position) + for dz in Int32(-1)...1 { + for dy in Int32(-1)...1 { + for dx in Int32(-1)...1 { + for j in grid[c &+ SIMD3(dx, dy, dz)] ?? [] where j != i { + let b = sprites[j] + let distance = simd_distance(a.position, b.position) + guard distance <= radius, !VoxelField.normalsDisagree(a.normal, b.normal) else { continue } + nearest.append((j, distance)) + } + } + } + } + nearest.sort { ($0.distance, $0.index) < ($1.distance, $1.index) } + for (j, _) in nearest.prefix(DotScheme.linksPerDot) { + pairs.insert(SIMD2(Int32(min(i, j)), Int32(max(i, j)))) + } + } + return pairs.sorted { ($0.x, $0.y) < ($1.x, $1.y) } + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Evidence.swift b/experiments/live-dots/Sources/LiveDotsCore/Evidence.swift new file mode 100644 index 000000000..a7afdfbc5 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/Evidence.swift @@ -0,0 +1,39 @@ +import simd + +public enum Evidence { + /// Opacity for a dot seen from `views` distinct directions: 45% at one, rising linearly to + /// 90% at four and holding there. Zero views draws nothing. + /// With `faceOn` false (an edge no view has seen within 30 degrees of face-on) the result + /// is capped at 60%. + public static func opacity(views: Int, faceOn: Bool = true) -> Float { + guard views > 0 else { return 0 } + let level = min(0.45 + 0.15 * Float(views - 1), 0.9) + return faceOn ? level : min(level, Tuning.obliqueEdgeOpacityCap) + } +} + +/// The distinct directions a point has been seen from. A direction is new only when it differs +/// from every stored one by more than `Tuning.viewSeparationDegrees`. +public struct ViewDirections: Sendable, Equatable { + public private(set) var directions: [SIMD3] = [] + /// Opacity stops rising at four views and the ring needs two, so eight is plenty. + public static let capacity = 8 + + public init() {} + + public var count: Int { directions.count } + + /// Adds `direction` (any length) if it is new. Returns whether it was. + @discardableResult + public mutating func insert(_ direction: SIMD3) -> Bool { + let length = simd_length(direction) + guard length > 1e-6, directions.count < Self.capacity else { return false } + let unit = direction / length + let limit = cos(Tuning.viewSeparationDegrees * Float.pi / 180) + for existing in directions where simd_dot(existing, unit) >= limit { + return false + } + directions.append(unit) + return true + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/FieldDot.swift b/experiments/live-dots/Sources/LiveDotsCore/FieldDot.swift new file mode 100644 index 000000000..cb9169d50 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/FieldDot.swift @@ -0,0 +1,51 @@ +import simd + +public enum DotKind: UInt8, Sendable { + /// A LiDAR voxel on a flat surface; one per 2 x 2 x 2 group is drawn. + case flat + /// A LiDAR voxel on a crease, an occupancy boundary or an image edge; every one is drawn. + case edge + /// A simulated ARKit raw feature point (no-LiDAR mode). + case feature + /// A simulated plane-detection dot on the wall (no-LiDAR mode). + case plane +} + +/// A dot a field wants drawn at the current keyframe, before timing and occlusion. +public struct FieldDot: Sendable, Equatable { + /// Stable for the life of the dot, so the timeline can tell a birth from a survivor. + public let id: UInt64 + public let position: SIMD3 + public let kind: DotKind + /// Distinct view directions so far; sets the opacity. + public let views: Int + /// On the thing standing in front of the wall (the bin), drawn in Hidden violet. + public let onOccluder: Bool + /// Some view saw it within 30 degrees of face-on. Only edge dots are ever false. + public let faceOn: Bool + /// Unit surface normal, zero when unknown (feature points). + public let normal: SIMD3 + + public init( + id: UInt64, position: SIMD3, kind: DotKind, views: Int, onOccluder: Bool, faceOn: Bool = true, + normal: SIMD3 = .zero + ) { + self.id = id + self.position = position + self.kind = kind + self.views = views + self.onOccluder = onOccluder + self.faceOn = faceOn + self.normal = normal + } + + public var opacity: Float { + Evidence.opacity(views: views, faceOn: faceOn) + } + + /// The brief's occluder test, plus a ground band so the lawn in front of the wall stays + /// Hologram: in front of the wall by more than 0.25 m and above the ground voxel layer. + public static func isOnOccluder(_ p: SIMD3) -> Bool { + p.z > Tuning.occluderMinZ && p.y > Tuning.groundBand + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/FixtureError.swift b/experiments/live-dots/Sources/LiveDotsCore/FixtureError.swift new file mode 100644 index 000000000..30c851933 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/FixtureError.swift @@ -0,0 +1,40 @@ +import Foundation + +/// Everything that can be wrong with a replay folder. Each message names the file and, for size +/// mismatches, both counts, so a broken fixture says exactly what to regenerate. +public enum FixtureError: Error, Equatable, CustomStringConvertible, LocalizedError { + case unreadable(path: String, reason: String) + case wrongFormat(path: String, found: String) + case unsupportedVersion(path: String, found: Int) + case malformed(path: String, detail: String) + case noDepth(path: String, keyframe: String) + case depthByteCount(path: String, found: Int, expected: Int, width: Int, height: Int) + case confidenceByteCount(path: String, found: Int, expected: Int, width: Int, height: Int) + case undecodableImage(path: String) + case imageSize(path: String, found: SIMD2, expected: SIMD2) + + public var description: String { + switch self { + case let .unreadable(path, reason): + "can't read \(path): \(reason)" + case let .wrongFormat(path, found): + "\(path) has format \"\(found)\", expected \"measure-lab-session\"" + case let .unsupportedVersion(path, found): + "\(path) has formatVersion \(found), this reader supports 2" + case let .malformed(path, detail): + "\(path) is malformed: \(detail)" + case let .noDepth(path, keyframe): + "\(path): keyframe \(keyframe) lists no depth, and LiDAR mode needs depth on every keyframe" + case let .depthByteCount(path, found, expected, width, height): + "\(path) has \(found) bytes, expected \(expected) for \(width) x \(height) Float32" + case let .confidenceByteCount(path, found, expected, width, height): + "\(path) has \(found) bytes, expected \(expected) for \(width) x \(height) UInt8" + case let .undecodableImage(path): + "\(path) is not a decodable image" + case let .imageSize(path, found, expected): + "\(path) is \(found.x) x \(found.y) pixels, session.json says \(expected.x) x \(expected.y)" + } + } + + public var errorDescription: String? { description } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/FogMask.swift b/experiments/live-dots/Sources/LiveDotsCore/FogMask.swift new file mode 100644 index 000000000..e51c8880f --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/FogMask.swift @@ -0,0 +1,33 @@ +import Foundation + +/// CPU copies of the fog shader's two scalar rules (Shaders.swift, `blurFragment` and +/// `lagFragment`), so the tests can pin them. See the header of Shaders.swift's fog section for +/// the whole layer and what it borrows. +public enum FogMask { + /// Reveal strength after blur: clamped to 1, then smoothstep from 0.12 to 0.55. + public static let revealLow: Float = 0.12 + public static let revealHigh: Float = 0.55 + /// The lagged mask eases up toward the reveal with this time constant, so the fog lifts a + /// beat after the dots arrive. + public static let lagTimeConstant: Float = 0.6 + /// Fog returns faster than it lifts. A single 600 ms lag both ways left the sky clear for most + /// of a second after the tilt cut, the reveal trailing a view that had moved on. + public static let returnTimeConstant: Float = 0.12 + /// Under Reduce Motion both directions are linear fades of these lengths instead. + public static let reducedFade: Float = 0.4 + public static let reducedReturn: Float = 0.12 + + public static func shape(_ accumulated: Float) -> Float { + let x = min(max((min(accumulated, 1) - revealLow) / (revealHigh - revealLow), 0), 1) + return x * x * (3 - 2 * x) + } + + public static func lagStep(from previous: Float, toward target: Float, dt: Float, reduceMotion: Bool) -> Float { + let lifting = target > previous + if reduceMotion { + let step = dt / (lifting ? reducedFade : reducedReturn) + return previous + min(max(target - previous, -step), step) + } + return previous + (target - previous) * (1 - exp(-dt / (lifting ? lagTimeConstant : returnTimeConstant))) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/FrameInput.swift b/experiments/live-dots/Sources/LiveDotsCore/FrameInput.swift new file mode 100644 index 000000000..68c679bf1 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/FrameInput.swift @@ -0,0 +1,14 @@ +/// One keyframe's inputs to a field. +public struct FrameInput: Sendable { + public let index: Int + public let keyframe: Keyframe + public let depth: DepthMap + public let gradient: GradientPyramid + + public init(index: Int, keyframe: Keyframe, depth: DepthMap, gradient: GradientPyramid) { + self.index = index + self.keyframe = keyframe + self.depth = depth + self.gradient = gradient + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/GradientPyramid.swift b/experiments/live-dots/Sources/LiveDotsCore/GradientPyramid.swift new file mode 100644 index 000000000..d0d78c2e7 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/GradientPyramid.swift @@ -0,0 +1,87 @@ +import Foundation + +/// Colour gradient of a keyframe image, measured at the scale of a voxel. +/// +/// Level 0 is a Sobel on the 640 x 480 JPEG; each further level halves the image by 2 x 2 +/// averaging first. A depth sample reads the level whose pixel is closest to one voxel (5 cm) at +/// its depth, so a 1 cm mortar joint averages away while a painted outline, a step in colour +/// wider than the pixel, keeps its full height. The magnitude is the Sobel of the strongest RGB +/// channel divided by 4, which reads a sharp step of height c (0 to 1) as c at every level. +public struct GradientPyramid: Sendable { + public struct Level: Sendable { + public let width: Int + public let height: Int + public let magnitude: [Float] + } + + public static let levelCount = 5 + public let levels: [Level] + + public init(image: RGBImage) { + var width = image.width, height = image.height + var planes: [[Float]] = (0..<3).map { channel in + (0..<(width * height)).map { Float(image.rgba[$0 * 4 + channel]) / 255 } + } + var levels: [Level] = [] + for level in 0.. Int { + let pixelsPerVoxel = Tuning.voxelSize * focal / max(depth, 0.01) + let level = Int(log2(max(pixelsPerVoxel, 1)).rounded()) + return min(max(level, 0), levelCount - 1) + } + + /// Bilinear magnitude at continuous level-0 pixel coordinates (u, v). + public func magnitude(u: Float, v: Float, level: Int) -> Float { + let l = levels[level] + let scale = Float(1 << level) + let x = min(max(u / scale - 0.5, 0), Float(l.width - 1)) + let y = min(max(v / scale - 0.5, 0), Float(l.height - 1)) + let x0 = Int(x), y0 = Int(y) + let x1 = min(x0 + 1, l.width - 1), y1 = min(y0 + 1, l.height - 1) + let fx = x - Float(x0), fy = y - Float(y0) + let m = l.magnitude + let top = m[y0 * l.width + x0] * (1 - fx) + m[y0 * l.width + x1] * fx + let bottom = m[y1 * l.width + x0] * (1 - fx) + m[y1 * l.width + x1] * fx + return top * (1 - fy) + bottom * fy + } + + private static func sobel(_ planes: [[Float]], width: Int, height: Int) -> [Float] { + var out = [Float](repeating: 0, count: width * height) + for plane in planes { + plane.withUnsafeBufferPointer { p in + for y in 0.. out[y0 + x] { out[y0 + x] = m } + } + } + } + } + return out + } + + private static func halve(_ planes: [[Float]], width: Int, height: Int) -> ([[Float]], Int, Int) { + let w = width / 2, h = height / 2 + let halved = planes.map { p in + (0..<(w * h)).map { i -> Float in + let x = (i % w) * 2, y = (i / w) * 2 + return (p[y * width + x] + p[y * width + x + 1] + p[(y + 1) * width + x] + p[(y + 1) * width + x + 1]) / 4 + } + } + return (halved, w, h) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Instruction.swift b/experiments/live-dots/Sources/LiveDotsCore/Instruction.swift new file mode 100644 index 000000000..5f3d37686 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/Instruction.swift @@ -0,0 +1,35 @@ +/// The one line of guidance at the bottom of the phone, as a small state machine driven by the +/// replay: point at the meter, hold still while it locks, walk left, walk right, recover from +/// the tilt, done. Every change is one of these transitions, so the 3 s minimum between other +/// changes never has anything to hold back on this replay. +public enum Instruction: Sendable, Equatable { + case pointAtMeter, holdStill, walkLeft, walkRight, tiltDown, done + + public var text: String { + switch self { + case .pointAtMeter: "Point at your electric meter" + case .holdStill: "Hold still" + case .walkLeft: "Walk slowly to your left" + case .walkRight: "Now walk to your right" + case .tiltDown: "Tilt back down to the wall" + case .done: "That's the whole wall" + } + } + + /// One instruction per keyframe: point at the meter for the first two, hold still on the + /// third, walk left until the camera has been left of x = -2.5 m, then walk right. The + /// fixture's last three keyframes tilt up at the sky: the first two ask to tilt back down, + /// and the last says the wall is done. + public static func sequence(for keyframes: [Keyframe]) -> [Instruction] { + var reachedLeft = false + let count = keyframes.count + return keyframes.enumerated().map { index, keyframe in + if keyframe.cameraPosition.x < -2.5 { reachedLeft = true } + if index == count - 1 { return .done } + if index >= count - 3 { return .tiltDown } + if index < Schedule.holdIndex { return .pointAtMeter } + if index == Schedule.holdIndex { return .holdStill } + return reachedLeft ? .walkRight : .walkLeft + } + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/RGBImage.swift b/experiments/live-dots/Sources/LiveDotsCore/RGBImage.swift new file mode 100644 index 000000000..65ac1ca00 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/RGBImage.swift @@ -0,0 +1,36 @@ +import CoreGraphics +import Foundation +import ImageIO + +/// A decoded 8-bit sRGB image, RGBA byte order, top row first. +public struct RGBImage: Sendable { + public let width: Int + public let height: Int + public let rgba: [UInt8] + + public init(width: Int, height: Int, rgba: [UInt8]) { + precondition(rgba.count == width * height * 4, "rgba must hold 4 bytes per pixel") + self.width = width + self.height = height + self.rgba = rgba + } + + public init(jpeg data: Data, path: String) throws(FixtureError) { + guard let source = CGImageSourceCreateWithData(data as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil), + let space = CGColorSpace(name: CGColorSpace.sRGB) + else { throw .undecodableImage(path: path) } + let width = image.width, height = image.height + var bytes = [UInt8](repeating: 0, count: width * height * 4) + let drawn = bytes.withUnsafeMutableBytes { raw -> Bool in + guard let context = CGContext( + data: raw.baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, + space: space, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) + else { return false } + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + return true + } + guard drawn else { throw .undecodableImage(path: path) } + self.init(width: width, height: height, rgba: bytes) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/RecognisedBox.swift b/experiments/live-dots/Sources/LiveDotsCore/RecognisedBox.swift new file mode 100644 index 000000000..422930c3d --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/RecognisedBox.swift @@ -0,0 +1,96 @@ +import simd + +/// A thing the app would recognise on the wall. Simulated: the rectangles are the synthetic +/// fixture's painted objects, taken from its generator, not detected. +public struct RecognisedBox: Sendable, Equatable { + public let caption: String + /// Wall-plane rectangle on z = 0: (minX, minY), (maxX, maxY). + public let min: SIMD2 + public let max: SIMD2 + /// The electric meter is the anchor: solid blue from the first keyframe it is in view. + public let isMeter: Bool + + public static let fixture: [RecognisedBox] = [ + RecognisedBox(caption: "Electric meter", min: SIMD2(-0.15, 1.3), max: SIMD2(0.15, 1.7), isMeter: true), + RecognisedBox(caption: "Gas meter", min: SIMD2(-1.35, 0.3), max: SIMD2(-1.05, 0.8), isMeter: false), + RecognisedBox(caption: "Door", min: SIMD2(-3.4, 0), max: SIMD2(-2.5, 2.05), isMeter: false), + RecognisedBox(caption: "Window", min: SIMD2(2.0, 0.9), max: SIMD2(3.0, 2.0), isMeter: false), + ] + + /// Corners in world space, clockwise from top-left as the wall is seen from the front. + public var corners: [SIMD3] { + [SIMD3(min.x, max.y, 0), SIMD3(max.x, max.y, 0), SIMD3(max.x, min.y, 0), SIMD3(min.x, min.y, 0)] + } + + var centre: SIMD3 { SIMD3((min + max) / 2, 0) } + + private func point(_ u: Float, _ v: Float) -> SIMD3 { + SIMD3(min.x + (max.x - min.x) * u, min.y + (max.y - min.y) * v, 0) + } + + private static func inFrame(_ p: SIMD3, _ projection: ScreenProjection) -> Bool { + guard let s = projection.screenPoint(p) else { return false } + return s.x >= 0 && s.y >= 0 && s.x <= projection.viewSize.x && s.y <= projection.viewSize.y + } + + /// Share of an 11 x 11 grid over the box that lands on screen. + public func visibleFraction(_ projection: ScreenProjection) -> Float { + var inside = 0 + for i in 0...10 { + for j in 0...10 where Self.inFrame(point(Float(i) / 10, Float(j) / 10), projection) { inside += 1 } + } + return Float(inside) / 121 + } + + /// Both vertical edges have at least half of their 11 samples on screen. + public func sidesInFrame(_ projection: ScreenProjection) -> Bool { + [Float(0), 1].allSatisfy { u in + (0...10).count { Self.inFrame(point(u, Float($0) / 10), projection) } >= 6 + } + } + + /// The camera sees the box within 30 degrees of face-on. + public func isFaceOn(from camera: SIMD3) -> Bool { + simd_dot(simd_normalize(camera - centre), SIMD3(0, 0, 1)) >= cos(Float.pi / 6) + } +} + +/// When each box appeared and when it settled, on the playback clock. Sticky: a box that has +/// appeared stays (it simply goes off screen), and a solid box never goes back to dashed. +public struct BoxState: Sendable, Equatable { + public var appearedAt: Float? + public var solidAt: Float? + + public init(appearedAt: Float? = nil, solidAt: Float? = nil) { + self.appearedAt = appearedAt + self.solidAt = solidAt + } + + /// Boxes other than the meter appear at 60% in view (dashed, not settled) and turn solid + /// the first time a keyframe sees them within 30 degrees of face-on with both side edges on + /// screen. The meter appears solid as soon as its centre is on screen. + public mutating func update(_ box: RecognisedBox, projection: ScreenProjection, at t: Float) { + if box.isMeter { + if appearedAt == nil, box.visibleFraction(projection) > 0, + let s = projection.screenPoint(box.centre), + s.x >= 0, s.y >= 0, s.x <= projection.viewSize.x, s.y <= projection.viewSize.y { + appearedAt = t + solidAt = t + } + return + } + if appearedAt == nil, box.visibleFraction(projection) >= 0.6 { appearedAt = t } + if appearedAt != nil, solidAt == nil, box.isFaceOn(from: projection.cameraPosition), box.sidesInFrame(projection) { + solidAt = t + } + } + + public static func timeline(for keyframes: [Keyframe], boxes: [RecognisedBox] = RecognisedBox.fixture) -> [[BoxState]] { + var states = boxes.map { _ in BoxState() } + return keyframes.enumerated().map { index, keyframe in + let projection = ScreenProjection(keyframe: keyframe, viewSize: SIMD2(390, 844)) + for b in boxes.indices { states[b].update(boxes[b], projection: projection, at: Schedule.start(of: index)) } + return states + } + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Replay.swift b/experiments/live-dots/Sources/LiveDotsCore/Replay.swift new file mode 100644 index 000000000..b780b8b1e --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/Replay.swift @@ -0,0 +1,135 @@ +import Foundation +import simd + +/// One recorded keyframe: its image, camera and depth file. +public struct Keyframe: Sendable, Equatable { + public let id: String + /// Relative to the session folder. + public let imagePath: String + public let width: Int + public let height: Int + /// fx, fy, cx, cy in pixels of the unrotated landscape JPEG. + public let intrinsics: SIMD4 + /// Column-major camera-to-world. Camera +x right, +y up, looking down -z. + public let cameraToWorld: simd_float4x4 + /// Seconds of device uptime at capture. + public let timestamp: Double + public let depth: DepthReference? + + public var cameraPosition: SIMD3 { + let c = cameraToWorld.columns.3 + return SIMD3(c.x, c.y, c.z) + } +} + +/// Where a keyframe's LiDAR depth lives: Float32 metres, little-endian, row-major, and UInt8 +/// confidence in the same layout. +public struct DepthReference: Sendable, Equatable { + public let file: String + public let confidenceFile: String? + public let width: Int + public let height: Int +} + +/// A "measure-lab-session" version 2 replay folder. Reads the keyframes and ignores every other +/// field, because the lab format keeps gaining fields this prototype has no use for. +public struct Replay: Sendable { + public let folder: URL + /// Sorted by timestamp, then id. + public let keyframes: [Keyframe] + + public static func load(folder: URL) throws(FixtureError) -> Replay { + let url = folder.appendingPathComponent("session.json") + let data = try read(url) + let file: SessionFile + do { + file = try JSONDecoder().decode(SessionFile.self, from: data) + } catch { + throw .malformed(path: url.path, detail: String(describing: error)) + } + guard file.format == "measure-lab-session" else { throw .wrongFormat(path: url.path, found: file.format) } + guard file.formatVersion == 2 else { throw .unsupportedVersion(path: url.path, found: file.formatVersion) } + guard !file.keyframes.isEmpty else { throw .malformed(path: url.path, detail: "keyframes is empty") } + + var keyframes: [Keyframe] = [] + for (index, entry) in file.keyframes.enumerated() { + guard entry.intrinsics.count == 4 else { + throw .malformed(path: url.path, detail: "keyframes[\(index)].intrinsics has \(entry.intrinsics.count) values, expected 4") + } + guard entry.pose.count == 16 else { + throw .malformed(path: url.path, detail: "keyframes[\(index)].pose has \(entry.pose.count) values, expected 16") + } + let p = entry.pose.map(Float.init) + let pose = simd_float4x4(columns: ( + SIMD4(p[0], p[1], p[2], p[3]), SIMD4(p[4], p[5], p[6], p[7]), + SIMD4(p[8], p[9], p[10], p[11]), SIMD4(p[12], p[13], p[14], p[15]) + )) + let k = entry.intrinsics.map(Float.init) + keyframes.append(Keyframe( + id: entry.id, imagePath: entry.img, width: entry.w, height: entry.h, + intrinsics: SIMD4(k[0], k[1], k[2], k[3]), cameraToWorld: pose, timestamp: entry.timestamp, + depth: entry.depth.map { DepthReference(file: $0.file, confidenceFile: $0.confidenceFile, width: $0.w, height: $0.h) } + )) + } + keyframes.sort { $0.timestamp != $1.timestamp ? $0.timestamp < $1.timestamp : $0.id < $1.id } + return Replay(folder: folder, keyframes: keyframes) + } + + /// The keyframe's depth, with intrinsics scaled from the JPEG's to the depth map's size. + public func depth(for keyframe: Keyframe) throws(FixtureError) -> DepthMap { + guard let reference = keyframe.depth else { + throw .noDepth(path: folder.appendingPathComponent("session.json").path, keyframe: keyframe.id) + } + let depthURL = folder.appendingPathComponent(reference.file) + let confidenceURL = reference.confidenceFile.map { folder.appendingPathComponent($0) } + return try DepthMap( + depth: Self.read(depthURL), depthPath: depthURL.path, + confidence: confidenceURL.map(Self.read), confidencePath: confidenceURL?.path, + width: reference.width, height: reference.height, + intrinsics: keyframe.intrinsics * SIMD4( + Float(reference.width) / Float(keyframe.width), Float(reference.height) / Float(keyframe.height), + Float(reference.width) / Float(keyframe.width), Float(reference.height) / Float(keyframe.height)) + ) + } + + public func image(for keyframe: Keyframe) throws(FixtureError) -> RGBImage { + let url = folder.appendingPathComponent(keyframe.imagePath) + let image = try RGBImage(jpeg: Self.read(url), path: url.path) + guard image.width == keyframe.width, image.height == keyframe.height else { + throw .imageSize(path: url.path, found: SIMD2(image.width, image.height), expected: SIMD2(keyframe.width, keyframe.height)) + } + return image + } + + private static func read(_ url: URL) throws(FixtureError) -> Data { + do { + return try Data(contentsOf: url) + } catch { + throw .unreadable(path: url.path, reason: error.localizedDescription) + } + } +} + +private struct SessionFile: Decodable { + let format: String + let formatVersion: Int + let keyframes: [Entry] + + struct Entry: Decodable { + let id: String + let img: String + let w: Int + let h: Int + let intrinsics: [Double] + let pose: [Double] + let timestamp: Double + let depth: Depth? + } + + struct Depth: Decodable { + let file: String + let confidenceFile: String? + let w: Int + let h: Int + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift b/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift new file mode 100644 index 000000000..2637afe10 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift @@ -0,0 +1,26 @@ +/// When each keyframe appears on the playback clock. Keyframes play at 4 per second, except the +/// third, which holds for the 1 s "Hold still" ring fill and the 150 ms bracket flash after it. +public enum Schedule { + public static let holdIndex = 2 + public static let holdDuration: Float = 1 + public static let flashDuration: Float = 0.15 + + static var interval: Float { 1 / Tuning.keyframesPerSecond } + + /// Playback seconds at which keyframe `index` appears. + public static func start(of index: Int) -> Float { + Float(index) * interval + (index > holdIndex ? holdDuration + flashDuration - interval : 0) + } + + /// The keyframe on screen at playback time `t`. + public static func keyframe(at t: Float, count: Int) -> Int { + var index = 0 + while index + 1 < count, start(of: index + 1) <= t { index += 1 } + return index + } + + /// When the last keyframe's slot ends. + public static func end(count: Int) -> Float { + start(of: count) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/ScreenProjection.swift b/experiments/live-dots/Sources/LiveDotsCore/ScreenProjection.swift new file mode 100644 index 000000000..16645b37c --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/ScreenProjection.swift @@ -0,0 +1,57 @@ +import simd + +/// Where world points land on a portrait phone screen that shows the keyframe's landscape sensor +/// image turned 90 degrees clockwise and scaled to fill (cropping the sides, as ARKit's display +/// transform does). The phone is held in portrait, so camera +x is world down: landscape pixel +/// (u, v) becomes portrait pixel (imageHeight - v, u). +public struct ScreenProjection: Sendable { + /// Screen size in any unit (points or pixels), origin top-left, y down. + public let viewSize: SIMD2 + /// Landscape image size, (640, 480) for the fixture. + public let imageSize: SIMD2 + public let intrinsics: SIMD4 + public let worldToCamera: simd_float4x4 + public let cameraPosition: SIMD3 + /// Screen units per portrait image pixel. + public let scale: Float + /// Screen position of the portrait image's top-left corner. + public let offset: SIMD2 + + public init(keyframe: Keyframe, viewSize: SIMD2) { + self.viewSize = viewSize + imageSize = SIMD2(Float(keyframe.width), Float(keyframe.height)) + intrinsics = keyframe.intrinsics + worldToCamera = keyframe.cameraToWorld.inverse + cameraPosition = keyframe.cameraPosition + let portrait = SIMD2(imageSize.y, imageSize.x) + scale = max(viewSize.x / portrait.x, viewSize.y / portrait.y) + offset = (viewSize - portrait * scale) / 2 + } + + public func portraitPixel(u: Float, v: Float) -> SIMD2 { + SIMD2(imageSize.y - v, u) + } + + /// Screen position of a world point, or nil behind the camera. + public func screenPoint(_ world: SIMD3) -> SIMD2? { + let camera = CameraMath.transform(worldToCamera, world) + guard let pixel = CameraMath.project(camera, intrinsics: intrinsics) else { return nil } + return offset + portraitPixel(u: pixel.u, v: pixel.v) * scale + } + + /// World to Metal clip space (x right, y up, depth fixed at 0.5), the same mapping as + /// `screenPoint`. Points behind the camera get w < 0 and the GPU clips them. + public var clipMatrix: simd_float4x4 { + let (fx, fy, cx, cy) = (intrinsics.x, intrinsics.y, intrinsics.z, intrinsics.w) + // Rows act on camera-space (x, y, z, 1); w is the depth along -z. + let w = SIMD4(0, 0, -1, 0) + let portraitX = SIMD4(0, fy, cy - imageSize.y, 0) + let portraitY = SIMD4(fx, 0, -cx, 0) + let screenX = offset.x * w + scale * portraitX + let screenY = offset.y * w + scale * portraitY + let clipX = (2 / viewSize.x) * screenX - w + let clipY = w - (2 / viewSize.y) * screenY + let cameraToClip = simd_float4x4(rows: [clipX, clipY, 0.5 * w, w]) + return cameraToClip * worldToCamera + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/SimulatedFeatures.swift b/experiments/live-dots/Sources/LiveDotsCore/SimulatedFeatures.swift new file mode 100644 index 000000000..de5de6830 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/SimulatedFeatures.swift @@ -0,0 +1,181 @@ +import simd + +/// The no-LiDAR dot field, simulated from the fixture's depth because the fixture has no +/// feature points: stand-ins for ARKit's `rawFeaturePoints` and for plane detection on the wall. +/// It reads depth, which a phone without LiDAR does not have, so it shows the look of this mode, +/// not its accuracy. +public struct FeatureField: Sendable { + struct Track { + let id: UInt64 + /// First observation; a feature dot never moves after birth. + let position: SIMD3 + var views = ViewDirections() + /// Keyframe indices it was matched in, newest last, trimmed to the window. + var observed: [Int] = [] + var lastSeen: Double + } + + private var tracks: [UInt64: Track] = [:] + /// 8 cm cells to the tracks inside them, for the re-observation match. + private var grid: [SIMD3: [UInt64]] = [:] + private var nextTrack: UInt64 = 0 + private var planeViews: [SIMD2: ViewDirections] = [:] + private var frameIndex = 0 + private var frameTime: Double = 0 + /// Candidates kept by the last `integrate`, after thinning. + public private(set) var lastSampleCount = 0 + + static let featureTag: UInt64 = 1 << 63 + static let planeTag: UInt64 = 1 << 62 + static let planeColumns = Int(((FixtureWall.xRange.upperBound - FixtureWall.xRange.lowerBound) / Tuning.planeCell).rounded()) + static let planeRows = Int(((FixtureWall.yRange.upperBound - FixtureWall.yRange.lowerBound) / Tuning.planeCell).rounded()) + + public init() {} + + public mutating func integrate(_ frame: FrameInput) { + frameIndex = frame.index + frameTime = frame.keyframe.timestamp + let camera = frame.keyframe.cameraPosition + let samples = strongGradientSamples(frame) + lastSampleCount = samples.count + for point in samples { + if let id = nearestTrack(to: point) { + tracks[id]?.lastSeen = frameTime + if tracks[id]?.observed.last != frameIndex { tracks[id]?.observed.append(frameIndex) } + tracks[id]?.views.insert(camera - point) + } else { + var track = Track(id: nextTrack, position: point, lastSeen: frameTime) + track.observed = [frameIndex] + track.views.insert(camera - point) + tracks[track.id] = track + grid[cell(point), default: []].append(track.id) + nextTrack += 1 + } + } + for id in tracks.keys.sorted() { + guard var track = tracks[id] else { continue } + track.observed.removeAll { $0 <= frameIndex - Tuning.featureWindow } + if frameTime - track.lastSeen > Tuning.featureLifetime { + tracks[id] = nil + grid[cell(track.position)]?.removeAll { $0 == id } + } else { + tracks[id] = track + } + } + coverPlane(frame) + } + + /// Depth samples whose image gradient reaches `Tuning.featureGradientThreshold`, merged to + /// one per 4 cm cell and thinned to the `Tuning.featureTarget` lowest cell hashes. + func strongGradientSamples(_ frame: FrameInput) -> [SIMD3] { + Array(Self.candidates(frame, threshold: Tuning.featureGradientThreshold).prefix(Tuning.featureTarget)) + } + + /// All candidate features before thinning, lowest cell hash first. + public static func candidates(_ frame: FrameInput, threshold: Float) -> [SIMD3] { + let depth = frame.depth + let pose = frame.keyframe.cameraToWorld + let imageScale = Float(frame.keyframe.width) / Float(depth.width) + let focal = frame.keyframe.intrinsics.x + var cells: [SIMD3: SIMD3] = [:] + for j in 0..= Tuning.minimumConfidence, Tuning.depthRange.contains(d) else { continue } + let u = Float(i) + 0.5, v = Float(j) + 0.5 + let level = GradientPyramid.level(depth: d, focal: focal) + guard frame.gradient.magnitude(u: u * imageScale, v: v * imageScale, level: level) >= threshold else { continue } + let world = CameraMath.transform(pose, CameraMath.unproject(u: u, v: v, depth: d, intrinsics: depth.intrinsics)) + let key = SIMD3((world / Tuning.featureCell).rounded(.down)) + if cells[key] == nil { cells[key] = world } + } + } + let ranked = cells.map { (point: $0.value, hash: StableHash.hash($0.key, seed: 0xFEA7)) }.sorted { $0.hash < $1.hash } + return ranked.map(\.point) + } + + private func cell(_ p: SIMD3) -> SIMD3 { + SIMD3((p / Tuning.featureMatchRadius).rounded(.down)) + } + + private func nearestTrack(to p: SIMD3) -> UInt64? { + let c = cell(p) + var best: (id: UInt64, distance: Float)? + for dz in Int32(-1)...1 { + for dy in Int32(-1)...1 { + for dx in Int32(-1)...1 { + for id in grid[c &+ SIMD3(dx, dy, dz)] ?? [] { + guard let track = tracks[id] else { continue } + let distance = simd_distance(track.position, p) + guard distance <= Tuning.featureMatchRadius else { continue } + if let current = best, current.distance < distance || (current.distance == distance && current.id < id) { continue } + best = (id, distance) + } + } + } + } + return best?.id + } + + /// Plane cells on the wall whose centre is in view within `Tuning.planeRange` and not hidden + /// behind something nearer. The depth check stands in for the plane's own extent, and keeps + /// plane dots off the bin. + mutating func coverPlane(_ frame: FrameInput) { + let depth = frame.depth + let worldToCamera = frame.keyframe.cameraToWorld.inverse + let camera = frame.keyframe.cameraPosition + for row in 0.. 0, abs(measured - pixel.depth) < Tuning.freeSpaceMargin + else { continue } + planeViews[SIMD2(Int32(column), Int32(row)), default: ViewDirections()].insert(camera - centre) + } + } + } + + static func planeCentre(column: Int, row: Int) -> SIMD3 { + SIMD3( + FixtureWall.xRange.lowerBound + (Float(column) + 0.5) * Tuning.planeCell, + FixtureWall.yRange.lowerBound + (Float(row) + 0.5) * Tuning.planeCell, 0) + } + + private func isAlive(_ track: Track) -> Bool { + track.observed.count >= Tuning.featureMinObservations && frameTime - track.lastSeen <= Tuning.featureLifetime + } + + public func dots() -> [FieldDot] { + var dots: [FieldDot] = [] + for track in tracks.values where isAlive(track) { + dots.append(FieldDot( + id: Self.featureTag | track.id, position: track.position, kind: .feature, + views: track.views.count, onOccluder: FieldDot.isOnOccluder(track.position))) + } + for (cell, views) in planeViews { + let key = SIMD3(cell.x, cell.y, 0) + let position = Self.planeCentre(column: Int(cell.x), row: Int(cell.y)) + + Jitter.offset(for: key, normal: SIMD3(0, 0, 1), cellSize: Tuning.planeCell) + dots.append(FieldDot( + id: Self.planeTag | UInt64(UInt32(cell.x)) << 16 | UInt64(UInt32(cell.y)), + position: position, kind: .plane, views: views.count, onOccluder: false, + normal: SIMD3(0, 0, 1))) + } + return dots + } + + public func seenWallCells() -> Set { + var cells = Set() + for cell in planeViews.keys { + let c = Self.planeCentre(column: Int(cell.x), row: Int(cell.y)) + if let wallCell = WallCell.containing(x: c.x, y: c.y) { cells.insert(wallCell) } + } + for track in tracks.values where isAlive(track) && abs(track.position.z) <= 0.1 { + if let wallCell = WallCell.containing(x: track.position.x, y: track.position.y) { cells.insert(wallCell) } + } + return cells + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/StableHash.swift b/experiments/live-dots/Sources/LiveDotsCore/StableHash.swift new file mode 100644 index 000000000..47e3ccac7 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/StableHash.swift @@ -0,0 +1,45 @@ +import simd + +/// Deterministic 64-bit hashing (SplitMix64 finaliser) for jitter, thinning and dot order. +/// Swift's `Hasher` is seeded per process, so it can't give a field that looks the same twice. +public enum StableHash { + public static func mix(_ value: UInt64) -> UInt64 { + var z = value &+ 0x9E37_79B9_7F4A_7C15 + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB + return z ^ (z >> 31) + } + + public static func hash(_ key: SIMD3, seed: UInt64 = 0) -> UInt64 { + var h = mix(seed) + h = mix(h ^ UInt64(UInt32(bitPattern: key.x))) + h = mix(h ^ UInt64(UInt32(bitPattern: key.y))) + h = mix(h ^ UInt64(UInt32(bitPattern: key.z))) + return h + } + + /// A value in [0, 1) from the top 24 bits. + public static func unit(_ h: UInt64) -> Float { + Float(h >> 40) / Float(1 << 24) + } +} + +public enum Jitter { + /// A fixed offset in the plane perpendicular to `normal`, uniform over a disc of radius + /// `Tuning.jitterFraction * cellSize`, from a hash of the cell index. + public static func offset(for key: SIMD3, normal: SIMD3, cellSize: Float) -> SIMD3 { + let (t, b) = tangentBasis(normal) + let h = StableHash.hash(key, seed: 0x6A17) + let angle = StableHash.unit(h) * 2 * Float.pi + let radius = Tuning.jitterFraction * cellSize * StableHash.unit(StableHash.mix(h)).squareRoot() + return (t * cos(angle) + b * sin(angle)) * radius + } + + static func tangentBasis(_ n: SIMD3) -> (SIMD3, SIMD3) { + let length = simd_length(n) + let normal = length > 1e-6 ? n / length : SIMD3(0, 0, 1) + let helper: SIMD3 = abs(normal.y) < 0.9 ? SIMD3(0, 1, 0) : SIMD3(1, 0, 0) + let t = simd_normalize(simd_cross(helper, normal)) + return (t, simd_cross(normal, t)) + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Tuning.swift b/experiments/live-dots/Sources/LiveDotsCore/Tuning.swift new file mode 100644 index 000000000..70f3468ef --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/Tuning.swift @@ -0,0 +1,81 @@ +import Foundation + +/// The prototype's rule values. Each is the brief's value unless its comment says why it differs. +public enum Tuning { + // MARK: LiDAR fusion + + public static let voxelSize: Float = 0.05 + /// ARKit confidence: 0 low, 1 medium, 2 high. + public static let minimumConfidence: UInt8 = 1 + public static let depthRange: ClosedRange = 0.3...6 + /// Jitter radius as a fraction of the cell it sits in, so the field never reads as a grid. + public static let jitterFraction: Float = 0.3 + /// A new view of a voxel counts only when it differs from every stored view by more than this. + public static let viewSeparationDegrees: Float = 15 + /// Face neighbours whose normals differ by more than this make a crease edge. + public static let creaseDegrees: Float = 35 + /// A face neighbour lies along the surface when its direction is within 60 degrees of the + /// tangent plane (|cos| to the normal under 0.5). Not in the brief; the brief names the idea. + public static let alongSurfaceCosine: Float = 0.5 + /// A neighbour voxel counts as seen-empty when a depth ray passes it by this much. + public static let freeSpaceMargin: Float = 0.1 + /// Scale-normalised colour gradient (0 to 1, the step height of the strongest channel) at + /// which a voxel is an image edge. Chosen from `LiveDots --gradient-report` on the synthetic + /// fixture: see the comment at the top of DotField.swift for the numbers. + public static let gradientThreshold: Float = 0.2 + /// Most dots drawn in one frame. Only dots in the phone's view count. + public static let dotCap = 6_000 + /// An edge dot stays at or below this opacity until some view saw it within + /// `faceOnDegrees` of its normal. The team's field data: window and door edges seen more + /// than 30 degrees off face-on carried 8 to 24 inches of error. + public static let obliqueEdgeOpacityCap: Float = 0.6 + public static let faceOnDegrees: Float = 30 + /// The cap applies only to edges on vertical-ish surfaces (normal within 45 degrees of + /// horizontal): the rule came from wall features, and a walking phone never sees the ground + /// face-on. + public static let cappedNormalMaxY: Float = 0.7071 + /// The occluder is any dot this far in front of the wall. The ground under it is excluded by + /// `groundBand`, which the brief does not mention: without it the lawn would be violet too. + public static let occluderMinZ: Float = 0.25 + public static let groundBand: Float = 0.1 + + // MARK: Drawing + + /// A dot hides while the current keyframe's depth shows something this much nearer. + public static let occlusionMargin: Float = 0.15 + + // MARK: Simulated no-LiDAR + + public static let featureTarget = 600 + /// Gradient a depth sample needs to become a simulated feature point: half the edge + /// threshold, so brick texture counts, as it does for ARKit's rawFeaturePoints. At 0.2 only + /// painted outlines qualified and 34 to 84 points lived in view. + public static let featureGradientThreshold: Float = 0.1 + public static let featureMatchRadius: Float = 0.08 + /// A feature lives while re-observed in 2 of the last 6 keyframes and dies 4 s after it was + /// last seen. The real app would use 3 of the last 10 frames at frame rate; the fixture's + /// keyframes are 0.5 m apart, and at 3 of 10 only 30 to 80 points stayed alive in view. + public static let featureWindow = 6 + public static let featureMinObservations = 2 + /// Seconds of capture time (the fixture's timestamps), not of playback. + public static let featureLifetime: Double = 4 + /// Candidate features are merged to one per cell of this size before thinning, so the same + /// wall spot tends to be picked again in the next keyframe, as a tracked ARKit feature is. + public static let featureCell: Float = 0.04 + public static let planeCell: Float = 0.12 + public static let planeRange: Float = 4 + + // MARK: Coverage ring and fog + + public static let coverageZone: ClosedRange = -3...3 + public static let coverageColumn: Float = 0.3 + public static let coverageMinViews = 2 + public static let fogCell: Float = 0.3 + + // MARK: Playback and motion + + public static let keyframesPerSecond: Float = 4 + public static let birthDuration: Float = 0.35 + public static let evidenceDuration: Float = 0.25 + public static let birthStartScale: Float = 0.6 +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/WallCell.swift b/experiments/live-dots/Sources/LiveDotsCore/WallCell.swift new file mode 100644 index 000000000..d47d976cb --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/WallCell.swift @@ -0,0 +1,37 @@ +/// The synthetic fixture's wall, z = 0, x from -6 to 6 m, 3 m high. Only the simulated no-LiDAR +/// plane dots and the fog comparison use it; the LiDAR field finds surfaces from depth alone. +public enum FixtureWall { + public static let xRange: ClosedRange = -6...6 + public static let yRange: ClosedRange = 0...3 +} + +/// A 30 cm square of the fixture wall, for the "unseen as fog" comparison veil. +public struct WallCell: Hashable, Sendable, Comparable { + public let column: Int + public let row: Int + + public static let size = Tuning.fogCell + public static let columns = Int(((FixtureWall.xRange.upperBound - FixtureWall.xRange.lowerBound) / size).rounded()) + public static let rows = Int(((FixtureWall.yRange.upperBound - FixtureWall.yRange.lowerBound) / size).rounded()) + + public static let all: [WallCell] = (0.. WallCell? { + let column = Int(((x - FixtureWall.xRange.lowerBound) / size).rounded(.down)) + let row = Int(((y - FixtureWall.yRange.lowerBound) / size).rounded(.down)) + guard (0.. Bool { + (a.row, a.column) < (b.row, b.column) + } +} diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift new file mode 100644 index 000000000..0bb451250 --- /dev/null +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift @@ -0,0 +1,170 @@ +import simd +import Testing +@testable import LiveDotsCore + +struct FieldRuleTests { + let camera = SIMD3(0.2, 1, 2.5) + + @Test func `a corner between two planes is a crease edge and their interiors are flat`() { + var field = VoxelField() + for a in Int32(0)..<10 { + for b in Int32(1)..<10 { + field.observe(VoxelKey(a, b, 0), normal: SIMD3(0, 0, 1), gradient: 0, samples: 4, camera: camera, frame: 0) + field.observe(VoxelKey(a, 0, b), normal: SIMD3(0, 1, 0), gradient: 0, samples: 4, camera: camera, frame: 0) + } + // The corner voxel holds samples of both planes, so its normal averages to 45 degrees. + field.observe(VoxelKey(a, 0, 0), normal: simd_normalize(SIMD3(0, 1, 1)), gradient: 0, samples: 4, camera: camera, frame: 0) + } + #expect(field.edgeReason(for: VoxelKey(5, 0, 0)) == .crease) + #expect(field.edgeReason(for: VoxelKey(5, 1, 0)) == .crease) + #expect(field.edgeReason(for: VoxelKey(5, 5, 0)) == nil) + #expect(field.edgeReason(for: VoxelKey(5, 0, 5)) == nil) + } + + @Test func `neighbours under 35 degrees apart are not a crease`() { + #expect(!VoxelField.normalsDisagree(SIMD3(0, 0, 1), SIMD3(0, sin(0.59), cos(0.59)))) // 33.8 degrees + #expect(VoxelField.normalsDisagree(SIMD3(0, 0, 1), SIMD3(0, sin(0.63), cos(0.63)))) // 36.1 degrees + } + + @Test func `seen-empty space along the surface makes a boundary, unseen space does not`() { + var field = VoxelField() + for x in Int32(0)..<10 { + for y in Int32(0)..<10 { + field.observe(VoxelKey(x, y, 0), normal: SIMD3(0, 0, 1), gradient: 0, samples: 4, camera: camera, frame: 0) + } + } + for y in Int32(0)..<10 { field.markFree(VoxelKey(10, y, 0)) } + // Empty space in front of the plane is not along the surface. + field.markFree(VoxelKey(5, 5, 1)) + + #expect(field.edgeReason(for: VoxelKey(9, 5, 0)) == .boundary) + #expect(field.edgeReason(for: VoxelKey(0, 5, 0)) == nil, "x = -1 was never seen, so the scan's frontier is not an edge") + #expect(field.edgeReason(for: VoxelKey(5, 5, 0)) == nil) + // An occupied voxel can't be marked free. + field.markFree(VoxelKey(4, 4, 0)) + #expect(!field.free.contains(VoxelKey(4, 4, 0))) + } + + @Test func `the gradient threshold makes an image edge`() { + var field = VoxelField() + for x in Int32(0)..<3 { + field.observe(VoxelKey(x, 0, 0), normal: SIMD3(0, 0, 1), gradient: x == 1 ? Tuning.gradientThreshold : 0.19, samples: 4, camera: camera, frame: 0) + } + #expect(field.edgeReason(for: VoxelKey(1, 0, 0)) == .imageEdge) + #expect(field.edgeReason(for: VoxelKey(0, 0, 0)) == nil) + } + + @Test func `edges stay edges and flat groups keep one representative`() { + var field = VoxelField() + for x in Int32(0)..<4 { + for y in Int32(0)..<4 { + field.observe(VoxelKey(x, y, 0), normal: SIMD3(0, 0, 1), gradient: x == 0 ? 1 : 0, samples: 4, camera: camera, frame: 0) + } + } + field.classify(frame: 0) + let first = field.dots() + #expect(first.count { $0.kind == .edge } == 4) + // x 0...3 by y 0...3 falls into four 2 x 2 x 2 groups, each with flat members. + #expect(first.count { $0.kind == .flat } == 4) + // A later keyframe with no gradient can't demote an edge. + field.observe(VoxelKey(0, 0, 0), normal: SIMD3(0, 0, 1), gradient: 0, samples: 4, camera: camera, frame: 1) + field.classify(frame: 1) + let second = field.dots() + #expect(Set(first.map(\.id)) == Set(second.map(\.id))) + } + + @Test(arguments: [(0, Float(0)), (1, 0.45), (2, 0.6), (3, 0.75), (4, 0.9), (7, 0.9)]) + func `evidence maps views to opacity`(views: Int, opacity: Float) { + #expect(abs(Evidence.opacity(views: views) - opacity) < 1e-6) + } + + /// An edge voxel seen from 40, -40 and 60 degrees off its normal (75% if uncapped), plus + /// `extra`. The normal is the wall's, or the ground's with `ground`. + func obliqueEdge(_ extra: Float?, ground: Bool = false) throws -> FieldDot { + var field = VoxelField() + let key = VoxelKey(0, 30, 0) + let centre = field.centre(of: key) + let normal: SIMD3 = ground ? SIMD3(0, 1, 0) : SIMD3(0, 0, 1) + let side = SIMD3(1, 0, 0) + for degrees in [40, -40, 60] + (extra.map { [$0] } ?? []) { + let camera = centre + 2.6 * (side * sin(degrees * .pi / 180) + normal * cos(degrees * .pi / 180)) + field.observe(key, normal: normal, gradient: 1, samples: 4, camera: camera, frame: 0) + } + field.classify(frame: 0) + return try #require(field.dots().first) + } + + @Test func `an edge seen only obliquely stays at 60 percent until a face-on view`() throws { + let oblique = try obliqueEdge(nil) + #expect(oblique.kind == .edge && oblique.views == 3 && !oblique.faceOn) + #expect(abs(oblique.opacity - 0.6) < 1e-6) + + let faced = try obliqueEdge(20) + #expect(faced.views == 4 && faced.faceOn) + #expect(abs(faced.opacity - 0.9) < 1e-6) + + // 28 degrees is within 15 of the 40-degree view, so it adds no view, yet it faces on. + let near = try obliqueEdge(28) + #expect(near.views == 3 && near.faceOn) + #expect(abs(near.opacity - 0.75) < 1e-6) + } + + @Test func `the oblique cap leaves edges on the ground alone`() throws { + let ground = try obliqueEdge(nil, ground: true) + #expect(ground.kind == .edge && ground.faceOn) + #expect(abs(ground.opacity - 0.75) < 1e-6) + } + + @Test func `flat dots ignore the oblique cap`() { + #expect(abs(Evidence.opacity(views: 4, faceOn: false) - 0.6) < 1e-6) + #expect(abs(FieldDot(id: 1, position: .zero, kind: .flat, views: 4, onOccluder: false).opacity - 0.9) < 1e-6) + } + + @Test func `a view counts as new only past 15 degrees`() { + func direction(_ degrees: Float) -> SIMD3 { + SIMD3(sin(degrees * .pi / 180), 0, cos(degrees * .pi / 180)) + } + var views = ViewDirections() + let added = [ + views.insert(direction(0) * 3), + views.insert(direction(14)), + views.insert(direction(16)), + views.insert(direction(29)), // 13 degrees from the 16-degree view + views.insert(direction(-16)), + views.insert(.zero), + ] + #expect(added == [true, false, true, false, true, false]) + #expect(views.count == 3) + } + + @Test func `coverage counts 30 cm columns with a two-view wall dot`() { + func dot(_ x: Float, views: Int, z: Float = 0, occluder: Bool = false) -> FieldDot { + FieldDot(id: UInt64(bitPattern: Int64(x * 1000)), position: SIMD3(x, 1, z), kind: .flat, views: views, onOccluder: occluder) + } + let dots = [ + dot(-2.95, views: 2), dot(-2.9, views: 3), // column 0, counted once + dot(0.1, views: 1), // one view: not yet + dot(2.99, views: 2), // column 19 + dot(3.5, views: 4), // outside the zone + dot(1.0, views: 4, z: 0.5), // off the wall plane + dot(1.5, views: 4, occluder: true), // on the bin + ] + #expect(Coverage.columnCount == 20) + #expect(abs(Coverage.fraction(of: dots) - 0.1) < 1e-6) + #expect(Coverage.fraction(of: []) == 0) + } + + @Test func `past the cap only flat dots are dropped`() { + func sprite(_ id: UInt64, _ kind: DotKind) -> (sprite: DotSprite, kind: DotKind) { + (DotSprite(id: id, position: .zero, kind: kind, onOccluder: false, birthTime: 0, fromOpacity: 0.45, toOpacity: 0.45, + opacityTime: 0, edgeSince: kind == .edge ? -.infinity : .infinity, deathTime: .infinity), kind) + } + let edges = (0..(500 * 4 / 640, 500 * 2 / 480, 320 * 4 / 640, 240 * 2 / 480) + #expect(simd_distance(depth.intrinsics, expected) < 1e-5) + } + + @Test func `the wrong format version is refused`() throws { + let url = folder.appendingPathComponent("session.json") + let text = try String(contentsOf: url, encoding: .utf8).replacingOccurrences(of: "\"formatVersion\": 2", with: "\"formatVersion\": 3") + try Data(text.utf8).write(to: url) + #expect(throws: FixtureError.unsupportedVersion(path: url.path, found: 3)) { try Replay.load(folder: folder) } + } +} diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/FogAndGuidanceTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/FogAndGuidanceTests.swift new file mode 100644 index 000000000..1b0512818 --- /dev/null +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/FogAndGuidanceTests.swift @@ -0,0 +1,85 @@ +import Foundation +import simd +import Testing +@testable import LiveDotsCore + +struct FogAndGuidanceTests { + @Test func `the reveal mask is smoothstepped from 0.12 to 0.55 after clamping to 1`() { + #expect(FogMask.shape(0) == 0) + #expect(FogMask.shape(0.12) == 0) + #expect(abs(FogMask.shape(0.335) - 0.5) < 1e-5) + #expect(FogMask.shape(0.55) == 1) + #expect(FogMask.shape(3.7) == 1) + } + + @Test func `the fog lifts with a 600 ms time constant and returns in 120 ms`() { + let lifted = FogMask.lagStep(from: 0, toward: 1, dt: 0.6, reduceMotion: false) + #expect(abs(lifted - (1 - exp(-1))) < 1e-5) + let returned = FogMask.lagStep(from: 1, toward: 0, dt: 0.12, reduceMotion: false) + #expect(abs(returned - exp(-1)) < 1e-5) + // Stepping in frames lands in the same place as one long step. + var stepped: Float = 0 + for _ in 0..<36 { stepped = FogMask.lagStep(from: stepped, toward: 1, dt: 1 / 60, reduceMotion: false) } + #expect(abs(stepped - lifted) < 1e-4) + } + + @Test func `under Reduce Motion the lift is a linear 400 ms fade`() { + #expect(abs(FogMask.lagStep(from: 0, toward: 1, dt: 0.2, reduceMotion: true) - 0.5) < 1e-6) + #expect(FogMask.lagStep(from: 0, toward: 1, dt: 0.5, reduceMotion: true) == 1) + } + + @Test func `a box appears at 60 percent in view and settles when seen face-on with both sides in frame`() throws { + let window = try #require(RecognisedBox.fixture.first { $0.caption == "Window" }) + var state = BoxState() + func view(_ x: Float, _ t: Float) { + state.update(window, projection: ScreenProjection(keyframe: .fixtureStyle(x: x), viewSize: SIMD2(390, 844)), at: t) + } + view(0.5, 1) // the window is off to the right + #expect(state == BoxState()) + view(1.9, 2) // most of it in view, its right edge cut off: dashed + #expect(state.appearedAt == 2 && state.solidAt == nil) + view(2.5, 3) // square on, both sides in frame: solid + #expect(state.appearedAt == 2 && state.solidAt == 3) + view(0.5, 4) // walking away changes nothing + #expect(state.appearedAt == 2 && state.solidAt == 3) + } + + @Test func `an oblique view never settles a box`() throws { + let window = try #require(RecognisedBox.fixture.first { $0.caption == "Window" }) + // 1.9 m to the side and 0.6 m out, turned to face it: 72 degrees off face-on. + let camera = SIMD3(4.4, 1.5, 0.6) + let forward = simd_normalize(SIMD3(2.5, 1.45, 0) - camera) + let right = simd_normalize(simd_cross(forward, SIMD3(0, 1, 0))), up = simd_cross(right, forward) + let keyframe = Keyframe( + id: "k", imagePath: "k.jpg", width: 640, height: 480, intrinsics: SIMD4(500, 500, 320, 240), + cameraToWorld: simd_float4x4(columns: (SIMD4(-up, 0), SIMD4(right, 0), SIMD4(-forward, 0), SIMD4(camera, 1))), + timestamp: 0, depth: nil) + var state = BoxState() + state.update(window, projection: ScreenProjection(keyframe: keyframe, viewSize: SIMD2(390, 844)), at: 1) + #expect(state.appearedAt == 1 && state.solidAt == nil) + } + + @Test func `the meter box is solid from the first keyframe it is in view`() throws { + let meter = try #require(RecognisedBox.fixture.first { $0.isMeter }) + var state = BoxState() + state.update(meter, projection: ScreenProjection(keyframe: .fixtureStyle(x: 0, pitch: 0), viewSize: SIMD2(390, 844)), at: 0) + #expect(state == BoxState(appearedAt: 0, solidAt: 0)) + } + + @Test func `guidance runs meter, hold, left, right, tilt, done`() { + let xs: [Float] = [0, 0, 0, 0, -1, -2.6, -1, 0, 1, 0.2, 0.2, 0.2] + let sequence = Instruction.sequence(for: xs.map { Keyframe.fixtureStyle(x: $0) }) + #expect(sequence == [ + .pointAtMeter, .pointAtMeter, .holdStill, .walkLeft, .walkLeft, .walkRight, .walkRight, .walkRight, .walkRight, + .tiltDown, .tiltDown, .done, + ]) + } + + @Test func `the hold keyframe stays on screen for the ring fill and the flash`() { + let hold = Schedule.start(of: Schedule.holdIndex) + #expect(abs(Schedule.start(of: Schedule.holdIndex + 1) - hold - Schedule.holdDuration - Schedule.flashDuration) < 1e-6) + #expect(abs(Schedule.start(of: 5) - Schedule.start(of: 4) - 0.25) < 1e-6) + #expect(Schedule.keyframe(at: hold + 1, count: 41) == Schedule.holdIndex) + #expect(Schedule.keyframe(at: 1e5, count: 41) == 40) + } +} diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/GeometryTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/GeometryTests.swift new file mode 100644 index 000000000..591bf2969 --- /dev/null +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/GeometryTests.swift @@ -0,0 +1,86 @@ +import Foundation +import simd +import Testing +@testable import LiveDotsCore + +struct GeometryTests { + let depthIntrinsics = SIMD4(200, 200, 128, 96) + + @Test func `unprojection matches a hand-computed point`() { + // Pixel (178, 46), centre (178.5, 46.5), 2 m: x = 50.5 * 2 / 200, y = -(-49.5) * 2 / 200. + let camera = CameraMath.unproject(u: 178.5, v: 46.5, depth: 2, intrinsics: depthIntrinsics) + #expect(simd_distance(camera, SIMD3(0.505, 0.495, -2)) < 1e-5) + + // The fixture's portrait pose: camera +x is world down, +y world right, +z world +z. + let pose = simd_float4x4(columns: (SIMD4(0, -1, 0, 0), SIMD4(1, 0, 0, 0), SIMD4(0, 0, 1, 0), SIMD4(0, 1.5, 2.6, 1))) + let world = CameraMath.transform(pose, camera) + #expect(simd_distance(world, SIMD3(0.495, 0.995, 0.6)) < 1e-5) + } + + @Test func `projection inverts unprojection`() throws { + let p = CameraMath.unproject(u: 17.25, v: 150.5, depth: 3.3, intrinsics: depthIntrinsics) + let pixel = try #require(CameraMath.project(p, intrinsics: depthIntrinsics)) + #expect(abs(pixel.u - 17.25) < 1e-3 && abs(pixel.v - 150.5) < 1e-3 && abs(pixel.depth - 3.3) < 1e-5) + #expect(CameraMath.project(SIMD3(0, 0, 1), intrinsics: depthIntrinsics) == nil) + } + + @Test func `the clip matrix lands where screenPoint does`() throws { + let keyframe = Keyframe.fixtureStyle(x: 0.7) + let projection = ScreenProjection(keyframe: keyframe, viewSize: SIMD2(390, 844)) + for world in [SIMD3(0.7, 1.5, 0), SIMD3(1.4, 0.2, 0.3), SIMD3(-0.1, 2.0, 0)] { + let screen = try #require(projection.screenPoint(world)) + let clip = projection.clipMatrix * SIMD4(world, 1) + let ndc = SIMD2(clip.x, clip.y) / clip.w + let fromClip = SIMD2((ndc.x + 1) / 2 * 390, (1 - ndc.y) / 2 * 844) + #expect(simd_distance(screen, fromClip) < 1e-2) + } + // Straight ahead of a camera aimed square at the wall is the screen's centre. + let centre = try #require(ScreenProjection(keyframe: .fixtureStyle(x: 0, pitch: 0), viewSize: SIMD2(390, 844)) + .screenPoint(SIMD3(0, 1.5, 0))) + #expect(simd_distance(centre, SIMD2(195, 422)) < 1e-3) + } + + @Test func `jitter is deterministic, in the tangent plane and within 30 percent of the cell`() { + let normal = simd_normalize(SIMD3(0.2, 0.3, 1)) + var longest: Float = 0 + for x in Int32(-20)..<20 { + for y in Int32(-10)..<10 { + let key = VoxelKey(x, y, 3) + let a = Jitter.offset(for: key, normal: normal, cellSize: 0.05) + #expect(a == Jitter.offset(for: key, normal: normal, cellSize: 0.05)) + #expect(simd_length(a) <= 0.3 * 0.05 + 1e-6) + #expect(abs(simd_dot(a, normal)) < 1e-5) + longest = max(longest, simd_length(a)) + } + } + // The offsets fill the disc rather than collapsing to its centre. + #expect(longest > 0.25 * 0.05) + } + + @Test func `easing curve runs from 0 to 1 and front-loads the motion`() { + let curve = CubicBezier.strongEaseOut + #expect(curve(0) == 0 && curve(1) == 1 && curve(-1) == 0 && curve(2) == 1) + var previous: Float = 0 + for step in 1...100 { + let value = curve(Float(step) / 100) + #expect(value >= previous) + previous = value + } + #expect(curve(0.25) > 0.7) + } +} + +extension Keyframe { + /// A keyframe like the fixture's walk: 640 x 480, phone in portrait at 1.5 m height and 2.6 m + /// from the wall, pitched down by `pitch` degrees. + static func fixtureStyle(x: Float, pitch: Float = 20, depth: DepthReference? = nil) -> Keyframe { + let angle = pitch * .pi / 180 + let forward = SIMD3(0, -sin(angle), -cos(angle)) + let right = simd_normalize(simd_cross(forward, SIMD3(0, 1, 0))) + let up = simd_cross(right, forward) + let pose = simd_float4x4(columns: (SIMD4(-up, 0), SIMD4(right, 0), SIMD4(-forward, 0), SIMD4(x, 1.5, 2.6, 1))) + return Keyframe( + id: "k", imagePath: "k.jpg", width: 640, height: 480, intrinsics: SIMD4(500, 500, 320, 240), + cameraToWorld: pose, timestamp: 0, depth: depth) + } +} diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/SchemeTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/SchemeTests.swift new file mode 100644 index 000000000..627ef0227 --- /dev/null +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/SchemeTests.swift @@ -0,0 +1,59 @@ +import simd +import Testing +@testable import LiveDotsCore + +struct SchemeTests { + func sprite(_ x: Float, kind: DotKind = .edge, normal: SIMD3 = SIMD3(0, 0, 1), y: Float = 1) -> DotSprite { + DotSprite( + id: UInt64(x * 1000), position: SIMD3(x, y, 0), kind: kind, onOccluder: false, birthTime: 0, + fromOpacity: 0.45, toOpacity: 0.45, opacityTime: 0, edgeSince: kind == .edge ? -.infinity : .infinity, + deathTime: .infinity, normal: normal) + } + + @Test func `constellation draws no flat or plane dots`() { + #expect(DotKind.allKinds.filter(DotScheme.constellation.draws) == [.edge, .feature]) + #expect(DotKind.allKinds.allSatisfy(DotScheme.hologram.draws)) + #expect(DotKind.allKinds.allSatisfy(DotScheme.ember.draws)) + } + + @Test func `constellation links follow an outline and skip other surfaces and flat dots`() { + let sprites = [ + sprite(0), sprite(0.05), sprite(0.10), sprite(0.15), sprite(0.20), // an outline, 5 cm apart + sprite(0.05, normal: SIMD3(0, 1, 0), y: 1.04), // 4 cm away, 90 degrees off + sprite(0.10, kind: .flat, y: 1.03), // flat, never linked + ] + let links = EdgeLinks.links(among: sprites) + #expect(links == [SIMD2(0, 1), SIMD2(1, 2), SIMD2(2, 3), SIMD2(3, 4)]) + } + + @Test func `an ember dot cools from its birth and seeing it again never re-warms it`() throws { + // A 32 x 32 grey image and an empty depth map: nothing occludes, and only the timeline matters. + let image = RGBImage(width: 32, height: 32, rgba: [UInt8](repeating: 128, count: 32 * 32 * 4)) + let depth = DepthMap( + meters: [Float](repeating: 0, count: 12), confidence: [UInt8](repeating: 2, count: 12), + width: 4, height: 3, intrinsics: SIMD4(500 * 4 / 640, 500 * 3 / 480, 2, 1.5)) + func frame(_ index: Int) -> FrameInput { + FrameInput(index: index, keyframe: .fixtureStyle(x: 0, pitch: 0), depth: depth, gradient: GradientPyramid(image: image)) + } + let position = SIMD3(0, 1.5, 0) + var builder = DotTimeline.Builder() + builder.append(frame(0), dots: [FieldDot(id: 1, position: position, kind: .flat, views: 1, onOccluder: false)], seen: [], instruction: .walkLeft) + // Keyframe 30, 7.5 s later: dot 1 is observed again, from a new direction, and has become + // an edge; dot 2 is new. + builder.append(frame(30), dots: [ + FieldDot(id: 1, position: position, kind: .edge, views: 2, onOccluder: false), + FieldDot(id: 2, position: position + SIMD3(0.1, 0, 0), kind: .flat, views: 1, onOccluder: false), + ], seen: [], instruction: .walkLeft) + let sprites = try #require(builder.states.last).sprites + let old = try #require(sprites.first { $0.id == 1 }), new = try #require(sprites.first { $0.id == 2 }) + let now = Schedule.start(of: 30) + #expect(old.birthTime == 0) + #expect(DotScheme.warmth(at: now, birth: old.birthTime) == 0) + #expect(DotScheme.warmth(at: now, birth: new.birthTime) == 1) + #expect(abs(DotScheme.warmth(at: now + 3, birth: new.birthTime) - 0.5) < 1e-6) + } +} + +extension DotKind { + static let allKinds: [DotKind] = [.flat, .edge, .feature, .plane] +} diff --git a/experiments/live-dots/fetch-fixtures.sh b/experiments/live-dots/fetch-fixtures.sh new file mode 100755 index 000000000..0370cda39 --- /dev/null +++ b/experiments/live-dots/fetch-fixtures.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Copies the synthetic replay fixtures from the guided-capture branch (PR #10) into Fixtures/, +# without adding ios/ to a sparse checkout. Fixtures/ is git-ignored. +set -eu +here="$(cd "$(dirname "$0")" && pwd)" +root="$(git -C "$here" rev-parse --show-toplevel)" +git -C "$root" fetch --quiet origin t3/ios-mvf +rm -rf "$here/Fixtures" +# ios/HouseScanUITests/Fixtures//... becomes Fixtures//... +git -C "$root" archive origin/t3/ios-mvf ios/HouseScanUITests/Fixtures | tar -x -C "$here" --strip-components=2 +ls "$here/Fixtures" diff --git a/experiments/live-dots/make-video.sh b/experiments/live-dots/make-video.sh new file mode 100755 index 000000000..1ac95e777 --- /dev/null +++ b/experiments/live-dots/make-video.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Renders the replay at 1170 x 2532, 30 fps, and encodes each run to H.264: with fog in both modes +# and with Reduce Motion, then without fog in both modes and in each dot scheme. +# LiveDots streams raw BGRA frames into ffmpeg, so no frame folder is written: as PNGs the +# ~320 frames of one mode take about 500 MB. +# Usage: ./make-video.sh [output folder] +set -euo pipefail +cd "$(dirname "$0")" +out="${1:-$HOME/house-scanning-data/reports/experience/prototype}" +mkdir -p "$out" +swift build -c release +render() { + .build/release/LiveDots --export - "$@" | + ffmpeg -y -loglevel error -f rawvideo -pix_fmt bgra -s 1170x2532 -r 30 -i - \ + -c:v libx264 -crf 23 -pix_fmt yuv420p "$file" + echo "wrote $file" +} +for mode in lidar nolidar; do + file="$out/fog-$mode.mp4" render --mode "$mode" +done +file="$out/fog-reduce-motion.mp4" render --mode lidar --reduce-motion +# Without fog, as the earlier recordings and the scheme comparison were made. +for mode in lidar nolidar; do + file="$out/live-dots-$mode.mp4" render --mode "$mode" --fog off +done +for scheme in a-hologram b-constellation c-ember; do + file="$out/scheme-$scheme.mp4" render --mode lidar --scheme "${scheme#?-}" --fog off +done diff --git a/experiments/live-dots/results/fog-early-keyframe02.jpg b/experiments/live-dots/results/fog-early-keyframe02.jpg new file mode 100644 index 000000000..89007aa78 Binary files /dev/null and b/experiments/live-dots/results/fog-early-keyframe02.jpg differ diff --git a/experiments/live-dots/results/fog-lidar.mp4 b/experiments/live-dots/results/fog-lidar.mp4 new file mode 100644 index 000000000..87df0b7ec Binary files /dev/null and b/experiments/live-dots/results/fog-lidar.mp4 differ diff --git a/experiments/live-dots/results/fog-nolidar-keyframe31.jpg b/experiments/live-dots/results/fog-nolidar-keyframe31.jpg new file mode 100644 index 000000000..7734ca9b2 Binary files /dev/null and b/experiments/live-dots/results/fog-nolidar-keyframe31.jpg differ diff --git a/experiments/live-dots/results/fog-nolidar.mp4 b/experiments/live-dots/results/fog-nolidar.mp4 new file mode 100644 index 000000000..18fea193d Binary files /dev/null and b/experiments/live-dots/results/fog-nolidar.mp4 differ diff --git a/experiments/live-dots/results/fog-reduce-motion.mp4 b/experiments/live-dots/results/fog-reduce-motion.mp4 new file mode 100644 index 000000000..176f137fc Binary files /dev/null and b/experiments/live-dots/results/fog-reduce-motion.mp4 differ diff --git a/experiments/live-dots/results/fog-tilt-keyframe39.jpg b/experiments/live-dots/results/fog-tilt-keyframe39.jpg new file mode 100644 index 000000000..a5d6a2524 Binary files /dev/null and b/experiments/live-dots/results/fog-tilt-keyframe39.jpg differ diff --git a/experiments/live-dots/results/fog-walk-keyframe31.jpg b/experiments/live-dots/results/fog-walk-keyframe31.jpg new file mode 100644 index 000000000..bc437508e Binary files /dev/null and b/experiments/live-dots/results/fog-walk-keyframe31.jpg differ diff --git a/experiments/live-dots/results/fog-window-keyframe33.jpg b/experiments/live-dots/results/fog-window-keyframe33.jpg new file mode 100644 index 000000000..796d3fe91 Binary files /dev/null and b/experiments/live-dots/results/fog-window-keyframe33.jpg differ diff --git a/experiments/live-dots/results/lidar-early-keyframe05.jpg b/experiments/live-dots/results/lidar-early-keyframe05.jpg new file mode 100644 index 000000000..84df8d97f Binary files /dev/null and b/experiments/live-dots/results/lidar-early-keyframe05.jpg differ diff --git a/experiments/live-dots/results/lidar-fog-keyframe31.jpg b/experiments/live-dots/results/lidar-fog-keyframe31.jpg new file mode 100644 index 000000000..356e93ba9 Binary files /dev/null and b/experiments/live-dots/results/lidar-fog-keyframe31.jpg differ diff --git a/experiments/live-dots/results/lidar-walk-bin-keyframe31.jpg b/experiments/live-dots/results/lidar-walk-bin-keyframe31.jpg new file mode 100644 index 000000000..56a07de08 Binary files /dev/null and b/experiments/live-dots/results/lidar-walk-bin-keyframe31.jpg differ diff --git a/experiments/live-dots/results/live-dots-lidar.mp4 b/experiments/live-dots/results/live-dots-lidar.mp4 new file mode 100644 index 000000000..a910f46d7 Binary files /dev/null and b/experiments/live-dots/results/live-dots-lidar.mp4 differ diff --git a/experiments/live-dots/results/nolidar-walk-keyframe31.jpg b/experiments/live-dots/results/nolidar-walk-keyframe31.jpg new file mode 100644 index 000000000..1bfb04974 Binary files /dev/null and b/experiments/live-dots/results/nolidar-walk-keyframe31.jpg differ diff --git a/experiments/live-dots/results/scheme-a-hologram.jpg b/experiments/live-dots/results/scheme-a-hologram.jpg new file mode 100644 index 000000000..56a07de08 Binary files /dev/null and b/experiments/live-dots/results/scheme-a-hologram.jpg differ diff --git a/experiments/live-dots/results/scheme-a-hologram.mp4 b/experiments/live-dots/results/scheme-a-hologram.mp4 new file mode 100644 index 000000000..53a13f2ce Binary files /dev/null and b/experiments/live-dots/results/scheme-a-hologram.mp4 differ diff --git a/experiments/live-dots/results/scheme-b-constellation.jpg b/experiments/live-dots/results/scheme-b-constellation.jpg new file mode 100644 index 000000000..0093e258c Binary files /dev/null and b/experiments/live-dots/results/scheme-b-constellation.jpg differ diff --git a/experiments/live-dots/results/scheme-b-constellation.mp4 b/experiments/live-dots/results/scheme-b-constellation.mp4 new file mode 100644 index 000000000..0b982eb38 Binary files /dev/null and b/experiments/live-dots/results/scheme-b-constellation.mp4 differ diff --git a/experiments/live-dots/results/scheme-c-ember.jpg b/experiments/live-dots/results/scheme-c-ember.jpg new file mode 100644 index 000000000..960f13a46 Binary files /dev/null and b/experiments/live-dots/results/scheme-c-ember.jpg differ diff --git a/experiments/live-dots/results/scheme-c-ember.mp4 b/experiments/live-dots/results/scheme-c-ember.mp4 new file mode 100644 index 000000000..ee36a3446 Binary files /dev/null and b/experiments/live-dots/results/scheme-c-ember.mp4 differ