experiments: live dots, a macOS prototype of the holographic capture view

experiments/live-dots/.gitignore

@@ -0,0 +1,3 @@
+Fixtures/
+.build/
+.swiftpm/

experiments/live-dots/Package.swift

@@ -0,0 +1,24 @@
+// swift-tools-version: 6.2
+import PackageDescription
+
+let package = Package(
+    name: "LiveDots",
+    platforms: [.macOS(.v15)],
+    targets: [
+        // Fixture decoding and the dot field's rules. No UI, no Metal, no main actor: it runs off
+        // the main thread while the app precomputes the replay, and the tests drive it directly.
+        .target(
+            name: "LiveDotsCore",
+            // Unoptimised, fusing the 41 keyframes takes about 40 s instead of under 1 s, which
+            // made plain `swift run LiveDots` sit on its loading bar.
+            swiftSettings: [.unsafeFlags(["-O"], .when(configuration: .debug))]
+        ),
+        .executableTarget(
+            name: "LiveDots",
+            dependencies: ["LiveDotsCore"],
+            swiftSettings: [.defaultIsolation(MainActor.self)]
+        ),
+        .testTarget(name: "LiveDotsCoreTests", dependencies: ["LiveDotsCore"]),
+    ],
+    swiftLanguageModes: [.v6]
+)

experiments/live-dots/README.md

@@ -0,0 +1,20 @@
+# Live dots
+
+**Question.** Do dots built from recorded depth read as sitting on the wall's edges and surfaces, denser on edges and absent where unseen, at 60 fps with 6,000 dots? It passes if the painted outlines and the bin's silhouette carry edge dots while under 5% of plain-brick voxels do, no dot lands on unseen wall, births ease in without a pop, and 6,000 dots render well inside 16.7 ms. The rules, sizes and colours are in the header of `Sources/LiveDotsCore/DotField.swift`; the fog's borrowings are in the header of its section in `Shaders.swift`.
+
+**Run.** Needs Xcode 27 and, for video, ffmpeg.
+
+```
+./fetch-fixtures.sh
+swift run LiveDots
+swift test
+./make-video.sh [output folder]
+```
+
+Flags: `--scheme hologram|constellation|ember` (three looks), `--fog on|off|veil` (fog that lifts a beat after dots arrive, or the old flat veil), `--still <n> --at <seconds>` (a still after playing to that moment). `make-video.sh` and `--still` produced `results/`, downscaled to 800 px.
+
+The fixture is the synthetic wall from #10, and the meter, gas meter, door and window boxes come from its scene, not a detector. The no-LiDAR mode is simulated from its depth: feature points use a 0.1 gradient, since ARKit finds them on brick texture too, and live while seen in 2 of the last 6 keyframes, where the real app would use 3 of 10 at frame rate.
+
+**Result.** It passes. At 0.2, 100% of outline voxels and 1% of brick voxels become edges, and nothing draws behind the bin. On an M4 Pro, 6,000 dots take 0.54 ms of GPU per 1170 × 2532 frame, 1.0 ms with fog.
+
+**What it changes.** Building this into `Map3DOverlay` (#21) needs a surface-sample iterator with per-frame deltas and a 5 cm point set beside the 10 cm map.

experiments/live-dots/Sources/LiveDots/Benchmark.swift

@@ -0,0 +1,48 @@
+import Foundation
+import LiveDotsCore
+import QuartzCore
+
+/// Frame cost with exactly 6,000 dots, every one mid-birth and mid-evidence-rise so the shader
+/// runs its full easing path, at the export size and at the app's Retina size.
+enum Benchmark {
+    static func run(fixture: String?) throws {
+        let data = try ReplayData.loadNow(folder: FixtureLocator.folder(fixture))
+        let renderer = try DotRenderer(data: data)
+        guard let fullest = data.lidar.states.max(by: { $0.sprites.count < $1.sprites.count }) else { return }
+        let time = fullest.time + 0.1
+        var sprites: [DotSprite] = []
+        var copy = 0
+        while sprites.count < Tuning.dotCap {
+            for s in fullest.sprites where sprites.count < Tuning.dotCap {
+                // Later copies shift 1 cm per pass so they don't stack on the same pixel.
+                sprites.append(DotSprite(
+                    id: s.id &+ UInt64(copy) << 56, position: s.position + SIMD3(0.01 * Float(copy), 0, 0), kind: s.kind,
+                    onOccluder: s.onOccluder, birthTime: time - 0.1, fromOpacity: 0.45, toOpacity: 0.9,
+                    opacityTime: time - 0.1, edgeSince: s.edgeSince, deathTime: .infinity))
+            }
+            copy += 1
+        }
+        renderer.spriteOverride = sprites
+        let edges = sprites.count { $0.edgeSince < .infinity }
+        print("keyframe \(fullest.index + 1): \(sprites.count) dots (\(edges) edges), each with a halo: \(2 * sprites.count) point sprites")
+
+        for (label, scale) in [("1170 x 2532 (export, 3x)", Float(3)), ("780 x 1688 (app window, 2x)", Float(2))] {
+            let target = try OffscreenTarget(renderer: renderer, scale: scale)
+            let request = FrameRequest(mode: .lidar, keyframe: fullest.index, time: time, reduceMotion: false, fog: .on)
+            for _ in 0..<30 { try target.render(request, readBack: false) }
+            var gpu: [Double] = [], wall: [Double] = []
+            for _ in 0..<300 {
+                let start = CACurrentMediaTime()
+                gpu.append(try target.render(request, readBack: false) * 1000)
+                wall.append((CACurrentMediaTime() - start) * 1000)
+            }
+            gpu.sort()
+            wall.sort()
+            func line(_ name: String, _ values: [Double]) -> String {
+                let mean = values.reduce(0, +) / Double(values.count)
+                return "\(name) mean \(String(format: "%.2f", mean)) ms, p95 \(String(format: "%.2f", values[values.count * 95 / 100])) ms"
+            }
+            print("\(label): \(line("GPU", gpu)); \(line("encode+submit+wait", wall))")
+        }
+    }
+}

experiments/live-dots/Sources/LiveDots/BoxOverlay.swift

@@ -0,0 +1,103 @@
+import LiveDotsCore
+import SwiftUI
+
+/// Brackets around the recognised things on the wall (simulated from the fixture's scene), in
+/// the manner of RoomPlan's outlines. The electric meter is solid Signal blue from the first
+/// keyframe it is in view; the others appear dashed at 60% Hologram (not settled) and turn solid
+/// at 90% once seen face-on. Appearing fades in over 200 ms, settling crossfades over 150 ms, and
+/// the meter flashes once when the hold completes. All of it is opacity, so Reduce Motion needs
+/// no variant.
+struct BoxOverlay: View {
+    var keyframe: Keyframe
+    var boxes: [BoxState]
+    var time: Float
+
+    static let signalBlue = Color(red: 0x1F / 255, green: 0x66 / 255, blue: 0xF2 / 255)
+    static let viewSize = SIMD2<Float>(390, 844)
+
+    var body: some View {
+        Canvas(rendersAsynchronously: false) { context, _ in
+            let projection = ScreenProjection(keyframe: keyframe, viewSize: Self.viewSize)
+            for (box, state) in zip(RecognisedBox.fixture, boxes) {
+                guard let appeared = state.appearedAt, time >= appeared else { continue }
+                let points = box.corners.compactMap(projection.screenPoint)
+                guard points.count == 4 else { continue }
+                let corners = Self.outset(points.map { CGPoint(x: CGFloat($0.x), y: CGFloat($0.y)) })
+                let shown = Double(CubicBezier.strongEaseOut((time - appeared) / 0.2))
+                let settled = state.solidAt.map { Double(CubicBezier.strongEaseOut((time - $0) / 0.15)) } ?? 0
+                let path = Self.bracket(corners)
+                if box.isMeter {
+                    context.stroke(path, with: .color(Self.signalBlue.opacity(shown)), style: Self.solid)
+                    let flash = flashStrength
+                    if flash > 0 {
+                        context.stroke(path, with: .color(.white.opacity(0.8 * flash)), style: StrokeStyle(lineWidth: 3, lineCap: .round, lineJoin: .round))
+                    }
+                } else {
+                    context.stroke(path, with: .color(Palette.hologram.opacity(0.6 * (1 - settled) * shown)), style: Self.dashed)
+                    context.stroke(path, with: .color(Palette.hologram.opacity(0.9 * settled * shown)), style: Self.solid)
+                }
+                drawCaption(box.caption, above: corners, opacity: shown, in: &context)
+            }
+        }
+        .accessibilityElement()
+        .accessibilityLabel(accessibilitySummary)
+    }
+
+    static let solid = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round)
+    static let dashed = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round, dash: [5, 4])
+
+    /// The 150 ms brighten after the hold's ring completes, a sine bump.
+    private var flashStrength: Double {
+        let start = Schedule.start(of: Schedule.holdIndex) + Schedule.holdDuration
+        let p = (time - start) / Schedule.flashDuration
+        guard p > 0, p < 1 else { return 0 }
+        return Double(sin(p * Float.pi))
+    }
+
+    /// Moves each corner 6 pt away from the box's centre. On the edge itself, a light bracket
+    /// vanished against the window's white frame.
+    static func outset(_ corners: [CGPoint]) -> [CGPoint] {
+        let cx = corners.map(\.x).reduce(0, +) / 4, cy = corners.map(\.y).reduce(0, +) / 4
+        return corners.map { p in
+            let dx = p.x - cx, dy = p.y - cy
+            let length = max((dx * dx + dy * dy).squareRoot(), 0.001)
+            return CGPoint(x: p.x + dx / length * 6, y: p.y + dy / length * 6)
+        }
+    }
+
+    /// Four L-shaped corners, each arm 18 pt or 40% of its edge, with a 4 pt rounded bend.
+    static func bracket(_ corners: [CGPoint]) -> Path {
+        var path = Path()
+        for i in corners.indices {
+            let corner = corners[i], previous = corners[(i + 3) % 4], next = corners[(i + 1) % 4]
+            func toward(_ other: CGPoint) -> CGPoint {
+                let dx = other.x - corner.x, dy = other.y - corner.y
+                let length = max((dx * dx + dy * dy).squareRoot(), 0.001)
+                let arm = min(18, 0.4 * length)
+                return CGPoint(x: corner.x + dx / length * arm, y: corner.y + dy / length * arm)
+            }
+            path.move(to: toward(previous))
+            path.addArc(tangent1End: corner, tangent2End: toward(next), radius: 4)
+            path.addLine(to: toward(next))
+        }
+        return path
+    }
+
+    private func drawCaption(_ caption: String, above corners: [CGPoint], opacity: Double, in context: inout GraphicsContext) {
+        let top = CGPoint(x: (corners[0].x + corners[1].x) / 2, y: min(corners[0].y, corners[1].y) - 13)
+        var layer = context
+        layer.opacity = opacity
+        layer.addFilter(.shadow(color: .black.opacity(0.4), radius: 6, y: 1))
+        let text = Text(caption)
+            .font(.system(size: 13, weight: .semibold, design: .rounded))
+            .foregroundStyle(.white)
+        layer.draw(text, at: top, anchor: .center)
+    }
+
+    private var accessibilitySummary: String {
+        let names = zip(RecognisedBox.fixture, boxes).compactMap { box, state in
+            state.appearedAt.map { time >= $0 ? box.caption : nil } ?? nil
+        }
+        return names.isEmpty ? "Nothing recognised yet" : "Recognised: " + names.joined(separator: ", ")
+    }
+}

experiments/live-dots/Sources/LiveDots/Command.swift

@@ -0,0 +1,100 @@
+import LiveDotsCore
+
+/// What the executable was asked to do. With no flags it opens the window.
+enum Command {
+    case app(fixture: String?)
+    /// Every frame of the replay at 30 fps to PNG files in a folder, or raw BGRA to stdout for "-".
+    case export(destination: String, options: RenderOptions)
+    /// One settled keyframe (1-based, as in the fixture's ids) to a PNG file.
+    case still(keyframe: Int, at: Float?, output: String, options: RenderOptions)
+    case gradientReport(fixture: String?)
+    case benchmark(fixture: String?)
+    case fieldReport(fixture: String?)
+
+    struct RenderOptions {
+        var fixture: String?
+        var mode: CaptureMode = .lidar
+        var reduceMotion = false
+        var fog: FogStyle = .on
+        var scheme: DotScheme = .hologram
+    }
+
+    struct UsageError: Error, CustomStringConvertible {
+        let problem: String
+        var description: String {
+            """
+            \(problem)
+            usage: LiveDots [--fixture <dir>]
+                   LiveDots --export <dir|-> --mode lidar|nolidar [--scheme hologram|constellation|ember] [--fog on|off|veil] [--reduce-motion] [--fixture <dir>]
+                   LiveDots --still <keyframe> [--at <seconds>] --out <file.png> --mode lidar|nolidar [--scheme ...] [--fog on|off|veil] [--reduce-motion] [--fixture <dir>]
+                   LiveDots --gradient-report [--fixture <dir>]
+                   LiveDots --benchmark [--fixture <dir>]
+                   LiveDots --field-report [--fixture <dir>]
+            """
+        }
+    }
+
+    static func parse(_ arguments: [String]) throws(UsageError) -> Command {
+        var options = RenderOptions()
+        var export: String?, still: Int?, output: String?, mode: String?, at: Float?
+        var report = false, benchmark = false, fieldReport = false
+        var rest = arguments[...]
+        func value(for flag: String) throws(UsageError) -> String {
+            guard let next = rest.popFirst() else { throw UsageError(problem: "\(flag) needs a value") }
+            return next
+        }
+        while let flag = rest.popFirst() {
+            switch flag {
+            case "--fixture": options.fixture = try value(for: flag)
+            case "--export": export = try value(for: flag)
+            case "--still":
+                let text = try value(for: flag)
+                guard let number = Int(text), number >= 1 else { throw UsageError(problem: "--still needs a keyframe number from 1, got \(text)") }
+                still = number
+            case "--out": output = try value(for: flag)
+            case "--at":
+                let text = try value(for: flag)
+                guard let seconds = Float(text), seconds >= 0 else { throw UsageError(problem: "--at needs seconds from 0, got \(text)") }
+                at = seconds
+            case "--mode": mode = try value(for: flag)
+            case "--scheme":
+                let name = try value(for: flag)
+                guard let scheme = DotScheme(rawValue: name) else {
+                    throw UsageError(problem: "--scheme must be hologram, constellation or ember, got \(name)")
+                }
+                options.scheme = scheme
+            case "--reduce-motion": options.reduceMotion = true
+            case "--fog":
+                let name = try value(for: flag)
+                guard let fog = FogStyle(rawValue: name) else { throw UsageError(problem: "--fog must be on, off or veil, got \(name)") }
+                options.fog = fog
+            case "--gradient-report": report = true
+            case "--benchmark": benchmark = true
+            case "--field-report": fieldReport = true
+            // Xcode and LaunchServices pass these to GUI launches.
+            case let other where other.hasPrefix("-NS") || other.hasPrefix("-Apple"): _ = rest.popFirst()
+            default: throw UsageError(problem: "unknown argument \(flag)")
+            }
+        }
+        if let mode {
+            switch mode {
+            case "lidar": options.mode = .lidar
+            case "nolidar": options.mode = .noLidar
+            default: throw UsageError(problem: "--mode must be lidar or nolidar, got \(mode)")
+            }
+        }
+        if report { return .gradientReport(fixture: options.fixture) }
+        if benchmark { return .benchmark(fixture: options.fixture) }
+        if fieldReport { return .fieldReport(fixture: options.fixture) }
+        if let export {
+            guard mode != nil else { throw UsageError(problem: "--export needs --mode") }
+            return .export(destination: export, options: options)
+        }
+        if let still {
+            guard mode != nil else { throw UsageError(problem: "--still needs --mode") }
+            guard let output else { throw UsageError(problem: "--still needs --out <file.png>") }
+            return .still(keyframe: still, at: at, output: output, options: options)
+        }
+        return .app(fixture: options.fixture)
+    }
+}

experiments/live-dots/Sources/LiveDots/ContentView.swift

@@ -0,0 +1,79 @@
+import LiveDotsCore
+import SwiftUI
+
+/// The phone on the left, the controls on the right.
+struct ContentView: View {
+    let fixtureArgument: String?
+
+    @State private var stage: Stage = .loading
+    @State private var progress = LoadProgress()
+    @Environment(\.accessibilityReduceMotion) private var systemReduceMotion
+
+    enum Stage {
+        case loading
+        case failed(String)
+        case ready(DotRenderer, ReplayPlayer, FrameTimer)
+    }
+
+    var body: some View {
+        HStack(alignment: .top, spacing: 32) {
+            PhoneFrame {
+                switch stage {
+                case .loading:
+                    ProgressView(value: progress.fraction) {
+                        Text("Fusing keyframes")
+                    }
+                    .progressViewStyle(.linear)
+                    .tint(Palette.hologram)
+                    .foregroundStyle(.white)
+                    .padding(48)
+                case let .failed(message):
+                    Text(message)
+                        .font(.callout)
+                        .foregroundStyle(.white)
+                        .padding(32)
+                case let .ready(renderer, player, timer):
+                    ZStack {
+                        DotsMetalView(renderer: renderer, player: player, timer: timer)
+                            .accessibilityHidden(true)
+                        TimelineView(.animation) { _ in
+                            let data = renderer.data, request = player.request
+                            PhoneChrome(
+                                state: data.timeline(request.mode).states[request.keyframe],
+                                keyframe: data.replay.keyframes[request.keyframe],
+                                boxes: data.boxes[request.keyframe], time: request.time)
+                        }
+                    }
+                }
+            }
+            Group {
+                if case let .ready(renderer, player, timer) = stage {
+                    ControlsPanel(player: player, data: renderer.data, timer: timer)
+                } else {
+                    Color.clear
+                }
+            }
+            .frame(width: 280)
+        }
+        .padding(28)
+        .background(Palette.window)
+        .preferredColorScheme(.dark)
+        .task { await load() }
+    }
+
+    private func load() async {
+        let progress = progress
+        do {
+            let folder = try FixtureLocator.folder(fixtureArgument)
+            let data = try await ReplayData.load(folder: folder) { done, total in
+                Task { @MainActor in progress.fraction = Double(done) / Double(total) }
+            }
+            let renderer = try DotRenderer(data: data)
+            let player = ReplayPlayer(keyframeCount: data.keyframeCount, reduceMotion: systemReduceMotion)
+            stage = .ready(renderer, player, FrameTimer())
+            player.togglePlayback()
+        } catch {
+            stage = .failed(String(describing: error))
+        }
+    }
+}