Show folder discovery progress before indexing begins

Sources/DigUpApp/Engine.swift

@@ -592,13 +592,19 @@ nonisolated final class Engine: @unchecked Sendable {
         func int(_ key: String) -> Int { event[key] as? Int ?? 0 }
         if event["code"] as? Bool == true { return handleCode(event) }
         switch event["event"] as? String {
+        case "scanning":
+            update {
+                if case .failed = $0.activity { return }
+                $0.activity = .syncing
+            }
         case "sync":
             if let started = event["started"] as? Double, let wanted = syncWantedAt,
                started >= wanted.timeIntervalSince1970 {
                 syncWantedAt = nil
             }
             applyCounts(event)
             if let leftOut = event["left_out"] as? [String: [String: Int]] { update { $0.leftOut = leftOut } }
+            update { if case .syncing = $0.activity { $0.activity = .idle } }
             if int("new") > 0 { update { $0.activity = .indexing(done: 0, total: int("new")) } }
             log("sync: \(int("added")) new · \(int("changed")) changed · \(int("moved")) moved · "
                 + "\(int("removed")) removed · \(int("unchanged")) unchanged in \(int("ms")) ms")
@@ -649,7 +655,10 @@ nonisolated final class Engine: @unchecked Sendable {
             log("indexer: the model or settings changed, indexing everything again (was: \(event["from"] as? String ?? "?"))")
         case "error":
             log("indexer: \(event["message"] as? String ?? "error")")
-            if event["fatal"] as? Bool == true { update { $0.activity = .failed("Indexing stopped") } }
+            update {
+                if event["fatal"] as? Bool == true { $0.activity = .failed("Indexing stopped") }
+                else if case .syncing = $0.activity { $0.activity = .idle }
+            }
         default:
             break
         }
@@ -725,6 +734,7 @@ nonisolated final class Engine: @unchecked Sendable {
             if case .indexing = $0.activity { $0.activity = .idle }
             if case .finishing = $0.activity { $0.activity = .idle }
             if case .starting = $0.activity { $0.activity = .idle }
+            if case .syncing = $0.activity { $0.activity = .idle }
             $0.code.indexing = nil
         }
         refreshSearcher(force: true)

Sources/DigUpKit/Crawler.swift

@@ -20,6 +20,18 @@ public struct CrawlReport: Sendable {
     public var leftOut: [String: [String: Int]] = [:]
 }
 
+/// Entries checked during a folder walk, including files that may later be skipped.
+/// The count is cumulative across roots and is not an indexing total.
+public struct CrawlProgress: Sendable, Equatable {
+    public let checked: Int
+    public let root: String?
+
+    public init(checked: Int, root: String?) {
+        self.checked = checked
+        self.root = root
+    }
+}
+
 /// Walks the chosen folders and applies `Rules`. Reads metadata only, never file contents (except the 600-byte
 /// sniff that tells a real MPEG-TS video from a TypeScript file).
 ///
@@ -30,7 +42,8 @@ public enum Crawler {
     /// gone, says no). Opening is enough: listing a cloud folder's top level could take a while.
     public static func canList(_ url: URL) -> Bool { FileSystem.canList(url.path) }
 
-    public static func crawl(_ roots: [URL], options: IndexOptions) -> CrawlReport {
+    public static func crawl(_ roots: [URL], options: IndexOptions,
+                             progress: ((CrawlProgress) -> Void)? = nil) -> CrawlReport {
         let fm = FileManager.default
         let home = fm.homeDirectoryForCurrentUser.path
         let keys = FileSystem.crawlKeys
@@ -45,8 +58,13 @@ public enum Crawler {
             report.leftOut[root, default: [:]][reason, default: 0] += 1
         }
         let rootPaths = Set(roots.map { $0.standardizedFileURL.path })
+        var checked = 0
+        let clock = ContinuousClock()
+        var lastUpdate = clock.now
 
         for root in roots.map({ $0.standardizedFileURL }) {
+            progress?(CrawlProgress(checked: checked, root: root.path))
+            lastUpdate = clock.now
             if Rules.isCacheDirectory(root) {
                 report.skippedFolders["cache folder", default: 0] += 1
                 continue
@@ -137,6 +155,14 @@ public enum Crawler {
             // URL resource values are autoreleased; draining per file keeps a big walk from piling them up (the app
             // crawls on a background queue whose pool only drains when the whole sync is done).
             while let url = autoreleasepool(invoking: { walker.nextObject() as? URL }) {
+                if let progress {
+                    checked += 1
+                    let now = clock.now
+                    if now - lastUpdate >= .milliseconds(250) {
+                        progress(CrawlProgress(checked: checked, root: root.path))
+                        lastUpdate = now
+                    }
+                }
                 autoreleasepool {
                     visit(url)
                 }
@@ -177,6 +203,7 @@ public enum Crawler {
             }
         }
         report.candidates = kept
+        progress?(CrawlProgress(checked: checked, root: nil))
         return report
     }
 }

Sources/DigUpKit/Indexer.swift

@@ -87,8 +87,8 @@ public final class Indexer {
 
     /// Crawls `roots` and reconciles the index: new files are queued, changed ones re-queued, moved ones re-pathed
     /// (no re-embedding), and files that are gone or newly excluded are removed.
-    public func sync(_ roots: [URL]) throws -> SyncSummary {
-        let crawl = Crawler.crawl(roots, options: options)
+    public func sync(_ roots: [URL], progress: ((CrawlProgress) -> Void)? = nil) throws -> SyncSummary {
+        let crawl = Crawler.crawl(roots, options: options, progress: progress)
         let existing = try store.files(under: roots.map { $0.standardizedFileURL.path })
         var byPath: [String: FileRecord] = [:]
         var byInode: [String: FileRecord] = [:]

Sources/digup/IndexService.swift

@@ -12,7 +12,7 @@ import WinSDK
 ///
 /// Protocol: one command per line on stdin, one JSON event per line on stdout.
 ///   stdin:  sync | sync code | stop | backfill on | backfill off      (EOF = stop)
-///   stdout: {"event": "missing" | "pruned" | "sync" | "file" | "rest" | "summary" | "waiting" | "idle" | "error" |
+///   stdout: {"event": "missing" | "pruned" | "scanning" | "sync" | "file" | "rest" | "summary" | "waiting" | "idle" | "error" |
 ///            "exit", ...}; events about the code index say "code": true
 ///
 /// The loop: sync, index what the sync just queued (if it's a few files: new files go first, on any power source),
@@ -219,7 +219,13 @@ final class IndexService: @unchecked Sendable {
             lock.withLock { yieldAt = Date() + max(took, 2) }
         }
         do {
-            let summary = try part.indexer.sync(availableRoots(part))
+            let roots = availableRoots(part)
+            emit(["event": "scanning", "checked": 0, "code": part.code])
+            let summary = try part.indexer.sync(roots) { progress in
+                var event: [String: Any] = ["event": "scanning", "checked": progress.checked, "code": part.code]
+                if let root = progress.root { event["root"] = root }
+                emit(event)
+            }
             if summary.queued.count <= newFilesLimit { newFiles.formUnion(summary.queued) }
             emitCounts("sync", [
                 "code": part.code,

Tests/DigUpKitTests/CrawlProgressTests.swift

@@ -0,0 +1,90 @@
+import Foundation
+import Testing
+@testable import DigUpKit
+
+@Suite struct CrawlProgressTests {
+    @Test func emptyFolderReportsStartAndFinal() throws {
+        let root = try makeFolder()
+        defer { try? FileManager.default.removeItem(at: root) }
+        var updates: [CrawlProgress] = []
+
+        let report = Crawler.crawl([root], options: IndexOptions()) { updates.append($0) }
+
+        #expect(report.candidates.isEmpty)
+        #expect(updates == [CrawlProgress(checked: 0, root: root.standardizedFileURL.path),
+                            CrawlProgress(checked: 0, root: nil)])
+    }
+
+    @Test func countsEnumeratedEntriesIncludingExcludedFoldersWithoutChangingCandidates() throws {
+        let root = try makeFolder()
+        defer { try? FileManager.default.removeItem(at: root) }
+        let excluded = root.appendingPathComponent("excluded")
+        try FileManager.default.createDirectory(at: excluded, withIntermediateDirectories: true)
+        try "keep".write(to: root.appendingPathComponent("keep.txt"), atomically: false, encoding: .utf8)
+        try "skip".write(to: excluded.appendingPathComponent("skip.txt"), atomically: false, encoding: .utf8)
+        try "skip".write(to: root.appendingPathComponent("other.xyz"), atomically: false, encoding: .utf8)
+        var options = IndexOptions()
+        options.excludedFolders = [excluded.path]
+        var updates: [CrawlProgress] = []
+
+        let baseline = Crawler.crawl([root], options: options)
+        let observed = Crawler.crawl([root], options: options) { updates.append($0) }
+
+        #expect(observed.candidates.map(\.path) == baseline.candidates.map(\.path))
+        #expect(observed.skipped == baseline.skipped)
+        #expect(observed.skippedFolders == baseline.skippedFolders)
+        #expect(observed.candidates.map(\.path) == [FolderSelection.realPath(root.appendingPathComponent("keep.txt").path)])
+        #expect(updates.first == CrawlProgress(checked: 0, root: root.standardizedFileURL.path))
+        #expect(updates.last == CrawlProgress(checked: 3, root: nil))
+        #expect(updates.map(\.checked) == updates.map(\.checked).sorted())
+    }
+
+    @Test func startsEachRootAndKeepsCumulativeCount() throws {
+        let first = try makeFolder()
+        let second = try makeFolder()
+        defer {
+            try? FileManager.default.removeItem(at: first)
+            try? FileManager.default.removeItem(at: second)
+        }
+        try "one".write(to: first.appendingPathComponent("one.txt"), atomically: false, encoding: .utf8)
+        try "two".write(to: second.appendingPathComponent("two.txt"), atomically: false, encoding: .utf8)
+        var updates: [CrawlProgress] = []
+
+        _ = Crawler.crawl([first, second], options: IndexOptions()) { updates.append($0) }
+
+        #expect(updates.first == CrawlProgress(checked: 0, root: first.standardizedFileURL.path))
+        #expect(updates.contains(CrawlProgress(checked: 1, root: second.standardizedFileURL.path)))
+        #expect(updates.last == CrawlProgress(checked: 2, root: nil))
+    }
+
+    @Test func indexerForwardsProgressBeforeWritingNewRows() throws {
+        let root = try makeFolder()
+        let index = try makeFolder()
+        defer { try? FileManager.default.removeItem(at: root) }
+        defer { try? FileManager.default.removeItem(at: index) }
+        try "note".write(to: root.appendingPathComponent("note.txt"), atomically: false, encoding: .utf8)
+        let store = try IndexStore(directory: index)
+        let indexer = try Indexer(store: store, options: IndexOptions(),
+                                  makeEmbedder: { throw WorkerError.notFound("no model needed to sync") })
+        var updates: [CrawlProgress] = []
+        var rowsDuringProgress: [Int] = []
+
+        let summary = try indexer.sync([root]) { update in
+            updates.append(update)
+            rowsDuringProgress.append((try? store.files(under: [root.path]).count) ?? -1)
+        }
+
+        #expect(summary.added == 1)
+        #expect(updates.first == CrawlProgress(checked: 0, root: root.standardizedFileURL.path))
+        #expect(updates.last?.root == nil)
+        #expect(updates.last?.checked == 1)
+        #expect(rowsDuringProgress.allSatisfy { $0 == 0 })
+        #expect(try store.files(under: [root.path]).count == 1)
+    }
+
+    private func makeFolder() throws -> URL {
+        let root = FileManager.default.temporaryDirectory.appendingPathComponent("DigUpCrawlProgress-\(UUID().uuidString)")
+        try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+        return root
+    }
+}