diff --git a/Sources/DigUpApp/Engine.swift b/Sources/DigUpApp/Engine.swift index dfd7c5ca..15d7d53a 100644 --- a/Sources/DigUpApp/Engine.swift +++ b/Sources/DigUpApp/Engine.swift @@ -592,6 +592,11 @@ nonisolated final class Engine: @unchecked Sendable { func int(_ key: String) -> Int { event[key] as? Int ?? 0 } if event["code"] as? Bool == true { return handleCode(event) } switch event["event"] as? String { + case "scanning": + update { + if case .failed = $0.activity { return } + $0.activity = .syncing + } case "sync": if let started = event["started"] as? Double, let wanted = syncWantedAt, started >= wanted.timeIntervalSince1970 { @@ -599,6 +604,7 @@ nonisolated final class Engine: @unchecked Sendable { } applyCounts(event) if let leftOut = event["left_out"] as? [String: [String: Int]] { update { $0.leftOut = leftOut } } + update { if case .syncing = $0.activity { $0.activity = .idle } } if int("new") > 0 { update { $0.activity = .indexing(done: 0, total: int("new")) } } log("sync: \(int("added")) new · \(int("changed")) changed · \(int("moved")) moved · " + "\(int("removed")) removed · \(int("unchanged")) unchanged in \(int("ms")) ms") @@ -649,7 +655,10 @@ nonisolated final class Engine: @unchecked Sendable { log("indexer: the model or settings changed, indexing everything again (was: \(event["from"] as? String ?? "?"))") case "error": log("indexer: \(event["message"] as? String ?? "error")") - if event["fatal"] as? Bool == true { update { $0.activity = .failed("Indexing stopped") } } + update { + if event["fatal"] as? Bool == true { $0.activity = .failed("Indexing stopped") } + else if case .syncing = $0.activity { $0.activity = .idle } + } default: break } @@ -725,6 +734,7 @@ nonisolated final class Engine: @unchecked Sendable { if case .indexing = $0.activity { $0.activity = .idle } if case .finishing = $0.activity { $0.activity = .idle } if case .starting = $0.activity { $0.activity = .idle } + if case .syncing = $0.activity { $0.activity = .idle } $0.code.indexing = nil } refreshSearcher(force: true) diff --git a/Sources/DigUpKit/Crawler.swift b/Sources/DigUpKit/Crawler.swift index 048f461a..fd8f878e 100644 --- a/Sources/DigUpKit/Crawler.swift +++ b/Sources/DigUpKit/Crawler.swift @@ -20,6 +20,18 @@ public struct CrawlReport: Sendable { public var leftOut: [String: [String: Int]] = [:] } +/// Entries checked during a folder walk, including files that may later be skipped. +/// The count is cumulative across roots and is not an indexing total. +public struct CrawlProgress: Sendable, Equatable { + public let checked: Int + public let root: String? + + public init(checked: Int, root: String?) { + self.checked = checked + self.root = root + } +} + /// Walks the chosen folders and applies `Rules`. Reads metadata only, never file contents (except the 600-byte /// sniff that tells a real MPEG-TS video from a TypeScript file). /// @@ -30,7 +42,8 @@ public enum Crawler { /// gone, says no). Opening is enough: listing a cloud folder's top level could take a while. public static func canList(_ url: URL) -> Bool { FileSystem.canList(url.path) } - public static func crawl(_ roots: [URL], options: IndexOptions) -> CrawlReport { + public static func crawl(_ roots: [URL], options: IndexOptions, + progress: ((CrawlProgress) -> Void)? = nil) -> CrawlReport { let fm = FileManager.default let home = fm.homeDirectoryForCurrentUser.path let keys = FileSystem.crawlKeys @@ -45,8 +58,13 @@ public enum Crawler { report.leftOut[root, default: [:]][reason, default: 0] += 1 } let rootPaths = Set(roots.map { $0.standardizedFileURL.path }) + var checked = 0 + let clock = ContinuousClock() + var lastUpdate = clock.now for root in roots.map({ $0.standardizedFileURL }) { + progress?(CrawlProgress(checked: checked, root: root.path)) + lastUpdate = clock.now if Rules.isCacheDirectory(root) { report.skippedFolders["cache folder", default: 0] += 1 continue @@ -137,6 +155,14 @@ public enum Crawler { // URL resource values are autoreleased; draining per file keeps a big walk from piling them up (the app // crawls on a background queue whose pool only drains when the whole sync is done). while let url = autoreleasepool(invoking: { walker.nextObject() as? URL }) { + if let progress { + checked += 1 + let now = clock.now + if now - lastUpdate >= .milliseconds(250) { + progress(CrawlProgress(checked: checked, root: root.path)) + lastUpdate = now + } + } autoreleasepool { visit(url) } @@ -177,6 +203,7 @@ public enum Crawler { } } report.candidates = kept + progress?(CrawlProgress(checked: checked, root: nil)) return report } } diff --git a/Sources/DigUpKit/Indexer.swift b/Sources/DigUpKit/Indexer.swift index 92c98822..2b69040c 100644 --- a/Sources/DigUpKit/Indexer.swift +++ b/Sources/DigUpKit/Indexer.swift @@ -87,8 +87,8 @@ public final class Indexer { /// Crawls `roots` and reconciles the index: new files are queued, changed ones re-queued, moved ones re-pathed /// (no re-embedding), and files that are gone or newly excluded are removed. - public func sync(_ roots: [URL]) throws -> SyncSummary { - let crawl = Crawler.crawl(roots, options: options) + public func sync(_ roots: [URL], progress: ((CrawlProgress) -> Void)? = nil) throws -> SyncSummary { + let crawl = Crawler.crawl(roots, options: options, progress: progress) let existing = try store.files(under: roots.map { $0.standardizedFileURL.path }) var byPath: [String: FileRecord] = [:] var byInode: [String: FileRecord] = [:] diff --git a/Sources/digup/IndexService.swift b/Sources/digup/IndexService.swift index 5e405bf4..db2a807f 100644 --- a/Sources/digup/IndexService.swift +++ b/Sources/digup/IndexService.swift @@ -12,7 +12,7 @@ import WinSDK /// /// Protocol: one command per line on stdin, one JSON event per line on stdout. /// stdin: sync | sync code | stop | backfill on | backfill off (EOF = stop) -/// stdout: {"event": "missing" | "pruned" | "sync" | "file" | "rest" | "summary" | "waiting" | "idle" | "error" | +/// stdout: {"event": "missing" | "pruned" | "scanning" | "sync" | "file" | "rest" | "summary" | "waiting" | "idle" | "error" | /// "exit", ...}; events about the code index say "code": true /// /// The loop: sync, index what the sync just queued (if it's a few files: new files go first, on any power source), @@ -219,7 +219,13 @@ final class IndexService: @unchecked Sendable { lock.withLock { yieldAt = Date() + max(took, 2) } } do { - let summary = try part.indexer.sync(availableRoots(part)) + let roots = availableRoots(part) + emit(["event": "scanning", "checked": 0, "code": part.code]) + let summary = try part.indexer.sync(roots) { progress in + var event: [String: Any] = ["event": "scanning", "checked": progress.checked, "code": part.code] + if let root = progress.root { event["root"] = root } + emit(event) + } if summary.queued.count <= newFilesLimit { newFiles.formUnion(summary.queued) } emitCounts("sync", [ "code": part.code, diff --git a/Tests/DigUpKitTests/CrawlProgressTests.swift b/Tests/DigUpKitTests/CrawlProgressTests.swift new file mode 100644 index 00000000..35654354 --- /dev/null +++ b/Tests/DigUpKitTests/CrawlProgressTests.swift @@ -0,0 +1,90 @@ +import Foundation +import Testing +@testable import DigUpKit + +@Suite struct CrawlProgressTests { + @Test func emptyFolderReportsStartAndFinal() throws { + let root = try makeFolder() + defer { try? FileManager.default.removeItem(at: root) } + var updates: [CrawlProgress] = [] + + let report = Crawler.crawl([root], options: IndexOptions()) { updates.append($0) } + + #expect(report.candidates.isEmpty) + #expect(updates == [CrawlProgress(checked: 0, root: root.standardizedFileURL.path), + CrawlProgress(checked: 0, root: nil)]) + } + + @Test func countsEnumeratedEntriesIncludingExcludedFoldersWithoutChangingCandidates() throws { + let root = try makeFolder() + defer { try? FileManager.default.removeItem(at: root) } + let excluded = root.appendingPathComponent("excluded") + try FileManager.default.createDirectory(at: excluded, withIntermediateDirectories: true) + try "keep".write(to: root.appendingPathComponent("keep.txt"), atomically: false, encoding: .utf8) + try "skip".write(to: excluded.appendingPathComponent("skip.txt"), atomically: false, encoding: .utf8) + try "skip".write(to: root.appendingPathComponent("other.xyz"), atomically: false, encoding: .utf8) + var options = IndexOptions() + options.excludedFolders = [excluded.path] + var updates: [CrawlProgress] = [] + + let baseline = Crawler.crawl([root], options: options) + let observed = Crawler.crawl([root], options: options) { updates.append($0) } + + #expect(observed.candidates.map(\.path) == baseline.candidates.map(\.path)) + #expect(observed.skipped == baseline.skipped) + #expect(observed.skippedFolders == baseline.skippedFolders) + #expect(observed.candidates.map(\.path) == [FolderSelection.realPath(root.appendingPathComponent("keep.txt").path)]) + #expect(updates.first == CrawlProgress(checked: 0, root: root.standardizedFileURL.path)) + #expect(updates.last == CrawlProgress(checked: 3, root: nil)) + #expect(updates.map(\.checked) == updates.map(\.checked).sorted()) + } + + @Test func startsEachRootAndKeepsCumulativeCount() throws { + let first = try makeFolder() + let second = try makeFolder() + defer { + try? FileManager.default.removeItem(at: first) + try? FileManager.default.removeItem(at: second) + } + try "one".write(to: first.appendingPathComponent("one.txt"), atomically: false, encoding: .utf8) + try "two".write(to: second.appendingPathComponent("two.txt"), atomically: false, encoding: .utf8) + var updates: [CrawlProgress] = [] + + _ = Crawler.crawl([first, second], options: IndexOptions()) { updates.append($0) } + + #expect(updates.first == CrawlProgress(checked: 0, root: first.standardizedFileURL.path)) + #expect(updates.contains(CrawlProgress(checked: 1, root: second.standardizedFileURL.path))) + #expect(updates.last == CrawlProgress(checked: 2, root: nil)) + } + + @Test func indexerForwardsProgressBeforeWritingNewRows() throws { + let root = try makeFolder() + let index = try makeFolder() + defer { try? FileManager.default.removeItem(at: root) } + defer { try? FileManager.default.removeItem(at: index) } + try "note".write(to: root.appendingPathComponent("note.txt"), atomically: false, encoding: .utf8) + let store = try IndexStore(directory: index) + let indexer = try Indexer(store: store, options: IndexOptions(), + makeEmbedder: { throw WorkerError.notFound("no model needed to sync") }) + var updates: [CrawlProgress] = [] + var rowsDuringProgress: [Int] = [] + + let summary = try indexer.sync([root]) { update in + updates.append(update) + rowsDuringProgress.append((try? store.files(under: [root.path]).count) ?? -1) + } + + #expect(summary.added == 1) + #expect(updates.first == CrawlProgress(checked: 0, root: root.standardizedFileURL.path)) + #expect(updates.last?.root == nil) + #expect(updates.last?.checked == 1) + #expect(rowsDuringProgress.allSatisfy { $0 == 0 }) + #expect(try store.files(under: [root.path]).count == 1) + } + + private func makeFolder() throws -> URL { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("DigUpCrawlProgress-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + return root + } +}