From 1e1525bf6bebbef436b785c7e2e5a7681a099687 Mon Sep 17 00:00:00 2001 From: Ben Siraphob Date: Fri, 23 Oct 2026 00:59:41 -0400 Subject: [PATCH 1/7] Reuse extracted text during document backfill --- Sources/DigUpKit/Indexer.swift | 89 +++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/Sources/DigUpKit/Indexer.swift b/Sources/DigUpKit/Indexer.swift index 2b69040c..45d52de3 100644 --- a/Sources/DigUpKit/Indexer.swift +++ b/Sources/DigUpKit/Indexer.swift @@ -196,7 +196,8 @@ public final class Indexer { var files = Set(jobs.map { $0.files[0].id }).count var states: [String: Int] = [:] var finished = Set() - try process(jobs, shouldStop: shouldStop) { index, job, results, seconds in + try process(jobs, shouldStop: shouldStop, documentCache: DocumentPreparationCache()) { + index, job, results, seconds in let file = job.files[0] let (state, note) = results[0] if state != "partial", finished.insert(file.id).inserted { @@ -219,6 +220,7 @@ public final class Indexer { /// (images scaled, screenshots OCR'd, PDF pages rendered, audio and video decoded). OCR alone is ~0.24 s per Retina /// screenshot, about as long as embedding it. private func process(_ jobs: [Job], shouldStop: () -> Bool, + documentCache: DocumentPreparationCache? = nil, committed: (Int, Job, [(String, String?)], Double) -> Void, skip: (Job) -> Bool = { _ in false }) throws { var ahead: PendingBatch? @@ -232,11 +234,15 @@ public final class Indexer { } for (index, job) in jobs.enumerated() { guard !shouldStop() else { break } + if skip(job) { + if let ahead { try? FileManager.default.removeItem(at: ahead.wait().folder) } + ahead = nil + continue + } let started = Date() - let prepared = (ahead ?? prepare(job)).wait() - ahead = index + 1 < jobs.count ? prepare(jobs[index + 1]) : nil + let prepared = (ahead ?? prepare(job, documentCache: documentCache)).wait() + ahead = index + 1 < jobs.count ? prepare(jobs[index + 1], documentCache: documentCache) : nil defer { try? FileManager.default.removeItem(at: prepared.folder) } - guard !skip(job) else { continue } // The helper's and the CLI's main thread has no run loop to drain what Foundation autoreleases while a job // commits: without a pool per job it all stayed until the process exited (~30 KB a text chunk). let modelWasRunning = worker != nil @@ -350,6 +356,57 @@ public final class Indexer { case code(CodeExtractor.File) } + /// The extracted chunks of the document currently in the preparation queue. A long document has one job per + /// `docStep`; without this cache every job reopened, reparsed and rechunked the whole file from its beginning. + /// Jobs for one file are contiguous, so replacing this one value bounds retained document text to one file. + final class DocumentPreparationCache: @unchecked Sendable { + private struct Key: Equatable { + let file: Int64 + let path: String + let size: Int64 + let modified: Double + let limit: Int + } + + struct Value: Sendable { + let chunks: [String] + let characters: Int + } + + private enum Load { + case document(Value) + case failure(ExtractError) + } + + private let lock = NSLock() + private var cached: (key: Key, load: Load)? + + func read(_ file: FileRecord, at url: URL, limit: Int) throws -> Value { + try lock.withLock { + let key = Key(file: file.id, path: file.path, size: file.size, modified: file.mtime, limit: limit) + if let cached, cached.key == key { + switch cached.load { + case .document(let value): return value + case .failure(let error): throw error + } + } + do { + let text = try DocExtractor.text(of: url) + let value = Value(chunks: DocExtractor.chunks(text, limit: limit), characters: text.count) + cached = (key, .document(value)) + return value + } catch let error as ExtractError { + cached = (key, .failure(error)) + throw error + } catch { + let error = ExtractError.unreadable("\(error)") + cached = (key, .failure(error)) + throw error + } + } + } + } + struct ReadyImage: Sendable { let slot: Int let image: ImageExtractor.Prepared @@ -388,17 +445,18 @@ public final class Indexer { readingLock.withLock { reading = (path, Date()) } } - private func prepare(_ job: Job) -> PendingBatch { + private func prepare(_ job: Job, documentCache: DocumentPreparationCache? = nil) -> PendingBatch { let pending = PendingBatch() jobsPrepared += 1 let (options, folder) = (options, scratch.appendingPathComponent("batch-\(jobsPrepared)")) preparing.async { - pending.fulfill(Self.prepare(job, options: options, into: folder)) + pending.fulfill(Self.prepare(job, options: options, into: folder, documentCache: documentCache)) } return pending } - static func prepare(_ job: Job, options: IndexOptions, into folder: URL) -> Prepared { + static func prepare(_ job: Job, options: IndexOptions, into folder: URL, + documentCache: DocumentPreparationCache? = nil) -> Prepared { let batch = job.files var prepared = Prepared(batch: batch, step: job.step, folder: folder) let url = URL(fileURLWithPath: batch[0].path) @@ -439,11 +497,18 @@ public final class Indexer { prepared.payload = .pdf(pages: read.pages, read: wanted, total: read.total, title: read.title) case .doc: // The chunks are cut the same way every time, so a later step goes on where the last one stopped. - let text = try DocExtractor.text(of: url) - let chunks = DocExtractor.chunks(text, limit: options.maxChunks + 1) - let read = (job.step ?? 0.. Date: Fri, 23 Oct 2026 01:02:58 -0400 Subject: [PATCH 2/7] Bound document inputs before extraction --- Sources/DigUpKit/Crawler.swift | 4 ++++ Sources/DigUpKit/Indexer.swift | 4 ++++ Sources/DigUpKit/Model.swift | 3 +++ 3 files changed, 11 insertions(+) diff --git a/Sources/DigUpKit/Crawler.swift b/Sources/DigUpKit/Crawler.swift index fd8f878e..71d8a0a2 100644 --- a/Sources/DigUpKit/Crawler.swift +++ b/Sources/DigUpKit/Crawler.swift @@ -132,6 +132,10 @@ public enum Crawler { leftOut("in the cloud, not downloaded", under: root.path) return } + if !options.code, kind == .doc, info.size > options.maxDocumentBytes { + leftOut("document over \(options.maxDocumentBytes >> 20) MB", under: root.path) + return + } if options.code { // Too big to be written by hand; a notebook's file holds its outputs too. let limit = url.pathExtension.lowercased() == "ipynb" ? options.maxNotebookBytes diff --git a/Sources/DigUpKit/Indexer.swift b/Sources/DigUpKit/Indexer.swift index 45d52de3..465c06b6 100644 --- a/Sources/DigUpKit/Indexer.swift +++ b/Sources/DigUpKit/Indexer.swift @@ -497,6 +497,10 @@ public final class Indexer { prepared.payload = .pdf(pages: read.pages, read: wanted, total: read.total, title: read.title) case .doc: // The chunks are cut the same way every time, so a later step goes on where the last one stopped. + let size = FileSystem.info(of: url.path)?.size ?? batch[0].size + guard size <= options.maxDocumentBytes else { + throw ExtractError.leftOut("document over \(options.maxDocumentBytes >> 20) MB") + } let document: DocumentPreparationCache.Value if let documentCache { document = try documentCache.read(batch[0], at: url, limit: options.maxChunks + 1) diff --git a/Sources/DigUpKit/Model.swift b/Sources/DigUpKit/Model.swift index 72cb6726..b9df810a 100644 --- a/Sources/DigUpKit/Model.swift +++ b/Sources/DigUpKit/Model.swift @@ -46,6 +46,9 @@ public struct IndexOptions: Sendable { public var maxCodeBytes = 1 << 20 /// A notebook's file holds its outputs too (pictures, tables): bigger than this, it isn't read. public var maxNotebookBytes = 20 << 20 + /// Documents larger than this are left out before extraction. The document index retains at most about 9 MB of + /// text, and loading a larger HTML or rich-text file can need several full-size intermediate strings. + public var maxDocumentBytes: Int64 = 64 << 20 public var includeCode = false public var includeDatasets = false public var minImageSide = 256 From 07e2fd4fba2ed9dc36161476ea0c9490f05d3ead Mon Sep 17 00:00:00 2001 From: Ben Siraphob Date: Fri, 23 Oct 2026 01:03:20 -0400 Subject: [PATCH 3/7] Keep preview rendering off the request loop --- Sources/digup/SearchService.swift | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Sources/digup/SearchService.swift b/Sources/digup/SearchService.swift index fca7b36f..4afe8bec 100644 --- a/Sources/digup/SearchService.swift +++ b/Sources/digup/SearchService.swift @@ -53,7 +53,12 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l private let previews: URL private var rendered: [String: String] = [:] private let renderLock = NSLock() - private let renderSlots = DispatchSemaphore(value: 2) + private let renderQueue: OperationQueue = { + let queue = OperationQueue() + queue.maxConcurrentOperationCount = 2 + queue.qualityOfService = .userInitiated + return queue + }() init(index: URL, roots: [String], skipped: [String], types: Set, codeRoots: [String], codeSkipped: [String], idleSeconds: Double) throws { @@ -97,10 +102,8 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l return 0 } if op == "render" { - renderSlots.wait() let request = UncheckedSendable(request) - Thread.detachNewThread { [self] in - defer { renderSlots.signal() } + renderQueue.addOperation { [self] in var reply: [String: Any] do { reply = try render(request.value) } catch { reply = ["ok": false, "error": String("\(error)".prefix(500))] From 72d87ad15171ea1406b68a182fb3268cc1e4d0d1 Mon Sep 17 00:00:00 2001 From: Ben Siraphob Date: Fri, 23 Oct 2026 01:04:20 -0400 Subject: [PATCH 4/7] Coalesce obsolete query embeddings --- Sources/digup/SearchService.swift | 35 +++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/Sources/digup/SearchService.swift b/Sources/digup/SearchService.swift index 4afe8bec..27d79d98 100644 --- a/Sources/digup/SearchService.swift +++ b/Sources/digup/SearchService.swift @@ -49,6 +49,14 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l private var warnedAboutSource = false private let lock = NSLock() private let output = NSLock() + private let typedLock = NSLock() + private var newestTyped: String? + private let requestQueue: OperationQueue = { + let queue = OperationQueue() + queue.maxConcurrentOperationCount = 1 + queue.qualityOfService = .userInitiated + return queue + }() /// Renders for previews: two at a time, beside the request loop, each made once per file version. private let previews: URL private var rendered: [String: String] = [:] @@ -97,6 +105,7 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l do { guard let op = request["op"] as? String else { throw CLIError("no op") } if op == "quit" { + requestQueue.cancelAllOperations() write(["id": id, "ok": true]) lock.withLock { stopEncoder() } return 0 @@ -113,13 +122,33 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l } continue } - reply = try lock.withLock { try handle(op, request) } + let request = UncheckedSendable(request) + let typed = op == "embed_ahead" + ? Self.vectorKey(request.value["q"] as? String ?? "", code: request.value["code"] as? Bool ?? false) + : nil + if let typed { typedLock.withLock { newestTyped = typed } } + requestQueue.addOperation { [self] in + var reply: [String: Any] + do { + if let typed, typedLock.withLock({ newestTyped }) != typed { + reply = ["ok": true] + } else { + reply = try lock.withLock { try handle(op, request.value) } + } + } catch { + reply = ["ok": false, "error": String("\(error)".prefix(500))] + } + reply["id"] = request.value["id"] ?? NSNull() + write(reply) + } + continue } catch { reply = ["ok": false, "error": String("\(error)".prefix(500))] } reply["id"] = id write(reply) } + requestQueue.waitUntilAllOperationsAreFinished() lock.withLock { stopEncoder() } return 0 } @@ -286,9 +315,11 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l // MARK: The query encoder /// The query's vector (cached), or nil when the encoder can't run: keyword results still work. + private static func vectorKey(_ text: String, code: Bool) -> String { (code ? "code\u{0}" : "") + text } + private func vector(for text: String, code: Bool) -> [Float]? { lastQuery = Date() - let key = (code ? "code\u{0}" : "") + text + let key = Self.vectorKey(text, code: code) if let cached = vectorCache[key] { return cached } guard let encoder = runningEncoder() else { return nil } do { From ef45048b883de39bc29981d87f20db3dcb93e9da Mon Sep 17 00:00:00 2001 From: Ben Siraphob Date: Fri, 23 Oct 2026 01:07:00 -0400 Subject: [PATCH 5/7] Bound portable image decode memory --- .../Platform/Portable/Images+Portable.swift | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/Sources/DigUpKit/Platform/Portable/Images+Portable.swift b/Sources/DigUpKit/Platform/Portable/Images+Portable.swift index 97a703b5..401399a8 100644 --- a/Sources/DigUpKit/Platform/Portable/Images+Portable.swift +++ b/Sources/DigUpKit/Platform/Portable/Images+Portable.swift @@ -8,9 +8,8 @@ extension ImageExtractor { /// No SVG here yet: stb and ffmpeg don't draw vector pictures. static let drawsVectorPictures = false - /// Writes a downscaled copy for the model: decoded in full, turned the way the camera held it, scaled so the long - /// side is `maxPixel`. stb reads JPEG, PNG, BMP, GIF, PSD, TGA and PNM here; HEIC, WebP, TIFF and AVIF go through - /// ffmpeg (`decodeWithFFmpeg`). RAW isn't read yet, and says so. + /// Writes a downscaled copy for the model. stb reads JPEG, PNG, BMP, GIF, PSD, TGA and PNM here; large pictures and + /// other formats go through ffmpeg, which scales while decoding. RAW isn't read yet, and says so. static func prepare(_ url: URL, maxPixel: Int, minSide: Int, to folder: URL, png: Bool) throws -> Prepared { let exif = EXIF.read(url) var image: RGBImage @@ -19,17 +18,24 @@ extension ImageExtractor { if stbi_info(url.path, &storedWidth, &storedHeight, &channels) != 0 { (width, height) = (Int(storedWidth), Int(storedHeight)) guard max(width, height) >= minSide else { throw ExtractError.tooSmall(width, height) } - guard let decoded = RGBImage.load(url.path) else { throw ExtractError.unreadable("can't decode image") } - image = decoded.oriented(exif?.orientation ?? 1) + guard let pixels = RGBImage.checkedPixelCount(width: width, height: height) else { + throw ExtractError.unreadable("image dimensions are too large") + } + if pixels > RGBImage.directDecodePixels { + image = try decodeWithFFmpeg(url, maxPixel: maxPixel, sourceSize: (width, height)) + } else { + guard let decoded = RGBImage.load(url.path) else { throw ExtractError.unreadable("can't decode image") } + image = decoded.oriented(exif?.orientation ?? 1) + } } else { if let note = notReadYet(url.pathExtension.lowercased()) { throw ExtractError.unreadable(note) } if exif?.tiled == true { throw ExtractError.unreadable("tiled TIFF pictures aren't read on this system yet") } - image = try decodeWithFFmpeg(url) - // Its size as shown (ImageIO reports it as stored: the two differ for a picture turned 90°). - (width, height) = (image.width, image.height) + let source = try sourceSize(url) + (width, height) = source guard max(width, height) >= minSide else { throw ExtractError.tooSmall(width, height) } + image = try decodeWithFFmpeg(url, maxPixel: maxPixel, sourceSize: source) } let longest = max(image.width, image.height) if longest > maxPixel { @@ -42,18 +48,26 @@ extension ImageExtractor { return Prepared(path: out.path, width: width, height: height, taken: exif?.dateTimeOriginal) } - /// The whole picture, through ffmpeg as 8-bit RGB (PPM down a pipe), turned the way the file says (irot/imir in + /// The picture, scaled through ffmpeg as 8-bit RGB (PPM down a pipe), turned the way the file says (irot/imir in /// HEIF, the EXIF orientation elsewhere) and, for a tiled HEIC, stitched whole. ffmpeg 8.1 and later do both /// (2026-10-11; 6.1 reads no HEIC and leaves WebP and TIFF unturned), so builds bring their own. A /// TIFF stored in tiles (ImageIO writes uncompressed ones that way) comes out black, so `prepare` doesn't ask. - static func decodeWithFFmpeg(_ url: URL) throws -> RGBImage { + static func decodeWithFFmpeg(_ url: URL, maxPixel: Int, sourceSize: (Int, Int)) throws -> RGBImage { guard FFmpeg.ffmpeg != nil else { - throw ExtractError.unreadable("ffmpeg isn't installed (HEIC, WebP and TIFF pictures need it)") + throw ExtractError.unreadable("ffmpeg isn't installed (large, HEIC, WebP and TIFF pictures need it)") + } + guard RGBImage.checkedPixelCount(width: sourceSize.0, height: sourceSize.1) != nil else { + throw ExtractError.unreadable("image dimensions are too large") } var data = Data() do { - try FFmpeg.run(FFmpeg.ffmpeg, ["-v", "error", "-nostdin", "-i", url.path, "-frames:v", "1", "-pix_fmt", - "rgb24", "-f", "image2pipe", "-c:v", "ppm", "-"]) { data.append($0) } + try FFmpeg.run(FFmpeg.ffmpeg, ["-v", "error", "-nostdin", "-max_pixels", "\(RGBImage.maxSourcePixels)", + "-i", url.path, "-frames:v", "1", "-vf", + "scale='min(iw,\(maxPixel))':'min(ih,\(maxPixel))':" + + "force_original_aspect_ratio=decrease", + "-pix_fmt", "rgb24", "-f", "image2pipe", "-c:v", "ppm", "-"]) { + data.append($0) + } } catch { let ext = url.pathExtension.lowercased() if ext == "heic" || ext == "heif" { throw ExtractError.unreadable("this ffmpeg can't read HEIC (8.1 can)") } @@ -63,6 +77,20 @@ extension ImageExtractor { return image } + private static func sourceSize(_ url: URL) throws -> (Int, Int) { + guard FFmpeg.ffprobe != nil else { + throw ExtractError.unreadable("ffprobe isn't installed (HEIC, WebP and TIFF pictures need it)") + } + let text = try FFmpeg.output(FFmpeg.ffprobe, ["-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=width,height", "-of", "csv=p=0:s=x", url.path]) + let dimensions = text.trimmingCharacters(in: .whitespacesAndNewlines).split(separator: "x") + guard dimensions.count == 2, let width = Int(dimensions[0]), let height = Int(dimensions[1]), + RGBImage.checkedPixelCount(width: width, height: height) != nil else { + throw ExtractError.unreadable("can't read image dimensions") + } + return (width, height) + } + /// Why a picture of a kind neither stb nor ffmpeg decodes isn't readable, for the file's note. static func notReadYet(_ ext: String) -> String? { let raw: Set = ["cr2", "nef", "arw", "dng", "raf", "orf", "rw2"] @@ -79,17 +107,28 @@ extension OCR { /// Packed 8-bit RGB in memory: what stb decodes to and encodes from. struct RGBImage { + static let maxSourcePixels = (256 << 20) / 3 + static let directDecodePixels = (64 << 20) / 3 + private static let maxSourceSide = 100_000 + var width: Int var height: Int var pixels: [UInt8] + static func checkedPixelCount(width: Int, height: Int) -> Int? { + guard width > 0, height > 0, width <= maxSourceSide, height <= maxSourceSide, + width <= maxSourcePixels / height else { return nil } + return width * height + } + /// Decoded from a file (nil when stb can't read it). static func load(_ path: String) -> RGBImage? { var width: Int32 = 0, height: Int32 = 0, channels: Int32 = 0 guard let pixels = stbi_load(path, &width, &height, &channels, 3) else { return nil } defer { stbi_image_free(pixels) } - return RGBImage(width: Int(width), height: Int(height), - pixels: Array(UnsafeBufferPointer(start: pixels, count: Int(width) * Int(height) * 3))) + let (w, h) = (Int(width), Int(height)) + guard let count = checkedPixelCount(width: w, height: h) else { return nil } + return RGBImage(width: w, height: h, pixels: Array(UnsafeBufferPointer(start: pixels, count: count * 3))) } /// Decoded from a file's bytes in memory (nil when stb can't read them). @@ -102,8 +141,9 @@ struct RGBImage { } guard let pixels else { return nil } defer { stbi_image_free(pixels) } - return RGBImage(width: Int(width), height: Int(height), - pixels: Array(UnsafeBufferPointer(start: pixels, count: Int(width) * Int(height) * 3))) + let (w, h) = (Int(width), Int(height)) + guard let count = checkedPixelCount(width: w, height: h) else { return nil } + return RGBImage(width: w, height: h, pixels: Array(UnsafeBufferPointer(start: pixels, count: count * 3))) } /// Turned the way the camera held it (EXIF orientation 1–8), as ImageIO's thumbnails are. From fb65682f1c3e06ca6006ffb2d0aff10e84e4c39b Mon Sep 17 00:00:00 2001 From: Ben Siraphob Date: Fri, 23 Oct 2026 01:07:33 -0400 Subject: [PATCH 6/7] Reload search vectors in place --- Sources/DigUpKit/Searcher.swift | 3 ++- Sources/digup/SearchService.swift | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/DigUpKit/Searcher.swift b/Sources/DigUpKit/Searcher.swift index 4592cbe5..b2616ec0 100644 --- a/Sources/DigUpKit/Searcher.swift +++ b/Sources/DigUpKit/Searcher.swift @@ -97,7 +97,8 @@ public final class Searcher { vectorsLoaded = false } - private func reloadFiles() throws { + /// Refreshes file kinds and vector provenance without bringing the vector matrix into memory. + public func reloadFiles() throws { fileKind.removeAll(keepingCapacity: true) try store.db.query("SELECT id, kind FROM files") { row in fileKind[row.int(0)] = FileKind(rawValue: row.text(1) ?? "") diff --git a/Sources/digup/SearchService.swift b/Sources/digup/SearchService.swift index 27d79d98..ba74f046 100644 --- a/Sources/digup/SearchService.swift +++ b/Sources/digup/SearchService.swift @@ -208,10 +208,9 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l let code = request["code"] as? Bool ?? false return ["text": try (code ? codeSearcher : searcher)?.text(ofSegment: segment) ?? NSNull()] case "reload": - let hadVectors = searcher.vectorsLoaded - searcher = try Searcher(store: store, loadVectors: hadVectors) - if let codeStore { - codeSearcher = try Searcher(store: codeStore, loadVectors: codeSearcher?.vectorsLoaded ?? false) + if searcher.vectorsLoaded { try searcher.reload() } else { try searcher.reloadFiles() } + if let codeSearcher { + if codeSearcher.vectorsLoaded { try codeSearcher.reload() } else { try codeSearcher.reloadFiles() } } else { try openCode() } From 4858f8c01e72bc831322cd8b071ecd6e4b52a8fb Mon Sep 17 00:00:00 2001 From: Ben Siraphob Date: Fri, 23 Oct 2026 01:08:30 -0400 Subject: [PATCH 7/7] Aggregate folder summaries in SQLite --- Sources/DigUpApp/Engine.swift | 21 +++++++------------- Sources/DigUpKit/IndexStore.swift | 32 +++++++++++++++++++++++++++++++ Sources/digup/SearchService.swift | 23 ++++++++-------------- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Sources/DigUpApp/Engine.swift b/Sources/DigUpApp/Engine.swift index 15d7d53a..806e8038 100644 --- a/Sources/DigUpApp/Engine.swift +++ b/Sources/DigUpApp/Engine.swift @@ -918,20 +918,13 @@ nonisolated final class Engine: @unchecked Sendable { var summary = RootSummary(path: root.path, missing: !AppSettings.isFolder(root), unreadable: unreadable.contains(root.path)) summary.leftOut = leftOut[root.path] ?? leftOut[FolderSelection.realPath(root.path)] ?? [:] - for file in (try? store?.files(under: [root.path])) ?? [] { - summary.files += 1 - summary.kinds[file.kind, default: 0] += 1 - switch file.state { - case "done": summary.indexed += 1 - case "partial": - summary.indexed += 1 - summary.unfinished += 1 - case "pending": summary.pending += 1 - case "failed": summary.failed += 1 - default: - summary.skipped += 1 - summary.skippedReasons[file.detail ?? "not read", default: 0] += 1 - } + if let counts = try? store?.fileSummary(under: root.path) { + (summary.files, summary.indexed, summary.pending) = + (counts.files, counts.indexed, counts.pending) + (summary.failed, summary.skipped, summary.unfinished) = + (counts.failed, counts.skipped, counts.unfinished) + summary.kinds = counts.kinds + summary.skippedReasons = counts.skippedReasons } return summary } diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index 7a9b8b82..ba2db6c5 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -249,6 +249,38 @@ public final class IndexStore { return out } + public struct FileSummary: Sendable { + public var files = 0, indexed = 0, pending = 0, failed = 0, skipped = 0, unfinished = 0 + public var kinds: [FileKind: Int] = [:] + public var skippedReasons: [String: Int] = [:] + } + + /// Counts the files under one root without materializing their paths and metadata. + public func fileSummary(under root: String) throws -> FileSummary { + var summary = FileSummary() + let ranges = Self.pathRanges(under: root) + let whereSQL = ranges.map { _ in "(path >= ? AND path < ?)" }.joined(separator: " OR ") + let values = ranges.flatMap { [SQLValue.text($0.from), .text($0.to)] } + try db.query("SELECT kind, state, detail, COUNT(*) FROM files WHERE \(whereSQL) GROUP BY kind, state, detail", + values) { row in + let count = Int(row.int(3)) + summary.files += count + summary.kinds[FileKind(rawValue: row.text(0) ?? "") ?? .doc, default: 0] += count + switch row.text(1) { + case "done": summary.indexed += count + case "partial": + summary.indexed += count + summary.unfinished += count + case "pending": summary.pending += count + case "failed": summary.failed += count + default: + summary.skipped += count + summary.skippedReasons[row.text(2) ?? "not read", default: 0] += count + } + } + return summary + } + /// Adds a file (pending) and returns its id. @discardableResult public func insert(_ file: Candidate) throws -> Int64 { diff --git a/Sources/digup/SearchService.swift b/Sources/digup/SearchService.swift index ba74f046..9acc7500 100644 --- a/Sources/digup/SearchService.swift +++ b/Sources/digup/SearchService.swift @@ -270,23 +270,16 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l var states = ["indexed": 0, "pending": 0, "failed": 0, "skipped": 0, "unfinished": 0] var kinds: [String: Int] = [:] var skippedReasons: [String: Int] = [:] // why files were skipped once read ("no text", "silent or shorter than 2 s") - let files = (try? store?.files(under: [root])) ?? [] - for file in files { - kinds[file.kind.rawValue, default: 0] += 1 - switch file.state { - case "done": states["indexed", default: 0] += 1 - case "partial": - states["indexed", default: 0] += 1 - states["unfinished", default: 0] += 1 - case "pending": states["pending", default: 0] += 1 - case "failed": states["failed", default: 0] += 1 - default: - states["skipped", default: 0] += 1 - skippedReasons[file.detail ?? "not read", default: 0] += 1 - } + var files = 0 + if let summary = try? store?.fileSummary(under: root) { + files = summary.files + states = ["indexed": summary.indexed, "pending": summary.pending, "failed": summary.failed, + "skipped": summary.skipped, "unfinished": summary.unfinished] + kinds = Dictionary(uniqueKeysWithValues: summary.kinds.map { ($0.key.rawValue, $0.value) }) + skippedReasons = summary.skippedReasons } fields.merge(states) { $1 } - fields["files"] = files.count + fields["files"] = files fields["kinds"] = kinds fields["skipped_reasons"] = skippedReasons return fields