Fix Han full-text matching and preserve tokenizer-budgeted text tails

README.md

@@ -205,11 +205,12 @@ carry their own PDF, video and image readers.
 **Will it slow my computer down?** Only the first pass is heavy. After that it reads new files only. The model runs in
 a separate process that quits when it's done, and the search side only loads while you're searching.
 
-**Which languages?** The model knows more than 100, and DigUp searches in all of them, across languages too. The evals
-cover only English, Bengali and Arabic because those are the languages of the files DigUp was tested on, and Google
-notes that the model isn't equally strong in every language. Two limits apply to exact words, not to meaning: the
-text inside screenshots is read only on a Mac, by Apple's text recognition, which knows about 25 languages, and
-languages written without spaces, like Chinese, Japanese and Thai, aren't split into words.
+**Which languages?** The model knows more than 100, and DigUp searches in all of them, across languages too. The semantic
+evals cover only English, Bengali and Arabic because those are the languages of the files DigUp was tested on, and
+Google notes that the model isn't equally strong in every language. Exact-word search reads screenshot text only on
+a Mac, using Apple's text recognition (about 25 languages). Han characters in text bodies match as contiguous
+sequences; other text written without spaces, such as Japanese kana and Thai, is not split into words. See
+[Han body search](docs/han-body-search.md) for index-upgrade details.
 
 **How do I remove it?**
 On a Mac, quit it, then delete DigUp.app and `~/Library/Application Support/DigUp`, which holds the model and the

Sources/DigUpKit/CodeExtractor.swift

@@ -23,9 +23,6 @@ public enum CodeExtractor {
         let cells: Int?
     }
 
-    /// The model reads 2,048 tokens of an input at most; a single line longer than this is cut here.
-    static let longestPiece = 4000
-
     static func read(_ url: URL, options: IndexOptions) throws -> File {
         let data = try Data(contentsOf: url)
         let (repo, title) = place(of: url.path)
@@ -117,8 +114,8 @@ public enum CodeExtractor {
             }
             var last = end
             while last > start + 1, blank(last - 1) { last -= 1 }   // blank lines at its end don't count
-            var body = lines[start..<last].joined(separator: "\n")
-            if body.count > longestPiece { body = String(body.prefix(longestPiece)) }
+            // The indexer splits complete prompts using the runtime tokenizer. Never discard a long line here.
+            let body = lines[start..<last].joined(separator: "\n")
             pieces.append(Piece(text: body, first: start + 1, last: last))
             start = natural ? end : max(end - 2, start + 1)
         }

Sources/DigUpKit/Extractors.swift

@@ -123,7 +123,9 @@ enum DocExtractor {
             var next = clean.index(end, offsetBy: -overlap, limitedBy: start) ?? end
             if next <= start { next = end }
             // Start the next chunk on a word boundary.
-            while next < end, !clean[next].isWhitespace { next = clean.index(after: next) }
+            while next < end, !clean[next].isWhitespace, !SearchText.isHan(clean[next]) {
+                next = clean.index(after: next)
+            }
             start = next < end ? next : end
         }
         return chunks

Sources/DigUpKit/IndexStore.swift

@@ -20,7 +20,7 @@ public struct FileRecord: Sendable {
 public final class IndexStore {
     public let directory: URL
     public let db: Database
-    static let schemaVersion: Int64 = 5
+    static let schemaVersion: Int64 = 6
 
     /// Keyword search's tokenizer. Marks (M*) count as letters, so Bengali vowel signs and Arabic harakat stay inside
     /// their words: by default SQLite splits on them ("কলকাতায়" became "কলক", "ত", "য").
@@ -101,6 +101,7 @@ public final class IndexStore {
             if (1...2).contains(version) { try migrateScreenshotText() }
             if (1...3).contains(version) { try migrateVectorNames() }
             if (1...4).contains(version) { try migrateLongFiles() }
+            if (1...5).contains(version) { try migrateHanBodies() }
         }
     }
 
@@ -138,6 +139,18 @@ public final class IndexStore {
             """)
     }
 
+    /// Schema 5 → 6: split Han bodies from stored originals, leaving names unspaced for filename search.
+    /// No source reads or requeue: disconnected files keep their cached text and vectors. Old silently truncated
+    /// vectors require an explicit reindex, since repairing them needs the original file and the model tokenizer.
+    private func migrateHanBodies() throws {
+        try db.execute("CREATE VIRTUAL TABLE fts_new USING fts5(\(Self.ftsColumns))")
+        try db.query("SELECT rowid, name, coalesce(original, body) FROM fts") { row in
+            try db.run("INSERT INTO fts_new(rowid, name, body, original) VALUES(?, ?, ?, ?)",
+                       [.int(row.int(0)), .text(row.text(1) ?? "")] + Self.keywordText(row.text(2) ?? ""))
+        }
+        try db.execute("DROP TABLE fts; ALTER TABLE fts_new RENAME TO fts; PRAGMA user_version = 6;")
+    }
+
     /// Schema 1 → 2: keyword search with whole Bengali and Arabic words (`tokenizer`) over normalized text
     /// (`SearchText`). The words are rebuilt from what's stored, without reading any file. PDFs whose text layer came
     /// out garbled (`PageTextProblem`) are queued again, to be read as pictures or by OCR.
@@ -280,9 +293,8 @@ public final class IndexStore {
     public func move(_ id: Int64, to path: String) throws {
         try db.run("UPDATE files SET path = ? WHERE id = ?", [.text(path), .int(id)])
         try db.run("""
-            UPDATE fts SET name = ?, body = ? WHERE rowid IN (SELECT id FROM segments WHERE file = ? AND kind = 'name')
-            """, [.text(SearchText.normalized(Self.nameText(path))), .text(SearchText.normalized(Self.folderText(path))),
-                  .int(id)])
+            UPDATE fts SET name = ?, body = ?, original = ? WHERE rowid IN (SELECT id FROM segments WHERE file = ? AND kind = 'name')
+            """, [.text(SearchText.normalized(Self.nameText(path)))] + Self.keywordText(Self.folderText(path)) + [.int(id)])
     }
 
     public func delete(_ id: Int64) throws {
@@ -439,15 +451,15 @@ public final class IndexStore {
 
     /// `body` and `original` for keyword search: the normalized text, and the text as written if that differs.
     static func keywordText(_ text: String) -> [SQLValue] {
-        let normalized = SearchText.normalized(text)
+        let normalized = SearchText.indexed(text)
         return [.text(normalized), normalized == text ? .null : .text(text)]
     }
 
     private func addNameRow(file: Int64, path: String) throws {
         try db.run("INSERT INTO segments(file, kind, modality) VALUES(?, 'name', 'none')", [.int(file)])
-        try db.run("INSERT INTO fts(rowid, name, body) VALUES(?, ?, ?)",
-                   [.int(db.lastInsertID), .text(SearchText.normalized(Self.nameText(path))),
-                    .text(SearchText.normalized(Self.folderText(path)))])
+        try db.run("INSERT INTO fts(rowid, name, body, original) VALUES(?, ?, ?, ?)",
+                   [.int(db.lastInsertID), .text(SearchText.normalized(Self.nameText(path)))]
+                    + Self.keywordText(Self.folderText(path)))
     }
 
     private func deleteSegments(of file: Int64, keepingName: Bool) throws {

Sources/DigUpKit/Indexer.swift

@@ -583,28 +583,41 @@ public final class Indexer {
         let wordPages = textPages.filter { SearchText.hasWords($0.text) }
         let imagePages = pages.filter { $0.imagePath != nil }
         let garbledPages = pages.filter { $0.garbled != nil }   // rendered too: among `imagePages`
-        var textVectors: [Int: [Float]] = [:]   // by page number
+        var textParts: [(page: Int, text: String)] = []
+        var textVectors: [Int: [Float]] = [:]   // by part, not page (a page can have several)
         var imageVectors: [Embedding] = []
+        var garbledParts: [(page: Int, text: String)] = []
         var garbledVectors: [[Float]] = []
         if !wordPages.isEmpty || !imagePages.isEmpty {
             let worker = try fullWorker()
-            let vectors = try worker.embedTexts(wordPages.map {
-                Prompts.document(title: title, text: String($0.text.prefix(4000)))
+            for page in textPages {
+                let parts = SearchText.hasWords(page.text)
+                    ? try worker.documentChunks(page.text, title: title) : [page.text]
+                textParts += parts.map { (page.number, $0) }
+            }
+            let worded = textParts.indices.filter { SearchText.hasWords(textParts[$0].text) }
+            let vectors = try worker.embedTexts(worded.map {
+                Prompts.document(title: title, text: textParts[$0].text)
             })
-            textVectors = Dictionary(uniqueKeysWithValues: zip(wordPages.map(\.number), vectors))
+            textVectors = Dictionary(uniqueKeysWithValues: zip(worded, vectors))
             imageVectors = try worker.embedImages(imagePages.compactMap(\.imagePath), budget: options.pageBudget)
-            garbledVectors = try worker.embedTexts(garbledPages.map {
-                Prompts.document(title: title, text: String($0.garbled!.prefix(4000)))
+            for page in garbledPages {
+                garbledParts += try worker.documentChunks(page.garbled!, title: title).map { (page.number, $0) }
+            }
+            garbledVectors = try worker.embedTexts(garbledParts.map {
+                Prompts.document(title: title, text: $0.text)
             })
+        } else {
+            textParts = textPages.map { ($0.number, $0.text) }
         }
         try store.db.transaction {
             // Meaning only: the garbled words are never shown or matched against what you type.
-            for (page, vector) in zip(garbledPages, garbledVectors) {
-                try store.addSegment(file: file.id, kind: .page, modality: .text, loc: Double(page.number), vector: vector)
+            for (part, vector) in zip(garbledParts, garbledVectors) {
+                try store.addSegment(file: file.id, kind: .page, modality: .text, loc: Double(part.page), vector: vector)
             }
-            for page in textPages {
-                try store.addSegment(file: file.id, kind: .page, modality: .text, loc: Double(page.number),
-                                     excerpt: excerpt(page.text), text: page.text, vector: textVectors[page.number])
+            for (index, part) in textParts.enumerated() {
+                try store.addSegment(file: file.id, kind: .page, modality: .text, loc: Double(part.page),
+                                     excerpt: excerpt(part.text), text: part.text, vector: textVectors[index])
             }
             for (page, embedding) in zip(imagePages, imageVectors) {
                 try store.addSegment(file: file.id, kind: .pageImage, modality: .image, loc: Double(page.number),
@@ -634,18 +647,24 @@ public final class Indexer {
         }
         let title = URL(fileURLWithPath: file.path).deletingPathExtension().lastPathComponent
         // Text that's mostly digits (a data dump saved as .txt) gets keyword search only.
-        let worded = chunks.indices.filter { SearchText.hasWords(chunks[$0]) }
-        let vectors = worded.isEmpty ? [] : try fullWorker().embedTexts(worded.map {
-            Prompts.document(title: title, text: chunks[$0])
+        let worker = chunks.contains(where: SearchText.hasWords) ? try fullWorker() : nil
+        var parts: [(offset: Int, text: String)] = []
+        for (offset, chunk) in chunks.enumerated() {
+            let split = SearchText.hasWords(chunk) ? try worker!.documentChunks(chunk, title: title) : [chunk]
+            parts += split.map { (offset, $0) }
+        }
+        let worded = parts.indices.filter { SearchText.hasWords(parts[$0].text) }
+        let vectors = worded.isEmpty ? [] : try worker!.embedTexts(worded.map {
+            Prompts.document(title: title, text: parts[$0].text)
         })
         let vector = Dictionary(uniqueKeysWithValues: zip(worded, vectors))
         let end = min(total, options.maxChunks)
         let unread = read.upperBound < end
         try store.db.transaction {
-            for (offset, chunk) in chunks.enumerated() {
+            for (index, part) in parts.enumerated() {
                 try store.addSegment(file: file.id, kind: .chunk, modality: .text,
-                                     loc: Double(read.lowerBound + offset + 1), excerpt: excerpt(chunk), text: chunk,
-                                     vector: vector[offset])
+                                     loc: Double(read.lowerBound + part.offset + 1), excerpt: excerpt(part.text), text: part.text,
+                                     vector: vector[index])
             }
             var info = ["chunks": "\(end)", "chars": "\(characters)"]
             info["read_to"] = unread ? "\(read.upperBound)" : nil
@@ -656,7 +675,7 @@ public final class Indexer {
         let span = later ? "chunks \(read.lowerBound + 1)–\(read.upperBound) of \(end)"
             : unread ? "\(chunks.count) of \(end) chunks" : "\(chunks.count) chunks"
         return (unread ? "partial" : "done",
-                span + (worded.count < chunks.count ? ", \(chunks.count - worded.count) for keywords only" : ""))
+                span + (worded.count < parts.count ? ", \(parts.count - worded.count) for keywords only" : ""))
     }
 
     /// A code file's stretches of lines (a notebook's of cells), each embedded with the file's path in its repo as its
@@ -666,14 +685,39 @@ public final class Indexer {
             try store.finish(file.id, state: "skipped", detail: "no code")
             return ("skipped", "no code")
         }
-        // Lines that are mostly numbers and symbols (a table of constants) get keyword search only.
-        let worded = code.pieces.indices.filter { SearchText.hasWords(code.pieces[$0].text) }
-        let vectors = worded.isEmpty ? [] : try fullWorker().embedTexts(worded.map {
-            Prompts.document(title: code.title, text: code.pieces[$0].text)
+        // Keep the extractor's coarse code boundaries, then apply the complete
+        // prompt's real token budget. Preserve source offsets through this split.
+        let worker = code.pieces.contains { SearchText.hasWords($0.text) } ? try fullWorker() : nil
+        var pieces: [CodeExtractor.Piece] = []
+        for piece in code.pieces {
+            guard SearchText.hasWords(piece.text), let worker else {
+                pieces.append(piece)
+                continue
+            }
+            let ranges = try TextBudget.ranges(piece.text, title: code.title, limit: worker.textTokenLimit,
+                                               count: worker.textTokenCount)
+            var countedThrough = piece.text.startIndex, lineOffset = 0
+            for range in ranges {
+                let text = String(piece.text[range])
+                var first = piece.first, last = piece.last
+                if code.cells == nil {
+                    // Ranges advance even with overlap; count each source prefix only once.
+                    lineOffset += piece.text[countedThrough..<range.lowerBound].filter { $0 == "\n" || $0 == "\r\n" }.count
+                    countedThrough = range.lowerBound
+                    first += lineOffset
+                    last = first + text.dropLast().filter { $0 == "\n" || $0 == "\r\n" }.count
+                }
+                pieces.append(CodeExtractor.Piece(text: text, first: first, last: last))
+            }
+        }
+        // Lines that are mostly numbers and symbols get keyword search only.
+        let worded = pieces.indices.filter { SearchText.hasWords(pieces[$0].text) }
+        let vectors = worded.isEmpty ? [] : try worker!.embedTexts(worded.map {
+            Prompts.document(title: code.title, text: pieces[$0].text)
         })
         let vector = Dictionary(uniqueKeysWithValues: zip(worded, vectors))
         try store.db.transaction {
-            for (offset, piece) in code.pieces.enumerated() {
+            for (offset, piece) in pieces.enumerated() {
                 try store.addSegment(file: file.id, kind: code.cells == nil ? .lines : .cells, modality: .text,
                                      loc: Double(piece.first), locEnd: Double(piece.last),
                                      excerpt: CodeExtractor.summary(piece.text), text: piece.text, vector: vector[offset])
@@ -683,8 +727,8 @@ public final class Indexer {
             info["repo"] = code.repo
             try store.finish(file.id, state: "done", info: info)
         }
-        return ("done", "\(code.pieces.count) stretches, \(code.cells.map { "\($0) cells" } ?? "\(code.lines) lines")"
-                + (worded.count < code.pieces.count ? ", \(code.pieces.count - worded.count) for keywords only" : ""))
+        return ("done", "\(pieces.count) stretches, \(code.cells.map { "\($0) cells" } ?? "\(code.lines) lines")"
+                + (worded.count < pieces.count ? ", \(pieces.count - worded.count) for keywords only" : ""))
     }
 
     private func audio(_ file: FileRecord, pieces: [MediaPiece], duration: Double) throws -> (String, String?) {