Find two-character Han words inside filenames

Sources/DigUpKit/Searcher.swift

@@ -171,6 +171,15 @@ public final class Searcher {
             }
         }
 
+        // Han words have no spaces: unicode61 keeps a whole filename run as one token, so a two-character
+        // lookup in its middle cannot match FTS. Read only name rows, without rebuilding indexes or embeddings.
+        let names = try hanFilenameMatches(text, allowed: allowed)
+        let nameStrength = Dictionary(names.map { ($0.file, $0.strength) }, uniquingKeysWith: max)
+        if !names.isEmpty {
+            words = names.map { ($0.file, $0.segment, nil) } + words.filter { nameStrength[$0.file] == nil }
+            for name in names { coverage[name.file] = 1 }
+        }
+
         // A file qualifies by standing out in meaning, or by containing at least half of the query's words.
         // Score: meaning z + a bonus that grows with word coverage, + 1 for the kind the query asks for.
         // One common word can't beat a strong meaning match; all the words of an exact lookup usually do.
@@ -187,6 +196,11 @@ public final class Searcher {
             scored[file] = score
         }
 
+        // A literal basename match is a lookup, not just another semantic hint. Put complete names ahead of
+        // partial names, and both ahead of content-only matches. Keep scores in rank order for strongCount().
+        let highest = scored.values.max() ?? 0
+        for (file, strength) in nameStrength { scored[file] = highest + Double(strength) }
+
         let wordRank = Dictionary(words.enumerated().map { ($0.element.file, ($0.offset + 1, $0.element)) },
                                   uniquingKeysWith: { first, _ in first })
         var hits: [SearchHit] = []
@@ -196,7 +210,7 @@ public final class Searcher {
             let semantic = best[file]
             let keyword = wordRank[file]
             // Point at the best meaning segment if it stands out; else at the segment the words matched.
-            let useMeaning = semantic.map { $0.z >= zGate || keyword == nil } ?? false
+            let useMeaning = nameStrength[file] == nil && (semantic.map { $0.z >= zGate || keyword == nil } ?? false)
             guard var segmentID = useMeaning ? semantic.map({ rowSegment[$0.row] }) : keyword?.1.segment else { continue }
             var focus: Double?
             if isCode {
@@ -246,6 +260,43 @@ public final class Searcher {
         return hits
     }
 
+    private struct FilenameMatch {
+        let file: Int64
+        let segment: Int64
+        let strength: Int
+    }
+
+    /// A cheap SQLite prefilter uses a literal Han run (never LIKE wildcards). Verify every query component against
+    /// the normalized basename, including numbers and extensions. Folders and document contents cannot qualify.
+    private func hanFilenameMatches(_ text: String, allowed: (Int64) -> Bool) throws -> [FilenameMatch] {
+        guard !isCode else { return [] }
+        let query = SearchText.normalized(text).lowercased().trimmingCharacters(in: .whitespacesAndNewlines)
+        var longest = "", run = ""
+        for scalar in query.unicodeScalars {
+            switch scalar.value {
+            case 0x3400...0x4DBF, 0x4E00...0x9FFF, 0xF900...0xFAFF, 0x20000...0x2FA1F, 0x30000...0x323AF:
+                run.unicodeScalars.append(scalar)
+                if run.unicodeScalars.count > longest.unicodeScalars.count { longest = run }
+            default:
+                run = ""
+            }
+        }
+        guard longest.unicodeScalars.count >= 2 else { return [] }
+        let terms = query.components(separatedBy: CharacterSet.alphanumerics.inverted).filter { !$0.isEmpty }
+        var matches: [FilenameMatch] = []
+        try store.db.query("""
+            SELECT s.file, s.id, fts.name FROM segments s JOIN fts ON fts.rowid = s.id
+            WHERE s.kind = 'name' AND instr(fts.name, ?) > 0
+            """, [.text(longest)]) { row in
+                let file = row.int(0)
+                guard allowed(file), let name = row.text(2)?.lowercased(),
+                      terms.allSatisfy({ name.contains($0) }) else { return }
+                let exact = name == query || (name as NSString).deletingPathExtension == query
+                matches.append(FilenameMatch(file: file, segment: row.int(1), strength: exact ? 2 : 1))
+            }
+        return matches.sorted { $0.strength > $1.strength || ($0.strength == $1.strength && $0.file < $1.file) }
+    }
+
     /// Code: the stretch to point at, and the line in it to open at. A name looked up points at the stretch that
     /// defines it, when one does (not the ones that call it). Otherwise the line is the first with the query's words,
     /// when they matched in this stretch; a match by meaning alone opens at the stretch's first line.

Tests/DigUpKitTests/HanFilenameTests.swift

@@ -0,0 +1,152 @@
+import Foundation
+import Testing
+@testable import DigUpKit
+
+struct HanFilenameTests {
+    @Test func traditionalFilenameRankingAndDeduplicationHoldWhileTypingAndAfterSubmission() throws {
+        try withStore { store in
+            for name in ["學生情緒輔導紀錄.txt", "其他文件.txt", "輔導.txt"] {
+                let id = try add(name, to: store, kind: .doc)
+                // A standalone keyword is already supported by FTS on v0.6.1.
+                // Deliberately avoid depending on Han body tokenization from #111.
+                try store.addSegment(file: id, kind: .chunk, modality: .text, loc: 1, text: "輔導")
+            }
+            let searcher = try Searcher(store: store, loadVectors: false)
+            for typing in [false, true] {
+                let hits = try searcher.search("輔導", queryVector: nil, prefixLast: typing)
+                #expect(hits.map { URL(fileURLWithPath: $0.path).lastPathComponent }
+                        == ["輔導.txt", "學生情緒輔導紀錄.txt", "其他文件.txt"], "typing: \(typing)")
+                try #require(hits.count == 3)
+                #expect(hits[0].segment == .name)
+                #expect(hits[1].segment == .name)
+                #expect(hits[2].segment == .chunk)
+                #expect(Set(hits.map(\.path)).count == hits.count)
+            }
+        }
+    }
+
+    func withStore(file: String = IndexStore.mainFile, _ body: (IndexStore) throws -> Void) throws {
+        let folder = FileManager.default.temporaryDirectory.appendingPathComponent("HanFilenameTests-\(UUID())")
+        defer { try? FileManager.default.removeItem(at: folder) }
+        try body(IndexStore(directory: folder, file: file))
+    }
+
+    @discardableResult
+    func add(_ name: String, to store: IndexStore, kind: FileKind = .pdf) throws -> Int64 {
+        try store.insert(Candidate(path: "/fixture/\(name)", kind: kind, size: 1, mtime: 1, inode: 1, device: 1))
+    }
+
+    @Test func findsTwoCharacterWordsAndInternalFilenameSubstringsWithoutVectors() throws {
+        try withStore { store in
+            try add("2026學生情緒輔導紀錄.pdf", to: store)
+            let searcher = try Searcher(store: store, loadVectors: false)
+            for query in ["輔導", "情緒", "情緒輔導", "學生情緒輔導", "２０２６學生", "輔導 PDF"] {
+                for typing in [false, true] {
+                    let hits = try searcher.search(query, queryVector: nil, prefixLast: typing)
+                    #expect(hits.first?.path == "/fixture/2026學生情緒輔導紀錄.pdf", "query: \(query), typing: \(typing)")
+                    #expect(hits.first?.segment == .name)
+                    #expect(hits.first?.keywordRank != nil)
+                }
+            }
+        }
+    }
+
+    @Test func exactStemRanksAheadOfPartialNamesAndContentMatches() throws {
+        try withStore { store in
+            try add("學生情緒輔導紀錄.pdf", to: store)
+            let content = try add("其他資料.pdf", to: store)
+            // A filename lookup must point at the file rather than a matching content page.
+            try store.db.run("INSERT INTO segments(file, kind, modality, loc) VALUES(?, 'page', 'none', 5)", [.int(content)])
+            try store.db.run("INSERT INTO fts(rowid, name, body) VALUES(?, '', '輔導')", [.int(store.db.lastInsertID)])
+            try add("輔導.pdf", to: store)
+            let hits = try Searcher(store: store).search("輔導", queryVector: nil)
+            #expect(hits.map { ($0.path as NSString).lastPathComponent } == ["輔導.pdf", "學生情緒輔導紀錄.pdf", "其他資料.pdf"])
+            #expect(hits.first?.loc == nil)
+        }
+    }
+
+    @Test func requiresAllFilenameTermsAndHonorsKindFilter() throws {
+        try withStore { store in
+            try add("2025學生輔導紀錄.pdf", to: store)
+            try add("2026學生輔導紀錄.pdf", to: store)
+            try add("2026學生輔導紀錄.txt", to: store, kind: .doc)
+            let searcher = try Searcher(store: store)
+            let hits = try searcher.search("輔導 2026", queryVector: nil, kinds: [.pdf])
+            #expect(hits.map(\.path) == ["/fixture/2026學生輔導紀錄.pdf"])
+            #expect(try searcher.search("輔導 不存在", queryVector: nil).isEmpty)
+            #expect(try searcher.search("輔導", queryVector: nil, kinds: [.video]).isEmpty)
+        }
+    }
+
+    @Test func readsExistingNameRowsAndTracksRenamesAndDeletion() throws {
+        try withStore { store in
+            let id = try add("學生情緒輔導紀錄.pdf", to: store)
+            let searcher = try Searcher(store: store, loadVectors: false)
+            #expect(try searcher.search("輔導", queryVector: nil).count == 1)
+            try store.move(id, to: "/fixture/學生生涯探索紀錄.pdf")
+            #expect(try searcher.search("輔導", queryVector: nil).isEmpty)
+            #expect(try searcher.search("探索", queryVector: nil).count == 1)
+            try store.delete(id)
+            #expect(try searcher.search("探索", queryVector: nil).isEmpty)
+        }
+    }
+
+    @Test func doesNotTreatFolderNamesOrLatinSubstringsAsHanFilenameMatches() throws {
+        try withStore { store in
+            try store.insert(Candidate(path: "/fixture/情緒輔導資料/別的檔案.pdf", kind: .pdf, size: 1, mtime: 1, inode: 1, device: 1))
+            try add("教育簡報.pdf", to: store)
+            try add("category.pdf", to: store)
+            let searcher = try Searcher(store: store)
+            #expect(try searcher.search("輔導", queryVector: nil).isEmpty)
+            #expect(try searcher.search("育簡", queryVector: nil).first?.path == "/fixture/教育簡報.pdf")
+            #expect(try searcher.search("tegory", queryVector: nil).isEmpty)
+        }
+    }
+
+    @Test func filenameLookupWinsOverSemanticPagesAndHasNoDuplicateHits() throws {
+        try withStore { store in
+            let named = try add("學生情緒輔導紀錄.pdf", to: store)
+            let query: [Float] = [1] + Array(repeating: 0, count: Vectors.dimension - 1)
+            let other: [Float] = [0, 1] + Array(repeating: 0, count: Vectors.dimension - 2)
+            try store.addSegment(file: named, kind: .page, modality: .text, loc: 3, text: "輔導", vector: query)
+            for n in 0..<64 {
+                let id = try add("其他檔案-\(n).pdf", to: store)
+                try store.addSegment(file: id, kind: .page, modality: .text, loc: 1, text: "other", vector: n == 0 ? query : other)
+            }
+            let hits = try Searcher(store: store).search("輔導", queryVector: query)
+            #expect(hits.first?.path == "/fixture/學生情緒輔導紀錄.pdf")
+            #expect(hits.first?.segment == .name)
+            #expect(hits.first?.loc == nil)
+            #expect(hits.filter { $0.path == "/fixture/學生情緒輔導紀錄.pdf" }.count == 1)
+            #expect(zip(hits, hits.dropFirst()).allSatisfy { $0.score >= $1.score })
+        }
+    }
+
+    @Test func exactNameIsNotLostBehindManyPartialMatches() throws {
+        try withStore { store in
+            try store.db.transaction {
+                for n in 0..<1_100 { try add("學生輔導紀錄-\(n).pdf", to: store) }
+                try add("輔導.pdf", to: store)
+            }
+            let hits = try Searcher(store: store, loadVectors: false).search("輔導", queryVector: nil, limit: 5)
+            #expect(hits.count == 5)
+            #expect(hits.first?.path == "/fixture/輔導.pdf")
+        }
+    }
+
+    @Test func mixedLatinCaseAndCompatibilityCharactersMatch() throws {
+        try withStore { store in
+            try add("２０２６ＣＡＦÉ學生輔導紀錄.pdf", to: store)
+            let hits = try Searcher(store: store).search("2026café學生輔導", queryVector: nil)
+            #expect(hits.count == 1)
+        }
+    }
+
+    @Test func codeIdentifierSearchKeepsItsExistingRules() throws {
+        try withStore(file: IndexStore.codeFile) { store in
+            try add("學生情緒輔導紀錄.swift", to: store)
+            #expect(try Searcher(store: store).search("輔導", queryVector: nil).isEmpty)
+        }
+    }
+
+}

docs/han-filename-search.md

@@ -0,0 +1,71 @@
+# Han filename lookups
+
+`輔導` should find `學生情緒輔導紀錄.pdf` even before its contents have been embedded. FTS5's unicode61 tokenizer
+keeps an unspaced Han run as one token, so matching the middle of that run needs a basename lookup.
+
+```
+query contains a run of at least two Han characters
+  → literal SQLite prefilter over existing name rows
+  → every query component must occur in the normalized basename
+  → complete basename/stem, then partial basename, then existing content/meaning matches
+```
+
+The prefilter uses a bound `instr` argument, not LIKE wildcards. Names retain the existing NFKC normalization;
+verification is case-insensitive and includes mixed letters, numbers and extensions. File-kind filters apply before
+matches are ranked. A filename lookup points to its name segment, rather than an unrelated semantic page in that file.
+Scores remain descending so the existing weaker-results fold can use them.
+
+There is no schema change, index rebuild, embedding change, or new dependency. Existing names are immediately
+searchable, including pending files, and moves/deletes are read from the live index. Latin-only queries and code
+identifier search keep their existing rules. This does not introduce Chinese content tokenization, Traditional/
+Simplified conversion, single-character substring lookup or Spotlight application/system actions.
+
+## Reproduce the focused evaluation
+
+Use a new, isolated folder and index:
+
+```sh
+python3 scripts/testbed/make_han_filenames.py /tmp/DigUpTestbed/han-filenames
+.build/release/digup index /tmp/DigUpTestbed/han-filenames --limit 0 --index /tmp/DigUpHanIndex
+DIGUP_TESTBED=/tmp/DigUpTestbed .build/release/digup eval evals/han-filenames.json --index /tmp/DigUpHanIndex
+```
+
+`--limit 0` syncs the actual fixture names without embedding their contents. This isolates filename recall from the
+model's guesses. Use the same index for the before/after binaries; the file generator contains only synthetic text.
+
+## Validation (macOS, Apple M2 Pro, 16 GB)
+
+Compared official v0.6.1 with this patch on the same indexes and model. Nine regression tests cover two-character and
+internal substrings, typing/settled queries, exact-name ranking, semantic-page competition, duplicate prevention,
+kind filters, all query components, existing names, rename/delete, full-width/case normalization, code isolation, and
+an exact name inserted after 1,100 partial matches. `swift test`: 101 passed (95 DigUpKit, 6 LlamaRuntime including
+actual reference vectors).
+
+| Evaluation | Queries | Before R@1 / R@5 / R@10 | After R@1 / R@5 / R@10 |
+| --- | ---: | --- | --- |
+| Han names, keywords only | 12 | .33 / .33 / .33 | 1.00 / 1.00 / 1.00 |
+| Upstream synthetic | 37 | .92 / .97 / .97 | .92 / .97 / .97 |
+| Upstream multilingual | 48 | .92 / .98 / .98 | .92 / .98 / .98 |
+| Upstream code | 41 | 1.00 / 1.00 / 1.00 | 1.00 / 1.00 / 1.00 |
+
+All 126 upstream queries produced byte-identical evaluation output before and after. The synthetic/media/code
+fixtures were generated by the upstream testbed scripts; the personal-file sampling step was omitted. These
+figures are measured results on this fixture, not a claim of perfect recall across arbitrary files or languages.
+
+A separate keyword-only index held 20,000 synthetic filenames, including 2,000 matching Han names. Twenty measured
+CLI runs per query alternated before/after after two warmups; times include process startup, opening the same index
+and returning up to 20 results. The extra basename verification costs time only for Han queries.
+
+| Query | Before median / p95 (ms) | After median / p95 (ms) |
+| --- | --- | --- |
+| 輔導 | 15.18 / 19.48 | 59.02 / 63.12 |
+| 情緒輔導 | 14.70 / 16.39 | 57.25 / 60.80 |
+| annual | 39.91 / 59.33 | 39.24 / 41.91 |
+
+## Independent regression coverage (2026-10-20)
+
+An additional test checks exact Traditional Chinese filenames before partial filenames before content-only matches,
+with no duplicate files, both while typing and after submission. It uses the public segment insertion API with a
+standalone body keyword that v0.6.1 already supports, and disables vectors. It does not require the Han body
+tokenization or index migration proposed in #111. Those changes and their body-search tests remain separate.
+Interface localization remains in #34.

evals/han-filenames.json

@@ -0,0 +1,89 @@
+{
+  "base": "~/DigUpTestbed/han-filenames",
+  "queries": [
+    {
+      "q": "輔導",
+      "expect": [
+        "exact/輔導.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "情緒",
+      "expect": [
+        "general/學生情緒輔導紀錄.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "情緒輔導",
+      "expect": [
+        "general/學生情緒輔導紀錄.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "學生情緒",
+      "expect": [
+        "general/學生情緒輔導紀錄.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "探索",
+      "expect": [
+        "other/生涯探索規劃.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "規劃",
+      "expect": [
+        "other/生涯探索規劃.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "教學",
+      "expect": [
+        "other/期末教學檢討.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "課程",
+      "expect": [
+        "other/特殊教育課程簡報.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "整理",
+      "expect": [
+        "other/資料整理會議紀錄.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "會議",
+      "expect": [
+        "other/資料整理會議紀錄.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "2026café學生輔導",
+      "expect": [
+        "mixed/２０２６ＣＡＦÉ學生輔導紀錄.txt"
+      ],
+      "tag": "han-name"
+    },
+    {
+      "q": "學生情緒輔導紀錄.txt",
+      "expect": [
+        "general/學生情緒輔導紀錄.txt"
+      ],
+      "tag": "han-name"
+    }
+  ]
+}