From c37d26c736ca765fcfd089304bffd588662d38c2 Mon Sep 17 00:00:00 2001 From: max0071001 <48833066+max0071001@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:28:32 +0800 Subject: [PATCH 1/2] Find Han substrings in filenames without reindexing --- Sources/DigUpKit/Searcher.swift | 53 ++++++++- Tests/DigUpKitTests/HanFilenameTests.swift | 130 +++++++++++++++++++++ docs/han-filename-search.md | 63 ++++++++++ evals/han-filenames.json | 89 ++++++++++++++ scripts/testbed/make_han_filenames.py | 23 ++++ 5 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 Tests/DigUpKitTests/HanFilenameTests.swift create mode 100644 docs/han-filename-search.md create mode 100644 evals/han-filenames.json create mode 100644 scripts/testbed/make_han_filenames.py diff --git a/Sources/DigUpKit/Searcher.swift b/Sources/DigUpKit/Searcher.swift index 4d7b6def..370abda0 100644 --- a/Sources/DigUpKit/Searcher.swift +++ b/Sources/DigUpKit/Searcher.swift @@ -171,6 +171,15 @@ public final class Searcher { } } + // Han words have no spaces: unicode61 keeps a whole filename run as one token, so a two-character + // lookup in its middle cannot match FTS. Read only name rows, without rebuilding indexes or embeddings. + let names = try hanFilenameMatches(text, allowed: allowed) + let nameStrength = Dictionary(names.map { ($0.file, $0.strength) }, uniquingKeysWith: max) + if !names.isEmpty { + words = names.map { ($0.file, $0.segment, nil) } + words.filter { nameStrength[$0.file] == nil } + for name in names { coverage[name.file] = 1 } + } + // A file qualifies by standing out in meaning, or by containing at least half of the query's words. // Score: meaning z + a bonus that grows with word coverage, + 1 for the kind the query asks for. // One common word can't beat a strong meaning match; all the words of an exact lookup usually do. @@ -187,6 +196,11 @@ public final class Searcher { scored[file] = score } + // A literal basename match is a lookup, not just another semantic hint. Put complete names ahead of + // partial names, and both ahead of content-only matches. Keep scores in rank order for strongCount(). + let highest = scored.values.max() ?? 0 + for (file, strength) in nameStrength { scored[file] = highest + Double(strength) } + let wordRank = Dictionary(words.enumerated().map { ($0.element.file, ($0.offset + 1, $0.element)) }, uniquingKeysWith: { first, _ in first }) var hits: [SearchHit] = [] @@ -196,7 +210,7 @@ public final class Searcher { let semantic = best[file] let keyword = wordRank[file] // Point at the best meaning segment if it stands out; else at the segment the words matched. - let useMeaning = semantic.map { $0.z >= zGate || keyword == nil } ?? false + let useMeaning = nameStrength[file] == nil && (semantic.map { $0.z >= zGate || keyword == nil } ?? false) guard var segmentID = useMeaning ? semantic.map({ rowSegment[$0.row] }) : keyword?.1.segment else { continue } var focus: Double? if isCode { @@ -246,6 +260,43 @@ public final class Searcher { return hits } + private struct FilenameMatch { + let file: Int64 + let segment: Int64 + let strength: Int + } + + /// A cheap SQLite prefilter uses a literal Han run (never LIKE wildcards). Verify every query component against + /// the normalized basename, including numbers and extensions. Folders and document contents cannot qualify. + private func hanFilenameMatches(_ text: String, allowed: (Int64) -> Bool) throws -> [FilenameMatch] { + guard !isCode else { return [] } + let query = SearchText.normalized(text).lowercased().trimmingCharacters(in: .whitespacesAndNewlines) + var longest = "", run = "" + for scalar in query.unicodeScalars { + switch scalar.value { + case 0x3400...0x4DBF, 0x4E00...0x9FFF, 0xF900...0xFAFF, 0x20000...0x2FA1F, 0x30000...0x323AF: + run.unicodeScalars.append(scalar) + if run.unicodeScalars.count > longest.unicodeScalars.count { longest = run } + default: + run = "" + } + } + guard longest.unicodeScalars.count >= 2 else { return [] } + let terms = query.components(separatedBy: CharacterSet.alphanumerics.inverted).filter { !$0.isEmpty } + var matches: [FilenameMatch] = [] + try store.db.query(""" + SELECT s.file, s.id, fts.name FROM segments s JOIN fts ON fts.rowid = s.id + WHERE s.kind = 'name' AND instr(fts.name, ?) > 0 + """, [.text(longest)]) { row in + let file = row.int(0) + guard allowed(file), let name = row.text(2)?.lowercased(), + terms.allSatisfy({ name.contains($0) }) else { return } + let exact = name == query || (name as NSString).deletingPathExtension == query + matches.append(FilenameMatch(file: file, segment: row.int(1), strength: exact ? 2 : 1)) + } + return matches.sorted { $0.strength > $1.strength || ($0.strength == $1.strength && $0.file < $1.file) } + } + /// Code: the stretch to point at, and the line in it to open at. A name looked up points at the stretch that /// defines it, when one does (not the ones that call it). Otherwise the line is the first with the query's words, /// when they matched in this stretch; a match by meaning alone opens at the stretch's first line. diff --git a/Tests/DigUpKitTests/HanFilenameTests.swift b/Tests/DigUpKitTests/HanFilenameTests.swift new file mode 100644 index 00000000..7b4f9943 --- /dev/null +++ b/Tests/DigUpKitTests/HanFilenameTests.swift @@ -0,0 +1,130 @@ +import Foundation +import Testing +@testable import DigUpKit + +struct HanFilenameTests { + func withStore(file: String = IndexStore.mainFile, _ body: (IndexStore) throws -> Void) throws { + let folder = FileManager.default.temporaryDirectory.appendingPathComponent("HanFilenameTests-\(UUID())") + defer { try? FileManager.default.removeItem(at: folder) } + try body(IndexStore(directory: folder, file: file)) + } + + @discardableResult + func add(_ name: String, to store: IndexStore, kind: FileKind = .pdf) throws -> Int64 { + try store.insert(Candidate(path: "/fixture/\(name)", kind: kind, size: 1, mtime: 1, inode: 1, device: 1)) + } + + @Test func findsTwoCharacterWordsAndInternalFilenameSubstringsWithoutVectors() throws { + try withStore { store in + try add("2026學生情緒輔導紀錄.pdf", to: store) + let searcher = try Searcher(store: store, loadVectors: false) + for query in ["輔導", "情緒", "情緒輔導", "學生情緒輔導", "２０２６學生", "輔導 PDF"] { + for typing in [false, true] { + let hits = try searcher.search(query, queryVector: nil, prefixLast: typing) + #expect(hits.first?.path == "/fixture/2026學生情緒輔導紀錄.pdf", "query: \(query), typing: \(typing)") + #expect(hits.first?.segment == .name) + #expect(hits.first?.keywordRank != nil) + } + } + } + } + + @Test func exactStemRanksAheadOfPartialNamesAndContentMatches() throws { + try withStore { store in + try add("學生情緒輔導紀錄.pdf", to: store) + let content = try add("其他資料.pdf", to: store) + // A filename lookup must point at the file rather than a matching content page. + try store.db.run("INSERT INTO segments(file, kind, modality, loc) VALUES(?, 'page', 'none', 5)", [.int(content)]) + try store.db.run("INSERT INTO fts(rowid, name, body) VALUES(?, '', '輔導')", [.int(store.db.lastInsertID)]) + try add("輔導.pdf", to: store) + let hits = try Searcher(store: store).search("輔導", queryVector: nil) + #expect(hits.map { ($0.path as NSString).lastPathComponent } == ["輔導.pdf", "學生情緒輔導紀錄.pdf", "其他資料.pdf"]) + #expect(hits.first?.loc == nil) + } + } + + @Test func requiresAllFilenameTermsAndHonorsKindFilter() throws { + try withStore { store in + try add("2025學生輔導紀錄.pdf", to: store) + try add("2026學生輔導紀錄.pdf", to: store) + try add("2026學生輔導紀錄.txt", to: store, kind: .doc) + let searcher = try Searcher(store: store) + let hits = try searcher.search("輔導 2026", queryVector: nil, kinds: [.pdf]) + #expect(hits.map(\.path) == ["/fixture/2026學生輔導紀錄.pdf"]) + #expect(try searcher.search("輔導 不存在", queryVector: nil).isEmpty) + #expect(try searcher.search("輔導", queryVector: nil, kinds: [.video]).isEmpty) + } + } + + @Test func readsExistingNameRowsAndTracksRenamesAndDeletion() throws { + try withStore { store in + let id = try add("學生情緒輔導紀錄.pdf", to: store) + let searcher = try Searcher(store: store, loadVectors: false) + #expect(try searcher.search("輔導", queryVector: nil).count == 1) + try store.move(id, to: "/fixture/學生生涯探索紀錄.pdf") + #expect(try searcher.search("輔導", queryVector: nil).isEmpty) + #expect(try searcher.search("探索", queryVector: nil).count == 1) + try store.delete(id) + #expect(try searcher.search("探索", queryVector: nil).isEmpty) + } + } + + @Test func doesNotTreatFolderNamesOrLatinSubstringsAsHanFilenameMatches() throws { + try withStore { store in + try store.insert(Candidate(path: "/fixture/情緒輔導資料/別的檔案.pdf", kind: .pdf, size: 1, mtime: 1, inode: 1, device: 1)) + try add("教育簡報.pdf", to: store) + try add("category.pdf", to: store) + let searcher = try Searcher(store: store) + #expect(try searcher.search("輔導", queryVector: nil).isEmpty) + #expect(try searcher.search("育簡", queryVector: nil).first?.path == "/fixture/教育簡報.pdf") + #expect(try searcher.search("tegory", queryVector: nil).isEmpty) + } + } + + @Test func filenameLookupWinsOverSemanticPagesAndHasNoDuplicateHits() throws { + try withStore { store in + let named = try add("學生情緒輔導紀錄.pdf", to: store) + let query: [Float] = [1] + Array(repeating: 0, count: Vectors.dimension - 1) + let other: [Float] = [0, 1] + Array(repeating: 0, count: Vectors.dimension - 2) + try store.addSegment(file: named, kind: .page, modality: .text, loc: 3, text: "輔導", vector: query) + for n in 0..<64 { + let id = try add("其他檔案-\(n).pdf", to: store) + try store.addSegment(file: id, kind: .page, modality: .text, loc: 1, text: "other", vector: n == 0 ? query : other) + } + let hits = try Searcher(store: store).search("輔導", queryVector: query) + #expect(hits.first?.path == "/fixture/學生情緒輔導紀錄.pdf") + #expect(hits.first?.segment == .name) + #expect(hits.first?.loc == nil) + #expect(hits.filter { $0.path == "/fixture/學生情緒輔導紀錄.pdf" }.count == 1) + #expect(zip(hits, hits.dropFirst()).allSatisfy { $0.score >= $1.score }) + } + } + + @Test func exactNameIsNotLostBehindManyPartialMatches() throws { + try withStore { store in + try store.db.transaction { + for n in 0..<1_100 { try add("學生輔導紀錄-\(n).pdf", to: store) } + try add("輔導.pdf", to: store) + } + let hits = try Searcher(store: store, loadVectors: false).search("輔導", queryVector: nil, limit: 5) + #expect(hits.count == 5) + #expect(hits.first?.path == "/fixture/輔導.pdf") + } + } + + @Test func mixedLatinCaseAndCompatibilityCharactersMatch() throws { + try withStore { store in + try add("２０２６ＣＡＦÉ學生輔導紀錄.pdf", to: store) + let hits = try Searcher(store: store).search("2026café學生輔導", queryVector: nil) + #expect(hits.count == 1) + } + } + + @Test func codeIdentifierSearchKeepsItsExistingRules() throws { + try withStore(file: IndexStore.codeFile) { store in + try add("學生情緒輔導紀錄.swift", to: store) + #expect(try Searcher(store: store).search("輔導", queryVector: nil).isEmpty) + } + } + +} diff --git a/docs/han-filename-search.md b/docs/han-filename-search.md new file mode 100644 index 00000000..acafd0fa --- /dev/null +++ b/docs/han-filename-search.md @@ -0,0 +1,63 @@ +# Han filename lookups + +`輔導` should find `學生情緒輔導紀錄.pdf` even before its contents have been embedded. FTS5's unicode61 tokenizer +keeps an unspaced Han run as one token, so matching the middle of that run needs a basename lookup. + +``` +query contains a run of at least two Han characters + → literal SQLite prefilter over existing name rows + → every query component must occur in the normalized basename + → complete basename/stem, then partial basename, then existing content/meaning matches +``` + +The prefilter uses a bound `instr` argument, not LIKE wildcards. Names retain the existing NFKC normalization; +verification is case-insensitive and includes mixed letters, numbers and extensions. File-kind filters apply before +matches are ranked. A filename lookup points to its name segment, rather than an unrelated semantic page in that file. +Scores remain descending so the existing weaker-results fold can use them. + +There is no schema change, index rebuild, embedding change, or new dependency. Existing names are immediately +searchable, including pending files, and moves/deletes are read from the live index. Latin-only queries and code +identifier search keep their existing rules. This does not introduce Chinese content tokenization, Traditional/ +Simplified conversion, single-character substring lookup or Spotlight application/system actions. + +## Reproduce the focused evaluation + +Use a new, isolated folder and index: + +```sh +python3 scripts/testbed/make_han_filenames.py /tmp/DigUpTestbed/han-filenames +.build/release/digup index /tmp/DigUpTestbed/han-filenames --limit 0 --index /tmp/DigUpHanIndex +DIGUP_TESTBED=/tmp/DigUpTestbed .build/release/digup eval evals/han-filenames.json --index /tmp/DigUpHanIndex +``` + +`--limit 0` syncs the actual fixture names without embedding their contents. This isolates filename recall from the +model's guesses. Use the same index for the before/after binaries; the file generator contains only synthetic text. + +## Validation (macOS, Apple M2 Pro, 16 GB) + +Compared official v0.6.1 with this patch on the same indexes and model. Nine regression tests cover two-character and +internal substrings, typing/settled queries, exact-name ranking, semantic-page competition, duplicate prevention, +kind filters, all query components, existing names, rename/delete, full-width/case normalization, code isolation, and +an exact name inserted after 1,100 partial matches. `swift test`: 101 passed (95 DigUpKit, 6 LlamaRuntime including +actual reference vectors). + +| Evaluation | Queries | Before R@1 / R@5 / R@10 | After R@1 / R@5 / R@10 | +| --- | ---: | --- | --- | +| Han names, keywords only | 12 | .33 / .33 / .33 | 1.00 / 1.00 / 1.00 | +| Upstream synthetic | 37 | .92 / .97 / .97 | .92 / .97 / .97 | +| Upstream multilingual | 48 | .92 / .98 / .98 | .92 / .98 / .98 | +| Upstream code | 41 | 1.00 / 1.00 / 1.00 | 1.00 / 1.00 / 1.00 | + +All 126 upstream queries produced byte-identical evaluation output before and after. The synthetic/media/code +fixtures were generated by the upstream testbed scripts; the personal-file sampling step was omitted. These +figures are measured results on this fixture, not a claim of perfect recall across arbitrary files or languages. + +A separate keyword-only index held 20,000 synthetic filenames, including 2,000 matching Han names. Twenty measured +CLI runs per query alternated before/after after two warmups; times include process startup, opening the same index +and returning up to 20 results. The extra basename verification costs time only for Han queries. + +| Query | Before median / p95 (ms) | After median / p95 (ms) | +| --- | --- | --- | +| 輔導 | 15.18 / 19.48 | 59.02 / 63.12 | +| 情緒輔導 | 14.70 / 16.39 | 57.25 / 60.80 | +| annual | 39.91 / 59.33 | 39.24 / 41.91 | diff --git a/evals/han-filenames.json b/evals/han-filenames.json new file mode 100644 index 00000000..1481012b --- /dev/null +++ b/evals/han-filenames.json @@ -0,0 +1,89 @@ +{ + "base": "~/DigUpTestbed/han-filenames", + "queries": [ + { + "q": "輔導", + "expect": [ + "exact/輔導.txt" + ], + "tag": "han-name" + }, + { + "q": "情緒", + "expect": [ + "general/學生情緒輔導紀錄.txt" + ], + "tag": "han-name" + }, + { + "q": "情緒輔導", + "expect": [ + "general/學生情緒輔導紀錄.txt" + ], + "tag": "han-name" + }, + { + "q": "學生情緒", + "expect": [ + "general/學生情緒輔導紀錄.txt" + ], + "tag": "han-name" + }, + { + "q": "探索", + "expect": [ + "other/生涯探索規劃.txt" + ], + "tag": "han-name" + }, + { + "q": "規劃", + "expect": [ + "other/生涯探索規劃.txt" + ], + "tag": "han-name" + }, + { + "q": "教學", + "expect": [ + "other/期末教學檢討.txt" + ], + "tag": "han-name" + }, + { + "q": "課程", + "expect": [ + "other/特殊教育課程簡報.txt" + ], + "tag": "han-name" + }, + { + "q": "整理", + "expect": [ + "other/資料整理會議紀錄.txt" + ], + "tag": "han-name" + }, + { + "q": "會議", + "expect": [ + "other/資料整理會議紀錄.txt" + ], + "tag": "han-name" + }, + { + "q": "2026café學生輔導", + "expect": [ + "mixed/２０２６ＣＡＦÉ學生輔導紀錄.txt" + ], + "tag": "han-name" + }, + { + "q": "學生情緒輔導紀錄.txt", + "expect": [ + "general/學生情緒輔導紀錄.txt" + ], + "tag": "han-name" + } + ] +} diff --git a/scripts/testbed/make_han_filenames.py b/scripts/testbed/make_han_filenames.py new file mode 100644 index 00000000..30271812 --- /dev/null +++ b/scripts/testbed/make_han_filenames.py @@ -0,0 +1,23 @@ +"""Create synthetic filename fixtures only; never reads user files. + +python3 scripts/testbed/make_han_filenames.py /han-filenames +Then sync names only with `digup index --limit 0 --index `. +Evaluate evals/han-filenames.json with DIGUP_TESTBED=. +""" +from pathlib import Path +import sys + +root = Path(sys.argv[1]) +for name in [ + "general/學生情緒輔導紀錄.txt", + "exact/輔導.txt", + "mixed/２０２６ＣＡＦÉ學生輔導紀錄.txt", + "other/生涯探索規劃.txt", + "other/期末教學檢討.txt", + "other/特殊教育課程簡報.txt", + "other/資料整理會議紀錄.txt", + "資料夾輔導/無關文件.txt", +]: + file = root / name + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text("Synthetic filename-search fixture. No personal data.\n", encoding="utf-8") From 3e7070e5bac0b03515c7777f638d3aab7c557217 Mon Sep 17 00:00:00 2001 From: max0071001 <48833066+max0071001@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:35:26 +0800 Subject: [PATCH 2/2] Cover Traditional Chinese filename ranking in typing and settled searches --- Tests/DigUpKitTests/HanFilenameTests.swift | 22 ++++++++++++++++++++++ docs/han-filename-search.md | 8 ++++++++ 2 files changed, 30 insertions(+) diff --git a/Tests/DigUpKitTests/HanFilenameTests.swift b/Tests/DigUpKitTests/HanFilenameTests.swift index 7b4f9943..d6a5a037 100644 --- a/Tests/DigUpKitTests/HanFilenameTests.swift +++ b/Tests/DigUpKitTests/HanFilenameTests.swift @@ -3,6 +3,28 @@ import Testing @testable import DigUpKit struct HanFilenameTests { + @Test func traditionalFilenameRankingAndDeduplicationHoldWhileTypingAndAfterSubmission() throws { + try withStore { store in + for name in ["學生情緒輔導紀錄.txt", "其他文件.txt", "輔導.txt"] { + let id = try add(name, to: store, kind: .doc) + // A standalone keyword is already supported by FTS on v0.6.1. + // Deliberately avoid depending on Han body tokenization from #111. + try store.addSegment(file: id, kind: .chunk, modality: .text, loc: 1, text: "輔導") + } + let searcher = try Searcher(store: store, loadVectors: false) + for typing in [false, true] { + let hits = try searcher.search("輔導", queryVector: nil, prefixLast: typing) + #expect(hits.map { URL(fileURLWithPath: $0.path).lastPathComponent } + == ["輔導.txt", "學生情緒輔導紀錄.txt", "其他文件.txt"], "typing: \(typing)") + try #require(hits.count == 3) + #expect(hits[0].segment == .name) + #expect(hits[1].segment == .name) + #expect(hits[2].segment == .chunk) + #expect(Set(hits.map(\.path)).count == hits.count) + } + } + } + func withStore(file: String = IndexStore.mainFile, _ body: (IndexStore) throws -> Void) throws { let folder = FileManager.default.temporaryDirectory.appendingPathComponent("HanFilenameTests-\(UUID())") defer { try? FileManager.default.removeItem(at: folder) } diff --git a/docs/han-filename-search.md b/docs/han-filename-search.md index acafd0fa..6b52af67 100644 --- a/docs/han-filename-search.md +++ b/docs/han-filename-search.md @@ -61,3 +61,11 @@ and returning up to 20 results. The extra basename verification costs time only | 輔導 | 15.18 / 19.48 | 59.02 / 63.12 | | 情緒輔導 | 14.70 / 16.39 | 57.25 / 60.80 | | annual | 39.91 / 59.33 | 39.24 / 41.91 | + +## Independent regression coverage (2026-10-20) + +An additional test checks exact Traditional Chinese filenames before partial filenames before content-only matches, +with no duplicate files, both while typing and after submission. It uses the public segment insertion API with a +standalone body keyword that v0.6.1 already supports, and disables vectors. It does not require the Han body +tokenization or index migration proposed in #111. Those changes and their body-search tests remain separate. +Interface localization remains in #34.