From 084681d0fa3e19409d76caafdef45e8140f7fe84 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Thu, 8 Oct 2026 19:01:51 +0300 Subject: [PATCH] feat(search): find-in-page candidates over a whole book findInBookCandidates returns the lines of a book that may contain a literal substring (nikud, final letters and case ignored), for the app to confirm. Null when the index can't answer: book not indexed, or a query too short to filter on. Signed-off-by: Elie Gambache --- .../search/LuceneSearchEngine.kt | 63 ++++++++++++++ .../seforimlibrary/search/SearchEngine.kt | 13 +++ .../search/LuceneSearchEngineTest.kt | 85 +++++++++++++++++++ 3 files changed, 161 insertions(+) diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index 4901a34c..9b60696b 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -219,6 +219,69 @@ class LuceneSearchEngine( return buildSnippetInternal(rawClean, anchorTerms, highlightTerms) } + override fun findInBookCandidates(query: String, bookId: Long): LongArray? { + val norm = HebrewTextUtils.normalizeHebrew(query) + if (norm.isBlank()) return LongArray(0) + val bookLines = BooleanQuery.Builder() + .add(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER) + .add(IntPoint.newExactQuery("book_id", bookId.toInt()), BooleanClause.Occur.FILTER) + .build() + // Nothing to filter on: every line would be a candidate, cheaper scanned from the DB + val clauses = literalPresenceClauses(analyzeToTerms(stdAnalyzer, norm).orEmpty()) + if (clauses.isEmpty()) return null + val builder = BooleanQuery.Builder().add(bookLines, BooleanClause.Occur.FILTER) + clauses.forEach { builder.add(it, BooleanClause.Occur.FILTER) } + + return withSearcher { searcher -> + var ids = LongArray(256) + var count = 0 + // Only line_id is read per hit: no scoring, no other stored field + val collector = object : Collector { + override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { + val storedFields = leafContext.reader().storedFields() + return object : LeafCollector { + override fun setScorer(scorer: Scorable) = Unit + + override fun collect(doc: Int) { + val lineId = storedFields.document(doc, setOf("line_id")) + .getField("line_id")?.numericValue()?.toLong() ?: return + if (count == ids.size) ids = ids.copyOf(count * 2) + ids[count++] = lineId + } + } + } + + override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES + } + // searcher has no executor, so slices run sequentially and can share one collector + searcher.search( + builder.build(), + object : CollectorManager { + override fun newCollector(): Collector = collector + override fun reduce(collectors: Collection) = Unit + } + ) + // No candidate may mean a book the index lacks (older index, book added later): unknown, not "no match" + if (count == 0 && searcher.count(bookLines) == 0) null else ids.copyOf(count) + } + } + + /** + * Filters every line holding the analyzed [tokens] as one literal substring satisfies. The + * substring may start and end mid-word: the first token is only a word suffix, the last a + * word prefix, the inner ones whole words. A first or lone token is matched anywhere in a + * word through its 4-grams, so one under 4 letters can't filter (a leading wildcard would + * scan the whole term dictionary); a last one under 3 neither, its prefix being too common. + */ + private fun literalPresenceClauses(tokens: List): List = + tokens.mapIndexedNotNull { i, token -> + when { + i in 1 until tokens.lastIndex -> TermQuery(Term("text", token)) + i > 0 && token.length >= 3 -> PrefixQuery(Term("text", token)) + else -> buildNgramPresenceForToken(token) + } + } + override fun close() { // Directory is closed automatically when readers are closed } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt index f2d6cef3..79d6cc4b 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt @@ -107,6 +107,19 @@ interface SearchEngine : Closeable { */ suspend fun semanticFind(query: String, bookId: Long, limit: Int): List = emptyList() + /** + * Literal find-in-page over a whole book, not just its loaded lines: the ids of the lines + * of [bookId] that may contain [query] as a substring (nikud, final letters and case + * ignored), in no particular order. + * + * A superset: the index knows which words a line holds, not where, so the caller confirms + * each candidate against the line's text. + * + * @return null when the index can't answer (the book isn't indexed, no index at all, or a + * query too short to filter on): the caller must then scan the book itself + */ + fun findInBookCandidates(query: String, bookId: Long): LongArray? = null + /** * Ensures the dense backend (embedding model + vector index) is loaded and reports whether * it is actually available. Useful for diagnostics and to decide whether semantic features diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt index f75ce602..223499de 100644 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt @@ -7,7 +7,12 @@ import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue import kotlin.test.assertFalse +import org.apache.lucene.analysis.Analyzer +import org.apache.lucene.analysis.LowerCaseFilter +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper +import org.apache.lucene.analysis.ngram.NGramTokenFilter import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.analysis.standard.StandardTokenizer import org.apache.lucene.document.Document import org.apache.lucene.document.Field import org.apache.lucene.document.IntPoint @@ -482,6 +487,46 @@ class LuceneSearchEngineTest { } } + // --- findInBookCandidates tests --- + + @Test + fun `findInBookCandidates finds a substring inside a word, nikud ignored`() { + withFindIndex { engine -> + assertEquals(setOf(1L), engine.findInBookCandidates("רֵאשִׁי", bookId = 1)!!.toSet()) + } + } + + @Test + fun `findInBookCandidates finds a query starting and ending mid-word`() { + withFindIndex { engine -> + // "שית" ends a word, "ברא" begins one, "אלהים" is whole + assertEquals(setOf(1L), engine.findInBookCandidates("שית ברא אלהים", bookId = 1)!!.toSet()) + } + } + + @Test + fun `findInBookCandidates can't answer for a book the index lacks`() { + withFindIndex { engine -> + assertNull(engine.findInBookCandidates("בראשית", bookId = 99)) + assertTrue(engine.findInBookCandidates("ויקרא", bookId = 1)!!.isEmpty()) + } + } + + @Test + fun `findInBookCandidates stays within the book`() { + withFindIndex { engine -> + assertEquals(setOf(4L), engine.findInBookCandidates("בראשית", bookId = 2)!!.toSet()) + } + } + + @Test + fun `findInBookCandidates can't answer when the query is too short to filter`() { + withFindIndex { engine -> + assertNull(engine.findInBookCandidates("של", bookId = 1)) + assertTrue(engine.findInBookCandidates(" ", bookId = 1)!!.isEmpty()) + } + } + // --- Helper methods --- private fun collectLineIds(session: SearchSession): Set = runBlocking { @@ -526,6 +571,46 @@ class LuceneSearchEngineTest { } } + private fun withFindIndex(block: (LuceneSearchEngine) -> Unit) { + val tempDir = createTempIndexDir() + try { + // Same analyzers as the generator: standard words + per-word 4-grams + val ngram4 = object : Analyzer() { + override fun createComponents(fieldName: String): TokenStreamComponents { + val src = StandardTokenizer() + return TokenStreamComponents(src, NGramTokenFilter(LowerCaseFilter(src), 4, 4, false)) + } + } + val analyzer = PerFieldAnalyzerWrapper(StandardAnalyzer(), mapOf("text_ng4" to ngram4)) + FSDirectory.open(tempDir).use { dir -> + IndexWriter(dir, IndexWriterConfig(analyzer)).use { writer -> + listOf( + Triple(1L, 1, "בראשית ברא אלהים"), + Triple(2L, 1, "והארץ היתה תהו"), + Triple(3L, 1, "ויברא אלהים את האדם"), + Triple(4L, 2, "בראשית רבה"), + ).forEach { (lineId, bookId, text) -> + val norm = HebrewTextUtils.normalizeHebrew(text) + writer.addDocument( + Document().apply { + add(StringField("type", "line", Field.Store.NO)) + add(StoredField("book_id", bookId)) + add(IntPoint("book_id", bookId)) + add(StoredField("line_id", lineId)) + add(IntPoint("line_id", lineId.toInt())) + add(TextField("text", norm, Field.Store.NO)) + add(TextField("text_ng4", norm, Field.Store.NO)) + }, + ) + } + } + } + block(LuceneSearchEngine(tempDir)) + } finally { + deleteDirectory(tempDir) + } + } + private fun createTempIndexDir(): Path { return Files.createTempDirectory("lucene_test_index") }