From 37122c6ee0a063316a27bcec7d1c929e6fb43540 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 12:20:39 -0500 Subject: [PATCH 01/27] feat(core): support unsegmented-script full-text search Signed-off-by: phernandez --- docs/semantic-search.md | 25 +-- ...7_add_script_ngrams_to_full_text_search.py | 118 +++++++++++++ src/basic_memory/models/search.py | 19 +++ .../repository/postgres_search_repository.py | 156 +++++++++++++----- src/basic_memory/repository/script_ngrams.py | 112 +++++++++++++ .../repository/search_repository_base.py | 19 ++- .../repository/sqlite_search_repository.py | 36 +++- tests/conftest.py | 4 + tests/repository/test_script_ngrams.py | 109 ++++++++++++ tests/repository/test_search_repository.py | 6 +- tests/test_script_ngram_search_migration.py | 99 +++++++++++ 11 files changed, 638 insertions(+), 65 deletions(-) create mode 100644 src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py create mode 100644 src/basic_memory/repository/script_ngrams.py create mode 100644 tests/repository/test_script_ngrams.py create mode 100644 tests/test_script_ngram_search_migration.py diff --git a/docs/semantic-search.md b/docs/semantic-search.md index 7984f6122..ccbf2ca29 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -203,17 +203,20 @@ use the same model contract. ### Chinese, Japanese, Korean and other unsegmented scripts -Keyword (full-text) search does not segment CJK text. Both backends tokenize a run of CJK -characters as a single token — SQLite's FTS5 `unicode61` tokenizer and Postgres's default text -search parser alike. Query terms are matched as prefixes, so `生存` finds a note containing -`生存竞争` (the run starts with the query) but not one containing only `适者生存`, where the -query sits in the middle of a run. - -Until a segmenting tokenizer option ships (tracked in -[#1294](https://github.com/basicmachines-co/basic-memory/issues/1294)), use semantic or hybrid -search with a multilingual embedding model for these languages — the Jina Chinese-English model -or multilingual E5 configured above both work — and expect keyword search to match whole runs -and run prefixes only. +Full-text search automatically adds an ordered script n-gram channel for writing systems that do +not reliably separate words with spaces. It covers Han, Hiragana, Katakana, Hangul, Bopomofo, +Thai, Lao, Tibetan, Myanmar, and Khmer text on both SQLite and Postgres. For example, `生存` +matches the middle of `适者生存`, while a query with the characters in a different order does +not. Mixed queries such as `OpenAI 适者生存` require both the word and script terms. + +There is no language, tokenizer, or embedding setting to enable. New and edited notes are indexed +automatically. After upgrading an existing local project, run `bm reindex --search` once to +populate script terms for notes already in the index. Hosted Cloud projects receive the same +Postgres analysis through the managed fleet reindex; Cloud users do not run a local command. + +This is lexical matching, independent of the configured embedding model. Vector and hybrid search +quality in these languages still depends on choosing a multilingual embedding model such as the +Jina Chinese-English model or multilingual E5 configured above. ### OpenAI diff --git a/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py b/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py new file mode 100644 index 000000000..00639c2ea --- /dev/null +++ b/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py @@ -0,0 +1,118 @@ +"""Add portable script n-grams to full-text search. + +Revision ID: d2e3f4a5b6c7 +Revises: bcdbd5a942ca +Create Date: 2026-08-29 00:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect + + +revision: str = "d2e3f4a5b6c7" +down_revision: Union[str, None] = "bcdbd5a942ca" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +SQLITE_COLUMNS = """ + id, title, content_stems, content_snippet, {script_column} permalink, + file_path, type, project_id, from_id, to_id, relation_type, entity_id, + category, metadata, created_at, updated_at +""" + + +def rebuild_sqlite_search_index(*, include_script_ngrams: bool) -> None: + """Copy the FTS5 table while changing its indexed-column contract.""" + search_index_exists = inspect(op.get_bind()).has_table("search_index") + script_definition = "script_ngrams," if include_script_ngrams else "" + op.execute(f""" + CREATE VIRTUAL TABLE search_index_rebuilt USING fts5( + id UNINDEXED, + title, + content_stems, + content_snippet, + {script_definition} + permalink, + file_path UNINDEXED, + type UNINDEXED, + project_id UNINDEXED, + from_id UNINDEXED, + to_id UNINDEXED, + relation_type UNINDEXED, + entity_id UNINDEXED, + category UNINDEXED, + metadata UNINDEXED, + created_at UNINDEXED, + updated_at UNINDEXED, + tokenize='unicode61 tokenchars 0x2F', + prefix='1,2,3,4' + ) + """) + + if not search_index_exists: + op.execute("ALTER TABLE search_index_rebuilt RENAME TO search_index") + return + + source_columns = SQLITE_COLUMNS.format(script_column="") + target_columns = SQLITE_COLUMNS.format( + script_column="script_ngrams," if include_script_ngrams else "" + ) + selected_columns = SQLITE_COLUMNS.format(script_column="'' AS script_ngrams,") + if not include_script_ngrams: + selected_columns = source_columns + op.execute( + f"INSERT INTO search_index_rebuilt ({target_columns}) " + f"SELECT {selected_columns} FROM search_index" + ) + op.execute("DROP TABLE search_index") + op.execute("ALTER TABLE search_index_rebuilt RENAME TO search_index") + + +def upgrade() -> None: + """Add the derived script channel; a reindex populates existing rows.""" + if op.get_bind().dialect.name == "sqlite": + rebuild_sqlite_search_index(include_script_ngrams=True) + return + + op.execute("ALTER TABLE search_index ADD COLUMN script_ngrams TEXT NOT NULL DEFAULT ''") + op.execute(""" + ALTER TABLE search_index + ADD COLUMN script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED + """) + op.execute(""" + CREATE INDEX idx_search_index_script_ngrams_fts + ON search_index USING gin(script_ngrams_index_col) + """) + op.execute( + "ALTER TABLE search_index_fts_chunks ADD COLUMN script_ngrams TEXT NOT NULL DEFAULT ''" + ) + op.execute(""" + ALTER TABLE search_index_fts_chunks + ADD COLUMN script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED + """) + op.execute(""" + CREATE INDEX idx_search_index_fts_chunks_script_ngrams_fts + ON search_index_fts_chunks USING gin(script_ngrams_index_col) + """) + + +def downgrade() -> None: + """Remove the script channel while preserving the existing word index.""" + if op.get_bind().dialect.name == "sqlite": + rebuild_sqlite_search_index(include_script_ngrams=False) + return + + op.execute("DROP INDEX IF EXISTS idx_search_index_fts_chunks_script_ngrams_fts") + op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS script_ngrams_index_col") + op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS script_ngrams") + op.execute("DROP INDEX IF EXISTS idx_search_index_script_ngrams_fts") + op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS script_ngrams_index_col") + op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS script_ngrams") diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index c9616d850..5501cd83a 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -21,6 +21,7 @@ title TEXT, content_stems TEXT, content_snippet TEXT, + script_ngrams TEXT NOT NULL DEFAULT '', permalink VARCHAR, file_path VARCHAR, type VARCHAR, @@ -39,6 +40,9 @@ coalesce(content_stems, '') ) ) STORED, + script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED, PRIMARY KEY (id, type, project_id), FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE ) @@ -48,6 +52,11 @@ CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col) """) +CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_script_ngrams_fts +ON search_index USING gin(script_ngrams_index_col) +""") + # Full note bodies are stored in bounded child rows so one unusually large note # cannot exceed PostgreSQL's per-tsvector size limit. CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL(""" @@ -57,9 +66,13 @@ search_index_type VARCHAR NOT NULL, chunk_index INTEGER NOT NULL, chunk_text TEXT NOT NULL, + script_ngrams TEXT NOT NULL DEFAULT '', textsearchable_index_col tsvector GENERATED ALWAYS AS ( to_tsvector('english', chunk_text) ) STORED, + script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED, PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index), FOREIGN KEY (search_index_id, search_index_type, project_id) REFERENCES search_index(id, type, project_id) @@ -72,6 +85,11 @@ ON search_index_fts_chunks USING gin(textsearchable_index_col) """) +CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_script_ngrams_fts +ON search_index_fts_chunks USING gin(script_ngrams_index_col) +""") + CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL(""" CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops) """) @@ -94,6 +112,7 @@ title, -- Title for searching content_stems, -- Main searchable content split into stems content_snippet, -- File content snippet for display + script_ngrams, -- Portable bigrams for scripts without word boundaries permalink, -- Stable identifier (now indexed for path search) file_path UNINDEXED, -- Physical location type UNINDEXED, -- entity/relation/observation diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index e90c75d70..542e352c8 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -21,6 +21,7 @@ from basic_memory.repository.rerank_provider_factory import create_rerank_provider from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_query import relaxed_query_words, relaxation_word_tokens +from basic_memory.repository.script_ngrams import analyze_script_query, build_script_ngrams from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.repository.search_repository_base import ( SearchRepositoryBase, @@ -282,6 +283,10 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: # Serialize JSON for raw SQL insert_data = search_index_row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + search_index_row.title, + search_index_row.content_stems, + ) insert_data = _strip_nul_from_row(insert_data) # Use upsert to handle race conditions during parallel indexing @@ -291,13 +296,13 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -308,6 +313,7 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + script_ngrams = EXCLUDED.script_ngrams, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -359,6 +365,7 @@ async def _replace_fts_chunks( "search_index_type": row.type, "chunk_index": chunk_index, "chunk_text": chunk_text.replace("\x00", ""), + "script_ngrams": build_script_ngrams(chunk_text.replace("\x00", "")), } for row in search_index_rows for chunk_index, chunk_text in _iter_fts_chunks(row.content_snippet) @@ -373,19 +380,22 @@ async def _replace_fts_chunks( search_index_id, search_index_type, chunk_index, - chunk_text + chunk_text, + script_ngrams ) SELECT :project_id, chunk.search_index_id, chunk.search_index_type, chunk.chunk_index, - chunk.chunk_text + chunk.chunk_text, + chunk.script_ngrams FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk( search_index_id INTEGER, search_index_type VARCHAR, chunk_index INTEGER, - chunk_text TEXT + chunk_text TEXT, + script_ngrams TEXT ) """), {"project_id": self.project_id, "chunks": json.dumps(chunks)}, @@ -901,6 +911,10 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non for row in search_index_rows: insert_data = row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + row.title, + row.content_stems, + ) insert_data_list.append(_strip_nul_from_row(insert_data)) # Use upsert to handle race conditions during parallel indexing @@ -910,13 +924,13 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -927,6 +941,7 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + script_ngrams = EXCLUDED.script_ngrams, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -977,6 +992,7 @@ async def _build_fts_query_parts( order_by_clause = "" from_clause = "search_index" document_vector_sql: str | None = None + script_tsqueries: list[str] = [] # Handle text search for title and content using tsvector if search_text: @@ -984,29 +1000,30 @@ async def _build_fts_query_parts( # For wildcard searches, don't add any text conditions pass else: - # Prepare search term for tsquery - processed_text = self._prepare_search_term(search_text.strip()) - params["text"] = processed_text - probe_texts = [processed_text] - if allow_relaxed: - relaxed_text = self._relaxed_tsquery_text(search_text) - if relaxed_text: - probe_texts.append(relaxed_text) - - candidate_operands: dict[str, None] = {} - for probe_text in probe_texts: - for operand, _representative in _tsquery_operands(probe_text): - candidate_operands.setdefault(operand, None) - if candidate_operands: - params["text_candidate"] = " | ".join(candidate_operands) - - # Trigger: PostgreSQL can extract a required-positive query tree. - # Why: OR-ing its operands is a safe indexed superset even when - # terms live in different chunks. Pure/optional negation returns - # ``T`` and must retain all project rows for correct semantics. - # Outcome: ordinary and required-positive NOT queries use both - # GIN indexes; only genuinely unindexable negation scans the project. - from_clause = """ + script_query = analyze_script_query(search_text.strip()) + if script_query.word_text: + processed_text = self._prepare_search_term(script_query.word_text) + params["text"] = processed_text + probe_texts = [processed_text] + if allow_relaxed: + relaxed_text = self._relaxed_tsquery_text(script_query.word_text) + if relaxed_text: + probe_texts.append(relaxed_text) + + candidate_operands: dict[str, None] = {} + for probe_text in probe_texts: + for operand, _representative in _tsquery_operands(probe_text): + candidate_operands.setdefault(operand, None) + if candidate_operands: + params["text_candidate"] = " | ".join(candidate_operands) + + # Trigger: PostgreSQL can extract a required-positive query tree. + # Why: OR-ing its operands is a safe indexed superset even when + # terms live in different chunks. Pure/optional negation returns + # ``T`` and must retain all project rows for correct semantics. + # Outcome: ordinary and required-positive NOT queries use both + # GIN indexes; only genuinely unindexable negation scans the project. + from_clause = """ search_index JOIN ( SELECT candidate_parent.project_id, @@ -1039,9 +1056,59 @@ async def _build_fts_query_parts( ON fts_candidate.project_id = search_index.project_id AND fts_candidate.id = search_index.id AND fts_candidate.type = search_index.type - """ - document_vector_sql = self._document_fts_vector_sql(probe_texts, params) - conditions.append(f"{document_vector_sql} @@ to_tsquery('english', :text)") + """ + document_vector_sql = self._document_fts_vector_sql(probe_texts, params) + conditions.append(f"{document_vector_sql} @@ to_tsquery('english', :text)") + + if script_query.gram_phrases: + script_tsqueries = [ + " <-> ".join(f"'{gram}'" for gram in phrase) + for phrase in script_query.gram_phrases + ] + for index, script_tsquery in enumerate(script_tsqueries): + params[f"script_text_{index}"] = script_tsquery + if document_vector_sql is None: + # Trigger: a query contains script grams but no word terms. + # Why: a correlated predicate alone can scan every tenant row. + # Outcome: start from the parent and child GIN indexes. + params["script_candidate_text"] = " | ".join( + f"({script_tsquery})" for script_tsquery in script_tsqueries + ) + from_clause = """ + search_index JOIN ( + SELECT + script_parent.project_id, + script_parent.id, + script_parent.type + FROM search_index AS script_parent + WHERE script_parent.project_id = :project_id + AND script_parent.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + UNION + SELECT + script_candidate.project_id, + script_candidate.search_index_id AS id, + script_candidate.search_index_type AS type + FROM search_index_fts_chunks AS script_candidate + WHERE script_candidate.project_id = :project_id + AND script_candidate.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + ) AS fts_candidate + ON fts_candidate.project_id = search_index.project_id + AND fts_candidate.id = search_index.id + AND fts_candidate.type = search_index.type + """ + conditions.extend( + "(search_index.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index}) OR EXISTS (" + "SELECT 1 FROM search_index_fts_chunks AS script_chunk " + "WHERE script_chunk.project_id = search_index.project_id " + "AND script_chunk.search_index_id = search_index.id " + "AND script_chunk.search_index_type = search_index.type " + "AND script_chunk.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index})))" + for index in range(len(script_tsqueries)) + ) # Handle title search if title: @@ -1199,9 +1266,9 @@ async def _build_fts_query_parts( # Build SQL with ts_rank() for scoring # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 - if search_text and search_text.strip() and search_text.strip() != "*": - assert document_vector_sql is not None - score_expr = ( + score_parts: list[str] = [] + if document_vector_sql is not None: + score_parts.append( "GREATEST(" f"ts_rank({document_vector_sql}, to_tsquery('english', :text)), " "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text)), " @@ -1214,8 +1281,21 @@ async def _build_fts_query_parts( "AND fts_chunk.textsearchable_index_col " "@@ to_tsquery('english', :text)), 0))" ) - else: - score_expr = "0" + score_parts.extend( + "GREATEST(" + "ts_rank(search_index.script_ngrams_index_col, " + f"to_tsquery('simple', :script_text_{index})), " + "COALESCE((SELECT MAX(ts_rank(script_rank.script_ngrams_index_col, " + f"to_tsquery('simple', :script_text_{index}))) " + "FROM search_index_fts_chunks AS script_rank " + "WHERE script_rank.project_id = search_index.project_id " + "AND script_rank.search_index_id = search_index.id " + "AND script_rank.search_index_type = search_index.type " + "AND script_rank.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index})), 0))" + for index in range(len(script_tsqueries)) + ) + score_expr = f"GREATEST({', '.join(score_parts)})" if score_parts else "0" return from_clause, where_clause, params, order_by_clause, score_expr diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py new file mode 100644 index 000000000..83074f5fd --- /dev/null +++ b/src/basic_memory/repository/script_ngrams.py @@ -0,0 +1,112 @@ +"""Application-owned lexical analysis for scripts without reliable word boundaries.""" + +import unicodedata +from dataclasses import dataclass + + +_SCRIPT_BOUNDARY = "bm_script_boundary" + + +@dataclass(frozen=True, slots=True) +class ScriptQuery: + """The word-oriented and script-oriented parts of one user query.""" + + word_text: str | None + gram_phrases: tuple[tuple[str, ...], ...] + + +def is_script_search_character(character: str) -> bool: + codepoint = ord(character) + return any( + lower <= codepoint <= upper + for lower, upper in ( + (0x1100, 0x11FF), # Hangul Jamo + (0x0E00, 0x0EFF), # Thai and Lao + (0x0F00, 0x0FFF), # Tibetan + (0x1000, 0x109F), # Myanmar + (0x1780, 0x17FF), # Khmer + (0x19E0, 0x19FF), # Khmer symbols + (0x3040, 0x30FF), # Hiragana and Katakana + (0x3100, 0x318F), # Bopomofo and Hangul compatibility Jamo + (0x31A0, 0x31BF), # Bopomofo extended + (0x31F0, 0x31FF), # Katakana phonetic extensions + (0x3400, 0x4DBF), # CJK unified ideographs extension A + (0x4E00, 0x9FFF), # CJK unified ideographs + (0xA960, 0xA97F), # Hangul Jamo extended A + (0xA9E0, 0xA9FF), # Myanmar extended B + (0xAA60, 0xAA7F), # Myanmar extended A + (0xAC00, 0xD7AF), # Hangul syllables + (0xD7B0, 0xD7FF), # Hangul Jamo extended B + (0xF900, 0xFAFF), # CJK compatibility ideographs + (0x1B000, 0x1B16F), # Kana supplements and extensions + (0x20000, 0x2FFFF), # Supplementary CJK ideographs + (0x30000, 0x323AF), # CJK unified ideographs extensions G-H + ) + ) + + +def script_runs(text: str) -> tuple[tuple[str, ...], ...]: + """Return normalized script runs as grapheme-like searchable units.""" + runs: list[tuple[str, ...]] = [] + current: list[str] = [] + for character in unicodedata.normalize("NFKC", text): + if current and unicodedata.category(character) in {"Mn", "Mc", "Me"}: + current[-1] += character + continue + if is_script_search_character(character): + current.append(character) + continue + if current: + runs.append(tuple(current)) + current = [] + if current: + runs.append(tuple(current)) + return tuple(runs) + + +def script_run_grams(run: tuple[str, ...]) -> tuple[str, ...]: + """Use bigrams for context and retain a searchable single-character run.""" + if len(run) == 1: + return run + return tuple(first + second for first, second in zip(run, run[1:], strict=False)) + + +def build_script_ngrams(*texts: str | None) -> str: + """Build portable index text without depending on a database tokenizer.""" + gram_runs = [ + " ".join(script_run_grams(run)) for text in texts if text for run in script_runs(text) + ] + return f" {_SCRIPT_BOUNDARY} ".join(gram_runs) + + +def analyze_script_query(text: str) -> ScriptQuery: + """Split a natural-language query into word text and ordered script grams.""" + normalized = unicodedata.normalize("NFKC", text) + # Explicit Boolean expressions keep the existing backend parser. Splitting one + # operand into a second SQL channel would otherwise change OR/NOT semantics. + if {token.upper() for token in normalized.split()} & {"AND", "OR", "NOT"}: + return ScriptQuery(word_text=normalized, gram_phrases=()) + + word_characters: list[str] = [] + in_script_run = False + for character in normalized: + if in_script_run and unicodedata.category(character) in {"Mn", "Mc", "Me"}: + continue + if is_script_search_character(character): + if not in_script_run: + word_characters.append(" ") + in_script_run = True + continue + in_script_run = False + word_characters.append(character) + + word_tokens = [ + token + for token in "".join(word_characters).split() + if any(character.isalnum() for character in token) + ] + word_text = " ".join(word_tokens) or None + return ScriptQuery( + word_text=word_text, + gram_phrases=tuple(script_run_grams(run) for run in script_runs(normalized)), + ) diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 81ed70c9f..9b6d7e205 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -32,6 +32,7 @@ validate_rerank_scores, ) from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.script_ngrams import build_script_ngrams from basic_memory.repository.search_trace import ( BelowThreshold, FilteredOut, @@ -993,18 +994,23 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: # The database driver/column type will handle conversion insert_data = search_index_row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + search_index_row.title, + search_index_row.content_stems, + search_index_row.content_snippet, + ) # Insert new record await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -1039,19 +1045,24 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non for row in search_index_rows: insert_data = row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + row.title, + row.content_stems, + row.content_snippet, + ) insert_data_list.append(insert_data) # Batch insert all records using executemany await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 9dced81f9..472970be0 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -29,6 +29,7 @@ from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_query import relaxed_query_words from basic_memory.repository.search_repository_base import SearchRepositoryBase +from basic_memory.repository.script_ngrams import analyze_script_query from basic_memory.repository.search_trace import ( SearchTraceCollector, build_fts_page_stage, @@ -793,15 +794,21 @@ async def _build_fts_query_parts( # For wildcard searches, don't add any text conditions - return all results pass else: - # Use _prepare_search_term to handle both Boolean and non-Boolean queries - processed_text = self._prepare_search_term(search_text.strip()) - params["text"] = processed_text - # content_stems is capped for Postgres index-row compatibility, while - # SQLite stores the complete note body in its FTS5 content_snippet column. - match_conditions.append( - "(search_index.title MATCH :text OR search_index.content_stems MATCH :text " - "OR search_index.content_snippet MATCH :text)" - ) + script_query = analyze_script_query(search_text.strip()) + if script_query.word_text: + # Use _prepare_search_term to handle both Boolean and non-Boolean queries. + params["text"] = self._prepare_search_term(script_query.word_text) + # content_stems is capped for Postgres index-row compatibility, while + # SQLite stores the complete note body in its FTS5 content_snippet column. + match_conditions.append( + "(search_index.title MATCH :text OR " + "search_index.content_stems MATCH :text OR " + "search_index.content_snippet MATCH :text)" + ) + for index, phrase in enumerate(script_query.gram_phrases): + param_name = f"script_phrase_{index}" + params[param_name] = f'"{" ".join(phrase)}"' + match_conditions.append(f"search_index.script_ngrams MATCH :{param_name}") # Handle title match search if title: @@ -964,6 +971,17 @@ async def _build_fts_query_parts( conditions.append(f"{compare_expr} {operator} :{value_param}") continue + # Trigger: SQLite rejects some Boolean combinations of MATCH predicates, + # including a word-field OR expression combined with the script channel. + # Why: each MATCH must be evaluated in an FTS-valid query context. + # Outcome: intersect rowid subqueries when a search has multiple FTS clauses. + if len(match_conditions) > 1: + conditions.extend( + f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_condition})" + for match_condition in match_conditions + ) + match_conditions = [] + # Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with # "unable to use function MATCH in the requested context". # Why: MATCH needs to run in an FTS-valid context. diff --git a/tests/conftest.py b/tests/conftest.py index ef6cdb9d1..3769bfd2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -153,7 +153,9 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No from basic_memory.models.search import ( CREATE_POSTGRES_SEARCH_INDEX_FTS, CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX, CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE, + CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS, CREATE_POSTGRES_SEARCH_INDEX_METADATA, CREATE_POSTGRES_SEARCH_INDEX_PERMALINK, CREATE_POSTGRES_SEARCH_INDEX_TABLE, @@ -168,8 +170,10 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No await conn.run_sync(Base.metadata.create_all) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK) await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE) diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py new file mode 100644 index 000000000..df147f159 --- /dev/null +++ b/tests/repository/test_script_ngrams.py @@ -0,0 +1,109 @@ +"""Portable script n-gram analysis and full-text search regressions.""" + +from datetime import datetime, timezone + +import pytest + +from basic_memory.repository.script_ngrams import ( + analyze_script_query, + build_script_ngrams, + script_run_grams, + script_runs, +) +from basic_memory.repository.search_index_row import SearchIndexRow + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("适者生存", (("适", "者", "生", "存"),)), + ("適者生存", (("適", "者", "生", "存"),)), + ("サバイバル", (("サ", "バ", "イ", "バ", "ル"),)), + ("생존 경쟁", (("생", "존"), ("경", "쟁"))), + ("ภาษาไทย", (("ภ", "า", "ษ", "า", "ไ", "ท", "ย"),)), + ("ＡＢＣ", ()), + ], +) +def test_script_runs_cover_cjk_scripts_and_normalize_width( + text: str, + expected: tuple[tuple[str, ...], ...], +) -> None: + assert script_runs(text) == expected + + +def test_script_run_grams_preserve_order_and_single_character_queries() -> None: + assert script_run_grams(("适", "者", "生", "存")) == ("适者", "者生", "生存") + assert script_run_grams(("猫",)) == ("猫",) + + +def test_script_runs_attach_combining_marks_to_the_previous_unit() -> None: + text = "漢\N{VARIATION SELECTOR-1}" + + assert script_runs(text) == ((text,),) + assert analyze_script_query(text).word_text is None + + +def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> None: + assert build_script_ngrams("适者", "生存") == "适者 bm_script_boundary 生存" + + +def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: + query = analyze_script_query("OpenAI 适者生存，サバイバル") + + assert query.word_text == "OpenAI" + assert query.gram_phrases == ( + ("适者", "者生", "生存"), + ("サバ", "バイ", "イバ", "バル"), + ) + + +def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: + query = analyze_script_query("OpenAI OR 适者生存") + + assert query.word_text == "OpenAI OR 适者生存" + assert query.gram_phrases == () + + +@pytest.mark.asyncio +async def test_search_matches_cjk_substring_without_matching_reordered_characters( + search_repository, +) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1294, + type="entity", + file_path="notes/evolution.md", + title="進化について", + content_stems="OpenAI 即适者生存的讨论", + content_snippet="OpenAI 即适者生存的讨论", + permalink="notes/evolution", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + assert [result.id for result in await search_repository.search("适者生存")] == [1294] + assert [result.id for result in await search_repository.search("OpenAI 适者生存")] == [1294] + assert await search_repository.search("适生者存") == [] + + +@pytest.mark.asyncio +async def test_search_matches_script_text_beyond_the_parent_fts_limit(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1295, + type="entity", + file_path="notes/long.md", + title="進化 Long note", + content_stems="bounded parent search text", + content_snippet=f"{'x' * 9_000} 适者生存", + permalink="notes/long", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + assert [result.id for result in await search_repository.search("适者生存")] == [1295] + assert [result.id for result in await search_repository.search("進化 适者生存")] == [1295] diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 8eb049033..933ae45ea 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -1265,10 +1265,10 @@ async def test_multiword_query_relaxes_to_or_when_strict_misses(search_repositor @pytest.mark.asyncio -async def test_cjk_compound_query_relaxes_with_backend_prefix_terms( +async def test_cjk_compound_query_matches_with_or_without_relaxation( search_repository, search_entity ): - """Whitespace-separated CJK terms should match indexed CJK compounds when relaxed.""" + """Script n-grams make whitespace-separated CJK terms strict matches.""" row = SearchIndexRow( project_id=search_repository.project_id, id=search_entity.id, @@ -1286,7 +1286,7 @@ async def test_cjk_compound_query_relaxes_with_backend_prefix_terms( await search_repository.index_item(row) strict = await search_repository.search(search_text="季度 报告") - assert strict == [] + assert any(r.entity_id == search_entity.id for r in strict) results = await search_repository.search(search_text="季度 报告", allow_relaxed=True) assert any(r.entity_id == search_entity.id for r in results) diff --git a/tests/test_script_ngram_search_migration.py b/tests/test_script_ngram_search_migration.py new file mode 100644 index 000000000..2ddfc2be3 --- /dev/null +++ b/tests/test_script_ngram_search_migration.py @@ -0,0 +1,99 @@ +"""Migration coverage for the portable script n-gram FTS channel.""" + +import sqlite3 +from importlib import import_module +from types import SimpleNamespace + +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import create_engine + + +migration = import_module( + "basic_memory.alembic.versions.d2e3f4a5b6c7_add_script_ngrams_to_full_text_search" +) + + +def test_sqlite_upgrade_and_downgrade_preserve_word_search_rows(tmp_path, monkeypatch) -> None: + database_path = tmp_path / "script-ngrams.db" + engine = create_engine(f"sqlite:///{database_path}") + with engine.begin() as connection: + connection.exec_driver_sql(""" + CREATE VIRTUAL TABLE search_index USING fts5( + id UNINDEXED, title, content_stems, content_snippet, permalink, + file_path UNINDEXED, type UNINDEXED, project_id UNINDEXED, + from_id UNINDEXED, to_id UNINDEXED, relation_type UNINDEXED, + entity_id UNINDEXED, category UNINDEXED, metadata UNINDEXED, + created_at UNINDEXED, updated_at UNINDEXED, + tokenize='unicode61 tokenchars 0x2F', prefix='1,2,3,4' + ) + """) + connection.exec_driver_sql(""" + INSERT INTO search_index (id, title, content_stems, project_id) + VALUES (1, 'Existing title', 'existing words', 7) + """) + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + migration.upgrade() + + with sqlite3.connect(database_path) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(search_index)")] + assert "script_ngrams" in columns + assert connection.execute( + "SELECT id, title, script_ngrams FROM search_index" + ).fetchall() == [(1, "Existing title", "")] + + with engine.begin() as connection: + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + migration.downgrade() + + with sqlite3.connect(database_path) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(search_index)")] + assert "script_ngrams" not in columns + assert connection.execute("SELECT id, title FROM search_index").fetchall() == [ + (1, "Existing title") + ] + + +def test_sqlite_upgrade_creates_search_index_for_a_fresh_database(tmp_path, monkeypatch) -> None: + database_path = tmp_path / "fresh-script-ngrams.db" + engine = create_engine(f"sqlite:///{database_path}") + with engine.begin() as connection: + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + migration.upgrade() + + with sqlite3.connect(database_path) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(search_index)")] + assert "script_ngrams" in columns + + +def test_postgres_upgrade_and_downgrade_manage_both_script_indexes(monkeypatch) -> None: + statements: list[str] = [] + monkeypatch.setattr( + migration.op, + "get_bind", + lambda: SimpleNamespace(dialect=SimpleNamespace(name="postgresql")), + ) + monkeypatch.setattr( + migration.op, "execute", lambda statement: statements.append(str(statement)) + ) + + migration.upgrade() + migration.downgrade() + + sql = "\n".join(statements) + assert "idx_search_index_script_ngrams_fts" in sql + assert "idx_search_index_fts_chunks_script_ngrams_fts" in sql + assert "to_tsvector('simple', script_ngrams)" in sql + assert "DROP COLUMN IF EXISTS script_ngrams" in sql From 917552b8b6280733720a1e039fc91e01930cf504 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 12:30:08 -0500 Subject: [PATCH 02/27] fix(core): preserve absent runtime search indexes Signed-off-by: phernandez --- ...6c7_add_script_ngrams_to_full_text_search.py | 17 +++++++++++------ tests/test_script_ngram_search_migration.py | 10 +++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py b/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py index 00639c2ea..913297889 100644 --- a/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py +++ b/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py @@ -9,7 +9,7 @@ from typing import Sequence, Union from alembic import op -from sqlalchemy import inspect +from sqlalchemy import text revision: str = "d2e3f4a5b6c7" @@ -27,7 +27,16 @@ def rebuild_sqlite_search_index(*, include_script_ngrams: bool) -> None: """Copy the FTS5 table while changing its indexed-column contract.""" - search_index_exists = inspect(op.get_bind()).has_table("search_index") + bind = op.get_bind() + search_index_sql: str | None = bind.execute( + text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search_index'") + ).scalar_one_or_none() + + # The runtime owns creation of the derived FTS table, so migrations only transform an + # existing FTS5 index. This also leaves unrelated physical tables with the same name alone. + if search_index_sql is None or "using fts5" not in search_index_sql.casefold(): + return + script_definition = "script_ngrams," if include_script_ngrams else "" op.execute(f""" CREATE VIRTUAL TABLE search_index_rebuilt USING fts5( @@ -53,10 +62,6 @@ def rebuild_sqlite_search_index(*, include_script_ngrams: bool) -> None: ) """) - if not search_index_exists: - op.execute("ALTER TABLE search_index_rebuilt RENAME TO search_index") - return - source_columns = SQLITE_COLUMNS.format(script_column="") target_columns = SQLITE_COLUMNS.format( script_column="script_ngrams," if include_script_ngrams else "" diff --git a/tests/test_script_ngram_search_migration.py b/tests/test_script_ngram_search_migration.py index 2ddfc2be3..7efe08ed8 100644 --- a/tests/test_script_ngram_search_migration.py +++ b/tests/test_script_ngram_search_migration.py @@ -62,7 +62,9 @@ def test_sqlite_upgrade_and_downgrade_preserve_word_search_rows(tmp_path, monkey ] -def test_sqlite_upgrade_creates_search_index_for_a_fresh_database(tmp_path, monkeypatch) -> None: +def test_sqlite_upgrade_leaves_runtime_search_index_creation_to_the_model( + tmp_path, monkeypatch +) -> None: database_path = tmp_path / "fresh-script-ngrams.db" engine = create_engine(f"sqlite:///{database_path}") with engine.begin() as connection: @@ -74,8 +76,10 @@ def test_sqlite_upgrade_creates_search_index_for_a_fresh_database(tmp_path, monk migration.upgrade() with sqlite3.connect(database_path) as connection: - columns = [row[1] for row in connection.execute("PRAGMA table_info(search_index)")] - assert "script_ngrams" in columns + search_index_exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'search_index'" + ).fetchone() + assert search_index_exists is None def test_postgres_upgrade_and_downgrade_manage_both_script_indexes(monkeypatch) -> None: From 0e40744f9ce5584a7f8f5db5b63b224c014706ed Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 12:36:58 -0500 Subject: [PATCH 03/27] fix(core): address multilingual FTS review findings Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 14 +++-- .../repository/sqlite_search_repository.py | 23 ++++---- tests/repository/test_script_ngrams.py | 52 +++++++++++++++++-- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 83074f5fd..01235027f 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -73,9 +73,15 @@ def script_run_grams(run: tuple[str, ...]) -> tuple[str, ...]: def build_script_ngrams(*texts: str | None) -> str: """Build portable index text without depending on a database tokenizer.""" - gram_runs = [ - " ".join(script_run_grams(run)) for text in texts if text for run in script_runs(text) - ] + gram_runs: list[str] = [] + for text in texts: + if not text: + continue + for run in script_runs(text): + # Unigrams make a single-character query searchable inside a longer run. Keeping + # bigrams together after them preserves ordered phrase matching for longer queries. + index_terms = run if len(run) == 1 else (*run, *script_run_grams(run)) + gram_runs.append(" ".join(index_terms)) return f" {_SCRIPT_BOUNDARY} ".join(gram_runs) @@ -84,7 +90,7 @@ def analyze_script_query(text: str) -> ScriptQuery: normalized = unicodedata.normalize("NFKC", text) # Explicit Boolean expressions keep the existing backend parser. Splitting one # operand into a second SQL channel would otherwise change OR/NOT semantics. - if {token.upper() for token in normalized.split()} & {"AND", "OR", "NOT"}: + if set(normalized.split()) & {"AND", "OR", "NOT"}: return ScriptQuery(word_text=normalized, gram_phrases=()) word_characters: list[str] = [] diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 472970be0..80906af5e 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -43,6 +43,9 @@ from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +SQLITE_WORD_COLUMNS = "{title content_stems content_snippet}" + + class SQLiteSearchRepository(SearchRepositoryBase): """SQLite FTS5 implementation of search repository. @@ -797,14 +800,11 @@ async def _build_fts_query_parts( script_query = analyze_script_query(search_text.strip()) if script_query.word_text: # Use _prepare_search_term to handle both Boolean and non-Boolean queries. - params["text"] = self._prepare_search_term(script_query.word_text) + prepared_text = self._prepare_search_term(script_query.word_text) + params["text"] = f"{SQLITE_WORD_COLUMNS}: ({prepared_text})" # content_stems is capped for Postgres index-row compatibility, while # SQLite stores the complete note body in its FTS5 content_snippet column. - match_conditions.append( - "(search_index.title MATCH :text OR " - "search_index.content_stems MATCH :text OR " - "search_index.content_snippet MATCH :text)" - ) + match_conditions.append("search_index MATCH :text") for index, phrase in enumerate(script_query.gram_phrases): param_name = f"script_phrase_{index}" params[param_name] = f'"{" ".join(phrase)}"' @@ -974,13 +974,14 @@ async def _build_fts_query_parts( # Trigger: SQLite rejects some Boolean combinations of MATCH predicates, # including a word-field OR expression combined with the script channel. # Why: each MATCH must be evaluated in an FTS-valid query context. - # Outcome: intersect rowid subqueries when a search has multiple FTS clauses. + # Outcome: keep one outer MATCH for bm25 ranking and intersect the rest by rowid. if len(match_conditions) > 1: + ranked_match, *additional_matches = match_conditions conditions.extend( f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_condition})" - for match_condition in match_conditions + for match_condition in additional_matches ) - match_conditions = [] + match_conditions = [ranked_match] # Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with # "unable to use function MATCH in the requested context". @@ -1110,7 +1111,7 @@ async def run_search(active_session: AsyncSession): relaxed = self._relaxed_fts_text(search_text) if allow_relaxed and not rows else None if relaxed and params.get("text"): relaxed_fallback_used = True - params["text"] = relaxed + params["text"] = f"{SQLITE_WORD_COLUMNS}: ({relaxed})" logger.debug( "Strict SQLite FTS returned 0 results; retrying relaxed FTS query " f"strict='{search_text}' relaxed='{relaxed}'" @@ -1226,7 +1227,7 @@ async def count( self._relaxed_fts_text(search_text) if allow_relaxed and total == 0 else None ) if relaxed and params.get("text"): - params["text"] = relaxed + params["text"] = f"{SQLITE_WORD_COLUMNS}: ({relaxed})" with logfire.span( "search.count.relaxed_fts_retry", backend="sqlite", diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index df147f159..ace958bbf 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -44,7 +44,7 @@ def test_script_runs_attach_combining_marks_to_the_previous_unit() -> None: def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> None: - assert build_script_ngrams("适者", "生存") == "适者 bm_script_boundary 生存" + assert build_script_ngrams("适者", "生存") == ("适 者 适者 bm_script_boundary 生 存 生存") def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: @@ -64,6 +64,13 @@ def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: assert query.gram_phrases == () +def test_analyze_script_query_treats_lowercase_boolean_words_as_natural_language() -> None: + query = analyze_script_query("OpenAI and 适者生存") + + assert query.word_text == "OpenAI and" + assert query.gram_phrases == (("适者", "者生", "生存"),) + + @pytest.mark.asyncio async def test_search_matches_cjk_substring_without_matching_reordered_characters( search_repository, @@ -75,8 +82,8 @@ async def test_search_matches_cjk_substring_without_matching_reordered_character type="entity", file_path="notes/evolution.md", title="進化について", - content_stems="OpenAI 即适者生存的讨论", - content_snippet="OpenAI 即适者生存的讨论", + content_stems="OpenAI and 即适者生存的讨论与黑猫", + content_snippet="OpenAI and 即适者生存的讨论与黑猫", permalink="notes/evolution", created_at=now, updated_at=now, @@ -85,9 +92,48 @@ async def test_search_matches_cjk_substring_without_matching_reordered_character assert [result.id for result in await search_repository.search("适者生存")] == [1294] assert [result.id for result in await search_repository.search("OpenAI 适者生存")] == [1294] + assert [result.id for result in await search_repository.search("OpenAI and 适者生存")] == [1294] + assert [result.id for result in await search_repository.search("猫")] == [1294] assert await search_repository.search("适生者存") == [] +@pytest.mark.asyncio +async def test_mixed_word_and_script_search_preserves_fts_ranking(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1296, + type="entity", + file_path="notes/strong-match.md", + title="OpenAI OpenAI OpenAI", + content_stems="OpenAI 即适者生存", + content_snippet="OpenAI 即适者生存", + permalink="notes/strong-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1297, + type="entity", + file_path="notes/weaker-match.md", + title="Weaker match", + content_stems="OpenAI 即适者生存", + content_snippet="OpenAI 即适者生存", + permalink="notes/weaker-match", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("OpenAI 适者生存") + + assert [result.id for result in results] == [1296, 1297] + assert all(result.score != 0.0 for result in results) + + @pytest.mark.asyncio async def test_search_matches_script_text_beyond_the_parent_fts_limit(search_repository) -> None: now = datetime.now(timezone.utc) From c9cf725137810d01362400273d8b1dd5be1bda28 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 12:50:10 -0500 Subject: [PATCH 04/27] fix(core): preserve structured FTS semantics Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 2 +- .../repository/sqlite_search_repository.py | 20 ++++-- tests/repository/test_script_ngrams.py | 71 +++++++++++++++++++ 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 01235027f..65f403009 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -90,7 +90,7 @@ def analyze_script_query(text: str) -> ScriptQuery: normalized = unicodedata.normalize("NFKC", text) # Explicit Boolean expressions keep the existing backend parser. Splitting one # operand into a second SQL channel would otherwise change OR/NOT semantics. - if set(normalized.split()) & {"AND", "OR", "NOT"}: + if '"' in normalized or set(normalized.split()) & {"AND", "OR", "NOT"}: return ScriptQuery(word_text=normalized, gram_phrases=()) word_characters: list[str] = [] diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 80906af5e..ba54998d3 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -798,17 +798,23 @@ async def _build_fts_query_parts( pass else: script_query = analyze_script_query(search_text.strip()) + params["text"] = "" + params["script_text"] = "" if script_query.word_text: # Use _prepare_search_term to handle both Boolean and non-Boolean queries. prepared_text = self._prepare_search_term(script_query.word_text) params["text"] = f"{SQLITE_WORD_COLUMNS}: ({prepared_text})" - # content_stems is capped for Postgres index-row compatibility, while - # SQLite stores the complete note body in its FTS5 content_snippet column. - match_conditions.append("search_index MATCH :text") - for index, phrase in enumerate(script_query.gram_phrases): - param_name = f"script_phrase_{index}" - params[param_name] = f'"{" ".join(phrase)}"' - match_conditions.append(f"search_index.script_ngrams MATCH :{param_name}") + if script_query.gram_phrases: + script_phrases = " AND ".join( + f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases + ) + script_clause = f"script_ngrams: ({script_phrases})" + params["script_text"] = ( + f" AND ({script_clause})" if script_query.word_text else script_clause + ) + # One table-level MATCH keeps every required word and script phrase in the + # active FTS5 context so bm25 ranks the complete natural-language query. + match_conditions.append("search_index MATCH (:text || :script_text)") # Handle title match search if title: diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index ace958bbf..fd18b4bd5 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -11,6 +11,7 @@ script_runs, ) from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository @pytest.mark.parametrize( @@ -71,6 +72,13 @@ def test_analyze_script_query_treats_lowercase_boolean_words_as_natural_language assert query.gram_phrases == (("适者", "者生", "生存"),) +def test_analyze_script_query_preserves_quoted_mixed_script_semantics() -> None: + query = analyze_script_query('"OpenAI 适者生存"') + + assert query.word_text == '"OpenAI 适者生存"' + assert query.gram_phrases == () + + @pytest.mark.asyncio async def test_search_matches_cjk_substring_without_matching_reordered_characters( search_repository, @@ -134,6 +142,69 @@ async def test_mixed_word_and_script_search_preserves_fts_ranking(search_reposit assert all(result.score != 0.0 for result in results) +@pytest.mark.asyncio +async def test_search_ranking_includes_every_script_run(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1300, + type="entity", + file_path="notes/multiple-runs.md", + title="Multiple runs", + content_stems="适者 生存 生存", + content_snippet="适者 生存 生存", + permalink="notes/multiple-runs", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + first_run_results = await search_repository.search("适者") + all_run_results = await search_repository.search("适者 生存") + + assert [result.id for result in all_run_results] == [1300] + assert all_run_results[0].score != first_run_results[0].score + + +@pytest.mark.asyncio +async def test_quoted_mixed_script_search_preserves_phrase_adjacency(search_repository) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific quoted FTS5 regression") + + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1298, + type="entity", + file_path="notes/adjacent.md", + title="Adjacent", + content_stems="OpenAI 适者生存", + content_snippet="OpenAI 适者生存", + permalink="notes/adjacent", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1299, + type="entity", + file_path="notes/separated.md", + title="Separated", + content_stems="OpenAI words far away from 适者生存", + content_snippet="OpenAI words far away from 适者生存", + permalink="notes/separated", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search('"OpenAI 适者生存"') + + assert [result.id for result in results] == [1298] + + @pytest.mark.asyncio async def test_search_matches_script_text_beyond_the_parent_fts_limit(search_repository) -> None: now = datetime.now(timezone.utc) From 8d568e84761241414e1b71b010689d1ffcd1a45d Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 12:57:37 -0500 Subject: [PATCH 05/27] fix(core): preserve word-query compatibility forms Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 7 +++-- tests/repository/test_script_ngrams.py | 31 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 65f403009..b613d0b50 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -91,14 +91,15 @@ def analyze_script_query(text: str) -> ScriptQuery: # Explicit Boolean expressions keep the existing backend parser. Splitting one # operand into a second SQL channel would otherwise change OR/NOT semantics. if '"' in normalized or set(normalized.split()) & {"AND", "OR", "NOT"}: - return ScriptQuery(word_text=normalized, gram_phrases=()) + return ScriptQuery(word_text=text, gram_phrases=()) word_characters: list[str] = [] in_script_run = False - for character in normalized: + for character in text: if in_script_run and unicodedata.category(character) in {"Mn", "Mc", "Me"}: continue - if is_script_search_character(character): + normalized_character = unicodedata.normalize("NFKC", character) + if any(is_script_search_character(unit) for unit in normalized_character): if not in_script_run: word_characters.append(" ") in_script_run = True diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index fd18b4bd5..37236914d 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -79,6 +79,13 @@ def test_analyze_script_query_preserves_quoted_mixed_script_semantics() -> None: assert query.gram_phrases == () +def test_analyze_script_query_preserves_compatibility_characters_in_word_text() -> None: + query = analyze_script_query("ＡＢＣ finance 适者生存") + + assert query.word_text == "ＡＢＣ finance" + assert query.gram_phrases == (("适者", "者生", "生存"),) + + @pytest.mark.asyncio async def test_search_matches_cjk_substring_without_matching_reordered_characters( search_repository, @@ -205,6 +212,30 @@ async def test_quoted_mixed_script_search_preserves_phrase_adjacency(search_repo assert [result.id for result in results] == [1298] +@pytest.mark.asyncio +async def test_word_search_preserves_nfkc_sensitive_compatibility_characters( + search_repository, +) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1301, + type="entity", + file_path="notes/compatibility.md", + title="Compatibility", + content_stems="ＡＢＣ finance", + content_snippet="ＡＢＣ finance", + permalink="notes/compatibility", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("ＡＢＣ finance") + + assert [result.id for result in results] == [1301] + + @pytest.mark.asyncio async def test_search_matches_script_text_beyond_the_parent_fts_limit(search_repository) -> None: now = datetime.now(timezone.utc) From dc91a736d29a7219f73e7df8b09d361744bc884e Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 13:07:21 -0500 Subject: [PATCH 06/27] fix(core): preserve word-only search ranking Signed-off-by: phernandez --- .../repository/sqlite_search_repository.py | 44 ++++++++++++++----- tests/repository/test_search_repository.py | 26 +++++++++++ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index ba54998d3..1a8404ffb 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -798,13 +798,16 @@ async def _build_fts_query_parts( pass else: script_query = analyze_script_query(search_text.strip()) - params["text"] = "" - params["script_text"] = "" - if script_query.word_text: - # Use _prepare_search_term to handle both Boolean and non-Boolean queries. - prepared_text = self._prepare_search_term(script_query.word_text) - params["text"] = f"{SQLITE_WORD_COLUMNS}: ({prepared_text})" + # Trigger: the query contains text from an unsegmented script. + # Why: the script channel needs one table-level MATCH alongside word fields. + # Outcome: mixed queries rank all terms together; word-only queries retain their + # established per-column matching and ranking behavior. if script_query.gram_phrases: + params["text"] = "" + params["script_text"] = "" + if script_query.word_text: + prepared_text = self._prepare_search_term(script_query.word_text) + params["text"] = f"{SQLITE_WORD_COLUMNS}: ({prepared_text})" script_phrases = " AND ".join( f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases ) @@ -812,9 +815,22 @@ async def _build_fts_query_parts( params["script_text"] = ( f" AND ({script_clause})" if script_query.word_text else script_clause ) - # One table-level MATCH keeps every required word and script phrase in the - # active FTS5 context so bm25 ranks the complete natural-language query. - match_conditions.append("search_index MATCH (:text || :script_text)") + match_conditions.append("search_index MATCH (:text || :script_text)") + else: + word_text = ( + script_query.word_text + if script_query.word_text is not None + else search_text.strip() + ) + processed_text = self._prepare_search_term(word_text) + params["text"] = processed_text + # content_stems is capped for Postgres index-row compatibility, while + # SQLite stores the complete note body in its FTS5 content_snippet column. + match_conditions.append( + "(search_index.title MATCH :text OR " + "search_index.content_stems MATCH :text OR " + "search_index.content_snippet MATCH :text)" + ) # Handle title match search if title: @@ -1117,7 +1133,9 @@ async def run_search(active_session: AsyncSession): relaxed = self._relaxed_fts_text(search_text) if allow_relaxed and not rows else None if relaxed and params.get("text"): relaxed_fallback_used = True - params["text"] = f"{SQLITE_WORD_COLUMNS}: ({relaxed})" + params["text"] = ( + f"{SQLITE_WORD_COLUMNS}: ({relaxed})" if "script_text" in params else relaxed + ) logger.debug( "Strict SQLite FTS returned 0 results; retrying relaxed FTS query " f"strict='{search_text}' relaxed='{relaxed}'" @@ -1233,7 +1251,11 @@ async def count( self._relaxed_fts_text(search_text) if allow_relaxed and total == 0 else None ) if relaxed and params.get("text"): - params["text"] = f"{SQLITE_WORD_COLUMNS}: ({relaxed})" + params["text"] = ( + f"{SQLITE_WORD_COLUMNS}: ({relaxed})" + if "script_text" in params + else relaxed + ) with logfire.span( "search.count.relaxed_fts_retry", backend="sqlite", diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 933ae45ea..ee9d40c8b 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -282,6 +282,32 @@ async def test_sqlite_text_search_matches_full_content_snippet(search_repository assert await search_repository.count(search_text=marker) == 1 +@pytest.mark.asyncio +async def test_sqlite_word_query_keeps_terms_in_one_search_column(search_repository, search_entity): + """Adding the script channel must not broaden established word-query matches.""" + if is_postgres_backend(search_repository): + pytest.skip("SQLite's per-column FTS matching is backend-specific") + + search_row = SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="alpha", + content_stems="beta", + content_snippet="beta", + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + project_id=search_repository.project_id, + ) + await search_repository.index_item(search_row) + + assert await search_repository.search(search_text="alpha beta") == [] + assert await search_repository.count(search_text="alpha beta") == 0 + + @pytest.mark.asyncio async def test_index_item_upsert_on_duplicate_permalink(search_repository, search_entity): """Test that indexing the same permalink twice uses upsert instead of failing. From eda1885c5af54fc9c9fb6894559e4772b2c4d118 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 13:13:04 -0500 Subject: [PATCH 07/27] fix(core): cover script continuations and ranking Signed-off-by: phernandez --- .../repository/postgres_search_repository.py | 4 +- src/basic_memory/repository/script_ngrams.py | 4 ++ tests/repository/test_script_ngrams.py | 50 ++++++++++++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 542e352c8..c235f2877 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -1295,7 +1295,9 @@ async def _build_fts_query_parts( f"@@ to_tsquery('simple', :script_text_{index})), 0))" for index in range(len(script_tsqueries)) ) - score_expr = f"GREATEST({', '.join(score_parts)})" if score_parts else "0" + # Each condition above is required, so every query component should contribute to + # relevance. Taking only the strongest rank makes additional script runs invisible. + score_expr = " + ".join(score_parts) if score_parts else "0" return from_clause, where_clause, params, order_by_clause, score_expr diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index b613d0b50..3c53b42b0 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -26,6 +26,10 @@ def is_script_search_character(character: str) -> bool: (0x1000, 0x109F), # Myanmar (0x1780, 0x17FF), # Khmer (0x19E0, 0x19FF), # Khmer symbols + (0x3005, 0x3007), # Ideographic iteration, closing, and zero marks + (0x3021, 0x3029), # Hangzhou numerals + (0x3031, 0x3035), # Vertical kana repeat marks + (0x3038, 0x303B), # Hangzhou tens and vertical iteration mark (0x3040, 0x30FF), # Hiragana and Katakana (0x3100, 0x318F), # Bopomofo and Hangul compatibility Jamo (0x31A0, 0x31BF), # Bopomofo extended diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 37236914d..fb5a25742 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -10,6 +10,7 @@ script_run_grams, script_runs, ) +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository @@ -22,6 +23,7 @@ ("サバイバル", (("サ", "バ", "イ", "バ", "ル"),)), ("생존 경쟁", (("생", "존"), ("경", "쟁"))), ("ภาษาไทย", (("ภ", "า", "ษ", "า", "ไ", "ท", "ย"),)), + ("時々", (("時", "々"),)), ("ＡＢＣ", ()), ], ) @@ -97,8 +99,8 @@ async def test_search_matches_cjk_substring_without_matching_reordered_character type="entity", file_path="notes/evolution.md", title="進化について", - content_stems="OpenAI and 即适者生存的讨论与黑猫", - content_snippet="OpenAI and 即适者生存的讨论与黑猫", + content_stems="OpenAI and 即适者生存的讨论与黑猫，時々更新", + content_snippet="OpenAI and 即适者生存的讨论与黑猫，時々更新", permalink="notes/evolution", created_at=now, updated_at=now, @@ -109,6 +111,7 @@ async def test_search_matches_cjk_substring_without_matching_reordered_character assert [result.id for result in await search_repository.search("OpenAI 适者生存")] == [1294] assert [result.id for result in await search_repository.search("OpenAI and 适者生存")] == [1294] assert [result.id for result in await search_repository.search("猫")] == [1294] + assert [result.id for result in await search_repository.search("時々")] == [1294] assert await search_repository.search("适生者存") == [] @@ -173,6 +176,49 @@ async def test_search_ranking_includes_every_script_run(search_repository) -> No assert all_run_results[0].score != first_run_results[0].score +@pytest.mark.asyncio +async def test_postgres_ranking_adds_contributions_from_every_script_run( + search_repository, +) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL combines independently ranked script phrases") + + now = datetime.now(timezone.utc) + shared_first_run = "適者 " * 5 + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1302, + type="entity", + file_path="notes/one-secondary-match.md", + title="One secondary match", + content_stems=f"{shared_first_run}生存", + content_snippet=f"{shared_first_run}生存", + permalink="notes/one-secondary-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1303, + type="entity", + file_path="notes/many-secondary-matches.md", + title="Many secondary matches", + content_stems=f"{shared_first_run}{'生存 ' * 5}", + content_snippet=f"{shared_first_run}{'生存 ' * 5}", + permalink="notes/many-secondary-matches", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("適者 生存") + + assert [result.id for result in results] == [1303, 1302] + assert results[0].score > results[1].score + + @pytest.mark.asyncio async def test_quoted_mixed_script_search_preserves_phrase_adjacency(search_repository) -> None: if not isinstance(search_repository, SQLiteSearchRepository): From 22f752f854014399e7da34e1c3d142f275240eb7 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 13:17:00 -0500 Subject: [PATCH 08/27] fix(core): handle mixed stopword script queries Signed-off-by: phernandez --- .../repository/postgres_search_repository.py | 74 +++++++++++-------- tests/repository/test_script_ngrams.py | 25 +++++++ 2 files changed, 67 insertions(+), 32 deletions(-) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index c235f2877..ebdfacb43 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -1058,7 +1058,16 @@ async def _build_fts_query_parts( AND fts_candidate.type = search_index.type """ document_vector_sql = self._document_fts_vector_sql(probe_texts, params) - conditions.append(f"{document_vector_sql} @@ to_tsquery('english', :text)") + word_condition = f"{document_vector_sql} @@ to_tsquery('english', :text)" + if script_query.gram_phrases: + # Trigger: PostgreSQL's English dictionary removes every word term. + # Why: an empty word query must not suppress a required script match. + # Outcome: only mixed queries treat the empty word channel as neutral; + # word-only stopword queries retain their established empty result. + word_condition = ( + f"(numnode(to_tsquery('english', :text)) = 0 OR {word_condition})" + ) + conditions.append(word_condition) if script_query.gram_phrases: script_tsqueries = [ @@ -1067,37 +1076,38 @@ async def _build_fts_query_parts( ] for index, script_tsquery in enumerate(script_tsqueries): params[f"script_text_{index}"] = script_tsquery - if document_vector_sql is None: - # Trigger: a query contains script grams but no word terms. - # Why: a correlated predicate alone can scan every tenant row. - # Outcome: start from the parent and child GIN indexes. - params["script_candidate_text"] = " | ".join( - f"({script_tsquery})" for script_tsquery in script_tsqueries - ) - from_clause = """ - search_index JOIN ( - SELECT - script_parent.project_id, - script_parent.id, - script_parent.type - FROM search_index AS script_parent - WHERE script_parent.project_id = :project_id - AND script_parent.script_ngrams_index_col - @@ to_tsquery('simple', :script_candidate_text) - UNION - SELECT - script_candidate.project_id, - script_candidate.search_index_id AS id, - script_candidate.search_index_type AS type - FROM search_index_fts_chunks AS script_candidate - WHERE script_candidate.project_id = :project_id - AND script_candidate.script_ngrams_index_col - @@ to_tsquery('simple', :script_candidate_text) - ) AS fts_candidate - ON fts_candidate.project_id = search_index.project_id - AND fts_candidate.id = search_index.id - AND fts_candidate.type = search_index.type - """ + # Trigger: a query contains script grams, with or without word terms. + # Why: every script phrase is required, while an English word clause can + # reduce to an empty tsquery after dictionary processing. + # Outcome: start from the parent and child script GIN indexes, then apply + # every word and script predicate below. + params["script_candidate_text"] = " | ".join( + f"({script_tsquery})" for script_tsquery in script_tsqueries + ) + from_clause = """ + search_index JOIN ( + SELECT + script_parent.project_id, + script_parent.id, + script_parent.type + FROM search_index AS script_parent + WHERE script_parent.project_id = :project_id + AND script_parent.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + UNION + SELECT + script_candidate.project_id, + script_candidate.search_index_id AS id, + script_candidate.search_index_type AS type + FROM search_index_fts_chunks AS script_candidate + WHERE script_candidate.project_id = :project_id + AND script_candidate.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + ) AS fts_candidate + ON fts_candidate.project_id = search_index.project_id + AND fts_candidate.id = search_index.id + AND fts_candidate.type = search_index.type + """ conditions.extend( "(search_index.script_ngrams_index_col " f"@@ to_tsquery('simple', :script_text_{index}) OR EXISTS (" diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index fb5a25742..0c5e43614 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -219,6 +219,31 @@ async def test_postgres_ranking_adds_contributions_from_every_script_run( assert results[0].score > results[1].score +@pytest.mark.asyncio +async def test_postgres_mixed_search_ignores_empty_stopword_query(search_repository) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL's English dictionary removes stopwords") + + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1304, + type="entity", + file_path="notes/stopword-and-script.md", + title="Script match", + content_stems="適者生存", + content_snippet="適者生存", + permalink="notes/stopword-and-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("the 適者生存") + + assert [result.id for result in results] == [1304] + + @pytest.mark.asyncio async def test_quoted_mixed_script_search_preserves_phrase_adjacency(search_repository) -> None: if not isinstance(search_repository, SQLiteSearchRepository): From 39de387cbc838f7979ce8c8224e25ebccb6ca68b Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 13:46:08 -0500 Subject: [PATCH 09/27] fix(core): preserve SQLite script query semantics Signed-off-by: phernandez --- .../repository/sqlite_search_repository.py | 51 +++++++++++++++---- tests/repository/test_script_ngrams.py | 39 ++++++++++++++ .../test_search_text_with_metadata_filters.py | 32 ++++++++++++ 3 files changed, 111 insertions(+), 11 deletions(-) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 1a8404ffb..494480fde 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -782,13 +782,15 @@ async def _build_fts_query_parts( search_item_types: Optional[List[SearchItemType]] = None, categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, - ) -> tuple[str, str, dict[str, Any], str]: + ) -> tuple[str, str, dict[str, Any], str, str]: """Build SQLite FTS FROM/WHERE params shared by search and count.""" conditions = [] match_conditions = [] params = {} order_by_clause = "" from_clause = "search_index" + score_expression = "bm25(search_index)" + preserve_match_score = False # Handle text search for title and content if search_text: @@ -803,11 +805,16 @@ async def _build_fts_query_parts( # Outcome: mixed queries rank all terms together; word-only queries retain their # established per-column matching and ranking behavior. if script_query.gram_phrases: + preserve_match_score = True params["text"] = "" params["script_text"] = "" if script_query.word_text: prepared_text = self._prepare_search_term(script_query.word_text) - params["text"] = f"{SQLITE_WORD_COLUMNS}: ({prepared_text})" + params["text"] = ( + f"(title: ({prepared_text}) OR " + f"content_stems: ({prepared_text}) OR " + f"content_snippet: ({prepared_text}))" + ) script_phrases = " AND ".join( f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases ) @@ -1007,13 +1014,23 @@ async def _build_fts_query_parts( # Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with # "unable to use function MATCH in the requested context". - # Why: MATCH needs to run in an FTS-valid context. - # Outcome: evaluate MATCH clauses in an FTS subquery and filter outer rows by rowid. + # Why: script queries need MATCH and bm25 together for ranking, while legacy + # word-column OR predicates cannot evaluate bm25 in the same derived query. + # Outcome: rank script matches before joining metadata; retain the established + # rowid-filter path for word-only searches. if metadata_filters and match_conditions: match_where = " AND ".join(match_conditions) - conditions.append( - f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})" - ) + if preserve_match_score: + from_clause = ( + "(SELECT search_index.*, bm25(search_index) AS fts_score " + f"FROM search_index WHERE {match_where}) AS search_index " + "JOIN entity ON search_index.entity_id = entity.id" + ) + score_expression = "search_index.fts_score" + else: + conditions.append( + f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})" + ) else: conditions.extend(match_conditions) @@ -1023,7 +1040,7 @@ async def _build_fts_query_parts( # Build WHERE clause where_clause = " AND ".join(conditions) if conditions else "1=1" - return from_clause, where_clause, params, order_by_clause + return from_clause, where_clause, params, order_by_clause, score_expression @override async def search( @@ -1074,7 +1091,13 @@ async def search( return dispatched # --- FTS mode (SQLite-specific) --- - from_clause, where_clause, params, order_by_clause = await self._build_fts_query_parts( + ( + from_clause, + where_clause, + params, + order_by_clause, + score_expression, + ) = await self._build_fts_query_parts( search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1107,7 +1130,7 @@ async def search( search_index.category, search_index.created_at, search_index.updated_at, - bm25(search_index) as score + {score_expression} as score FROM {from_clause} WHERE {where_clause} ORDER BY score ASC {order_by_clause} @@ -1230,7 +1253,13 @@ async def count( min_similarity=min_similarity, ) - from_clause, where_clause, params, _order_by_clause = await self._build_fts_query_parts( + ( + from_clause, + where_clause, + params, + _order_by_clause, + _score_expression, + ) = await self._build_fts_query_parts( search_text=search_text, permalink=permalink, permalink_match=permalink_match, diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 0c5e43614..95ba94624 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -152,6 +152,45 @@ async def test_mixed_word_and_script_search_preserves_fts_ranking(search_reposit assert all(result.score != 0.0 for result in results) +@pytest.mark.asyncio +async def test_sqlite_mixed_search_requires_all_words_in_one_column(search_repository) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite preserves its established per-column word matching") + + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1305, + type="entity", + file_path="notes/same-column.md", + title="Same column", + content_stems="alpha beta 適者生存", + content_snippet="alpha beta 適者生存", + permalink="notes/same-column", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1306, + type="entity", + file_path="notes/split-columns.md", + title="alpha", + content_stems="beta 適者生存", + content_snippet="beta 適者生存", + permalink="notes/split-columns", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("alpha beta 適者生存") + + assert [result.id for result in results] == [1305] + + @pytest.mark.asyncio async def test_search_ranking_includes_every_script_run(search_repository) -> None: now = datetime.now(timezone.utc) diff --git a/tests/repository/test_search_text_with_metadata_filters.py b/tests/repository/test_search_text_with_metadata_filters.py index 7ac878ab3..6eef72b10 100644 --- a/tests/repository/test_search_text_with_metadata_filters.py +++ b/tests/repository/test_search_text_with_metadata_filters.py @@ -7,6 +7,7 @@ from basic_memory import db from basic_memory.models.knowledge import Entity from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository from basic_memory.schemas.search import SearchItemType @@ -63,3 +64,34 @@ async def test_search_text_and_metadata_filters_work_together(search_repository, ) assert {row.id for row in results} == {active.id} + + +@pytest.mark.asyncio +async def test_sqlite_script_search_with_metadata_filters_preserves_ranking( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific FTS5 ranking regression") + + stronger = await _index_entity( + search_repository, + session_maker, + "適者生存 適者生存 適者生存", + "active", + ) + weaker = await _index_entity( + search_repository, + session_maker, + "適者生存", + "active", + ) + + results = await search_repository.search( + search_text="適者生存", + metadata_filters={"status": "active"}, + ) + + assert [row.id for row in results] == [stronger.id, weaker.id] + assert results[0].score != results[1].score + assert all(row.score != 0.0 for row in results) From 5c8ab1e48356f7c5cca6c7bfb9a0b3ef6b2b6fc2 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 14:16:20 -0500 Subject: [PATCH 10/27] fix(core): index accepted-note script grams Signed-off-by: phernandez --- .../accepted_note_search_repository.py | 21 +++++++--- .../test_accepted_note_search_repository.py | 41 ++++++++++++++++++- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/repository/accepted_note_search_repository.py b/src/basic_memory/repository/accepted_note_search_repository.py index 30d06bdd9..0ecbc6257 100644 --- a/src/basic_memory/repository/accepted_note_search_repository.py +++ b/src/basic_memory/repository/accepted_note_search_repository.py @@ -13,6 +13,7 @@ ProjectIndexExternalVectorCleaner, delete_project_index_vector_rows, ) +from basic_memory.repository.script_ngrams import build_script_ngrams type SearchIndexSqlValue = str | int | datetime | None type SearchIndexSqlParams = dict[str, SearchIndexSqlValue] @@ -28,14 +29,15 @@ INSERT_ACCEPTED_NOTE_SEARCH_SQL = text( """ INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, + permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, - :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, + :permalink, :file_path, :type, :metadata, NULL, NULL, NULL, :entity_id, NULL, :created_at, :updated_at, @@ -47,14 +49,15 @@ UPSERT_ACCEPTED_NOTE_SEARCH_SQL = text( """ INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, + permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, - CAST(:metadata AS jsonb), + :id, :title, :content_stems, :content_snippet, :script_ngrams, + :permalink, :file_path, :type, CAST(:metadata AS jsonb), NULL, NULL, NULL, :entity_id, NULL, :created_at, :updated_at, @@ -65,6 +68,7 @@ title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + script_ngrams = EXCLUDED.script_ngrams, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -95,6 +99,11 @@ def accepted_note_search_insert_params( "title": row.title, "content_stems": row.content_stems, "content_snippet": row.content_snippet, + "script_ngrams": build_script_ngrams( + row.title, + row.content_stems, + row.content_snippet, + ), "permalink": row.permalink, "file_path": row.file_path, "type": row.item_type, diff --git a/tests/repository/test_accepted_note_search_repository.py b/tests/repository/test_accepted_note_search_repository.py index bbdf3bf0d..73981aa11 100644 --- a/tests/repository/test_accepted_note_search_repository.py +++ b/tests/repository/test_accepted_note_search_repository.py @@ -6,10 +6,12 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession +from basic_memory import db from basic_memory.indexing.accepted_note_search import build_accepted_note_search_row from basic_memory.repository.accepted_note_search_repository import ( AcceptedNoteSearchRepository, ) +from basic_memory.repository.script_ngrams import build_script_ngrams class _Dialect: @@ -47,7 +49,7 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: entity_metadata={"tags": ["strategy"]}, permalink="main/project-plan", file_path="notes/project-plan.md", - search_content="Main body", + search_content="Main body 适者生存", created_at=created_at, updated_at=updated_at, project_id=7, @@ -62,11 +64,17 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: assert delete_params == {"entity_id": 42, "project_id": 7} assert "CAST(:metadata AS jsonb)" in insert_sql assert "ON CONFLICT (permalink, project_id)" in insert_sql + assert "script_ngrams = EXCLUDED.script_ngrams" in insert_sql assert insert_params == { "id": 42, "title": "Project Plan", "content_stems": row.content_stems, - "content_snippet": "Main body", + "content_snippet": "Main body 适者生存", + "script_ngrams": build_script_ngrams( + row.title, + row.content_stems, + row.content_snippet, + ), "permalink": "main/project-plan", "file_path": "notes/project-plan.md", "type": "entity", @@ -102,6 +110,7 @@ async def test_refresh_entity_uses_plain_insert_for_sqlite_virtual_table() -> No assert "ON CONFLICT" not in insert_sql assert "CAST(:metadata AS jsonb)" not in insert_sql assert ":metadata" in insert_sql + assert ":script_ngrams" in insert_sql @pytest.mark.asyncio @@ -126,3 +135,31 @@ async def test_refresh_entity_rejects_cross_project_rows() -> None: await repository.refresh_entity(cast(AsyncSession, session), row) assert session.executed == [] + + +@pytest.mark.asyncio +async def test_refresh_entity_is_immediately_searchable_by_script_substring( + search_repository, + session_maker, +) -> None: + repository = AcceptedNoteSearchRepository(project_id=search_repository.project_id) + now = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + row = build_accepted_note_search_row( + entity_id=42, + title="Evolution", + note_type="note", + entity_metadata=None, + permalink="main/evolution", + file_path="notes/evolution.md", + search_content="即适者生存的讨论", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, row) + + results = await search_repository.search("适者生存") + + assert [result.id for result in results] == [42] From b99ba4c19388a026bc6520c6dbb9213d27c1a6d7 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 14:48:53 -0500 Subject: [PATCH 11/27] fix(core): bound accepted-note script indexing Signed-off-by: phernandez --- .../accepted_note_search_repository.py | 63 +++++++++++++++++-- .../repository/postgres_fts_chunks.py | 19 ++++++ .../repository/postgres_search_repository.py | 20 +----- src/basic_memory/repository/script_ngrams.py | 4 +- .../test_accepted_note_search_repository.py | 53 ++++++++++++++-- tests/repository/test_script_ngrams.py | 29 +++++++++ 6 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 src/basic_memory/repository/postgres_fts_chunks.py diff --git a/src/basic_memory/repository/accepted_note_search_repository.py b/src/basic_memory/repository/accepted_note_search_repository.py index 0ecbc6257..d74974a17 100644 --- a/src/basic_memory/repository/accepted_note_search_repository.py +++ b/src/basic_memory/repository/accepted_note_search_repository.py @@ -13,6 +13,7 @@ ProjectIndexExternalVectorCleaner, delete_project_index_vector_rows, ) +from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks from basic_memory.repository.script_ngrams import build_script_ngrams type SearchIndexSqlValue = str | int | datetime | None @@ -82,6 +83,33 @@ """ ) +INSERT_ACCEPTED_NOTE_FTS_CHUNKS_SQL = text( + """ + INSERT INTO search_index_fts_chunks ( + project_id, + search_index_id, + search_index_type, + chunk_index, + chunk_text, + script_ngrams + ) + SELECT + :project_id, + chunk.search_index_id, + chunk.search_index_type, + chunk.chunk_index, + chunk.chunk_text, + chunk.script_ngrams + FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk( + search_index_id INTEGER, + search_index_type VARCHAR, + chunk_index INTEGER, + chunk_text TEXT, + script_ngrams TEXT + ) + """ +) + def accepted_note_search_insert_statement(session: AsyncSession): """Return the insert statement supported by the active search table backend.""" @@ -92,18 +120,19 @@ def accepted_note_search_insert_statement(session: AsyncSession): def accepted_note_search_insert_params( row: AcceptedNoteSearchRow, + *, + include_full_content_grams: bool, ) -> SearchIndexSqlParams: """Build SQL parameters for one accepted-note search row.""" + script_texts = (row.title, row.content_stems) + if include_full_content_grams: + script_texts = (*script_texts, row.content_snippet) return { "id": row.id, "title": row.title, "content_stems": row.content_stems, "content_snippet": row.content_snippet, - "script_ngrams": build_script_ngrams( - row.title, - row.content_stems, - row.content_snippet, - ), + "script_ngrams": build_script_ngrams(*script_texts), "permalink": row.permalink, "file_path": row.file_path, "type": row.item_type, @@ -143,10 +172,32 @@ async def refresh_entity( DELETE_ACCEPTED_NOTE_SEARCH_SQL, {"entity_id": row.entity_id, "project_id": row.project_id}, ) + is_sqlite = session.get_bind().dialect.name == "sqlite" await session.execute( accepted_note_search_insert_statement(session), - accepted_note_search_insert_params(row), + accepted_note_search_insert_params( + row, + include_full_content_grams=is_sqlite, + ), ) + if is_sqlite: + return + + chunks = [ + { + "search_index_id": row.id, + "search_index_type": row.item_type, + "chunk_index": chunk_index, + "chunk_text": chunk_text, + "script_ngrams": build_script_ngrams(chunk_text), + } + for chunk_index, chunk_text in split_postgres_fts_chunks(row.content_snippet) + ] + if chunks: + await session.execute( + INSERT_ACCEPTED_NOTE_FTS_CHUNKS_SQL, + {"project_id": row.project_id, "chunks": json.dumps(chunks)}, + ) async def delete_entity( self, diff --git a/src/basic_memory/repository/postgres_fts_chunks.py b/src/basic_memory/repository/postgres_fts_chunks.py new file mode 100644 index 000000000..7cea623e2 --- /dev/null +++ b/src/basic_memory/repository/postgres_fts_chunks.py @@ -0,0 +1,19 @@ +"""Bounded PostgreSQL full-text search chunks.""" + +POSTGRES_FTS_CHUNK_SIZE = 8_000 +# PostgreSQL ignores lexemes at 2 KiB and above. A 2,048-character overlap is +# therefore conservative for every indexable lexeme, including multi-byte text: +# any token split at one 8,000-character edge is complete in the next chunk. +POSTGRES_FTS_CHUNK_OVERLAP = 2_048 + + +def split_postgres_fts_chunks(content: str | None) -> list[tuple[int, str]]: + """Split full note text without losing an indexable lexeme at a chunk edge.""" + if not content: + return [] + + step = POSTGRES_FTS_CHUNK_SIZE - POSTGRES_FTS_CHUNK_OVERLAP + return [ + (chunk_index, content[start : start + POSTGRES_FTS_CHUNK_SIZE]) + for chunk_index, start in enumerate(range(0, len(content), step)) + ] diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index ebdfacb43..85e6d2e70 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -43,14 +43,10 @@ resolve_semantic_vector_index_name, ) from basic_memory.repository.pgvector_index import PgVectorIndex +from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode -POSTGRES_FTS_CHUNK_SIZE = 8_000 -# PostgreSQL ignores lexemes at 2 KiB and above. A 2,048-character overlap is -# therefore conservative for every indexable lexeme, including multi-byte text: -# any token split at one 8,000-character edge is complete in the next chunk. -POSTGRES_FTS_CHUNK_OVERLAP = 2_048 _TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") _TSQUERY_WORD_PATTERN = re.compile(r"[^\W_]+(?:'[^\W_]+)?", re.UNICODE) _QUOTED_QUERY_PATTERN = re.compile(r'"([^"]*)"') @@ -58,18 +54,6 @@ _TSQUERY_METACHARACTERS = frozenset("&|!:<>") -def _iter_fts_chunks(content: str | None) -> list[tuple[int, str]]: - """Split full note text without losing an indexable lexeme at a chunk edge.""" - if not content: - return [] - - step = POSTGRES_FTS_CHUNK_SIZE - POSTGRES_FTS_CHUNK_OVERLAP - return [ - (chunk_index, content[start : start + POSTGRES_FTS_CHUNK_SIZE]) - for chunk_index, start in enumerate(range(0, len(content), step)) - ] - - def _tsquery_operands(processed_text: str) -> list[tuple[str, str]]: """Return unique (query operand, representative text) pairs in source order.""" operands: dict[str, str] = {} @@ -368,7 +352,7 @@ async def _replace_fts_chunks( "script_ngrams": build_script_ngrams(chunk_text.replace("\x00", "")), } for row in search_index_rows - for chunk_index, chunk_text in _iter_fts_chunks(row.content_snippet) + for chunk_index, chunk_text in split_postgres_fts_chunks(row.content_snippet) ] if not chunks: return diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 3c53b42b0..748178874 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -94,7 +94,9 @@ def analyze_script_query(text: str) -> ScriptQuery: normalized = unicodedata.normalize("NFKC", text) # Explicit Boolean expressions keep the existing backend parser. Splitting one # operand into a second SQL channel would otherwise change OR/NOT semantics. - if '"' in normalized or set(normalized.split()) & {"AND", "OR", "NOT"}: + # Compatibility forms are natural-language text because neither backend parses + # their normalized equivalents as operators. + if '"' in text or set(text.split()) & {"AND", "OR", "NOT"}: return ScriptQuery(word_text=text, gram_phrases=()) word_characters: list[str] = [] diff --git a/tests/repository/test_accepted_note_search_repository.py b/tests/repository/test_accepted_note_search_repository.py index 73981aa11..f8a8a4151 100644 --- a/tests/repository/test_accepted_note_search_repository.py +++ b/tests/repository/test_accepted_note_search_repository.py @@ -1,5 +1,6 @@ """Tests for accepted-note search repository operations.""" +import json from datetime import UTC, datetime from typing import Any, cast @@ -12,6 +13,7 @@ AcceptedNoteSearchRepository, ) from basic_memory.repository.script_ngrams import build_script_ngrams +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository class _Dialect: @@ -57,7 +59,7 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: await repository.refresh_entity(cast(AsyncSession, session), row) - assert len(session.executed) == 2 + assert len(session.executed) == 3 delete_sql, delete_params = session.executed[0] insert_sql, insert_params = session.executed[1] assert "DELETE FROM search_index" in delete_sql @@ -70,11 +72,7 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: "title": "Project Plan", "content_stems": row.content_stems, "content_snippet": "Main body 适者生存", - "script_ngrams": build_script_ngrams( - row.title, - row.content_stems, - row.content_snippet, - ), + "script_ngrams": build_script_ngrams(row.title, row.content_stems), "permalink": "main/project-plan", "file_path": "notes/project-plan.md", "type": "entity", @@ -84,6 +82,18 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: "updated_at": updated_at, "project_id": 7, } + chunk_sql, chunk_params = session.executed[2] + assert "INSERT INTO search_index_fts_chunks" in chunk_sql + assert chunk_params["project_id"] == 7 + assert json.loads(chunk_params["chunks"]) == [ + { + "search_index_id": 42, + "search_index_type": "entity", + "chunk_index": 0, + "chunk_text": "Main body 适者生存", + "script_ngrams": build_script_ngrams("Main body 适者生存"), + } + ] @pytest.mark.asyncio @@ -163,3 +173,34 @@ async def test_refresh_entity_is_immediately_searchable_by_script_substring( results = await search_repository.search("适者生存") assert [result.id for result in results] == [42] + + +@pytest.mark.asyncio +async def test_postgres_refresh_entity_chunks_large_script_content( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL stores full note bodies in bounded FTS chunks") + + repository = AcceptedNoteSearchRepository(project_id=search_repository.project_id) + now = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + row = build_accepted_note_search_row( + entity_id=43, + title="Long evolution note", + note_type="note", + entity_metadata=None, + permalink="main/long-evolution", + file_path="notes/long-evolution.md", + search_content=f"{'進化' * 5_000}适者生存", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, row) + + results = await search_repository.search("适者生存") + + assert [result.id for result in results] == [43] diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 95ba94624..8c81c72b4 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -88,6 +88,35 @@ def test_analyze_script_query_preserves_compatibility_characters_in_word_text() assert query.gram_phrases == (("适者", "者生", "生存"),) +def test_analyze_script_query_treats_compatibility_boolean_text_as_natural_language() -> None: + query = analyze_script_query("适者 ＡＮＤ 生存") + + assert query.word_text == "ＡＮＤ" + assert query.gram_phrases == (("适者",), ("生存",)) + + +@pytest.mark.asyncio +async def test_compatibility_boolean_text_uses_script_substring_search(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1307, + type="entity", + file_path="notes/compatibility-boolean.md", + title="Compatibility Boolean", + content_stems="不适者 ＡＮＤ 生存者", + content_snippet="不适者 ＡＮＤ 生存者", + permalink="notes/compatibility-boolean", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("适者 ＡＮＤ 生存") + + assert [result.id for result in results] == [1307] + + @pytest.mark.asyncio async def test_search_matches_cjk_substring_without_matching_reordered_characters( search_repository, From d9f06474e9bf3ae853c12ef1e5e1127c03f735d8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 15:14:23 -0500 Subject: [PATCH 12/27] fix(core): align boolean whitespace handling Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 6 ++-- tests/repository/test_script_ngrams.py | 30 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 748178874..521343f09 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -96,7 +96,7 @@ def analyze_script_query(text: str) -> ScriptQuery: # operand into a second SQL channel would otherwise change OR/NOT semantics. # Compatibility forms are natural-language text because neither backend parses # their normalized equivalents as operators. - if '"' in text or set(text.split()) & {"AND", "OR", "NOT"}: + if '"' in text or any(f" {operator} " in text for operator in ("AND", "OR", "NOT")): return ScriptQuery(word_text=text, gram_phrases=()) word_characters: list[str] = [] @@ -113,8 +113,10 @@ def analyze_script_query(text: str) -> ScriptQuery: in_script_run = False word_characters.append(character) + # Operator-shaped tokens outside the literal-space syntax above are words. Lowercase + # keeps the case-insensitive FTS match while preventing reparsing after reconstruction. word_tokens = [ - token + token.lower() if token in {"AND", "OR", "NOT"} else token for token in "".join(word_characters).split() if any(character.isalnum() for character in token) ] diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 8c81c72b4..021b2fe2c 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -95,6 +95,14 @@ def test_analyze_script_query_treats_compatibility_boolean_text_as_natural_langu assert query.gram_phrases == (("适者",), ("生存",)) +@pytest.mark.parametrize("separator", ["\t", "\n"]) +def test_analyze_script_query_matches_backend_boolean_whitespace(separator: str) -> None: + query = analyze_script_query(f"适者{separator}AND{separator}生存") + + assert query.word_text == "and" + assert query.gram_phrases == (("适者",), ("生存",)) + + @pytest.mark.asyncio async def test_compatibility_boolean_text_uses_script_substring_search(search_repository) -> None: now = datetime.now(timezone.utc) @@ -117,6 +125,28 @@ async def test_compatibility_boolean_text_uses_script_substring_search(search_re assert [result.id for result in results] == [1307] +@pytest.mark.asyncio +async def test_non_space_boolean_text_uses_script_substring_search(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1308, + type="entity", + file_path="notes/non-space-boolean.md", + title="Non-space Boolean", + content_stems="不适者\tAND\t生存者", + content_snippet="不适者\tAND\t生存者", + permalink="notes/non-space-boolean", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("适者\tAND\t生存") + + assert [result.id for result in results] == [1308] + + @pytest.mark.asyncio async def test_search_matches_cjk_substring_without_matching_reordered_characters( search_repository, From 2eb08e7130df409e3b5070e7199b0a7f56da1619 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 15:43:04 -0500 Subject: [PATCH 13/27] fix(core): preserve filtered script candidates Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 7 +- .../repository/sqlite_search_repository.py | 3 +- tests/repository/test_script_ngrams.py | 83 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 521343f09..68f88d497 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -120,8 +120,11 @@ def analyze_script_query(text: str) -> ScriptQuery: for token in "".join(word_characters).split() if any(character.isalnum() for character in token) ] - word_text = " ".join(word_tokens) or None + gram_phrases = tuple(script_run_grams(run) for run in script_runs(normalized)) + # Preserve punctuation-only input as an explicit backend query. Dropping it would make + # the repositories confuse user text with the intentional no-predicate wildcard path. + word_text = " ".join(word_tokens) or (text if not gram_phrases else None) return ScriptQuery( word_text=word_text, - gram_phrases=tuple(script_run_grams(run) for run in script_runs(normalized)), + gram_phrases=gram_phrases, ) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 494480fde..e15a5e24a 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -1022,7 +1022,8 @@ async def _build_fts_query_parts( match_where = " AND ".join(match_conditions) if preserve_match_score: from_clause = ( - "(SELECT search_index.*, bm25(search_index) AS fts_score " + "(SELECT search_index.rowid AS rowid, search_index.*, " + "bm25(search_index) AS fts_score " f"FROM search_index WHERE {match_where}) AS search_index " "JOIN entity ON search_index.entity_id = entity.id" ) diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 021b2fe2c..26161a118 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -4,6 +4,8 @@ import pytest +from basic_memory import db +from basic_memory.models import Entity from basic_memory.repository.script_ngrams import ( analyze_script_query, build_script_ngrams, @@ -103,6 +105,14 @@ def test_analyze_script_query_matches_backend_boolean_whitespace(separator: str) assert query.gram_phrases == (("适者",), ("生存",)) +@pytest.mark.parametrize("text", ["!!!", "😀"]) +def test_analyze_script_query_preserves_punctuation_only_text(text: str) -> None: + query = analyze_script_query(text) + + assert query.word_text == text + assert query.gram_phrases == () + + @pytest.mark.asyncio async def test_compatibility_boolean_text_uses_script_substring_search(search_repository) -> None: now = datetime.now(timezone.utc) @@ -147,6 +157,79 @@ async def test_non_space_boolean_text_uses_script_substring_search(search_reposi assert [result.id for result in results] == [1308] +@pytest.mark.asyncio +@pytest.mark.parametrize("query", ["!!!", "😀"]) +async def test_punctuation_only_search_does_not_return_every_row( + search_repository, + query: str, +) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1310, + type="entity", + file_path="notes/punctuation-decoy.md", + title="Punctuation decoy", + content_stems="ordinary searchable words", + content_snippet="ordinary searchable words", + permalink="notes/punctuation-decoy", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + assert await search_repository.search(query) == [] + + +@pytest.mark.asyncio +async def test_sqlite_script_search_combines_metadata_and_title_filters( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific FTS5 rowid regression") + + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + entity = Entity( + project_id=search_repository.project_id, + title="Evolution match", + note_type="note", + permalink="notes/evolution-filtered", + file_path="notes/evolution-filtered.md", + content_type="text/markdown", + entity_metadata={"region": "asia"}, + created_at=now, + updated_at=now, + ) + session.add(entity) + await session.flush() + entity_id = entity.id + + row = SearchIndexRow( + project_id=search_repository.project_id, + id=entity_id, + type="entity", + entity_id=entity_id, + file_path="notes/evolution-filtered.md", + title="Evolution match", + content_stems="不适者生存者", + content_snippet="不适者生存者", + permalink="notes/evolution-filtered", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search( + "适者", + title="Evolution match", + metadata_filters={"region": "asia"}, + ) + + assert [result.id for result in results] == [entity_id] + + @pytest.mark.asyncio async def test_search_matches_cjk_substring_without_matching_reordered_characters( search_repository, From dc8edfb0bd7ddef22e7fc1bb5f2e8c7d337117cb Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 15:49:08 -0500 Subject: [PATCH 14/27] fix(core): preserve boundary boolean operators Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 3 ++- tests/repository/test_script_ngrams.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 68f88d497..702e05fc2 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -96,7 +96,8 @@ def analyze_script_query(text: str) -> ScriptQuery: # operand into a second SQL channel would otherwise change OR/NOT semantics. # Compatibility forms are natural-language text because neither backend parses # their normalized equivalents as operators. - if '"' in text or any(f" {operator} " in text for operator in ("AND", "OR", "NOT")): + padded_text = f" {text} " + if '"' in text or any(f" {operator} " in padded_text for operator in ("AND", "OR", "NOT")): return ScriptQuery(word_text=text, gram_phrases=()) word_characters: list[str] = [] diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 26161a118..d1f14f5c0 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -69,6 +69,14 @@ def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: assert query.gram_phrases == () +@pytest.mark.parametrize("text", ["NOT 生存", "生存 OR"]) +def test_analyze_script_query_preserves_boundary_boolean_operators(text: str) -> None: + query = analyze_script_query(text) + + assert query.word_text == text + assert query.gram_phrases == () + + def test_analyze_script_query_treats_lowercase_boolean_words_as_natural_language() -> None: query = analyze_script_query("OpenAI and 适者生存") From 831a0c577739ea339f260a7129a22288c9247f8c Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 15:59:11 -0500 Subject: [PATCH 15/27] docs(core): correct script reindex command Signed-off-by: phernandez --- docs/semantic-search.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/semantic-search.md b/docs/semantic-search.md index ccbf2ca29..1aaa4af6f 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -210,9 +210,10 @@ matches the middle of `适者生存`, while a query with the characters in a dif not. Mixed queries such as `OpenAI 适者生存` require both the word and script terms. There is no language, tokenizer, or embedding setting to enable. New and edited notes are indexed -automatically. After upgrading an existing local project, run `bm reindex --search` once to -populate script terms for notes already in the index. Hosted Cloud projects receive the same -Postgres analysis through the managed fleet reindex; Cloud users do not run a local command. +automatically. After upgrading an existing local project, run `bm reindex --full --search` once +to force every existing note through search indexing and populate its script terms. Hosted Cloud +projects receive the same Postgres analysis through the managed full fleet reindex; Cloud users +do not run a local command. This is lexical matching, independent of the configured embedding model. Vector and hybrid search quality in these languages still depends on choosing a multilingual embedding model such as the From 154783dc2a5018c22adaafd07f036f2a9196a86a Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 16:32:36 -0500 Subject: [PATCH 16/27] fix(core): preserve adjoining mixed-script tokens Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 38 +++++++++++++++----- tests/repository/test_script_ngrams.py | 29 +++++++++++++++ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 702e05fc2..c180b4a30 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -101,19 +101,41 @@ def analyze_script_query(text: str) -> ScriptQuery: return ScriptQuery(word_text=text, gram_phrases=()) word_characters: list[str] = [] - in_script_run = False + token_characters: list[str] = [] + token_has_script = False + token_has_word = False for character in text: - if in_script_run and unicodedata.category(character) in {"Mn", "Mc", "Me"}: - continue + category = unicodedata.category(character) normalized_character = unicodedata.normalize("NFKC", character) - if any(is_script_search_character(unit) for unit in normalized_character): - if not in_script_run: - word_characters.append(" ") - in_script_run = True + character_has_script = any( + is_script_search_character(unit) for unit in normalized_character + ) + character_has_word = any( + unit.isalnum() and not is_script_search_character(unit) for unit in normalized_character + ) + if character.isalnum() or category in {"Mn", "Mc", "Me"}: + token_characters.append(character) + token_has_script = token_has_script or character_has_script + token_has_word = token_has_word or character_has_word continue - in_script_run = False + + if token_characters: + # A mixed token is one lexeme in the existing FTS index. Keeping it intact + # preserves exact adjoining text while the script channel supplies substring terms. + if token_has_word: + word_characters.extend(token_characters) + elif token_has_script: + word_characters.append(" ") + token_characters = [] + token_has_script = False + token_has_word = False word_characters.append(character) + if token_has_word: + word_characters.extend(token_characters) + elif token_has_script: + word_characters.append(" ") + # Operator-shaped tokens outside the literal-space syntax above are words. Lowercase # keeps the case-insensitive FTS match while preventing reparsing after reconstruction. word_tokens = [ diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index d1f14f5c0..719061e3e 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -62,6 +62,13 @@ def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: ) +def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> None: + query = analyze_script_query("foo適者bar") + + assert query.word_text == "foo適者bar" + assert query.gram_phrases == (("適者",),) + + def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: query = analyze_script_query("OpenAI OR 适者生存") @@ -265,6 +272,28 @@ async def test_search_matches_cjk_substring_without_matching_reordered_character assert await search_repository.search("适生者存") == [] +@pytest.mark.asyncio +async def test_search_preserves_adjoining_word_and_script_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1311, + type="entity", + file_path="notes/adjoining-script.md", + title="Adjoining script", + content_stems="foo適者bar", + content_snippet="foo適者bar", + permalink="notes/adjoining-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo適者bar") + + assert [result.id for result in results] == [1311] + + @pytest.mark.asyncio async def test_mixed_word_and_script_search_preserves_fts_ranking(search_repository) -> None: now = datetime.now(timezone.utc) From 3d14332a131121820de6ce18a1ee11c89f2d0e91 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 17:00:59 -0500 Subject: [PATCH 17/27] fix(core): isolate relaxed script search channels Signed-off-by: phernandez --- .../repository/sqlite_search_repository.py | 18 +++++++++--- tests/repository/test_script_ngrams.py | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index e15a5e24a..296a289a4 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -1113,6 +1113,9 @@ async def search( # set limit on search query params["limit"] = limit params["offset"] = offset + relaxed_search_text = search_text + if search_text and "script_text" in params: + relaxed_search_text = analyze_script_query(search_text.strip()).word_text sql = f""" SELECT @@ -1154,7 +1157,9 @@ async def run_search(active_session: AsyncSession): # vector-only. # Outcome: one retry with OR-joined prefix terms; bm25 still # ranks multi-term matches first. - relaxed = self._relaxed_fts_text(search_text) if allow_relaxed and not rows else None + relaxed = ( + self._relaxed_fts_text(relaxed_search_text) if allow_relaxed and not rows else None + ) if relaxed and params.get("text"): relaxed_fallback_used = True params["text"] = ( @@ -1167,7 +1172,7 @@ async def run_search(active_session: AsyncSession): with logfire.span( "search.relaxed_fts_retry", backend="sqlite", - token_count=len(relaxed_query_words(search_text) or ()), + token_count=len(relaxed_query_words(relaxed_search_text) or ()), limit=limit, offset=offset, ): @@ -1273,12 +1278,17 @@ async def count( ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" logger.trace(f"Count {sql} params: {params}") + relaxed_search_text = search_text + if search_text and "script_text" in params: + relaxed_search_text = analyze_script_query(search_text.strip()).word_text try: async with db.scoped_session(self.session_maker) as session: result = await session.execute(text(sql), params) total = int(result.scalar_one()) relaxed = ( - self._relaxed_fts_text(search_text) if allow_relaxed and total == 0 else None + self._relaxed_fts_text(relaxed_search_text) + if allow_relaxed and total == 0 + else None ) if relaxed and params.get("text"): params["text"] = ( @@ -1289,7 +1299,7 @@ async def count( with logfire.span( "search.count.relaxed_fts_retry", backend="sqlite", - token_count=len(relaxed_query_words(search_text) or ()), + token_count=len(relaxed_query_words(relaxed_search_text) or ()), ): result = await session.execute(text(sql), params) total = int(result.scalar_one()) diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 719061e3e..dcd362e29 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -370,6 +370,35 @@ async def test_sqlite_mixed_search_requires_all_words_in_one_column(search_repos assert [result.id for result in results] == [1305] +@pytest.mark.asyncio +async def test_sqlite_relaxed_search_keeps_word_and_script_channels_distinct( + search_repository, +) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific relaxed FTS5 regression") + + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1312, + type="entity", + file_path="notes/script-only.md", + title="Script only", + content_stems="適者", + content_snippet="適者", + permalink="notes/script-only", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("missing 適者", allow_relaxed=True) + total = await search_repository.count("missing 適者", allow_relaxed=True) + + assert results == [] + assert total == 0 + + @pytest.mark.asyncio async def test_search_ranking_includes_every_script_run(search_repository) -> None: now = datetime.now(timezone.utc) From 8556e4c1235324da0a1fa25bb8ce4aa880cb2927 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 17:32:17 -0500 Subject: [PATCH 18/27] fix(core): preserve mixed search replacement invariants Signed-off-by: phernandez --- .../accepted_note_search_repository.py | 19 +++++++ src/basic_memory/repository/script_ngrams.py | 45 +++------------ .../test_accepted_note_search_repository.py | 56 ++++++++++++++++++- tests/repository/test_script_ngrams.py | 29 ++++++++++ 4 files changed, 109 insertions(+), 40 deletions(-) diff --git a/src/basic_memory/repository/accepted_note_search_repository.py b/src/basic_memory/repository/accepted_note_search_repository.py index d74974a17..f566f6e63 100644 --- a/src/basic_memory/repository/accepted_note_search_repository.py +++ b/src/basic_memory/repository/accepted_note_search_repository.py @@ -110,6 +110,15 @@ """ ) +DELETE_ACCEPTED_NOTE_FTS_CHUNKS_SQL = text( + """ + DELETE FROM search_index_fts_chunks + WHERE project_id = :project_id + AND search_index_id = :search_index_id + AND search_index_type = :search_index_type + """ +) + def accepted_note_search_insert_statement(session: AsyncSession): """Return the insert statement supported by the active search table backend.""" @@ -183,6 +192,16 @@ async def refresh_entity( if is_sqlite: return + # An upsert can move an older permalink owner's chunks through ON UPDATE CASCADE. + # Clear the final parent key before installing this accepted note's replacements. + await session.execute( + DELETE_ACCEPTED_NOTE_FTS_CHUNKS_SQL, + { + "project_id": row.project_id, + "search_index_id": row.id, + "search_index_type": row.item_type, + }, + ) chunks = [ { "search_index_id": row.id, diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index c180b4a30..f69acb3c4 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -100,48 +100,17 @@ def analyze_script_query(text: str) -> ScriptQuery: if '"' in text or any(f" {operator} " in padded_text for operator in ("AND", "OR", "NOT")): return ScriptQuery(word_text=text, gram_phrases=()) - word_characters: list[str] = [] - token_characters: list[str] = [] - token_has_script = False - token_has_word = False - for character in text: - category = unicodedata.category(character) - normalized_character = unicodedata.normalize("NFKC", character) - character_has_script = any( - is_script_search_character(unit) for unit in normalized_character - ) - character_has_word = any( - unit.isalnum() and not is_script_search_character(unit) for unit in normalized_character - ) - if character.isalnum() or category in {"Mn", "Mc", "Me"}: - token_characters.append(character) - token_has_script = token_has_script or character_has_script - token_has_word = token_has_word or character_has_word - continue - - if token_characters: - # A mixed token is one lexeme in the existing FTS index. Keeping it intact - # preserves exact adjoining text while the script channel supplies substring terms. - if token_has_word: - word_characters.extend(token_characters) - elif token_has_script: - word_characters.append(" ") - token_characters = [] - token_has_script = False - token_has_word = False - word_characters.append(character) - - if token_has_word: - word_characters.extend(token_characters) - elif token_has_script: - word_characters.append(" ") - # Operator-shaped tokens outside the literal-space syntax above are words. Lowercase # keeps the case-insensitive FTS match while preventing reparsing after reconstruction. + # Whitespace-delimited mixed tokens stay intact because punctuation and script characters + # both contribute positions to the existing backend tokenizers. word_tokens = [ token.lower() if token in {"AND", "OR", "NOT"} else token - for token in "".join(word_characters).split() - if any(character.isalnum() for character in token) + for token in text.split() + if any( + character.isalnum() and not is_script_search_character(character) + for character in unicodedata.normalize("NFKC", token) + ) ] gram_phrases = tuple(script_run_grams(run) for run in script_runs(normalized)) # Preserve punctuation-only input as an explicit backend query. Dropping it would make diff --git a/tests/repository/test_accepted_note_search_repository.py b/tests/repository/test_accepted_note_search_repository.py index f8a8a4151..6341cf380 100644 --- a/tests/repository/test_accepted_note_search_repository.py +++ b/tests/repository/test_accepted_note_search_repository.py @@ -59,7 +59,7 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: await repository.refresh_entity(cast(AsyncSession, session), row) - assert len(session.executed) == 3 + assert len(session.executed) == 4 delete_sql, delete_params = session.executed[0] insert_sql, insert_params = session.executed[1] assert "DELETE FROM search_index" in delete_sql @@ -82,7 +82,14 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: "updated_at": updated_at, "project_id": 7, } - chunk_sql, chunk_params = session.executed[2] + chunk_delete_sql, chunk_delete_params = session.executed[2] + assert "DELETE FROM search_index_fts_chunks" in chunk_delete_sql + assert chunk_delete_params == { + "project_id": 7, + "search_index_id": 42, + "search_index_type": "entity", + } + chunk_sql, chunk_params = session.executed[3] assert "INSERT INTO search_index_fts_chunks" in chunk_sql assert chunk_params["project_id"] == 7 assert json.loads(chunk_params["chunks"]) == [ @@ -204,3 +211,48 @@ async def test_postgres_refresh_entity_chunks_large_script_content( results = await search_repository.search("适者生存") assert [result.id for result in results] == [43] + + +@pytest.mark.asyncio +async def test_postgres_refresh_entity_replaces_cascaded_permalink_chunks( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL cascades chunk parent keys during permalink upserts") + + repository = AcceptedNoteSearchRepository(project_id=search_repository.project_id) + now = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + old_row = build_accepted_note_search_row( + entity_id=44, + title="Old owner", + note_type="note", + entity_metadata=None, + permalink="main/reassigned", + file_path="notes/old-owner.md", + search_content="旧所有者内容", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + new_row = build_accepted_note_search_row( + entity_id=45, + title="New owner", + note_type="note", + entity_metadata=None, + permalink="main/reassigned", + file_path="notes/new-owner.md", + search_content="新所有者适者生存", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, old_row) + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, new_row) + + results = await search_repository.search("适者生存") + + assert [result.id for result in results] == [45] diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index dcd362e29..d27ad4655 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -69,6 +69,13 @@ def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> Non assert query.gram_phrases == (("適者",),) +def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: + query = analyze_script_query("foo-適者-bar") + + assert query.word_text == "foo-適者-bar" + assert query.gram_phrases == (("適者",),) + + def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: query = analyze_script_query("OpenAI OR 适者生存") @@ -294,6 +301,28 @@ async def test_search_preserves_adjoining_word_and_script_token(search_repositor assert [result.id for result in results] == [1311] +@pytest.mark.asyncio +async def test_search_preserves_punctuation_separated_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1313, + type="entity", + file_path="notes/punctuation-separated-script.md", + title="Punctuation-separated script", + content_stems="foo-適者-bar", + content_snippet="foo-適者-bar", + permalink="notes/punctuation-separated-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo-適者-bar") + + assert [result.id for result in results] == [1313] + + @pytest.mark.asyncio async def test_mixed_word_and_script_search_preserves_fts_ranking(search_repository) -> None: now = datetime.now(timezone.utc) From 9bf2ed82cc702bc493a1e37bee7110aad1e7a921 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 18:06:24 -0500 Subject: [PATCH 19/27] fix(core): separate mixed-script query channels Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 43 ++++++++++++++------ tests/repository/test_script_ngrams.py | 33 ++++++++++++++- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index f69acb3c4..4a73d29d6 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -100,18 +100,37 @@ def analyze_script_query(text: str) -> ScriptQuery: if '"' in text or any(f" {operator} " in padded_text for operator in ("AND", "OR", "NOT")): return ScriptQuery(word_text=text, gram_phrases=()) - # Operator-shaped tokens outside the literal-space syntax above are words. Lowercase - # keeps the case-insensitive FTS match while preventing reparsing after reconstruction. - # Whitespace-delimited mixed tokens stay intact because punctuation and script characters - # both contribute positions to the existing backend tokenizers. - word_tokens = [ - token.lower() if token in {"AND", "OR", "NOT"} else token - for token in text.split() - if any( - character.isalnum() and not is_script_search_character(character) - for character in unicodedata.normalize("NFKC", token) - ) - ] + word_tokens: list[str] = [] + for token in text.split(): + normalized_token = unicodedata.normalize("NFKC", token) + if not any(is_script_search_character(character) for character in normalized_token): + if any(character.isalnum() for character in normalized_token): + # Operator-shaped tokens outside the literal-space syntax above are words. + # Lowercase prevents the backend from reparsing them after reconstruction. + word_tokens.append(token.lower() if token in {"AND", "OR", "NOT"} else token) + continue + + # A backend word token that starts with Latin text can be matched by its prefix even + # when a longer script run follows. Text after that script run is not a separate word + # token unless punctuation creates a new boundary, so requiring it causes false misses. + word_prefix: list[str] = [] + script_seen = False + for character in normalized_token: + if is_script_search_character(character): + script_seen = True + continue + if character.isalnum() or ( + word_prefix and unicodedata.category(character) in {"Mn", "Mc", "Me"} + ): + if not script_seen: + word_prefix.append(character) + continue + if word_prefix: + word_tokens.append("".join(word_prefix)) + word_prefix = [] + script_seen = False + if word_prefix: + word_tokens.append("".join(word_prefix)) gram_phrases = tuple(script_run_grams(run) for run in script_runs(normalized)) # Preserve punctuation-only input as an explicit backend query. Dropping it would make # the repositories confuse user text with the intentional no-predicate wildcard path. diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index d27ad4655..44b408c41 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -65,14 +65,21 @@ def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> None: query = analyze_script_query("foo適者bar") - assert query.word_text == "foo適者bar" + assert query.word_text == "foo" assert query.gram_phrases == (("適者",),) def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: query = analyze_script_query("foo-適者-bar") - assert query.word_text == "foo-適者-bar" + assert query.word_text == "foo bar" + assert query.gram_phrases == (("適者",),) + + +def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: + query = analyze_script_query("foo適者") + + assert query.word_text == "foo" assert query.gram_phrases == (("適者",),) @@ -301,6 +308,28 @@ async def test_search_preserves_adjoining_word_and_script_token(search_repositor assert [result.id for result in results] == [1311] +@pytest.mark.asyncio +async def test_search_matches_script_substring_inside_longer_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1312, + type="entity", + file_path="notes/longer-adjoining-script.md", + title="Longer adjoining script", + content_stems="foo不適者bar", + content_snippet="foo不適者bar", + permalink="notes/longer-adjoining-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo適者") + + assert [result.id for result in results] == [1312] + + @pytest.mark.asyncio async def test_search_preserves_punctuation_separated_mixed_token(search_repository) -> None: now = datetime.now(timezone.utc) From 4e8e6c330b68a340197be8aa1b073a606a943235 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 18:34:05 -0500 Subject: [PATCH 20/27] fix(core): preserve mixed-token compatibility text Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 14 ++++++++-- tests/repository/test_script_ngrams.py | 29 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 4a73d29d6..e3c43c4c3 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -115,11 +115,19 @@ def analyze_script_query(text: str) -> ScriptQuery: # token unless punctuation creates a new boundary, so requiring it causes false misses. word_prefix: list[str] = [] script_seen = False - for character in normalized_token: - if is_script_search_character(character): + for character in token: + normalized_character = unicodedata.normalize("NFKC", character) + character_has_script = any( + is_script_search_character(unit) for unit in normalized_character + ) + character_has_word = any( + unit.isalnum() and not is_script_search_character(unit) + for unit in normalized_character + ) + if character_has_script: script_seen = True continue - if character.isalnum() or ( + if character_has_word or ( word_prefix and unicodedata.category(character) in {"Mn", "Mc", "Me"} ): if not script_seen: diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 44b408c41..5a8e524ef 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -83,6 +83,13 @@ def test_analyze_script_query_does_not_require_script_substring_in_word_channel( assert query.gram_phrases == (("適者",),) +def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: + query = analyze_script_query("ＡＢＣ適者") + + assert query.word_text == "ＡＢＣ" + assert query.gram_phrases == (("適者",),) + + def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: query = analyze_script_query("OpenAI OR 适者生存") @@ -330,6 +337,28 @@ async def test_search_matches_script_substring_inside_longer_mixed_token(search_ assert [result.id for result in results] == [1312] +@pytest.mark.asyncio +async def test_search_preserves_compatibility_bytes_in_mixed_prefix(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1314, + type="entity", + file_path="notes/compatibility-prefix-script.md", + title="Compatibility prefix script", + content_stems="ＡＢＣ適者", + content_snippet="ＡＢＣ適者", + permalink="notes/compatibility-prefix-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("ＡＢＣ適者") + + assert [result.id for result in results] == [1314] + + @pytest.mark.asyncio async def test_search_preserves_punctuation_separated_mixed_token(search_repository) -> None: now = datetime.now(timezone.utc) From ab1aedaff07647233a8cbaaa03c48b94888ee531 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 19:03:30 -0500 Subject: [PATCH 21/27] fix(core): index mixed-token word fragments Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 87 +++++++++++--------- tests/repository/test_script_ngrams.py | 75 +++++++++++++++-- 2 files changed, 114 insertions(+), 48 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index e3c43c4c3..d880a5422 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -75,6 +75,37 @@ def script_run_grams(run: tuple[str, ...]) -> tuple[str, ...]: return tuple(first + second for first, second in zip(run, run[1:], strict=False)) +def mixed_token_word_terms(text: str) -> tuple[str, ...]: + """Encode word fragments from tokens that also contain script characters.""" + terms: list[str] = [] + for token in text.split(): + normalized_token = unicodedata.normalize("NFKC", token) + if not any(is_script_search_character(character) for character in normalized_token): + continue + + word_fragment: list[str] = [] + for character in normalized_token: + if is_script_search_character(character): + if word_fragment: + fragment = "".join(word_fragment).casefold() + terms.append(f"bmword{fragment.encode('utf-8').hex()}") + word_fragment = [] + continue + if character.isalnum() or ( + word_fragment and unicodedata.category(character) in {"Mn", "Mc", "Me"} + ): + word_fragment.append(character) + continue + if word_fragment: + fragment = "".join(word_fragment).casefold() + terms.append(f"bmword{fragment.encode('utf-8').hex()}") + word_fragment = [] + if word_fragment: + fragment = "".join(word_fragment).casefold() + terms.append(f"bmword{fragment.encode('utf-8').hex()}") + return tuple(terms) + + def build_script_ngrams(*texts: str | None) -> str: """Build portable index text without depending on a database tokenizer.""" gram_runs: list[str] = [] @@ -86,6 +117,7 @@ def build_script_ngrams(*texts: str | None) -> str: # bigrams together after them preserves ordered phrase matching for longer queries. index_terms = run if len(run) == 1 else (*run, *script_run_grams(run)) gram_runs.append(" ".join(index_terms)) + gram_runs.extend(mixed_token_word_terms(text)) return f" {_SCRIPT_BOUNDARY} ".join(gram_runs) @@ -100,46 +132,21 @@ def analyze_script_query(text: str) -> ScriptQuery: if '"' in text or any(f" {operator} " in padded_text for operator in ("AND", "OR", "NOT")): return ScriptQuery(word_text=text, gram_phrases=()) - word_tokens: list[str] = [] - for token in text.split(): - normalized_token = unicodedata.normalize("NFKC", token) - if not any(is_script_search_character(character) for character in normalized_token): - if any(character.isalnum() for character in normalized_token): - # Operator-shaped tokens outside the literal-space syntax above are words. - # Lowercase prevents the backend from reparsing them after reconstruction. - word_tokens.append(token.lower() if token in {"AND", "OR", "NOT"} else token) - continue - - # A backend word token that starts with Latin text can be matched by its prefix even - # when a longer script run follows. Text after that script run is not a separate word - # token unless punctuation creates a new boundary, so requiring it causes false misses. - word_prefix: list[str] = [] - script_seen = False - for character in token: - normalized_character = unicodedata.normalize("NFKC", character) - character_has_script = any( - is_script_search_character(unit) for unit in normalized_character - ) - character_has_word = any( - unit.isalnum() and not is_script_search_character(unit) - for unit in normalized_character - ) - if character_has_script: - script_seen = True - continue - if character_has_word or ( - word_prefix and unicodedata.category(character) in {"Mn", "Mc", "Me"} - ): - if not script_seen: - word_prefix.append(character) - continue - if word_prefix: - word_tokens.append("".join(word_prefix)) - word_prefix = [] - script_seen = False - if word_prefix: - word_tokens.append("".join(word_prefix)) - gram_phrases = tuple(script_run_grams(run) for run in script_runs(normalized)) + # Mixed tokens use the same application-owned auxiliary channel as script grams. This + # avoids assuming that either backend exposes word fragments on both sides of a script run. + word_tokens = [ + token.lower() if token in {"AND", "OR", "NOT"} else token + for token in text.split() + if not any( + is_script_search_character(character) + for character in unicodedata.normalize("NFKC", token) + ) + and any(character.isalnum() for character in unicodedata.normalize("NFKC", token)) + ] + gram_phrases = ( + *(script_run_grams(run) for run in script_runs(normalized)), + *((term,) for term in mixed_token_word_terms(text)), + ) # Preserve punctuation-only input as an explicit backend query. Dropping it would make # the repositories confuse user text with the intentional no-predicate wildcard path. word_text = " ".join(word_tokens) or (text if not gram_phrases else None) diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 5a8e524ef..300b4592e 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -9,6 +9,7 @@ from basic_memory.repository.script_ngrams import ( analyze_script_query, build_script_ngrams, + mixed_token_word_terms, script_run_grams, script_runs, ) @@ -52,6 +53,14 @@ def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> Non assert build_script_ngrams("适者", "生存") == ("适 者 适者 bm_script_boundary 生 存 生存") +def test_mixed_token_word_terms_encode_all_word_fragments() -> None: + assert mixed_token_word_terms("foo不適者bar ＡＢＣ適者") == ( + "bmword666f6f", + "bmword626172", + "bmword616263", + ) + + def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: query = analyze_script_query("OpenAI 适者生存，サバイバル") @@ -65,29 +74,44 @@ def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> None: query = analyze_script_query("foo適者bar") - assert query.word_text == "foo" - assert query.gram_phrases == (("適者",),) + assert query.word_text is None + assert query.gram_phrases == ( + ("適者",), + ("bmword666f6f",), + ("bmword626172",), + ) def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: query = analyze_script_query("foo-適者-bar") - assert query.word_text == "foo bar" - assert query.gram_phrases == (("適者",),) + assert query.word_text is None + assert query.gram_phrases == ( + ("適者",), + ("bmword666f6f",), + ("bmword626172",), + ) def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: query = analyze_script_query("foo適者") - assert query.word_text == "foo" - assert query.gram_phrases == (("適者",),) + assert query.word_text is None + assert query.gram_phrases == (("適者",), ("bmword666f6f",)) def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: query = analyze_script_query("ＡＢＣ適者") - assert query.word_text == "ＡＢＣ" - assert query.gram_phrases == (("適者",),) + assert query.word_text is None + assert query.gram_phrases == (("適者",), ("bmword616263",)) + + +def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: + query = analyze_script_query("適者OpenAI") + + assert query.word_text is None + assert query.gram_phrases == (("適者",), ("bmword6f70656e6169",)) def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: @@ -359,6 +383,41 @@ async def test_search_preserves_compatibility_bytes_in_mixed_prefix(search_repos assert [result.id for result in results] == [1314] +@pytest.mark.asyncio +async def test_search_requires_trailing_word_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + matching_row = SearchIndexRow( + project_id=search_repository.project_id, + id=1315, + type="entity", + file_path="notes/trailing-mixed-word.md", + title="Trailing mixed word", + content_stems="適者OpenAI", + content_snippet="適者OpenAI", + permalink="notes/trailing-mixed-word", + created_at=now, + updated_at=now, + ) + nonmatching_row = SearchIndexRow( + project_id=search_repository.project_id, + id=1316, + type="entity", + file_path="notes/script-only.md", + title="Script only", + content_stems="適者", + content_snippet="適者", + permalink="notes/script-only", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(matching_row) + await search_repository.index_item(nonmatching_row) + + results = await search_repository.search("適者OpenAI") + + assert [result.id for result in results] == [1315] + + @pytest.mark.asyncio async def test_search_preserves_punctuation_separated_mixed_token(search_repository) -> None: now = datetime.now(timezone.utc) From e880d19b806f0d0716982d4e1bb33e14626a394c Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 19:05:12 -0500 Subject: [PATCH 22/27] fix(core): bound mixed-token search terms Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 7 +++-- tests/repository/test_script_ngrams.py | 29 +++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index d880a5422..8ac4f4122 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -1,5 +1,6 @@ """Application-owned lexical analysis for scripts without reliable word boundaries.""" +import hashlib import unicodedata from dataclasses import dataclass @@ -88,7 +89,7 @@ def mixed_token_word_terms(text: str) -> tuple[str, ...]: if is_script_search_character(character): if word_fragment: fragment = "".join(word_fragment).casefold() - terms.append(f"bmword{fragment.encode('utf-8').hex()}") + terms.append(f"bmword{hashlib.sha256(fragment.encode()).hexdigest()}") word_fragment = [] continue if character.isalnum() or ( @@ -98,11 +99,11 @@ def mixed_token_word_terms(text: str) -> tuple[str, ...]: continue if word_fragment: fragment = "".join(word_fragment).casefold() - terms.append(f"bmword{fragment.encode('utf-8').hex()}") + terms.append(f"bmword{hashlib.sha256(fragment.encode()).hexdigest()}") word_fragment = [] if word_fragment: fragment = "".join(word_fragment).casefold() - terms.append(f"bmword{fragment.encode('utf-8').hex()}") + terms.append(f"bmword{hashlib.sha256(fragment.encode()).hexdigest()}") return tuple(terms) diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 300b4592e..6a0b89b37 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -55,9 +55,9 @@ def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> Non def test_mixed_token_word_terms_encode_all_word_fragments() -> None: assert mixed_token_word_terms("foo不適者bar ＡＢＣ適者") == ( - "bmword666f6f", - "bmword626172", - "bmword616263", + "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", + "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", ) @@ -77,8 +77,8 @@ def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> Non assert query.word_text is None assert query.gram_phrases == ( ("適者",), - ("bmword666f6f",), - ("bmword626172",), + ("bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",), + ("bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9",), ) @@ -88,8 +88,8 @@ def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> N assert query.word_text is None assert query.gram_phrases == ( ("適者",), - ("bmword666f6f",), - ("bmword626172",), + ("bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",), + ("bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9",), ) @@ -97,21 +97,30 @@ def test_analyze_script_query_does_not_require_script_substring_in_word_channel( query = analyze_script_query("foo適者") assert query.word_text is None - assert query.gram_phrases == (("適者",), ("bmword666f6f",)) + assert query.gram_phrases == ( + ("適者",), + ("bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",), + ) def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: query = analyze_script_query("ＡＢＣ適者") assert query.word_text is None - assert query.gram_phrases == (("適者",), ("bmword616263",)) + assert query.gram_phrases == ( + ("適者",), + ("bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",), + ) def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: query = analyze_script_query("適者OpenAI") assert query.word_text is None - assert query.gram_phrases == (("適者",), ("bmword6f70656e6169",)) + assert query.gram_phrases == ( + ("適者",), + ("bmword7d3194f79e645c42e4396dda38be04766810ec6a00d00aced3ffc2a0a1f1a9ef",), + ) def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: From 20dba6b533ed0440ad7f79a36415c6217c3d90e8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 19:33:52 -0500 Subject: [PATCH 23/27] fix(core): cover script join controls and extended kana Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 5 ++ tests/repository/test_script_ngrams.py | 71 ++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 8ac4f4122..0c99edad5 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -43,6 +43,7 @@ def is_script_search_character(character: str) -> bool: (0xAC00, 0xD7AF), # Hangul syllables (0xD7B0, 0xD7FF), # Hangul Jamo extended B (0xF900, 0xFAFF), # CJK compatibility ideographs + (0x1AFF0, 0x1AFFF), # Katakana extended B (0x1B000, 0x1B16F), # Kana supplements and extensions (0x20000, 0x2FFFF), # Supplementary CJK ideographs (0x30000, 0x323AF), # CJK unified ideographs extensions G-H @@ -55,6 +56,10 @@ def script_runs(text: str) -> tuple[tuple[str, ...], ...]: runs: list[tuple[str, ...]] = [] current: list[str] = [] for character in unicodedata.normalize("NFKC", text): + # Join controls shape neighboring script characters without introducing a searchable + # unit. Keeping the current run open preserves ordered matching across the control. + if character in {"\u200c", "\u200d"}: + continue if current and unicodedata.category(character) in {"Mn", "Mc", "Me"}: current[-1] += character continue diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 6a0b89b37..ab573a16c 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -27,6 +27,7 @@ ("생존 경쟁", (("생", "존"), ("경", "쟁"))), ("ภาษาไทย", (("ภ", "า", "ษ", "า", "ไ", "ท", "ย"),)), ("時々", (("時", "々"),)), + ("\U0001aff0\U0001aff3\U0001affd", (("\U0001aff0", "\U0001aff3", "\U0001affd"),)), ("ＡＢＣ", ()), ], ) @@ -49,6 +50,14 @@ def test_script_runs_attach_combining_marks_to_the_previous_unit() -> None: assert analyze_script_query(text).word_text is None +@pytest.mark.parametrize("join_control", ["\u200c", "\u200d"]) +def test_script_runs_keep_join_controls_inside_ordered_runs(join_control: str) -> None: + text = f"ក{join_control}ខ" + + assert script_runs(text) == (("ក", "ខ"),) + assert analyze_script_query(text).gram_phrases == (("កខ",),) + + def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> None: assert build_script_ngrams("适者", "生存") == ("适 者 适者 bm_script_boundary 生 存 生存") @@ -326,6 +335,68 @@ async def test_search_matches_cjk_substring_without_matching_reordered_character assert await search_repository.search("适生者存") == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("join_control", ["\u200c", "\u200d"]) +async def test_search_preserves_order_across_join_controls( + search_repository, + join_control: str, +) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1317, + type="entity", + file_path="notes/join-control-match.md", + title="Join control match", + content_stems=f"ក{join_control}ខ", + content_snippet=f"ក{join_control}ខ", + permalink="notes/join-control-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1318, + type="entity", + file_path="notes/join-control-reversed.md", + title="Join control reversed", + content_stems=f"ខ{join_control}ក", + content_snippet=f"ខ{join_control}ក", + permalink="notes/join-control-reversed", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search(f"ក{join_control}ខ") + + assert [result.id for result in results] == [1317] + + +@pytest.mark.asyncio +async def test_search_matches_katakana_extended_b_substring(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1319, + type="entity", + file_path="notes/katakana-extended-b.md", + title="Katakana extended B", + content_stems="\U0001aff0\U0001aff3\U0001affd", + content_snippet="\U0001aff0\U0001aff3\U0001affd", + permalink="notes/katakana-extended-b", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("\U0001aff3") + + assert [result.id for result in results] == [1319] + + @pytest.mark.asyncio async def test_search_preserves_adjoining_word_and_script_token(search_repository) -> None: now = datetime.now(timezone.utc) From 1e6d50f0320d4a31b5e84c70ac2dc952337dda8e Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 20:09:13 -0500 Subject: [PATCH 24/27] fix(core): preserve mixed-token search semantics Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 81 +++++++++--- tests/repository/test_script_ngrams.py | 122 +++++++++++++++---- 2 files changed, 157 insertions(+), 46 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 0c99edad5..2e2b4e05a 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -82,34 +82,77 @@ def script_run_grams(run: tuple[str, ...]) -> tuple[str, ...]: def mixed_token_word_terms(text: str) -> tuple[str, ...]: - """Encode word fragments from tokens that also contain script characters.""" + """Encode prefix-searchable word fragments and their order around script runs.""" terms: list[str] = [] for token in text.split(): normalized_token = unicodedata.normalize("NFKC", token) if not any(is_script_search_character(character) for character in normalized_token): continue - word_fragment: list[str] = [] + components: list[tuple[str, str]] = [] + component_kind: str | None = None + component_characters: list[str] = [] for character in normalized_token: - if is_script_search_character(character): - if word_fragment: - fragment = "".join(word_fragment).casefold() - terms.append(f"bmword{hashlib.sha256(fragment.encode()).hexdigest()}") - word_fragment = [] + if character in {"\u200c", "\u200d"}: continue - if character.isalnum() or ( - word_fragment and unicodedata.category(character) in {"Mn", "Mc", "Me"} - ): - word_fragment.append(character) + if component_characters and unicodedata.category(character) in {"Mn", "Mc", "Me"}: + component_characters.append(character) continue - if word_fragment: - fragment = "".join(word_fragment).casefold() - terms.append(f"bmword{hashlib.sha256(fragment.encode()).hexdigest()}") - word_fragment = [] - if word_fragment: - fragment = "".join(word_fragment).casefold() - terms.append(f"bmword{hashlib.sha256(fragment.encode()).hexdigest()}") - return tuple(terms) + + next_kind = ( + "script" + if is_script_search_character(character) + else "word" + if character.isalnum() + else None + ) + if next_kind != component_kind and component_characters: + components.append((component_kind or "word", "".join(component_characters))) + component_characters = [] + component_kind = next_kind + if next_kind is not None: + component_characters.append(character) + if component_characters: + components.append((component_kind or "word", "".join(component_characters))) + + word_prefixes: dict[int, tuple[str, ...]] = {} + script_terms: dict[int, tuple[str, ...]] = {} + for index, (kind, value) in enumerate(components): + if kind == "word": + normalized_word = value.casefold() + prefixes = tuple( + normalized_word[:length] for length in range(1, len(normalized_word) + 1) + ) + word_prefixes[index] = prefixes + terms.extend( + f"bmword{hashlib.sha256(prefix.encode()).hexdigest()}" for prefix in prefixes + ) + continue + + run = tuple(unit for script_run in script_runs(value) for unit in script_run) + script_terms[index] = (*run, *script_run_grams(run)) + + for index, ((first_kind, _), (second_kind, _)) in enumerate( + zip(components, components[1:], strict=False) + ): + if first_kind == "word" and second_kind == "script": + ordered_pairs = ( + f"word-script\0{prefix}\0{script_term}" + for prefix in word_prefixes[index] + for script_term in script_terms[index + 1] + ) + elif first_kind == "script" and second_kind == "word": + ordered_pairs = ( + f"script-word\0{script_term}\0{prefix}" + for script_term in script_terms[index] + for prefix in word_prefixes[index + 1] + ) + else: + continue + terms.extend( + f"bmseq{hashlib.sha256(pair.encode()).hexdigest()}" for pair in ordered_pairs + ) + return tuple(dict.fromkeys(terms)) def build_script_ngrams(*texts: str | None) -> str: diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index ab573a16c..f71ef2e52 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -63,11 +63,12 @@ def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> Non def test_mixed_token_word_terms_encode_all_word_fragments() -> None: - assert mixed_token_word_terms("foo不適者bar ＡＢＣ適者") == ( - "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", - "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - ) + terms = mixed_token_word_terms("foo不適者bar ＡＢＣ適者") + + assert "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" in terms + assert "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" in terms + assert "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" in terms + assert any(term.startswith("bmseq") for term in terms) def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: @@ -84,52 +85,61 @@ def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> Non query = analyze_script_query("foo適者bar") assert query.word_text is None - assert query.gram_phrases == ( - ("適者",), - ("bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",), - ("bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9",), - ) + assert query.gram_phrases[0] == ("適者",) + assert ( + "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + ) in query.gram_phrases + assert ( + "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", + ) in query.gram_phrases + assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: query = analyze_script_query("foo-適者-bar") assert query.word_text is None - assert query.gram_phrases == ( - ("適者",), - ("bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",), - ("bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9",), - ) + assert query.gram_phrases[0] == ("適者",) + assert ( + "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + ) in query.gram_phrases + assert ( + "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", + ) in query.gram_phrases + assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: query = analyze_script_query("foo適者") assert query.word_text is None - assert query.gram_phrases == ( - ("適者",), - ("bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",), - ) + assert query.gram_phrases[0] == ("適者",) + assert ( + "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + ) in query.gram_phrases + assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: query = analyze_script_query("ＡＢＣ適者") assert query.word_text is None - assert query.gram_phrases == ( - ("適者",), - ("bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",), - ) + assert query.gram_phrases[0] == ("適者",) + assert ( + "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + ) in query.gram_phrases + assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: query = analyze_script_query("適者OpenAI") assert query.word_text is None - assert query.gram_phrases == ( - ("適者",), - ("bmword7d3194f79e645c42e4396dda38be04766810ec6a00d00aced3ffc2a0a1f1a9ef",), - ) + assert query.gram_phrases[0] == ("適者",) + assert ( + "bmword7d3194f79e645c42e4396dda38be04766810ec6a00d00aced3ffc2a0a1f1a9ef", + ) in query.gram_phrases + assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: @@ -441,6 +451,64 @@ async def test_search_matches_script_substring_inside_longer_mixed_token(search_ assert [result.id for result in results] == [1312] +@pytest.mark.asyncio +async def test_search_preserves_prefix_matching_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1320, + type="entity", + file_path="notes/mixed-prefix.md", + title="Mixed prefix", + content_stems="foobar不適者", + content_snippet="foobar不適者", + permalink="notes/mixed-prefix", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo適者") + + assert [result.id for result in results] == [1320] + + +@pytest.mark.asyncio +async def test_search_preserves_word_fragment_order_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1321, + type="entity", + file_path="notes/mixed-order-match.md", + title="Mixed order match", + content_stems="foo適者bar", + content_snippet="foo適者bar", + permalink="notes/mixed-order-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1322, + type="entity", + file_path="notes/mixed-order-reversed.md", + title="Mixed order reversed", + content_stems="bar適者foo", + content_snippet="bar適者foo", + permalink="notes/mixed-order-reversed", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("foo適者bar") + + assert [result.id for result in results] == [1321] + + @pytest.mark.asyncio async def test_search_preserves_compatibility_bytes_in_mixed_prefix(search_repository) -> None: now = datetime.now(timezone.utc) From ad91d8a81fd9d48f95e02a84afbf3815b2b48a14 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 23:11:51 -0500 Subject: [PATCH 25/27] fix(core): bound mixed-token ordering terms Signed-off-by: phernandez --- src/basic_memory/repository/script_ngrams.py | 72 ++++++++++++-------- tests/repository/test_script_ngrams.py | 55 +++++++++++++-- 2 files changed, 91 insertions(+), 36 deletions(-) diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 2e2b4e05a..7760860ea 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -6,6 +6,7 @@ _SCRIPT_BOUNDARY = "bm_script_boundary" +MIXED_WORD_PREFIX_LIMIT = 64 @dataclass(frozen=True, slots=True) @@ -116,41 +117,52 @@ def mixed_token_word_terms(text: str) -> tuple[str, ...]: components.append((component_kind or "word", "".join(component_characters))) word_prefixes: dict[int, tuple[str, ...]] = {} - script_terms: dict[int, tuple[str, ...]] = {} for index, (kind, value) in enumerate(components): - if kind == "word": - normalized_word = value.casefold() - prefixes = tuple( - normalized_word[:length] for length in range(1, len(normalized_word) + 1) - ) - word_prefixes[index] = prefixes - terms.extend( - f"bmword{hashlib.sha256(prefix.encode()).hexdigest()}" for prefix in prefixes - ) + if kind != "word": continue - run = tuple(unit for script_run in script_runs(value) for unit in script_run) - script_terms[index] = (*run, *script_run_grams(run)) - - for index, ((first_kind, _), (second_kind, _)) in enumerate( - zip(components, components[1:], strict=False) - ): - if first_kind == "word" and second_kind == "script": - ordered_pairs = ( - f"word-script\0{prefix}\0{script_term}" - for prefix in word_prefixes[index] - for script_term in script_terms[index + 1] - ) - elif first_kind == "script" and second_kind == "word": - ordered_pairs = ( - f"script-word\0{script_term}\0{prefix}" - for script_term in script_terms[index] - for prefix in word_prefixes[index + 1] - ) - else: + normalized_word = value.casefold() + prefix_count = min(len(normalized_word), MIXED_WORD_PREFIX_LIMIT) + prefixes = tuple(normalized_word[:length] for length in range(1, prefix_count + 1)) + word_prefixes[index] = prefixes + terms.extend( + f"bmword{hashlib.sha256(prefix.encode()).hexdigest()}" for prefix in prefixes + ) + if len(normalized_word) > MIXED_WORD_PREFIX_LIMIT: + terms.append(f"bmwordexact{hashlib.sha256(normalized_word.encode()).hexdigest()}") + + # A word's direction and ordinal distance from the nearest script component preserve + # complete component order without multiplying every prefix by every script gram. + after_distance = 0 + has_script_before = False + for index, (kind, _) in enumerate(components): + if kind == "script": + after_distance = 0 + has_script_before = True + continue + if not has_script_before: + continue + after_distance += 1 + terms.extend( + "bmpos" + hashlib.sha256(f"after\0{after_distance}\0{prefix}".encode()).hexdigest() + for prefix in word_prefixes[index] + ) + + before_distance = 0 + has_script_after = False + for index in range(len(components) - 1, -1, -1): + kind, _ = components[index] + if kind == "script": + before_distance = 0 + has_script_after = True + continue + if not has_script_after: continue + before_distance += 1 terms.extend( - f"bmseq{hashlib.sha256(pair.encode()).hexdigest()}" for pair in ordered_pairs + "bmpos" + + hashlib.sha256(f"before\0{before_distance}\0{prefix}".encode()).hexdigest() + for prefix in word_prefixes[index] ) return tuple(dict.fromkeys(terms)) diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index f71ef2e52..a6d6e3028 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -7,6 +7,7 @@ from basic_memory import db from basic_memory.models import Entity from basic_memory.repository.script_ngrams import ( + MIXED_WORD_PREFIX_LIMIT, analyze_script_query, build_script_ngrams, mixed_token_word_terms, @@ -68,7 +69,13 @@ def test_mixed_token_word_terms_encode_all_word_fragments() -> None: assert "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" in terms assert "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" in terms assert "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" in terms - assert any(term.startswith("bmseq") for term in terms) + assert any(term.startswith("bmpos") for term in terms) + + +def test_mixed_token_word_terms_bound_long_fragment_expansion() -> None: + terms = mixed_token_word_terms(f"{'a' * 500}{'漢字' * 250}") + + assert len(terms) <= MIXED_WORD_PREFIX_LIMIT * 2 + 1 def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: @@ -92,7 +99,7 @@ def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> Non assert ( "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", ) in query.gram_phrases - assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) + assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: @@ -106,7 +113,7 @@ def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> N assert ( "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", ) in query.gram_phrases - assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) + assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: @@ -117,7 +124,7 @@ def test_analyze_script_query_does_not_require_script_substring_in_word_channel( assert ( "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", ) in query.gram_phrases - assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) + assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: @@ -128,7 +135,7 @@ def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> assert ( "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", ) in query.gram_phrases - assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) + assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: @@ -139,7 +146,7 @@ def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> No assert ( "bmword7d3194f79e645c42e4396dda38be04766810ec6a00d00aced3ffc2a0a1f1a9ef", ) in query.gram_phrases - assert any(phrase[0].startswith("bmseq") for phrase in query.gram_phrases) + assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: @@ -509,6 +516,42 @@ async def test_search_preserves_word_fragment_order_in_mixed_token(search_reposi assert [result.id for result in results] == [1321] +@pytest.mark.asyncio +async def test_search_preserves_same_side_word_fragment_order(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1323, + type="entity", + file_path="notes/same-side-order-match.md", + title="Same-side order match", + content_stems="foo-bar-baz適者", + content_snippet="foo-bar-baz適者", + permalink="notes/same-side-order-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1324, + type="entity", + file_path="notes/same-side-order-reversed.md", + title="Same-side order reversed", + content_stems="bar-foo-baz適者", + content_snippet="bar-foo-baz適者", + permalink="notes/same-side-order-reversed", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("foo-bar-baz適者") + + assert [result.id for result in results] == [1323] + + @pytest.mark.asyncio async def test_search_preserves_compatibility_bytes_in_mixed_prefix(search_repository) -> None: now = datetime.now(timezone.utc) From 3d9b3e15ff5949a94e4d6decb935c462653307d9 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 23:23:24 -0500 Subject: [PATCH 26/27] fix(core): bind mixed-token blocks to script roles Signed-off-by: phernandez --- .../repository/postgres_search_repository.py | 5 +- src/basic_memory/repository/script_ngrams.py | 60 +++++----- .../repository/sqlite_search_repository.py | 5 +- tests/repository/test_script_ngrams.py | 111 +++++++++++++----- 4 files changed, 120 insertions(+), 61 deletions(-) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 85e6d2e70..363f4bc74 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -1055,7 +1055,10 @@ async def _build_fts_query_parts( if script_query.gram_phrases: script_tsqueries = [ - " <-> ".join(f"'{gram}'" for gram in phrase) + " <-> ".join( + f"'{gram}':*" if gram.startswith("bmprefix") else f"'{gram}'" + for gram in phrase + ) for phrase in script_query.gram_phrases ] for index, script_tsquery in enumerate(script_tsqueries): diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index 7760860ea..c993c1228 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -6,7 +6,7 @@ _SCRIPT_BOUNDARY = "bm_script_boundary" -MIXED_WORD_PREFIX_LIMIT = 64 +MIXED_WORD_BLOCK_BYTES = 24 @dataclass(frozen=True, slots=True) @@ -116,23 +116,7 @@ def mixed_token_word_terms(text: str) -> tuple[str, ...]: if component_characters: components.append((component_kind or "word", "".join(component_characters))) - word_prefixes: dict[int, tuple[str, ...]] = {} - for index, (kind, value) in enumerate(components): - if kind != "word": - continue - - normalized_word = value.casefold() - prefix_count = min(len(normalized_word), MIXED_WORD_PREFIX_LIMIT) - prefixes = tuple(normalized_word[:length] for length in range(1, prefix_count + 1)) - word_prefixes[index] = prefixes - terms.extend( - f"bmword{hashlib.sha256(prefix.encode()).hexdigest()}" for prefix in prefixes - ) - if len(normalized_word) > MIXED_WORD_PREFIX_LIMIT: - terms.append(f"bmwordexact{hashlib.sha256(normalized_word.encode()).hexdigest()}") - - # A word's direction and ordinal distance from the nearest script component preserve - # complete component order without multiplying every prefix by every script gram. + word_roles: dict[int, list[tuple[str, int]]] = {} after_distance = 0 has_script_before = False for index, (kind, _) in enumerate(components): @@ -143,10 +127,7 @@ def mixed_token_word_terms(text: str) -> tuple[str, ...]: if not has_script_before: continue after_distance += 1 - terms.extend( - "bmpos" + hashlib.sha256(f"after\0{after_distance}\0{prefix}".encode()).hexdigest() - for prefix in word_prefixes[index] - ) + word_roles.setdefault(index, []).append(("after", after_distance)) before_distance = 0 has_script_after = False @@ -159,11 +140,36 @@ def mixed_token_word_terms(text: str) -> tuple[str, ...]: if not has_script_after: continue before_distance += 1 - terms.extend( - "bmpos" - + hashlib.sha256(f"before\0{before_distance}\0{prefix}".encode()).hexdigest() - for prefix in word_prefixes[index] - ) + word_roles.setdefault(index, []).append(("before", before_distance)) + + # Fixed-size blocks preserve arbitrary-length prefix matching while keeping every + # generated lexeme and the total auxiliary representation linear in the input size. + for index, roles in word_roles.items(): + word_bytes = components[index][1].casefold().encode() + for direction, distance in roles: + terms.extend( + f"bmprefix{direction}{distance}x{block_index}x" + f"{word_bytes[start : start + MIXED_WORD_BLOCK_BYTES].hex()}" + for block_index, start in enumerate( + range(0, len(word_bytes), MIXED_WORD_BLOCK_BYTES) + ) + ) + + # Role terms bind the positional word blocks to their neighboring script component. + # A separate token cannot satisfy these terms merely by containing the same script gram. + for index, (kind, value) in enumerate(components): + if kind != "script": + continue + run = tuple(unit for script_run in script_runs(value) for unit in script_run) + run_terms = (*run, *script_run_grams(run)) + if index > 0 and components[index - 1][0] == "word": + terms.extend( + "bmrolebefore" + hashlib.sha256(term.encode()).hexdigest() for term in run_terms + ) + if index + 1 < len(components) and components[index + 1][0] == "word": + terms.extend( + "bmroleafter" + hashlib.sha256(term.encode()).hexdigest() for term in run_terms + ) return tuple(dict.fromkeys(terms)) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 296a289a4..757679635 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -816,7 +816,10 @@ async def _build_fts_query_parts( f"content_snippet: ({prepared_text}))" ) script_phrases = " AND ".join( - f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases + f'"{" ".join(phrase)}"*' + if len(phrase) == 1 and phrase[0].startswith("bmprefix") + else f'"{" ".join(phrase)}"' + for phrase in script_query.gram_phrases ) script_clause = f"script_ngrams: ({script_phrases})" params["script_text"] = ( diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index a6d6e3028..5e7f05769 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -7,7 +7,7 @@ from basic_memory import db from basic_memory.models import Entity from basic_memory.repository.script_ngrams import ( - MIXED_WORD_PREFIX_LIMIT, + MIXED_WORD_BLOCK_BYTES, analyze_script_query, build_script_ngrams, mixed_token_word_terms, @@ -66,16 +66,17 @@ def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> Non def test_mixed_token_word_terms_encode_all_word_fragments() -> None: terms = mixed_token_word_terms("foo不適者bar ＡＢＣ適者") - assert "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" in terms - assert "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" in terms - assert "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" in terms - assert any(term.startswith("bmpos") for term in terms) + assert "bmprefixbefore1x0x666f6f" in terms + assert "bmprefixafter1x0x626172" in terms + assert "bmprefixbefore1x0x616263" in terms + assert any(term.startswith("bmrole") for term in terms) def test_mixed_token_word_terms_bound_long_fragment_expansion() -> None: terms = mixed_token_word_terms(f"{'a' * 500}{'漢字' * 250}") - assert len(terms) <= MIXED_WORD_PREFIX_LIMIT * 2 + 1 + word_block_count = (500 + MIXED_WORD_BLOCK_BYTES - 1) // MIXED_WORD_BLOCK_BYTES + assert len(terms) <= word_block_count + 4 def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: @@ -93,13 +94,9 @@ def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> Non assert query.word_text is None assert query.gram_phrases[0] == ("適者",) - assert ( - "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - ) in query.gram_phrases - assert ( - "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", - ) in query.gram_phrases - assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) + assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases + assert ("bmprefixafter1x0x626172",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: @@ -107,13 +104,9 @@ def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> N assert query.word_text is None assert query.gram_phrases[0] == ("適者",) - assert ( - "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - ) in query.gram_phrases - assert ( - "bmwordfcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", - ) in query.gram_phrases - assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) + assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases + assert ("bmprefixafter1x0x626172",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: @@ -121,10 +114,8 @@ def test_analyze_script_query_does_not_require_script_substring_in_word_channel( assert query.word_text is None assert query.gram_phrases[0] == ("適者",) - assert ( - "bmword2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - ) in query.gram_phrases - assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) + assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: @@ -132,10 +123,8 @@ def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> assert query.word_text is None assert query.gram_phrases[0] == ("適者",) - assert ( - "bmwordba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - ) in query.gram_phrases - assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) + assert ("bmprefixbefore1x0x616263",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: @@ -143,10 +132,8 @@ def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> No assert query.word_text is None assert query.gram_phrases[0] == ("適者",) - assert ( - "bmword7d3194f79e645c42e4396dda38be04766810ec6a00d00aced3ffc2a0a1f1a9ef", - ) in query.gram_phrases - assert any(phrase[0].startswith("bmpos") for phrase in query.gram_phrases) + assert ("bmprefixafter1x0x6f70656e6169",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: @@ -480,6 +467,30 @@ async def test_search_preserves_prefix_matching_in_mixed_token(search_repository assert [result.id for result in results] == [1320] +@pytest.mark.asyncio +async def test_search_preserves_long_prefix_matching_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + indexed_prefix = "a" * 70 + query_prefix = "a" * 65 + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1325, + type="entity", + file_path="notes/long-mixed-prefix.md", + title="Long mixed prefix", + content_stems=f"{indexed_prefix}適者", + content_snippet=f"{indexed_prefix}適者", + permalink="notes/long-mixed-prefix", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search(f"{query_prefix}適者") + + assert [result.id for result in results] == [1325] + + @pytest.mark.asyncio async def test_search_preserves_word_fragment_order_in_mixed_token(search_repository) -> None: now = datetime.now(timezone.utc) @@ -516,6 +527,42 @@ async def test_search_preserves_word_fragment_order_in_mixed_token(search_reposi assert [result.id for result in results] == [1321] +@pytest.mark.asyncio +async def test_search_binds_word_positions_to_their_script_run(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1326, + type="entity", + file_path="notes/script-role-match.md", + title="Script role match", + content_stems="foo適者bar", + content_snippet="foo適者bar", + permalink="notes/script-role-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1327, + type="entity", + file_path="notes/script-role-decoy.md", + title="Script role decoy", + content_stems="foo生存bar 適者", + content_snippet="foo生存bar 適者", + permalink="notes/script-role-decoy", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("foo適者bar") + + assert [result.id for result in results] == [1326] + + @pytest.mark.asyncio async def test_search_preserves_same_side_word_fragment_order(search_repository) -> None: now = datetime.now(timezone.utc) From 62b1e941185d14a78d9f0c3df3ef410bdd30eabb Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 10:29:30 -0500 Subject: [PATCH 27/27] fix(core): keep mixed-script tokens on word search path Signed-off-by: phernandez --- .../repository/postgres_search_repository.py | 5 +- src/basic_memory/repository/script_ngrams.py | 120 +--------- .../repository/sqlite_search_repository.py | 5 +- tests/repository/test_script_ngrams.py | 224 +++--------------- 4 files changed, 52 insertions(+), 302 deletions(-) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 363f4bc74..85e6d2e70 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -1055,10 +1055,7 @@ async def _build_fts_query_parts( if script_query.gram_phrases: script_tsqueries = [ - " <-> ".join( - f"'{gram}':*" if gram.startswith("bmprefix") else f"'{gram}'" - for gram in phrase - ) + " <-> ".join(f"'{gram}'" for gram in phrase) for phrase in script_query.gram_phrases ] for index, script_tsquery in enumerate(script_tsqueries): diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py index c993c1228..656b57715 100644 --- a/src/basic_memory/repository/script_ngrams.py +++ b/src/basic_memory/repository/script_ngrams.py @@ -1,12 +1,10 @@ """Application-owned lexical analysis for scripts without reliable word boundaries.""" -import hashlib import unicodedata from dataclasses import dataclass _SCRIPT_BOUNDARY = "bm_script_boundary" -MIXED_WORD_BLOCK_BYTES = 24 @dataclass(frozen=True, slots=True) @@ -82,97 +80,6 @@ def script_run_grams(run: tuple[str, ...]) -> tuple[str, ...]: return tuple(first + second for first, second in zip(run, run[1:], strict=False)) -def mixed_token_word_terms(text: str) -> tuple[str, ...]: - """Encode prefix-searchable word fragments and their order around script runs.""" - terms: list[str] = [] - for token in text.split(): - normalized_token = unicodedata.normalize("NFKC", token) - if not any(is_script_search_character(character) for character in normalized_token): - continue - - components: list[tuple[str, str]] = [] - component_kind: str | None = None - component_characters: list[str] = [] - for character in normalized_token: - if character in {"\u200c", "\u200d"}: - continue - if component_characters and unicodedata.category(character) in {"Mn", "Mc", "Me"}: - component_characters.append(character) - continue - - next_kind = ( - "script" - if is_script_search_character(character) - else "word" - if character.isalnum() - else None - ) - if next_kind != component_kind and component_characters: - components.append((component_kind or "word", "".join(component_characters))) - component_characters = [] - component_kind = next_kind - if next_kind is not None: - component_characters.append(character) - if component_characters: - components.append((component_kind or "word", "".join(component_characters))) - - word_roles: dict[int, list[tuple[str, int]]] = {} - after_distance = 0 - has_script_before = False - for index, (kind, _) in enumerate(components): - if kind == "script": - after_distance = 0 - has_script_before = True - continue - if not has_script_before: - continue - after_distance += 1 - word_roles.setdefault(index, []).append(("after", after_distance)) - - before_distance = 0 - has_script_after = False - for index in range(len(components) - 1, -1, -1): - kind, _ = components[index] - if kind == "script": - before_distance = 0 - has_script_after = True - continue - if not has_script_after: - continue - before_distance += 1 - word_roles.setdefault(index, []).append(("before", before_distance)) - - # Fixed-size blocks preserve arbitrary-length prefix matching while keeping every - # generated lexeme and the total auxiliary representation linear in the input size. - for index, roles in word_roles.items(): - word_bytes = components[index][1].casefold().encode() - for direction, distance in roles: - terms.extend( - f"bmprefix{direction}{distance}x{block_index}x" - f"{word_bytes[start : start + MIXED_WORD_BLOCK_BYTES].hex()}" - for block_index, start in enumerate( - range(0, len(word_bytes), MIXED_WORD_BLOCK_BYTES) - ) - ) - - # Role terms bind the positional word blocks to their neighboring script component. - # A separate token cannot satisfy these terms merely by containing the same script gram. - for index, (kind, value) in enumerate(components): - if kind != "script": - continue - run = tuple(unit for script_run in script_runs(value) for unit in script_run) - run_terms = (*run, *script_run_grams(run)) - if index > 0 and components[index - 1][0] == "word": - terms.extend( - "bmrolebefore" + hashlib.sha256(term.encode()).hexdigest() for term in run_terms - ) - if index + 1 < len(components) and components[index + 1][0] == "word": - terms.extend( - "bmroleafter" + hashlib.sha256(term.encode()).hexdigest() for term in run_terms - ) - return tuple(dict.fromkeys(terms)) - - def build_script_ngrams(*texts: str | None) -> str: """Build portable index text without depending on a database tokenizer.""" gram_runs: list[str] = [] @@ -184,7 +91,6 @@ def build_script_ngrams(*texts: str | None) -> str: # bigrams together after them preserves ordered phrase matching for longer queries. index_terms = run if len(run) == 1 else (*run, *script_run_grams(run)) gram_runs.append(" ".join(index_terms)) - gram_runs.extend(mixed_token_word_terms(text)) return f" {_SCRIPT_BOUNDARY} ".join(gram_runs) @@ -199,21 +105,19 @@ def analyze_script_query(text: str) -> ScriptQuery: if '"' in text or any(f" {operator} " in padded_text for operator in ("AND", "OR", "NOT")): return ScriptQuery(word_text=text, gram_phrases=()) - # Mixed tokens use the same application-owned auxiliary channel as script grams. This - # avoids assuming that either backend exposes word fragments on both sides of a script run. - word_tokens = [ - token.lower() if token in {"AND", "OR", "NOT"} else token - for token in text.split() - if not any( - is_script_search_character(character) - for character in unicodedata.normalize("NFKC", token) + word_tokens: list[str] = [] + for token in text.split(): + normalized_token = unicodedata.normalize("NFKC", token) + # Keep mixed-script tokens on the established word-search path. The auxiliary channel + # adds script substring recall without inventing new cross-script token semantics. + has_word_character = any( + character.isalnum() and not is_script_search_character(character) + for character in normalized_token ) - and any(character.isalnum() for character in unicodedata.normalize("NFKC", token)) - ] - gram_phrases = ( - *(script_run_grams(run) for run in script_runs(normalized)), - *((term,) for term in mixed_token_word_terms(text)), - ) + if has_word_character: + word_tokens.append(token.lower() if token in {"AND", "OR", "NOT"} else token) + + gram_phrases = tuple(script_run_grams(run) for run in script_runs(normalized)) # Preserve punctuation-only input as an explicit backend query. Dropping it would make # the repositories confuse user text with the intentional no-predicate wildcard path. word_text = " ".join(word_tokens) or (text if not gram_phrases else None) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 757679635..296a289a4 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -816,10 +816,7 @@ async def _build_fts_query_parts( f"content_snippet: ({prepared_text}))" ) script_phrases = " AND ".join( - f'"{" ".join(phrase)}"*' - if len(phrase) == 1 and phrase[0].startswith("bmprefix") - else f'"{" ".join(phrase)}"' - for phrase in script_query.gram_phrases + f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases ) script_clause = f"script_ngrams: ({script_phrases})" params["script_text"] = ( diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py index 5e7f05769..5867a513c 100644 --- a/tests/repository/test_script_ngrams.py +++ b/tests/repository/test_script_ngrams.py @@ -7,10 +7,8 @@ from basic_memory import db from basic_memory.models import Entity from basic_memory.repository.script_ngrams import ( - MIXED_WORD_BLOCK_BYTES, analyze_script_query, build_script_ngrams, - mixed_token_word_terms, script_run_grams, script_runs, ) @@ -63,22 +61,6 @@ def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> Non assert build_script_ngrams("适者", "生存") == ("适 者 适者 bm_script_boundary 生 存 生存") -def test_mixed_token_word_terms_encode_all_word_fragments() -> None: - terms = mixed_token_word_terms("foo不適者bar ＡＢＣ適者") - - assert "bmprefixbefore1x0x666f6f" in terms - assert "bmprefixafter1x0x626172" in terms - assert "bmprefixbefore1x0x616263" in terms - assert any(term.startswith("bmrole") for term in terms) - - -def test_mixed_token_word_terms_bound_long_fragment_expansion() -> None: - terms = mixed_token_word_terms(f"{'a' * 500}{'漢字' * 250}") - - word_block_count = (500 + MIXED_WORD_BLOCK_BYTES - 1) // MIXED_WORD_BLOCK_BYTES - assert len(terms) <= word_block_count + 4 - - def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: query = analyze_script_query("OpenAI 适者生存，サバイバル") @@ -92,48 +74,36 @@ def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> None: query = analyze_script_query("foo適者bar") - assert query.word_text is None - assert query.gram_phrases[0] == ("適者",) - assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases - assert ("bmprefixafter1x0x626172",) in query.gram_phrases - assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + assert query.word_text == "foo適者bar" + assert query.gram_phrases == (("適者",),) def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: query = analyze_script_query("foo-適者-bar") - assert query.word_text is None - assert query.gram_phrases[0] == ("適者",) - assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases - assert ("bmprefixafter1x0x626172",) in query.gram_phrases - assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + assert query.word_text == "foo-適者-bar" + assert query.gram_phrases == (("適者",),) -def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: +def test_analyze_script_query_keeps_mixed_token_on_word_channel() -> None: query = analyze_script_query("foo適者") - assert query.word_text is None - assert query.gram_phrases[0] == ("適者",) - assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases - assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + assert query.word_text == "foo適者" + assert query.gram_phrases == (("適者",),) -def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: +def test_analyze_script_query_preserves_compatibility_text_in_mixed_token() -> None: query = analyze_script_query("ＡＢＣ適者") - assert query.word_text is None - assert query.gram_phrases[0] == ("適者",) - assert ("bmprefixbefore1x0x616263",) in query.gram_phrases - assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + assert query.word_text == "ＡＢＣ適者" + assert query.gram_phrases == (("適者",),) -def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: +def test_analyze_script_query_retains_trailing_word_in_word_channel() -> None: query = analyze_script_query("適者OpenAI") - assert query.word_text is None - assert query.gram_phrases[0] == ("適者",) - assert ("bmprefixafter1x0x6f70656e6169",) in query.gram_phrases - assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + assert query.word_text == "適者OpenAI" + assert query.gram_phrases == (("適者",),) def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: @@ -424,122 +394,18 @@ async def test_search_preserves_adjoining_word_and_script_token(search_repositor @pytest.mark.asyncio -async def test_search_matches_script_substring_inside_longer_mixed_token(search_repository) -> None: - now = datetime.now(timezone.utc) - row = SearchIndexRow( - project_id=search_repository.project_id, - id=1312, - type="entity", - file_path="notes/longer-adjoining-script.md", - title="Longer adjoining script", - content_stems="foo不適者bar", - content_snippet="foo不適者bar", - permalink="notes/longer-adjoining-script", - created_at=now, - updated_at=now, - ) - await search_repository.index_item(row) - - results = await search_repository.search("foo適者") - - assert [result.id for result in results] == [1312] - - -@pytest.mark.asyncio -async def test_search_preserves_prefix_matching_in_mixed_token(search_repository) -> None: - now = datetime.now(timezone.utc) - row = SearchIndexRow( - project_id=search_repository.project_id, - id=1320, - type="entity", - file_path="notes/mixed-prefix.md", - title="Mixed prefix", - content_stems="foobar不適者", - content_snippet="foobar不適者", - permalink="notes/mixed-prefix", - created_at=now, - updated_at=now, - ) - await search_repository.index_item(row) - - results = await search_repository.search("foo適者") - - assert [result.id for result in results] == [1320] - - -@pytest.mark.asyncio -async def test_search_preserves_long_prefix_matching_in_mixed_token(search_repository) -> None: - now = datetime.now(timezone.utc) - indexed_prefix = "a" * 70 - query_prefix = "a" * 65 - row = SearchIndexRow( - project_id=search_repository.project_id, - id=1325, - type="entity", - file_path="notes/long-mixed-prefix.md", - title="Long mixed prefix", - content_stems=f"{indexed_prefix}適者", - content_snippet=f"{indexed_prefix}適者", - permalink="notes/long-mixed-prefix", - created_at=now, - updated_at=now, - ) - await search_repository.index_item(row) - - results = await search_repository.search(f"{query_prefix}適者") - - assert [result.id for result in results] == [1325] - - -@pytest.mark.asyncio -async def test_search_preserves_word_fragment_order_in_mixed_token(search_repository) -> None: - now = datetime.now(timezone.utc) - rows = [ - SearchIndexRow( - project_id=search_repository.project_id, - id=1321, - type="entity", - file_path="notes/mixed-order-match.md", - title="Mixed order match", - content_stems="foo適者bar", - content_snippet="foo適者bar", - permalink="notes/mixed-order-match", - created_at=now, - updated_at=now, - ), - SearchIndexRow( - project_id=search_repository.project_id, - id=1322, - type="entity", - file_path="notes/mixed-order-reversed.md", - title="Mixed order reversed", - content_stems="bar適者foo", - content_snippet="bar適者foo", - permalink="notes/mixed-order-reversed", - created_at=now, - updated_at=now, - ), - ] - await search_repository.bulk_index_items(rows) - - results = await search_repository.search("foo適者bar") - - assert [result.id for result in results] == [1321] - - -@pytest.mark.asyncio -async def test_search_binds_word_positions_to_their_script_run(search_repository) -> None: +async def test_mixed_token_query_does_not_combine_terms_across_tokens(search_repository) -> None: now = datetime.now(timezone.utc) rows = [ SearchIndexRow( project_id=search_repository.project_id, id=1326, type="entity", - file_path="notes/script-role-match.md", - title="Script role match", + file_path="notes/mixed-token-match.md", + title="Mixed token match", content_stems="foo適者bar", content_snippet="foo適者bar", - permalink="notes/script-role-match", + permalink="notes/mixed-token-match", created_at=now, updated_at=now, ), @@ -547,11 +413,11 @@ async def test_search_binds_word_positions_to_their_script_run(search_repository project_id=search_repository.project_id, id=1327, type="entity", - file_path="notes/script-role-decoy.md", - title="Script role decoy", - content_stems="foo生存bar 適者", - content_snippet="foo生存bar 適者", - permalink="notes/script-role-decoy", + file_path="notes/distributed-token-decoy.md", + title="Distributed token decoy", + content_stems="foo生存bar x適者y", + content_snippet="foo生存bar x適者y", + permalink="notes/distributed-token-decoy", created_at=now, updated_at=now, ), @@ -564,39 +430,25 @@ async def test_search_binds_word_positions_to_their_script_run(search_repository @pytest.mark.asyncio -async def test_search_preserves_same_side_word_fragment_order(search_repository) -> None: +async def test_search_matches_script_substring_inside_longer_mixed_token(search_repository) -> None: now = datetime.now(timezone.utc) - rows = [ - SearchIndexRow( - project_id=search_repository.project_id, - id=1323, - type="entity", - file_path="notes/same-side-order-match.md", - title="Same-side order match", - content_stems="foo-bar-baz適者", - content_snippet="foo-bar-baz適者", - permalink="notes/same-side-order-match", - created_at=now, - updated_at=now, - ), - SearchIndexRow( - project_id=search_repository.project_id, - id=1324, - type="entity", - file_path="notes/same-side-order-reversed.md", - title="Same-side order reversed", - content_stems="bar-foo-baz適者", - content_snippet="bar-foo-baz適者", - permalink="notes/same-side-order-reversed", - created_at=now, - updated_at=now, - ), - ] - await search_repository.bulk_index_items(rows) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1312, + type="entity", + file_path="notes/longer-adjoining-script.md", + title="Longer adjoining script", + content_stems="foo不適者bar", + content_snippet="foo不適者bar", + permalink="notes/longer-adjoining-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) - results = await search_repository.search("foo-bar-baz適者") + results = await search_repository.search("適者") - assert [result.id for result in results] == [1323] + assert [result.id for result in results] == [1312] @pytest.mark.asyncio