feat(core): add CJK substring search support

CHANGELOG.md

@@ -2,6 +2,31 @@
 
 ## Unreleased
 
+### Features
+
+- **#1294**: Chinese, Japanese, and Korean terms now match anywhere in an
+  indexed note, not only at the start of a CJK run. Search indexing derives an
+  index-only stream of overlapping CJK bigrams (`search_tokens` as an extra
+  SQLite FTS5 column; `search_tokens`/`chunk_tokens` text columns with
+  `simple`-configuration generated `tsvector`s and GIN indexes on PostgreSQL),
+  and CJK queries are rendered as exact adjacency phrases against those
+  columns. Bigrams never cross a field or chunk boundary, so scattered
+  characters do not satisfy a contiguous term. Display text is unchanged:
+  titles, permalinks, `content_stems`, snippets, and chunk text keep their
+  original bytes, and pure non-CJK queries take the same SQL and ranking paths
+  as before.
+
+  The migration adds schema only — it does not backfill derived tokens.
+  Existing installations must repopulate the search index once after
+  upgrading:
+
+  ```text
+  basic-memory reindex --full --search
+  ```
+
+  New installs and any note written after the upgrade index CJK tokens
+  automatically.
+
 ## v0.23.2 (2026-08-25)
 
 Patch release fixing case-duplicate folder creation.

src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py

@@ -0,0 +1,176 @@
+"""Add CJK search token indexes
+
+Revision ID: 8a7b6c5d4e3f
+Revises: 7f6a2b8c9d10
+Create Date: 2026-08-29 23:30:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy import text
+from sqlalchemy.engine import Connection
+
+
+# revision identifiers, used by Alembic.
+revision: str = "8a7b6c5d4e3f"
+down_revision: Union[str, None] = "7f6a2b8c9d10"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _has_fts5_search_index(connection: Connection) -> bool:
+    """Whether a real FTS5 search_index virtual table exists on this connection.
+
+    Trigger: SQLite creates search_index as an FTS5 virtual table at runtime
+    via SearchRepository.init_search_index, not through Alembic, so fresh
+    installs hit this migration before the table exists. Some migration
+    tests also stand up a minimal plain `search_index` table to exercise an
+    earlier migration's repair SQL in isolation.
+    Why: recreating a table Alembic never created would make a fresh install
+    diverge from every other install; recreating a same-named table that
+    isn't actually the FTS5 index would destroy unrelated data for no schema
+    benefit.
+    Outcome: only a genuine FTS5 search_index gets dropped and rebuilt with
+    search_tokens. A missing table, or a differently-shaped one, is left
+    alone -- the runtime creates the real one with the current schema on
+    first use.
+    """
+    row = connection.execute(
+        text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search_index'")
+    ).fetchone()
+    return row is not None and row[0] is not None and "fts5" in row[0].lower()
+
+
+def upgrade() -> None:
+    """Add derived CJK search-token storage for SQLite and PostgreSQL.
+
+    Schema only: the Python bigram transform (repository/search_query.py
+    cjk_search_tokens, added in the prior commit) is deliberately not
+    duplicated in SQL or PL/pgSQL. Existing rows stay stale until the forced
+    reindex below runs, consistent with this project's derived-state
+    convergence model.
+    """
+    connection = op.get_bind()
+    if connection.dialect.name == "sqlite" and _has_fts5_search_index(connection):
+        # search_index is a derived FTS5 virtual table with no foreign keys to
+        # entity/note_content/observation/relation (FTS5 can't carry one), so
+        # dropping and recreating it cannot touch canonical source data -- it
+        # only empties the derived index, which the reindex below repopulates.
+        op.execute("DROP TABLE IF EXISTS search_index")
+        op.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+                -- Core entity fields
+                id UNINDEXED,          -- Row ID
+                title,                 -- Title for searching
+                content_stems,         -- Main searchable content split into stems
+                content_snippet,       -- File content snippet for display
+                search_tokens,         -- Derived CJK bigram tokens (cjk_search_tokens)
+                permalink,             -- Stable identifier (now indexed for path search)
+                file_path UNINDEXED,   -- Physical location
+                type UNINDEXED,        -- entity/relation/observation
+
+                -- Project context
+                project_id UNINDEXED,  -- Project identifier
+
+                -- Relation fields
+                from_id UNINDEXED,     -- Source entity
+                to_id UNINDEXED,       -- Target entity
+                relation_type UNINDEXED, -- Type of relation
+
+                -- Observation fields
+                entity_id UNINDEXED,   -- Parent entity
+                category UNINDEXED,    -- Observation category
+
+                -- Common fields
+                metadata UNINDEXED,    -- JSON metadata
+                created_at UNINDEXED,  -- Creation timestamp
+                updated_at UNINDEXED,  -- Last update
+
+                -- Configuration
+                tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+                prefix='1,2,3,4'                    -- Support longer prefixes for paths
+            );
+        """)
+    elif connection.dialect.name == "postgresql":
+        op.execute("ALTER TABLE search_index ADD COLUMN IF NOT EXISTS search_tokens TEXT")
+        op.execute("""
+            ALTER TABLE search_index ADD COLUMN IF NOT EXISTS search_tokens_index_col tsvector
+            GENERATED ALWAYS AS (
+                to_tsvector('simple', coalesce(search_tokens, ''))
+            ) STORED
+        """)
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_search_index_cjk_fts
+            ON search_index USING gin(search_tokens_index_col)
+        """)
+
+        op.execute("ALTER TABLE search_index_fts_chunks ADD COLUMN IF NOT EXISTS chunk_tokens TEXT")
+        op.execute("""
+            ALTER TABLE search_index_fts_chunks
+            ADD COLUMN IF NOT EXISTS chunk_tokens_index_col tsvector
+            GENERATED ALWAYS AS (
+                to_tsvector('simple', coalesce(chunk_tokens, ''))
+            ) STORED
+        """)
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_cjk_fts
+            ON search_index_fts_chunks USING gin(chunk_tokens_index_col)
+        """)
+
+    print("\nCJK search index added. Run: basic-memory reindex --full --search\n")
+
+
+def downgrade() -> None:
+    """Remove the CJK search-token storage, restoring the prior schema exactly."""
+    connection = op.get_bind()
+    if connection.dialect.name == "sqlite" and _has_fts5_search_index(connection):
+        op.execute("DROP TABLE IF EXISTS search_index")
+        op.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+                -- Core entity fields
+                id UNINDEXED,          -- Row ID
+                title,                 -- Title for searching
+                content_stems,         -- Main searchable content split into stems
+                content_snippet,       -- File content snippet for display
+                permalink,             -- Stable identifier (now indexed for path search)
+                file_path UNINDEXED,   -- Physical location
+                type UNINDEXED,        -- entity/relation/observation
+
+                -- Project context
+                project_id UNINDEXED,  -- Project identifier
+
+                -- Relation fields
+                from_id UNINDEXED,     -- Source entity
+                to_id UNINDEXED,       -- Target entity
+                relation_type UNINDEXED, -- Type of relation
+
+                -- Observation fields
+                entity_id UNINDEXED,   -- Parent entity
+                category UNINDEXED,    -- Observation category
+
+                -- Common fields
+                metadata UNINDEXED,    -- JSON metadata
+                created_at UNINDEXED,  -- Creation timestamp
+                updated_at UNINDEXED,  -- Last update
+
+                -- Configuration
+                tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+                prefix='1,2,3,4'                    -- Support longer prefixes for paths
+            );
+        """)
+    elif connection.dialect.name == "postgresql":
+        op.execute("DROP INDEX IF EXISTS idx_search_index_fts_chunks_cjk_fts")
+        op.execute("""
+            ALTER TABLE search_index_fts_chunks
+            DROP COLUMN IF EXISTS chunk_tokens_index_col
+        """)
+        op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS chunk_tokens")
+
+        op.execute("DROP INDEX IF EXISTS idx_search_index_cjk_fts")
+        op.execute("""
+            ALTER TABLE search_index
+            DROP COLUMN IF EXISTS search_tokens_index_col
+        """)
+        op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS search_tokens")

src/basic_memory/indexing/accepted_note_search.py

@@ -9,6 +9,7 @@
 
 from basic_memory.file_utils import ParseError, remove_frontmatter
 from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow
+from basic_memory.repository.search_query import cjk_search_tokens
 from basic_memory.schemas.base import normalize_note_type
 
 MAX_ACCEPTED_SEARCH_CONTENT_STEMS_SIZE = 6000
@@ -112,17 +113,20 @@ def build_accepted_note_search_row(
     item_type: str = "entity",
 ) -> AcceptedNoteSearchRow:
     """Build the hot entity search row for one accepted note snapshot."""
+    title_text = strip_search_text(title)
+    content_stems = accepted_note_content_stems(
+        title=title,
+        search_content=search_content,
+        permalink=permalink,
+        file_path=file_path,
+        tags=accepted_note_tags(entity_metadata),
+    )
     return AcceptedNoteSearchRow(
         id=entity_id,
-        title=strip_search_text(title),
-        content_stems=accepted_note_content_stems(
-            title=title,
-            search_content=search_content,
-            permalink=permalink,
-            file_path=file_path,
-            tags=accepted_note_tags(entity_metadata),
-        ),
+        title=title_text,
+        content_stems=content_stems,
         content_snippet=strip_search_text(search_content),
+        search_tokens=cjk_search_tokens(title_text, permalink, content_stems),
         permalink=permalink,
         file_path=Path(file_path).as_posix(),
         item_type=item_type,

src/basic_memory/models/search.py

@@ -39,6 +39,10 @@
             coalesce(content_stems, '')
         )
     ) STORED,
+    search_tokens TEXT,
+    search_tokens_index_col tsvector GENERATED ALWAYS AS (
+        to_tsvector('simple', coalesce(search_tokens, ''))
+    ) STORED,
     PRIMARY KEY (id, type, project_id),
     FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE
 )
@@ -48,6 +52,14 @@
 CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col)
 """)
 
+# Cross-CJK lexical candidates: search_tokens holds whitespace-separated
+# overlapping bigrams for CJK runs (see repository/search_query.py
+# cjk_search_tokens), indexed with the 'simple' parser so Postgres does not
+# stem or stopword-filter the bigrams.
+CREATE_POSTGRES_SEARCH_INDEX_CJK_FTS = DDL("""
+CREATE INDEX IF NOT EXISTS idx_search_index_cjk_fts ON search_index USING gin(search_tokens_index_col)
+""")
+
 # Full note bodies are stored in bounded child rows so one unusually large note
 # cannot exceed PostgreSQL's per-tsvector size limit.
 CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL("""
@@ -60,6 +72,10 @@
     textsearchable_index_col tsvector GENERATED ALWAYS AS (
         to_tsvector('english', chunk_text)
     ) STORED,
+    chunk_tokens TEXT,
+    chunk_tokens_index_col tsvector GENERATED ALWAYS AS (
+        to_tsvector('simple', coalesce(chunk_tokens, ''))
+    ) STORED,
     PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index),
     FOREIGN KEY (search_index_id, search_index_type, project_id)
         REFERENCES search_index(id, type, project_id)
@@ -72,6 +88,11 @@
 ON search_index_fts_chunks USING gin(textsearchable_index_col)
 """)
 
+CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_CJK_FTS = DDL("""
+CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_cjk_fts
+ON search_index_fts_chunks USING gin(chunk_tokens_index_col)
+""")
+
 CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL("""
 CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops)
 """)
@@ -94,6 +115,7 @@
     title,                 -- Title for searching
     content_stems,         -- Main searchable content split into stems
     content_snippet,       -- File content snippet for display
+    search_tokens,         -- Derived CJK bigram tokens (cjk_search_tokens)
     permalink,             -- Stable identifier (now indexed for path search)
     file_path UNINDEXED,   -- Physical location
     type UNINDEXED,        -- entity/relation/observation

src/basic_memory/repository/accepted_note_search_repository.py

@@ -28,13 +28,13 @@
 INSERT_ACCEPTED_NOTE_SEARCH_SQL = text(
     """
     INSERT INTO search_index (
-        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+        id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata,
         from_id, to_id, relation_type,
         entity_id, category,
         created_at, updated_at,
         project_id
     ) VALUES (
-        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type,
+        :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type,
         :metadata,
         NULL, NULL, NULL,
         :entity_id, NULL,
@@ -47,13 +47,13 @@
 UPSERT_ACCEPTED_NOTE_SEARCH_SQL = text(
     """
     INSERT INTO search_index (
-        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+        id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata,
         from_id, to_id, relation_type,
         entity_id, category,
         created_at, updated_at,
         project_id
     ) VALUES (
-        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type,
+        :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type,
         CAST(:metadata AS jsonb),
         NULL, NULL, NULL,
         :entity_id, NULL,
@@ -65,6 +65,7 @@
         title = EXCLUDED.title,
         content_stems = EXCLUDED.content_stems,
         content_snippet = EXCLUDED.content_snippet,
+        search_tokens = EXCLUDED.search_tokens,
         file_path = EXCLUDED.file_path,
         type = EXCLUDED.type,
         metadata = EXCLUDED.metadata,
@@ -95,6 +96,7 @@ def accepted_note_search_insert_params(
         "title": row.title,
         "content_stems": row.content_stems,
         "content_snippet": row.content_snippet,
+        "search_tokens": row.search_tokens,
         "permalink": row.permalink,
         "file_path": row.file_path,
         "type": row.item_type,

src/basic_memory/repository/accepted_note_search_row.py

@@ -12,6 +12,7 @@ class AcceptedNoteSearchRow:
     title: str
     content_stems: str
     content_snippet: str
+    search_tokens: str
     permalink: str | None
     file_path: str
     item_type: str