feat(index): full-text search with BM25 ranking (#70)

src/BLite.Core/BLiteMigration.cs

@@ -290,6 +290,9 @@ private static async Task RecreateIndexAsync(DynamicCollection col, IndexMetadat
             case IndexType.Spatial:
                 await col.CreateSpatialIndexAsync(field, idx.Name);
                 break;
+            case IndexType.FullText:
+                await col.CreateFullTextIndexAsync(field, idx.Name);
+                break;
         }
     }
 

src/BLite.Core/Collections/IDocumentCollection.cs

@@ -165,6 +165,21 @@ Task<ICollectionIndex<TId, T>> CreateIndexAsync<TKey>(
         bool unique = false,
         CancellationToken ct = default);
 
+    /// <inheritdoc cref="DocumentCollection{TId, T}.CreateFullTextIndexAsync"/>
+    [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
+    Task<ICollectionIndex<TId, T>> CreateFullTextIndexAsync(
+        Expression<Func<T, string?>> keySelector,
+        string? name = null,
+        CancellationToken ct = default);
+
+    /// <inheritdoc cref="DocumentCollection{TId, T}.FullTextSearchAsync"/>
+    IAsyncEnumerable<FullTextSearchResult<T>> FullTextSearchAsync(
+        string indexName,
+        string query,
+        int k,
+        bool matchAllTerms = true,
+        CancellationToken ct = default);
+
     [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
     Task<ICollectionIndex<TId, T>> CreateVectorIndexAsync<TKey>(
         Expression<Func<T, TKey>> keySelector,

src/BLite.Core/Indexing/CollectionSecondaryIndex.cs

@@ -47,6 +47,7 @@ static CollectionSecondaryIndex()
     private readonly BTreeIndex? _btreeIndex;
     private readonly VectorSearchIndex? _vectorIndex;
     private readonly RTreeIndex? _spatialIndex;
+    private readonly FullTextIndex? _fullTextIndex;
     private readonly IDocumentMapper<TId, T> _mapper;
     private bool _disposed;
     private long _documentCount;
@@ -61,7 +62,7 @@ static CollectionSecondaryIndex()
     /// </summary>
     public BTreeIndex? BTreeIndex => _btreeIndex;
 
-    public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? 0;
+    public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? _fullTextIndex?.RootPageId ?? 0;
 
     public CollectionSecondaryIndex(
         CollectionIndexDefinition<T> definition,
@@ -86,6 +87,14 @@ public CollectionSecondaryIndex(
             _btreeIndex = null;
             _vectorIndex = null;
         }
+        else if (indexOptions.Type == IndexType.FullText)
+        {
+            _fullTextIndex = new FullTextIndex(storage, indexOptions, rootPageId);
+            _btreeIndex = null;
+            _vectorIndex = null;
+            _spatialIndex = null;
+            KeyFormat = FullTextIndex.FormatVersion;
+        }
         else
         {
             _btreeIndex = new BTreeIndex(storage, indexOptions, rootPageId);
@@ -214,6 +223,13 @@ private void InsertKey(object keyValue, T document, DocumentLocation location, I
                 _spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
             }
         }
+        else if (_fullTextIndex != null)
+        {
+            if (keyValue is string text)
+            {
+                _fullTextIndex.Insert(text, location, transaction);
+            }
+        }
         else if (_btreeIndex != null)
         {
             // BTree Index logic
@@ -264,6 +280,13 @@ private void DeleteKey(object keyValue, T document, DocumentLocation location, I
                 _spatialIndex.Delete(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
             }
         }
+        else if (_fullTextIndex != null)
+        {
+            if (keyValue is string text)
+            {
+                _fullTextIndex.Delete(text, location, transaction);
+            }
+        }
         else if (_btreeIndex != null)
         {
             var userKey = ConvertToIndexKey(keyValue);
@@ -359,6 +382,40 @@ public IEnumerable<DocumentLocation> Within((double Latitude, double Longitude)
         return _spatialIndex.Search(area, transaction);
     }
 
+    /// <summary>
+    /// Locations of the documents whose indexed text contains every term of <paramref name="query"/>.
+    /// </summary>
+    public IReadOnlyList<DocumentLocation> FullTextMatch(string query, ITransaction? transaction = null)
+    {
+        if (_fullTextIndex == null)
+            throw new InvalidOperationException("This index is not a full-text index.");
+
+        return _fullTextIndex.Match(query, transaction);
+    }
+
+    /// <summary>
+    /// The <paramref name="k"/> best BM25 matches for <paramref name="query"/>, best first.
+    /// </summary>
+    public IReadOnlyList<(DocumentLocation Location, double Score)> FullTextSearch(string query, int k, bool matchAllTerms = true, ITransaction? transaction = null)
+    {
+        if (_fullTextIndex == null)
+            throw new InvalidOperationException("This index is not a full-text index.");
+
+        return _fullTextIndex.Search(query, k, matchAllTerms, transaction);
+    }
+
+    /// <summary>
+    /// The BM25 matches for <paramref name="query"/>, best first, starting with the
+    /// <paramref name="k"/> best and continuing past them for as long as the caller enumerates.
+    /// </summary>
+    internal IEnumerable<(DocumentLocation Location, double Score)> FullTextSearchExpanding(string query, int k, bool matchAllTerms, ITransaction? transaction = null)
+    {
+        if (_fullTextIndex == null)
+            throw new InvalidOperationException("This index is not a full-text index.");
+
+        return _fullTextIndex.SearchExpanding(query, k, matchAllTerms, transaction);
+    }
+
     /// <summary>
     /// Scans a range of keys (O(log n + k) where k is result count)
     /// </summary>
@@ -773,10 +830,27 @@ private static decimal ToDecimalClamped(float f)
 
     /// <summary>
     /// True when this B-tree index keys a <c>decimal</c> property and its pages still use the legacy
-    /// format, i.e. hold <c>(double)value</c> keys that the exact decimal encoding cannot match.
+    /// format, i.e. hold <c>(double)value</c> keys that the exact decimal encoding cannot match; or
+    /// when this full-text index was written with another analyzer version or holds keys it did not
+    /// write (e.g. from an older BLite that read it as a plain B-tree).
     /// </summary>
     internal bool NeedsKeyFormatMigration =>
-        _btreeIndex != null && KeyFormat < IndexMetadata.CurrentKeyFormat && IsDecimalKeyed;
+        (_btreeIndex != null && KeyFormat < IndexMetadata.CurrentKeyFormat && IsDecimalKeyed)
+        || (_fullTextIndex != null && (KeyFormat != FullTextIndex.FormatVersion || _fullTextIndex.HasForeignKeys()));
+
+    /// <summary>The key format a rebuilt copy of this index is written in.</summary>
+    internal byte TargetKeyFormat => _fullTextIndex != null ? FullTextIndex.FormatVersion : IndexMetadata.CurrentKeyFormat;
+
+    /// <summary>
+    /// Whether the indexed text of <paramref name="document"/> contains every term of
+    /// <paramref name="query"/> (or, with <paramref name="matchAllTerms"/> false, at least one).
+    /// Used to re-check full-text hits, so a stale posting can never surface another document.
+    /// </summary>
+    internal bool FullTextContains(T document, string query, bool matchAllTerms)
+    {
+        var text = _definition.TryGetKey(document, out var key) ? key as string : null;
+        return matchAllTerms ? Text.FullTextAnalyzer.Matches(text, query) : Text.FullTextAnalyzer.MatchesAny(text, query);
+    }
 
     // ── ICollectionIndex metadata pass-throughs ─────────────────────────────
 
@@ -875,6 +949,7 @@ internal IReadOnlyList<uint> CollectAllPages(ulong transactionId)
         if (_btreeIndex != null) return _btreeIndex.CollectAllPages(transactionId);
         if (_vectorIndex != null) return _vectorIndex.CollectAllPages(transactionId);
         if (_spatialIndex != null) return _spatialIndex.CollectAllPages(transactionId);
+        if (_fullTextIndex != null) return _fullTextIndex.CollectAllPages(transactionId);
         return Array.Empty<uint>();
     }
 }

src/BLite.Core/Indexing/FullTextExtensions.cs

@@ -0,0 +1,23 @@
+using BLite.Core.Text;
+
+namespace BLite.Core;
+
+public static class FullTextExtensions
+{
+    /// <summary>
+    /// Whether <paramref name="text"/> contains every term of <paramref name="query"/>. Terms are
+    /// compared after removing accents, lowercasing and treating anything that is not a letter or
+    /// digit as a separator, so <c>"Café, au lait!"</c> matches the query <c>"cafe LAIT"</c>.
+    /// A query without any term matches nothing.
+    /// <para>
+    /// The LINQ query provider serves this from the property's full-text index when there is one;
+    /// otherwise it is evaluated per document with the same rules.
+    /// </para>
+    /// </summary>
+    /// <param name="text">The string property of the entity.</param>
+    /// <param name="query">The terms to look for.</param>
+    public static bool FullTextMatch(this string? text, string query)
+    {
+        return FullTextAnalyzer.Matches(text, query);
+    }
+}