From 8cff3de985729edb050edffdaf7ebefe0305f090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 14:21:00 +0200 Subject: [PATCH 01/15] fix(index): fill model-declared indexes from existing documents (#171) An index declared in OnModelCreating was created empty when the collection already held documents, so queries routed to it missed every older document. This hit the usual way of adding an index: a new app version that adds HasIndex, HasVectorIndex or HasSpatialIndex to an existing collection. ApplyIndexBuilder now builds a missing index unregistered, fills it from the primary index, and only then registers and persists it. A crash before that leaves no empty index in the metadata, and the next open builds it again. If a document cannot be loaded or indexed, the open fails with an InvalidOperationException naming the index, and nothing is persisted. Existing indexes are still a cheap name lookup on every open. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 35 +++---- .../Indexing/CollectionIndexManager.cs | 51 ++++++++++ tests/BLite.Shared/IndexEvolutionContexts.cs | 58 +++++++++++ tests/BLite.Tests/IndexBackfillTests.cs | 98 +++++++++++++++++++ 4 files changed, 225 insertions(+), 17 deletions(-) create mode 100644 tests/BLite.Shared/IndexEvolutionContexts.cs create mode 100644 tests/BLite.Tests/IndexBackfillTests.cs diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 19d77c2..2f47927 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -1070,28 +1070,29 @@ public IEnumerable GetIndexes() [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) { - // Use the IndexManager directly to ensure the index exists - // We need to convert the LambdaExpression to a typed expression if possible, - // or add an untyped CreateIndexAsync to IndexManager. + var name = CollectionIndexManager.ResolveIndexName(builder.KeySelector, builder.Name); + if (_indexManager.GetIndex(name) != null) + return; - // For now, let's use a dynamic approach or cast if we know it's Func - if (builder.Type == IndexType.Vector) - { - _indexManager.CreateVectorIndexUntyped(builder.KeySelector, builder.Dimensions, builder.Metric, builder.Name); - } - else if (builder.Type == IndexType.Spatial) - { - _indexManager.CreateSpatialIndexUntyped(builder.KeySelector, builder.Name); - } - else if (builder.KeySelector is System.Linq.Expressions.Expression> selector) + // The collection may already hold documents, e.g. when a new app version adds the index. + // Fill the index before registering it, so neither a crash nor a document that cannot be + // indexed leaves an empty index persisted that later opens would trust. + var definition = CollectionIndexManager.BuildDefinition( + builder.KeySelector, name, builder.IsUnique, builder.Type, builder.Dimensions, builder.Metric); + var index = _indexManager.CreateUnregistered(definition); + try { - _indexManager.EnsureIndex(selector, builder.Name, builder.IsUnique); + // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. + Task.Run(() => RebuildIndexAsync(index, skipUnreadableDocuments: false)).GetAwaiter().GetResult(); } - else + catch (Exception ex) { - // Try to rebuild the expression or use untyped version - _indexManager.EnsureIndexUntyped(builder.KeySelector, builder.Name, builder.IsUnique); + index.FreeAllPages(_storage); + index.Dispose(); + throw new InvalidOperationException( + $"Failed to build index '{name}' of collection '{_collectionName}' from its existing documents; it is built again on the next open.", ex); } + _indexManager.Register(index); } /// diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index d465365..674646a 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -110,6 +110,57 @@ public CollectionSecondaryIndex CreateIndex(CollectionIndexDefinition } } + /// The name an index on gets when none is given. + internal static string ResolveIndexName(LambdaExpression keySelector, string? name) + => name ?? GenerateIndexName(ExpressionAnalyzer.ExtractPropertyPaths(keySelector)); + + /// + /// Builds an index definition from an untyped key selector, as declared in the model. + /// + [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] + internal static CollectionIndexDefinition BuildDefinition( + LambdaExpression keySelector, + string? name, + bool unique, + IndexType type, + int dimensions, + VectorMetric metric) + { + var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector); + var body = keySelector.Body.Type == typeof(object) + ? keySelector.Body + : Expression.Convert(keySelector.Body, typeof(object)); + var lambda = Expression.Lambda>(body, keySelector.Parameters); + + return new CollectionIndexDefinition( + name ?? GenerateIndexName(propertyPaths), propertyPaths, lambda, unique, type, false, dimensions, metric); + } + + /// + /// Creates an index for without registering or persisting it, + /// so it can be filled before queries or the collection metadata see it. The index owns its + /// root page: hand it to or release it with + /// . + /// + internal CollectionSecondaryIndex CreateUnregistered(CollectionIndexDefinition definition) + => new(definition, _storage, _mapper); + + /// Registers an index built by and persists it. + internal void Register(CollectionSecondaryIndex index) + { + lock (_lock) + { + if (_disposed) + throw new ObjectDisposedException(nameof(CollectionIndexManager)); + if (_indexes.ContainsKey(index.Name)) + throw new InvalidOperationException($"Index '{index.Name}' already exists"); + + _indexes[index.Name] = index; + _cachedIndexInfo = null; // invalidate cache + SaveMetadata(); + } + } + // ... methods ... /// diff --git a/tests/BLite.Shared/IndexEvolutionContexts.cs b/tests/BLite.Shared/IndexEvolutionContexts.cs new file mode 100644 index 0000000..bbcb44a --- /dev/null +++ b/tests/BLite.Shared/IndexEvolutionContexts.cs @@ -0,0 +1,58 @@ +using BLite.Bson; +using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Indexing; +using BLite.Core.Metadata; + +namespace BLite.Shared; + +// Contexts for adding model-declared indexes to collections that already hold documents. + +/// "App v1": the collections without any secondary index. +public partial class IndexEvoV1DbContext : DocumentDbContext +{ + public DocumentCollection People { get; set; } = null!; + public DocumentCollection Vectors { get; set; } = null!; + public DocumentCollection Places { get; set; } = null!; + + public IndexEvoV1DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("evo_people"); + modelBuilder.Entity().ToCollection("evo_vectors"); + modelBuilder.Entity().ToCollection("evo_places"); + } +} + +/// "App v2": the same collections with a B-Tree, a vector and a spatial index. +public partial class IndexEvoV2DbContext : DocumentDbContext +{ + public DocumentCollection People { get; set; } = null!; + public DocumentCollection Vectors { get; set; } = null!; + public DocumentCollection Places { get; set; } = null!; + + public IndexEvoV2DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("evo_people").HasIndex(p => p.Age); + modelBuilder.Entity().ToCollection("evo_vectors") + .HasVectorIndex(x => x.Embedding, dimensions: 3, metric: VectorMetric.L2, name: "idx_vector"); + modelBuilder.Entity().ToCollection("evo_places").HasSpatialIndex(x => x.Location, name: "idx_spatial"); + } +} + +/// Declares a vector index whose dimension does not match the stored embeddings. +public partial class IndexEvoBadVectorDbContext : DocumentDbContext +{ + public DocumentCollection Vectors { get; set; } = null!; + + public IndexEvoBadVectorDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("evo_vectors") + .HasVectorIndex(x => x.Embedding, dimensions: 4, metric: VectorMetric.L2, name: "idx_vector"); + } +} diff --git a/tests/BLite.Tests/IndexBackfillTests.cs b/tests/BLite.Tests/IndexBackfillTests.cs new file mode 100644 index 0000000..714ffb4 --- /dev/null +++ b/tests/BLite.Tests/IndexBackfillTests.cs @@ -0,0 +1,98 @@ +using BLite.Core; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// An index declared in the model of a collection that already holds documents must be filled +/// from them when it is first created (#171). +/// +public class IndexBackfillTests : IDisposable +{ + private readonly string _dbPath; + + public IndexBackfillTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"idx_backfill_{Guid.NewGuid():N}.db"); + } + + public void Dispose() + { + foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ } + } + + private async Task SeedAsync() + { + using var v1 = new IndexEvoV1DbContext(_dbPath); + for (int i = 0; i < 100; i++) + await v1.People.InsertAsync(new PersonV2 { Name = $"p{i}", Age = i % 10 }); + await v1.Vectors.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] }); + await v1.Vectors.InsertAsync(new VectorEntity { Title = "B", Embedding = [5f, 5f, 5f] }); + await v1.Places.InsertAsync(new GeoEntity { Name = "Milan", Location = (45.46, 9.19) }); + await v1.Places.InsertAsync(new GeoEntity { Name = "New York", Location = (40.71, -74.01) }); + } + + [Fact] + public async Task AddedBTreeIndex_CoversExistingDocuments() + { + await SeedAsync(); + + using var v2 = new IndexEvoV2DbContext(_dbPath); + + Assert.Equal(10, v2.People.AsQueryable().Where(p => p.Age == 3).ToList().Count); + Assert.Equal(10, (await v2.People.QueryIndexAsync("idx_Age", 3, 3).ToListAsync()).Count); + } + + [Fact] + public async Task AddedVectorIndex_CoversExistingDocuments() + { + await SeedAsync(); + + using var v2 = new IndexEvoV2DbContext(_dbPath); + + var nearest = await v2.Vectors.VectorSearchAsync("idx_vector", [4f, 4f, 4f], 1).ToListAsync(); + Assert.Equal("B", Assert.Single(nearest).Title); + } + + [Fact] + public async Task AddedSpatialIndex_CoversExistingDocuments() + { + await SeedAsync(); + + using var v2 = new IndexEvoV2DbContext(_dbPath); + + var inBox = await v2.Places.WithinAsync("idx_spatial", (44.0, 8.0), (46.0, 10.0)).ToListAsync(); + Assert.Equal("Milan", Assert.Single(inBox).Name); + } + + [Fact] + public async Task AddedIndex_IsFilledOnce_AndMaintainedAfterwards() + { + await SeedAsync(); + using (var v2 = new IndexEvoV2DbContext(_dbPath)) + await v2.People.InsertAsync(new PersonV2 { Name = "late", Age = 3 }); + + using var reopened = new IndexEvoV2DbContext(_dbPath); + + Assert.Equal(11, (await reopened.People.QueryIndexAsync("idx_Age", 3, 3).ToListAsync()).Count); + } + + [Fact] + public async Task IndexThatCannotBeFilled_FailsTheOpen_AndIsNotPersisted() + { + await SeedAsync(); + + // The stored embeddings have 3 dimensions; the declared index expects 4. + var ex = Assert.Throws(() => new IndexEvoBadVectorDbContext(_dbPath)); + Assert.Contains("idx_vector", ex.Message); + + // Nothing was registered, so the data stays readable and the next open tries again. + using (var v1 = new IndexEvoV1DbContext(_dbPath)) + { + Assert.DoesNotContain(v1.Vectors.GetIndexes(), i => i.Name == "idx_vector"); + Assert.Equal(2, v1.Vectors.AsQueryable().ToList().Count); + } + Assert.Throws(() => new IndexEvoBadVectorDbContext(_dbPath)); + } +} From 1c04a620b9d6f37a924534fcf91d39366e277c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 15:35:55 +0200 Subject: [PATCH 02/15] fix(index): address review of model-declared index backfill (#171) - The error when an existing document cannot be indexed now says that nothing was saved and that every open fails the same way until the documents can be indexed or the index is removed from OnModelCreating, and includes the underlying message. - Removes CollectionIndexManager.EnsureIndexUntyped and the public CreateVectorIndexUntyped / CreateSpatialIndexUntyped, which BuildDefinition made dead code. - The CreateUnregistered doc comment notes that a spatial index cannot release its pages yet. Opening a context whose model declares a new index over documents it cannot index now throws; before, the index was created empty. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 4 +- .../Indexing/CollectionIndexManager.cs | 75 +------------------ tests/BLite.Tests/IndexBackfillTests.cs | 2 + 3 files changed, 7 insertions(+), 74 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 2f47927..796fbed 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -1090,7 +1090,9 @@ internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) index.FreeAllPages(_storage); index.Dispose(); throw new InvalidOperationException( - $"Failed to build index '{name}' of collection '{_collectionName}' from its existing documents; it is built again on the next open.", ex); + $"Cannot build index '{name}' of collection '{_collectionName}' from the documents already in it: {ex.Message} " + + "Nothing was saved, but every open fails the same way until those documents can be indexed or the index " + + "is removed from the model (OnModelCreating).", ex); } _indexManager.Register(index); } diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index 674646a..280f44f 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -140,7 +140,8 @@ internal static CollectionIndexDefinition BuildDefinition( /// Creates an index for without registering or persisting it, /// so it can be filled before queries or the collection metadata see it. The index owns its /// root page: hand it to or release it with - /// . + /// (a spatial index cannot release + /// its pages yet, so an abandoned one leaks them). /// internal CollectionSecondaryIndex CreateUnregistered(CollectionIndexDefinition definition) => new(definition, _storage, _mapper); @@ -239,78 +240,6 @@ public CollectionSecondaryIndex EnsureIndex( } } - [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] - internal CollectionSecondaryIndex EnsureIndexUntyped( - LambdaExpression keySelector, - string? name = null, - bool unique = false) - { - // Convert LambdaExpression to Expression> properly by sharing parameters - var body = keySelector.Body; - if (body.Type != typeof(object)) - { - body = Expression.Convert(body, typeof(object)); - } - - var lambda = Expression.Lambda>(body, keySelector.Parameters); - - return EnsureIndex(lambda, name, unique); - } - - [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] - public CollectionSecondaryIndex CreateVectorIndexUntyped( - LambdaExpression keySelector, - int dimensions, - VectorMetric metric = VectorMetric.Cosine, - string? name = null) - { - var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector); - var indexName = name ?? GenerateIndexName(propertyPaths); - - lock (_lock) - { - if (_indexes.TryGetValue(indexName, out var existing)) - return existing; - - var body = keySelector.Body; - if (body.Type != typeof(object)) - { - body = Expression.Convert(body, typeof(object)); - } - - var lambda = Expression.Lambda>(body, keySelector.Parameters); - - var definition = new CollectionIndexDefinition(indexName, propertyPaths, lambda, false, IndexType.Vector, false, dimensions, metric); - return CreateIndex(definition); - } - } - - [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] - public CollectionSecondaryIndex CreateSpatialIndexUntyped( - LambdaExpression keySelector, - string? name = null) - { - var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector); - var indexName = name ?? GenerateIndexName(propertyPaths); - - lock (_lock) - { - if (_indexes.TryGetValue(indexName, out var existing)) - return existing; - - var body = keySelector.Body; - if (body.Type != typeof(object)) - { - body = Expression.Convert(body, typeof(object)); - } - - var lambda = Expression.Lambda>(body, keySelector.Parameters); - - var definition = new CollectionIndexDefinition(indexName, propertyPaths, lambda, false, IndexType.Spatial); - return CreateIndex(definition); - } - } - /// /// Drops an existing index by name /// diff --git a/tests/BLite.Tests/IndexBackfillTests.cs b/tests/BLite.Tests/IndexBackfillTests.cs index 714ffb4..c1ef444 100644 --- a/tests/BLite.Tests/IndexBackfillTests.cs +++ b/tests/BLite.Tests/IndexBackfillTests.cs @@ -86,6 +86,8 @@ public async Task IndexThatCannotBeFilled_FailsTheOpen_AndIsNotPersisted() // The stored embeddings have 3 dimensions; the declared index expects 4. var ex = Assert.Throws(() => new IndexEvoBadVectorDbContext(_dbPath)); Assert.Contains("idx_vector", ex.Message); + Assert.Contains("removed from the model", ex.Message); + Assert.IsType(ex.InnerException); // the dimension mismatch // Nothing was registered, so the data stays readable and the next open tries again. using (var v1 = new IndexEvoV1DbContext(_dbPath)) From 1cb3a8e4e22fcbccdbc02e268d08a88450fddf5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 19:00:15 +0200 Subject: [PATCH 03/15] fix(index): address follow-ups on model-declared index backfill (#171) - Restore the public CreateVectorIndexUntyped and CreateSpatialIndexUntyped on CollectionIndexManager, unchanged but marked [Obsolete]; the internal EnsureIndexUntyped stays removed. - The backfill failure message ends the inner message with exactly one full stop before "Nothing was saved". Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 2 +- .../Indexing/CollectionIndexManager.cs | 56 +++++++++++++++++++ tests/BLite.Tests/IndexBackfillTests.cs | 1 + 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 796fbed..ce52d91 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -1090,7 +1090,7 @@ internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) index.FreeAllPages(_storage); index.Dispose(); throw new InvalidOperationException( - $"Cannot build index '{name}' of collection '{_collectionName}' from the documents already in it: {ex.Message} " + + $"Cannot build index '{name}' of collection '{_collectionName}' from the documents already in it: {ex.Message.TrimEnd('.')}. " + "Nothing was saved, but every open fails the same way until those documents can be indexed or the index " + "is removed from the model (OnModelCreating).", ex); } diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index 280f44f..672d776 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -240,6 +240,62 @@ public CollectionSecondaryIndex EnsureIndex( } } + [Obsolete("Use model HasVectorIndex/HasSpatialIndex or CreateIndex(definition); will be removed in 6.0")] + [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] + public CollectionSecondaryIndex CreateVectorIndexUntyped( + LambdaExpression keySelector, + int dimensions, + VectorMetric metric = VectorMetric.Cosine, + string? name = null) + { + var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector); + var indexName = name ?? GenerateIndexName(propertyPaths); + + lock (_lock) + { + if (_indexes.TryGetValue(indexName, out var existing)) + return existing; + + var body = keySelector.Body; + if (body.Type != typeof(object)) + { + body = Expression.Convert(body, typeof(object)); + } + + var lambda = Expression.Lambda>(body, keySelector.Parameters); + + var definition = new CollectionIndexDefinition(indexName, propertyPaths, lambda, false, IndexType.Vector, false, dimensions, metric); + return CreateIndex(definition); + } + } + + [Obsolete("Use model HasVectorIndex/HasSpatialIndex or CreateIndex(definition); will be removed in 6.0")] + [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] + public CollectionSecondaryIndex CreateSpatialIndexUntyped( + LambdaExpression keySelector, + string? name = null) + { + var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector); + var indexName = name ?? GenerateIndexName(propertyPaths); + + lock (_lock) + { + if (_indexes.TryGetValue(indexName, out var existing)) + return existing; + + var body = keySelector.Body; + if (body.Type != typeof(object)) + { + body = Expression.Convert(body, typeof(object)); + } + + var lambda = Expression.Lambda>(body, keySelector.Parameters); + + var definition = new CollectionIndexDefinition(indexName, propertyPaths, lambda, false, IndexType.Spatial); + return CreateIndex(definition); + } + } + /// /// Drops an existing index by name /// diff --git a/tests/BLite.Tests/IndexBackfillTests.cs b/tests/BLite.Tests/IndexBackfillTests.cs index c1ef444..9d59801 100644 --- a/tests/BLite.Tests/IndexBackfillTests.cs +++ b/tests/BLite.Tests/IndexBackfillTests.cs @@ -88,6 +88,7 @@ public async Task IndexThatCannotBeFilled_FailsTheOpen_AndIsNotPersisted() Assert.Contains("idx_vector", ex.Message); Assert.Contains("removed from the model", ex.Message); Assert.IsType(ex.InnerException); // the dimension mismatch + Assert.Contains($"{ex.InnerException!.Message.TrimEnd('.')}. Nothing was saved", ex.Message); // Nothing was registered, so the data stays readable and the next open tries again. using (var v1 = new IndexEvoV1DbContext(_dbPath)) From 4a15f87b4e88239994c5e881eca483dee9e90663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:20:42 +0200 Subject: [PATCH 04/15] fix(index): free a partially built model index and roll back a failed registration (#171) Address review of #175 on top of #170: - ApplyIndexBuilder fills the new index through the same helper as the vacuum rebuild (FillUnregisteredIndexAsync): its own transaction, pages collected through it before the rollback, then freed. Before, a fill that failed after splits freed only the root, so every failing open leaked the rest of the partial tree and grew the file. - PublishFilledIndex hands a filled index to Register or ReplaceIndex and frees it if it did not get registered; used by both callers. - CollectionIndexManager: Register and ReplaceIndex share PublishLocked, which restores the previous registration (or none) when the metadata cannot be saved, and refreshes the unique-index flag. Register skipped that refresh, so a unique index added by the model was not enforced until the next open. - CreateUnregisteredCopy is gone; callers use CreateUnregistered with the existing definition, which now checks for disposal. Tests: a fill that fails on the last of 3000 documents does not grow the file over repeated opens; a failed metadata save leaves the index unregistered and frees its pages; a unique model index is enforced in the session that adds it. All three fail without these changes. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 56 ++++++--- .../Indexing/CollectionIndexManager.cs | 91 +++++++------- tests/BLite.Tests/IndexBackfillTests.cs | 117 ++++++++++++++++++ 3 files changed, 201 insertions(+), 63 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index fe1c910..3d39d94 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -724,7 +724,7 @@ public async Task VacuumAsync(VacuumOptions? options = null, CancellationToken c // RTreeIndex cannot enumerate its pages yet, so a rebuild would leak the // whole old tree on every vacuum. Spatial indexes are kept as they are. if (index.Definition.Type == IndexType.Spatial) continue; - await RebuildIndexFromScratchAsync(index.Name, ct).ConfigureAwait(false); + await RebuildIndexFromScratchAsync(index, ct).ConfigureAwait(false); } } @@ -1088,18 +1088,16 @@ internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) try { // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. - Task.Run(() => RebuildIndexAsync(index, skipUnreadableDocuments: false)).GetAwaiter().GetResult(); + Task.Run(() => FillUnregisteredIndexAsync(index, CancellationToken.None)).GetAwaiter().GetResult(); } catch (Exception ex) { - index.FreeAllPages(_storage); - index.Dispose(); throw new InvalidOperationException( $"Cannot build index '{name}' of collection '{_collectionName}' from the documents already in it: {ex.Message.TrimEnd('.')}. " + "Nothing was saved, but every open fails the same way until those documents can be indexed or the index " + "is removed from the model (OnModelCreating).", ex); } - _indexManager.Register(index); + PublishFilledIndex(index, _indexManager.Register); } /// @@ -1603,55 +1601,75 @@ public async IAsyncEnumerable QueryIndexAsync( } /// - /// Populates an empty copy of index from the documents and swaps it in + /// Populates an empty copy of from the documents and swaps it in /// only after the rebuild committed; the old tree is then freed. If any document cannot be /// loaded or indexed, the rebuild fails and the existing index stays in place untouched, /// instead of being replaced by an incomplete one. /// - private async Task RebuildIndexFromScratchAsync(string name, CancellationToken ct) + private async Task RebuildIndexFromScratchAsync(CollectionSecondaryIndex current, CancellationToken ct) + { + var fresh = _indexManager.CreateUnregistered(current.Definition); + await FillUnregisteredIndexAsync(fresh, ct).ConfigureAwait(false); + PublishFilledIndex(fresh, copy => _indexManager.ReplaceIndex(current.Name, copy)); + } + + /// + /// Fills an index that is not registered yet (see ) + /// from every document, in its own transaction, and commits it. If a document cannot be loaded + /// or indexed, or the commit fails, every page of the index is freed, including the pages its + /// splits added, the index is disposed and the exception propagates. + /// + private async Task FillUnregisteredIndexAsync(CollectionSecondaryIndex index, CancellationToken ct) { - var fresh = _indexManager.CreateUnregisteredCopy(name); var transaction = _storage.BeginTransaction(IsolationLevel.ReadCommitted); try { - await FillIndexAsync(fresh, transaction, skipUnreadableDocuments: false, ct).ConfigureAwait(false); + await FillIndexAsync(index, transaction, skipUnreadableDocuments: false, ct).ConfigureAwait(false); await transaction.CommitAsync(ct); } catch { - // Collect the copy's pages through the transaction before rolling it back: pages added + // Collect the index's pages through the transaction before rolling it back: pages added // by splits are allocated outside the transaction but linked only from nodes written // in it, so after the rollback they could not be found any more. IReadOnlyList pages = Array.Empty(); - try { pages = fresh.CollectAllPages(transaction.TransactionId); } + try { pages = index.CollectAllPages(transaction.TransactionId); } catch { /* best-effort: never mask the original failure */ } try { await transaction.RollbackAsync(); } catch { /* best-effort: never mask the original failure */ } - FreeUnregisteredCopy(fresh, pages); + FreeUnregisteredIndex(index, pages); throw; } + } + /// + /// Hands an index filled by to + /// ( or ). + /// If that fails and the index did not get registered, nothing refers to it any more, so its + /// pages are freed and it is disposed. + /// + private void PublishFilledIndex(CollectionSecondaryIndex index, Action> register) + { try { - _indexManager.ReplaceIndex(name, fresh); + register(index); } catch { - // The copy is committed but, unless the swap itself went through, nothing refers to it. - if (!ReferenceEquals(_indexManager.GetIndex(name), fresh)) + if (!ReferenceEquals(_indexManager.GetIndex(index.Name), index)) { IReadOnlyList pages = Array.Empty(); - try { pages = fresh.CollectAllPages(0); } + try { pages = index.CollectAllPages(0); } catch { /* best-effort: never mask the original failure */ } - FreeUnregisteredCopy(fresh, pages); + FreeUnregisteredIndex(index, pages); } throw; } } - private void FreeUnregisteredCopy(CollectionSecondaryIndex copy, IReadOnlyList pages) + private void FreeUnregisteredIndex(CollectionSecondaryIndex index, IReadOnlyList pages) { try { @@ -1659,7 +1677,7 @@ private void FreeUnregisteredCopy(CollectionSecondaryIndex copy, IReadOn _storage.FreePage(pageId); } catch { /* best-effort: never mask the original failure */ } - copy.Dispose(); + index.Dispose(); } private async Task RebuildIndexAsync(CollectionSecondaryIndex index, CancellationToken ct = default, bool skipUnreadableDocuments = true) diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index dd035f9..1b75c05 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -153,14 +153,24 @@ internal static CollectionIndexDefinition BuildDefinition( /// /// Creates an index for without registering or persisting it, /// so it can be filled before queries or the collection metadata see it. The index owns its - /// root page: hand it to or release it with - /// (a spatial index cannot release - /// its pages yet, so an abandoned one leaks them). + /// root page: hand it to or , or release its + /// pages (a spatial index cannot enumerate its pages yet, so an abandoned one leaks them). /// internal CollectionSecondaryIndex CreateUnregistered(CollectionIndexDefinition definition) - => new(definition, _storage, _mapper); + { + lock (_lock) + { + if (_disposed) + throw new ObjectDisposedException(nameof(CollectionIndexManager)); + + return new CollectionSecondaryIndex(definition, _storage, _mapper); + } + } - /// Registers an index built by and persists it. + /// + /// Registers an index built by and persists it. If the + /// metadata cannot be saved, the index is not registered and remains the caller's to release. + /// internal void Register(CollectionSecondaryIndex index) { lock (_lock) @@ -170,9 +180,7 @@ internal void Register(CollectionSecondaryIndex index) if (_indexes.ContainsKey(index.Name)) throw new InvalidOperationException($"Index '{index.Name}' already exists"); - _indexes[index.Name] = index; - _cachedIndexInfo = null; // invalidate cache - SaveMetadata(); + PublishLocked(index.Name, index, previous: null); } } @@ -352,32 +360,13 @@ internal CollectionSecondaryIndex RecreateIndex(string name) if (!_indexes.TryGetValue(name, out var old)) throw new InvalidOperationException($"Index '{name}' does not exist"); - var fresh = CreateUnregisteredCopy(name); + var fresh = CreateUnregistered(old.Definition); fresh.KeyFormat = old.KeyFormat; ReplaceIndex(name, fresh); return fresh; } } - /// - /// Creates an empty index with the same definition as without - /// registering it, so queries keep using the existing index while the copy is populated. - /// The copy owns its root page: hand it to or release it with - /// . - /// - internal CollectionSecondaryIndex CreateUnregisteredCopy(string name) - { - lock (_lock) - { - if (_disposed) - throw new ObjectDisposedException(nameof(CollectionIndexManager)); - if (!_indexes.TryGetValue(name, out var old)) - throw new InvalidOperationException($"Index '{name}' does not exist"); - - return new CollectionSecondaryIndex(old.Definition, _storage, _mapper); - } - } - /// /// Registers under , persists its root page /// and frees the pages of the index it replaces. If the metadata cannot be saved, the old @@ -392,29 +381,43 @@ internal void ReplaceIndex(string name, CollectionSecondaryIndex fresh) if (!_indexes.TryGetValue(name, out var old)) throw new InvalidOperationException($"Index '{name}' does not exist"); - _indexes[name] = fresh; - _cachedIndexInfo = null; // invalidate cache - RefreshHasUniqueBTreeIndex(); - // Persist the new root before freeing the old tree: a crash in between leaks the old // pages, whereas the reverse order would let the next open free them a second time. - try - { - SaveMetadata(); - } - catch - { - // Keep the old index registered, matching the metadata on disk; the caller still - // owns the copy. - _indexes[name] = old; - UpdateMetadata(); - throw; - } + PublishLocked(name, fresh, old); old.FreeAllPages(_storage); old.Dispose(); } } + /// + /// Registers under in place of + /// (null: none) and saves the metadata. If the save fails, the + /// previous registration is restored, so memory keeps matching the metadata on disk, and the + /// exception propagates. Must be called under . + /// + private void PublishLocked(string name, CollectionSecondaryIndex index, CollectionSecondaryIndex? previous) + { + _indexes[name] = index; + _cachedIndexInfo = null; // invalidate cache + RefreshHasUniqueBTreeIndex(); + + try + { + SaveMetadata(); + } + catch + { + if (previous != null) + _indexes[name] = previous; + else + _indexes.Remove(name); + _cachedIndexInfo = null; + RefreshHasUniqueBTreeIndex(); + UpdateMetadata(); + throw; + } + } + /// Records the key format an index's pages are written with and persists it. internal void SetKeyFormat(string name, byte keyFormat) { diff --git a/tests/BLite.Tests/IndexBackfillTests.cs b/tests/BLite.Tests/IndexBackfillTests.cs index 9d59801..7fec0a4 100644 --- a/tests/BLite.Tests/IndexBackfillTests.cs +++ b/tests/BLite.Tests/IndexBackfillTests.cs @@ -1,4 +1,11 @@ +using System.Linq.Expressions; +using System.Reflection; +using BLite.Bson; using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Indexing; +using BLite.Core.Metadata; +using BLite.Core.Storage; using BLite.Shared; namespace BLite.Tests; @@ -98,4 +105,114 @@ public async Task IndexThatCannotBeFilled_FailsTheOpen_AndIsNotPersisted() } Assert.Throws(() => new IndexEvoBadVectorDbContext(_dbPath)); } + + /// + /// A fill that fails after the index has split must free every page it added, not only the + /// root (#171 review): the error repeats on every open until the data is fixed, so a leak would + /// grow the file each time. + /// + [Fact] + public async Task IndexThatFailsLate_FreesEveryPageItAdded_SoRepeatedOpensDoNotGrowTheFile() + { + using (var v1 = new IndexEvoV1DbContext(_dbPath)) + { + await v1.Vectors.InsertBulkAsync(Enumerable.Range(0, 3000) + .Select(i => new VectorEntity { Title = $"v{i}", Embedding = [i, i % 7, i % 13] }).ToList()); + + // Give the last document in primary-key order a wrong-dimension embedding, so the fill + // fails only after thousands of inserts. + var primary = (BTreeIndex)typeof(DocumentCollection) + .GetField("_primaryIndex", BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(v1.Vectors)!; + var lastId = primary.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Backward).First().Key.As(); + var last = (await v1.Vectors.FindByIdAsync(lastId))!; + last.Embedding = [1f, 2f]; + await v1.Vectors.UpdateAsync(last); + } + + var sizes = new List(); + for (int open = 0; open < 4; open++) + { + var ex = Assert.Throws(() => new IndexEvoV2DbContext(_dbPath)); + Assert.Contains("idx_vector", ex.Message); + sizes.Add(new FileInfo(_dbPath).Length); + } + + // The first failed open may extend the file; the following ones reuse the freed pages. + Assert.All(sizes.Skip(1), size => Assert.Equal(sizes[0], size)); + using var reopened = new IndexEvoV1DbContext(_dbPath); + Assert.DoesNotContain(reopened.Vectors.GetIndexes(), i => i.Name == "idx_vector"); + } + + /// + /// If the filled index cannot be persisted, it is not left registered in memory either, and + /// its pages are freed; the next open builds it again. + /// + [Fact] + public async Task AddedIndex_WhoseMetadataCannotBeSaved_IsNotRegistered_AndItsPagesAreFreed() + { + using (var v1 = new IndexEvoV1DbContext(_dbPath)) + { + await v1.People.InsertBulkAsync(Enumerable.Range(0, 3000) + .Select(i => new PersonV2 { Name = $"p{i}", Age = i % 90 }).ToList()); + await v1.CheckpointAsync(); + + // Hold the metadata lock so saving the new index times out. + var metadataLock = (SemaphoreSlim)typeof(StorageEngine) + .GetField("_metadataLock", BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(v1.Storage)!; + Expression> age = p => p.Age; + metadataLock.Wait(); + try + { + Assert.Throws(() => v1.People.ApplyIndexBuilder(new IndexBuilder(age, "idx_Age", unique: false))); + } + finally + { + metadataLock.Release(); + } + + Assert.Null(v1.People.IndexManager.GetIndex("idx_Age")); + Assert.DoesNotContain(v1.People.GetIndexes(), i => i.Name == "idx_Age"); + Assert.DoesNotContain(v1.Storage.GetCollectionMetadata("evo_people")!.Indexes, i => i.Name == "idx_Age"); + await AssertIndexPagesWereFreedAsync(v1); + } + + using var v2 = new IndexEvoV2DbContext(_dbPath); + Assert.Equal(34, (await v2.People.QueryIndexAsync("idx_Age", 3, 3).ToListAsync()).Count); + } + + /// + /// A unique index added by the model is enforced in the session that creates it, not only after + /// the next open. + /// + [Fact] + public async Task AddedUniqueIndex_IsEnforcedRightAway() + { + await SeedAsync(); + using var v1 = new IndexEvoV1DbContext(_dbPath); + Expression> name = p => p.Name; + + v1.People.ApplyIndexBuilder(new IndexBuilder(name, "idx_Name", unique: true)); + + await Assert.ThrowsAnyAsync(async () => await v1.People.InsertAsync(new PersonV2 { Name = "p99", Age = 1 })); + await v1.People.InsertAsync(new PersonV2 { Name = "new", Age = 1 }); + Assert.Single(v1.People.AsQueryable().ToList(), p => p.Name == "p99"); + } + + /// + /// A 3000-entry B-Tree spans dozens of pages. Each must be back on the free list (handed out + /// next, below the high-water mark) or queued for the next WAL truncation. + /// + private static async Task AssertIndexPagesWereFreedAsync(IndexEvoV1DbContext db) + { + var storage = db.Storage; + await db.CheckpointAsync(); + var flags = BindingFlags.NonPublic | BindingFlags.Instance; + int queued = ((HashSet)typeof(StorageEngine).GetField("_pendingFreePages", flags)!.GetValue(storage)!).Count; + var highWater = ((IPageStorage)typeof(StorageEngine).GetField("_pageFile", flags)!.GetValue(storage)!).NextPageId; + int reused = 0; + while (storage.AllocatePage() < highWater) reused++; + Assert.True(reused + queued > 20, $"only {reused} pages were returned to the free list and {queued} queued"); + } } From 758717bf735839141ae1133a093a8ba568c79c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:28:02 +0200 Subject: [PATCH 05/15] fix(index): migrate legacy decimal indexes into a copy and free it on failure (#171) MigrateLegacyDecimalIndexes replaced the index with an empty tree (RecreateIndex) and then filled it in place. A rebuild that failed after splits rolled back and left those pages linked from nowhere, and the next open freed only the root of the emptied tree, so every failing open leaked the partial tree and grew the file. The legacy tree was also gone after the first failure. The migration now uses the same helpers as model indexes and the vacuum rebuild: it fills an unregistered copy (FillUnregisteredIndexAsync) set to the current key format and swaps it in with ReplaceIndex through PublishFilledIndex, so the new format and root are one metadata save. On failure the copy is freed and the old index stays registered in its old format with its data, and the next open retries. RecreateIndex has no callers left and is removed. Test: a legacy-format decimal index over 3000 documents whose last one cannot be read; four failing opens do not grow the file, the stored index keeps its root and legacy format, and the migration succeeds once the document is repaired. Fails without this change. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 17 +++-- .../Indexing/CollectionIndexManager.cs | 20 ------ tests/BLite.Tests/DecimalIndexTests.cs | 67 +++++++++++++++++++ 3 files changed, 77 insertions(+), 27 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 3d39d94..c25c5c4 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -870,10 +870,12 @@ internal DocumentCollection(StorageEngine storage, ITransactionHolder transactio /// /// Decimal properties used to be indexed as (double)value. An index whose persisted - /// predates the exact decimal encoding is replaced by an - /// empty tree, repopulated from the documents and then marked with the current format, the first - /// time the collection is opened. The format is recorded only after the rebuild committed, so a - /// crash in between just repeats the migration on the next open. No-op for every other index. + /// predates the exact decimal encoding is rebuilt from the + /// documents into a copy that queries don't see, the first time the collection is opened. Only + /// after the copy committed is it swapped in, with the current format, in one metadata save, and + /// the old tree freed. If the rebuild fails or is interrupted, the copy's pages are freed and the + /// old index stays registered in its old format, so the next open repeats the migration. + /// No-op for every other index. /// private void MigrateLegacyDecimalIndexes() { @@ -884,12 +886,13 @@ private void MigrateLegacyDecimalIndexes() var name = index.Name; try { - var fresh = _indexManager.RecreateIndex(name); + var fresh = _indexManager.CreateUnregistered(index.Definition); + fresh.KeyFormat = IndexMetadata.CurrentKeyFormat; // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. // A document that cannot be loaded fails the migration rather than silently leaving // the rebuilt index incomplete. - Task.Run(() => RebuildIndexAsync(fresh, skipUnreadableDocuments: false)).GetAwaiter().GetResult(); - _indexManager.SetKeyFormat(name, IndexMetadata.CurrentKeyFormat); + Task.Run(() => FillUnregisteredIndexAsync(fresh, CancellationToken.None)).GetAwaiter().GetResult(); + PublishFilledIndex(fresh, copy => _indexManager.ReplaceIndex(name, copy)); } catch (Exception ex) { diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index 1b75c05..3f9d2e0 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -347,26 +347,6 @@ public bool DropIndex(string name) } } - /// - /// Replaces an index with an empty one of the same definition, freeing the old tree's pages. - /// The new index keeps the old : the - /// caller repopulates it and then records the new format with , so an - /// interrupted rebuild is simply redone on the next open. - /// - internal CollectionSecondaryIndex RecreateIndex(string name) - { - lock (_lock) - { - if (!_indexes.TryGetValue(name, out var old)) - throw new InvalidOperationException($"Index '{name}' does not exist"); - - var fresh = CreateUnregistered(old.Definition); - fresh.KeyFormat = old.KeyFormat; - ReplaceIndex(name, fresh); - return fresh; - } - } - /// /// Registers under , persists its root page /// and frees the pages of the index it replaces. If the metadata cannot be saved, the old diff --git a/tests/BLite.Tests/DecimalIndexTests.cs b/tests/BLite.Tests/DecimalIndexTests.cs index d025193..fd69a20 100644 --- a/tests/BLite.Tests/DecimalIndexTests.cs +++ b/tests/BLite.Tests/DecimalIndexTests.cs @@ -1,5 +1,7 @@ +using System.Reflection; using BLite.Bson; using BLite.Core; +using BLite.Core.Collections; using BLite.Core.Indexing; using BLite.Core.Query; using BLite.Core.Query.Blql; @@ -220,6 +222,71 @@ public async Task KeyFormat_RoundTripsThroughMetadata() Assert.Equal(IndexMetadata.CurrentKeyFormat, priceIndex.KeyFormat); } + /// + /// A migration that fails late, after the rebuilt tree has split, keeps the legacy index as it + /// was and frees every page of the partial rebuild: it is retried on each open without growing + /// the file, and succeeds once the document can be read again. + /// + [Fact] + public async Task LegacyIndexMigration_ThatFailsLate_KeepsTheOldIndex_AndDoesNotGrowTheFile() + { + const int count = 3000; + uint legacyRoot; + uint corruptedPage; + byte[] intact; + using (var db = new TestDbContext(_dbPath)) + { + await db.Products.InsertBulkAsync(Enumerable.Range(1, count) + .Select(i => new Product { Id = i, Title = $"P{i}", Price = i + 0.01m }).ToList()); + db.Products.IndexManager.SetKeyFormat(PriceIndex(db).Name, IndexMetadata.LegacyKeyFormat); + legacyRoot = PriceIndex(db).RootPageId; + await db.CheckpointAsync(); + + // Make the last document in primary-key order unreadable, so the rebuild on open fails + // only after thousands of inserts. + var primary = (BTreeIndex)typeof(DocumentCollection) + .GetField("_primaryIndex", BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(db.Products)!; + var location = primary.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Backward).First().Location; + corruptedPage = location.PageId; + intact = new byte[db.Storage.PageSize]; + db.Storage.ReadPage(corruptedPage, null, intact); + var corrupt = (byte[])intact.Clone(); + var slot = SlotEntry.ReadFrom(corrupt.AsSpan(SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size)); + corrupt.AsSpan(slot.Offset + 4, 12).Fill(0xFF); // invalid element types + db.Storage.WritePageImmediate(corruptedPage, corrupt); + } + + var sizes = new List(); + for (int open = 0; open < 4; open++) + { + var ex = Assert.Throws(() => new TestDbContext(_dbPath)); + Assert.Contains("retried on the next open", ex.Message); + sizes.Add(new FileInfo(_dbPath).Length); + } + // The first failed open may extend the file; the following ones reuse the freed pages. + Assert.All(sizes.Skip(1), size => Assert.Equal(sizes[0], size)); + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + // The legacy index is still the registered one, so the next open retries the migration. + var stored = storage.GetCollectionMetadata("products_collection")!.Indexes + .Single(i => i.PropertyPaths.Single() == nameof(Product.Price)); + Assert.Equal(IndexMetadata.LegacyKeyFormat, stored.KeyFormat); + Assert.Equal(legacyRoot, stored.RootPageId); + storage.WritePageImmediate(corruptedPage, intact); + } + + using (var db = new TestDbContext(_dbPath)) + { + var index = PriceIndex(db); + Assert.Equal(IndexMetadata.CurrentKeyFormat, index.KeyFormat); + Assert.Equal(count, AllKeys(index).Count); + Assert.Equal(count, (await db.Products.AsQueryable().Where(p => p.Price >= 0m).ToListAsync()).Count); + Assert.Equal(1500, Assert.Single(await db.Products.AsQueryable().Where(p => p.Price == 1500.01m).ToListAsync()).Id); + } + } + private static List AllKeys(CollectionSecondaryIndex index) => index.BTreeIndex!.Range(IndexKey.NullSentinelNext, IndexKey.MaxKey).Select(e => e.Key).ToList(); From 9f48116ea89ec75bd5bc3290ee096de59a587cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 14:37:43 +0200 Subject: [PATCH 06/15] feat(index): full-text search with BM25 ranking (#70) Adds a full-text index type for string properties, stored as a companion inverted index in one B-Tree per index: - postings [0x01][term][0x00][location][tf][doc length] -> location, so BM25 scores come from the postings alone and a delete recomputes the exact keys from the old text; - one statistics entry [0x02][documents][terms], kept in the same transaction as the postings. Text is analyzed with TextNormalizer (accents removed, lowercased, punctuation as separators) and split into terms of at most 32 UTF-8 bytes. API: - HasFullTextIndex(x => x.Text) and CreateFullTextIndexAsync(x => x.Text) (default name idx_fts_{property}), filled from existing documents before the index is registered; - LINQ x.Text.FullTextMatch("terms"): documents containing every term, served from the index and evaluated with the same rules in memory; - FullTextSearchAsync(index, query, k, matchAllTerms) returning the top k documents with their BM25 score; - DynamicCollection.CreateFullTextIndexAsync / FullTextSearchAsync. Equality, range and IN lookups no longer pick a vector, spatial or full-text index just because it covers the property. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/BLiteMigration.cs | 3 + .../Collections/DocumentCollection.cs | 77 +++- src/BLite.Core/DynamicCollection.cs | 105 ++++- .../Indexing/CollectionIndexManager.cs | 15 +- .../Indexing/CollectionSecondaryIndex.cs | 47 ++- src/BLite.Core/Indexing/FullTextExtensions.cs | 23 ++ src/BLite.Core/Indexing/FullTextIndex.cs | 221 ++++++++++ .../Indexing/FullTextSearchResult.cs | 6 + src/BLite.Core/Indexing/IndexOptions.cs | 5 +- src/BLite.Core/Metadata/EntityTypeBuilder.cs | 11 + src/BLite.Core/Query/IndexOptimizer.cs | 38 +- .../Storage/StorageEngine.Memory.cs | 7 + src/BLite.Core/Text/FullTextAnalyzer.cs | 95 +++++ tests/BLite.Shared/FullTextContexts.cs | 44 ++ tests/BLite.Tests/FullTextSearchTests.cs | 381 ++++++++++++++++++ 15 files changed, 1062 insertions(+), 16 deletions(-) create mode 100644 src/BLite.Core/Indexing/FullTextExtensions.cs create mode 100644 src/BLite.Core/Indexing/FullTextIndex.cs create mode 100644 src/BLite.Core/Indexing/FullTextSearchResult.cs create mode 100644 src/BLite.Core/Text/FullTextAnalyzer.cs create mode 100644 tests/BLite.Shared/FullTextContexts.cs create mode 100644 tests/BLite.Tests/FullTextSearchTests.cs diff --git a/src/BLite.Core/BLiteMigration.cs b/src/BLite.Core/BLiteMigration.cs index 3388358..cbec33d 100644 --- a/src/BLite.Core/BLiteMigration.cs +++ b/src/BLite.Core/BLiteMigration.cs @@ -290,6 +290,9 @@ private static async Task RecreateIndexAsync(DynamicCollection col, IndexMetadat case IndexType.Spatial: await col.CreateSpatialIndexAsync(field, idx.Name); break; + case IndexType.FullText: + await col.CreateFullTextIndexAsync(field, idx.Name); + break; } } diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index edab085..4e11399 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -992,6 +992,38 @@ public async Task> CreateIndexAsync( } } + /// + /// Creates a full-text index on a string property, filled from the documents already in the + /// collection, or returns the existing index of that name. Query it with + /// in LINQ or rank with + /// . The default name is idx_fts_{property}. + /// + [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] + public async Task> CreateFullTextIndexAsync( + System.Linq.Expressions.Expression> keySelector, + string? name = null, + CancellationToken ct = default) + { + if (keySelector == null) throw new ArgumentNullException(nameof(keySelector)); + + // Block writers while the index is filled: a document inserted meanwhile would miss the + // index, which only starts receiving writes once it is registered. + if (!await _collectionLock.WaitAsync(WriteLockTimeoutMs, ct).ConfigureAwait(false)) + throw new TimeoutException("Timed out acquiring collection write lock (CreateFullTextIndexAsync)."); + try + { + var builder = new Metadata.IndexBuilder(keySelector, name, false, IndexType.FullText); + await Task.Run(() => ApplyIndexBuilder(builder), ct).ConfigureAwait(false); + } + finally + { + _collectionLock.Release(); + } + + var indexName = CollectionIndexManager.ResolveIndexName(keySelector, name, IndexType.FullText); + return await BindReaderAndReturn(_indexManager.GetIndex(indexName)!); + } + /// /// Asynchronously creates a vector (HNSW) index for similarity search. /// @@ -1075,7 +1107,7 @@ public IEnumerable GetIndexes() [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) { - var name = CollectionIndexManager.ResolveIndexName(builder.KeySelector, builder.Name); + var name = CollectionIndexManager.ResolveIndexName(builder.KeySelector, builder.Name, builder.Type); if (_indexManager.GetIndex(name) != null) return; @@ -3437,6 +3469,7 @@ internal async Task CountByPredicateAsync( if (indexOpt != null && !indexOpt.IsVectorSearch && !indexOpt.IsSpatialSearch + && !indexOpt.IsFullTextSearch && !indexOpt.HasResiduePredicate) { var index = _indexManager.GetIndex(indexOpt.IndexName); @@ -3533,7 +3566,7 @@ public IAsyncEnumerable FindAsync( // Path: IndexOptimizer → CollectionSecondaryIndex.Seek (TryFindFirst) → FindByLocationAsync. var indexOpt = Query.IndexOptimizer.TryOptimize(predicate, GetIndexes(), ConverterRegistry); if (indexOpt != null && !indexOpt.IsRange && !indexOpt.IsVectorSearch && !indexOpt.IsSpatialSearch - && indexOpt.MinValue != null) + && !indexOpt.IsFullTextSearch && indexOpt.MinValue != null) { var index = _indexManager.GetIndex(indexOpt.IndexName); if (index != null) @@ -3634,6 +3667,15 @@ Func GetCompiled() => if (++yielded >= fetchLimit) yield break; } } + else if (indexOpt.IsFullTextSearch) + { + var ftsIndex = _indexManager.GetIndex(indexOpt.IndexName)!; + foreach (var location in ftsIndex.FullTextMatch(indexOpt.FullTextQuery!, transaction)) + { + var item = await FindByLocationAsync(location, transaction?.TransactionId ?? 0UL, ct); + if (item != null && (indexOpt.FilterCompleteness == Query.IndexOptimizer.FilterCompleteness.Exact || GetCompiled()(item))) { yield return item; if (++yielded >= fetchLimit) yield break; } + } + } else if (indexOpt.IsSpatialSearch) { var spatialSeq = indexOpt.SpatialType == Query.IndexOptimizer.SpatialQueryType.Near @@ -3861,6 +3903,37 @@ public async IAsyncEnumerable VectorSearchAsync( } } + /// + /// Returns the documents that best match in a + /// full-text index, best first, each with its BM25 relevance score. + /// + /// The name of the full-text index to search. + /// The terms to look for, analyzed like the indexed text. + /// Maximum number of documents to return. + /// + /// true (the default) to return only documents containing every query term; + /// false to return documents containing any of them, ranked by relevance. + /// + /// Cancellation token. + public async IAsyncEnumerable> FullTextSearchAsync( + string indexName, + string query, + int k, + bool matchAllTerms = true, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) + { + var index = _indexManager.GetIndex(indexName); + if (index == null) + throw new ArgumentException($"Index '{indexName}' not found.", nameof(indexName)); + + foreach (var (location, score) in index.FullTextSearch(query, k, matchAllTerms)) + { + ct.ThrowIfCancellationRequested(); + var doc = await FindByLocationAsync(location, 0UL, ct); + if (doc != null) yield return new FullTextSearchResult(doc, score); + } + } + /// /// Asynchronously finds all documents located within a specified radius of a geographic center /// point using a spatial index. diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index 67966dd..f6fcf27 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -28,6 +28,7 @@ public sealed record DynamicIndexDescriptor( { public bool IsVector => Type == IndexType.Vector; public bool IsSpatial => Type == IndexType.Spatial; + public bool IsFullText => Type == IndexType.FullText; public bool IsStandard => Type == IndexType.BTree; } @@ -59,7 +60,7 @@ public sealed class DynamicCollection : IDisposable private int _retentionRunning; // 0=idle, 1=running (Interlocked flag) // ── Discriminated union for secondary indexes ───────────────────────────── - private enum DynamicIndexKind { BTree, Vector, Spatial } + private enum DynamicIndexKind { BTree, Vector, Spatial, FullText } private sealed class DynamicSecondaryIndex { @@ -69,7 +70,8 @@ private sealed class DynamicSecondaryIndex public BTreeIndex? BTree { get; } public VectorSearchIndex? Vector { get; } public RTreeIndex? Spatial { get; } - public uint RootPageId => BTree?.RootPageId ?? Vector?.RootPageId ?? Spatial?.RootPageId ?? 0; + public FullTextIndex? FullText { get; } + public uint RootPageId => BTree?.RootPageId ?? Vector?.RootPageId ?? Spatial?.RootPageId ?? FullText?.RootPageId ?? 0; public DynamicSecondaryIndex(BTreeIndex btree, string fieldPath, IndexOptions options) { Kind = DynamicIndexKind.BTree; BTree = btree; FieldPath = fieldPath; Options = options; } @@ -79,6 +81,9 @@ public DynamicSecondaryIndex(VectorSearchIndex vector, string fieldPath, IndexOp public DynamicSecondaryIndex(RTreeIndex spatial, string fieldPath, IndexOptions options) { Kind = DynamicIndexKind.Spatial; Spatial = spatial; FieldPath = fieldPath; Options = options; } + + public DynamicSecondaryIndex(FullTextIndex fullText, string fieldPath, IndexOptions options) + { Kind = DynamicIndexKind.FullText; FullText = fullText; FieldPath = fieldPath; Options = options; } } // Secondary indexes: name → DynamicSecondaryIndex @@ -158,6 +163,13 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction _secondaryIndexes[idxMeta.Name] = new DynamicSecondaryIndex(spatial, fieldPath, opts); break; } + case IndexType.FullText: + { + var opts = IndexOptions.CreateBTree(idxMeta.PropertyPaths); + var fullText = new FullTextIndex(_storage, opts, idxMeta.RootPageId); + _secondaryIndexes[idxMeta.Name] = new DynamicSecondaryIndex(fullText, fieldPath, opts); + break; + } } } } @@ -985,6 +997,30 @@ public async IAsyncEnumerable WithinAsync(string indexName, (doubl } } + /// + /// Returns the documents that best match in a + /// full-text index, best first, each with its BM25 relevance score. + /// The field must have been indexed via . + /// + /// The name of the full-text index to search. + /// The terms to look for, analyzed like the indexed text. + /// Maximum number of documents to return. + /// + /// true (the default) to return only documents containing every query term; + /// false to return documents containing any of them, ranked by relevance. + /// + public async IAsyncEnumerable> FullTextSearchAsync(string indexName, string query, int k, bool matchAllTerms = true) + { + if (!_secondaryIndexes.TryGetValue(indexName, out var entry) || entry.Kind != DynamicIndexKind.FullText || entry.FullText == null) + throw new ArgumentException($"Full-text index '{indexName}' not found on collection '{_collectionName}'"); + + foreach (var (location, score) in entry.FullText.Search(query, k, matchAllTerms, null)) + { + var doc = ReadDocumentAt(location, 0UL); + if (doc != null) yield return new FullTextSearchResult(doc, score); + } + } + /// /// Asynchronously yields documents matching the specified predicate. /// @@ -1587,6 +1623,7 @@ public async Task VacuumAsync(VacuumOptions? options = null, CancellationToken c DynamicIndexKind.BTree => new DynamicSecondaryIndex(new BTreeIndex(_storage, entry.Options), entry.FieldPath, entry.Options), DynamicIndexKind.Vector => new DynamicSecondaryIndex(new VectorSearchIndex(_storage, entry.Options), entry.FieldPath, entry.Options), DynamicIndexKind.Spatial => new DynamicSecondaryIndex(new RTreeIndex(_storage, entry.Options, 0), entry.FieldPath, entry.Options), + DynamicIndexKind.FullText => new DynamicSecondaryIndex(new FullTextIndex(_storage, entry.Options), entry.FieldPath, entry.Options), _ => entry }; rebuildEntries[name] = freshEntry; @@ -1737,6 +1774,56 @@ public async Task CreateSpatialIndexAsync(string fieldPath, string? name = null) PersistIndexMetadata(); } + /// + /// Creates a full-text index on a string field (dot-notation for nested fields), filled from + /// the documents already in the collection. Search it with . + /// The default name is idx_fts_{field}. + /// + public async Task CreateFullTextIndexAsync(string fieldPath, string? name = null) + { + name ??= $"idx_fts_{fieldPath.ToLowerInvariant()}"; + fieldPath = fieldPath.ToLowerInvariant(); + + // Block writers while the index is filled: a document inserted meanwhile would miss the + // index, which only starts receiving writes once it is registered. + if (!await _collectionLock.WaitAsync(WriteLockTimeoutMs).ConfigureAwait(false)) + throw new TimeoutException("Timed out acquiring collection write lock (CreateFullTextIndexAsync)."); + try + { + if (_secondaryIndexes.ContainsKey(name)) + throw new InvalidOperationException($"Index '{name}' already exists"); + + RegisterNestedPathKeys(fieldPath); + + var opts = IndexOptions.CreateBTree(fieldPath); + var entry = new DynamicSecondaryIndex(new FullTextIndex(_storage, opts), fieldPath, opts); + + // Fill before registering, so a crash or a failure leaves no partial index persisted. + var transaction = _storage.BeginTransaction(IsolationLevel.ReadCommitted); + try + { + foreach (var e in _primaryIndex.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transaction.TransactionId)) + { + var doc = ReadDocumentAt(e.Location, transaction.TransactionId); + if (doc != null) IndexInsert(entry, doc, e.Location, transaction); + } + await transaction.CommitAsync().ConfigureAwait(false); + } + catch + { + await transaction.RollbackAsync().ConfigureAwait(false); + throw; + } + + _secondaryIndexes[name] = entry; + PersistIndexMetadata(); + } + finally + { + _collectionLock.Release(); + } + } + /// Drops a secondary index by name. public bool DropIndex(string name) { @@ -1795,8 +1882,9 @@ public IReadOnlyList GetIndexDescriptors() { var type = kvp.Value.Kind switch { - DynamicIndexKind.Vector => IndexType.Vector, - DynamicIndexKind.Spatial => IndexType.Spatial, + DynamicIndexKind.Vector => IndexType.Vector, + DynamicIndexKind.Spatial => IndexType.Spatial, + DynamicIndexKind.FullText => IndexType.FullText, _ => IndexType.BTree }; return new DynamicIndexDescriptor( @@ -1838,6 +1926,9 @@ internal void PersistIndexMetadata() case DynamicIndexKind.Spatial: idxMeta.Type = IndexType.Spatial; break; + case DynamicIndexKind.FullText: + idxMeta.Type = IndexType.FullText; + break; } metadata.Indexes.Add(idxMeta); @@ -1867,6 +1958,9 @@ private void IndexInsert(DynamicSecondaryIndex idx, BsonDocument document, Docum var coords = ExtractCoordinates(val); if (coords.HasValue) idx.Spatial!.Insert(GeoBox.FromPoint(new GeoPoint(coords.Value.Lat, coords.Value.Lon)), location, transaction); break; + case DynamicIndexKind.FullText: + if (val.IsString) idx.FullText!.Insert(val.AsString, location, transaction); + break; } } @@ -1889,6 +1983,9 @@ private void IndexDelete(DynamicSecondaryIndex idx, BsonDocument document, Docum var coords = ExtractCoordinates(val); if (coords.HasValue) idx.Spatial!.Delete(GeoBox.FromPoint(new GeoPoint(coords.Value.Lat, coords.Value.Lon)), location, transaction); break; + case DynamicIndexKind.FullText: + if (val.IsString) idx.FullText!.Delete(val.AsString, location, transaction); + break; } } diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index 8ea9aa0..43ae3f7 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -128,9 +128,18 @@ public CollectionSecondaryIndex CreateIndex(CollectionIndexDefinition } } - /// The name an index on gets when none is given. - internal static string ResolveIndexName(LambdaExpression keySelector, string? name) - => name ?? GenerateIndexName(ExpressionAnalyzer.ExtractPropertyPaths(keySelector)); + /// + /// The name an index on gets when none is given. Full-text + /// indexes get their own prefix, so they can sit next to a B-Tree index on the same property. + /// + internal static string ResolveIndexName(LambdaExpression keySelector, string? name, IndexType type = IndexType.BTree) + { + if (name != null) return name; + var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector); + return type == IndexType.FullText + ? $"idx_fts_{string.Join("_", propertyPaths)}" + : GenerateIndexName(propertyPaths); + } /// /// Builds an index definition from an untyped key selector, as declared in the model. diff --git a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs index 2c7a65c..f548450 100644 --- a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs +++ b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs @@ -47,6 +47,7 @@ static CollectionSecondaryIndex() private readonly BTreeIndex? _btreeIndex; private readonly VectorSearchIndex? _vectorIndex; private readonly RTreeIndex? _spatialIndex; + private readonly FullTextIndex? _fullTextIndex; private readonly IDocumentMapper _mapper; private bool _disposed; private long _documentCount; @@ -61,7 +62,7 @@ static CollectionSecondaryIndex() /// public BTreeIndex? BTreeIndex => _btreeIndex; - public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? 0; + public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? _fullTextIndex?.RootPageId ?? 0; public CollectionSecondaryIndex( CollectionIndexDefinition definition, @@ -86,6 +87,13 @@ public CollectionSecondaryIndex( _btreeIndex = null; _vectorIndex = null; } + else if (indexOptions.Type == IndexType.FullText) + { + _fullTextIndex = new FullTextIndex(storage, indexOptions, rootPageId); + _btreeIndex = null; + _vectorIndex = null; + _spatialIndex = null; + } else { _btreeIndex = new BTreeIndex(storage, indexOptions, rootPageId); @@ -214,6 +222,13 @@ private void InsertKey(object keyValue, T document, DocumentLocation location, I _spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction); } } + else if (_fullTextIndex != null) + { + if (keyValue is string text) + { + _fullTextIndex.Insert(text, location, transaction); + } + } else if (_btreeIndex != null) { // BTree Index logic @@ -264,6 +279,13 @@ private void DeleteKey(object keyValue, T document, DocumentLocation location, I _spatialIndex.Delete(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction); } } + else if (_fullTextIndex != null) + { + if (keyValue is string text) + { + _fullTextIndex.Delete(text, location, transaction); + } + } else if (_btreeIndex != null) { var userKey = ConvertToIndexKey(keyValue); @@ -359,6 +381,28 @@ public IEnumerable Within((double Latitude, double Longitude) return _spatialIndex.Search(area, transaction); } + /// + /// Locations of the documents whose indexed text contains every term of . + /// + public IReadOnlyList FullTextMatch(string query, ITransaction? transaction = null) + { + if (_fullTextIndex == null) + throw new InvalidOperationException("This index is not a full-text index."); + + return _fullTextIndex.Match(query, transaction); + } + + /// + /// The best BM25 matches for , best first. + /// + public IReadOnlyList<(DocumentLocation Location, double Score)> FullTextSearch(string query, int k, bool matchAllTerms = true, ITransaction? transaction = null) + { + if (_fullTextIndex == null) + throw new InvalidOperationException("This index is not a full-text index."); + + return _fullTextIndex.Search(query, k, matchAllTerms, transaction); + } + /// /// Scans a range of keys (O(log n + k) where k is result count) /// @@ -800,6 +844,7 @@ internal IReadOnlyList CollectAllPages(ulong transactionId) if (_btreeIndex != null) return _btreeIndex.CollectAllPages(transactionId); if (_vectorIndex != null) return _vectorIndex.CollectAllPages(transactionId); if (_spatialIndex != null) return _spatialIndex.CollectAllPages(transactionId); + if (_fullTextIndex != null) return _fullTextIndex.CollectAllPages(transactionId); return Array.Empty(); } } diff --git a/src/BLite.Core/Indexing/FullTextExtensions.cs b/src/BLite.Core/Indexing/FullTextExtensions.cs new file mode 100644 index 0000000..28c942a --- /dev/null +++ b/src/BLite.Core/Indexing/FullTextExtensions.cs @@ -0,0 +1,23 @@ +using BLite.Core.Text; + +namespace BLite.Core; + +public static class FullTextExtensions +{ + /// + /// Whether contains every term of . Terms are + /// compared after removing accents, lowercasing and treating anything that is not a letter or + /// digit as a separator, so "Café, au lait!" matches the query "cafe LAIT". + /// A query without any term matches nothing. + /// + /// The LINQ query provider serves this from the property's full-text index when there is one; + /// otherwise it is evaluated per document with the same rules. + /// + /// + /// The string property of the entity. + /// The terms to look for. + public static bool FullTextMatch(this string? text, string query) + { + return FullTextAnalyzer.Matches(text, query); + } +} diff --git a/src/BLite.Core/Indexing/FullTextIndex.cs b/src/BLite.Core/Indexing/FullTextIndex.cs new file mode 100644 index 0000000..1705f5c --- /dev/null +++ b/src/BLite.Core/Indexing/FullTextIndex.cs @@ -0,0 +1,221 @@ +using System.Buffers.Binary; +using System.Text; +using BLite.Core.Storage; +using BLite.Core.Text; +using BLite.Core.Transactions; + +namespace BLite.Core.Indexing; + +/// +/// Inverted index for full-text search, stored in a single B-Tree. +/// +/// Posting, one per distinct term of a document: +/// [0x01][term UTF-8][0x00][location (6)][term frequency u16][document length u32], +/// pointing at the document. The term frequency and length make BM25 scoring possible from the +/// postings alone; deleting a document recomputes the same keys from its old text. +/// Statistics, a single entry: [0x02][documents u64][terms u64], the number of +/// indexed documents and their total length, maintained in the same transaction as the +/// postings. +/// +/// Terms come from ; a document without any term is not indexed. +/// +internal sealed class FullTextIndex +{ + private const byte PostingTag = 0x01; + private const byte StatsTag = 0x02; + private const byte TermEnd = 0x00; + + // BM25 parameters (Robertson et al.; the usual defaults, also used by SQLite FTS5 and Lucene). + private const double K1 = 1.2; + private const double B = 0.75; + + private static readonly IndexKey StatsMin = IndexKey.FromOwnedArray([StatsTag]); + private static readonly IndexKey StatsMax = IndexKey.FromOwnedArray([StatsTag + 1]); + + private readonly BTreeIndex _tree; + + public FullTextIndex(StorageEngine storage, IndexOptions options, uint rootPageId = 0) + { + _tree = new BTreeIndex(storage, options, rootPageId); + } + + public uint RootPageId => _tree.RootPageId; + + public IReadOnlyList CollectAllPages() => _tree.CollectAllPages(); + + /// The index pages as sees them. + public IReadOnlyList CollectAllPages(ulong transactionId) => _tree.CollectAllPages(transactionId); + + public void Insert(string? text, DocumentLocation location, ITransaction? transaction) + { + var terms = FullTextAnalyzer.Analyze(text, out var length); + if (length == 0) return; + + var txnId = transaction?.TransactionId; + foreach (var pair in terms) + _tree.Insert(PostingKey(pair.Key, location, pair.Value, length), location, txnId); + AdjustStats(1, length, txnId); + } + + public void Delete(string? text, DocumentLocation location, ITransaction? transaction) + { + var terms = FullTextAnalyzer.Analyze(text, out var length); + if (length == 0) return; + + var txnId = transaction?.TransactionId; + bool found = false; + foreach (var pair in terms) + found |= _tree.Delete(PostingKey(pair.Key, location, pair.Value, length), location, txnId); + + // A document that was never indexed must not change the statistics. + if (found) + AdjustStats(-1, -length, txnId); + } + + /// Locations of the documents that contain every term of . + public IReadOnlyList Match(string? query, ITransaction? transaction) + { + var terms = FullTextAnalyzer.QueryTerms(query); + if (terms.Count == 0) return Array.Empty(); + + var txnId = transaction?.TransactionId; + // Start from the rarest term so the candidate list is as short as possible. + var ordered = terms + .Select(t => (Term: t, Count: _tree.CountRange(TermMin(t), TermMax(t), txnId))) + .OrderBy(t => t.Count) + .Select(t => t.Term) + .ToList(); + + var result = Postings(ordered[0], txnId).Select(p => p.Location).ToList(); + for (int i = 1; i < ordered.Count && result.Count > 0; i++) + { + var present = new HashSet(Postings(ordered[i], txnId).Select(p => Pack(p.Location))); + result.RemoveAll(l => !present.Contains(Pack(l))); + } + return result; + } + + /// + /// The documents with the highest BM25 score for , + /// best first. With only documents containing every query term + /// qualify; otherwise any term is enough and documents with more of them score higher. + /// + public IReadOnlyList<(DocumentLocation Location, double Score)> Search(string? query, int k, bool matchAllTerms, ITransaction? transaction) + { + var terms = FullTextAnalyzer.QueryTerms(query); + if (terms.Count == 0 || k <= 0) return Array.Empty<(DocumentLocation, double)>(); + + var txnId = transaction?.TransactionId; + var (documents, totalLength) = ReadStats(txnId); + if (documents == 0) return Array.Empty<(DocumentLocation, double)>(); + double averageLength = (double)totalLength / documents; + + var scores = new Dictionary(); + foreach (var term in terms) + { + var postings = Postings(term, txnId).ToList(); + if (postings.Count == 0) + { + if (matchAllTerms) return Array.Empty<(DocumentLocation, double)>(); + continue; + } + + double idf = Math.Log(1 + (documents - postings.Count + 0.5) / (postings.Count + 0.5)); + foreach (var p in postings) + { + double weight = idf * p.Frequency * (K1 + 1) + / (p.Frequency + K1 * (1 - B + B * p.Length / averageLength)); + var key = Pack(p.Location); + scores[key] = scores.TryGetValue(key, out var s) + ? (s.Location, s.Score + weight, s.Terms + 1) + : (p.Location, weight, 1); + } + } + + return scores.Values + .Where(s => !matchAllTerms || s.Terms == terms.Count) + .OrderByDescending(s => s.Score) + .ThenBy(s => Pack(s.Location)) + .Take(k) + .Select(s => (s.Location, s.Score)) + .ToList(); + } + + private IEnumerable<(DocumentLocation Location, int Frequency, int Length)> Postings(string term, ulong? txnId) + { + int offset = 2 + Encoding.UTF8.GetByteCount(term) + DocumentLocation.SerializedSize; + foreach (var entry in _tree.Range(TermMin(term), TermMax(term), IndexDirection.Forward, txnId)) + { + var key = entry.Key.Data; + yield return (entry.Location, + BinaryPrimitives.ReadUInt16BigEndian(key.Slice(offset)), + (int)BinaryPrimitives.ReadUInt32BigEndian(key.Slice(offset + 2))); + } + } + + private (long Documents, long Length) ReadStats(ulong? txnId) + { + foreach (var entry in _tree.Range(StatsMin, StatsMax, IndexDirection.Forward, txnId)) + { + var key = entry.Key.Data; + return ((long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)), + (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(9))); + } + return (0, 0); + } + + private void AdjustStats(long documents, long length, ulong? txnId) + { + long currentDocuments = 0, currentLength = 0; + foreach (var entry in _tree.Range(StatsMin, StatsMax, IndexDirection.Forward, txnId)) + { + var key = entry.Key.Data; + currentDocuments = (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)); + currentLength = (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(9)); + _tree.Delete(entry.Key, entry.Location, txnId); + break; + } + + long newDocuments = currentDocuments + documents; + long newLength = currentLength + length; + if (newDocuments <= 0) return; + + var stats = new byte[17]; + stats[0] = StatsTag; + BinaryPrimitives.WriteUInt64BigEndian(stats.AsSpan(1), (ulong)newDocuments); + BinaryPrimitives.WriteUInt64BigEndian(stats.AsSpan(9), (ulong)Math.Max(0, newLength)); + _tree.Insert(IndexKey.FromOwnedArray(stats), default, txnId); + } + + private static IndexKey PostingKey(string term, DocumentLocation location, int frequency, int length) + { + int termBytes = Encoding.UTF8.GetByteCount(term); + var key = new byte[2 + termBytes + DocumentLocation.SerializedSize + 2 + 4]; + key[0] = PostingTag; + Encoding.UTF8.GetBytes(term, 0, term.Length, key, 1); + key[1 + termBytes] = TermEnd; + int offset = 2 + termBytes; + location.WriteTo(key.AsSpan(offset, DocumentLocation.SerializedSize)); + offset += DocumentLocation.SerializedSize; + BinaryPrimitives.WriteUInt16BigEndian(key.AsSpan(offset), (ushort)Math.Min(frequency, ushort.MaxValue)); + BinaryPrimitives.WriteUInt32BigEndian(key.AsSpan(offset + 2), (uint)length); + return IndexKey.FromOwnedArray(key); + } + + // [0x01][term][0x00] .. [0x01][term][0x01]: every posting of exactly this term. A longer term + // sharing the prefix has a letter or digit (> 0x01) where these bounds have 0x00 / 0x01. + private static IndexKey TermMin(string term) => TermBound(term, TermEnd); + private static IndexKey TermMax(string term) => TermBound(term, TermEnd + 1); + + private static IndexKey TermBound(string term, int last) + { + int termBytes = Encoding.UTF8.GetByteCount(term); + var key = new byte[2 + termBytes]; + key[0] = PostingTag; + Encoding.UTF8.GetBytes(term, 0, term.Length, key, 1); + key[1 + termBytes] = (byte)last; + return IndexKey.FromOwnedArray(key); + } + + private static ulong Pack(DocumentLocation location) => ((ulong)location.PageId << 16) | location.SlotIndex; +} diff --git a/src/BLite.Core/Indexing/FullTextSearchResult.cs b/src/BLite.Core/Indexing/FullTextSearchResult.cs new file mode 100644 index 0000000..18c0675 --- /dev/null +++ b/src/BLite.Core/Indexing/FullTextSearchResult.cs @@ -0,0 +1,6 @@ +namespace BLite.Core; + +/// A document returned by a ranked full-text search, with its BM25 relevance score. +/// The matching document. +/// The BM25 score; higher is more relevant. Scores compare only within one search. +public readonly record struct FullTextSearchResult(T Document, double Score); diff --git a/src/BLite.Core/Indexing/IndexOptions.cs b/src/BLite.Core/Indexing/IndexOptions.cs index 387c028..f9bfdad 100644 --- a/src/BLite.Core/Indexing/IndexOptions.cs +++ b/src/BLite.Core/Indexing/IndexOptions.cs @@ -18,7 +18,10 @@ public enum IndexType : byte Vector = 4, /// Geospatial index (R-Tree) for spatial queries - Spatial = 5 + Spatial = 5, + + /// Inverted index for full-text search over a string property + FullText = 6 } /// diff --git a/src/BLite.Core/Metadata/EntityTypeBuilder.cs b/src/BLite.Core/Metadata/EntityTypeBuilder.cs index 3805f1f..ca43631 100644 --- a/src/BLite.Core/Metadata/EntityTypeBuilder.cs +++ b/src/BLite.Core/Metadata/EntityTypeBuilder.cs @@ -81,6 +81,17 @@ public EntityTypeBuilder HasSpatialIndex(Expression> keyS return this; } + /// + /// Declares a full-text index on a string property, queried with + /// in LINQ or ranked with + /// DocumentCollection.FullTextSearchAsync. The default name is idx_fts_{property}. + /// + public EntityTypeBuilder HasFullTextIndex(Expression> keySelector, string? name = null) + { + Indexes.Add(new IndexBuilder(keySelector, name, false, IndexType.FullText)); + return this; + } + public EntityTypeBuilder HasKey(Expression> keySelector) { PrimaryKeySelector = keySelector; diff --git a/src/BLite.Core/Query/IndexOptimizer.cs b/src/BLite.Core/Query/IndexOptimizer.cs index f467d7b..d6b6738 100644 --- a/src/BLite.Core/Query/IndexOptimizer.cs +++ b/src/BLite.Core/Query/IndexOptimizer.cs @@ -58,6 +58,9 @@ public class OptimizationResult /// public MethodCallExpression? VectorSearchCall { get; set; } + public bool IsFullTextSearch { get; set; } + public string? FullTextQuery { get; set; } + public bool IsSpatialSearch { get; set; } public (double Latitude, double Longitude) SpatialPoint { get; set; } public double RadiusKm { get; set; } @@ -178,7 +181,8 @@ public enum FilterCompleteness } if (left != null && right != null && left.IndexName == right.IndexName - && !left.IsSpatialSearch && !right.IsSpatialSearch) + && !left.IsSpatialSearch && !right.IsSpatialSearch + && !left.IsFullTextSearch && !right.IsFullTextSearch) { // Both sides are covered by the same index — combine into a single range. // Propagate the stricter completeness: if either branch required a post-filter @@ -255,7 +259,7 @@ public enum FilterCompleteness bareMember.Type == typeof(bool)) { CollectionIndexInfo? ix = null; - foreach (var idx in indexes) { if (Matches(idx, bareMember.Member.Name)) { ix = idx; break; } } + foreach (var idx in indexes) { if (SupportsKeyLookup(idx) && Matches(idx, bareMember.Member.Name)) { ix = idx; break; } } if (ix != null) return new OptimizationResult { IndexName = ix.Name, MinValue = true, MaxValue = true, IsRange = false, FilterCompleteness = FilterCompleteness.Exact }; } @@ -266,7 +270,7 @@ notExpr.Operand is MemberExpression notMember && notMember.Type == typeof(bool)) { CollectionIndexInfo? ix = null; - foreach (var idx in indexes) { if (Matches(idx, notMember.Member.Name)) { ix = idx; break; } } + foreach (var idx in indexes) { if (SupportsKeyLookup(idx) && Matches(idx, notMember.Member.Name)) { ix = idx; break; } } if (ix != null) return new OptimizationResult { IndexName = ix.Name, MinValue = false, MaxValue = false, IsRange = false, FilterCompleteness = FilterCompleteness.Exact }; } @@ -276,7 +280,7 @@ notExpr.Operand is MemberExpression notMember && if (propertyName != null) { CollectionIndexInfo? index = null; - foreach (var idx in indexes) { if (Matches(idx, propertyName)) { index = idx; break; } } + foreach (var idx in indexes) { if (SupportsKeyLookup(idx) && Matches(idx, propertyName)) { index = idx; break; } } if (index != null) { var result = new OptimizationResult { IndexName = index.Name }; @@ -346,7 +350,7 @@ notExpr.Operand is MemberExpression notMember && if (TryParseContainsInPredicate(expression, parameter, registry, out var inPropertyPath, out var inValues)) { CollectionIndexInfo? index = null; - foreach (var idx in indexes) { if (Matches(idx, inPropertyPath)) { index = idx; break; } } + foreach (var idx in indexes) { if (SupportsKeyLookup(idx) && Matches(idx, inPropertyPath)) { index = idx; break; } } if (index != null) { return new OptimizationResult @@ -389,6 +393,23 @@ notExpr.Operand is MemberExpression notMember && // Handle Method Calls (VectorSearch, Near, Within) if (expression is MethodCallExpression mcall) { + // FullTextMatch(this string text, string query) + if (mcall.Method.Name == "FullTextMatch" && mcall.Method.DeclaringType == typeof(FullTextExtensions) + && mcall.Arguments[0] is MemberExpression tMember && tMember.Expression == parameter) + { + CollectionIndexInfo? index = null; + foreach (var idx in indexes) { if (idx.Type == IndexType.FullText && Matches(idx, tMember.Member.Name)) { index = idx; break; } } + if (index != null) + { + return new OptimizationResult + { + IndexName = index.Name, + IsFullTextSearch = true, + FullTextQuery = EvaluateExpression(mcall.Arguments[1]) + }; + } + } + // VectorSearch(this float[] vector, float[] query, int k) if (mcall.Method.Name == "VectorSearch" && mcall.Arguments[0] is MemberExpression vMember && vMember.Expression == parameter) { @@ -573,6 +594,13 @@ private static bool TryWalkMemberChain(Expression expr, out object? value) return true; } + /// + /// Whether equality, range and IN lookups can use the index. Vector, spatial and full-text + /// indexes do not store property values in key order. + /// + private static bool SupportsKeyLookup(CollectionIndexInfo index) + => index.Type is not (IndexType.Vector or IndexType.Spatial or IndexType.FullText); + private static bool Matches(CollectionIndexInfo index, string propertyName) { if (index.PropertyPaths == null || index.PropertyPaths.Length == 0) return false; diff --git a/src/BLite.Core/Storage/StorageEngine.Memory.cs b/src/BLite.Core/Storage/StorageEngine.Memory.cs index b90b521..14a17ff 100644 --- a/src/BLite.Core/Storage/StorageEngine.Memory.cs +++ b/src/BLite.Core/Storage/StorageEngine.Memory.cs @@ -441,6 +441,13 @@ internal IReadOnlyCollection FreeCollectionPagesCore(string collectionName toFree.Add(pageId); break; } + case IndexType.FullText: + { + var ftsIndex = new FullTextIndex(this, IndexOptions.CreateBTree(idx.PropertyPaths), idx.RootPageId); + foreach (var pageId in ftsIndex.CollectAllPages()) + toFree.Add(pageId); + break; + } case IndexType.Spatial: { var opts = IndexOptions.CreateSpatial(idx.PropertyPaths); diff --git a/src/BLite.Core/Text/FullTextAnalyzer.cs b/src/BLite.Core/Text/FullTextAnalyzer.cs new file mode 100644 index 0000000..262b074 --- /dev/null +++ b/src/BLite.Core/Text/FullTextAnalyzer.cs @@ -0,0 +1,95 @@ +using System.Text; + +namespace BLite.Core.Text; + +/// +/// Splits text into the terms a full-text index stores and a full-text query looks up. +/// +/// Text is normalized with (accents removed, lowercased, +/// every character that is not a letter or digit treated as a separator) and split into terms. +/// A term longer than bytes of UTF-8 is cut to that length. +/// Documents and queries go through the same steps, so a document matches a query exactly when +/// it contains every query term, whether or not an index evaluates the query. +/// +/// +/// Scripts written without spaces between words, such as Chinese or Japanese, are not segmented: +/// each run of letters is one term. +/// +/// +internal static class FullTextAnalyzer +{ + /// Longest term stored, in bytes of UTF-8; keeps index keys small. + internal const int MaxTermBytes = 32; + + /// + /// Returns each distinct term of with its number of occurrences, and + /// the total number of terms in . + /// + public static Dictionary Analyze(string? text, out int length) + { + var frequencies = new Dictionary(StringComparer.Ordinal); + length = 0; + foreach (var term in Terms(text)) + { + frequencies.TryGetValue(term, out var count); + frequencies[term] = count + 1; + length++; + } + return frequencies; + } + + /// The distinct terms of a query, in the order they first appear. + public static IReadOnlyList QueryTerms(string? query) + { + var seen = new HashSet(StringComparer.Ordinal); + var terms = new List(); + foreach (var term in Terms(query)) + { + if (seen.Add(term)) + terms.Add(term); + } + return terms; + } + + /// Whether contains every term of . + public static bool Matches(string? text, string? query) + { + var queryTerms = QueryTerms(query); + if (queryTerms.Count == 0) + return false; + + var textTerms = new HashSet(Terms(text), StringComparer.Ordinal); + foreach (var term in queryTerms) + { + if (!textTerms.Contains(term)) + return false; + } + return true; + } + + private static IEnumerable Terms(string? text) + { + var normalized = TextNormalizer.Normalize(text); + if (normalized.Length == 0) + yield break; + + foreach (var token in normalized.Split(' ')) + { + if (token.Length > 0) + yield return Truncate(token); + } + } + + private static string Truncate(string term) + { + if (Encoding.UTF8.GetByteCount(term) <= MaxTermBytes) + return term; + + var bytes = Encoding.UTF8.GetBytes(term); + int cut = MaxTermBytes; + // Back off to the start of a character: UTF-8 continuation bytes are 10xxxxxx. + while (cut > 0 && (bytes[cut] & 0xC0) == 0x80) + cut--; + return Encoding.UTF8.GetString(bytes, 0, cut); + } +} diff --git a/tests/BLite.Shared/FullTextContexts.cs b/tests/BLite.Shared/FullTextContexts.cs new file mode 100644 index 0000000..f502bb6 --- /dev/null +++ b/tests/BLite.Shared/FullTextContexts.cs @@ -0,0 +1,44 @@ +using BLite.Bson; +using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Metadata; + +namespace BLite.Shared; + +public class Article +{ + public ObjectId Id { get; set; } + public string Title { get; set; } = string.Empty; + public string? Body { get; set; } + public int Year { get; set; } +} + +/// +/// Articles with full-text indexes on Body and Title, next to a B-Tree index on +/// Title for equality and range lookups. +/// +public partial class FullTextDbContext : DocumentDbContext +{ + public DocumentCollection Articles { get; set; } = null!; + + public FullTextDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity
().ToCollection("articles") + .HasIndex(a => a.Title) + .HasFullTextIndex(a => a.Title) + .HasFullTextIndex(a => a.Body); + } +} + +/// The same collection without any secondary index. +public partial class FullTextNoIndexDbContext : DocumentDbContext +{ + public DocumentCollection Articles { get; set; } = null!; + + public FullTextNoIndexDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity
().ToCollection("articles"); +} diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs new file mode 100644 index 0000000..2a8dc87 --- /dev/null +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -0,0 +1,381 @@ +using BLite.Bson; +using BLite.Core; +using System.Linq.Expressions; +using BLite.Core.Indexing; +using BLite.Core.Query; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// Full-text indexes: analysis, LINQ FullTextMatch, BM25-ranked search, index maintenance +/// and the dynamic API (#70). +/// +public class FullTextSearchTests : IDisposable +{ + private readonly string _dbPath; + + public FullTextSearchTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"fts_{Guid.NewGuid():N}.db"); + } + + public void Dispose() + { + foreach (var path in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".fresh.db") }) + foreach (var f in new[] { path, Path.ChangeExtension(path, ".wal") }) + try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ } + } + + private static Article A(string title, string? body, int year = 2020) => new() { Title = title, Body = body, Year = year }; + + private static async Task SeedAsync(FullTextDbContext db) + { + await db.Articles.InsertAsync(A("fox", "The quick brown fox jumps over the lazy dog", 2001)); + await db.Articles.InsertAsync(A("fox again", "A quick fox. Another quick fox!", 1999)); + await db.Articles.InsertAsync(A("dog", "The dog sleeps all day long in the sun", 2010)); + await db.Articles.InsertAsync(A("cafe", "Café au lait, s'il vous plaît", 2015)); + await db.Articles.InsertAsync(A("empty", null, 2020)); + } + + private static List Titles(IEnumerable
articles) => articles.Select(a => a.Title).OrderBy(t => t).ToList(); + + // ── In-memory matching ────────────────────────────────────────────────── + + [Fact] + public void FullTextMatch_InMemory_IgnoresCaseAccentsAndPunctuation() + { + Assert.True("Café, au LAIT!".FullTextMatch("cafe lait")); + Assert.True("The quick brown fox".FullTextMatch("FOX quick")); + Assert.False("The quick brown fox".FullTextMatch("quick dog")); + Assert.False("The quick brown fox".FullTextMatch(" ,; ")); + Assert.False(((string?)null).FullTextMatch("fox")); + } + + // ── LINQ ──────────────────────────────────────────────────────────────── + + [Fact] + public async Task Linq_FullTextMatch_ReturnsDocumentsContainingEveryTerm() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + + Assert.Equal(new[] { "fox", "fox again" }, Titles(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("quick fox")).ToList())); + Assert.Equal(new[] { "fox" }, Titles(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("fox dog")).ToList())); + Assert.Equal(new[] { "cafe" }, Titles(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("CAFE")).ToList())); + Assert.Empty(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("cat")).ToList()); + } + + [Fact] + public async Task Linq_FullTextMatch_CombinesWithOtherConditions() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + + var and = db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("fox") && a.Year > 2000).ToList(); + var or = db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("sun") || a.Year == 1999).ToList(); + var not = db.Articles.AsQueryable().Where(a => a.Body != null && !a.Body.FullTextMatch("fox")).ToList(); + + Assert.Equal(new[] { "fox" }, Titles(and)); + Assert.Equal(new[] { "dog", "fox again" }, Titles(or)); + Assert.Equal(new[] { "cafe", "dog" }, Titles(not)); + } + + [Fact] + public async Task Linq_CountAndFirst_UseTheFullTextIndex() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + + Assert.Equal(2, db.Articles.AsQueryable().Count(a => a.Body.FullTextMatch("quick"))); + Assert.Equal("dog", db.Articles.AsQueryable().First(a => a.Body.FullTextMatch("sleeps")).Title); + Assert.Null(db.Articles.AsQueryable().FirstOrDefault(a => a.Body.FullTextMatch("cat"))); + } + + [Fact] + public async Task Linq_EqualityOnAPropertyWithBothIndexes_UsesTheBTree() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + + Assert.Equal("fox again", db.Articles.AsQueryable().Single(a => a.Title == "fox again").Title); + Assert.Equal(2, db.Articles.AsQueryable().Count(a => a.Title == "fox" || a.Title == "dog")); + Assert.Equal(new[] { "fox", "fox again" }, Titles(db.Articles.AsQueryable().Where(a => a.Title.FullTextMatch("fox")).ToList())); + } + + // ── Query planning ────────────────────────────────────────────────────── + + private static readonly CollectionIndexInfo[] PlannerIndexes = + [ + new() { Name = "idx_fts_Title", PropertyPaths = ["Title"], Type = IndexType.FullText }, + new() { Name = "idx_Title", PropertyPaths = ["Title"], Type = IndexType.BTree }, + new() { Name = "idx_fts_Body", PropertyPaths = ["Body"], Type = IndexType.FullText }, + ]; + + private static IndexOptimizer.OptimizationResult? Plan(Expression> predicate) + => IndexOptimizer.TryOptimize
(new QueryModel { WhereClause = predicate }, PlannerIndexes); + + [Fact] + public void Planner_ServesFullTextMatch_FromTheFullTextIndex() + { + var plan = Plan(a => a.Body.FullTextMatch("quick fox")); + + Assert.NotNull(plan); + Assert.True(plan.IsFullTextSearch); + Assert.Equal("idx_fts_Body", plan.IndexName); + Assert.Equal("quick fox", plan.FullTextQuery); + Assert.Equal(IndexOptimizer.FilterCompleteness.Exact, plan.FilterCompleteness); + } + + [Fact] + public void Planner_KeyLookups_SkipFullTextIndexes() + { + var plan = Plan(a => a.Title == "fox"); + + Assert.NotNull(plan); + Assert.False(plan.IsFullTextSearch); + Assert.Equal("idx_Title", plan.IndexName); + } + + [Fact] + public void Planner_FullTextMatchWithAnotherCondition_PostFilters() + { + var plan = Plan(a => a.Body.FullTextMatch("fox") && a.Year > 2000); + + Assert.NotNull(plan); + Assert.True(plan.IsFullTextSearch); + Assert.Equal(IndexOptimizer.FilterCompleteness.PartialAnd, plan.FilterCompleteness); + } + + [Fact] + public void Planner_FullTextMatchWithoutAnIndex_HasNoPlan() + { + var indexes = new[] { new CollectionIndexInfo { Name = "idx_Title", PropertyPaths = ["Title"], Type = IndexType.BTree } }; + + Assert.Null(IndexOptimizer.TryOptimize
( + new QueryModel { WhereClause = (Expression>)(a => a.Body.FullTextMatch("fox")) }, indexes)); + } + + // ── Ranked search ─────────────────────────────────────────────────────── + + [Fact] + public async Task FullTextSearch_RanksByBm25() + { + using var db = new FullTextDbContext(_dbPath); + await db.Articles.InsertAsync(A("once, long", "fox " + string.Join(" ", Enumerable.Repeat("filler", 30)))); + await db.Articles.InsertAsync(A("twice, short", "fox fox")); + await db.Articles.InsertAsync(A("none", "nothing to see here")); + + var results = await db.Articles.FullTextSearchAsync("idx_fts_Body", "fox", 10).ToListAsync(); + + Assert.Equal(new[] { "twice, short", "once, long" }, results.Select(r => r.Document.Title)); + Assert.True(results[0].Score > results[1].Score); + Assert.All(results, r => Assert.True(r.Score > 0)); + } + + [Fact] + public async Task FullTextSearch_AnyTerm_RanksDocumentsWithMoreAndRarerTermsFirst() + { + // Every body has two terms, so only term rarity and the number of matching terms differ. + using var db = new FullTextDbContext(_dbPath); + await db.Articles.InsertAsync(A("common", "apple filler")); + await db.Articles.InsertAsync(A("common 2", "apple filler")); + await db.Articles.InsertAsync(A("common 3", "apple filler")); + await db.Articles.InsertAsync(A("rare", "durian filler")); + await db.Articles.InsertAsync(A("both", "apple durian")); + + var all = await db.Articles.FullTextSearchAsync("idx_fts_Body", "apple durian", 10).ToListAsync(); + var any = await db.Articles.FullTextSearchAsync("idx_fts_Body", "apple durian", 10, matchAllTerms: false).ToListAsync(); + var top2 = await db.Articles.FullTextSearchAsync("idx_fts_Body", "apple durian", 2, matchAllTerms: false).ToListAsync(); + + Assert.Equal(new[] { "both" }, all.Select(r => r.Document.Title)); + Assert.Equal(5, any.Count); + Assert.Equal(new[] { "both", "rare" }, any.Take(2).Select(r => r.Document.Title)); + Assert.Equal(new[] { "both", "rare" }, top2.Select(r => r.Document.Title)); + } + + [Fact] + public async Task FullTextSearch_ScoresAfterUpdatesAndDeletes_MatchAFreshIndex() + { + using (var db = new FullTextDbContext(_dbPath)) + { + var keep = A("keep", "red green blue"); + var change = A("change", "red red"); + var gone = A("gone", "green green green blue"); + await db.Articles.InsertAsync(keep); + await db.Articles.InsertAsync(change); + await db.Articles.InsertAsync(gone); + await db.Articles.InsertAsync(A("gone too", "blue")); + change.Body = "blue green with many more words here"; + await db.Articles.UpdateAsync(change); + await db.Articles.DeleteAsync(gone.Id); + await db.Articles.DeleteAsync(db.Articles.AsQueryable().Single(a => a.Title == "gone too").Id); + } + + var freshPath = Path.ChangeExtension(_dbPath, ".fresh.db"); + using (var fresh = new FullTextDbContext(freshPath)) + { + await fresh.Articles.InsertAsync(A("keep", "red green blue")); + await fresh.Articles.InsertAsync(A("change", "blue green with many more words here")); + } + + foreach (var query in new[] { "blue", "green", "red", "blue green" }) + { + using var db = new FullTextDbContext(_dbPath); + using var fresh = new FullTextDbContext(freshPath); + var actual = await db.Articles.FullTextSearchAsync("idx_fts_Body", query, 10, matchAllTerms: false).ToListAsync(); + var expected = await fresh.Articles.FullTextSearchAsync("idx_fts_Body", query, 10, matchAllTerms: false).ToListAsync(); + + Assert.Equal(expected.Select(r => r.Document.Title), actual.Select(r => r.Document.Title)); + Assert.Equal(expected.Select(r => Math.Round(r.Score, 9)), actual.Select(r => Math.Round(r.Score, 9))); + } + } + + // ── Maintenance ───────────────────────────────────────────────────────── + + [Fact] + public async Task UpdatesAndDeletes_AreReflected() + { + using var db = new FullTextDbContext(_dbPath); + var a = A("a", "alpha beta"); + var b = A("b", "gamma"); + await db.Articles.InsertAsync(a); + await db.Articles.InsertAsync(b); + + a.Body = "delta"; + await db.Articles.UpdateAsync(a); + b.Body = "epsilon " + new string('x', 3000); // no longer fits its slot + await db.Articles.UpdateAsync(b); + await db.Articles.InsertAsync(A("c", "zeta")); + await db.Articles.DeleteAsync(db.Articles.AsQueryable().Single(x => x.Title == "c").Id); + await db.Articles.InsertAsync(A("d", "unrelated words")); // may reuse the deleted slot + + List Match(string q) => Titles(db.Articles.AsQueryable().Where(x => x.Body.FullTextMatch(q)).ToList()); + Assert.Empty(Match("alpha")); + Assert.Equal(new[] { "a" }, Match("delta")); + Assert.Empty(Match("gamma")); + Assert.Equal(new[] { "b" }, Match("epsilon")); + Assert.Empty(Match("zeta")); + Assert.Empty(await db.Articles.FullTextSearchAsync("idx_fts_Body", "zeta", 10).ToListAsync()); + } + + [Fact] + public async Task RolledBackWrites_LeaveTheIndexUnchanged() + { + using var db = new FullTextDbContext(_dbPath); + var kept = A("kept", "persistent text"); + await db.Articles.InsertAsync(kept); + + using (var txn = await db.BeginTransactionAsync()) + { + await db.Articles.InsertAsync(A("rolled back", "ephemeral text"), txn); + await db.Articles.DeleteAsync(kept.Id, txn); + await txn.RollbackAsync(); + } + + Assert.Equal(new[] { "kept" }, Titles(db.Articles.AsQueryable().Where(x => x.Body.FullTextMatch("text")).ToList())); + var results = await db.Articles.FullTextSearchAsync("idx_fts_Body", "text", 10).ToListAsync(); + Assert.Equal("kept", Assert.Single(results).Document.Title); + } + + [Fact] + public async Task Index_SurvivesReopen() + { + double score; + using (var db = new FullTextDbContext(_dbPath)) + { + await SeedAsync(db); + score = (await db.Articles.FullTextSearchAsync("idx_fts_Body", "dog", 10).ToListAsync())[0].Score; + } + + using var reopened = new FullTextDbContext(_dbPath); + var results = await reopened.Articles.FullTextSearchAsync("idx_fts_Body", "dog", 10).ToListAsync(); + + Assert.Equal(2, results.Count); + Assert.Equal(score, results[0].Score, 9); + Assert.Contains(reopened.Articles.GetIndexes(), i => i.Name == "idx_fts_Body" && i.Type == IndexType.FullText); + } + + [Fact] + public async Task IndexAddedToExistingData_CoversIt() + { + using (var v1 = new FullTextNoIndexDbContext(_dbPath)) + { + await v1.Articles.InsertAsync(A("old", "written before the index existed")); + } + + using (var v1 = new FullTextNoIndexDbContext(_dbPath)) + { + await v1.Articles.CreateFullTextIndexAsync(a => a.Title, "idx_title_runtime"); + await v1.Articles.CreateFullTextIndexAsync(a => a.Title, "idx_title_runtime"); // existing: no-op + Assert.Single(await v1.Articles.FullTextSearchAsync("idx_title_runtime", "old", 10).ToListAsync()); + } + + using var v2 = new FullTextDbContext(_dbPath); + Assert.Equal(new[] { "old" }, Titles(v2.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("index existed")).ToList())); + } + + [Fact] + public async Task VacuumRebuild_KeepsResultsAndScores() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + var before = await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick fox", 10).ToListAsync(); + + await db.Articles.VacuumAsync(new VacuumOptions { RebuildIndexes = true }); + await db.Articles.VacuumAsync(new VacuumOptions { RebuildIndexes = true }); + var after = await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick fox", 10).ToListAsync(); + + Assert.Equal(before.Select(r => r.Document.Title), after.Select(r => r.Document.Title)); + Assert.Equal(before.Select(r => Math.Round(r.Score, 9)), after.Select(r => Math.Round(r.Score, 9))); + } + + [Fact] + public async Task DroppedIndex_FallsBackToScanning() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + + Assert.True(await db.Articles.DropIndexAsync("idx_fts_Body")); + + Assert.Equal(new[] { "fox", "fox again" }, Titles(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("quick fox")).ToList())); + await Assert.ThrowsAsync(async () => await db.Articles.FullTextSearchAsync("idx_fts_Body", "fox", 1).ToListAsync()); + } + + // ── DynamicCollection ─────────────────────────────────────────────────── + + [Fact] + public async Task Dynamic_CreateSearchUpdateDeleteAndReopen() + { + BsonId movedId; + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + BsonDocument Doc(string title, string text) => col.CreateDocument(["title", "text"], b => b + .AddString("title", title).AddString("text", text)); + + await col.InsertAsync(Doc("before index", "indexed later on")); + await col.CreateFullTextIndexAsync("text"); + movedId = await col.InsertAsync(Doc("moved", "original words")); + var goneId = await col.InsertAsync(Doc("gone", "words to forget")); + + await col.UpdateAsync(movedId, Doc("moved", "replacement words")); + await col.DeleteAsync(goneId); + + Assert.Contains(col.GetIndexDescriptors(), d => d.Name == "idx_fts_text" && d.IsFullText); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + async Task> Search(string q) => + (await col.FullTextSearchAsync("idx_fts_text", q, 10).ToListAsync()) + .Select(r => r.Document.TryGetString("title", out var t) ? t : null).ToList(); + + Assert.Equal(new[] { "before index" }, await Search("later")); + Assert.Equal(new[] { "moved" }, await Search("words")); + Assert.Empty(await Search("original")); + Assert.Empty(await Search("forget")); + } + } +} From 76855bd8f1381132224d6274519263924078366a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 16:09:54 +0200 Subject: [PATCH 07/15] fix(fts): address review of full-text search (#70) - Tag postings 0x10 and statistics 0x20, outside the range of the generic [0x00..0x03] keys an older BLite writes when it reads the tree as a plain B-Tree; validate the statistics key length. - Record FullTextIndex.FormatVersion as the index KeyFormat and rebuild the index on open (typed and dynamic) when it differs or the tree holds keys the index did not write, since deletes rely on reproducing the keys. - Re-check every hit against the document text, for LINQ FullTextMatch and FullTextSearchAsync (typed and dynamic), so a stale posting never surfaces the document that now holds its location. - Search with matchAllTerms intersects rarest-first before scoring, and falls back to statistics estimated from the postings when the entry is missing (e.g. lost to a concurrent transaction's update). - CreateFullTextIndexAsync throws when the name belongs to a non-full-text index, passes its token to the rebuild, and is declared on IDocumentCollection; DynamicCollection.FullTextSearchAsync takes a token. - Document the concurrent-transaction statistics caveat, batching inserts and the CJK / supplementary-plane limitation. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 21 +- .../Collections/IDocumentCollection.cs | 15 + src/BLite.Core/DynamicCollection.cs | 57 +++- .../Indexing/CollectionSecondaryIndex.cs | 22 +- src/BLite.Core/Indexing/FullTextIndex.cs | 143 +++++++-- src/BLite.Core/Metadata/EntityTypeBuilder.cs | 5 + src/BLite.Core/Text/FullTextAnalyzer.cs | 20 +- tests/BLite.Tests/FullTextSearchTests.cs | 288 ++++++++++++++++++ 8 files changed, 537 insertions(+), 34 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 4e11399..63190ff 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -884,7 +884,7 @@ private void MigrateLegacyDecimalIndexes() try { var fresh = _indexManager.CreateUnregistered(index.Definition); - fresh.KeyFormat = IndexMetadata.CurrentKeyFormat; + fresh.KeyFormat = fresh.TargetKeyFormat; // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. // A document that cannot be loaded fails the migration rather than silently leaving // the rebuilt index incomplete. @@ -894,7 +894,7 @@ private void MigrateLegacyDecimalIndexes() catch (Exception ex) { throw new InvalidOperationException( - $"Failed to rebuild index '{name}' of collection '{_collectionName}' with the exact decimal key format; the migration is retried on the next open.", ex); + $"Failed to rebuild index '{name}' of collection '{_collectionName}' in its current key format; the migration is retried on the next open.", ex); } } } @@ -1012,8 +1012,12 @@ public async Task> CreateFullTextIndexAsync( throw new TimeoutException("Timed out acquiring collection write lock (CreateFullTextIndexAsync)."); try { + var existing = _indexManager.GetIndex(CollectionIndexManager.ResolveIndexName(keySelector, name, IndexType.FullText)); + if (existing != null && existing.Definition.Type != IndexType.FullText) + throw new InvalidOperationException($"Index '{existing.Name}' already exists and is not a full-text index."); + var builder = new Metadata.IndexBuilder(keySelector, name, false, IndexType.FullText); - await Task.Run(() => ApplyIndexBuilder(builder), ct).ConfigureAwait(false); + await Task.Run(() => ApplyIndexBuilder(builder, ct), ct).ConfigureAwait(false); } finally { @@ -1105,7 +1109,7 @@ public IEnumerable GetIndexes() } [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] - internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) + internal void ApplyIndexBuilder(Metadata.IndexBuilder builder, CancellationToken ct = default) { var name = CollectionIndexManager.ResolveIndexName(builder.KeySelector, builder.Name, builder.Type); if (_indexManager.GetIndex(name) != null) @@ -1120,7 +1124,7 @@ internal void ApplyIndexBuilder(Metadata.IndexBuilder builder) try { // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. - Task.Run(() => FillUnregisteredIndexAsync(index, CancellationToken.None)).GetAwaiter().GetResult(); + Task.Run(() => FillUnregisteredIndexAsync(index, ct)).GetAwaiter().GetResult(); } catch (Exception ex) { @@ -3669,11 +3673,13 @@ Func GetCompiled() => } else if (indexOpt.IsFullTextSearch) { + // Every hit is re-checked with the full predicate: a posting left behind by an + // analyzer change must not surface the document that now holds its location. var ftsIndex = _indexManager.GetIndex(indexOpt.IndexName)!; foreach (var location in ftsIndex.FullTextMatch(indexOpt.FullTextQuery!, transaction)) { var item = await FindByLocationAsync(location, transaction?.TransactionId ?? 0UL, ct); - if (item != null && (indexOpt.FilterCompleteness == Query.IndexOptimizer.FilterCompleteness.Exact || GetCompiled()(item))) { yield return item; if (++yielded >= fetchLimit) yield break; } + if (item != null && GetCompiled()(item)) { yield return item; if (++yielded >= fetchLimit) yield break; } } } else if (indexOpt.IsSpatialSearch) @@ -3930,7 +3936,8 @@ public async IAsyncEnumerable> FullTextSearchAsync( { ct.ThrowIfCancellationRequested(); var doc = await FindByLocationAsync(location, 0UL, ct); - if (doc != null) yield return new FullTextSearchResult(doc, score); + if (doc != null && index.FullTextContains(doc, query, matchAllTerms)) + yield return new FullTextSearchResult(doc, score); } } diff --git a/src/BLite.Core/Collections/IDocumentCollection.cs b/src/BLite.Core/Collections/IDocumentCollection.cs index ad615cf..845bc3b 100644 --- a/src/BLite.Core/Collections/IDocumentCollection.cs +++ b/src/BLite.Core/Collections/IDocumentCollection.cs @@ -107,6 +107,21 @@ Task> CreateIndexAsync( bool unique = false, CancellationToken ct = default); + /// + [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] + Task> CreateFullTextIndexAsync( + Expression> keySelector, + string? name = null, + CancellationToken ct = default); + + /// + IAsyncEnumerable> FullTextSearchAsync( + string indexName, + string query, + int k, + bool matchAllTerms = true, + CancellationToken ct = default); + [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] Task> CreateVectorIndexAsync( Expression> keySelector, diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index f6fcf27..0688ba5 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using BLite.Bson; +using BLite.Core.Text; using BLite.Core.Audit; using BLite.Core.CDC; using BLite.Core.Collections; @@ -115,6 +116,7 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction _isPageLocked = _storage.IsPageLocked; // Load or create collection metadata + var staleFullTextIndexes = new List(); var metadata = _storage.GetCollectionMetadata(_collectionName); uint primaryRootPageId = 0; @@ -168,6 +170,8 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction var opts = IndexOptions.CreateBTree(idxMeta.PropertyPaths); var fullText = new FullTextIndex(_storage, opts, idxMeta.RootPageId); _secondaryIndexes[idxMeta.Name] = new DynamicSecondaryIndex(fullText, fieldPath, opts); + if (idxMeta.KeyFormat != FullTextIndex.FormatVersion || fullText.HasForeignKeys()) + staleFullTextIndexes.Add(idxMeta.Name); break; } } @@ -205,10 +209,47 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction if (rootsChanged) _storage.SaveCollectionMetadata(metadata); + // A full-text index written with another analyzer or key layout is rebuilt: deleting a + // document relies on reproducing the keys it was indexed under. + foreach (var name in staleFullTextIndexes) + RebuildFullTextIndex(name); + // Rebuild the free-space index from existing page headers on cold start. RebuildFreeSpaceIndex(); } + /// + /// Rebuilds full-text index into a fresh tree, swaps it in once the + /// rebuild committed, persists it with the current format version and frees the old tree. + /// + private void RebuildFullTextIndex(string name) + { + var old = _secondaryIndexes[name]; + var fresh = new DynamicSecondaryIndex(new FullTextIndex(_storage, old.Options), old.FieldPath, old.Options); + var transaction = _storage.BeginTransaction(IsolationLevel.ReadCommitted); + try + { + foreach (var e in _primaryIndex.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transaction.TransactionId)) + { + var doc = ReadDocumentAt(e.Location, transaction.TransactionId); + if (doc != null) IndexInsert(fresh, doc, e.Location, transaction); + } + // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. + Task.Run(async () => await transaction.CommitAsync()).GetAwaiter().GetResult(); + } + catch + { + Task.Run(async () => await transaction.RollbackAsync()).GetAwaiter().GetResult(); + throw; + } + + var oldPages = old.FullText!.CollectAllPages(); + _secondaryIndexes[name] = fresh; + PersistIndexMetadata(); + foreach (var pageId in oldPages) + _storage.FreePage(pageId); + } + private void RebuildFreeSpaceIndex() { // Read only the fixed-size page header (24 bytes) rather than renting a full-page @@ -1009,15 +1050,26 @@ public async IAsyncEnumerable WithinAsync(string indexName, (doubl /// true (the default) to return only documents containing every query term; /// false to return documents containing any of them, ranked by relevance. /// - public async IAsyncEnumerable> FullTextSearchAsync(string indexName, string query, int k, bool matchAllTerms = true) + public async IAsyncEnumerable> FullTextSearchAsync( + string indexName, + string query, + int k, + bool matchAllTerms = true, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) { if (!_secondaryIndexes.TryGetValue(indexName, out var entry) || entry.Kind != DynamicIndexKind.FullText || entry.FullText == null) throw new ArgumentException($"Full-text index '{indexName}' not found on collection '{_collectionName}'"); foreach (var (location, score) in entry.FullText.Search(query, k, matchAllTerms, null)) { + ct.ThrowIfCancellationRequested(); var doc = ReadDocumentAt(location, 0UL); - if (doc != null) yield return new FullTextSearchResult(doc, score); + if (doc == null) continue; + + // Re-check the hit, so a stale posting can never surface another document. + var text = TryGetNestedValue(doc, entry.FieldPath, out var val) && val.IsString ? val.AsString : null; + if (matchAllTerms ? FullTextAnalyzer.Matches(text, query) : FullTextAnalyzer.MatchesAny(text, query)) + yield return new FullTextSearchResult(doc, score); } } @@ -1928,6 +1980,7 @@ internal void PersistIndexMetadata() break; case DynamicIndexKind.FullText: idxMeta.Type = IndexType.FullText; + idxMeta.KeyFormat = FullTextIndex.FormatVersion; break; } diff --git a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs index f548450..3fc779e 100644 --- a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs +++ b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs @@ -93,6 +93,7 @@ public CollectionSecondaryIndex( _btreeIndex = null; _vectorIndex = null; _spatialIndex = null; + KeyFormat = FullTextIndex.FormatVersion; } else { @@ -742,10 +743,27 @@ private static decimal ToDecimalClamped(float f) /// /// True when this B-tree index keys a decimal property and its pages still use the legacy - /// format, i.e. hold (double)value keys that the exact decimal encoding cannot match. + /// format, i.e. hold (double)value keys that the exact decimal encoding cannot match; or + /// when this full-text index was written with another analyzer version or holds keys it did not + /// write (e.g. from an older BLite that read it as a plain B-tree). /// internal bool NeedsKeyFormatMigration => - _btreeIndex != null && KeyFormat < IndexMetadata.CurrentKeyFormat && IsDecimalKeyed; + (_btreeIndex != null && KeyFormat < IndexMetadata.CurrentKeyFormat && IsDecimalKeyed) + || (_fullTextIndex != null && (KeyFormat != FullTextIndex.FormatVersion || _fullTextIndex.HasForeignKeys())); + + /// The key format a rebuilt copy of this index is written in. + internal byte TargetKeyFormat => _fullTextIndex != null ? FullTextIndex.FormatVersion : IndexMetadata.CurrentKeyFormat; + + /// + /// Whether the indexed text of contains every term of + /// (or, with false, at least one). + /// Used to re-check full-text hits, so a stale posting can never surface another document. + /// + internal bool FullTextContains(T document, string query, bool matchAllTerms) + { + var text = _definition.TryGetKey(document, out var key) ? key as string : null; + return matchAllTerms ? Text.FullTextAnalyzer.Matches(text, query) : Text.FullTextAnalyzer.MatchesAny(text, query); + } // ── ICollectionIndex metadata pass-throughs ───────────────────────────── diff --git a/src/BLite.Core/Indexing/FullTextIndex.cs b/src/BLite.Core/Indexing/FullTextIndex.cs index 1705f5c..d566881 100644 --- a/src/BLite.Core/Indexing/FullTextIndex.cs +++ b/src/BLite.Core/Indexing/FullTextIndex.cs @@ -10,20 +10,36 @@ namespace BLite.Core.Indexing; /// Inverted index for full-text search, stored in a single B-Tree. /// /// Posting, one per distinct term of a document: -/// [0x01][term UTF-8][0x00][location (6)][term frequency u16][document length u32], +/// [0x10][term UTF-8][0x00][location (6)][term frequency u16][document length u32], /// pointing at the document. The term frequency and length make BM25 scoring possible from the /// postings alone; deleting a document recomputes the same keys from its old text. -/// Statistics, a single entry: [0x02][documents u64][terms u64], the number of +/// Statistics, a single entry: [0x20][documents u64][terms u64], the number of /// indexed documents and their total length, maintained in the same transaction as the /// postings. /// +/// The tags cannot start a key that an older BLite (which reads this tree as a plain B-Tree index) +/// writes: those begin with 0x00-0x03, so such keys stay outside every range read here, +/// and detects them so the index is rebuilt on open. /// Terms come from ; a document without any term is not indexed. +/// +/// Every write updates the single statistics entry, so two explicit transactions writing to the +/// same collection concurrently lose one of the updates (BLite has no write-write conflict +/// detection between transactions). Searches stay correct, but scores drift until the index is +/// rebuilt. +/// ///
internal sealed class FullTextIndex { - private const byte PostingTag = 0x01; - private const byte StatsTag = 0x02; + /// + /// Version of the analyzer and key layout, recorded as the index's KeyFormat. An index + /// written with another version is rebuilt on open, since deletes rely on reproducing the keys. + /// + internal const byte FormatVersion = 1; + + private const byte PostingTag = 0x10; + private const byte StatsTag = 0x20; private const byte TermEnd = 0x00; + private const int StatsKeyLength = 17; // BM25 parameters (Robertson et al.; the usual defaults, also used by SQLite FTS5 and Lucene). private const double K1 = 1.2; @@ -31,6 +47,7 @@ internal sealed class FullTextIndex private static readonly IndexKey StatsMin = IndexKey.FromOwnedArray([StatsTag]); private static readonly IndexKey StatsMax = IndexKey.FromOwnedArray([StatsTag + 1]); + private static readonly IndexKey AfterPostings = IndexKey.FromOwnedArray([PostingTag + 1]); private readonly BTreeIndex _tree; @@ -46,6 +63,30 @@ public FullTextIndex(StorageEngine storage, IndexOptions options, uint rootPageI /// The index pages as sees them. public IReadOnlyList CollectAllPages(ulong transactionId) => _tree.CollectAllPages(transactionId); + /// + /// Whether the tree holds a key this index did not write, such as one inserted by an older BLite + /// that read the tree as a plain B-Tree index. Such an index is also missing the postings of those + /// writes, so it must be rebuilt. Reads at most the first key and the keys after the postings. + /// + public bool HasForeignKeys() + { + foreach (var entry in _tree.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, null)) + { + var first = entry.Key.Data; + if (first.Length == 0 || (first[0] != PostingTag && first[0] != StatsTag)) return true; + break; + } + + // After the postings only the single statistics entry may follow. + int count = 0; + foreach (var entry in _tree.Range(AfterPostings, IndexKey.MaxKey, IndexDirection.Forward, null)) + { + var key = entry.Key.Data; + if (++count > 1 || key.Length != StatsKeyLength || key[0] != StatsTag) return true; + } + return false; + } + public void Insert(string? text, DocumentLocation location, ITransaction? transaction) { var terms = FullTextAnalyzer.Analyze(text, out var length); @@ -106,41 +147,97 @@ public IReadOnlyList Match(string? query, ITransaction? transa if (terms.Count == 0 || k <= 0) return Array.Empty<(DocumentLocation, double)>(); var txnId = transaction?.TransactionId; - var (documents, totalLength) = ReadStats(txnId); - if (documents == 0) return Array.Empty<(DocumentLocation, double)>(); - double averageLength = (double)totalLength / documents; - var scores = new Dictionary(); - foreach (var term in terms) + // Rarest term first: with matchAllTerms its postings bound the candidate set. + var ordered = terms + .Select(t => (Term: t, Count: _tree.CountRange(TermMin(t), TermMax(t), txnId))) + .OrderBy(t => t.Count) + .ToList(); + if (matchAllTerms && ordered[0].Count == 0) return Array.Empty<(DocumentLocation, double)>(); + + var perTerm = new List<(int DocumentFrequency, List<(DocumentLocation Location, int Frequency, int Length)> Postings)>(); + Dictionary? candidates = null; + foreach (var (term, count) in ordered) { - var postings = Postings(term, txnId).ToList(); - if (postings.Count == 0) + if (count == 0) continue; + + // Once the candidates are few, look each one up instead of reading every posting of a + // much more common term. + var postings = candidates != null && candidates.Count * 8 < count + ? candidates.Values.Select(l => TryPosting(term, l, txnId)).Where(p => p.HasValue).Select(p => p!.Value).ToList() + : Postings(term, txnId).ToList(); + + if (matchAllTerms) { - if (matchAllTerms) return Array.Empty<(DocumentLocation, double)>(); - continue; + var next = new Dictionary(); + foreach (var p in postings) + { + var key = Pack(p.Location); + if (candidates == null || candidates.ContainsKey(key)) next[key] = p.Location; + } + if (next.Count == 0) return Array.Empty<(DocumentLocation, double)>(); + candidates = next; } + perTerm.Add((count, postings)); + } + if (perTerm.Count == 0) return Array.Empty<(DocumentLocation, double)>(); + + var (documents, totalLength) = ReadStats(txnId); + if (documents <= 0 || totalLength <= 0) + { + // No statistics (e.g. a concurrent transaction's update won): estimate them from the + // postings at hand rather than return nothing while Match still finds the documents. + var lengths = perTerm.SelectMany(t => t.Postings).GroupBy(p => Pack(p.Location)).Select(g => g.First().Length).ToList(); + documents = Math.Max(1, perTerm.Max(t => t.DocumentFrequency)); + totalLength = Math.Max(1, (long)(lengths.Average() * documents)); + } + double averageLength = (double)totalLength / documents; - double idf = Math.Log(1 + (documents - postings.Count + 0.5) / (postings.Count + 0.5)); + var scores = new Dictionary(); + foreach (var (df, postings) in perTerm) + { + double idf = Math.Log(1 + (documents - df + 0.5) / (df + 0.5)); foreach (var p in postings) { + var key = Pack(p.Location); + if (candidates != null && !candidates.ContainsKey(key)) continue; double weight = idf * p.Frequency * (K1 + 1) / (p.Frequency + K1 * (1 - B + B * p.Length / averageLength)); - var key = Pack(p.Location); - scores[key] = scores.TryGetValue(key, out var s) - ? (s.Location, s.Score + weight, s.Terms + 1) - : (p.Location, weight, 1); + scores[key] = scores.TryGetValue(key, out var s) ? (s.Location, s.Score + weight) : (p.Location, weight); } } return scores.Values - .Where(s => !matchAllTerms || s.Terms == terms.Count) .OrderByDescending(s => s.Score) .ThenBy(s => Pack(s.Location)) .Take(k) - .Select(s => (s.Location, s.Score)) .ToList(); } + /// The posting of for the document at , if any. + private (DocumentLocation Location, int Frequency, int Length)? TryPosting(string term, DocumentLocation location, ulong? txnId) + { + int termBytes = Encoding.UTF8.GetByteCount(term); + int prefixLength = 2 + termBytes + DocumentLocation.SerializedSize; + var min = new byte[prefixLength]; + min[0] = PostingTag; + Encoding.UTF8.GetBytes(term, 0, term.Length, min, 1); + min[1 + termBytes] = TermEnd; + location.WriteTo(min.AsSpan(2 + termBytes)); + var max = new byte[prefixLength + 6]; + min.CopyTo(max, 0); + max.AsSpan(prefixLength).Fill(0xFF); + + foreach (var entry in _tree.Range(IndexKey.FromOwnedArray(min), IndexKey.FromOwnedArray(max), IndexDirection.Forward, txnId)) + { + var key = entry.Key.Data; + return (entry.Location, + BinaryPrimitives.ReadUInt16BigEndian(key.Slice(prefixLength)), + (int)BinaryPrimitives.ReadUInt32BigEndian(key.Slice(prefixLength + 2))); + } + return null; + } + private IEnumerable<(DocumentLocation Location, int Frequency, int Length)> Postings(string term, ulong? txnId) { int offset = 2 + Encoding.UTF8.GetByteCount(term) + DocumentLocation.SerializedSize; @@ -158,6 +255,7 @@ public IReadOnlyList Match(string? query, ITransaction? transa foreach (var entry in _tree.Range(StatsMin, StatsMax, IndexDirection.Forward, txnId)) { var key = entry.Key.Data; + if (key.Length != StatsKeyLength) continue; // not written by this index: ignore return ((long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)), (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(9))); } @@ -170,6 +268,7 @@ private void AdjustStats(long documents, long length, ulong? txnId) foreach (var entry in _tree.Range(StatsMin, StatsMax, IndexDirection.Forward, txnId)) { var key = entry.Key.Data; + if (key.Length != StatsKeyLength) continue; // not written by this index: ignore currentDocuments = (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)); currentLength = (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(9)); _tree.Delete(entry.Key, entry.Location, txnId); @@ -180,7 +279,7 @@ private void AdjustStats(long documents, long length, ulong? txnId) long newLength = currentLength + length; if (newDocuments <= 0) return; - var stats = new byte[17]; + var stats = new byte[StatsKeyLength]; stats[0] = StatsTag; BinaryPrimitives.WriteUInt64BigEndian(stats.AsSpan(1), (ulong)newDocuments); BinaryPrimitives.WriteUInt64BigEndian(stats.AsSpan(9), (ulong)Math.Max(0, newLength)); @@ -202,7 +301,7 @@ private static IndexKey PostingKey(string term, DocumentLocation location, int f return IndexKey.FromOwnedArray(key); } - // [0x01][term][0x00] .. [0x01][term][0x01]: every posting of exactly this term. A longer term + // [0x10][term][0x00] .. [0x10][term][0x01]: every posting of exactly this term. A longer term // sharing the prefix has a letter or digit (> 0x01) where these bounds have 0x00 / 0x01. private static IndexKey TermMin(string term) => TermBound(term, TermEnd); private static IndexKey TermMax(string term) => TermBound(term, TermEnd + 1); diff --git a/src/BLite.Core/Metadata/EntityTypeBuilder.cs b/src/BLite.Core/Metadata/EntityTypeBuilder.cs index ca43631..a5a35ac 100644 --- a/src/BLite.Core/Metadata/EntityTypeBuilder.cs +++ b/src/BLite.Core/Metadata/EntityTypeBuilder.cs @@ -85,6 +85,11 @@ public EntityTypeBuilder HasSpatialIndex(Expression> keyS /// Declares a full-text index on a string property, queried with /// in LINQ or ranked with /// DocumentCollection.FullTextSearchAsync. The default name is idx_fts_{property}. + /// + /// Each document touches one index page per distinct term, so single-document writes cost + /// noticeably more than without the index; insert and update in batches + /// (InsertBulkAsync or one explicit transaction) where possible. + /// /// public EntityTypeBuilder HasFullTextIndex(Expression> keySelector, string? name = null) { diff --git a/src/BLite.Core/Text/FullTextAnalyzer.cs b/src/BLite.Core/Text/FullTextAnalyzer.cs index 262b074..14cec41 100644 --- a/src/BLite.Core/Text/FullTextAnalyzer.cs +++ b/src/BLite.Core/Text/FullTextAnalyzer.cs @@ -13,7 +13,9 @@ namespace BLite.Core.Text; /// /// /// Scripts written without spaces between words, such as Chinese or Japanese, are not segmented: -/// each run of letters is one term. +/// each run of letters is one term (and is then cut to , about ten +/// ideographs). Characters outside the Basic Multilingual Plane, such as emoji or supplementary +/// letters, are treated as separators. /// /// internal static class FullTextAnalyzer @@ -67,6 +69,22 @@ public static bool Matches(string? text, string? query) return true; } + /// Whether contains at least one term of . + public static bool MatchesAny(string? text, string? query) + { + var queryTerms = QueryTerms(query); + if (queryTerms.Count == 0) + return false; + + var textTerms = new HashSet(Terms(text), StringComparer.Ordinal); + foreach (var term in queryTerms) + { + if (textTerms.Contains(term)) + return true; + } + return false; + } + private static IEnumerable Terms(string? text) { var normalized = TextNormalizer.Normalize(text); diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs index 2a8dc87..cb1e2d0 100644 --- a/tests/BLite.Tests/FullTextSearchTests.cs +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -1,8 +1,10 @@ using BLite.Bson; using BLite.Core; +using BLite.Core.Collections; using System.Linq.Expressions; using BLite.Core.Indexing; using BLite.Core.Query; +using BLite.Core.Storage; using BLite.Shared; namespace BLite.Tests; @@ -147,6 +149,21 @@ public void Planner_FullTextMatchWithAnotherCondition_PostFilters() Assert.Equal(IndexOptimizer.FilterCompleteness.PartialAnd, plan.FilterCompleteness); } + [Fact] + public void Planner_FullTextMatchUnderOrOrNot_IsNotServedByTheIndex() + { + // The index only knows which documents contain the terms: these need a scan. + foreach (var plan in new[] + { + Plan(a => a.Body.FullTextMatch("sun") || a.Year == 1999), + Plan(a => a.Body.FullTextMatch("sun") || a.Body.FullTextMatch("fox")), + Plan(a => !a.Body.FullTextMatch("fox")), + }) + { + Assert.False(plan?.IsFullTextSearch ?? false); + } + } + [Fact] public void Planner_FullTextMatchWithoutAnIndex_HasNoPlan() { @@ -342,6 +359,202 @@ public async Task DroppedIndex_FallsBackToScanning() await Assert.ThrowsAsync(async () => await db.Articles.FullTextSearchAsync("idx_fts_Body", "fox", 1).ToListAsync()); } + // ── On-disk robustness ────────────────────────────────────────────────── + + private static CollectionSecondaryIndex BodyIndex(FullTextDbContext db) + => db.Articles.IndexManager.GetIndex("idx_fts_Body")!; + + /// A second handle on the tree of idx_fts_Body, to write keys the index never would. + private static BTreeIndex RawTree(FullTextDbContext db) + => new(db.Storage, IndexOptions.CreateBTree("Body"), BodyIndex(db).RootPageId); + + private static async Task WriteAsync(FullTextDbContext db, Action write) + { + using var txn = db.BeginTransaction(); + write(txn.TransactionId); + await txn.CommitAsync(); + } + + /// Indexes at , whatever document is stored there. + private static async Task AddPostingsAsync(FullTextDbContext db, string text, DocumentLocation location) + { + using var txn = db.BeginTransaction(); + new FullTextIndex(db.Storage, IndexOptions.CreateBTree("Body"), BodyIndex(db).RootPageId).Insert(text, location, txn); + await txn.CommitAsync(); + } + + private static List<(string Title, double Score)> Ranked(IEnumerable> results) + => results.Select(r => (r.Document.Title, Math.Round(r.Score, 9))).ToList(); + + [Fact] + public async Task IndexesWrittenByThisVersion_AreNotRebuiltOnOpen() + { + uint body, title, empty; + using (var db = new FullTextDbContext(_dbPath)) + { + await SeedAsync(db); + body = BodyIndex(db).RootPageId; + title = db.Articles.IndexManager.GetIndex("idx_fts_Title")!.RootPageId; + } + var emptyPath = Path.ChangeExtension(_dbPath, ".fresh.db"); + using (var db = new FullTextNoIndexDbContext(emptyPath)) + { + await db.Articles.CreateFullTextIndexAsync(a => a.Title, "idx_empty"); + empty = db.Articles.IndexManager.GetIndex("idx_empty")!.RootPageId; + } + + using (var reopened = new FullTextDbContext(_dbPath)) + { + Assert.Equal(body, BodyIndex(reopened).RootPageId); + Assert.Equal(title, reopened.Articles.IndexManager.GetIndex("idx_fts_Title")!.RootPageId); + Assert.Equal(FullTextIndex.FormatVersion, BodyIndex(reopened).KeyFormat); + } + using (var reopened = new FullTextNoIndexDbContext(emptyPath)) + Assert.Equal(empty, reopened.Articles.IndexManager.GetIndex("idx_empty")!.RootPageId); + } + + [Fact] + public async Task KeysWrittenByAnOlderVersion_AreIgnored_AndTheIndexIsRebuiltOnOpen() + { + List<(string, double)> before; + using (var db = new FullTextDbContext(_dbPath)) + { + await SeedAsync(db); + var expected = Ranked(await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick fox", 10).ToListAsync()); + + // An older BLite reads a full-text index as a plain B-Tree and inserts its generic string + // keys ([0x02][string]), most of them shorter than the statistics entry. + await WriteAsync(db, txn => + { + RawTree(db).Insert(new IndexKey("a"), new DocumentLocation(1, 1), txn); + RawTree(db).Insert(new IndexKey("The quick brown fox jumps over the lazy dog"), new DocumentLocation(1, 2), txn); + }); + + Assert.Equal(expected, Ranked(await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick fox", 10).ToListAsync())); + await db.Articles.InsertAsync(A("later", "a quick reply")); + Assert.Equal(new[] { "fox", "fox again", "later" }, Titles(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("quick")).ToList())); + before = Ranked(await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick", 10).ToListAsync()); + Assert.True(BodyIndex(db).NeedsKeyFormatMigration); + } + + using var reopened = new FullTextDbContext(_dbPath); + Assert.False(BodyIndex(reopened).NeedsKeyFormatMigration); + Assert.All(RawTree(reopened).Range(IndexKey.MinKey, IndexKey.MaxKey).ToList(), e => Assert.True(e.Key.Data[0] >= 0x10)); + Assert.Equal(before, Ranked(await reopened.Articles.FullTextSearchAsync("idx_fts_Body", "quick", 10).ToListAsync())); + } + + [Fact] + public async Task IndexFromAnotherFormatVersion_IsRebuiltOnOpen() + { + using (var db = new FullTextDbContext(_dbPath)) + { + await SeedAsync(db); + // Postings the current analyzer would not produce for that document, as if written by + // another version of it: deleting the document would leave them behind. + var dog = Assert.Single(BodyIndex(db).FullTextMatch("sleeps")); + await AddPostingsAsync(db, "ghost", dog); + db.Articles.IndexManager.SetKeyFormat("idx_fts_Body", 0); + } + + var freshPath = Path.ChangeExtension(_dbPath, ".fresh.db"); + using (var fresh = new FullTextDbContext(freshPath)) + await SeedAsync(fresh); + + using var db2 = new FullTextDbContext(_dbPath); + using var fresh2 = new FullTextDbContext(freshPath); + Assert.Equal(FullTextIndex.FormatVersion, BodyIndex(db2).KeyFormat); + Assert.Empty(BodyIndex(db2).FullTextMatch("ghost")); + foreach (var query in new[] { "quick fox", "dog" }) + { + Assert.Equal( + Ranked(await fresh2.Articles.FullTextSearchAsync("idx_fts_Body", query, 10).ToListAsync()), + Ranked(await db2.Articles.FullTextSearchAsync("idx_fts_Body", query, 10).ToListAsync())); + } + } + + [Fact] + public async Task StalePostings_NeverSurfaceTheDocumentAtTheirLocation() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + var dog = Assert.Single(BodyIndex(db).FullTextMatch("sleeps")); + await AddPostingsAsync(db, "ghost", dog); + + Assert.Single(BodyIndex(db).FullTextMatch("ghost")); // the index alone would return it + Assert.Empty(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("ghost")).ToList()); + Assert.Equal(0, db.Articles.AsQueryable().Count(a => a.Body.FullTextMatch("ghost"))); + Assert.Null(db.Articles.AsQueryable().FirstOrDefault(a => a.Body.FullTextMatch("ghost sleeps"))); + Assert.Empty(await db.Articles.FullTextSearchAsync("idx_fts_Body", "ghost", 10).ToListAsync()); + Assert.Empty(await db.Articles.FullTextSearchAsync("idx_fts_Body", "ghost sleeps", 10).ToListAsync()); + var any = await db.Articles.FullTextSearchAsync("idx_fts_Body", "ghost sleeps", 10, matchAllTerms: false).ToListAsync(); + Assert.Equal("dog", Assert.Single(any).Document.Title); // through "sleeps", which it does contain + } + + [Fact] + public async Task MissingStatistics_SearchStillFindsAndRanksTheDocuments() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + await WriteAsync(db, txn => + { + var tree = RawTree(db); + var stats = tree.Range(IndexKey.FromOwnedArray([0x20]), IndexKey.FromOwnedArray([0x21]), IndexDirection.Forward, txn).ToList(); + Assert.True(tree.Delete(Assert.Single(stats).Key, Assert.Single(stats).Location, txn)); + }); + + var all = await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick fox", 10).ToListAsync(); + var any = await db.Articles.FullTextSearchAsync("idx_fts_Body", "sun fox", 10, matchAllTerms: false).ToListAsync(); + + Assert.Equal(new[] { "fox", "fox again" }, Titles(all.Select(r => r.Document))); + Assert.Equal(new[] { "dog", "fox", "fox again" }, Titles(any.Select(r => r.Document))); + Assert.All(all.Concat(any), r => Assert.True(r.Score > 0 && !double.IsInfinity(r.Score))); + } + + [Fact] + public async Task CreateFullTextIndex_UnderTheNameOfAnotherIndex_Throws() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + IDocumentCollection articles = db.Articles; + + var ex = await Assert.ThrowsAsync(() => articles.CreateFullTextIndexAsync(a => a.Body, "idx_Title")); + + Assert.Contains("idx_Title", ex.Message); + Assert.Equal(IndexType.BTree, db.Articles.GetIndexes().Single(i => i.Name == "idx_Title").Type); + Assert.Equal("fox again", db.Articles.AsQueryable().Single(a => a.Title == "fox again").Title); + } + + [Fact] + public async Task TermsLongerThan32Bytes_AreCutAtACharacterBoundary() + { + // 'ж' takes two bytes of UTF-8 and '漢' three, so neither 32-byte cut falls on a character start. + var cyrillic = "a" + new string('ж', 20); // 41 bytes; the term kept is "a" + 15 × 'ж' (31 bytes) + var cjk = new string('漢', 15); // 45 bytes; the term kept is 10 × '漢' (30 bytes) + var text = cyrillic + " " + cjk; + var stored = "a" + new string('ж', 15); + + using var db = new FullTextDbContext(_dbPath); + var article = A("long", text); + await db.Articles.InsertAsync(article); + await db.Articles.InsertAsync(A("other", "unrelated")); + + List Match(string q) => Titles(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch(q)).ToList()); + Assert.Equal(new[] { "long" }, Match(cyrillic)); + Assert.Equal(new[] { "long" }, Match(cjk)); + Assert.Equal(new[] { "long" }, Match(stored)); // the same term once cut + Assert.Equal(new[] { "long" }, Match(new string('漢', 11) + "字")); // differs only past the cut + Assert.Empty(Match("a" + new string('ж', 14))); // a different, shorter term + Assert.True(text.FullTextMatch(stored)); + Assert.Equal("long", Assert.Single(await db.Articles.FullTextSearchAsync("idx_fts_Body", cjk + " " + cyrillic, 10).ToListAsync()).Document.Title); + + // Deleting recomputes the same cut keys: nothing is left in the index itself. + article.Body = "replaced"; + await db.Articles.UpdateAsync(article); + Assert.Empty(BodyIndex(db).FullTextMatch(cyrillic)); + Assert.Empty(BodyIndex(db).FullTextMatch(cjk)); + Assert.Equal(new[] { "long" }, Match("replaced")); + } + // ── DynamicCollection ─────────────────────────────────────────────────── [Fact] @@ -378,4 +591,79 @@ BsonDocument Doc(string title, string text) => col.CreateDocument(["title", "tex Assert.Empty(await Search("forget")); } } + + /// The full-text index of , as persisted. + private static FullTextIndex StoredFullTextIndex(StorageEngine storage, string collection, string index) + => new(storage, IndexOptions.CreateBTree("text"), storage.GetCollectionMetadata(collection)!.Indexes.Single(i => i.Name == index).RootPageId); + + private static async Task SeedNotesAsync(string path) + { + using var engine = new BLiteEngine(path); + var col = engine.GetOrCreateCollection("notes"); + await col.InsertAsync(col.CreateDocument(["title", "text"], b => b.AddString("title", "kept").AddString("text", "plain words"))); + await col.InsertAsync(col.CreateDocument(["title", "text"], b => b.AddString("title", "other").AddString("text", "more words"))); + await col.CreateFullTextIndexAsync("text"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Dynamic_IndexFromAnotherFormatVersionOrWithForeignKeys_IsRebuiltOnOpen(bool foreignKey) + { + await SeedNotesAsync(_dbPath); + using (var engine = new BLiteEngine(_dbPath)) // the collection itself is not opened + { + var storage = engine.Storage; + var index = StoredFullTextIndex(storage, "notes", "idx_fts_text"); + var kept = Assert.Single(index.Match("plain", null)); + var txn = storage.BeginTransaction(); + index.Insert("ghost", kept, txn); + if (foreignKey) + new BTreeIndex(storage, IndexOptions.CreateBTree("text"), index.RootPageId).Insert(new IndexKey("plain words"), kept, txn.TransactionId); + await txn.CommitAsync(); + + if (!foreignKey) + { + var metadata = storage.GetCollectionMetadata("notes")!; + metadata.Indexes.Single(i => i.Name == "idx_fts_text").KeyFormat = 0; + storage.SaveCollectionMetadata(metadata); + } + Assert.Single(StoredFullTextIndex(storage, "notes", "idx_fts_text").Match("ghost", null)); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + var rebuilt = StoredFullTextIndex(engine.Storage, "notes", "idx_fts_text"); + + Assert.Empty(rebuilt.Match("ghost", null)); + Assert.False(rebuilt.HasForeignKeys()); + Assert.Equal(FullTextIndex.FormatVersion, engine.Storage.GetCollectionMetadata("notes")!.Indexes.Single(i => i.Name == "idx_fts_text").KeyFormat); + Assert.Equal(2, (await col.FullTextSearchAsync("idx_fts_text", "words", 10).ToListAsync()).Count); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + // Rebuilt once: the next open keeps the tree. + var root = engine.Storage.GetCollectionMetadata("notes")!.Indexes.Single(i => i.Name == "idx_fts_text").RootPageId; + engine.GetOrCreateCollection("notes"); + Assert.Equal(root, engine.Storage.GetCollectionMetadata("notes")!.Indexes.Single(i => i.Name == "idx_fts_text").RootPageId); + } + } + + [Fact] + public async Task Dynamic_StalePostings_NeverSurfaceTheDocumentAtTheirLocation() + { + await SeedNotesAsync(_dbPath); + using var engine = new BLiteEngine(_dbPath); + var index = StoredFullTextIndex(engine.Storage, "notes", "idx_fts_text"); + var txn = engine.Storage.BeginTransaction(); + index.Insert("ghost", Assert.Single(index.Match("plain", null)), txn); + await txn.CommitAsync(); + + var col = engine.GetOrCreateCollection("notes"); + Assert.Empty(await col.FullTextSearchAsync("idx_fts_text", "ghost", 10).ToListAsync()); + var any = await col.FullTextSearchAsync("idx_fts_text", "ghost plain", 10, matchAllTerms: false).ToListAsync(); + Assert.Equal("kept", Assert.Single(any).Document.TryGetString("title", out var t) ? t : null); + } } From ae69af6c1847d1b8d18106f02df8cb1bad8da055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 16:11:20 +0200 Subject: [PATCH 08/15] docs(fts): batching advice and name-collision exception on CreateFullTextIndexAsync (#70) Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Collections/DocumentCollection.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 63190ff..acd23ee 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -997,7 +997,13 @@ public async Task> CreateIndexAsync( /// collection, or returns the existing index of that name. Query it with /// in LINQ or rank with /// . The default name is idx_fts_{property}. + /// + /// Each document touches one index page per distinct term, so single-document writes cost + /// noticeably more than without the index; insert and update in batches + /// (InsertBulkAsync or one explicit transaction) where possible. + /// /// + /// An index of that name exists and is not a full-text index. [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] public async Task> CreateFullTextIndexAsync( System.Linq.Expressions.Expression> keySelector, From 903f7fe5b4b6b9c641a15c6cc5a4014eba0a0a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 17:05:20 +0200 Subject: [PATCH 09/15] test(fts): a failed rebuild triggered by foreign keys is retried on the next open (#70) A full-text index whose format is current but whose tree holds keys it did not write (e.g. from an older BLite reading it as a plain B-Tree) is rebuilt on open. MigrateLegacyDecimalIndexes (on #175's base) rebuilds it into an unregistered copy and swaps that in only after it committed, so a rebuild that fails leaves the old index registered as it was: still holding the foreign keys, still needing the migration. - MigrateLegacyDecimalIndexes doc comment covers full-text indexes. - Test: corrupt a document so the rebuild on open fails; over three failing opens the stored root and format are unchanged, the foreign key is still there and the file does not grow; after repairing the document the next open swaps in the rebuilt index and a further open keeps it. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 9 +-- tests/BLite.Tests/FullTextSearchTests.cs | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index acd23ee..85fac86 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -867,12 +867,13 @@ internal DocumentCollection(StorageEngine storage, ITransactionHolder transactio /// /// Decimal properties used to be indexed as (double)value. An index whose persisted - /// predates the exact decimal encoding is rebuilt from the + /// predates the exact decimal encoding, or a full-text + /// index written in another format or holding keys it did not write, is rebuilt from the /// documents into a copy that queries don't see, the first time the collection is opened. Only - /// after the copy committed is it swapped in, with the current format, in one metadata save, and + /// after the copy committed is it swapped in, with its target format, in one metadata save, and /// the old tree freed. If the rebuild fails or is interrupted, the copy's pages are freed and the - /// old index stays registered in its old format, so the next open repeats the migration. - /// No-op for every other index. + /// old index stays registered as it was, so it still needs the migration (whatever triggered it) + /// and the next open repeats it. No-op for every other index. /// private void MigrateLegacyDecimalIndexes() { diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs index cb1e2d0..a4c1e52 100644 --- a/tests/BLite.Tests/FullTextSearchTests.cs +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -472,6 +472,67 @@ public async Task IndexFromAnotherFormatVersion_IsRebuiltOnOpen() } } + [Fact] + public async Task FailedRebuildTriggeredByForeignKeys_IsRetriedOnTheNextOpen() + { + DocumentLocation dog; + byte[] intact; + uint root; + using (var db = new FullTextDbContext(_dbPath)) + { + await SeedAsync(db); + // Only the foreign key triggers the rebuild: the persisted format is already current. + await WriteAsync(db, txn => RawTree(db).Insert(new IndexKey("a"), new DocumentLocation(1, 1), txn)); + Assert.Equal(FullTextIndex.FormatVersion, BodyIndex(db).KeyFormat); + Assert.True(BodyIndex(db).NeedsKeyFormatMigration); + root = BodyIndex(db).RootPageId; + await db.CheckpointAsync(); + + // Make one document unreadable, so the rebuild on the next open fails. + dog = Assert.Single(BodyIndex(db).FullTextMatch("sleeps")); + intact = new byte[db.Storage.PageSize]; + db.Storage.ReadPage(dog.PageId, null, intact); + var corrupt = (byte[])intact.Clone(); + var slot = SlotEntry.ReadFrom(corrupt.AsSpan(SlottedPageHeader.Size + dog.SlotIndex * SlotEntry.Size)); + corrupt.AsSpan(slot.Offset + 4, 12).Fill(0xFF); // invalid element types + db.Storage.WritePageImmediate(dog.PageId, corrupt); + } + + var sizes = new List(); + for (int open = 0; open < 3; open++) + { + var ex = Assert.Throws(() => new FullTextDbContext(_dbPath)); + Assert.Contains("idx_fts_Body", ex.Message); + sizes.Add(new FileInfo(_dbPath).Length); + + using var engine = new BLiteEngine(_dbPath); // the collection itself is not opened + // The old index is still the registered one, foreign key included, so it still needs + // the migration (format current, foreign keys present) and the next open retries it. + var stored = engine.Storage.GetCollectionMetadata("articles")!.Indexes.Single(i => i.Name == "idx_fts_Body"); + Assert.Equal(root, stored.RootPageId); + Assert.Equal(FullTextIndex.FormatVersion, stored.KeyFormat); + Assert.True(new FullTextIndex(engine.Storage, IndexOptions.CreateBTree("Body"), stored.RootPageId).HasForeignKeys()); + if (open == 2) engine.Storage.WritePageImmediate(dog.PageId, intact); + } + // The copy each failed open built is freed, so later opens reuse its pages. + Assert.All(sizes.Skip(1), size => Assert.Equal(sizes[0], size)); + + uint migrated; + using (var repaired = new FullTextDbContext(_dbPath)) + { + migrated = BodyIndex(repaired).RootPageId; + Assert.NotEqual(root, migrated); // the rebuilt copy was swapped in + Assert.Equal(FullTextIndex.FormatVersion, BodyIndex(repaired).KeyFormat); + Assert.False(BodyIndex(repaired).NeedsKeyFormatMigration); + } + + using var reopened = new FullTextDbContext(_dbPath); + Assert.Equal(migrated, BodyIndex(reopened).RootPageId); // not rebuilt again + Assert.Equal(dog, Assert.Single(BodyIndex(reopened).FullTextMatch("sleeps"))); + Assert.Equal(new[] { "fox", "fox again" }, + Titles((await reopened.Articles.FullTextSearchAsync("idx_fts_Body", "quick fox", 10).ToListAsync()).Select(r => r.Document))); + } + [Fact] public async Task StalePostings_NeverSurfaceTheDocumentAtTheirLocation() { From 406d1db86edf513988d5b4826a52022889767511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 17:07:42 +0200 Subject: [PATCH 10/15] fix(fts): return k re-checked hits and each location once (#70) - FullTextSearchAsync (typed and dynamic) keeps taking candidates past the first k, in widening windows, until k hits survive the re-check against the document; the first window costs one Search as before. - Postings yields one posting per location, so a stale posting sharing its term and location with the current document's no longer makes Match return the location twice or Search score it twice. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 11 +++-- src/BLite.Core/DynamicCollection.cs | 12 ++++-- .../Indexing/CollectionSecondaryIndex.cs | 12 ++++++ src/BLite.Core/Indexing/FullTextIndex.cs | 38 +++++++++++++++++ tests/BLite.Tests/FullTextSearchTests.cs | 42 +++++++++++++++++++ 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 85fac86..71b1213 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -3939,12 +3939,17 @@ public async IAsyncEnumerable> FullTextSearchAsync( if (index == null) throw new ArgumentException($"Index '{indexName}' not found.", nameof(indexName)); - foreach (var (location, score) in index.FullTextSearch(query, k, matchAllTerms)) + // Hits are re-checked against the document, so a stale posting can never surface another + // document; candidates past the first k are taken until k hits survive the re-check. + int returned = 0; + foreach (var (location, score) in index.FullTextSearchExpanding(query, k, matchAllTerms)) { ct.ThrowIfCancellationRequested(); var doc = await FindByLocationAsync(location, 0UL, ct); - if (doc != null && index.FullTextContains(doc, query, matchAllTerms)) - yield return new FullTextSearchResult(doc, score); + if (doc == null || !index.FullTextContains(doc, query, matchAllTerms)) continue; + + yield return new FullTextSearchResult(doc, score); + if (++returned == k) yield break; } } diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index 0688ba5..f56d0bd 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -1060,16 +1060,20 @@ public async IAsyncEnumerable> FullTextSearch if (!_secondaryIndexes.TryGetValue(indexName, out var entry) || entry.Kind != DynamicIndexKind.FullText || entry.FullText == null) throw new ArgumentException($"Full-text index '{indexName}' not found on collection '{_collectionName}'"); - foreach (var (location, score) in entry.FullText.Search(query, k, matchAllTerms, null)) + int returned = 0; + foreach (var (location, score) in entry.FullText.SearchExpanding(query, k, matchAllTerms, null)) { ct.ThrowIfCancellationRequested(); var doc = ReadDocumentAt(location, 0UL); if (doc == null) continue; - // Re-check the hit, so a stale posting can never surface another document. + // Re-check the hit, so a stale posting can never surface another document; candidates + // past the first k are taken until k hits survive the re-check. var text = TryGetNestedValue(doc, entry.FieldPath, out var val) && val.IsString ? val.AsString : null; - if (matchAllTerms ? FullTextAnalyzer.Matches(text, query) : FullTextAnalyzer.MatchesAny(text, query)) - yield return new FullTextSearchResult(doc, score); + if (!(matchAllTerms ? FullTextAnalyzer.Matches(text, query) : FullTextAnalyzer.MatchesAny(text, query))) continue; + + yield return new FullTextSearchResult(doc, score); + if (++returned == k) yield break; } } diff --git a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs index 3fc779e..22acf4f 100644 --- a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs +++ b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs @@ -404,6 +404,18 @@ public IReadOnlyList FullTextMatch(string query, ITransaction? return _fullTextIndex.Search(query, k, matchAllTerms, transaction); } + /// + /// The BM25 matches for , best first, starting with the + /// best and continuing past them for as long as the caller enumerates. + /// + internal IEnumerable<(DocumentLocation Location, double Score)> FullTextSearchExpanding(string query, int k, bool matchAllTerms, ITransaction? transaction = null) + { + if (_fullTextIndex == null) + throw new InvalidOperationException("This index is not a full-text index."); + + return _fullTextIndex.SearchExpanding(query, k, matchAllTerms, transaction); + } + /// /// Scans a range of keys (O(log n + k) where k is result count) /// diff --git a/src/BLite.Core/Indexing/FullTextIndex.cs b/src/BLite.Core/Indexing/FullTextIndex.cs index d566881..7b8f258 100644 --- a/src/BLite.Core/Indexing/FullTextIndex.cs +++ b/src/BLite.Core/Indexing/FullTextIndex.cs @@ -214,6 +214,32 @@ public IReadOnlyList Match(string? query, ITransaction? transa .ToList(); } + /// + /// Like , but keeps going past the first hits, best first, + /// for as long as the caller enumerates: when a window is used up it searches again with twice as + /// many and skips the hits already returned. Callers that drop hits (re-checking them against the + /// document) can so still return results. The first window costs exactly one + /// . + /// + public IEnumerable<(DocumentLocation Location, double Score)> SearchExpanding(string? query, int k, bool matchAllTerms, ITransaction? transaction) + { + HashSet? returned = null; + for (int window = k; window > 0;) + { + var hits = Search(query, window, matchAllTerms, transaction); + foreach (var hit in hits) + { + if (returned == null || !returned.Contains(Pack(hit.Location))) + yield return hit; + } + if (hits.Count < window || window == int.MaxValue) yield break; + + returned ??= new HashSet(); + foreach (var hit in hits) returned.Add(Pack(hit.Location)); + window = window > int.MaxValue / 2 ? int.MaxValue : window * 2; + } + } + /// The posting of for the document at , if any. private (DocumentLocation Location, int Frequency, int Length)? TryPosting(string term, DocumentLocation location, ulong? txnId) { @@ -238,11 +264,23 @@ public IReadOnlyList Match(string? query, ITransaction? transa return null; } + /// + /// The postings of , one per location. A stale posting left at a location + /// that now holds another document containing the term differs only in its frequency and length, + /// so it sorts next to the current one and is skipped: the first posting of a location wins. + /// private IEnumerable<(DocumentLocation Location, int Frequency, int Length)> Postings(string term, ulong? txnId) { int offset = 2 + Encoding.UTF8.GetByteCount(term) + DocumentLocation.SerializedSize; + ulong previous = 0; + bool first = true; foreach (var entry in _tree.Range(TermMin(term), TermMax(term), IndexDirection.Forward, txnId)) { + var packed = Pack(entry.Location); + if (!first && packed == previous) continue; + first = false; + previous = packed; + var key = entry.Key.Data; yield return (entry.Location, BinaryPrimitives.ReadUInt16BigEndian(key.Slice(offset)), diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs index a4c1e52..1aca7b9 100644 --- a/tests/BLite.Tests/FullTextSearchTests.cs +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -551,6 +551,41 @@ public async Task StalePostings_NeverSurfaceTheDocumentAtTheirLocation() Assert.Equal("dog", Assert.Single(any).Document.Title); // through "sleeps", which it does contain } + [Fact] + public async Task StaleHitsInTheTopK_AreReplacedByTheNextCandidates() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + // A stale posting that outranks every document really containing "quick". + var dog = Assert.Single(BodyIndex(db).FullTextMatch("sleeps")); + await AddPostingsAsync(db, "quick quick quick quick quick", dog); + Assert.Equal(dog, BodyIndex(db).FullTextSearch("quick", 1)[0].Location); + + var top1 = await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick", 1).ToListAsync(); + var top2 = await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick", 2).ToListAsync(); + var top5 = await db.Articles.FullTextSearchAsync("idx_fts_Body", "quick", 5).ToListAsync(); + + Assert.Equal("fox again", Assert.Single(top1).Document.Title); + Assert.Equal(new[] { "fox again", "fox" }, top2.Select(r => r.Document.Title)); + Assert.Equal(new[] { "fox again", "fox" }, top5.Select(r => r.Document.Title)); + } + + [Fact] + public async Task StalePostingAtTheLocationOfADocumentWithTheSameTerm_IsCountedOnce() + { + using var db = new FullTextDbContext(_dbPath); + await SeedAsync(db); + var dog = Assert.Single(BodyIndex(db).FullTextMatch("sleeps")); + var before = Ranked(await db.Articles.FullTextSearchAsync("idx_fts_Body", "sleeps", 10).ToListAsync()); + // Same term and location as the real posting, another frequency and length. + await AddPostingsAsync(db, "sleeps", dog); + + Assert.Equal(dog, Assert.Single(BodyIndex(db).FullTextMatch("sleeps"))); + Assert.Equal("dog", Assert.Single(db.Articles.AsQueryable().Where(a => a.Body.FullTextMatch("sleeps")).ToList()).Title); + Assert.Equal("dog", Assert.Single(await db.Articles.FullTextSearchAsync("idx_fts_Body", "sleeps", 10).ToListAsync()).Document.Title); + Assert.Equal(before.Select(r => r.Title), Ranked(await db.Articles.FullTextSearchAsync("idx_fts_Body", "sleeps", 10).ToListAsync()).Select(r => r.Title)); + } + [Fact] public async Task MissingStatistics_SearchStillFindsAndRanksTheDocuments() { @@ -726,5 +761,12 @@ public async Task Dynamic_StalePostings_NeverSurfaceTheDocumentAtTheirLocation() Assert.Empty(await col.FullTextSearchAsync("idx_fts_text", "ghost", 10).ToListAsync()); var any = await col.FullTextSearchAsync("idx_fts_text", "ghost plain", 10, matchAllTerms: false).ToListAsync(); Assert.Equal("kept", Assert.Single(any).Document.TryGetString("title", out var t) ? t : null); + + // A stale hit ranked first is replaced by the next candidate, so k hits still come back. + txn = engine.Storage.BeginTransaction(); + index.Insert("more more more more", Assert.Single(index.Match("plain", null)), txn); + await txn.CommitAsync(); + var top1 = await col.FullTextSearchAsync("idx_fts_text", "more", 1).ToListAsync(); + Assert.Equal("other", Assert.Single(top1).Document.TryGetString("title", out var o) ? o : null); } } From b1c739806999d45a55aed6ddc8c52ad402ca25c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 18:55:26 +0200 Subject: [PATCH 11/15] docs(fts): rarest-first saves decoding and allocation, not I/O (#70) CountRange walks every leaf entry of each query term, so looking up the few candidates in a common term's postings avoids decoding them, not reading their pages. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Indexing/FullTextIndex.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/BLite.Core/Indexing/FullTextIndex.cs b/src/BLite.Core/Indexing/FullTextIndex.cs index 7b8f258..0e0b943 100644 --- a/src/BLite.Core/Indexing/FullTextIndex.cs +++ b/src/BLite.Core/Indexing/FullTextIndex.cs @@ -120,7 +120,8 @@ public IReadOnlyList Match(string? query, ITransaction? transa if (terms.Count == 0) return Array.Empty(); var txnId = transaction?.TransactionId; - // Start from the rarest term so the candidate list is as short as possible. + // Start from the rarest term so the candidate list is as short as possible. Counting a term + // (CountRange) still walks all of its leaf entries, without decoding or allocating them. var ordered = terms .Select(t => (Term: t, Count: _tree.CountRange(TermMin(t), TermMax(t), txnId))) .OrderBy(t => t.Count) @@ -148,7 +149,8 @@ public IReadOnlyList Match(string? query, ITransaction? transa var txnId = transaction?.TransactionId; - // Rarest term first: with matchAllTerms its postings bound the candidate set. + // Rarest term first: with matchAllTerms its postings bound the candidate set. Counting a term + // (CountRange) still walks all of its leaf entries, without decoding or allocating them. var ordered = terms .Select(t => (Term: t, Count: _tree.CountRange(TermMin(t), TermMax(t), txnId))) .OrderBy(t => t.Count) @@ -161,8 +163,9 @@ public IReadOnlyList Match(string? query, ITransaction? transa { if (count == 0) continue; - // Once the candidates are few, look each one up instead of reading every posting of a - // much more common term. + // Once the candidates are few, look each one up instead of decoding every posting of a + // much more common term. This saves decoding and allocation, not I/O: CountRange above + // has already read that term's leaf pages. var postings = candidates != null && candidates.Count * 8 < count ? candidates.Values.Select(l => TryPosting(term, l, txnId)).Where(p => p.HasValue).Select(p => p!.Value).ToList() : Postings(term, txnId).ToList(); From 179378dcb65881984d94bcc3b0cd77005f9825fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:16:36 +0200 Subject: [PATCH 12/15] docs(fts): a stale posting at a document's location can be the one scored (#70) - Postings documents that the first posting of a location in key order (lowest frequency, then shortest length) is returned, which can be a stale one, so its frequency and length can be used for BM25 scoring, and CountRange counts both towards the document frequency. Matches and once-per-location results are unaffected. - The test for a stale posting at a document's location notes that it checks membership and order, not scores. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Indexing/FullTextIndex.cs | 9 ++++++++- tests/BLite.Tests/FullTextSearchTests.cs | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/BLite.Core/Indexing/FullTextIndex.cs b/src/BLite.Core/Indexing/FullTextIndex.cs index 0e0b943..2c3be76 100644 --- a/src/BLite.Core/Indexing/FullTextIndex.cs +++ b/src/BLite.Core/Indexing/FullTextIndex.cs @@ -270,8 +270,15 @@ public IReadOnlyList Match(string? query, ITransaction? transa /// /// The postings of , one per location. A stale posting left at a location /// that now holds another document containing the term differs only in its frequency and length, - /// so it sorts next to the current one and is skipped: the first posting of a location wins. + /// so it sorts next to the current one and only the first posting of the location is returned. /// + /// + /// "First" is key order, i.e. lowest frequency, then shortest length, not the current posting, so + /// the stale one can be the posting used for scoring (as in ), and + /// CountRange counts both towards the term's document frequency. That only skews BM25 + /// scores while the stale posting exists: which documents match, and that each comes back once, + /// do not depend on it, and hits are re-checked against the document. + /// private IEnumerable<(DocumentLocation Location, int Frequency, int Length)> Postings(string term, ulong? txnId) { int offset = 2 + Encoding.UTF8.GetByteCount(term) + DocumentLocation.SerializedSize; diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs index 1aca7b9..b9cb0d5 100644 --- a/tests/BLite.Tests/FullTextSearchTests.cs +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -570,6 +570,7 @@ public async Task StaleHitsInTheTopK_AreReplacedByTheNextCandidates() Assert.Equal(new[] { "fox again", "fox" }, top5.Select(r => r.Document.Title)); } + // Checks membership and order only: the stale posting may be the one scored (see FullTextIndex.Postings). [Fact] public async Task StalePostingAtTheLocationOfADocumentWithTheSameTerm_IsCountedOnce() { From f1cd8f06a0fe5106d6a017f56100ee6dd9ba3540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:39:38 +0200 Subject: [PATCH 13/15] fix(fts): free a failed dynamic full-text rebuild and keep the old index (#70) DynamicCollection rebuilt a stale full-text index on open into a fresh tree and, if a document could not be read or indexed, rolled back and threw without freeing that tree. Pages added by splits are allocated outside the transaction, so every failing open leaked the partial tree and grew the file. CreateFullTextIndexAsync leaked the same way. - FillUnregisteredFullTextIndexAsync fills the new index in its own transaction; on failure it collects the index's pages through the transaction before the rollback and frees them. Used by the rebuild on open and by CreateFullTextIndexAsync. - PublishFullTextIndex registers the filled index and persists the metadata; if that fails it restores the previous registration and frees the new index. - Each full-text index keeps its own KeyFormat and PersistIndexMetadata writes it, so persisting after one index's rebuild no longer marks another index that still awaits its rebuild as current (e.g. a crash between two rebuilds). Test: a dynamic full-text index over 3000 documents with a foreign key and an unreadable last document; four failing opens do not grow the file and keep the old index (same root, still searchable, still flagged for rebuild), and the rebuild succeeds once the document is repaired. Fails without this change. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/DynamicCollection.cs | 109 ++++++++++++++++------- tests/BLite.Tests/FullTextSearchTests.cs | 81 +++++++++++++++++ 2 files changed, 160 insertions(+), 30 deletions(-) diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index f56d0bd..da5a920 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -72,6 +72,12 @@ private sealed class DynamicSecondaryIndex public VectorSearchIndex? Vector { get; } public RTreeIndex? Spatial { get; } public FullTextIndex? FullText { get; } + /// + /// Key format of a full-text index's pages, persisted as its . + /// Kept per index so persisting the metadata never marks an index that still awaits its + /// rebuild as current. + /// + public byte KeyFormat { get; init; } = FullTextIndex.FormatVersion; public uint RootPageId => BTree?.RootPageId ?? Vector?.RootPageId ?? Spatial?.RootPageId ?? FullText?.RootPageId ?? 0; public DynamicSecondaryIndex(BTreeIndex btree, string fieldPath, IndexOptions options) @@ -169,7 +175,7 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction { var opts = IndexOptions.CreateBTree(idxMeta.PropertyPaths); var fullText = new FullTextIndex(_storage, opts, idxMeta.RootPageId); - _secondaryIndexes[idxMeta.Name] = new DynamicSecondaryIndex(fullText, fieldPath, opts); + _secondaryIndexes[idxMeta.Name] = new DynamicSecondaryIndex(fullText, fieldPath, opts) { KeyFormat = idxMeta.KeyFormat }; if (idxMeta.KeyFormat != FullTextIndex.FormatVersion || fullText.HasForeignKeys()) staleFullTextIndexes.Add(idxMeta.Name); break; @@ -221,33 +227,91 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction /// /// Rebuilds full-text index into a fresh tree, swaps it in once the /// rebuild committed, persists it with the current format version and frees the old tree. + /// If the rebuild or the swap fails, the fresh tree's pages are freed and the old index stays + /// persisted as it was, so the next open retries. /// private void RebuildFullTextIndex(string name) { var old = _secondaryIndexes[name]; var fresh = new DynamicSecondaryIndex(new FullTextIndex(_storage, old.Options), old.FieldPath, old.Options); + // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. + Task.Run(() => FillUnregisteredFullTextIndexAsync(fresh)).GetAwaiter().GetResult(); + + var oldPages = old.FullText!.CollectAllPages(); + PublishFullTextIndex(name, fresh, old); + foreach (var pageId in oldPages) + _storage.FreePage(pageId); + } + + /// + /// Fills a full-text index that is not registered yet from every document, in its own + /// transaction, and commits it. If a document cannot be read or indexed, or the commit fails, + /// every page of the index is freed, including the pages its splits added, and the exception + /// propagates. + /// + private async Task FillUnregisteredFullTextIndexAsync(DynamicSecondaryIndex index) + { var transaction = _storage.BeginTransaction(IsolationLevel.ReadCommitted); try { foreach (var e in _primaryIndex.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transaction.TransactionId)) { var doc = ReadDocumentAt(e.Location, transaction.TransactionId); - if (doc != null) IndexInsert(fresh, doc, e.Location, transaction); + if (doc != null) IndexInsert(index, doc, e.Location, transaction); } - // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait. - Task.Run(async () => await transaction.CommitAsync()).GetAwaiter().GetResult(); + await transaction.CommitAsync().ConfigureAwait(false); } catch { - Task.Run(async () => await transaction.RollbackAsync()).GetAwaiter().GetResult(); + // Collect the pages through the transaction before rolling it back: pages added by + // splits are allocated outside the transaction but linked only from nodes written in + // it, so after the rollback they could not be found any more. + IReadOnlyList pages = Array.Empty(); + try { pages = index.FullText!.CollectAllPages(transaction.TransactionId); } + catch { /* best-effort: never mask the original failure */ } + + try { await transaction.RollbackAsync().ConfigureAwait(false); } + catch { /* best-effort: never mask the original failure */ } + + FreePagesBestEffort(pages); throw; } + } - var oldPages = old.FullText!.CollectAllPages(); - _secondaryIndexes[name] = fresh; - PersistIndexMetadata(); - foreach (var pageId in oldPages) - _storage.FreePage(pageId); + /// + /// Registers the filled full-text index under + /// in place of (null: none) and persists the metadata. If that + /// fails, the previous registration is restored, the index's pages are freed and the exception + /// propagates. + /// + private void PublishFullTextIndex(string name, DynamicSecondaryIndex index, DynamicSecondaryIndex? previous) + { + _secondaryIndexes[name] = index; + try + { + PersistIndexMetadata(); + } + catch + { + if (previous != null) _secondaryIndexes[name] = previous; + else _secondaryIndexes.Remove(name); + + IReadOnlyList pages = Array.Empty(); + try { pages = index.FullText!.CollectAllPages(); } + catch { /* best-effort: never mask the original failure */ } + FreePagesBestEffort(pages); + throw; + } + } + + private void FreePagesBestEffort(IReadOnlyList pages) + { + try + { + foreach (var pageId in pages) + _storage.FreePage(pageId); + } + catch { /* best-effort: never mask the original failure */ } } private void RebuildFreeSpaceIndex() @@ -1854,25 +1918,10 @@ public async Task CreateFullTextIndexAsync(string fieldPath, string? name = null var opts = IndexOptions.CreateBTree(fieldPath); var entry = new DynamicSecondaryIndex(new FullTextIndex(_storage, opts), fieldPath, opts); - // Fill before registering, so a crash or a failure leaves no partial index persisted. - var transaction = _storage.BeginTransaction(IsolationLevel.ReadCommitted); - try - { - foreach (var e in _primaryIndex.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transaction.TransactionId)) - { - var doc = ReadDocumentAt(e.Location, transaction.TransactionId); - if (doc != null) IndexInsert(entry, doc, e.Location, transaction); - } - await transaction.CommitAsync().ConfigureAwait(false); - } - catch - { - await transaction.RollbackAsync().ConfigureAwait(false); - throw; - } - - _secondaryIndexes[name] = entry; - PersistIndexMetadata(); + // Fill before registering, so a crash or a failure leaves no partial index persisted; + // a failed fill or registration frees the index's pages. + await FillUnregisteredFullTextIndexAsync(entry).ConfigureAwait(false); + PublishFullTextIndex(name, entry, previous: null); } finally { @@ -1984,7 +2033,7 @@ internal void PersistIndexMetadata() break; case DynamicIndexKind.FullText: idxMeta.Type = IndexType.FullText; - idxMeta.KeyFormat = FullTextIndex.FormatVersion; + idxMeta.KeyFormat = idx.KeyFormat; break; } diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs index b9cb0d5..1e24813 100644 --- a/tests/BLite.Tests/FullTextSearchTests.cs +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -748,6 +748,87 @@ public async Task Dynamic_IndexFromAnotherFormatVersionOrWithForeignKeys_IsRebui } } + /// + /// A rebuild on open that fails late, after the new tree has split, frees every page of the + /// partial rebuild and keeps the old index: it is retried on each open without growing the + /// file, and succeeds once the document can be read again. + /// + [Fact] + public async Task Dynamic_RebuildOnOpen_ThatFailsLate_KeepsTheOldIndex_AndDoesNotGrowTheFile() + { + const int count = 3000; + uint root; + DocumentLocation last; + byte[] intact; + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + await col.InsertBulkAsync(Enumerable.Range(0, count).Select(i => col.CreateDocument(["title", "text"], b => b + .AddString("title", $"n{i}").AddString("text", $"note number{i} about topic{i % 50} and more words"))).ToList()); + await col.CreateFullTextIndexAsync("text"); + + var storage = engine.Storage; + var index = StoredFullTextIndex(storage, "notes", "idx_fts_text"); + root = index.RootPageId; + // A foreign key makes the next open rebuild the index. + var txn = storage.BeginTransaction(); + new BTreeIndex(storage, IndexOptions.CreateBTree("text"), root).Insert(new IndexKey("plain"), new DocumentLocation(1, 1), txn.TransactionId); + await txn.CommitAsync(); + await engine.CheckpointAsync(); + + // Make the last document in primary-key order unreadable, so the rebuild fails only + // after thousands of inserts. + var primary = new BTreeIndex(storage, IndexOptions.CreateUnique("_id"), storage.GetCollectionMetadata("notes")!.PrimaryRootPageId); + last = primary.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Backward).First().Location; + intact = new byte[storage.PageSize]; + storage.ReadPage(last.PageId, null, intact); + var corrupt = (byte[])intact.Clone(); + var slot = SlotEntry.ReadFrom(corrupt.AsSpan(SlottedPageHeader.Size + last.SlotIndex * SlotEntry.Size)); + corrupt.AsSpan(slot.Offset + 4, 12).Fill(0xFF); // invalid element types + storage.WritePageImmediate(last.PageId, corrupt); + } + + var sizes = new List(); + for (int open = 0; open < 4; open++) + { + using (var engine = new BLiteEngine(_dbPath)) + Assert.ThrowsAny(() => engine.GetOrCreateCollection("notes")); + sizes.Add(new FileInfo(_dbPath).Length); + + using (var engine = new BLiteEngine(_dbPath)) // the collection itself is not opened + { + // The old index is still the persisted one, foreign key and all, so the next open retries. + var stored = StoredFullTextIndex(engine.Storage, "notes", "idx_fts_text"); + Assert.Equal(root, stored.RootPageId); + Assert.True(stored.HasForeignKeys()); + Assert.Single(stored.Match("number7", null)); + } + } + // The first failed open may extend the file; the following ones reuse the freed pages. + Assert.All(sizes.Skip(1), size => Assert.Equal(sizes[0], size)); + + using (var engine = new BLiteEngine(_dbPath)) + engine.Storage.WritePageImmediate(last.PageId, intact); + + uint rebuilt; + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + var stored = StoredFullTextIndex(engine.Storage, "notes", "idx_fts_text"); + rebuilt = stored.RootPageId; + Assert.NotEqual(root, rebuilt); + Assert.False(stored.HasForeignKeys()); + Assert.Equal(count / 50, (await col.FullTextSearchAsync("idx_fts_text", "topic7", count).ToListAsync()).Count); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + // Rebuilt once: the next open keeps the tree. + engine.GetOrCreateCollection("notes"); + Assert.Equal(rebuilt, StoredFullTextIndex(engine.Storage, "notes", "idx_fts_text").RootPageId); + } + } + [Fact] public async Task Dynamic_StalePostings_NeverSurfaceTheDocumentAtTheirLocation() { From e67d375e19a68edf3e7a01c8a086cb969db4dce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 17:34:32 +0200 Subject: [PATCH 14/15] feat(fts): record full-text indexes as a required feature of the collection (#70) Uses the per-collection required features from #194, so BLite versions that have that check but not full-text indexes refuse the collection instead of misreading it. - CollectionFeatures.FullTextIndexes: id "fulltext-index", description "full-text indexes (BLite 5.3 or later)", requirement Open. A version without full-text indexes reads the index as a B-Tree: its planner answers equality, range and IN lookups on the property from the postings, so even read-only results are wrong. It is a known feature in this version. - SyncIndexFeatures records the feature while the collection has a full-text index and removes it with the last one, leaving every other feature alone. Called whenever the index list is written: typed UpdateMetadata (CreateFullTextIndexAsync, model-declared indexes via ApplyIndexBuilder, rebuilds, DropIndex) and dynamic PersistIndexMetadata. - A read-write open records the feature again if a release that predates it rewrote the record and dropped the section (5.2.x does); a read-only open leaves the file alone. - CollectionFeatures and ReadCollectionMetadata document that the index-type rule only helps while an old writer keeps the index entry (a 5.2.x dynamic collection drops both), and that new per-index data goes only in trailing sections. - CreateFullTextIndexAsync (typed and dynamic) notes: do not open a database with full-text indexes for writing with BLite < 5.3. Tests: the feature is recorded for model-declared, typed and dynamic indexes, survives reopen, stays while another full-text index is left and is cleared with the last one; it is recorded again on a read-write open after being dropped, not on a read-only one; a version without full-text indexes (known features and index types passed in) refuses to open the collection even read-only, from the feature alone and from the index type alone. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 5 + src/BLite.Core/DynamicCollection.cs | 10 ++ .../Indexing/CollectionIndexManager.cs | 5 +- src/BLite.Core/Storage/CollectionFeatures.cs | 52 ++++++- .../Storage/StorageEngine.Collections.cs | 3 + tests/BLite.Tests/FullTextSearchTests.cs | 144 ++++++++++++++++++ 6 files changed, 212 insertions(+), 7 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 71b1213..fd85467 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -1003,6 +1003,11 @@ public async Task> CreateIndexAsync( /// noticeably more than without the index; insert and update in batches /// (InsertBulkAsync or one explicit transaction) where possible. /// + /// + /// Compatibility: the collection records that it requires full-text indexes, so BLite 5.3 and + /// later versions without them refuse to open it. Older releases cannot refuse: do not open a + /// database with full-text indexes for writing with BLite < 5.3. + /// /// /// An index of that name exists and is not a full-text index. [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")] diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index da5a920..06619dc 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -210,6 +210,10 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction rootsChanged = true; } } + // Re-record the features the indexes require if a release that predates them rewrote the + // record and dropped them (a read-only open cannot, and the indexes still protect it). + if (!_storage.IsReadOnly && CollectionFeatures.SyncIndexFeatures(metadata)) + rootsChanged = true; // Only the root ids are patched in place so index entries this constructor does not // restore (unknown types, no property paths) are preserved as stored. if (rootsChanged) @@ -1898,6 +1902,11 @@ public async Task CreateSpatialIndexAsync(string fieldPath, string? name = null) /// Creates a full-text index on a string field (dot-notation for nested fields), filled from /// the documents already in the collection. Search it with . /// The default name is idx_fts_{field}. + /// + /// Compatibility: the collection records that it requires full-text indexes, so BLite 5.3 and + /// later versions without them refuse to open it. Older releases cannot refuse: do not open a + /// database with full-text indexes for writing with BLite < 5.3. + /// /// public async Task CreateFullTextIndexAsync(string fieldPath, string? name = null) { @@ -2039,6 +2048,7 @@ internal void PersistIndexMetadata() metadata.Indexes.Add(idxMeta); } + CollectionFeatures.SyncIndexFeatures(metadata); _storage.SaveCollectionMetadata(metadata); } diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs index 43ae3f7..bc3a616 100644 --- a/src/BLite.Core/Indexing/CollectionIndexManager.cs +++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs @@ -63,7 +63,9 @@ public CollectionIndexManager(StorageEngine storage, IDocumentMapper map RefreshHasUniqueBTreeIndex(); - if (rootAllocated) + // Re-record the features the indexes require if a release that predates them rewrote the + // record and dropped them (a read-only open cannot, and the indexes still protect it). + if (rootAllocated || (!_storage.IsReadOnly && CollectionFeatures.SyncIndexFeatures(_metadata))) SaveMetadata(); } @@ -93,6 +95,7 @@ private void UpdateMetadata() KeyFormat = index.KeyFormat }); } + CollectionFeatures.SyncIndexFeatures(_metadata); } /// diff --git a/src/BLite.Core/Storage/CollectionFeatures.cs b/src/BLite.Core/Storage/CollectionFeatures.cs index be5ee42..9d63d93 100644 --- a/src/BLite.Core/Storage/CollectionFeatures.cs +++ b/src/BLite.Core/Storage/CollectionFeatures.cs @@ -45,13 +45,26 @@ internal sealed record RequiredFeature(string Id, FeatureRequirement Requirement /// /// Only versions that contain this check can refuse; releases before it ignore the record and may /// also drop it when they rewrite the collection's metadata. An index of a type newer than any this -/// version knows is refused the same way, so a new index type stays protected even after such a -/// release dropped the record. +/// version knows is refused the same way, which keeps a new index type protected after such a +/// release dropped the record, but only while that release kept the index entry: a 5.2.x dynamic +/// collection rewrites the index list from the indexes it loaded, so its index changes drop both. +/// New per-index data must therefore go only in trailing sections of the record, never inline in +/// the index list (as the vector dimensions and metric are): a version that misreads the index list +/// cannot load the record at all, and so never gets to refuse it. /// internal static class CollectionFeatures { - // Features this version implements. Empty until a feature that needs protecting ships. - private static readonly HashSet s_known = new(StringComparer.Ordinal); + /// + /// Full-text indexes (#70), recorded while the collection has at least one. Needed to open: a + /// version without them reads a full-text index as a B-Tree index, so its query planner would + /// answer equality, range and IN lookups on the property from the postings, and its writes + /// would leave keys and stale postings behind. + /// + internal static readonly RequiredFeature FullTextIndexes = + new("fulltext-index", FeatureRequirement.Open, "full-text indexes (BLite 5.3 or later)"); + + // Features this version implements. + private static readonly HashSet s_known = new(StringComparer.Ordinal) { FullTextIndexes.Id }; // Highest IndexType this version knows; larger values come from newer versions. 0 is accepted: // older releases could persist an index whose type was never set. @@ -68,6 +81,24 @@ internal static class CollectionFeatures /// Whether this version implements the feature with the given id. internal static bool IsKnown(string id) => s_known.Contains(id); + /// + /// Records the features the indexes in require and removes those + /// none of them requires any more (today: while there is a + /// full-text index). Call after changing the index list, before saving the record. Other + /// features, including ones this version does not know, are left as they are. + /// + /// Whether changed. + internal static bool SyncIndexFeatures(CollectionMetadata metadata) + { + bool needed = metadata.Indexes.Any(i => i.Type == IndexType.FullText); + bool recorded = metadata.RequiredFeatures.Any(f => f.Id == FullTextIndexes.Id); + if (needed == recorded) return false; + + if (needed) metadata.RequiredFeatures.Add(FullTextIndexes); + else metadata.RequiredFeatures.RemoveAll(f => f.Id == FullTextIndexes.Id); + return true; + } + /// /// Whether this version may open the collection: every feature it requires is known, except /// features when the database is open read-only. @@ -110,11 +141,20 @@ internal static void EnsureCanWrite(CollectionMetadata metadata) } private static List Unsupported(CollectionMetadata metadata, bool readOnly) + => Unsupported(metadata, readOnly, s_known, s_maxIndexType); + + /// + /// The features that keep a version knowing only features and index + /// types up to from opening (or, with + /// false, modifying) the collection. Takes the version's knowledge as parameters so tests can + /// ask what an older version would refuse. + /// + internal static List Unsupported(CollectionMetadata metadata, bool readOnly, ICollection known, byte maxIndexType) { var result = new List(); foreach (var feature in metadata.RequiredFeatures) { - if (IsKnown(feature.Id)) continue; + if (known.Contains(feature.Id)) continue; if (readOnly && feature.Requirement == FeatureRequirement.Write) continue; result.Add(feature); } @@ -124,7 +164,7 @@ private static List Unsupported(CollectionMetadata metadata, bo foreach (var index in metadata.Indexes) { var type = (byte)index.Type; - if (type > s_maxIndexType) + if (type > maxIndexType) result.Add(new RequiredFeature($"index-type-{type}", FeatureRequirement.Open, $"index '{index.Name}' of type {type}, which this version does not know")); } diff --git a/src/BLite.Core/Storage/StorageEngine.Collections.cs b/src/BLite.Core/Storage/StorageEngine.Collections.cs index c1c6c10..8f9b83a 100644 --- a/src/BLite.Core/Storage/StorageEngine.Collections.cs +++ b/src/BLite.Core/Storage/StorageEngine.Collections.cs @@ -179,6 +179,9 @@ public sealed partial class StorageEngine /// Deserialises one collection record after its name. The only parser of this record: every /// section after the index list is optional and read only while bytes remain, so records written /// by older releases load with defaults for the sections they lack. + /// New per-index data goes only in a new trailing section, never inline in the index list: an + /// older version would misread the list and fail to load the record, so it could not refuse + /// the collection (see ). /// private static CollectionMetadata ReadCollectionMetadata(string name, BinaryReader reader) { diff --git a/tests/BLite.Tests/FullTextSearchTests.cs b/tests/BLite.Tests/FullTextSearchTests.cs index 1e24813..1b92c5d 100644 --- a/tests/BLite.Tests/FullTextSearchTests.cs +++ b/tests/BLite.Tests/FullTextSearchTests.cs @@ -851,4 +851,148 @@ public async Task Dynamic_StalePostings_NeverSurfaceTheDocumentAtTheirLocation() var top1 = await col.FullTextSearchAsync("idx_fts_text", "more", 1).ToListAsync(); Assert.Equal("other", Assert.Single(top1).Document.TryGetString("title", out var o) ? o : null); } + + // ── Required feature ──────────────────────────────────────────────────── + + private static bool RequiresFullText(StorageEngine storage, string collection) + => storage.GetCollectionMetadata(collection)!.RequiredFeatures.Contains(CollectionFeatures.FullTextIndexes); + + [Fact] + public async Task RequiredFeature_IsRecordedForModelDeclaredIndexes_AndClearedWithTheLastOne() + { + using (var db = new FullTextDbContext(_dbPath)) + { + await SeedAsync(db); + Assert.True(RequiresFullText(db.Storage, "articles")); + } + + using (var db = new FullTextNoIndexDbContext(_dbPath)) // the indexes stay, so does the feature + { + Assert.True(RequiresFullText(db.Storage, "articles")); + Assert.True(await db.Articles.DropIndexAsync("idx_fts_Title")); + Assert.True(RequiresFullText(db.Storage, "articles")); // idx_fts_Body is left + Assert.True(await db.Articles.DropIndexAsync("idx_fts_Body")); + Assert.False(RequiresFullText(db.Storage, "articles")); + } + + using (var db = new FullTextNoIndexDbContext(_dbPath)) + Assert.False(RequiresFullText(db.Storage, "articles")); + } + + [Fact] + public async Task RequiredFeature_IsRecordedByCreateFullTextIndexAsync_AndSurvivesReopen() + { + using (var db = new FullTextNoIndexDbContext(_dbPath)) + { + await db.Articles.InsertAsync(A("fox", "The quick brown fox")); + Assert.False(RequiresFullText(db.Storage, "articles")); + await db.Articles.CreateFullTextIndexAsync(a => a.Body); + Assert.True(RequiresFullText(db.Storage, "articles")); + } + + using (var db = new FullTextNoIndexDbContext(_dbPath)) + { + Assert.True(RequiresFullText(db.Storage, "articles")); + Assert.True(await db.Articles.DropIndexAsync("idx_fts_Body")); + Assert.False(RequiresFullText(db.Storage, "articles")); + } + } + + [Fact] + public async Task Dynamic_RequiredFeature_IsRecordedOnCreate_SurvivesReopen_AndIsClearedWithTheLastIndex() + { + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + await col.InsertAsync(col.CreateDocument(["title", "text"], b => b.AddString("title", "a").AddString("text", "plain words"))); + await col.CreateIndexAsync("title"); + Assert.False(RequiresFullText(engine.Storage, "notes")); + await col.CreateFullTextIndexAsync("text"); + await col.CreateFullTextIndexAsync("title", "idx_fts_title"); + Assert.True(RequiresFullText(engine.Storage, "notes")); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("notes"); + Assert.True(RequiresFullText(engine.Storage, "notes")); + Assert.True(col.DropIndex("idx_fts_text")); + Assert.True(RequiresFullText(engine.Storage, "notes")); + Assert.True(col.DropIndex("idx_fts_title")); + Assert.False(RequiresFullText(engine.Storage, "notes")); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + engine.GetOrCreateCollection("notes"); + Assert.False(RequiresFullText(engine.Storage, "notes")); + } + } + + /// + /// A release that predates the feature rewrites the record without it (5.2.x drops the trailing + /// section); the next read-write open records it again, a read-only open leaves the file alone. + /// + [Fact] + public async Task RequiredFeature_DroppedByAnOlderRelease_IsRecordedAgainOnTheNextReadWriteOpen() + { + static void DropFeature(string path, string collection) + { + using var storage = new StorageEngine(path, PageFileConfig.Default); + var meta = storage.GetCollectionMetadata(collection)!; + Assert.Equal(1, meta.RequiredFeatures.RemoveAll(f => f.Id == CollectionFeatures.FullTextIndexes.Id)); + storage.SaveCollectionMetadata(meta); + } + + using (var db = new FullTextDbContext(_dbPath)) + await SeedAsync(db); + await SeedNotesAsync(_dbPath); + DropFeature(_dbPath, "articles"); + DropFeature(_dbPath, "notes"); + + var readOnly = PageFileConfig.Default with { Access = System.IO.MemoryMappedFiles.MemoryMappedFileAccess.Read, AllowMultiProcessAccess = true }; + using (var engine = new BLiteEngine(_dbPath, readOnly)) + { + engine.GetOrCreateCollection("notes"); + Assert.False(RequiresFullText(engine.Storage, "notes")); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + engine.GetOrCreateCollection("notes"); + Assert.True(RequiresFullText(engine.Storage, "notes")); + } + // Last: the typed context drops "notes", which it does not map. + using (var db = new FullTextNoIndexDbContext(_dbPath)) + Assert.True(RequiresFullText(db.Storage, "articles")); + } + + /// + /// What a version without full-text indexes makes of the record: it refuses to open the + /// collection, read-only included, from the recorded feature alone (a version that knew type 6 + /// but not the feature) and from the index type alone (one whose metadata lost the feature). + /// + [Fact] + public async Task RequiredFeature_IsNeededToOpen_SoAVersionWithoutFullTextRefusesEvenReadOnly() + { + using (var db = new FullTextDbContext(_dbPath)) + await SeedAsync(db); + await SeedNotesAsync(_dbPath); + + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + foreach (var name in new[] { "articles", "notes" }) + { + var meta = storage.GetCollectionMetadata(name)!; + Assert.Equal(FeatureRequirement.Open, Assert.Single(meta.RequiredFeatures, f => f.Id == "fulltext-index").Requirement); + Assert.True(CollectionFeatures.CanOpen(meta, readOnly: false)); // this version knows it + + var featureOnly = CollectionFeatures.Unsupported(meta, readOnly: true, new HashSet(), (byte)IndexType.FullText); + Assert.Equal(CollectionFeatures.FullTextIndexes, Assert.Single(featureOnly)); + + meta.RequiredFeatures.Clear(); + var typeOnly = CollectionFeatures.Unsupported(meta, readOnly: true, new HashSet(), (byte)IndexType.Spatial); + Assert.NotEmpty(typeOnly); + Assert.All(typeOnly, f => Assert.Equal(FeatureRequirement.Open, f.Requirement)); + } + } } From 8d33f58e3fdb28b12166868eab49b4c593fe94d8 Mon Sep 17 00:00:00 2001 From: MrDevRobot Date: Thu, 15 Oct 2026 16:36:24 +0200 Subject: [PATCH 15/15] docs(storage): drop the format-rule note duplicated by #219's remarks Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Storage/StorageEngine.Collections.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/BLite.Core/Storage/StorageEngine.Collections.cs b/src/BLite.Core/Storage/StorageEngine.Collections.cs index 185b2be..26c21d0 100644 --- a/src/BLite.Core/Storage/StorageEngine.Collections.cs +++ b/src/BLite.Core/Storage/StorageEngine.Collections.cs @@ -207,9 +207,6 @@ public sealed partial class StorageEngine /// Deserialises one collection record after its name. The only parser of this record: every /// section after the index list is optional and read only while bytes remain, so records written /// by older releases load with defaults for the sections they lack. - /// New per-index data goes only in a new trailing section, never inline in the index list: an - /// older version would misread the list and fail to load the record, so it could not refuse - /// the collection (see ). /// /// /// Format rule: new data goes only in new trailing sections, after the last one read here; new