diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs
index 2ab1a32..edab085 100644
--- a/src/BLite.Core/Collections/DocumentCollection.cs
+++ b/src/BLite.Core/Collections/DocumentCollection.cs
@@ -721,7 +721,7 @@ public async Task VacuumAsync(VacuumOptions? options = null, CancellationToken c
{
foreach (var index in _indexManager.GetAllIndexes())
{
- await RebuildIndexFromScratchAsync(index.Name, ct).ConfigureAwait(false);
+ await RebuildIndexFromScratchAsync(index, ct).ConfigureAwait(false);
}
}
@@ -867,10 +867,12 @@ internal DocumentCollection(StorageEngine storage, ITransactionHolder transactio
///
/// Decimal properties used to be indexed as (double)value. An index whose persisted
- /// predates the exact decimal encoding is replaced by an
- /// empty tree, repopulated from the documents and then marked with the current format, the first
- /// time the collection is opened. The format is recorded only after the rebuild committed, so a
- /// crash in between just repeats the migration on the next open. No-op for every other index.
+ /// predates the exact decimal encoding is rebuilt from the
+ /// documents into a copy that queries don't see, the first time the collection is opened. Only
+ /// after the copy committed is it swapped in, with the current format, in one metadata save, and
+ /// the old tree freed. If the rebuild fails or is interrupted, the copy's pages are freed and the
+ /// old index stays registered in its old format, so the next open repeats the migration.
+ /// No-op for every other index.
///
private void MigrateLegacyDecimalIndexes()
{
@@ -881,12 +883,13 @@ private void MigrateLegacyDecimalIndexes()
var name = index.Name;
try
{
- var fresh = _indexManager.RecreateIndex(name);
+ var fresh = _indexManager.CreateUnregistered(index.Definition);
+ fresh.KeyFormat = IndexMetadata.CurrentKeyFormat;
// Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait.
// A document that cannot be loaded fails the migration rather than silently leaving
// the rebuilt index incomplete.
- Task.Run(() => RebuildIndexAsync(fresh, skipUnreadableDocuments: false)).GetAwaiter().GetResult();
- _indexManager.SetKeyFormat(name, IndexMetadata.CurrentKeyFormat);
+ Task.Run(() => FillUnregisteredIndexAsync(fresh, CancellationToken.None)).GetAwaiter().GetResult();
+ PublishFilledIndex(fresh, copy => _indexManager.ReplaceIndex(name, copy));
}
catch (Exception ex)
{
@@ -1072,28 +1075,29 @@ public IEnumerable GetIndexes()
[RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
internal void ApplyIndexBuilder(Metadata.IndexBuilder builder)
{
- // Use the IndexManager directly to ensure the index exists
- // We need to convert the LambdaExpression to a typed expression if possible,
- // or add an untyped CreateIndexAsync to IndexManager.
+ var name = CollectionIndexManager.ResolveIndexName(builder.KeySelector, builder.Name);
+ if (_indexManager.GetIndex(name) != null)
+ return;
- // For now, let's use a dynamic approach or cast if we know it's Func
- if (builder.Type == IndexType.Vector)
- {
- _indexManager.CreateVectorIndexUntyped(builder.KeySelector, builder.Dimensions, builder.Metric, builder.Name);
- }
- else if (builder.Type == IndexType.Spatial)
- {
- _indexManager.CreateSpatialIndexUntyped(builder.KeySelector, builder.Name);
- }
- else if (builder.KeySelector is System.Linq.Expressions.Expression> selector)
+ // The collection may already hold documents, e.g. when a new app version adds the index.
+ // Fill the index before registering it, so neither a crash nor a document that cannot be
+ // indexed leaves an empty index persisted that later opens would trust.
+ var definition = CollectionIndexManager.BuildDefinition(
+ builder.KeySelector, name, builder.IsUnique, builder.Type, builder.Dimensions, builder.Metric);
+ var index = _indexManager.CreateUnregistered(definition);
+ try
{
- _indexManager.EnsureIndex(selector, builder.Name, builder.IsUnique);
+ // Run on the thread pool so a captured SynchronizationContext cannot deadlock the wait.
+ Task.Run(() => FillUnregisteredIndexAsync(index, CancellationToken.None)).GetAwaiter().GetResult();
}
- else
+ catch (Exception ex)
{
- // Try to rebuild the expression or use untyped version
- _indexManager.EnsureIndexUntyped(builder.KeySelector, builder.Name, builder.IsUnique);
+ throw new InvalidOperationException(
+ $"Cannot build index '{name}' of collection '{_collectionName}' from the documents already in it: {ex.Message.TrimEnd('.')}. " +
+ "Nothing was saved, but every open fails the same way until those documents can be indexed or the index " +
+ "is removed from the model (OnModelCreating).", ex);
}
+ PublishFilledIndex(index, _indexManager.Register);
}
///
@@ -1597,55 +1601,75 @@ public async IAsyncEnumerable QueryIndexAsync(
}
///
- /// Populates an empty copy of index from the documents and swaps it in
+ /// Populates an empty copy of from the documents and swaps it in
/// only after the rebuild committed; the old tree is then freed. If any document cannot be
/// loaded or indexed, the rebuild fails and the existing index stays in place untouched,
/// instead of being replaced by an incomplete one.
///
- private async Task RebuildIndexFromScratchAsync(string name, CancellationToken ct)
+ private async Task RebuildIndexFromScratchAsync(CollectionSecondaryIndex current, CancellationToken ct)
+ {
+ var fresh = _indexManager.CreateUnregistered(current.Definition);
+ await FillUnregisteredIndexAsync(fresh, ct).ConfigureAwait(false);
+ PublishFilledIndex(fresh, copy => _indexManager.ReplaceIndex(current.Name, copy));
+ }
+
+ ///
+ /// Fills an index that is not registered yet (see )
+ /// from every document, in its own transaction, and commits it. If a document cannot be loaded
+ /// or indexed, or the commit fails, every page of the index is freed, including the pages its
+ /// splits added, the index is disposed and the exception propagates.
+ ///
+ private async Task FillUnregisteredIndexAsync(CollectionSecondaryIndex index, CancellationToken ct)
{
- var fresh = _indexManager.CreateUnregisteredCopy(name);
var transaction = _storage.BeginTransaction(IsolationLevel.ReadCommitted);
try
{
- await FillIndexAsync(fresh, transaction, skipUnreadableDocuments: false, ct).ConfigureAwait(false);
+ await FillIndexAsync(index, transaction, skipUnreadableDocuments: false, ct).ConfigureAwait(false);
await transaction.CommitAsync(ct);
}
catch
{
- // Collect the copy's pages through the transaction before rolling it back: pages added
+ // Collect the index's pages through the transaction before rolling it back: pages added
// by splits are allocated outside the transaction but linked only from nodes written
// in it, so after the rollback they could not be found any more.
IReadOnlyList pages = Array.Empty();
- try { pages = fresh.CollectAllPages(transaction.TransactionId); }
+ try { pages = index.CollectAllPages(transaction.TransactionId); }
catch { /* best-effort: never mask the original failure */ }
try { await transaction.RollbackAsync(); }
catch { /* best-effort: never mask the original failure */ }
- FreeUnregisteredCopy(fresh, pages);
+ FreeUnregisteredIndex(index, pages);
throw;
}
+ }
+ ///
+ /// Hands an index filled by to
+ /// ( or ).
+ /// If that fails and the index did not get registered, nothing refers to it any more, so its
+ /// pages are freed and it is disposed.
+ ///
+ private void PublishFilledIndex(CollectionSecondaryIndex index, Action> register)
+ {
try
{
- _indexManager.ReplaceIndex(name, fresh);
+ register(index);
}
catch
{
- // The copy is committed but, unless the swap itself went through, nothing refers to it.
- if (!ReferenceEquals(_indexManager.GetIndex(name), fresh))
+ if (!ReferenceEquals(_indexManager.GetIndex(index.Name), index))
{
IReadOnlyList pages = Array.Empty();
- try { pages = fresh.CollectAllPages(0); }
+ try { pages = index.CollectAllPages(0); }
catch { /* best-effort: never mask the original failure */ }
- FreeUnregisteredCopy(fresh, pages);
+ FreeUnregisteredIndex(index, pages);
}
throw;
}
}
- private void FreeUnregisteredCopy(CollectionSecondaryIndex copy, IReadOnlyList pages)
+ private void FreeUnregisteredIndex(CollectionSecondaryIndex index, IReadOnlyList pages)
{
try
{
@@ -1653,7 +1677,7 @@ private void FreeUnregisteredCopy(CollectionSecondaryIndex copy, IReadOn
_storage.FreePage(pageId);
}
catch { /* best-effort: never mask the original failure */ }
- copy.Dispose();
+ index.Dispose();
}
private async Task RebuildIndexAsync(CollectionSecondaryIndex index, CancellationToken ct = default, bool skipUnreadableDocuments = true)
diff --git a/src/BLite.Core/Indexing/CollectionIndexManager.cs b/src/BLite.Core/Indexing/CollectionIndexManager.cs
index 7dddfb0..8ea9aa0 100644
--- a/src/BLite.Core/Indexing/CollectionIndexManager.cs
+++ b/src/BLite.Core/Indexing/CollectionIndexManager.cs
@@ -128,6 +128,66 @@ public CollectionSecondaryIndex CreateIndex(CollectionIndexDefinition
}
}
+ /// The name an index on gets when none is given.
+ internal static string ResolveIndexName(LambdaExpression keySelector, string? name)
+ => name ?? GenerateIndexName(ExpressionAnalyzer.ExtractPropertyPaths(keySelector));
+
+ ///
+ /// Builds an index definition from an untyped key selector, as declared in the model.
+ ///
+ [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
+ internal static CollectionIndexDefinition BuildDefinition(
+ LambdaExpression keySelector,
+ string? name,
+ bool unique,
+ IndexType type,
+ int dimensions,
+ VectorMetric metric)
+ {
+ var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
+ var body = keySelector.Body.Type == typeof(object)
+ ? keySelector.Body
+ : Expression.Convert(keySelector.Body, typeof(object));
+ var lambda = Expression.Lambda>(body, keySelector.Parameters);
+
+ return new CollectionIndexDefinition(
+ name ?? GenerateIndexName(propertyPaths), propertyPaths, lambda, unique, type, false, dimensions, metric);
+ }
+
+ ///
+ /// Creates an index for without registering or persisting it,
+ /// so it can be filled before queries or the collection metadata see it. The index owns its
+ /// root page: hand it to or , or release its
+ /// pages.
+ ///
+ internal CollectionSecondaryIndex CreateUnregistered(CollectionIndexDefinition definition)
+ {
+ lock (_lock)
+ {
+ if (_disposed)
+ throw new ObjectDisposedException(nameof(CollectionIndexManager));
+
+ return new CollectionSecondaryIndex(definition, _storage, _mapper);
+ }
+ }
+
+ ///
+ /// Registers an index built by and persists it. If the
+ /// metadata cannot be saved, the index is not registered and remains the caller's to release.
+ ///
+ internal void Register(CollectionSecondaryIndex index)
+ {
+ lock (_lock)
+ {
+ if (_disposed)
+ throw new ObjectDisposedException(nameof(CollectionIndexManager));
+ if (_indexes.ContainsKey(index.Name))
+ throw new InvalidOperationException($"Index '{index.Name}' already exists");
+
+ PublishLocked(index.Name, index, previous: null);
+ }
+ }
+
// ... methods ...
///
@@ -206,24 +266,7 @@ public CollectionSecondaryIndex EnsureIndex(
}
}
- [RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
- internal CollectionSecondaryIndex EnsureIndexUntyped(
- LambdaExpression keySelector,
- string? name = null,
- bool unique = false)
- {
- // Convert LambdaExpression to Expression> properly by sharing parameters
- var body = keySelector.Body;
- if (body.Type != typeof(object))
- {
- body = Expression.Convert(body, typeof(object));
- }
-
- var lambda = Expression.Lambda>(body, keySelector.Parameters);
-
- return EnsureIndex(lambda, name, unique);
- }
-
+ [Obsolete("Use model HasVectorIndex/HasSpatialIndex or CreateIndex(definition); will be removed in 6.0")]
[RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
public CollectionSecondaryIndex CreateVectorIndexUntyped(
LambdaExpression keySelector,
@@ -252,6 +295,7 @@ public CollectionSecondaryIndex CreateVectorIndexUntyped(
}
}
+ [Obsolete("Use model HasVectorIndex/HasSpatialIndex or CreateIndex(definition); will be removed in 6.0")]
[RequiresDynamicCode("Index creation compiles key selector expressions using Expression.Compile() which requires dynamic code generation.")]
public CollectionSecondaryIndex CreateSpatialIndexUntyped(
LambdaExpression keySelector,
@@ -307,45 +351,6 @@ public bool DropIndex(string name)
}
}
- ///
- /// Replaces an index with an empty one of the same definition, freeing the old tree's pages.
- /// The new index keeps the old : the
- /// caller repopulates it and then records the new format with , so an
- /// interrupted rebuild is simply redone on the next open.
- ///
- internal CollectionSecondaryIndex RecreateIndex(string name)
- {
- lock (_lock)
- {
- if (!_indexes.TryGetValue(name, out var old))
- throw new InvalidOperationException($"Index '{name}' does not exist");
-
- var fresh = CreateUnregisteredCopy(name);
- fresh.KeyFormat = old.KeyFormat;
- ReplaceIndex(name, fresh);
- return fresh;
- }
- }
-
- ///
- /// Creates an empty index with the same definition as without
- /// registering it, so queries keep using the existing index while the copy is populated.
- /// The copy owns its root page: hand it to or release it with
- /// .
- ///
- internal CollectionSecondaryIndex CreateUnregisteredCopy(string name)
- {
- lock (_lock)
- {
- if (_disposed)
- throw new ObjectDisposedException(nameof(CollectionIndexManager));
- if (!_indexes.TryGetValue(name, out var old))
- throw new InvalidOperationException($"Index '{name}' does not exist");
-
- return new CollectionSecondaryIndex(old.Definition, _storage, _mapper);
- }
- }
-
///
/// Registers under , persists its root page
/// and frees the pages of the index it replaces. If the metadata cannot be saved, the old
@@ -360,29 +365,43 @@ internal void ReplaceIndex(string name, CollectionSecondaryIndex fresh)
if (!_indexes.TryGetValue(name, out var old))
throw new InvalidOperationException($"Index '{name}' does not exist");
- _indexes[name] = fresh;
- _cachedIndexInfo = null; // invalidate cache
- RefreshHasUniqueBTreeIndex();
-
// Persist the new root before freeing the old tree: a crash in between leaks the old
// pages, whereas the reverse order would let the next open free them a second time.
- try
- {
- SaveMetadata();
- }
- catch
- {
- // Keep the old index registered, matching the metadata on disk; the caller still
- // owns the copy.
- _indexes[name] = old;
- UpdateMetadata();
- throw;
- }
+ PublishLocked(name, fresh, old);
old.FreeAllPages(_storage);
old.Dispose();
}
}
+ ///
+ /// Registers under in place of
+ /// (null: none) and saves the metadata. If the save fails, the
+ /// previous registration is restored, so memory keeps matching the metadata on disk, and the
+ /// exception propagates. Must be called under .
+ ///
+ private void PublishLocked(string name, CollectionSecondaryIndex index, CollectionSecondaryIndex? previous)
+ {
+ _indexes[name] = index;
+ _cachedIndexInfo = null; // invalidate cache
+ RefreshHasUniqueBTreeIndex();
+
+ try
+ {
+ SaveMetadata();
+ }
+ catch
+ {
+ if (previous != null)
+ _indexes[name] = previous;
+ else
+ _indexes.Remove(name);
+ _cachedIndexInfo = null;
+ RefreshHasUniqueBTreeIndex();
+ UpdateMetadata();
+ throw;
+ }
+ }
+
/// Records the key format an index's pages are written with and persists it.
internal void SetKeyFormat(string name, byte keyFormat)
{
diff --git a/tests/BLite.Shared/IndexEvolutionContexts.cs b/tests/BLite.Shared/IndexEvolutionContexts.cs
new file mode 100644
index 0000000..bbcb44a
--- /dev/null
+++ b/tests/BLite.Shared/IndexEvolutionContexts.cs
@@ -0,0 +1,58 @@
+using BLite.Bson;
+using BLite.Core;
+using BLite.Core.Collections;
+using BLite.Core.Indexing;
+using BLite.Core.Metadata;
+
+namespace BLite.Shared;
+
+// Contexts for adding model-declared indexes to collections that already hold documents.
+
+/// "App v1": the collections without any secondary index.
+public partial class IndexEvoV1DbContext : DocumentDbContext
+{
+ public DocumentCollection People { get; set; } = null!;
+ public DocumentCollection Vectors { get; set; } = null!;
+ public DocumentCollection Places { get; set; } = null!;
+
+ public IndexEvoV1DbContext(string path) : base(path) => InitializeCollections();
+
+ protected override void OnModelCreating(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity().ToCollection("evo_people");
+ modelBuilder.Entity().ToCollection("evo_vectors");
+ modelBuilder.Entity().ToCollection("evo_places");
+ }
+}
+
+/// "App v2": the same collections with a B-Tree, a vector and a spatial index.
+public partial class IndexEvoV2DbContext : DocumentDbContext
+{
+ public DocumentCollection People { get; set; } = null!;
+ public DocumentCollection Vectors { get; set; } = null!;
+ public DocumentCollection Places { get; set; } = null!;
+
+ public IndexEvoV2DbContext(string path) : base(path) => InitializeCollections();
+
+ protected override void OnModelCreating(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity().ToCollection("evo_people").HasIndex(p => p.Age);
+ modelBuilder.Entity().ToCollection("evo_vectors")
+ .HasVectorIndex(x => x.Embedding, dimensions: 3, metric: VectorMetric.L2, name: "idx_vector");
+ modelBuilder.Entity().ToCollection("evo_places").HasSpatialIndex(x => x.Location, name: "idx_spatial");
+ }
+}
+
+/// Declares a vector index whose dimension does not match the stored embeddings.
+public partial class IndexEvoBadVectorDbContext : DocumentDbContext
+{
+ public DocumentCollection Vectors { get; set; } = null!;
+
+ public IndexEvoBadVectorDbContext(string path) : base(path) => InitializeCollections();
+
+ protected override void OnModelCreating(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity().ToCollection("evo_vectors")
+ .HasVectorIndex(x => x.Embedding, dimensions: 4, metric: VectorMetric.L2, name: "idx_vector");
+ }
+}
diff --git a/tests/BLite.Tests/DecimalIndexTests.cs b/tests/BLite.Tests/DecimalIndexTests.cs
index d025193..fd69a20 100644
--- a/tests/BLite.Tests/DecimalIndexTests.cs
+++ b/tests/BLite.Tests/DecimalIndexTests.cs
@@ -1,5 +1,7 @@
+using System.Reflection;
using BLite.Bson;
using BLite.Core;
+using BLite.Core.Collections;
using BLite.Core.Indexing;
using BLite.Core.Query;
using BLite.Core.Query.Blql;
@@ -220,6 +222,71 @@ public async Task KeyFormat_RoundTripsThroughMetadata()
Assert.Equal(IndexMetadata.CurrentKeyFormat, priceIndex.KeyFormat);
}
+ ///
+ /// A migration that fails late, after the rebuilt tree has split, keeps the legacy index as it
+ /// was and frees every page of the partial rebuild: it is retried on each open without growing
+ /// the file, and succeeds once the document can be read again.
+ ///
+ [Fact]
+ public async Task LegacyIndexMigration_ThatFailsLate_KeepsTheOldIndex_AndDoesNotGrowTheFile()
+ {
+ const int count = 3000;
+ uint legacyRoot;
+ uint corruptedPage;
+ byte[] intact;
+ using (var db = new TestDbContext(_dbPath))
+ {
+ await db.Products.InsertBulkAsync(Enumerable.Range(1, count)
+ .Select(i => new Product { Id = i, Title = $"P{i}", Price = i + 0.01m }).ToList());
+ db.Products.IndexManager.SetKeyFormat(PriceIndex(db).Name, IndexMetadata.LegacyKeyFormat);
+ legacyRoot = PriceIndex(db).RootPageId;
+ await db.CheckpointAsync();
+
+ // Make the last document in primary-key order unreadable, so the rebuild on open fails
+ // only after thousands of inserts.
+ var primary = (BTreeIndex)typeof(DocumentCollection)
+ .GetField("_primaryIndex", BindingFlags.NonPublic | BindingFlags.Instance)!
+ .GetValue(db.Products)!;
+ var location = primary.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Backward).First().Location;
+ corruptedPage = location.PageId;
+ intact = new byte[db.Storage.PageSize];
+ db.Storage.ReadPage(corruptedPage, null, intact);
+ var corrupt = (byte[])intact.Clone();
+ var slot = SlotEntry.ReadFrom(corrupt.AsSpan(SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size));
+ corrupt.AsSpan(slot.Offset + 4, 12).Fill(0xFF); // invalid element types
+ db.Storage.WritePageImmediate(corruptedPage, corrupt);
+ }
+
+ var sizes = new List();
+ for (int open = 0; open < 4; open++)
+ {
+ var ex = Assert.Throws(() => new TestDbContext(_dbPath));
+ Assert.Contains("retried on the next open", ex.Message);
+ sizes.Add(new FileInfo(_dbPath).Length);
+ }
+ // The first failed open may extend the file; the following ones reuse the freed pages.
+ Assert.All(sizes.Skip(1), size => Assert.Equal(sizes[0], size));
+
+ using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default))
+ {
+ // The legacy index is still the registered one, so the next open retries the migration.
+ var stored = storage.GetCollectionMetadata("products_collection")!.Indexes
+ .Single(i => i.PropertyPaths.Single() == nameof(Product.Price));
+ Assert.Equal(IndexMetadata.LegacyKeyFormat, stored.KeyFormat);
+ Assert.Equal(legacyRoot, stored.RootPageId);
+ storage.WritePageImmediate(corruptedPage, intact);
+ }
+
+ using (var db = new TestDbContext(_dbPath))
+ {
+ var index = PriceIndex(db);
+ Assert.Equal(IndexMetadata.CurrentKeyFormat, index.KeyFormat);
+ Assert.Equal(count, AllKeys(index).Count);
+ Assert.Equal(count, (await db.Products.AsQueryable().Where(p => p.Price >= 0m).ToListAsync()).Count);
+ Assert.Equal(1500, Assert.Single(await db.Products.AsQueryable().Where(p => p.Price == 1500.01m).ToListAsync()).Id);
+ }
+ }
+
private static List AllKeys(CollectionSecondaryIndex index)
=> index.BTreeIndex!.Range(IndexKey.NullSentinelNext, IndexKey.MaxKey).Select(e => e.Key).ToList();
diff --git a/tests/BLite.Tests/IndexBackfillTests.cs b/tests/BLite.Tests/IndexBackfillTests.cs
new file mode 100644
index 0000000..7fec0a4
--- /dev/null
+++ b/tests/BLite.Tests/IndexBackfillTests.cs
@@ -0,0 +1,218 @@
+using System.Linq.Expressions;
+using System.Reflection;
+using BLite.Bson;
+using BLite.Core;
+using BLite.Core.Collections;
+using BLite.Core.Indexing;
+using BLite.Core.Metadata;
+using BLite.Core.Storage;
+using BLite.Shared;
+
+namespace BLite.Tests;
+
+///
+/// An index declared in the model of a collection that already holds documents must be filled
+/// from them when it is first created (#171).
+///
+public class IndexBackfillTests : IDisposable
+{
+ private readonly string _dbPath;
+
+ public IndexBackfillTests()
+ {
+ _dbPath = Path.Combine(Path.GetTempPath(), $"idx_backfill_{Guid.NewGuid():N}.db");
+ }
+
+ public void Dispose()
+ {
+ foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") })
+ try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ }
+ }
+
+ private async Task SeedAsync()
+ {
+ using var v1 = new IndexEvoV1DbContext(_dbPath);
+ for (int i = 0; i < 100; i++)
+ await v1.People.InsertAsync(new PersonV2 { Name = $"p{i}", Age = i % 10 });
+ await v1.Vectors.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] });
+ await v1.Vectors.InsertAsync(new VectorEntity { Title = "B", Embedding = [5f, 5f, 5f] });
+ await v1.Places.InsertAsync(new GeoEntity { Name = "Milan", Location = (45.46, 9.19) });
+ await v1.Places.InsertAsync(new GeoEntity { Name = "New York", Location = (40.71, -74.01) });
+ }
+
+ [Fact]
+ public async Task AddedBTreeIndex_CoversExistingDocuments()
+ {
+ await SeedAsync();
+
+ using var v2 = new IndexEvoV2DbContext(_dbPath);
+
+ Assert.Equal(10, v2.People.AsQueryable().Where(p => p.Age == 3).ToList().Count);
+ Assert.Equal(10, (await v2.People.QueryIndexAsync("idx_Age", 3, 3).ToListAsync()).Count);
+ }
+
+ [Fact]
+ public async Task AddedVectorIndex_CoversExistingDocuments()
+ {
+ await SeedAsync();
+
+ using var v2 = new IndexEvoV2DbContext(_dbPath);
+
+ var nearest = await v2.Vectors.VectorSearchAsync("idx_vector", [4f, 4f, 4f], 1).ToListAsync();
+ Assert.Equal("B", Assert.Single(nearest).Title);
+ }
+
+ [Fact]
+ public async Task AddedSpatialIndex_CoversExistingDocuments()
+ {
+ await SeedAsync();
+
+ using var v2 = new IndexEvoV2DbContext(_dbPath);
+
+ var inBox = await v2.Places.WithinAsync("idx_spatial", (44.0, 8.0), (46.0, 10.0)).ToListAsync();
+ Assert.Equal("Milan", Assert.Single(inBox).Name);
+ }
+
+ [Fact]
+ public async Task AddedIndex_IsFilledOnce_AndMaintainedAfterwards()
+ {
+ await SeedAsync();
+ using (var v2 = new IndexEvoV2DbContext(_dbPath))
+ await v2.People.InsertAsync(new PersonV2 { Name = "late", Age = 3 });
+
+ using var reopened = new IndexEvoV2DbContext(_dbPath);
+
+ Assert.Equal(11, (await reopened.People.QueryIndexAsync("idx_Age", 3, 3).ToListAsync()).Count);
+ }
+
+ [Fact]
+ public async Task IndexThatCannotBeFilled_FailsTheOpen_AndIsNotPersisted()
+ {
+ await SeedAsync();
+
+ // The stored embeddings have 3 dimensions; the declared index expects 4.
+ var ex = Assert.Throws(() => new IndexEvoBadVectorDbContext(_dbPath));
+ Assert.Contains("idx_vector", ex.Message);
+ Assert.Contains("removed from the model", ex.Message);
+ Assert.IsType(ex.InnerException); // the dimension mismatch
+ Assert.Contains($"{ex.InnerException!.Message.TrimEnd('.')}. Nothing was saved", ex.Message);
+
+ // Nothing was registered, so the data stays readable and the next open tries again.
+ using (var v1 = new IndexEvoV1DbContext(_dbPath))
+ {
+ Assert.DoesNotContain(v1.Vectors.GetIndexes(), i => i.Name == "idx_vector");
+ Assert.Equal(2, v1.Vectors.AsQueryable().ToList().Count);
+ }
+ Assert.Throws(() => new IndexEvoBadVectorDbContext(_dbPath));
+ }
+
+ ///
+ /// A fill that fails after the index has split must free every page it added, not only the
+ /// root (#171 review): the error repeats on every open until the data is fixed, so a leak would
+ /// grow the file each time.
+ ///
+ [Fact]
+ public async Task IndexThatFailsLate_FreesEveryPageItAdded_SoRepeatedOpensDoNotGrowTheFile()
+ {
+ using (var v1 = new IndexEvoV1DbContext(_dbPath))
+ {
+ await v1.Vectors.InsertBulkAsync(Enumerable.Range(0, 3000)
+ .Select(i => new VectorEntity { Title = $"v{i}", Embedding = [i, i % 7, i % 13] }).ToList());
+
+ // Give the last document in primary-key order a wrong-dimension embedding, so the fill
+ // fails only after thousands of inserts.
+ var primary = (BTreeIndex)typeof(DocumentCollection)
+ .GetField("_primaryIndex", BindingFlags.NonPublic | BindingFlags.Instance)!
+ .GetValue(v1.Vectors)!;
+ var lastId = primary.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Backward).First().Key.As();
+ var last = (await v1.Vectors.FindByIdAsync(lastId))!;
+ last.Embedding = [1f, 2f];
+ await v1.Vectors.UpdateAsync(last);
+ }
+
+ var sizes = new List();
+ for (int open = 0; open < 4; open++)
+ {
+ var ex = Assert.Throws(() => new IndexEvoV2DbContext(_dbPath));
+ Assert.Contains("idx_vector", ex.Message);
+ sizes.Add(new FileInfo(_dbPath).Length);
+ }
+
+ // The first failed open may extend the file; the following ones reuse the freed pages.
+ Assert.All(sizes.Skip(1), size => Assert.Equal(sizes[0], size));
+ using var reopened = new IndexEvoV1DbContext(_dbPath);
+ Assert.DoesNotContain(reopened.Vectors.GetIndexes(), i => i.Name == "idx_vector");
+ }
+
+ ///
+ /// If the filled index cannot be persisted, it is not left registered in memory either, and
+ /// its pages are freed; the next open builds it again.
+ ///
+ [Fact]
+ public async Task AddedIndex_WhoseMetadataCannotBeSaved_IsNotRegistered_AndItsPagesAreFreed()
+ {
+ using (var v1 = new IndexEvoV1DbContext(_dbPath))
+ {
+ await v1.People.InsertBulkAsync(Enumerable.Range(0, 3000)
+ .Select(i => new PersonV2 { Name = $"p{i}", Age = i % 90 }).ToList());
+ await v1.CheckpointAsync();
+
+ // Hold the metadata lock so saving the new index times out.
+ var metadataLock = (SemaphoreSlim)typeof(StorageEngine)
+ .GetField("_metadataLock", BindingFlags.NonPublic | BindingFlags.Instance)!
+ .GetValue(v1.Storage)!;
+ Expression> age = p => p.Age;
+ metadataLock.Wait();
+ try
+ {
+ Assert.Throws(() => v1.People.ApplyIndexBuilder(new IndexBuilder(age, "idx_Age", unique: false)));
+ }
+ finally
+ {
+ metadataLock.Release();
+ }
+
+ Assert.Null(v1.People.IndexManager.GetIndex("idx_Age"));
+ Assert.DoesNotContain(v1.People.GetIndexes(), i => i.Name == "idx_Age");
+ Assert.DoesNotContain(v1.Storage.GetCollectionMetadata("evo_people")!.Indexes, i => i.Name == "idx_Age");
+ await AssertIndexPagesWereFreedAsync(v1);
+ }
+
+ using var v2 = new IndexEvoV2DbContext(_dbPath);
+ Assert.Equal(34, (await v2.People.QueryIndexAsync("idx_Age", 3, 3).ToListAsync()).Count);
+ }
+
+ ///
+ /// A unique index added by the model is enforced in the session that creates it, not only after
+ /// the next open.
+ ///
+ [Fact]
+ public async Task AddedUniqueIndex_IsEnforcedRightAway()
+ {
+ await SeedAsync();
+ using var v1 = new IndexEvoV1DbContext(_dbPath);
+ Expression> name = p => p.Name;
+
+ v1.People.ApplyIndexBuilder(new IndexBuilder(name, "idx_Name", unique: true));
+
+ await Assert.ThrowsAnyAsync(async () => await v1.People.InsertAsync(new PersonV2 { Name = "p99", Age = 1 }));
+ await v1.People.InsertAsync(new PersonV2 { Name = "new", Age = 1 });
+ Assert.Single(v1.People.AsQueryable().ToList(), p => p.Name == "p99");
+ }
+
+ ///
+ /// A 3000-entry B-Tree spans dozens of pages. Each must be back on the free list (handed out
+ /// next, below the high-water mark) or queued for the next WAL truncation.
+ ///
+ private static async Task AssertIndexPagesWereFreedAsync(IndexEvoV1DbContext db)
+ {
+ var storage = db.Storage;
+ await db.CheckpointAsync();
+ var flags = BindingFlags.NonPublic | BindingFlags.Instance;
+ int queued = ((HashSet)typeof(StorageEngine).GetField("_pendingFreePages", flags)!.GetValue(storage)!).Count;
+ var highWater = ((IPageStorage)typeof(StorageEngine).GetField("_pageFile", flags)!.GetValue(storage)!).NextPageId;
+ int reused = 0;
+ while (storage.AllocatePage() < highWater) reused++;
+ Assert.True(reused + queued > 20, $"only {reused} pages were returned to the free list and {queued} queued");
+ }
+}