Skip to content

fix(index): update and delete spatial and vector index entries (#164) #172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrdevrobot merged 4 commits into from
Oct 13, 2026
Merged

fix(index): update and delete spatial and vector index entries (#164) #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
df76234
fix(index): update and delete spatial and vector index entries (#164)
CaffeinatedCoder Oct 12, 2026
c8251e1
Merge remote-tracking branch 'upstream/main' into fix/164-spatial-vec…
CaffeinatedCoder Oct 12, 2026
623014d
fix(index): address review of spatial/vector index maintenance (#164)
CaffeinatedCoder Oct 12, 2026
58d9c71
fix(index): address follow-ups on spatial/vector index maintenance (#…
CaffeinatedCoder Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions src/BLite.Core/Collections/DocumentCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2809,8 +2809,9 @@ private async Task<bool> UpdateAtLocationCore(TId id, T entity, byte[] docData,
}
else
{
// Delete old + insert new
await DeleteCore(id, transaction, notifyCdc: false);
// Delete old + insert new. Secondary indexes are moved below, so an entry whose
// key did not change is re-pointed rather than deleted and inserted again.
await DeleteCore(id, transaction, notifyCdc: false, deleteFromIndexes: false);

DocumentLocation newLocation;
if (bytesWritten + SlotEntry.Size <= _maxDocumentSizeForSinglePage)
Expand All @@ -2827,9 +2828,7 @@ private async Task<bool> UpdateAtLocationCore(TId id, T entity, byte[] docData,
}

_primaryIndex.Insert(key, newLocation, transaction.TransactionId);
// DeleteCore already removed all secondary index entries for this document.
// We only need to insert the new entries at the new physical location.
_indexManager.InsertIntoAll(entity, newLocation, transaction);
_indexManager.UpdateInAll(oldEntity, entity, oldLocation, newLocation, transaction);

// Notify CDC
await NotifyCdc(OperationType.Update, id, transaction, docData);
Expand Down Expand Up @@ -3089,17 +3088,20 @@ private async Task<int> DeleteBulkInternal(IEnumerable<TId> ids, ITransaction tr
return deleteCount;
}

private async Task<bool> DeleteCore(TId id, ITransaction transaction, bool notifyCdc = true)
private async Task<bool> DeleteCore(TId id, ITransaction transaction, bool notifyCdc = true, bool deleteFromIndexes = true)
{
var key = _mapper.ToIndexKey(id);
if (!_primaryIndex.TryFind(key, out var location, transaction.TransactionId))
return false;

// Notify secondary indexes BEFORE deleting document from storage
var entity = await FindByLocation(location, transaction);
if (entity != null)
if (deleteFromIndexes)
{
_indexManager.DeleteFromAll(entity, location, transaction);
var entity = await FindByLocation(location, transaction);
if (entity != null)
{
_indexManager.DeleteFromAll(entity, location, transaction);
}
}

// Read page
Expand Down
40 changes: 30 additions & 10 deletions src/BLite.Core/DynamicCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1081,10 +1081,7 @@ private async Task UpdateAtLocationCore(BsonId id, IndexKey key, BsonDocument ne
_primaryIndex.Insert(key, newLocation, transaction.TransactionId);

foreach (var (_, idx) in _secondaryIndexes)
{
if (oldDoc != null) IndexDelete(idx, oldDoc, oldLocation, transaction);
IndexInsert(idx, newDocument, newLocation, transaction);
}
IndexUpdate(idx, oldDoc, newDocument, oldLocation, newLocation, transaction);

await NotifyCdcAsync(OperationType.Update, id, transaction, newDocument.RawData);
}
Expand Down Expand Up @@ -1308,10 +1305,7 @@ public async ValueTask<int> UpdateBulkAsync(IEnumerable<(BsonId Id, BsonDocument
_primaryIndex.Insert(key, newLocation, transaction.TransactionId);

foreach (var (_, idx) in _secondaryIndexes)
{
if (oldDoc != null) IndexDelete(idx, oldDoc, oldLocation, transaction);
IndexInsert(idx, newDoc, newLocation, transaction);
}
IndexUpdate(idx, oldDoc, newDoc, oldLocation, newLocation, transaction);

await NotifyCdcAsync(OperationType.Update, id, transaction, newDoc.RawData);
count++;
Expand Down Expand Up @@ -1884,11 +1878,37 @@ private void IndexDelete(DynamicSecondaryIndex idx, BsonDocument document, Docum
var key = BsonValueToIndexKey(val);
if (key.HasValue) idx.BTree!.Delete(key.Value, location, transaction.TransactionId);
break;
// Vector and Spatial indexes do not support individual entry deletion;
// their rebuild is handled at the collection level (compaction/reindex).
case DynamicIndexKind.Vector:
var floats = ExtractFloatVector(val);
if (floats != null) idx.Vector!.Delete(floats, location, transaction);
break;
case DynamicIndexKind.Spatial:
var coords = ExtractCoordinates(val);
if (coords.HasValue) idx.Spatial!.Delete(GeoBox.FromPoint(new GeoPoint(coords.Value.Lat, coords.Value.Lon)), location, transaction);
break;
}
}

/// <summary>
/// Moves a document's entry from <paramref name="oldDocument"/> at <paramref name="oldLocation"/>
/// to <paramref name="newDocument"/> at <paramref name="newLocation"/>. An unchanged vector keeps
/// its HNSW node, which is re-pointed instead of being replaced by a new one.
/// </summary>
private void IndexUpdate(DynamicSecondaryIndex idx, BsonDocument? oldDocument, BsonDocument newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction)
{
if (oldDocument != null && idx.Kind == DynamicIndexKind.Vector
&& TryGetNestedValue(oldDocument, idx.FieldPath, out var oldVal)
&& TryGetNestedValue(newDocument, idx.FieldPath, out var newVal)
&& ExtractFloatVector(oldVal) is { } oldVector
&& ExtractFloatVector(newVal) is { } newVector
&& oldVector.AsSpan().SequenceEqual(newVector)
&& idx.Vector!.Relocate(oldVector, oldLocation, newLocation, transaction))
return;

if (oldDocument != null) IndexDelete(idx, oldDocument, oldLocation, transaction);
IndexInsert(idx, newDocument, newLocation, transaction);
}

#endregion

#region Internal storage operations
Expand Down
199 changes: 141 additions & 58 deletions src/BLite.Core/Indexing/CollectionSecondaryIndex.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,52 +103,21 @@ public void Insert(T document, DocumentLocation location, ITransaction transacti
if (!_definition.TryGetKey(document, out var keyValue) || keyValue == null)
return; // Skip null keys or failed traversal

if (_vectorIndex != null)
{
// Vector Index Support
if (keyValue is float[] singleVector)
{
_vectorIndex.Insert(singleVector, location, transaction);
}
else if (keyValue is IEnumerable<float[]> vectors)
{
foreach (var v in vectors)
{
_vectorIndex.Insert(v, location, transaction);
}
}
}
else if (_spatialIndex != null)
{
// Geospatial Index Support
if (keyValue is ValueTuple<double, double> t)
{
_spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
}
}
else if (_btreeIndex != null)
{
// BTree Index logic
var userKey = ConvertToIndexKey(keyValue);
var documentId = _mapper.GetId(document);
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
_btreeIndex.Insert(compositeKey, location, transaction?.TransactionId);
}
InsertKey(keyValue, document, location, transaction);

System.Threading.Interlocked.Increment(ref _documentCount);
}

/// <summary>
/// Updates a document in this index (delete old, insert new).
/// This optimization applies only for in-place updates where the physical location
/// is unchanged and the old index entry remains valid. For non-in-place (relocating)
/// updates, callers must use <c>InsertInAll</c> directly after <c>DeleteFromAll</c>
/// has already been called (e.g., via <c>DeleteCore</c>).
/// Updates a document in this index, for both in-place and relocating updates.
/// If the key is unchanged and the document did not move, the existing entry is kept;
/// if only the location changed, the entry is re-pointed to <paramref name="newLocation"/>;
/// otherwise the old entry is removed and the new one inserted.
/// </summary>
/// <param name="oldDocument">Old version of document</param>
/// <param name="newDocument">New version of document</param>
/// <param name="oldLocation">Physical location of old document</param>
/// <param name="newLocation">Physical location of new document (must equal oldLocation for this method)</param>
/// <param name="newLocation">Physical location of new document (may differ from <paramref name="oldLocation"/>)</param>
/// <param name="transaction">Optional transaction</param>
public void Update(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction)
{
Expand All @@ -161,28 +130,37 @@ public void Update(T oldDocument, T newDocument, DocumentLocation oldLocation, D
_definition.TryGetKey(oldDocument, out var oldKey);
_definition.TryGetKey(newDocument, out var newKey);

// If keys are the same, no index update needed (optimization for in-place updates
// where the existing entry at the same location remains valid).
if (Equals(oldKey, newKey))
bool moved = oldLocation.PageId != newLocation.PageId || oldLocation.SlotIndex != newLocation.SlotIndex;

if (KeysEqual(oldKey, newKey))
{
// Same key at the same location: the existing entry stays valid.
if (!moved || oldKey == null)
return;

// Same key, new location: re-point the entry. An unchanged vector keeps its HNSW
// node instead of leaving a deleted node behind.
if (_vectorIndex == null || !TryRelocateVectors(oldKey, oldLocation, newLocation, transaction))
{
DeleteKey(oldKey, oldDocument, oldLocation, transaction);
InsertKey(newKey!, newDocument, newLocation, transaction);
}
return;

var documentId = _mapper.GetId(oldDocument);
}

// Delete old entry if it had a key
if (oldKey != null)
{
var oldUserKey = ConvertToIndexKey(oldKey);
var oldCompositeKey = CreateCompositeKey(oldUserKey, _mapper.ToIndexKey(documentId));
_btreeIndex?.Delete(oldCompositeKey, oldLocation, transaction?.TransactionId);
}
DeleteKey(oldKey, oldDocument, oldLocation, transaction);

// Insert new entry if it has a key
if (newKey != null)
{
var newUserKey = ConvertToIndexKey(newKey);
var newCompositeKey = CreateCompositeKey(newUserKey, _mapper.ToIndexKey(documentId));
_btreeIndex?.Insert(newCompositeKey, newLocation, transaction?.TransactionId);
}
InsertKey(newKey, newDocument, newLocation, transaction);

// Keep the count in line with Insert/Delete, which skip documents without a key.
if (oldKey == null && newKey != null)
System.Threading.Interlocked.Increment(ref _documentCount);
else if (oldKey != null && newKey == null)
System.Threading.Interlocked.Decrement(ref _documentCount);
}

/// <summary>
Expand All @@ -200,16 +178,121 @@ public void Delete(T document, DocumentLocation location, ITransaction transacti
if (!_definition.TryGetKey(document, out var keyValue) || keyValue == null)
return; // Nothing to delete

var userKey = ConvertToIndexKey(keyValue);
var documentId = _mapper.GetId(document);

// Create composite key and delete
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
_btreeIndex?.Delete(compositeKey, location, transaction?.TransactionId);
DeleteKey(keyValue, document, location, transaction);

System.Threading.Interlocked.Decrement(ref _documentCount);
}

private void InsertKey(object keyValue, T document, DocumentLocation location, ITransaction? transaction)
{
if (_vectorIndex != null)
{
// Vector Index Support
if (keyValue is float[] singleVector)
{
_vectorIndex.Insert(singleVector, location, transaction);
}
else if (keyValue is IEnumerable<float[]> vectors)
{
foreach (var v in vectors)
{
_vectorIndex.Insert(v, location, transaction);
}
}
}
else if (_spatialIndex != null)
{
// Geospatial Index Support
if (keyValue is ValueTuple<double, double> t)
{
_spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
}
}
else if (_btreeIndex != null)
{
// BTree Index logic
var userKey = ConvertToIndexKey(keyValue);
var documentId = _mapper.GetId(document);
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
_btreeIndex.Insert(compositeKey, location, transaction?.TransactionId);
}
}

/// <summary>Re-points every vector of <paramref name="keyValue"/>; false if any node was missing.</summary>
private bool TryRelocateVectors(object keyValue, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction? transaction)
{
if (keyValue is float[] singleVector)
return _vectorIndex!.Relocate(singleVector, oldLocation, newLocation, transaction);

if (keyValue is IEnumerable<float[]> vectors)
{
bool all = true;
foreach (var v in vectors)
all &= _vectorIndex!.Relocate(v, oldLocation, newLocation, transaction);
return all;
}

return false;
}

private void DeleteKey(object keyValue, T document, DocumentLocation location, ITransaction? transaction)
{
if (_vectorIndex != null)
{
if (keyValue is float[] singleVector)
{
_vectorIndex.Delete(singleVector, location, transaction);
}
else if (keyValue is IEnumerable<float[]> vectors)
{
foreach (var v in vectors)
{
_vectorIndex.Delete(v, location, transaction);
}
}
}
else if (_spatialIndex != null)
{
if (keyValue is ValueTuple<double, double> t)
{
_spatialIndex.Delete(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
}
}
else if (_btreeIndex != null)
{
var userKey = ConvertToIndexKey(keyValue);
var documentId = _mapper.GetId(document);
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
_btreeIndex.Delete(compositeKey, location, transaction?.TransactionId);
}
}

/// <summary>
/// Key equality for skipping index maintenance. Vectors compare by value: every load
/// materializes a new array, so reference equality would rewrite the vector index on
/// every update.
/// </summary>
private static bool KeysEqual(object? oldKey, object? newKey)
{
if (oldKey is float[] oldVector && newKey is float[] newVector)
return oldVector.AsSpan().SequenceEqual(newVector);

if (oldKey is IEnumerable<float[]> oldVectors && newKey is IEnumerable<float[]> newVectors)
{
using var oldEnum = oldVectors.GetEnumerator();
using var newEnum = newVectors.GetEnumerator();
while (true)
{
bool hasOld = oldEnum.MoveNext();
if (hasOld != newEnum.MoveNext()) return false;
if (!hasOld) return true;
if (!oldEnum.Current.AsSpan().SequenceEqual(newEnum.Current)) return false;
}
}

return Equals(oldKey, newKey);
}

/// <summary>
/// Seeks a single document by exact key match (O(log n))
/// </summary>
Expand Down
Loading
Loading