fix(index): split B-Tree nodes by size and find duplicates across splits (#217, #213)

src/BLite.Core/Collections/DocumentCollection.cs

@@ -3035,7 +3035,7 @@ private async Task<bool> UpdateAtLocationCore(TId id, T entity, byte[] docData,
         // Unique indexes only where the key changed, so documents that already share a value
         // can still be updated.
         _indexManager.EnsureUniqueInAll(entity, oldEntity, transaction);
-        _indexManager.ValidateKeysInAll(entity);
+        _indexManager.ValidateKeysInAll(entity, oldEntity);
 
         // Read old page
         var pageBuffer = ArrayPool<byte>.Shared.Rent(_storage.PageSize);
@@ -3066,7 +3066,9 @@ private async Task<bool> UpdateAtLocationCore(TId id, T entity, byte[] docData,
             {
                 // Delete old + insert new. Secondary indexes are moved below, so an entry whose
                 // key did not change is re-pointed rather than deleted and inserted again.
-                await DeleteCore(id, transaction, notifyCdc: false, deleteFromIndexes: false);
+                // The primary entry is kept and re-pointed below: the id is unchanged, and one
+                // stored before MaxKeySize was lowered (#217) could not be inserted again.
+                await DeleteCore(id, transaction, notifyCdc: false, deleteFromIndexes: false, deleteFromPrimary: false);
 
                 DocumentLocation newLocation;
                 if (bytesWritten + SlotEntry.Size <= _maxDocumentSizeForSinglePage)
@@ -3082,7 +3084,8 @@ private async Task<bool> UpdateAtLocationCore(TId id, T entity, byte[] docData,
                     newLocation = new DocumentLocation(newPageId, newSlotIndex);
                 }
 
-                _primaryIndex.Insert(key, newLocation, transaction.TransactionId);
+                if (!_primaryIndex.Relocate(key, oldLocation, newLocation, transaction.TransactionId))
+                    _primaryIndex.Insert(key, newLocation, transaction.TransactionId);
                 _indexManager.UpdateInAll(oldEntity, entity, oldLocation, newLocation, transaction);
 
                 // Notify CDC
@@ -3377,7 +3380,7 @@ private async Task<int> DeleteBulkInternal(IEnumerable<TId> ids, ITransaction tr
         return deleteCount;
     }
 
-    private async Task<bool> DeleteCore(TId id, ITransaction transaction, bool notifyCdc = true, bool deleteFromIndexes = true)
+    private async Task<bool> DeleteCore(TId id, ITransaction transaction, bool notifyCdc = true, bool deleteFromIndexes = true, bool deleteFromPrimary = true)
     {
         var key = _mapper.ToIndexKey(id);
         if (!_primaryIndex.TryFind(key, out var location, transaction.TransactionId))
@@ -3434,7 +3437,8 @@ private async Task<bool> DeleteCore(TId id, ITransaction transaction, bool notif
             _fsi.Update(location.PageId, compactedHeader.AvailableFreeSpace);
 
             // Remove from primary index
-            _primaryIndex.Delete(key, location, transaction.TransactionId);
+            if (deleteFromPrimary)
+                _primaryIndex.Delete(key, location, transaction.TransactionId);
 
             // Notify CDC
             if (notifyCdc) 

src/BLite.Core/DynamicCollection.cs

@@ -1267,16 +1267,28 @@ private async Task UpdateAtLocationCore(BsonId id, IndexKey key, BsonDocument ne
         // Everything that can reject the document is checked before any page is written.
         ValidateWrite(id, key, newDocument, oldDoc, isInsert: false, transaction);
 
-        DeleteSlot(oldLocation, transaction);
-
+        // A document that is not larger than before stays in its slot, so the entries of keys
+        // that did not change stay valid and are left alone (see IndexUpdate).
         var docData = newDocument.RawData;
-        var pageId = FindPageWithSpace(docData.Length + SlotEntry.Size, transaction.TransactionId);
-        if (pageId == 0) pageId = AllocateNewDataPage(transaction);
-        var slotIndex = InsertIntoPage(pageId, docData, transaction);
-        var newLocation = new DocumentLocation(pageId, slotIndex);
+        DocumentLocation newLocation;
+        if (TryUpdateInPlace(oldLocation, docData, transaction))
+        {
+            newLocation = oldLocation;
+        }
+        else
+        {
+            DeleteSlot(oldLocation, transaction);
 
-        _primaryIndex.Delete(key, oldLocation, transaction.TransactionId);
-        _primaryIndex.Insert(key, newLocation, transaction.TransactionId);
+            var pageId = FindPageWithSpace(docData.Length + SlotEntry.Size, transaction.TransactionId);
+            if (pageId == 0) pageId = AllocateNewDataPage(transaction);
+            var slotIndex = InsertIntoPage(pageId, docData, transaction);
+            newLocation = new DocumentLocation(pageId, slotIndex);
+
+            // The id is unchanged: re-point its entry rather than inserting it again, which
+            // also lets an id stored before MaxKeySize was lowered (#217) follow its document.
+            if (!_primaryIndex.Relocate(key, oldLocation, newLocation, transaction.TransactionId))
+                _primaryIndex.Insert(key, newLocation, transaction.TransactionId);
+        }
 
         foreach (var (_, idx) in _secondaryIndexes)
             IndexUpdate(idx, oldDoc, newDocument, oldLocation, newLocation, transaction);
@@ -2127,7 +2139,9 @@ private void ValidateWrite(BsonId id, IndexKey key, BsonDocument document, BsonD
             throw new BLiteValidationException(
                 $"Document size {size} exceeds maximum single page size. Overflow not yet supported in DynamicCollection.");
 
-        if (key.Data.Length > _primaryIndex.MaxKeySize)
+        // An update keeps the id it was found by, so only a new id is checked: one stored before
+        // MaxKeySize was lowered (#217) must not make its document impossible to update.
+        if (isInsert && key.Data.Length > _primaryIndex.MaxKeySize)
             throw new BLiteValidationException(
                 $"Id of {key.Data.Length} bytes is too large for the primary index of collection '{_collectionName}' (maximum {_primaryIndex.MaxKeySize} bytes).");
 
@@ -2145,20 +2159,21 @@ private void ValidateWrite(BsonId id, IndexKey key, BsonDocument document, BsonD
                 {
                     if (BsonValueToIndexKey(val) is not { } indexKey) break;
 
+                    // On update an unchanged key keeps its entry (IndexUpdate leaves it in place
+                    // or re-points it), so it is neither size-checked again (a key stored before
+                    // MaxKeySize was lowered, #217, stays updatable) nor checked for uniqueness
+                    // (documents that already share a value can still be updated).
+                    if (oldDocument != null
+                        && TryGetBTreeKey(oldDocument, idx, out var oldKey)
+                        && oldKey.Data.SequenceEqual(indexKey.Data))
+                        break;
+
                     if (indexKey.Data.Length > idx.BTree!.MaxKeySize)
                         throw new BLiteValidationException(
                             $"Key of {indexKey.Data.Length} bytes is too large for index '{name}' (maximum {idx.BTree.MaxKeySize} bytes).");
 
                     if (!idx.Options.Unique) break;
 
-                    // On update an unchanged key keeps its entry (IndexUpdate deletes it first),
-                    // so documents that already share a value can still be updated.
-                    if (oldDocument != null
-                        && TryGetNestedValue(oldDocument, idx.FieldPath, out var oldVal)
-                        && BsonValueToIndexKey(oldVal) is { } oldKey
-                        && oldKey.Data.SequenceEqual(indexKey.Data))
-                        break;
-
                     if (idx.BTree.TryFind(indexKey, out _, transaction.TransactionId))
                         throw new BLiteValidationException($"Duplicate key violation for unique index '{name}'");
                     break;
@@ -2248,10 +2263,69 @@ private void IndexUpdate(DynamicSecondaryIndex idx, BsonDocument? oldDocument, B
             && idx.Vector!.Relocate(oldVector, oldLocation, newLocation, transaction))
             return;
 
+        // An unchanged B-Tree key keeps its entry: left alone when the document stayed in its
+        // slot, re-pointed when it moved. Deleting and inserting it again would cost a search
+        // through every copy of a duplicated key (keys here carry no document id), and would
+        // fail for a key stored before MaxKeySize was lowered (#217).
+        if (oldDocument != null && idx.Kind == DynamicIndexKind.BTree
+            && TryGetBTreeKey(oldDocument, idx, out var oldKey)
+            && TryGetBTreeKey(newDocument, idx, out var newKey)
+            && oldKey.Data.SequenceEqual(newKey.Data))
+        {
+            bool moved = oldLocation.PageId != newLocation.PageId || oldLocation.SlotIndex != newLocation.SlotIndex;
+            if (!moved || idx.BTree!.Relocate(oldKey, oldLocation, newLocation, transaction.TransactionId))
+                return;
+        }
+
         if (oldDocument != null) IndexDelete(idx, oldDocument, oldLocation, transaction);
         IndexInsert(idx, newDocument, newLocation, transaction);
     }
 
+    /// <summary>The key <paramref name="document"/> has in the B-Tree index <paramref name="idx"/>, if any.</summary>
+    private bool TryGetBTreeKey(BsonDocument document, DynamicSecondaryIndex idx, out IndexKey key)
+    {
+        key = default;
+        if (!TryGetNestedValue(document, idx.FieldPath, out var val) || BsonValueToIndexKey(val) is not { } k)
+            return false;
+        key = k;
+        return true;
+    }
+
+    /// <summary>
+    /// Overwrites the document at <paramref name="location"/> with <paramref name="data"/> when
+    /// it fits in the slot it already has, so the document keeps its location. Returns
+    /// <c>false</c>, writing nothing, when it does not fit or the slot is not a live data slot.
+    /// </summary>
+    private bool TryUpdateInPlace(DocumentLocation location, ReadOnlyMemory<byte> data, ITransaction transaction)
+    {
+        var buffer = ArrayPool<byte>.Shared.Rent(_storage.PageSize);
+        try
+        {
+            _storage.ReadPage(location.PageId, transaction.TransactionId, buffer);
+            var header = SlottedPageHeader.ReadFrom(buffer);
+            if (header.PageType != PageType.Data || location.SlotIndex >= header.SlotCount)
+                return false;
+
+            var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
+            var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
+            if (slot.Flags != SlotFlags.None || data.Length > slot.Length)
+                return false;
+
+            data.Span.CopyTo(buffer.AsSpan(slot.Offset, data.Length));
+            // Erase the bytes the shorter document no longer uses, as DeleteSlot does.
+            buffer.AsSpan(slot.Offset + data.Length, slot.Length - data.Length).Clear();
+            slot.Length = (ushort)data.Length;
+            slot.WriteTo(buffer.AsSpan(slotOffset));
+
+            _storage.WritePage(location.PageId, transaction.TransactionId, buffer.AsSpan(0, _storage.PageSize));
+            return true;
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(buffer);
+        }
+    }
+
     #endregion
 
     #region Internal storage operations

src/BLite.Core/Indexing/BTreeCursor.cs

@@ -54,7 +54,12 @@ public bool MoveToFirst()
             pageId = BitConverter.ToUInt32(_pageBuffer.AsSpan(dataOffset, 4));
         }
 
-        return PositionAtStart();
+        if (PositionAtStart()) return true;
+
+        // The leaf is empty (left unmerged, #217): MoveNext skips to the next one with entries.
+        _isValid = true;
+        _currentEntryIndex = -1;
+        return MoveNext();
     }
 
     public bool MoveToLast()
@@ -89,22 +94,29 @@ public bool MoveToLast()
             pageId = lastPointer;
         }
 
-        return PositionAtEnd();
+        if (PositionAtEnd()) return true;
+
+        // The leaf is empty (left unmerged, #217): MovePrev skips to the previous one with entries.
+        _isValid = true;
+        _currentEntryIndex = 0;
+        return MovePrev();
     }
 
     public bool Seek(IndexKey key)
     {
-        // Use Index to find leaf
-        var leafPageId = _index.FindLeafNode(key, _transactionId);
+        // Use Index to find leaf: the leftmost one that can hold the key, so that no copy of a
+        // duplicated key sits in an earlier leaf (#213).
+        var leafPageId = _index.FindLeafNode(key, _transactionId, leftmost: true);
         LoadPage(leafPageId);
         ParseEntries();
 
         // Binary search in entries
         var idx = _currentEntries.BinarySearch(new IndexEntry(key, default(DocumentLocation)));
-        
+
         if (idx >= 0)
         {
-            // Found exact match
+            // Found exact match; with duplicates, position on the first copy.
+            while (idx > 0 && _currentEntries[idx - 1].Key.Equals(key)) idx--;
             _currentEntryIndex = idx;
             _isValid = true;
             return true;
@@ -121,16 +133,18 @@ public bool Seek(IndexKey key)
             }
             else
             {
-                // Key is larger than max in this page, move to next page
-                if (_currentHeader.NextLeafPageId != 0)
+                // Key is larger than max in this page, move to the next page that has entries
+                // (a leaf can be left empty when it could not be merged, #217).
+                while (_currentHeader.NextLeafPageId != 0)
                 {
                     LoadPage(_currentHeader.NextLeafPageId);
                     ParseEntries();
                     _currentEntryIndex = 0;
                     if (_currentEntries.Count > 0)
                     {
+                        // The key can start the next leaf when it equals the separator.
                         _isValid = true;
-                        return false;
+                        return _currentEntries[0].Key.Equals(key);
                     }
                 }
 
@@ -151,11 +165,11 @@ public bool MoveNext()
             return true;
         }
 
-        // Move to next page
-        if (_currentHeader.NextLeafPageId != 0)
+        // Move to the next page that has entries
+        while (_currentHeader.NextLeafPageId != 0)
         {
             LoadPage(_currentHeader.NextLeafPageId);
-            return PositionAtStart();
+            if (PositionAtStart()) return true;
         }
 
         _isValid = false;
@@ -172,11 +186,11 @@ public bool MovePrev()
             return true;
         }
 
-        // Move to prev page
-        if (_currentHeader.PrevLeafPageId != 0)
+        // Move to the previous page that has entries
+        while (_currentHeader.PrevLeafPageId != 0)
         {
             LoadPage(_currentHeader.PrevLeafPageId);
-            return PositionAtEnd();
+            if (PositionAtEnd()) return true;
         }
 
         _isValid = false;