From 91b118ecd6042625a728ef2bc1e0745824bb4716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 20:24:29 +0200 Subject: [PATCH 1/2] fix(storage): stop the collection metadata chain from growing with every save (#195) SaveCollectionMetadata marked the old record deleted and appended the new one, and nothing reused that space: every save permanently added a record's worth of bytes to the chain at page 1, about one page per 430 saves, and every metadata read and write walked the whole chain. With one save per insert into an int/long-keyed collection (GetNextSequence), inserts got linearly slower. - A record that is no larger than the stored one is overwritten in place, in a single page write. This covers sequence values and root page ids. - A larger or new record first compacts the chain (live records packed from page 1 on, in order; deleted records, leftover bytes and duplicate copies dropped; pages no longer needed freed). Then it is written to a page with room, or to a new page linked at the end, before the old one is marked deleted, so a crash in between leaves the old record rather than neither. - After an in-place write, the chain is compacted when that frees a page, so chains grown by earlier releases shrink on their next metadata save. - Compaction rewrites pages front to back and records only move toward page 1, so a crash part-way can duplicate a record but never lose one. GetAllCollectionsMetadata lists each collection once, and DeleteCollectionMetadata deletes every copy. Co-Authored-By: Claude Opus 5.5 --- .../Storage/StorageEngine.Collections.cs | 329 ++++++++++------ tests/BLite.Tests/MetadataChainTests.cs | 363 ++++++++++++++++++ 2 files changed, 583 insertions(+), 109 deletions(-) create mode 100644 tests/BLite.Tests/MetadataChainTests.cs diff --git a/src/BLite.Core/Storage/StorageEngine.Collections.cs b/src/BLite.Core/Storage/StorageEngine.Collections.cs index a5b503c..e7eabb4 100644 --- a/src/BLite.Core/Storage/StorageEngine.Collections.cs +++ b/src/BLite.Core/Storage/StorageEngine.Collections.cs @@ -364,147 +364,252 @@ public void SaveCollectionMetadata(CollectionMetadata metadata) writer.Write(idx.KeyFormat); var newData = stream.ToArray(); + if (newData.Length + SlotEntry.Size > PageSize - SlottedPageHeader.Size) + throw new InvalidOperationException( + $"Collection metadata for '{metadata.Name}' ({newData.Length} bytes) does not fit in a page."); - // ── Pass 1: walk the chain to find the existing entry and the first page - // with enough free space to store the new serialised record. ── - - var loadedPages = new List<(uint PageId, byte[] Buffer)>(); - uint chainPageId = 1; - int foundOnPageIdx = -1; // index into loadedPages - int foundSlotIndex = -1; - int bestWriteIdx = -1; // first page with enough room + var chain = LoadMetadataChain(); + if (chain.Count == 0) + throw new InvalidOperationException("Page 1 is not a collection metadata page."); + var (foundPage, foundSlot) = FindMetadataRecord(chain, metadata.Name); - while (chainPageId != 0) + // ── 1. No larger than the stored record: overwrite it where it is ───── + // A single page write. Covers the frequent saves (sequence values, root page ids) whose + // record keeps its size, so they no longer leave a deleted record behind (#195). + if (foundPage >= 0) { - var buf = new byte[PageSize]; - ReadPage(chainPageId, null, buf); - int thisIdx = loadedPages.Count; - loadedPages.Add((chainPageId, buf)); - - var hdr = SlottedPageHeader.ReadFrom(buf); - if (hdr.PageType != PageType.Collection) - break; - - // Locate existing slot for this collection name - if (foundSlotIndex == -1) + var (pageId, buffer) = chain[foundPage]; + var slot = ReadSlot(buffer, foundSlot); + if (newData.Length <= slot.Length) { - for (ushort i = 0; i < hdr.SlotCount; i++) - { - var so = SlottedPageHeader.Size + (i * SlotEntry.Size); - var slot = SlotEntry.ReadFrom(buf.AsSpan(so)); - if ((slot.Flags & SlotFlags.Deleted) != 0) continue; - - try - { - using var ms = new MemoryStream(buf, slot.Offset, slot.Length, false); - using var rdr = new BinaryReader(ms); - var existingName = rdr.ReadString(); - if (string.Equals(existingName, metadata.Name, StringComparison.OrdinalIgnoreCase)) - { - foundOnPageIdx = thisIdx; - foundSlotIndex = i; - break; - } - } - catch { } - } + newData.CopyTo(buffer.AsSpan(slot.Offset)); + slot.Length = (ushort)newData.Length; + WriteSlot(buffer, foundSlot, slot); + WritePageImmediate(pageId, buffer); + + // Shrink a chain grown by earlier releases (or by records that grew) once a page can go. + CompactMetadataChain(chain, onlyIfShorter: true); + return; } + } - // First page with enough room for the new entry - if (bestWriteIdx == -1 && hdr.AvailableFreeSpace >= newData.Length + SlotEntry.Size) - bestWriteIdx = thisIdx; + // ── 2. Larger or new: reclaim dead space first, so the record goes into free space in the + // chain before a page is appended ──────────────────────────────────────────────── + chain = CompactMetadataChain(chain, onlyIfShorter: false); + (foundPage, foundSlot) = FindMetadataRecord(chain, metadata.Name); - chainPageId = hdr.NextOverflowPage; + // ── 3. Write the new record, then delete the old one: a crash in between leaves the old + // record in place rather than neither ───────────────────────────────────────────── + int target = chain.FindIndex(p => + SlottedPageHeader.ReadFrom(p.Buffer).AvailableFreeSpace >= newData.Length + SlotEntry.Size); + + if (target < 0) + { + // No room anywhere: append a page holding the record, then link it from the last page. + var newPageId = AllocatePage(); + var newPage = new byte[PageSize]; + NewMetadataPage(newPageId, 0).WriteTo(newPage); + AppendRecord(newPage, newData); + WritePageImmediate(newPageId, newPage); + + var (lastId, lastBuf) = chain[chain.Count - 1]; + var lastHdr = SlottedPageHeader.ReadFrom(lastBuf); + lastHdr.NextOverflowPage = newPageId; + lastHdr.WriteTo(lastBuf); + WritePageImmediate(lastId, lastBuf); + } + else + { + AppendRecord(chain[target].Buffer, newData); + if (target == foundPage) + { + // Same page: one write covers both the new record and the deletion. + MarkSlotDeleted(chain[foundPage].Buffer, foundSlot); + foundPage = -1; + } + WritePageImmediate(chain[target].PageId, chain[target].Buffer); } - // ── Pass 2: mark old entry as Deleted (in its in-memory buffer) ── - if (foundSlotIndex >= 0) + if (foundPage >= 0) { - var (_, fbuf) = loadedPages[foundOnPageIdx]; - var so = SlottedPageHeader.Size + (foundSlotIndex * SlotEntry.Size); - var slot = SlotEntry.ReadFrom(fbuf.AsSpan(so)); - slot.Flags |= SlotFlags.Deleted; - slot.WriteTo(fbuf.AsSpan(so)); + MarkSlotDeleted(chain[foundPage].Buffer, foundSlot); + WritePageImmediate(chain[foundPage].PageId, chain[foundPage].Buffer); } + } // end try + finally { _metadataLock.Release(); } + } - // ── Pass 3: resolve write-target page ── - byte[] targetBuf; - uint targetPageId; + // ── Collection metadata chain (page 1 and its overflow pages) ──────────── - if (bestWriteIdx >= 0) + /// Reads the Collection pages of the metadata chain, starting at page 1. + private List<(uint PageId, byte[] Buffer)> LoadMetadataChain() + { + var chain = new List<(uint PageId, byte[] Buffer)>(); + uint pageId = 1; + while (pageId != 0) { - (targetPageId, targetBuf) = loadedPages[bestWriteIdx]; + var buffer = new byte[PageSize]; + ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + if (header.PageType != PageType.Collection) + break; + chain.Add((pageId, buffer)); + pageId = header.NextOverflowPage; } - else + return chain; + } + + /// Finds the first live record of the named collection, or (-1, -1). + private static (int Page, int Slot) FindMetadataRecord(List<(uint PageId, byte[] Buffer)> chain, string name) + { + for (int p = 0; p < chain.Count; p++) { - // No existing page has room → allocate a new Collection overflow page - targetPageId = AllocatePage(); - targetBuf = new byte[PageSize]; - var newHdr = new SlottedPageHeader + var buffer = chain[p].Buffer; + var header = SlottedPageHeader.ReadFrom(buffer); + for (int i = 0; i < header.SlotCount; i++) { - PageId = targetPageId, - PageType = PageType.Collection, - SlotCount = 0, - FreeSpaceStart = SlottedPageHeader.Size, - FreeSpaceEnd = (ushort)PageSize, - NextOverflowPage = 0, - TransactionId = 0 - }; - newHdr.WriteTo(targetBuf); + var slot = ReadSlot(buffer, i); + if ((slot.Flags & SlotFlags.Deleted) != 0) continue; + if (string.Equals(TryReadRecordName(buffer, slot), name, StringComparison.OrdinalIgnoreCase)) + return (p, i); + } + } + return (-1, -1); + } - // Append new page to the end of the chain - var (lastPid, lastBuf) = loadedPages[loadedPages.Count - 1]; - var lastHdr = SlottedPageHeader.ReadFrom(lastBuf); - lastHdr.NextOverflowPage = targetPageId; - lastHdr.WriteTo(lastBuf); - WritePageImmediate(lastPid, lastBuf); + /// + /// Packs the live records of the chain into as few pages as possible, starting at page 1 and in + /// their current order, and frees the pages no longer needed. Deleted records, bytes left behind + /// by records overwritten with shorter ones, and duplicates of a record (kept: the first, as + /// lookups do) are dropped. + /// + /// + /// Only rewrite when that frees at least one page; otherwise whenever there is anything to drop. + /// + /// The chain as it is afterwards. + /// + /// Pages are rewritten front to back and every record lands on its own page or an earlier one, + /// so all records are on some page after each write. A crash part-way can leave a record on + /// two pages; the copies are identical, readers take the first, and the next compaction drops + /// the other. + /// + private List<(uint PageId, byte[] Buffer)> CompactMetadataChain(List<(uint PageId, byte[] Buffer)> chain, bool onlyIfShorter) + { + var records = new List(); + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + bool anythingToDrop = false; - // Flush the deletion page if it differs from lastPage - if (foundSlotIndex >= 0 && foundOnPageIdx != loadedPages.Count - 1) + foreach (var (_, buffer) in chain) + { + var header = SlottedPageHeader.ReadFrom(buffer); + int liveBytes = 0; + for (int i = 0; i < header.SlotCount; i++) { - var (fpid, fbuf) = loadedPages[foundOnPageIdx]; - WritePageImmediate(fpid, fbuf); - } + var slot = ReadSlot(buffer, i); + if ((slot.Flags & SlotFlags.Deleted) != 0) { anythingToDrop = true; continue; } + + var name = TryReadRecordName(buffer, slot); + if (name != null && !names.Add(name)) { anythingToDrop = true; continue; } - loadedPages.Add((targetPageId, targetBuf)); + records.Add(buffer.AsSpan(slot.Offset, slot.Length).ToArray()); + liveBytes += slot.Length; + } + if (PageSize - header.FreeSpaceEnd > liveBytes) anythingToDrop = true; } - // If deletion page != target page, flush it now - if (foundSlotIndex >= 0) + // Greedy packing in order needs no more pages than any in-order layout, the current one included. + var pages = new List> { new() }; + int room = PageSize - SlottedPageHeader.Size; + foreach (var record in records) { - var (fpid, fbuf) = loadedPages[foundOnPageIdx]; - if (fpid != targetPageId) - WritePageImmediate(fpid, fbuf); + int need = record.Length + SlotEntry.Size; + if (need > room) + { + pages.Add(new List()); + room = PageSize - SlottedPageHeader.Size; + } + pages[pages.Count - 1].Add(record); + room -= need; } - // ── Pass 4: insert new slot into targetBuf ── - var th = SlottedPageHeader.ReadFrom(targetBuf); + if (pages.Count >= chain.Count && (onlyIfShorter || !anythingToDrop)) + return chain; - // Defensive check (should not trigger given the logic above) - if (th.AvailableFreeSpace < newData.Length + SlotEntry.Size) - throw new InvalidOperationException( - $"Not enough space in Collection page {targetPageId} to save collection metadata."); + var result = new List<(uint PageId, byte[] Buffer)>(pages.Count); + for (int p = 0; p < pages.Count; p++) + { + var pageId = chain[p].PageId; + var next = p + 1 < pages.Count ? chain[p + 1].PageId : 0u; + var buffer = new byte[PageSize]; + NewMetadataPage(pageId, next).WriteTo(buffer); + foreach (var record in pages[p]) + AppendRecord(buffer, record); + WritePageImmediate(pageId, buffer); + result.Add((pageId, buffer)); + } - int docOff = th.FreeSpaceEnd - newData.Length; - newData.CopyTo(targetBuf.AsSpan(docOff)); + // Page 1 is always kept; the pages after the new end of the chain are no longer reachable. + for (int p = pages.Count; p < chain.Count; p++) + FreePage(chain[p].PageId); - int newSlotOff = SlottedPageHeader.Size + (th.SlotCount * SlotEntry.Size); - var newSlot = new SlotEntry + return result; + } + + private SlottedPageHeader NewMetadataPage(uint pageId, uint nextPageId) => new() + { + PageId = pageId, + PageType = PageType.Collection, + SlotCount = 0, + FreeSpaceStart = SlottedPageHeader.Size, + FreeSpaceEnd = (ushort)PageSize, + NextOverflowPage = nextPageId, + TransactionId = 0 + }; + + /// Adds as a new slot; the caller checked that it fits. + private static void AppendRecord(byte[] page, byte[] record) + { + var header = SlottedPageHeader.ReadFrom(page); + int offset = header.FreeSpaceEnd - record.Length; + record.CopyTo(page.AsSpan(offset)); + WriteSlot(page, header.SlotCount, new SlotEntry { - Offset = (ushort)docOff, - Length = (ushort)newData.Length, + Offset = (ushort)offset, + Length = (ushort)record.Length, Flags = SlotFlags.None - }; - newSlot.WriteTo(targetBuf.AsSpan(newSlotOff)); + }); + header.SlotCount++; + header.FreeSpaceEnd = (ushort)offset; + header.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + (header.SlotCount * SlotEntry.Size)); + header.WriteTo(page); + } - th.SlotCount++; - th.FreeSpaceEnd = (ushort)docOff; - th.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + (th.SlotCount * SlotEntry.Size)); - th.WriteTo(targetBuf); + private static void MarkSlotDeleted(byte[] page, int slotIndex) + { + var slot = ReadSlot(page, slotIndex); + slot.Flags |= SlotFlags.Deleted; + WriteSlot(page, slotIndex, slot); + } - WritePageImmediate(targetPageId, targetBuf); - } // end try - finally { _metadataLock.Release(); } + private static SlotEntry ReadSlot(byte[] page, int slotIndex) + => SlotEntry.ReadFrom(page.AsSpan(SlottedPageHeader.Size + (slotIndex * SlotEntry.Size))); + + private static void WriteSlot(byte[] page, int slotIndex, SlotEntry slot) + => slot.WriteTo(page.AsSpan(SlottedPageHeader.Size + (slotIndex * SlotEntry.Size))); + + /// The collection name a record starts with, or null if the record is malformed. + private static string? TryReadRecordName(byte[] page, SlotEntry slot) + { + try + { + using var ms = new MemoryStream(page, slot.Offset, slot.Length, false); + using var reader = new BinaryReader(ms); + return reader.ReadString(); + } + catch + { + return null; + } } /// @@ -544,10 +649,11 @@ public void DeleteCollectionMetadata(string name) if (!string.Equals(collName, name, StringComparison.OrdinalIgnoreCase)) continue; + // Keep going: a crash while the chain was compacted can leave a second, + // identical copy further on (see CompactMetadataChain). slot.Flags |= SlotFlags.Deleted; slot.WriteTo(buffer.AsSpan(slotOffset)); WritePageImmediate(pageId, buffer); - return; } catch { } } @@ -561,6 +667,9 @@ public void DeleteCollectionMetadata(string name) public IReadOnlyList GetAllCollectionsMetadata() { var result = new List(); + // One entry per collection: the first copy, as GetCollectionMetadata returns it (a crash + // while the chain was compacted can leave a second one; see CompactMetadataChain). + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); var buffer = new byte[PageSize]; uint pageId = 1; @@ -584,7 +693,9 @@ public IReadOnlyList GetAllCollectionsMetadata() using var reader = new BinaryReader(ms); var collName = reader.ReadString(); + if (seen.Contains(collName)) continue; result.Add(ReadCollectionMetadata(collName, reader)); + seen.Add(collName); } catch { diff --git a/tests/BLite.Tests/MetadataChainTests.cs b/tests/BLite.Tests/MetadataChainTests.cs new file mode 100644 index 0000000..d4f56da --- /dev/null +++ b/tests/BLite.Tests/MetadataChainTests.cs @@ -0,0 +1,363 @@ +using BLite.Core; +using BLite.Core.Retention; +using BLite.Core.Storage; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// The collection metadata chain (page 1 and its overflow pages) must not grow with every save, +/// and a chain grown by older releases shrinks again (#195). +/// +public class MetadataChainTests : IDisposable +{ + private readonly string _dbPath; + + public MetadataChainTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"blite_metadata_chain_{Guid.NewGuid():N}.db"); + } + + public void Dispose() + { + foreach (var path in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + { + try { if (File.Exists(path)) File.Delete(path); } catch { } + } + } + + private sealed record ChainInfo(List Pages, int Live, int Deleted, List LiveLengths) + { + /// Pages the live records need when packed in order. + public int MinimalPages(int pageSize) + { + int pages = 1, room = pageSize - SlottedPageHeader.Size; + foreach (var length in LiveLengths) + { + if (length + SlotEntry.Size > room) { pages++; room = pageSize - SlottedPageHeader.Size; } + room -= length + SlotEntry.Size; + } + return pages; + } + } + + private static ChainInfo Inspect(StorageEngine storage) + { + var pages = new List(); + var lengths = new List(); + int live = 0, deleted = 0; + var buffer = new byte[storage.PageSize]; + uint pageId = 1; + while (pageId != 0) + { + storage.ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + if (header.PageType != PageType.Collection) break; + pages.Add(pageId); + for (int i = 0; i < header.SlotCount; i++) + { + var slot = SlotEntry.ReadFrom(buffer.AsSpan(SlottedPageHeader.Size + i * SlotEntry.Size)); + if ((slot.Flags & SlotFlags.Deleted) != 0) deleted++; + else { live++; lengths.Add(slot.Length); } + } + pageId = header.NextOverflowPage; + } + return new ChainInfo(pages, live, deleted, lengths); + } + + private async Task SeedAsync(params string[] collections) + { + using var engine = new BLiteEngine(_dbPath); + foreach (var name in collections) + await engine.InsertAsync(name, engine.CreateDocument(["name"], b => b.AddString("name", name + "-1"))); + } + + private static string[] Names(int count) => Enumerable.Range(0, count).Select(i => $"c{i}").ToArray(); + + // ── No growth ──────────────────────────────────────────────────────────── + + [Fact] + public async Task SequenceSaves_RewriteTheRecordInPlace() + { + await SeedAsync(Names(8)); + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + var before = Inspect(storage); + for (int i = 1; i <= 5000; i++) + Assert.Equal(i, storage.GetNextSequence("c3")); + + var after = Inspect(storage); + Assert.Equal(before.Pages, after.Pages); + Assert.Equal(8, after.Live); + Assert.Equal(0, after.Deleted); + } + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + Assert.Equal(5001, storage.GetNextSequence("c3")); + } + + [Fact] + public async Task TypedInsertsWithGeneratedIntIds_KeepTheChainAtOnePage() + { + using (var db = new TestDbContext(_dbPath)) + { + for (int i = 0; i < 2000; i++) + await db.Products.InsertAsync(new Product { Title = $"p{i}", Price = i }); + } + + using (var db = new TestDbContext(_dbPath)) + { + var ids = db.Products.AsQueryable().Select(p => p.Id).ToList().OrderBy(id => id).ToList(); + Assert.Equal(Enumerable.Range(1, 2000), ids); + } + + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + var chain = Inspect(storage); + Assert.Single(chain.Pages); + Assert.True(chain.Deleted <= 1); // at most the record the last growing save replaced + } + + [Fact] + public async Task RecordsThatShrinkAndGrow_ReuseTheSpaceTheyLeave() + { + await SeedAsync(Names(8)); + + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + var pagesBefore = Inspect(storage).Pages; + for (int i = 0; i < 500; i++) + { + var meta = storage.GetCollectionMetadata("c5")!; + meta.GeneralRetentionPolicy = new RetentionPolicy + { + MaxAgeMs = i, + TimestampField = new string('t', i % 2 == 0 ? 400 : 10), // alternately grows and shrinks + }; + storage.SaveCollectionMetadata(meta); + } + + var chain = Inspect(storage); + Assert.Equal(pagesBefore, chain.Pages); + Assert.Equal(8, chain.Live); + var last = storage.GetCollectionMetadata("c5")!; + Assert.Equal(499, last.GeneralRetentionPolicy!.MaxAgeMs); + Assert.Equal(new string('t', 10), last.GeneralRetentionPolicy.TimestampField); + } + + // ── Chains that span pages ─────────────────────────────────────────────── + + [Fact] + public async Task ChainOfSeveralPages_GrowsOnlyWithItsLiveRecords_AndShrinksWhenTheyGo() + { + // Records of ~1.2 KB: 40 collections need several 16 KB pages. + var names = Names(40); + await SeedAsync(names); + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + foreach (var name in names) + { + var meta = storage.GetCollectionMetadata(name)!; + meta.GeneralRetentionPolicy = new RetentionPolicy { TimestampField = new string('x', 1100) }; + storage.SaveCollectionMetadata(meta); + } + var grown = Inspect(storage); + Assert.True(grown.Pages.Count >= 3); + Assert.Equal(grown.MinimalPages(storage.PageSize), grown.Pages.Count); + + // Rewrite every record many times: the chain keeps its length. + for (int round = 0; round < 20; round++) + foreach (var name in names) + storage.GetNextSequence(name); + var full = Inspect(storage); + Assert.Equal(grown.Pages, full.Pages); + Assert.Equal(40, full.Live); + + // Drop most collections: the next save packs the rest into page 1 and frees the others. + foreach (var name in names.Skip(10)) + storage.DeleteCollectionMetadata(name); + storage.GetNextSequence("c0"); + + var shrunk = Inspect(storage); + Assert.Equal(new uint[] { 1 }, shrunk.Pages); + Assert.Equal(10, shrunk.Live); + Assert.Equal(0, shrunk.Deleted); + + // The freed pages are reused before the file grows. + var freed = full.Pages.Skip(1).ToHashSet(); + Assert.Contains(storage.AllocatePage(), freed); + } + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + var all = storage.GetAllCollectionsMetadata(); + Assert.Equal(names.Take(10), all.Select(m => m.Name).OrderBy(n => int.Parse(n[1..]))); + Assert.All(all, m => Assert.Equal(new string('x', 1100), m.GeneralRetentionPolicy!.TimestampField)); + Assert.Equal(20, all.Single(m => m.Name == "c1").SequenceValue); + } + + using (var engine = new BLiteEngine(_dbPath)) + Assert.Single(await engine.GetOrCreateCollection("c3").FindAllAsync().ToListAsync()); + } + + // ── Chains grown by older releases ─────────────────────────────────────── + + /// + /// Saves the record of times the way + /// releases before #195 did: the old slot is marked deleted and a copy is appended to the first + /// page with room, or to a new page linked at the end. + /// + private static void GrowLikeOlderReleases(StorageEngine storage, string collection, int saves) + { + for (int n = 0; n < saves; n++) + { + var chain = new List<(uint Id, byte[] Buffer)>(); + byte[]? record = null; + uint pageId = 1; + while (pageId != 0) + { + var buffer = new byte[storage.PageSize]; + storage.ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + chain.Add((pageId, buffer)); + for (int i = 0; i < header.SlotCount && record == null; i++) + { + var offset = SlottedPageHeader.Size + i * SlotEntry.Size; + var slot = SlotEntry.ReadFrom(buffer.AsSpan(offset)); + if ((slot.Flags & SlotFlags.Deleted) != 0) continue; + using var reader = new BinaryReader(new MemoryStream(buffer, slot.Offset, slot.Length, false)); + if (reader.ReadString() != collection) continue; + record = buffer.AsSpan(slot.Offset, slot.Length).ToArray(); + slot.Flags |= SlotFlags.Deleted; + slot.WriteTo(buffer.AsSpan(offset)); + storage.WritePageImmediate(pageId, buffer); + } + pageId = header.NextOverflowPage; + } + + var target = chain.FirstOrDefault(p => + SlottedPageHeader.ReadFrom(p.Buffer).AvailableFreeSpace >= record!.Length + SlotEntry.Size); + if (target.Buffer == null) + { + var id = storage.AllocatePage(); + var page = new byte[storage.PageSize]; + new SlottedPageHeader + { + PageId = id, PageType = PageType.Collection, FreeSpaceStart = SlottedPageHeader.Size, + FreeSpaceEnd = (ushort)storage.PageSize + }.WriteTo(page); + var last = chain[^1]; + var lastHeader = SlottedPageHeader.ReadFrom(last.Buffer); + lastHeader.NextOverflowPage = id; + lastHeader.WriteTo(last.Buffer); + storage.WritePageImmediate(last.Id, last.Buffer); + target = (id, page); + } + + var h = SlottedPageHeader.ReadFrom(target.Buffer); + var dataOffset = h.FreeSpaceEnd - record!.Length; + record.CopyTo(target.Buffer.AsSpan(dataOffset)); + new SlotEntry { Offset = (ushort)dataOffset, Length = (ushort)record.Length } + .WriteTo(target.Buffer.AsSpan(SlottedPageHeader.Size + h.SlotCount * SlotEntry.Size)); + h.SlotCount++; + h.FreeSpaceEnd = (ushort)dataOffset; + h.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + h.SlotCount * SlotEntry.Size); + h.WriteTo(target.Buffer); + storage.WritePageImmediate(target.Id, target.Buffer); + } + } + + [Fact] + public async Task ChainGrownByOlderReleases_ShrinksBackOnTheNextSave() + { + await SeedAsync(Names(8)); + uint[] grownPages; + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + GrowLikeOlderReleases(storage, "c3", 3000); + var grown = Inspect(storage); + Assert.True(grown.Pages.Count >= 7, $"expected a grown chain, got {grown.Pages.Count} pages"); + Assert.Equal(3000, grown.Deleted); + grownPages = grown.Pages.ToArray(); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + // The collection reads as before; the next metadata save (an index here) repairs the chain. + Assert.Single(await engine.GetOrCreateCollection("c3").FindAllAsync().ToListAsync()); + await engine.GetOrCreateCollection("c4").CreateIndexAsync("name"); + } + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + var healed = Inspect(storage); + Assert.Equal(new uint[] { 1 }, healed.Pages); + Assert.Equal(8, healed.Live); + Assert.True(healed.Deleted <= 1); // the record c4 replaced, until the next compaction + Assert.Equal(Names(8), storage.GetAllCollectionsMetadata().Select(m => m.Name).OrderBy(n => n)); + Assert.Single(storage.GetCollectionMetadata("c4")!.Indexes); + Assert.Contains(storage.AllocatePage(), grownPages.Skip(1)); + } + } + + // ── Duplicates a crash during compaction can leave ─────────────────────── + + [Fact] + public async Task DuplicateRecord_IsListedOnce_DeletedWithTheOriginal_AndDroppedByTheNextCompaction() + { + await SeedAsync(Names(3)); + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + + // A second, identical copy of c1's record on a page further on, as a crash part-way + // through a compaction leaves it. + var page1 = new byte[storage.PageSize]; + storage.ReadPage(1, null, page1); + byte[]? record = null; + var header1 = SlottedPageHeader.ReadFrom(page1); + for (int i = 0; i < header1.SlotCount; i++) + { + var slot = SlotEntry.ReadFrom(page1.AsSpan(SlottedPageHeader.Size + i * SlotEntry.Size)); + using var reader = new BinaryReader(new MemoryStream(page1, slot.Offset, slot.Length, false)); + if (reader.ReadString() == "c1") record = page1.AsSpan(slot.Offset, slot.Length).ToArray(); + } + var extraId = storage.AllocatePage(); + var extra = new byte[storage.PageSize]; + var extraHeader = new SlottedPageHeader + { + PageId = extraId, PageType = PageType.Collection, SlotCount = 1, + FreeSpaceStart = SlottedPageHeader.Size + SlotEntry.Size, + FreeSpaceEnd = (ushort)(storage.PageSize - record!.Length) + }; + extraHeader.WriteTo(extra); + record.CopyTo(extra.AsSpan(extraHeader.FreeSpaceEnd)); + new SlotEntry { Offset = extraHeader.FreeSpaceEnd, Length = (ushort)record.Length } + .WriteTo(extra.AsSpan(SlottedPageHeader.Size)); + storage.WritePageImmediate(extraId, extra); + header1.NextOverflowPage = extraId; + header1.WriteTo(page1); + storage.WritePageImmediate(1, page1); + + Assert.Equal(3, storage.GetAllCollectionsMetadata().Count); + + // The next save drops the copy and the page that held it. + storage.GetNextSequence("c2"); + var compacted = Inspect(storage); + Assert.Equal(new uint[] { 1 }, compacted.Pages); + Assert.Equal(3, compacted.Live); + + // With a copy back, a delete removes both. + var againId = storage.AllocatePage(); + extraHeader.PageId = againId; + extraHeader.WriteTo(extra); + storage.WritePageImmediate(againId, extra); + storage.ReadPage(1, null, page1); + header1 = SlottedPageHeader.ReadFrom(page1); + header1.NextOverflowPage = againId; + header1.WriteTo(page1); + storage.WritePageImmediate(1, page1); + Assert.Equal(3, storage.GetAllCollectionsMetadata().Count); + + storage.DeleteCollectionMetadata("c1"); + Assert.Null(storage.GetCollectionMetadata("c1")); + Assert.Equal(new[] { "c0", "c2" }, storage.GetAllCollectionsMetadata().Select(m => m.Name).OrderBy(n => n)); + } +} From 445d68356b129f9c572d6be1c13ca8d3c4ee20dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:44:10 +0200 Subject: [PATCH 2/2] fix(storage): confirm metadata misses under the lock while the chain is compacted (#195) Follow-up to #196. - GetCollectionMetadata: the lock-free walk can miss a record that a concurrent compaction moves onto a page the walk already read (and rewrite or free the page it was on). Callers treat null as "new collection" and save fresh metadata over the existing record, so a miss (or an unparsable page) is now repeated under _metadataLock. - GetAllCollectionsMetadata walks under _metadataLock: a missing entry in a full listing cannot be detected after the fact. - CompactMetadataChain plans from slot lengths and copies nothing unless it rewrites; a single-page chain returns at once, so in-place saves (GetNextSequence) no longer copy every live record. - Document that a crash between the last compaction write and the FreePage loop leaks the tail pages. - Tests: the reader/compaction race for both readers (deterministic, via an intercepting IPageStorage), and the up-front size check. Co-Authored-By: Claude Opus 5.5 --- .../Storage/StorageEngine.Collections.cs | 104 +++++++--- tests/BLite.Tests/MetadataChainTests.cs | 196 ++++++++++++++++++ 2 files changed, 271 insertions(+), 29 deletions(-) diff --git a/src/BLite.Core/Storage/StorageEngine.Collections.cs b/src/BLite.Core/Storage/StorageEngine.Collections.cs index e7eabb4..08a3ed5 100644 --- a/src/BLite.Core/Storage/StorageEngine.Collections.cs +++ b/src/BLite.Core/Storage/StorageEngine.Collections.cs @@ -125,6 +125,34 @@ public class IndexMetadata public sealed partial class StorageEngine { public CollectionMetadata? GetCollectionMetadata(string name) + { + // Lock-free first: saves only rewrite or move records, so a record this walk finds is the + // collection's. A miss proves nothing, though: a concurrent compaction (any save, under + // _metadataLock) can move the record onto a page the walk has already read and rewrite or + // free the page it was on. Callers take null for "new collection" and save fresh metadata + // over the existing record, so a miss, or a walk that runs into a page it cannot parse, is + // repeated under the lock. + try + { + var metadata = FindCollectionMetadata(name); + if (metadata != null) + return metadata; + } + catch + { + // Retried under the lock below, where an error is real and propagates. + } + + if (!_metadataLock.Wait(_config.LockTimeout.WriteTimeoutMs)) + throw new TimeoutException("Timed out acquiring metadata lock (GetCollectionMetadata)."); + try + { + return FindCollectionMetadata(name); + } + finally { _metadataLock.Release(); } + } + + private CollectionMetadata? FindCollectionMetadata(string name) { var buffer = new byte[PageSize]; uint pageId = 1; @@ -364,6 +392,7 @@ public void SaveCollectionMetadata(CollectionMetadata metadata) writer.Write(idx.KeyFormat); var newData = stream.ToArray(); + // Checked before anything is written or allocated: the stored record stays as it is. if (newData.Length + SlotEntry.Size > PageSize - SlottedPageHeader.Size) throw new InvalidOperationException( $"Collection metadata for '{metadata.Name}' ({newData.Length} bytes) does not fit in a page."); @@ -491,16 +520,26 @@ private static (int Page, int Slot) FindMetadataRecord(List<(uint PageId, byte[] /// Pages are rewritten front to back and every record lands on its own page or an earlier one, /// so all records are on some page after each write. A crash part-way can leave a record on /// two pages; the copies are identical, readers take the first, and the next compaction drops - /// the other. + /// the other. A crash after the last page is rewritten but before the pages past the new end + /// are freed leaves those pages unreachable and off the free list: they leak (file space only, + /// no records are lost), which is accepted to keep the rewrite free of extra bookkeeping. /// private List<(uint PageId, byte[] Buffer)> CompactMetadataChain(List<(uint PageId, byte[] Buffer)> chain, bool onlyIfShorter) { - var records = new List(); - var names = new HashSet(StringComparer.OrdinalIgnoreCase); + // A single page cannot get shorter. This is the common case: every in-place save asks. + if (onlyIfShorter && chain.Count <= 1) + return chain; + + // Plan from the slot tables alone: which records stay, and the page each one lands on. + var keep = new List<(int SourcePage, SlotEntry Slot, int TargetPage)>(); + var names = new HashSet(StringComparer.OrdinalIgnoreCase); bool anythingToDrop = false; + int pageCount = 1; + int room = PageSize - SlottedPageHeader.Size; - foreach (var (_, buffer) in chain) + for (int p = 0; p < chain.Count; p++) { + var buffer = chain[p].Buffer; var header = SlottedPageHeader.ReadFrom(buffer); int liveBytes = 0; for (int i = 0; i < header.SlotCount; i++) @@ -510,46 +549,45 @@ private static (int Page, int Slot) FindMetadataRecord(List<(uint PageId, byte[] var name = TryReadRecordName(buffer, slot); if (name != null && !names.Add(name)) { anythingToDrop = true; continue; } - - records.Add(buffer.AsSpan(slot.Offset, slot.Length).ToArray()); liveBytes += slot.Length; - } - if (PageSize - header.FreeSpaceEnd > liveBytes) anythingToDrop = true; - } - // Greedy packing in order needs no more pages than any in-order layout, the current one included. - var pages = new List> { new() }; - int room = PageSize - SlottedPageHeader.Size; - foreach (var record in records) - { - int need = record.Length + SlotEntry.Size; - if (need > room) - { - pages.Add(new List()); - room = PageSize - SlottedPageHeader.Size; + // Greedy packing in order needs no more pages than any in-order layout, the current one included. + int need = slot.Length + SlotEntry.Size; + if (need > room) + { + pageCount++; + room = PageSize - SlottedPageHeader.Size; + } + room -= need; + keep.Add((p, slot, pageCount - 1)); } - pages[pages.Count - 1].Add(record); - room -= need; + if (PageSize - header.FreeSpaceEnd > liveBytes) anythingToDrop = true; } - if (pages.Count >= chain.Count && (onlyIfShorter || !anythingToDrop)) + if (pageCount >= chain.Count && (onlyIfShorter || !anythingToDrop)) return chain; - var result = new List<(uint PageId, byte[] Buffer)>(pages.Count); - for (int p = 0; p < pages.Count; p++) + // The records are copied out of the buffers loaded before the first write; each page is + // rebuilt in a fresh buffer, so those stay intact while the chain is rewritten. + var result = new List<(uint PageId, byte[] Buffer)>(pageCount); + int k = 0; + for (int p = 0; p < pageCount; p++) { var pageId = chain[p].PageId; - var next = p + 1 < pages.Count ? chain[p + 1].PageId : 0u; + var next = p + 1 < pageCount ? chain[p + 1].PageId : 0u; var buffer = new byte[PageSize]; NewMetadataPage(pageId, next).WriteTo(buffer); - foreach (var record in pages[p]) - AppendRecord(buffer, record); + for (; k < keep.Count && keep[k].TargetPage == p; k++) + { + var (source, slot, _) = keep[k]; + AppendRecord(buffer, chain[source].Buffer.AsSpan(slot.Offset, slot.Length)); + } WritePageImmediate(pageId, buffer); result.Add((pageId, buffer)); } // Page 1 is always kept; the pages after the new end of the chain are no longer reachable. - for (int p = pages.Count; p < chain.Count; p++) + for (int p = pageCount; p < chain.Count; p++) FreePage(chain[p].PageId); return result; @@ -567,7 +605,7 @@ private static (int Page, int Slot) FindMetadataRecord(List<(uint PageId, byte[] }; /// Adds as a new slot; the caller checked that it fits. - private static void AppendRecord(byte[] page, byte[] record) + private static void AppendRecord(byte[] page, ReadOnlySpan record) { var header = SlottedPageHeader.ReadFrom(page); int offset = header.FreeSpaceEnd - record.Length; @@ -666,6 +704,12 @@ public void DeleteCollectionMetadata(string name) public IReadOnlyList GetAllCollectionsMetadata() { + // Under the lock: a concurrent compaction can move records onto a page this walk has + // already read, and unlike a single lookup the result would not show that they are missing. + if (!_metadataLock.Wait(_config.LockTimeout.WriteTimeoutMs)) + throw new TimeoutException("Timed out acquiring metadata lock (GetAllCollectionsMetadata)."); + try + { var result = new List(); // One entry per collection: the first copy, as GetCollectionMetadata returns it (a crash // while the chain was compacted can leave a second one; see CompactMetadataChain). @@ -707,6 +751,8 @@ public IReadOnlyList GetAllCollectionsMetadata() } return result; + } // end try + finally { _metadataLock.Release(); } } /// diff --git a/tests/BLite.Tests/MetadataChainTests.cs b/tests/BLite.Tests/MetadataChainTests.cs index d4f56da..bece2ae 100644 --- a/tests/BLite.Tests/MetadataChainTests.cs +++ b/tests/BLite.Tests/MetadataChainTests.cs @@ -1,6 +1,7 @@ using BLite.Core; using BLite.Core.Retention; using BLite.Core.Storage; +using BLite.Core.Transactions; using BLite.Shared; namespace BLite.Tests; @@ -360,4 +361,199 @@ public async Task DuplicateRecord_IsListedOnce_DeletedWithTheOriginal_AndDropped Assert.Null(storage.GetCollectionMetadata("c1")); Assert.Equal(new[] { "c0", "c2" }, storage.GetAllCollectionsMetadata().Select(m => m.Name).OrderBy(n => n)); } + + // ── Readers that do not take the metadata lock ─────────────────────────── + + /// In-memory page storage that runs a callback once, right after the next read of a page. + private sealed class InterceptingPageStorage : IPageStorage + { + private readonly MemoryPageStorage _inner; + private uint _pageId; + private Action? _afterRead; + + public InterceptingPageStorage(int pageSize) + { + _inner = new MemoryPageStorage(pageSize); + _inner.Open(); + } + + public void AfterNextReadOf(uint pageId, Action action) + { + _pageId = pageId; + Volatile.Write(ref _afterRead, action); + } + + public void ReadPage(uint pageId, Span destination) + { + _inner.ReadPage(pageId, destination); + if (pageId == _pageId && Interlocked.Exchange(ref _afterRead, null) is { } action) + action(); + } + + public int PageSize => _inner.PageSize; + public uint NextPageId => _inner.NextPageId; + public void Open() { } + public void ReadPageHeader(uint pageId, Span destination) => _inner.ReadPageHeader(pageId, destination); + public ValueTask ReadPageAsync(uint pageId, Memory destination, CancellationToken cancellationToken = default) + => _inner.ReadPageAsync(pageId, destination, cancellationToken); + public void WritePage(uint pageId, ReadOnlySpan source) => _inner.WritePage(pageId, source); + public uint AllocatePage() => _inner.AllocatePage(); + public void FreePage(uint pageId) => _inner.FreePage(pageId); + public void Flush() => _inner.Flush(); + public Task FlushAsync(CancellationToken cancellationToken = default) => _inner.FlushAsync(cancellationToken); + public Task BackupAsync(string destinationPath, CancellationToken cancellationToken = default) + => _inner.BackupAsync(destinationPath, cancellationToken); + public Task TruncateToMinimumAsync(CancellationToken cancellationToken = default) + => _inner.TruncateToMinimumAsync(cancellationToken); + public void Dispose() => _inner.Dispose(); + } + + /// The live collection names on each page of the chain. + private static List<(uint PageId, List Names)> Layout(StorageEngine storage) + { + var layout = new List<(uint, List)>(); + var buffer = new byte[storage.PageSize]; + uint pageId = 1; + while (pageId != 0) + { + storage.ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + if (header.PageType != PageType.Collection) break; + var names = new List(); + for (int i = 0; i < header.SlotCount; i++) + { + var slot = SlotEntry.ReadFrom(buffer.AsSpan(SlottedPageHeader.Size + i * SlotEntry.Size)); + if ((slot.Flags & SlotFlags.Deleted) != 0) continue; + using var reader = new BinaryReader(new MemoryStream(buffer, slot.Offset, slot.Length, false)); + names.Add(reader.ReadString()); + } + layout.Add((pageId, names)); + pageId = header.NextOverflowPage; + } + return layout; + } + + /// + /// A chain of three pages whose live records fit in two: all of page 1 but its first record is + /// deleted. The next save of that first record compacts the chain: the records of page 2 move + /// to page 1 (but its last), page 3's move to page 2, and page 3 is freed. + /// + private static (StorageEngine Storage, InterceptingPageStorage Pages, List<(uint PageId, List Names)> Layout) + ChainAboutToBeCompacted() + { + var pages = new InterceptingPageStorage(16384); + var storage = new StorageEngine(pages, new MemoryWriteAheadLog()); + foreach (var name in Names(36)) + { + storage.SaveCollectionMetadata(new CollectionMetadata + { + Name = name, + GeneralRetentionPolicy = new RetentionPolicy { TimestampField = new string('x', 1100) } + }); + } + + var layout = Layout(storage); + Assert.Equal(3, layout.Count); + foreach (var name in layout[0].Names.Skip(1)) + storage.DeleteCollectionMetadata(name); + return (storage, pages, Layout(storage)); + } + + [Theory] + [InlineData(0)] // the reader then reads page 2 after the rewrite: the record has left it for page 1 + [InlineData(1)] // the reader then follows the old link to page 3, which the compaction freed + public void GetCollectionMetadata_FindsARecordThatACompactionMovesBehindTheReader(int compactAfterPage) + { + var (storage, pages, layout) = ChainAboutToBeCompacted(); + using var _ = storage; + var trigger = layout[0].Names[0]; + var moved = layout[compactAfterPage + 1].Names[0]; // moves onto the page the reader has just read + + // A save of another collection compacts the chain while the reader is between two pages. + pages.AfterNextReadOf(layout[compactAfterPage].PageId, + () => Assert.True(Task.Run(() => storage.GetNextSequence(trigger)).Wait(TimeSpan.FromSeconds(10)))); + var metadata = storage.GetCollectionMetadata(moved); + + Assert.Equal(2, Layout(storage).Count); // the compaction ran mid-walk + Assert.NotNull(metadata); + Assert.Equal(moved, metadata!.Name); + Assert.Equal(new string('x', 1100), metadata.GeneralRetentionPolicy!.TimestampField); + } + + [Fact] + public void GetAllCollectionsMetadata_ListsRecordsThatACompactionMovesBehindTheReader() + { + var (storage, pages, layout) = ChainAboutToBeCompacted(); + using var _ = storage; + var live = layout.SelectMany(p => p.Names).OrderBy(n => n).ToList(); + + // Same race as above, while the reader is between page 1 and page 2. The reader holds the + // metadata lock, so the save waits for it (well within the lock timeout) instead of running. + Task? compaction = null; + pages.AfterNextReadOf(1, () => + { + compaction = Task.Run(() => storage.GetNextSequence(layout[0].Names[0])); + compaction.Wait(TimeSpan.FromMilliseconds(100)); + }); + var all = storage.GetAllCollectionsMetadata(); + Assert.NotNull(compaction); + compaction!.Wait(); + + Assert.Equal(live, all.Select(m => m.Name).OrderBy(n => n)); + Assert.Equal(2, Layout(storage).Count); + } + + // ── Records too large for a page ───────────────────────────────────────── + + private static int RecordLength(StorageEngine storage, string collection) + { + var buffer = new byte[storage.PageSize]; + uint pageId = 1; + while (pageId != 0) + { + storage.ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + for (int i = 0; i < header.SlotCount; i++) + { + var slot = SlotEntry.ReadFrom(buffer.AsSpan(SlottedPageHeader.Size + i * SlotEntry.Size)); + if ((slot.Flags & SlotFlags.Deleted) != 0) continue; + using var reader = new BinaryReader(new MemoryStream(buffer, slot.Offset, slot.Length, false)); + if (reader.ReadString() == collection) return slot.Length; + } + pageId = header.NextOverflowPage; + } + throw new InvalidOperationException($"No record for {collection}."); + } + + [Fact] + public async Task RecordTooLargeForAPage_IsRefusedBeforeAnythingIsWritten() + { + await SeedAsync(Names(3)); + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + int max = storage.PageSize - SlottedPageHeader.Size - SlotEntry.Size; + + var meta = storage.GetCollectionMetadata("c1")!; + meta.GeneralRetentionPolicy = new RetentionPolicy { TimestampField = new string('t', 200) }; + storage.SaveCollectionMetadata(meta); + int fill = 200 + max - RecordLength(storage, "c1"); // 2-byte length prefix either way + + // The largest record a page holds is stored (on a page of its own)... + meta.GeneralRetentionPolicy.TimestampField = new string('t', fill); + storage.SaveCollectionMetadata(meta); + Assert.Equal(max, RecordLength(storage, "c1")); + var before = Inspect(storage); + Assert.Equal(2, before.Pages.Count); + + // ...one byte more is refused, and the chain and the stored record stay exactly as they were. + meta.GeneralRetentionPolicy.TimestampField = new string('t', fill + 1); + var error = Assert.Throws(() => storage.SaveCollectionMetadata(meta)); + Assert.Contains("does not fit in a page", error.Message); + + var after = Inspect(storage); + Assert.Equal(before.Pages, after.Pages); + Assert.Equal(before.Live, after.Live); + Assert.Equal(before.Deleted, after.Deleted); + Assert.Equal(new string('t', fill), storage.GetCollectionMetadata("c1")!.GeneralRetentionPolicy!.TimestampField); + Assert.Equal(3, storage.GetAllCollectionsMetadata().Count); + } }