From 91b118ecd6042625a728ef2bc1e0745824bb4716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 20:24:29 +0200 Subject: [PATCH] fix(storage): stop the collection metadata chain from growing with every save (#195) SaveCollectionMetadata marked the old record deleted and appended the new one, and nothing reused that space: every save permanently added a record's worth of bytes to the chain at page 1, about one page per 430 saves, and every metadata read and write walked the whole chain. With one save per insert into an int/long-keyed collection (GetNextSequence), inserts got linearly slower. - A record that is no larger than the stored one is overwritten in place, in a single page write. This covers sequence values and root page ids. - A larger or new record first compacts the chain (live records packed from page 1 on, in order; deleted records, leftover bytes and duplicate copies dropped; pages no longer needed freed). Then it is written to a page with room, or to a new page linked at the end, before the old one is marked deleted, so a crash in between leaves the old record rather than neither. - After an in-place write, the chain is compacted when that frees a page, so chains grown by earlier releases shrink on their next metadata save. - Compaction rewrites pages front to back and records only move toward page 1, so a crash part-way can duplicate a record but never lose one. GetAllCollectionsMetadata lists each collection once, and DeleteCollectionMetadata deletes every copy. Co-Authored-By: Claude Opus 5.5 --- .../Storage/StorageEngine.Collections.cs | 329 ++++++++++------ tests/BLite.Tests/MetadataChainTests.cs | 363 ++++++++++++++++++ 2 files changed, 583 insertions(+), 109 deletions(-) create mode 100644 tests/BLite.Tests/MetadataChainTests.cs diff --git a/src/BLite.Core/Storage/StorageEngine.Collections.cs b/src/BLite.Core/Storage/StorageEngine.Collections.cs index a5b503c..e7eabb4 100644 --- a/src/BLite.Core/Storage/StorageEngine.Collections.cs +++ b/src/BLite.Core/Storage/StorageEngine.Collections.cs @@ -364,147 +364,252 @@ public void SaveCollectionMetadata(CollectionMetadata metadata) writer.Write(idx.KeyFormat); var newData = stream.ToArray(); + if (newData.Length + SlotEntry.Size > PageSize - SlottedPageHeader.Size) + throw new InvalidOperationException( + $"Collection metadata for '{metadata.Name}' ({newData.Length} bytes) does not fit in a page."); - // ── Pass 1: walk the chain to find the existing entry and the first page - // with enough free space to store the new serialised record. ── - - var loadedPages = new List<(uint PageId, byte[] Buffer)>(); - uint chainPageId = 1; - int foundOnPageIdx = -1; // index into loadedPages - int foundSlotIndex = -1; - int bestWriteIdx = -1; // first page with enough room + var chain = LoadMetadataChain(); + if (chain.Count == 0) + throw new InvalidOperationException("Page 1 is not a collection metadata page."); + var (foundPage, foundSlot) = FindMetadataRecord(chain, metadata.Name); - while (chainPageId != 0) + // ── 1. No larger than the stored record: overwrite it where it is ───── + // A single page write. Covers the frequent saves (sequence values, root page ids) whose + // record keeps its size, so they no longer leave a deleted record behind (#195). + if (foundPage >= 0) { - var buf = new byte[PageSize]; - ReadPage(chainPageId, null, buf); - int thisIdx = loadedPages.Count; - loadedPages.Add((chainPageId, buf)); - - var hdr = SlottedPageHeader.ReadFrom(buf); - if (hdr.PageType != PageType.Collection) - break; - - // Locate existing slot for this collection name - if (foundSlotIndex == -1) + var (pageId, buffer) = chain[foundPage]; + var slot = ReadSlot(buffer, foundSlot); + if (newData.Length <= slot.Length) { - for (ushort i = 0; i < hdr.SlotCount; i++) - { - var so = SlottedPageHeader.Size + (i * SlotEntry.Size); - var slot = SlotEntry.ReadFrom(buf.AsSpan(so)); - if ((slot.Flags & SlotFlags.Deleted) != 0) continue; - - try - { - using var ms = new MemoryStream(buf, slot.Offset, slot.Length, false); - using var rdr = new BinaryReader(ms); - var existingName = rdr.ReadString(); - if (string.Equals(existingName, metadata.Name, StringComparison.OrdinalIgnoreCase)) - { - foundOnPageIdx = thisIdx; - foundSlotIndex = i; - break; - } - } - catch { } - } + newData.CopyTo(buffer.AsSpan(slot.Offset)); + slot.Length = (ushort)newData.Length; + WriteSlot(buffer, foundSlot, slot); + WritePageImmediate(pageId, buffer); + + // Shrink a chain grown by earlier releases (or by records that grew) once a page can go. + CompactMetadataChain(chain, onlyIfShorter: true); + return; } + } - // First page with enough room for the new entry - if (bestWriteIdx == -1 && hdr.AvailableFreeSpace >= newData.Length + SlotEntry.Size) - bestWriteIdx = thisIdx; + // ── 2. Larger or new: reclaim dead space first, so the record goes into free space in the + // chain before a page is appended ──────────────────────────────────────────────── + chain = CompactMetadataChain(chain, onlyIfShorter: false); + (foundPage, foundSlot) = FindMetadataRecord(chain, metadata.Name); - chainPageId = hdr.NextOverflowPage; + // ── 3. Write the new record, then delete the old one: a crash in between leaves the old + // record in place rather than neither ───────────────────────────────────────────── + int target = chain.FindIndex(p => + SlottedPageHeader.ReadFrom(p.Buffer).AvailableFreeSpace >= newData.Length + SlotEntry.Size); + + if (target < 0) + { + // No room anywhere: append a page holding the record, then link it from the last page. + var newPageId = AllocatePage(); + var newPage = new byte[PageSize]; + NewMetadataPage(newPageId, 0).WriteTo(newPage); + AppendRecord(newPage, newData); + WritePageImmediate(newPageId, newPage); + + var (lastId, lastBuf) = chain[chain.Count - 1]; + var lastHdr = SlottedPageHeader.ReadFrom(lastBuf); + lastHdr.NextOverflowPage = newPageId; + lastHdr.WriteTo(lastBuf); + WritePageImmediate(lastId, lastBuf); + } + else + { + AppendRecord(chain[target].Buffer, newData); + if (target == foundPage) + { + // Same page: one write covers both the new record and the deletion. + MarkSlotDeleted(chain[foundPage].Buffer, foundSlot); + foundPage = -1; + } + WritePageImmediate(chain[target].PageId, chain[target].Buffer); } - // ── Pass 2: mark old entry as Deleted (in its in-memory buffer) ── - if (foundSlotIndex >= 0) + if (foundPage >= 0) { - var (_, fbuf) = loadedPages[foundOnPageIdx]; - var so = SlottedPageHeader.Size + (foundSlotIndex * SlotEntry.Size); - var slot = SlotEntry.ReadFrom(fbuf.AsSpan(so)); - slot.Flags |= SlotFlags.Deleted; - slot.WriteTo(fbuf.AsSpan(so)); + MarkSlotDeleted(chain[foundPage].Buffer, foundSlot); + WritePageImmediate(chain[foundPage].PageId, chain[foundPage].Buffer); } + } // end try + finally { _metadataLock.Release(); } + } - // ── Pass 3: resolve write-target page ── - byte[] targetBuf; - uint targetPageId; + // ── Collection metadata chain (page 1 and its overflow pages) ──────────── - if (bestWriteIdx >= 0) + /// Reads the Collection pages of the metadata chain, starting at page 1. + private List<(uint PageId, byte[] Buffer)> LoadMetadataChain() + { + var chain = new List<(uint PageId, byte[] Buffer)>(); + uint pageId = 1; + while (pageId != 0) { - (targetPageId, targetBuf) = loadedPages[bestWriteIdx]; + var buffer = new byte[PageSize]; + ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + if (header.PageType != PageType.Collection) + break; + chain.Add((pageId, buffer)); + pageId = header.NextOverflowPage; } - else + return chain; + } + + /// Finds the first live record of the named collection, or (-1, -1). + private static (int Page, int Slot) FindMetadataRecord(List<(uint PageId, byte[] Buffer)> chain, string name) + { + for (int p = 0; p < chain.Count; p++) { - // No existing page has room → allocate a new Collection overflow page - targetPageId = AllocatePage(); - targetBuf = new byte[PageSize]; - var newHdr = new SlottedPageHeader + var buffer = chain[p].Buffer; + var header = SlottedPageHeader.ReadFrom(buffer); + for (int i = 0; i < header.SlotCount; i++) { - PageId = targetPageId, - PageType = PageType.Collection, - SlotCount = 0, - FreeSpaceStart = SlottedPageHeader.Size, - FreeSpaceEnd = (ushort)PageSize, - NextOverflowPage = 0, - TransactionId = 0 - }; - newHdr.WriteTo(targetBuf); + var slot = ReadSlot(buffer, i); + if ((slot.Flags & SlotFlags.Deleted) != 0) continue; + if (string.Equals(TryReadRecordName(buffer, slot), name, StringComparison.OrdinalIgnoreCase)) + return (p, i); + } + } + return (-1, -1); + } - // Append new page to the end of the chain - var (lastPid, lastBuf) = loadedPages[loadedPages.Count - 1]; - var lastHdr = SlottedPageHeader.ReadFrom(lastBuf); - lastHdr.NextOverflowPage = targetPageId; - lastHdr.WriteTo(lastBuf); - WritePageImmediate(lastPid, lastBuf); + /// + /// Packs the live records of the chain into as few pages as possible, starting at page 1 and in + /// their current order, and frees the pages no longer needed. Deleted records, bytes left behind + /// by records overwritten with shorter ones, and duplicates of a record (kept: the first, as + /// lookups do) are dropped. + /// + /// + /// Only rewrite when that frees at least one page; otherwise whenever there is anything to drop. + /// + /// The chain as it is afterwards. + /// + /// Pages are rewritten front to back and every record lands on its own page or an earlier one, + /// so all records are on some page after each write. A crash part-way can leave a record on + /// two pages; the copies are identical, readers take the first, and the next compaction drops + /// the other. + /// + private List<(uint PageId, byte[] Buffer)> CompactMetadataChain(List<(uint PageId, byte[] Buffer)> chain, bool onlyIfShorter) + { + var records = new List(); + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + bool anythingToDrop = false; - // Flush the deletion page if it differs from lastPage - if (foundSlotIndex >= 0 && foundOnPageIdx != loadedPages.Count - 1) + foreach (var (_, buffer) in chain) + { + var header = SlottedPageHeader.ReadFrom(buffer); + int liveBytes = 0; + for (int i = 0; i < header.SlotCount; i++) { - var (fpid, fbuf) = loadedPages[foundOnPageIdx]; - WritePageImmediate(fpid, fbuf); - } + var slot = ReadSlot(buffer, i); + if ((slot.Flags & SlotFlags.Deleted) != 0) { anythingToDrop = true; continue; } + + var name = TryReadRecordName(buffer, slot); + if (name != null && !names.Add(name)) { anythingToDrop = true; continue; } - loadedPages.Add((targetPageId, targetBuf)); + records.Add(buffer.AsSpan(slot.Offset, slot.Length).ToArray()); + liveBytes += slot.Length; + } + if (PageSize - header.FreeSpaceEnd > liveBytes) anythingToDrop = true; } - // If deletion page != target page, flush it now - if (foundSlotIndex >= 0) + // Greedy packing in order needs no more pages than any in-order layout, the current one included. + var pages = new List> { new() }; + int room = PageSize - SlottedPageHeader.Size; + foreach (var record in records) { - var (fpid, fbuf) = loadedPages[foundOnPageIdx]; - if (fpid != targetPageId) - WritePageImmediate(fpid, fbuf); + int need = record.Length + SlotEntry.Size; + if (need > room) + { + pages.Add(new List()); + room = PageSize - SlottedPageHeader.Size; + } + pages[pages.Count - 1].Add(record); + room -= need; } - // ── Pass 4: insert new slot into targetBuf ── - var th = SlottedPageHeader.ReadFrom(targetBuf); + if (pages.Count >= chain.Count && (onlyIfShorter || !anythingToDrop)) + return chain; - // Defensive check (should not trigger given the logic above) - if (th.AvailableFreeSpace < newData.Length + SlotEntry.Size) - throw new InvalidOperationException( - $"Not enough space in Collection page {targetPageId} to save collection metadata."); + var result = new List<(uint PageId, byte[] Buffer)>(pages.Count); + for (int p = 0; p < pages.Count; p++) + { + var pageId = chain[p].PageId; + var next = p + 1 < pages.Count ? chain[p + 1].PageId : 0u; + var buffer = new byte[PageSize]; + NewMetadataPage(pageId, next).WriteTo(buffer); + foreach (var record in pages[p]) + AppendRecord(buffer, record); + WritePageImmediate(pageId, buffer); + result.Add((pageId, buffer)); + } - int docOff = th.FreeSpaceEnd - newData.Length; - newData.CopyTo(targetBuf.AsSpan(docOff)); + // Page 1 is always kept; the pages after the new end of the chain are no longer reachable. + for (int p = pages.Count; p < chain.Count; p++) + FreePage(chain[p].PageId); - int newSlotOff = SlottedPageHeader.Size + (th.SlotCount * SlotEntry.Size); - var newSlot = new SlotEntry + return result; + } + + private SlottedPageHeader NewMetadataPage(uint pageId, uint nextPageId) => new() + { + PageId = pageId, + PageType = PageType.Collection, + SlotCount = 0, + FreeSpaceStart = SlottedPageHeader.Size, + FreeSpaceEnd = (ushort)PageSize, + NextOverflowPage = nextPageId, + TransactionId = 0 + }; + + /// Adds as a new slot; the caller checked that it fits. + private static void AppendRecord(byte[] page, byte[] record) + { + var header = SlottedPageHeader.ReadFrom(page); + int offset = header.FreeSpaceEnd - record.Length; + record.CopyTo(page.AsSpan(offset)); + WriteSlot(page, header.SlotCount, new SlotEntry { - Offset = (ushort)docOff, - Length = (ushort)newData.Length, + Offset = (ushort)offset, + Length = (ushort)record.Length, Flags = SlotFlags.None - }; - newSlot.WriteTo(targetBuf.AsSpan(newSlotOff)); + }); + header.SlotCount++; + header.FreeSpaceEnd = (ushort)offset; + header.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + (header.SlotCount * SlotEntry.Size)); + header.WriteTo(page); + } - th.SlotCount++; - th.FreeSpaceEnd = (ushort)docOff; - th.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + (th.SlotCount * SlotEntry.Size)); - th.WriteTo(targetBuf); + private static void MarkSlotDeleted(byte[] page, int slotIndex) + { + var slot = ReadSlot(page, slotIndex); + slot.Flags |= SlotFlags.Deleted; + WriteSlot(page, slotIndex, slot); + } - WritePageImmediate(targetPageId, targetBuf); - } // end try - finally { _metadataLock.Release(); } + private static SlotEntry ReadSlot(byte[] page, int slotIndex) + => SlotEntry.ReadFrom(page.AsSpan(SlottedPageHeader.Size + (slotIndex * SlotEntry.Size))); + + private static void WriteSlot(byte[] page, int slotIndex, SlotEntry slot) + => slot.WriteTo(page.AsSpan(SlottedPageHeader.Size + (slotIndex * SlotEntry.Size))); + + /// The collection name a record starts with, or null if the record is malformed. + private static string? TryReadRecordName(byte[] page, SlotEntry slot) + { + try + { + using var ms = new MemoryStream(page, slot.Offset, slot.Length, false); + using var reader = new BinaryReader(ms); + return reader.ReadString(); + } + catch + { + return null; + } } /// @@ -544,10 +649,11 @@ public void DeleteCollectionMetadata(string name) if (!string.Equals(collName, name, StringComparison.OrdinalIgnoreCase)) continue; + // Keep going: a crash while the chain was compacted can leave a second, + // identical copy further on (see CompactMetadataChain). slot.Flags |= SlotFlags.Deleted; slot.WriteTo(buffer.AsSpan(slotOffset)); WritePageImmediate(pageId, buffer); - return; } catch { } } @@ -561,6 +667,9 @@ public void DeleteCollectionMetadata(string name) public IReadOnlyList GetAllCollectionsMetadata() { var result = new List(); + // One entry per collection: the first copy, as GetCollectionMetadata returns it (a crash + // while the chain was compacted can leave a second one; see CompactMetadataChain). + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); var buffer = new byte[PageSize]; uint pageId = 1; @@ -584,7 +693,9 @@ public IReadOnlyList GetAllCollectionsMetadata() using var reader = new BinaryReader(ms); var collName = reader.ReadString(); + if (seen.Contains(collName)) continue; result.Add(ReadCollectionMetadata(collName, reader)); + seen.Add(collName); } catch { diff --git a/tests/BLite.Tests/MetadataChainTests.cs b/tests/BLite.Tests/MetadataChainTests.cs new file mode 100644 index 0000000..d4f56da --- /dev/null +++ b/tests/BLite.Tests/MetadataChainTests.cs @@ -0,0 +1,363 @@ +using BLite.Core; +using BLite.Core.Retention; +using BLite.Core.Storage; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// The collection metadata chain (page 1 and its overflow pages) must not grow with every save, +/// and a chain grown by older releases shrinks again (#195). +/// +public class MetadataChainTests : IDisposable +{ + private readonly string _dbPath; + + public MetadataChainTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"blite_metadata_chain_{Guid.NewGuid():N}.db"); + } + + public void Dispose() + { + foreach (var path in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + { + try { if (File.Exists(path)) File.Delete(path); } catch { } + } + } + + private sealed record ChainInfo(List Pages, int Live, int Deleted, List LiveLengths) + { + /// Pages the live records need when packed in order. + public int MinimalPages(int pageSize) + { + int pages = 1, room = pageSize - SlottedPageHeader.Size; + foreach (var length in LiveLengths) + { + if (length + SlotEntry.Size > room) { pages++; room = pageSize - SlottedPageHeader.Size; } + room -= length + SlotEntry.Size; + } + return pages; + } + } + + private static ChainInfo Inspect(StorageEngine storage) + { + var pages = new List(); + var lengths = new List(); + int live = 0, deleted = 0; + var buffer = new byte[storage.PageSize]; + uint pageId = 1; + while (pageId != 0) + { + storage.ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + if (header.PageType != PageType.Collection) break; + pages.Add(pageId); + for (int i = 0; i < header.SlotCount; i++) + { + var slot = SlotEntry.ReadFrom(buffer.AsSpan(SlottedPageHeader.Size + i * SlotEntry.Size)); + if ((slot.Flags & SlotFlags.Deleted) != 0) deleted++; + else { live++; lengths.Add(slot.Length); } + } + pageId = header.NextOverflowPage; + } + return new ChainInfo(pages, live, deleted, lengths); + } + + private async Task SeedAsync(params string[] collections) + { + using var engine = new BLiteEngine(_dbPath); + foreach (var name in collections) + await engine.InsertAsync(name, engine.CreateDocument(["name"], b => b.AddString("name", name + "-1"))); + } + + private static string[] Names(int count) => Enumerable.Range(0, count).Select(i => $"c{i}").ToArray(); + + // ── No growth ──────────────────────────────────────────────────────────── + + [Fact] + public async Task SequenceSaves_RewriteTheRecordInPlace() + { + await SeedAsync(Names(8)); + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + var before = Inspect(storage); + for (int i = 1; i <= 5000; i++) + Assert.Equal(i, storage.GetNextSequence("c3")); + + var after = Inspect(storage); + Assert.Equal(before.Pages, after.Pages); + Assert.Equal(8, after.Live); + Assert.Equal(0, after.Deleted); + } + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + Assert.Equal(5001, storage.GetNextSequence("c3")); + } + + [Fact] + public async Task TypedInsertsWithGeneratedIntIds_KeepTheChainAtOnePage() + { + using (var db = new TestDbContext(_dbPath)) + { + for (int i = 0; i < 2000; i++) + await db.Products.InsertAsync(new Product { Title = $"p{i}", Price = i }); + } + + using (var db = new TestDbContext(_dbPath)) + { + var ids = db.Products.AsQueryable().Select(p => p.Id).ToList().OrderBy(id => id).ToList(); + Assert.Equal(Enumerable.Range(1, 2000), ids); + } + + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + var chain = Inspect(storage); + Assert.Single(chain.Pages); + Assert.True(chain.Deleted <= 1); // at most the record the last growing save replaced + } + + [Fact] + public async Task RecordsThatShrinkAndGrow_ReuseTheSpaceTheyLeave() + { + await SeedAsync(Names(8)); + + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + var pagesBefore = Inspect(storage).Pages; + for (int i = 0; i < 500; i++) + { + var meta = storage.GetCollectionMetadata("c5")!; + meta.GeneralRetentionPolicy = new RetentionPolicy + { + MaxAgeMs = i, + TimestampField = new string('t', i % 2 == 0 ? 400 : 10), // alternately grows and shrinks + }; + storage.SaveCollectionMetadata(meta); + } + + var chain = Inspect(storage); + Assert.Equal(pagesBefore, chain.Pages); + Assert.Equal(8, chain.Live); + var last = storage.GetCollectionMetadata("c5")!; + Assert.Equal(499, last.GeneralRetentionPolicy!.MaxAgeMs); + Assert.Equal(new string('t', 10), last.GeneralRetentionPolicy.TimestampField); + } + + // ── Chains that span pages ─────────────────────────────────────────────── + + [Fact] + public async Task ChainOfSeveralPages_GrowsOnlyWithItsLiveRecords_AndShrinksWhenTheyGo() + { + // Records of ~1.2 KB: 40 collections need several 16 KB pages. + var names = Names(40); + await SeedAsync(names); + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + foreach (var name in names) + { + var meta = storage.GetCollectionMetadata(name)!; + meta.GeneralRetentionPolicy = new RetentionPolicy { TimestampField = new string('x', 1100) }; + storage.SaveCollectionMetadata(meta); + } + var grown = Inspect(storage); + Assert.True(grown.Pages.Count >= 3); + Assert.Equal(grown.MinimalPages(storage.PageSize), grown.Pages.Count); + + // Rewrite every record many times: the chain keeps its length. + for (int round = 0; round < 20; round++) + foreach (var name in names) + storage.GetNextSequence(name); + var full = Inspect(storage); + Assert.Equal(grown.Pages, full.Pages); + Assert.Equal(40, full.Live); + + // Drop most collections: the next save packs the rest into page 1 and frees the others. + foreach (var name in names.Skip(10)) + storage.DeleteCollectionMetadata(name); + storage.GetNextSequence("c0"); + + var shrunk = Inspect(storage); + Assert.Equal(new uint[] { 1 }, shrunk.Pages); + Assert.Equal(10, shrunk.Live); + Assert.Equal(0, shrunk.Deleted); + + // The freed pages are reused before the file grows. + var freed = full.Pages.Skip(1).ToHashSet(); + Assert.Contains(storage.AllocatePage(), freed); + } + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + var all = storage.GetAllCollectionsMetadata(); + Assert.Equal(names.Take(10), all.Select(m => m.Name).OrderBy(n => int.Parse(n[1..]))); + Assert.All(all, m => Assert.Equal(new string('x', 1100), m.GeneralRetentionPolicy!.TimestampField)); + Assert.Equal(20, all.Single(m => m.Name == "c1").SequenceValue); + } + + using (var engine = new BLiteEngine(_dbPath)) + Assert.Single(await engine.GetOrCreateCollection("c3").FindAllAsync().ToListAsync()); + } + + // ── Chains grown by older releases ─────────────────────────────────────── + + /// + /// Saves the record of times the way + /// releases before #195 did: the old slot is marked deleted and a copy is appended to the first + /// page with room, or to a new page linked at the end. + /// + private static void GrowLikeOlderReleases(StorageEngine storage, string collection, int saves) + { + for (int n = 0; n < saves; n++) + { + var chain = new List<(uint Id, byte[] Buffer)>(); + byte[]? record = null; + uint pageId = 1; + while (pageId != 0) + { + var buffer = new byte[storage.PageSize]; + storage.ReadPage(pageId, null, buffer); + var header = SlottedPageHeader.ReadFrom(buffer); + chain.Add((pageId, buffer)); + for (int i = 0; i < header.SlotCount && record == null; i++) + { + var offset = SlottedPageHeader.Size + i * SlotEntry.Size; + var slot = SlotEntry.ReadFrom(buffer.AsSpan(offset)); + if ((slot.Flags & SlotFlags.Deleted) != 0) continue; + using var reader = new BinaryReader(new MemoryStream(buffer, slot.Offset, slot.Length, false)); + if (reader.ReadString() != collection) continue; + record = buffer.AsSpan(slot.Offset, slot.Length).ToArray(); + slot.Flags |= SlotFlags.Deleted; + slot.WriteTo(buffer.AsSpan(offset)); + storage.WritePageImmediate(pageId, buffer); + } + pageId = header.NextOverflowPage; + } + + var target = chain.FirstOrDefault(p => + SlottedPageHeader.ReadFrom(p.Buffer).AvailableFreeSpace >= record!.Length + SlotEntry.Size); + if (target.Buffer == null) + { + var id = storage.AllocatePage(); + var page = new byte[storage.PageSize]; + new SlottedPageHeader + { + PageId = id, PageType = PageType.Collection, FreeSpaceStart = SlottedPageHeader.Size, + FreeSpaceEnd = (ushort)storage.PageSize + }.WriteTo(page); + var last = chain[^1]; + var lastHeader = SlottedPageHeader.ReadFrom(last.Buffer); + lastHeader.NextOverflowPage = id; + lastHeader.WriteTo(last.Buffer); + storage.WritePageImmediate(last.Id, last.Buffer); + target = (id, page); + } + + var h = SlottedPageHeader.ReadFrom(target.Buffer); + var dataOffset = h.FreeSpaceEnd - record!.Length; + record.CopyTo(target.Buffer.AsSpan(dataOffset)); + new SlotEntry { Offset = (ushort)dataOffset, Length = (ushort)record.Length } + .WriteTo(target.Buffer.AsSpan(SlottedPageHeader.Size + h.SlotCount * SlotEntry.Size)); + h.SlotCount++; + h.FreeSpaceEnd = (ushort)dataOffset; + h.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + h.SlotCount * SlotEntry.Size); + h.WriteTo(target.Buffer); + storage.WritePageImmediate(target.Id, target.Buffer); + } + } + + [Fact] + public async Task ChainGrownByOlderReleases_ShrinksBackOnTheNextSave() + { + await SeedAsync(Names(8)); + uint[] grownPages; + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + GrowLikeOlderReleases(storage, "c3", 3000); + var grown = Inspect(storage); + Assert.True(grown.Pages.Count >= 7, $"expected a grown chain, got {grown.Pages.Count} pages"); + Assert.Equal(3000, grown.Deleted); + grownPages = grown.Pages.ToArray(); + } + + using (var engine = new BLiteEngine(_dbPath)) + { + // The collection reads as before; the next metadata save (an index here) repairs the chain. + Assert.Single(await engine.GetOrCreateCollection("c3").FindAllAsync().ToListAsync()); + await engine.GetOrCreateCollection("c4").CreateIndexAsync("name"); + } + + using (var storage = new StorageEngine(_dbPath, PageFileConfig.Default)) + { + var healed = Inspect(storage); + Assert.Equal(new uint[] { 1 }, healed.Pages); + Assert.Equal(8, healed.Live); + Assert.True(healed.Deleted <= 1); // the record c4 replaced, until the next compaction + Assert.Equal(Names(8), storage.GetAllCollectionsMetadata().Select(m => m.Name).OrderBy(n => n)); + Assert.Single(storage.GetCollectionMetadata("c4")!.Indexes); + Assert.Contains(storage.AllocatePage(), grownPages.Skip(1)); + } + } + + // ── Duplicates a crash during compaction can leave ─────────────────────── + + [Fact] + public async Task DuplicateRecord_IsListedOnce_DeletedWithTheOriginal_AndDroppedByTheNextCompaction() + { + await SeedAsync(Names(3)); + using var storage = new StorageEngine(_dbPath, PageFileConfig.Default); + + // A second, identical copy of c1's record on a page further on, as a crash part-way + // through a compaction leaves it. + var page1 = new byte[storage.PageSize]; + storage.ReadPage(1, null, page1); + byte[]? record = null; + var header1 = SlottedPageHeader.ReadFrom(page1); + for (int i = 0; i < header1.SlotCount; i++) + { + var slot = SlotEntry.ReadFrom(page1.AsSpan(SlottedPageHeader.Size + i * SlotEntry.Size)); + using var reader = new BinaryReader(new MemoryStream(page1, slot.Offset, slot.Length, false)); + if (reader.ReadString() == "c1") record = page1.AsSpan(slot.Offset, slot.Length).ToArray(); + } + var extraId = storage.AllocatePage(); + var extra = new byte[storage.PageSize]; + var extraHeader = new SlottedPageHeader + { + PageId = extraId, PageType = PageType.Collection, SlotCount = 1, + FreeSpaceStart = SlottedPageHeader.Size + SlotEntry.Size, + FreeSpaceEnd = (ushort)(storage.PageSize - record!.Length) + }; + extraHeader.WriteTo(extra); + record.CopyTo(extra.AsSpan(extraHeader.FreeSpaceEnd)); + new SlotEntry { Offset = extraHeader.FreeSpaceEnd, Length = (ushort)record.Length } + .WriteTo(extra.AsSpan(SlottedPageHeader.Size)); + storage.WritePageImmediate(extraId, extra); + header1.NextOverflowPage = extraId; + header1.WriteTo(page1); + storage.WritePageImmediate(1, page1); + + Assert.Equal(3, storage.GetAllCollectionsMetadata().Count); + + // The next save drops the copy and the page that held it. + storage.GetNextSequence("c2"); + var compacted = Inspect(storage); + Assert.Equal(new uint[] { 1 }, compacted.Pages); + Assert.Equal(3, compacted.Live); + + // With a copy back, a delete removes both. + var againId = storage.AllocatePage(); + extraHeader.PageId = againId; + extraHeader.WriteTo(extra); + storage.WritePageImmediate(againId, extra); + storage.ReadPage(1, null, page1); + header1 = SlottedPageHeader.ReadFrom(page1); + header1.NextOverflowPage = againId; + header1.WriteTo(page1); + storage.WritePageImmediate(1, page1); + Assert.Equal(3, storage.GetAllCollectionsMetadata().Count); + + storage.DeleteCollectionMetadata("c1"); + Assert.Null(storage.GetCollectionMetadata("c1")); + Assert.Equal(new[] { "c0", "c2" }, storage.GetAllCollectionsMetadata().Select(m => m.Name).OrderBy(n => n)); + } +}