feat(storage): per-collection required features, refused by versions that lack them (#187)

src/BLite.Core/BLiteEngine.cs

@@ -356,10 +356,20 @@ public IReadOnlyList<string> ListCollections()
         // registered in the in-memory dictionary.  DynamicCollection reads its
         // own metadata from storage, so the idType argument is irrelevant for
         // existing collections (only matters when creating brand-new ones).
+        // A collection that requires features this version cannot maintain is listed but
+        // not opened: opening it throws, and it must not make the whole listing fail.
+        var unopened = new List<string>();
         foreach (var meta in _storage.GetAllCollectionsMetadata())
+        {
+            if (!_collections.ContainsKey(meta.Name) && !CollectionFeatures.CanOpen(meta, _storage.IsReadOnly))
+            {
+                unopened.Add(meta.Name);
+                continue;
+            }
             _collections.GetOrAdd(meta.Name, n => new DynamicCollection(_storage, this, n, BsonIdType.ObjectId, _freeSpaceIndexes.GetIndex()));
+        }
 
-        return _collections.Keys.ToList();
+        return _collections.Keys.Concat(unopened).ToList();
     }
 
     /// <summary>
@@ -532,18 +542,22 @@ public async Task VacuumAsync(VacuumOptions? options = null, CancellationToken c
         {
             // Ensure the collection is loaded from disk (it may exist on disk but not yet
             // in the in-memory map) before vacuum; silently skip if it doesn't exist at all.
+            // A collection ListCollections could not open still exists: opening it below
+            // reports the features this version lacks.
             ListCollections(); // warms up _collections from the on-disk catalog
-            if (!_collections.ContainsKey(options.CollectionName))
+            if (!_collections.ContainsKey(options.CollectionName)
+                && _storage.GetCollectionMetadata(options.CollectionName) == null)
                 return;
 
             targets = new[] { GetOrCreateCollection(options.CollectionName) };
         }
         else
         {
-            // Warm up: load any on-disk collections not yet in the in-memory map, then
-            // snapshot the full set to avoid mutation during iteration.
-            ListCollections();
-            targets = _collections.Values.ToArray();
+            // Open every collection, including on-disk ones not yet in the in-memory map, and
+            // snapshot the set to avoid mutation during iteration. A collection that requires
+            // features this version lacks throws here, before anything is vacuumed: skipping it
+            // would leave its deleted documents unerased.
+            targets = ListCollections().Select(name => GetOrCreateCollection(name)).ToArray();
         }
 
         foreach (var collection in targets)

src/BLite.Core/DocumentDbContext.cs

@@ -348,6 +348,10 @@ private void DropOrphanCollections()
         {
             if (registeredNames.Contains(metadata.Name)) continue;
 
+            // Created by a newer version with features this one cannot maintain: dropping it would
+            // destroy that version's data (and leak the pages this one cannot find), so leave it.
+            if (!CollectionFeatures.CanWrite(metadata)) continue;
+
             // Orphan — free its pages, delete its metadata, and remove any per-collection file.
             _storage.FreeCollectionPagesCore(metadata.Name, _freeSpaceIndexes.InvalidatePages);
             _storage.DeleteCollectionMetadata(metadata.Name);

src/BLite.Core/DynamicCollection.cs

@@ -115,6 +115,10 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction
 
         if (metadata != null)
         {
+            // Refuse a collection that requires features this version cannot maintain, before
+            // anything below writes to it.
+            CollectionFeatures.EnsureCanOpen(metadata, _storage.IsReadOnly);
+
             primaryRootPageId = metadata.PrimaryRootPageId;
             _isTimeSeries = metadata.IsTimeSeries;
 

src/BLite.Core/Indexing/CollectionIndexManager.cs

@@ -40,6 +40,10 @@ public CollectionIndexManager(StorageEngine storage, IDocumentMapper<TId, T> map
 
         // Load existing metadata via storage
         _metadata = _storage.GetCollectionMetadata(_collectionName) ?? new CollectionMetadata { Name = _collectionName };
+
+        // Refuse a collection that requires features this version cannot maintain, before anything
+        // below writes to it.
+        CollectionFeatures.EnsureCanOpen(_metadata, _storage.IsReadOnly);
 
         // Initialize indexes from metadata. Root page ids are fixed for the lifetime of an index
         // (a root split rewrites the root page in place), so they only need persisting when the

src/BLite.Core/Storage/CollectionFeatures.cs

@@ -0,0 +1,137 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using BLite.Core.Indexing;
+
+namespace BLite.Core.Storage;
+
+/// <summary>What a BLite version must support a <see cref="RequiredFeature"/> for.</summary>
+internal enum FeatureRequirement : byte
+{
+    /// <summary>
+    /// Needed to open the collection at all, because a version without the feature would read it
+    /// wrongly (e.g. serve queries from an index it misinterprets). Any value other than
+    /// <see cref="Write"/>, including ones added later, is treated this way.
+    /// </summary>
+    Open = 1,
+
+    /// <summary>
+    /// Needed only to modify the collection: reading is correct without the feature, but writes
+    /// would leave the data it depends on out of date. A version without it may still open the
+    /// collection when the database is opened read-only.
+    /// </summary>
+    Write = 2,
+}
+
+/// <summary>
+/// A capability a collection depends on, recorded in its <see cref="CollectionMetadata"/> by the
+/// version that started using it.
+/// </summary>
+/// <param name="Id">Stable identifier that versions implementing the feature recognise.</param>
+/// <param name="Requirement">Whether the feature is needed to open the collection or only to write to it.</param>
+/// <param name="Description">
+/// Human-readable name for error messages, including the first version that supports it, e.g.
+/// <c>"full-text indexes (BLite 5.3 or later)"</c>. Stored with the feature so that versions which
+/// do not know it can still tell the user what is missing.
+/// </param>
+internal sealed record RequiredFeature(string Id, FeatureRequirement Requirement, string Description);
+
+/// <summary>
+/// Per-collection required features (#187). A collection that uses something older versions cannot
+/// maintain (a new index type, a new document encoding, ...) records it in
+/// <see cref="CollectionMetadata.RequiredFeatures"/>; a version that finds a feature it does not
+/// know refuses to open or modify the collection instead of silently damaging it.
+/// </summary>
+/// <remarks>
+/// Only versions that contain this check can refuse; releases before it ignore the record and may
+/// also drop it when they rewrite the collection's metadata. An index of a type newer than any this
+/// version knows is refused the same way, so a new index type stays protected even after such a
+/// release dropped the record.
+/// </remarks>
+internal static class CollectionFeatures
+{
+    // Features this version implements. Empty until a feature that needs protecting ships.
+    private static readonly HashSet<string> s_known = new(StringComparer.Ordinal);
+
+    // Highest IndexType this version knows; larger values come from newer versions. 0 is accepted:
+    // older releases could persist an index whose type was never set.
+    private static readonly byte s_maxIndexType =
+#if NET5_0_OR_GREATER
+        Enum.GetValues<IndexType>().Max(t => (byte)t);
+#else
+        Enum.GetValues(typeof(IndexType)).Cast<IndexType>().Max(t => (byte)t);
+#endif
+
+    private static readonly string s_version =
+        typeof(CollectionFeatures).Assembly.GetName().Version?.ToString(3) ?? "0.0.0";
+
+    /// <summary>Whether this version implements the feature with the given id.</summary>
+    internal static bool IsKnown(string id) => s_known.Contains(id);
+
+    /// <summary>
+    /// Whether this version may open the collection: every feature it requires is known, except
+    /// <see cref="FeatureRequirement.Write"/> features when the database is open read-only.
+    /// </summary>
+    internal static bool CanOpen(CollectionMetadata metadata, bool readOnly)
+        => Unsupported(metadata, readOnly).Count == 0;
+
+    /// <summary>Whether this version may modify the collection: every feature it requires is known.</summary>
+    internal static bool CanWrite(CollectionMetadata metadata)
+        => Unsupported(metadata, readOnly: false).Count == 0;
+
+    /// <summary>Throws unless <see cref="CanOpen"/> allows opening the collection.</summary>
+    /// <exception cref="NotSupportedException">The collection requires features this version does not support.</exception>
+    internal static void EnsureCanOpen(CollectionMetadata metadata, bool readOnly)
+    {
+        var unsupported = Unsupported(metadata, readOnly);
+        if (unsupported.Count == 0) return;
+
+        throw unsupported.All(f => f.Requirement == FeatureRequirement.Write)
+            ? new NotSupportedException(
+                $"Collection '{metadata.Name}' requires features that BLite {s_version} cannot maintain: " +
+                $"{Describe(unsupported)}. Open the database read-only to read it, or use a BLite version " +
+                "that supports these features.")
+            : new NotSupportedException(
+                $"Collection '{metadata.Name}' requires features that BLite {s_version} does not support: " +
+                $"{Describe(unsupported)}. Use a BLite version that supports these features.");
+    }
+
+    /// <summary>Throws unless <see cref="CanWrite"/> allows modifying the collection.</summary>
+    /// <exception cref="NotSupportedException">The collection requires features this version does not support.</exception>
+    internal static void EnsureCanWrite(CollectionMetadata metadata)
+    {
+        var unsupported = Unsupported(metadata, readOnly: false);
+        if (unsupported.Count == 0) return;
+
+        throw new NotSupportedException(
+            $"Collection '{metadata.Name}' requires features that BLite {s_version} does not support: " +
+            $"{Describe(unsupported)}. It cannot be modified by this version; use a BLite version that " +
+            "supports these features.");
+    }
+
+    private static List<RequiredFeature> Unsupported(CollectionMetadata metadata, bool readOnly)
+    {
+        var result = new List<RequiredFeature>();
+        foreach (var feature in metadata.RequiredFeatures)
+        {
+            if (IsKnown(feature.Id)) continue;
+            if (readOnly && feature.Requirement == FeatureRequirement.Write) continue;
+            result.Add(feature);
+        }
+
+        // A version that does not know an index type cannot query or maintain the index, so it is
+        // treated as a feature needed to open the collection.
+        foreach (var index in metadata.Indexes)
+        {
+            var type = (byte)index.Type;
+            if (type > s_maxIndexType)
+                result.Add(new RequiredFeature($"index-type-{type}", FeatureRequirement.Open,
+                    $"index '{index.Name}' of type {type}, which this version does not know"));
+        }
+        return result;
+    }
+
+    private static string Describe(IEnumerable<RequiredFeature> features)
+        => string.Join(", ", features.Select(f =>
+            string.IsNullOrWhiteSpace(f.Description) ? $"'{f.Id}'" : f.Description));
+}

src/BLite.Core/Storage/StorageEngine.Collections.cs

@@ -93,6 +93,15 @@ public class CollectionMetadata
 
     // ── Generalized Retention Policy ──
     public RetentionPolicy? GeneralRetentionPolicy { get; set; }
+
+    // ── Required features ──
+    /// <summary>
+    /// Features this collection depends on that older versions cannot maintain; a version that
+    /// does not know one of them refuses to open or modify the collection (see
+    /// <see cref="CollectionFeatures"/>). Persisted as stored, including features this version does
+    /// not know, so a rewrite of the record never drops them.
+    /// </summary>
+    internal List<RequiredFeature> RequiredFeatures { get; } = new();
 }
 
 public class IndexMetadata
@@ -276,6 +285,19 @@ private static CollectionMetadata ReadCollectionMetadata(string name, BinaryRead
                     metadata.Indexes[j].KeyFormat = format;
             }
         }
+
+        // ── Required features (backward-compatible; absent = none) ────────
+        if (reader.BaseStream.Position < reader.BaseStream.Length)
+        {
+            int featureCount = reader.ReadInt32();
+            for (int j = 0; j < featureCount; j++)
+            {
+                var id          = reader.ReadString();
+                var requirement = (FeatureRequirement)reader.ReadByte();
+                var description = reader.ReadString();
+                metadata.RequiredFeatures.Add(new RequiredFeature(id, requirement, description));
+            }
+        }
         return metadata;
     }
 
@@ -363,6 +385,17 @@ public void SaveCollectionMetadata(CollectionMetadata metadata)
         foreach (var idx in metadata.Indexes)
             writer.Write(idx.KeyFormat);
 
+        // ── Required features (id, requirement, description) ──────────────────
+        // Written after every older section, so releases that stop reading at the key formats
+        // load the record unchanged. New sections go after this one.
+        writer.Write(metadata.RequiredFeatures.Count);
+        foreach (var feature in metadata.RequiredFeatures)
+        {
+            writer.Write(feature.Id);
+            writer.Write((byte)feature.Requirement);
+            writer.Write(feature.Description);
+        }
+
         var newData = stream.ToArray();
         if (newData.Length + SlotEntry.Size > PageSize - SlottedPageHeader.Size)
             throw new InvalidOperationException(
@@ -373,6 +406,15 @@ public void SaveCollectionMetadata(CollectionMetadata metadata)
             throw new InvalidOperationException("Page 1 is not a collection metadata page.");
         var (foundPage, foundSlot) = FindMetadataRecord(chain, metadata.Name);
 
+        // Never rewrite the record of a collection that requires features this version lacks
+        // (see CollectionFeatures). Checked before every write path, the in-place one included.
+        // Nothing has been written yet.
+        if (foundPage >= 0
+            && TryReadRecord(chain[foundPage].Buffer, ReadSlot(chain[foundPage].Buffer, foundSlot)) is { } stored)
+        {
+            CollectionFeatures.EnsureCanWrite(stored);
+        }
+
         // ── 1. No larger than the stored record: overwrite it where it is ─────
         // A single page write. Covers the frequent saves (sequence values, root page ids) whose
         // record keeps its size, so they no longer leave a deleted record behind (#195).
@@ -612,6 +654,21 @@ private static void WriteSlot(byte[] page, int slotIndex, SlotEntry slot)
         }
     }
 
+    /// <summary>Parses the collection record in <paramref name="slot"/>, or returns <c>null</c> if it is malformed.</summary>
+    private static CollectionMetadata? TryReadRecord(byte[] page, SlotEntry slot)
+    {
+        try
+        {
+            using var ms     = new MemoryStream(page, slot.Offset, slot.Length, false);
+            using var reader = new BinaryReader(ms);
+            return ReadCollectionMetadata(reader.ReadString(), reader);
+        }
+        catch
+        {
+            return null;
+        }
+    }
+
     /// <summary>
     /// Marks the collection's metadata slot as deleted so that it no longer
     /// appears in <see cref="GetAllCollectionsMetadata"/> or
@@ -623,6 +680,11 @@ public void DeleteCollectionMetadata(string name)
             throw new TimeoutException("Timed out acquiring metadata lock (DeleteCollectionMetadata).");
         try
         {
+        // Never drop the record of a collection that requires features this version lacks
+        // (see CollectionFeatures).
+        if (GetCollectionMetadata(name) is { } stored)
+            CollectionFeatures.EnsureCanWrite(stored);
+
         var buffer = new byte[PageSize];
         uint pageId = 1;
 

src/BLite.Core/Storage/StorageEngine.Memory.cs

@@ -323,6 +323,10 @@ internal IReadOnlyCollection<uint> FreeCollectionPagesCore(string collectionName
         var metadata = GetCollectionMetadata(collectionName);
         if (metadata == null) return Array.Empty<uint>();
 
+        // The pages of a feature this version lacks (e.g. an index type it does not know) cannot be
+        // found here; refuse instead of leaking them and dropping the rest.
+        CollectionFeatures.EnsureCanWrite(metadata);
+
         // Collect page IDs in a set first, then free them all — avoids modifying storage
         // while iterating over B-tree nodes (which also read from the same storage).
         var toFree = new HashSet<uint>();

src/BLite.Core/Storage/StorageEngine.cs

@@ -116,6 +116,14 @@ public sealed partial class StorageEngine : IDisposable
     internal LockTimeout LockTimeout => _config.LockTimeout;
     internal bool UsesSeparateCollectionFiles => _collectionFiles != null;
 
+    /// <summary>
+    /// Whether the database was opened read-only (<see cref="PageFileConfig.Access"/> is
+    /// <see cref="System.IO.MemoryMappedFiles.MemoryMappedFileAccess.Read"/>). Collections that
+    /// require features this version cannot maintain may still be opened for reading then
+    /// (see <see cref="CollectionFeatures"/>).
+    /// </summary>
+    internal bool IsReadOnly => _config.Access == System.IO.MemoryMappedFiles.MemoryMappedFileAccess.Read;
+
     /// <summary>
     /// Returns the semaphore a <see cref="Collections.DocumentCollection{TId,T}"/> must hold for
     /// write operations. In single-file mode every collection of this engine gets the same