From b17b7aa0209a5e410ef6226e5ccf79dab6cd950a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 16:50:21 +0200 Subject: [PATCH 1/6] perf(query): visit raw documents in place during BSON scans ScanAsync, CountScanAsync and the BSON-predicate path of FetchAsync copied every document out of its page (ReadRawBytesAt) before looking at it, and kept every page they had read in a dictionary until the scan ended. VisitRawDocumentsAsync hands single-page documents to the caller straight from the page buffer, reassembling only overflow documents, and keeps at most 64 page buffers, reusing them once the window is full. Documents are still checked against their primary-key entry before they are visited. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 93 +++++++++++++------ 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 19d77c22..911af037 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -535,9 +535,19 @@ private static bool TryReadIndexKeyFromReader(ref BsonSpanReader reader, BsonTyp return false; } - private async IAsyncEnumerable<(IndexEntry Entry, byte[] RawBytes)> EnumerateOwnedRawDocumentsAsync( + /// Processes the raw BSON of one document in place; returns whether to yield . + private delegate bool RawDocumentVisitor(ReadOnlySpan bson, out TResult result); + + /// + /// Visits the raw BSON of every document in primary-key order. Single-page documents are + /// handed to straight from the page buffer, without a copy; only + /// overflow documents are reassembled. Each page is read once and kept until the scan ends: + /// primary-key order is not page order (ObjectIds created in the same second are not + /// monotonic), so a bounded window of pages would read the same pages over and over. + /// + private async IAsyncEnumerable VisitRawDocumentsAsync( ulong txnId, - bool includeOverflow, + RawDocumentVisitor visit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) { var pageCache = new Dictionary(); @@ -550,27 +560,42 @@ private static bool TryReadIndexKeyFromReader(ref BsonSpanReader reader, BsonTyp { ct.ThrowIfCancellationRequested(); - if (!pageCache.TryGetValue(entry.Location.PageId, out var cachedBuffer)) + if (!pageCache.TryGetValue(entry.Location.PageId, out var pageBuffer)) { - cachedBuffer = ArrayPool.Shared.Rent(_storage.PageSize); - _storage.ReadPage(entry.Location.PageId, txnId, cachedBuffer); - pageCache[entry.Location.PageId] = cachedBuffer; + pageBuffer = ArrayPool.Shared.Rent(_storage.PageSize); + _storage.ReadPage(entry.Location.PageId, txnId, pageBuffer); + pageCache[entry.Location.PageId] = pageBuffer; } - var rawBytes = ReadRawBytesAt(entry.Location, txnId, includeOverflow, cachedBuffer); - if (rawBytes is null) continue; - if (!MatchesIndexEntryKey(rawBytes, entry.Key)) continue; - - yield return (entry, rawBytes); + if (TryVisitRawDocument(entry, txnId, pageBuffer, visit, out var result)) + yield return result; } } finally { - foreach (var buf in pageCache.Values) - ArrayPool.Shared.Return(buf); + foreach (var buf in pageCache.Values) ArrayPool.Shared.Return(buf); } } + /// + /// Applies to the document points at, if that + /// slot still holds the document with this primary key. + /// + private bool TryVisitRawDocument(IndexEntry entry, ulong txnId, byte[] pageBuffer, RawDocumentVisitor visit, out TResult result) + { + result = default!; + + var pageType = (PageType)pageBuffer[4]; + if (pageType == PageType.Free || pageType == PageType.Empty || pageType == PageType.TimeSeries) + return false; + + if (TryReadInlineRawBytes(pageBuffer, entry.Location.SlotIndex, out var inline)) + return MatchesIndexEntryKey(inline, entry.Key) && visit(inline, out result); + + var reassembled = ReadRawBytesAt(entry.Location, txnId, includeOverflow: true, pageBuffer); + return reassembled is not null && MatchesIndexEntryKey(reassembled, entry.Key) && visit(reassembled, out result); + } + /// /// Extracts a UTC-ticks timestamp from raw BSON bytes using the given field name. /// Resolves the field name against the Key Dictionary for correct C-BSON lookup, @@ -1119,10 +1144,15 @@ public async IAsyncEnumerable ScanAsync( try { - await foreach (var (_, rawBytes) in EnumerateOwnedRawDocumentsAsync(txnId, includeOverflow: true, ct).ConfigureAwait(false)) + await foreach (var document in VisitRawDocumentsAsync(txnId, (ReadOnlySpan bson, out T doc) => + { + doc = default!; + if (!predicate(new BsonSpanReader(bson, keyMap))) return false; + doc = _mapper.Deserialize(new BsonSpanReader(bson, keyMap)); + return true; + }, ct).ConfigureAwait(false)) { - if (!predicate(new BsonSpanReader(rawBytes, keyMap))) continue; - yield return _mapper.Deserialize(new BsonSpanReader(rawBytes, keyMap)); + yield return document; } } finally @@ -1284,10 +1314,13 @@ public async IAsyncEnumerable ScanAsync( var txnId = 0UL; var keyMap = _storage.GetKeyReverseMap(); - await foreach (var (_, rawBytes) in EnumerateOwnedRawDocumentsAsync(txnId, includeOverflow: true, ct).ConfigureAwait(false)) + await foreach (var result in VisitRawDocumentsAsync(txnId, (ReadOnlySpan bson, out TResult projected) => + { + projected = projector(new BsonSpanReader(bson, keyMap))!; + return projected is not null; + }, ct).ConfigureAwait(false)) { - var result = projector(new BsonSpanReader(rawBytes, keyMap)); - if (result is not null) yield return result; + yield return result; } } @@ -1310,16 +1343,20 @@ internal async Task CountScanAsync( int count = 0; var keyMap = _storage.GetKeyReverseMap(); - await foreach (var (_, rawBytes) in EnumerateOwnedRawDocumentsAsync(txnId, includeOverflow: true, ct).ConfigureAwait(false)) - { - try + await foreach (var _ in VisitRawDocumentsAsync(txnId, (ReadOnlySpan bson, out bool matched) => { - if (predicate(new BsonSpanReader(rawBytes, keyMap))) count++; - } - catch - { - // Malformed BSON — skip silently. - } + try + { + matched = predicate(new BsonSpanReader(bson, keyMap)); + } + catch + { + matched = false; // Malformed BSON — skip silently. + } + return matched; + }, ct).ConfigureAwait(false)) + { + count++; } return count; From 5b92e84f689800ca5840ea37a2cd27654ebc340d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 16:50:21 +0200 Subject: [PATCH 2/6] perf(query): compute LINQ Sum/Average/Min/Max without materialising entities An aggregate with a selector always materialised every matching entity and ran Enumerable over them. The BSON push-down meant to avoid that never ran: it cast ScanAsync's IAsyncEnumerable to IEnumerable, got null and fell through. It also had other semantics than LINQ (0 instead of an exception on empty input, unchecked int sums, Take/Skip ignored). Sum/Average/Min/Max(x => x.Property), applied straight to Where clauses over the collection, now run in DocumentCollection.TryAggregateAsync: - int/long property with a B-Tree index, and no filter or one the same index covers exactly: the values are decoded from the index keys, with no document reads; Min/Max read a single entry. - otherwise, when no index serves the filter: the field is read from the raw BSON of each document in primary-key order, with the filter as a BSON predicate. A missing field, a BSON type the mapper would not read unchanged, a null in a non-nullable property or a renamed property ([Column]/[BsonProperty]/[JsonPropertyName]) gives up, and the query runs the usual way. LinqAggregate folds the values like Enumerable: checked int/long sums, float sums in double, first of equal Min/Max values wins (-0.0/0.0, 1.0m/1.00m), NaN smallest, nullable selectors skip nulls, empty input throws or returns null as LINQ does. Index keys come in value order, so an index sum range-checks only the total: it returns the exact sum where LINQ would overflow on an intermediate running total in document order. Any other operator before the aggregate (Take, Skip, OrderBy, Select, ...) keeps the existing path. Decimal and floating-point properties never use the index: decimal keys drop trailing zeros and floating-point sums depend on the order of the values. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 285 ++++++++++++++ src/BLite.Core/Indexing/BTreeIndex.cs | 38 +- .../Indexing/CollectionSecondaryIndex.cs | 40 +- src/BLite.Core/Indexing/IndexKey.cs | 2 +- src/BLite.Core/Query/BTreeQueryProvider.cs | 185 ++------- src/BLite.Core/Query/LinqAggregate.cs | 167 +++++++++ .../LinqAggregateBenchmarks.cs | 89 +++++ tests/BLite.Shared/AggregateContexts.cs | 77 ++++ tests/BLite.Tests/LinqAggregateTests.cs | 354 ++++++++++++++++++ 9 files changed, 1070 insertions(+), 167 deletions(-) create mode 100644 src/BLite.Core/Query/LinqAggregate.cs create mode 100644 tests/BLite.Benchmark/LinqAggregateBenchmarks.cs create mode 100644 tests/BLite.Shared/AggregateContexts.cs create mode 100644 tests/BLite.Tests/LinqAggregateTests.cs diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 911af037..fd6c21fd 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -3418,6 +3418,291 @@ internal async Task CountByPredicateAsync( return count; } + /// + /// Computes Sum, Average, Min or Max of over + /// the documents matching without materialising them, with the + /// result LINQ would compute over the entities. Returns Handled = false when the query + /// must run the usual way. + /// + /// An int/long property with a B-Tree index, and no filter or one the same + /// index covers exactly, is aggregated from the index keys alone (Min/Max + /// read one entry). + /// Otherwise, when no index serves the filter, the property is read from the raw BSON + /// of each document in primary-key order, the order a scan yields the entities in. A + /// document without the field, or holding a type the entity would not be read from + /// unchanged, ends the attempt so the query runs the usual way. + /// + /// + /// A plain property access, x => x.Property. + /// The result type of the LINQ operator. + [RequiresDynamicCode("Aggregate push-down uses index optimization and Expression.Compile() which require dynamic code generation.")] + [RequiresUnreferencedCode("Aggregate push-down uses reflection to resolve members at runtime. Ensure all entity types are preserved.")] + internal Task<(bool Handled, object? Value)> TryAggregateAsync( + AggregateOperator op, + LambdaExpression selector, + LambdaExpression? whereClause, + Type resultType, + CancellationToken ct = default) + { + if (selector.Body is not MemberExpression { Member: System.Reflection.PropertyInfo property } member + || member.Expression != selector.Parameters[0] + || selector.ReturnType != property.PropertyType + || ConverterRegistry.HasConverter(property.Name)) + return Task.FromResult<(bool, object?)>(default); + + var valueType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; + bool nullable = valueType != property.PropertyType; + + if (valueType == typeof(int)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(long)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(double)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(float)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(decimal)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + return Task.FromResult<(bool, object?)>(default); + } + + [RequiresDynamicCode("Aggregate push-down uses index optimization and Expression.Compile() which require dynamic code generation.")] + [RequiresUnreferencedCode("Aggregate push-down uses reflection to resolve members at runtime. Ensure all entity types are preserved.")] + private async Task<(bool Handled, object? Value)> TryAggregateAsync( + AggregateOperator op, + System.Reflection.PropertyInfo property, + bool nullable, + LambdaExpression? whereClause, + Type resultType, + CancellationToken ct) + where TValue : struct, IComparable + { + if (LinqAggregate.ResultType(op, nullable) != resultType) + return default; + + var plan = whereClause == null ? null : Query.IndexOptimizer.TryOptimize(whereClause, GetIndexes(), ConverterRegistry); + + // Index keys hold the exact int/long value. Not for the other types: floating-point sums + // depend on the order of the values, decimal keys drop trailing zeros (1.50 → 1.5), and the + // first of equal Min/Max values wins (-0.0 vs 0.0, 1.0 vs 1.00). + if (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long)) + { + var index = _indexManager.GetAllIndexes().FirstOrDefault(i => + i.Definition.Type == IndexType.BTree + && !i.Definition.IsPrimary + && i.Definition.PropertyPaths.Length == 1 + && i.Definition.PropertyPaths[0] == property.Name); + + if (index != null) + { + if (whereClause == null) + return AggregateIndexKeys(op, nullable, index, null, null, true, true); + + if (plan != null + && plan.IndexName == index.Name + && !plan.HasResiduePredicate + && !plan.IsVectorSearch + && !plan.IsSpatialSearch + && plan.InValues == null + && plan.MinValue is not DBNull + && plan.MaxValue is not DBNull) + return AggregateIndexKeys(op, nullable, index, plan.MinValue, plan.MaxValue, plan.StartInclusive, plan.EndInclusive); + } + } + + // An index serves the filter: fetching the matching documents through it is cheaper than + // scanning them all, and keeps the order the values are aggregated in unchanged. + if (plan != null || IsRenamedInBson(property)) + return default; + + BsonReaderPredicate? predicate = null; + if (whereClause != null) + { + predicate = BsonExpressionEvaluator.TryCompile(whereClause, ConverterRegistry, _storage.GetKeyMap()); + if (predicate == null) return default; + } + + return await AggregateRawFieldAsync(op, nullable, ProjectionAnalyzer.ToBsonName(property.Name), predicate, ct).ConfigureAwait(false); + } + + /// Aggregates the values of the index entries in the given range from their keys. + private static (bool Handled, object? Value) AggregateIndexKeys( + AggregateOperator op, + bool nullable, + CollectionSecondaryIndex index, + object? min, + object? max, + bool startInclusive, + bool endInclusive) + where TValue : struct, IComparable + { + var aggregate = new LinqAggregate(op, nullable, exactTotal: true); + + if (op == AggregateOperator.Max) + { + // Keys are [value][document id]: the last entry holds the largest value. + var last = index.LastKey(min, max, startInclusive, endInclusive); + if (last is { } key) + { + if (!IndexKeyValueVisitor.TryDecode(key.Data, out var value)) return default; + aggregate.Add(value); + } + return (true, aggregate.Result()); + } + + var visitor = new IndexKeyValueVisitor(aggregate, stopAfterFirst: op == AggregateOperator.Min); + index.ScanKeys(min, max, startInclusive, endInclusive, ref visitor); + return visitor.Malformed ? default : (true, aggregate.Result()); + } + + /// + /// Feeds the value prefix of each [value][document id] key of an int or + /// long index to a . + /// + private struct IndexKeyValueVisitor : BTreeIndex.IKeyVisitor + where TValue : struct, IComparable + { + private readonly LinqAggregate _aggregate; + private readonly bool _stopAfterFirst; + + public IndexKeyValueVisitor(LinqAggregate aggregate, bool stopAfterFirst) + { + _aggregate = aggregate; + _stopAfterFirst = stopAfterFirst; + Malformed = false; + } + + /// A key did not start with an encoded value of the expected type. + public bool Malformed { get; private set; } + + public bool Visit(ReadOnlySpan key) + { + if (!TryDecode(key, out var value)) + { + Malformed = true; + return false; + } + _aggregate.Add(value); + return !_stopAfterFirst; + } + + /// Decodes the / prefix of a key. + public static bool TryDecode(ReadOnlySpan key, out TValue value) + { + value = default; + if (key.Length == 0 || key[0] != IndexKey.KeyPrefix) return false; + + if (typeof(TValue) == typeof(int)) + { + if (key.Length < 5) return false; + value = (TValue)(object)(int)(BinaryPrimitives.ReadUInt32BigEndian(key.Slice(1)) ^ 0x8000_0000u); + return true; + } + if (typeof(TValue) == typeof(long)) + { + if (key.Length < 9) return false; + value = (TValue)(object)(long)(BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)) ^ 0x8000_0000_0000_0000UL); + return true; + } + return false; + } + } + + /// + /// Whether is stored under a name other than the convention + /// follows, through the attributes the mapper + /// generator honours ([BsonProperty], [JsonPropertyName], [Column]). + /// + private static bool IsRenamedInBson(System.Reflection.PropertyInfo property) + { + foreach (var attribute in property.GetCustomAttributes(inherit: true)) + { + var name = attribute.GetType().Name; + if (name is "BsonPropertyAttribute" or "BsonProperty" + or "JsonPropertyNameAttribute" or "JsonPropertyName" + or "ColumnAttribute" or "Column") + return true; + } + return false; + } + + /// + /// Aggregates field of every document matching + /// from raw BSON, in primary-key order. Gives up (Handled = + /// false) on a document without the field, with a BSON type other than the one the mapper + /// writes for (or null for a nullable property), or that + /// fails to evaluate, since materialising the entity could then yield a different value. + /// + private async Task<(bool Handled, object? Value)> AggregateRawFieldAsync( + AggregateOperator op, + bool nullable, + string fieldName, + BsonReaderPredicate? predicate, + CancellationToken ct) + where TValue : struct, IComparable + { + if (!_storage.GetKeyMap().TryGetValue(fieldName, out var fieldId)) + return default; // never written: every document lacks it + + var expectedType = + typeof(TValue) == typeof(int) ? BsonType.Int32 : + typeof(TValue) == typeof(long) ? BsonType.Int64 : + typeof(TValue) == typeof(decimal) ? BsonType.Decimal128 : + BsonType.Double; // double and float + var keyMap = _storage.GetKeyReverseMap(); + var aggregate = new LinqAggregate(op, nullable); + + // Adds the document's value if it matches; false to give up. + bool Fold(ReadOnlySpan bson) + { + try + { + if (predicate != null && !predicate(new BsonSpanReader(bson, keyMap))) + return true; + + var reader = new BsonSpanReader(bson, keyMap); + reader.ReadDocumentSize(); + if (!reader.TrySeekToField(fieldId, out var type) && !SeekField(ref reader, fieldName, out type)) + return false; + + if (type == BsonType.Null) + return nullable; + if (type != expectedType) + return false; + + aggregate.Add(ReadValue(ref reader)); + return true; + } + catch + { + return false; + } + } + + // The scan yields only for a document that ends the attempt. + await foreach (var _ in VisitRawDocumentsAsync(0UL, (ReadOnlySpan bson, out bool giveUp) => giveUp = !Fold(bson), ct).ConfigureAwait(false)) + return default; + + return (true, aggregate.Result()); + + static bool SeekField(ref BsonSpanReader reader, string name, out BsonType type) + { + while (reader.Remaining > 1) + { + type = reader.ReadBsonType(); + if (type == BsonType.EndOfDocument) break; + if (reader.ReadElementHeader() == name) return true; + reader.SkipValue(type); + } + type = BsonType.EndOfDocument; + return false; + } + + static TV ReadValue(ref BsonSpanReader reader) + { + if (typeof(TV) == typeof(int)) return (TV)(object)reader.ReadInt32(); + if (typeof(TV) == typeof(long)) return (TV)(object)reader.ReadInt64(); + if (typeof(TV) == typeof(double)) return (TV)(object)reader.ReadDouble(); + if (typeof(TV) == typeof(float)) return (TV)(object)(float)reader.ReadDouble(); + return (TV)(object)reader.ReadDecimal128(); + } + } + /// /// Queries the collection using an expression tree predicate, enabling index optimisation. /// When the predicate references an indexed field the query engine performs a BTree lookup diff --git a/src/BLite.Core/Indexing/BTreeIndex.cs b/src/BLite.Core/Indexing/BTreeIndex.cs index e657c248..f58f8040 100644 --- a/src/BLite.Core/Indexing/BTreeIndex.cs +++ b/src/BLite.Core/Indexing/BTreeIndex.cs @@ -277,9 +277,34 @@ public IEnumerable Range(IndexKey minKey, IndexKey maxKey, IndexDire /// Equivalent to Range(min, max).Count() but with zero per-entry heap allocations. /// internal int CountRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId = null) + { + var counter = new CountingVisitor(); + ScanRange(minKey, maxKey, transactionId, ref counter); + return counter.Count; + } + + /// Receives the keys of a , in key order. + internal interface IKeyVisitor + { + /// Handles one key; returns false to stop the scan. + bool Visit(ReadOnlySpan key); + } + + private struct CountingVisitor : IKeyVisitor + { + public int Count; + public bool Visit(ReadOnlySpan key) { Count++; return true; } + } + + /// + /// Hands every leaf key in [, ] (inclusive) + /// to straight from the page buffer, in ascending order, without + /// allocating per entry. The key span is only valid during the call. + /// + internal void ScanRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId, ref TVisitor visitor) + where TVisitor : struct, IKeyVisitor { var txnId = transactionId ?? 0; - int count = 0; var pageBuffer = System.Buffers.ArrayPool.Shared.Rent(_storage.PageSize); try { @@ -313,13 +338,16 @@ internal int CountRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId = } // inRange: compare against maxKey only. - if (IndexKey.CompareRaw(entryKeySpan, maxKey.Data) <= 0) - count++; - else + if (IndexKey.CompareRaw(entryKeySpan, maxKey.Data) > 0) { stopScan = true; // past maxKey — no more matches in any leaf break; } + if (!visitor.Visit(entryKeySpan)) + { + stopScan = true; // the visitor has seen enough + break; + } dataOffset += 4 + keyLength + DocumentLocation.SerializedSize; } @@ -332,9 +360,9 @@ internal int CountRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId = { System.Buffers.ArrayPool.Shared.Return(pageBuffer); } - return count; } + /// /// Finds the first leaf entry in [minKey, maxKey] (inclusive) without allocating an /// iterator state machine. Equivalent to Range(min,max).FirstOrDefault() but with diff --git a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs index 0afe4b91..56f3e617 100644 --- a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs +++ b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs @@ -360,6 +360,44 @@ public int CountRange(object? minKey, object? maxKey, bool startInclusive, bool // [userKey][realId] are <= this boundary, so they are included. // Exclusive end: lower composite boundary [userKey][MinId] — all real entries // [userKey][realId] are > this boundary, so they are excluded. + var (actualMinKey, actualMaxKey) = CompositeRange(minKey, maxKey, startInclusive, endInclusive); + return _btreeIndex.CountRange(actualMinKey, actualMaxKey, transaction?.TransactionId); + } + + /// + /// Hands the stored keys of the entries in the range to , in ascending + /// order and without allocating per entry; bounds as for . Each key is + /// [user key][document id], so the indexed value is its prefix. + /// + internal void ScanKeys(object? minKey, object? maxKey, bool startInclusive, bool endInclusive, ref TVisitor visitor, ITransaction? transaction = null) + where TVisitor : struct, BTreeIndex.IKeyVisitor + { + if (_btreeIndex == null) return; + + var (actualMinKey, actualMaxKey) = CompositeRange(minKey, maxKey, startInclusive, endInclusive); + _btreeIndex.ScanRange(actualMinKey, actualMaxKey, transaction?.TransactionId, ref visitor); + } + + /// + /// The stored key of the last entry in the range (bounds as for ), read + /// from the end of the range; null when the range is empty. + /// + internal IndexKey? LastKey(object? minKey, object? maxKey, bool startInclusive, bool endInclusive, ITransaction? transaction = null) + { + if (_btreeIndex == null) return null; + + var (actualMinKey, actualMaxKey) = CompositeRange(minKey, maxKey, startInclusive, endInclusive); + foreach (var entry in _btreeIndex.Range(actualMinKey, actualMaxKey, IndexDirection.Backward, transaction?.TransactionId)) + return entry.Key; + return null; + } + + /// + /// The composite B-Tree bounds covering every entry whose user key lies between + /// and (null = unbounded). + /// + private (IndexKey Min, IndexKey Max) CompositeRange(object? minKey, object? maxKey, bool startInclusive, bool endInclusive) + { IndexKey actualMinKey = IndexKey.MinKey; IndexKey actualMaxKey = s_maxBoundaryKey; @@ -379,7 +417,7 @@ public int CountRange(object? minKey, object? maxKey, bool startInclusive, bool actualMaxKey = CreateCompositeKeyBoundary(userMax, useMinObjectId: !endInclusive); } - return _btreeIndex.CountRange(actualMinKey, actualMaxKey, transaction?.TransactionId); + return (actualMinKey, actualMaxKey); } /// diff --git a/src/BLite.Core/Indexing/IndexKey.cs b/src/BLite.Core/Indexing/IndexKey.cs index 3b77927d..2d08f05b 100644 --- a/src/BLite.Core/Indexing/IndexKey.cs +++ b/src/BLite.Core/Indexing/IndexKey.cs @@ -17,7 +17,7 @@ public struct IndexKey : IEquatable, IComparable // Guarantees NullSentinel (0x00) and NullSentinelNext (0x01) sort strictly before // all encoded user values, including int.MinValue which encodes to { 0x00, 0x00, 0x00, 0x00 } // without the prefix. - private const byte KeyPrefix = 0x02; + internal const byte KeyPrefix = 0x02; // Prefix of decimal keys. Distinct from KeyPrefix so that an index whose keys were written by an // older version (decimal stored as double under KeyPrefix) can be told apart from a current one. diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index aa8a65c0..7ff8919d 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -240,18 +240,18 @@ private async Task ExecuteAsyncCore(Expression expression, Can if (typeof(TResult) == typeof(object)) return (TResult)(object)count; } - // ── Fast path: Sum / Average / Min / Max via BSON field-projection scan ─ - // Reads only the target field from raw BSON — T is never fully instantiated. - // Fires for both no-WHERE and WHERE variants: - // - No WHERE: projector reads only the selector field. - // - With WHERE: BsonProjectionCompiler merges WHERE + SELECT fields into one - // BSON pass; documents that fail the WHERE return null and are skipped. - // HasComplexOperators is intentionally NOT checked here: VisitAggregate always - // sets it so that queries that cannot be pushed down fall back to EnumerableRewriter. - if (model.AggregateOp is not null && model.AggregateSelector is not null) + // ── Fast path: Sum / Average / Min / Max of a property without materialising T ─ + // From index keys or raw BSON, with the result LINQ computes over the entities. Only for + // an aggregate applied straight to Where clauses over the collection: any other operator + // in between (Take, Skip, OrderBy, Select, Distinct, ...) changes which values are + // aggregated or their order, so those queries run the usual way. + if (model.AggregateOp is not null && model.AggregateSelector is not null + && IsAggregateOverFilteredRoot(expression) + && Enum.TryParse(model.AggregateOp, out var aggregateOp)) { - if (TryBsonAggregate(model.AggregateOp, model.AggregateSelector, model.WhereClause, out var aggResult)) - return aggResult; + var (handled, value) = await _collection.TryAggregateAsync( + aggregateOp, model.AggregateSelector, model.WhereClause, typeof(TResult), cancellationToken).ConfigureAwait(false); + if (handled) return (TResult)value!; } // ── Push-down SELECT (+ optional WHERE) ─────────────────────────────── @@ -427,163 +427,28 @@ private static TResult TerminalReturn(IEnumerable data) $"BLite query pipeline: unsupported terminal result type '{typeof(TResult)}'."); } - // ─── BSON-level aggregate push-down ────────────────────────────────────── + // ─── Aggregate push-down ───────────────────────────────────────────────── /// - /// Attempts to compute Sum or Average entirely by scanning raw BSON bytes, - /// without deserialising the full entity . + /// Whether is Sum/Average/Min/Max with a + /// selector, applied to zero or more Where(x => ...) calls over the collection itself. /// - /// - /// true and a populated on success; - /// false to signal the caller should fall through to the standard path. - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private bool TryBsonAggregate( - string aggregateOp, - LambdaExpression selector, - LambdaExpression? whereClause, - out TResult result) + private static bool IsAggregateOverFilteredRoot(Expression expression) { - result = default!; - var fieldType = selector.ReturnType; - - // Only numeric types are supported; others fall through to EnumerableRewriter. - if (fieldType != typeof(decimal) && fieldType != typeof(double) && - fieldType != typeof(float) && fieldType != typeof(int) && - fieldType != typeof(long)) + if (expression is not MethodCallExpression call + || call.Method.DeclaringType != typeof(Queryable) + || call.Arguments.Count != 2) return false; - // Compile a BSON projector for the target field. - // When whereClause is provided, BsonProjectionCompiler merges WHERE + SELECT fields - // into one BSON pass; documents that fail the WHERE return null and are skipped - // by ScanAsync. - object? projector; - try - { - var compileMethod = s_compiledSelectMethods.GetOrAdd( - fieldType, t => s_tryCompileMethod.MakeGenericMethod(typeof(T), t)); - projector = compileMethod.Invoke(null, [selector, whereClause, _collection.GetKeyMap()]); - } - catch { return false; } - - if (projector is null) return false; // field shape too complex for BSON push-down - - // Execute the BSON scan — yields only non-null field values. - System.Collections.IEnumerable? values; - try - { - var scanMethod = s_compiledScanMethods.GetOrAdd( - fieldType, t => s_scanProjectorMethod.MakeGenericMethod(t)); - values = scanMethod.Invoke(_collection, [projector]) as System.Collections.IEnumerable; - } - catch { return false; } - - if (values is null) return false; - - try { return AggregateValues(aggregateOp, values, fieldType, out result); } - catch { return false; } - } - - private static bool AggregateValues( - string op, - System.Collections.IEnumerable values, - Type fieldType, - out TResult result) - { - result = default!; - - if (fieldType == typeof(decimal)) - { - var typed = (IEnumerable)values; - decimal agg = op switch - { - "Sum" => typed.Sum(), - "Average" => typed.DefaultIfEmpty().Average(), - "Min" => typed.DefaultIfEmpty().Min(), - "Max" => typed.DefaultIfEmpty().Max(), - _ => throw new NotSupportedException($"Unsupported aggregate op: {op}") - }; - if (typeof(TResult) == typeof(decimal)) { result = (TResult)(object)agg; return true; } - } - else if (fieldType == typeof(double)) + var source = call.Arguments[0]; + while (source is MethodCallExpression where + && where.Method.DeclaringType == typeof(Queryable) + && where.Method.Name == nameof(Queryable.Where) + && where.Arguments[1] is UnaryExpression { Operand: LambdaExpression { Parameters.Count: 1 } }) { - var typed = (IEnumerable)values; - double agg = op switch - { - "Sum" => typed.Sum(), - "Average" => typed.DefaultIfEmpty().Average(), - "Min" => typed.DefaultIfEmpty().Min(), - "Max" => typed.DefaultIfEmpty().Max(), - _ => throw new NotSupportedException($"Unsupported aggregate op: {op}") - }; - if (typeof(TResult) == typeof(double)) { result = (TResult)(object)agg; return true; } - } - else if (fieldType == typeof(float)) - { - var typed = (IEnumerable)values; - float agg = op switch - { - "Sum" => typed.Sum(), - "Average" => typed.DefaultIfEmpty().Average(), - "Min" => typed.DefaultIfEmpty().Min(), - "Max" => typed.DefaultIfEmpty().Max(), - _ => throw new NotSupportedException($"Unsupported aggregate op: {op}") - }; - if (typeof(TResult) == typeof(float)) { result = (TResult)(object)agg; return true; } - } - else if (fieldType == typeof(int)) - { - var typed = (IEnumerable)values; - if (op == "Sum") - { - long sum = 0; - foreach (var v in typed) sum += v; - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)sum; return true; } - if (typeof(TResult) == typeof(int)) { result = (TResult)(object)(int)sum; return true; } - } - else if (op == "Average") - { - double avg = typed.DefaultIfEmpty().Average(); - if (typeof(TResult) == typeof(double)) { result = (TResult)(object)avg; return true; } - } - else if (op == "Min") - { - int min = typed.DefaultIfEmpty().Min(); - if (typeof(TResult) == typeof(int)) { result = (TResult)(object)min; return true; } - } - else if (op == "Max") - { - int max = typed.DefaultIfEmpty().Max(); - if (typeof(TResult) == typeof(int)) { result = (TResult)(object)max; return true; } - } - } - else if (fieldType == typeof(long)) - { - var typed = (IEnumerable)values; - if (op == "Sum") - { - long sum = 0; - foreach (var v in typed) sum += v; - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)sum; return true; } - } - else if (op == "Average") - { - double avg = typed.DefaultIfEmpty().Select(x => (double)x).Average(); - if (typeof(TResult) == typeof(double)) { result = (TResult)(object)avg; return true; } - } - else if (op == "Min") - { - long min = typed.DefaultIfEmpty().Min(); - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)min; return true; } - } - else if (op == "Max") - { - long max = typed.DefaultIfEmpty().Max(); - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)max; return true; } - } + source = where.Arguments[0]; } - return false; + return source is ConstantExpression { Value: IQueryable }; } // ─── Push-down SELECT helper ────────────────────────────────────────────── diff --git a/src/BLite.Core/Query/LinqAggregate.cs b/src/BLite.Core/Query/LinqAggregate.cs new file mode 100644 index 00000000..6e69638f --- /dev/null +++ b/src/BLite.Core/Query/LinqAggregate.cs @@ -0,0 +1,167 @@ +namespace BLite.Core.Query; + +/// The LINQ aggregate operators that can run without materialising entities. +internal enum AggregateOperator +{ + Sum, + Average, + Min, + Max, +} + +/// +/// Folds the values of one numeric property the way the Enumerable operator does when LINQ +/// runs it over the materialised entities, so a pushed-down aggregate returns the same value and +/// throws the same exceptions: +/// +/// Sum of int accumulates in a checked int, of long in a checked +/// long, of float in a double (cast back at the end); Average of +/// int/long sums in a checked long. +/// Min/Max compare with (as +/// Comparer<T>.Default) and only replace on a strict improvement, so the first of +/// equal values wins (-0.0 vs 0.0, 1.0m vs 1.00m) and NaN is the +/// smallest value. +/// Nullable selectors skip null; on no values Sum is 0 and the others +/// return null. Non-nullable Average/Min/Max over no values throw +/// . +/// +/// Values must be added in the order LINQ would see them, since floating-point sums and checked +/// overflow depend on it. With exactTotal, integer sums are instead computed exactly and only +/// the total is range-checked: for values read in a different order (from an index) this throws +/// exactly when the result does not fit. +/// +internal sealed class LinqAggregate where TValue : struct, IComparable +{ + private readonly AggregateOperator _op; + private readonly bool _nullable; + private readonly bool _exactTotal; + + private long _count; + private TValue _best; + private int _sumInt32; + private long _sumInt64; + private decimal _sumOverflow; // exactTotal: the part of a long sum beyond the long range + private double _sumDouble; + private decimal _sumDecimal; + + public LinqAggregate(AggregateOperator op, bool nullable, bool exactTotal = false) + { + _op = op; + _nullable = nullable; + _exactTotal = exactTotal; + } + + /// + /// The result type of the Queryable operator for a + /// (or nullable) selector, e.g. double for Average of int. + /// + public static Type ResultType(AggregateOperator op, bool nullable) + { + if (op == AggregateOperator.Average && (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long))) + return nullable ? typeof(double?) : typeof(double); + return nullable ? typeof(TValue?) : typeof(TValue); + } + + /// Adds the value of the next document (nullable selectors pass null as such). + public void Add(TValue? value) + { + if (value.HasValue) Add(value.GetValueOrDefault()); + } + + /// Adds the value of the next document. + public void Add(TValue value) + { + switch (_op) + { + case AggregateOperator.Min: + if (_count == 0 || value.CompareTo(_best) < 0) _best = value; + break; + case AggregateOperator.Max: + if (_count == 0 || value.CompareTo(_best) > 0) _best = value; + break; + default: + Accumulate(value); + break; + } + _count++; + } + + private void Accumulate(TValue value) + { + if (typeof(TValue) == typeof(int)) + { + int v = (int)(object)value; + if (_op == AggregateOperator.Sum && !_exactTotal) _sumInt32 = checked(_sumInt32 + v); + else _sumInt64 = checked(_sumInt64 + v); + } + else if (typeof(TValue) == typeof(long)) + { + long v = (long)(object)value; + if (!_exactTotal) + { + _sumInt64 = checked(_sumInt64 + v); + } + else + { + long sum = unchecked(_sumInt64 + v); + // Signed overflow: both operands share a sign the result does not have. + if (((_sumInt64 ^ sum) & (v ^ sum)) < 0) + { + _sumOverflow += (decimal)_sumInt64 + v; + sum = 0; + } + _sumInt64 = sum; + } + } + else if (typeof(TValue) == typeof(double)) + { + _sumDouble += (double)(object)value; + } + else if (typeof(TValue) == typeof(float)) + { + _sumDouble += (float)(object)value; + } + else if (typeof(TValue) == typeof(decimal)) + { + _sumDecimal += (decimal)(object)value; + } + else + { + throw new NotSupportedException($"Aggregating {typeof(TValue)} is not supported."); + } + } + + /// + /// The aggregate, boxed as (a nullable result boxes its value or is + /// null). + /// + public object? Result() + { + switch (_op) + { + case AggregateOperator.Sum: + if (typeof(TValue) == typeof(int)) return _exactTotal ? checked((int)_sumInt64) : _sumInt32; + if (typeof(TValue) == typeof(long)) return ExactInt64(); + if (typeof(TValue) == typeof(double)) return _sumDouble; + if (typeof(TValue) == typeof(float)) return (float)_sumDouble; + return _sumDecimal; + + case AggregateOperator.Average: + if (_count == 0) return NoElements(); + if (typeof(TValue) == typeof(int)) return (double)_sumInt64 / _count; + if (typeof(TValue) == typeof(long)) return (double)ExactInt64() / _count; + if (typeof(TValue) == typeof(double)) return _sumDouble / _count; + if (typeof(TValue) == typeof(float)) return (float)(_sumDouble / _count); + return _sumDecimal / _count; + + default: + return _count == 0 ? NoElements() : _best; + } + } + + private long ExactInt64() => _sumOverflow == 0 ? _sumInt64 : checked((long)(_sumOverflow + _sumInt64)); + + private object? NoElements() => _nullable + ? null + : throw new InvalidOperationException("Sequence contains no elements"); +} diff --git a/tests/BLite.Benchmark/LinqAggregateBenchmarks.cs b/tests/BLite.Benchmark/LinqAggregateBenchmarks.cs new file mode 100644 index 00000000..b48c29c1 --- /dev/null +++ b/tests/BLite.Benchmark/LinqAggregateBenchmarks.cs @@ -0,0 +1,89 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BLite.Shared; +using System.IO; + +namespace BLite.Benchmark; + +/// +/// LINQ Sum / Average / Max over 10 000 documents, +/// against materialising the entities and aggregating them with LINQ to Objects (what every +/// aggregate with a selector did before). Covers each push-down path: +/// +/// index keys: int property with a B-Tree index, with and without a range filter; +/// index boundary: Max of an indexed int reads one entry; +/// raw BSON: double and decimal properties without an index, with and +/// without a filter on a non-indexed field. +/// +/// +[SimpleJob] +[InProcess] +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +public class LinqAggregateBenchmarks +{ + private const int DocCount = 10_000; + + private string _dbPath = null!; + private AggregateDbContext _ctx = null!; + + [GlobalSetup] + public async Task Setup() + { + _dbPath = Path.Combine(AppContext.BaseDirectory, $"linq_agg_{Guid.NewGuid():N}.db"); + _ctx = new AggregateDbContext(_dbPath); + await _ctx.Readings.InsertBulkAsync(Enumerable.Range(0, DocCount).Select(i => new Reading + { + Tag = "abcd"[i % 4].ToString(), + Count = (i * 7919) % 1000, + Total = i * 1_000L, + Ratio = Math.Sin(i) * 100, + Weight = i / 3f, + Price = i * 1.25m, + })); + } + + [GlobalCleanup] + public void Cleanup() + { + _ctx?.Dispose(); + foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + if (File.Exists(f)) File.Delete(f); + } + + // ── int property with an index ────────────────────────────────────────── + + [Benchmark(Baseline = true, Description = "Sum(int) – materialise + LINQ to Objects")] + [BenchmarkCategory("IndexedInt")] + public int SumIndexed_Materialised() => _ctx.Readings.AsQueryable().ToList().Sum(r => r.Count); + + [Benchmark(Description = "Sum(int) – index keys")] + [BenchmarkCategory("IndexedInt")] + public int SumIndexed() => _ctx.Readings.AsQueryable().Sum(r => r.Count); + + [Benchmark(Description = "Where(range).Sum(int) – index keys")] + [BenchmarkCategory("IndexedInt")] + public int SumIndexedRange() => _ctx.Readings.AsQueryable().Where(r => r.Count >= 250 && r.Count < 500).Sum(r => r.Count); + + [Benchmark(Description = "Max(int) – index boundary")] + [BenchmarkCategory("IndexedInt")] + public int MaxIndexed() => _ctx.Readings.AsQueryable().Max(r => r.Count); + + // ── properties without an index ───────────────────────────────────────── + + [Benchmark(Baseline = true, Description = "Sum(double) – materialise + LINQ to Objects")] + [BenchmarkCategory("RawBson")] + public double SumDouble_Materialised() => _ctx.Readings.AsQueryable().ToList().Sum(r => r.Ratio); + + [Benchmark(Description = "Sum(double) – raw BSON")] + [BenchmarkCategory("RawBson")] + public double SumDouble() => _ctx.Readings.AsQueryable().Sum(r => r.Ratio); + + [Benchmark(Description = "Average(decimal) – raw BSON")] + [BenchmarkCategory("RawBson")] + public decimal AverageDecimal() => _ctx.Readings.AsQueryable().Average(r => r.Price); + + [Benchmark(Description = "Where(tag).Sum(double) – raw BSON")] + [BenchmarkCategory("RawBson")] + public double SumDoubleFiltered() => _ctx.Readings.AsQueryable().Where(r => r.Tag == "b").Sum(r => r.Ratio); +} diff --git a/tests/BLite.Shared/AggregateContexts.cs b/tests/BLite.Shared/AggregateContexts.cs new file mode 100644 index 00000000..f4e8804a --- /dev/null +++ b/tests/BLite.Shared/AggregateContexts.cs @@ -0,0 +1,77 @@ +using System.ComponentModel.DataAnnotations.Schema; +using BLite.Bson; +using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Metadata; + +namespace BLite.Shared; + +/// One property per numeric type LINQ aggregates, plain and nullable. +public class Reading +{ + public ObjectId Id { get; set; } + public string Tag { get; set; } = string.Empty; + public int Count { get; set; } + public long Total { get; set; } + public int Units { get; set; } + public double Ratio { get; set; } + public float Weight { get; set; } + public decimal Price { get; set; } + public int? MaybeCount { get; set; } + public long? MaybeTotal { get; set; } + public double? MaybeRatio { get; set; } + public float? MaybeWeight { get; set; } + public decimal? MaybePrice { get; set; } + + /// + /// Stored as "score", while is stored as "level": each field + /// carries the conventional name of the other property. + /// + [Column("score")] + public int Level { get; set; } + + [Column("level")] + public int Score { get; set; } +} + +/// +/// with B-Tree indexes on Count (int), Total (long) and +/// MaybeCount (int?), and none on the other properties. +/// +public partial class AggregateDbContext : DocumentDbContext +{ + public DocumentCollection Readings { get; set; } = null!; + + public AggregateDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("readings") + .HasIndex(r => r.Count) + .HasIndex(r => r.Total) + .HasIndex(r => r.MaybeCount); + } +} + +/// A later version of with properties (initialised) that older documents lack. +public class ReadingV2 +{ + public ObjectId Id { get; set; } + public string Tag { get; set; } = string.Empty; + public int Count { get; set; } + public long Total { get; set; } + public int? MaybeCount { get; set; } + public int Bonus { get; set; } = 7; + public double? Factor { get; set; } = 1.5; +} + +/// Reads the readings collection as . +public partial class AggregateV2DbContext : DocumentDbContext +{ + public DocumentCollection Readings { get; set; } = null!; + + public AggregateV2DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().ToCollection("readings"); +} diff --git a/tests/BLite.Tests/LinqAggregateTests.cs b/tests/BLite.Tests/LinqAggregateTests.cs new file mode 100644 index 00000000..02f9859b --- /dev/null +++ b/tests/BLite.Tests/LinqAggregateTests.cs @@ -0,0 +1,354 @@ +using System.Linq.Expressions; +using System.Reflection; +using BLite.Bson; +using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Query; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// LINQ Sum, Average, Min and Max computed from index keys or raw BSON +/// return exactly what LINQ returns over the materialised entities, or throw the same exception. +/// +public class LinqAggregateTests : IDisposable +{ + private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"agg_{Guid.NewGuid():N}.db"); + + public void Dispose() + { + foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ } + } + + private static readonly string[] Operators = ["Sum", "Average", "Min", "Max"]; + private static readonly string[] Indexed = ["Count", "Total", "MaybeCount"]; + private static readonly string[] Properties = + ["Count", "Total", "Units", "Ratio", "Weight", "Price", "MaybeCount", "MaybeTotal", "MaybeRatio", "MaybeWeight", "MaybePrice"]; + + private static Reading Make(int i) => new() + { + Tag = "abc"[i % 3].ToString(), + Count = (i * 7) % 13 - 4, + Total = i * 1_000_000_007L - 5_000_000_000L, + Units = i * 3 - 50, + Ratio = Math.Sin(i) * 100 / 3, + Weight = (float)(i / 7.0), + // Same values at different scales: 1.5 / 1.50 / 1.500. + Price = (i % 3) switch { 0 => i * 1.5m, 1 => i * 1.50m, _ => i * 1.500m }, + MaybeCount = i % 4 == 0 ? null : i - 20, + MaybeTotal = i % 4 == 1 ? null : i * 3L, + MaybeRatio = i % 4 == 2 ? null : 1.0 / (i + 1), + MaybeWeight = i % 4 == 3 ? null : (float)(i * 0.1), + MaybePrice = i % 4 == 0 ? null : i * 0.10m, + Level = i, + Score = 100 + i, + }; + + private static readonly (string Name, Expression>? Where)[] Filters = + [ + ("none", null), + ("tag", r => r.Tag == "b"), + ("count range", r => r.Count >= -1 && r.Count < 5), + ("count range and tag", r => r.Count > 0 && r.Tag == "c"), + ("no match", r => r.Tag == "zzz"), + ("empty count range", r => r.Count > 1000), + ]; + + /// Whether the aggregate should be computed without materialising the entities. + private static bool ExpectPushedDown(string property, string filter) => filter switch + { + "none" or "tag" or "no match" => true, // index keys, or raw BSON without/with a predicate + "count range" or "empty count range" => property == "Count", // the Count index covers the filter + _ => false, // another index serves the filter + }; + + [Fact] + public async Task EveryOperatorPropertyAndFilter_MatchesLinqOverTheEntities() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 40; i++) + await db.Readings.InsertAsync(Make(i)); + var entities = db.Readings.AsQueryable().ToList(); + + var failures = new List(); + foreach (var op in Operators) + foreach (var property in Properties) + foreach (var (filter, where) in Filters) + { + var selector = Selector(property); + var label = $"{op}({property}) where {filter}"; + + // LINQ over the matching entities, in the order the collection returns them for the filter + // (primary-key order, or index order when an index serves it). + var matches = where == null ? entities : db.Readings.AsQueryable().Where((Expression>)where).ToList(); + var expected = Run(() => Execute(matches.AsQueryable(), op, selector, null)); + var actual = Run(() => Execute(db.Readings.AsQueryable(), op, selector, where)); + if (!Same(expected, actual)) + failures.Add($"{label}: expected {Show(expected)}, got {Show(actual)}"); + + var resultType = AggregateCall(entities.AsQueryable(), op, selector, where).Type; + var direct = await RunAsync(() => db.Readings.TryAggregateAsync(Enum.Parse(op), selector, where, resultType)); + bool handled = direct.Error != null || direct.Value.Handled; + if (handled != ExpectPushedDown(property, filter)) + failures.Add($"{label}: pushed down = {handled}, expected {ExpectPushedDown(property, filter)}"); + else if (handled && !Same(expected, (direct.Value.Value, direct.Error))) + failures.Add($"{label} (pushed down): expected {Show(expected)}, got {Show((direct.Value.Value, direct.Error))}"); + } + + Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures)); + } + + [Theory] + [InlineData(new[] { 0.0, -0.0, double.NaN, 1.0, -0.0 })] + [InlineData(new[] { -0.0, 0.0, 2.5, double.NaN })] + [InlineData(new[] { double.NaN, double.NaN })] + [InlineData(new[] { double.PositiveInfinity, double.NegativeInfinity, 1.0 })] + [InlineData(new[] { 1e308, 1e308, -1e308 })] + [InlineData(new[] { 447711.6875, 7923489.5, 16777216.0 })] // float Average: (float)(sum / n) != (float)sum / n + public async Task FloatingPointEdgeCases_MatchLinq(double[] values) + { + using var db = new AggregateDbContext(_dbPath); + foreach (var v in values) + await db.Readings.InsertAsync(new Reading { Ratio = v, MaybeRatio = v, Weight = (float)v, MaybeWeight = (float)v }); + var entities = db.Readings.AsQueryable().ToList(); + + foreach (var op in Operators) + foreach (var property in new[] { "Ratio", "MaybeRatio", "Weight", "MaybeWeight" }) + { + var selector = Selector(property); + var expected = Run(() => Execute(entities.AsQueryable(), op, selector, null)); + var direct = await db.Readings.TryAggregateAsync(Enum.Parse(op), selector, null, AggregateCall(entities.AsQueryable(), op, selector, null).Type); + + Assert.True(direct.Handled, $"{op}({property})"); + Assert.True(Same(expected, (direct.Value, null)), $"{op}({property}): expected {Show(expected)}, got {Show((direct.Value, null))}"); + } + } + + [Fact] + public async Task DecimalScale_MatchesLinq() + { + using var db = new AggregateDbContext(_dbPath); + foreach (var price in new[] { 1.00m, 1.0m, 2.50m, 1m }) + await db.Readings.InsertAsync(new Reading { Price = price, MaybePrice = price }); + + var entities = db.Readings.AsQueryable().ToList(); + string Text(decimal? d) => d?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "null"; + + Assert.Equal(Text(entities.Sum(r => r.Price)), Text(db.Readings.AsQueryable().Sum(r => r.Price))); + Assert.Equal(Text(entities.Min(r => r.Price)), Text(db.Readings.AsQueryable().Min(r => r.Price))); // first of the equal minima + Assert.Equal(Text(entities.Max(r => r.MaybePrice)), Text(db.Readings.AsQueryable().Max(r => r.MaybePrice))); + Assert.Equal(Text(entities.Average(r => r.MaybePrice)), Text(db.Readings.AsQueryable().Average(r => r.MaybePrice))); + } + + [Fact] + public async Task IntegerOverflow_ThrowsLikeLinq() + { + using var db = new AggregateDbContext(_dbPath); + await db.Readings.InsertAsync(new Reading { Units = int.MaxValue, Count = int.MaxValue, Total = long.MaxValue }); + await db.Readings.InsertAsync(new Reading { Units = 1, Count = 1, Total = 1 }); + + Assert.Throws(() => db.Readings.AsQueryable().Sum(r => r.Units)); // raw BSON + Assert.Throws(() => db.Readings.AsQueryable().Sum(r => r.Count)); // index keys + Assert.Throws(() => db.Readings.AsQueryable().Sum(r => r.Total)); + Assert.Throws(() => db.Readings.AsQueryable().Average(r => r.Total)); + Assert.Equal((int.MaxValue + 1.0) / 2, db.Readings.AsQueryable().Average(r => r.Units)); + Assert.Equal((int.MaxValue + 1.0) / 2, db.Readings.AsQueryable().Average(r => r.Count)); + } + + [Fact] + public async Task IndexKeySums_CheckOnlyTheTotalForOverflow() + { + // Index keys come in value order, not document order, so an index sum cannot reproduce an + // overflow of a running total that LINQ would hit before a later negative value: it returns + // the exact total when that fits, and throws when it does not. + using var db = new AggregateDbContext(_dbPath); + foreach (var count in new[] { int.MaxValue, 1, -2 }) + await db.Readings.InsertAsync(new Reading { Count = count, Units = count }); + + // Whether LINQ overflows depends on the document order (int.MaxValue + 1 before the -2). + var entities = db.Readings.AsQueryable().ToList(); + Assert.True(Same(Run(() => entities.Sum(r => r.Units)), Run(() => db.Readings.AsQueryable().Sum(r => r.Units)))); // raw BSON, document order + Assert.Equal(int.MaxValue - 1, db.Readings.AsQueryable().Sum(r => r.Count)); // index keys: the exact total + } + + [Fact] + public async Task DocumentsWithoutTheField_UseTheEntityValue() + { + using (var v1 = new AggregateDbContext(_dbPath)) + { + for (int i = 0; i < 5; i++) + await v1.Readings.InsertAsync(Make(i)); + } + + // Whatever value the mapper gives a missing property, the aggregate must use the same one. + using var v2 = new AggregateV2DbContext(_dbPath); + var entities = v2.Readings.AsQueryable().ToList(); + + Assert.False((await v2.Readings.TryAggregateAsync(AggregateOperator.Sum, (Expression>)(r => r.Bonus), null, typeof(int))).Handled); + Assert.False((await v2.Readings.TryAggregateAsync(AggregateOperator.Max, (Expression>)(r => r.Factor), null, typeof(double?))).Handled); + Assert.Equal(entities.Sum(r => r.Bonus), v2.Readings.AsQueryable().Sum(r => r.Bonus)); + Assert.Equal(entities.Max(r => r.Factor), v2.Readings.AsQueryable().Max(r => r.Factor)); + Assert.Equal(entities.Sum(r => r.Count), v2.Readings.AsQueryable().Sum(r => r.Count)); + } + + [Theory] + [InlineData("units", "null")] // null in a non-nullable int + [InlineData("units", "int64")] // an Int64 in an int field + [InlineData("ratio", "null")] + [InlineData("ratio", "int32")] // an Int32 in a double field + public async Task FieldsWithAnotherBsonType_UseTheEntityValue(string field, string kind) + { + using (var db = new AggregateDbContext(_dbPath)) + { + for (int i = 0; i < 4; i++) + await db.Readings.InsertAsync(Make(i)); + } + + // Written through the schemaless API, which stores whatever it is given. + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("readings"); + await col.InsertAsync(col.CreateDocument(["_id", "tag", field], b => + { + b.AddObjectId("_id", ObjectId.NewObjectId()).AddString("tag", "x"); + switch (kind) + { + case "null": b.AddNull(field); break; + case "int64": b.AddInt64(field, 5L); break; + default: b.AddInt32(field, 2); break; + } + })); + } + + using var reopened = new AggregateDbContext(_dbPath); + var property = field == "units" ? "Units" : "Ratio"; + var resultType = field == "units" ? typeof(int) : typeof(double); + + // A null the entity cannot hold makes materialising throw; the aggregate must throw alike. + var entities = Run(() => reopened.Readings.AsQueryable().ToList()); + var expected = entities.Error != null + ? entities + : Run(() => Execute(((List)entities.Value!).AsQueryable(), "Sum", Selector(property), null)); + + Assert.False((await reopened.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector(property), null, resultType)).Handled); + Assert.True(Same(expected, Run(() => Execute(reopened.Readings.AsQueryable(), "Sum", Selector(property), null))), Show(expected)); + } + + [Fact] + public async Task RenamedProperty_IsNotReadFromAnotherField() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 5; i++) + await db.Readings.InsertAsync(Make(i)); + + Assert.False((await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Level"), null, typeof(int))).Handled); + Assert.Equal(10, db.Readings.AsQueryable().Sum(r => r.Level)); + Assert.Equal(510, db.Readings.AsQueryable().Sum(r => r.Score)); + } + + [Fact] + public async Task OtherOperatorsBeforeTheAggregate_AreApplied() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 20; i++) + await db.Readings.InsertAsync(Make(i)); + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + + Assert.Equal(entities.Take(3).Sum(r => r.Count), q.Take(3).Sum(r => r.Count)); + Assert.Equal(entities.Skip(5).Max(r => r.Count), q.Skip(5).Max(r => r.Count)); + Assert.Equal(entities.OrderBy(r => r.Ratio).Take(4).Sum(r => r.Ratio), q.OrderBy(r => r.Ratio).Take(4).Sum(r => r.Ratio)); + Assert.Equal(entities.Where(r => r.Tag == "a").Where(r => r.Units > -40).Sum(r => r.Units), + q.Where(r => r.Tag == "a").Where(r => r.Units > -40).Sum(r => r.Units)); + Assert.Equal(entities.Where(r => r.Count > 0).Sum(r => r.Count), await q.Where(r => r.Count > 0).SumAsync(r => r.Count)); + Assert.Equal(entities.Average(r => r.Units), await q.AverageAsync(r => r.Units)); + } + + [Fact] + public async Task IndexAggregates_FollowUpdatesAndDeletes() + { + using var db = new AggregateDbContext(_dbPath); + var readings = Enumerable.Range(0, 10).Select(Make).ToList(); + foreach (var r in readings) await db.Readings.InsertAsync(r); + + readings[3].Count = 500; + await db.Readings.UpdateAsync(readings[3]); + await db.Readings.DeleteAsync(readings[4].Id); + readings.RemoveAt(4); + + Assert.Equal(readings.Sum(r => r.Count), db.Readings.AsQueryable().Sum(r => r.Count)); + Assert.Equal(500, db.Readings.AsQueryable().Max(r => r.Count)); + Assert.Equal(readings.Min(r => r.Count), db.Readings.AsQueryable().Min(r => r.Count)); + Assert.Equal(readings.Where(r => r.Count < 500).Max(r => r.Count), db.Readings.AsQueryable().Where(r => r.Count < 500).Max(r => r.Count)); + } + + // ── helpers ───────────────────────────────────────────────────────────── + + private static LambdaExpression Selector(string property) + { + var p = Expression.Parameter(typeof(Reading), "r"); + return Expression.Lambda(Expression.Property(p, property), p); + } + + private static MethodCallExpression AggregateCall(IQueryable source, string op, LambdaExpression selector, LambdaExpression? where) + { + var src = where == null + ? source.Expression + : Expression.Call(typeof(Queryable), nameof(Queryable.Where), [typeof(TEntity)], source.Expression, Expression.Quote(where)); + Type[] typeArgs = op is "Min" or "Max" ? [typeof(TEntity), selector.ReturnType] : [typeof(TEntity)]; + return Expression.Call(typeof(Queryable), op, typeArgs, src, Expression.Quote(selector)); + } + + private static readonly MethodInfo s_execute = typeof(IQueryProvider).GetMethods().Single(m => m.Name == "Execute" && m.IsGenericMethodDefinition); + + private static object? Execute(IQueryable source, string op, LambdaExpression selector, LambdaExpression? where) + { + var call = AggregateCall(source, op, selector, where); + return s_execute.MakeGenericMethod(call.Type).Invoke(source.Provider, [call]); + } + + private static (object? Value, Type? Error) Run(Func action) + { + try { return (action(), null); } + catch (Exception ex) { return (null, Unwrap(ex).GetType()); } + } + + private static async Task<((bool Handled, object? Value) Value, Type? Error)> RunAsync(Func> action) + { + try { return (await action(), null); } + catch (Exception ex) { return (default, Unwrap(ex).GetType()); } + } + + private static Exception Unwrap(Exception ex) + { + while (ex is TargetInvocationException or AggregateException && ex.InnerException != null) + ex = ex.InnerException!; + return ex; + } + + /// Same exception type, or values of the same type with identical bits. + private static bool Same((object? Value, Type? Error) expected, (object? Value, Type? Error) actual) + { + if (expected.Error != null || actual.Error != null) return expected.Error == actual.Error; + return (expected.Value, actual.Value) switch + { + (null, null) => true, + (double x, double y) => BitConverter.DoubleToInt64Bits(x) == BitConverter.DoubleToInt64Bits(y), + (float x, float y) => BitConverter.SingleToInt32Bits(x) == BitConverter.SingleToInt32Bits(y), + (decimal x, decimal y) => decimal.GetBits(x).SequenceEqual(decimal.GetBits(y)), + var (x, y) => x?.GetType() == y?.GetType() && Equals(x, y), + }; + } + + private static string Show((object? Value, Type? Error) result) + => result.Error != null ? result.Error.Name + : result.Value switch + { + null => "null", + double d => d.ToString("R") + (BitConverter.DoubleToInt64Bits(d) < 0 ? " (neg)" : ""), + float f => f.ToString("R"), + var v => $"{v} ({v.GetType().Name})", + }; +} From 19c25ad7c195bca187abc379b04ef709c780442b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:49:47 +0200 Subject: [PATCH 3/6] fix(query): read aggregates only from fields the mapper stores verbatim (#183) Address the review of the Sum/Average/Min/Max push-down: - Generated mappers now override a new DocumentMapperBase.GetVerbatimFieldName(property): the BSON field a property is restored from unchanged, or null. A property qualifies when both accessors are compiler-implemented and it has a setter, no converter, is scalar and is not passed to the constructor. Ignored, computed and get-only properties are not mapped at all. The key maps to "_id"; renamed properties map to their attribute name. - TryAggregateAsync takes the field from that metadata instead of lowercase(Name) / "Id" -> "_id", and uses neither the raw field nor an index key without it. IsRenamedInBson (attribute-name sniffing) is gone; renamed properties are now pushed down correctly. - Index Sum/Average of int/long keep the sums of positive and negative values apart. Every running total LINQ computes lies between the two, so when both fit LINQ cannot overflow in any order and the result is exact; otherwise the aggregate runs in the order LINQ sees the entities (raw BSON or materialised), so it throws exactly where LINQ throws. - The index path requires the index selector to be x => x.Property of the aggregated type and rejects keys without a document id after the value. - The raw-field fallback seeks from the first element explicitly when the offset table has no entry for the field. - Remove TryPushDownSelect and its reflection caches: it invoked ScanAsync(projector, ct) with one argument and cast an IAsyncEnumerable to IEnumerable, so it always failed and fell through after compiling a projector. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Collections/BaseMappers.cs | 12 + .../Collections/DocumentCollection.cs | 108 +++++--- src/BLite.Core/Query/BTreeQueryProvider.cs | 87 +------ src/BLite.Core/Query/LinqAggregate.cs | 85 +++--- src/BLite.SourceGenerators/CodeGenerator.cs | 37 +++ src/BLite.SourceGenerators/EntityAnalyzer.cs | 1 + .../Helpers/SyntaxHelper.cs | 41 +++ .../Models/EntityInfo.cs | 6 + tests/BLite.Shared/AggregateContexts.cs | 101 +++++++ tests/BLite.Tests/LinqAggregateTests.cs | 246 +++++++++++++++++- 10 files changed, 561 insertions(+), 163 deletions(-) diff --git a/src/BLite.Core/Collections/BaseMappers.cs b/src/BLite.Core/Collections/BaseMappers.cs index 092d846a..a5c79e5d 100644 --- a/src/BLite.Core/Collections/BaseMappers.cs +++ b/src/BLite.Core/Collections/BaseMappers.cs @@ -22,6 +22,18 @@ public abstract class DocumentMapperBase : IDocumentMapper where public virtual IndexKey ToIndexKey(TId id) => IndexKey.Create(id); public virtual TId FromIndexKey(IndexKey key) => key.As(); + /// + /// The BSON field stores in, when + /// gives the property exactly the value of that field (a plain + /// auto-property with no converter), so the value of the materialised property can be read from + /// the raw document instead; null when that does not hold or is not known. + /// + /// + /// Generated mappers override this. The default returns null, so queries that could read a + /// property from raw BSON materialise the entities instead. + /// + public virtual string? GetVerbatimFieldName(string propertyName) => null; + #pragma warning disable IL2026 public virtual IEnumerable UsedKeys => GetSchema().GetAllKeys(); #pragma warning restore IL2026 diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 30e836a7..440f65e4 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -3502,10 +3502,18 @@ internal async Task CountByPredicateAsync( /// the documents matching without materialising them, with the /// result LINQ would compute over the entities. Returns Handled = false when the query /// must run the usual way. + /// + /// Only for a property the mapper restores verbatim from a BSON field + /// (): the stored value, in the + /// document or in an index key, is then the value of the materialised property. Ignored, + /// computed and converted properties, and ones whose accessors have logic, are materialised. + /// /// - /// An int/long property with a B-Tree index, and no filter or one the same - /// index covers exactly, is aggregated from the index keys alone (Min/Max - /// read one entry). + /// An int/long property with a B-Tree index on exactly that property, and + /// no filter or one the same index covers exactly, is aggregated from the index keys + /// alone (Min/Max read one entry). Keys are in value order, so a + /// Sum/Average for which LINQ could overflow in one order but not in + /// another is not taken from them. /// Otherwise, when no index serves the filter, the property is read from the raw BSON /// of each document in primary-key order, the order a scan yields the entities in. A /// document without the field, or holding a type the entity would not be read from @@ -3554,6 +3562,13 @@ internal async Task CountByPredicateAsync( if (LinqAggregate.ResultType(op, nullable) != resultType) return default; + // The field the mapper restores the property from unchanged. Without it, neither the raw + // field nor an index key (computed from the property when the document was written) is + // known to be the value the materialised entity returns. + var fieldName = (_mapper as DocumentMapperBase)?.GetVerbatimFieldName(property.Name); + if (fieldName == null) + return default; + var plan = whereClause == null ? null : Query.IndexOptimizer.TryOptimize(whereClause, GetIndexes(), ConverterRegistry); // Index keys hold the exact int/long value. Not for the other types: floating-point sums @@ -3565,14 +3580,17 @@ internal async Task CountByPredicateAsync( i.Definition.Type == IndexType.BTree && !i.Definition.IsPrimary && i.Definition.PropertyPaths.Length == 1 - && i.Definition.PropertyPaths[0] == property.Name); + && i.Definition.PropertyPaths[0] == property.Name + && KeysAreValuesOf(i.Definition, property, typeof(TValue))); if (index != null) { + (bool Handled, object? Value) fromKeys = default; if (whereClause == null) - return AggregateIndexKeys(op, nullable, index, null, null, true, true); - - if (plan != null + { + fromKeys = AggregateIndexKeys(op, nullable, index, null, null, true, true); + } + else if (plan != null && plan.IndexName == index.Name && !plan.HasResiduePredicate && !plan.IsVectorSearch @@ -3580,13 +3598,18 @@ internal async Task CountByPredicateAsync( && plan.InValues == null && plan.MinValue is not DBNull && plan.MaxValue is not DBNull) - return AggregateIndexKeys(op, nullable, index, plan.MinValue, plan.MaxValue, plan.StartInclusive, plan.EndInclusive); + { + fromKeys = AggregateIndexKeys(op, nullable, index, plan.MinValue, plan.MaxValue, plan.StartInclusive, plan.EndInclusive); + } + if (fromKeys.Handled) + return fromKeys; + // Otherwise aggregate the values in the order LINQ sees them, below. } } // An index serves the filter: fetching the matching documents through it is cheaper than // scanning them all, and keeps the order the values are aggregated in unchanged. - if (plan != null || IsRenamedInBson(property)) + if (plan != null) return default; BsonReaderPredicate? predicate = null; @@ -3596,7 +3619,25 @@ internal async Task CountByPredicateAsync( if (predicate == null) return default; } - return await AggregateRawFieldAsync(op, nullable, ProjectionAnalyzer.ToBsonName(property.Name), predicate, ct).ConfigureAwait(false); + return await AggregateRawFieldAsync(op, nullable, fieldName, predicate, ct).ConfigureAwait(false); + } + + /// + /// Whether the keys of the index are the values of + /// itself (x => x.Property), encoded as + /// : the user key is encoded from the boxed value the selector + /// returns, so this fixes the key layout the index path decodes. + /// + private static bool KeysAreValuesOf(CollectionIndexDefinition definition, System.Reflection.PropertyInfo property, Type valueType) + { + var body = definition.KeySelectorExpression.Body; + while (body is UnaryExpression { NodeType: ExpressionType.Convert } convert && convert.Type == typeof(object)) + body = convert.Operand; + + return body is MemberExpression { Member: System.Reflection.PropertyInfo keyProperty } access + && access.Expression == definition.KeySelectorExpression.Parameters[0] + && keyProperty.Name == property.Name + && (Nullable.GetUnderlyingType(access.Type) ?? access.Type) == valueType; } /// Aggregates the values of the index entries in the given range from their keys. @@ -3610,7 +3651,7 @@ private static (bool Handled, object? Value) AggregateIndexKeys( bool endInclusive) where TValue : struct, IComparable { - var aggregate = new LinqAggregate(op, nullable, exactTotal: true); + var aggregate = new LinqAggregate(op, nullable, anyOrder: true); if (op == AggregateOperator.Max) { @@ -3626,7 +3667,7 @@ private static (bool Handled, object? Value) AggregateIndexKeys( var visitor = new IndexKeyValueVisitor(aggregate, stopAfterFirst: op == AggregateOperator.Min); index.ScanKeys(min, max, startInclusive, endInclusive, ref visitor); - return visitor.Malformed ? default : (true, aggregate.Result()); + return visitor.Malformed || aggregate.DependsOnOrder ? default : (true, aggregate.Result()); } /// @@ -3657,10 +3698,17 @@ public bool Visit(ReadOnlySpan key) return false; } _aggregate.Add(value); - return !_stopAfterFirst; + // Past an order-dependent sum the remaining keys cannot change the outcome. + return !_stopAfterFirst && !_aggregate.DependsOnOrder; } - /// Decodes the / prefix of a key. + /// + /// Decodes the / prefix of a + /// [value][document id] key. The index was checked to be keyed by a + /// (); a key not laid out that way + /// (no prefix byte, or no document id after the value) is reported as malformed rather than + /// decoded. + /// public static bool TryDecode(ReadOnlySpan key, out TValue value) { value = default; @@ -3668,13 +3716,13 @@ public static bool TryDecode(ReadOnlySpan key, out TValue value) if (typeof(TValue) == typeof(int)) { - if (key.Length < 5) return false; + if (key.Length <= 5) return false; value = (TValue)(object)(int)(BinaryPrimitives.ReadUInt32BigEndian(key.Slice(1)) ^ 0x8000_0000u); return true; } if (typeof(TValue) == typeof(long)) { - if (key.Length < 9) return false; + if (key.Length <= 9) return false; value = (TValue)(object)(long)(BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)) ^ 0x8000_0000_0000_0000UL); return true; } @@ -3682,24 +3730,6 @@ public static bool TryDecode(ReadOnlySpan key, out TValue value) } } - /// - /// Whether is stored under a name other than the convention - /// follows, through the attributes the mapper - /// generator honours ([BsonProperty], [JsonPropertyName], [Column]). - /// - private static bool IsRenamedInBson(System.Reflection.PropertyInfo property) - { - foreach (var attribute in property.GetCustomAttributes(inherit: true)) - { - var name = attribute.GetType().Name; - if (name is "BsonPropertyAttribute" or "BsonProperty" - or "JsonPropertyNameAttribute" or "JsonPropertyName" - or "ColumnAttribute" or "Column") - return true; - } - return false; - } - /// /// Aggregates field of every document matching /// from raw BSON, in primary-key order. Gives up (Handled = @@ -3736,8 +3766,14 @@ bool Fold(ReadOnlySpan bson) var reader = new BsonSpanReader(bson, keyMap); reader.ReadDocumentSize(); - if (!reader.TrySeekToField(fieldId, out var type) && !SeekField(ref reader, fieldName, out type)) - return false; + var firstElement = reader.Position; + if (!reader.TrySeekToField(fieldId, out var type)) + { + // No usable offset table entry: look for the field from the first element on. + reader.Seek(firstElement); + if (!SeekField(ref reader, fieldName, out type)) + return false; + } if (type == BsonType.Null) return nullable; diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index 5d001bc9..11b0ab77 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -19,31 +19,6 @@ public class BTreeQueryProvider : IQueryProvider, IAsyncQueryProvider, I private readonly DocumentCollection _collection; private readonly ValueConverterRegistry _converterRegistry; - // ── Reflection cache (computed once per TId+T combination) ──────────────── - // BsonProjectionCompiler.TryCompile generic method definition. - // Targets the 3-parameter overload (selectLambda, whereLambda, keyMap) so that - // the key map can be threaded in for offset-table fast paths. -#pragma warning disable IL2026, IL2075 - private static readonly MethodInfo s_tryCompileMethod = - typeof(BsonProjectionCompiler) - .GetMethods(BindingFlags.Static | BindingFlags.Public) - .Single(m => m.Name == nameof(BsonProjectionCompiler.TryCompile) - && m.GetParameters().Length == 3); - - // DocumentCollection.ScanAsync(BsonReaderProjector, CancellationToken) — generic overload. - private static readonly MethodInfo s_scanProjectorMethod = - typeof(DocumentCollection) - .GetMethods(BindingFlags.Public | BindingFlags.Instance) - .Single(m => m.Name == "ScanAsync" && m.IsGenericMethodDefinition); -#pragma warning restore IL2026, IL2075 - - // ── Per-projection-type MakeGenericMethod cache (Fase 3) ───────────────── - // Keyed on TProj. Avoids calling MakeGenericMethod on every query execution. - private static readonly System.Collections.Concurrent.ConcurrentDictionary - s_compiledSelectMethods = new(); - private static readonly System.Collections.Concurrent.ConcurrentDictionary - s_compiledScanMethods = new(); - public BTreeQueryProvider(DocumentCollection collection, ValueConverterRegistry? registry = null) { _collection = collection; @@ -254,18 +229,6 @@ private async Task ExecuteAsyncCore(Expression expression, Can if (handled) return (TResult)value!; } - // ── Push-down SELECT (+ optional WHERE) ─────────────────────────────── - // Single-pass BSON projection: no T instantiation, no EnumerableRewriter. - // Only applicable when there is no post-scan ordering or paging. - if (model.SelectClause is not null - && model.OrderByClause is null - && !model.Take.HasValue - && !model.Skip.HasValue) - { - var pushed = TryPushDownSelect(model.SelectClause, model.WhereClause); - if (pushed is not null) return pushed; - } - // 2. Data fetching. // When Take (+ optional Skip) is present without complex operators or ordering that // requires a full scan first, cap how many rows we need to materialise. @@ -367,7 +330,7 @@ private TResult ExecutePipeline(QueryModel model, IEnumerable source if (model.Skip.HasValue) data = data.Skip(model.Skip.Value); if (model.Take.HasValue) data = data.Take(model.Take.Value); - // SELECT (with ORDER BY / SKIP / TAKE — rare path, push-down already handled the common case) + // SELECT if (model.SelectClause != null) return ProjectEnumerable(data, model.SelectClause); @@ -457,54 +420,6 @@ private static bool IsAggregateOverFilteredRoot(Expression expression) return source is ConstantExpression { Value: IQueryable }; } - // ─── Push-down SELECT helper ────────────────────────────────────────────── - - /// - /// Attempts to execute the query entirely via a single-pass BSON projection scan, - /// bypassing full deserialisation. - /// - /// The projected IEnumerable<TProj>, or null if push-down is - /// not applicable (caller should fall through to the standard path). - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private TResult? TryPushDownSelect(LambdaExpression selectLambda, - LambdaExpression? whereLambda = null) - { - // We need TResult = IEnumerable for some projection type TProj. - var resultType = typeof(TResult); - if (!resultType.IsGenericType) return default; - if (resultType.GetGenericTypeDefinition() != typeof(IEnumerable<>)) return default; - - var projType = resultType.GetGenericArguments()[0]; - if (projType == typeof(T)) return default; // not a real projection - - // Compile the push-down projector via reflection (TProj is only known at runtime). - // BsonProjectionCompiler.TryCompile(selectLambda, whereLambda?) handles - // both the pure-SELECT and the WHERE+SELECT cases in a single BSON pass. - object? projector; - try - { - var compileMethod = s_compiledSelectMethods.GetOrAdd( - projType, - t => s_tryCompileMethod.MakeGenericMethod(typeof(T), t)); - projector = compileMethod.Invoke(null, [selectLambda, whereLambda, _collection.GetKeyMap()]); - } - catch { return default; } - - if (projector is null) return default; // compilation soft-failed - - // Invoke _collection.Scan(projector) — MethodInfo cached per TProj. - try - { - var scanMethod = s_compiledScanMethods.GetOrAdd( - projType, - t => s_scanProjectorMethod.MakeGenericMethod(t)); - var result = scanMethod.Invoke(_collection, [projector]); - return (TResult?)result; - } - catch { return default; } - } - // ─── Expression-tree helpers ────────────────────────────────────────────── /// diff --git a/src/BLite.Core/Query/LinqAggregate.cs b/src/BLite.Core/Query/LinqAggregate.cs index 6e69638f..25902f28 100644 --- a/src/BLite.Core/Query/LinqAggregate.cs +++ b/src/BLite.Core/Query/LinqAggregate.cs @@ -26,29 +26,50 @@ internal enum AggregateOperator /// . /// /// Values must be added in the order LINQ would see them, since floating-point sums and checked -/// overflow depend on it. With exactTotal, integer sums are instead computed exactly and only -/// the total is range-checked: for values read in a different order (from an index) this throws -/// exactly when the result does not fit. +/// overflow depend on it. With anyOrder, int/long values may come in another +/// order (from an index): Sum/Average then keep the sums of the positive and of the +/// negative values apart. Every running total LINQ computes, in whatever order, lies between those +/// two, so when both fit the accumulator LINQ cannot overflow and the result is exact; otherwise +/// whether LINQ throws depends on the order and is set. /// internal sealed class LinqAggregate where TValue : struct, IComparable { private readonly AggregateOperator _op; private readonly bool _nullable; - private readonly bool _exactTotal; + private readonly bool _anyOrder; private long _count; private TValue _best; private int _sumInt32; private long _sumInt64; - private decimal _sumOverflow; // exactTotal: the part of a long sum beyond the long range + private long _sumPositive; // anyOrder: the sums of the positive and the negative values + private long _sumNegative; + private bool _accumulatorOverflow; private double _sumDouble; private decimal _sumDecimal; - public LinqAggregate(AggregateOperator op, bool nullable, bool exactTotal = false) + public LinqAggregate(AggregateOperator op, bool nullable, bool anyOrder = false) { _op = op; _nullable = nullable; - _exactTotal = exactTotal; + _anyOrder = anyOrder; + } + + /// + /// With anyOrder: whether the values added so far could make LINQ overflow in some order + /// but not in another, so must not be used; the values have to be aggregated + /// in the order LINQ sees them. + /// + public bool DependsOnOrder + { + get + { + if (!_anyOrder || _op is AggregateOperator.Min or AggregateOperator.Max) return false; + if (_accumulatorOverflow) return true; + // Sum of int accumulates in an int; Average of int and everything of long in a long. + return typeof(TValue) == typeof(int) && _op == AggregateOperator.Sum + && (_sumPositive > int.MaxValue || _sumNegative < int.MinValue); + } } /// @@ -88,31 +109,32 @@ public void Add(TValue value) private void Accumulate(TValue value) { - if (typeof(TValue) == typeof(int)) - { - int v = (int)(object)value; - if (_op == AggregateOperator.Sum && !_exactTotal) _sumInt32 = checked(_sumInt32 + v); - else _sumInt64 = checked(_sumInt64 + v); - } - else if (typeof(TValue) == typeof(long)) + if (_anyOrder && (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long))) { - long v = (long)(object)value; - if (!_exactTotal) + long v = typeof(TValue) == typeof(int) ? (int)(object)value : (long)(object)value; + if (v >= 0) { - _sumInt64 = checked(_sumInt64 + v); + long sum = unchecked(_sumPositive + v); + if (sum < _sumPositive) _accumulatorOverflow = true; + else _sumPositive = sum; } else { - long sum = unchecked(_sumInt64 + v); - // Signed overflow: both operands share a sign the result does not have. - if (((_sumInt64 ^ sum) & (v ^ sum)) < 0) - { - _sumOverflow += (decimal)_sumInt64 + v; - sum = 0; - } - _sumInt64 = sum; + long sum = unchecked(_sumNegative + v); + if (sum > _sumNegative) _accumulatorOverflow = true; + else _sumNegative = sum; } } + else if (typeof(TValue) == typeof(int)) + { + int v = (int)(object)value; + if (_op == AggregateOperator.Sum) _sumInt32 = checked(_sumInt32 + v); + else _sumInt64 = checked(_sumInt64 + v); + } + else if (typeof(TValue) == typeof(long)) + { + _sumInt64 = checked(_sumInt64 + (long)(object)value); + } else if (typeof(TValue) == typeof(double)) { _sumDouble += (double)(object)value; @@ -137,19 +159,21 @@ private void Accumulate(TValue value) /// public object? Result() { + if (DependsOnOrder) + throw new InvalidOperationException("The result depends on the order of the values."); + switch (_op) { case AggregateOperator.Sum: - if (typeof(TValue) == typeof(int)) return _exactTotal ? checked((int)_sumInt64) : _sumInt32; - if (typeof(TValue) == typeof(long)) return ExactInt64(); + if (typeof(TValue) == typeof(int)) return _anyOrder ? (int)(_sumPositive + _sumNegative) : _sumInt32; + if (typeof(TValue) == typeof(long)) return Int64Sum(); if (typeof(TValue) == typeof(double)) return _sumDouble; if (typeof(TValue) == typeof(float)) return (float)_sumDouble; return _sumDecimal; case AggregateOperator.Average: if (_count == 0) return NoElements(); - if (typeof(TValue) == typeof(int)) return (double)_sumInt64 / _count; - if (typeof(TValue) == typeof(long)) return (double)ExactInt64() / _count; + if (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long)) return (double)Int64Sum() / _count; if (typeof(TValue) == typeof(double)) return _sumDouble / _count; if (typeof(TValue) == typeof(float)) return (float)(_sumDouble / _count); return _sumDecimal / _count; @@ -159,7 +183,8 @@ private void Accumulate(TValue value) } } - private long ExactInt64() => _sumOverflow == 0 ? _sumInt64 : checked((long)(_sumOverflow + _sumInt64)); + // anyOrder: a positive and a negative long, so their sum cannot overflow. + private long Int64Sum() => _anyOrder ? _sumPositive + _sumNegative : _sumInt64; private object? NoElements() => _nullable ? null diff --git a/src/BLite.SourceGenerators/CodeGenerator.cs b/src/BLite.SourceGenerators/CodeGenerator.cs index 5cb63411..f00e565b 100644 --- a/src/BLite.SourceGenerators/CodeGenerator.cs +++ b/src/BLite.SourceGenerators/CodeGenerator.cs @@ -174,6 +174,8 @@ public static string GenerateMapperClass(EntityInfo entity, string mapperNamespa { sb.AppendLine(); GenerateIdAccessors(sb, entity); + sb.AppendLine(); + GenerateVerbatimFieldNames(sb, entity); } // Emit PersonalData static members (WP1) @@ -1075,6 +1077,41 @@ private static void GenerateIdAccessors(StringBuilder sb, EntityInfo entity) } } + /// + /// Emits GetVerbatimFieldName: for each scalar property that Deserialize restores + /// unchanged from its field, the field name, so queries can read the value from raw BSON. A + /// property qualifies when it is an auto-property with a setter, has no converter, and is not + /// passed to the constructor (which could transform it); computed, ignored and get-only + /// properties are not in or have no setter. + /// + private static void GenerateVerbatimFieldNames(StringBuilder sb, EntityInfo entity) + { + var ctorParams = entity.SelectedConstructorParameters != null + ? new HashSet(entity.SelectedConstructorParameters.Select(p => p.MatchedPropertyName)) + : new HashSet(); + + var verbatim = entity.Properties + .Where(p => p.IsAutoProperty + && p.HasAnySetter + && p.ConverterTypeName == null + && !p.IsCollection && !p.IsDictionary && !p.IsNestedObject && !p.IsEnum + && !ctorParams.Contains(p.Name)) + .ToList(); + + sb.AppendLine($" public override string? GetVerbatimFieldName(string propertyName)"); + sb.AppendLine($" {{"); + sb.AppendLine($" switch (propertyName)"); + sb.AppendLine($" {{"); + foreach (var prop in verbatim) + { + var fieldName = prop.IsKey ? BLiteConventions.BsonIdFieldName : prop.BsonFieldName; + sb.AppendLine($" case \"{prop.Name}\": return \"{fieldName}\";"); + } + sb.AppendLine($" default: return null;"); + sb.AppendLine($" }}"); + sb.AppendLine($" }}"); + } + private static string GetBaseMapperClass(PropertyInfo? keyProp, EntityInfo entity) { if (keyProp?.ConverterTypeName != null) diff --git a/src/BLite.SourceGenerators/EntityAnalyzer.cs b/src/BLite.SourceGenerators/EntityAnalyzer.cs index 9946080a..bfff24a0 100644 --- a/src/BLite.SourceGenerators/EntityAnalyzer.cs +++ b/src/BLite.SourceGenerators/EntityAnalyzer.cs @@ -185,6 +185,7 @@ private static void AnalyzeProperties(INamedTypeSymbol typeSymbol, List f.AssociatedSymbol?.Equals(property, SymbolEqualityComparer.Default) == true); } + /// + /// True when every accessor of is compiler-implemented + /// ({ get; set; }, { get; init; }, { get; private set; }), so the getter + /// returns exactly the value last assigned. False for accessors with a body, including ones + /// using the field keyword, for partial and abstract properties, and for record + /// positional parameters. + /// + public static bool IsAutoProperty(IPropertySymbol property) + { + if (property.IsAbstract || property.IsExtern) + return false; + + if (property.DeclaringSyntaxReferences.IsEmpty) + { + // Compiled assembly: auto-implemented accessors carry [CompilerGenerated]. A private + // setter may be invisible here (SetMethod == null); the getter decides then. + return IsCompilerGenerated(property.GetMethod) + && (property.SetMethod == null || IsCompilerGenerated(property.SetMethod)); + } + + foreach (var reference in property.DeclaringSyntaxReferences) + { + if (reference.GetSyntax() is not PropertyDeclarationSyntax declaration + || declaration.ExpressionBody != null + || declaration.AccessorList == null + || declaration.Modifiers.Any(SyntaxKind.PartialKeyword)) + return false; + + foreach (var accessor in declaration.AccessorList.Accessors) + { + if (accessor.Body != null || accessor.ExpressionBody != null) + return false; + } + } + return true; + + static bool IsCompilerGenerated(IMethodSymbol? accessor) + => accessor != null && accessor.GetAttributes().Any(a => + a.AttributeClass?.ToDisplayString() == "System.Runtime.CompilerServices.CompilerGeneratedAttribute"); + } + /// /// Returns true when is (or implements) IDictionary<TKey, TValue>. /// Populates and with the two type arguments. diff --git a/src/BLite.SourceGenerators/Models/EntityInfo.cs b/src/BLite.SourceGenerators/Models/EntityInfo.cs index c0ed88d7..51d77e53 100644 --- a/src/BLite.SourceGenerators/Models/EntityInfo.cs +++ b/src/BLite.SourceGenerators/Models/EntityInfo.cs @@ -95,6 +95,12 @@ public class PropertyInfo public bool HasInitOnlySetter { get; set; } public bool HasAnySetter { get; set; } public bool IsReadOnlyGetter { get; set; } + + /// + /// True when every accessor is compiler-implemented ({ get; set; }, { get; init; }, + /// { get; private set; }): the getter returns exactly what was last assigned. + /// + public bool IsAutoProperty { get; set; } public string? BackingFieldName { get; set; } /// diff --git a/tests/BLite.Shared/AggregateContexts.cs b/tests/BLite.Shared/AggregateContexts.cs index f4e8804a..0ea5a868 100644 --- a/tests/BLite.Shared/AggregateContexts.cs +++ b/tests/BLite.Shared/AggregateContexts.cs @@ -75,3 +75,104 @@ public partial class AggregateV2DbContext : DocumentDbContext protected override void OnModelCreating(ModelBuilder modelBuilder) => modelBuilder.Entity().ToCollection("readings"); } + +/// Stores hidden, unmapped and gone, which no longer reads. +public class ShapeV1 +{ + public ObjectId Id { get; set; } + public int Count { get; set; } + public int Hidden { get; set; } + public int Unmapped { get; set; } + public int Gone { get; set; } +} + +public partial class ShapeV1DbContext : DocumentDbContext +{ + public DocumentCollection Shapes { get; set; } = null!; + + public ShapeV1DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().ToCollection("shapes"); +} + +/// +/// A later version of : the fields older documents still hold are no longer +/// the values of these properties. +/// +public class ShapeV2 +{ + public ObjectId Id { get; set; } + public int Count { get; set; } + + [BsonIgnore] + public int Hidden { get; set; } + + [NotMapped] + public int Unmapped { get; set; } + + public int Gone => Count * 10; +} + +public partial class ShapeV2DbContext : DocumentDbContext +{ + public DocumentCollection Shapes { get; set; } = null!; + + public ShapeV2DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().ToCollection("shapes"); +} + +/// Accessors with logic: the stored value is not what the materialised getter returns. +public class Doubled +{ + private int _g; + private long _h; + + public ObjectId Id { get; set; } + + /// Stores 2 * _g; reading it back sets _g to that, so the getter doubles it again. + public int G { get => _g * 2; set => _g = value; } + + /// The same, with the logic in the setter of a property with a B-Tree index. + public long H { get => _h; set => _h = value + 1; } + + public int Plain { get; set; } +} + +/// A key named Code next to an ordinary property named Id. +public class Coded +{ + [System.ComponentModel.DataAnnotations.Key] + public int Code { get; set; } + + public int Id { get; set; } +} + +/// Its constructor transforms the value it is passed when the entity is read back. +public class Scaled +{ + public Scaled(int amount) => Amount = amount * 3; + + public ObjectId Id { get; set; } + public int Amount { get; set; } +} + +public partial class VerbatimDbContext : DocumentDbContext +{ + public DocumentCollection Doubled { get; set; } = null!; + public DocumentCollection Coded { get; set; } = null!; + public DocumentCollection Scaled { get; set; } = null!; + + public VerbatimDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("doubled") + .HasIndex(d => d.G) + .HasIndex(d => d.H); + modelBuilder.Entity().ToCollection("coded"); + modelBuilder.Entity().ToCollection("scaled"); + } +} diff --git a/tests/BLite.Tests/LinqAggregateTests.cs b/tests/BLite.Tests/LinqAggregateTests.cs index 02f9859b..a36aeb21 100644 --- a/tests/BLite.Tests/LinqAggregateTests.cs +++ b/tests/BLite.Tests/LinqAggregateTests.cs @@ -157,20 +157,71 @@ public async Task IntegerOverflow_ThrowsLikeLinq() Assert.Equal((int.MaxValue + 1.0) / 2, db.Readings.AsQueryable().Average(r => r.Count)); } + [Theory] + // In primary-key order. LINQ overflows on a running total although the total fits. + [InlineData(new[] { int.MaxValue, 1, -2 }, true)] + [InlineData(new[] { int.MinValue, -1, 5 }, true)] + // The positive or negative values alone overflow, but no running total in this order does. + [InlineData(new[] { 5, int.MinValue, -1 }, false)] + [InlineData(new[] { -2, int.MaxValue, 1 }, false)] + public async Task IndexKeySums_OverflowWhereLinqDoes(int[] values, bool linqOverflows) + { + // Index keys come in value order, not the order LINQ sees the entities in, and whether LINQ + // overflows depends on that order: such sums must not be taken from the keys. + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < values.Length; i++) + { + long total = values[i] switch { int.MaxValue => long.MaxValue, int.MinValue => long.MinValue, var v => v }; + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = values[i], Total = total }); + } + + var entities = db.Readings.AsQueryable().ToList(); + Assert.Equal(values, entities.Select(r => r.Count)); // primary-key order + Assert.Equal(linqOverflows, Run(() => entities.Sum(r => r.Count)).Error == typeof(OverflowException)); + + var q = db.Readings.AsQueryable(); + Assert.True(Same(Run(() => entities.Sum(r => r.Count)), Run(() => q.Sum(r => r.Count)))); + Assert.True(Same(Run(() => entities.Sum(r => r.Total)), Run(() => q.Sum(r => r.Total)))); + Assert.True(Same(Run(() => entities.Average(r => r.Total)), Run(() => q.Average(r => r.Total)))); + Assert.Equal(entities.Average(r => r.Count), q.Average(r => r.Count)); + + // With a filter the index covers, the entities come in index order. + var filtered = q.Where(r => r.Count < 100).ToList(); + Assert.True(Same(Run(() => filtered.Sum(r => r.Count)), Run(() => q.Where(r => r.Count < 100).Sum(r => r.Count)))); + var filteredTotal = q.Where(r => r.Total < 100).ToList(); + Assert.True(Same(Run(() => filteredTotal.Sum(r => r.Total)), Run(() => q.Where(r => r.Total < 100).Sum(r => r.Total)))); + } + + [Fact] + public async Task IndexOnAConvertedProperty_IsNotDecodedAsThePropertyType() + { + // The keys of this index are longs although its property path is the int Units. + using var db = new AggregateDbContext(_dbPath); + await db.Readings.CreateIndexAsync(r => (long)r.Units); + for (int i = 0; i < 10; i++) + await db.Readings.InsertAsync(Make(i)); + + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + Assert.Equal(entities.Sum(r => r.Units), q.Sum(r => r.Units)); + Assert.Equal(entities.Min(r => r.Units), q.Min(r => r.Units)); + Assert.Equal(entities.Max(r => r.Units), q.Max(r => r.Units)); + } + [Fact] - public async Task IndexKeySums_CheckOnlyTheTotalForOverflow() + public async Task IndexKeySums_ThatCannotOverflow_AreTakenFromTheKeys() { - // Index keys come in value order, not document order, so an index sum cannot reproduce an - // overflow of a running total that LINQ would hit before a later negative value: it returns - // the exact total when that fits, and throws when it does not. using var db = new AggregateDbContext(_dbPath); - foreach (var count in new[] { int.MaxValue, 1, -2 }) - await db.Readings.InsertAsync(new Reading { Count = count, Units = count }); + foreach (var count in new[] { int.MaxValue - 10, 4, -7, int.MinValue + 3, 6 }) + await db.Readings.InsertAsync(new Reading { Count = count, Total = count }); - // Whether LINQ overflows depends on the document order (int.MaxValue + 1 before the -2). var entities = db.Readings.AsQueryable().ToList(); - Assert.True(Same(Run(() => entities.Sum(r => r.Units)), Run(() => db.Readings.AsQueryable().Sum(r => r.Units)))); // raw BSON, document order - Assert.Equal(int.MaxValue - 1, db.Readings.AsQueryable().Sum(r => r.Count)); // index keys: the exact total + var sum = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Count"), null, typeof(int)); + Assert.True(sum.Handled); + Assert.Equal(entities.Sum(r => r.Count), sum.Value); + var average = await db.Readings.TryAggregateAsync(AggregateOperator.Average, Selector("Total"), null, typeof(double)); + Assert.True(average.Handled); + Assert.Equal(entities.Average(r => r.Total), average.Value); } [Fact] @@ -237,17 +288,190 @@ await col.InsertAsync(col.CreateDocument(["_id", "tag", field], b => } [Fact] - public async Task RenamedProperty_IsNotReadFromAnotherField() + public async Task RenamedProperty_IsReadFromItsOwnField() { using var db = new AggregateDbContext(_dbPath); for (int i = 0; i < 5; i++) await db.Readings.InsertAsync(Make(i)); - Assert.False((await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Level"), null, typeof(int))).Handled); + // Level is stored as "score" and Score as "level". + var level = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Level"), null, typeof(int)); + var score = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Score"), null, typeof(int)); + Assert.Equal((true, (object?)10), level); + Assert.Equal((true, (object?)510), score); Assert.Equal(10, db.Readings.AsQueryable().Sum(r => r.Level)); Assert.Equal(510, db.Readings.AsQueryable().Sum(r => r.Score)); } + [Fact] + public async Task FieldAfterOthers_InDocumentsWithoutOffsetTable_IsFound() + { + using (var db = new AggregateDbContext(_dbPath)) + { + for (int i = 0; i < 4; i++) + await db.Readings.InsertAsync(Make(i)); + } + + // The schemaless API writes no offset table, so the field is looked up by name, after the + // fields before it. + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("readings"); + for (int i = 0; i < 3; i++) + { + var units = 1000 * (i + 1); + await col.InsertAsync(col.CreateDocument(["_id", "tag", "units"], b => + b.AddObjectId("_id", ObjectId.NewObjectId()).AddString("tag", "x").AddInt32("units", units))); + } + } + + using var reopened = new AggregateDbContext(_dbPath); + var entities = reopened.Readings.AsQueryable().ToList(); + Assert.Equal(7, entities.Count); + + var sum = await reopened.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Units"), null, typeof(int)); + Assert.True(sum.Handled); + Assert.Equal(entities.Sum(r => r.Units), sum.Value); + Expression> tagX = r => r.Tag == "x"; + var filtered = await reopened.Readings.TryAggregateAsync(AggregateOperator.Max, Selector("Units"), tagX, typeof(int)); + Assert.Equal((true, (object?)3000), filtered); + } + + /// Writes five documents and reopens them as . + private async Task ShapesWrittenBeforeTheyWereIgnoredAsync() + { + using (var v1 = new ShapeV1DbContext(_dbPath)) + { + for (int i = 0; i < 5; i++) + await v1.Shapes.InsertAsync(new ShapeV1 { Count = i, Hidden = 100 + i, Unmapped = 200 + i, Gone = 1000 + i }); + } + return new ShapeV2DbContext(_dbPath); + } + + [Fact] + public async Task IgnoredProperty_OverDocumentsThatStillHoldTheField_IsMaterialised() + { + using var db = await ShapesWrittenBeforeTheyWereIgnoredAsync(); + var shapes = db.Shapes.AsQueryable().ToList(); + var q = db.Shapes.AsQueryable(); + + Assert.Equal(0, shapes.Sum(s => s.Hidden)); + Assert.Equal(shapes.Sum(s => s.Hidden), q.Sum(s => s.Hidden)); // [BsonIgnore] + Assert.Equal(shapes.Max(s => s.Hidden), q.Max(s => s.Hidden)); + Assert.Equal(shapes.Sum(s => s.Unmapped), q.Sum(s => s.Unmapped)); // [NotMapped] + Assert.Equal(shapes.Sum(s => s.Count), q.Sum(s => s.Count)); + } + + [Fact] + public async Task ComputedProperty_OverDocumentsThatStoredIt_IsMaterialised() + { + using var db = await ShapesWrittenBeforeTheyWereIgnoredAsync(); + var shapes = db.Shapes.AsQueryable().ToList(); + var q = db.Shapes.AsQueryable(); + + Assert.Equal(100, shapes.Sum(s => s.Gone)); + Assert.Equal(shapes.Sum(s => s.Gone), q.Sum(s => s.Gone)); + Assert.Equal(shapes.Min(s => s.Gone), q.Min(s => s.Gone)); + Assert.Equal(shapes.Average(s => s.Gone), q.Average(s => s.Gone)); + } + + [Fact] + public async Task AccessorsWithLogic_AreMaterialised() + { + using var db = new VerbatimDbContext(_dbPath); + for (int i = 1; i <= 4; i++) + await db.Doubled.InsertAsync(new Doubled { G = i * 5, H = i * 7, Plain = i }); + + var doubled = db.Doubled.AsQueryable().ToList(); + var q = db.Doubled.AsQueryable(); + + // G: getter with logic, indexed (index keys) and filtered on another property (raw BSON). + Assert.Equal(doubled.Sum(d => d.G), q.Sum(d => d.G)); + Assert.Equal(doubled.Min(d => d.G), q.Min(d => d.G)); + Assert.Equal(doubled.Max(d => d.G), q.Max(d => d.G)); + Assert.Equal(doubled.Average(d => d.G), q.Average(d => d.G)); + Assert.Equal(doubled.Where(d => d.Plain > 1).Sum(d => d.G), q.Where(d => d.Plain > 1).Sum(d => d.G)); + Assert.Equal(doubled.Where(d => d.Plain > 1).Max(d => d.G), q.Where(d => d.Plain > 1).Max(d => d.G)); + // The index on G holds the stored values, so the filter matches what the index says (as on + // main); the aggregate must still use the materialised values of those documents. + Assert.Equal(q.Where(d => d.G > 15).ToList().Sum(d => d.G), q.Where(d => d.G > 15).Sum(d => d.G)); + + // H: setter with logic, indexed long. + Assert.Equal(doubled.Sum(d => d.H), q.Sum(d => d.H)); + Assert.Equal(doubled.Min(d => d.H), q.Min(d => d.H)); + Assert.Equal(doubled.Max(d => d.H), q.Max(d => d.H)); + Assert.Equal(doubled.Average(d => d.H), q.Average(d => d.H)); + + // A plain property next to them is still aggregated without materialising. + var plain = await db.Doubled.TryAggregateAsync(AggregateOperator.Sum, (Expression>)(d => d.Plain), null, typeof(int)); + Assert.Equal((true, (object?)10), plain); + } + + [Fact] + public async Task PropertyNamedId_ThatIsNotTheKey_IsReadFromItsOwnField() + { + using var db = new VerbatimDbContext(_dbPath); + for (int i = 1; i <= 4; i++) + await db.Coded.InsertAsync(new Coded { Code = i, Id = i * 100 }); + + var coded = db.Coded.AsQueryable().ToList(); + Assert.Equal(1000, coded.Sum(c => c.Id)); + Assert.Equal(coded.Sum(c => c.Id), db.Coded.AsQueryable().Sum(c => c.Id)); + Assert.Equal(coded.Max(c => c.Code), db.Coded.AsQueryable().Max(c => c.Code)); + + // Stored as "id", not "_id": still read from raw BSON. + var id = await db.Coded.TryAggregateAsync(AggregateOperator.Sum, (Expression>)(c => c.Id), null, typeof(int)); + Assert.Equal((true, (object?)1000), id); + } + + [Fact] + public async Task PropertySetThroughTheConstructor_IsMaterialised() + { + using var db = new VerbatimDbContext(_dbPath); + for (int i = 1; i <= 4; i++) + await db.Scaled.InsertAsync(new Scaled(i)); + + var scaled = db.Scaled.AsQueryable().ToList(); + Assert.Equal(scaled.Sum(s => s.Amount), db.Scaled.AsQueryable().Sum(s => s.Amount)); + Assert.Equal(scaled.Max(s => s.Amount), db.Scaled.AsQueryable().Max(s => s.Amount)); + } + + [Fact] + public void GeneratedMappers_NameTheFieldOnlyForPropertiesStoredVerbatim() + { + var reading = MapperOf(); + Assert.Equal("_id", reading.GetVerbatimFieldName("Id")); + Assert.Equal("count", reading.GetVerbatimFieldName("Count")); + Assert.Equal("maybeprice", reading.GetVerbatimFieldName("MaybePrice")); + Assert.Equal("score", reading.GetVerbatimFieldName("Level")); // [Column("score")] + Assert.Equal("level", reading.GetVerbatimFieldName("Score")); + Assert.Null(reading.GetVerbatimFieldName("NoSuchProperty")); + + var shape = MapperOf(); + Assert.Equal("count", shape.GetVerbatimFieldName("Count")); + Assert.Null(shape.GetVerbatimFieldName("Hidden")); // [BsonIgnore] + Assert.Null(shape.GetVerbatimFieldName("Unmapped")); // [NotMapped] + Assert.Null(shape.GetVerbatimFieldName("Gone")); // computed + + var doubled = MapperOf(); + Assert.Null(doubled.GetVerbatimFieldName("G")); // getter with logic + Assert.Null(doubled.GetVerbatimFieldName("H")); // setter with logic + Assert.Equal("plain", doubled.GetVerbatimFieldName("Plain")); + + var coded = MapperOf(); + Assert.Equal("_id", coded.GetVerbatimFieldName("Code")); + Assert.Equal("id", coded.GetVerbatimFieldName("Id")); + + Assert.Null(MapperOf().GetVerbatimFieldName("Amount")); // set through the constructor + } + + private static DocumentMapperBase MapperOf() where TEntity : class + { + var type = typeof(VerbatimDbContext).Assembly.GetTypes() + .First(t => !t.IsAbstract && typeof(DocumentMapperBase).IsAssignableFrom(t)); + return (DocumentMapperBase)Activator.CreateInstance(type)!; + } + [Fact] public async Task OtherOperatorsBeforeTheAggregate_AreApplied() { From a45fd4579b1ef3fff152f494939f05d403539c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 17:38:14 +0200 Subject: [PATCH 4/6] test(query): make the non-overflowing index-sum test deterministic (#183) The data's negative values summed to int.MinValue - 4, so whether the sum came from the index depended on the order the random ObjectIds put the documents in: the index path correctly declined (the outcome depends on the order), and when the raw scan then hit an overflow in that order it declined too, because LINQ throws there. Use values whose positive and negative sums are exactly int.MaxValue and int.MinValue, the edge of the gate, and explicit ascending ids. Co-Authored-By: Claude Opus 5.5 --- tests/BLite.Tests/LinqAggregateTests.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/BLite.Tests/LinqAggregateTests.cs b/tests/BLite.Tests/LinqAggregateTests.cs index a36aeb21..38a21679 100644 --- a/tests/BLite.Tests/LinqAggregateTests.cs +++ b/tests/BLite.Tests/LinqAggregateTests.cs @@ -211,9 +211,17 @@ public async Task IndexOnAConvertedProperty_IsNotDecodedAsThePropertyType() [Fact] public async Task IndexKeySums_ThatCannotOverflow_AreTakenFromTheKeys() { + // The positive values sum to exactly int.MaxValue and the negative ones to exactly + // int.MinValue, so no running total overflows in any order and the sum is taken from the + // keys. (One more below int.MinValue and the result would depend on the order: see + // IndexKeySums_OverflowWhereLinqDoes.) Explicit ids keep the document order fixed. + int[] counts = [int.MaxValue - 10, 4, -7, int.MinValue + 7, 6]; + Assert.Equal(int.MaxValue, counts.Where(c => c > 0).Sum(c => (long)c)); + Assert.Equal(int.MinValue, counts.Where(c => c < 0).Sum(c => (long)c)); + using var db = new AggregateDbContext(_dbPath); - foreach (var count in new[] { int.MaxValue - 10, 4, -7, int.MinValue + 3, 6 }) - await db.Readings.InsertAsync(new Reading { Count = count, Total = count }); + for (int i = 0; i < counts.Length; i++) + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = counts[i], Total = counts[i] }); var entities = db.Readings.AsQueryable().ToList(); var sum = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Count"), null, typeof(int)); From eb21083a0765b5febd69fd1e9993c847d3a7ac2c Mon Sep 17 00:00:00 2001 From: MrDevRobot Date: Thu, 15 Oct 2026 16:16:18 +0200 Subject: [PATCH 5/6] fix(query): don't take a filter bound of another type as an index-key range The optimizer keeps the constant's type, so r.Count < 9.5 or r.Count > 4L gave a double or long bound that was encoded and compared bytewise against the int keys: Where(r => r.Count < 9.5).Max(r => r.Count) returned 19 instead of 9. Use the keys only when each bound is null or exactly TValue; other filters post-filter the documents as before. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Collections/DocumentCollection.cs | 8 ++++++-- tests/BLite.Tests/LinqAggregateTests.cs | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index fcb0bcc8..c598b48f 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -3593,9 +3593,13 @@ internal async Task CountByPredicateAsync( && !plan.IsVectorSearch && !plan.IsSpatialSearch && plan.InValues == null - && plan.MinValue is not DBNull - && plan.MaxValue is not DBNull) + && plan.MinValue is null or TValue + && plan.MaxValue is null or TValue) { + // The optimizer keeps the constant's own type (r.Count < 9.5 gives a double + // bound), and keys carry no type tag, so a bound of another type would be + // compared bytewise against the int/long keys. Such filters take the path below, + // which post-filters the documents the index returns. fromKeys = AggregateIndexKeys(op, nullable, index, plan.MinValue, plan.MaxValue, plan.StartInclusive, plan.EndInclusive); } if (fromKeys.Handled) diff --git a/tests/BLite.Tests/LinqAggregateTests.cs b/tests/BLite.Tests/LinqAggregateTests.cs index 38a21679..16915acc 100644 --- a/tests/BLite.Tests/LinqAggregateTests.cs +++ b/tests/BLite.Tests/LinqAggregateTests.cs @@ -208,6 +208,22 @@ public async Task IndexOnAConvertedProperty_IsNotDecodedAsThePropertyType() Assert.Equal(entities.Max(r => r.Units), q.Max(r => r.Units)); } + [Fact] + public async Task FilterBoundOfAnotherType_IsNotUsedAsAKeyRange() + { + // The optimizer keeps the constant's type, so these bounds are a double and a long against + // the int Count index; encoded as keys they would bound the wrong range. + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 20; i++) + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = i, Total = i }); + + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + Assert.Equal(entities.Where(r => r.Count < 9.5).Max(r => r.Count), q.Where(r => r.Count < 9.5).Max(r => r.Count)); + Assert.Equal(entities.Where(r => r.Count > 4L).Sum(r => r.Count), q.Where(r => r.Count > 4L).Sum(r => r.Count)); + Assert.Equal(entities.Where(r => r.Count > 4).Sum(r => r.Count), q.Where(r => r.Count > 4).Sum(r => r.Count)); + } + [Fact] public async Task IndexKeySums_ThatCannotOverflow_AreTakenFromTheKeys() { From bbb72ec4ac0f52e3b825209340643e03e2645f50 Mon Sep 17 00:00:00 2001 From: MrDevRobot Date: Thu, 15 Oct 2026 16:19:11 +0200 Subject: [PATCH 6/6] test(query): aggregate over a filter that queries the collection again Co-Authored-By: Claude Opus 5.5 --- tests/BLite.Tests/LinqAggregateTests.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/BLite.Tests/LinqAggregateTests.cs b/tests/BLite.Tests/LinqAggregateTests.cs index 16915acc..5522b62e 100644 --- a/tests/BLite.Tests/LinqAggregateTests.cs +++ b/tests/BLite.Tests/LinqAggregateTests.cs @@ -224,6 +224,23 @@ public async Task FilterBoundOfAnotherType_IsNotUsedAsAKeyRange() Assert.Equal(entities.Where(r => r.Count > 4).Sum(r => r.Count), q.Where(r => r.Count > 4).Sum(r => r.Count)); } + [Fact] + public async Task FilterWithASubqueryOverTheCollection_MatchesLinq() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 20; i++) + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = i % 7, Total = i }); + + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + Assert.Equal( + entities.Where(r => entities.Any(o => o.Count == r.Count + 1)).Sum(r => r.Count), + q.Where(r => q.Any(o => o.Count == r.Count + 1)).Sum(r => r.Count)); + Assert.Equal( + entities.Where(r => r.Count > 2 && entities.Count(o => o.Total < r.Total) > 3).Max(r => r.Total), + q.Where(r => r.Count > 2 && q.Count(o => o.Total < r.Total) > 3).Max(r => r.Total)); + } + [Fact] public async Task IndexKeySums_ThatCannotOverflow_AreTakenFromTheKeys() {