diff --git a/src/BLite.Core/Collections/BaseMappers.cs b/src/BLite.Core/Collections/BaseMappers.cs index 092d846a..a5c79e5d 100644 --- a/src/BLite.Core/Collections/BaseMappers.cs +++ b/src/BLite.Core/Collections/BaseMappers.cs @@ -22,6 +22,18 @@ public abstract class DocumentMapperBase : IDocumentMapper where public virtual IndexKey ToIndexKey(TId id) => IndexKey.Create(id); public virtual TId FromIndexKey(IndexKey key) => key.As(); + /// + /// The BSON field stores in, when + /// gives the property exactly the value of that field (a plain + /// auto-property with no converter), so the value of the materialised property can be read from + /// the raw document instead; null when that does not hold or is not known. + /// + /// + /// Generated mappers override this. The default returns null, so queries that could read a + /// property from raw BSON materialise the entities instead. + /// + public virtual string? GetVerbatimFieldName(string propertyName) => null; + #pragma warning disable IL2026 public virtual IEnumerable UsedKeys => GetSchema().GetAllKeys(); #pragma warning restore IL2026 diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 83a2b22d..638e84ca 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -535,9 +535,19 @@ private static bool TryReadIndexKeyFromReader(ref BsonSpanReader reader, BsonTyp return false; } - private async IAsyncEnumerable<(IndexEntry Entry, byte[] RawBytes)> EnumerateOwnedRawDocumentsAsync( + /// Processes the raw BSON of one document in place; returns whether to yield . + private delegate bool RawDocumentVisitor(ReadOnlySpan bson, out TResult result); + + /// + /// Visits the raw BSON of every document in primary-key order. Single-page documents are + /// handed to straight from the page buffer, without a copy; only + /// overflow documents are reassembled. Each page is read once and kept until the scan ends: + /// primary-key order is not page order (ObjectIds created in the same second are not + /// monotonic), so a bounded window of pages would read the same pages over and over. + /// + private async IAsyncEnumerable VisitRawDocumentsAsync( ulong txnId, - bool includeOverflow, + RawDocumentVisitor visit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) { var pageCache = new Dictionary(); @@ -550,27 +560,42 @@ private static bool TryReadIndexKeyFromReader(ref BsonSpanReader reader, BsonTyp { ct.ThrowIfCancellationRequested(); - if (!pageCache.TryGetValue(entry.Location.PageId, out var cachedBuffer)) + if (!pageCache.TryGetValue(entry.Location.PageId, out var pageBuffer)) { - cachedBuffer = ArrayPool.Shared.Rent(_storage.PageSize); - _storage.ReadPage(entry.Location.PageId, txnId, cachedBuffer); - pageCache[entry.Location.PageId] = cachedBuffer; + pageBuffer = ArrayPool.Shared.Rent(_storage.PageSize); + _storage.ReadPage(entry.Location.PageId, txnId, pageBuffer); + pageCache[entry.Location.PageId] = pageBuffer; } - var rawBytes = ReadRawBytesAt(entry.Location, txnId, includeOverflow, cachedBuffer); - if (rawBytes is null) continue; - if (!MatchesIndexEntryKey(rawBytes, entry.Key)) continue; - - yield return (entry, rawBytes); + if (TryVisitRawDocument(entry, txnId, pageBuffer, visit, out var result)) + yield return result; } } finally { - foreach (var buf in pageCache.Values) - ArrayPool.Shared.Return(buf); + foreach (var buf in pageCache.Values) ArrayPool.Shared.Return(buf); } } + /// + /// Applies to the document points at, if that + /// slot still holds the document with this primary key. + /// + private bool TryVisitRawDocument(IndexEntry entry, ulong txnId, byte[] pageBuffer, RawDocumentVisitor visit, out TResult result) + { + result = default!; + + var pageType = (PageType)pageBuffer[4]; + if (pageType == PageType.Free || pageType == PageType.Empty || pageType == PageType.TimeSeries) + return false; + + if (TryReadInlineRawBytes(pageBuffer, entry.Location.SlotIndex, out var inline)) + return MatchesIndexEntryKey(inline, entry.Key) && visit(inline, out result); + + var reassembled = ReadRawBytesAt(entry.Location, txnId, includeOverflow: true, pageBuffer); + return reassembled is not null && MatchesIndexEntryKey(reassembled, entry.Key) && visit(reassembled, out result); + } + /// /// Extracts a UTC-ticks timestamp from raw BSON bytes using the given field name. /// Resolves the field name against the Key Dictionary for correct C-BSON lookup, @@ -1131,10 +1156,15 @@ public async IAsyncEnumerable ScanAsync( try { - await foreach (var (_, rawBytes) in EnumerateOwnedRawDocumentsAsync(txnId, includeOverflow: true, ct).ConfigureAwait(false)) + await foreach (var document in VisitRawDocumentsAsync(txnId, (ReadOnlySpan bson, out T doc) => + { + doc = default!; + if (!predicate(new BsonSpanReader(bson, keyMap))) return false; + doc = _mapper.Deserialize(new BsonSpanReader(bson, keyMap)); + return true; + }, ct).ConfigureAwait(false)) { - if (!predicate(new BsonSpanReader(rawBytes, keyMap))) continue; - yield return _mapper.Deserialize(new BsonSpanReader(rawBytes, keyMap)); + yield return document; } } finally @@ -1296,10 +1326,13 @@ public async IAsyncEnumerable ScanAsync( var txnId = 0UL; var keyMap = _storage.GetKeyReverseMap(); - await foreach (var (_, rawBytes) in EnumerateOwnedRawDocumentsAsync(txnId, includeOverflow: true, ct).ConfigureAwait(false)) + await foreach (var result in VisitRawDocumentsAsync(txnId, (ReadOnlySpan bson, out TResult projected) => + { + projected = projector(new BsonSpanReader(bson, keyMap))!; + return projected is not null; + }, ct).ConfigureAwait(false)) { - var result = projector(new BsonSpanReader(rawBytes, keyMap)); - if (result is not null) yield return result; + yield return result; } } @@ -1322,16 +1355,20 @@ internal async Task CountScanAsync( int count = 0; var keyMap = _storage.GetKeyReverseMap(); - await foreach (var (_, rawBytes) in EnumerateOwnedRawDocumentsAsync(txnId, includeOverflow: true, ct).ConfigureAwait(false)) - { - try + await foreach (var _ in VisitRawDocumentsAsync(txnId, (ReadOnlySpan bson, out bool matched) => { - if (predicate(new BsonSpanReader(rawBytes, keyMap))) count++; - } - catch - { - // Malformed BSON — skip silently. - } + try + { + matched = predicate(new BsonSpanReader(bson, keyMap)); + } + catch + { + matched = false; // Malformed BSON — skip silently. + } + return matched; + }, ct).ConfigureAwait(false)) + { + count++; } return count; @@ -3583,6 +3620,331 @@ internal async Task CountByPredicateAsync( return count; } + /// + /// Computes Sum, Average, Min or Max of over + /// the documents matching without materialising them, with the + /// result LINQ would compute over the entities. Returns Handled = false when the query + /// must run the usual way. + /// + /// Only for a property the mapper restores verbatim from a BSON field + /// (): the stored value, in the + /// document or in an index key, is then the value of the materialised property. Ignored, + /// computed and converted properties, and ones whose accessors have logic, are materialised. + /// + /// + /// An int/long property with a B-Tree index on exactly that property, and + /// no filter or one the same index covers exactly, is aggregated from the index keys + /// alone (Min/Max read one entry). Keys are in value order, so a + /// Sum/Average for which LINQ could overflow in one order but not in + /// another is not taken from them. + /// Otherwise, when no index serves the filter, the property is read from the raw BSON + /// of each document in primary-key order, the order a scan yields the entities in. A + /// document without the field, or holding a type the entity would not be read from + /// unchanged, ends the attempt so the query runs the usual way. + /// + /// + /// A plain property access, x => x.Property. + /// The result type of the LINQ operator. + [RequiresDynamicCode("Aggregate push-down uses index optimization and Expression.Compile() which require dynamic code generation.")] + [RequiresUnreferencedCode("Aggregate push-down uses reflection to resolve members at runtime. Ensure all entity types are preserved.")] + internal Task<(bool Handled, object? Value)> TryAggregateAsync( + AggregateOperator op, + LambdaExpression selector, + LambdaExpression? whereClause, + Type resultType, + CancellationToken ct = default) + { + if (selector.Body is not MemberExpression { Member: System.Reflection.PropertyInfo property } member + || member.Expression != selector.Parameters[0] + || selector.ReturnType != property.PropertyType + || ConverterRegistry.HasConverter(property.Name)) + return Task.FromResult<(bool, object?)>(default); + + var valueType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; + bool nullable = valueType != property.PropertyType; + + if (valueType == typeof(int)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(long)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(double)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(float)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + if (valueType == typeof(decimal)) return TryAggregateAsync(op, property, nullable, whereClause, resultType, ct); + return Task.FromResult<(bool, object?)>(default); + } + + [RequiresDynamicCode("Aggregate push-down uses index optimization and Expression.Compile() which require dynamic code generation.")] + [RequiresUnreferencedCode("Aggregate push-down uses reflection to resolve members at runtime. Ensure all entity types are preserved.")] + private async Task<(bool Handled, object? Value)> TryAggregateAsync( + AggregateOperator op, + System.Reflection.PropertyInfo property, + bool nullable, + LambdaExpression? whereClause, + Type resultType, + CancellationToken ct) + where TValue : struct, IComparable + { + if (LinqAggregate.ResultType(op, nullable) != resultType) + return default; + + // The field the mapper restores the property from unchanged. Without it, neither the raw + // field nor an index key (computed from the property when the document was written) is + // known to be the value the materialised entity returns. + var fieldName = (_mapper as DocumentMapperBase)?.GetVerbatimFieldName(property.Name); + if (fieldName == null) + return default; + + var plan = whereClause == null ? null : Query.IndexOptimizer.TryOptimize(whereClause, GetIndexes(), ConverterRegistry); + + // Index keys hold the exact int/long value. Not for the other types: floating-point sums + // depend on the order of the values, decimal keys drop trailing zeros (1.50 → 1.5), and the + // first of equal Min/Max values wins (-0.0 vs 0.0, 1.0 vs 1.00). + if (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long)) + { + var index = _indexManager.GetAllIndexes().FirstOrDefault(i => + i.Definition.Type == IndexType.BTree + && !i.Definition.IsPrimary + && i.Definition.PropertyPaths.Length == 1 + && i.Definition.PropertyPaths[0] == property.Name + && KeysAreValuesOf(i.Definition, property, typeof(TValue))); + + if (index != null) + { + (bool Handled, object? Value) fromKeys = default; + if (whereClause == null) + { + fromKeys = AggregateIndexKeys(op, nullable, index, null, null, true, true); + } + else if (plan != null + && plan.IndexName == index.Name + && !plan.HasResiduePredicate + && !plan.IsVectorSearch + && !plan.IsSpatialSearch + && plan.InValues == null + && plan.MinValue is null or TValue + && plan.MaxValue is null or TValue) + { + // The optimizer keeps the constant's own type (r.Count < 9.5 gives a double + // bound), and keys carry no type tag, so a bound of another type would be + // compared bytewise against the int/long keys. Such filters take the path below, + // which post-filters the documents the index returns. + fromKeys = AggregateIndexKeys(op, nullable, index, plan.MinValue, plan.MaxValue, plan.StartInclusive, plan.EndInclusive); + } + if (fromKeys.Handled) + return fromKeys; + // Otherwise aggregate the values in the order LINQ sees them, below. + } + } + + // An index serves the filter: fetching the matching documents through it is cheaper than + // scanning them all, and keeps the order the values are aggregated in unchanged. + if (plan != null) + return default; + + BsonReaderPredicate? predicate = null; + if (whereClause != null) + { + predicate = BsonExpressionEvaluator.TryCompile(whereClause, ConverterRegistry, _storage.GetKeyMap()); + if (predicate == null) return default; + } + + return await AggregateRawFieldAsync(op, nullable, fieldName, predicate, ct).ConfigureAwait(false); + } + + /// + /// Whether the keys of the index are the values of + /// itself (x => x.Property), encoded as + /// : the user key is encoded from the boxed value the selector + /// returns, so this fixes the key layout the index path decodes. + /// + private static bool KeysAreValuesOf(CollectionIndexDefinition definition, System.Reflection.PropertyInfo property, Type valueType) + { + var body = definition.KeySelectorExpression.Body; + while (body is UnaryExpression { NodeType: ExpressionType.Convert } convert && convert.Type == typeof(object)) + body = convert.Operand; + + return body is MemberExpression { Member: System.Reflection.PropertyInfo keyProperty } access + && access.Expression == definition.KeySelectorExpression.Parameters[0] + && keyProperty.Name == property.Name + && (Nullable.GetUnderlyingType(access.Type) ?? access.Type) == valueType; + } + + /// Aggregates the values of the index entries in the given range from their keys. + private static (bool Handled, object? Value) AggregateIndexKeys( + AggregateOperator op, + bool nullable, + CollectionSecondaryIndex index, + object? min, + object? max, + bool startInclusive, + bool endInclusive) + where TValue : struct, IComparable + { + var aggregate = new LinqAggregate(op, nullable, anyOrder: true); + + if (op == AggregateOperator.Max) + { + // Keys are [value][document id]: the last entry holds the largest value. + var last = index.LastKey(min, max, startInclusive, endInclusive); + if (last is { } key) + { + if (!IndexKeyValueVisitor.TryDecode(key.Data, out var value)) return default; + aggregate.Add(value); + } + return (true, aggregate.Result()); + } + + var visitor = new IndexKeyValueVisitor(aggregate, stopAfterFirst: op == AggregateOperator.Min); + index.ScanKeys(min, max, startInclusive, endInclusive, ref visitor); + return visitor.Malformed || aggregate.DependsOnOrder ? default : (true, aggregate.Result()); + } + + /// + /// Feeds the value prefix of each [value][document id] key of an int or + /// long index to a . + /// + private struct IndexKeyValueVisitor : BTreeIndex.IKeyVisitor + where TValue : struct, IComparable + { + private readonly LinqAggregate _aggregate; + private readonly bool _stopAfterFirst; + + public IndexKeyValueVisitor(LinqAggregate aggregate, bool stopAfterFirst) + { + _aggregate = aggregate; + _stopAfterFirst = stopAfterFirst; + Malformed = false; + } + + /// A key did not start with an encoded value of the expected type. + public bool Malformed { get; private set; } + + public bool Visit(ReadOnlySpan key) + { + if (!TryDecode(key, out var value)) + { + Malformed = true; + return false; + } + _aggregate.Add(value); + // Past an order-dependent sum the remaining keys cannot change the outcome. + return !_stopAfterFirst && !_aggregate.DependsOnOrder; + } + + /// + /// Decodes the / prefix of a + /// [value][document id] key. The index was checked to be keyed by a + /// (); a key not laid out that way + /// (no prefix byte, or no document id after the value) is reported as malformed rather than + /// decoded. + /// + public static bool TryDecode(ReadOnlySpan key, out TValue value) + { + value = default; + if (key.Length == 0 || key[0] != IndexKey.KeyPrefix) return false; + + if (typeof(TValue) == typeof(int)) + { + if (key.Length <= 5) return false; + value = (TValue)(object)(int)(BinaryPrimitives.ReadUInt32BigEndian(key.Slice(1)) ^ 0x8000_0000u); + return true; + } + if (typeof(TValue) == typeof(long)) + { + if (key.Length <= 9) return false; + value = (TValue)(object)(long)(BinaryPrimitives.ReadUInt64BigEndian(key.Slice(1)) ^ 0x8000_0000_0000_0000UL); + return true; + } + return false; + } + } + + /// + /// Aggregates field of every document matching + /// from raw BSON, in primary-key order. Gives up (Handled = + /// false) on a document without the field, with a BSON type other than the one the mapper + /// writes for (or null for a nullable property), or that + /// fails to evaluate, since materialising the entity could then yield a different value. + /// + private async Task<(bool Handled, object? Value)> AggregateRawFieldAsync( + AggregateOperator op, + bool nullable, + string fieldName, + BsonReaderPredicate? predicate, + CancellationToken ct) + where TValue : struct, IComparable + { + if (!_storage.GetKeyMap().TryGetValue(fieldName, out var fieldId)) + return default; // never written: every document lacks it + + var expectedType = + typeof(TValue) == typeof(int) ? BsonType.Int32 : + typeof(TValue) == typeof(long) ? BsonType.Int64 : + typeof(TValue) == typeof(decimal) ? BsonType.Decimal128 : + BsonType.Double; // double and float + var keyMap = _storage.GetKeyReverseMap(); + var aggregate = new LinqAggregate(op, nullable); + + // Adds the document's value if it matches; false to give up. + bool Fold(ReadOnlySpan bson) + { + try + { + if (predicate != null && !predicate(new BsonSpanReader(bson, keyMap))) + return true; + + var reader = new BsonSpanReader(bson, keyMap); + reader.ReadDocumentSize(); + var firstElement = reader.Position; + if (!reader.TrySeekToField(fieldId, out var type)) + { + // No usable offset table entry: look for the field from the first element on. + reader.Seek(firstElement); + if (!SeekField(ref reader, fieldName, out type)) + return false; + } + + if (type == BsonType.Null) + return nullable; + if (type != expectedType) + return false; + + aggregate.Add(ReadValue(ref reader)); + return true; + } + catch + { + return false; + } + } + + // The scan yields only for a document that ends the attempt. + await foreach (var _ in VisitRawDocumentsAsync(0UL, (ReadOnlySpan bson, out bool giveUp) => giveUp = !Fold(bson), ct).ConfigureAwait(false)) + return default; + + return (true, aggregate.Result()); + + static bool SeekField(ref BsonSpanReader reader, string name, out BsonType type) + { + while (reader.Remaining > 1) + { + type = reader.ReadBsonType(); + if (type == BsonType.EndOfDocument) break; + if (reader.ReadElementHeader() == name) return true; + reader.SkipValue(type); + } + type = BsonType.EndOfDocument; + return false; + } + + static TV ReadValue(ref BsonSpanReader reader) + { + if (typeof(TV) == typeof(int)) return (TV)(object)reader.ReadInt32(); + if (typeof(TV) == typeof(long)) return (TV)(object)reader.ReadInt64(); + if (typeof(TV) == typeof(double)) return (TV)(object)reader.ReadDouble(); + if (typeof(TV) == typeof(float)) return (TV)(object)(float)reader.ReadDouble(); + return (TV)(object)reader.ReadDecimal128(); + } + } + /// /// Queries the collection using an expression tree predicate, enabling index optimisation. /// When the predicate references an indexed field the query engine performs a BTree lookup diff --git a/src/BLite.Core/Indexing/BTreeIndex.cs b/src/BLite.Core/Indexing/BTreeIndex.cs index def4b6d0..0694756a 100644 --- a/src/BLite.Core/Indexing/BTreeIndex.cs +++ b/src/BLite.Core/Indexing/BTreeIndex.cs @@ -293,9 +293,34 @@ public IEnumerable Range(IndexKey minKey, IndexKey maxKey, IndexDire /// Equivalent to Range(min, max).Count() but with zero per-entry heap allocations. /// internal int CountRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId = null) + { + var counter = new CountingVisitor(); + ScanRange(minKey, maxKey, transactionId, ref counter); + return counter.Count; + } + + /// Receives the keys of a , in key order. + internal interface IKeyVisitor + { + /// Handles one key; returns false to stop the scan. + bool Visit(ReadOnlySpan key); + } + + private struct CountingVisitor : IKeyVisitor + { + public int Count; + public bool Visit(ReadOnlySpan key) { Count++; return true; } + } + + /// + /// Hands every leaf key in [, ] (inclusive) + /// to straight from the page buffer, in ascending order, without + /// allocating per entry. The key span is only valid during the call. + /// + internal void ScanRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId, ref TVisitor visitor) + where TVisitor : struct, IKeyVisitor { var txnId = transactionId ?? 0; - int count = 0; var pageBuffer = System.Buffers.ArrayPool.Shared.Rent(_storage.PageSize); try { @@ -329,13 +354,16 @@ internal int CountRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId = } // inRange: compare against maxKey only. - if (IndexKey.CompareRaw(entryKeySpan, maxKey.Data) <= 0) - count++; - else + if (IndexKey.CompareRaw(entryKeySpan, maxKey.Data) > 0) { stopScan = true; // past maxKey — no more matches in any leaf break; } + if (!visitor.Visit(entryKeySpan)) + { + stopScan = true; // the visitor has seen enough + break; + } dataOffset += 4 + keyLength + DocumentLocation.SerializedSize; } @@ -348,9 +376,9 @@ internal int CountRange(IndexKey minKey, IndexKey maxKey, ulong? transactionId = { System.Buffers.ArrayPool.Shared.Return(pageBuffer); } - return count; } + /// /// Finds the first leaf entry in [minKey, maxKey] (inclusive) without allocating an /// iterator state machine. Equivalent to Range(min,max).FirstOrDefault() but with diff --git a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs index ec59904d..09bf7384 100644 --- a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs +++ b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs @@ -445,6 +445,44 @@ public int CountRange(object? minKey, object? maxKey, bool startInclusive, bool // [userKey][realId] are <= this boundary, so they are included. // Exclusive end: lower composite boundary [userKey][MinId] — all real entries // [userKey][realId] are > this boundary, so they are excluded. + var (actualMinKey, actualMaxKey) = CompositeRange(minKey, maxKey, startInclusive, endInclusive); + return _btreeIndex.CountRange(actualMinKey, actualMaxKey, transaction?.TransactionId); + } + + /// + /// Hands the stored keys of the entries in the range to , in ascending + /// order and without allocating per entry; bounds as for . Each key is + /// [user key][document id], so the indexed value is its prefix. + /// + internal void ScanKeys(object? minKey, object? maxKey, bool startInclusive, bool endInclusive, ref TVisitor visitor, ITransaction? transaction = null) + where TVisitor : struct, BTreeIndex.IKeyVisitor + { + if (_btreeIndex == null) return; + + var (actualMinKey, actualMaxKey) = CompositeRange(minKey, maxKey, startInclusive, endInclusive); + _btreeIndex.ScanRange(actualMinKey, actualMaxKey, transaction?.TransactionId, ref visitor); + } + + /// + /// The stored key of the last entry in the range (bounds as for ), read + /// from the end of the range; null when the range is empty. + /// + internal IndexKey? LastKey(object? minKey, object? maxKey, bool startInclusive, bool endInclusive, ITransaction? transaction = null) + { + if (_btreeIndex == null) return null; + + var (actualMinKey, actualMaxKey) = CompositeRange(minKey, maxKey, startInclusive, endInclusive); + foreach (var entry in _btreeIndex.Range(actualMinKey, actualMaxKey, IndexDirection.Backward, transaction?.TransactionId)) + return entry.Key; + return null; + } + + /// + /// The composite B-Tree bounds covering every entry whose user key lies between + /// and (null = unbounded). + /// + private (IndexKey Min, IndexKey Max) CompositeRange(object? minKey, object? maxKey, bool startInclusive, bool endInclusive) + { IndexKey actualMinKey = IndexKey.MinKey; IndexKey actualMaxKey = s_maxBoundaryKey; @@ -464,7 +502,7 @@ public int CountRange(object? minKey, object? maxKey, bool startInclusive, bool actualMaxKey = CreateCompositeKeyBoundary(userMax, useMinObjectId: !endInclusive); } - return _btreeIndex.CountRange(actualMinKey, actualMaxKey, transaction?.TransactionId); + return (actualMinKey, actualMaxKey); } /// diff --git a/src/BLite.Core/Indexing/IndexKey.cs b/src/BLite.Core/Indexing/IndexKey.cs index 7ddaa072..35b2269f 100644 --- a/src/BLite.Core/Indexing/IndexKey.cs +++ b/src/BLite.Core/Indexing/IndexKey.cs @@ -17,7 +17,7 @@ public struct IndexKey : IEquatable, IComparable // Guarantees NullSentinel (0x00) and NullSentinelNext (0x01) sort strictly before // all encoded user values, including int.MinValue which encodes to { 0x00, 0x00, 0x00, 0x00 } // without the prefix. - private const byte KeyPrefix = 0x02; + internal const byte KeyPrefix = 0x02; // Prefix of decimal keys. Distinct from KeyPrefix so that an index whose keys were written by an // older version (decimal stored as double under KeyPrefix) can be told apart from a current one. diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index 3309a2b1..f38c8dfe 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -1,978 +1,749 @@ -using System.Collections.Concurrent; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using BLite.Bson; -using BLite.Core.Audit; -using BLite.Core.Collections; -using BLite.Core.Indexing; -using BLite.Core.Metadata; -using BLite.Core.Metrics; -using static BLite.Core.Query.IndexOptimizer; - -namespace BLite.Core.Query; - -public class BTreeQueryProvider : IQueryProvider, IAsyncQueryProvider, IBTreeQueryCore where T : class -{ - private readonly DocumentCollection _collection; - private readonly ValueConverterRegistry _converterRegistry; - - // ── Reflection cache (computed once per TId+T combination) ──────────────── - // BsonProjectionCompiler.TryCompile generic method definition. - // Targets the 3-parameter overload (selectLambda, whereLambda, keyMap) so that - // the key map can be threaded in for offset-table fast paths. -#pragma warning disable IL2026, IL2075 - private static readonly MethodInfo s_tryCompileMethod = - typeof(BsonProjectionCompiler) - .GetMethods(BindingFlags.Static | BindingFlags.Public) - .Single(m => m.Name == nameof(BsonProjectionCompiler.TryCompile) - && m.GetParameters().Length == 3); - - // DocumentCollection.ScanAsync(BsonReaderProjector, CancellationToken) — generic overload. - private static readonly MethodInfo s_scanProjectorMethod = - typeof(DocumentCollection) - .GetMethods(BindingFlags.Public | BindingFlags.Instance) - .Single(m => m.Name == "ScanAsync" && m.IsGenericMethodDefinition); -#pragma warning restore IL2026, IL2075 - - // ── Per-projection-type MakeGenericMethod cache (Fase 3) ───────────────── - // Keyed on TProj. Avoids calling MakeGenericMethod on every query execution. - private static readonly System.Collections.Concurrent.ConcurrentDictionary - s_compiledSelectMethods = new(); - private static readonly System.Collections.Concurrent.ConcurrentDictionary - s_compiledScanMethods = new(); - - public BTreeQueryProvider(DocumentCollection collection, ValueConverterRegistry? registry = null) - { - _collection = collection; - _converterRegistry = registry ?? ValueConverterRegistry.Empty; - } - - // NOTE: CreateQuery implements IQueryProvider.CreateQuery which cannot be annotated. - // The body uses dynamic code (Compile, MakeGenericMethod) suppressed via pragma. -#pragma warning disable IL3050, IL2026, IL2075 - public IQueryable CreateQuery(Expression expression) - { - // IQueryProvider.CreateQuery(Expression) is called only for operators that change the element - // type at runtime (e.g. GroupBy, Cast). For the common case (same T) create directly. - var elementType = expression.Type.IsGenericType - ? expression.Type.GetGenericArguments()[0] - : typeof(T); - - if (elementType == typeof(T)) - return new BTreeQueryable(this, expression); - - // Rare path: element type differs from T. - // Cache a compiled delegate Func, Expression, IQueryable>. - // Using a 2-param delegate (provider, expr) avoids capturing 'this' in the static cache - // — which would be a bug: the cached lambda would call the FIRST provider instance forever. - var del = s_createQueryDelegates.GetOrAdd(elementType, t => - { - var method = typeof(BTreeQueryProvider) - .GetMethod(nameof(CreateQueryTyped), BindingFlags.NonPublic | BindingFlags.Instance)! - .MakeGenericMethod(t); - var providerParam = Expression.Parameter(typeof(BTreeQueryProvider), "p"); - var exprParam = Expression.Parameter(typeof(Expression), "e"); - var callExpr = Expression.Call(providerParam, method, exprParam); - var castResult = Expression.Convert(callExpr, typeof(IQueryable)); - return Expression.Lambda, Expression, IQueryable>>( - castResult, providerParam, exprParam).Compile(); - }); - return del(this, expression); - } -#pragma warning restore IL3050, IL2026, IL2075 - - private BTreeQueryable CreateQueryTyped(Expression expression) - => new(this, expression); - - private static readonly System.Collections.Concurrent.ConcurrentDictionary< - Type, Func, Expression, IQueryable>> - s_createQueryDelegates = new(); - - // Explicit implementation satisfies IQueryProvider contract. - // The runtime object is always BTreeQueryable : IBLiteQueryable, - // so callers can safely use AsAsyncEnumerable() or the typed LINQ extensions. - IQueryable IQueryProvider.CreateQuery(Expression expression) - => new BTreeQueryable(this, expression); - - /// Typed overload — returns the richer interface. - public IBLiteQueryable CreateQuery(Expression expression) - => new BTreeQueryable(this, expression); - - // NOTE: Execute/Execute implement IQueryProvider methods which cannot be annotated. - // Use pragma to suppress warnings about calling dynamic code methods within the body. -#pragma warning disable IL3050, IL2026 - public object? Execute(Expression expression) - { - return Execute(expression); - } - - public TResult Execute(Expression expression) - { - // Run on the thread pool to avoid deadlocks when called from a thread that - // has a SynchronizationContext (ASP.NET classic, WPF, Blazor). - // Without Task.Run, ExecuteAsync awaits FindAllAsync which uses SemaphoreSlim.WaitAsync; - // its continuation would try to resume on the captured context that is already blocked - // by this .GetAwaiter().GetResult() call → deadlock. - // BTreeQueryable.GetAsyncEnumerator applies the same pattern for the async path. - return Task.Run(() => ExecuteAsync(expression, CancellationToken.None)) - .GetAwaiter().GetResult(); - } -#pragma warning restore IL3050, IL2026 - - // Explicit IAsyncQueryProvider implementation — delegates to the private async path, - // allowing BTreeQueryable to call ExecuteAsync directly (no double Task.Run). - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - Task IAsyncQueryProvider.ExecuteAsync(Expression expression, CancellationToken ct) - => ExecuteAsync(expression, ct); - - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private async Task ExecuteAsync(Expression expression, CancellationToken cancellationToken) - { - // ── AUDIT: start ───────────────────────────────────────────────────── - var auditOpts = _collection.Storage.AuditOptions; - var auditVsw = (auditOpts is not null && (auditOpts.Sink is not null || auditOpts.EnableMetrics)) - ? ValueStopwatch.StartNew() - : default; - -#if NET5_0_OR_GREATER - Activity? activity = auditOpts?.EnableDiagnosticSource == true - ? BLiteDiagnostics.ActivitySource.StartActivity(BLiteDiagnostics.QueryActivityName) - : null; - activity?.SetTag("db.system", "blite"); - activity?.SetTag("db.collection.name", _collection.CollectionName); -#endif - // ──────────────────────────────────────────────────────────────────── - - try - { - var result = await ExecuteAsyncCore(expression, cancellationToken).ConfigureAwait(false); - - // ── AUDIT: emit on success ──────────────────────────────────────── - if (auditVsw.IsActive) - { - var elapsed = auditVsw.GetElapsed(); - var opts = _collection.Storage.AuditOptions!; - var userId = (opts.ContextProvider ?? AmbientAuditContext.Instance).GetCurrentUserId(); - - var evt = new QueryAuditEvent( - CollectionName: _collection.CollectionName, - // Phase 1: strategy resolution requires BTreeQueryVisitor integration - // (index vs full-scan detection). Deferred to a future release. - Strategy: QueryStrategy.Unknown, - IndexName: null, - ResultCount: -1, - Elapsed: elapsed, - UserId: userId); - - _collection.Storage.AuditSink?.OnQuery(evt); - _collection.Storage.AuditMetrics?.RecordQuery(QueryStrategy.Unknown, elapsed); - - // Slow-query detection - if (opts.SlowOperationThreshold is { } threshold && elapsed > threshold) - { - _collection.Storage.AuditSink?.OnSlowOperation(new SlowOperationEvent( - SlowOperationType.Query, - CollectionName: _collection.CollectionName, - Elapsed: elapsed, - Detail: null)); - } - } - -#if NET5_0_OR_GREATER - // Always dispose the activity (even when metrics/sink not active) so ActivityStopped fires. - activity?.SetTag("db.blite.query_strategy", QueryStrategy.Unknown.ToString()); - activity?.Dispose(); - activity = null; -#endif - // ──────────────────────────────────────────────────────────────── - - return result; - } - catch - { -#if NET5_0_OR_GREATER - activity?.SetStatus(ActivityStatusCode.Error); - activity?.Dispose(); -#endif - throw; - } - } - - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private async Task ExecuteAsyncCore(Expression expression, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - // 1. Parse the LINQ expression tree into a flat QueryModel. - var visitor = new BTreeExpressionVisitor(); - visitor.Visit(expression); - var model = visitor.GetModel(); - - // ── Element terminals: First / FirstOrDefault / Single / SingleOrDefault / Any ── - // With only a WHERE in front, at most one (Single*: two) matching document decides the - // result, so fetch just that many through the index-aware FetchAsync. Anything else in - // front (OrderBy, Select, Take, ...), and Last / LastOrDefault, which need every match, - // go through the general fallback. - if (model.ElementTerminal is { } terminal) - { - if (terminal is not ("Last" or "LastOrDefault") - && !model.HasComplexOperators && model.SelectClause == null && model.OrderByClause == null - && !model.Take.HasValue && !model.Skip.HasValue && model.AggregateOp == null && !model.IsCountOnly) - { - bool single = terminal is "Single" or "SingleOrDefault"; - var found = new List(single ? 2 : 1); - await foreach (var item in _collection.FetchAsync(model.WhereClause, single ? 2 : 1, cancellationToken)) - found.Add(item); - - return terminal switch - { - "First" => (TResult)(object)found.First()!, - "FirstOrDefault" => (TResult)(object?)found.FirstOrDefault()!, - "Single" => (TResult)(object)found.Single()!, - "SingleOrDefault" => (TResult)(object?)found.SingleOrDefault()!, - _ => (TResult)(object)(found.Count > 0), - }; - } - - model.HasComplexOperators = true; - } - - // ── Fast path: Count() / LongCount() terminal with no WHERE ───────────── - // Uses the primary BTree key scan (O(n) key reads, zero document reads). - // Disabled when Take/Skip are present — those shaping operators reduce the - // logical row-set before counting and must not be bypassed. - if (model.IsCountOnly && model.WhereClause == null && !model.HasComplexOperators - && !model.Take.HasValue && !model.Skip.HasValue) - { - var count = await _collection.CountAsync(cancellationToken); - if (typeof(TResult) == typeof(int)) return (TResult)(object)count; - if (typeof(TResult) == typeof(long)) return (TResult)(object)(long)count; - if (typeof(TResult) == typeof(object)) return (TResult)(object)count; - } - - // ── Fast path: Count() / LongCount() terminal with WHERE predicate ──────── - // Uses index key-only scan for indexed predicates (zero document reads) or a - // streaming count for non-indexed predicates (no large List accumulation). - // Disabled when Take/Skip are present for the same reason as above. - if (model.IsCountOnly && model.WhereClause != null && !model.HasComplexOperators - && !model.Take.HasValue && !model.Skip.HasValue) - { - var count = await _collection.CountByPredicateAsync(model.WhereClause, cancellationToken); - if (typeof(TResult) == typeof(int)) return (TResult)(object)count; - if (typeof(TResult) == typeof(long)) return (TResult)(object)(long)count; - if (typeof(TResult) == typeof(object)) return (TResult)(object)count; - } - - // ── Fast path: Sum / Average / Min / Max via BSON field-projection scan ─ - // Reads only the target field from raw BSON — T is never fully instantiated. - // Fires for both no-WHERE and WHERE variants: - // - No WHERE: projector reads only the selector field. - // - With WHERE: BsonProjectionCompiler merges WHERE + SELECT fields into one - // BSON pass; documents that fail the WHERE return null and are skipped. - // HasComplexOperators is intentionally NOT checked here: VisitAggregate always - // sets it so that queries that cannot be pushed down fall back to EnumerableRewriter. - if (model.AggregateOp is not null && model.AggregateSelector is not null) - { - if (TryBsonAggregate(model.AggregateOp, model.AggregateSelector, model.WhereClause, out var aggResult)) - return aggResult; - } - - // ── Push-down SELECT (+ optional WHERE) ─────────────────────────────── - // Single-pass BSON projection: no T instantiation, no EnumerableRewriter. - // Only applicable when the projection is the whole query: no post-scan ordering, - // paging or other operators. A terminal after the Select (Any, Max, Count(pred), ...) - // runs the usual way. - if (model.SelectClause is not null - && model.OrderByClause is null - && !model.Take.HasValue - && !model.Skip.HasValue - && !model.HasComplexOperators - && TryPushDownSelect(model.SelectClause, model.WhereClause, out var pushed)) - { - return pushed; - } - - // ── Several query sources ───────────────────────────────────────────── - // A query that reads a queryable outside its main chain (q.Where(...).Concat(q), a - // self-join, a subquery inside a lambda) cannot be served by one filtered or paged - // fetch: the model mixes the clauses of the different uses, and EnumerableRewriter - // replaces every use of the root with the same source. Replay the whole query over - // the whole collection. - if (HasSecondaryQuery(expression)) - { - var all = new List(); - await foreach (var item in _collection.FetchAsync(null, int.MaxValue, cancellationToken)) - all.Add(item); - return ExecuteViaEnumerableRewriter(expression, all); - } - - // 2. Data fetching. - // When Take (+ optional Skip) is present without complex operators or ordering that - // requires a full scan first, cap how many rows we need to materialise. - int fetchLimit = (model.Take.HasValue && model.OrderByClause == null && !model.HasComplexOperators) - ? (model.Skip.GetValueOrDefault() + model.Take.Value) - : int.MaxValue; - - // ── Fast path: OrderBy(indexedField).Skip(S).Take(N) without WHERE ─────────── - // Reads only skip+take entries from the index in sorted order, skipping the - // O(n log n) in-memory sort over all documents. Skip is performed at the - // index-entry level (no document reads for skipped positions), so only the - // requested page window is ever deserialised. - if (model.WhereClause == null && model.OrderByClause != null && model.Take.HasValue - && TryGetIndexWindow(expression, model.OrderByClause, out int windowEnd, out bool plainWindow)) - { - var orderOpt = IndexOptimizer.TryOptimizeOrderBy(model, _collection.GetIndexes()); - if (orderOpt is not null) - { - bool ascending = !model.OrderDescending; - if (plainWindow && !model.HasComplexOperators - && (model.SelectClause == null || IsSequenceResult())) - { - int skip = model.Skip ?? 0; - int take = model.Take.Value; - var topN = new List(take); - await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, skip, take, cancellationToken)) - topN.Add(item); - if (model.SelectClause != null) - return ProjectEnumerable(topN, model.SelectClause); - return TerminalReturn(topN); - } - - // Operators after the window (a terminal with a predicate or selector, a scalar - // after Select, ...): read the first windowEnd entries in index order and replay the - // whole query over them. OrderBy is a stable sort, so the replay keeps the index - // order and its Skip/Take select the same window. - // This needs the root queryable to be the query's only source: EnumerableRewriter - // replaces every use of it with the prefix. A query that reads a queryable again - // (Concat(q), Join(q, ...)) took the HasSecondaryQuery path above. - var prefix = new List(windowEnd); - await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, 0, windowEnd, cancellationToken)) - prefix.Add(item); - return ExecuteViaEnumerableRewriter(expression, prefix); - } - } - - // ── General path: FetchAsync picks index / BSON scan / full scan ───── - // FetchAsync always applies the WHERE clause internally (all three strategies - // filter before yielding), so no residual WHERE step is needed afterwards. - var sourceList = new List(); - bool whereAlreadyApplied = model.WhereClause != null; - - await foreach (var item in _collection.FetchAsync(model.WhereClause, fetchLimit, cancellationToken)) - sourceList.Add(item); - - IEnumerable sourceData = sourceList; - - // ── Complex-operator fallback ────────────────────────────────────────── - // GroupBy, Join, Sum/Average/Min/Max with selectors — operators the direct - // pipeline cannot model. We still benefit from index / BSON-scan fetch, - // then let EnumerableRewriter translate the remaining Queryable calls to - // Enumerable equivalents and compile the residual expression once. - // FetchAsync applied the WHERE, so the replay drops the root-chain Where calls folded into - // it. Re-evaluating them would change nothing except for VectorSearch, which only the index - // can evaluate. Every other VectorSearch in the expression is still evaluated and throws. - // The same holds for a terminal predicate folded into the WHERE (First(p) after OrderBy). - if (whereAlreadyApplied) - expression = DropAppliedWheres(DropFoldedTerminalPredicate(expression, model.FoldedTerminal)).Expression; - - if (model.HasComplexOperators) - return ExecuteViaEnumerableRewriter(expression, sourceData); - - // When OrderBy is chained after Select, the key-selector parameter type is the - // projected type (e.g. DateTimeOffset), not T. The direct pipeline builds a - // Func from that lambda, which throws ArgumentException at runtime. - // Fall back to EnumerableRewriter which rewrites the full expression tree correctly. - if (model.OrderByClause != null && model.OrderByClause.Parameters[0].Type != typeof(T)) - return ExecuteViaEnumerableRewriter(expression, sourceData); - - // The direct pipeline can only project into a sequence. A scalar terminal after a - // Select (e.g. Take(n).Select(...).Count()) needs the full expression. - if (model.SelectClause != null && !IsSequenceResult()) - return ExecuteViaEnumerableRewriter(expression, sourceData); - - // 3. Direct pipeline — no expression-tree rewrite, no EnumerableRewriter, no Compile(). - return ExecutePipeline(model, sourceData, whereAlreadyApplied); - } - - /// - /// Applies WHERE / ORDER BY / SKIP / TAKE / SELECT directly on the materialized - /// source without rewriting the original expression tree. - /// Terminal results (IEnumerable<T>, List<T>, scalar aggregates) - /// are dispatched via a type-switch to avoid any reflection call per invocation. - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private TResult ExecutePipeline(QueryModel model, IEnumerable source, bool whereAlreadyApplied) - { - IEnumerable data = source; - - // WHERE (residual — when the BSON scan didn't already filter) - if (!whereAlreadyApplied && model.WhereClause != null) - { - var pred = model.WhereClause.Compile() as Func - ?? (Func)model.WhereClause.Compile(); - data = data.Where(pred); - } - - // ORDER BY — compile key selector to Func (boxing) to avoid - // MakeGenericMethod for Enumerable.OrderBy with runtime-only TKey. - if (model.OrderByClause != null) - { - var param = model.OrderByClause.Parameters[0]; - var boxed = Expression.Lambda>( - Expression.Convert(model.OrderByClause.Body, typeof(object)), param).Compile(); - data = model.OrderDescending - ? data.OrderByDescending(x => (IComparable?)boxed(x)) - : data.OrderBy(x => (IComparable?)boxed(x)); - } - - // SKIP / TAKE - if (model.Skip.HasValue) data = data.Skip(model.Skip.Value); - if (model.Take.HasValue) data = data.Take(model.Take.Value); - - // SELECT (with ORDER BY / SKIP / TAKE — rare path, push-down already handled the common case) - if (model.SelectClause != null) - return ProjectEnumerable(data, model.SelectClause); - - // Terminal dispatch — no reflection, no Compile(), no MakeGenericMethod. - return TerminalReturn(data); - } - - /// - /// Projects via the given SELECT lambda. - /// The generic dispatch (ProjectTyped<TProj>) is resolved via a cached - /// : MakeGenericMethod is called once per projection type. - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private TResult ProjectEnumerable(IEnumerable source, LambdaExpression selectLambda) - { - if (!IsSequenceResult()) - throw new NotSupportedException( - $"BLite query pipeline: cannot project into result type '{typeof(TResult)}'."); - - var projType = typeof(TResult).GetGenericArguments()[0]; - var method = s_projectMethodCache.GetOrAdd( - projType, - t => typeof(BTreeQueryProvider) - .GetMethod(nameof(ProjectTyped), BindingFlags.NonPublic | BindingFlags.Static)! - .MakeGenericMethod(t)); - - return (TResult)method.Invoke(null, [source, selectLambda])!; - } - - /// Compiles as Func<T, TProj> and projects the source. - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - private static IEnumerable ProjectTyped(IEnumerable source, LambdaExpression selectLambda) - { - var selector = (Func)selectLambda.Compile(); - return source.Select(selector); - } - - private static readonly System.Collections.Concurrent.ConcurrentDictionary - s_projectMethodCache = new(); - - /// Whether is IEnumerable<TElement>, i.e. the query is enumerated rather than ended by a terminal. - private static bool IsSequenceResult() - => typeof(TResult).IsGenericType && typeof(TResult).GetGenericTypeDefinition() == typeof(IEnumerable<>); - - /// - /// Checks that starts with OrderBy/OrderByDescending - /// on the collection (the call whose key selector is ), followed by - /// Skip/Take calls that bound it, so that the result only depends on the first - /// documents in that order. - /// - /// - /// true when the window is exactly Take(n) or Skip(s).Take(n), as - /// and describe it, and only - /// Select or a plain Count()/LongCount() follows it. - /// - private static bool TryGetIndexWindow(Expression expression, LambdaExpression orderBy, - out int windowEnd, out bool plainWindow) - { - windowEnd = 0; - plainWindow = false; - - // Collect the Queryable calls from the root outwards. - var calls = new List(); - var current = expression; - while (current is MethodCallExpression call && call.Method.DeclaringType == typeof(Queryable)) - { - calls.Add(call); - current = call.Arguments[0]; - } - if (current is not ConstantExpression { Value: IQueryable } || calls.Count < 2) return false; - calls.Reverse(); - - var first = calls[0]; - if (first.Method.Name is not ("OrderBy" or "OrderByDescending") - || first.Arguments[1] is not UnaryExpression { Operand: var key } - || !ReferenceEquals(key, orderBy)) - return false; - - long start = 0, length = long.MaxValue; - int i = 1; - for (; i < calls.Count && calls[i].Method.Name is "Skip" or "Take"; i++) - { - if (calls[i].Arguments[1] is not ConstantExpression { Value: int n }) return false; - n = Math.Max(n, 0); - if (calls[i].Method.Name == "Skip") - { - start += n; - if (length != long.MaxValue) length = Math.Max(0, length - n); - } - else - { - length = Math.Min(length, n); - } - } - - // No Take bounds the window, or a ThenBy refines the order of the documents at its edge. - if (length == long.MaxValue) return false; - if (i < calls.Count && calls[i].Method.Name is "ThenBy" or "ThenByDescending") return false; - - long end = start + length; - if (end > int.MaxValue) return false; - windowEnd = (int)end; - - var window = calls.GetRange(1, i - 1).ConvertAll(c => c.Method.Name); - plainWindow = (window.Count == 1 && window[0] == "Take" - || window.Count == 2 && window[0] == "Skip" && window[1] == "Take") - && calls.Skip(i).All(c => c.Method.Name == "Select" - || c.Method.Name is "Count" or "LongCount" && c.Arguments.Count == 1); - return true; - } - - /// - /// Whether an operator of the main chain (the Queryable calls from the result down - /// to the root) has an argument other than its source that holds a queryable or calls a - /// Queryable operator: a second use of the root (Concat(q), Join(q, ...), - /// Union(q.Select(...))), another source, or a subquery inside a lambda. - /// - private static bool HasSecondaryQuery(Expression expression) - { - var finder = new SecondaryQueryFinder(); - for (var current = expression; - current is MethodCallExpression call && call.Method.DeclaringType == typeof(Queryable); - current = call.Arguments[0]) - { - for (int i = 1; i < call.Arguments.Count && !finder.Found; i++) - finder.Visit(call.Arguments[i]); - if (finder.Found) return true; - } - return false; - } - - private sealed class SecondaryQueryFinder : ExpressionVisitor - { - public bool Found { get; private set; } - - public override Expression? Visit(Expression? node) => Found ? node : base.Visit(node); - - protected override Expression VisitConstant(ConstantExpression node) - { - if (node.Value is IQueryable) Found = true; - return node; - } - - protected override Expression VisitMethodCall(MethodCallExpression node) - { - if (node.Method.DeclaringType == typeof(Queryable)) - { - Found = true; - return node; - } - return base.VisitMethodCall(node); - } - } - - /// - /// Type-switch dispatch for terminal results — first-match, no reflection per call. - /// Covers all standard LINQ terminal operators: First/Single/Count/Any/All/ToList/ToArray. - /// - private static TResult TerminalReturn(IEnumerable data) - { - if (typeof(TResult) == typeof(IEnumerable)) return (TResult)data; - if (typeof(TResult) == typeof(List)) return (TResult)(object)data.ToList(); - if (typeof(TResult) == typeof(T[])) return (TResult)(object)data.ToArray(); - if (typeof(TResult) == typeof(T)) return (TResult)(object)data.First(); - if (typeof(TResult) == typeof(int)) return (TResult)(object)data.Count(); - if (typeof(TResult) == typeof(long)) return (TResult)(object)(long)data.Count(); - if (typeof(TResult) == typeof(bool)) return (TResult)(object)data.Any(); - if (typeof(TResult) == typeof(object)) return (TResult)(object)(data.ToList()); - - // Unknown terminal: materialise and attempt a cast — handles e.g. First. - var list = data.ToList(); - if (list.Count > 0 && list[0] is TResult first) return first; - if (list is TResult asResult) return asResult; - - throw new NotSupportedException( - $"BLite query pipeline: unsupported terminal result type '{typeof(TResult)}'."); - } - - // ─── BSON-level aggregate push-down ────────────────────────────────────── - - /// - /// Attempts to compute Sum or Average entirely by scanning raw BSON bytes, - /// without deserialising the full entity . - /// - /// - /// true and a populated on success; - /// false to signal the caller should fall through to the standard path. - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private bool TryBsonAggregate( - string aggregateOp, - LambdaExpression selector, - LambdaExpression? whereClause, - out TResult result) - { - result = default!; - var fieldType = selector.ReturnType; - - // Only numeric types are supported; others fall through to EnumerableRewriter. - if (fieldType != typeof(decimal) && fieldType != typeof(double) && - fieldType != typeof(float) && fieldType != typeof(int) && - fieldType != typeof(long)) - return false; - - // Compile a BSON projector for the target field. - // When whereClause is provided, BsonProjectionCompiler merges WHERE + SELECT fields - // into one BSON pass; documents that fail the WHERE return null and are skipped - // by ScanAsync. - object? projector; - try - { - var compileMethod = s_compiledSelectMethods.GetOrAdd( - fieldType, t => s_tryCompileMethod.MakeGenericMethod(typeof(T), t)); - projector = compileMethod.Invoke(null, [selector, whereClause, _collection.GetKeyMap()]); - } - catch { return false; } - - if (projector is null) return false; // field shape too complex for BSON push-down - - // Execute the BSON scan — yields only non-null field values. - System.Collections.IEnumerable? values; - try - { - var scanMethod = s_compiledScanMethods.GetOrAdd( - fieldType, t => s_scanProjectorMethod.MakeGenericMethod(t)); - values = scanMethod.Invoke(_collection, [projector]) as System.Collections.IEnumerable; - } - catch { return false; } - - if (values is null) return false; - - try { return AggregateValues(aggregateOp, values, fieldType, out result); } - catch { return false; } - } - - private static bool AggregateValues( - string op, - System.Collections.IEnumerable values, - Type fieldType, - out TResult result) - { - result = default!; - - if (fieldType == typeof(decimal)) - { - var typed = (IEnumerable)values; - decimal agg = op switch - { - "Sum" => typed.Sum(), - "Average" => typed.DefaultIfEmpty().Average(), - "Min" => typed.DefaultIfEmpty().Min(), - "Max" => typed.DefaultIfEmpty().Max(), - _ => throw new NotSupportedException($"Unsupported aggregate op: {op}") - }; - if (typeof(TResult) == typeof(decimal)) { result = (TResult)(object)agg; return true; } - } - else if (fieldType == typeof(double)) - { - var typed = (IEnumerable)values; - double agg = op switch - { - "Sum" => typed.Sum(), - "Average" => typed.DefaultIfEmpty().Average(), - "Min" => typed.DefaultIfEmpty().Min(), - "Max" => typed.DefaultIfEmpty().Max(), - _ => throw new NotSupportedException($"Unsupported aggregate op: {op}") - }; - if (typeof(TResult) == typeof(double)) { result = (TResult)(object)agg; return true; } - } - else if (fieldType == typeof(float)) - { - var typed = (IEnumerable)values; - float agg = op switch - { - "Sum" => typed.Sum(), - "Average" => typed.DefaultIfEmpty().Average(), - "Min" => typed.DefaultIfEmpty().Min(), - "Max" => typed.DefaultIfEmpty().Max(), - _ => throw new NotSupportedException($"Unsupported aggregate op: {op}") - }; - if (typeof(TResult) == typeof(float)) { result = (TResult)(object)agg; return true; } - } - else if (fieldType == typeof(int)) - { - var typed = (IEnumerable)values; - if (op == "Sum") - { - long sum = 0; - foreach (var v in typed) sum += v; - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)sum; return true; } - if (typeof(TResult) == typeof(int)) { result = (TResult)(object)(int)sum; return true; } - } - else if (op == "Average") - { - double avg = typed.DefaultIfEmpty().Average(); - if (typeof(TResult) == typeof(double)) { result = (TResult)(object)avg; return true; } - } - else if (op == "Min") - { - int min = typed.DefaultIfEmpty().Min(); - if (typeof(TResult) == typeof(int)) { result = (TResult)(object)min; return true; } - } - else if (op == "Max") - { - int max = typed.DefaultIfEmpty().Max(); - if (typeof(TResult) == typeof(int)) { result = (TResult)(object)max; return true; } - } - } - else if (fieldType == typeof(long)) - { - var typed = (IEnumerable)values; - if (op == "Sum") - { - long sum = 0; - foreach (var v in typed) sum += v; - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)sum; return true; } - } - else if (op == "Average") - { - double avg = typed.DefaultIfEmpty().Select(x => (double)x).Average(); - if (typeof(TResult) == typeof(double)) { result = (TResult)(object)avg; return true; } - } - else if (op == "Min") - { - long min = typed.DefaultIfEmpty().Min(); - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)min; return true; } - } - else if (op == "Max") - { - long max = typed.DefaultIfEmpty().Max(); - if (typeof(TResult) == typeof(long)) { result = (TResult)(object)max; return true; } - } - } - return false; - } - - // ─── Push-down SELECT helper ────────────────────────────────────────────── - - /// - /// Attempts to execute the query entirely via a single-pass BSON projection scan, - /// bypassing full deserialisation. - /// - /// - /// true and the projected IEnumerable<TProj> in ; - /// false if push-down is not applicable (caller should fall through to the standard path). - /// default(TResult) cannot signal this: for a value-type result such as bool it - /// is a valid answer (#181). - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private bool TryPushDownSelect(LambdaExpression selectLambda, - LambdaExpression? whereLambda, - out TResult result) - { - result = default!; - - // We need TResult = IEnumerable for some projection type TProj. - if (!IsSequenceResult()) return false; - - var projType = typeof(TResult).GetGenericArguments()[0]; - if (projType == typeof(T)) return false; // not a real projection - - // Compile the push-down projector via reflection (TProj is only known at runtime). - // BsonProjectionCompiler.TryCompile(selectLambda, whereLambda?) handles - // both the pure-SELECT and the WHERE+SELECT cases in a single BSON pass. - object? projector; - try - { - var compileMethod = s_compiledSelectMethods.GetOrAdd( - projType, - t => s_tryCompileMethod.MakeGenericMethod(typeof(T), t)); - projector = compileMethod.Invoke(null, [selectLambda, whereLambda, _collection.GetKeyMap()]); - } - catch { return false; } - - if (projector is null) return false; // compilation soft-failed - - // Invoke _collection.Scan(projector) — MethodInfo cached per TProj. - try - { - var scanMethod = s_compiledScanMethods.GetOrAdd( - projType, - t => s_scanProjectorMethod.MakeGenericMethod(t)); - if (scanMethod.Invoke(_collection, [projector]) is not TResult scanned) return false; - result = scanned; - return true; - } - catch { return false; } - } - - // ─── Expression-tree helpers ────────────────────────────────────────────── - - /// - /// Removes the Queryable.Where(x => ...) calls on the root chain whose input is the - /// fetched documents or a subset of them (only filtering, ordering and paging operators lie - /// between the root and the Where). folds each of these - /// into , which FetchAsync applied, so every element - /// reaching them already satisfies their predicate and dropping them leaves the result unchanged. - /// - /// - /// The rewritten expression, and whether its elements are still a subset of the fetched documents. - /// - private static (Expression Expression, bool IsFetchedSubset) DropAppliedWheres(Expression expression) - { - if (expression is ConstantExpression { Value: IQueryable }) - return (expression, true); - - if (expression is not MethodCallExpression call - || call.Method.DeclaringType != typeof(Queryable) - || call.Arguments.Count == 0) - return (expression, false); - - var (source, isFetchedSubset) = DropAppliedWheres(call.Arguments[0]); - - if (isFetchedSubset - && call.Method.Name == nameof(Queryable.Where) - && call.Arguments[1] is UnaryExpression { Operand: LambdaExpression { Parameters.Count: 1 } }) - return (source, true); - - var rewritten = source == call.Arguments[0] - ? call - : call.Update(call.Object, [source, .. call.Arguments.Skip(1)]); - return (rewritten, isFetchedSubset && s_subsetPreservingOperators.Contains(call.Method.Name)); - } - - /// - /// Replaces the terminal whose predicate folded into - /// (Count(p), First(p), ...) with its - /// parameterless overload. The visitor folds a predicate only when the terminal's source is the - /// root followed by plain Where and ordering calls, so the elements reaching the terminal are - /// fetched documents, which all satisfy the predicate already. - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private static Expression DropFoldedTerminalPredicate(Expression expression, MethodCallExpression? foldedTerminal) - { - if (foldedTerminal is null || !ReferenceEquals(expression, foldedTerminal)) - return expression; - - var name = foldedTerminal.Method.Name; - if (!s_parameterlessTerminals.TryGetValue(name, out var definition)) - { - definition = typeof(Queryable).GetMethods().Single(m => m.Name == name && m.GetParameters().Length == 1); - s_parameterlessTerminals[name] = definition; - } - - var parameterless = definition.MakeGenericMethod(foldedTerminal.Method.GetGenericArguments()); - return Expression.Call(parameterless, foldedTerminal.Arguments[0]); - } - - /// Generic definitions of the parameterless Queryable terminals, by name. - private static readonly ConcurrentDictionary s_parameterlessTerminals = new(); - - /// Queryable operators whose output elements are a subset of their source's. - private static readonly HashSet s_subsetPreservingOperators = - [ - nameof(Queryable.Where), - nameof(Queryable.OrderBy), nameof(Queryable.OrderByDescending), - nameof(Queryable.ThenBy), nameof(Queryable.ThenByDescending), - nameof(Queryable.Skip), nameof(Queryable.Take), - nameof(Queryable.SkipWhile), nameof(Queryable.TakeWhile), - nameof(Queryable.Distinct), nameof(Queryable.Reverse), - ]; - - /// - /// Fallback for queries containing operators the direct pipeline cannot handle - /// (GroupBy, Join, Sum/Average/Min/Max with selectors, etc.). - /// The source data has already been fetched (and filtered by index/BSON-scan), - /// so we only need to rewrite the remaining Queryable.* calls to - /// Enumerable.* equivalents before compiling and executing. - /// - /// - /// The compiled Func<IEnumerable<T>, TResult> is cached per expression shape - /// (keyed by TResult.FullName + expression.ToString()), so the expensive - /// call is paid only once per unique query shape. - /// - [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] - private TResult ExecuteViaEnumerableRewriter(Expression expression, IEnumerable sourceData) - { - var rootFinder = new RootFinder(); - rootFinder.Visit(expression); - var root = rootFinder.Root; - if (root == null) - throw new InvalidOperationException("Could not find root IQueryable in expression."); - - var cacheKey = typeof(TResult).FullName + "\x00" + expression.ToString(); - - var cachedDelegate = s_enumRewriterCache.GetOrAdd(cacheKey, _ => - { - var sourceParam = Expression.Parameter(typeof(IEnumerable), "_src"); - var rewriter = new EnumerableRewriter(root, sourceParam); - var rewritten = rewriter.Visit(expression); - if (rewritten.Type != typeof(TResult)) - rewritten = Expression.Convert(rewritten, typeof(TResult)); - return Expression.Lambda, TResult>>(rewritten, sourceParam).Compile(); - }); - - return ((Func, TResult>)cachedDelegate)(sourceData); - } - - /// Cache of compiled EnumerableRewriter delegates keyed by expression shape. - private static readonly ConcurrentDictionary s_enumRewriterCache = new(); - - private sealed class RootFinder : ExpressionVisitor - { - public IQueryable? Root { get; private set; } - - protected override Expression VisitConstant(ConstantExpression node) - { - if (Root == null && node.Value is IQueryable q) - Root = q; - return base.VisitConstant(node); - } - } - - // ─── IBTreeQueryCore implementation ─────────────────────────────────── - - internal DocumentCollection Collection => _collection; - - IAsyncEnumerable IBTreeQueryCore.ScanAsync(BsonReaderPredicate predicate, CancellationToken ct) - => _collection.ScanAsync(predicate, ct); - - IAsyncEnumerable IBTreeQueryCore.ScanAsync(IndexQueryPlan plan, CancellationToken ct) - => _collection.ScanAsync(plan, ct); - - IEnumerable IBTreeQueryCore.GetIndexes() - => _collection.GetIndexes(); - - Task IBTreeQueryCore.CountAsync(CancellationToken ct) - => _collection.CountAsync(ct); - - ValueTask IBTreeQueryCore.MinBoundaryAsync(IndexMinMax plan, CancellationToken ct) - => _collection.MinBoundaryAsync(plan, ct); - - ValueTask IBTreeQueryCore.MaxBoundaryAsync(IndexMinMax plan, CancellationToken ct) - => _collection.MaxBoundaryAsync(plan, ct); -} +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using BLite.Bson; +using BLite.Core.Audit; +using BLite.Core.Collections; +using BLite.Core.Indexing; +using BLite.Core.Metadata; +using BLite.Core.Metrics; +using static BLite.Core.Query.IndexOptimizer; + +namespace BLite.Core.Query; + +public class BTreeQueryProvider : IQueryProvider, IAsyncQueryProvider, IBTreeQueryCore where T : class +{ + private readonly DocumentCollection _collection; + private readonly ValueConverterRegistry _converterRegistry; + + public BTreeQueryProvider(DocumentCollection collection, ValueConverterRegistry? registry = null) + { + _collection = collection; + _converterRegistry = registry ?? ValueConverterRegistry.Empty; + } + + // NOTE: CreateQuery implements IQueryProvider.CreateQuery which cannot be annotated. + // The body uses dynamic code (Compile, MakeGenericMethod) suppressed via pragma. +#pragma warning disable IL3050, IL2026, IL2075 + public IQueryable CreateQuery(Expression expression) + { + // IQueryProvider.CreateQuery(Expression) is called only for operators that change the element + // type at runtime (e.g. GroupBy, Cast). For the common case (same T) create directly. + var elementType = expression.Type.IsGenericType + ? expression.Type.GetGenericArguments()[0] + : typeof(T); + + if (elementType == typeof(T)) + return new BTreeQueryable(this, expression); + + // Rare path: element type differs from T. + // Cache a compiled delegate Func, Expression, IQueryable>. + // Using a 2-param delegate (provider, expr) avoids capturing 'this' in the static cache + // — which would be a bug: the cached lambda would call the FIRST provider instance forever. + var del = s_createQueryDelegates.GetOrAdd(elementType, t => + { + var method = typeof(BTreeQueryProvider) + .GetMethod(nameof(CreateQueryTyped), BindingFlags.NonPublic | BindingFlags.Instance)! + .MakeGenericMethod(t); + var providerParam = Expression.Parameter(typeof(BTreeQueryProvider), "p"); + var exprParam = Expression.Parameter(typeof(Expression), "e"); + var callExpr = Expression.Call(providerParam, method, exprParam); + var castResult = Expression.Convert(callExpr, typeof(IQueryable)); + return Expression.Lambda, Expression, IQueryable>>( + castResult, providerParam, exprParam).Compile(); + }); + return del(this, expression); + } +#pragma warning restore IL3050, IL2026, IL2075 + + private BTreeQueryable CreateQueryTyped(Expression expression) + => new(this, expression); + + private static readonly System.Collections.Concurrent.ConcurrentDictionary< + Type, Func, Expression, IQueryable>> + s_createQueryDelegates = new(); + + // Explicit implementation satisfies IQueryProvider contract. + // The runtime object is always BTreeQueryable : IBLiteQueryable, + // so callers can safely use AsAsyncEnumerable() or the typed LINQ extensions. + IQueryable IQueryProvider.CreateQuery(Expression expression) + => new BTreeQueryable(this, expression); + + /// Typed overload — returns the richer interface. + public IBLiteQueryable CreateQuery(Expression expression) + => new BTreeQueryable(this, expression); + + // NOTE: Execute/Execute implement IQueryProvider methods which cannot be annotated. + // Use pragma to suppress warnings about calling dynamic code methods within the body. +#pragma warning disable IL3050, IL2026 + public object? Execute(Expression expression) + { + return Execute(expression); + } + + public TResult Execute(Expression expression) + { + // Run on the thread pool to avoid deadlocks when called from a thread that + // has a SynchronizationContext (ASP.NET classic, WPF, Blazor). + // Without Task.Run, ExecuteAsync awaits FindAllAsync which uses SemaphoreSlim.WaitAsync; + // its continuation would try to resume on the captured context that is already blocked + // by this .GetAwaiter().GetResult() call → deadlock. + // BTreeQueryable.GetAsyncEnumerator applies the same pattern for the async path. + return Task.Run(() => ExecuteAsync(expression, CancellationToken.None)) + .GetAwaiter().GetResult(); + } +#pragma warning restore IL3050, IL2026 + + // Explicit IAsyncQueryProvider implementation — delegates to the private async path, + // allowing BTreeQueryable to call ExecuteAsync directly (no double Task.Run). + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] + Task IAsyncQueryProvider.ExecuteAsync(Expression expression, CancellationToken ct) + => ExecuteAsync(expression, ct); + + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] + private async Task ExecuteAsync(Expression expression, CancellationToken cancellationToken) + { + // ── AUDIT: start ───────────────────────────────────────────────────── + var auditOpts = _collection.Storage.AuditOptions; + var auditVsw = (auditOpts is not null && (auditOpts.Sink is not null || auditOpts.EnableMetrics)) + ? ValueStopwatch.StartNew() + : default; + +#if NET5_0_OR_GREATER + Activity? activity = auditOpts?.EnableDiagnosticSource == true + ? BLiteDiagnostics.ActivitySource.StartActivity(BLiteDiagnostics.QueryActivityName) + : null; + activity?.SetTag("db.system", "blite"); + activity?.SetTag("db.collection.name", _collection.CollectionName); +#endif + // ──────────────────────────────────────────────────────────────────── + + try + { + var result = await ExecuteAsyncCore(expression, cancellationToken).ConfigureAwait(false); + + // ── AUDIT: emit on success ──────────────────────────────────────── + if (auditVsw.IsActive) + { + var elapsed = auditVsw.GetElapsed(); + var opts = _collection.Storage.AuditOptions!; + var userId = (opts.ContextProvider ?? AmbientAuditContext.Instance).GetCurrentUserId(); + + var evt = new QueryAuditEvent( + CollectionName: _collection.CollectionName, + // Phase 1: strategy resolution requires BTreeQueryVisitor integration + // (index vs full-scan detection). Deferred to a future release. + Strategy: QueryStrategy.Unknown, + IndexName: null, + ResultCount: -1, + Elapsed: elapsed, + UserId: userId); + + _collection.Storage.AuditSink?.OnQuery(evt); + _collection.Storage.AuditMetrics?.RecordQuery(QueryStrategy.Unknown, elapsed); + + // Slow-query detection + if (opts.SlowOperationThreshold is { } threshold && elapsed > threshold) + { + _collection.Storage.AuditSink?.OnSlowOperation(new SlowOperationEvent( + SlowOperationType.Query, + CollectionName: _collection.CollectionName, + Elapsed: elapsed, + Detail: null)); + } + } + +#if NET5_0_OR_GREATER + // Always dispose the activity (even when metrics/sink not active) so ActivityStopped fires. + activity?.SetTag("db.blite.query_strategy", QueryStrategy.Unknown.ToString()); + activity?.Dispose(); + activity = null; +#endif + // ──────────────────────────────────────────────────────────────── + + return result; + } + catch + { +#if NET5_0_OR_GREATER + activity?.SetStatus(ActivityStatusCode.Error); + activity?.Dispose(); +#endif + throw; + } + } + + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] + private async Task ExecuteAsyncCore(Expression expression, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + // 1. Parse the LINQ expression tree into a flat QueryModel. + var visitor = new BTreeExpressionVisitor(); + visitor.Visit(expression); + var model = visitor.GetModel(); + + // ── Element terminals: First / FirstOrDefault / Single / SingleOrDefault / Any ── + // With only a WHERE in front, at most one (Single*: two) matching document decides the + // result, so fetch just that many through the index-aware FetchAsync. Anything else in + // front (OrderBy, Select, Take, ...), and Last / LastOrDefault, which need every match, + // go through the general fallback. + if (model.ElementTerminal is { } terminal) + { + if (terminal is not ("Last" or "LastOrDefault") + && !model.HasComplexOperators && model.SelectClause == null && model.OrderByClause == null + && !model.Take.HasValue && !model.Skip.HasValue && model.AggregateOp == null && !model.IsCountOnly) + { + bool single = terminal is "Single" or "SingleOrDefault"; + var found = new List(single ? 2 : 1); + await foreach (var item in _collection.FetchAsync(model.WhereClause, single ? 2 : 1, cancellationToken)) + found.Add(item); + + return terminal switch + { + "First" => (TResult)(object)found.First()!, + "FirstOrDefault" => (TResult)(object?)found.FirstOrDefault()!, + "Single" => (TResult)(object)found.Single()!, + "SingleOrDefault" => (TResult)(object?)found.SingleOrDefault()!, + _ => (TResult)(object)(found.Count > 0), + }; + } + + model.HasComplexOperators = true; + } + + // ── Fast path: Count() / LongCount() terminal with no WHERE ───────────── + // Uses the primary BTree key scan (O(n) key reads, zero document reads). + // Disabled when Take/Skip are present — those shaping operators reduce the + // logical row-set before counting and must not be bypassed. + if (model.IsCountOnly && model.WhereClause == null && !model.HasComplexOperators + && !model.Take.HasValue && !model.Skip.HasValue) + { + var count = await _collection.CountAsync(cancellationToken); + if (typeof(TResult) == typeof(int)) return (TResult)(object)count; + if (typeof(TResult) == typeof(long)) return (TResult)(object)(long)count; + if (typeof(TResult) == typeof(object)) return (TResult)(object)count; + } + + // ── Fast path: Count() / LongCount() terminal with WHERE predicate ──────── + // Uses index key-only scan for indexed predicates (zero document reads) or a + // streaming count for non-indexed predicates (no large List accumulation). + // Disabled when Take/Skip are present for the same reason as above. + if (model.IsCountOnly && model.WhereClause != null && !model.HasComplexOperators + && !model.Take.HasValue && !model.Skip.HasValue) + { + var count = await _collection.CountByPredicateAsync(model.WhereClause, cancellationToken); + if (typeof(TResult) == typeof(int)) return (TResult)(object)count; + if (typeof(TResult) == typeof(long)) return (TResult)(object)(long)count; + if (typeof(TResult) == typeof(object)) return (TResult)(object)count; + } + + // ── Fast path: Sum / Average / Min / Max of a property without materialising T ─ + // From index keys or raw BSON, with the result LINQ computes over the entities. Only for + // an aggregate applied straight to Where clauses over the collection: any other operator + // in between (Take, Skip, OrderBy, Select, Distinct, ...) changes which values are + // aggregated or their order, so those queries run the usual way. + if (model.AggregateOp is not null && model.AggregateSelector is not null + && IsAggregateOverFilteredRoot(expression) + && Enum.TryParse(model.AggregateOp, out var aggregateOp)) + { + var (handled, value) = await _collection.TryAggregateAsync( + aggregateOp, model.AggregateSelector, model.WhereClause, typeof(TResult), cancellationToken).ConfigureAwait(false); + if (handled) return (TResult)value!; + } + + // ── Several query sources ───────────────────────────────────────────── + // A query that reads a queryable outside its main chain (q.Where(...).Concat(q), a + // self-join, a subquery inside a lambda) cannot be served by one filtered or paged + // fetch: the model mixes the clauses of the different uses, and EnumerableRewriter + // replaces every use of the root with the same source. Replay the whole query over + // the whole collection. + if (HasSecondaryQuery(expression)) + { + var all = new List(); + await foreach (var item in _collection.FetchAsync(null, int.MaxValue, cancellationToken)) + all.Add(item); + return ExecuteViaEnumerableRewriter(expression, all); + } + + // 2. Data fetching. + // When Take (+ optional Skip) is present without complex operators or ordering that + // requires a full scan first, cap how many rows we need to materialise. + int fetchLimit = (model.Take.HasValue && model.OrderByClause == null && !model.HasComplexOperators) + ? (model.Skip.GetValueOrDefault() + model.Take.Value) + : int.MaxValue; + + // ── Fast path: OrderBy(indexedField).Skip(S).Take(N) without WHERE ─────────── + // Reads only skip+take entries from the index in sorted order, skipping the + // O(n log n) in-memory sort over all documents. Skip is performed at the + // index-entry level (no document reads for skipped positions), so only the + // requested page window is ever deserialised. + if (model.WhereClause == null && model.OrderByClause != null && model.Take.HasValue + && TryGetIndexWindow(expression, model.OrderByClause, out int windowEnd, out bool plainWindow)) + { + var orderOpt = IndexOptimizer.TryOptimizeOrderBy(model, _collection.GetIndexes()); + if (orderOpt is not null) + { + bool ascending = !model.OrderDescending; + if (plainWindow && !model.HasComplexOperators + && (model.SelectClause == null || IsSequenceResult())) + { + int skip = model.Skip ?? 0; + int take = model.Take.Value; + var topN = new List(take); + await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, skip, take, cancellationToken)) + topN.Add(item); + if (model.SelectClause != null) + return ProjectEnumerable(topN, model.SelectClause); + return TerminalReturn(topN); + } + + // Operators after the window (a terminal with a predicate or selector, a scalar + // after Select, ...): read the first windowEnd entries in index order and replay the + // whole query over them. OrderBy is a stable sort, so the replay keeps the index + // order and its Skip/Take select the same window. + // This needs the root queryable to be the query's only source: EnumerableRewriter + // replaces every use of it with the prefix. A query that reads a queryable again + // (Concat(q), Join(q, ...)) took the HasSecondaryQuery path above. + var prefix = new List(windowEnd); + await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, 0, windowEnd, cancellationToken)) + prefix.Add(item); + return ExecuteViaEnumerableRewriter(expression, prefix); + } + } + + // ── General path: FetchAsync picks index / BSON scan / full scan ───── + // FetchAsync always applies the WHERE clause internally (all three strategies + // filter before yielding), so no residual WHERE step is needed afterwards. + var sourceList = new List(); + bool whereAlreadyApplied = model.WhereClause != null; + + await foreach (var item in _collection.FetchAsync(model.WhereClause, fetchLimit, cancellationToken)) + sourceList.Add(item); + + IEnumerable sourceData = sourceList; + + // ── Complex-operator fallback ────────────────────────────────────────── + // GroupBy, Join, Sum/Average/Min/Max with selectors — operators the direct + // pipeline cannot model. We still benefit from index / BSON-scan fetch, + // then let EnumerableRewriter translate the remaining Queryable calls to + // Enumerable equivalents and compile the residual expression once. + // FetchAsync applied the WHERE, so the replay drops the root-chain Where calls folded into + // it. Re-evaluating them would change nothing except for VectorSearch, which only the index + // can evaluate. Every other VectorSearch in the expression is still evaluated and throws. + // The same holds for a terminal predicate folded into the WHERE (First(p) after OrderBy). + if (whereAlreadyApplied) + expression = DropAppliedWheres(DropFoldedTerminalPredicate(expression, model.FoldedTerminal)).Expression; + + if (model.HasComplexOperators) + return ExecuteViaEnumerableRewriter(expression, sourceData); + + // When OrderBy is chained after Select, the key-selector parameter type is the + // projected type (e.g. DateTimeOffset), not T. The direct pipeline builds a + // Func from that lambda, which throws ArgumentException at runtime. + // Fall back to EnumerableRewriter which rewrites the full expression tree correctly. + if (model.OrderByClause != null && model.OrderByClause.Parameters[0].Type != typeof(T)) + return ExecuteViaEnumerableRewriter(expression, sourceData); + + // The direct pipeline can only project into a sequence. A scalar terminal after a + // Select (e.g. Take(n).Select(...).Count()) needs the full expression. + if (model.SelectClause != null && !IsSequenceResult()) + return ExecuteViaEnumerableRewriter(expression, sourceData); + + // 3. Direct pipeline — no expression-tree rewrite, no EnumerableRewriter, no Compile(). + return ExecutePipeline(model, sourceData, whereAlreadyApplied); + } + + /// + /// Applies WHERE / ORDER BY / SKIP / TAKE / SELECT directly on the materialized + /// source without rewriting the original expression tree. + /// Terminal results (IEnumerable<T>, List<T>, scalar aggregates) + /// are dispatched via a type-switch to avoid any reflection call per invocation. + /// + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] + private TResult ExecutePipeline(QueryModel model, IEnumerable source, bool whereAlreadyApplied) + { + IEnumerable data = source; + + // WHERE (residual — when the BSON scan didn't already filter) + if (!whereAlreadyApplied && model.WhereClause != null) + { + var pred = model.WhereClause.Compile() as Func + ?? (Func)model.WhereClause.Compile(); + data = data.Where(pred); + } + + // ORDER BY — compile key selector to Func (boxing) to avoid + // MakeGenericMethod for Enumerable.OrderBy with runtime-only TKey. + if (model.OrderByClause != null) + { + var param = model.OrderByClause.Parameters[0]; + var boxed = Expression.Lambda>( + Expression.Convert(model.OrderByClause.Body, typeof(object)), param).Compile(); + data = model.OrderDescending + ? data.OrderByDescending(x => (IComparable?)boxed(x)) + : data.OrderBy(x => (IComparable?)boxed(x)); + } + + // SKIP / TAKE + if (model.Skip.HasValue) data = data.Skip(model.Skip.Value); + if (model.Take.HasValue) data = data.Take(model.Take.Value); + + // SELECT + if (model.SelectClause != null) + return ProjectEnumerable(data, model.SelectClause); + + // Terminal dispatch — no reflection, no Compile(), no MakeGenericMethod. + return TerminalReturn(data); + } + + /// + /// Projects via the given SELECT lambda. + /// The generic dispatch (ProjectTyped<TProj>) is resolved via a cached + /// : MakeGenericMethod is called once per projection type. + /// + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] + private TResult ProjectEnumerable(IEnumerable source, LambdaExpression selectLambda) + { + if (!IsSequenceResult()) + throw new NotSupportedException( + $"BLite query pipeline: cannot project into result type '{typeof(TResult)}'."); + + var projType = typeof(TResult).GetGenericArguments()[0]; + var method = s_projectMethodCache.GetOrAdd( + projType, + t => typeof(BTreeQueryProvider) + .GetMethod(nameof(ProjectTyped), BindingFlags.NonPublic | BindingFlags.Static)! + .MakeGenericMethod(t)); + + return (TResult)method.Invoke(null, [source, selectLambda])!; + } + + /// Compiles as Func<T, TProj> and projects the source. + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + private static IEnumerable ProjectTyped(IEnumerable source, LambdaExpression selectLambda) + { + var selector = (Func)selectLambda.Compile(); + return source.Select(selector); + } + + private static readonly System.Collections.Concurrent.ConcurrentDictionary + s_projectMethodCache = new(); + + /// Whether is IEnumerable<TElement>, i.e. the query is enumerated rather than ended by a terminal. + private static bool IsSequenceResult() + => typeof(TResult).IsGenericType && typeof(TResult).GetGenericTypeDefinition() == typeof(IEnumerable<>); + + /// + /// Checks that starts with OrderBy/OrderByDescending + /// on the collection (the call whose key selector is ), followed by + /// Skip/Take calls that bound it, so that the result only depends on the first + /// documents in that order. + /// + /// + /// true when the window is exactly Take(n) or Skip(s).Take(n), as + /// and describe it, and only + /// Select or a plain Count()/LongCount() follows it. + /// + private static bool TryGetIndexWindow(Expression expression, LambdaExpression orderBy, + out int windowEnd, out bool plainWindow) + { + windowEnd = 0; + plainWindow = false; + + // Collect the Queryable calls from the root outwards. + var calls = new List(); + var current = expression; + while (current is MethodCallExpression call && call.Method.DeclaringType == typeof(Queryable)) + { + calls.Add(call); + current = call.Arguments[0]; + } + if (current is not ConstantExpression { Value: IQueryable } || calls.Count < 2) return false; + calls.Reverse(); + + var first = calls[0]; + if (first.Method.Name is not ("OrderBy" or "OrderByDescending") + || first.Arguments[1] is not UnaryExpression { Operand: var key } + || !ReferenceEquals(key, orderBy)) + return false; + + long start = 0, length = long.MaxValue; + int i = 1; + for (; i < calls.Count && calls[i].Method.Name is "Skip" or "Take"; i++) + { + if (calls[i].Arguments[1] is not ConstantExpression { Value: int n }) return false; + n = Math.Max(n, 0); + if (calls[i].Method.Name == "Skip") + { + start += n; + if (length != long.MaxValue) length = Math.Max(0, length - n); + } + else + { + length = Math.Min(length, n); + } + } + + // No Take bounds the window, or a ThenBy refines the order of the documents at its edge. + if (length == long.MaxValue) return false; + if (i < calls.Count && calls[i].Method.Name is "ThenBy" or "ThenByDescending") return false; + + long end = start + length; + if (end > int.MaxValue) return false; + windowEnd = (int)end; + + var window = calls.GetRange(1, i - 1).ConvertAll(c => c.Method.Name); + plainWindow = (window.Count == 1 && window[0] == "Take" + || window.Count == 2 && window[0] == "Skip" && window[1] == "Take") + && calls.Skip(i).All(c => c.Method.Name == "Select" + || c.Method.Name is "Count" or "LongCount" && c.Arguments.Count == 1); + return true; + } + + /// + /// Whether an operator of the main chain (the Queryable calls from the result down + /// to the root) has an argument other than its source that holds a queryable or calls a + /// Queryable operator: a second use of the root (Concat(q), Join(q, ...), + /// Union(q.Select(...))), another source, or a subquery inside a lambda. + /// + private static bool HasSecondaryQuery(Expression expression) + { + var finder = new SecondaryQueryFinder(); + for (var current = expression; + current is MethodCallExpression call && call.Method.DeclaringType == typeof(Queryable); + current = call.Arguments[0]) + { + for (int i = 1; i < call.Arguments.Count && !finder.Found; i++) + finder.Visit(call.Arguments[i]); + if (finder.Found) return true; + } + return false; + } + + private sealed class SecondaryQueryFinder : ExpressionVisitor + { + public bool Found { get; private set; } + + public override Expression? Visit(Expression? node) => Found ? node : base.Visit(node); + + protected override Expression VisitConstant(ConstantExpression node) + { + if (node.Value is IQueryable) Found = true; + return node; + } + + protected override Expression VisitMethodCall(MethodCallExpression node) + { + if (node.Method.DeclaringType == typeof(Queryable)) + { + Found = true; + return node; + } + return base.VisitMethodCall(node); + } + } + + /// + /// Type-switch dispatch for terminal results — first-match, no reflection per call. + /// Covers all standard LINQ terminal operators: First/Single/Count/Any/All/ToList/ToArray. + /// + private static TResult TerminalReturn(IEnumerable data) + { + if (typeof(TResult) == typeof(IEnumerable)) return (TResult)data; + if (typeof(TResult) == typeof(List)) return (TResult)(object)data.ToList(); + if (typeof(TResult) == typeof(T[])) return (TResult)(object)data.ToArray(); + if (typeof(TResult) == typeof(T)) return (TResult)(object)data.First(); + if (typeof(TResult) == typeof(int)) return (TResult)(object)data.Count(); + if (typeof(TResult) == typeof(long)) return (TResult)(object)(long)data.Count(); + if (typeof(TResult) == typeof(bool)) return (TResult)(object)data.Any(); + if (typeof(TResult) == typeof(object)) return (TResult)(object)(data.ToList()); + + // Unknown terminal: materialise and attempt a cast — handles e.g. First. + var list = data.ToList(); + if (list.Count > 0 && list[0] is TResult first) return first; + if (list is TResult asResult) return asResult; + + throw new NotSupportedException( + $"BLite query pipeline: unsupported terminal result type '{typeof(TResult)}'."); + } + + // ─── Aggregate push-down ───────────────────────────────────────────────── + + /// + /// Whether is Sum/Average/Min/Max with a + /// selector, applied to zero or more Where(x => ...) calls over the collection itself. + /// + private static bool IsAggregateOverFilteredRoot(Expression expression) + { + if (expression is not MethodCallExpression call + || call.Method.DeclaringType != typeof(Queryable) + || call.Arguments.Count != 2) + return false; + + var source = call.Arguments[0]; + while (source is MethodCallExpression where + && where.Method.DeclaringType == typeof(Queryable) + && where.Method.Name == nameof(Queryable.Where) + && where.Arguments[1] is UnaryExpression { Operand: LambdaExpression { Parameters.Count: 1 } }) + { + source = where.Arguments[0]; + } + return source is ConstantExpression { Value: IQueryable }; + } + + // ─── Expression-tree helpers ────────────────────────────────────────────── + + /// + /// Removes the Queryable.Where(x => ...) calls on the root chain whose input is the + /// fetched documents or a subset of them (only filtering, ordering and paging operators lie + /// between the root and the Where). folds each of these + /// into , which FetchAsync applied, so every element + /// reaching them already satisfies their predicate and dropping them leaves the result unchanged. + /// + /// + /// The rewritten expression, and whether its elements are still a subset of the fetched documents. + /// + private static (Expression Expression, bool IsFetchedSubset) DropAppliedWheres(Expression expression) + { + if (expression is ConstantExpression { Value: IQueryable }) + return (expression, true); + + if (expression is not MethodCallExpression call + || call.Method.DeclaringType != typeof(Queryable) + || call.Arguments.Count == 0) + return (expression, false); + + var (source, isFetchedSubset) = DropAppliedWheres(call.Arguments[0]); + + if (isFetchedSubset + && call.Method.Name == nameof(Queryable.Where) + && call.Arguments[1] is UnaryExpression { Operand: LambdaExpression { Parameters.Count: 1 } }) + return (source, true); + + var rewritten = source == call.Arguments[0] + ? call + : call.Update(call.Object, [source, .. call.Arguments.Skip(1)]); + return (rewritten, isFetchedSubset && s_subsetPreservingOperators.Contains(call.Method.Name)); + } + + /// + /// Replaces the terminal whose predicate folded into + /// (Count(p), First(p), ...) with its + /// parameterless overload. The visitor folds a predicate only when the terminal's source is the + /// root followed by plain Where and ordering calls, so the elements reaching the terminal are + /// fetched documents, which all satisfy the predicate already. + /// + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] + private static Expression DropFoldedTerminalPredicate(Expression expression, MethodCallExpression? foldedTerminal) + { + if (foldedTerminal is null || !ReferenceEquals(expression, foldedTerminal)) + return expression; + + var name = foldedTerminal.Method.Name; + if (!s_parameterlessTerminals.TryGetValue(name, out var definition)) + { + definition = typeof(Queryable).GetMethods().Single(m => m.Name == name && m.GetParameters().Length == 1); + s_parameterlessTerminals[name] = definition; + } + + var parameterless = definition.MakeGenericMethod(foldedTerminal.Method.GetGenericArguments()); + return Expression.Call(parameterless, foldedTerminal.Arguments[0]); + } + + /// Generic definitions of the parameterless Queryable terminals, by name. + private static readonly ConcurrentDictionary s_parameterlessTerminals = new(); + + /// Queryable operators whose output elements are a subset of their source's. + private static readonly HashSet s_subsetPreservingOperators = + [ + nameof(Queryable.Where), + nameof(Queryable.OrderBy), nameof(Queryable.OrderByDescending), + nameof(Queryable.ThenBy), nameof(Queryable.ThenByDescending), + nameof(Queryable.Skip), nameof(Queryable.Take), + nameof(Queryable.SkipWhile), nameof(Queryable.TakeWhile), + nameof(Queryable.Distinct), nameof(Queryable.Reverse), + ]; + + /// + /// Fallback for queries containing operators the direct pipeline cannot handle + /// (GroupBy, Join, Sum/Average/Min/Max with selectors, etc.). + /// The source data has already been fetched (and filtered by index/BSON-scan), + /// so we only need to rewrite the remaining Queryable.* calls to + /// Enumerable.* equivalents before compiling and executing. + /// + /// + /// The compiled Func<IEnumerable<T>, TResult> is cached per expression shape + /// (keyed by TResult.FullName + expression.ToString()), so the expensive + /// call is paid only once per unique query shape. + /// + [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] + private TResult ExecuteViaEnumerableRewriter(Expression expression, IEnumerable sourceData) + { + var rootFinder = new RootFinder(); + rootFinder.Visit(expression); + var root = rootFinder.Root; + if (root == null) + throw new InvalidOperationException("Could not find root IQueryable in expression."); + + var cacheKey = typeof(TResult).FullName + "\x00" + expression.ToString(); + + var cachedDelegate = s_enumRewriterCache.GetOrAdd(cacheKey, _ => + { + var sourceParam = Expression.Parameter(typeof(IEnumerable), "_src"); + var rewriter = new EnumerableRewriter(root, sourceParam); + var rewritten = rewriter.Visit(expression); + if (rewritten.Type != typeof(TResult)) + rewritten = Expression.Convert(rewritten, typeof(TResult)); + return Expression.Lambda, TResult>>(rewritten, sourceParam).Compile(); + }); + + return ((Func, TResult>)cachedDelegate)(sourceData); + } + + /// Cache of compiled EnumerableRewriter delegates keyed by expression shape. + private static readonly ConcurrentDictionary s_enumRewriterCache = new(); + + private sealed class RootFinder : ExpressionVisitor + { + public IQueryable? Root { get; private set; } + + protected override Expression VisitConstant(ConstantExpression node) + { + if (Root == null && node.Value is IQueryable q) + Root = q; + return base.VisitConstant(node); + } + } + + // ─── IBTreeQueryCore implementation ─────────────────────────────────── + + internal DocumentCollection Collection => _collection; + + IAsyncEnumerable IBTreeQueryCore.ScanAsync(BsonReaderPredicate predicate, CancellationToken ct) + => _collection.ScanAsync(predicate, ct); + + IAsyncEnumerable IBTreeQueryCore.ScanAsync(IndexQueryPlan plan, CancellationToken ct) + => _collection.ScanAsync(plan, ct); + + IEnumerable IBTreeQueryCore.GetIndexes() + => _collection.GetIndexes(); + + Task IBTreeQueryCore.CountAsync(CancellationToken ct) + => _collection.CountAsync(ct); + + ValueTask IBTreeQueryCore.MinBoundaryAsync(IndexMinMax plan, CancellationToken ct) + => _collection.MinBoundaryAsync(plan, ct); + + ValueTask IBTreeQueryCore.MaxBoundaryAsync(IndexMinMax plan, CancellationToken ct) + => _collection.MaxBoundaryAsync(plan, ct); +} diff --git a/src/BLite.Core/Query/LinqAggregate.cs b/src/BLite.Core/Query/LinqAggregate.cs new file mode 100644 index 00000000..25902f28 --- /dev/null +++ b/src/BLite.Core/Query/LinqAggregate.cs @@ -0,0 +1,192 @@ +namespace BLite.Core.Query; + +/// The LINQ aggregate operators that can run without materialising entities. +internal enum AggregateOperator +{ + Sum, + Average, + Min, + Max, +} + +/// +/// Folds the values of one numeric property the way the Enumerable operator does when LINQ +/// runs it over the materialised entities, so a pushed-down aggregate returns the same value and +/// throws the same exceptions: +/// +/// Sum of int accumulates in a checked int, of long in a checked +/// long, of float in a double (cast back at the end); Average of +/// int/long sums in a checked long. +/// Min/Max compare with (as +/// Comparer<T>.Default) and only replace on a strict improvement, so the first of +/// equal values wins (-0.0 vs 0.0, 1.0m vs 1.00m) and NaN is the +/// smallest value. +/// Nullable selectors skip null; on no values Sum is 0 and the others +/// return null. Non-nullable Average/Min/Max over no values throw +/// . +/// +/// Values must be added in the order LINQ would see them, since floating-point sums and checked +/// overflow depend on it. With anyOrder, int/long values may come in another +/// order (from an index): Sum/Average then keep the sums of the positive and of the +/// negative values apart. Every running total LINQ computes, in whatever order, lies between those +/// two, so when both fit the accumulator LINQ cannot overflow and the result is exact; otherwise +/// whether LINQ throws depends on the order and is set. +/// +internal sealed class LinqAggregate where TValue : struct, IComparable +{ + private readonly AggregateOperator _op; + private readonly bool _nullable; + private readonly bool _anyOrder; + + private long _count; + private TValue _best; + private int _sumInt32; + private long _sumInt64; + private long _sumPositive; // anyOrder: the sums of the positive and the negative values + private long _sumNegative; + private bool _accumulatorOverflow; + private double _sumDouble; + private decimal _sumDecimal; + + public LinqAggregate(AggregateOperator op, bool nullable, bool anyOrder = false) + { + _op = op; + _nullable = nullable; + _anyOrder = anyOrder; + } + + /// + /// With anyOrder: whether the values added so far could make LINQ overflow in some order + /// but not in another, so must not be used; the values have to be aggregated + /// in the order LINQ sees them. + /// + public bool DependsOnOrder + { + get + { + if (!_anyOrder || _op is AggregateOperator.Min or AggregateOperator.Max) return false; + if (_accumulatorOverflow) return true; + // Sum of int accumulates in an int; Average of int and everything of long in a long. + return typeof(TValue) == typeof(int) && _op == AggregateOperator.Sum + && (_sumPositive > int.MaxValue || _sumNegative < int.MinValue); + } + } + + /// + /// The result type of the Queryable operator for a + /// (or nullable) selector, e.g. double for Average of int. + /// + public static Type ResultType(AggregateOperator op, bool nullable) + { + if (op == AggregateOperator.Average && (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long))) + return nullable ? typeof(double?) : typeof(double); + return nullable ? typeof(TValue?) : typeof(TValue); + } + + /// Adds the value of the next document (nullable selectors pass null as such). + public void Add(TValue? value) + { + if (value.HasValue) Add(value.GetValueOrDefault()); + } + + /// Adds the value of the next document. + public void Add(TValue value) + { + switch (_op) + { + case AggregateOperator.Min: + if (_count == 0 || value.CompareTo(_best) < 0) _best = value; + break; + case AggregateOperator.Max: + if (_count == 0 || value.CompareTo(_best) > 0) _best = value; + break; + default: + Accumulate(value); + break; + } + _count++; + } + + private void Accumulate(TValue value) + { + if (_anyOrder && (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long))) + { + long v = typeof(TValue) == typeof(int) ? (int)(object)value : (long)(object)value; + if (v >= 0) + { + long sum = unchecked(_sumPositive + v); + if (sum < _sumPositive) _accumulatorOverflow = true; + else _sumPositive = sum; + } + else + { + long sum = unchecked(_sumNegative + v); + if (sum > _sumNegative) _accumulatorOverflow = true; + else _sumNegative = sum; + } + } + else if (typeof(TValue) == typeof(int)) + { + int v = (int)(object)value; + if (_op == AggregateOperator.Sum) _sumInt32 = checked(_sumInt32 + v); + else _sumInt64 = checked(_sumInt64 + v); + } + else if (typeof(TValue) == typeof(long)) + { + _sumInt64 = checked(_sumInt64 + (long)(object)value); + } + else if (typeof(TValue) == typeof(double)) + { + _sumDouble += (double)(object)value; + } + else if (typeof(TValue) == typeof(float)) + { + _sumDouble += (float)(object)value; + } + else if (typeof(TValue) == typeof(decimal)) + { + _sumDecimal += (decimal)(object)value; + } + else + { + throw new NotSupportedException($"Aggregating {typeof(TValue)} is not supported."); + } + } + + /// + /// The aggregate, boxed as (a nullable result boxes its value or is + /// null). + /// + public object? Result() + { + if (DependsOnOrder) + throw new InvalidOperationException("The result depends on the order of the values."); + + switch (_op) + { + case AggregateOperator.Sum: + if (typeof(TValue) == typeof(int)) return _anyOrder ? (int)(_sumPositive + _sumNegative) : _sumInt32; + if (typeof(TValue) == typeof(long)) return Int64Sum(); + if (typeof(TValue) == typeof(double)) return _sumDouble; + if (typeof(TValue) == typeof(float)) return (float)_sumDouble; + return _sumDecimal; + + case AggregateOperator.Average: + if (_count == 0) return NoElements(); + if (typeof(TValue) == typeof(int) || typeof(TValue) == typeof(long)) return (double)Int64Sum() / _count; + if (typeof(TValue) == typeof(double)) return _sumDouble / _count; + if (typeof(TValue) == typeof(float)) return (float)(_sumDouble / _count); + return _sumDecimal / _count; + + default: + return _count == 0 ? NoElements() : _best; + } + } + + // anyOrder: a positive and a negative long, so their sum cannot overflow. + private long Int64Sum() => _anyOrder ? _sumPositive + _sumNegative : _sumInt64; + + private object? NoElements() => _nullable + ? null + : throw new InvalidOperationException("Sequence contains no elements"); +} diff --git a/src/BLite.SourceGenerators/CodeGenerator.cs b/src/BLite.SourceGenerators/CodeGenerator.cs index 5cb63411..f00e565b 100644 --- a/src/BLite.SourceGenerators/CodeGenerator.cs +++ b/src/BLite.SourceGenerators/CodeGenerator.cs @@ -174,6 +174,8 @@ public static string GenerateMapperClass(EntityInfo entity, string mapperNamespa { sb.AppendLine(); GenerateIdAccessors(sb, entity); + sb.AppendLine(); + GenerateVerbatimFieldNames(sb, entity); } // Emit PersonalData static members (WP1) @@ -1075,6 +1077,41 @@ private static void GenerateIdAccessors(StringBuilder sb, EntityInfo entity) } } + /// + /// Emits GetVerbatimFieldName: for each scalar property that Deserialize restores + /// unchanged from its field, the field name, so queries can read the value from raw BSON. A + /// property qualifies when it is an auto-property with a setter, has no converter, and is not + /// passed to the constructor (which could transform it); computed, ignored and get-only + /// properties are not in or have no setter. + /// + private static void GenerateVerbatimFieldNames(StringBuilder sb, EntityInfo entity) + { + var ctorParams = entity.SelectedConstructorParameters != null + ? new HashSet(entity.SelectedConstructorParameters.Select(p => p.MatchedPropertyName)) + : new HashSet(); + + var verbatim = entity.Properties + .Where(p => p.IsAutoProperty + && p.HasAnySetter + && p.ConverterTypeName == null + && !p.IsCollection && !p.IsDictionary && !p.IsNestedObject && !p.IsEnum + && !ctorParams.Contains(p.Name)) + .ToList(); + + sb.AppendLine($" public override string? GetVerbatimFieldName(string propertyName)"); + sb.AppendLine($" {{"); + sb.AppendLine($" switch (propertyName)"); + sb.AppendLine($" {{"); + foreach (var prop in verbatim) + { + var fieldName = prop.IsKey ? BLiteConventions.BsonIdFieldName : prop.BsonFieldName; + sb.AppendLine($" case \"{prop.Name}\": return \"{fieldName}\";"); + } + sb.AppendLine($" default: return null;"); + sb.AppendLine($" }}"); + sb.AppendLine($" }}"); + } + private static string GetBaseMapperClass(PropertyInfo? keyProp, EntityInfo entity) { if (keyProp?.ConverterTypeName != null) diff --git a/src/BLite.SourceGenerators/EntityAnalyzer.cs b/src/BLite.SourceGenerators/EntityAnalyzer.cs index 9946080a..bfff24a0 100644 --- a/src/BLite.SourceGenerators/EntityAnalyzer.cs +++ b/src/BLite.SourceGenerators/EntityAnalyzer.cs @@ -185,6 +185,7 @@ private static void AnalyzeProperties(INamedTypeSymbol typeSymbol, List f.AssociatedSymbol?.Equals(property, SymbolEqualityComparer.Default) == true); } + /// + /// True when every accessor of is compiler-implemented + /// ({ get; set; }, { get; init; }, { get; private set; }), so the getter + /// returns exactly the value last assigned. False for accessors with a body, including ones + /// using the field keyword, for partial and abstract properties, and for record + /// positional parameters. + /// + public static bool IsAutoProperty(IPropertySymbol property) + { + if (property.IsAbstract || property.IsExtern) + return false; + + if (property.DeclaringSyntaxReferences.IsEmpty) + { + // Compiled assembly: auto-implemented accessors carry [CompilerGenerated]. A private + // setter may be invisible here (SetMethod == null); the getter decides then. + return IsCompilerGenerated(property.GetMethod) + && (property.SetMethod == null || IsCompilerGenerated(property.SetMethod)); + } + + foreach (var reference in property.DeclaringSyntaxReferences) + { + if (reference.GetSyntax() is not PropertyDeclarationSyntax declaration + || declaration.ExpressionBody != null + || declaration.AccessorList == null + || declaration.Modifiers.Any(SyntaxKind.PartialKeyword)) + return false; + + foreach (var accessor in declaration.AccessorList.Accessors) + { + if (accessor.Body != null || accessor.ExpressionBody != null) + return false; + } + } + return true; + + static bool IsCompilerGenerated(IMethodSymbol? accessor) + => accessor != null && accessor.GetAttributes().Any(a => + a.AttributeClass?.ToDisplayString() == "System.Runtime.CompilerServices.CompilerGeneratedAttribute"); + } + /// /// Returns true when is (or implements) IDictionary<TKey, TValue>. /// Populates and with the two type arguments. diff --git a/src/BLite.SourceGenerators/Models/EntityInfo.cs b/src/BLite.SourceGenerators/Models/EntityInfo.cs index c0ed88d7..51d77e53 100644 --- a/src/BLite.SourceGenerators/Models/EntityInfo.cs +++ b/src/BLite.SourceGenerators/Models/EntityInfo.cs @@ -95,6 +95,12 @@ public class PropertyInfo public bool HasInitOnlySetter { get; set; } public bool HasAnySetter { get; set; } public bool IsReadOnlyGetter { get; set; } + + /// + /// True when every accessor is compiler-implemented ({ get; set; }, { get; init; }, + /// { get; private set; }): the getter returns exactly what was last assigned. + /// + public bool IsAutoProperty { get; set; } public string? BackingFieldName { get; set; } /// diff --git a/tests/BLite.Benchmark/LinqAggregateBenchmarks.cs b/tests/BLite.Benchmark/LinqAggregateBenchmarks.cs new file mode 100644 index 00000000..b48c29c1 --- /dev/null +++ b/tests/BLite.Benchmark/LinqAggregateBenchmarks.cs @@ -0,0 +1,89 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BLite.Shared; +using System.IO; + +namespace BLite.Benchmark; + +/// +/// LINQ Sum / Average / Max over 10 000 documents, +/// against materialising the entities and aggregating them with LINQ to Objects (what every +/// aggregate with a selector did before). Covers each push-down path: +/// +/// index keys: int property with a B-Tree index, with and without a range filter; +/// index boundary: Max of an indexed int reads one entry; +/// raw BSON: double and decimal properties without an index, with and +/// without a filter on a non-indexed field. +/// +/// +[SimpleJob] +[InProcess] +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +public class LinqAggregateBenchmarks +{ + private const int DocCount = 10_000; + + private string _dbPath = null!; + private AggregateDbContext _ctx = null!; + + [GlobalSetup] + public async Task Setup() + { + _dbPath = Path.Combine(AppContext.BaseDirectory, $"linq_agg_{Guid.NewGuid():N}.db"); + _ctx = new AggregateDbContext(_dbPath); + await _ctx.Readings.InsertBulkAsync(Enumerable.Range(0, DocCount).Select(i => new Reading + { + Tag = "abcd"[i % 4].ToString(), + Count = (i * 7919) % 1000, + Total = i * 1_000L, + Ratio = Math.Sin(i) * 100, + Weight = i / 3f, + Price = i * 1.25m, + })); + } + + [GlobalCleanup] + public void Cleanup() + { + _ctx?.Dispose(); + foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + if (File.Exists(f)) File.Delete(f); + } + + // ── int property with an index ────────────────────────────────────────── + + [Benchmark(Baseline = true, Description = "Sum(int) – materialise + LINQ to Objects")] + [BenchmarkCategory("IndexedInt")] + public int SumIndexed_Materialised() => _ctx.Readings.AsQueryable().ToList().Sum(r => r.Count); + + [Benchmark(Description = "Sum(int) – index keys")] + [BenchmarkCategory("IndexedInt")] + public int SumIndexed() => _ctx.Readings.AsQueryable().Sum(r => r.Count); + + [Benchmark(Description = "Where(range).Sum(int) – index keys")] + [BenchmarkCategory("IndexedInt")] + public int SumIndexedRange() => _ctx.Readings.AsQueryable().Where(r => r.Count >= 250 && r.Count < 500).Sum(r => r.Count); + + [Benchmark(Description = "Max(int) – index boundary")] + [BenchmarkCategory("IndexedInt")] + public int MaxIndexed() => _ctx.Readings.AsQueryable().Max(r => r.Count); + + // ── properties without an index ───────────────────────────────────────── + + [Benchmark(Baseline = true, Description = "Sum(double) – materialise + LINQ to Objects")] + [BenchmarkCategory("RawBson")] + public double SumDouble_Materialised() => _ctx.Readings.AsQueryable().ToList().Sum(r => r.Ratio); + + [Benchmark(Description = "Sum(double) – raw BSON")] + [BenchmarkCategory("RawBson")] + public double SumDouble() => _ctx.Readings.AsQueryable().Sum(r => r.Ratio); + + [Benchmark(Description = "Average(decimal) – raw BSON")] + [BenchmarkCategory("RawBson")] + public decimal AverageDecimal() => _ctx.Readings.AsQueryable().Average(r => r.Price); + + [Benchmark(Description = "Where(tag).Sum(double) – raw BSON")] + [BenchmarkCategory("RawBson")] + public double SumDoubleFiltered() => _ctx.Readings.AsQueryable().Where(r => r.Tag == "b").Sum(r => r.Ratio); +} diff --git a/tests/BLite.Shared/AggregateContexts.cs b/tests/BLite.Shared/AggregateContexts.cs new file mode 100644 index 00000000..0ea5a868 --- /dev/null +++ b/tests/BLite.Shared/AggregateContexts.cs @@ -0,0 +1,178 @@ +using System.ComponentModel.DataAnnotations.Schema; +using BLite.Bson; +using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Metadata; + +namespace BLite.Shared; + +/// One property per numeric type LINQ aggregates, plain and nullable. +public class Reading +{ + public ObjectId Id { get; set; } + public string Tag { get; set; } = string.Empty; + public int Count { get; set; } + public long Total { get; set; } + public int Units { get; set; } + public double Ratio { get; set; } + public float Weight { get; set; } + public decimal Price { get; set; } + public int? MaybeCount { get; set; } + public long? MaybeTotal { get; set; } + public double? MaybeRatio { get; set; } + public float? MaybeWeight { get; set; } + public decimal? MaybePrice { get; set; } + + /// + /// Stored as "score", while is stored as "level": each field + /// carries the conventional name of the other property. + /// + [Column("score")] + public int Level { get; set; } + + [Column("level")] + public int Score { get; set; } +} + +/// +/// with B-Tree indexes on Count (int), Total (long) and +/// MaybeCount (int?), and none on the other properties. +/// +public partial class AggregateDbContext : DocumentDbContext +{ + public DocumentCollection Readings { get; set; } = null!; + + public AggregateDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("readings") + .HasIndex(r => r.Count) + .HasIndex(r => r.Total) + .HasIndex(r => r.MaybeCount); + } +} + +/// A later version of with properties (initialised) that older documents lack. +public class ReadingV2 +{ + public ObjectId Id { get; set; } + public string Tag { get; set; } = string.Empty; + public int Count { get; set; } + public long Total { get; set; } + public int? MaybeCount { get; set; } + public int Bonus { get; set; } = 7; + public double? Factor { get; set; } = 1.5; +} + +/// Reads the readings collection as . +public partial class AggregateV2DbContext : DocumentDbContext +{ + public DocumentCollection Readings { get; set; } = null!; + + public AggregateV2DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().ToCollection("readings"); +} + +/// Stores hidden, unmapped and gone, which no longer reads. +public class ShapeV1 +{ + public ObjectId Id { get; set; } + public int Count { get; set; } + public int Hidden { get; set; } + public int Unmapped { get; set; } + public int Gone { get; set; } +} + +public partial class ShapeV1DbContext : DocumentDbContext +{ + public DocumentCollection Shapes { get; set; } = null!; + + public ShapeV1DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().ToCollection("shapes"); +} + +/// +/// A later version of : the fields older documents still hold are no longer +/// the values of these properties. +/// +public class ShapeV2 +{ + public ObjectId Id { get; set; } + public int Count { get; set; } + + [BsonIgnore] + public int Hidden { get; set; } + + [NotMapped] + public int Unmapped { get; set; } + + public int Gone => Count * 10; +} + +public partial class ShapeV2DbContext : DocumentDbContext +{ + public DocumentCollection Shapes { get; set; } = null!; + + public ShapeV2DbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().ToCollection("shapes"); +} + +/// Accessors with logic: the stored value is not what the materialised getter returns. +public class Doubled +{ + private int _g; + private long _h; + + public ObjectId Id { get; set; } + + /// Stores 2 * _g; reading it back sets _g to that, so the getter doubles it again. + public int G { get => _g * 2; set => _g = value; } + + /// The same, with the logic in the setter of a property with a B-Tree index. + public long H { get => _h; set => _h = value + 1; } + + public int Plain { get; set; } +} + +/// A key named Code next to an ordinary property named Id. +public class Coded +{ + [System.ComponentModel.DataAnnotations.Key] + public int Code { get; set; } + + public int Id { get; set; } +} + +/// Its constructor transforms the value it is passed when the entity is read back. +public class Scaled +{ + public Scaled(int amount) => Amount = amount * 3; + + public ObjectId Id { get; set; } + public int Amount { get; set; } +} + +public partial class VerbatimDbContext : DocumentDbContext +{ + public DocumentCollection Doubled { get; set; } = null!; + public DocumentCollection Coded { get; set; } = null!; + public DocumentCollection Scaled { get; set; } = null!; + + public VerbatimDbContext(string path) : base(path) => InitializeCollections(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().ToCollection("doubled") + .HasIndex(d => d.G) + .HasIndex(d => d.H); + modelBuilder.Entity().ToCollection("coded"); + modelBuilder.Entity().ToCollection("scaled"); + } +} diff --git a/tests/BLite.Tests/LinqAggregateTests.cs b/tests/BLite.Tests/LinqAggregateTests.cs new file mode 100644 index 00000000..5522b62e --- /dev/null +++ b/tests/BLite.Tests/LinqAggregateTests.cs @@ -0,0 +1,619 @@ +using System.Linq.Expressions; +using System.Reflection; +using BLite.Bson; +using BLite.Core; +using BLite.Core.Collections; +using BLite.Core.Query; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// LINQ Sum, Average, Min and Max computed from index keys or raw BSON +/// return exactly what LINQ returns over the materialised entities, or throw the same exception. +/// +public class LinqAggregateTests : IDisposable +{ + private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"agg_{Guid.NewGuid():N}.db"); + + public void Dispose() + { + foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ } + } + + private static readonly string[] Operators = ["Sum", "Average", "Min", "Max"]; + private static readonly string[] Indexed = ["Count", "Total", "MaybeCount"]; + private static readonly string[] Properties = + ["Count", "Total", "Units", "Ratio", "Weight", "Price", "MaybeCount", "MaybeTotal", "MaybeRatio", "MaybeWeight", "MaybePrice"]; + + private static Reading Make(int i) => new() + { + Tag = "abc"[i % 3].ToString(), + Count = (i * 7) % 13 - 4, + Total = i * 1_000_000_007L - 5_000_000_000L, + Units = i * 3 - 50, + Ratio = Math.Sin(i) * 100 / 3, + Weight = (float)(i / 7.0), + // Same values at different scales: 1.5 / 1.50 / 1.500. + Price = (i % 3) switch { 0 => i * 1.5m, 1 => i * 1.50m, _ => i * 1.500m }, + MaybeCount = i % 4 == 0 ? null : i - 20, + MaybeTotal = i % 4 == 1 ? null : i * 3L, + MaybeRatio = i % 4 == 2 ? null : 1.0 / (i + 1), + MaybeWeight = i % 4 == 3 ? null : (float)(i * 0.1), + MaybePrice = i % 4 == 0 ? null : i * 0.10m, + Level = i, + Score = 100 + i, + }; + + private static readonly (string Name, Expression>? Where)[] Filters = + [ + ("none", null), + ("tag", r => r.Tag == "b"), + ("count range", r => r.Count >= -1 && r.Count < 5), + ("count range and tag", r => r.Count > 0 && r.Tag == "c"), + ("no match", r => r.Tag == "zzz"), + ("empty count range", r => r.Count > 1000), + ]; + + /// Whether the aggregate should be computed without materialising the entities. + private static bool ExpectPushedDown(string property, string filter) => filter switch + { + "none" or "tag" or "no match" => true, // index keys, or raw BSON without/with a predicate + "count range" or "empty count range" => property == "Count", // the Count index covers the filter + _ => false, // another index serves the filter + }; + + [Fact] + public async Task EveryOperatorPropertyAndFilter_MatchesLinqOverTheEntities() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 40; i++) + await db.Readings.InsertAsync(Make(i)); + var entities = db.Readings.AsQueryable().ToList(); + + var failures = new List(); + foreach (var op in Operators) + foreach (var property in Properties) + foreach (var (filter, where) in Filters) + { + var selector = Selector(property); + var label = $"{op}({property}) where {filter}"; + + // LINQ over the matching entities, in the order the collection returns them for the filter + // (primary-key order, or index order when an index serves it). + var matches = where == null ? entities : db.Readings.AsQueryable().Where((Expression>)where).ToList(); + var expected = Run(() => Execute(matches.AsQueryable(), op, selector, null)); + var actual = Run(() => Execute(db.Readings.AsQueryable(), op, selector, where)); + if (!Same(expected, actual)) + failures.Add($"{label}: expected {Show(expected)}, got {Show(actual)}"); + + var resultType = AggregateCall(entities.AsQueryable(), op, selector, where).Type; + var direct = await RunAsync(() => db.Readings.TryAggregateAsync(Enum.Parse(op), selector, where, resultType)); + bool handled = direct.Error != null || direct.Value.Handled; + if (handled != ExpectPushedDown(property, filter)) + failures.Add($"{label}: pushed down = {handled}, expected {ExpectPushedDown(property, filter)}"); + else if (handled && !Same(expected, (direct.Value.Value, direct.Error))) + failures.Add($"{label} (pushed down): expected {Show(expected)}, got {Show((direct.Value.Value, direct.Error))}"); + } + + Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures)); + } + + [Theory] + [InlineData(new[] { 0.0, -0.0, double.NaN, 1.0, -0.0 })] + [InlineData(new[] { -0.0, 0.0, 2.5, double.NaN })] + [InlineData(new[] { double.NaN, double.NaN })] + [InlineData(new[] { double.PositiveInfinity, double.NegativeInfinity, 1.0 })] + [InlineData(new[] { 1e308, 1e308, -1e308 })] + [InlineData(new[] { 447711.6875, 7923489.5, 16777216.0 })] // float Average: (float)(sum / n) != (float)sum / n + public async Task FloatingPointEdgeCases_MatchLinq(double[] values) + { + using var db = new AggregateDbContext(_dbPath); + foreach (var v in values) + await db.Readings.InsertAsync(new Reading { Ratio = v, MaybeRatio = v, Weight = (float)v, MaybeWeight = (float)v }); + var entities = db.Readings.AsQueryable().ToList(); + + foreach (var op in Operators) + foreach (var property in new[] { "Ratio", "MaybeRatio", "Weight", "MaybeWeight" }) + { + var selector = Selector(property); + var expected = Run(() => Execute(entities.AsQueryable(), op, selector, null)); + var direct = await db.Readings.TryAggregateAsync(Enum.Parse(op), selector, null, AggregateCall(entities.AsQueryable(), op, selector, null).Type); + + Assert.True(direct.Handled, $"{op}({property})"); + Assert.True(Same(expected, (direct.Value, null)), $"{op}({property}): expected {Show(expected)}, got {Show((direct.Value, null))}"); + } + } + + [Fact] + public async Task DecimalScale_MatchesLinq() + { + using var db = new AggregateDbContext(_dbPath); + foreach (var price in new[] { 1.00m, 1.0m, 2.50m, 1m }) + await db.Readings.InsertAsync(new Reading { Price = price, MaybePrice = price }); + + var entities = db.Readings.AsQueryable().ToList(); + string Text(decimal? d) => d?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "null"; + + Assert.Equal(Text(entities.Sum(r => r.Price)), Text(db.Readings.AsQueryable().Sum(r => r.Price))); + Assert.Equal(Text(entities.Min(r => r.Price)), Text(db.Readings.AsQueryable().Min(r => r.Price))); // first of the equal minima + Assert.Equal(Text(entities.Max(r => r.MaybePrice)), Text(db.Readings.AsQueryable().Max(r => r.MaybePrice))); + Assert.Equal(Text(entities.Average(r => r.MaybePrice)), Text(db.Readings.AsQueryable().Average(r => r.MaybePrice))); + } + + [Fact] + public async Task IntegerOverflow_ThrowsLikeLinq() + { + using var db = new AggregateDbContext(_dbPath); + await db.Readings.InsertAsync(new Reading { Units = int.MaxValue, Count = int.MaxValue, Total = long.MaxValue }); + await db.Readings.InsertAsync(new Reading { Units = 1, Count = 1, Total = 1 }); + + Assert.Throws(() => db.Readings.AsQueryable().Sum(r => r.Units)); // raw BSON + Assert.Throws(() => db.Readings.AsQueryable().Sum(r => r.Count)); // index keys + Assert.Throws(() => db.Readings.AsQueryable().Sum(r => r.Total)); + Assert.Throws(() => db.Readings.AsQueryable().Average(r => r.Total)); + Assert.Equal((int.MaxValue + 1.0) / 2, db.Readings.AsQueryable().Average(r => r.Units)); + Assert.Equal((int.MaxValue + 1.0) / 2, db.Readings.AsQueryable().Average(r => r.Count)); + } + + [Theory] + // In primary-key order. LINQ overflows on a running total although the total fits. + [InlineData(new[] { int.MaxValue, 1, -2 }, true)] + [InlineData(new[] { int.MinValue, -1, 5 }, true)] + // The positive or negative values alone overflow, but no running total in this order does. + [InlineData(new[] { 5, int.MinValue, -1 }, false)] + [InlineData(new[] { -2, int.MaxValue, 1 }, false)] + public async Task IndexKeySums_OverflowWhereLinqDoes(int[] values, bool linqOverflows) + { + // Index keys come in value order, not the order LINQ sees the entities in, and whether LINQ + // overflows depends on that order: such sums must not be taken from the keys. + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < values.Length; i++) + { + long total = values[i] switch { int.MaxValue => long.MaxValue, int.MinValue => long.MinValue, var v => v }; + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = values[i], Total = total }); + } + + var entities = db.Readings.AsQueryable().ToList(); + Assert.Equal(values, entities.Select(r => r.Count)); // primary-key order + Assert.Equal(linqOverflows, Run(() => entities.Sum(r => r.Count)).Error == typeof(OverflowException)); + + var q = db.Readings.AsQueryable(); + Assert.True(Same(Run(() => entities.Sum(r => r.Count)), Run(() => q.Sum(r => r.Count)))); + Assert.True(Same(Run(() => entities.Sum(r => r.Total)), Run(() => q.Sum(r => r.Total)))); + Assert.True(Same(Run(() => entities.Average(r => r.Total)), Run(() => q.Average(r => r.Total)))); + Assert.Equal(entities.Average(r => r.Count), q.Average(r => r.Count)); + + // With a filter the index covers, the entities come in index order. + var filtered = q.Where(r => r.Count < 100).ToList(); + Assert.True(Same(Run(() => filtered.Sum(r => r.Count)), Run(() => q.Where(r => r.Count < 100).Sum(r => r.Count)))); + var filteredTotal = q.Where(r => r.Total < 100).ToList(); + Assert.True(Same(Run(() => filteredTotal.Sum(r => r.Total)), Run(() => q.Where(r => r.Total < 100).Sum(r => r.Total)))); + } + + [Fact] + public async Task IndexOnAConvertedProperty_IsNotDecodedAsThePropertyType() + { + // The keys of this index are longs although its property path is the int Units. + using var db = new AggregateDbContext(_dbPath); + await db.Readings.CreateIndexAsync(r => (long)r.Units); + for (int i = 0; i < 10; i++) + await db.Readings.InsertAsync(Make(i)); + + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + Assert.Equal(entities.Sum(r => r.Units), q.Sum(r => r.Units)); + Assert.Equal(entities.Min(r => r.Units), q.Min(r => r.Units)); + Assert.Equal(entities.Max(r => r.Units), q.Max(r => r.Units)); + } + + [Fact] + public async Task FilterBoundOfAnotherType_IsNotUsedAsAKeyRange() + { + // The optimizer keeps the constant's type, so these bounds are a double and a long against + // the int Count index; encoded as keys they would bound the wrong range. + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 20; i++) + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = i, Total = i }); + + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + Assert.Equal(entities.Where(r => r.Count < 9.5).Max(r => r.Count), q.Where(r => r.Count < 9.5).Max(r => r.Count)); + Assert.Equal(entities.Where(r => r.Count > 4L).Sum(r => r.Count), q.Where(r => r.Count > 4L).Sum(r => r.Count)); + Assert.Equal(entities.Where(r => r.Count > 4).Sum(r => r.Count), q.Where(r => r.Count > 4).Sum(r => r.Count)); + } + + [Fact] + public async Task FilterWithASubqueryOverTheCollection_MatchesLinq() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 20; i++) + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = i % 7, Total = i }); + + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + Assert.Equal( + entities.Where(r => entities.Any(o => o.Count == r.Count + 1)).Sum(r => r.Count), + q.Where(r => q.Any(o => o.Count == r.Count + 1)).Sum(r => r.Count)); + Assert.Equal( + entities.Where(r => r.Count > 2 && entities.Count(o => o.Total < r.Total) > 3).Max(r => r.Total), + q.Where(r => r.Count > 2 && q.Count(o => o.Total < r.Total) > 3).Max(r => r.Total)); + } + + [Fact] + public async Task IndexKeySums_ThatCannotOverflow_AreTakenFromTheKeys() + { + // The positive values sum to exactly int.MaxValue and the negative ones to exactly + // int.MinValue, so no running total overflows in any order and the sum is taken from the + // keys. (One more below int.MinValue and the result would depend on the order: see + // IndexKeySums_OverflowWhereLinqDoes.) Explicit ids keep the document order fixed. + int[] counts = [int.MaxValue - 10, 4, -7, int.MinValue + 7, 6]; + Assert.Equal(int.MaxValue, counts.Where(c => c > 0).Sum(c => (long)c)); + Assert.Equal(int.MinValue, counts.Where(c => c < 0).Sum(c => (long)c)); + + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < counts.Length; i++) + await db.Readings.InsertAsync(new Reading { Id = new ObjectId(i + 1, 0), Count = counts[i], Total = counts[i] }); + + var entities = db.Readings.AsQueryable().ToList(); + var sum = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Count"), null, typeof(int)); + Assert.True(sum.Handled); + Assert.Equal(entities.Sum(r => r.Count), sum.Value); + var average = await db.Readings.TryAggregateAsync(AggregateOperator.Average, Selector("Total"), null, typeof(double)); + Assert.True(average.Handled); + Assert.Equal(entities.Average(r => r.Total), average.Value); + } + + [Fact] + public async Task DocumentsWithoutTheField_UseTheEntityValue() + { + using (var v1 = new AggregateDbContext(_dbPath)) + { + for (int i = 0; i < 5; i++) + await v1.Readings.InsertAsync(Make(i)); + } + + // Whatever value the mapper gives a missing property, the aggregate must use the same one. + using var v2 = new AggregateV2DbContext(_dbPath); + var entities = v2.Readings.AsQueryable().ToList(); + + Assert.False((await v2.Readings.TryAggregateAsync(AggregateOperator.Sum, (Expression>)(r => r.Bonus), null, typeof(int))).Handled); + Assert.False((await v2.Readings.TryAggregateAsync(AggregateOperator.Max, (Expression>)(r => r.Factor), null, typeof(double?))).Handled); + Assert.Equal(entities.Sum(r => r.Bonus), v2.Readings.AsQueryable().Sum(r => r.Bonus)); + Assert.Equal(entities.Max(r => r.Factor), v2.Readings.AsQueryable().Max(r => r.Factor)); + Assert.Equal(entities.Sum(r => r.Count), v2.Readings.AsQueryable().Sum(r => r.Count)); + } + + [Theory] + [InlineData("units", "null")] // null in a non-nullable int + [InlineData("units", "int64")] // an Int64 in an int field + [InlineData("ratio", "null")] + [InlineData("ratio", "int32")] // an Int32 in a double field + public async Task FieldsWithAnotherBsonType_UseTheEntityValue(string field, string kind) + { + using (var db = new AggregateDbContext(_dbPath)) + { + for (int i = 0; i < 4; i++) + await db.Readings.InsertAsync(Make(i)); + } + + // Written through the schemaless API, which stores whatever it is given. + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("readings"); + await col.InsertAsync(col.CreateDocument(["_id", "tag", field], b => + { + b.AddObjectId("_id", ObjectId.NewObjectId()).AddString("tag", "x"); + switch (kind) + { + case "null": b.AddNull(field); break; + case "int64": b.AddInt64(field, 5L); break; + default: b.AddInt32(field, 2); break; + } + })); + } + + using var reopened = new AggregateDbContext(_dbPath); + var property = field == "units" ? "Units" : "Ratio"; + var resultType = field == "units" ? typeof(int) : typeof(double); + + // A null the entity cannot hold makes materialising throw; the aggregate must throw alike. + var entities = Run(() => reopened.Readings.AsQueryable().ToList()); + var expected = entities.Error != null + ? entities + : Run(() => Execute(((List)entities.Value!).AsQueryable(), "Sum", Selector(property), null)); + + Assert.False((await reopened.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector(property), null, resultType)).Handled); + Assert.True(Same(expected, Run(() => Execute(reopened.Readings.AsQueryable(), "Sum", Selector(property), null))), Show(expected)); + } + + [Fact] + public async Task RenamedProperty_IsReadFromItsOwnField() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 5; i++) + await db.Readings.InsertAsync(Make(i)); + + // Level is stored as "score" and Score as "level". + var level = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Level"), null, typeof(int)); + var score = await db.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Score"), null, typeof(int)); + Assert.Equal((true, (object?)10), level); + Assert.Equal((true, (object?)510), score); + Assert.Equal(10, db.Readings.AsQueryable().Sum(r => r.Level)); + Assert.Equal(510, db.Readings.AsQueryable().Sum(r => r.Score)); + } + + [Fact] + public async Task FieldAfterOthers_InDocumentsWithoutOffsetTable_IsFound() + { + using (var db = new AggregateDbContext(_dbPath)) + { + for (int i = 0; i < 4; i++) + await db.Readings.InsertAsync(Make(i)); + } + + // The schemaless API writes no offset table, so the field is looked up by name, after the + // fields before it. + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("readings"); + for (int i = 0; i < 3; i++) + { + var units = 1000 * (i + 1); + await col.InsertAsync(col.CreateDocument(["_id", "tag", "units"], b => + b.AddObjectId("_id", ObjectId.NewObjectId()).AddString("tag", "x").AddInt32("units", units))); + } + } + + using var reopened = new AggregateDbContext(_dbPath); + var entities = reopened.Readings.AsQueryable().ToList(); + Assert.Equal(7, entities.Count); + + var sum = await reopened.Readings.TryAggregateAsync(AggregateOperator.Sum, Selector("Units"), null, typeof(int)); + Assert.True(sum.Handled); + Assert.Equal(entities.Sum(r => r.Units), sum.Value); + Expression> tagX = r => r.Tag == "x"; + var filtered = await reopened.Readings.TryAggregateAsync(AggregateOperator.Max, Selector("Units"), tagX, typeof(int)); + Assert.Equal((true, (object?)3000), filtered); + } + + /// Writes five documents and reopens them as . + private async Task ShapesWrittenBeforeTheyWereIgnoredAsync() + { + using (var v1 = new ShapeV1DbContext(_dbPath)) + { + for (int i = 0; i < 5; i++) + await v1.Shapes.InsertAsync(new ShapeV1 { Count = i, Hidden = 100 + i, Unmapped = 200 + i, Gone = 1000 + i }); + } + return new ShapeV2DbContext(_dbPath); + } + + [Fact] + public async Task IgnoredProperty_OverDocumentsThatStillHoldTheField_IsMaterialised() + { + using var db = await ShapesWrittenBeforeTheyWereIgnoredAsync(); + var shapes = db.Shapes.AsQueryable().ToList(); + var q = db.Shapes.AsQueryable(); + + Assert.Equal(0, shapes.Sum(s => s.Hidden)); + Assert.Equal(shapes.Sum(s => s.Hidden), q.Sum(s => s.Hidden)); // [BsonIgnore] + Assert.Equal(shapes.Max(s => s.Hidden), q.Max(s => s.Hidden)); + Assert.Equal(shapes.Sum(s => s.Unmapped), q.Sum(s => s.Unmapped)); // [NotMapped] + Assert.Equal(shapes.Sum(s => s.Count), q.Sum(s => s.Count)); + } + + [Fact] + public async Task ComputedProperty_OverDocumentsThatStoredIt_IsMaterialised() + { + using var db = await ShapesWrittenBeforeTheyWereIgnoredAsync(); + var shapes = db.Shapes.AsQueryable().ToList(); + var q = db.Shapes.AsQueryable(); + + Assert.Equal(100, shapes.Sum(s => s.Gone)); + Assert.Equal(shapes.Sum(s => s.Gone), q.Sum(s => s.Gone)); + Assert.Equal(shapes.Min(s => s.Gone), q.Min(s => s.Gone)); + Assert.Equal(shapes.Average(s => s.Gone), q.Average(s => s.Gone)); + } + + [Fact] + public async Task AccessorsWithLogic_AreMaterialised() + { + using var db = new VerbatimDbContext(_dbPath); + for (int i = 1; i <= 4; i++) + await db.Doubled.InsertAsync(new Doubled { G = i * 5, H = i * 7, Plain = i }); + + var doubled = db.Doubled.AsQueryable().ToList(); + var q = db.Doubled.AsQueryable(); + + // G: getter with logic, indexed (index keys) and filtered on another property (raw BSON). + Assert.Equal(doubled.Sum(d => d.G), q.Sum(d => d.G)); + Assert.Equal(doubled.Min(d => d.G), q.Min(d => d.G)); + Assert.Equal(doubled.Max(d => d.G), q.Max(d => d.G)); + Assert.Equal(doubled.Average(d => d.G), q.Average(d => d.G)); + Assert.Equal(doubled.Where(d => d.Plain > 1).Sum(d => d.G), q.Where(d => d.Plain > 1).Sum(d => d.G)); + Assert.Equal(doubled.Where(d => d.Plain > 1).Max(d => d.G), q.Where(d => d.Plain > 1).Max(d => d.G)); + // The index on G holds the stored values, so the filter matches what the index says (as on + // main); the aggregate must still use the materialised values of those documents. + Assert.Equal(q.Where(d => d.G > 15).ToList().Sum(d => d.G), q.Where(d => d.G > 15).Sum(d => d.G)); + + // H: setter with logic, indexed long. + Assert.Equal(doubled.Sum(d => d.H), q.Sum(d => d.H)); + Assert.Equal(doubled.Min(d => d.H), q.Min(d => d.H)); + Assert.Equal(doubled.Max(d => d.H), q.Max(d => d.H)); + Assert.Equal(doubled.Average(d => d.H), q.Average(d => d.H)); + + // A plain property next to them is still aggregated without materialising. + var plain = await db.Doubled.TryAggregateAsync(AggregateOperator.Sum, (Expression>)(d => d.Plain), null, typeof(int)); + Assert.Equal((true, (object?)10), plain); + } + + [Fact] + public async Task PropertyNamedId_ThatIsNotTheKey_IsReadFromItsOwnField() + { + using var db = new VerbatimDbContext(_dbPath); + for (int i = 1; i <= 4; i++) + await db.Coded.InsertAsync(new Coded { Code = i, Id = i * 100 }); + + var coded = db.Coded.AsQueryable().ToList(); + Assert.Equal(1000, coded.Sum(c => c.Id)); + Assert.Equal(coded.Sum(c => c.Id), db.Coded.AsQueryable().Sum(c => c.Id)); + Assert.Equal(coded.Max(c => c.Code), db.Coded.AsQueryable().Max(c => c.Code)); + + // Stored as "id", not "_id": still read from raw BSON. + var id = await db.Coded.TryAggregateAsync(AggregateOperator.Sum, (Expression>)(c => c.Id), null, typeof(int)); + Assert.Equal((true, (object?)1000), id); + } + + [Fact] + public async Task PropertySetThroughTheConstructor_IsMaterialised() + { + using var db = new VerbatimDbContext(_dbPath); + for (int i = 1; i <= 4; i++) + await db.Scaled.InsertAsync(new Scaled(i)); + + var scaled = db.Scaled.AsQueryable().ToList(); + Assert.Equal(scaled.Sum(s => s.Amount), db.Scaled.AsQueryable().Sum(s => s.Amount)); + Assert.Equal(scaled.Max(s => s.Amount), db.Scaled.AsQueryable().Max(s => s.Amount)); + } + + [Fact] + public void GeneratedMappers_NameTheFieldOnlyForPropertiesStoredVerbatim() + { + var reading = MapperOf(); + Assert.Equal("_id", reading.GetVerbatimFieldName("Id")); + Assert.Equal("count", reading.GetVerbatimFieldName("Count")); + Assert.Equal("maybeprice", reading.GetVerbatimFieldName("MaybePrice")); + Assert.Equal("score", reading.GetVerbatimFieldName("Level")); // [Column("score")] + Assert.Equal("level", reading.GetVerbatimFieldName("Score")); + Assert.Null(reading.GetVerbatimFieldName("NoSuchProperty")); + + var shape = MapperOf(); + Assert.Equal("count", shape.GetVerbatimFieldName("Count")); + Assert.Null(shape.GetVerbatimFieldName("Hidden")); // [BsonIgnore] + Assert.Null(shape.GetVerbatimFieldName("Unmapped")); // [NotMapped] + Assert.Null(shape.GetVerbatimFieldName("Gone")); // computed + + var doubled = MapperOf(); + Assert.Null(doubled.GetVerbatimFieldName("G")); // getter with logic + Assert.Null(doubled.GetVerbatimFieldName("H")); // setter with logic + Assert.Equal("plain", doubled.GetVerbatimFieldName("Plain")); + + var coded = MapperOf(); + Assert.Equal("_id", coded.GetVerbatimFieldName("Code")); + Assert.Equal("id", coded.GetVerbatimFieldName("Id")); + + Assert.Null(MapperOf().GetVerbatimFieldName("Amount")); // set through the constructor + } + + private static DocumentMapperBase MapperOf() where TEntity : class + { + var type = typeof(VerbatimDbContext).Assembly.GetTypes() + .First(t => !t.IsAbstract && typeof(DocumentMapperBase).IsAssignableFrom(t)); + return (DocumentMapperBase)Activator.CreateInstance(type)!; + } + + [Fact] + public async Task OtherOperatorsBeforeTheAggregate_AreApplied() + { + using var db = new AggregateDbContext(_dbPath); + for (int i = 0; i < 20; i++) + await db.Readings.InsertAsync(Make(i)); + var entities = db.Readings.AsQueryable().ToList(); + var q = db.Readings.AsQueryable(); + + Assert.Equal(entities.Take(3).Sum(r => r.Count), q.Take(3).Sum(r => r.Count)); + Assert.Equal(entities.Skip(5).Max(r => r.Count), q.Skip(5).Max(r => r.Count)); + Assert.Equal(entities.OrderBy(r => r.Ratio).Take(4).Sum(r => r.Ratio), q.OrderBy(r => r.Ratio).Take(4).Sum(r => r.Ratio)); + Assert.Equal(entities.Where(r => r.Tag == "a").Where(r => r.Units > -40).Sum(r => r.Units), + q.Where(r => r.Tag == "a").Where(r => r.Units > -40).Sum(r => r.Units)); + Assert.Equal(entities.Where(r => r.Count > 0).Sum(r => r.Count), await q.Where(r => r.Count > 0).SumAsync(r => r.Count)); + Assert.Equal(entities.Average(r => r.Units), await q.AverageAsync(r => r.Units)); + } + + [Fact] + public async Task IndexAggregates_FollowUpdatesAndDeletes() + { + using var db = new AggregateDbContext(_dbPath); + var readings = Enumerable.Range(0, 10).Select(Make).ToList(); + foreach (var r in readings) await db.Readings.InsertAsync(r); + + readings[3].Count = 500; + await db.Readings.UpdateAsync(readings[3]); + await db.Readings.DeleteAsync(readings[4].Id); + readings.RemoveAt(4); + + Assert.Equal(readings.Sum(r => r.Count), db.Readings.AsQueryable().Sum(r => r.Count)); + Assert.Equal(500, db.Readings.AsQueryable().Max(r => r.Count)); + Assert.Equal(readings.Min(r => r.Count), db.Readings.AsQueryable().Min(r => r.Count)); + Assert.Equal(readings.Where(r => r.Count < 500).Max(r => r.Count), db.Readings.AsQueryable().Where(r => r.Count < 500).Max(r => r.Count)); + } + + // ── helpers ───────────────────────────────────────────────────────────── + + private static LambdaExpression Selector(string property) + { + var p = Expression.Parameter(typeof(Reading), "r"); + return Expression.Lambda(Expression.Property(p, property), p); + } + + private static MethodCallExpression AggregateCall(IQueryable source, string op, LambdaExpression selector, LambdaExpression? where) + { + var src = where == null + ? source.Expression + : Expression.Call(typeof(Queryable), nameof(Queryable.Where), [typeof(TEntity)], source.Expression, Expression.Quote(where)); + Type[] typeArgs = op is "Min" or "Max" ? [typeof(TEntity), selector.ReturnType] : [typeof(TEntity)]; + return Expression.Call(typeof(Queryable), op, typeArgs, src, Expression.Quote(selector)); + } + + private static readonly MethodInfo s_execute = typeof(IQueryProvider).GetMethods().Single(m => m.Name == "Execute" && m.IsGenericMethodDefinition); + + private static object? Execute(IQueryable source, string op, LambdaExpression selector, LambdaExpression? where) + { + var call = AggregateCall(source, op, selector, where); + return s_execute.MakeGenericMethod(call.Type).Invoke(source.Provider, [call]); + } + + private static (object? Value, Type? Error) Run(Func action) + { + try { return (action(), null); } + catch (Exception ex) { return (null, Unwrap(ex).GetType()); } + } + + private static async Task<((bool Handled, object? Value) Value, Type? Error)> RunAsync(Func> action) + { + try { return (await action(), null); } + catch (Exception ex) { return (default, Unwrap(ex).GetType()); } + } + + private static Exception Unwrap(Exception ex) + { + while (ex is TargetInvocationException or AggregateException && ex.InnerException != null) + ex = ex.InnerException!; + return ex; + } + + /// Same exception type, or values of the same type with identical bits. + private static bool Same((object? Value, Type? Error) expected, (object? Value, Type? Error) actual) + { + if (expected.Error != null || actual.Error != null) return expected.Error == actual.Error; + return (expected.Value, actual.Value) switch + { + (null, null) => true, + (double x, double y) => BitConverter.DoubleToInt64Bits(x) == BitConverter.DoubleToInt64Bits(y), + (float x, float y) => BitConverter.SingleToInt32Bits(x) == BitConverter.SingleToInt32Bits(y), + (decimal x, decimal y) => decimal.GetBits(x).SequenceEqual(decimal.GetBits(y)), + var (x, y) => x?.GetType() == y?.GetType() && Equals(x, y), + }; + } + + private static string Show((object? Value, Type? Error) result) + => result.Error != null ? result.Error.Name + : result.Value switch + { + null => "null", + double d => d.ToString("R") + (BitConverter.DoubleToInt64Bits(d) < 0 ? " (neg)" : ""), + float f => f.ToString("R"), + var v => $"{v} ({v.GetType().Name})", + }; +}