From 95f3b2be22e01d54f42906acd6f37a4dae19e64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 19:29:47 +0200 Subject: [PATCH 1/4] fix(query): return the real result of terminals after Select and after an indexed top-N window (#181) - TryPushDownSelect reports success with a bool and an out value instead of using default(TResult) as "not applicable". For a value-type result (Select(...).Any(), .Max(), .Count(pred), ...) default is false or 0, which the provider returned as the query result. The push-down now only runs when the projection is the whole query. - A scalar terminal after Select on the direct pipeline (e.g. Take(n).Select(...).Count()) runs through EnumerableRewriter. ProjectEnumerable used to return default for a non-sequence result; it now throws if it is ever reached that way. - The OrderBy(indexed).Skip/Take fast path takes its shortcut only for the plain window: Take, or Skip then Take, directly on the ordered collection, followed by nothing but Select or a plain Count/LongCount. Anything else after the window (Sum/Max with a selector, Any/Count with a predicate, First, a scalar after Select, another Skip/Take, Take-then-Skip) is replayed over the first skip+take entries in index order. Before, those operators were ignored or misread: OrderBy(Age).Take(10).Max(p => p.Age) returned 10. A ThenBy right after the window turns the shortcut off. - EnumerableRewriter maps non-generic Queryable overloads (parameterless Sum and Average over int/long/double/decimal/...) instead of failing to find an Enumerable counterpart. SelectTerminalTests compares 55 query shapes against LINQ-to-Objects; 39 of them fail on main. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Query/BTreeQueryProvider.cs | 161 +++++++++++++++++---- src/BLite.Core/Query/EnumerableRewriter.cs | 15 +- tests/BLite.Tests/SelectTerminalTests.cs | 114 +++++++++++++++ 3 files changed, 256 insertions(+), 34 deletions(-) create mode 100644 tests/BLite.Tests/SelectTerminalTests.cs diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index aa8a65c0..8b3514ce 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -256,14 +256,17 @@ private async Task ExecuteAsyncCore(Expression expression, Can // ── Push-down SELECT (+ optional WHERE) ─────────────────────────────── // Single-pass BSON projection: no T instantiation, no EnumerableRewriter. - // Only applicable when there is no post-scan ordering or paging. + // Only applicable when the projection is the whole query: no post-scan ordering, + // paging or other operators. A terminal after the Select (Any, Max, Count(pred), ...) + // runs the usual way. if (model.SelectClause is not null && model.OrderByClause is null && !model.Take.HasValue - && !model.Skip.HasValue) + && !model.Skip.HasValue + && !model.HasComplexOperators + && TryPushDownSelect(model.SelectClause, model.WhereClause, out var pushed)) { - var pushed = TryPushDownSelect(model.SelectClause, model.WhereClause); - if (pushed is not null) return pushed; + return pushed; } // 2. Data fetching. @@ -278,20 +281,34 @@ private async Task ExecuteAsyncCore(Expression expression, Can // O(n log n) in-memory sort over all documents. Skip is performed at the // index-entry level (no document reads for skipped positions), so only the // requested page window is ever deserialised. - if (model.WhereClause == null && model.OrderByClause != null && model.Take.HasValue) + if (model.WhereClause == null && model.OrderByClause != null && model.Take.HasValue + && TryGetIndexWindow(expression, model.OrderByClause, out int windowEnd, out bool plainWindow)) { var orderOpt = IndexOptimizer.TryOptimizeOrderBy(model, _collection.GetIndexes()); if (orderOpt is not null) { bool ascending = !model.OrderDescending; - int skip = model.Skip ?? 0; - int take = model.Take.Value; - var topN = new List(take); - await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, skip, take, cancellationToken)) - topN.Add(item); - if (model.SelectClause != null) - return ProjectEnumerable(topN, model.SelectClause); - return TerminalReturn(topN); + if (plainWindow && !model.HasComplexOperators + && (model.SelectClause == null || IsSequenceResult())) + { + int skip = model.Skip ?? 0; + int take = model.Take.Value; + var topN = new List(take); + await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, skip, take, cancellationToken)) + topN.Add(item); + if (model.SelectClause != null) + return ProjectEnumerable(topN, model.SelectClause); + return TerminalReturn(topN); + } + + // Operators after the window (a terminal with a predicate or selector, a scalar + // after Select, ...): read the first windowEnd entries in index order and replay the + // whole query over them. OrderBy is a stable sort, so the replay keeps the index + // order and its Skip/Take select the same window. + var prefix = new List(windowEnd); + await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, 0, windowEnd, cancellationToken)) + prefix.Add(item); + return ExecuteViaEnumerableRewriter(expression, prefix); } } @@ -321,6 +338,11 @@ private async Task ExecuteAsyncCore(Expression expression, Can if (model.OrderByClause != null && model.OrderByClause.Parameters[0].Type != typeof(T)) return ExecuteViaEnumerableRewriter(expression, sourceData); + // The direct pipeline can only project into a sequence. A scalar terminal after a + // Select (e.g. Take(n).Select(...).Count()) needs the full expression. + if (model.SelectClause != null && !IsSequenceResult()) + return ExecuteViaEnumerableRewriter(expression, sourceData); + // 3. Direct pipeline — no expression-tree rewrite, no EnumerableRewriter, no Compile(). return ExecutePipeline(model, sourceData, whereAlreadyApplied); } @@ -378,11 +400,11 @@ private TResult ExecutePipeline(QueryModel model, IEnumerable source [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] private TResult ProjectEnumerable(IEnumerable source, LambdaExpression selectLambda) { - var resultType = typeof(TResult); - if (!resultType.IsGenericType || resultType.GetGenericTypeDefinition() != typeof(IEnumerable<>)) - return default!; + if (!IsSequenceResult()) + throw new NotSupportedException( + $"BLite query pipeline: cannot project into result type '{typeof(TResult)}'."); - var projType = resultType.GetGenericArguments()[0]; + var projType = typeof(TResult).GetGenericArguments()[0]; var method = s_projectMethodCache.GetOrAdd( projType, t => typeof(BTreeQueryProvider) @@ -403,6 +425,77 @@ private static IEnumerable ProjectTyped(IEnumerable source, Lam private static readonly System.Collections.Concurrent.ConcurrentDictionary s_projectMethodCache = new(); + /// Whether is IEnumerable<TElement>, i.e. the query is enumerated rather than ended by a terminal. + private static bool IsSequenceResult() + => typeof(TResult).IsGenericType && typeof(TResult).GetGenericTypeDefinition() == typeof(IEnumerable<>); + + /// + /// Checks that starts with OrderBy/OrderByDescending + /// on the collection (the call whose key selector is ), followed by + /// Skip/Take calls that bound it, so that the result only depends on the first + /// documents in that order. + /// + /// + /// true when the window is exactly Take(n) or Skip(s).Take(n), as + /// and describe it, and only + /// Select or a plain Count()/LongCount() follows it. + /// + private static bool TryGetIndexWindow(Expression expression, LambdaExpression orderBy, + out int windowEnd, out bool plainWindow) + { + windowEnd = 0; + plainWindow = false; + + // Collect the Queryable calls from the root outwards. + var calls = new List(); + var current = expression; + while (current is MethodCallExpression call && call.Method.DeclaringType == typeof(Queryable)) + { + calls.Add(call); + current = call.Arguments[0]; + } + if (current is not ConstantExpression { Value: IQueryable } || calls.Count < 2) return false; + calls.Reverse(); + + var first = calls[0]; + if (first.Method.Name is not ("OrderBy" or "OrderByDescending") + || first.Arguments[1] is not UnaryExpression { Operand: var key } + || !ReferenceEquals(key, orderBy)) + return false; + + long start = 0, length = long.MaxValue; + int i = 1; + for (; i < calls.Count && calls[i].Method.Name is "Skip" or "Take"; i++) + { + if (calls[i].Arguments[1] is not ConstantExpression { Value: int n }) return false; + n = Math.Max(n, 0); + if (calls[i].Method.Name == "Skip") + { + start += n; + if (length != long.MaxValue) length = Math.Max(0, length - n); + } + else + { + length = Math.Min(length, n); + } + } + + // No Take bounds the window, or a ThenBy refines the order of the documents at its edge. + if (length == long.MaxValue) return false; + if (i < calls.Count && calls[i].Method.Name is "ThenBy" or "ThenByDescending") return false; + + long end = start + length; + if (end > int.MaxValue) return false; + windowEnd = (int)end; + + var window = calls.GetRange(1, i - 1).ConvertAll(c => c.Method.Name); + plainWindow = (window.Count == 1 && window[0] == "Take" + || window.Count == 2 && window[0] == "Skip" && window[1] == "Take") + && calls.Skip(i).All(c => c.Method.Name == "Select" + || c.Method.Name is "Count" or "LongCount" && c.Arguments.Count == 1); + return true; + } + /// /// Type-switch dispatch for terminal results — first-match, no reflection per call. /// Covers all standard LINQ terminal operators: First/Single/Count/Any/All/ToList/ToArray. @@ -592,20 +685,25 @@ private static bool AggregateValues( /// Attempts to execute the query entirely via a single-pass BSON projection scan, /// bypassing full deserialisation. /// - /// The projected IEnumerable<TProj>, or null if push-down is - /// not applicable (caller should fall through to the standard path). + /// + /// true and the projected IEnumerable<TProj> in ; + /// false if push-down is not applicable (caller should fall through to the standard path). + /// default(TResult) cannot signal this: for a value-type result such as bool it + /// is a valid answer (#181). + /// [RequiresDynamicCode("BLite LINQ queries use Expression.Compile() and MakeGenericMethod which require dynamic code generation.")] [RequiresUnreferencedCode("BLite LINQ queries use reflection to resolve methods and types at runtime. Ensure all entity types and their members are preserved.")] - private TResult? TryPushDownSelect(LambdaExpression selectLambda, - LambdaExpression? whereLambda = null) + private bool TryPushDownSelect(LambdaExpression selectLambda, + LambdaExpression? whereLambda, + out TResult result) { + result = default!; + // We need TResult = IEnumerable for some projection type TProj. - var resultType = typeof(TResult); - if (!resultType.IsGenericType) return default; - if (resultType.GetGenericTypeDefinition() != typeof(IEnumerable<>)) return default; + if (!IsSequenceResult()) return false; - var projType = resultType.GetGenericArguments()[0]; - if (projType == typeof(T)) return default; // not a real projection + var projType = typeof(TResult).GetGenericArguments()[0]; + if (projType == typeof(T)) return false; // not a real projection // Compile the push-down projector via reflection (TProj is only known at runtime). // BsonProjectionCompiler.TryCompile(selectLambda, whereLambda?) handles @@ -618,9 +716,9 @@ private static bool AggregateValues( t => s_tryCompileMethod.MakeGenericMethod(typeof(T), t)); projector = compileMethod.Invoke(null, [selectLambda, whereLambda, _collection.GetKeyMap()]); } - catch { return default; } + catch { return false; } - if (projector is null) return default; // compilation soft-failed + if (projector is null) return false; // compilation soft-failed // Invoke _collection.Scan(projector) — MethodInfo cached per TProj. try @@ -628,10 +726,11 @@ private static bool AggregateValues( var scanMethod = s_compiledScanMethods.GetOrAdd( projType, t => s_scanProjectorMethod.MakeGenericMethod(t)); - var result = scanMethod.Invoke(_collection, [projector]); - return (TResult?)result; + if (scanMethod.Invoke(_collection, [projector]) is not TResult scanned) return false; + result = scanned; + return true; } - catch { return default; } + catch { return false; } } // ─── Expression-tree helpers ────────────────────────────────────────────── diff --git a/src/BLite.Core/Query/EnumerableRewriter.cs b/src/BLite.Core/Query/EnumerableRewriter.cs index 01c89bfb..51b9ec81 100644 --- a/src/BLite.Core/Query/EnumerableRewriter.cs +++ b/src/BLite.Core/Query/EnumerableRewriter.cs @@ -78,9 +78,18 @@ protected override Expression VisitMethodCall(MethodCallExpression node) { if (candidate.GetGenericArguments().Length != typeArgs.Length) continue; if (candidate.GetParameters().Length != args.Length) continue; - MethodInfo closed = null!; - try { closed = candidate.MakeGenericMethod(typeArgs); } - catch { continue; } + MethodInfo closed; + if (typeArgs.Length == 0) + { + // Non-generic overloads such as Sum(IEnumerable) / Average(IEnumerable): + // MakeGenericMethod would throw on them. + closed = candidate; + } + else + { + try { closed = candidate.MakeGenericMethod(typeArgs); } + catch { continue; } + } try { return Expression.Call(closed, args); } catch { continue; } } diff --git a/tests/BLite.Tests/SelectTerminalTests.cs b/tests/BLite.Tests/SelectTerminalTests.cs new file mode 100644 index 00000000..cdf5c45c --- /dev/null +++ b/tests/BLite.Tests/SelectTerminalTests.cs @@ -0,0 +1,114 @@ +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// Terminals after Select, and after an indexed OrderBy(...).Take(...), +/// must return what LINQ-to-Objects returns (#181). +/// +public class SelectTerminalTests : IDisposable +{ + private readonly string _dbPath; + private readonly TestDbContext _db; + private readonly List _people; + + public SelectTerminalTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"blite_select_terminal_{Guid.NewGuid()}.db"); + _db = new TestDbContext(_dbPath); + _people = Enumerable.Range(0, 300) + .Select(i => new PersonV2 { Name = $"p{i}", Age = i % 50 }) + .ToList(); + _db.PeopleV2.InsertBulkAsync(_people).GetAwaiter().GetResult(); + } + + public void Dispose() + { + _db.Dispose(); + if (File.Exists(_dbPath)) File.Delete(_dbPath); + } + + // Every Age value occurs exactly 6 times. Windows over OrderBy(p => p.Age) either select Age only + // or end on a multiple of 6, so ties at the window edge cannot change the expected result. + public static IEnumerable Queries() => Cases.Keys.Select(k => new object[] { k }); + + private static readonly Dictionary, object?>> Cases = new() + { + // Select, then a scalar terminal (the issue's repro). + ["Select.Any"] = q => q.Select(p => p.Age).Any(), + ["Select.Any(pred)"] = q => q.Select(p => p.Age).Any(a => a == 7), + ["Select.Any(pred) none"] = q => q.Select(p => p.Age).Any(a => a == 700), + ["Select.All(pred)"] = q => q.Select(p => p.Age).All(a => a < 50), + ["Select.Count(pred)"] = q => q.Select(p => p.Age).Count(a => a == 7), + ["Select.LongCount(pred)"] = q => q.Select(p => p.Age).LongCount(a => a == 7), + ["Select.Max"] = q => q.Select(p => p.Age).Max(), + ["Select.Min"] = q => q.Select(p => p.Age).Min(), + ["Select.Sum"] = q => q.Select(p => p.Age).Sum(), + ["Select.Average"] = q => q.Select(p => p.Age).Average(), + ["Select.Contains"] = q => q.Select(p => p.Age).Contains(49), + ["Select.ElementAt via OrderBy"] = q => q.Select(p => p.Age).OrderBy(a => a).ElementAt(10), + ["Select.First(pred)"] = q => q.Select(p => p.Age).First(a => a == 7), + ["Select.FirstOrDefault none"] = q => q.Select(p => p.Age).FirstOrDefault(a => a == 700), + ["Select.Single(pred)"] = q => q.Select(p => p.Name).Single(n => n == "p7"), + ["Select.Distinct.Count"] = q => q.Select(p => p.Age).Distinct().Count(), + ["Where.Select.Any"] = q => q.Where(p => p.Age == 7).Select(p => p.Name).Any(), + ["Where.Select.Any none"] = q => q.Where(p => p.Age == 700).Select(p => p.Name).Any(), + ["Where.Select.Max"] = q => q.Where(p => p.Age > 10).Select(p => p.Age).Max(), + ["Where.Select.Count(pred)"] = q => q.Where(p => p.Age > 10).Select(p => p.Age).Count(a => a < 20), + ["Select.Select.Sum"] = q => q.Select(p => p.Age).Select(a => a * 2).Sum(), + ["Select(anon).Any"] = q => q.Select(p => new { p.Age }).Any(x => x.Age == 7), + + // Select after paging, then a scalar terminal. + ["Take.Select.Count"] = q => q.Take(10).Select(p => p.Age).Count(), + ["Skip.Select.LongCount"] = q => q.Skip(290).Select(p => p.Age).LongCount(), + ["OrderBy.Select.Take.Count"] = q => q.OrderBy(p => p.Name).Select(p => p.Age).Take(10).Count(), + ["OrderBy.Take.Select.Any"] = q => q.OrderBy(p => p.Name).Take(10).Select(p => p.Age).Any(), + + // OrderBy on the indexed Age, then Take: the index top-N fast path. + ["OrderByIdx.Take.Select.Sum"] = q => q.OrderBy(p => p.Age).Take(10).Select(p => p.Age).Sum(), + ["OrderByIdx.Take.Select.Max"] = q => q.OrderBy(p => p.Age).Take(10).Select(p => p.Age).Max(), + ["OrderByIdx.Take.Select.Count"] = q => q.OrderBy(p => p.Age).Take(10).Select(p => p.Age).Count(), + ["OrderByIdx.Skip.Take.Select.Sum"] = q => q.OrderBy(p => p.Age).Skip(20).Take(10).Select(p => p.Age).Sum(), + ["OrderByIdx.Take.Sum(sel)"] = q => q.OrderBy(p => p.Age).Take(10).Sum(p => p.Age), + ["OrderByIdx.Take.Max(sel)"] = q => q.OrderBy(p => p.Age).Take(10).Max(p => p.Age), + ["OrderByIdx.Take.Any(pred)"] = q => q.OrderBy(p => p.Age).Take(10).Any(p => p.Age > 5), + ["OrderByIdx.Take.Count(pred)"] = q => q.OrderBy(p => p.Age).Take(10).Count(p => p.Age == 0), + ["OrderByIdx.Take.Count"] = q => q.OrderBy(p => p.Age).Take(10).Count(), + ["OrderByIdx.Take.Last.Age"] = q => q.OrderBy(p => p.Age).Take(10).Select(p => p.Age).Last(), + ["OrderByDescIdx.Take.Select.Min"] = q => q.OrderByDescending(p => p.Age).Take(10).Select(p => p.Age).Min(), + ["OrderByIdx.Skip.Take.First.Age"] = q => q.OrderBy(p => p.Age).Skip(20).Take(10).Select(p => p.Age).First(), + + // Window shapes the index top-N shortcut must not take, or must replay. + ["OrderByIdx.Take.Skip.List"] = q => string.Join(",", q.OrderBy(p => p.Age).Take(10).Skip(5).Select(p => p.Age).ToList()), + ["OrderByIdx.Skip.Take.Skip.Take"] = q => string.Join(",", q.OrderBy(p => p.Age).Skip(10).Take(20).Skip(5).Take(3).Select(p => p.Age).ToList()), + ["OrderByIdx.Skip.Take.Select.Skip"] = q => string.Join(",", q.OrderBy(p => p.Age).Skip(5).Take(10).Select(p => p.Age).Skip(2).ToList()), + ["OrderByIdx.Take.Select.Take"] = q => string.Join(",", q.OrderBy(p => p.Age).Take(10).Select(p => p.Age).Take(8).ToList()), + ["OrderByIdx.ThenBy.Take.List"] = q => string.Join(",", q.OrderBy(p => p.Age).ThenBy(p => p.Name).Take(7).Select(p => p.Name).ToList()), + ["OrderByIdx.Take.ThenBy.List"] = q => string.Join(",", q.OrderBy(p => p.Age).Take(12).OrderBy(p => p.Age).ThenByDescending(p => p.Name).Select(p => p.Name).ToList()), + ["Select(anon).OrderBy.Take.List"] = q => string.Join(",", q.Select(p => new { p.Age, p.Name }).OrderBy(x => x.Age).Take(3).Select(x => x.Age).ToList()), + ["OrderByIdx.Skip.Take.First"] = q => q.OrderBy(p => p.Age).Skip(3).Take(10).First().Age, + ["OrderByIdx.Take(0).Any"] = q => q.OrderBy(p => p.Age).Take(0).Select(p => p.Age).Any(), + + // Parameterless Sum/Average after Select (non-generic Queryable overloads). + ["Select(long).Sum"] = q => q.Select(p => (long)p.Age).Sum(), + ["Select(double).Average"] = q => q.Select(p => (double)p.Age).Average(), + ["Select(decimal?).Sum"] = q => q.Select(p => (decimal?)p.Age).Sum(), + + // Shapes that already worked, kept as guards. + ["Select.Count"] = q => q.Select(p => p.Age).Count(), + ["Select.ToList.Count"] = q => q.Select(p => p.Age).ToList().Count, + ["Where.Select.ToList"] = q => string.Join(",", q.Where(p => p.Age == 7).Select(p => p.Name).ToList().OrderBy(n => n)), + ["OrderByIdx.Take.Select.List"]= q => string.Join(",", q.OrderBy(p => p.Age).Take(12).Select(p => p.Age).ToList()), + ["OrderByIdx.Skip.Take.List"] = q => string.Join(",", q.OrderBy(p => p.Age).Skip(20).Take(10).Select(p => p.Age).ToList()), + }; + + [Theory] + [MemberData(nameof(Queries))] + public void Terminal_MatchesLinqToObjects(string name) + { + var query = Cases[name]; + var expected = query(_people.AsQueryable()); + var actual = query(_db.PeopleV2.AsQueryable()); + Assert.Equal(expected, actual); + } +} From aebaaf20506c0e9c3cb011c6f63d01b557c66ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:42:13 +0200 Subject: [PATCH 2/4] fix(query): do not replay the index window when the root is used again after it (#181) - TryGetIndexWindow returns false when an operator after the window has an argument holding a queryable (Concat(q), Join(q, ...), Union(q.Select(...))). EnumerableRewriter replaces every use of the root with the window prefix, so that second use read the prefix instead of the collection: q.OrderBy(p => p.Age).Take(5).Concat(q).Count() returned 10 instead of 305. Such queries take the general path. - The comment on the replay states the single-source requirement. - SelectTerminalTests: four shapes that use the root again after the window. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Query/BTreeQueryProvider.cs | 30 ++++++++++++++++++++++ tests/BLite.Tests/SelectTerminalTests.cs | 7 +++++ 2 files changed, 37 insertions(+) diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index 8b3514ce..33d5f587 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -305,6 +305,9 @@ private async Task ExecuteAsyncCore(Expression expression, Can // after Select, ...): read the first windowEnd entries in index order and replay the // whole query over them. OrderBy is a stable sort, so the replay keeps the index // order and its Skip/Take select the same window. + // This needs the root queryable to be the query's only source: EnumerableRewriter + // replaces every use of it with the prefix, so TryGetIndexWindow rejects a query + // that reads a queryable again after the window (Concat(q), Join(q, ...)). var prefix = new List(windowEnd); await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, 0, windowEnd, cancellationToken)) prefix.Add(item); @@ -484,6 +487,12 @@ private static bool TryGetIndexWindow(Expression expression, LambdaExpression or if (length == long.MaxValue) return false; if (i < calls.Count && calls[i].Method.Name is "ThenBy" or "ThenByDescending") return false; + // A second query source after the window (Concat(q), Join(q, ...), Union(q.Select(...))): + // the replay would read the window prefix for it, since every use of the root is replaced. + for (int j = i; j < calls.Count; j++) + for (int a = 1; a < calls[j].Arguments.Count; a++) + if (ContainsQueryableConstant(calls[j].Arguments[a])) return false; + long end = start + length; if (end > int.MaxValue) return false; windowEnd = (int)end; @@ -496,6 +505,27 @@ private static bool TryGetIndexWindow(Expression expression, LambdaExpression or return true; } + /// Whether contains a constant holding an , i.e. a query source. + private static bool ContainsQueryableConstant(Expression expression) + { + var finder = new QueryableConstantFinder(); + finder.Visit(expression); + return finder.Found; + } + + private sealed class QueryableConstantFinder : ExpressionVisitor + { + public bool Found { get; private set; } + + public override Expression? Visit(Expression? node) => Found ? node : base.Visit(node); + + protected override Expression VisitConstant(ConstantExpression node) + { + if (node.Value is IQueryable) Found = true; + return node; + } + } + /// /// Type-switch dispatch for terminal results — first-match, no reflection per call. /// Covers all standard LINQ terminal operators: First/Single/Count/Any/All/ToList/ToArray. diff --git a/tests/BLite.Tests/SelectTerminalTests.cs b/tests/BLite.Tests/SelectTerminalTests.cs index cdf5c45c..dd6bfc2d 100644 --- a/tests/BLite.Tests/SelectTerminalTests.cs +++ b/tests/BLite.Tests/SelectTerminalTests.cs @@ -89,6 +89,13 @@ public void Dispose() ["OrderByIdx.Skip.Take.First"] = q => q.OrderBy(p => p.Age).Skip(3).Take(10).First().Age, ["OrderByIdx.Take(0).Any"] = q => q.OrderBy(p => p.Age).Take(0).Select(p => p.Age).Any(), + // The root queryable used a second time after the window: that use reads the whole + // collection, not the window prefix the replay is given. + ["OrderByIdx.Take.Concat(root).Count"] = q => q.OrderBy(p => p.Age).Take(5).Concat(q).Count(), + ["OrderByIdx.Take.Select.Concat(root).Count"] = q => q.OrderBy(p => p.Age).Take(5).Select(p => p.Age).Concat(q.Select(p => p.Age)).Count(), + ["OrderByIdx.Take.Select.Union(root).Count"] = q => q.OrderBy(p => p.Age).Take(5).Select(p => p.Age).Union(q.Select(p => p.Age)).Count(), + ["OrderByIdx.Take.Join(root).Count"] = q => q.OrderBy(p => p.Age).Take(5).Join(q, a => a.Age, b => b.Age, (a, b) => b.Name).Count(), + // Parameterless Sum/Average after Select (non-generic Queryable overloads). ["Select(long).Sum"] = q => q.Select(p => (long)p.Age).Sum(), ["Select(double).Average"] = q => q.Select(p => (double)p.Age).Average(), From 68d5e89ab1307576c5c8e1a7eb6f6d6c17f153ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:46:52 +0200 Subject: [PATCH 3/4] fix(query): keep lambdas inline when EnumerableRewriter replays a query (#181) EnumerableRewriter compiled each quoted lambda on its own and passed the delegate as a constant. A lambda that refers to the parameter of an outer lambda, such as the predicate of a subquery, q.Count(x => q.Any(y => y.Age == x.Age + 1)), cannot be compiled on its own and threw InvalidOperationException ("variable 'x' ... is not defined"). The rewriter now passes the unquoted lambda, which is compiled with the enclosing expression. This already failed on the general path; the window replay now routes OrderBy(idx).Take(n).Count(pred) through the rewriter too, where it used to ignore the predicate. SelectTerminalTests covers both, plus a closure subquery after the window, which must read the collection, not the prefix. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Query/EnumerableRewriter.cs | 9 ++++----- tests/BLite.Tests/SelectTerminalTests.cs | 6 ++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/BLite.Core/Query/EnumerableRewriter.cs b/src/BLite.Core/Query/EnumerableRewriter.cs index 51b9ec81..d7f70086 100644 --- a/src/BLite.Core/Query/EnumerableRewriter.cs +++ b/src/BLite.Core/Query/EnumerableRewriter.cs @@ -60,12 +60,11 @@ protected override Expression VisitMethodCall(MethodCallExpression node) for (int i = 0; i < node.Arguments.Count; i++) { var arg = Visit(node.Arguments[i]); - // Strip Quote wrapping around lambda arguments. + // Strip Quote wrapping around lambda arguments. The lambda stays in the tree and is + // compiled with the enclosing expression: compiling it on its own would fail when + // it refers to the parameter of an outer lambda (q.Count(x => q.Any(y => y.Age == x.Age))). if (arg is UnaryExpression { NodeType: ExpressionType.Quote } quote) - { - var lambda = (LambdaExpression)quote.Operand; - arg = Expression.Constant(lambda.Compile()); - } + arg = quote.Operand; args[i] = arg; } diff --git a/tests/BLite.Tests/SelectTerminalTests.cs b/tests/BLite.Tests/SelectTerminalTests.cs index dd6bfc2d..36060522 100644 --- a/tests/BLite.Tests/SelectTerminalTests.cs +++ b/tests/BLite.Tests/SelectTerminalTests.cs @@ -96,6 +96,12 @@ public void Dispose() ["OrderByIdx.Take.Select.Union(root).Count"] = q => q.OrderBy(p => p.Age).Take(5).Select(p => p.Age).Union(q.Select(p => p.Age)).Count(), ["OrderByIdx.Take.Join(root).Count"] = q => q.OrderBy(p => p.Age).Take(5).Join(q, a => a.Age, b => b.Age, (a, b) => b.Name).Count(), + // A lambda that captures the queryable reads it through a closure field, not a constant, + // so the replay does not replace it with the prefix: it runs as its own query. + ["OrderByIdx.Take.Count(closure root)"] = q => q.OrderBy(p => p.Age).Take(12).Count(x => q.Any(y => y.Age == 49)), + ["OrderByIdx.Take.Count(correlated closure)"] = q => q.OrderBy(p => p.Age).Take(12).Count(x => q.Any(y => y.Age == x.Age + 1)), + ["Count(correlated closure)"] = q => q.Count(x => q.Any(y => y.Age == x.Age + 1)), + // Parameterless Sum/Average after Select (non-generic Queryable overloads). ["Select(long).Sum"] = q => q.Select(p => (long)p.Age).Sum(), ["Select(double).Average"] = q => q.Select(p => (double)p.Age).Average(), From 515c88fd21eb18c3213e521201ad7f1684483b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Tue, 13 Oct 2026 15:48:39 +0200 Subject: [PATCH 4/4] fix(query): replay over the whole collection when a query reads a second source (#181) ExecuteAsyncCore fetched one source for the whole query, filtered by the model's WHERE, and EnumerableRewriter replaced every use of the root with it. BTreeExpressionVisitor also collects Where/Take/... from the other arguments of operators it does not model, including subqueries inside their lambdas. So a second use of the root got the first use's filter, or the other way round: q.Where(p => p.Age < 10).Concat(q).Count() 120, not 360 q.Concat(q.Where(p => p.Age < 10)).Count() 120, not 360 q.Where(p => p.Age == 1).Join(q, ...).Count() 0, not 36 q.Any(x => x.Age == 0 && q.Where(y => y.Age == 49).Any()) false, not true These are on main too. When an operator of the main chain has an argument other than its source that holds a queryable or calls a Queryable operator, the provider now reads the whole collection and replays the query over it. This also covers the index window replay, so the check added to TryGetIndexWindow in the previous commit moves here. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Query/BTreeQueryProvider.cs | 58 ++++++++++++++++------ tests/BLite.Tests/SelectTerminalTests.cs | 10 ++++ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index 33d5f587..0b822799 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -269,6 +269,20 @@ private async Task ExecuteAsyncCore(Expression expression, Can return pushed; } + // ── Several query sources ───────────────────────────────────────────── + // A query that reads a queryable outside its main chain (q.Where(...).Concat(q), a + // self-join, a subquery inside a lambda) cannot be served by one filtered or paged + // fetch: the model mixes the clauses of the different uses, and EnumerableRewriter + // replaces every use of the root with the same source. Replay the whole query over + // the whole collection. + if (HasSecondaryQuery(expression)) + { + var all = new List(); + await foreach (var item in _collection.FetchAsync(null, int.MaxValue, cancellationToken)) + all.Add(item); + return ExecuteViaEnumerableRewriter(expression, all); + } + // 2. Data fetching. // When Take (+ optional Skip) is present without complex operators or ordering that // requires a full scan first, cap how many rows we need to materialise. @@ -306,8 +320,8 @@ private async Task ExecuteAsyncCore(Expression expression, Can // whole query over them. OrderBy is a stable sort, so the replay keeps the index // order and its Skip/Take select the same window. // This needs the root queryable to be the query's only source: EnumerableRewriter - // replaces every use of it with the prefix, so TryGetIndexWindow rejects a query - // that reads a queryable again after the window (Concat(q), Join(q, ...)). + // replaces every use of it with the prefix. A query that reads a queryable again + // (Concat(q), Join(q, ...)) took the HasSecondaryQuery path above. var prefix = new List(windowEnd); await foreach (var item in _collection.QueryIndexAsync(orderOpt.IndexName, null, null, ascending, 0, windowEnd, cancellationToken)) prefix.Add(item); @@ -487,12 +501,6 @@ private static bool TryGetIndexWindow(Expression expression, LambdaExpression or if (length == long.MaxValue) return false; if (i < calls.Count && calls[i].Method.Name is "ThenBy" or "ThenByDescending") return false; - // A second query source after the window (Concat(q), Join(q, ...), Union(q.Select(...))): - // the replay would read the window prefix for it, since every use of the root is replaced. - for (int j = i; j < calls.Count; j++) - for (int a = 1; a < calls[j].Arguments.Count; a++) - if (ContainsQueryableConstant(calls[j].Arguments[a])) return false; - long end = start + length; if (end > int.MaxValue) return false; windowEnd = (int)end; @@ -505,15 +513,27 @@ private static bool TryGetIndexWindow(Expression expression, LambdaExpression or return true; } - /// Whether contains a constant holding an , i.e. a query source. - private static bool ContainsQueryableConstant(Expression expression) + /// + /// Whether an operator of the main chain (the Queryable calls from the result down + /// to the root) has an argument other than its source that holds a queryable or calls a + /// Queryable operator: a second use of the root (Concat(q), Join(q, ...), + /// Union(q.Select(...))), another source, or a subquery inside a lambda. + /// + private static bool HasSecondaryQuery(Expression expression) { - var finder = new QueryableConstantFinder(); - finder.Visit(expression); - return finder.Found; + var finder = new SecondaryQueryFinder(); + for (var current = expression; + current is MethodCallExpression call && call.Method.DeclaringType == typeof(Queryable); + current = call.Arguments[0]) + { + for (int i = 1; i < call.Arguments.Count && !finder.Found; i++) + finder.Visit(call.Arguments[i]); + if (finder.Found) return true; + } + return false; } - private sealed class QueryableConstantFinder : ExpressionVisitor + private sealed class SecondaryQueryFinder : ExpressionVisitor { public bool Found { get; private set; } @@ -524,6 +544,16 @@ protected override Expression VisitConstant(ConstantExpression node) if (node.Value is IQueryable) Found = true; return node; } + + protected override Expression VisitMethodCall(MethodCallExpression node) + { + if (node.Method.DeclaringType == typeof(Queryable)) + { + Found = true; + return node; + } + return base.VisitMethodCall(node); + } } /// diff --git a/tests/BLite.Tests/SelectTerminalTests.cs b/tests/BLite.Tests/SelectTerminalTests.cs index 36060522..40f8fce4 100644 --- a/tests/BLite.Tests/SelectTerminalTests.cs +++ b/tests/BLite.Tests/SelectTerminalTests.cs @@ -102,6 +102,16 @@ public void Dispose() ["OrderByIdx.Take.Count(correlated closure)"] = q => q.OrderBy(p => p.Age).Take(12).Count(x => q.Any(y => y.Age == x.Age + 1)), ["Count(correlated closure)"] = q => q.Count(x => q.Any(y => y.Age == x.Age + 1)), + // Outside the window path: a Where on one use of the root, or inside a lambda, must not + // filter the other uses. + ["Where.Concat(root).Count"] = q => q.Where(p => p.Age < 10).Concat(q).Count(), + ["Concat(root.Where).Count"] = q => q.Concat(q.Where(p => p.Age < 10)).Count(), + ["Where.Join(root).Count"] = q => q.Where(p => p.Age == 1).Join(q, a => a.Age, b => b.Age - 1, (a, b) => b.Name).Count(), + ["Concat(root).Sum(sel)"] = q => q.Concat(q).Sum(p => p.Age), + ["Take.Concat(root.Take).Count"] = q => q.Take(5).Concat(q.Take(3)).Count(), + ["OrderByIdx.Take.Concat(root.Where).Sum"] = q => q.OrderBy(p => p.Age).Take(5).Concat(q.Where(p => p.Age >= 10)).Sum(p => p.Age), + ["Any(closure Where)"] = q => q.Any(x => x.Age == 0 && q.Where(y => y.Age == 49).Any()), + // Parameterless Sum/Average after Select (non-generic Queryable overloads). ["Select(long).Sum"] = q => q.Select(p => (long)p.Age).Sum(), ["Select(double).Average"] = q => q.Select(p => (double)p.Age).Average(),