From 224b332157e646cc08fbf77971b7320b66b966c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 14:11:13 +0200 Subject: [PATCH 1/3] fix(query): evaluate Near/Within for real and refine indexed Near to the circle (#165) The Near and Within LINQ markers returned true when evaluated in memory, so every document matched inside an ||, under a negation, or on any other path that falls back to the compiled predicate. Indexed Near returned the whole bounding box of the circle, including points up to sqrt(2) x radius away. - Near computes the great-circle distance and Within tests the box, matching BLQL's $geoNear and $geoWithin. - RTreeIndex.SearchNear checks each bounding-box hit against the exact distance, using the indexed point, so indexed Near stays an exact plan. Typed and dynamic NearAsync use it. - VectorSearch throws NotSupportedException when evaluated per document: top-k membership depends on the whole collection. The planner lets a vector term drive any && plan and post-filters with the VectorSearch call replaced by true, so `VectorSearch(...) && x` keeps working, while `VectorSearch(...) || x` now fails loudly instead of matching everything. - Two spatial terms on the same index are no longer merged into a B-Tree range. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 6 +- src/BLite.Core/DynamicCollection.cs | 3 +- .../Indexing/CollectionSecondaryIndex.cs | 6 +- .../Indexing/GeoSpatialExtensions.cs | 11 +- src/BLite.Core/Indexing/RTreeIndex.cs | 27 +- .../Indexing/VectorSearchExtensions.cs | 18 +- src/BLite.Core/Query/IndexOptimizer.cs | 508 ++++++++++-------- .../BLite.Tests/NearWithinCorrectnessTests.cs | 180 +++++++ 8 files changed, 508 insertions(+), 251 deletions(-) create mode 100644 tests/BLite.Tests/NearWithinCorrectnessTests.cs diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 19d77c22..5304f160 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -3522,8 +3522,12 @@ Func GetCompiled() => { if (indexOpt.IsVectorSearch) { + // The index satisfied the VectorSearch term; the post-filter checks the rest. + var residual = indexOpt.FilterCompleteness == Query.IndexOptimizer.FilterCompleteness.Exact + ? null + : ((System.Linq.Expressions.Expression>)Query.IndexOptimizer.WithoutCall(whereClause, indexOpt.VectorSearchCall!)).Compile(); await foreach (var item in VectorSearchAsync(indexOpt.IndexName, indexOpt.VectorQuery!, indexOpt.K, ct: ct)) - if (indexOpt.FilterCompleteness == Query.IndexOptimizer.FilterCompleteness.Exact || GetCompiled()(item)) { yield return item; if (++yielded >= fetchLimit) yield break; } + if (residual == null || residual(item)) { yield return item; if (++yielded >= fetchLimit) yield break; } } else if (indexOpt.IsSpatialSearch) { diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index 3a20770d..6aace815 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -957,8 +957,7 @@ public async IAsyncEnumerable NearAsync(string indexName, (double if (!_secondaryIndexes.TryGetValue(indexName, out var entry) || entry.Kind != DynamicIndexKind.Spatial || entry.Spatial == null) throw new ArgumentException($"Spatial index '{indexName}' not found on collection '{_collectionName}'"); - var queryBox = SpatialMath.BoundingBox(center.Latitude, center.Longitude, radiusKm); - foreach (var loc in entry.Spatial.Search(queryBox, null)) + foreach (var loc in entry.Spatial.SearchNear(center.Latitude, center.Longitude, radiusKm, null)) { var doc = ReadDocumentAt(loc, 0UL); if (doc != null) yield return doc; diff --git a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs index 0afe4b91..13ee328c 100644 --- a/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs +++ b/src/BLite.Core/Indexing/CollectionSecondaryIndex.cs @@ -255,11 +255,7 @@ public IEnumerable Near((double Latitude, double Longitude) ce if (_spatialIndex == null) throw new InvalidOperationException("This index is not a spatial index."); - var queryBox = SpatialMath.BoundingBox(center.Latitude, center.Longitude, radiusKm); - foreach (var loc in _spatialIndex.Search(queryBox, transaction)) - { - yield return loc; - } + return _spatialIndex.SearchNear(center.Latitude, center.Longitude, radiusKm, transaction); } /// diff --git a/src/BLite.Core/Indexing/GeoSpatialExtensions.cs b/src/BLite.Core/Indexing/GeoSpatialExtensions.cs index 3535feb7..eeb5286b 100644 --- a/src/BLite.Core/Indexing/GeoSpatialExtensions.cs +++ b/src/BLite.Core/Indexing/GeoSpatialExtensions.cs @@ -4,7 +4,8 @@ public static class GeoSpatialExtensions { /// /// Performs a geospatial proximity search (Near) on a coordinate tuple property. - /// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available. + /// The LINQ query provider serves it from an R-Tree index when one exists; otherwise it is + /// evaluated per document using the great-circle (Haversine) distance. /// /// The coordinate tuple (Latitude, Longitude) property of the entity. /// The center point (Latitude, Longitude) for the proximity search. @@ -12,12 +13,13 @@ public static class GeoSpatialExtensions /// True if the point is within the specified radius. public static bool Near(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) center, double radiusKm) { - return true; + return Indexing.SpatialMath.DistanceKm(point.Latitude, point.Longitude, center.Latitude, center.Longitude) <= radiusKm; } /// /// Performs a geospatial bounding box search (Within) on a coordinate tuple property. - /// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available. + /// The LINQ query provider serves it from an R-Tree index when one exists; otherwise it is + /// evaluated per document. The box includes its edges. /// /// The coordinate tuple (Latitude, Longitude) property of the entity. /// The minimum (Latitude, Longitude) of the bounding box. @@ -25,6 +27,7 @@ public static bool Near(this (double Latitude, double Longitude) point, (double /// True if the point is within the specified bounding box. public static bool Within(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) min, (double Latitude, double Longitude) max) { - return true; + return point.Latitude >= min.Latitude && point.Latitude <= max.Latitude && + point.Longitude >= min.Longitude && point.Longitude <= max.Longitude; } } diff --git a/src/BLite.Core/Indexing/RTreeIndex.cs b/src/BLite.Core/Indexing/RTreeIndex.cs index 2ffb2258..a222e3c4 100644 --- a/src/BLite.Core/Indexing/RTreeIndex.cs +++ b/src/BLite.Core/Indexing/RTreeIndex.cs @@ -50,6 +50,31 @@ private uint InitializeNewIndex() } public IEnumerable Search(GeoBox area, ITransaction? transaction = null) + { + foreach (var (_, location) in SearchEntries(area, transaction)) + yield return location; + } + + /// + /// Locations of the indexed points within of the center. The tree + /// is searched with the circle's bounding box, then each hit is checked against the + /// great-circle distance, so points in the box corners are left out. + /// + public IEnumerable SearchNear(double latitude, double longitude, double radiusKm, ITransaction? transaction = null) + { + var box = SpatialMath.BoundingBox(latitude, longitude, radiusKm); + foreach (var (mbr, location) in SearchEntries(box, transaction)) + { + if (SpatialMath.DistanceKm(mbr.MinLat, mbr.MinLon, latitude, longitude) <= radiusKm) + yield return location; + } + } + + /// + /// Like , but also yields each leaf entry's box, which for an indexed + /// point is the point itself. + /// + private IEnumerable<(GeoBox Mbr, DocumentLocation Location)> SearchEntries(GeoBox area, ITransaction? transaction = null) { var stack = new Stack(); stack.Push(_rootPageId); @@ -73,7 +98,7 @@ public IEnumerable Search(GeoBox area, ITransaction? transacti { if (isLeaf) { - yield return pointer; + yield return (mbr, pointer); } else { diff --git a/src/BLite.Core/Indexing/VectorSearchExtensions.cs b/src/BLite.Core/Indexing/VectorSearchExtensions.cs index f61e2f74..3b6184c6 100644 --- a/src/BLite.Core/Indexing/VectorSearchExtensions.cs +++ b/src/BLite.Core/Indexing/VectorSearchExtensions.cs @@ -4,23 +4,33 @@ public static class VectorSearchExtensions { /// /// Performs a similarity search on a vector property. - /// This method is a marker for the LINQ query provider and is optimized using HNSW indexes if available. + /// This method is a marker for the LINQ query provider, which serves it from the property's + /// HNSW index. It must appear on its own or as a term of a top-level &&. /// /// The vector property of the entity. /// The query vector to compare against. /// Number of nearest neighbors to return. - /// True if the document is part of the top-k results (always returns true when evaluated in memory for compilation purposes). + /// Never returns. + /// + /// Always: whether a document is among the k nearest depends on the whole collection, so the + /// call cannot be evaluated one document at a time. + /// public static bool VectorSearch(this float[] vector, float[] query, int k) { - return true; + throw NotEvaluable(); } /// /// Performs a similarity search on a collection of vector properties. /// Used for entities with multiple vectors per document. /// + /// Always; see . public static bool VectorSearch(this IEnumerable vectors, float[] query, int k) { - return true; + throw NotEvaluable(); } + + private static NotSupportedException NotEvaluable() => new( + "VectorSearch selects the k nearest neighbours of the whole collection and cannot be evaluated per document. " + + "Use it on a property with a vector index, either on its own or combined with other conditions using &&."); } diff --git a/src/BLite.Core/Query/IndexOptimizer.cs b/src/BLite.Core/Query/IndexOptimizer.cs index c58dfa0f..ed04b204 100644 --- a/src/BLite.Core/Query/IndexOptimizer.cs +++ b/src/BLite.Core/Query/IndexOptimizer.cs @@ -9,12 +9,12 @@ namespace BLite.Core.Query; internal static class IndexOptimizer { - public class OptimizationResult - { - public string IndexName { get; set; } = ""; - public object? MinValue { get; set; } - public object? MaxValue { get; set; } - public IReadOnlyList? InValues { get; set; } + public class OptimizationResult + { + public string IndexName { get; set; } = ""; + public object? MinValue { get; set; } + public object? MaxValue { get; set; } + public IReadOnlyList? InValues { get; set; } /// /// Describes how completely the index scan covers the WHERE expression. @@ -51,6 +51,13 @@ public class OptimizationResult public float[]? VectorQuery { get; set; } public int K { get; set; } + /// + /// The VectorSearch call a vector plan serves from the index. It cannot be + /// evaluated per document, so a post-filter must treat it as satisfied; see + /// . + /// + public MethodCallExpression? VectorSearchCall { get; set; } + public bool IsSpatialSearch { get; set; } public (double Latitude, double Longitude) SpatialPoint { get; set; } public double RadiusKm { get; set; } @@ -153,15 +160,25 @@ public enum FilterCompleteness } [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] - private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter, IEnumerable indexes, ValueConverterRegistry? registry = null) - { - // ... (Existing AndAlso logic remains the same) ... - if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso) - { + private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter, IEnumerable indexes, ValueConverterRegistry? registry = null) + { + // ... (Existing AndAlso logic remains the same) ... + if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso) + { var left = OptimizeExpression(binary.Left, parameter, indexes, registry); var right = OptimizeExpression(binary.Right, parameter, indexes, registry); - if (left != null && right != null && left.IndexName == right.IndexName) + // A vector term must drive the plan: the post-filter cannot evaluate it per document. + if (left?.IsVectorSearch == true || right?.IsVectorSearch == true) + { + var vector = left?.IsVectorSearch == true ? left : right!; + vector.FilterCompleteness = FilterCompleteness.PartialAnd; + vector.HasResiduePredicate = true; + return vector; + } + + if (left != null && right != null && left.IndexName == right.IndexName + && !left.IsSpatialSearch && !right.IsSpatialSearch) { // Both sides are covered by the same index — combine into a single range. // Propagate the stricter completeness: if either branch required a post-filter @@ -188,46 +205,46 @@ public enum FilterCompleteness // does not fully satisfy the WHERE; caller must post-filter. if (left != null) { left.FilterCompleteness = FilterCompleteness.PartialAnd; left.HasResiduePredicate = true; return left; } if (right != null) { right.FilterCompleteness = FilterCompleteness.PartialAnd; right.HasResiduePredicate = true; return right; } - return null; - } - - // Handle OR over exact matches on the same indexed field: - // x => x.Prop == a || x.Prop == b → multi-point index probes. - if (expression is BinaryExpression orBinary && orBinary.NodeType == ExpressionType.OrElse) - { - var left = OptimizeExpression(orBinary.Left, parameter, indexes, registry); - var right = OptimizeExpression(orBinary.Right, parameter, indexes, registry); - - if (left != null && right != null && - left.IndexName == right.IndexName && - left.FilterCompleteness == FilterCompleteness.Exact && - right.FilterCompleteness == FilterCompleteness.Exact && - TryGetPointValues(left, out var leftValues) && - TryGetPointValues(right, out var rightValues)) - { - var merged = new List(leftValues.Count + rightValues.Count); - var seen = new HashSet(s_inValueComparer); - foreach (var v in leftValues) - { - if (seen.Add(v)) merged.Add(v); - } - foreach (var v in rightValues) - { - if (seen.Add(v)) merged.Add(v); - } - - return new OptimizationResult - { - IndexName = left.IndexName, - InValues = merged, - IsRange = false, - FilterCompleteness = FilterCompleteness.Exact, - StartInclusive = true, - EndInclusive = true - }; - } - return null; - } + return null; + } + + // Handle OR over exact matches on the same indexed field: + // x => x.Prop == a || x.Prop == b → multi-point index probes. + if (expression is BinaryExpression orBinary && orBinary.NodeType == ExpressionType.OrElse) + { + var left = OptimizeExpression(orBinary.Left, parameter, indexes, registry); + var right = OptimizeExpression(orBinary.Right, parameter, indexes, registry); + + if (left != null && right != null && + left.IndexName == right.IndexName && + left.FilterCompleteness == FilterCompleteness.Exact && + right.FilterCompleteness == FilterCompleteness.Exact && + TryGetPointValues(left, out var leftValues) && + TryGetPointValues(right, out var rightValues)) + { + var merged = new List(leftValues.Count + rightValues.Count); + var seen = new HashSet(s_inValueComparer); + foreach (var v in leftValues) + { + if (seen.Add(v)) merged.Add(v); + } + foreach (var v in rightValues) + { + if (seen.Add(v)) merged.Add(v); + } + + return new OptimizationResult + { + IndexName = left.IndexName, + InValues = merged, + IsRange = false, + FilterCompleteness = FilterCompleteness.Exact, + StartInclusive = true, + EndInclusive = true + }; + } + return null; + } // Handle bare bool member: e => e.IsActive (equivalent to e.IsActive == true) // Handle logical NOT over bool member: e => !e.IsActive (equivalent to e.IsActive == false) @@ -322,29 +339,29 @@ notExpr.Operand is MemberExpression notMember && } return result; } - } - - if (TryParseContainsInPredicate(expression, parameter, registry, out var inPropertyPath, out var inValues)) - { - CollectionIndexInfo? index = null; - foreach (var idx in indexes) { if (Matches(idx, inPropertyPath)) { index = idx; break; } } - if (index != null) - { - return new OptimizationResult - { - IndexName = index.Name, - InValues = inValues, - IsRange = false, - FilterCompleteness = FilterCompleteness.Exact, - StartInclusive = true, - EndInclusive = true - }; - } - } - - // Handle StartsWith - if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" && call.Object is MemberExpression member) - { + } + + if (TryParseContainsInPredicate(expression, parameter, registry, out var inPropertyPath, out var inValues)) + { + CollectionIndexInfo? index = null; + foreach (var idx in indexes) { if (Matches(idx, inPropertyPath)) { index = idx; break; } } + if (index != null) + { + return new OptimizationResult + { + IndexName = index.Name, + InValues = inValues, + IsRange = false, + FilterCompleteness = FilterCompleteness.Exact, + StartInclusive = true, + EndInclusive = true + }; + } + } + + // Handle StartsWith + if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" && call.Object is MemberExpression member) + { if (member.Expression == parameter && call.Arguments[0] is ConstantExpression constant && constant.Value is string prefix) { CollectionIndexInfo? index = null; @@ -381,7 +398,8 @@ notExpr.Operand is MemberExpression notMember && IndexName = index.Name, IsVectorSearch = true, VectorQuery = query, - K = k + K = k, + VectorSearchCall = mcall }; } } @@ -429,26 +447,48 @@ notExpr.Operand is MemberExpression notMember && } } - return null; - } - - private static bool TryGetPointValues(OptimizationResult result, out IReadOnlyList values) - { - if (result.InValues != null) - { - values = result.InValues; - return true; - } - - if (!result.IsRange && Equals(result.MinValue, result.MaxValue)) - { - values = [result.MinValue]; - return true; - } - - values = Array.Empty(); - return false; - } + return null; + } + + /// + /// Returns with replaced by true, + /// for post-filtering the candidates of an index that already satisfied that call. + /// + public static LambdaExpression WithoutCall(LambdaExpression predicate, Expression call) + => Expression.Lambda(predicate.Type, new ReplaceNodeVisitor(call, Expression.Constant(true)).Visit(predicate.Body)!, predicate.Parameters); + + private sealed class ReplaceNodeVisitor : ExpressionVisitor + { + private readonly Expression _target; + private readonly Expression _replacement; + + public ReplaceNodeVisitor(Expression target, Expression replacement) + { + _target = target; + _replacement = replacement; + } + + public override Expression? Visit(Expression? node) + => ReferenceEquals(node, _target) ? _replacement : base.Visit(node); + } + + private static bool TryGetPointValues(OptimizationResult result, out IReadOnlyList values) + { + if (result.InValues != null) + { + values = result.InValues; + return true; + } + + if (!result.IsRange && Equals(result.MinValue, result.MaxValue)) + { + values = [result.MinValue]; + return true; + } + + values = Array.Empty(); + return false; + } private static string IncrementPrefix(string prefix) { @@ -537,7 +577,7 @@ static string Normalize(string s) => } [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] - private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression, ParameterExpression parameter, ValueConverterRegistry? registry = null) + private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression, ParameterExpression parameter, ValueConverterRegistry? registry = null) { if (expression is BinaryExpression binary) { @@ -634,137 +674,137 @@ private static (string? propertyName, object? value, ExpressionType op) ParseSim } } - return (null, null, ExpressionType.Default); - } - - [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] - private static bool TryParseContainsInPredicate( - Expression expression, - ParameterExpression parameter, - ValueConverterRegistry? registry, - out string propertyPath, - out IReadOnlyList values) - { - propertyPath = string.Empty; - values = Array.Empty(); - - if (expression is not MethodCallExpression call || call.Method.Name != "Contains") - return false; - - static Expression UnwrapConvert(Expression e) - { - while (e is UnaryExpression { NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked } u) - e = u.Operand; - return e; - } - - Expression? collectionExpr = null; - Expression? memberExpr = null; - - if (call.Object != null && call.Arguments.Count == 1) - { - // Instance Contains: list.Contains(x.Prop) - collectionExpr = call.Object; - memberExpr = call.Arguments[0]; - } - else if (call.Object == null && (call.Arguments.Count == 2 || call.Arguments.Count == 3)) - { - // Enumerable.Contains(list, x.Prop) or MemoryExtensions.Contains(span, x.Prop, comparer) - // where the comparer argument is typically null. - if (call.Arguments.Count == 3) - { - object? comparer; - try - { - comparer = EvaluateExpression(call.Arguments[2]); - } - catch - { - return false; - } - - if (comparer != null) - return false; - } - - collectionExpr = call.Arguments[0]; - memberExpr = call.Arguments[1]; - } - else - { - return false; - } - - var unwrappedMember = UnwrapConvert(memberExpr); - if (unwrappedMember is not MemberExpression member) - return false; - - var extractedPath = ExtractMemberPath(member, parameter); - if (extractedPath == null) - return false; - propertyPath = extractedPath; - - if (collectionExpr is MethodCallExpression { Method.Name: "op_Implicit", Object: null } implicitCall && - implicitCall.Arguments.Count == 1) - { - collectionExpr = implicitCall.Arguments[0]; - } - - object? enumerableObj; - try - { - enumerableObj = EvaluateExpression(collectionExpr); - } - catch - { - return false; - } - - if (enumerableObj is not System.Collections.IEnumerable enumerable || enumerableObj is string) - return false; - - var list = new List(); - var seen = new HashSet(s_inValueComparer); - foreach (var raw in enumerable) - { - var converted = TryApplyConverter(propertyPath, raw, registry); - if (!IsIndexableValue(converted)) - return false; - var normalized = converted ?? DBNull.Value; - if (seen.Add(normalized)) - { - list.Add(normalized); - } - } - - values = list; - return true; - } - - private static readonly IEqualityComparer s_inValueComparer = new InValueComparer(); - - private sealed class InValueComparer : IEqualityComparer - { - bool IEqualityComparer.Equals(object? x, object? y) - { - if (ReferenceEquals(x, y)) return true; - if (x is null || y is null) return false; - if (x is byte[] xb && y is byte[] yb) return xb.AsSpan().SequenceEqual(yb); - return x.Equals(y); - } - - int IEqualityComparer.GetHashCode(object? obj) - { - if (obj is null) return 0; - if (obj is byte[] bytes) - { - var hc = new HashCode(); - foreach (var b in bytes) hc.Add(b); - return hc.ToHashCode(); - } - return obj.GetHashCode(); - } - } + return (null, null, ExpressionType.Default); + } + + [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] + private static bool TryParseContainsInPredicate( + Expression expression, + ParameterExpression parameter, + ValueConverterRegistry? registry, + out string propertyPath, + out IReadOnlyList values) + { + propertyPath = string.Empty; + values = Array.Empty(); + + if (expression is not MethodCallExpression call || call.Method.Name != "Contains") + return false; + + static Expression UnwrapConvert(Expression e) + { + while (e is UnaryExpression { NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked } u) + e = u.Operand; + return e; + } + + Expression? collectionExpr = null; + Expression? memberExpr = null; + + if (call.Object != null && call.Arguments.Count == 1) + { + // Instance Contains: list.Contains(x.Prop) + collectionExpr = call.Object; + memberExpr = call.Arguments[0]; + } + else if (call.Object == null && (call.Arguments.Count == 2 || call.Arguments.Count == 3)) + { + // Enumerable.Contains(list, x.Prop) or MemoryExtensions.Contains(span, x.Prop, comparer) + // where the comparer argument is typically null. + if (call.Arguments.Count == 3) + { + object? comparer; + try + { + comparer = EvaluateExpression(call.Arguments[2]); + } + catch + { + return false; + } + + if (comparer != null) + return false; + } + + collectionExpr = call.Arguments[0]; + memberExpr = call.Arguments[1]; + } + else + { + return false; + } + + var unwrappedMember = UnwrapConvert(memberExpr); + if (unwrappedMember is not MemberExpression member) + return false; + + var extractedPath = ExtractMemberPath(member, parameter); + if (extractedPath == null) + return false; + propertyPath = extractedPath; + + if (collectionExpr is MethodCallExpression { Method.Name: "op_Implicit", Object: null } implicitCall && + implicitCall.Arguments.Count == 1) + { + collectionExpr = implicitCall.Arguments[0]; + } + + object? enumerableObj; + try + { + enumerableObj = EvaluateExpression(collectionExpr); + } + catch + { + return false; + } + + if (enumerableObj is not System.Collections.IEnumerable enumerable || enumerableObj is string) + return false; + + var list = new List(); + var seen = new HashSet(s_inValueComparer); + foreach (var raw in enumerable) + { + var converted = TryApplyConverter(propertyPath, raw, registry); + if (!IsIndexableValue(converted)) + return false; + var normalized = converted ?? DBNull.Value; + if (seen.Add(normalized)) + { + list.Add(normalized); + } + } + + values = list; + return true; + } + + private static readonly IEqualityComparer s_inValueComparer = new InValueComparer(); + + private sealed class InValueComparer : IEqualityComparer + { + bool IEqualityComparer.Equals(object? x, object? y) + { + if (ReferenceEquals(x, y)) return true; + if (x is null || y is null) return false; + if (x is byte[] xb && y is byte[] yb) return xb.AsSpan().SequenceEqual(yb); + return x.Equals(y); + } + + int IEqualityComparer.GetHashCode(object? obj) + { + if (obj is null) return 0; + if (obj is byte[] bytes) + { + var hc = new HashCode(); + foreach (var b in bytes) hc.Add(b); + return hc.ToHashCode(); + } + return obj.GetHashCode(); + } + } private static object? TryApplyConverter(string propertyPath, object? value, ValueConverterRegistry? registry) { @@ -774,12 +814,12 @@ private sealed class InValueComparer : IEqualityComparer return registry.TryConvert(topProp, value, out var pv) ? pv : value; } - private static readonly HashSet _knownBsonPrimitives = - [ - typeof(int), typeof(long), typeof(double), typeof(decimal), - typeof(bool), typeof(string), typeof(DateTime), typeof(DateTimeOffset), - typeof(ObjectId), typeof(Guid), typeof(byte[]) - ]; + private static readonly HashSet _knownBsonPrimitives = + [ + typeof(int), typeof(long), typeof(double), typeof(decimal), + typeof(bool), typeof(string), typeof(DateTime), typeof(DateTimeOffset), + typeof(ObjectId), typeof(Guid), typeof(byte[]) + ]; /// /// Returns true when can be used directly as an index diff --git a/tests/BLite.Tests/NearWithinCorrectnessTests.cs b/tests/BLite.Tests/NearWithinCorrectnessTests.cs new file mode 100644 index 00000000..f8fe064f --- /dev/null +++ b/tests/BLite.Tests/NearWithinCorrectnessTests.cs @@ -0,0 +1,180 @@ +using BLite.Core; +using BLite.Core.Indexing; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// Near / Within must give the same answer whether or not an index serves them, +/// and VectorSearch must not silently match every document outside an index plan (#165). +/// +public class NearWithinCorrectnessTests : IDisposable +{ + private static readonly (double, double) Milan = (45.4642, 9.1899); + private static readonly (double, double) NewYork = (40.7128, -74.0060); + + private readonly string _dbPath; + private readonly TestDbContext _db; + + public NearWithinCorrectnessTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"near_within_{Guid.NewGuid():N}.db"); + _db = new TestDbContext(_dbPath); + } + + public void Dispose() + { + _db.Dispose(); + foreach (var f in new[] { _dbPath, Path.ChangeExtension(_dbPath, ".wal") }) + try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ } + } + + // ── In-memory evaluation ──────────────────────────────────────────────── + + [Fact] + public void Near_InMemory_UsesGreatCircleDistance() + { + Assert.False(NewYork.Near(Milan, 10.0)); + Assert.True(NewYork.Near(Milan, 6500.0)); + } + + [Fact] + public void Within_InMemory_IncludesEdges() + { + Assert.True((45.0, 9.0).Within((44.0, 8.0), (46.0, 10.0))); + Assert.True((44.0, 10.0).Within((44.0, 8.0), (46.0, 10.0))); + Assert.False(NewYork.Within((44.0, 8.0), (46.0, 10.0))); + } + + [Fact] + public void VectorSearch_InMemory_Throws() + { + Assert.Throws(() => new[] { 1f }.VectorSearch([1f], 1)); + } + + // ── LINQ ──────────────────────────────────────────────────────────────── + + [Fact] + public async Task Near_InsideOr_OnlyMatchesNearbyDocuments() + { + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "New York", Location = NewYork }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "Milan", Location = (45.47, 9.19) }); + + var results = _db.GeoItems.AsQueryable() + .Where(p => p.Location.Near(Milan, 10.0) || p.Name == "nobody") + .ToList(); + + Assert.Equal("Milan", Assert.Single(results).Name); + } + + [Fact] + public async Task Within_InsideOr_OnlyMatchesDocumentsInTheBox() + { + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "New York", Location = NewYork }); + (double, double) min = (44.0, 8.0), max = (46.0, 10.0); + + var results = _db.GeoItems.AsQueryable() + .Where(p => p.Location.Within(min, max) || p.Name == "nobody") + .ToList(); + + Assert.Empty(results); + } + + [Fact] + public async Task NegatedNear_FullScan_ReturnsOnlyFarDocuments() + { + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "New York", Location = NewYork }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "Milan", Location = (45.47, 9.19) }); + + var results = _db.GeoItems.AsQueryable().Where(p => !p.Location.Near(Milan, 10.0)).ToList(); + + Assert.Equal("New York", Assert.Single(results).Name); + } + + [Fact] + public async Task IndexedNear_ExcludesBoxCorner() + { + // (0.85, 0.85) is 133.7 km from (0, 0): inside the 100 km bounding box, outside the circle. + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "corner", Location = (0.85, 0.85) }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "inside", Location = (0.5, 0.5) }); + var center = (0.0, 0.0); + + var viaLinq = _db.GeoItems.AsQueryable().Where(p => p.Location.Near(center, 100.0)).ToList(); + var viaAsync = await _db.GeoItems.NearAsync("idx_spatial", center, 100.0).ToListAsync(); + var count = _db.GeoItems.AsQueryable().Count(p => p.Location.Near(center, 100.0)); + + Assert.Equal("inside", Assert.Single(viaLinq).Name); + Assert.Equal("inside", Assert.Single(viaAsync).Name); + Assert.Equal(1, count); + } + + [Fact] + public async Task NearAndWithin_OnTheSameIndex_ReturnTheirIntersection() + { + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "both", Location = (0.1, 0.1) }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "near only", Location = (-0.1, -0.1) }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "box only", Location = (5.0, 5.0) }); + var center = (0.0, 0.0); + (double, double) min = (0.0, 0.0), max = (10.0, 10.0); + + var results = _db.GeoItems.AsQueryable() + .Where(p => p.Location.Near(center, 50.0) && p.Location.Within(min, max)) + .ToList(); + + Assert.Equal("both", Assert.Single(results).Name); + } + + [Fact] + public async Task VectorSearch_WithAnotherCondition_FiltersTheNearestNeighbours() + { + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] }); + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "B", Embedding = [1f, 1f, 1f] }); + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "B", Embedding = [50f, 50f, 50f] }); + float[] query = [0f, 0f, 0f]; + + var results = _db.VectorItems.AsQueryable() + .Where(x => x.Title == "B" && x.Embedding.VectorSearch(query, 2)) + .ToList(); + + Assert.Equal(1f, Assert.Single(results).Embedding[0]); + } + + [Fact] + public async Task VectorSearch_InsideOr_Throws() + { + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] }); + float[] query = [0f, 0f, 0f]; + + Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Embedding.VectorSearch(query, 1) || x.Title == "A") + .ToList()); + } + + // ── DynamicCollection ─────────────────────────────────────────────────── + + [Fact] + public async Task Dynamic_Near_ExcludesBoxCorner() + { + var path = Path.ChangeExtension(_dbPath, ".dyn.db"); + try + { + using var engine = new BLiteEngine(path); + var col = engine.GetOrCreateCollection("places"); + await col.CreateSpatialIndexAsync("pos"); + await col.InsertAsync(col.CreateDocument(["name", "pos"], b => b + .AddString("name", "corner").AddCoordinates("pos", (0.85, 0.85)))); + await col.InsertAsync(col.CreateDocument(["name", "pos"], b => b + .AddString("name", "inside").AddCoordinates("pos", (0.5, 0.5)))); + + var results = await col.NearAsync("idx_spatial_pos", (0.0, 0.0), 100.0).ToListAsync(); + + Assert.True(Assert.Single(results).TryGetString("name", out var name)); + Assert.Equal("inside", name); + } + finally + { + foreach (var f in new[] { path, Path.ChangeExtension(path, ".wal") }) + try { if (File.Exists(f)) File.Delete(f); } catch { /* best-effort */ } + } + } +} From 97bfb7d0e9b8e5da85f135b081058a00cd875162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 15:44:07 +0200 Subject: [PATCH 2/3] fix(query): address review of Near/Within/VectorSearch (#165) - The in-memory replay of a query whose WHERE FetchAsync already applied (GroupBy, Join, aggregates with a selector, OrderBy after Select) treats VectorSearch as satisfied: the index served it. Before, the replay re-evaluated it and threw. - The OrElse merge no longer turns two spatial or vector terms on one index into an IN probe on that index, so Near(a) || Near(b) is evaluated in memory and returns documents near either point. - The VectorSearch error says a query can contain only one VectorSearch. - The vector residual predicate is compiled lazily, like GetCompiled(). - Within documents that the box does not wrap across the antimeridian. - IndexOptimizer.cs keeps its original mixed line endings; only the real changes remain in its diff. BREAKING CHANGE: VectorSearch throws NotSupportedException when it is evaluated per document (outside an index plan, inside ||, or on a property without a vector index) instead of returning true for every document. Co-Authored-By: Claude Opus 5.5 --- .../Collections/DocumentCollection.cs | 13 +- .../Indexing/GeoSpatialExtensions.cs | 4 +- .../Indexing/VectorSearchExtensions.cs | 3 +- src/BLite.Core/Query/BTreeQueryProvider.cs | 16 + src/BLite.Core/Query/IndexOptimizer.cs | 510 +++++++++--------- .../BLite.Tests/NearWithinCorrectnessTests.cs | 49 ++ 6 files changed, 335 insertions(+), 260 deletions(-) diff --git a/src/BLite.Core/Collections/DocumentCollection.cs b/src/BLite.Core/Collections/DocumentCollection.cs index 5304f160..ff32de6b 100644 --- a/src/BLite.Core/Collections/DocumentCollection.cs +++ b/src/BLite.Core/Collections/DocumentCollection.cs @@ -3523,11 +3523,16 @@ Func GetCompiled() => if (indexOpt.IsVectorSearch) { // The index satisfied the VectorSearch term; the post-filter checks the rest. - var residual = indexOpt.FilterCompleteness == Query.IndexOptimizer.FilterCompleteness.Exact - ? null - : ((System.Linq.Expressions.Expression>)Query.IndexOptimizer.WithoutCall(whereClause, indexOpt.VectorSearchCall!)).Compile(); + // Compiled lazily, like GetCompiled(), only when a candidate needs it. + bool exact = indexOpt.FilterCompleteness == Query.IndexOptimizer.FilterCompleteness.Exact; + Func? residual = null; await foreach (var item in VectorSearchAsync(indexOpt.IndexName, indexOpt.VectorQuery!, indexOpt.K, ct: ct)) - if (residual == null || residual(item)) { yield return item; if (++yielded >= fetchLimit) yield break; } + { + if (!exact && !(residual ??= ((System.Linq.Expressions.Expression>)Query.IndexOptimizer.WithoutCall(whereClause, indexOpt.VectorSearchCall!)).Compile())(item)) + continue; + yield return item; + if (++yielded >= fetchLimit) yield break; + } } else if (indexOpt.IsSpatialSearch) { diff --git a/src/BLite.Core/Indexing/GeoSpatialExtensions.cs b/src/BLite.Core/Indexing/GeoSpatialExtensions.cs index eeb5286b..ef5aa491 100644 --- a/src/BLite.Core/Indexing/GeoSpatialExtensions.cs +++ b/src/BLite.Core/Indexing/GeoSpatialExtensions.cs @@ -19,7 +19,9 @@ public static bool Near(this (double Latitude, double Longitude) point, (double /// /// Performs a geospatial bounding box search (Within) on a coordinate tuple property. /// The LINQ query provider serves it from an R-Tree index when one exists; otherwise it is - /// evaluated per document. The box includes its edges. + /// evaluated per document. The box includes its edges and does not wrap across the ±180° + /// meridian: a longitude greater than the + /// longitude matches nothing. /// /// The coordinate tuple (Latitude, Longitude) property of the entity. /// The minimum (Latitude, Longitude) of the bounding box. diff --git a/src/BLite.Core/Indexing/VectorSearchExtensions.cs b/src/BLite.Core/Indexing/VectorSearchExtensions.cs index 3b6184c6..9012cc01 100644 --- a/src/BLite.Core/Indexing/VectorSearchExtensions.cs +++ b/src/BLite.Core/Indexing/VectorSearchExtensions.cs @@ -32,5 +32,6 @@ public static bool VectorSearch(this IEnumerable vectors, float[] query private static NotSupportedException NotEvaluable() => new( "VectorSearch selects the k nearest neighbours of the whole collection and cannot be evaluated per document. " + - "Use it on a property with a vector index, either on its own or combined with other conditions using &&."); + "Use it on a property with a vector index, either on its own or combined with other conditions using &&; " + + "a query can contain only one VectorSearch."); } diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index aa8a65c0..39d3b77b 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -311,6 +311,11 @@ private async Task ExecuteAsyncCore(Expression expression, Can // pipeline cannot model. We still benefit from index / BSON-scan fetch, // then let EnumerableRewriter translate the remaining Queryable calls to // Enumerable equivalents and compile the residual expression once. + // FetchAsync applied the WHERE. A VectorSearch term in it was served by the index (otherwise + // the fetch would have thrown evaluating it), so the in-memory replay treats it as satisfied. + if (whereAlreadyApplied) + expression = VectorSearchSatisfiedRewriter.Instance.Visit(expression); + if (model.HasComplexOperators) return ExecuteViaEnumerableRewriter(expression, sourceData); @@ -636,6 +641,17 @@ private static bool AggregateValues( // ─── Expression-tree helpers ────────────────────────────────────────────── + /// Replaces every VectorSearch call with true. + private sealed class VectorSearchSatisfiedRewriter : ExpressionVisitor + { + public static readonly VectorSearchSatisfiedRewriter Instance = new(); + + protected override Expression VisitMethodCall(MethodCallExpression node) + => node.Method.DeclaringType == typeof(VectorSearchExtensions) + ? Expression.Constant(true) + : base.VisitMethodCall(node); + } + /// /// Fallback for queries containing operators the direct pipeline cannot handle /// (GroupBy, Join, Sum/Average/Min/Max with selectors, etc.). diff --git a/src/BLite.Core/Query/IndexOptimizer.cs b/src/BLite.Core/Query/IndexOptimizer.cs index ed04b204..b89dae65 100644 --- a/src/BLite.Core/Query/IndexOptimizer.cs +++ b/src/BLite.Core/Query/IndexOptimizer.cs @@ -9,12 +9,12 @@ namespace BLite.Core.Query; internal static class IndexOptimizer { - public class OptimizationResult - { - public string IndexName { get; set; } = ""; - public object? MinValue { get; set; } - public object? MaxValue { get; set; } - public IReadOnlyList? InValues { get; set; } + public class OptimizationResult + { + public string IndexName { get; set; } = ""; + public object? MinValue { get; set; } + public object? MaxValue { get; set; } + public IReadOnlyList? InValues { get; set; } /// /// Describes how completely the index scan covers the WHERE expression. @@ -160,11 +160,11 @@ public enum FilterCompleteness } [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] - private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter, IEnumerable indexes, ValueConverterRegistry? registry = null) - { - // ... (Existing AndAlso logic remains the same) ... - if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso) - { + private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter, IEnumerable indexes, ValueConverterRegistry? registry = null) + { + // ... (Existing AndAlso logic remains the same) ... + if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso) + { var left = OptimizeExpression(binary.Left, parameter, indexes, registry); var right = OptimizeExpression(binary.Right, parameter, indexes, registry); @@ -205,46 +205,48 @@ public enum FilterCompleteness // does not fully satisfy the WHERE; caller must post-filter. if (left != null) { left.FilterCompleteness = FilterCompleteness.PartialAnd; left.HasResiduePredicate = true; return left; } if (right != null) { right.FilterCompleteness = FilterCompleteness.PartialAnd; right.HasResiduePredicate = true; return right; } - return null; - } - - // Handle OR over exact matches on the same indexed field: - // x => x.Prop == a || x.Prop == b → multi-point index probes. - if (expression is BinaryExpression orBinary && orBinary.NodeType == ExpressionType.OrElse) - { - var left = OptimizeExpression(orBinary.Left, parameter, indexes, registry); - var right = OptimizeExpression(orBinary.Right, parameter, indexes, registry); - - if (left != null && right != null && - left.IndexName == right.IndexName && - left.FilterCompleteness == FilterCompleteness.Exact && - right.FilterCompleteness == FilterCompleteness.Exact && - TryGetPointValues(left, out var leftValues) && - TryGetPointValues(right, out var rightValues)) - { - var merged = new List(leftValues.Count + rightValues.Count); - var seen = new HashSet(s_inValueComparer); - foreach (var v in leftValues) - { - if (seen.Add(v)) merged.Add(v); - } - foreach (var v in rightValues) - { - if (seen.Add(v)) merged.Add(v); - } - - return new OptimizationResult - { - IndexName = left.IndexName, - InValues = merged, - IsRange = false, - FilterCompleteness = FilterCompleteness.Exact, - StartInclusive = true, - EndInclusive = true - }; - } - return null; - } + return null; + } + + // Handle OR over exact matches on the same indexed field: + // x => x.Prop == a || x.Prop == b → multi-point index probes. + if (expression is BinaryExpression orBinary && orBinary.NodeType == ExpressionType.OrElse) + { + var left = OptimizeExpression(orBinary.Left, parameter, indexes, registry); + var right = OptimizeExpression(orBinary.Right, parameter, indexes, registry); + + if (left != null && right != null && + left.IndexName == right.IndexName && + !left.IsSpatialSearch && !right.IsSpatialSearch && + !left.IsVectorSearch && !right.IsVectorSearch && + left.FilterCompleteness == FilterCompleteness.Exact && + right.FilterCompleteness == FilterCompleteness.Exact && + TryGetPointValues(left, out var leftValues) && + TryGetPointValues(right, out var rightValues)) + { + var merged = new List(leftValues.Count + rightValues.Count); + var seen = new HashSet(s_inValueComparer); + foreach (var v in leftValues) + { + if (seen.Add(v)) merged.Add(v); + } + foreach (var v in rightValues) + { + if (seen.Add(v)) merged.Add(v); + } + + return new OptimizationResult + { + IndexName = left.IndexName, + InValues = merged, + IsRange = false, + FilterCompleteness = FilterCompleteness.Exact, + StartInclusive = true, + EndInclusive = true + }; + } + return null; + } // Handle bare bool member: e => e.IsActive (equivalent to e.IsActive == true) // Handle logical NOT over bool member: e => !e.IsActive (equivalent to e.IsActive == false) @@ -339,29 +341,29 @@ notExpr.Operand is MemberExpression notMember && } return result; } - } - - if (TryParseContainsInPredicate(expression, parameter, registry, out var inPropertyPath, out var inValues)) - { - CollectionIndexInfo? index = null; - foreach (var idx in indexes) { if (Matches(idx, inPropertyPath)) { index = idx; break; } } - if (index != null) - { - return new OptimizationResult - { - IndexName = index.Name, - InValues = inValues, - IsRange = false, - FilterCompleteness = FilterCompleteness.Exact, - StartInclusive = true, - EndInclusive = true - }; - } - } - - // Handle StartsWith - if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" && call.Object is MemberExpression member) - { + } + + if (TryParseContainsInPredicate(expression, parameter, registry, out var inPropertyPath, out var inValues)) + { + CollectionIndexInfo? index = null; + foreach (var idx in indexes) { if (Matches(idx, inPropertyPath)) { index = idx; break; } } + if (index != null) + { + return new OptimizationResult + { + IndexName = index.Name, + InValues = inValues, + IsRange = false, + FilterCompleteness = FilterCompleteness.Exact, + StartInclusive = true, + EndInclusive = true + }; + } + } + + // Handle StartsWith + if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" && call.Object is MemberExpression member) + { if (member.Expression == parameter && call.Arguments[0] is ConstantExpression constant && constant.Value is string prefix) { CollectionIndexInfo? index = null; @@ -447,48 +449,48 @@ notExpr.Operand is MemberExpression notMember && } } - return null; - } - - /// - /// Returns with replaced by true, - /// for post-filtering the candidates of an index that already satisfied that call. - /// - public static LambdaExpression WithoutCall(LambdaExpression predicate, Expression call) - => Expression.Lambda(predicate.Type, new ReplaceNodeVisitor(call, Expression.Constant(true)).Visit(predicate.Body)!, predicate.Parameters); - - private sealed class ReplaceNodeVisitor : ExpressionVisitor - { - private readonly Expression _target; - private readonly Expression _replacement; - - public ReplaceNodeVisitor(Expression target, Expression replacement) - { - _target = target; - _replacement = replacement; - } - - public override Expression? Visit(Expression? node) - => ReferenceEquals(node, _target) ? _replacement : base.Visit(node); - } - - private static bool TryGetPointValues(OptimizationResult result, out IReadOnlyList values) - { - if (result.InValues != null) - { - values = result.InValues; - return true; - } - - if (!result.IsRange && Equals(result.MinValue, result.MaxValue)) - { - values = [result.MinValue]; - return true; - } - - values = Array.Empty(); - return false; - } + return null; + } + + /// + /// Returns with replaced by true, + /// for post-filtering the candidates of an index that already satisfied that call. + /// + public static LambdaExpression WithoutCall(LambdaExpression predicate, Expression call) + => Expression.Lambda(predicate.Type, new ReplaceNodeVisitor(call, Expression.Constant(true)).Visit(predicate.Body)!, predicate.Parameters); + + private sealed class ReplaceNodeVisitor : ExpressionVisitor + { + private readonly Expression _target; + private readonly Expression _replacement; + + public ReplaceNodeVisitor(Expression target, Expression replacement) + { + _target = target; + _replacement = replacement; + } + + public override Expression? Visit(Expression? node) + => ReferenceEquals(node, _target) ? _replacement : base.Visit(node); + } + + private static bool TryGetPointValues(OptimizationResult result, out IReadOnlyList values) + { + if (result.InValues != null) + { + values = result.InValues; + return true; + } + + if (!result.IsRange && Equals(result.MinValue, result.MaxValue)) + { + values = [result.MinValue]; + return true; + } + + values = Array.Empty(); + return false; + } private static string IncrementPrefix(string prefix) { @@ -577,7 +579,7 @@ static string Normalize(string s) => } [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] - private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression, ParameterExpression parameter, ValueConverterRegistry? registry = null) + private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression, ParameterExpression parameter, ValueConverterRegistry? registry = null) { if (expression is BinaryExpression binary) { @@ -674,137 +676,137 @@ private static (string? propertyName, object? value, ExpressionType op) ParseSim } } - return (null, null, ExpressionType.Default); - } - - [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] - private static bool TryParseContainsInPredicate( - Expression expression, - ParameterExpression parameter, - ValueConverterRegistry? registry, - out string propertyPath, - out IReadOnlyList values) - { - propertyPath = string.Empty; - values = Array.Empty(); - - if (expression is not MethodCallExpression call || call.Method.Name != "Contains") - return false; - - static Expression UnwrapConvert(Expression e) - { - while (e is UnaryExpression { NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked } u) - e = u.Operand; - return e; - } - - Expression? collectionExpr = null; - Expression? memberExpr = null; - - if (call.Object != null && call.Arguments.Count == 1) - { - // Instance Contains: list.Contains(x.Prop) - collectionExpr = call.Object; - memberExpr = call.Arguments[0]; - } - else if (call.Object == null && (call.Arguments.Count == 2 || call.Arguments.Count == 3)) - { - // Enumerable.Contains(list, x.Prop) or MemoryExtensions.Contains(span, x.Prop, comparer) - // where the comparer argument is typically null. - if (call.Arguments.Count == 3) - { - object? comparer; - try - { - comparer = EvaluateExpression(call.Arguments[2]); - } - catch - { - return false; - } - - if (comparer != null) - return false; - } - - collectionExpr = call.Arguments[0]; - memberExpr = call.Arguments[1]; - } - else - { - return false; - } - - var unwrappedMember = UnwrapConvert(memberExpr); - if (unwrappedMember is not MemberExpression member) - return false; - - var extractedPath = ExtractMemberPath(member, parameter); - if (extractedPath == null) - return false; - propertyPath = extractedPath; - - if (collectionExpr is MethodCallExpression { Method.Name: "op_Implicit", Object: null } implicitCall && - implicitCall.Arguments.Count == 1) - { - collectionExpr = implicitCall.Arguments[0]; - } - - object? enumerableObj; - try - { - enumerableObj = EvaluateExpression(collectionExpr); - } - catch - { - return false; - } - - if (enumerableObj is not System.Collections.IEnumerable enumerable || enumerableObj is string) - return false; - - var list = new List(); - var seen = new HashSet(s_inValueComparer); - foreach (var raw in enumerable) - { - var converted = TryApplyConverter(propertyPath, raw, registry); - if (!IsIndexableValue(converted)) - return false; - var normalized = converted ?? DBNull.Value; - if (seen.Add(normalized)) - { - list.Add(normalized); - } - } - - values = list; - return true; - } - - private static readonly IEqualityComparer s_inValueComparer = new InValueComparer(); - - private sealed class InValueComparer : IEqualityComparer - { - bool IEqualityComparer.Equals(object? x, object? y) - { - if (ReferenceEquals(x, y)) return true; - if (x is null || y is null) return false; - if (x is byte[] xb && y is byte[] yb) return xb.AsSpan().SequenceEqual(yb); - return x.Equals(y); - } - - int IEqualityComparer.GetHashCode(object? obj) - { - if (obj is null) return 0; - if (obj is byte[] bytes) - { - var hc = new HashCode(); - foreach (var b in bytes) hc.Add(b); - return hc.ToHashCode(); - } - return obj.GetHashCode(); - } - } + return (null, null, ExpressionType.Default); + } + + [RequiresDynamicCode("Index optimization may use Expression.Compile() to evaluate complex expressions.")] + private static bool TryParseContainsInPredicate( + Expression expression, + ParameterExpression parameter, + ValueConverterRegistry? registry, + out string propertyPath, + out IReadOnlyList values) + { + propertyPath = string.Empty; + values = Array.Empty(); + + if (expression is not MethodCallExpression call || call.Method.Name != "Contains") + return false; + + static Expression UnwrapConvert(Expression e) + { + while (e is UnaryExpression { NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked } u) + e = u.Operand; + return e; + } + + Expression? collectionExpr = null; + Expression? memberExpr = null; + + if (call.Object != null && call.Arguments.Count == 1) + { + // Instance Contains: list.Contains(x.Prop) + collectionExpr = call.Object; + memberExpr = call.Arguments[0]; + } + else if (call.Object == null && (call.Arguments.Count == 2 || call.Arguments.Count == 3)) + { + // Enumerable.Contains(list, x.Prop) or MemoryExtensions.Contains(span, x.Prop, comparer) + // where the comparer argument is typically null. + if (call.Arguments.Count == 3) + { + object? comparer; + try + { + comparer = EvaluateExpression(call.Arguments[2]); + } + catch + { + return false; + } + + if (comparer != null) + return false; + } + + collectionExpr = call.Arguments[0]; + memberExpr = call.Arguments[1]; + } + else + { + return false; + } + + var unwrappedMember = UnwrapConvert(memberExpr); + if (unwrappedMember is not MemberExpression member) + return false; + + var extractedPath = ExtractMemberPath(member, parameter); + if (extractedPath == null) + return false; + propertyPath = extractedPath; + + if (collectionExpr is MethodCallExpression { Method.Name: "op_Implicit", Object: null } implicitCall && + implicitCall.Arguments.Count == 1) + { + collectionExpr = implicitCall.Arguments[0]; + } + + object? enumerableObj; + try + { + enumerableObj = EvaluateExpression(collectionExpr); + } + catch + { + return false; + } + + if (enumerableObj is not System.Collections.IEnumerable enumerable || enumerableObj is string) + return false; + + var list = new List(); + var seen = new HashSet(s_inValueComparer); + foreach (var raw in enumerable) + { + var converted = TryApplyConverter(propertyPath, raw, registry); + if (!IsIndexableValue(converted)) + return false; + var normalized = converted ?? DBNull.Value; + if (seen.Add(normalized)) + { + list.Add(normalized); + } + } + + values = list; + return true; + } + + private static readonly IEqualityComparer s_inValueComparer = new InValueComparer(); + + private sealed class InValueComparer : IEqualityComparer + { + bool IEqualityComparer.Equals(object? x, object? y) + { + if (ReferenceEquals(x, y)) return true; + if (x is null || y is null) return false; + if (x is byte[] xb && y is byte[] yb) return xb.AsSpan().SequenceEqual(yb); + return x.Equals(y); + } + + int IEqualityComparer.GetHashCode(object? obj) + { + if (obj is null) return 0; + if (obj is byte[] bytes) + { + var hc = new HashCode(); + foreach (var b in bytes) hc.Add(b); + return hc.ToHashCode(); + } + return obj.GetHashCode(); + } + } private static object? TryApplyConverter(string propertyPath, object? value, ValueConverterRegistry? registry) { @@ -814,12 +816,12 @@ private sealed class InValueComparer : IEqualityComparer return registry.TryConvert(topProp, value, out var pv) ? pv : value; } - private static readonly HashSet _knownBsonPrimitives = - [ - typeof(int), typeof(long), typeof(double), typeof(decimal), - typeof(bool), typeof(string), typeof(DateTime), typeof(DateTimeOffset), - typeof(ObjectId), typeof(Guid), typeof(byte[]) - ]; + private static readonly HashSet _knownBsonPrimitives = + [ + typeof(int), typeof(long), typeof(double), typeof(decimal), + typeof(bool), typeof(string), typeof(DateTime), typeof(DateTimeOffset), + typeof(ObjectId), typeof(Guid), typeof(byte[]) + ]; /// /// Returns true when can be used directly as an index diff --git a/tests/BLite.Tests/NearWithinCorrectnessTests.cs b/tests/BLite.Tests/NearWithinCorrectnessTests.cs index f8fe064f..022f5366 100644 --- a/tests/BLite.Tests/NearWithinCorrectnessTests.cs +++ b/tests/BLite.Tests/NearWithinCorrectnessTests.cs @@ -150,6 +150,55 @@ public async Task VectorSearch_InsideOr_Throws() .ToList()); } + [Fact] + public async Task VectorSearch_FollowedByGroupByOrAggregate_UsesTheIndexResult() + { + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] }); + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [1f, 1f, 1f] }); + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "B", Embedding = [50f, 50f, 50f] }); + float[] query = [0f, 0f, 0f]; + + var groups = _db.VectorItems.AsQueryable() + .Where(x => x.Embedding.VectorSearch(query, 2)) + .GroupBy(x => x.Title) + .Select(g => new { g.Key, Count = g.Count() }) + .ToList(); + var sum = _db.VectorItems.AsQueryable() + .Where(x => x.Embedding.VectorSearch(query, 2)) + .Sum(x => (int)x.Embedding[0]); + + Assert.Equal("A", Assert.Single(groups).Key); + Assert.Equal(2, groups[0].Count); + Assert.Equal(1, sum); + } + + [Fact] + public async Task NearOrNear_OnTheSameIndex_ReturnsDocumentsNearEither() + { + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "Milan", Location = (45.47, 9.19) }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "New York", Location = (40.72, -74.0) }); + await _db.GeoItems.InsertAsync(new GeoEntity { Name = "Tokyo", Location = (35.68, 139.69) }); + + var results = _db.GeoItems.AsQueryable() + .Where(p => p.Location.Near(Milan, 10.0) || p.Location.Near(NewYork, 10.0)) + .ToList() + .Select(p => p.Name).OrderBy(n => n).ToList(); + + Assert.Equal(new[] { "Milan", "New York" }, results); + } + + [Fact] + public async Task TwoVectorSearches_ExplainThatOnlyOneIsSupported() + { + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] }); + float[] q1 = [0f, 0f, 0f], q2 = [1f, 1f, 1f]; + + var ex = Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Embedding.VectorSearch(q1, 1) && x.Embedding.VectorSearch(q2, 1)) + .ToList()); + Assert.Contains("only one VectorSearch", ex.Message); + } + // ── DynamicCollection ─────────────────────────────────────────────────── [Fact] From db38b781c5f3c37b4b1d747f68ef527fa7158d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 12 Oct 2026 17:06:19 +0200 Subject: [PATCH 3/3] fix(query): only skip VectorSearch in the Where that FetchAsync applied (#165) - The in-memory replay drops the root-chain Where calls folded into the WHERE that FetchAsync applied, as long as only filtering, ordering and paging operators lie between them and the root. It no longer replaces every VectorSearch in the expression with true, so a VectorSearch in a projection, group, aggregate selector or Count predicate throws again. - Regression tests for the five probes from the review, and for a VectorSearch Where followed by Select and OrderBy. Co-Authored-By: Claude Opus 5.5 --- src/BLite.Core/Query/BTreeQueryProvider.cs | 54 +++++++++--- .../BLite.Tests/NearWithinCorrectnessTests.cs | 83 +++++++++++++++++++ 2 files changed, 127 insertions(+), 10 deletions(-) diff --git a/src/BLite.Core/Query/BTreeQueryProvider.cs b/src/BLite.Core/Query/BTreeQueryProvider.cs index 39d3b77b..32965ad9 100644 --- a/src/BLite.Core/Query/BTreeQueryProvider.cs +++ b/src/BLite.Core/Query/BTreeQueryProvider.cs @@ -311,10 +311,11 @@ private async Task ExecuteAsyncCore(Expression expression, Can // pipeline cannot model. We still benefit from index / BSON-scan fetch, // then let EnumerableRewriter translate the remaining Queryable calls to // Enumerable equivalents and compile the residual expression once. - // FetchAsync applied the WHERE. A VectorSearch term in it was served by the index (otherwise - // the fetch would have thrown evaluating it), so the in-memory replay treats it as satisfied. + // FetchAsync applied the WHERE, so the replay drops the root-chain Where calls folded into + // it. Re-evaluating them would change nothing except for VectorSearch, which only the index + // can evaluate. Every other VectorSearch in the expression is still evaluated and throws. if (whereAlreadyApplied) - expression = VectorSearchSatisfiedRewriter.Instance.Visit(expression); + expression = DropAppliedWheres(expression).Expression; if (model.HasComplexOperators) return ExecuteViaEnumerableRewriter(expression, sourceData); @@ -641,17 +642,50 @@ private static bool AggregateValues( // ─── Expression-tree helpers ────────────────────────────────────────────── - /// Replaces every VectorSearch call with true. - private sealed class VectorSearchSatisfiedRewriter : ExpressionVisitor + /// + /// Removes the Queryable.Where(x => ...) calls on the root chain whose input is the + /// fetched documents or a subset of them (only filtering, ordering and paging operators lie + /// between the root and the Where). folds each of these + /// into , which FetchAsync applied, so every element + /// reaching them already satisfies their predicate and dropping them leaves the result unchanged. + /// + /// + /// The rewritten expression, and whether its elements are still a subset of the fetched documents. + /// + private static (Expression Expression, bool IsFetchedSubset) DropAppliedWheres(Expression expression) { - public static readonly VectorSearchSatisfiedRewriter Instance = new(); + if (expression is ConstantExpression { Value: IQueryable }) + return (expression, true); - protected override Expression VisitMethodCall(MethodCallExpression node) - => node.Method.DeclaringType == typeof(VectorSearchExtensions) - ? Expression.Constant(true) - : base.VisitMethodCall(node); + if (expression is not MethodCallExpression call + || call.Method.DeclaringType != typeof(Queryable) + || call.Arguments.Count == 0) + return (expression, false); + + var (source, isFetchedSubset) = DropAppliedWheres(call.Arguments[0]); + + if (isFetchedSubset + && call.Method.Name == nameof(Queryable.Where) + && call.Arguments[1] is UnaryExpression { Operand: LambdaExpression { Parameters.Count: 1 } }) + return (source, true); + + var rewritten = source == call.Arguments[0] + ? call + : call.Update(call.Object, [source, .. call.Arguments.Skip(1)]); + return (rewritten, isFetchedSubset && s_subsetPreservingOperators.Contains(call.Method.Name)); } + /// Queryable operators whose output elements are a subset of their source's. + private static readonly HashSet s_subsetPreservingOperators = + [ + nameof(Queryable.Where), + nameof(Queryable.OrderBy), nameof(Queryable.OrderByDescending), + nameof(Queryable.ThenBy), nameof(Queryable.ThenByDescending), + nameof(Queryable.Skip), nameof(Queryable.Take), + nameof(Queryable.SkipWhile), nameof(Queryable.TakeWhile), + nameof(Queryable.Distinct), nameof(Queryable.Reverse), + ]; + /// /// Fallback for queries containing operators the direct pipeline cannot handle /// (GroupBy, Join, Sum/Average/Min/Max with selectors, etc.). diff --git a/tests/BLite.Tests/NearWithinCorrectnessTests.cs b/tests/BLite.Tests/NearWithinCorrectnessTests.cs index 022f5366..9fc5ae69 100644 --- a/tests/BLite.Tests/NearWithinCorrectnessTests.cs +++ b/tests/BLite.Tests/NearWithinCorrectnessTests.cs @@ -199,6 +199,89 @@ public async Task TwoVectorSearches_ExplainThatOnlyOneIsSupported() Assert.Contains("only one VectorSearch", ex.Message); } + // Only the VectorSearch in a Where that FetchAsync applied is served by the index. One in a + // projection, a group or an aggregate after the fetch must still throw, not match everything. + + private static readonly float[] Q = [0f, 0f, 0f]; + + private async Task InsertVectorProbeDataAsync() + { + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [0f, 0f, 0f] }); + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "A", Embedding = [1f, 1f, 1f] }); + await _db.VectorItems.InsertAsync(new VectorEntity { Title = "B", Embedding = [50f, 50f, 50f] }); + } + + [Fact] + public async Task VectorSearch_InWhereThenProjectionOrderedAfterSelect_UsesTheIndexResult() + { + await InsertVectorProbeDataAsync(); + + var titles = _db.VectorItems.AsQueryable() + .Where(x => x.Embedding.VectorSearch(Q, 2)) + .Select(x => new { x.Title }) + .OrderBy(a => a.Title) + .ToList(); + + Assert.Equal(new[] { "A", "A" }, titles.Select(a => a.Title)); + } + + [Fact] + public async Task VectorSearch_InProjection_AfterIndexedVectorWhere_Throws() + { + await InsertVectorProbeDataAsync(); + float[] q2 = [1f, 1f, 1f]; + + Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Embedding.VectorSearch(Q, 2)) + .Select(x => new { x.Title, ok = x.Embedding.VectorSearch(q2, 1) }) + .OrderBy(a => a.Title) + .ToList()); + } + + [Fact] + public async Task VectorSearch_InProjection_AfterOtherWhere_Throws() + { + await InsertVectorProbeDataAsync(); + + Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Title != null) + .Select(x => new { x.Title, ok = x.Embedding.VectorSearch(Q, 1) }) + .OrderBy(a => a.Title) + .ToList()); + } + + [Fact] + public async Task VectorSearch_InGroupCount_AfterOtherWhere_Throws() + { + await InsertVectorProbeDataAsync(); + + Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Title != null) + .GroupBy(x => x.Title) + .Select(g => new { g.Key, Count = g.Count(x => x.Embedding.VectorSearch(Q, 1)) }) + .ToList()); + } + + [Fact] + public async Task VectorSearch_InSumSelector_AfterOtherWhere_Throws() + { + await InsertVectorProbeDataAsync(); + + Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Title != null) + .Sum(x => x.Embedding.VectorSearch(Q, 1) ? 1 : 0)); + } + + [Fact] + public async Task VectorSearch_InCountPredicate_AfterOtherWhere_Throws() + { + await InsertVectorProbeDataAsync(); + + Assert.Throws(() => _db.VectorItems.AsQueryable() + .Where(x => x.Title != null) + .Count(x => x.Embedding.VectorSearch(Q, 1))); + } + // ── DynamicCollection ─────────────────────────────────────────────────── [Fact]