diff --git a/fit/Couchbase.FitPerformer/PerformerService.cs b/fit/Couchbase.FitPerformer/PerformerService.cs index d6a813baa..f2cefcbe7 100644 --- a/fit/Couchbase.FitPerformer/PerformerService.cs +++ b/fit/Couchbase.FitPerformer/PerformerService.cs @@ -180,6 +180,7 @@ public override Task performerCapsFetch(PerformerCap response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkBucketSettingsNumVbuckets); response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkZoneAwareReadFromReplica); response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkGetReplica); + response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkSearchCustomScriptQueries); response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkPrefilterVectorSearch); response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkCouchbase2Observability); response.SdkImplementationCaps.Add(Grpc.Protocol.Sdk.Caps.SdkObservabilityRfcRev24); diff --git a/fit/Couchbase.FitPerformer/Workload/SearchHelper.cs b/fit/Couchbase.FitPerformer/Workload/SearchHelper.cs index 5ba4cdf40..4ee183391 100644 --- a/fit/Couchbase.FitPerformer/Workload/SearchHelper.cs +++ b/fit/Couchbase.FitPerformer/Workload/SearchHelper.cs @@ -20,6 +20,8 @@ using BooleanFieldQuery = Couchbase.Search.Queries.Simple.BooleanFieldQuery; using BooleanQuery = Couchbase.Search.Queries.Compound.BooleanQuery; using ConjunctionQuery = Couchbase.Search.Queries.Compound.ConjunctionQuery; +using CustomFilterQuery = Couchbase.Search.Queries.Script.CustomFilterQuery; +using CustomScoreQuery = Couchbase.Search.Queries.Script.CustomScoreQuery; using DateRangeQuery = Couchbase.Search.Queries.Range.DateRangeQuery; using DisjunctionQuery = Couchbase.Search.Queries.Compound.DisjunctionQuery; using DocIdQuery = Couchbase.Search.Queries.Simple.DocIdQuery; @@ -302,8 +304,8 @@ private static Couchbase.Search.ISearchQuery ProtoSearchQueryToCore(Couchbase.Gr { //--------Initialise BooleanQuery-------- var coreQuery = new BooleanQuery(); - if (protoQuery.Boolean.HasBoost) coreQuery.Boost(protoQuery.Boolean.Boost); - if (protoQuery.Boolean.HasShouldMin) coreQuery.ShouldMin((int)protoQuery.Boolean.ShouldMin); + if (innerProtoQuery.Boolean.HasBoost) coreQuery.Boost(innerProtoQuery.Boolean.Boost); + if (innerProtoQuery.Boolean.HasShouldMin) coreQuery.ShouldMin((int)innerProtoQuery.Boolean.ShouldMin); //--------Recurse to add converted queries to the above compound queries-------- if (innerProtoQuery.Boolean.Must.Count > 0) @@ -461,6 +463,26 @@ private static Couchbase.Search.ISearchQuery ProtoSearchQueryToCore(Couchbase.Gr { return new MatchNoneQuery(); } + case SearchQuery.QueryOneofCase.CustomScore: + { + var proto = innerProtoQuery.CustomScore; + // A null inner query would fall back to the root and recurse forever. + if (proto.Query is null) throw new UnsupportedException("CustomScoreQuery has no inner query"); + var coreQuery = new CustomScoreQuery(ProtoSearchQueryToCore(protoQuery, proto.Query), proto.Source); + if (proto.Fields.Count > 0) coreQuery.Fields(proto.Fields.ToArray()); + if (proto.Parameters is not null) coreQuery.Parameters(StructToDictionary(proto.Parameters)); + return coreQuery; + } + case SearchQuery.QueryOneofCase.CustomFilter: + { + var proto = innerProtoQuery.CustomFilter; + // A null inner query would fall back to the root and recurse forever. + if (proto.Query is null) throw new UnsupportedException("CustomFilterQuery has no inner query"); + var coreQuery = new CustomFilterQuery(ProtoSearchQueryToCore(protoQuery, proto.Query), proto.Source); + if (proto.Fields.Count > 0) coreQuery.Fields(proto.Fields.ToArray()); + if (proto.Parameters is not null) coreQuery.Parameters(StructToDictionary(proto.Parameters)); + return coreQuery; + } case SearchQuery.QueryOneofCase.MatchPhrase: { var coreQuery = new MatchPhraseQuery(innerProtoQuery.MatchPhrase.MatchPhrase); @@ -549,4 +571,18 @@ private static Couchbase.Search.ISearchQuery ProtoSearchQueryToCore(Couchbase.Gr throw new UnsupportedException($"Could not determine Search Query type: {protoQuery}"); } + + private static Dictionary StructToDictionary(Struct protoStruct) => + protoStruct.Fields.ToDictionary(field => field.Key, field => ProtoValueToObject(field.Value)); + + private static object? ProtoValueToObject(Value value) => value.KindCase switch + { + Value.KindOneofCase.NullValue => null, + Value.KindOneofCase.NumberValue => value.NumberValue, + Value.KindOneofCase.StringValue => value.StringValue, + Value.KindOneofCase.BoolValue => value.BoolValue, + Value.KindOneofCase.StructValue => StructToDictionary(value.StructValue), + Value.KindOneofCase.ListValue => value.ListValue.Values.Select(ProtoValueToObject).ToList(), + _ => throw new UnsupportedException($"Unsupported protobuf Value kind: {value.KindCase}") + }; } \ No newline at end of file diff --git a/fit/Couchbase.FitPerformer/gRPC/sdk.caps.proto b/fit/Couchbase.FitPerformer/gRPC/sdk.caps.proto index b3b10ccf0..495590d86 100644 --- a/fit/Couchbase.FitPerformer/gRPC/sdk.caps.proto +++ b/fit/Couchbase.FitPerformer/gRPC/sdk.caps.proto @@ -162,4 +162,18 @@ enum Caps { // The SDK has support for the Collection.GetReplica API. This includes the new DocumentNotFoundOnReplicaException, // ReplicaIndexOutOfBoundsException and ReplicaIndexCurrentlyUnavailableException exceptions. SDK_GET_REPLICA = 43; + + // The performer supports the ScopeEventingFunctionManager API. + // This is everything under sdk.scope.eventing_function_manager + SDK_SCOPE_EVENTING_FUNCTION_MANAGER = 44; + + // The SDK raises InvalidArgumentException for an HTTP 400 from any HTTP service when no more specific error applies + // (SDK-RFC 58 revision 22). + SDK_HTTP_400_INVALID_ARGUMENT = 45; + + // The SDK has support for the eventing-specific error types defined in SDK-RFC 62 (see shared.exceptions.proto). + SDK_EVENTING_ERRORS = 46; + + // The SDK has support for the CustomScoreQuery and CustomFilterQuery SearchQuery types + SDK_SEARCH_CUSTOM_SCRIPT_QUERIES = 47; } diff --git a/fit/Couchbase.FitPerformer/gRPC/sdk.search.proto b/fit/Couchbase.FitPerformer/gRPC/sdk.search.proto index ba95a1f93..717b7132c 100644 --- a/fit/Couchbase.FitPerformer/gRPC/sdk.search.proto +++ b/fit/Couchbase.FitPerformer/gRPC/sdk.search.proto @@ -7,9 +7,11 @@ option go_package = "github.com/couchbaselabs/transactions-fit-performer/protoco option java_multiple_files = true; option ruby_package = "FIT::Protocol::SDK::Search"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/struct.proto"; + import "shared.content.proto"; import "shared.basic.proto"; -import "google/protobuf/timestamp.proto"; import "streams.top_level.proto"; import "shared.transcoders.proto"; @@ -484,6 +486,20 @@ message MatchAllQuery { message MatchNoneQuery { } +message CustomScoreQuery { + SearchQuery query = 1; + string source = 2; + repeated string fields = 3; + optional google.protobuf.Struct parameters = 4; +} + +message CustomFilterQuery { + SearchQuery query = 1; + string source = 2; + repeated string fields = 3; + optional google.protobuf.Struct parameters = 4; +} + message SearchQuery { oneof query { MatchQuery match = 1; @@ -507,5 +523,9 @@ message SearchQuery { PhraseQuery phrase = 17; MatchAllQuery match_all = 18; MatchNoneQuery match_none = 19; + + // Support declared using SDK_SEARCH_CUSTOM_SCRIPT_QUERIES SDK cap. + CustomScoreQuery custom_score = 21; + CustomFilterQuery custom_filter = 22; } } diff --git a/src/Couchbase/Cluster.cs b/src/Couchbase/Cluster.cs index b059578fa..41766804c 100644 --- a/src/Couchbase/Cluster.cs +++ b/src/Couchbase/Cluster.cs @@ -26,6 +26,7 @@ using Couchbase.Management.Users; using Couchbase.Query; using Couchbase.Search; +using Couchbase.Search.Queries.Script; using Microsoft.Extensions.Logging; using AnalyticsOptions = Couchbase.Analytics.AnalyticsOptions; using Couchbase.Core.RateLimiting; @@ -51,6 +52,8 @@ public partial class Cluster : ICluster, IBootstrappable private readonly object _syncObject = new object(); private bool _disposed; private readonly ClusterContext _context; + // Test seam: lets unit tests supply the GlobalConfig that capability checks read. + internal ClusterContext Context => _context; private bool _hasBootStrapped = false; // Warn once on entering the failed state, not on every bootstrapper poll (2.5s by @@ -589,6 +592,12 @@ public async Task SearchAsync( "Score fusion is not available on this version of Couchbase Server"); } + if (searchRequest.ContainsCustomScriptQuery()) + { + _context.GlobalConfig?.AssertClusterCap(ClusterCapabilities.SEARCH_UDF_QUERY, + "Custom score or filter queries are not available on this version of Couchbase Server"); + } + FtsSearchRequest ftsSearchRequest = new() { Index = indexName, diff --git a/src/Couchbase/Core/Configuration/Server/BucketConfig.cs b/src/Couchbase/Core/Configuration/Server/BucketConfig.cs index c2dda684d..478077eb8 100644 --- a/src/Couchbase/Core/Configuration/Server/BucketConfig.cs +++ b/src/Couchbase/Core/Configuration/Server/BucketConfig.cs @@ -599,6 +599,7 @@ internal class ClusterCapabilities public const string SCOPED_SEARCH_INDEX = "search.scopedSearchIndex"; public const string VECTOR_SEARCH = "search.vectorSearch"; public const string SCORE_FUSION = "search.scoreFusion"; + public const string SEARCH_UDF_QUERY = "search.udfQuery"; [JsonPropertyName("clusterCapabilitiesVer")] public IEnumerable Version { get; set; } [JsonPropertyName("clusterCapabilities")] public Dictionary> Capabilities { get; set; } diff --git a/src/Couchbase/Search/Queries/Script/CustomFilterQuery.cs b/src/Couchbase/Search/Queries/Script/CustomFilterQuery.cs new file mode 100644 index 000000000..a3b9b215a --- /dev/null +++ b/src/Couchbase/Search/Queries/Script/CustomFilterQuery.cs @@ -0,0 +1,128 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Couchbase.Core.Compatibility; +using Newtonsoft.Json.Linq; + +namespace Couchbase.Search.Queries.Script +{ + /// + /// A query that passes the results of an inner query through a user-defined JavaScript filtering function. + /// Only the results for which the function returns true are kept. Their scores are not changed. + /// + /// + /// + /// Requires Couchbase Server 8.5.0 or later, Enterprise Edition. Custom script queries are disabled on the + /// Search service by default; enable them by setting customScriptQueriesEnabled to "true" with + /// PUT /api/managerOptions on a Search node. The SDK cannot detect this setting, so while it is disabled + /// the server rejects the query. Not available over couchbase2://. + /// + /// + /// The function is called once for each result, with doc and params: + /// + /// doc._id: the document ID. + /// doc.score: the score from the inner query. + /// doc.fields: only the fields requested with ; undefined if + /// none are requested. See for how text values appear. + /// params: the object given to , the same for every + /// result; null if none is given. + /// + /// An exception in the function fails the whole query. + /// + /// + [InterfaceStability(Level.Uncommitted)] + public sealed class CustomFilterQuery : ISearchQuery + { + private readonly CustomScriptQuery _query; + + /// + /// Creates a . + /// + /// The inner query whose results are passed through the filtering function. + /// + /// A JavaScript function that takes (doc, params) and returns a boolean, for example + /// function filter(doc, params) { return doc.score >= 0.5; }. + /// + public CustomFilterQuery(ISearchQuery query, string source) + { + _query = new CustomScriptQuery(query, source); + } + + /// + /// The document fields made available to the filtering function, as doc.fields. + /// The function sees no fields unless this is set. Pass "*" for all fields. + /// + /// + /// The values come from the Search index, not from the stored document, so a field must be indexed to be + /// available. Text values are the terms from the field's analyzer, not the original text. Use the + /// keyword analyzer to get the exact text. + /// + /// This for chaining. + public CustomFilterQuery Fields(params string[] fields) + { + _query.SetFields(fields); + return this; + } + + /// + /// Parameters made available to the filtering function, as params. + /// Values may be any JSON-serializable value, including nested dictionaries and lists. + /// The same parameters are given to the function for every result. + /// + /// + /// The values are serialized with Newtonsoft.Json, not with the cluster's serializer. Use plain .NET values; + /// System.Text.Json types such as JsonElement are not serialized correctly. + /// + /// This for chaining. + public CustomFilterQuery Parameters(IDictionary parameters) + { + _query.SetParameters(parameters); + return this; + } + + /// + /// Has no effect. The Search service ignores a boost on this query type, so apply the boost to the inner + /// query instead. + /// + /// This , unchanged. + [Obsolete("CustomFilterQuery does not support boost. Apply the boost to the inner query instead.")] + public ISearchQuery Boost(double boost) => this; + + [RequiresUnreferencedCode(SearchClient.SearchRequiresUnreferencedMembersWarning)] + [RequiresDynamicCode(SearchClient.SearchRequiresDynamicCodeWarning)] + public JObject Export() => _query.Export("custom_filter"); + + public ReadOnly AsReadOnly() => + new(_query.Query, _query.Source, _query.Fields, _query.Parameters); + + public record ReadOnly( + ISearchQuery Query, + string Source, + IReadOnlyList? Fields, + IReadOnlyDictionary? Parameters); + } +} + +#region [ License information ] + +/* ************************************************************ + * + * @author Couchbase + * @copyright 2026 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ************************************************************/ + +#endregion diff --git a/src/Couchbase/Search/Queries/Script/CustomScoreQuery.cs b/src/Couchbase/Search/Queries/Script/CustomScoreQuery.cs new file mode 100644 index 000000000..95b221c51 --- /dev/null +++ b/src/Couchbase/Search/Queries/Script/CustomScoreQuery.cs @@ -0,0 +1,128 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Couchbase.Core.Compatibility; +using Newtonsoft.Json.Linq; + +namespace Couchbase.Search.Queries.Script +{ + /// + /// A query that passes the results of an inner query through a user-defined JavaScript scoring function. + /// Each result is given the score that the function returns. + /// + /// + /// + /// Requires Couchbase Server 8.5.0 or later, Enterprise Edition. Custom script queries are disabled on the + /// Search service by default; enable them by setting customScriptQueriesEnabled to "true" with + /// PUT /api/managerOptions on a Search node. The SDK cannot detect this setting, so while it is disabled + /// the server rejects the query. Not available over couchbase2://. + /// + /// + /// The function is called once for each result, with doc and params: + /// + /// doc._id: the document ID. + /// doc.score: the score from the inner query. + /// doc.fields: only the fields requested with ; undefined if + /// none are requested. See for how text values appear. + /// params: the object given to , the same for every + /// result; null if none is given. + /// + /// An exception in the function fails the whole query. + /// + /// + [InterfaceStability(Level.Uncommitted)] + public sealed class CustomScoreQuery : ISearchQuery + { + private readonly CustomScriptQuery _query; + + /// + /// Creates a . + /// + /// The inner query whose results are passed through the scoring function. + /// + /// A JavaScript function that takes (doc, params) and returns a number, for example + /// function score(doc, params) { return doc.score * 2; }. + /// + public CustomScoreQuery(ISearchQuery query, string source) + { + _query = new CustomScriptQuery(query, source); + } + + /// + /// The document fields made available to the scoring function, as doc.fields. + /// The function sees no fields unless this is set. Pass "*" for all fields. + /// + /// + /// The values come from the Search index, not from the stored document, so a field must be indexed to be + /// available. Text values are the terms from the field's analyzer, not the original text. Use the + /// keyword analyzer to get the exact text. + /// + /// This for chaining. + public CustomScoreQuery Fields(params string[] fields) + { + _query.SetFields(fields); + return this; + } + + /// + /// Parameters made available to the scoring function, as params. + /// Values may be any JSON-serializable value, including nested dictionaries and lists. + /// The same parameters are given to the function for every result. + /// + /// + /// The values are serialized with Newtonsoft.Json, not with the cluster's serializer. Use plain .NET values; + /// System.Text.Json types such as JsonElement are not serialized correctly. + /// + /// This for chaining. + public CustomScoreQuery Parameters(IDictionary parameters) + { + _query.SetParameters(parameters); + return this; + } + + /// + /// Has no effect. The Search service ignores a boost on this query type, so apply the boost to the inner + /// query instead. + /// + /// This , unchanged. + [Obsolete("CustomScoreQuery does not support boost. Apply the boost to the inner query instead.")] + public ISearchQuery Boost(double boost) => this; + + [RequiresUnreferencedCode(SearchClient.SearchRequiresUnreferencedMembersWarning)] + [RequiresDynamicCode(SearchClient.SearchRequiresDynamicCodeWarning)] + public JObject Export() => _query.Export("custom_score"); + + public ReadOnly AsReadOnly() => + new(_query.Query, _query.Source, _query.Fields, _query.Parameters); + + public record ReadOnly( + ISearchQuery Query, + string Source, + IReadOnlyList? Fields, + IReadOnlyDictionary? Parameters); + } +} + +#region [ License information ] + +/* ************************************************************ + * + * @author Couchbase + * @copyright 2026 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ************************************************************/ + +#endregion diff --git a/src/Couchbase/Search/Queries/Script/CustomScriptQuery.cs b/src/Couchbase/Search/Queries/Script/CustomScriptQuery.cs new file mode 100644 index 000000000..5e8551538 --- /dev/null +++ b/src/Couchbase/Search/Queries/Script/CustomScriptQuery.cs @@ -0,0 +1,103 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Newtonsoft.Json.Linq; + +namespace Couchbase.Search.Queries.Script +{ + /// + /// Shared state and JSON export for and , + /// which differ only in their top-level JSON key. + /// + internal sealed class CustomScriptQuery + { + public CustomScriptQuery(ISearchQuery query, string source) + { + Query = query ?? throw new ArgumentNullException(nameof(query)); + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + if (string.IsNullOrWhiteSpace(source)) + { + throw new ArgumentException("The source must not be empty or whitespace.", nameof(source)); + } + Source = source; + } + + public ISearchQuery Query { get; } + + public string Source { get; } + + public IReadOnlyList? Fields { get; private set; } + + public IReadOnlyDictionary? Parameters { get; private set; } + + // Both setters copy, so changing the caller's collection afterwards does not change the query. + public void SetFields(string[] fields) + { + if (fields is null) + { + throw new ArgumentNullException(nameof(fields)); + } + Fields = fields.ToArray(); + } + + public void SetParameters(IDictionary parameters) + { + if (parameters is null) + { + throw new ArgumentNullException(nameof(parameters)); + } + Parameters = new Dictionary(parameters); + } + + [RequiresUnreferencedCode(SearchClient.SearchRequiresUnreferencedMembersWarning)] + [RequiresDynamicCode(SearchClient.SearchRequiresDynamicCodeWarning)] + public JObject Export(string queryType) + { + var body = new JObject + { + { "query", Query.Export() }, + { "source", Source } + }; + + // The RFC says to omit both when not set, so an empty value is treated as not set. + if (Fields is { Count: > 0 }) + { + body.Add("fields", new JArray(Fields)); + } + if (Parameters is { Count: > 0 }) + { + body.Add("params", JObject.FromObject(Parameters)); + } + + return new JObject { { queryType, body } }; + } + } +} + +#region [ License information ] + +/* ************************************************************ + * + * @author Couchbase + * @copyright 2026 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ************************************************************/ + +#endregion diff --git a/src/Couchbase/Search/Queries/Script/CustomScriptQueryDetection.cs b/src/Couchbase/Search/Queries/Script/CustomScriptQueryDetection.cs new file mode 100644 index 000000000..9e0014dc9 --- /dev/null +++ b/src/Couchbase/Search/Queries/Script/CustomScriptQueryDetection.cs @@ -0,0 +1,54 @@ +#nullable enable +using System.Linq; +using Couchbase.Search.Queries.Compound; + +namespace Couchbase.Search.Queries.Script +{ + /// + /// Finds and anywhere in a search request, so the + /// SDK can check for the cluster capability before it sends anything (SDK-RFC 52). + /// + internal static class CustomScriptQueryDetection + { + // A user's own ISearchQuery implementation is opaque, so only the SDK's compound types are searched. + internal static bool ContainsCustomScriptQuery(this ISearchQuery? query) => query switch + { + CustomScoreQuery or CustomFilterQuery => true, + ConjunctionQuery conjunction => conjunction.Any(ContainsCustomScriptQuery), + DisjunctionQuery disjunction => disjunction.Any(ContainsCustomScriptQuery), + BooleanQuery boolean => boolean.AsReadOnly() is var parts + && (parts.MustQueries.ContainsCustomScriptQuery() + || parts.ShouldQueries.ContainsCustomScriptQuery() + || parts.MustNotQueries.ContainsCustomScriptQuery()), + _ => false + }; + + // Includes vector query prefilters, which are tested by FIT even though the RFC names only compound queries. + internal static bool ContainsCustomScriptQuery(this SearchRequest request) => + request.SearchQuery.ContainsCustomScriptQuery() + || request.VectorSearch?.VectorQueries?.Any(vectorQuery => vectorQuery.Filter.ContainsCustomScriptQuery()) == true; + } +} + +#region [ License information ] + +/* ************************************************************ + * + * @author Couchbase + * @copyright 2026 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ************************************************************/ + +#endregion diff --git a/src/Couchbase/Stellar/Search/StellarSearchClient.cs b/src/Couchbase/Stellar/Search/StellarSearchClient.cs index bede0e83c..882bd81d6 100644 --- a/src/Couchbase/Stellar/Search/StellarSearchClient.cs +++ b/src/Couchbase/Stellar/Search/StellarSearchClient.cs @@ -12,6 +12,7 @@ using Couchbase.KeyValue; using Couchbase.Protostellar.Search.V1; using Couchbase.Search; +using Couchbase.Search.Queries.Script; using Couchbase.Search.Queries.Vector; using Couchbase.Stellar.Core; using Couchbase.Stellar.Core.Retry; @@ -58,6 +59,12 @@ public async Task QueryAsync(string indexName, ISearchQuery query throw new FeatureNotAvailableException("Score fusion is not available over couchbase2."); } + // The protocol has no custom score or filter query types. QueryConverter would also reject them, but this error names the feature. + if (query.ContainsCustomScriptQuery()) + { + throw new FeatureNotAvailableException("Custom score or filter queries are not available over couchbase2."); + } + var searchQuery = QueryConverter(query); var searchQueryRequest = new Couchbase.Protostellar.Search.V1.SearchQueryRequest @@ -429,6 +436,9 @@ private Couchbase.Protostellar.Search.V1.Query QueryConverter(ISearchQuery searc protoQuery.MatchQuery.Operator = coreQuery.MatchOperator.Value.ToProto(); } break; + default: + throw new FeatureNotAvailableException( + $"{couchbaseQuery.GetType().Name} is not supported over couchbase2."); } return protoQuery; diff --git a/tests/Couchbase.UnitTests/Core/Configuration/Server/BucketConfigTests.cs b/tests/Couchbase.UnitTests/Core/Configuration/Server/BucketConfigTests.cs index 07988cac7..d2d7551a9 100644 --- a/tests/Couchbase.UnitTests/Core/Configuration/Server/BucketConfigTests.cs +++ b/tests/Couchbase.UnitTests/Core/Configuration/Server/BucketConfigTests.cs @@ -690,6 +690,7 @@ public void Test_BucketConfig_Without_Product() [Theory] [InlineData(ClusterCapabilities.SCORE_FUSION, true)] [InlineData(ClusterCapabilities.VECTOR_SEARCH, true)] + [InlineData(ClusterCapabilities.SEARCH_UDF_QUERY, true)] [InlineData("search.somethingElse", false)] public void Test_HasClusterCap(string capability, bool expected) { @@ -697,7 +698,7 @@ public void Test_HasClusterCap(string capability, bool expected) { ClusterCapabilities = new Dictionary> { - ["search"] = new[] { "vectorSearch", "scopedSearchIndex", "scoreFusion" } + ["search"] = new[] { "vectorSearch", "scopedSearchIndex", "scoreFusion", "udfQuery" } } }; diff --git a/tests/Couchbase.UnitTests/Search/CustomScriptQueryCapabilityTests.cs b/tests/Couchbase.UnitTests/Search/CustomScriptQueryCapabilityTests.cs new file mode 100644 index 000000000..c19b93622 --- /dev/null +++ b/tests/Couchbase.UnitTests/Search/CustomScriptQueryCapabilityTests.cs @@ -0,0 +1,133 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Couchbase.Core.Configuration.Server; +using Couchbase.Core.Exceptions; +using Couchbase.Core.Retry; +using Couchbase.Core.Retry.Search; +using Couchbase.KeyValue; +using Couchbase.Search; +using Couchbase.Search.Queries.Compound; +using Couchbase.Search.Queries.Script; +using Couchbase.Search.Queries.Simple; +using Couchbase.Search.Queries.Vector; +using Couchbase.Test.Common.Utils; +using Moq; +using Xunit; + +namespace Couchbase.UnitTests.Search +{ + public class CustomScriptQueryCapabilityTests + { + private const string Source = "function f(doc, params) { return true; }"; + + private static ISearchQuery Score() => new CustomScoreQuery(new MatchAllQuery(), Source); + private static ISearchQuery Filter() => new CustomFilterQuery(new MatchAllQuery(), Source); + private static ISearchQuery Match() => new MatchQuery("michael"); + + private static SearchRequest Prefilter(ISearchQuery filter) => SearchRequest.Create(VectorSearch.Create( + new VectorQuery("vector_field", new VectorQueryOptions(Filter: filter)).WithVector(new[] { 0.1f }))); + + public static TheoryData RequestsWithCustomScriptQuery() => new() + { + { "score", SearchRequest.Create(Score()) }, + { "filter", SearchRequest.Create(Filter()) }, + { "conjunction", SearchRequest.Create(new ConjunctionQuery(Match(), Score())) }, + { "disjunction", SearchRequest.Create(new DisjunctionQuery(Match(), Filter())) }, + { "boolean must", SearchRequest.Create(new BooleanQuery().Must(Score()).Should(Match())) }, + { "boolean should", SearchRequest.Create(new BooleanQuery().Must(Match()).Should(Filter())) }, + { "boolean must not", SearchRequest.Create(new BooleanQuery().Must(Match()).MustNot(Score())) }, + { "nested compound", SearchRequest.Create(new ConjunctionQuery(new DisjunctionQuery(Match(), Filter()))) }, + { "vector prefilter", Prefilter(Filter()) }, + { "vector prefilter compound", Prefilter(new ConjunctionQuery(Match(), Score())) }, + { "query and vector", SearchRequest.Create(Match()).WithVectorSearch(Prefilter(Filter()).VectorSearch!) } + }; + + public static TheoryData RequestsWithoutCustomScriptQuery() => new() + { + { "match", SearchRequest.Create(Match()) }, + { "compound", SearchRequest.Create(new BooleanQuery().Must(Match()).Should(new ConjunctionQuery(Match()))) }, + { "vector", SearchRequest.Create(VectorSearch.Create(new VectorQuery("vector_field").WithVector(new[] { 0.1f }))) }, + { "vector prefilter", Prefilter(Match()) } + }; + + [Theory] + [MemberData(nameof(RequestsWithCustomScriptQuery))] + public void ContainsCustomScriptQuery_Finds_It(string name, SearchRequest request) + { + Assert.True(request.ContainsCustomScriptQuery(), name); + } + + [Theory] + [MemberData(nameof(RequestsWithoutCustomScriptQuery))] + public void ContainsCustomScriptQuery_When_Absent_Returns_False(string name, SearchRequest request) + { + Assert.False(request.ContainsCustomScriptQuery(), name); + } + + [Theory] + [MemberData(nameof(RequestsWithCustomScriptQuery))] + public async Task SearchAsync_Without_Capability_Throws_Before_Sending(string name, SearchRequest request) + { + var searchClient = new Mock(); + using var cluster = CreateCluster(searchClient.Object, "vectorSearch", "scopedSearchIndex", "scoreFusion"); + + var ex = await Assert.ThrowsAsync( + () => cluster.SearchAsync("index", request)); + + Assert.Contains(ClusterCapabilities.SEARCH_UDF_QUERY, ex.Message); + searchClient.Verify(m => m.QueryAsync(It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never, name); + } + + [Theory] + [MemberData(nameof(RequestsWithCustomScriptQuery))] + public async Task SearchAsync_With_Capability_Sends_The_Request(string name, SearchRequest request) + { + var searchResult = new Mock(); + searchResult.SetupGet(m => m.RetryReason).Returns(RetryReason.NoRetry); + var searchClient = new Mock(); + searchClient + .Setup(m => m.QueryAsync(It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(searchResult.Object); + using var cluster = CreateCluster(searchClient.Object, "vectorSearch", "scopedSearchIndex", "scoreFusion", "udfQuery"); + + var result = await cluster.SearchAsync("index", request); + + Assert.Same(searchResult.Object, result); + searchClient.Verify(m => m.QueryAsync(It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), Times.Once, name); + } + + [Fact] + public async Task SearchQueryAsync_Without_Capability_Throws() + { + using var cluster = CreateCluster(new Mock().Object, "vectorSearch"); + + await Assert.ThrowsAsync( + () => cluster.SearchQueryAsync("index", new ConjunctionQuery(Match(), Filter()))); + } + + private static Cluster CreateCluster(ISearchClient searchClient, params string[] searchCapabilities) + { +#pragma warning disable CS0618 // Type or member is obsolete + var options = new ClusterOptions().WithCredentials("u", "p") + .WithConnectionString("couchbase://localhost"); +#pragma warning restore CS0618 // Type or member is obsolete + + var cluster = new Cluster(options) + { + LazySearchClient = LazyServiceShimFactory.Create(searchClient) + }; + cluster.Context.GlobalConfig = new BucketConfig + { + ClusterCapabilities = new Dictionary> + { + ["search"] = searchCapabilities + } + }; + return cluster; + } + } +} diff --git a/tests/Couchbase.UnitTests/Search/CustomScriptQueryTests.cs b/tests/Couchbase.UnitTests/Search/CustomScriptQueryTests.cs new file mode 100644 index 000000000..b4d57613d --- /dev/null +++ b/tests/Couchbase.UnitTests/Search/CustomScriptQueryTests.cs @@ -0,0 +1,209 @@ +using System; +using System.Collections.Generic; +using Couchbase.Search; +using Couchbase.Search.Queries.Compound; +using Couchbase.Search.Queries.Script; +using Couchbase.Search.Queries.Simple; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Couchbase.UnitTests.Search +{ + public class CustomScriptQueryTests + { + private const string Source = "function f(doc, params) { return doc.score; }"; + + public static IEnumerable QueryTypes() + { + yield return new object[] { "custom_score" }; + yield return new object[] { "custom_filter" }; + } + + // Builds either query type, so each test covers both with the same code. + private static ISearchQuery Create(string queryType, ISearchQuery inner, string source, + string[] fields = null, IDictionary parameters = null) + { + if (queryType == "custom_score") + { + var query = new CustomScoreQuery(inner, source); + if (fields != null) query.Fields(fields); + if (parameters != null) query.Parameters(parameters); + return query; + } + else + { + var query = new CustomFilterQuery(inner, source); + if (fields != null) query.Fields(fields); + if (parameters != null) query.Parameters(parameters); + return query; + } + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Export_WithOnlyQueryAndSource_OmitsFieldsAndParams(string queryType) + { + var query = Create(queryType, new MatchAllQuery(), Source); + + var expected = new JObject + { + { queryType, new JObject + { + { "query", new JObject { { "match_all", null } } }, + { "source", Source } + }} + }; + + Assert.Equal(expected.ToString(Formatting.None), query.Export().ToString(Formatting.None)); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Export_WithFieldsAndNestedParams_WritesBoth(string queryType) + { + var parameters = new Dictionary + { + { "factor", 2 }, + { "floor", 0.5 }, + { "name", "michael" }, + { "enabled", true }, + { "missing", null }, + { "tags", new List { "a", 1 } }, + { "boost", new Dictionary + { + { "equal", new Dictionary { { "name", "michael" } } } + }} + }; + + var query = Create(queryType, new MatchQuery("michael").Field("name"), Source, + new[] { "name", "price" }, parameters); + + var expected = JObject.Parse($$""" + { + "{{queryType}}": { + "query": { "match": "michael", "prefix_length": 0, "fuzziness": 0, "field": "name" }, + "source": "{{Source}}", + "fields": [ "name", "price" ], + "params": { + "factor": 2, + "floor": 0.5, + "name": "michael", + "enabled": true, + "missing": null, + "tags": [ "a", 1 ], + "boost": { "equal": { "name": "michael" } } + } + } + } + """); + + Assert.True(JToken.DeepEquals(expected, query.Export()), query.Export().ToString()); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Export_WithEmptyFieldsAndParams_OmitsBoth(string queryType) + { + var query = Create(queryType, new MatchAllQuery(), Source, + Array.Empty(), new Dictionary()); + + var body = (JObject)query.Export()[queryType]!; + + Assert.Null(body["fields"]); + Assert.Null(body["params"]); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Export_WithCompoundInnerQuery_ExportsItInPlace(string queryType) + { + var inner = new ConjunctionQuery(new MatchQuery("michael"), new MatchAllQuery()); + + var query = Create(queryType, inner, Source); + + Assert.True(JToken.DeepEquals(inner.Export(), query.Export()[queryType]!["query"])); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Fields_AndParameters_AreCopied(string queryType) + { + var fields = new[] { "name" }; + var parameters = new Dictionary { { "factor", 2 } }; + var query = Create(queryType, new MatchAllQuery(), Source, fields, parameters); + + fields[0] = "changed"; + parameters["factor"] = 3; + parameters["added"] = true; + + var body = query.Export()[queryType]!; + Assert.Equal("name", (string)body["fields"]![0]); + Assert.Equal(2, (int)body["params"]!["factor"]); + Assert.Null(body["params"]!["added"]); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Ctor_WhenQueryIsNull_ThrowsArgumentNullException(string queryType) + { + Assert.Throws(() => Create(queryType, null, Source)); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Ctor_WhenSourceIsNull_ThrowsArgumentNullException(string queryType) + { + Assert.Throws(() => Create(queryType, new MatchAllQuery(), null)); + } + + [Theory] + [MemberData(nameof(QueryTypes))] + public void Ctor_WhenSourceIsEmptyOrWhitespace_ThrowsArgumentException(string queryType) + { + Assert.Throws(() => Create(queryType, new MatchAllQuery(), "")); + Assert.Throws(() => Create(queryType, new MatchAllQuery(), " ")); + } + + [Fact] + public void Fields_WhenNull_ThrowsArgumentNullException() + { + Assert.Throws(() => new CustomScoreQuery(new MatchAllQuery(), Source).Fields(null)); + Assert.Throws(() => new CustomFilterQuery(new MatchAllQuery(), Source).Fields(null)); + } + + [Fact] + public void Parameters_WhenNull_ThrowsArgumentNullException() + { + Assert.Throws(() => new CustomScoreQuery(new MatchAllQuery(), Source).Parameters(null)); + Assert.Throws(() => new CustomFilterQuery(new MatchAllQuery(), Source).Parameters(null)); + } + + // FTS silently ignores a boost on these query types (checked against 8.5.0-1308), so, as in Java, Boost does nothing. + [Theory] + [MemberData(nameof(QueryTypes))] + public void Boost_HasNoEffect(string queryType) + { + var query = Create(queryType, new MatchAllQuery(), Source); + var expected = query.Export().ToString(); + + Assert.Same(query, query.Boost(2)); + Assert.Equal(expected, query.Export().ToString()); + } + + [Fact] + public void AsReadOnly_ReturnsTheConfiguredValues() + { + var inner = new MatchAllQuery(); + var readOnly = new CustomScoreQuery(inner, Source) + .Fields("name") + .Parameters(new Dictionary { { "factor", 2 } }) + .AsReadOnly(); + + Assert.Same(inner, readOnly.Query); + Assert.Equal(Source, readOnly.Source); + Assert.Equal(new[] { "name" }, readOnly.Fields); + Assert.Equal(2, readOnly.Parameters!["factor"]); + } + } +} diff --git a/tests/Couchbase.UnitTests/Stellar/Search/StellarSearchClientTests.cs b/tests/Couchbase.UnitTests/Stellar/Search/StellarSearchClientTests.cs index a775185fa..7e6cfb168 100644 --- a/tests/Couchbase.UnitTests/Stellar/Search/StellarSearchClientTests.cs +++ b/tests/Couchbase.UnitTests/Stellar/Search/StellarSearchClientTests.cs @@ -3,8 +3,11 @@ using System.Threading.Tasks; using Couchbase.Core.Exceptions; using Couchbase.Search; +using Couchbase.Search.Queries.Compound; +using Couchbase.Search.Queries.Script; using Couchbase.Search.Queries.Simple; using Couchbase.Stellar.Search; +using Newtonsoft.Json.Linq; using Xunit; namespace Couchbase.UnitTests.Stellar.Search; @@ -41,6 +44,51 @@ await Assert.ThrowsAsync( () => client.QueryAsync("indexname", new TermQuery("term"), options)); } + public static TheoryData CustomScriptQueries() => + [ + new CustomScoreQuery(new MatchAllQuery(), "function f(doc, params) { return doc.score; }"), + new ConjunctionQuery(new TermQuery("term"), + new CustomFilterQuery(new MatchAllQuery(), "function f(doc, params) { return true; }")) + ]; + + [Theory] + [MemberData(nameof(CustomScriptQueries))] + public async Task Throw_FeatureNotAvailableException_When_Custom_Script_Query_Used(ISearchQuery query) + { + var client = CreateClient(); + + // The protocol has no custom score or filter query types, so this fails before any RPC is attempted. + await Assert.ThrowsAsync( + () => client.QueryAsync("indexname", query)); + } + + public static TheoryData UnconvertibleQueries() => + [ + new UserDefinedQuery(), + new ConjunctionQuery(new TermQuery("term"), new UserDefinedQuery()), + new BooleanQuery().Must(new TermQuery("term")).Should(new UserDefinedQuery()) + ]; + + [Theory] + [MemberData(nameof(UnconvertibleQueries))] + public async Task Throw_FeatureNotAvailableException_When_Query_Type_Cannot_Be_Converted(ISearchQuery query) + { + var client = CreateClient(); + + // The converter has no protocol mapping for these, so it must fail rather than send an empty query. + var ex = await Assert.ThrowsAsync( + () => client.QueryAsync("indexname", query)); + + Assert.Contains(nameof(UserDefinedQuery), ex.Message); + } + + private sealed class UserDefinedQuery : ISearchQuery + { + public ISearchQuery Boost(double boost) => this; + + public JObject Export() => new() { { "term", "term" } }; + } + private static StellarSearchClient CreateClient() => new(new ClusterTests().CreateClusterFromMocks()); }