diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 58e02d8e..4b302778 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -200,16 +200,16 @@ def query_timdex(query) Rails.cache.fetch("#{cache_key}/#{@active_tab}", expires_in: 12.hours) do raw = if Feature.enabled?(:geodata) execute_geospatial_query(query) + elsif tuning_request?(query) + execute_tuning_query(query) else TimdexBase::Client.query(TimdexSearch::BaseQuery, variables: query) end - # The response type is a GraphQL::Client::Response, which is not directly serializable, so we - # convert it to a hash. - { - data: raw.data.to_h, - errors: raw.errors.details.to_h - } + # The response is either a GraphQL::Client::Response (for most queries), which is not directly serializable, or + # it is already a hash (for tuning queries). These two formats are standardized into a common shape in + # process_timdex_response. + process_timdex_response(raw, query) end end @@ -225,7 +225,8 @@ def query_primo(per_page, offset) end def execute_geospatial_query(query) - query = query.except('queryMode') + query = query.except('queryMode', 'semanticDropBoostThreshold', 'semanticMustBoostThreshold', + 'semanticShortQueryMaxTokens') if query['geobox'] == 'true' && query[:geodistance] == 'true' TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query) @@ -238,6 +239,29 @@ def execute_geospatial_query(query) end end + # The parameters we use for measuring relevance while tuning the platform are not included in the public schema, so + # we cannot rely on the graphql-client gem (which validates queries based on that schema) while using those + # parameters. As a result, we rely on a fallback method of querying the API via Net::HTTP for these queries. + def execute_tuning_query(query_vars) + uri = URI(ENV.fetch('TIMDEX_GRAPHQL', '')) + + req = Net::HTTP::Post.new(uri) + req['Content-Type'] = 'application/json' + req['Accept'] = 'application/json' + req['User-Agent'] = 'MIT Libraries Client' + + req.body = JSON.generate( + query: TimdexTuning::TUNING_QUERY, + variables: query_vars + ) + + res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(req) + end + + JSON.parse(res.body) + end + def extract_errors(response) response[:errors]['data'] if response.is_a?(Hash) && response.key?(:errors) && response[:errors].key?('data') end @@ -453,4 +477,24 @@ def valid_token? def show_nls_warning? @natural_language_search_optin && primo_tabs.include?(@active_tab) end + + def tuning_request?(query) + tuning_params = %w[semanticMustBoostThreshold semanticDropBoostThreshold semanticShortQueryMaxTokens] + + tuning_params.intersect?(query.keys) + end + + def process_timdex_response(raw, query) + if tuning_request?(query) + { + data: (raw['data'].nil? ? {} : raw['data']), + errors: (raw['errors'].nil? ? {} : { 'data' => raw['errors'] }) + } + else + { + data: raw.data.to_h, + errors: raw.errors.details.to_h + } + end + end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index b085ceb8..9be0d2fc 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -20,7 +20,8 @@ def index_page_title def results_page_title(query, character_limit = 50) return index_page_title unless query.present? - ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode] + ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode semanticDropBoostThreshold + semanticMustBoostThreshold semanticShortQueryMaxTokens] terms = query.reject { |term| ignored_terms.include? term }.values.join(' ') terms = "#{terms.first(character_limit)}..." if terms.length > character_limit "#{terms} | #{index_page_title}" diff --git a/app/models/enhancer.rb b/app/models/enhancer.rb index b21099bf..7ee32a4a 100644 --- a/app/models/enhancer.rb +++ b/app/models/enhancer.rb @@ -1,7 +1,8 @@ class Enhancer attr_accessor :enhanced_query - QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title queryMode].freeze + QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title + queryMode semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %i[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb index 76491ca0..f7c4596c 100644 --- a/app/models/query_builder.rb +++ b/app/models/query_builder.rb @@ -7,6 +7,7 @@ class QueryBuilder GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude geodistanceLongitude geodistanceDistance].freeze VALID_QUERY_MODES = %w[keyword semantic hybrid].freeze + TOKENIZATION_PARAMS = %w[semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze def initialize(enhanced_query) @query = {} @@ -22,6 +23,7 @@ def initialize(enhanced_query) extract_geosearch(enhanced_query) extract_filters(enhanced_query) evaluate_query_mode(enhanced_query) + extract_tokenization_params(enhanced_query) @query['index'] = ENV.fetch('TIMDEX_INDEX', nil) @query['booleanType'] = enhanced_query[:booleanType] @query.compact! @@ -59,9 +61,20 @@ def extract_filters(enhanced_query) end end - # The GraphQL API requires that lat/long in geospatial fields be floats - def coerce_to_float?(geo_param) - geo_param.to_s.include?('Longitude') || geo_param.to_s.include?('Latitude') + # We treat the tokenization parameters separately because we need to ensure that floats and integers are formatted + # correctly. + def extract_tokenization_params(enhanced_query) + TOKENIZATION_PARAMS.each do |tp| + next unless enhanced_query[tp.to_sym].present? + + @query[tp] = coerce_to_float?(tp) ? enhanced_query[tp.to_sym]&.strip.to_f : enhanced_query[tp.to_sym]&.strip.to_i + end + end + + # The GraphQL API requires that some parameters - lat/long in geospatial fields and boost thresholds for tuning - be + # floats. + def coerce_to_float?(param) + param.to_s.include?('Longitude') || param.to_s.include?('Latitude') || param.to_s.include?('BoostThreshold') end # Determine the query mode from URL parameter or config, with fallback to 'keyword' diff --git a/app/models/timdex_tuning.rb b/app/models/timdex_tuning.rb new file mode 100644 index 00000000..6f42e774 --- /dev/null +++ b/app/models/timdex_tuning.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +# This class exists because we rely on some undocumented parameters for tuning system performance, and using those +# parameters in conjunction with the graphql-client gem results in schema validation errors (because the parameters +# are not found in the schema). +class TimdexTuning + TUNING_QUERY = <<-GRAPHQL + query TimdexPlaygroundQuery( + $q: String, + $citation: String, + $contributors: String, + $fundingInformation: String, + $identifiers: String, + $locations: String, + $subjects: String, + $title: String, + $index: String, + $from: String, + $booleanType: String, + $queryMode: String, + $fulltext: Boolean, + $perPage: Int, + $accessToFilesFilter: [String!], + $contentTypeFilter: [String!], + $contributorsFilter: [String!], + $formatFilter: [String!], + $languagesFilter: [String!], + $literaryFormFilter: String, + $placesFilter: [String!], + $sourceFilter: [String!], + $subjectsFilter: [String!], + $useGlobalScoring: Boolean, + $semanticDropBoostThreshold: Float, + $semanticMustBoostThreshold: Float, + $semanticShortQueryMaxTokens: Int + ) { + search( + searchterm: $q + citation: $citation + contributors: $contributors + fundingInformation: $fundingInformation + identifiers: $identifiers + locations: $locations + subjects: $subjects + title: $title + index: $index + from: $from + booleanType: $booleanType + queryMode: $queryMode + fulltext: $fulltext + perPage: $perPage + accessToFilesFilter: $accessToFilesFilter + contentTypeFilter: $contentTypeFilter + contributorsFilter: $contributorsFilter + formatFilter: $formatFilter + languagesFilter: $languagesFilter + literaryFormFilter: $literaryFormFilter + placesFilter: $placesFilter + sourceFilter: $sourceFilter + subjectsFilter: $subjectsFilter + useGlobalScoring: $useGlobalScoring + semanticDropBoostThreshold: $semanticDropBoostThreshold + semanticMustBoostThreshold: $semanticMustBoostThreshold + semanticShortQueryMaxTokens: $semanticShortQueryMaxTokens + ) { + hits + records { + timdexRecordId + identifiers { + kind + value + } + title + source + contentType + contributors { + kind + value + } + publicationInformation + dates { + kind + value + range { + gte + lte + } + } + links { + kind + restrictions + text + url + } + notes { + kind + value + } + highlight { + matchedField + matchedPhrases + } + provider + rights { + kind + description + uri + } + sourceLink + summary + subjects { + kind + value + } + citation + } + aggregations { + accessToFiles { + key + docCount + } + contentType { + key + docCount + } + contributors { + key + docCount + } + format { + key + docCount + } + languages { + key + docCount + } + literaryForm { + key + docCount + } + places { + key + docCount + } + source { + key + docCount + } + subjects { + key + docCount + } + } + } + } + GRAPHQL +end diff --git a/test/controllers/search_controller_test.rb b/test/controllers/search_controller_test.rb index 010c527e..c4af5ed5 100644 --- a/test/controllers/search_controller_test.rb +++ b/test/controllers/search_controller_test.rb @@ -1183,6 +1183,26 @@ def source_filter_count(controller) assert_select '.pagination-container .current', text: /21 - 40 of 800/ end + # test 'results can include tuning parameters' do + # query = 'optical networks in space' + # must_default = 0.7 + # drop_default = 0.1 + # must_alt = 0.9 + # drop_alt = 0.4 + + # VCR.use_cassette('default tuning for stock query') do + # get "/results?q=#{query}&semanticMustBoostThreshold=#{must_default}&semanticDropBoostThreshold=#{drop_default}&tab=timdex" + # assert_response :success + # end + + # VCR.use_cassette('alternate tuning for stock query') do + # get "/results?q=#{query}&semanticMustBoostThreshold=#{must_alt}&semanticDropBoostThreshold=#{drop_alt}&tab=timdex" + # assert_response :success + # end + + # # Assert result counts are different + # end + test 'results can be returned in JSON format when env is set and valid token is provided' do secret_value = 'sooper_sekret' quepid_ua = 'Quepid/1.0 (Web Scraper)' diff --git a/test/helpers/application_helper_test.rb b/test/helpers/application_helper_test.rb index a2a62000..74d6f28a 100644 --- a/test/helpers/application_helper_test.rb +++ b/test/helpers/application_helper_test.rb @@ -54,6 +54,12 @@ class ApplicationHelperTest < ActionView::TestCase assert_equal 'Search MIT Libraries', results_page_title(no_query) end + test 'results_page_title ignores tuning parameters' do + query = { q: 'National Parks Service', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7', + semanticShortQueryMaxTokens: '5' } + assert_equal 'National Parks Service | Search MIT Libraries', results_page_title(query) + end + test 'record_page_title includes record title' do record = { 'title' => 'The Waves' } assert_equal 'The Waves | Search MIT Libraries', record_page_title(record) diff --git a/test/models/query_builder_test.rb b/test/models/query_builder_test.rb index 41b689fc..8f364b6b 100644 --- a/test/models/query_builder_test.rb +++ b/test/models/query_builder_test.rb @@ -123,6 +123,7 @@ class QueryBuilderTest < ActiveSupport::TestCase assert_equal expected, QueryBuilder.new(search).query end + # Query mode behavior test 'query builder defaults to keyword queryMode' do search = { q: 'blah' } assert_equal('keyword', QueryBuilder.new(search).query['queryMode']) @@ -185,4 +186,28 @@ class QueryBuilderTest < ActiveSupport::TestCase search = { q: 'blah', queryMode: ' semantic ' } assert_equal('semantic', QueryBuilder.new(search).query['queryMode']) end + + # Tuning parameter behavior + test 'query builder handles tuning parameters' do + expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX', + 'semanticDropBoostThreshold' => 0.1, 'semanticMustBoostThreshold' => 0.7, + 'semanticShortQueryMaxTokens' => 5 } + search = { q: 'blah', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7', + semanticShortQueryMaxTokens: '5' } + assert_equal(expected, QueryBuilder.new(search).query) + end + + test 'query builder converts semanticShortQueryMaxTokens to integer' do + expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX', + 'semanticShortQueryMaxTokens' => 5 } + search = { q: 'blah', semanticShortQueryMaxTokens: '5.0' } + assert_equal(expected, QueryBuilder.new(search).query) + end + + test 'query builder converts drop and boost thresholds to floats' do + expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX', + 'semanticDropBoostThreshold' => 0.0, 'semanticMustBoostThreshold' => 1.0 } + search = { q: 'blah', semanticDropBoostThreshold: '0', semanticMustBoostThreshold: '1' } + assert_equal(expected, QueryBuilder.new(search).query) + end end