Add support for query tokenization parameters

app/controllers/search_controller.rb

@@ -200,16 +200,16 @@ def query_timdex(query)
     Rails.cache.fetch("#{cache_key}/#{@active_tab}", expires_in: 12.hours) do
       raw = if Feature.enabled?(:geodata)
               execute_geospatial_query(query)
+            elsif tuning_request?(query)
+              execute_tuning_query(query)
             else
               TimdexBase::Client.query(TimdexSearch::BaseQuery, variables: query)
             end
 
-      # The response type is a GraphQL::Client::Response, which is not directly serializable, so we
-      # convert it to a hash.
-      {
-        data: raw.data.to_h,
-        errors: raw.errors.details.to_h
-      }
+      # The response is either a GraphQL::Client::Response (for most queries), which is not directly serializable, or
+      # it is already a hash (for tuning queries). These two formats are standardized into a common shape in
+      # process_timdex_response.
+      process_timdex_response(raw, query)
     end
   end
 
@@ -225,7 +225,8 @@ def query_primo(per_page, offset)
   end
 
   def execute_geospatial_query(query)
-    query = query.except('queryMode')
+    query = query.except('queryMode', 'semanticDropBoostThreshold', 'semanticMustBoostThreshold',
+                         'semanticShortQueryMaxTokens')
 
     if query['geobox'] == 'true' && query[:geodistance] == 'true'
       TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query)
@@ -238,6 +239,29 @@ def execute_geospatial_query(query)
     end
   end
 
+  # The parameters we use for measuring relevance while tuning the platform are not included in the public schema, so
+  # we cannot rely on the graphql-client gem (which validates queries based on that schema) while using those
+  # parameters. As a result, we rely on a fallback method of querying the API via Net::HTTP for these queries.
+  def execute_tuning_query(query_vars)
+    uri = URI(ENV.fetch('TIMDEX_GRAPHQL', ''))
+
+    req = Net::HTTP::Post.new(uri)
+    req['Content-Type'] = 'application/json'
+    req['Accept'] = 'application/json'
+    req['User-Agent'] = 'MIT Libraries Client'
+
+    req.body = JSON.generate(
+      query: TimdexTuning::TUNING_QUERY,
+      variables: query_vars
+    )
+
+    res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+      http.request(req)
+    end
+
+    JSON.parse(res.body)
+  end
+
   def extract_errors(response)
     response[:errors]['data'] if response.is_a?(Hash) && response.key?(:errors) && response[:errors].key?('data')
   end
@@ -453,4 +477,24 @@ def valid_token?
   def show_nls_warning?
     @natural_language_search_optin && primo_tabs.include?(@active_tab)
   end
+
+  def tuning_request?(query)
+    tuning_params = %w[semanticMustBoostThreshold semanticDropBoostThreshold semanticShortQueryMaxTokens]
+
+    tuning_params.intersect?(query.keys)
+  end
+
+  def process_timdex_response(raw, query)
+    if tuning_request?(query)
+      {
+        data: (raw['data'].nil? ? {} : raw['data']),
+        errors: (raw['errors'].nil? ? {} : { 'data' => raw['errors'] })
+      }
+    else
+      {
+        data: raw.data.to_h,
+        errors: raw.errors.details.to_h
+      }
+    end
+  end
 end

app/helpers/application_helper.rb

@@ -20,7 +20,8 @@ def index_page_title
   def results_page_title(query, character_limit = 50)
     return index_page_title unless query.present?
 
-    ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode]
+    ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode semanticDropBoostThreshold
+                       semanticMustBoostThreshold semanticShortQueryMaxTokens]
     terms = query.reject { |term| ignored_terms.include? term }.values.join(' ')
     terms = "#{terms.first(character_limit)}..." if terms.length > character_limit
     "#{terms} | #{index_page_title}"

app/models/enhancer.rb

@@ -1,7 +1,8 @@
 class Enhancer
   attr_accessor :enhanced_query
 
-  QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title queryMode].freeze
+  QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title
+                    queryMode semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze
   FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter
                      literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze
   GEO_PARAMS = %i[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude

app/models/query_builder.rb

@@ -7,6 +7,7 @@ class QueryBuilder
   GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude
                   geodistanceLongitude geodistanceDistance].freeze
   VALID_QUERY_MODES = %w[keyword semantic hybrid].freeze
+  TOKENIZATION_PARAMS = %w[semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze
 
   def initialize(enhanced_query)
     @query = {}
@@ -22,6 +23,7 @@ def initialize(enhanced_query)
     extract_geosearch(enhanced_query)
     extract_filters(enhanced_query)
     evaluate_query_mode(enhanced_query)
+    extract_tokenization_params(enhanced_query)
     @query['index'] = ENV.fetch('TIMDEX_INDEX', nil)
     @query['booleanType'] = enhanced_query[:booleanType]
     @query.compact!
@@ -59,9 +61,20 @@ def extract_filters(enhanced_query)
     end
   end
 
-  # The GraphQL API requires that lat/long in geospatial fields be floats
-  def coerce_to_float?(geo_param)
-    geo_param.to_s.include?('Longitude') || geo_param.to_s.include?('Latitude')
+  # We treat the tokenization parameters separately because we need to ensure that floats and integers are formatted
+  # correctly.
+  def extract_tokenization_params(enhanced_query)
+    TOKENIZATION_PARAMS.each do |tp|
+      next unless enhanced_query[tp.to_sym].present?
+
+      @query[tp] = coerce_to_float?(tp) ? enhanced_query[tp.to_sym]&.strip.to_f : enhanced_query[tp.to_sym]&.strip.to_i
+    end
+  end
+
+  # The GraphQL API requires that some parameters - lat/long in geospatial fields and boost thresholds for tuning - be
+  # floats.
+  def coerce_to_float?(param)
+    param.to_s.include?('Longitude') || param.to_s.include?('Latitude') || param.to_s.include?('BoostThreshold')
   end
 
   # Determine the query mode from URL parameter or config, with fallback to 'keyword'

app/models/timdex_tuning.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+# This class exists because we rely on some undocumented parameters for tuning system performance, and using those
+# parameters in conjunction with the graphql-client gem results in schema validation errors (because the parameters
+# are not found in the schema).
+class TimdexTuning
+  TUNING_QUERY = <<-GRAPHQL
+    query TimdexPlaygroundQuery(
+      $q: String,
+      $citation: String,
+      $contributors: String,
+      $fundingInformation: String,
+      $identifiers: String,
+      $locations: String,
+      $subjects: String,
+      $title: String,
+      $index: String,
+      $from: String,
+      $booleanType: String,
+      $queryMode: String,
+      $fulltext: Boolean,
+      $perPage: Int,
+      $accessToFilesFilter: [String!],
+      $contentTypeFilter: [String!],
+      $contributorsFilter: [String!],
+      $formatFilter: [String!],
+      $languagesFilter: [String!],
+      $literaryFormFilter: String,
+      $placesFilter: [String!],
+      $sourceFilter: [String!],
+      $subjectsFilter: [String!],
+      $useGlobalScoring: Boolean,
+      $semanticDropBoostThreshold: Float,
+      $semanticMustBoostThreshold: Float,
+      $semanticShortQueryMaxTokens: Int
+    ) {
+      search(
+        searchterm: $q
+        citation: $citation
+        contributors: $contributors
+        fundingInformation: $fundingInformation
+        identifiers: $identifiers
+        locations: $locations
+        subjects: $subjects
+        title: $title
+        index: $index
+        from: $from
+        booleanType: $booleanType
+        queryMode: $queryMode
+        fulltext: $fulltext
+        perPage: $perPage
+        accessToFilesFilter: $accessToFilesFilter
+        contentTypeFilter: $contentTypeFilter
+        contributorsFilter: $contributorsFilter
+        formatFilter: $formatFilter
+        languagesFilter: $languagesFilter
+        literaryFormFilter: $literaryFormFilter
+        placesFilter: $placesFilter
+        sourceFilter: $sourceFilter
+        subjectsFilter: $subjectsFilter
+        useGlobalScoring: $useGlobalScoring
+        semanticDropBoostThreshold: $semanticDropBoostThreshold
+        semanticMustBoostThreshold: $semanticMustBoostThreshold
+        semanticShortQueryMaxTokens: $semanticShortQueryMaxTokens
+      ) {
+        hits
+        records {
+          timdexRecordId
+          identifiers {
+            kind
+            value
+          }
+          title
+          source
+          contentType
+          contributors {
+            kind
+            value
+          }
+          publicationInformation
+          dates {
+            kind
+            value
+            range {
+              gte
+              lte
+            }
+          }
+          links {
+            kind
+            restrictions
+            text
+            url
+          }
+          notes {
+            kind
+            value
+          }
+          highlight {
+            matchedField
+            matchedPhrases
+          }
+          provider
+          rights {
+            kind
+            description
+            uri
+          }
+          sourceLink
+          summary
+          subjects {
+            kind
+            value
+          }
+          citation
+        }
+        aggregations {
+          accessToFiles {
+            key
+            docCount
+          }
+          contentType {
+            key
+            docCount
+          }
+          contributors {
+            key
+            docCount
+          }
+          format {
+            key
+            docCount
+          }
+          languages {
+            key
+            docCount
+          }
+          literaryForm {
+            key
+            docCount
+          }
+          places {
+            key
+            docCount
+          }
+          source {
+            key
+            docCount
+          }
+          subjects {
+            key
+            docCount
+          }
+        }
+      }
+    }
+  GRAPHQL
+end

test/controllers/search_controller_test.rb

@@ -1183,6 +1183,26 @@ def source_filter_count(controller)
     assert_select '.pagination-container .current', text: /21 - 40 of 800/
   end
 
+  # test 'results can include tuning parameters' do
+  #   query = 'optical networks in space'
+  #   must_default = 0.7
+  #   drop_default = 0.1
+  #   must_alt = 0.9
+  #   drop_alt = 0.4
+
+  #   VCR.use_cassette('default tuning for stock query') do
+  #     get "/results?q=#{query}&semanticMustBoostThreshold=#{must_default}&semanticDropBoostThreshold=#{drop_default}&tab=timdex"
+  #     assert_response :success
+  #   end
+
+  #   VCR.use_cassette('alternate tuning for stock query') do
+  #     get "/results?q=#{query}&semanticMustBoostThreshold=#{must_alt}&semanticDropBoostThreshold=#{drop_alt}&tab=timdex"
+  #     assert_response :success
+  #   end
+
+  #   # Assert result counts are different
+  # end
+
   test 'results can be returned in JSON format when env is set and valid token is provided' do
     secret_value = 'sooper_sekret'
     quepid_ua = 'Quepid/1.0 (Web Scraper)'

test/helpers/application_helper_test.rb

@@ -54,6 +54,12 @@ class ApplicationHelperTest < ActionView::TestCase
     assert_equal 'Search MIT Libraries', results_page_title(no_query)
   end
 
+  test 'results_page_title ignores tuning parameters' do
+    query = { q: 'National Parks Service', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7',
+              semanticShortQueryMaxTokens: '5' }
+    assert_equal 'National Parks Service | Search MIT Libraries', results_page_title(query)
+  end
+
   test 'record_page_title includes record title' do
     record = { 'title' => 'The Waves' }
     assert_equal 'The Waves | Search MIT Libraries', record_page_title(record)

test/models/query_builder_test.rb

@@ -123,6 +123,7 @@ class QueryBuilderTest < ActiveSupport::TestCase
     assert_equal expected, QueryBuilder.new(search).query
   end
 
+  # Query mode behavior
   test 'query builder defaults to keyword queryMode' do
     search = { q: 'blah' }
     assert_equal('keyword', QueryBuilder.new(search).query['queryMode'])
@@ -185,4 +186,28 @@ class QueryBuilderTest < ActiveSupport::TestCase
     search = { q: 'blah', queryMode: '  semantic  ' }
     assert_equal('semantic', QueryBuilder.new(search).query['queryMode'])
   end
+
+  # Tuning parameter behavior
+  test 'query builder handles tuning parameters' do
+    expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX',
+                 'semanticDropBoostThreshold' => 0.1, 'semanticMustBoostThreshold' => 0.7,
+                 'semanticShortQueryMaxTokens' => 5 }
+    search = { q: 'blah', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7',
+               semanticShortQueryMaxTokens: '5' }
+    assert_equal(expected, QueryBuilder.new(search).query)
+  end
+
+  test 'query builder converts semanticShortQueryMaxTokens to integer' do
+    expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX',
+                 'semanticShortQueryMaxTokens' => 5 }
+    search = { q: 'blah', semanticShortQueryMaxTokens: '5.0' }
+    assert_equal(expected, QueryBuilder.new(search).query)
+  end
+
+  test 'query builder converts drop and boost thresholds to floats' do
+    expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX',
+                 'semanticDropBoostThreshold' => 0.0, 'semanticMustBoostThreshold' => 1.0 }
+    search = { q: 'blah', semanticDropBoostThreshold: '0', semanticMustBoostThreshold: '1' }
+    assert_equal(expected, QueryBuilder.new(search).query)
+  end
 end