web_search: Serply as an API key service

src/tools/web_search_services.cpp

@@ -332,6 +332,24 @@ std::vector<WebSearchService> make_catalogue() {
             V("", "On (default)"), V("false", "Off")}});
         c.push_back(std::move(s));
     }
+    {
+        WebSearchService s;
+        s.id = "serply-api"; s.label = "Serply API key"; s.needs_key = true;
+        s.signup = "https://serply.io";
+        s.note = "Google results as JSON.";
+        s.max_count = 10;
+        // Without gl Serply serves from whichever proxy region is free, so the
+        // results can change country between calls. req_serply sends gl=us
+        // for the default, so United States takes the first row.
+        auto country = country_dial("Google's gl parameter.");
+        country.values.erase(country.values.begin() + 1);
+        country.values[0].label = "United States (default)";
+        s.dials.push_back(std::move(country));
+        s.dials.push_back(freshness_dial("freshness",
+            "Google's tbs=qdr: time filter.",
+            hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y")));
+        c.push_back(std::move(s));
+    }
     {
         WebSearchService s;
         s.id = "perplexity-api"; s.label = "Perplexity API key"; s.needs_key = true;
@@ -654,6 +672,27 @@ HttpRequest req_serper(const WebSearchQuery& q, std::string_view key) {
                      {{"x-api-key", std::string{key}}});
 }
 
+// Serply. [L] GET /v1/search/?q=&num= ; X-Api-Key ; num stops at 10 ; gl and
+// tbs pass through to Google ; hl does not change the results, so no language.
+// gl is always sent, us by default: without it the proxy region is random.
+// Cloudflare in front of it refuses some default client user-agents, so the
+// request names itself.
+HttpRequest req_serply(const WebSearchQuery& q, std::string_view key) {
+    HttpRequest r;
+    r.method = "GET";
+    r.url = "https://api.serply.io/v1/search/?q=" + url_escape(q.text)
+          + "&num=" + std::to_string(q.count);
+    auto country = q.dial("country");
+    r.url += "&gl=" + (country.empty() ? std::string{"us"}
+                                       : url_escape(lower(std::string{country})));
+    if (auto v = q.dial("freshness"); !v.empty())
+        r.url += "&tbs=" + url_escape(v);
+    r.headers.push_back({"accept", "application/json"});
+    r.headers.push_back({"user-agent", "mcp-cpp"});
+    r.headers.push_back({"x-api-key", std::string{key}});
+    return r;
+}
+
 // Perplexity Search. [D] POST /search ; Bearer ; max_results 1-20 without a
 // tier ; country is ISO alpha-2 ; search_language_filter is a list of ISO 639-1
 // codes ; search_recency_filter is hour|day|week|month|year ; search_type is
@@ -877,6 +916,7 @@ HttpRequest build_web_search_request(const WebSearchService& svc,
     if (id == "brave-api")      return req_brave_api(q, key);
     if (id == "exa-api")        return req_exa(q, key);
     if (id == "serper-api")     return req_serper(q, key);
+    if (id == "serply-api")     return req_serply(q, key);
     if (id == "perplexity-api") return req_perplexity(q, key);
     return {};   // unreachable for a catalogue id; empty url fails the send
 }
@@ -984,6 +1024,14 @@ JsonHits parse_json_hits(std::string_view service_id, std::string_view body,
         // Serper omits `organic` when Google found nothing.
         const json& o = field(j, "organic");
         out.hits = collect(o, "link", "title", {"snippet"}, count);
+    } else if (service_id == "serply-api") {
+        // Now and then a row's `link` is an opaque token, not a URL. [L]
+        // Such rows are dropped before counting, so they never become hits.
+        json rows = json::array();
+        for (const auto& r : *need(field(j, "results")))
+            if (r.is_object() && str_at(r, "link").rfind("http", 0) == 0)
+                rows.push_back(r);
+        out.hits = collect(rows, "link", "title", {"description"}, count);
     } else if (service_id == "perplexity-api") {
         out.hits = collect(*need(field(j, "results")), "url", "title",
                            {"snippet"}, count);

tests/web_search_services_test.cpp

@@ -99,7 +99,7 @@ const char* kTavily =
 
 TEST_CASE("web search: search catalogue is well formed") {
     const auto& all = web_search_services();
-    CHECK(all.size() == 10);
+    CHECK(all.size() == 11);
     for (const auto& s : all) {
         INFO(s.id);
         CHECK(!s.id.empty());
@@ -169,6 +169,7 @@ TEST_CASE("web search: dials offer each vendor's documented values") {
     CHECK(dial_ids("exa-api", "type") == exa_types);
     CHECK(dial_ids("exa-api", "freshness") == exa_ages);
     CHECK(dial_ids("serper-api", "freshness") == hourly_tbs);
+    CHECK(dial_ids("serply-api", "freshness") == hourly_tbs);
     // Brave's "no country" sits right under the default.
     const auto brave = dial_ids("brave-api", "country");
     REQUIRE(brave.size() >= 2);
@@ -470,6 +471,47 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") {
           ["search_domain_filter"] == deny);
 }
 
+TEST_CASE("web search: Serply request and reply") {
+    auto q = query(); q.language = "fr";
+    q.dials = {{"country", "NZ"}, {"freshness", "qdr:d"}};
+    auto r = build_web_search_request(svc("serply-api"), "sy-k", q);
+    CHECK(r.method == "GET");
+    CHECK(r.url == "https://api.serply.io/v1/search/?q=rust%20async&num=5"
+                   "&gl=nz&tbs=qdr%3Ad");
+    CHECK(header(r, "x-api-key") == "sy-k");
+    CHECK(!header(r, "user-agent").empty());
+    CHECK(r.url.find("sy-k") == std::string::npos);
+    CHECK(r.url.find("hl=") == std::string::npos);    // no language parameter
+    CHECK(svc("serply-api").max_count == 10);
+    CHECK(!has_native_exclude("serply-api"));
+
+    // No country set: gl=us, since without gl the proxy region is random.
+    // United States is the default row, not a second "US" row.
+    auto plain = build_web_search_request(svc("serply-api"), "sy-k", query());
+    CHECK(plain.url.find("&gl=us") != std::string::npos);
+    const auto countries = dial_ids("serply-api", "country");
+    REQUIRE(!countries.empty());
+    CHECK(countries[0].empty());
+    CHECK(std::find(countries.begin(), countries.end(), "US") == countries.end());
+
+    auto h = parse_json_hits("serply-api",
+        R"({"results":[{"title":"S","link":"https://s.test/","description":"sd",)"
+        R"("position":1,"result_type":"organic"}],"total":1})", 5);
+    REQUIRE(h.ok);
+    REQUIRE(h.hits.size() == 1);
+    CHECK(h.hits[0].url == "https://s.test/");
+    CHECK(h.hits[0].snippet == "sd");
+    // A 2xx without `results` is not the documented shape.
+    CHECK(!parse_json_hits("serply-api", R"({"detail":"x"})", 5).ok);
+    // A row whose `link` is a token, not a URL, is skipped and not counted.
+    auto t = parse_json_hits("serply-api",
+        R"({"results":[{"title":"T","link":"CAESBggA","description":"td"},)"
+        R"({"title":"S","link":"https://s.test/","description":"sd"}]})", 1);
+    REQUIRE(t.ok);
+    REQUIRE(t.hits.size() == 1);
+    CHECK(t.hits[0].url == "https://s.test/");
+}
+
 TEST_CASE("web search: reply parsers") {
     auto t = parse_json_hits("tavily-free", kTavily, 10);
     REQUIRE(t.ok);
@@ -582,6 +624,9 @@ TEST_CASE("web search: failures are told apart") {
     CHECK(classify_http(tavily, 401, "") == WebSearchFailure::BadKey);
     CHECK(classify_http(tavily, 432, "") == WebSearchFailure::QuotaExhausted);
     CHECK(classify_http(svc("exa-api"), 402, "") == WebSearchFailure::QuotaExhausted);
+    // Serply's gateway at a zero balance, expired free credits included.
+    CHECK(classify_http(svc("serply-api"), 402,
+          R"({"detail":"Insufficient prepaid credits"})") == WebSearchFailure::QuotaExhausted);
     // Brave's API answers a rejected token with 422, recorded live.
     CHECK(classify_http(svc("brave-api"), 422,
           R"({"error":{"code":"SUBSCRIPTION_TOKEN_INVALID"}})") == WebSearchFailure::BadKey);