diff --git a/README.md b/README.md index d51db54..0e2ee10 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,33 @@ async fn main() -> Result<(), Box> { [Google search documentation](https://serpapi.com/search-api). More hands on examples are available below. +### Response formats + +Use `search` for structured results decoded into a `serde_json::Value`: + +```rust +let results = client.search(parameter).await?; +``` + +Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents. +It returns clean headings, links and tables behind a YAML frontmatter while using +roughly half the tokens of the JSON output: + +```rust +let markdown = client.md(parameter).await?; +``` + +Use `html` when you need the raw response from the search engine: + +```rust +let raw_html = client.html(parameter).await?; +``` + +`md` and `html` always force the output format, so any `output` search parameter is ignored. +Archived results are also available as Markdown with `client.search_archive_md(&id).await?`. + +Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output). + #### Documentations * [Full documentation on SerpApi.com](https://serpapi.com) @@ -139,6 +166,12 @@ println!("{}", archived_results); assert_eq!(archive_id, search_id); ``` +The same archived search is available as Markdown: + +```rust +let markdown = client.search_archive_md(&id).await.expect("request"); +``` + ### Account API ```rust let client = Client::new(HashMap::::new()); @@ -150,11 +183,15 @@ let account = client.account(parameter).await.expect("request"); It returns your account information. ### Technical features +- Search results as JSON with `search`, Markdown with `md`, or raw search engine HTML with `html` - Dynamic JSON decoding using Serde JSON - Asyncronous HTTP request handle method using tokio and reqwest - Async tests using Tokio ### Changes log +- Unreleased: + - Add Markdown output for LLMs and AI agents with `client.md(parameter)` and `client.search_archive_md(&id)`. + - `client.html(parameter)` now calls the search endpoint with `output=html` which returns the raw search engine HTML. - 1.1.0: Always reuse the same client object instead of creating a new one for each search. - This is a breaking change for the API because the client must be unwrapped in the main function. ```rust diff --git a/README.md.erb b/README.md.erb index 1728d0a..2f258d1 100644 --- a/README.md.erb +++ b/README.md.erb @@ -102,6 +102,33 @@ async fn main() -> Result<(), Box> { [Google search documentation](https://serpapi.com/search-api). More hands on examples are available below. +### Response formats + +Use `search` for structured results decoded into a `serde_json::Value`: + +```rust +let results = client.search(parameter).await?; +``` + +Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents. +It returns clean headings, links and tables behind a YAML frontmatter while using +roughly half the tokens of the JSON output: + +```rust +let markdown = client.md(parameter).await?; +``` + +Use `html` when you need the raw response from the search engine: + +```rust +let raw_html = client.html(parameter).await?; +``` + +`md` and `html` always force the output format, so any `output` search parameter is ignored. +Archived results are also available as Markdown with `client.search_archive_md(&id).await?`. + +Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output). + #### Documentations * [Full documentation on SerpApi.com](https://serpapi.com) @@ -154,6 +181,12 @@ println!("{}", archived_results); assert_eq!(archive_id, search_id); ``` +The same archived search is available as Markdown: + +```rust +let markdown = client.search_archive_md(&id).await.expect("request"); +``` + ### Account API ```rust let client = Client::new(HashMap::::new()); @@ -165,11 +198,15 @@ let account = client.account(parameter).await.expect("request"); It returns your account information. ### Technical features +- Search results as JSON with `search`, Markdown with `md`, or raw search engine HTML with `html` - Dynamic JSON decoding using Serde JSON - Asyncronous HTTP request handle method using tokio and reqwest - Async tests using Tokio ### Changes log +- Unreleased: + - Add Markdown output for LLMs and AI agents with `client.md(parameter)` and `client.search_archive_md(&id)`. + - `client.html(parameter)` now calls the search endpoint with `output=html` which returns the raw search engine HTML. - 1.1.0: Always reuse the same client object instead of creating a new one for each search. - This is a breaking change for the API because the client must be unwrapped in the main function. ```rust diff --git a/src/lib.rs b/src/lib.rs index 34e812d..58fa421 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,4 +5,7 @@ //! SerpApi.com enables to do localized search, leverage advanced search engine features and a lot more... //! A completed documentation is available at [SerpApi](https://serpapi.com). //! +//! Search results are available as JSON with `search`, as [Markdown](https://serpapi.com/markdown-output) +//! optimized for LLMs and AI agents with `md`, or as raw search engine HTML with `html`. +//! pub mod serpapi; diff --git a/src/serpapi.rs b/src/serpapi.rs index ae9a5ba..759b168 100644 --- a/src/serpapi.rs +++ b/src/serpapi.rs @@ -67,11 +67,47 @@ impl Client { Ok(results) } - // execute a search and return the result as raw HTML formatted as String + /// execute a search on serpapi.com + /// and return the results as Markdown formatted as String. + /// The Markdown output is optimized for LLMs and AI agents. + /// It holds a YAML frontmatter followed by headings, links and tables + /// using about half the tokens of the JSON output. + /// see: https://serpapi.com/markdown-output /// # Arguments - /// * `parameter` html search parameter + /// * `parameter` search parameter, the output is always set to md. + /// /// # Examples: + /// ```no_run + /// use std::collections::HashMap; + /// use serpapi::serpapi::Client; + /// + /// #[tokio::main] + /// async fn main() { + /// let mut default = HashMap::::new(); + /// default.insert("engine".to_string(), "google".to_string()); + /// default.insert("api_key".to_string(), "secret_api_key".to_string()); + /// // initialize the serpapi client + /// let client = Client::new(default).unwrap(); + /// let mut parameter = HashMap::::new(); + /// parameter.insert("q".to_string(), "coffee".to_string()); + /// // md returns the search results as a Markdown String. + /// let markdown = client.md(parameter).await.expect("request"); + /// assert!(markdown.starts_with("---")); + /// } /// ``` + pub async fn md( + &self, + parameter: HashMap, + ) -> Result> { + let body = self.text("/search", force_output(parameter, "md")).await?; + Ok(body) + } + + // execute a search and return the result as raw HTML formatted as String + /// # Arguments + /// * `parameter` html search parameter, the output is always set to html. + /// # Examples: + /// ```no_run /// use std::collections::HashMap; /// use serpapi::serpapi::Client; /// @@ -93,7 +129,9 @@ impl Client { &self, parameter: HashMap, ) -> Result> { - let body = self.get("/html", parameter).await?; + let body = self + .text("/search", force_output(parameter, "html")) + .await?; Ok(body) } @@ -136,6 +174,21 @@ impl Client { Ok(results) } + /// Retrieve a search result from the Search Archive API as Markdown. + /// see: https://serpapi.com/markdown-output + /// # Arguments + /// * `search_id` from the original search: `results["search_metadata"]["id"]` + pub async fn search_archive_md( + &self, + search_id: &str, + ) -> Result> { + let mut endpoint = "/searches/".to_string(); + endpoint.push_str(search_id); + endpoint.push_str(".md"); + let body = self.text(&endpoint, HashMap::new()).await?; + Ok(body) + } + // Get account information using Account API pub async fn account( &self, @@ -152,15 +205,44 @@ impl Client { ) -> Result> { let body = self.get(endpoint, parameter).await?; //debug: println!("Body:\n{}", body); - let value: serde_json::Value = serde_json::from_str(&body).unwrap(); + // a non JSON body means the output is html or md, see: Client::html and Client::md + let value: serde_json::Value = serde_json::from_str(&body)?; Ok(value) } + /// execute a request and return the body as String. + /// SerpApi reports errors as JSON even when html or md output is requested, + /// so a JSON response to a text request is reported as an error. + /// # Arguments + /// * `endpoint` HTTP service URI + /// * `parameter` search parameter + pub async fn text( + &self, + endpoint: &str, + parameter: HashMap, + ) -> Result> { + let (content_type, body) = self.raw(endpoint, parameter).await?; + if content_type.starts_with("application/json") { + return Err(format!("search failed on {} with: {}", endpoint, body).into()); + } + Ok(body) + } + pub async fn get( &self, endpoint: &str, parameter: HashMap, ) -> Result> { + let (_content_type, body) = self.raw(endpoint, parameter).await?; + Ok(body) + } + + /// execute a request and return the content type along with the body. + async fn raw( + &self, + endpoint: &str, + parameter: HashMap, + ) -> Result<(String, String), Box> { let mut query = HashMap::::new(); query.insert("source".to_string(), "rust".to_string()); for (key, value) in self.parameter.iter() { @@ -175,7 +257,20 @@ impl Client { let mut url = HOST.to_string(); url.push_str(endpoint); let res = self.http.get(url).query(&query).send().await?; + let content_type = res + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_lowercase(); let body = res.text().await?; - Ok(body) + Ok((content_type, body)) } } + +/// force the output format whatever the caller provides. +/// the format drives the return type: json -> serde_json::Value, html / md -> String. +fn force_output(mut parameter: HashMap, format: &str) -> HashMap { + parameter.insert("output".to_string(), format.to_string()); + parameter +} diff --git a/tests/serpapi-test.rs b/tests/serpapi-test.rs index 8642e63..f67aede 100644 --- a/tests/serpapi-test.rs +++ b/tests/serpapi-test.rs @@ -56,6 +56,42 @@ async fn html() { assert!(html.len() > 100); } +#[tokio::test] +async fn markdown() { + let mut default = HashMap::::new(); + default.insert("engine".to_string(), "google".to_string()); + default.insert("api_key".to_string(), api_key()); + + // initialize the search engine + let client = Client::new(default).unwrap(); + + let mut parameter = HashMap::::new(); + parameter.insert("q".to_string(), "coffee".to_string()); + parameter.insert( + "location".to_string(), + "Austin, TX, Texas, United States".to_string(), + ); + // md returns the search results as a Markdown String. + let markdown = client.md(parameter).await.expect("request"); + // the Markdown output starts with a YAML frontmatter + assert!(markdown.starts_with("---")); + assert!(markdown.contains("coffee")); +} + +#[tokio::test] +async fn markdown_ignores_the_output_parameter() { + let mut default = HashMap::::new(); + default.insert("engine".to_string(), "google".to_string()); + default.insert("api_key".to_string(), api_key()); + let client = Client::new(default).unwrap(); + + let mut parameter = HashMap::::new(); + parameter.insert("q".to_string(), "coffee".to_string()); + parameter.insert("output".to_string(), "json".to_string()); + let markdown = client.md(parameter).await.expect("request"); + assert!(markdown.starts_with("---")); +} + #[tokio::test] async fn location() { let default = HashMap::::new(); @@ -107,3 +143,23 @@ async fn search_archive() { println!("{}", archived_results); assert_eq!(archive_id, search_id); } + +#[tokio::test] +async fn search_archive_md() { + let mut default = HashMap::::new(); + default.insert("engine".to_string(), "google".to_string()); + default.insert("api_key".to_string(), api_key()); + let client = Client::new(default).unwrap(); + + let mut parameter = HashMap::::new(); + parameter.insert("q".to_string(), "coffee".to_string()); + let initial_results = client.search(parameter).await.expect("request"); + let id = initial_results["search_metadata"]["id"] + .as_str() + .expect("search id"); + + // search in archive as Markdown + let markdown = client.search_archive_md(id).await.expect("request"); + assert!(markdown.starts_with("---")); + assert!(markdown.contains(id)); +}