Add initial Markdown Output support

README.md

@@ -87,6 +87,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 [Google search documentation](https://serpapi.com/search-api).
 More hands on examples are available below.
 
+### Response formats
+
+Use `search` for structured results decoded into a `serde_json::Value`:
+
+```rust
+let results = client.search(parameter).await?;
+```
+
+Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents.
+It returns clean headings, links and tables behind a YAML frontmatter while using
+roughly half the tokens of the JSON output:
+
+```rust
+let markdown = client.md(parameter).await?;
+```
+
+Use `html` when you need the raw response from the search engine:
+
+```rust
+let raw_html = client.html(parameter).await?;
+```
+
+`md` and `html` always force the output format, so any `output` search parameter is ignored.
+Archived results are also available as Markdown with `client.search_archive_md(&id).await?`.
+
+Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output).
+
 #### Documentations
 
  * [Full documentation on SerpApi.com](https://serpapi.com)
@@ -139,6 +166,12 @@ println!("{}", archived_results);
 assert_eq!(archive_id, search_id);
 ```
 
+The same archived search is available as Markdown:
+
+```rust
+let markdown = client.search_archive_md(&id).await.expect("request");
+```
+
 ### Account API
 ```rust
 let client = Client::new(HashMap::<String, String>::new());
@@ -150,11 +183,15 @@ let account = client.account(parameter).await.expect("request");
 It returns your account information.
 
 ### Technical features
+- Search results as JSON with `search`, Markdown with `md`, or raw search engine HTML with `html`
 - Dynamic JSON decoding using Serde JSON
 - Asyncronous HTTP request handle method using tokio and reqwest
 - Async tests using Tokio
 
 ### Changes log
+- Unreleased:
+  - Add Markdown output for LLMs and AI agents with `client.md(parameter)` and `client.search_archive_md(&id)`.
+  - `client.html(parameter)` now calls the search endpoint with `output=html` which returns the raw search engine HTML.
 - 1.1.0: Always reuse the same client object instead of creating a new one for each search.
   - This is a breaking change for the API because the client must be unwrapped in the main function.
     ```rust

README.md.erb

@@ -102,6 +102,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 [Google search documentation](https://serpapi.com/search-api).
 More hands on examples are available below.
 
+### Response formats
+
+Use `search` for structured results decoded into a `serde_json::Value`:
+
+```rust
+let results = client.search(parameter).await?;
+```
+
+Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents.
+It returns clean headings, links and tables behind a YAML frontmatter while using
+roughly half the tokens of the JSON output:
+
+```rust
+let markdown = client.md(parameter).await?;
+```
+
+Use `html` when you need the raw response from the search engine:
+
+```rust
+let raw_html = client.html(parameter).await?;
+```
+
+`md` and `html` always force the output format, so any `output` search parameter is ignored.
+Archived results are also available as Markdown with `client.search_archive_md(&id).await?`.
+
+Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output).
+
 #### Documentations
 
  * [Full documentation on SerpApi.com](https://serpapi.com)
@@ -154,6 +181,12 @@ println!("{}", archived_results);
 assert_eq!(archive_id, search_id);
 ```
 
+The same archived search is available as Markdown:
+
+```rust
+let markdown = client.search_archive_md(&id).await.expect("request");
+```
+
 ### Account API
 ```rust
 let client = Client::new(HashMap::<String, String>::new());
@@ -165,11 +198,15 @@ let account = client.account(parameter).await.expect("request");
 It returns your account information.
 
 ### Technical features
+- Search results as JSON with `search`, Markdown with `md`, or raw search engine HTML with `html`
 - Dynamic JSON decoding using Serde JSON
 - Asyncronous HTTP request handle method using tokio and reqwest
 - Async tests using Tokio
 
 ### Changes log
+- Unreleased:
+  - Add Markdown output for LLMs and AI agents with `client.md(parameter)` and `client.search_archive_md(&id)`.
+  - `client.html(parameter)` now calls the search endpoint with `output=html` which returns the raw search engine HTML.
 - 1.1.0: Always reuse the same client object instead of creating a new one for each search.
   - This is a breaking change for the API because the client must be unwrapped in the main function.
     ```rust

src/lib.rs

@@ -5,4 +5,7 @@
 //! SerpApi.com enables to do localized search, leverage advanced search engine features and a lot more...
 //! A completed documentation is available at [SerpApi](https://serpapi.com).
 //!
+//! Search results are available as JSON with `search`, as [Markdown](https://serpapi.com/markdown-output)
+//! optimized for LLMs and AI agents with `md`, or as raw search engine HTML with `html`.
+//!
 pub mod serpapi;

src/serpapi.rs

@@ -67,11 +67,47 @@ impl Client {
         Ok(results)
     }
 
-    // execute a search and return the result as raw HTML formatted as String
+    /// execute a search on serpapi.com
+    ///  and return the results as Markdown formatted as String.
+    ///  The Markdown output is optimized for LLMs and AI agents.
+    ///  It holds a YAML frontmatter followed by headings, links and tables
+    ///  using about half the tokens of the JSON output.
+    ///  see: https://serpapi.com/markdown-output
     /// # Arguments
-    /// * `parameter` html search parameter
+    ///  * `parameter` search parameter, the output is always set to md.
+    ///
     /// # Examples:
+    /// ```no_run
+    /// use std::collections::HashMap;
+    /// use serpapi::serpapi::Client;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///  let mut default = HashMap::<String, String>::new();
+    ///  default.insert("engine".to_string(), "google".to_string());
+    ///  default.insert("api_key".to_string(), "secret_api_key".to_string());
+    ///  // initialize the serpapi client
+    ///  let client = Client::new(default).unwrap();
+    ///  let mut parameter = HashMap::<String, String>::new();
+    ///  parameter.insert("q".to_string(), "coffee".to_string());
+    ///  // md returns the search results as a Markdown String.
+    ///  let markdown = client.md(parameter).await.expect("request");
+    ///  assert!(markdown.starts_with("---"));
+    /// }
     /// ```
+    pub async fn md(
+        &self,
+        parameter: HashMap<String, String>,
+    ) -> Result<String, Box<dyn std::error::Error>> {
+        let body = self.text("/search", force_output(parameter, "md")).await?;
+        Ok(body)
+    }
+
+    // execute a search and return the result as raw HTML formatted as String
+    /// # Arguments
+    /// * `parameter` html search parameter, the output is always set to html.
+    /// # Examples:
+    /// ```no_run
     /// use std::collections::HashMap;
     /// use serpapi::serpapi::Client;
     ///
@@ -93,7 +129,9 @@ impl Client {
         &self,
         parameter: HashMap<String, String>,
     ) -> Result<String, Box<dyn std::error::Error>> {
-        let body = self.get("/html", parameter).await?;
+        let body = self
+            .text("/search", force_output(parameter, "html"))
+            .await?;
         Ok(body)
     }
 
@@ -136,6 +174,21 @@ impl Client {
         Ok(results)
     }
 
+    /// Retrieve a search result from the Search Archive API as Markdown.
+    ///  see: https://serpapi.com/markdown-output
+    /// # Arguments
+    /// * `search_id` from the original search: `results["search_metadata"]["id"]`
+    pub async fn search_archive_md(
+        &self,
+        search_id: &str,
+    ) -> Result<String, Box<dyn std::error::Error>> {
+        let mut endpoint = "/searches/".to_string();
+        endpoint.push_str(search_id);
+        endpoint.push_str(".md");
+        let body = self.text(&endpoint, HashMap::new()).await?;
+        Ok(body)
+    }
+
     // Get account information using Account API
     pub async fn account(
         &self,
@@ -152,15 +205,44 @@ impl Client {
     ) -> Result<serde_json::Value, Box<dyn std::error::Error>> {
         let body = self.get(endpoint, parameter).await?;
         //debug: println!("Body:\n{}", body);
-        let value: serde_json::Value = serde_json::from_str(&body).unwrap();
+        //  a non JSON body means the output is html or md, see: Client::html and Client::md
+        let value: serde_json::Value = serde_json::from_str(&body)?;
         Ok(value)
     }
 
+    /// execute a request and return the body as String.
+    ///  SerpApi reports errors as JSON even when html or md output is requested,
+    ///  so a JSON response to a text request is reported as an error.
+    /// # Arguments
+    /// * `endpoint` HTTP service URI
+    /// * `parameter` search parameter
+    pub async fn text(
+        &self,
+        endpoint: &str,
+        parameter: HashMap<String, String>,
+    ) -> Result<String, Box<dyn std::error::Error>> {
+        let (content_type, body) = self.raw(endpoint, parameter).await?;
+        if content_type.starts_with("application/json") {
+            return Err(format!("search failed on {} with: {}", endpoint, body).into());
+        }
+        Ok(body)
+    }
+
     pub async fn get(
         &self,
         endpoint: &str,
         parameter: HashMap<String, String>,
     ) -> Result<String, Box<dyn std::error::Error>> {
+        let (_content_type, body) = self.raw(endpoint, parameter).await?;
+        Ok(body)
+    }
+
+    /// execute a request and return the content type along with the body.
+    async fn raw(
+        &self,
+        endpoint: &str,
+        parameter: HashMap<String, String>,
+    ) -> Result<(String, String), Box<dyn std::error::Error>> {
         let mut query = HashMap::<String, String>::new();
         query.insert("source".to_string(), "rust".to_string());
         for (key, value) in self.parameter.iter() {
@@ -175,7 +257,20 @@ impl Client {
         let mut url = HOST.to_string();
         url.push_str(endpoint);
         let res = self.http.get(url).query(&query).send().await?;
+        let content_type = res
+            .headers()
+            .get(reqwest::header::CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok())
+            .unwrap_or("")
+            .to_lowercase();
         let body = res.text().await?;
-        Ok(body)
+        Ok((content_type, body))
     }
 }
+
+/// force the output format whatever the caller provides.
+///  the format drives the return type: json -> serde_json::Value, html / md -> String.
+fn force_output(mut parameter: HashMap<String, String>, format: &str) -> HashMap<String, String> {
+    parameter.insert("output".to_string(), format.to_string());
+    parameter
+}

tests/serpapi-test.rs

@@ -56,6 +56,42 @@ async fn html() {
     assert!(html.len() > 100);
 }
 
+#[tokio::test]
+async fn markdown() {
+    let mut default = HashMap::<String, String>::new();
+    default.insert("engine".to_string(), "google".to_string());
+    default.insert("api_key".to_string(), api_key());
+
+    // initialize the search engine
+    let client = Client::new(default).unwrap();
+
+    let mut parameter = HashMap::<String, String>::new();
+    parameter.insert("q".to_string(), "coffee".to_string());
+    parameter.insert(
+        "location".to_string(),
+        "Austin, TX, Texas, United States".to_string(),
+    );
+    // md returns the search results as a Markdown String.
+    let markdown = client.md(parameter).await.expect("request");
+    // the Markdown output starts with a YAML frontmatter
+    assert!(markdown.starts_with("---"));
+    assert!(markdown.contains("coffee"));
+}
+
+#[tokio::test]
+async fn markdown_ignores_the_output_parameter() {
+    let mut default = HashMap::<String, String>::new();
+    default.insert("engine".to_string(), "google".to_string());
+    default.insert("api_key".to_string(), api_key());
+    let client = Client::new(default).unwrap();
+
+    let mut parameter = HashMap::<String, String>::new();
+    parameter.insert("q".to_string(), "coffee".to_string());
+    parameter.insert("output".to_string(), "json".to_string());
+    let markdown = client.md(parameter).await.expect("request");
+    assert!(markdown.starts_with("---"));
+}
+
 #[tokio::test]
 async fn location() {
     let default = HashMap::<String, String>::new();
@@ -107,3 +143,23 @@ async fn search_archive() {
     println!("{}", archived_results);
     assert_eq!(archive_id, search_id);
 }
+
+#[tokio::test]
+async fn search_archive_md() {
+    let mut default = HashMap::<String, String>::new();
+    default.insert("engine".to_string(), "google".to_string());
+    default.insert("api_key".to_string(), api_key());
+    let client = Client::new(default).unwrap();
+
+    let mut parameter = HashMap::<String, String>::new();
+    parameter.insert("q".to_string(), "coffee".to_string());
+    let initial_results = client.search(parameter).await.expect("request");
+    let id = initial_results["search_metadata"]["id"]
+        .as_str()
+        .expect("search id");
+
+    // search in archive as Markdown
+    let markdown = client.search_archive_md(id).await.expect("request");
+    assert!(markdown.starts_with("---"));
+    assert!(markdown.contains(id));
+}