feat(eval): add markkit_qa LLM eval via OpenRouter and Anthropic

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
+- `markkit_qa` LLM eval (`just eval-llm`): models answer 23 questions about the eval pages from
+  markkit markdown, main-content markdown or raw HTML, scored on correct answers with token usage
+  per input. Targets: Meta Muse Spark 1.3 contributor via OpenRouter and Claude Sonnet 5.5;
+  key-gated (`OPENROUTER_API_KEY`, `ANTHROPIC_API_KEY`, from Doppler).
 - XLSB (Excel binary workbook) support in the `spreadsheet` feature, via a new bounds-checked
   reader: values, shared strings, dates and durations, hidden-sheet skipping, row/column caps and
   the workbook title. Previously XLSB returned `UnsupportedFormat`.

crates/markkit-eval/Cargo.toml

@@ -23,10 +23,13 @@ path = "src/main.rs"
 # Mira eval framework (crates.io), imported as `mira`. `macros` provides the
 # #[eval] discovery attribute. In-process subject only: no mira-everruns.
 mira-eval = { version = "0.5", features = ["macros"] }
-# HTML only, no HTTP client: the eval never fetches (pages are embedded).
+# HTML only, no HTTP client: pages are embedded, markkit never fetches.
 markkit = { path = "../markkit", default-features = false, features = ["html"] }
 serde.workspace = true
 serde_json.workspace = true
+# LLM evals (markkit_qa) call Anthropic / OpenRouter. Eval crate only: the
+# markkit library itself stays without a network stack by default.
+reqwest.workspace = true
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/markkit-eval/data/qa-tasks.jsonl

@@ -0,0 +1,24 @@
+# markkit_qa dataset: one question per line over data/pages/. answers = accepted substrings (case-insensitive). See knowledge/operations/eval.md
+{"id": "news_article_q1", "page": "news_article.html", "question": "By how many acres will Riverside Park be expanded?", "answers": ["18"]}
+{"id": "news_article_q2", "page": "news_article.html", "question": "What was the council vote count?", "answers": ["7–2", "7-2", "7 to 2"]}
+{"id": "news_article_q3", "page": "news_article.html", "question": "How much is the project expected to cost?", "answers": ["14.2"]}
+{"id": "docs_page_q1", "page": "docs_page.html", "question": "What is the default request timeout?", "answers": ["30s", "30 s", "30 seconds"]}
+{"id": "docs_page_q2", "page": "docs_page.html", "question": "Which file does Tideline read its settings from?", "answers": ["tideline.toml"]}
+{"id": "docs_page_q3", "page": "docs_page.html", "question": "What happens to unknown configuration keys?", "answers": ["reject"]}
+{"id": "blog_post_q1", "page": "blog_post.html", "question": "How long did a clean CI build take before the migration?", "answers": ["47"]}
+{"id": "blog_post_q2", "page": "blog_post.html", "question": "What percentage of builds hit the remote cache after the migration?", "answers": ["92"]}
+{"id": "product_page_q1", "page": "product_page.html", "question": "How much does the backpack weigh?", "answers": ["1.2 kg", "1.2kg"]}
+{"id": "product_page_q2", "page": "product_page.html", "question": "What are the backpack's dimensions?", "answers": ["64"]}
+{"id": "product_page_q3", "page": "product_page.html", "question": "What is the price?", "answers": ["189"]}
+{"id": "recipe_q1", "page": "recipe.html", "question": "How much spaghetti does the recipe use?", "answers": ["350"]}
+{"id": "recipe_q2", "page": "recipe.html", "question": "What replaces the parmesan in the dairy-free version?", "answers": ["breadcrumb"]}
+{"id": "wiki_article_q1", "page": "wiki_article.html", "question": "Who was the architect of the lighthouse?", "answers": ["Sostratus"]}
+{"id": "wiki_article_q2", "page": "wiki_article.html", "question": "About how tall was the lighthouse?", "answers": ["100"]}
+{"id": "forum_thread_q1", "page": "forum_thread.html", "question": "How much extra space does the accepted solution use?", "answers": ["O(1)", "constant"]}
+{"id": "forum_thread_q2", "page": "forum_thread.html", "question": "What is the drawback of the recursive version?", "answers": ["stack", "recursion limit"]}
+{"id": "changelog_q1", "page": "changelog.html", "question": "Which issue number tracked the crash when pasting large images?", "answers": ["1182"]}
+{"id": "changelog_q2", "page": "changelog.html", "question": "What did the v2.3.1 security fix sanitize?", "answers": ["SVG"]}
+{"id": "data_table_q1", "page": "data_table.html", "question": "How many TWh did wind generate in Q4?", "answers": ["24.9"]}
+{"id": "data_table_q2", "page": "data_table.html", "question": "What was total generation in Q4, in TWh?", "answers": ["72.4"]}
+{"id": "legacy_layout_q1", "page": "legacy_layout.html", "question": "On which day does the club meet?", "answers": ["Tuesday"]}
+{"id": "legacy_layout_q2", "page": "legacy_layout.html", "question": "How much are the annual dues?", "answers": ["40"]}

crates/markkit-eval/src/lib.rs

@@ -12,3 +12,5 @@
 pub mod checks;
 pub mod dataset;
 pub mod mira_study;
+pub mod provider;
+pub mod qa;

crates/markkit-eval/src/main.rs

@@ -18,6 +18,11 @@ fn markkit_extract() -> Eval {
     mira_study::extract_eval()
 }
 
+#[eval]
+fn markkit_qa() -> Eval {
+    markkit_eval::qa::qa_eval()
+}
+
 fn main() -> std::io::Result<()> {
     mira::Study::registered().serve_blocking()
 }

crates/markkit-eval/src/provider.rs

@@ -0,0 +1,246 @@
+// Minimal single-turn LLM clients for the LLM-backed evals.
+//
+// Decisions:
+// - Two wire formats only: Anthropic Messages and OpenAI-style Chat
+//   Completions (OpenRouter). No tools, no streaming, one user turn: the QA
+//   eval asks one question about one document.
+// - Keys come from the environment (`ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`),
+//   typically injected by `doppler run --`. Keys never leave the study process
+//   (mira targets carry no secrets) and are never logged.
+// - Base URLs are overridable (`ANTHROPIC_BASE_URL`, `OPENROUTER_BASE_URL`) so
+//   tests and proxies can point elsewhere.
+// - Request bodies and response parsing are pure functions, unit-tested
+//   offline; only `complete` touches the network.
+
+use std::time::Duration;
+
+use serde_json::{Value, json};
+
+/// Provider ids understood by [`Client::for_provider`] (mira `Target::provider`).
+pub const ANTHROPIC: &str = "anthropic";
+pub const OPENROUTER: &str = "openrouter";
+
+const ANTHROPIC_URL: &str = "https://api.anthropic.com";
+const OPENROUTER_URL: &str = "https://openrouter.ai/api/v1";
+const ANTHROPIC_VERSION: &str = "2023-06-01";
+const MAX_TOKENS: u32 = 1024;
+const TIMEOUT: Duration = Duration::from_secs(300);
+
+/// One model reply.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct Completion {
+    pub text: String,
+    pub input_tokens: u64,
+    pub output_tokens: u64,
+}
+
+/// Wire format of a provider.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Wire {
+    Anthropic,
+    ChatCompletions,
+}
+
+/// A configured client for one provider.
+pub struct Client {
+    wire: Wire,
+    base_url: String,
+    key: String,
+    http: reqwest::Client,
+}
+
+impl Client {
+    /// Client for a mira target provider id, reading key and base URL from the
+    /// environment. Errors name the missing variable, never its value.
+    pub fn for_provider(provider: &str) -> Result<Self, String> {
+        let (wire, key_env, url_env, default_url) = match provider {
+            ANTHROPIC => (
+                Wire::Anthropic,
+                "ANTHROPIC_API_KEY",
+                "ANTHROPIC_BASE_URL",
+                ANTHROPIC_URL,
+            ),
+            OPENROUTER => (
+                Wire::ChatCompletions,
+                "OPENROUTER_API_KEY",
+                "OPENROUTER_BASE_URL",
+                OPENROUTER_URL,
+            ),
+            other => return Err(format!("unsupported provider {other:?}")),
+        };
+        let key = std::env::var(key_env)
+            .ok()
+            .filter(|k| !k.trim().is_empty())
+            .ok_or_else(|| format!("{key_env} not set"))?;
+        let base_url = std::env::var(url_env)
+            .ok()
+            .filter(|u| !u.trim().is_empty())
+            .unwrap_or_else(|| default_url.to_string());
+        let http = reqwest::Client::builder()
+            .timeout(TIMEOUT)
+            .build()
+            .map_err(|e| format!("http client: {e}"))?;
+        Ok(Self {
+            wire,
+            base_url: base_url.trim_end_matches('/').to_string(),
+            key,
+            http,
+        })
+    }
+
+    /// Send one system + user turn to `model`.
+    pub async fn complete(
+        &self,
+        model: &str,
+        system: &str,
+        user: &str,
+    ) -> Result<Completion, String> {
+        let body = request_body(self.wire, model, system, user);
+        let req = match self.wire {
+            Wire::Anthropic => self
+                .http
+                .post(format!("{}/v1/messages", self.base_url))
+                .header("x-api-key", &self.key)
+                .header("anthropic-version", ANTHROPIC_VERSION),
+            Wire::ChatCompletions => self
+                .http
+                .post(format!("{}/chat/completions", self.base_url))
+                .bearer_auth(&self.key),
+        };
+        let resp = req
+            .json(&body)
+            .send()
+            .await
+            .map_err(|e| format!("request failed: {e}"))?;
+        let status = resp.status();
+        let value: Value = resp
+            .json()
+            .await
+            .map_err(|e| format!("bad response body: {e}"))?;
+        if !status.is_success() {
+            return Err(format!("HTTP {status}: {}", error_message(&value)));
+        }
+        parse_response(self.wire, &value)
+    }
+}
+
+/// JSON request body for one system + user turn.
+pub fn request_body(wire: Wire, model: &str, system: &str, user: &str) -> Value {
+    match wire {
+        Wire::Anthropic => json!({
+            "model": model,
+            "max_tokens": MAX_TOKENS,
+            "system": system,
+            "messages": [{"role": "user", "content": user}],
+        }),
+        Wire::ChatCompletions => json!({
+            "model": model,
+            "max_tokens": MAX_TOKENS,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+        }),
+    }
+}
+
+/// Extract text and token usage from a successful response.
+pub fn parse_response(wire: Wire, v: &Value) -> Result<Completion, String> {
+    let u64_at = |p: &str| v.pointer(p).and_then(Value::as_u64).unwrap_or(0);
+    match wire {
+        Wire::Anthropic => {
+            let blocks = v
+                .get("content")
+                .and_then(Value::as_array)
+                .ok_or("response missing content")?;
+            let text = blocks
+                .iter()
+                .filter(|b| b.get("type").and_then(Value::as_str) == Some("text"))
+                .filter_map(|b| b.get("text").and_then(Value::as_str))
+                .collect::<Vec<_>>()
+                .join("");
+            Ok(Completion {
+                text,
+                input_tokens: u64_at("/usage/input_tokens"),
+                output_tokens: u64_at("/usage/output_tokens"),
+            })
+        }
+        Wire::ChatCompletions => {
+            let text = v
+                .pointer("/choices/0/message/content")
+                .and_then(Value::as_str)
+                .ok_or("response missing choices[0].message.content")?
+                .to_string();
+            Ok(Completion {
+                text,
+                input_tokens: u64_at("/usage/prompt_tokens"),
+                output_tokens: u64_at("/usage/completion_tokens"),
+            })
+        }
+    }
+}
+
+/// Best-effort error text from an error body (both wire formats use
+/// `error.message`).
+fn error_message(v: &Value) -> String {
+    v.pointer("/error/message")
+        .and_then(Value::as_str)
+        .map(str::to_string)
+        .unwrap_or_else(|| v.to_string().chars().take(300).collect())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn anthropic_body_and_parse() {
+        let b = request_body(Wire::Anthropic, "m", "sys", "hi");
+        assert_eq!(b["system"], "sys");
+        assert_eq!(b["messages"][0]["content"], "hi");
+        let v = json!({"content": [{"type": "thinking", "thinking": "x"}, {"type": "text", "text": "42"}],
+            "usage": {"input_tokens": 10, "output_tokens": 2}});
+        let c = parse_response(Wire::Anthropic, &v).unwrap();
+        assert_eq!(
+            c,
+            Completion {
+                text: "42".into(),
+                input_tokens: 10,
+                output_tokens: 2
+            }
+        );
+    }
+
+    #[test]
+    fn chat_body_and_parse() {
+        let b = request_body(Wire::ChatCompletions, "meta/x", "sys", "hi");
+        assert_eq!(b["messages"][0]["role"], "system");
+        assert_eq!(b["messages"][1]["content"], "hi");
+        let v = json!({"choices": [{"message": {"content": "24.9"}}],
+            "usage": {"prompt_tokens": 7, "completion_tokens": 3}});
+        let c = parse_response(Wire::ChatCompletions, &v).unwrap();
+        assert_eq!(c.text, "24.9");
+        assert_eq!((c.input_tokens, c.output_tokens), (7, 3));
+    }
+
+    #[test]
+    fn malformed_responses_are_errors() {
+        assert!(parse_response(Wire::Anthropic, &json!({})).is_err());
+        assert!(parse_response(Wire::ChatCompletions, &json!({"choices": []})).is_err());
+    }
+
+    #[test]
+    fn error_message_prefers_error_field_and_truncates() {
+        assert_eq!(
+            error_message(&json!({"error": {"message": "bad key"}})),
+            "bad key"
+        );
+        let long = json!({"x": "y".repeat(1000)});
+        assert!(error_message(&long).len() <= 300);
+    }
+
+    #[test]
+    fn unknown_provider_is_rejected() {
+        assert!(Client::for_provider("openai").is_err());
+    }
+}