diff --git a/CHANGELOG.md b/CHANGELOG.md index 562590d..3bfad2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +- `markkit_qa` LLM eval (`just eval-llm`): models answer 23 questions about the eval pages from + markkit markdown, main-content markdown or raw HTML, scored on correct answers with token usage + per input. Targets: Meta Muse Spark 1.3 contributor via OpenRouter and Claude Sonnet 5.5; + key-gated (`OPENROUTER_API_KEY`, `ANTHROPIC_API_KEY`, from Doppler). - XLSB (Excel binary workbook) support in the `spreadsheet` feature, via a new bounds-checked reader: values, shared strings, dates and durations, hidden-sheet skipping, row/column caps and the workbook title. Previously XLSB returned `UnsupportedFormat`. diff --git a/Cargo.lock b/Cargo.lock index 1021dc3..237374f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,6 +1328,7 @@ version = "0.1.1" dependencies = [ "markkit", "mira-eval", + "reqwest", "serde", "serde_json", "tokio", diff --git a/crates/markkit-eval/Cargo.toml b/crates/markkit-eval/Cargo.toml index 657ab01..8a8f84b 100644 --- a/crates/markkit-eval/Cargo.toml +++ b/crates/markkit-eval/Cargo.toml @@ -23,10 +23,13 @@ path = "src/main.rs" # Mira eval framework (crates.io), imported as `mira`. `macros` provides the # #[eval] discovery attribute. In-process subject only: no mira-everruns. mira-eval = { version = "0.5", features = ["macros"] } -# HTML only, no HTTP client: the eval never fetches (pages are embedded). +# HTML only, no HTTP client: pages are embedded, markkit never fetches. markkit = { path = "../markkit", default-features = false, features = ["html"] } serde.workspace = true serde_json.workspace = true +# LLM evals (markkit_qa) call Anthropic / OpenRouter. Eval crate only: the +# markkit library itself stays without a network stack by default. +reqwest.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/markkit-eval/data/qa-tasks.jsonl b/crates/markkit-eval/data/qa-tasks.jsonl new file mode 100644 index 0000000..20fcc93 --- /dev/null +++ b/crates/markkit-eval/data/qa-tasks.jsonl @@ -0,0 +1,24 @@ +# markkit_qa dataset: one question per line over data/pages/. answers = accepted substrings (case-insensitive). See knowledge/operations/eval.md +{"id": "news_article_q1", "page": "news_article.html", "question": "By how many acres will Riverside Park be expanded?", "answers": ["18"]} +{"id": "news_article_q2", "page": "news_article.html", "question": "What was the council vote count?", "answers": ["7–2", "7-2", "7 to 2"]} +{"id": "news_article_q3", "page": "news_article.html", "question": "How much is the project expected to cost?", "answers": ["14.2"]} +{"id": "docs_page_q1", "page": "docs_page.html", "question": "What is the default request timeout?", "answers": ["30s", "30 s", "30 seconds"]} +{"id": "docs_page_q2", "page": "docs_page.html", "question": "Which file does Tideline read its settings from?", "answers": ["tideline.toml"]} +{"id": "docs_page_q3", "page": "docs_page.html", "question": "What happens to unknown configuration keys?", "answers": ["reject"]} +{"id": "blog_post_q1", "page": "blog_post.html", "question": "How long did a clean CI build take before the migration?", "answers": ["47"]} +{"id": "blog_post_q2", "page": "blog_post.html", "question": "What percentage of builds hit the remote cache after the migration?", "answers": ["92"]} +{"id": "product_page_q1", "page": "product_page.html", "question": "How much does the backpack weigh?", "answers": ["1.2 kg", "1.2kg"]} +{"id": "product_page_q2", "page": "product_page.html", "question": "What are the backpack's dimensions?", "answers": ["64"]} +{"id": "product_page_q3", "page": "product_page.html", "question": "What is the price?", "answers": ["189"]} +{"id": "recipe_q1", "page": "recipe.html", "question": "How much spaghetti does the recipe use?", "answers": ["350"]} +{"id": "recipe_q2", "page": "recipe.html", "question": "What replaces the parmesan in the dairy-free version?", "answers": ["breadcrumb"]} +{"id": "wiki_article_q1", "page": "wiki_article.html", "question": "Who was the architect of the lighthouse?", "answers": ["Sostratus"]} +{"id": "wiki_article_q2", "page": "wiki_article.html", "question": "About how tall was the lighthouse?", "answers": ["100"]} +{"id": "forum_thread_q1", "page": "forum_thread.html", "question": "How much extra space does the accepted solution use?", "answers": ["O(1)", "constant"]} +{"id": "forum_thread_q2", "page": "forum_thread.html", "question": "What is the drawback of the recursive version?", "answers": ["stack", "recursion limit"]} +{"id": "changelog_q1", "page": "changelog.html", "question": "Which issue number tracked the crash when pasting large images?", "answers": ["1182"]} +{"id": "changelog_q2", "page": "changelog.html", "question": "What did the v2.3.1 security fix sanitize?", "answers": ["SVG"]} +{"id": "data_table_q1", "page": "data_table.html", "question": "How many TWh did wind generate in Q4?", "answers": ["24.9"]} +{"id": "data_table_q2", "page": "data_table.html", "question": "What was total generation in Q4, in TWh?", "answers": ["72.4"]} +{"id": "legacy_layout_q1", "page": "legacy_layout.html", "question": "On which day does the club meet?", "answers": ["Tuesday"]} +{"id": "legacy_layout_q2", "page": "legacy_layout.html", "question": "How much are the annual dues?", "answers": ["40"]} diff --git a/crates/markkit-eval/src/lib.rs b/crates/markkit-eval/src/lib.rs index 8d185c3..d408210 100644 --- a/crates/markkit-eval/src/lib.rs +++ b/crates/markkit-eval/src/lib.rs @@ -12,3 +12,5 @@ pub mod checks; pub mod dataset; pub mod mira_study; +pub mod provider; +pub mod qa; diff --git a/crates/markkit-eval/src/main.rs b/crates/markkit-eval/src/main.rs index a7b837b..b1688a4 100644 --- a/crates/markkit-eval/src/main.rs +++ b/crates/markkit-eval/src/main.rs @@ -18,6 +18,11 @@ fn markkit_extract() -> Eval { mira_study::extract_eval() } +#[eval] +fn markkit_qa() -> Eval { + markkit_eval::qa::qa_eval() +} + fn main() -> std::io::Result<()> { mira::Study::registered().serve_blocking() } diff --git a/crates/markkit-eval/src/provider.rs b/crates/markkit-eval/src/provider.rs new file mode 100644 index 0000000..c098e42 --- /dev/null +++ b/crates/markkit-eval/src/provider.rs @@ -0,0 +1,246 @@ +// Minimal single-turn LLM clients for the LLM-backed evals. +// +// Decisions: +// - Two wire formats only: Anthropic Messages and OpenAI-style Chat +// Completions (OpenRouter). No tools, no streaming, one user turn: the QA +// eval asks one question about one document. +// - Keys come from the environment (`ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`), +// typically injected by `doppler run --`. Keys never leave the study process +// (mira targets carry no secrets) and are never logged. +// - Base URLs are overridable (`ANTHROPIC_BASE_URL`, `OPENROUTER_BASE_URL`) so +// tests and proxies can point elsewhere. +// - Request bodies and response parsing are pure functions, unit-tested +// offline; only `complete` touches the network. + +use std::time::Duration; + +use serde_json::{Value, json}; + +/// Provider ids understood by [`Client::for_provider`] (mira `Target::provider`). +pub const ANTHROPIC: &str = "anthropic"; +pub const OPENROUTER: &str = "openrouter"; + +const ANTHROPIC_URL: &str = "https://api.anthropic.com"; +const OPENROUTER_URL: &str = "https://openrouter.ai/api/v1"; +const ANTHROPIC_VERSION: &str = "2023-06-01"; +const MAX_TOKENS: u32 = 1024; +const TIMEOUT: Duration = Duration::from_secs(300); + +/// One model reply. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Completion { + pub text: String, + pub input_tokens: u64, + pub output_tokens: u64, +} + +/// Wire format of a provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Wire { + Anthropic, + ChatCompletions, +} + +/// A configured client for one provider. +pub struct Client { + wire: Wire, + base_url: String, + key: String, + http: reqwest::Client, +} + +impl Client { + /// Client for a mira target provider id, reading key and base URL from the + /// environment. Errors name the missing variable, never its value. + pub fn for_provider(provider: &str) -> Result { + let (wire, key_env, url_env, default_url) = match provider { + ANTHROPIC => ( + Wire::Anthropic, + "ANTHROPIC_API_KEY", + "ANTHROPIC_BASE_URL", + ANTHROPIC_URL, + ), + OPENROUTER => ( + Wire::ChatCompletions, + "OPENROUTER_API_KEY", + "OPENROUTER_BASE_URL", + OPENROUTER_URL, + ), + other => return Err(format!("unsupported provider {other:?}")), + }; + let key = std::env::var(key_env) + .ok() + .filter(|k| !k.trim().is_empty()) + .ok_or_else(|| format!("{key_env} not set"))?; + let base_url = std::env::var(url_env) + .ok() + .filter(|u| !u.trim().is_empty()) + .unwrap_or_else(|| default_url.to_string()); + let http = reqwest::Client::builder() + .timeout(TIMEOUT) + .build() + .map_err(|e| format!("http client: {e}"))?; + Ok(Self { + wire, + base_url: base_url.trim_end_matches('/').to_string(), + key, + http, + }) + } + + /// Send one system + user turn to `model`. + pub async fn complete( + &self, + model: &str, + system: &str, + user: &str, + ) -> Result { + let body = request_body(self.wire, model, system, user); + let req = match self.wire { + Wire::Anthropic => self + .http + .post(format!("{}/v1/messages", self.base_url)) + .header("x-api-key", &self.key) + .header("anthropic-version", ANTHROPIC_VERSION), + Wire::ChatCompletions => self + .http + .post(format!("{}/chat/completions", self.base_url)) + .bearer_auth(&self.key), + }; + let resp = req + .json(&body) + .send() + .await + .map_err(|e| format!("request failed: {e}"))?; + let status = resp.status(); + let value: Value = resp + .json() + .await + .map_err(|e| format!("bad response body: {e}"))?; + if !status.is_success() { + return Err(format!("HTTP {status}: {}", error_message(&value))); + } + parse_response(self.wire, &value) + } +} + +/// JSON request body for one system + user turn. +pub fn request_body(wire: Wire, model: &str, system: &str, user: &str) -> Value { + match wire { + Wire::Anthropic => json!({ + "model": model, + "max_tokens": MAX_TOKENS, + "system": system, + "messages": [{"role": "user", "content": user}], + }), + Wire::ChatCompletions => json!({ + "model": model, + "max_tokens": MAX_TOKENS, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + }), + } +} + +/// Extract text and token usage from a successful response. +pub fn parse_response(wire: Wire, v: &Value) -> Result { + let u64_at = |p: &str| v.pointer(p).and_then(Value::as_u64).unwrap_or(0); + match wire { + Wire::Anthropic => { + let blocks = v + .get("content") + .and_then(Value::as_array) + .ok_or("response missing content")?; + let text = blocks + .iter() + .filter(|b| b.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|b| b.get("text").and_then(Value::as_str)) + .collect::>() + .join(""); + Ok(Completion { + text, + input_tokens: u64_at("/usage/input_tokens"), + output_tokens: u64_at("/usage/output_tokens"), + }) + } + Wire::ChatCompletions => { + let text = v + .pointer("/choices/0/message/content") + .and_then(Value::as_str) + .ok_or("response missing choices[0].message.content")? + .to_string(); + Ok(Completion { + text, + input_tokens: u64_at("/usage/prompt_tokens"), + output_tokens: u64_at("/usage/completion_tokens"), + }) + } + } +} + +/// Best-effort error text from an error body (both wire formats use +/// `error.message`). +fn error_message(v: &Value) -> String { + v.pointer("/error/message") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or_else(|| v.to_string().chars().take(300).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn anthropic_body_and_parse() { + let b = request_body(Wire::Anthropic, "m", "sys", "hi"); + assert_eq!(b["system"], "sys"); + assert_eq!(b["messages"][0]["content"], "hi"); + let v = json!({"content": [{"type": "thinking", "thinking": "x"}, {"type": "text", "text": "42"}], + "usage": {"input_tokens": 10, "output_tokens": 2}}); + let c = parse_response(Wire::Anthropic, &v).unwrap(); + assert_eq!( + c, + Completion { + text: "42".into(), + input_tokens: 10, + output_tokens: 2 + } + ); + } + + #[test] + fn chat_body_and_parse() { + let b = request_body(Wire::ChatCompletions, "meta/x", "sys", "hi"); + assert_eq!(b["messages"][0]["role"], "system"); + assert_eq!(b["messages"][1]["content"], "hi"); + let v = json!({"choices": [{"message": {"content": "24.9"}}], + "usage": {"prompt_tokens": 7, "completion_tokens": 3}}); + let c = parse_response(Wire::ChatCompletions, &v).unwrap(); + assert_eq!(c.text, "24.9"); + assert_eq!((c.input_tokens, c.output_tokens), (7, 3)); + } + + #[test] + fn malformed_responses_are_errors() { + assert!(parse_response(Wire::Anthropic, &json!({})).is_err()); + assert!(parse_response(Wire::ChatCompletions, &json!({"choices": []})).is_err()); + } + + #[test] + fn error_message_prefers_error_field_and_truncates() { + assert_eq!( + error_message(&json!({"error": {"message": "bad key"}})), + "bad key" + ); + let long = json!({"x": "y".repeat(1000)}); + assert!(error_message(&long).len() <= 300); + } + + #[test] + fn unknown_provider_is_rejected() { + assert!(Client::for_provider("openai").is_err()); + } +} diff --git a/crates/markkit-eval/src/qa.rs b/crates/markkit-eval/src/qa.rs new file mode 100644 index 0000000..e1bfd0c --- /dev/null +++ b/crates/markkit-eval/src/qa.rs @@ -0,0 +1,326 @@ +// markkit_qa: downstream usefulness of markkit output for an LLM. +// +// A target model answers a factual question about a page, given the page as +// context. The `input` axis picks what the model reads: markkit's markdown +// (`markdown`), markkit with main-content detection (`main_content`), or the +// raw page (`html`, the baseline). Scored by whether the reply contains an +// accepted answer; the transcript carries input tokens, so reports show the +// accuracy/token trade-off per input. +// +// Decisions: +// - Questions reuse the `markkit_extract` pages (data/pages/), one sample per +// question, so a wrong answer points at one fact on one page. +// - Answers are accepted substrings, matched case-insensitively with +// whitespace collapsed. Short factual answers keep this deterministic; no +// LLM judge needed to score. +// - Targets are key-gated (`ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`), so an +// offline run skips every case and stays green. Keys come from Doppler +// (`doppler run -- just eval-llm`). +// - Provider/HTTP failures are infra errors (N/A), never scored against the +// model or markkit. A markkit extraction failure is scored (empty context). + +use std::future::Future; + +use markkit::Markkit; +use mira::subject::{Subject, subject_fn}; +use mira::{Eval, Sample, Score, Target, Transcript}; +use serde::{Deserialize, Serialize}; + +use crate::dataset::page; +use crate::provider::{self, Client, Completion}; + +/// Embedded question list. +pub const QA_TASKS: &str = include_str!("../data/qa-tasks.jsonl"); + +/// Values of the `input` axis. +pub const INPUTS: [&str; 3] = ["markdown", "main_content", "html"]; + +/// Scorer name. +pub const ANSWER_SCORER: &str = "answer_correct"; + +/// System prompt for every question. +pub const SYSTEM_PROMPT: &str = "Answer the question using only the document. \ +Reply with the shortest possible answer and no explanation. \ +If the document does not contain the answer, reply UNKNOWN."; + +/// One question about one page. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QaTask { + pub id: String, + /// Page file name under `data/pages/`. + pub page: String, + pub question: String, + /// Accepted answer substrings (any one matches). + pub answers: Vec, +} + +/// Default model matrix. Decision (2026-10-27, maintainer): Meta Muse Spark +/// 1.3 contributor via OpenRouter, plus Anthropic direct. Keys from Doppler. +pub fn default_targets() -> Vec { + vec![ + Target::anthropic("claude-sonnet-5-5"), + Target::cloud( + provider::OPENROUTER, + "meta/muse-spark-1.3-contributor", + "OPENROUTER_API_KEY", + ), + ] +} + +/// Parse JSONL questions (blank and `#`/`//` lines skipped; malformed = error). +pub fn parse_qa(text: &str) -> Result, String> { + text.lines() + .enumerate() + .map(|(i, l)| (i + 1, l.trim())) + .filter(|(_, l)| !l.is_empty() && !l.starts_with('#') && !l.starts_with("//")) + .map(|(n, l)| serde_json::from_str(l).map_err(|e| format!("line {n}: {e}"))) + .collect() +} + +/// One sample per question; the page rides in `files`, the task in metadata. +pub fn qa_samples(jsonl: &str) -> Result, String> { + parse_qa(jsonl)? + .into_iter() + .map(|t| { + let html = page(&t.page).ok_or_else(|| format!("{}: unknown page {}", t.id, t.page))?; + if t.answers.iter().all(|a| a.trim().is_empty()) { + return Err(format!("{}: no answers", t.id)); + } + let page_tag = t.page.trim_end_matches(".html").to_string(); + let value = serde_json::to_value(&t).map_err(|e| e.to_string())?; + Ok(Sample::new(&t.id, &t.question) + .tag(page_tag) + .meta("page", t.page.clone()) + .meta("qa", value) + .file(crate::mira_study::PAGE_FILE, html)) + }) + .collect() +} + +fn qa_from_sample(sample: &Sample) -> Result { + let v = sample + .metadata + .get("qa") + .cloned() + .ok_or("sample missing qa metadata")?; + serde_json::from_value(v).map_err(|e| format!("bad qa metadata: {e}")) +} + +/// Document text the model reads for an `input` axis value. `Err` for an +/// unknown input; a markkit failure yields an empty document (scored). +pub fn document_for(input: &str, html: &str) -> Result { + let markkit = match input { + "html" => return Ok(html.to_string()), + "markdown" => Markkit::builder().build(), + "main_content" => Markkit::builder().main_content(true).build(), + other => return Err(format!("unknown input {other:?}")), + }; + Ok(markkit.extract_html_sync(html).unwrap_or_default()) +} + +/// User turn: the document, then the question. +pub fn user_prompt(document: &str, question: &str) -> String { + format!("\n{document}\n\n\nQuestion: {question}") +} + +/// Run one QA case with `ask` standing in for the model call. +pub async fn run_qa(sample: &Sample, input: &str, ask: F) -> Transcript +where + F: FnOnce(String) -> Fut, + Fut: Future>, +{ + let task = match qa_from_sample(sample) { + Ok(t) => t, + Err(e) => return Transcript::infra_error(e), + }; + let Some(html) = sample.files.get(crate::mira_study::PAGE_FILE) else { + return Transcript::infra_error("sample missing page.html"); + }; + let document = match document_for(input, html) { + Ok(d) => d, + Err(e) => return Transcript::infra_error(e), + }; + let doc_bytes = document.len(); + let start = std::time::Instant::now(); + let completion = match ask(user_prompt(&document, &task.question)).await { + Ok(c) => c, + Err(e) => return Transcript::infra_error(format!("model call failed: {e}")), + }; + let mut t = Transcript::response(completion.text.trim().to_string()); + t.iterations = 1; + t.usage.input_tokens = completion.input_tokens; + t.usage.output_tokens = completion.output_tokens; + t.timing.duration_ms = start.elapsed().as_millis() as u64; + t.record_metric("doc_bytes", doc_bytes as f64); + t.record_metric("input_tokens", completion.input_tokens as f64); + t +} + +/// Subject: build the case's document and ask the case's target model. +fn qa_subject() -> impl Subject { + subject_fn(|sample: Sample, cx| async move { + let input = cx.param("input").unwrap_or("markdown").to_string(); + let client = match Client::for_provider(&cx.target.provider) { + Ok(c) => c, + Err(e) => return Transcript::infra_error(e), + }; + let model = cx.target.model.clone(); + run_qa(&sample, &input, |user| async move { + client.complete(&model, SYSTEM_PROMPT, &user).await + }) + .await + }) +} + +/// Lowercase and collapse whitespace, for answer matching. +fn normalize(s: &str) -> String { + s.split_whitespace() + .collect::>() + .join(" ") + .to_lowercase() +} + +/// True when `reply` contains any accepted answer. +pub fn answer_matches(reply: &str, answers: &[String]) -> bool { + let reply = normalize(reply); + answers + .iter() + .map(|a| normalize(a)) + .any(|a| !a.is_empty() && reply.contains(&a)) +} + +/// Pass iff the reply contains an accepted answer. +pub fn answer_scorer() -> Box { + mira::scorer::scorer(ANSWER_SCORER, |sample: &Sample, t: &Transcript| { + if t.errored_infra() { + return Score::na(ANSWER_SCORER, "subject infra error"); + } + let task = match qa_from_sample(sample) { + Ok(t) => t, + Err(e) => return Score::na(ANSWER_SCORER, e), + }; + let reply: String = t.final_response.chars().take(200).collect(); + if answer_matches(&t.final_response, &task.answers) { + Score::pass(ANSWER_SCORER, format!("answered {reply:?}")) + } else { + Score::fail( + ANSWER_SCORER, + format!("answered {reply:?}; expected one of {:?}", task.answers), + ) + } + }) +} + +/// The QA eval. Select inputs with `--axis input=markdown`, pages with +/// `--tag `, models with `--targets`. +pub fn qa_eval() -> Eval { + let samples = qa_samples(QA_TASKS).expect("embedded QA dataset is valid (tested)"); + let mut b = Eval::new("markkit_qa") + .describe("LLM answers questions from markkit markdown vs raw HTML") + .subject(qa_subject()) + .scorer(answer_scorer()) + .max_turns(1) + .axis("input", INPUTS) + .targets(default_targets()); + for s in samples { + b = b.add_sample(s); + } + b.build() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeSet; + + fn ok(text: &str) -> Result { + Ok(Completion { + text: text.into(), + input_tokens: 100, + output_tokens: 3, + }) + } + + fn sample(id: &str) -> Sample { + qa_samples(QA_TASKS) + .unwrap() + .into_iter() + .find(|s| s.id == id) + .unwrap() + } + + #[test] + fn dataset_is_valid_and_answers_are_on_the_page() { + let tasks = parse_qa(QA_TASKS).unwrap(); + assert!(tasks.len() >= 20); + let mut ids = BTreeSet::new(); + for t in &tasks { + assert!(ids.insert(t.id.clone()), "duplicate {}", t.id); + let html = page(&t.page).unwrap_or_else(|| panic!("{}: page", t.id)); + // At least one accepted answer is literally in the source, so the + // question is answerable from the raw page. + let text = normalize(&html.replace("–", "–")); + assert!( + t.answers.iter().any(|a| text.contains(&normalize(a))), + "{}: no answer found on page", + t.id + ); + } + assert_eq!(qa_samples(QA_TASKS).unwrap().len(), tasks.len()); + } + + #[test] + fn answer_matching() { + let a = vec!["24.9".to_string(), "7–2".to_string()]; + assert!(answer_matches("Wind: 24.9 TWh", &a)); + assert!(answer_matches(" 7–2 ", &a)); + assert!(!answer_matches("UNKNOWN", &a)); + assert!(answer_matches("Tideline.TOML", &["tideline.toml".into()])); + assert!(!answer_matches("anything", &[" ".into()])); + } + + #[test] + fn documents_per_input() { + let html = page("data_table.html").unwrap(); + assert_eq!(document_for("html", html).unwrap(), html); + let md = document_for("markdown", html).unwrap(); + assert!(md.contains("| Wind | 21.4 |") && !md.contains("")); + assert!(document_for("main_content", html).is_ok()); + assert!(document_for("pdf", html).is_err()); + } + + #[tokio::test] + async fn run_qa_scores_right_and_wrong_answers() { + let s = sample("data_table_q1"); + let t = run_qa(&s, "markdown", |user| async move { + assert!(user.contains("Question: How many TWh did wind generate in Q4?")); + assert!(user.contains("| Wind |")); + ok("24.9") + }) + .await; + assert_eq!(t.usage.input_tokens, 100); + assert!(t.metrics.contains_key("doc_bytes")); + let scorer = answer_scorer(); + assert!(scorer.score(&s, &t).await.pass); + let wrong = run_qa(&s, "markdown", |_| async { ok("12.8") }).await; + assert!(!scorer.score(&s, &wrong).await.pass); + } + + #[tokio::test] + async fn provider_failure_and_bad_input_are_infra() { + let s = sample("data_table_q1"); + let t = run_qa(&s, "markdown", |_| async { Err("HTTP 401".to_string()) }).await; + assert!(t.errored_infra()); + assert!(answer_scorer().score(&s, &t).await.na); + let t = run_qa(&s, "bogus", |_| async { ok("24.9") }).await; + assert!(t.errored_infra()); + } + + #[test] + fn targets_are_key_gated() { + for t in default_targets() { + assert!([provider::ANTHROPIC, provider::OPENROUTER].contains(&t.provider.as_str())); + } + let _ = qa_eval(); + } +} diff --git a/justfile b/justfile index b45cc65..33f7250 100644 --- a/justfile +++ b/justfile @@ -136,6 +136,12 @@ eval-list: eval *args: mira run --study-bin markkit-eval markkit_extract {{args}} +# Run the LLM QA eval (markkit markdown vs raw HTML as model context). Needs +# ANTHROPIC_API_KEY / OPENROUTER_API_KEY; unkeyed targets are skipped. Keys live +# in Doppler: doppler run -- just eval-llm +eval-llm *args: + mira run --study-bin markkit-eval markkit_qa {{args}} + # ─── Security & Audit ─────────────────────────────────────── # Run cargo-deny checks diff --git a/knowledge/log.md b/knowledge/log.md index f0d3d06..7c8a573 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -23,6 +23,7 @@ * **Email input**: `email` feature on mail-parser 0.11 (1.0 was two days old and a rewrite, so not adopted yet). Headers folded against line injection; attachments listed, never parsed; shares the call's `Limits` budget. T7/S18 extended. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). * **ODT input**: `odt` feature reusing the DOCX archive/XML guards; style inheritance for bold/italic/strike and outline levels, lists, tables with capped repeats; notes, annotations and tracked changes skipped; shares the call's `Limits` budget; hidden text (`text:display="none"` styles/sections, hidden fields) skipped per S20. T6 and T9 cover ODT. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). * **EPUB input**: `epub` feature (implies `html`); spine chapters through the HTML parser, each once, traversal-safe href resolution, DRM-encrypted spine documents rejected; shares the call's `Limits` budget and hidden-content rules. T6 covers EPUB. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). +* **LLM QA eval**: `markkit_qa` asks target models (OpenRouter `meta/muse-spark-1.3-contributor`, Anthropic `claude-sonnet-5-5`; keys from Doppler) 23 questions over the eval pages with an `input` axis (markdown, main_content, raw html). See [Evaluation Harness](operations/eval.md). ## 2026-10-26 diff --git a/knowledge/operations/eval.md b/knowledge/operations/eval.md index 6d0e214..a3deebd 100644 --- a/knowledge/operations/eval.md +++ b/knowledge/operations/eval.md @@ -12,11 +12,14 @@ tags: ## Status -`markkit_extract` implemented: an offline, deterministic extraction-quality -eval on the [mira](https://github.com/everruns/mira) eval framework. LLM-backed -evals (LLM-as-judge, downstream QA over a model matrix) are planned as separate -evals in the same study; see [Planned evals](#planned-evals). Modeled on -everruns/bashkit's `bashkit-eval` study. +Two evals on the [mira](https://github.com/everruns/mira) eval framework, +modeled on everruns/bashkit's `bashkit-eval` study: + +- `markkit_extract`: offline, deterministic extraction quality. +- `markkit_qa`: LLM answers questions from markkit output vs raw HTML, over a + key-gated model matrix; see [LLM QA Eval](#llm-qa-eval-markkit_qa). + +LLM-as-judge is planned; see [Planned evals](#planned-evals). ## Architecture @@ -178,14 +181,42 @@ mira report `just eval`, `just eval-list` wrap these. `mira run` exits non-zero while any page fails, which is expected until the gaps above close. +## LLM QA Eval (`markkit_qa`) + +Downstream usefulness: does an LLM answer correctly from markkit's output, and +at what token cost? Code: `src/qa.rs` (subject, scorer, eval) and +`src/provider.rs` (HTTP clients). + +- **Samples**: `data/qa-tasks.jsonl`, 23 factual questions over the same ten + pages (`{"id", "page", "question", "answers"}`); tag = page name. A test + checks every question has an accepted answer literally on its page. +- **`input` axis**: `markdown` (default markkit), `main_content`, `html` (raw + page, the baseline). The model gets `SYSTEM_PROMPT` (answer only from the + document, shortest answer, else `UNKNOWN`) and one user turn + `...` + question. +- **Scorer** `answer_correct`: pass iff the reply contains an accepted answer + (case-insensitive, whitespace collapsed). Deterministic; no judge model. +- **Metrics**: usage tokens, `doc_bytes`, `input_tokens`, latency. Compare + `input` values for accuracy vs input tokens. +- **Targets** (maintainer decision 2026-10-27): `openrouter/meta/muse-spark-1.3-contributor` + (OpenRouter Chat Completions, `OPENROUTER_API_KEY`) and + `anthropic/claude-sonnet-5-5` (Messages API, `ANTHROPIC_API_KEY`). Key-gated: + unkeyed targets are skipped, so offline runs and CI stay green. +- **Keys** live in Doppler: `doppler run -- just eval-llm`. Keys stay in the + study process (mira targets carry none) and are never logged; error texts + name the missing variable only. `ANTHROPIC_BASE_URL` / `OPENROUTER_BASE_URL` + override endpoints. +- **Failures**: HTTP/provider errors and unknown inputs are infra (N/A); + a markkit extraction failure gives the model an empty document (scored). +- **Status**: implemented and smoke-tested end to end against a local mock + endpoint; no keyed run yet (the default cloud environment has no Doppler + access and blocks `api.doppler.com` / `openrouter.ai`). + ## Planned evals - **`markkit_judge`**: LLM-as-judge grading each page's markdown against its source HTML on the four metrics; judge model from the target matrix, gated on API-key env vars so an offline run skips it. -- **`markkit_qa`**: downstream usefulness. A target model answers questions - about a page given markkit's markdown; an axis compares against raw HTML - input (accuracy and input tokens). - **More formats**: DOCX and spreadsheet pages through `extract_document_sync`. ## See also