feat(eval): rebuild markkit-eval as a mira study

.gitignore

@@ -43,3 +43,6 @@ crates/markkit-js/index.js
 crates/markkit-js/index.d.ts
 crates/markkit-js/package-lock.json
 crates/markkit-js/*.tgz
+
+# mira saved eval runs (see mira.toml)
+crates/markkit-eval/results/*/

AGENTS.md

@@ -40,7 +40,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn
 | operations/maintenance | Maintenance invocation contract + checklist + workflow credential audit |
 | operations/release-process | Agent-driven release, crates.io + PyPI + npm |
 | operations/performance-results | Benchmark harness, metrics, committed results |
-| operations/eval | LLM-as-judge eval design (harness is a stub) |
+| operations/eval | mira eval study: dataset, checks, metrics, baseline |
 
 ### Documentation
 
@@ -86,7 +86,7 @@ just pre-pr        # Pre-PR checks
 | `markkit` | Core extraction library (HTML, DOCX, spreadsheets) |
 | `markkit-cli` | `markkit` CLI |
 | `markkit-bench` | Benchmark harness: ops/s, throughput, allocations; results committed |
-| `markkit-eval` | LLM evaluation harness (stub) |
+| `markkit-eval` | Extraction-quality eval, mira study (`just eval`) |
 | `markkit-python` | Python bindings (PyO3), module `markkit` |
 | `markkit-js` | Node.js bindings (NAPI-RS), package `@everruns/markkit` |
 | `markkit-wasm` | Browser/edge WebAssembly (wasm-bindgen), package `@everruns/markkit-wasm` |

CHANGELOG.md

@@ -126,6 +126,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
   `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. Block variants gain a
   `source` field (use `..` in patterns, `Node::paragraph`/`Node::heading` to construct). DOCX `gridSpan` is
   carried as `colspan`.
+- `markkit-eval` is now a [mira](https://github.com/everruns/mira) eval study. `just eval` runs
+  `markkit_extract`, an offline extraction-quality eval over ten realistic HTML pages scored for
+  completeness, accuracy, formatting and noise (needs `cargo install mira-cli`). A `mode` axis
+  compares default extraction with `main_content`; dated results are committed in
+  `crates/markkit-eval/results/`.
 - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI.
 - CI workflows declare read-only default permissions and do not persist checkout credentials.
 - CI runs every example; criterion benchmarks run with `just bench-criterion`.

crates/markkit-eval/Cargo.toml

@@ -1,19 +1,32 @@
-# LLM evaluation harness for markkit extraction quality
-# See knowledge/operations/eval.md for design
+# markkit-eval: mira eval study for markkit extraction quality.
+# Reimplemented on the mira eval framework (github.com/everruns/mira):
+# markkit extraction runs as a mira `Subject`; dataset pages are mira
+# `Sample`s; deterministic expectation checks run as mira `Scorer`s, one per
+# quality metric. The `mira` CLI hosts this binary over stdio and owns
+# selection, scheduling, saved runs, and reporting.
+# See knowledge/operations/eval.md for design decisions.
 
 [package]
 name = "markkit-eval"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
 authors.workspace = true
-description = "LLM evaluation harness for markkit extraction quality"
+description = "Mira eval study for markkit extraction quality"
+publish = false
 
 [[bin]]
 name = "markkit-eval"
 path = "src/main.rs"
 
 [dependencies]
+# Mira eval framework (crates.io), imported as `mira`. `macros` provides the
+# #[eval] discovery attribute. In-process subject only: no mira-everruns.
+mira-eval = { version = "0.5", features = ["macros"] }
+# HTML only, no HTTP client: the eval never fetches (pages are embedded).
+markkit = { path = "../markkit", default-features = false, features = ["html"] }
+serde.workspace = true
+serde_json.workspace = true
+
+[dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
-clap.workspace = true
-anyhow.workspace = true

crates/markkit-eval/data/extract-tasks.jsonl

@@ -0,0 +1,11 @@
+# markkit_extract dataset: one task per line. Schema and check types: knowledge/operations/eval.md
+{"id": "news_article", "category": "news", "description": "News article with cookie banner, site nav, inline ad, newsletter form, related-stories aside and footer", "page": "news_article.html", "expectations": [{"check": "heading", "level": 1, "text": "City Council Approves Riverside Park Expansion", "weight": 2}, {"check": "contains", "text": "voted 7–2 on Tuesday to expand Riverside Park"}, {"check": "contains", "text": "**18 acres**"}, {"check": "contains", "text": "$14.2 million"}, {"check": "line", "text": "> This is the largest addition to our public green space in fifty years."}, {"check": "contains", "text": "*spring 2028*"}, {"check": "link", "text": "city parks website", "url": "https://harbor.test/parks/riverside"}, {"check": "heading", "level": 2, "text": "What happens to the rail yard"}, {"check": "contains", "text": "two historic signal towers"}, {"check": "contains", "text": "![Aerial view of Riverside Park](https://cdn.harborgazette.test/img/riverside-park.jpg)"}, {"check": "order", "first": "$14.2 million", "then": "signal towers"}, {"check": "not_contains", "text": "We use cookies"}, {"check": "not_contains", "text": "Accept all cookies"}, {"check": "not_contains", "text": "Subscribe"}, {"check": "not_contains", "text": "Refinance today"}, {"check": "not_contains", "text": "Advertisement"}, {"check": "not_contains", "text": "Related stories"}, {"check": "not_contains", "text": "New bike lanes"}, {"check": "not_contains", "text": "All rights reserved"}, {"check": "not_contains", "text": "dataLayer"}, {"check": "not_contains", "text": "Get the morning briefing"}]}
+{"id": "docs_page", "category": "docs", "description": "Docs page with sidebar nav, option table, TOML and Rust code blocks, callout, edit link and pager", "page": "docs_page.html", "expectations": [{"check": "heading", "level": 1, "text": "Configuration", "weight": 2}, {"check": "contains", "text": "`tideline.toml`"}, {"check": "line", "text": "| Key | Type | Default | Description |"}, {"check": "line", "text": "| `workers` | integer | 4 | Number of worker threads |"}, {"check": "line", "text": "| `retry` | boolean | true | Retry failed requests once |"}, {"check": "line", "text": "```toml"}, {"check": "line", "text": "```rust"}, {"check": "contains", "text": "let cfg = Config::load(\"tideline.toml\")?;"}, {"check": "contains", "text": "fn main() -> anyhow::Result<()> {"}, {"check": "contains", "text": "unknown keys are rejected at startup"}, {"check": "link", "text": "environment variables", "url": "/docs/env"}, {"check": "order", "first": "## Options", "then": "## Example"}, {"check": "order", "first": "## Example", "then": "## Loading from code"}, {"check": "not_contains", "text": "Getting started"}, {"check": "not_contains", "text": "Quickstart"}, {"check": "not_contains", "text": "Search docs"}, {"check": "not_contains", "text": "Edit this page"}, {"check": "not_contains", "text": "Last updated"}, {"check": "not_contains", "text": "Previous:"}, {"check": "not_contains", "text": "Built with Tideline Docs"}]}
+{"id": "blog_post", "category": "blog", "description": "Engineering blog post with share buttons, author bio, comment section and newsletter popup", "page": "blog_post.html", "expectations": [{"check": "heading", "level": 1, "text": "Why we moved our build to Bazel", "weight": 2}, {"check": "contains", "text": "1.4 million lines of code"}, {"check": "contains", "text": "**47 minutes**"}, {"check": "heading", "level": 2, "text": "What we tried first"}, {"check": "line", "text": "1. Caching `node_modules` between CI runs"}, {"check": "line", "text": "3. Upgrading CI runners to 32 cores"}, {"check": "heading", "level": 2, "text": "Results"}, {"check": "contains", "text": "92% of builds hit the remote cache"}, {"check": "order", "first": "What we tried first", "then": "Results"}, {"check": "not_contains", "text": "We're hiring"}, {"check": "not_contains", "text": "Share on X"}, {"check": "not_contains", "text": "Share on LinkedIn"}, {"check": "not_contains", "text": "14 Comments"}, {"check": "not_contains", "text": "Did you consider Buck2"}, {"check": "not_contains", "text": "Log in to comment"}, {"check": "not_contains", "text": "Subscribe to our newsletter"}]}
+{"id": "product_page", "category": "commerce", "description": "Product page with nav, breadcrumbs, buy box form, spec table, reviews, recommendations and promo banner", "page": "product_page.html", "expectations": [{"check": "heading", "level": 1, "text": "Northwind Trail 40L Backpack", "weight": 2}, {"check": "contains", "text": "$189.00"}, {"check": "contains", "text": "ventilated back panel and a removable rain cover"}, {"check": "line", "text": "- Recycled 210D ripstop nylon"}, {"check": "line", "text": "- Hydration sleeve fits a 3 L reservoir"}, {"check": "contains", "text": "Adjustable torso length (15–21 in)"}, {"check": "contains", "text": "1.2 kg"}, {"check": "contains", "text": "64 × 32 × 24 cm"}, {"check": "line", "text": "| Weight | 1.2 kg |"}, {"check": "contains", "text": "4.6 out of 5 stars"}, {"check": "order", "first": "## Features", "then": "## Specifications"}, {"check": "not_contains", "text": "Cart (0)"}, {"check": "not_contains", "text": "Add to cart"}, {"check": "not_contains", "text": "Home / "}, {"check": "not_contains", "text": "You may also like"}, {"check": "not_contains", "text": "Ridgeline 30L"}, {"check": "not_contains", "text": "TRAIL10"}, {"check": "not_contains", "text": "1-800-555-0199"}]}
+{"id": "recipe", "category": "recipe", "description": "Recipe with sponsored ad slots, ingredient list, numbered steps and a rating widget", "page": "recipe.html", "expectations": [{"check": "heading", "level": 1, "text": "Weeknight Lemon Garlic Pasta", "weight": 2}, {"check": "heading", "level": 2, "text": "Ingredients"}, {"check": "heading", "level": 2, "text": "Instructions"}, {"check": "line", "text": "- 350 g spaghetti"}, {"check": "line", "text": "- 40 g grated parmesan"}, {"check": "line", "text": "1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water."}, {"check": "line", "text": "4. Loosen with pasta water as needed and serve immediately."}, {"check": "contains", "text": "Serves: 4"}, {"check": "contains", "text": "replace the parmesan with toasted breadcrumbs"}, {"check": "order", "first": "Warm the olive oil", "then": "Toss the pasta"}, {"check": "not_contains", "text": "Sponsored"}, {"check": "not_contains", "text": "MealBox"}, {"check": "not_contains", "text": "Chef's knife"}, {"check": "not_contains", "text": "Rate this recipe"}, {"check": "not_contains", "text": "Follow us on"}]}
+{"id": "wiki_article", "category": "reference", "description": "Encyclopedia article with infobox, inline citations, table of contents, edit links, categories and footer", "page": "wiki_article.html", "expectations": [{"check": "heading", "level": 1, "text": "Lighthouse of Alexandria", "weight": 2}, {"check": "contains", "text": "built by the Ptolemaic Kingdom on the island of Pharos"}, {"check": "contains", "text": "Seven Wonders of the Ancient World"}, {"check": "heading", "level": 2, "text": "Construction"}, {"check": "heading", "level": 3, "text": "Design"}, {"check": "heading", "level": 2, "text": "Destruction"}, {"check": "contains", "text": "Sostratus of Cnidus"}, {"check": "contains", "text": "between 956 and 1323 AD"}, {"check": "contains", "text": "c. 280 BC"}, {"check": "order", "first": "Construction was completed", "then": "badly damaged by earthquakes"}, {"check": "not_contains", "text": "Log in"}, {"check": "not_contains", "text": "Random article"}, {"check": "not_contains", "text": "[edit]"}, {"check": "not_contains", "text": "From OpenWiki, the free encyclopedia"}, {"check": "not_contains", "text": "Contents"}, {"check": "not_contains", "text": "Categories:"}, {"check": "not_contains", "text": "last edited on"}]}
+{"id": "forum_thread", "category": "forum", "description": "Q&A thread with vote widgets, tags, accepted answer with code, user cards and sidebar", "page": "forum_thread.html", "expectations": [{"check": "heading", "level": 1, "text": "How do I reverse a linked list in place?", "weight": 2}, {"check": "contains", "text": "reverse it without allocating a new list"}, {"check": "line", "text": "```python"}, {"check": "contains", "text": "        nxt = node.next"}, {"check": "contains", "text": "def reverse(head):"}, {"check": "contains", "text": "O(n) time and O(1) extra space"}, {"check": "contains", "text": "recursive version also works"}, {"check": "order", "first": "without allocating a new list", "then": "You overwrite"}, {"check": "order", "first": "O(1) extra space", "then": "recursive version"}, {"check": "not_contains", "text": "Viewed 48k times"}, {"check": "not_contains", "text": "Log in"}, {"check": "not_contains", "text": "Hot Network Questions"}, {"check": "not_contains", "text": "CSS grid"}, {"check": "not_contains", "text": "user contributions licensed"}, {"check": "not_contains", "text": "▲"}]}
+{"id": "changelog", "category": "changelog", "description": "Release notes with nested lists, strikethrough, issue links and call-to-action box", "page": "changelog.html", "expectations": [{"check": "heading", "level": 1, "text": "Changelog", "weight": 2}, {"check": "heading", "level": 2, "text": "v2.4.0 — 2026-04-02"}, {"check": "heading", "level": 3, "text": "Added"}, {"check": "heading", "level": 3, "text": "Fixed"}, {"check": "heading", "level": 2, "text": "v2.3.1 — 2026-03-10"}, {"check": "line", "text": "- Offline mode for the desktop editor"}, {"check": "contains", "text": "  - Custom cover images"}, {"check": "link", "text": "#1182", "url": "https://git.test/quillstack/issues/1182"}, {"check": "contains", "text": "~~Spellcheck ignored code blocks~~"}, {"check": "contains", "text": "Sanitize SVG uploads to prevent stored XSS"}, {"check": "order", "first": "v2.4.0", "then": "v2.3.1"}, {"check": "not_contains", "text": "Pricing"}, {"check": "not_contains", "text": "Start your free trial"}, {"check": "not_contains", "text": "Status"}]}
+{"id": "data_table", "category": "data", "description": "Data report with a captioned table including thead, tbody and tfoot", "page": "data_table.html", "expectations": [{"check": "heading", "level": 1, "text": "Electricity Generation by Source, 2025", "weight": 2}, {"check": "contains", "text": "terawatt-hours (TWh)"}, {"check": "line", "text": "| Source | Q1 | Q2 | Q3 | Q4 |", "weight": 2}, {"check": "line", "text": "| Wind | 21.4 | 15.2 | 12.8 | 24.9 |"}, {"check": "line", "text": "| Nuclear | 12.0 | 11.6 | 11.9 | 12.1 |"}, {"check": "contains", "text": "Natural gas", "metric": "completeness"}, {"check": "contains", "text": "Total"}, {"check": "contains", "text": "72.4"}, {"check": "contains", "text": "Quarterly generation"}, {"check": "order", "first": "| Wind |", "then": "| Solar |"}, {"check": "link", "text": "Download the data (CSV)", "url": "/reports/2025-energy.csv"}, {"check": "not_contains", "text": "Reports"}, {"check": "not_contains", "text": "Share:"}, {"check": "not_contains", "text": "independent project"}]}
+{"id": "legacy_layout", "category": "legacy", "description": "1990s table-layout page with font tags, unclosed paragraphs, line breaks and a hit counter", "page": "legacy_layout.html", "expectations": [{"check": "contains", "text": "Pine Hollow Model Railroad Club", "weight": 2}, {"check": "contains", "text": "**Tuesday at 7 PM**"}, {"check": "contains", "text": "Grange Hall, 18 Mill Road"}, {"check": "contains", "text": "600 feet of track"}, {"check": "contains", "text": "Boston & Maine railroad in 1952"}, {"check": "contains", "text": "Annual dues are $40."}, {"check": "link", "text": "walt@pinehollow.test", "url": "mailto:walt@pinehollow.test"}, {"check": "order", "first": "Welcome to the home", "then": "New members are always welcome"}, {"check": "not_contains", "text": "MENU", "weight": 2}, {"check": "not_contains", "text": "Guestbook"}, {"check": "not_contains", "text": "visitor number"}, {"check": "not_contains", "text": "Netscape Navigator"}]}

crates/markkit-eval/data/pages/blog_post.html

@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<html>
+<head><title>Why we moved our build to Bazel - Lattice Engineering</title></head>
+<body>
+<div id="top-bar"><a href="/">Lattice Engineering Blog</a> | <a href="/careers">We're hiring!</a></div>
+<div class="post">
+  <h1>Why we moved our build to Bazel</h1>
+  <div class="post-meta">Posted by Jordan Alvarez on 9 January 2026 &middot; 6 min read</div>
+  <div class="share-buttons">
+    <a href="https://twitter.test/share?u=bazel">Share on X</a>
+    <a href="https://linkedin.test/share?u=bazel">Share on LinkedIn</a>
+  </div>
+  <p>Our monorepo had grown to 1.4 million lines of code, and a clean CI build took <strong>47 minutes</strong>. Incremental builds were unreliable, so engineers ran clean builds out of habit.</p>
+  <h2>What we tried first</h2>
+  <ol>
+    <li>Caching <code>node_modules</code> between CI runs</li>
+    <li>Splitting the test suite across 12 shards</li>
+    <li>Upgrading CI runners to 32 cores</li>
+  </ol>
+  <p>Each change helped a little, but none fixed the root problem: the build had no accurate dependency graph.</p>
+  <h2>Results</h2>
+  <p>After the migration, the median CI build takes 6 minutes, and 92% of builds hit the remote cache.</p>
+  <p>Thanks to the developer productivity team for leading the migration.</p>
+  <div class="author-bio">
+    <img src="/avatars/jordan.png" alt="Jordan Alvarez">
+    <p>Jordan Alvarez is a staff engineer on the platform team.</p>
+  </div>
+</div>
+<section id="comments" class="comments">
+  <h3>14 Comments</h3>
+  <div class="comment"><b>devops_dan</b> said: Great write-up! Did you consider Buck2?</div>
+  <div class="comment"><b>sarahk</b> said: How long did the migration take?</div>
+  <a href="/login">Log in to comment</a>
+</section>
+<div class="popup-overlay"><p>Subscribe to our newsletter for weekly posts!</p></div>
+<footer><p>&copy; Lattice Inc.</p></footer>
+</body>
+</html>