From 658cdef9667dd9c7ff4943932f4970c0fe457292 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:37:21 +0000 Subject: [PATCH 1/3] feat(eval): rebuild markkit-eval as a mira study markkit-eval was a CLI stub. It is now a mira study binary (modeled on bashkit-eval) advertising markkit_extract: ten realistic HTML pages run through markkit in-process and scored by typed deterministic checks, with one scorer per quality metric (completeness, accuracy, formatting, noise). Offline and key-free; the single target is markkit itself. Baseline: 72.5% mean weighted rate, noise 34.3%. Gaps recorded in knowledge/operations/eval.md. --- .gitignore | 3 + AGENTS.md | 4 +- CHANGELOG.md | 3 + Cargo.lock | 64 ++++- crates/markkit-eval/Cargo.toml | 23 +- crates/markkit-eval/data/extract-tasks.jsonl | 11 + crates/markkit-eval/data/pages/blog_post.html | 38 +++ crates/markkit-eval/data/pages/changelog.html | 38 +++ .../markkit-eval/data/pages/data_table.html | 30 ++ crates/markkit-eval/data/pages/docs_page.html | 60 ++++ .../markkit-eval/data/pages/forum_thread.html | 46 +++ .../data/pages/legacy_layout.html | 27 ++ .../markkit-eval/data/pages/news_article.html | 61 ++++ .../markkit-eval/data/pages/product_page.html | 40 +++ crates/markkit-eval/data/pages/recipe.html | 37 +++ .../markkit-eval/data/pages/wiki_article.html | 37 +++ crates/markkit-eval/src/checks.rs | 223 ++++++++++++++ crates/markkit-eval/src/dataset.rs | 245 ++++++++++++++++ crates/markkit-eval/src/lib.rs | 14 + crates/markkit-eval/src/main.rs | 35 +-- crates/markkit-eval/src/mira_study.rs | 271 ++++++++++++++++++ justfile | 13 +- knowledge/foundations/architecture.md | 2 +- knowledge/log.md | 1 + knowledge/operations/eval.md | 161 +++++++++-- knowledge/operations/testing.md | 3 + mira.toml | 6 + 27 files changed, 1448 insertions(+), 48 deletions(-) create mode 100644 crates/markkit-eval/data/extract-tasks.jsonl create mode 100644 crates/markkit-eval/data/pages/blog_post.html create mode 100644 crates/markkit-eval/data/pages/changelog.html create mode 100644 crates/markkit-eval/data/pages/data_table.html create mode 100644 crates/markkit-eval/data/pages/docs_page.html create mode 100644 crates/markkit-eval/data/pages/forum_thread.html create mode 100644 crates/markkit-eval/data/pages/legacy_layout.html create mode 100644 crates/markkit-eval/data/pages/news_article.html create mode 100644 crates/markkit-eval/data/pages/product_page.html create mode 100644 crates/markkit-eval/data/pages/recipe.html create mode 100644 crates/markkit-eval/data/pages/wiki_article.html create mode 100644 crates/markkit-eval/src/checks.rs create mode 100644 crates/markkit-eval/src/dataset.rs create mode 100644 crates/markkit-eval/src/lib.rs create mode 100644 crates/markkit-eval/src/mira_study.rs create mode 100644 mira.toml diff --git a/.gitignore b/.gitignore index 211833a..018bf98 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ crates/markkit-js/index.js crates/markkit-js/index.d.ts crates/markkit-js/package-lock.json crates/markkit-js/*.tgz + +# mira saved eval runs (see mira.toml) +crates/markkit-eval/results/ diff --git a/AGENTS.md b/AGENTS.md index 9204780..0cd4ed5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | operations/examples | `examples/` contents and requirements | | operations/maintenance | Maintenance invocation contract + checklist + workflow credential audit | | operations/release-process | Agent-driven release, crates.io + PyPI + npm | -| operations/eval | LLM-as-judge eval design (harness is a stub) | +| operations/eval | mira eval study: dataset, checks, metrics, baseline | ### Documentation @@ -78,7 +78,7 @@ just pre-pr # Pre-PR checks | `markkit` | Core extraction library (HTML, DOCX, spreadsheets) | | `markkit-cli` | `markkit` CLI | | `markkit-bench` | Benchmark harness (stub; real benches are criterion in `markkit`) | -| `markkit-eval` | LLM evaluation harness (stub) | +| `markkit-eval` | Extraction-quality eval, mira study (`just eval`) | | `markkit-python` | Python bindings (PyO3), module `markkit` | | `markkit-js` | Node.js bindings (NAPI-RS), package `@everruns/markkit` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..790b33e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- `markkit-eval` is now a [mira](https://github.com/everruns/mira) eval study. `just eval` runs + `markkit_extract`, an offline extraction-quality eval over ten realistic HTML pages scored for + completeness, accuracy, formatting and noise (needs `cargo install mira-cli`). - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. - CI runs every example; `just bench` now runs the criterion benchmarks. diff --git a/Cargo.lock b/Cargo.lock index 01303b3..1020d0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,17 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "atoi_simd" version = "0.18.1" @@ -1032,6 +1043,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -1224,8 +1244,10 @@ dependencies = [ name = "markkit-eval" version = "0.1.1" dependencies = [ - "anyhow", - "clap", + "markkit", + "mira-eval", + "serde", + "serde_json", "tokio", ] @@ -1276,6 +1298,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mira-eval" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75db8c713ee4b8ed2fd8eb1797dd3129b0e545a039e9cef45ac19458447cc16c" +dependencies = [ + "async-trait", + "inventory", + "mira-macros", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "mira-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfd4d98e649e99b006acace5360e65ca6fd0bc6593e6502514592316ab38530" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -2157,6 +2206,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -2375,6 +2434,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", diff --git a/crates/markkit-eval/Cargo.toml b/crates/markkit-eval/Cargo.toml index c8d5062..657ab01 100644 --- a/crates/markkit-eval/Cargo.toml +++ b/crates/markkit-eval/Cargo.toml @@ -1,5 +1,10 @@ -# LLM evaluation harness for markkit extraction quality -# See knowledge/operations/eval.md for design +# markkit-eval: mira eval study for markkit extraction quality. +# Reimplemented on the mira eval framework (github.com/everruns/mira): +# markkit extraction runs as a mira `Subject`; dataset pages are mira +# `Sample`s; deterministic expectation checks run as mira `Scorer`s, one per +# quality metric. The `mira` CLI hosts this binary over stdio and owns +# selection, scheduling, saved runs, and reporting. +# See knowledge/operations/eval.md for design decisions. [package] name = "markkit-eval" @@ -7,13 +12,21 @@ version.workspace = true edition.workspace = true license.workspace = true authors.workspace = true -description = "LLM evaluation harness for markkit extraction quality" +description = "Mira eval study for markkit extraction quality" +publish = false [[bin]] name = "markkit-eval" path = "src/main.rs" [dependencies] +# Mira eval framework (crates.io), imported as `mira`. `macros` provides the +# #[eval] discovery attribute. In-process subject only: no mira-everruns. +mira-eval = { version = "0.5", features = ["macros"] } +# HTML only, no HTTP client: the eval never fetches (pages are embedded). +markkit = { path = "../markkit", default-features = false, features = ["html"] } +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -clap.workspace = true -anyhow.workspace = true diff --git a/crates/markkit-eval/data/extract-tasks.jsonl b/crates/markkit-eval/data/extract-tasks.jsonl new file mode 100644 index 0000000..69f3c52 --- /dev/null +++ b/crates/markkit-eval/data/extract-tasks.jsonl @@ -0,0 +1,11 @@ +# markkit_extract dataset: one task per line. Schema and check types: knowledge/operations/eval.md +{"id": "news_article", "category": "news", "description": "News article with cookie banner, site nav, inline ad, newsletter form, related-stories aside and footer", "page": "news_article.html", "expectations": [{"check": "heading", "level": 1, "text": "City Council Approves Riverside Park Expansion", "weight": 2}, {"check": "contains", "text": "voted 7–2 on Tuesday to expand Riverside Park"}, {"check": "contains", "text": "**18 acres**"}, {"check": "contains", "text": "$14.2 million"}, {"check": "line", "text": "> This is the largest addition to our public green space in fifty years."}, {"check": "contains", "text": "*spring 2028*"}, {"check": "link", "text": "city parks website", "url": "https://harbor.test/parks/riverside"}, {"check": "heading", "level": 2, "text": "What happens to the rail yard"}, {"check": "contains", "text": "two historic signal towers"}, {"check": "contains", "text": "![Aerial view of Riverside Park](https://cdn.harborgazette.test/img/riverside-park.jpg)"}, {"check": "order", "first": "$14.2 million", "then": "signal towers"}, {"check": "not_contains", "text": "We use cookies"}, {"check": "not_contains", "text": "Accept all cookies"}, {"check": "not_contains", "text": "Subscribe"}, {"check": "not_contains", "text": "Refinance today"}, {"check": "not_contains", "text": "Advertisement"}, {"check": "not_contains", "text": "Related stories"}, {"check": "not_contains", "text": "New bike lanes"}, {"check": "not_contains", "text": "All rights reserved"}, {"check": "not_contains", "text": "dataLayer"}, {"check": "not_contains", "text": "Get the morning briefing"}]} +{"id": "docs_page", "category": "docs", "description": "Docs page with sidebar nav, option table, TOML and Rust code blocks, callout, edit link and pager", "page": "docs_page.html", "expectations": [{"check": "heading", "level": 1, "text": "Configuration", "weight": 2}, {"check": "contains", "text": "`tideline.toml`"}, {"check": "line", "text": "| Key | Type | Default | Description |"}, {"check": "line", "text": "| `workers` | integer | 4 | Number of worker threads |"}, {"check": "line", "text": "| `retry` | boolean | true | Retry failed requests once |"}, {"check": "line", "text": "```toml"}, {"check": "line", "text": "```rust"}, {"check": "contains", "text": "let cfg = Config::load(\"tideline.toml\")?;"}, {"check": "contains", "text": "fn main() -> anyhow::Result<()> {"}, {"check": "contains", "text": "unknown keys are rejected at startup"}, {"check": "link", "text": "environment variables", "url": "/docs/env"}, {"check": "order", "first": "## Options", "then": "## Example"}, {"check": "order", "first": "## Example", "then": "## Loading from code"}, {"check": "not_contains", "text": "Getting started"}, {"check": "not_contains", "text": "Quickstart"}, {"check": "not_contains", "text": "Search docs"}, {"check": "not_contains", "text": "Edit this page"}, {"check": "not_contains", "text": "Last updated"}, {"check": "not_contains", "text": "Previous:"}, {"check": "not_contains", "text": "Built with Tideline Docs"}]} +{"id": "blog_post", "category": "blog", "description": "Engineering blog post with share buttons, author bio, comment section and newsletter popup", "page": "blog_post.html", "expectations": [{"check": "heading", "level": 1, "text": "Why we moved our build to Bazel", "weight": 2}, {"check": "contains", "text": "1.4 million lines of code"}, {"check": "contains", "text": "**47 minutes**"}, {"check": "heading", "level": 2, "text": "What we tried first"}, {"check": "line", "text": "1. Caching `node_modules` between CI runs"}, {"check": "line", "text": "3. Upgrading CI runners to 32 cores"}, {"check": "heading", "level": 2, "text": "Results"}, {"check": "contains", "text": "92% of builds hit the remote cache"}, {"check": "order", "first": "What we tried first", "then": "Results"}, {"check": "not_contains", "text": "We're hiring"}, {"check": "not_contains", "text": "Share on X"}, {"check": "not_contains", "text": "Share on LinkedIn"}, {"check": "not_contains", "text": "14 Comments"}, {"check": "not_contains", "text": "Did you consider Buck2"}, {"check": "not_contains", "text": "Log in to comment"}, {"check": "not_contains", "text": "Subscribe to our newsletter"}]} +{"id": "product_page", "category": "commerce", "description": "Product page with nav, breadcrumbs, buy box form, spec table, reviews, recommendations and promo banner", "page": "product_page.html", "expectations": [{"check": "heading", "level": 1, "text": "Northwind Trail 40L Backpack", "weight": 2}, {"check": "contains", "text": "$189.00"}, {"check": "contains", "text": "ventilated back panel and a removable rain cover"}, {"check": "line", "text": "- Recycled 210D ripstop nylon"}, {"check": "line", "text": "- Hydration sleeve fits a 3 L reservoir"}, {"check": "contains", "text": "Adjustable torso length (15–21 in)"}, {"check": "contains", "text": "1.2 kg"}, {"check": "contains", "text": "64 × 32 × 24 cm"}, {"check": "line", "text": "| Weight | 1.2 kg |"}, {"check": "contains", "text": "4.6 out of 5 stars"}, {"check": "order", "first": "## Features", "then": "## Specifications"}, {"check": "not_contains", "text": "Cart (0)"}, {"check": "not_contains", "text": "Add to cart"}, {"check": "not_contains", "text": "Home / "}, {"check": "not_contains", "text": "You may also like"}, {"check": "not_contains", "text": "Ridgeline 30L"}, {"check": "not_contains", "text": "TRAIL10"}, {"check": "not_contains", "text": "1-800-555-0199"}]} +{"id": "recipe", "category": "recipe", "description": "Recipe with sponsored ad slots, ingredient list, numbered steps and a rating widget", "page": "recipe.html", "expectations": [{"check": "heading", "level": 1, "text": "Weeknight Lemon Garlic Pasta", "weight": 2}, {"check": "heading", "level": 2, "text": "Ingredients"}, {"check": "heading", "level": 2, "text": "Instructions"}, {"check": "line", "text": "- 350 g spaghetti"}, {"check": "line", "text": "- 40 g grated parmesan"}, {"check": "line", "text": "1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water."}, {"check": "line", "text": "4. Loosen with pasta water as needed and serve immediately."}, {"check": "contains", "text": "Serves: 4"}, {"check": "contains", "text": "replace the parmesan with toasted breadcrumbs"}, {"check": "order", "first": "Warm the olive oil", "then": "Toss the pasta"}, {"check": "not_contains", "text": "Sponsored"}, {"check": "not_contains", "text": "MealBox"}, {"check": "not_contains", "text": "Chef's knife"}, {"check": "not_contains", "text": "Rate this recipe"}, {"check": "not_contains", "text": "Follow us on"}]} +{"id": "wiki_article", "category": "reference", "description": "Encyclopedia article with infobox, inline citations, table of contents, edit links, categories and footer", "page": "wiki_article.html", "expectations": [{"check": "heading", "level": 1, "text": "Lighthouse of Alexandria", "weight": 2}, {"check": "contains", "text": "built by the Ptolemaic Kingdom on the island of Pharos"}, {"check": "contains", "text": "Seven Wonders of the Ancient World"}, {"check": "heading", "level": 2, "text": "Construction"}, {"check": "heading", "level": 3, "text": "Design"}, {"check": "heading", "level": 2, "text": "Destruction"}, {"check": "contains", "text": "Sostratus of Cnidus"}, {"check": "contains", "text": "between 956 and 1323 AD"}, {"check": "contains", "text": "c. 280 BC"}, {"check": "order", "first": "Construction was completed", "then": "badly damaged by earthquakes"}, {"check": "not_contains", "text": "Log in"}, {"check": "not_contains", "text": "Random article"}, {"check": "not_contains", "text": "[edit]"}, {"check": "not_contains", "text": "From OpenWiki, the free encyclopedia"}, {"check": "not_contains", "text": "Contents"}, {"check": "not_contains", "text": "Categories:"}, {"check": "not_contains", "text": "last edited on"}]} +{"id": "forum_thread", "category": "forum", "description": "Q&A thread with vote widgets, tags, accepted answer with code, user cards and sidebar", "page": "forum_thread.html", "expectations": [{"check": "heading", "level": 1, "text": "How do I reverse a linked list in place?", "weight": 2}, {"check": "contains", "text": "reverse it without allocating a new list"}, {"check": "line", "text": "```python"}, {"check": "contains", "text": " nxt = node.next"}, {"check": "contains", "text": "def reverse(head):"}, {"check": "contains", "text": "O(n) time and O(1) extra space"}, {"check": "contains", "text": "recursive version also works"}, {"check": "order", "first": "without allocating a new list", "then": "You overwrite"}, {"check": "order", "first": "O(1) extra space", "then": "recursive version"}, {"check": "not_contains", "text": "Viewed 48k times"}, {"check": "not_contains", "text": "Log in"}, {"check": "not_contains", "text": "Hot Network Questions"}, {"check": "not_contains", "text": "CSS grid"}, {"check": "not_contains", "text": "user contributions licensed"}, {"check": "not_contains", "text": "▲"}]} +{"id": "changelog", "category": "changelog", "description": "Release notes with nested lists, strikethrough, issue links and call-to-action box", "page": "changelog.html", "expectations": [{"check": "heading", "level": 1, "text": "Changelog", "weight": 2}, {"check": "heading", "level": 2, "text": "v2.4.0 — 2026-04-02"}, {"check": "heading", "level": 3, "text": "Added"}, {"check": "heading", "level": 3, "text": "Fixed"}, {"check": "heading", "level": 2, "text": "v2.3.1 — 2026-03-10"}, {"check": "line", "text": "- Offline mode for the desktop editor"}, {"check": "contains", "text": " - Custom cover images"}, {"check": "link", "text": "#1182", "url": "https://git.test/quillstack/issues/1182"}, {"check": "contains", "text": "~~Spellcheck ignored code blocks~~"}, {"check": "contains", "text": "Sanitize SVG uploads to prevent stored XSS"}, {"check": "order", "first": "v2.4.0", "then": "v2.3.1"}, {"check": "not_contains", "text": "Pricing"}, {"check": "not_contains", "text": "Start your free trial"}, {"check": "not_contains", "text": "Status"}]} +{"id": "data_table", "category": "data", "description": "Data report with a captioned table including thead, tbody and tfoot", "page": "data_table.html", "expectations": [{"check": "heading", "level": 1, "text": "Electricity Generation by Source, 2025", "weight": 2}, {"check": "contains", "text": "terawatt-hours (TWh)"}, {"check": "line", "text": "| Source | Q1 | Q2 | Q3 | Q4 |", "weight": 2}, {"check": "line", "text": "| Wind | 21.4 | 15.2 | 12.8 | 24.9 |"}, {"check": "line", "text": "| Nuclear | 12.0 | 11.6 | 11.9 | 12.1 |"}, {"check": "contains", "text": "Natural gas", "metric": "completeness"}, {"check": "contains", "text": "Total"}, {"check": "contains", "text": "72.4"}, {"check": "contains", "text": "Quarterly generation"}, {"check": "order", "first": "| Wind |", "then": "| Solar |"}, {"check": "link", "text": "Download the data (CSV)", "url": "/reports/2025-energy.csv"}, {"check": "not_contains", "text": "Reports"}, {"check": "not_contains", "text": "Share:"}, {"check": "not_contains", "text": "independent project"}]} +{"id": "legacy_layout", "category": "legacy", "description": "1990s table-layout page with font tags, unclosed paragraphs, line breaks and a hit counter", "page": "legacy_layout.html", "expectations": [{"check": "contains", "text": "Pine Hollow Model Railroad Club", "weight": 2}, {"check": "contains", "text": "**Tuesday at 7 PM**"}, {"check": "contains", "text": "Grange Hall, 18 Mill Road"}, {"check": "contains", "text": "600 feet of track"}, {"check": "contains", "text": "Boston & Maine railroad in 1952"}, {"check": "contains", "text": "Annual dues are $40."}, {"check": "link", "text": "walt@pinehollow.test", "url": "mailto:walt@pinehollow.test"}, {"check": "order", "first": "Welcome to the home", "then": "New members are always welcome"}, {"check": "not_contains", "text": "MENU", "weight": 2}, {"check": "not_contains", "text": "Guestbook"}, {"check": "not_contains", "text": "visitor number"}, {"check": "not_contains", "text": "Netscape Navigator"}]} diff --git a/crates/markkit-eval/data/pages/blog_post.html b/crates/markkit-eval/data/pages/blog_post.html new file mode 100644 index 0000000..06e6115 --- /dev/null +++ b/crates/markkit-eval/data/pages/blog_post.html @@ -0,0 +1,38 @@ + + +Why we moved our build to Bazel - Lattice Engineering + +
Lattice Engineering Blog | We're hiring!
+
+

Why we moved our build to Bazel

+
Posted by Jordan Alvarez on 9 January 2026 · 6 min read
+ +

Our monorepo had grown to 1.4 million lines of code, and a clean CI build took 47 minutes. Incremental builds were unreliable, so engineers ran clean builds out of habit.

+

What we tried first

+
    +
  1. Caching node_modules between CI runs
    2. +
  2. Splitting the test suite across 12 shards
    3. +
  3. Upgrading CI runners to 32 cores
    4. +
+

Each change helped a little, but none fixed the root problem: the build had no accurate dependency graph.

+

Results

+

After the migration, the median CI build takes 6 minutes, and 92% of builds hit the remote cache.

+

Thanks to the developer productivity team for leading the migration.

+
+ Jordan Alvarez +

Jordan Alvarez is a staff engineer on the platform team.

+
+
+
+

14 Comments

+
devops_dan said: Great write-up! Did you consider Buck2?
+
sarahk said: How long did the migration take?
+ Log in to comment +
+

Subscribe to our newsletter for weekly posts!

+ + + diff --git a/crates/markkit-eval/data/pages/changelog.html b/crates/markkit-eval/data/pages/changelog.html new file mode 100644 index 0000000..21b7088 --- /dev/null +++ b/crates/markkit-eval/data/pages/changelog.html @@ -0,0 +1,38 @@ + + +Changelog - Quillstack + +
+
+

Changelog

+

All notable changes to Quillstack are documented here. Subscribe via RSS.

+
+

v2.4.0 — 2026-04-02

+

Added

+
    +
  • Offline mode for the desktop editor
    • +
  • Export to EPUB +
      +
    • Custom cover images
      • +
    • Table of contents generation
      • +
    +
    • +
+

Fixed

+
    +
  • Crash when pasting images larger than 20 MB (#1182)
    • +
  • Spellcheck ignored code blocks Spellcheck now skips code blocks
    • +
+
+
+

v2.3.1 — 2026-03-10

+

Security

+
    +
  • Sanitize SVG uploads to prevent stored XSS
    • +
+
+

Ready to write? Start your free trial

+
+ + + diff --git a/crates/markkit-eval/data/pages/data_table.html b/crates/markkit-eval/data/pages/data_table.html new file mode 100644 index 0000000..357081b --- /dev/null +++ b/crates/markkit-eval/data/pages/data_table.html @@ -0,0 +1,30 @@ + + +Quarterly Energy Report - Grid Watch + +
+
+

Electricity Generation by Source, 2025

+

Figures are in terawatt-hours (TWh). Source: national grid operator, published February 2026.

+ + + + + + + + + + + + + + +
Quarterly generation
SourceQ1Q2Q3Q4
Wind21.415.212.824.9
Solar4.111.713.33.6
Natural gas30.222.526.031.8
Nuclear12.011.611.912.1
Total67.761.064.072.4
+

Wind overtook natural gas in no quarter, but the gap narrowed to 6.9 TWh in Q4.

+

Download the data (CSV)

+
+
Share: X Email
+ + + diff --git a/crates/markkit-eval/data/pages/docs_page.html b/crates/markkit-eval/data/pages/docs_page.html new file mode 100644 index 0000000..2a78ebd --- /dev/null +++ b/crates/markkit-eval/data/pages/docs_page.html @@ -0,0 +1,60 @@ + + +Configuration - Tideline Docs + +
Tideline
+
+ +
+

Configuration

+

Tideline reads its settings from tideline.toml in the project root. Every key is optional.

+

Options

+ + + + + + + + + +
KeyTypeDefaultDescription
workersinteger4Number of worker threads
timeoutstring30sRequest timeout
retrybooleantrueRetry failed requests once
+

Example

+ 
[server]
+workers = 8
+timeout = "45s"
+
+[cache]
+enabled = true
+
+

Loading from code

+

Use Config::load to read the file manually:

+ 
use tideline::Config;
+
+fn main() -> anyhow::Result<()> {
+    let cfg = Config::load("tideline.toml")?;
+    println!("{} workers", cfg.workers);
+    Ok(())
+}
+
+
+

Warning: unknown keys are rejected at startup.

+
+

See environment variables to override settings per deployment.

+
+ Edit this page on GitHub + Last updated on Feb 2, 2026 +
+
← Previous: Quickstart Next: Environment →
+
+
+ + + diff --git a/crates/markkit-eval/data/pages/forum_thread.html b/crates/markkit-eval/data/pages/forum_thread.html new file mode 100644 index 0000000..522a41d --- /dev/null +++ b/crates/markkit-eval/data/pages/forum_thread.html @@ -0,0 +1,46 @@ + + +How do I reverse a linked list in place? - DevAnswers + +
+
+

How do I reverse a linked list in place?

+
Asked 3 years ago · Viewed 48k times
+
312
+
+

I have a singly linked list and want to reverse it without allocating a new list. My attempt loses the rest of the list after the first node:

+ 
while node:
+    node.next = prev
+    prev = node
+    node = node.next
+
+
python linked-list
+
+
+

2 Answers

+
+
518 Accepted
+
+

You overwrite node.next before saving it. Keep a reference to the next node first:

+ 
def reverse(head):
+    prev = None
+    node = head
+    while node:
+        nxt = node.next
+        node.next = prev
+        prev = node
+        node = nxt
+    return prev
+

This runs in O(n) time and O(1) extra space.

+
+
answered by ada
+
+
+
41
+

A recursive version also works, but it uses O(n) stack space and can hit the recursion limit on long lists.

+
+
+

Hot Network Questions

+ + + diff --git a/crates/markkit-eval/data/pages/legacy_layout.html b/crates/markkit-eval/data/pages/legacy_layout.html new file mode 100644 index 0000000..ad6699f --- /dev/null +++ b/crates/markkit-eval/data/pages/legacy_layout.html @@ -0,0 +1,27 @@ + +Pine Hollow Model Railroad Club + + + + + + +
+MENU
+Home
+Our Layout
+Join
+Guestbook +		 +
Pine Hollow Model Railroad Club
+

Welcome to the home of the Pine Hollow Model Railroad Club! We meet every Tuesday at 7 PM in the basement of the Grange Hall, 18 Mill Road. +

Our HO scale layout has over 600 feet of track and models the Boston & Maine railroad in 1952. +

New members are always welcome.
+Annual dues are $40.
+Contact our secretary, Walt Brennan, at walt@pinehollow.test. +

+

You are visitor number 004127 since 1998. +

Best viewed in Netscape Navigator 4.0 at 800x600. +

+ + diff --git a/crates/markkit-eval/data/pages/news_article.html b/crates/markkit-eval/data/pages/news_article.html new file mode 100644 index 0000000..27fe862 --- /dev/null +++ b/crates/markkit-eval/data/pages/news_article.html @@ -0,0 +1,61 @@ + + + + + City Council Approves Riverside Park Expansion | The Harbor Gazette + + + + + +
+ + +
+
+
+

City Council Approves Riverside Park Expansion

+

By Maria Chen ·

+
+ Aerial view of Riverside Park +
The current park covers 42 acres along the east bank.
+
+

The city council voted 7–2 on Tuesday to expand Riverside Park by 18 acres, converting a former rail yard into wetlands and walking trails.

+
AdvertisementRefinance today at 2.9% APR
+

The project is expected to cost $14.2 million, with most of the funding coming from a state conservation grant approved last year.

+
+

This is the largest addition to our public green space in fifty years.

+
+

Construction will begin in the fall and is scheduled to finish by spring 2028. Residents can review the plans at the city parks website.

+

What happens to the rail yard

+

Contaminated soil will be removed in the first phase. The two historic signal towers will be preserved as observation decks.

+ +
+ +
+ + + + diff --git a/crates/markkit-eval/data/pages/product_page.html b/crates/markkit-eval/data/pages/product_page.html new file mode 100644 index 0000000..721cb7e --- /dev/null +++ b/crates/markkit-eval/data/pages/product_page.html @@ -0,0 +1,40 @@ + + +Northwind Trail 40L Backpack | Summit Outfitters + +
+ +
+
Home / Packs / Northwind Trail 40L
+
+

Northwind Trail 40L Backpack

+

$189.00

+

A lightweight pack for multi-day hikes, with a ventilated back panel and a removable rain cover.

+
+ + +
+

Features

+ +

Specifications

+ + + + +
Weight1.2 kg
Capacity40 L
Dimensions64 × 32 × 24 cm
+

Customer reviews

+

4.6 out of 5 stars (212 reviews)

+

Comfortable on a 4-day trip. The hip belt pockets are big enough for a phone.

+
+ +
Free shipping on orders over $75! Use code TRAIL10 for 10% off.
+ + + diff --git a/crates/markkit-eval/data/pages/recipe.html b/crates/markkit-eval/data/pages/recipe.html new file mode 100644 index 0000000..d78ce8b --- /dev/null +++ b/crates/markkit-eval/data/pages/recipe.html @@ -0,0 +1,37 @@ + + +Weeknight Lemon Garlic Pasta - Pantry Table + +
+
+

Weeknight Lemon Garlic Pasta

+

Ready in 20 minutes with ingredients you probably already have.

+
Sponsored: Try MealBox free for 2 weeks
+ +

Ingredients

+ +

Instructions

+
    +
  1. Cook the spaghetti in salted water until al dente. Reserve 1 cup of pasta water.
    2. +
  2. Warm the olive oil and garlic over low heat until fragrant, about 3 minutes.
    3. +
  3. Toss the pasta with the garlic oil, lemon zest, lemon juice and parmesan.
    4. +
  4. Loosen with pasta water as needed and serve immediately.
    5. +
+

Notes

+

For a dairy-free version, replace the parmesan with toasted breadcrumbs.

+
Sponsored: Chef's knife sale, 40% off
+

Rate this recipe:

+
+ + + diff --git a/crates/markkit-eval/data/pages/wiki_article.html b/crates/markkit-eval/data/pages/wiki_article.html new file mode 100644 index 0000000..7f10e80 --- /dev/null +++ b/crates/markkit-eval/data/pages/wiki_article.html @@ -0,0 +1,37 @@ + + +Lighthouse of Alexandria - OpenWiki + +
OpenWiki Log in Random article
+
+

Lighthouse of Alexandria

+
From OpenWiki, the free encyclopedia
+ + + + + +
Lighthouse of Alexandria
LocationPharos, Alexandria, Egypt
Builtc. 280 BC
Heightabout 100 m
+

The Lighthouse of Alexandria was a lighthouse built by the Ptolemaic Kingdom on the island of Pharos.[1] It was one of the Seven Wonders of the Ancient World.

+
+

Contents

+ +
+

Construction[edit]

+

Construction was completed during the reign of Ptolemy II Philadelphus. The architect was Sostratus of Cnidus.

+

Design

+

The tower had three stages: a square lower section, an octagonal middle section and a circular top.

+

Destruction[edit]

+

The lighthouse was badly damaged by earthquakes between 956 and 1323 AD and became an abandoned ruin.

+

References

+
    +
  1. Clayton, Peter; Price, Martin (1988). The Seven Wonders of the Ancient World. Routledge.
    2. +
+
Categories: Lighthouses | Seven Wonders of the Ancient World
+
+ + + diff --git a/crates/markkit-eval/src/checks.rs b/crates/markkit-eval/src/checks.rs new file mode 100644 index 0000000..fbd69d5 --- /dev/null +++ b/crates/markkit-eval/src/checks.rs @@ -0,0 +1,223 @@ +// Deterministic expectation checks over extracted markdown. +// +// Pure string logic, no regex: page text is data, never a pattern. Each check +// returns pass/fail plus a short detail for the report. Per-metric aggregation +// (weighted pass rate) lives here too so scorers and tests share it. + +use std::collections::BTreeMap; + +use crate::dataset::{Check, Expectation, Metric}; + +/// Outcome of one expectation. +#[derive(Debug, Clone)] +pub struct CheckResult { + pub metric: Metric, + pub weight: f64, + pub passed: bool, + pub detail: String, +} + +/// Run one check against `markdown`. +pub fn run_check(check: &Check, markdown: &str) -> (bool, String) { + match check { + Check::Contains { text } => ( + markdown.contains(text.as_str()), + format!("contains {text:?}"), + ), + Check::NotContains { text } => ( + !markdown.contains(text.as_str()), + format!("not_contains {text:?}"), + ), + Check::Line { text } => ( + markdown.lines().any(|l| l.trim_end() == text), + format!("line {text:?}"), + ), + Check::Heading { level, text } => { + let want = format!("{} {}", "#".repeat(usize::from(*level)), text); + ( + (1..=6).contains(level) && markdown.lines().any(|l| l.trim_end() == want), + format!("heading {want:?}"), + ) + } + Check::Link { text, url } => { + let want = format!("[{text}]({url})"); + (markdown.contains(&want), format!("link {want:?}")) + } + Check::Order { first, then } => { + let ok = match (markdown.find(first.as_str()), markdown.find(then.as_str())) { + (Some(a), Some(b)) => a < b, + _ => false, + }; + (ok, format!("order {first:?} before {then:?}")) + } + } +} + +/// Run every expectation. +pub fn evaluate(expectations: &[Expectation], markdown: &str) -> Vec { + expectations + .iter() + .map(|e| { + let (passed, detail) = run_check(&e.check, markdown); + CheckResult { + metric: e.metric(), + weight: e.weight, + passed, + detail, + } + }) + .collect() +} + +/// Weighted pass rate of `results`; `None` when there is nothing to score. +pub fn weighted_rate<'a>(results: impl IntoIterator) -> Option { + let (mut total, mut passed) = (0.0, 0.0); + for r in results { + total += r.weight; + if r.passed { + passed += r.weight; + } + } + (total > 0.0).then(|| passed / total) +} + +/// Weighted pass rate per metric (metrics without checks are absent). +pub fn per_metric(results: &[CheckResult]) -> BTreeMap { + Metric::ALL + .into_iter() + .filter_map(|m| weighted_rate(results.iter().filter(|r| r.metric == m)).map(|v| (m, v))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + const MD: &str = + "# Title\n\nIntro with [docs](https://x.test/docs).\n\n## Usage \n\n- one\n- two\n"; + + fn c(check: Check) -> bool { + run_check(&check, MD).0 + } + + fn s(v: &str) -> String { + v.to_string() + } + + #[test] + fn contains_and_not_contains() { + assert!(c(Check::Contains { + text: s("Intro with") + })); + assert!(!c(Check::Contains { text: s("missing") })); + assert!(c(Check::NotContains { text: s("Cookie") })); + assert!(!c(Check::NotContains { text: s("Title") })); + } + + #[test] + fn line_matches_whole_line_ignoring_trailing_space() { + assert!(c(Check::Line { text: s("- one") })); + assert!(c(Check::Line { + text: s("## Usage") + })); + assert!(!c(Check::Line { text: s("one") })); + } + + #[test] + fn heading_level_and_text_exact() { + assert!(c(Check::Heading { + level: 1, + text: s("Title") + })); + assert!(c(Check::Heading { + level: 2, + text: s("Usage") + })); + assert!(!c(Check::Heading { + level: 2, + text: s("Title") + })); + assert!(!c(Check::Heading { + level: 1, + text: s("Tit") + })); + // Out-of-range levels never pass, even on a matching-looking line. + assert!( + !run_check( + &Check::Heading { + level: 0, + text: s("x") + }, + " x" + ) + .0 + ); + assert!( + !run_check( + &Check::Heading { + level: 7, + text: s("x") + }, + "####### x" + ) + .0 + ); + } + + #[test] + fn link_needs_exact_text_and_url() { + assert!(c(Check::Link { + text: s("docs"), + url: s("https://x.test/docs") + })); + assert!(!c(Check::Link { + text: s("docs"), + url: s("https://x.test/") + })); + assert!(!c(Check::Link { + text: s("Docs"), + url: s("https://x.test/docs") + })); + } + + #[test] + fn order_requires_both_and_sequence() { + assert!(c(Check::Order { + first: s("Title"), + then: s("Usage") + })); + assert!(!c(Check::Order { + first: s("Usage"), + then: s("Title") + })); + assert!(!c(Check::Order { + first: s("Title"), + then: s("absent") + })); + assert!(!c(Check::Order { + first: s("absent"), + then: s("Title") + })); + } + + #[test] + fn weighted_rates_per_metric() { + let r = |metric, weight, passed| CheckResult { + metric, + weight, + passed, + detail: String::new(), + }; + let results = vec![ + r(Metric::Completeness, 3.0, true), + r(Metric::Completeness, 1.0, false), + r(Metric::Noise, 1.0, false), + ]; + assert_eq!(weighted_rate(&results), Some(0.6)); + let m = per_metric(&results); + assert_eq!(m[&Metric::Completeness], 0.75); + assert_eq!(m[&Metric::Noise], 0.0); + assert!(!m.contains_key(&Metric::Formatting)); + assert_eq!(weighted_rate(&[]), None); + } +} diff --git a/crates/markkit-eval/src/dataset.rs b/crates/markkit-eval/src/dataset.rs new file mode 100644 index 0000000..b346fb3 --- /dev/null +++ b/crates/markkit-eval/src/dataset.rs @@ -0,0 +1,245 @@ +// Eval dataset: extraction tasks (JSONL) and the pages they extract. +// +// Decisions: +// - Pages live as real `.html` files under `data/pages/` (reviewable, open in a +// browser) and are embedded with `include_str!` via `PAGES`, so the study has +// no runtime path dependence. A test keeps `PAGES` and the directory in sync. +// - Expectations are typed (`check` tag) instead of bashkit's `kind:arg` +// strings: page text routinely contains `:`, which made string parsing +// ambiguous for two-argument checks (`link`, `order`). +// - Each expectation belongs to one quality metric (completeness, accuracy, +// formatting, noise); the default follows the check kind. Scorers aggregate +// per metric. See knowledge/operations/eval.md. + +use serde::{Deserialize, Serialize}; + +/// One extraction task: a page plus what ideal markdown for it must satisfy. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExtractTask { + pub id: String, + pub category: String, + pub description: String, + /// Page file name under `data/pages/` (a key of [`PAGES`]). + pub page: String, + pub expectations: Vec, +} + +/// Quality dimension an expectation measures. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Metric { + /// Important content is preserved. + Completeness, + /// Content is faithful: right text, right order, right targets. + Accuracy, + /// Markdown structure is right (headings, lists, tables, code fences). + Formatting, + /// Non-content (nav, ads, banners, comments) is excluded. + Noise, +} + +impl Metric { + pub const ALL: [Metric; 4] = [ + Metric::Completeness, + Metric::Accuracy, + Metric::Formatting, + Metric::Noise, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Metric::Completeness => "completeness", + Metric::Accuracy => "accuracy", + Metric::Formatting => "formatting", + Metric::Noise => "noise", + } + } +} + +/// A deterministic check against the extracted markdown. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "check", rename_all = "snake_case")] +pub enum Check { + /// `text` occurs somewhere in the output. + Contains { text: String }, + /// `text` does not occur anywhere in the output. + NotContains { text: String }, + /// Some output line equals `text` (trailing whitespace ignored). + Line { text: String }, + /// An ATX heading of exactly `level` with exactly `text`. + Heading { level: u8, text: String }, + /// An inline link `[text](url)`. + Link { text: String, url: String }, + /// Both occur and the first occurrence of `first` precedes `then`. + Order { first: String, then: String }, +} + +impl Check { + /// Metric a check measures when the task does not say. + pub fn default_metric(&self) -> Metric { + match self { + Check::Contains { .. } => Metric::Completeness, + Check::NotContains { .. } => Metric::Noise, + Check::Line { .. } | Check::Heading { .. } => Metric::Formatting, + Check::Link { .. } | Check::Order { .. } => Metric::Accuracy, + } + } +} + +/// A check with its metric and weight. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Expectation { + #[serde(flatten)] + pub check: Check, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metric: Option, + #[serde(default = "default_weight")] + pub weight: f64, +} + +fn default_weight() -> f64 { + 1.0 +} + +impl Expectation { + pub fn metric(&self) -> Metric { + self.metric.unwrap_or_else(|| self.check.default_metric()) + } +} + +/// Embedded task list. +pub const TASKS: &str = include_str!("../data/extract-tasks.jsonl"); + +/// Embedded pages, keyed by file name under `data/pages/`. +pub const PAGES: &[(&str, &str)] = &[ + ( + "changelog.html", + include_str!("../data/pages/changelog.html"), + ), + ( + "data_table.html", + include_str!("../data/pages/data_table.html"), + ), + ( + "docs_page.html", + include_str!("../data/pages/docs_page.html"), + ), + ( + "forum_thread.html", + include_str!("../data/pages/forum_thread.html"), + ), + ( + "legacy_layout.html", + include_str!("../data/pages/legacy_layout.html"), + ), + ( + "news_article.html", + include_str!("../data/pages/news_article.html"), + ), + ( + "product_page.html", + include_str!("../data/pages/product_page.html"), + ), + ("recipe.html", include_str!("../data/pages/recipe.html")), + ( + "blog_post.html", + include_str!("../data/pages/blog_post.html"), + ), + ( + "wiki_article.html", + include_str!("../data/pages/wiki_article.html"), + ), +]; + +/// Look up an embedded page by file name. +pub fn page(name: &str) -> Option<&'static str> { + PAGES + .iter() + .find(|(n, _)| *n == name) + .map(|(_, html)| *html) +} + +/// Parse JSONL tasks. Blank and `#` / `//` comment lines are skipped; a +/// malformed line is an error (a silently dropped task would skew scores). +pub fn parse_tasks(text: &str) -> Result, String> { + text.lines() + .enumerate() + .map(|(i, l)| (i + 1, l.trim())) + .filter(|(_, l)| !l.is_empty() && !l.starts_with('#') && !l.starts_with("//")) + .map(|(n, l)| serde_json::from_str(l).map_err(|e| format!("line {n}: {e}"))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeSet; + + #[test] + fn tasks_parse_and_resolve_pages() { + let tasks = parse_tasks(TASKS).expect("dataset parses"); + assert!(tasks.len() >= 10); + let mut ids = BTreeSet::new(); + for t in &tasks { + assert!(ids.insert(t.id.clone()), "duplicate id {}", t.id); + assert!(page(&t.page).is_some(), "{}: unknown page {}", t.id, t.page); + assert!(!t.expectations.is_empty(), "{}: no expectations", t.id); + for e in &t.expectations { + assert!(e.weight > 0.0, "{}: non-positive weight", t.id); + } + } + } + + #[test] + fn every_task_covers_completeness_and_noise() { + // Each page has chrome to strip and content to keep; a task that only + // checks one side cannot tell "kept everything" from "kept nothing". + for t in parse_tasks(TASKS).unwrap() { + let metrics: BTreeSet<_> = t.expectations.iter().map(|e| e.metric()).collect(); + assert!(metrics.contains(&Metric::Completeness), "{}", t.id); + assert!(metrics.contains(&Metric::Noise), "{}", t.id); + } + } + + #[test] + fn pages_table_matches_data_dir() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/data/pages"); + let on_disk: BTreeSet = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + let embedded: BTreeSet = PAGES.iter().map(|(n, _)| n.to_string()).collect(); + assert_eq!(on_disk, embedded, "PAGES out of sync with data/pages/"); + let used: BTreeSet = parse_tasks(TASKS) + .unwrap() + .into_iter() + .map(|t| t.page) + .collect(); + assert_eq!(used, embedded, "every page is used by a task"); + } + + #[test] + fn malformed_line_is_an_error() { + let err = parse_tasks("{\"id\":\"x\"}\n").unwrap_err(); + assert!(err.starts_with("line 1:"), "{err}"); + } + + #[test] + fn unknown_check_is_an_error() { + let line = r#"{"id":"x","category":"c","description":"d","page":"p","expectations":[{"check":"eval","text":"x"}]}"#; + assert!(parse_tasks(line).is_err()); + } + + #[test] + fn default_metric_follows_check_and_override_wins() { + let line = r#"{"id":"x","category":"c","description":"d","page":"p","expectations":[ + {"check":"contains","text":"a"}, + {"check":"contains","text":"b","metric":"accuracy","weight":2}]}"# + .replace('\n', ""); + let t = &parse_tasks(&line).unwrap()[0]; + assert_eq!(t.expectations[0].metric(), Metric::Completeness); + assert_eq!(t.expectations[0].weight, 1.0); + assert_eq!(t.expectations[1].metric(), Metric::Accuracy); + assert_eq!(t.expectations[1].weight, 2.0); + } +} diff --git a/crates/markkit-eval/src/lib.rs b/crates/markkit-eval/src/lib.rs new file mode 100644 index 0000000..8d185c3 --- /dev/null +++ b/crates/markkit-eval/src/lib.rs @@ -0,0 +1,14 @@ +//! Mira eval study for markkit extraction quality. +//! +//! Run through the `mira` host CLI (`cargo install mira-cli`): +//! +//! ```text +//! mira list --study-bin markkit-eval +//! mira run --study-bin markkit-eval markkit_extract +//! ``` +//! +//! See `knowledge/operations/eval.md` for the design. + +pub mod checks; +pub mod dataset; +pub mod mira_study; diff --git a/crates/markkit-eval/src/main.rs b/crates/markkit-eval/src/main.rs index e76d643..a7b837b 100644 --- a/crates/markkit-eval/src/main.rs +++ b/crates/markkit-eval/src/main.rs @@ -1,22 +1,23 @@ -use anyhow::Result; -use clap::Parser; +// markkit-eval: mira eval study host. +// +// Advertises markkit's evals to the `mira` CLI over stdio: +// +// mira list --study-bin markkit-eval +// mira run --study-bin markkit-eval markkit_extract +// mira run --study-bin markkit-eval markkit_extract --tag news --format html --out report.html +// +// The `#[eval]` wrappers live in the bin crate so their inventory registrations +// are guaranteed to link into this binary; the work is in +// `markkit_eval::mira_study`. See knowledge/operations/eval.md. -#[derive(Parser)] -#[command(name = "markkit-eval", about = "Evaluate markkit extraction quality")] -struct Cli { - /// Provider (anthropic, openai) - #[arg(long, default_value = "anthropic")] - provider: String, +use markkit_eval::mira_study; +use mira::{Eval, eval}; - /// Evaluation dataset path - #[arg(long)] - dataset: Option, +#[eval] +fn markkit_extract() -> Eval { + mira_study::extract_eval() } -#[tokio::main] -async fn main() -> Result<()> { - let _cli = Cli::parse(); - // TODO: Implement evaluation harness - println!("markkit-eval: evaluation harness not yet implemented"); - Ok(()) +fn main() -> std::io::Result<()> { + mira::Study::registered().serve_blocking() } diff --git a/crates/markkit-eval/src/mira_study.rs b/crates/markkit-eval/src/mira_study.rs new file mode 100644 index 0000000..abb4747 --- /dev/null +++ b/crates/markkit-eval/src/mira_study.rs @@ -0,0 +1,271 @@ +// Mira integration: turns the markkit extraction eval into a mira `Study`. +// +// Three pieces wire markkit into mira (github.com/everruns/mira): +// +// 1. Samples - each JSONL `ExtractTask` becomes a mira `Sample`. The task +// rides in `sample.metadata["task"]` (source of truth for subject and +// scorers); the page HTML in `sample.files["page.html"]`. +// 2. Subject - `extract_subject` runs markkit on the page in-process and puts +// the markdown in `transcript.final_response`, with size/time metrics. +// 3. Scorers - `expectations_scorer` (pass iff every check passes, value = +// weighted pass rate) plus one scorer per quality metric (completeness, +// accuracy, formatting, noise), so reports break quality down by metric. +// +// Decision: the thing under test is markkit, not a model, so the eval's only +// target is `Target::cli("markkit")`: always available, no API key, fully +// offline and deterministic. LLM-backed evals (judge, downstream QA) add model +// targets in their own evals. See knowledge/operations/eval.md. + +use std::time::Instant; + +use markkit::Markkit; +use mira::subject::{Subject, subject_fn}; +use mira::{Eval, Sample, Score, Target, Transcript}; + +use crate::checks::{evaluate, per_metric, weighted_rate}; +use crate::dataset::{ExtractTask, Metric, TASKS, page, parse_tasks}; + +/// Workspace path the page is seeded under (visible in mira reports). +pub const PAGE_FILE: &str = "page.html"; + +/// Name of the all-checks scorer. +pub const EXPECTATIONS_SCORER: &str = "markkit_expectations"; + +/// The single target: markkit itself. +pub fn markkit_target() -> Target { + Target::cli("markkit") +} + +/// Build one sample per task. Fails on a malformed dataset or unknown page. +pub fn extract_samples(jsonl: &str) -> Result, String> { + parse_tasks(jsonl)? + .into_iter() + .map(|task| { + let html = page(&task.page) + .ok_or_else(|| format!("{}: unknown page {}", task.id, task.page))?; + let task_value = serde_json::to_value(&task).map_err(|e| e.to_string())?; + Ok(Sample::new(&task.id, &task.description) + .tag(&task.category) + .tag("html") + .meta("category", task.category.clone()) + .meta("page", task.page.clone()) + .meta("task", task_value) + .file(PAGE_FILE, html)) + }) + .collect() +} + +fn task_from_sample(sample: &Sample) -> Result { + let value = sample + .metadata + .get("task") + .cloned() + .ok_or("sample missing task metadata")?; + serde_json::from_value(value).map_err(|e| format!("bad task metadata: {e}")) +} + +/// Run markkit with default settings on a page and describe the run. +pub fn extract_transcript(html: &str) -> Transcript { + let markkit = Markkit::builder().build(); + let start = Instant::now(); + let result = markkit.extract_html_sync(html); + let elapsed = start.elapsed(); + + let markdown = match result { + Ok(md) => md, + // markkit rejecting a page is markkit's failure, not infra: score it. + Err(e) => return Transcript::failed(format!("extraction failed: {e}")), + }; + + let mut t = Transcript::response(markdown.clone()); + t.timing.duration_ms = elapsed.as_millis() as u64; + t.record_metric("extract_us", elapsed.as_micros() as f64); + t.record_metric("input_bytes", html.len() as f64); + t.record_metric("output_bytes", markdown.len() as f64); + if !html.is_empty() { + t.record_metric("size_ratio", markdown.len() as f64 / html.len() as f64); + } + t +} + +/// Subject: extract the sample's page with markkit. +fn extract_subject() -> impl Subject { + subject_fn(|sample: Sample, _cx| async move { + match sample.files.get(PAGE_FILE) { + Some(html) => extract_transcript(html), + None => Transcript::infra_error("sample missing page.html"), + } + }) +} + +/// Score `transcript` against the sample's expectations, optionally restricted +/// to one metric. `None` metric = every check, pass iff all pass. +fn score(scorer: &str, metric: Option, sample: &Sample, t: &Transcript) -> Score { + if t.errored_infra() { + return Score::na(scorer, "subject infra error"); + } + let task = match task_from_sample(sample) { + Ok(task) => task, + Err(e) => return Score::na(scorer, e), + }; + // A failed extraction produced no markdown: every check runs against "", + // so content checks fail and noise checks pass, like an empty output would. + let results = evaluate(&task.expectations, &t.final_response); + let selected: Vec<_> = results + .iter() + .filter(|r| metric.is_none_or(|m| r.metric == m)) + .collect(); + let Some(value) = weighted_rate(selected.iter().copied()) else { + return Score::na(scorer, "no checks for this metric"); + }; + let failed: Vec<&str> = selected + .iter() + .filter(|r| !r.passed) + .map(|r| r.detail.as_str()) + .collect(); + let passed = selected.len() - failed.len(); + let mut reason = format!("{passed}/{} checks passed", selected.len()); + if metric.is_none() { + let parts: Vec = per_metric(&results) + .into_iter() + .map(|(m, v)| format!("{} {:.0}%", m.as_str(), v * 100.0)) + .collect(); + reason.push_str(&format!(" ({})", parts.join(", "))); + } + if !failed.is_empty() { + reason.push_str(&format!("; failed: {}", failed.join("; "))); + } + Score { + scorer: scorer.to_string(), + value, + pass: failed.is_empty(), + na: false, + reason, + } +} + +/// All checks: pass iff every check passes; value = weighted pass rate. +pub fn expectations_scorer() -> Box { + mira::scorer::scorer(EXPECTATIONS_SCORER, |sample: &Sample, t: &Transcript| { + score(EXPECTATIONS_SCORER, None, sample, t) + }) +} + +/// One metric's checks; N/A when the task has none for it. +pub fn metric_scorer(metric: Metric) -> Box { + let name = metric.as_str(); + mira::scorer::scorer(name, move |sample: &Sample, t: &Transcript| { + score(name, Some(metric), sample, t) + }) +} + +/// The extraction-quality eval: every dataset page through markkit, scored by +/// deterministic checks. Slice with `mira run --tag `. +pub fn extract_eval() -> Eval { + let samples = extract_samples(TASKS).expect("embedded dataset is valid (tested)"); + let mut b = Eval::new("markkit_extract") + .describe("HTML extraction quality: completeness, accuracy, formatting, noise") + .subject(extract_subject()) + .scorer(expectations_scorer()) + .max_turns(1) + .target(markkit_target()); + for m in Metric::ALL { + b = b.scorer(metric_scorer(m)); + } + for s in samples { + b = b.add_sample(s); + } + b.build() +} + +#[cfg(test)] +mod tests { + use super::*; + use mira::RunCx; + + #[test] + fn samples_carry_task_and_page() { + let samples = extract_samples(TASKS).unwrap(); + assert!(samples.len() >= 10); + for s in &samples { + let task = task_from_sample(s).unwrap(); + assert_eq!(s.files[PAGE_FILE], page(&task.page).unwrap()); + assert!(s.tags.contains(&task.category), "{}", s.id); + } + } + + #[test] + fn unknown_page_is_rejected() { + let line = r#"{"id":"x","category":"c","description":"d","page":"nope.html","expectations":[{"check":"contains","text":"a"}]}"#; + assert!(extract_samples(line).unwrap_err().contains("unknown page")); + } + + #[test] + fn eval_builds() { + let _ = extract_eval(); + } + + fn sample(expectations: &str) -> Sample { + let line = format!( + r#"{{"id":"t","category":"c","description":"d","page":"recipe.html","expectations":{expectations}}}"# + ); + extract_samples(&line).unwrap().remove(0) + } + + #[test] + fn scorer_pass_partial_and_metric_breakdown() { + let s = sample( + r#"[{"check":"contains","text":"alpha"},{"check":"contains","text":"zeta"},{"check":"not_contains","text":"ad"}]"#, + ); + let t = Transcript::response("# alpha beta"); + let all = score(EXPECTATIONS_SCORER, None, &s, &t); + assert!(!all.pass); + assert!((all.value - 2.0 / 3.0).abs() < 1e-9); + assert!(all.reason.contains("completeness 50%"), "{}", all.reason); + assert!(all.reason.contains("contains \"zeta\""), "{}", all.reason); + + let noise = score("noise", Some(Metric::Noise), &s, &t); + assert!(noise.pass && noise.value == 1.0); + let fmt = score("formatting", Some(Metric::Formatting), &s, &t); + assert!(fmt.na, "no formatting checks => N/A"); + } + + #[test] + fn infra_error_is_na_but_extraction_failure_is_scored() { + let s = sample(r#"[{"check":"contains","text":"alpha"}]"#); + assert!(score("x", None, &s, &Transcript::infra_error("boom")).na); + let failed = score("x", None, &s, &Transcript::failed("extraction failed")); + assert!(!failed.na && !failed.pass); + } + + #[test] + fn extraction_failure_is_not_infra() { + // Over the input limit: markkit refuses, which must count against it. + let markkit_limit = Markkit::builder().build().config().max_input_size; + let huge = "a".repeat(markkit_limit + 1); + let t = extract_transcript(&huge); + assert!(t.error.is_some() && !t.errored_infra()); + } + + #[tokio::test] + async fn subject_extracts_page_and_records_metrics() { + let s = sample(r#"[{"check":"contains","text":"x"}]"#); + let t = extract_subject() + .run(&s, &RunCx::new(markkit_target())) + .await; + assert!(t.error.is_none()); + assert!(!t.final_response.is_empty()); + for m in ["extract_us", "input_bytes", "output_bytes", "size_ratio"] { + assert!(t.metrics.contains_key(m), "missing metric {m}"); + } + } + + #[tokio::test] + async fn subject_without_page_is_infra() { + let s = Sample::new("bare", "no page"); + let t = extract_subject() + .run(&s, &RunCx::new(markkit_target())) + .await; + assert!(t.errored_infra()); + } +} diff --git a/justfile b/justfile index 85d67fd..d9f7f9b 100644 --- a/justfile +++ b/justfile @@ -89,9 +89,18 @@ bench-harness *args: # ─── Eval ──────────────────────────────────────────────────── -# Run the LLM evaluation harness (stub, not yet implemented) +# Evals run through the mira host CLI (cargo install mira-cli); markkit-eval is +# the study binary. Saved runs: crates/markkit-eval/results (mira.toml). +# See knowledge/operations/eval.md. + +# List advertised evals, samples, scorers, and targets +eval-list: + mira list --study-bin markkit-eval + +# Run the extraction-quality eval. Extra mira flags pass through, e.g. +# just eval --tag docs --format html --out report.html eval *args: - cargo run -p markkit-eval --release -- {{args}} + mira run --study-bin markkit-eval markkit_extract {{args}} # ─── Security & Audit ─────────────────────────────────────── diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 9781979..a4fa603 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -41,7 +41,7 @@ Input (bytes/URL/file) | `markkit` | Core library – parsing, conversion, API | | `markkit-cli` | Command-line tool | | `markkit-bench` | Performance benchmarks | -| `markkit-eval` | LLM-based quality evaluation | +| `markkit-eval` | Extraction-quality eval study (mira) | | `markkit-python` | Python bindings (PyO3) | | `markkit-js` | Node.js bindings (NAPI-RS) | diff --git a/knowledge/log.md b/knowledge/log.md index 1b2ff07..5fd654c 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Eval harness on mira**: `markkit-eval` is now a mira study (modeled on bashkit-eval). `markkit_extract` scores markkit's HTML output on ten realistic pages with typed deterministic checks and one scorer per metric; baseline 72.5% mean, noise 34.3%, with gaps recorded (row-header tables lose values, `tfoot`/`caption` dropped, asides and ads kept). See [Evaluation Harness](operations/eval.md). * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). * **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). diff --git a/knowledge/operations/eval.md b/knowledge/operations/eval.md index 1af0b2c..bd6a5b7 100644 --- a/knowledge/operations/eval.md +++ b/knowledge/operations/eval.md @@ -1,45 +1,168 @@ --- type: Test Strategy title: Evaluation Harness -description: LLM-as-judge extraction quality evaluation with its methodology, metrics, and providers. +description: markkit-eval as a mira study, its extraction-quality eval, dataset format, check types, metrics, and baseline. tags: - markkit - eval - - llm + - mira --- # Evaluation Harness ## Status -Not implemented. `crates/markkit-eval` is a CLI stub (`--provider`, `--dataset`) -that prints a placeholder and exits 0. This document is the design it will follow. +`markkit_extract` implemented: an offline, deterministic extraction-quality +eval on the [mira](https://github.com/everruns/mira) eval framework. LLM-backed +evals (LLM-as-judge, downstream QA over a model matrix) are planned as separate +evals in the same study; see [Planned evals](#planned-evals). Modeled on +everruns/bashkit's `bashkit-eval` study. -## Overview +## Architecture -`markkit-eval` measures extraction quality using LLM-as-judge -to compare markkit output against reference extractions. +`markkit-eval` is a **mira study**: a binary that advertises its evals to the +`mira` host CLI over stdio. Mira owns selection, scheduling, saved runs and +reporting (terminal, JSON/JSONL/CSV/JUnit/Markdown/HTML). markkit supplies the +subject under test and the scoring. -## Methodology +``` +mira host CLI ──spawns──▶ markkit-eval (study binary) + │ │ + │ selection (--tag) ├─ Sample (one per JSONL task, page seeded) + │ saved runs / resume ├─ Subject (markkit extract_html_sync) + │ reports └─ Scorers (all checks + one per metric) +``` -1. Curate a dataset of (URL, reference markdown) pairs -2. Extract markdown from each URL using markkit -3. Score extraction quality via LLM comparison -4. Track metrics over time +Code: `crates/markkit-eval/src/` (`dataset.rs`, `checks.rs`, `mira_study.rs`; +`main.rs` holds the `#[eval]` wrappers so their registrations link into the +binary). + +### Key Design Decisions + +1. **The target is markkit, not a model.** `markkit_extract` has one target, + `Target::cli("markkit")`: always available, no API key, offline and + deterministic. Model targets belong to the LLM evals only. +2. **In-process subject.** The subject calls `Markkit::extract_html_sync` with + default settings; no CLI round-trip. Depends only on `mira-eval` (no + `mira-everruns`) and `markkit` with `html` only (no HTTP client). +3. **Typed checks, not `kind:arg` strings.** Page text routinely contains `:`, + which makes string parsing of two-argument checks (`link`, `order`) + ambiguous. Checks are a tagged JSON enum. No regex: page text is data, + never a pattern. +4. **Per-metric scorers.** Every expectation belongs to one metric + (completeness, accuracy, formatting, noise). Mira reports one score per + metric per page plus the all-checks score, so a regression shows which + dimension moved. +5. **Pages are real `.html` files** under `data/pages/` (reviewable, open in a + browser), embedded with `include_str!` through `dataset::PAGES`. A test keeps + the table, the directory and the tasks in sync. +6. **Expectations describe ideal markdown, not current output.** Failing + checks are the point: they are the extraction gaps. Tests do not gate on the + score; they gate on dataset validity and scorer behavior. +7. **Extraction errors are scored, not infra.** markkit refusing a page is + markkit's failure (`Transcript::failed`, checks run against empty output). + Only a malformed sample is an infra error (N/A). + +## Dataset Format (JSONL) + +`crates/markkit-eval/data/extract-tasks.jsonl`, one task per line (`#` and +`//` comment lines skipped; a malformed line is an error, never dropped): + +```json +{"id": "recipe", "category": "recipe", "description": "Recipe with ad slots ...", + "page": "recipe.html", + "expectations": [ + {"check": "heading", "level": 1, "text": "Weeknight Lemon Garlic Pasta", "weight": 2}, + {"check": "line", "text": "- 350 g spaghetti"}, + {"check": "not_contains", "text": "Sponsored"} + ]} +``` + +`page` = file under `data/pages/`; `category` becomes a mira tag (`--tag +recipe`), as does `html`; `weight` defaults to 1; `metric` overrides the +check's default metric. Every task must check both completeness and noise +(test-enforced), so "kept everything" and "kept nothing" both fail. + +## Check Types + +| Check | Fields | Passes when | Default metric | +|-------|--------|-------------|----------------| +| `contains` | `text` | `text` occurs in the output | completeness | +| `not_contains` | `text` | `text` does not occur | noise | +| `line` | `text` | some line equals `text` (trailing whitespace ignored) | formatting | +| `heading` | `level`, `text` | line `#{level} {text}`; level outside 1-6 never passes | formatting | +| `link` | `text`, `url` | `[text](url)` occurs | accuracy | +| `order` | `first`, `then` | both occur, first occurrence of `first` precedes `then` | accuracy | ## Metrics -- **Completeness**: Is all important content preserved? -- **Accuracy**: Is the content faithful to the original? -- **Formatting**: Is the markdown well-structured? -- **Noise**: Is non-content (nav, ads, etc.) excluded? +- **Completeness**: important content preserved. +- **Accuracy**: content faithful (order, link targets). +- **Formatting**: markdown structure right (headings, lists, tables, fences). +- **Noise**: non-content (nav, ads, banners, comments, widgets) excluded. + +Scorers: `markkit_expectations` (pass iff every check passes; value = weighted +pass rate; reason lists per-metric rates and failed checks) and one scorer per +metric (N/A when a task has no check for it). Subject metrics +(`transcript.metrics`): `extract_us`, `input_bytes`, `output_bytes`, +`size_ratio`. + +## Dataset + +Ten pages, one per category, each with realistic chrome around the content: +`news` (cookie banner, inline ad, newsletter form, related aside), `docs` +(sidebar, option table, TOML/Rust code, edit link, pager), `blog` (share +buttons, comments, popup), `commerce` (breadcrumbs, buy box, row-header spec +table, recommendations, promo), `recipe` (ad slots, lists, rating widget), +`reference` (infobox, citations, TOC, edit links, categories), `forum` (votes, +accepted answer with code, sidebar), `changelog` (nested lists, strikethrough, +issue links, CTA), `data` (captioned table with thead/tbody/tfoot), `legacy` +(table layout, font tags, unclosed `

`, hit counter). + +## Baseline (2026-10-26) + +0/10 pages pass every check. Mean weighted rate 72.5%: completeness 92.3%, +accuracy 100%, formatting 93.3%, noise 34.3%. Gaps found: + +- **Noise**: no readability heuristic, so `