feat(eval): add public WCXB and AgentDocBench regression benchmarks

.github/workflows/public-benchmarks.yml

@@ -0,0 +1,74 @@
+name: Public benchmarks
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  schedule:
+    - cron: "30 5 * * *"
+  workflow_dispatch:
+    inputs:
+      scope:
+        description: "Dataset selection (test quality is report-only)"
+        type: choice
+        options: [pr, dev, test]
+        default: pr
+
+permissions:
+  contents: read
+
+concurrency:
+  group: public-benchmarks-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    name: Public benchmark regressions
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      # Input is data, never shell code; prepare/run additionally validate choices.
+      BENCHMARK_SCOPE: ${{ github.event_name == 'schedule' && 'dev' || inputs.scope || 'pr' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Cache verified corpora
+        uses: actions/cache@v4
+        with:
+          path: |
+            target/public-benchmarks/wcxb
+            target/public-benchmarks/agentdocbench
+          key: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-${{ env.BENCHMARK_SCOPE }}
+          restore-keys: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-
+      - name: Install scorer dependencies
+        run: python3 -m pip install --require-hashes -r scripts/public_benchmarks/requirements.txt
+      - name: Test benchmark contracts
+        run: python3 -m unittest discover -s scripts/tests -p 'test_public_benchmarks.py'
+      - name: Build release CLI
+        run: cargo build --locked --release -p markkit-cli
+      - name: Install verified Mira host
+        run: python3 scripts/public_benchmarks/install_mira.py
+      - name: Prepare pinned input selection
+        run: python3 scripts/public_benchmarks/prepare.py --scope "$BENCHMARK_SCOPE"
+      - name: Run and gate through Mira
+        run: python3 scripts/public_benchmarks/run.py --scope "$BENCHMARK_SCOPE" --mira target/public-benchmarks/bin/mira
+      - name: Job summary
+        if: always()
+        run: |
+          report="target/public-benchmarks/report-$BENCHMARK_SCOPE/summary.md"
+          if [ -f "$report" ]; then cat "$report" >> "$GITHUB_STEP_SUMMARY"; fi
+      - name: Upload reports and provenance
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: public-benchmarks-${{ env.BENCHMARK_SCOPE }}
+          path: target/public-benchmarks/report-${{ env.BENCHMARK_SCOPE }}/
+          if-no-files-found: warn
+          retention-days: 14

AGENTS.md

@@ -78,6 +78,9 @@ just check-okf     # knowledge/ OKF conformance
 just examples      # Run every example
 just bench         # Benchmark harness, saves results to crates/markkit-bench/results
 just bench-criterion # Criterion micro-benches (crates/markkit/benches)
+just eval-public-setup   # Pinned Mira + isolated Python scorer environment
+just eval-public-prepare # Download and verify public corpora (PR slice)
+just eval-public        # Release WCXB + AgentDocBench runs and regression gate
 just pre-pr        # Pre-PR checks
 ```
 

CHANGELOG.md

@@ -15,6 +15,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
+- Public WCXB and AgentDocBench evaluations through Mira (`just eval-public`), with
+  pinned/verified corpora, a release-build baseline regression gate, per-type/format
+  quality and conversion timings. PR/push runs use a fixed slice; nightly runs cover
+  the full development selection after excluding 139 upstream dev/test duplicates.
+
 - `markkit_qa` LLM eval (`just eval-llm`): models answer 23 questions about the eval pages from
   markkit markdown, main-content markdown or raw HTML, scored on correct answers with token usage
   per input. Targets: Meta Muse Spark 1.3 contributor via OpenRouter and Claude Sonnet 5.5;

NOTICE

@@ -34,3 +34,11 @@ Markkit relies on several permissively licensed crates:
 * serde / serde_json (MIT/Apache-2.0) – Serialization
 
 Full dependency license details are available via `cargo deny check licenses`.
+
+=== Public Benchmark Tooling ===
+
+scripts/public_benchmarks/vendor includes the Mira Python SDK (Everruns, MIT),
+the AgentDocBench scorer (SylphxAI, MIT), and the WCXB scorer (Murrough Foley,
+CC-BY-4.0), each with its license. Input documents retain the individual
+licenses and attribution in scripts/public_benchmarks/CORPUS-NOTICE and
+agentdocbench-corpus.json. The downloaded corpora are not included in packages.

README.md

@@ -329,8 +329,14 @@ just lint       # Run clippy
 just check      # fmt, clippy, tests, knowledge, examples
 just pre-pr     # Full pre-PR validation
 just bench      # Benchmarks, results saved to crates/markkit-bench/results
+just eval-public-setup   # Pinned Mira host + isolated scorer environment
+just eval-public-prepare # Download and verify WCXB + AgentDocBench inputs
+just eval-public        # Release quality/timing run and regression gate
 ```
 
+Public benchmark selections, scoring and baseline policy:
+[`scripts/public_benchmarks/README.md`](scripts/public_benchmarks/README.md).
+
 Coding agents: start with [AGENTS.md](AGENTS.md).
 
 ## Contributing

crates/markkit-eval/results/public-dev-2026-10-28.json

@@ -0,0 +1,110 @@
+{
+  "case_count": 2754,
+  "manifest_sha256": "1a696cff621de426697a1bc0720aeebd2b82d59da96700296084089ce9c0c05a",
+  "provenance": {
+    "binary_sha256": "390d54760c41a108b4ed4f94c6aa99b993bbf04d8a36e94e4ecb861e66e56918",
+    "build_profile": "release",
+    "command": [
+      "/workspace/work/markkit-tools/bin/mira",
+      "run",
+      "--study-python",
+      "scripts/public_benchmarks/study.py",
+      "--max-concurrent",
+      "1",
+      "--max-retries",
+      "0",
+      "--timeout",
+      "300",
+      "--group-by",
+      "category",
+      "--format",
+      "json",
+      "--out",
+      "/workspace/markkit/target/public-benchmarks/report-dev/mira.json"
+    ],
+    "core_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "cpu_count": 5,
+    "machine": "x86_64",
+    "mira_version": "mira 0.5.0",
+    "pins": {
+      "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3",
+      "mira": "v0.5.0",
+      "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc",
+      "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e",
+      "samples_per_type": 10,
+      "seed": "markkit-public-pilot-v1:",
+      "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9"
+    },
+    "platform": "Linux-6.18.55-x86_64-with-glibc2.41",
+    "python": "3.12.14",
+    "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)",
+    "source_dirty": true,
+    "source_revision": "0e1a9df33e6f3ec0967be5ae7f6b8016a1f585ba",
+    "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup"
+  },
+  "quality": {
+    "agentdocbench/category/cjk": 0.749,
+    "agentdocbench/category/docx": 0.9988666666666667,
+    "agentdocbench/category/epub": 0.9891333333333333,
+    "agentdocbench/category/financial": 0.423325,
+    "agentdocbench/category/form": 0.7486666666666667,
+    "agentdocbench/category/html": 1.0,
+    "agentdocbench/category/paper": 0.8231999999999999,
+    "agentdocbench/category/report": 0.87095,
+    "agentdocbench/category/scanned": 0.0,
+    "agentdocbench/category/slides": 0.674775,
+    "agentdocbench/category/spreadsheet": 1.0,
+    "agentdocbench/category/two-column": 0.7465666666666667,
+    "agentdocbench/format/csv": 1.0,
+    "agentdocbench/format/docx": 0.9988666666666667,
+    "agentdocbench/format/epub": 0.9891333333333333,
+    "agentdocbench/format/html": 1.0,
+    "agentdocbench/format/pdf": 0.5835739130434783,
+    "agentdocbench/format/pptx": 0.9572,
+    "agentdocbench/format/xlsx": 1.0,
+    "agentdocbench/no_ocr": 0.8204075757575757,
+    "agentdocbench/overall": 0.7520402777777778,
+    "wcxb/default/overall": 0.7365896180583379,
+    "wcxb/default/type/article": 0.843921522694064,
+    "wcxb/default/type/collection": 0.5524797998200086,
+    "wcxb/default/type/documentation": 0.8332947902465088,
+    "wcxb/default/type/forum": 0.4542585663534178,
+    "wcxb/default/type/listing": 0.5627205295435425,
+    "wcxb/default/type/product": 0.49295278236969814,
+    "wcxb/default/type/service": 0.7542204440443699,
+    "wcxb/main_content/overall": 0.6533818407195447,
+    "wcxb/main_content/type/article": 0.8493421740411403,
+    "wcxb/main_content/type/collection": 0.3748191171053075,
+    "wcxb/main_content/type/documentation": 0.8255895901445667,
+    "wcxb/main_content/type/forum": 0.27239940390104234,
+    "wcxb/main_content/type/listing": 0.3627308921901081,
+    "wcxb/main_content/type/product": 0.4259976957269795,
+    "wcxb/main_content/type/service": 0.4277338914909828
+  },
+  "regressions": [],
+  "scope": "dev",
+  "test_quality_report_only": false,
+  "wcxb_empty_references": [
+    "0585",
+    "0602",
+    "0618",
+    "2840",
+    "2906",
+    "2966",
+    "3008",
+    "4577",
+    "4681",
+    "4720",
+    "4721",
+    "4722",
+    "4750",
+    "4763",
+    "4788",
+    "4802",
+    "4829",
+    "4850",
+    "4868",
+    "4871",
+    "4893"
+  ]
+}

crates/markkit-eval/results/public-dev-2026-10-28.md

@@ -0,0 +1,55 @@
+# Public benchmarks (dev)
+
+2754 cases; 0 quality regressions.
+
+| Metric | Score |
+|---|---:|
+| agentdocbench/category/cjk | 0.7490 |
+| agentdocbench/category/docx | 0.9989 |
+| agentdocbench/category/epub | 0.9891 |
+| agentdocbench/category/financial | 0.4233 |
+| agentdocbench/category/form | 0.7487 |
+| agentdocbench/category/html | 1.0000 |
+| agentdocbench/category/paper | 0.8232 |
+| agentdocbench/category/report | 0.8710 |
+| agentdocbench/category/scanned | 0.0000 |
+| agentdocbench/category/slides | 0.6748 |
+| agentdocbench/category/spreadsheet | 1.0000 |
+| agentdocbench/category/two-column | 0.7466 |
+| agentdocbench/format/csv | 1.0000 |
+| agentdocbench/format/docx | 0.9989 |
+| agentdocbench/format/epub | 0.9891 |
+| agentdocbench/format/html | 1.0000 |
+| agentdocbench/format/pdf | 0.5836 |
+| agentdocbench/format/pptx | 0.9572 |
+| agentdocbench/format/xlsx | 1.0000 |
+| agentdocbench/no_ocr | 0.8204 |
+| agentdocbench/overall | 0.7520 |
+| wcxb/default/overall | 0.7366 |
+| wcxb/default/type/article | 0.8439 |
+| wcxb/default/type/collection | 0.5525 |
+| wcxb/default/type/documentation | 0.8333 |
+| wcxb/default/type/forum | 0.4543 |
+| wcxb/default/type/listing | 0.5627 |
+| wcxb/default/type/product | 0.4930 |
+| wcxb/default/type/service | 0.7542 |
+| wcxb/main_content/overall | 0.6534 |
+| wcxb/main_content/type/article | 0.8493 |
+| wcxb/main_content/type/collection | 0.3748 |
+| wcxb/main_content/type/documentation | 0.8256 |
+| wcxb/main_content/type/forum | 0.2724 |
+| wcxb/main_content/type/listing | 0.3627 |
+| wcxb/main_content/type/product | 0.4260 |
+| wcxb/main_content/type/service | 0.4277 |
+
+WCXB empty/absent full-text references: 21 pages. Upstream empty-reference F1 behavior is preserved; these cases remain in the denominator and snippet scores still apply.
+
+## Conversion timing
+
+Median of three fresh processes after one warm-up, sequential, startup included.
+
+| Suite / mode | Median per-document ms | Documents |
+|---|---:|---:|
+| agentdocbench / default | 15.199 | 38 |
+| wcxb / default | 7.877 | 1358 |
+| wcxb / main_content | 8.184 | 1358 |

crates/markkit-eval/results/public-pr-2026-10-28.json

@@ -0,0 +1,90 @@
+{
+  "case_count": 178,
+  "manifest_sha256": "3ec67e21d5d7befd4e8cd6d3d9b1efe02bc3e2689f5c57e5ae197b65aa67e9fc",
+  "provenance": {
+    "binary_sha256": "390d54760c41a108b4ed4f94c6aa99b993bbf04d8a36e94e4ecb861e66e56918",
+    "build_profile": "release",
+    "command": [
+      "/workspace/work/markkit-tools/bin/mira",
+      "run",
+      "--study-python",
+      "scripts/public_benchmarks/study.py",
+      "--max-concurrent",
+      "1",
+      "--max-retries",
+      "0",
+      "--timeout",
+      "300",
+      "--group-by",
+      "category",
+      "--format",
+      "json",
+      "--out",
+      "/workspace/markkit/target/public-benchmarks/report-pr/mira.json"
+    ],
+    "core_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "cpu_count": 5,
+    "machine": "x86_64",
+    "mira_version": "mira 0.5.0",
+    "pins": {
+      "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3",
+      "mira": "v0.5.0",
+      "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc",
+      "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e",
+      "samples_per_type": 10,
+      "seed": "markkit-public-pilot-v1:",
+      "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9"
+    },
+    "platform": "Linux-6.18.55-x86_64-with-glibc2.41",
+    "python": "3.12.14",
+    "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)",
+    "source_dirty": false,
+    "source_revision": "0e1a9df33e6f3ec0967be5ae7f6b8016a1f585ba",
+    "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup"
+  },
+  "quality": {
+    "agentdocbench/category/cjk": 0.749,
+    "agentdocbench/category/docx": 0.9988666666666667,
+    "agentdocbench/category/epub": 0.9891333333333333,
+    "agentdocbench/category/financial": 0.423325,
+    "agentdocbench/category/form": 0.7486666666666667,
+    "agentdocbench/category/html": 1.0,
+    "agentdocbench/category/paper": 0.8231999999999999,
+    "agentdocbench/category/report": 0.87095,
+    "agentdocbench/category/scanned": 0.0,
+    "agentdocbench/category/slides": 0.674775,
+    "agentdocbench/category/spreadsheet": 1.0,
+    "agentdocbench/category/two-column": 0.7465666666666667,
+    "agentdocbench/format/csv": 1.0,
+    "agentdocbench/format/docx": 0.9988666666666667,
+    "agentdocbench/format/epub": 0.9891333333333333,
+    "agentdocbench/format/html": 1.0,
+    "agentdocbench/format/pdf": 0.5835739130434783,
+    "agentdocbench/format/pptx": 0.9572,
+    "agentdocbench/format/xlsx": 1.0,
+    "agentdocbench/no_ocr": 0.8204075757575757,
+    "agentdocbench/overall": 0.7520402777777778,
+    "wcxb/default/overall": 0.6770940834789322,
+    "wcxb/default/type/article": 0.8593022688301003,
+    "wcxb/default/type/collection": 0.5618264586879532,
+    "wcxb/default/type/documentation": 0.849698642479727,
+    "wcxb/default/type/forum": 0.4559187546618804,
+    "wcxb/default/type/listing": 0.6660601098285603,
+    "wcxb/default/type/product": 0.5220558838570278,
+    "wcxb/default/type/service": 0.8247964660072756,
+    "wcxb/main_content/overall": 0.4999264653091694,
+    "wcxb/main_content/type/article": 0.9687139672138977,
+    "wcxb/main_content/type/collection": 0.484765233729096,
+    "wcxb/main_content/type/documentation": 0.7157843169028515,
+    "wcxb/main_content/type/forum": 0.10889386123735083,
+    "wcxb/main_content/type/listing": 0.33261257051933946,
+    "wcxb/main_content/type/product": 0.4767832465487646,
+    "wcxb/main_content/type/service": 0.41193206101288565
+  },
+  "regressions": [],
+  "scope": "pr",
+  "test_quality_report_only": false,
+  "wcxb_empty_references": [
+    "4829"
+  ]
+}