diff --git a/.github/workflows/public-benchmarks.yml b/.github/workflows/public-benchmarks.yml new file mode 100644 index 0000000..871f2f5 --- /dev/null +++ b/.github/workflows/public-benchmarks.yml @@ -0,0 +1,74 @@ +name: Public benchmarks + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: "30 5 * * *" + workflow_dispatch: + inputs: + scope: + description: "Dataset selection (test quality is report-only)" + type: choice + options: [pr, dev, test] + default: pr + +permissions: + contents: read + +concurrency: + group: public-benchmarks-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + benchmark: + name: Public benchmark regressions + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + # Input is data, never shell code; prepare/run additionally validate choices. + BENCHMARK_SCOPE: ${{ github.event_name == 'schedule' && 'dev' || inputs.scope || 'pr' }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Cache verified corpora + uses: actions/cache@v4 + with: + path: | + target/public-benchmarks/wcxb + target/public-benchmarks/agentdocbench + key: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-${{ env.BENCHMARK_SCOPE }} + restore-keys: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}- + - name: Install scorer dependencies + run: python3 -m pip install --require-hashes -r scripts/public_benchmarks/requirements.txt + - name: Test benchmark contracts + run: python3 -m unittest discover -s scripts/tests -p 'test_public_benchmarks.py' + - name: Build release CLI + run: cargo build --locked --release -p markkit-cli + - name: Install verified Mira host + run: python3 scripts/public_benchmarks/install_mira.py + - name: Prepare pinned input selection + run: python3 scripts/public_benchmarks/prepare.py --scope "$BENCHMARK_SCOPE" + - name: Run and gate through Mira + run: python3 scripts/public_benchmarks/run.py --scope "$BENCHMARK_SCOPE" --mira target/public-benchmarks/bin/mira + - name: Job summary + if: always() + run: | + report="target/public-benchmarks/report-$BENCHMARK_SCOPE/summary.md" + if [ -f "$report" ]; then cat "$report" >> "$GITHUB_STEP_SUMMARY"; fi + - name: Upload reports and provenance + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: public-benchmarks-${{ env.BENCHMARK_SCOPE }} + path: target/public-benchmarks/report-${{ env.BENCHMARK_SCOPE }}/ + if-no-files-found: warn + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 760fc07..edeca4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,6 +78,9 @@ just check-okf # knowledge/ OKF conformance just examples # Run every example just bench # Benchmark harness, saves results to crates/markkit-bench/results just bench-criterion # Criterion micro-benches (crates/markkit/benches) +just eval-public-setup # Pinned Mira + isolated Python scorer environment +just eval-public-prepare # Download and verify public corpora (PR slice) +just eval-public # Release WCXB + AgentDocBench runs and regression gate just pre-pr # Pre-PR checks ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index d235fd6..56a610a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +- Public WCXB and AgentDocBench evaluations through Mira (`just eval-public`), with + pinned/verified corpora, a release-build baseline regression gate, per-type/format + quality and conversion timings. PR/push runs use a fixed slice; nightly runs cover + the full development selection after excluding 139 upstream dev/test duplicates. + - `markkit_qa` LLM eval (`just eval-llm`): models answer 23 questions about the eval pages from markkit markdown, main-content markdown or raw HTML, scored on correct answers with token usage per input. Targets: Meta Muse Spark 1.3 contributor via OpenRouter and Claude Sonnet 5.5; diff --git a/NOTICE b/NOTICE index 1854781..a751509 100644 --- a/NOTICE +++ b/NOTICE @@ -34,3 +34,11 @@ Markkit relies on several permissively licensed crates: * serde / serde_json (MIT/Apache-2.0) – Serialization Full dependency license details are available via `cargo deny check licenses`. + +=== Public Benchmark Tooling === + +scripts/public_benchmarks/vendor includes the Mira Python SDK (Everruns, MIT), +the AgentDocBench scorer (SylphxAI, MIT), and the WCXB scorer (Murrough Foley, +CC-BY-4.0), each with its license. Input documents retain the individual +licenses and attribution in scripts/public_benchmarks/CORPUS-NOTICE and +agentdocbench-corpus.json. The downloaded corpora are not included in packages. diff --git a/README.md b/README.md index 139ab0c..9602b1a 100644 --- a/README.md +++ b/README.md @@ -329,8 +329,14 @@ just lint # Run clippy just check # fmt, clippy, tests, knowledge, examples just pre-pr # Full pre-PR validation just bench # Benchmarks, results saved to crates/markkit-bench/results +just eval-public-setup # Pinned Mira host + isolated scorer environment +just eval-public-prepare # Download and verify WCXB + AgentDocBench inputs +just eval-public # Release quality/timing run and regression gate ``` +Public benchmark selections, scoring and baseline policy: +[`scripts/public_benchmarks/README.md`](scripts/public_benchmarks/README.md). + Coding agents: start with [AGENTS.md](AGENTS.md). ## Contributing diff --git a/crates/markkit-eval/results/public-dev-2026-10-28.json b/crates/markkit-eval/results/public-dev-2026-10-28.json new file mode 100644 index 0000000..40c5396 --- /dev/null +++ b/crates/markkit-eval/results/public-dev-2026-10-28.json @@ -0,0 +1,110 @@ +{ + "case_count": 2754, + "manifest_sha256": "1a696cff621de426697a1bc0720aeebd2b82d59da96700296084089ce9c0c05a", + "provenance": { + "binary_sha256": "390d54760c41a108b4ed4f94c6aa99b993bbf04d8a36e94e4ecb861e66e56918", + "build_profile": "release", + "command": [ + "/workspace/work/markkit-tools/bin/mira", + "run", + "--study-python", + "scripts/public_benchmarks/study.py", + "--max-concurrent", + "1", + "--max-retries", + "0", + "--timeout", + "300", + "--group-by", + "category", + "--format", + "json", + "--out", + "/workspace/markkit/target/public-benchmarks/report-dev/mira.json" + ], + "core_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "cpu_count": 5, + "machine": "x86_64", + "mira_version": "mira 0.5.0", + "pins": { + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "samples_per_type": 10, + "seed": "markkit-public-pilot-v1:", + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9" + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "python": "3.12.14", + "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "source_dirty": true, + "source_revision": "0e1a9df33e6f3ec0967be5ae7f6b8016a1f585ba", + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup" + }, + "quality": { + "agentdocbench/category/cjk": 0.749, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/financial": 0.423325, + "agentdocbench/category/form": 0.7486666666666667, + "agentdocbench/category/html": 1.0, + "agentdocbench/category/paper": 0.8231999999999999, + "agentdocbench/category/report": 0.87095, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/slides": 0.674775, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/two-column": 0.7465666666666667, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0, + "agentdocbench/format/pdf": 0.5835739130434783, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/no_ocr": 0.8204075757575757, + "agentdocbench/overall": 0.7520402777777778, + "wcxb/default/overall": 0.7365896180583379, + "wcxb/default/type/article": 0.843921522694064, + "wcxb/default/type/collection": 0.5524797998200086, + "wcxb/default/type/documentation": 0.8332947902465088, + "wcxb/default/type/forum": 0.4542585663534178, + "wcxb/default/type/listing": 0.5627205295435425, + "wcxb/default/type/product": 0.49295278236969814, + "wcxb/default/type/service": 0.7542204440443699, + "wcxb/main_content/overall": 0.6533818407195447, + "wcxb/main_content/type/article": 0.8493421740411403, + "wcxb/main_content/type/collection": 0.3748191171053075, + "wcxb/main_content/type/documentation": 0.8255895901445667, + "wcxb/main_content/type/forum": 0.27239940390104234, + "wcxb/main_content/type/listing": 0.3627308921901081, + "wcxb/main_content/type/product": 0.4259976957269795, + "wcxb/main_content/type/service": 0.4277338914909828 + }, + "regressions": [], + "scope": "dev", + "test_quality_report_only": false, + "wcxb_empty_references": [ + "0585", + "0602", + "0618", + "2840", + "2906", + "2966", + "3008", + "4577", + "4681", + "4720", + "4721", + "4722", + "4750", + "4763", + "4788", + "4802", + "4829", + "4850", + "4868", + "4871", + "4893" + ] +} diff --git a/crates/markkit-eval/results/public-dev-2026-10-28.md b/crates/markkit-eval/results/public-dev-2026-10-28.md new file mode 100644 index 0000000..f3226c9 --- /dev/null +++ b/crates/markkit-eval/results/public-dev-2026-10-28.md @@ -0,0 +1,55 @@ +# Public benchmarks (dev) + +2754 cases; 0 quality regressions. + +| Metric | Score | +|---|---:| +| agentdocbench/category/cjk | 0.7490 | +| agentdocbench/category/docx | 0.9989 | +| agentdocbench/category/epub | 0.9891 | +| agentdocbench/category/financial | 0.4233 | +| agentdocbench/category/form | 0.7487 | +| agentdocbench/category/html | 1.0000 | +| agentdocbench/category/paper | 0.8232 | +| agentdocbench/category/report | 0.8710 | +| agentdocbench/category/scanned | 0.0000 | +| agentdocbench/category/slides | 0.6748 | +| agentdocbench/category/spreadsheet | 1.0000 | +| agentdocbench/category/two-column | 0.7466 | +| agentdocbench/format/csv | 1.0000 | +| agentdocbench/format/docx | 0.9989 | +| agentdocbench/format/epub | 0.9891 | +| agentdocbench/format/html | 1.0000 | +| agentdocbench/format/pdf | 0.5836 | +| agentdocbench/format/pptx | 0.9572 | +| agentdocbench/format/xlsx | 1.0000 | +| agentdocbench/no_ocr | 0.8204 | +| agentdocbench/overall | 0.7520 | +| wcxb/default/overall | 0.7366 | +| wcxb/default/type/article | 0.8439 | +| wcxb/default/type/collection | 0.5525 | +| wcxb/default/type/documentation | 0.8333 | +| wcxb/default/type/forum | 0.4543 | +| wcxb/default/type/listing | 0.5627 | +| wcxb/default/type/product | 0.4930 | +| wcxb/default/type/service | 0.7542 | +| wcxb/main_content/overall | 0.6534 | +| wcxb/main_content/type/article | 0.8493 | +| wcxb/main_content/type/collection | 0.3748 | +| wcxb/main_content/type/documentation | 0.8256 | +| wcxb/main_content/type/forum | 0.2724 | +| wcxb/main_content/type/listing | 0.3627 | +| wcxb/main_content/type/product | 0.4260 | +| wcxb/main_content/type/service | 0.4277 | + +WCXB empty/absent full-text references: 21 pages. Upstream empty-reference F1 behavior is preserved; these cases remain in the denominator and snippet scores still apply. + +## Conversion timing + +Median of three fresh processes after one warm-up, sequential, startup included. + +| Suite / mode | Median per-document ms | Documents | +|---|---:|---:| +| agentdocbench / default | 15.199 | 38 | +| wcxb / default | 7.877 | 1358 | +| wcxb / main_content | 8.184 | 1358 | diff --git a/crates/markkit-eval/results/public-pr-2026-10-28.json b/crates/markkit-eval/results/public-pr-2026-10-28.json new file mode 100644 index 0000000..f705158 --- /dev/null +++ b/crates/markkit-eval/results/public-pr-2026-10-28.json @@ -0,0 +1,90 @@ +{ + "case_count": 178, + "manifest_sha256": "3ec67e21d5d7befd4e8cd6d3d9b1efe02bc3e2689f5c57e5ae197b65aa67e9fc", + "provenance": { + "binary_sha256": "390d54760c41a108b4ed4f94c6aa99b993bbf04d8a36e94e4ecb861e66e56918", + "build_profile": "release", + "command": [ + "/workspace/work/markkit-tools/bin/mira", + "run", + "--study-python", + "scripts/public_benchmarks/study.py", + "--max-concurrent", + "1", + "--max-retries", + "0", + "--timeout", + "300", + "--group-by", + "category", + "--format", + "json", + "--out", + "/workspace/markkit/target/public-benchmarks/report-pr/mira.json" + ], + "core_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "cpu_count": 5, + "machine": "x86_64", + "mira_version": "mira 0.5.0", + "pins": { + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "samples_per_type": 10, + "seed": "markkit-public-pilot-v1:", + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9" + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "python": "3.12.14", + "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "source_dirty": false, + "source_revision": "0e1a9df33e6f3ec0967be5ae7f6b8016a1f585ba", + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup" + }, + "quality": { + "agentdocbench/category/cjk": 0.749, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/financial": 0.423325, + "agentdocbench/category/form": 0.7486666666666667, + "agentdocbench/category/html": 1.0, + "agentdocbench/category/paper": 0.8231999999999999, + "agentdocbench/category/report": 0.87095, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/slides": 0.674775, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/two-column": 0.7465666666666667, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0, + "agentdocbench/format/pdf": 0.5835739130434783, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/no_ocr": 0.8204075757575757, + "agentdocbench/overall": 0.7520402777777778, + "wcxb/default/overall": 0.6770940834789322, + "wcxb/default/type/article": 0.8593022688301003, + "wcxb/default/type/collection": 0.5618264586879532, + "wcxb/default/type/documentation": 0.849698642479727, + "wcxb/default/type/forum": 0.4559187546618804, + "wcxb/default/type/listing": 0.6660601098285603, + "wcxb/default/type/product": 0.5220558838570278, + "wcxb/default/type/service": 0.8247964660072756, + "wcxb/main_content/overall": 0.4999264653091694, + "wcxb/main_content/type/article": 0.9687139672138977, + "wcxb/main_content/type/collection": 0.484765233729096, + "wcxb/main_content/type/documentation": 0.7157843169028515, + "wcxb/main_content/type/forum": 0.10889386123735083, + "wcxb/main_content/type/listing": 0.33261257051933946, + "wcxb/main_content/type/product": 0.4767832465487646, + "wcxb/main_content/type/service": 0.41193206101288565 + }, + "regressions": [], + "scope": "pr", + "test_quality_report_only": false, + "wcxb_empty_references": [ + "4829" + ] +} diff --git a/crates/markkit-eval/results/public-pr-2026-10-28.md b/crates/markkit-eval/results/public-pr-2026-10-28.md new file mode 100644 index 0000000..8d59495 --- /dev/null +++ b/crates/markkit-eval/results/public-pr-2026-10-28.md @@ -0,0 +1,55 @@ +# Public benchmarks (pr) + +178 cases; 0 quality regressions. + +| Metric | Score | +|---|---:| +| agentdocbench/category/cjk | 0.7490 | +| agentdocbench/category/docx | 0.9989 | +| agentdocbench/category/epub | 0.9891 | +| agentdocbench/category/financial | 0.4233 | +| agentdocbench/category/form | 0.7487 | +| agentdocbench/category/html | 1.0000 | +| agentdocbench/category/paper | 0.8232 | +| agentdocbench/category/report | 0.8710 | +| agentdocbench/category/scanned | 0.0000 | +| agentdocbench/category/slides | 0.6748 | +| agentdocbench/category/spreadsheet | 1.0000 | +| agentdocbench/category/two-column | 0.7466 | +| agentdocbench/format/csv | 1.0000 | +| agentdocbench/format/docx | 0.9989 | +| agentdocbench/format/epub | 0.9891 | +| agentdocbench/format/html | 1.0000 | +| agentdocbench/format/pdf | 0.5836 | +| agentdocbench/format/pptx | 0.9572 | +| agentdocbench/format/xlsx | 1.0000 | +| agentdocbench/no_ocr | 0.8204 | +| agentdocbench/overall | 0.7520 | +| wcxb/default/overall | 0.6771 | +| wcxb/default/type/article | 0.8593 | +| wcxb/default/type/collection | 0.5618 | +| wcxb/default/type/documentation | 0.8497 | +| wcxb/default/type/forum | 0.4559 | +| wcxb/default/type/listing | 0.6661 | +| wcxb/default/type/product | 0.5221 | +| wcxb/default/type/service | 0.8248 | +| wcxb/main_content/overall | 0.4999 | +| wcxb/main_content/type/article | 0.9687 | +| wcxb/main_content/type/collection | 0.4848 | +| wcxb/main_content/type/documentation | 0.7158 | +| wcxb/main_content/type/forum | 0.1089 | +| wcxb/main_content/type/listing | 0.3326 | +| wcxb/main_content/type/product | 0.4768 | +| wcxb/main_content/type/service | 0.4119 | + +WCXB empty/absent full-text references: 1 pages. Upstream empty-reference F1 behavior is preserved; these cases remain in the denominator and snippet scores still apply. + +## Conversion timing + +Median of three fresh processes after one warm-up, sequential, startup included. + +| Suite / mode | Median per-document ms | Documents | +|---|---:|---:| +| agentdocbench / default | 16.551 | 38 | +| wcxb / default | 8.567 | 70 | +| wcxb / main_content | 8.774 | 70 | diff --git a/justfile b/justfile index 33f7250..2804f07 100644 --- a/justfile +++ b/justfile @@ -142,6 +142,21 @@ eval *args: eval-llm *args: mira run --study-bin markkit-eval markkit_qa {{args}} +# Install pinned public-benchmark scorer dependencies and verified Mira (Linux x86_64). +eval-public-setup: + python3 -m venv target/public-benchmarks/venv + target/public-benchmarks/venv/bin/python -m pip install --require-hashes -r scripts/public_benchmarks/requirements.txt + target/public-benchmarks/venv/bin/python scripts/public_benchmarks/install_mira.py + +# Download and verify the fixed PR sample, or --scope dev / test. Evaluation is offline. +eval-public-prepare *args: + target/public-benchmarks/venv/bin/python scripts/public_benchmarks/prepare.py {{args}} + +# Release CLI, public quality scores, conversion timing and canonical regression gate. +eval-public *args: + cargo build --locked --release -p markkit-cli + target/public-benchmarks/venv/bin/python scripts/public_benchmarks/run.py {{args}} + # ─── Security & Audit ─────────────────────────────────────── # Run cargo-deny checks diff --git a/knowledge/log.md b/knowledge/log.md index 2ac99c6..898343f 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -3,6 +3,7 @@ ## 2026-10-28 * **Footnote safety**: a blank line separates footnote markers from their indented bodies, preventing Markdown readers without footnotes from treating body text as a reference-link destination. Fixes the nightly renderer fuzz failure; regression checks both reader modes. See [Threat Model](security/threat-model.md) S17. +* **Public benchmarks**: WCXB and AgentDocBench run through the Mira Python SDK with exact upstream scorers, pinned corpus blobs, release CLI timing and a canonical score regression gate. PRs use 178 cases; scheduled dev uses 2,754 after excluding 139 HTML duplicates shared with WCXB test. Test evaluation is manual/report-only for quality. See [Evaluation Harness](operations/eval.md) and [Performance Results](operations/performance-results.md). ## 2026-10-27 diff --git a/knowledge/operations/eval.md b/knowledge/operations/eval.md index a3deebd..96542e1 100644 --- a/knowledge/operations/eval.md +++ b/knowledge/operations/eval.md @@ -212,6 +212,49 @@ at what token cost? Code: `src/qa.rs` (subject, scorer, eval) and endpoint; no keyed run yet (the default cloud environment has no Doppler access and blocks `api.doppler.com` / `openrouter.ai`). +## Public benchmarks + +`scripts/public_benchmarks` adds two companion Mira Python evals, +`markkit_wcxb` and `markkit_agentdocbench`, driven through the same host CLI. +The existing Rust fixture and QA evals remain separate. Code and run instructions: +[`scripts/public_benchmarks/README.md`](../../scripts/public_benchmarks/README.md). + +- **Corpora**: immutable source commits in `pins.json`; WCXB compressed HTML and + annotations locked by Git blob ID, AgentDocBench documents by upstream SHA-256 + and labels/references by blob ID. Prepared files are SHA-256 checked before the + study advertises cases. Missing/corrupt data fails setup, never skips samples. +- **Split integrity**: WCXB upstream has 139 identical HTML files in both splits; + its keyed metadata also overwrites those dev entries. The committed manifest + keys annotations by split and excludes shared HTML from all dev selections. + Selections: fixed 70-page PR slice; full 1,358-page dev; 511-page upstream test. +- **Subjects**: release Markkit CLI, all formats, no URL fetching during execution; + WCXB `mode=default,main_content`, AgentDocBench default only. One warm-up plus + three fresh-process conversions, sequential, identical output required. +- **Scoring**: unchanged upstream Python scorers; Markdown projected to visible + text before WCXB word-F1/snippet scoring, original Markdown for AgentDocBench. + AgentDocBench overall is category macro, not document mean; scans remain zero + in the full score, with a separate no-OCR slice. Component N/A is preserved. +- **Gates**: runner requires exact case coverage and successful conversions, then + compares canonical aggregates and per-type/category/format scores on the fixed + baseline subset, tolerance 0.005 absolute. Full dev reports its whole-set score + but gates the same fixed subset. Test scope gates execution/coverage only. + Rebaselining requires explicit `--write-baseline` on PR scope; CI never does it. + Corpus/scorer contract changes invalidate the old baseline. +- **CI**: `public-benchmarks.yml` runs 178 cases for PR/push, 2,754 on schedule; + manual dispatch supports all scopes (test is 1,060 cases). Reports include + source/binary hashes, profile, compiler/platform, commands, manifests and notices. + They upload on failure too. No keys/models/GPU or third-party conversion service. +- **Tests**: `scripts/tests/test_public_benchmarks.py` covers split leakage, + cache/checksum failures, path traversal, score normalization, category weighting, + missing/duplicate/N/A cases and regression gates. The public CI job installs + scorer dependencies and runs every test; the stdlib script suite skips the + Markdown-specific tests when that optional dependency is absent. + +Vendored SDK/scorers retain MIT/CC-BY-4.0 notices; each AgentDocBench input keeps +its own attribution in `CORPUS-NOTICE`. These are development tools only, with no +new dependency in the published library. Keep internal structural and security +tests: public sparse checks do not replace exact Markdown expectations. + ## Planned evals - **`markkit_judge`**: LLM-as-judge grading each page's markdown against its diff --git a/knowledge/operations/performance-results.md b/knowledge/operations/performance-results.md index f08f8bf..0ca417e 100644 --- a/knowledge/operations/performance-results.md +++ b/knowledge/operations/performance-results.md @@ -67,6 +67,13 @@ to `--`, overridable with `--moniker`, sanitized to ## Contract +Public corpus conversion timing is separate from these in-process generated +workloads. `just eval-public` builds the release CLI and records the median of +three fresh processes after one untimed warm-up, sequentially; startup is included, +downloads/scoring are excluded. Public reports pair duration with quality and +record source/binary/compiler/platform provenance. Latency is reported, not gated +across CI machines. See [Evaluation Harness](eval.md) Public benchmarks. + - Release builds only (`--release`); debug numbers are meaningless. - Commit saved runs for performance-relevant PRs and before releases. - Compare runs from the same moniker only; CPU, OS and container differ. diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 68dd4a4..bf0e7e5 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -33,6 +33,7 @@ tags: | `crates/markkit/benches/extraction.rs` | Criterion micro-benchmarks | | `crates/markkit-bench/` | Benchmark harness; `cargo test` runs every case once | | `crates/markkit-eval/src/**` `#[cfg(test)]` | Eval dataset validity, check semantics, scorers, subject | +| `scripts/tests/test_public_benchmarks.py` | Pinned public corpus integrity, split leakage, canonical scores and regression gate | Office fixtures are generated in code (`tests/common`), not committed binaries, so each security test states exactly which bytes it feeds the parser. @@ -70,6 +71,12 @@ each security test states exactly which bytes it feeds the parser. cargo-fuzz target nightly and uploads crashes; `coverage.yml` uploads tarpaulin coverage (`--engine llvm`; ptrace segfaults on threaded socket tests). All must pass before merge. +`public-benchmarks.yml` runs the WCXB and AgentDocBench Mira studies in release +mode on every PR/push and nightly, enforcing full coverage and fixed-slice quality +regressions. It installs the optional scorer dependencies, runs all public-tool +contract tests, and uploads canonical scores/timing/provenance even on failure. +See [Evaluation Harness](eval.md) Public benchmarks for split and baseline policy. + ## See also - [Threat Model](../security/threat-model.md), threat-to-test map diff --git a/scripts/public_benchmarks/CORPUS-NOTICE b/scripts/public_benchmarks/CORPUS-NOTICE new file mode 100644 index 0000000..7f481b3 --- /dev/null +++ b/scripts/public_benchmarks/CORPUS-NOTICE @@ -0,0 +1,382 @@ +AgentDocBench corpus release notice + +These 38 documents are redistributed byte-for-byte, without modification, +under their individual licenses, not the license of the anymd software. +Original source URLs, license evidence and attribution are retained below +and in corpus.json. SHA256SUMS records the checksum of each release asset. + +License links: +CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ +CC BY-SA 4.0: https://creativecommons.org/licenses/by-sa/4.0/ +Japan PDL 1.0: https://www.stat.go.jp/info/riyou.html +Taiwan OGDL 1.0: https://data.gov.tw/license +US federal government works: public domain under 17 U.S.C. 105. +Project Gutenberg EPUBs: public domain in the USA; embedded Project +Gutenberg license and trademark notices remain intact. Check local +copyright rules before redistributing them outside the USA. + +paper-evsched.pdf + Title: Grid-constrained online scheduling of flexible electric vehicle charging + Attribution: Emily van Huffelen, Roel Brouwer, Marjan van den Akker (Utrecht University) (arXiv:2403.03109v1) + License: CC-BY-4.0 + Source: https://arxiv.org/pdf/2403.03109v1 + Evidence: https://arxiv.org/abs/2403.03109v1 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2403.03109&metadataPrefix=arXiv + SHA-256: 402de06ee9e5df0ec8ded433eb9440b802f02cc2e44acd8eaa89977892322cd2 + Changes: none + +paper-rankone.pdf + Title: Probabilistic bounds on best rank-one approximation ratio + Attribution: Khazhgali Kozhasov, Josué Tonelli-Cueto (arXiv:2201.02191v2) + License: CC-BY-4.0 + Source: https://arxiv.org/pdf/2201.02191v2 + Evidence: https://arxiv.org/abs/2201.02191v2 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2201.02191&metadataPrefix=arXiv + SHA-256: 328a7069a9ea1f569158ead666352111ea4eccd695c6e00d56ddf0306f554c46 + Changes: none + +paper-ulam.pdf + Title: Diametric problem for permutations with the Ulam metric (optimal anticodes) + Attribution: Pat Devlin, Leo Douhovnikoff (Swarthmore College) (arXiv:2403.02276v1) + License: CC-BY-4.0 + Source: https://arxiv.org/pdf/2403.02276v1 + Evidence: https://arxiv.org/abs/2403.02276v1 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2403.02276&metadataPrefix=arXiv + SHA-256: fbc9633924adb8d5cc2b2ee72fdcaea52ebcb7772edc55c38d8b9d2820fab66a + Changes: none + +twocol-bygpt5.pdf + Title: ByGPT5: End-to-End Style-conditioned Poetry Generation with Token-free Language Models + Attribution: Jonas Belouadi, Steffen Eger (arXiv:2212.10474v2) + License: CC-BY-4.0 + Source: https://arxiv.org/pdf/2212.10474v2 + Evidence: https://arxiv.org/abs/2212.10474v2 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2212.10474&metadataPrefix=arXiv + SHA-256: d07da94752759e80f573979603e0d5f1c060083459e83f1a57771a59d97fad55 + Changes: none + +twocol-smile.pdf + Title: SMiLE: Schema-augmented Multi-level Contrastive Learning for Knowledge Graph Link Prediction + Attribution: Miao Peng, Ben Liu, Qianqian Xie, Wenjie Xu, Hua Wang, Min Peng (arXiv:2210.04870v3) + License: CC-BY-4.0 + Source: https://arxiv.org/pdf/2210.04870v3 + Evidence: https://arxiv.org/abs/2210.04870v3 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2210.04870&metadataPrefix=arXiv + SHA-256: cff9c2ff011e00aa50a7e8615d85bd928048eb4a8eb7d29d6f31f101200fbe54 + Changes: none + +twocol-uscore.pdf + Title: USCORE: An Effective Approach to Fully Unsupervised Evaluation Metrics for Machine Translation + Attribution: Jonas Belouadi, Steffen Eger (arXiv:2202.10062v3) + License: CC-BY-4.0 + Source: https://arxiv.org/pdf/2202.10062v3 + Evidence: https://arxiv.org/abs/2202.10062v3 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2202.10062&metadataPrefix=arXiv + SHA-256: 7a46f079adac9b3ed19d76fe224c1c0523c8a5113281662022c188abe5a85987 + Changes: none + +report-crs-dod-approps.pdf + Title: FY2026 Department of Defense Appropriations: In Brief (CRS R48891, version 4) + Attribution: Congressional Research Service (Cameron M. Keys) + License: public-domain-us-gov + Source: https://www.congress.gov/crs_external_products/R/PDF/R48891/R48891.4.pdf + Evidence: In-document disclaimer (last page): 'CRS Reports, as a work of the United States Government, are not subject to copyright protection in the United States.' + SHA-256: 3467a0131f7900f8caaf7c106b1748973b25fb28e7dd891c6ef5836c98273682 + Changes: none + +report-plz.pdf + Title: U.S. Access Board Technical Guide: Passenger Loading Zones + Attribution: U.S. Access Board + License: public-domain-us-gov + Source: https://www.access-board.gov/files/ada/guides/plz.pdf + Evidence: Work of the U.S. Access Board, a US federal agency (17 U.S.C. 105) + SHA-256: 0f8daeba6773577fac1d443828d6ff4bf8755e03d119d1ca13ca3827ffd8f66e + Changes: none + +fed-h8.pdf + Title: H.8 Assets and Liabilities of Commercial Banks in the United States, September 18, 2026 + Attribution: Board of Governors of the Federal Reserve System + License: public-domain-us-gov + Source: https://www.federalreserve.gov/releases/h8/20260918/h8.pdf + Evidence: Work of the Board of Governors of the Federal Reserve System, a US federal agency (17 U.S.C. 105); https://www.federalreserve.gov/disclaimer.htm + SHA-256: a56c5e4c4ac5391770abff75d3d9f9ab8d2d3cd9a9f9e057ad83f9828841a768 + Changes: none + +fin-bls-realer.pdf + Title: Real Earnings – July 2026 (USDL-26-1379) + Attribution: U.S. Bureau of Labor Statistics + License: public-domain-us-gov + Source: https://web.archive.org/web/20260819200140id_/https://www.bls.gov/news.release/archives/realer_08122026.pdf + Evidence: https://www.bls.gov/opub/copyright-information.htm (BLS material is in the public domain); US federal work, 17 U.S.C. 105 + SHA-256: 58686e3cf434e81e853b4d8f01b6e3ab55f16e84556516ce74eb33a649950a78 + Changes: none + +fin-census-income.pdf + Title: Household Income in States and Metropolitan Areas: 2024 (ACSBR-025) + Attribution: U.S. Census Bureau (Gloria G. Guzman) + License: public-domain-us-gov + Source: https://www2.census.gov/library/publications/2025/demo/acsbr-025.pdf + Evidence: US federal work (17 U.S.C. 105), U.S. Census Bureau publication; landing page https://www.census.gov/library/publications/2025/acs/acsbr-025.html + SHA-256: 5de1b0d5ec469344896bdf8485d3e93206351c0f1f9e51029b8d57d2df9054b9 + Changes: none + +invoice-borderless.pdf + Title: Inventory Reconciliation Report SPARSE-2024-INV-1234 (markitdown test file) + Attribution: Microsoft Corporation (markitdown test files) + License: MIT + Source: https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/SPARSE-2024-INV-1234_borderless_table.pdf + Evidence: https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT) + SHA-256: 784fef202eb748d5d274eaaf2ded51dfd0d0d1e92225e84fd33dafc2333d15d3 + Changes: none + +form-1040es.pdf + Title: 2024 Form 1040-ES: Estimated Tax for Individuals + Attribution: Internal Revenue Service + License: public-domain-us-gov + Source: https://www.irs.gov/pub/irs-prior/f1040es--2024.pdf + Evidence: IRS form, US federal government work (17 U.S.C. 105) + SHA-256: 860a291f7e24b2e146b4c164e41cf79f0f0afa9fadde2690b474c2bf21300ee6 + Changes: none + +form-w4.pdf + Title: 2024 Form W-4: Employee's Withholding Certificate + Attribution: Internal Revenue Service + License: public-domain-us-gov + Source: https://www.irs.gov/pub/irs-prior/fw4--2024.pdf + Evidence: IRS form, US federal government work (17 U.S.C. 105) + SHA-256: e60e3af7512073bffbd7ef3c4f3a2fc9593d2aed2e420718d4ccf17659e2c3ed + Changes: none + +irs-w9.pdf + Title: Form W-9 (Rev. March 2024): Request for Taxpayer Identification Number and Certification + Attribution: Internal Revenue Service + License: public-domain-us-gov + Source: https://www.irs.gov/pub/irs-prior/fw9--2024.pdf + Evidence: IRS form, US federal government work (17 U.S.C. 105); https://www.irs.gov/privacy-disclosure/irs-privacy-policy + SHA-256: 2d420cbb4123dcf1fb82595b2359cfbb5d81f00b9df9d359fcc7af361d093f53 + Changes: none + +scan-nasa-budget.pdf + Title: Office Memorandum: NASA 1961 Estimates and BOB Staff Proposal to BOB Director (J. L. Mitchell to G. B. Kistiakowsky, 14 Nov 1959) + Attribution: Jesse L. Mitchell (Office of the Special Assistant for Science and Technology) / Dwight D. Eisenhower Presidential Library, NARA + License: public-domain-us-gov + Source: https://web.archive.org/web/20250515183040id_/https://www.eisenhowerlibrary.gov/sites/default/files/file/nasa_Binder20.pdf + Evidence: https://www.eisenhowerlibrary.gov/research/online-documents ("All of these items are in the public domain"); authored by US federal government employees in the course of duty (17 U.S.C. 105) + SHA-256: f0637cdd4026471afa0cf1ab199a4030844ee619f0a6dd727ceab5f29717e847 + Changes: none + +scan-nasa-notes.pdf + Title: Memorandum for Dr. Killian: Notes on "NASA" (Robert Piland, 10 April 1958) + Attribution: Robert Piland (White House science advisory staff) / Dwight D. Eisenhower Presidential Library, NARA + License: public-domain-us-gov + Source: https://web.archive.org/web/20250515183039id_/https://www.eisenhowerlibrary.gov/sites/default/files/file/nasa_Binder18.pdf + Evidence: https://www.eisenhowerlibrary.gov/research/online-documents ("All of these items are in the public domain"); authored by US federal government employees in the course of duty (17 U.S.C. 105) + SHA-256: 0d0ae5bc4a51949159beea4455d198852087e7fd6b95e9d6730a6248700975bf + Changes: none + +scan-oklahoma-address.pdf + Title: Text of the Address on "Our Future Security", Oklahoma City, November 13, 1957 (White House press release) + Attribution: The White House (President Dwight D. Eisenhower) / Dwight D. Eisenhower Presidential Library, NARA + License: public-domain-us-gov + Source: https://web.archive.org/web/20250515183607id_/https://www.eisenhowerlibrary.gov/sites/default/files/research/online-documents/sputnik/11-13-57.pdf + Evidence: https://www.eisenhowerlibrary.gov/research/online-documents ("All of these items are in the public domain"); authored by US federal government employees in the course of duty (17 U.S.C. 105) + SHA-256: f9da22937bda1617e919bb4a3b902551187bb484a07af254ccff22647ae81848 + Changes: none + +cjk-jp-stat-elderly.pdf + Title: 統計トピックスNo.142 統計からみた我が国の高齢者 Ⅰ 高齢者の人口 + Attribution: 総務省統計局 (Statistics Bureau of Japan) + License: PDL-1.0 + Source: https://web.archive.org/web/20241117224402id_/https://www.stat.go.jp/data/topics/pdf/topi142_01.pdf + Evidence: https://www.stat.go.jp/info/riyou.html (統計局ホームページ利用規約: 公共データ利用規約（第1.0版）(PDL1.0) に準拠; 'クリエイティブ・コモンズ・ライセンスの表示4.0 国際 ... と互換性があり'; numeric data/simple tables not copyrightable) + SHA-256: 5565b9c00fff2065023da95dd313a5f5b950d75b162669a167538152fcd4efe1 + Changes: none + +cjk-tw-archives.pdf + Title: 參考文獻格式 (reference citation format guide) + Attribution: 國家發展委員會檔案管理局 (National Archives Administration, National Development Council, Taiwan) + License: CC-BY-4.0 + Source: https://www.archives.gov.tw/wSite/public/Attachment/0/f1715062099077.pdf + Evidence: https://www.archives.gov.tw/tw/arctw/625-10226.html (政府網站資料開放宣告: all site materials released under CC BY 4.0, attribution required) + SHA-256: b28e5e6d91923d27d6fa2d29b42018584e71b3a38ff04aa9acc638b0c11e1bd3 + Changes: none + +cjk-tw-dgbas-cpi.pdf + Title: 115 年 8 月物價變動概況 (DGBAS price statistics press release, Aug 2026) + Attribution: 行政院主計總處 (Directorate-General of Budget, Accounting and Statistics, Executive Yuan, Taiwan) + License: Taiwan-OGDL-1.0 + Source: https://ws.dgbas.gov.tw/001/Upload/463/relfile/10980/236675/news11508.pdf + Evidence: https://www.stat.gov.tw/cp.aspx?n=3164 (行政院主計總處 政府網站資料開放宣告: 採政府資料開放授權條款-第1版發布); release page https://www.stat.gov.tw/News_Content.aspx?n=2668&s=236675 + SHA-256: 203a27ef3771ffc870f3694c860aff660b8cb06e5bf3e4351bfeba8d02074630 + Changes: none + +slides-epa-1bp.pptx + Title: 1-Bromopropane (1-BP) Proposed Rulemaking Under TSCA Section 6(a) - Public Webinar, August 28, 2024 + Attribution: U.S. Environmental Protection Agency + License: public-domain-us-gov + Source: https://web.archive.org/web/20241116234835id_/https://www.epa.gov/system/files/documents/2024-09/1-bp-public-webinar-slides_08.28.24.pptx + Evidence: Work of US EPA staff (Office of Chemical Safety and Pollution Prevention), 17 U.S.C. 105; EPA site copyright note: https://www.epa.gov/web-policies-and-procedures/epa-disclaimers + SHA-256: c02b2620a068190818dddfbe7e15b083c21fd0f99f441f166762441a6734bf46 + Changes: none + +slides-markitdown-pptx.pptx + Title: markitdown test deck (AutoGen) + Attribution: Microsoft (markitdown contributors) + License: MIT + Source: https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.pptx + Evidence: https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT, repository test file) + SHA-256: f0b9e5252aec3730c91f276a7c3b8f4a9893a7b66540f6c6af160a49855ca507 + Changes: none + +slides-nasa-clarreo.pdf + Title: CLARREO Pathfinder Mission Overview and Status (CEOS WGCV IVOS 2022) + Attribution: Yolanda L. Shea, Rajendra Bhatt / NASA Langley Research Center + License: public-domain-us-gov + Source: https://ntrs.nasa.gov/api/citations/20220013266/downloads/Shea_etal_CEOS_WGCV_IVOS_2022_STRIVES.pdf + Evidence: https://ntrs.nasa.gov/citations/20220013266 (NTRS copyright determination GOV_PUBLIC_USE_PERMITTED: 'Work of the US Gov. Public Use Permitted'; both authors are NASA LaRC civil servants) + SHA-256: bb04eb9ea712c6960198b5153ae119ff58d3b5a4ce4d775af5299ba38c392a82 + Changes: none + +slides-nist-pqc-update.pdf + Title: NIST Post-Quantum Cryptography Update (Dustin Moody, ISPAB Oct 2023) + Attribution: Dustin Moody / NIST Computer Security Division + License: public-domain-us-gov + Source: https://web.archive.org/web/20241002031327id_/https://csrc.nist.gov/csrc/media/Presentations/2023/nist-post-quantum-cryptography-update/2a-Moody_NIST_PQC_2.pdf + Evidence: Work of a NIST employee, 17 U.S.C. 105; https://www.nist.gov/oism/copyrights and https://csrc.nist.gov/presentations/2023/nist-post-quantum-cryptography-update + SHA-256: 1c50a396a1eefc93ece22e43d6b389e6338c40685cb625bb29ff7d29453a5ee5 + Changes: none + +csv-census-popest-2023.csv + Title: Annual Estimates of the Resident Population and Population Change, States, 2020-2023 (NST-EST2023-POPCHG2020_2023) + Attribution: U.S. Census Bureau, Population Division + License: public-domain-us-gov + Source: https://web.archive.org/web/20240623113748id_/https://www2.census.gov/programs-surveys/popest/datasets/2020-2023/state/totals/NST-EST2023-POPCHG2020_2023.csv + Evidence: US Census Bureau data product, a work of the US Government (17 U.S.C. 105); https://www.census.gov/about/policies/citation.html + SHA-256: c7f1d6d644007e129baaf1be509b83c914f4a1e167e3407af65207dd14d8a9e5 + Changes: none + +csv-usgs-quakes-week.csv + Title: USGS earthquake feed: M4.5+ past 7 days, snapshot 2025-01-07 + Attribution: U.S. Geological Survey, Earthquake Hazards Program + License: public-domain-us-gov + Source: https://web.archive.org/web/20250107124642id_/https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_week.csv + Evidence: https://www.usgs.gov/information-policies-and-instructions/copyrights-and-credits (USGS-authored information is in the US public domain) + SHA-256: 707c6820e37b3a433ed4f46aec942bf8e1b18924e02025cdbf6348d098decade + Changes: none + +sheet-ers-crop-insurance.xlsx + Title: Federal crop insurance premiums and subsidies, 2000-25 (ERS chart data file 102257) + Attribution: USDA Economic Research Service + License: public-domain-us-gov + Source: https://web.archive.org/web/20260629195302id_/https://www.ers.usda.gov/media/29251/102257-chart-data-file.xlsx?v=88166 + Evidence: USDA web policy (ERS follows it): 'Most information presented on the USDA Web site is considered public domain information' https://www.usda.gov/about-usda/policies-and-links ; https://www.ers.usda.gov/about-ers/policies-and-standards/website + SHA-256: edcffe753bec87bd69fe41c5da7bb5a71344169d8819da22cef9822d0a05ac50 + Changes: none + +sheet-markitdown-xlsx.xlsx + Title: markitdown test workbook + Attribution: Microsoft (markitdown contributors) + License: MIT + Source: https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.xlsx + Evidence: https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT, repository test file) + SHA-256: a867be6ece38a4224bcfe34312e5c296ed520a76a431e0454603ceb171fbd4af + Changes: none + +docx-hhs-market-research.docx + Title: HHS Market Research Report Template (Tab D, final 1/18/2022) + Attribution: U.S. Department of Health and Human Services + License: public-domain-us-gov + Source: https://web.archive.org/web/20241008230234id_/https://www.cms.gov/files/document/tab-d-hhs-market-research-template-final-1182022.docx + Evidence: Work of the US Department of Health and Human Services, published on cms.gov (17 U.S.C. 105); https://www.hhs.gov/web/policies-and-standards/index.html + SHA-256: 89e4f8e3b3ac964f04f85385310b97f70bd0ab33e741a4fe26b84c493c170e1e + Changes: none + +docx-markitdown.docx + Title: markitdown test document (AutoGen) + Attribution: Microsoft (markitdown contributors) + License: MIT + Source: https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.docx + Evidence: https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT, repository test file) + SHA-256: ee1974633f3b1e8bb54201abce779a755cb1cd5259641a08cf7b1b9b36dec42b + Changes: none + +docx-nsf-aaac-minutes.docx + Title: Minutes of the Meeting of the Astronomy and Astrophysics Advisory Committee (AAAC), 15 December 2023 + Attribution: U.S. National Science Foundation (AAAC) + License: public-domain-us-gov + Source: https://web.archive.org/web/20260426103635id_/https://nsf-gov-resources.nsf.gov/attachments/308681/public/AAAC_Meeting_Minutes_Dec_15_2023.docx + Evidence: Federal advisory committee minutes published by NSF (17 U.S.C. 105); NSF site policy: https://www.nsf.gov/policies/site-policies + SHA-256: de2aa4a0731d343ddf71b1a2f27a500146ebc2a90df928193ae2a3bbdf2fc9a9 + Changes: none + +epub-gutenberg-jekyll.epub + Title: The Strange Case of Dr. Jekyll and Mr. Hyde (Project Gutenberg #43) + Attribution: Robert Louis Stevenson; Project Gutenberg + License: public-domain + Source: https://web.archive.org/web/20250115052559id_/https://www.gutenberg.org/cache/epub/43/pg43-images-3.epub + Evidence: EPUB content.opf Public domain in the USA.; https://www.gutenberg.org/ebooks/43 (Robert Louis Stevenson, 1886) + SHA-256: b7b093591573ff4562dc799c3940be5ddab4e0ac686cdb062bb6853bb2c4d66e + Changes: none + +epub-gutenberg-relativity.epub + Title: Relativity: The Special and General Theory (Project Gutenberg #5001) + Attribution: Albert Einstein, translated by Robert W. Lawson; Project Gutenberg + License: public-domain + Source: https://web.archive.org/web/20241012103314id_/https://www.gutenberg.org/cache/epub/5001/pg5001-images-3.epub + Evidence: EPUB content.opf Public domain in the USA.; https://www.gutenberg.org/ebooks/5001 (Einstein, tr. R. W. Lawson, 1920) + SHA-256: 698baa25db272591503de1ed8df5a594595dc4dc1f6713fad1b6b4be97875c09 + Changes: none + +epub-markitdown.epub + Title: Test EPUB Document (markitdown test file) + Attribution: Microsoft (markitdown test suite) + License: MIT + Source: https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.epub + Evidence: https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE + SHA-256: 33e65f39f54bd24f74d5f333d24213a68c62ee2f2854d12f8c4c59b15a4031a7 + Changes: none + +html-bls-ooh-software-developers.html + Title: Software Developers, Quality Assurance Analysts, and Testers - Occupational Outlook Handbook (BLS) + Attribution: U.S. Bureau of Labor Statistics + License: public-domain-us-gov + Source: https://web.archive.org/web/20260910003547id_/https://www.bls.gov/ooh/computer-and-information-technology/software-developers.htm + Evidence: https://www.bls.gov/opub/copyright-information.htm (BLS material is in the public domain except photographs/graphics credited to other sources; this page's text credits none); U.S. Department of Labor work + SHA-256: bf71737fceeb5e1891815ec1881c20844365779e696eb1b6d52dd4de1f2ce34a + Changes: none + +html-markitdown-wikipedia.html + Title: Microsoft - Wikipedia (markitdown test_wikipedia.html) + Attribution: Wikipedia contributors (article 'Microsoft'); Microsoft markitdown + License: CC-BY-SA-4.0 (Wikipedia content in an MIT test file) + Source: https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test_wikipedia.html + Evidence: https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT repo); article text CC BY-SA 4.0 per https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License + SHA-256: 4c1f6c5ca1147455b8df8b19cd957844cad37d23d541c039af421a7634074ea6 + Changes: none + +html-wikipedia-roman-numerals.html + Title: Roman numerals - Wikipedia (revision 1374259684) + Attribution: Wikipedia contributors (article 'Roman numerals', oldid 1374259684) + License: CC-BY-SA-4.0 + Source: https://web.archive.org/web/20260924023032id_/https://en.wikipedia.org/wiki/Roman_numerals + Evidence: https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License (footer of page states CC BY-SA 4.0) + SHA-256: 4addcd13f130fbac818f8acbd57f2aa5d7aea9ad8a0f2687efa3cd629ee0a575 + Changes: none + +Microsoft markitdown test files (including slides-markitdown-pptx): + + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/scripts/public_benchmarks/README.md b/scripts/public_benchmarks/README.md new file mode 100644 index 0000000..a56905e --- /dev/null +++ b/scripts/public_benchmarks/README.md @@ -0,0 +1,130 @@ +# Public extraction benchmarks + +WCXB measures main-content extraction from saved HTML; AgentDocBench measures +text, reading order and table fidelity across native documents and PDFs. Both run +through Mira, locally and in `public-benchmarks.yml`, without API keys or a GPU. + +## Run + +From the repository root, with Rust stable and Python 3.12+: + +```bash +just eval-public-setup +just eval-public-prepare +just eval-public + +# Larger development run, excluding test duplicates +just eval-public-prepare --scope dev +just eval-public --scope dev + +# Milestone evaluation; test quality is reported, not used to tune a PR gate +just eval-public-prepare --scope test +just eval-public --scope test +``` + +The setup recipe creates an isolated Python environment with hash-locked +dependencies and installs checksum-verified Mira 0.5.0 for Linux x86_64. On other +platforms, install that Mira version separately, create the Python environment +with the same requirements, and pass `--mira /absolute/path/to/mira` to the run. +The runner builds the release CLI itself; it does not accept a debug binary. + +Preparation downloads fixed inputs into `target/public-benchmarks`. The study +validates every prepared file before advertising cases and runs offline. Cached +objects are reused only after checksum verification. Missing files, broken +checksums, failed conversions, missing cases and unavailable required scores +fail the run. A scanned PDF successfully converted to empty text is a scored +quality failure (zero), not an unavailable case. + +## Selections and upstream data overlap + +| Scope | WCXB pages | With two HTML modes and 38 AgentDocBench documents | +|---|---:|---:| +| `pr` | 70, ten per type, fixed by seeded SHA-256 selection | 178 cases | +| `dev` | 1,358 development pages with test inputs removed | 2,754 cases | +| `test` | All 511 upstream test pages | 1,060 cases | + +The upstream WCXB snapshot contains 1,497 dev and 511 test annotations, but 139 +gzipped HTML files are byte-identical across splits. Its metadata dictionary +also overwrites the dev entries for those IDs. `wcxb-corpus.json` locks every +annotation and HTML blob separately by split; development selections exclude +all 139 shared inputs. Do not call the unfiltered upstream split independent. + +Some pinned annotations have empty or absent `main_content` despite required +snippets. The adapter matches upstream's empty-reference F1 behavior (zero for +nonempty output), keeps every case in the denominator, and still scores snippets. +Reports identify the affected IDs/count; no missing text is invented or dropped. + +The PR slice uses `SHA256(seed + id)` ordering per page type. IDs, source commits +and the seed are committed. Dataset updates change the corpus/scorer contract +and require explicit preparation and rebaselining; no live pages are fetched. + +## Scores and regression policy + +The unchanged upstream Python scorers live in `vendor/`. WCXB expects plain +text: the adapter renders Markdown with markdown-it-py (CommonMark + tables), +collects visible text and calls upstream word-F1/snippet functions. Apply this +same projection to Markdown-producing competitors. + +AgentDocBench scores the original Markdown. Each document averages its +applicable component metrics; each category averages its documents; the overall +score averages the 12 category means. The supplementary no-OCR score averages +the remaining categories after removing the three image-only PDFs. Format +slices are document means and are not the official overall score. + +Mira scorers expose continuous values; their pass flag means the score is valid. +The runner separately gates canonical aggregate and per-type/category/format +scores against `baseline.json`, allowing at most **0.005 absolute regression** +(half a percentage point). A full dev run must include every baseline case and +gates that identical subset; its full-set score is reported separately. Test +runs enforce coverage and execution but report quality without baseline gating. +These checks never treat a skipped or broken case as an improved denominator. + +To deliberately replace the reviewed PR baseline after inspecting results: + +```bash +just eval-public --write-baseline +``` + +This flag works only with `--scope pr`; CI never passes it. Commit the changed +baseline with the reason for adopting it. Corpus/scorer changes cannot silently +use an older contract. Component scores remain available in saved reports even +when the gate uses a category's canonical aggregate. + +## Timing and artifacts + +Every case gets one untimed warm-up conversion and three timed fresh CLI +processes. Execution is sequential; the median includes process startup and +excludes preparation, scoring and report persistence. Output must be identical +across all four conversions. Compare timings only on matched hardware and +startup boundaries; latency is reported rather than gated across CI machines. + +`target/public-benchmarks/report-/` contains Mira JSON, canonical JSON and +Markdown summaries, every per-case quality metric and conversion duration, +source/binary/compiler/platform provenance, the input manifest and notices. +Mira also saves its normal resumable run under `crates/markkit-eval/results/`. +CI uploads the reports even when the regression gate fails and writes a job +summary. PRs and pushes run the fixed selection; nightly runs use `dev`; +workflow dispatch also supports `test`. The recorded release baseline summary +is committed under `crates/markkit-eval/results/`. + +Retain internal structural, golden and security tests. WCXB labels full content, +while AgentDocBench has only a few native-format examples and sparse checks for +some formats. A perfect score on these samples does not establish complete +format support. Suggested future competitor runs are Trafilatura for WCXB and +MarkItDown for native formats on the same pinned inputs and machine. + +## Sources and licenses + +- [WCXB](https://github.com/Murrough-Foley/web-content-extraction-benchmark), + Murrough Foley, CC-BY-4.0. Source blob IDs are in `wcxb-corpus.json`; + see `vendor/wcxb/LICENSE`. +- [AgentDocBench](https://github.com/SylphxAI/anymd/tree/main/bench), SylphxAI, + MIT scorer. Individual documents retain the licenses and attribution in + `agentdocbench-corpus.json` and `CORPUS-NOTICE`. +- [Mira Python SDK](https://github.com/everruns/mira/tree/v0.5.0/sdks/python), + Everruns contributors, MIT. SDK/scorer sources are vendored unchanged with + their licenses; exact source revisions are in `pins.json`. + +Document corpora and third-party executables are downloaded, not committed. +This tool adds no Python, network or third-party scorer dependency to Markkit's +published library or bindings. diff --git a/scripts/public_benchmarks/agentdocbench-assets.json b/scripts/public_benchmarks/agentdocbench-assets.json new file mode 100644 index 0000000..4b34c83 --- /dev/null +++ b/scripts/public_benchmarks/agentdocbench-assets.json @@ -0,0 +1,68 @@ +{ + "reference/cjk-jp-stat-elderly.txt": "10e0ef761fb518580bdb716e6f2b296f927df188", + "reference/cjk-tw-archives.txt": "1c6f191846d30b38eaa803f0082f41b9d1733a35", + "reference/cjk-tw-dgbas-cpi.txt": "2fff628c219f9d8b84dc4bdd2d957db7c95e821b", + "reference/docx-hhs-market-research.txt": "e06cf9ce30e70e0b9922976daeaac68e9662a01e", + "reference/docx-markitdown.txt": "026e64975416f02fc035f830e4a0c07270b4b8b4", + "reference/docx-nsf-aaac-minutes.txt": "640513ba249aca6dcf988a3d8deb60df99226937", + "reference/epub-gutenberg-jekyll.txt": "1791cd6706ed6244124d24bfde0548df1eb59f82", + "reference/epub-gutenberg-relativity.txt": "cd2bbd32d2c26c325180452f57114382aef201c0", + "reference/epub-markitdown.txt": "e1301f3502944e32bb6d9ff3d78fc6a0b453bdf9", + "reference/fed-h8.txt": "f4f2b65e3a51160c0141258012ba232c8234f082", + "reference/fin-bls-realer.txt": "fc0f68eb7ae0f40e67f5dfb4d8241f215a5583b3", + "reference/fin-census-income.txt": "959d6da49781deb159599b8aff8ba90bc6582fff", + "reference/form-1040es.txt": "167ad1bdfc16a6453c630f64e66d87ec0449b302", + "reference/form-w4.txt": "90a2d1425ac7b8036f571c5b635f8fbb1613f579", + "reference/invoice-borderless.txt": "ab960633ec97e4229b8dc629b557b829bd6c5c32", + "reference/irs-w9.txt": "e61f93ca1d72a9d9c0271ceda01c71b8c4fea700", + "reference/paper-evsched.txt": "9e98fc272c1be58807707269e3aceae8541f6c26", + "reference/paper-rankone.txt": "5d1e3e3e900c2a8d43acd85ac3e283e5df58d9a2", + "reference/paper-ulam.txt": "3da839d54b9df4b87c33557a8c72c24e6d7d43ac", + "reference/report-crs-dod-approps.txt": "f5c621733412556f03d226c5554bb0f22cf1eaf3", + "reference/report-plz.txt": "2a92e7626a125699adb9fe134b2f30164d6035b4", + "reference/slides-epa-1bp.txt": "958447af74674b034083e38d363d1eb9c148ce80", + "reference/slides-markitdown-pptx.txt": "a31d782396865fe7503fe0f7e16cb300b0583a53", + "reference/slides-nasa-clarreo.txt": "ef70e97c9905ee7c1f109041138da9ba4f7bc58c", + "reference/slides-nist-pqc-update.txt": "94a678a63085eb67f57b3b57d1fd32dc7acb6281", + "reference/twocol-bygpt5.txt": "e6daf06c2b6cf1e393fc905705aae301c7ab7729", + "reference/twocol-smile.txt": "8807614827b730f2c21fe8cc3000416f9d0604f8", + "reference/twocol-uscore.txt": "5375a5dc1d2c7025918e6a9fc99ed106441aeaac", + "truth/cjk-jp-stat-elderly.json": "db348e69d5ea614147e65e01d489f67aa0a856f0", + "truth/cjk-tw-archives.json": "36f489c919af502614a64f7daadc9cd0d2dfd219", + "truth/cjk-tw-dgbas-cpi.json": "312016f4777302d12899285525d04c602f95e871", + "truth/csv-census-popest-2023.json": "5ce2bbeacdb468123f26d4ada1e4805f9f7e0549", + "truth/csv-usgs-quakes-week.json": "ee71b5badccd6943efa0276d77a57599a53d573a", + "truth/docx-hhs-market-research.json": "f022cdcace2aff327c8aeb5c61dab6faaee9ad28", + "truth/docx-markitdown.json": "72585277e90619f66134d5233e06fc838f7b9054", + "truth/docx-nsf-aaac-minutes.json": "4169e94a0c8341d3290972076bffb7b60aacef6b", + "truth/epub-gutenberg-jekyll.json": "8fda37b13af0be3c38b3a95efe4898d7f21468c7", + "truth/epub-gutenberg-relativity.json": "92ad6955e546ee505aeee3927e043d70849c81dd", + "truth/epub-markitdown.json": "0d99f82f5cac52117b2f7a22112a61165b5a669c", + "truth/fed-h8.json": "fe3892ce94fa682592444012e1477662cfac9602", + "truth/fin-bls-realer.json": "947eed9d3b5c183380f2b5451c23039fd8da6421", + "truth/fin-census-income.json": "13184dd7577a9a18ca0d1411c960af46bb6efbfa", + "truth/form-1040es.json": "a7caf9e5b4661c46e42b933552c78022fcadf572", + "truth/form-w4.json": "232d9d928e08d90a50c3c93bf8bb592a7702600b", + "truth/html-bls-ooh-software-developers.json": "01c084ab79e3db23e584d524f60f7735fa4ac5cc", + "truth/html-markitdown-wikipedia.json": "8039048292be858f0ef6d8363ef473fee20e0c60", + "truth/html-wikipedia-roman-numerals.json": "6a379dfd553ef0c9ac017dec296d3eba9766401a", + "truth/invoice-borderless.json": "73956cc9093b86ca39354ca2a67d9eb516eee90d", + "truth/irs-w9.json": "6e2f6c4013096ef5031f633cad3b31e9bf814627", + "truth/paper-evsched.json": "f7e174ae93e5ac76002b5f762c8fd7a7d9b4d4ba", + "truth/paper-rankone.json": "0fd986ad43e211dd7ec70b78b297c29449da2cb2", + "truth/paper-ulam.json": "8993370b9904da8e52803e4b1b92c4e99b828b3e", + "truth/report-crs-dod-approps.json": "9159e5e87e1e20ff4ea571654ff77bb6c36f5d18", + "truth/report-plz.json": "716a68da0999029ee23b544082614b004c0608f7", + "truth/scan-nasa-budget.json": "484a18b84dda73e5e252bd7759078e1648416587", + "truth/scan-nasa-notes.json": "248985e93942f8264f1c5724a196c8d4b7f576ba", + "truth/scan-oklahoma-address.json": "9aa8f136659cefdd491c8de5c07e74d5b2db7280", + "truth/sheet-ers-crop-insurance.json": "f3355faa33fc580f726b23cb222ecbaaf36142bf", + "truth/sheet-markitdown-xlsx.json": "09fc908867390f85585d57a9c22265d9d2b1f2c4", + "truth/slides-epa-1bp.json": "bef01bd9ad0152dab6d2cab43544d8e288defd09", + "truth/slides-markitdown-pptx.json": "27fb3a6b532eaf77f3325df318625cfe88817ee9", + "truth/slides-nasa-clarreo.json": "24a608e5154b6b55cf89362aa023df11aec3f43c", + "truth/slides-nist-pqc-update.json": "fbec7429a4041b1dca31ad4436db8b0357cdd597", + "truth/twocol-bygpt5.json": "137d05e7768d13026c0027c9deff7cf9142d5c96", + "truth/twocol-smile.json": "407bb6f777c1ae7544da77cf246dd10e9b8711d2", + "truth/twocol-uscore.json": "b35447773743ef548ac776629641f4bae160c57e" +} diff --git a/scripts/public_benchmarks/agentdocbench-corpus.json b/scripts/public_benchmarks/agentdocbench-corpus.json new file mode 100644 index 0000000..22790a1 --- /dev/null +++ b/scripts/public_benchmarks/agentdocbench-corpus.json @@ -0,0 +1,689 @@ +{ + "description": "AgentDocBench corpus. Every file is pinned by SHA-256 and downloaded by bench/fetch.py (or copied from bench/files/). See bench/README.md.", + "docs": [ + { + "id": "paper-evsched", + "file": "paper-evsched.pdf", + "category": "paper", + "format": "pdf", + "title": "Grid-constrained online scheduling of flexible electric vehicle charging", + "language": "en", + "pages": 23, + "bytes": 375898, + "sha256": "402de06ee9e5df0ec8ded433eb9440b802f02cc2e44acd8eaa89977892322cd2", + "url": "https://arxiv.org/pdf/2403.03109v1", + "license": "CC-BY-4.0", + "license_evidence": "https://arxiv.org/abs/2403.03109v1 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2403.03109&metadataPrefix=arXiv ", + "attribution": "Emily van Huffelen, Roel Brouwer, Marjan van den Akker (Utrecht University) (arXiv:2403.03109v1)", + "ocr": false, + "challenge": "single-column OR paper with 9 numeric tables incl. row-grouped tables with merged first-column labels, some math", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/paper-evsched.pdf" + }, + { + "id": "paper-rankone", + "file": "paper-rankone.pdf", + "category": "paper", + "format": "pdf", + "title": "Probabilistic bounds on best rank-one approximation ratio", + "language": "en", + "pages": 27, + "bytes": 347808, + "sha256": "328a7069a9ea1f569158ead666352111ea4eccd695c6e00d56ddf0306f554c46", + "url": "https://arxiv.org/pdf/2201.02191v2", + "license": "CC-BY-4.0", + "license_evidence": "https://arxiv.org/abs/2201.02191v2 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2201.02191&metadataPrefix=arXiv ", + "attribution": "Khazhgali Kozhasov, Josué Tonelli-Cueto (arXiv:2201.02191v2)", + "ocr": false, + "challenge": "single-column, very math-heavy (27 pp of propositions, proofs, multi-line display equations, legacy Type1 math fonts); no tables", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/paper-rankone.pdf" + }, + { + "id": "paper-ulam", + "file": "paper-ulam.pdf", + "category": "paper", + "format": "pdf", + "title": "Diametric problem for permutations with the Ulam metric (optimal anticodes)", + "language": "en", + "pages": 11, + "bytes": 158704, + "sha256": "fbc9633924adb8d5cc2b2ee72fdcaea52ebcb7772edc55c38d8b9d2820fab66a", + "url": "https://arxiv.org/pdf/2403.02276v1", + "license": "CC-BY-4.0", + "license_evidence": "https://arxiv.org/abs/2403.02276v1 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2403.02276&metadataPrefix=arXiv ", + "attribution": "Pat Devlin, Leo Douhovnikoff (Swarthmore College) (arXiv:2403.02276v1)", + "ocr": false, + "challenge": "single-column combinatorics paper: theorems/proofs, display math, a math-symbol table (∉, ∪, \\) with double rules", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/paper-ulam.pdf" + }, + { + "id": "twocol-bygpt5", + "file": "twocol-bygpt5.pdf", + "category": "two-column", + "format": "pdf", + "title": "ByGPT5: End-to-End Style-conditioned Poetry Generation with Token-free Language Models", + "language": "en", + "pages": 16, + "bytes": 2152156, + "sha256": "d07da94752759e80f573979603e0d5f1c060083459e83f1a57771a59d97fad55", + "url": "https://arxiv.org/pdf/2212.10474v2", + "license": "CC-BY-4.0", + "license_evidence": "https://arxiv.org/abs/2212.10474v2 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2212.10474&metadataPrefix=arXiv ", + "attribution": "Jonas Belouadi, Steffen Eger (arXiv:2212.10474v2)", + "ocr": false, + "challenge": "ACL two-column; small-caps model names, tables with ✓/✗ marks and row spans, figures interleaved between columns", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/twocol-bygpt5.pdf" + }, + { + "id": "twocol-smile", + "file": "twocol-smile.pdf", + "category": "two-column", + "format": "pdf", + "title": "SMiLE: Schema-augmented Multi-level Contrastive Learning for Knowledge Graph Link Prediction", + "language": "en", + "pages": 13, + "bytes": 881763, + "sha256": "cff9c2ff011e00aa50a7e8615d85bd928048eb4a8eb7d29d6f31f101200fbe54", + "url": "https://arxiv.org/pdf/2210.04870v3", + "license": "CC-BY-4.0", + "license_evidence": "https://arxiv.org/abs/2210.04870v3 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2210.04870&metadataPrefix=arXiv ", + "attribution": "Miao Peng, Ben Liu, Qianqian Xie, Wenjie Xu, Hua Wang, Min Peng (arXiv:2210.04870v3)", + "ocr": false, + "challenge": "EMNLP-Findings two-column; column-spanning tables, inline/display math, large figure on page 1 whose labels pollute the text layer", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/twocol-smile.pdf" + }, + { + "id": "twocol-uscore", + "file": "twocol-uscore.pdf", + "category": "two-column", + "format": "pdf", + "title": "USCORE: An Effective Approach to Fully Unsupervised Evaluation Metrics for Machine Translation", + "language": "en", + "pages": 17, + "bytes": 552061, + "sha256": "7a46f079adac9b3ed19d76fe224c1c0523c8a5113281662022c188abe5a85987", + "url": "https://arxiv.org/pdf/2202.10062v3", + "license": "CC-BY-4.0", + "license_evidence": "https://arxiv.org/abs/2202.10062v3 (license link http://creativecommons.org/licenses/by/4.0/); confirmed via https://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:2202.10062&metadataPrefix=arXiv ", + "attribution": "Jonas Belouadi, Steffen Eger (arXiv:2202.10062v3)", + "ocr": false, + "challenge": "EACL two-column; wide multi-group results tables with subscript/superscript metric names, figures with scattered labels", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/twocol-uscore.pdf" + }, + { + "id": "report-crs-dod-approps", + "file": "report-crs-dod-approps.pdf", + "category": "report", + "format": "pdf", + "title": "FY2026 Department of Defense Appropriations: In Brief (CRS R48891, version 4)", + "language": "en", + "pages": 12, + "bytes": 765186, + "sha256": "3467a0131f7900f8caaf7c106b1748973b25fb28e7dd891c6ef5836c98273682", + "url": "https://www.congress.gov/crs_external_products/R/PDF/R48891/R48891.4.pdf", + "license": "public-domain-us-gov", + "license_evidence": "In-document disclaimer (last page): 'CRS Reports, as a work of the United States Government, are not subject to copyright protection in the United States.'", + "attribution": "Congressional Research Service (Cameron M. Keys)", + "ocr": false, + "challenge": "report with table of contents duplicating headings, heavy footnotes on every page, and a table with 7 stacked multi-line column heads, wrapped row labels, italic subset rows and superscript footnote letters", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/report-crs-dod-approps.pdf" + }, + { + "id": "report-plz", + "file": "report-plz.pdf", + "category": "report", + "format": "pdf", + "title": "U.S. Access Board Technical Guide: Passenger Loading Zones", + "language": "en", + "pages": 5, + "bytes": 527558, + "sha256": "0f8daeba6773577fac1d443828d6ff4bf8755e03d119d1ca13ca3827ffd8f66e", + "url": "https://www.access-board.gov/files/ada/guides/plz.pdf", + "license": "public-domain-us-gov", + "license_evidence": "Work of the U.S. Access Board, a US federal agency (17 U.S.C. 105)", + "attribution": "U.S. Access Board", + "ocr": false, + "challenge": "designed guide: icons, photo with caption, labelled diagram with callout boxes in two columns, Q&A section, running headers", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/report-plz.pdf" + }, + { + "id": "fed-h8", + "file": "fed-h8.pdf", + "category": "financial", + "format": "pdf", + "title": "H.8 Assets and Liabilities of Commercial Banks in the United States, September 18, 2026", + "language": "en", + "pages": 22, + "bytes": 186358, + "sha256": "a56c5e4c4ac5391770abff75d3d9f9ab8d2d3cd9a9f9e057ad83f9828841a768", + "url": "https://www.federalreserve.gov/releases/h8/20260918/h8.pdf", + "license": "public-domain-us-gov", + "license_evidence": "Work of the Board of Governors of the Federal Reserve System, a US federal agency (17 U.S.C. 105); https://www.federalreserve.gov/disclaimer.htm", + "attribution": "Board of Governors of the Federal Reserve System", + "ocr": false, + "challenge": "22 pages of dense landscape statistical tables (16 columns), wrapped row labels, line-number column, superscript footnote markers, two-line column heads", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/fed-h8.pdf" + }, + { + "id": "fin-bls-realer", + "file": "fin-bls-realer.pdf", + "category": "financial", + "format": "pdf", + "title": "Real Earnings – July 2026 (USDL-26-1379)", + "language": "en", + "pages": 5, + "bytes": 392212, + "sha256": "58686e3cf434e81e853b4d8f01b6e3ab55f16e84556516ce74eb33a649950a78", + "url": "https://web.archive.org/web/20260819200140id_/https://www.bls.gov/news.release/archives/realer_08122026.pdf", + "license": "public-domain-us-gov", + "license_evidence": "https://www.bls.gov/opub/copyright-information.htm (BLS material is in the public domain); US federal work, 17 U.S.C. 105", + "attribution": "U.S. Bureau of Labor Statistics", + "ocr": false, + "challenge": "leader-dot tables with section rows and superscript footnotes, bar charts whose data labels spill into text, two-column technical note", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/fin-bls-realer.pdf" + }, + { + "id": "fin-census-income", + "file": "fin-census-income.pdf", + "category": "financial", + "format": "pdf", + "title": "Household Income in States and Metropolitan Areas: 2024 (ACSBR-025)", + "language": "en", + "pages": 10, + "bytes": 1477850, + "sha256": "5de1b0d5ec469344896bdf8485d3e93206351c0f1f9e51029b8d57d2df9054b9", + "url": "https://www2.census.gov/library/publications/2025/demo/acsbr-025.pdf", + "license": "public-domain-us-gov", + "license_evidence": "US federal work (17 U.S.C. 105), U.S. Census Bureau publication; landing page https://www.census.gov/library/publications/2025/acs/acsbr-025.html", + "attribution": "U.S. Census Bureau (Gloria G. Guzman)", + "ocr": false, + "challenge": "designed multi-column brief (2-3 columns) with shaded definition sidebars and text box, maps/charts, three appendix tables with two-level headers and leader dots (53-row state table)", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/fin-census-income.pdf" + }, + { + "id": "invoice-borderless", + "file": "invoice-borderless.pdf", + "category": "financial", + "format": "pdf", + "title": "Inventory Reconciliation Report SPARSE-2024-INV-1234 (markitdown test file)", + "language": "en", + "pages": 2, + "bytes": 19649, + "sha256": "784fef202eb748d5d274eaaf2ded51dfd0d0d1e92225e84fd33dafc2333d15d3", + "url": "https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/SPARSE-2024-INV-1234_borderless_table.pdf", + "license": "MIT", + "license_evidence": "https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT)", + "attribution": "Microsoft Corporation (markitdown test files)", + "ocr": false, + "challenge": "borderless sparse table with split rows and empty cells, second ruled table after a page break, chart image", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/invoice-borderless.pdf" + }, + { + "id": "form-1040es", + "file": "form-1040es.pdf", + "category": "form", + "format": "pdf", + "title": "2024 Form 1040-ES: Estimated Tax for Individuals", + "language": "en", + "pages": 12, + "bytes": 305018, + "sha256": "860a291f7e24b2e146b4c164e41cf79f0f0afa9fadde2690b474c2bf21300ee6", + "url": "https://www.irs.gov/pub/irs-prior/f1040es--2024.pdf", + "license": "public-domain-us-gov", + "license_evidence": "IRS form, US federal government work (17 U.S.C. 105)", + "attribution": "Internal Revenue Service", + "ocr": false, + "challenge": "fillable package (113 fields incl. payment vouchers), two-column instructions, standard-deduction chart, side-by-side tax rate schedules with multi-level headers, worksheets with dot leaders", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/form-1040es.pdf" + }, + { + "id": "form-w4", + "file": "form-w4.pdf", + "category": "form", + "format": "pdf", + "title": "2024 Form W-4: Employee's Withholding Certificate", + "language": "en", + "pages": 4, + "bytes": 189925, + "sha256": "e60e3af7512073bffbd7ef3c4f3a2fc9593d2aed2e420718d4ccf17659e2c3ed", + "url": "https://www.irs.gov/pub/irs-prior/fw4--2024.pdf", + "license": "public-domain-us-gov", + "license_evidence": "IRS form, US federal government work (17 U.S.C. 105)", + "attribution": "Internal Revenue Service", + "ocr": false, + "challenge": "fillable form (29 fields), step boxes with dot leaders, two-column instructions page 2, three 13-column numeric tax tables with spanning header on page 4", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/form-w4.pdf" + }, + { + "id": "irs-w9", + "file": "irs-w9.pdf", + "category": "form", + "format": "pdf", + "title": "Form W-9 (Rev. March 2024): Request for Taxpayer Identification Number and Certification", + "language": "en", + "pages": 6, + "bytes": 140815, + "sha256": "2d420cbb4123dcf1fb82595b2359cfbb5d81f00b9df9d359fcc7af361d093f53", + "url": "https://www.irs.gov/pub/irs-prior/fw9--2024.pdf", + "license": "public-domain-us-gov", + "license_evidence": "IRS form, US federal government work (17 U.S.C. 105); https://www.irs.gov/privacy-disclosure/irs-privacy-policy", + "attribution": "Internal Revenue Service", + "ocr": false, + "challenge": "fillable form (AcroForm/XFA, 23 fields), dense form grid page 1, two-column instructions, boxed two-column tables with bullets, footnote markers", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/irs-w9.pdf" + }, + { + "id": "scan-nasa-budget", + "file": "scan-nasa-budget.pdf", + "category": "scanned", + "format": "pdf", + "title": "Office Memorandum: NASA 1961 Estimates and BOB Staff Proposal to BOB Director (J. L. Mitchell to G. B. Kistiakowsky, 14 Nov 1959)", + "language": "en", + "pages": 3, + "bytes": 1010906, + "sha256": "f0637cdd4026471afa0cf1ab199a4030844ee619f0a6dd727ceab5f29717e847", + "url": "https://web.archive.org/web/20250515183040id_/https://www.eisenhowerlibrary.gov/sites/default/files/file/nasa_Binder20.pdf", + "license": "public-domain-us-gov", + "license_evidence": "https://www.eisenhowerlibrary.gov/research/online-documents (\"All of these items are in the public domain\"); authored by US federal government employees in the course of duty (17 U.S.C. 105)", + "attribution": "Jesse L. Mitchell (Office of the Special Assistant for Science and Technology) / Dwight D. Eisenhower Presidential Library, NARA", + "ocr": true, + "challenge": "image-only typed scan on blue paper, borderless typed numeric tables (3-col and 4-col), handwritten corrections, library stamp", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/scan-nasa-budget.pdf" + }, + { + "id": "scan-nasa-notes", + "file": "scan-nasa-notes.pdf", + "category": "scanned", + "format": "pdf", + "title": "Memorandum for Dr. Killian: Notes on \"NASA\" (Robert Piland, 10 April 1958)", + "language": "en", + "pages": 3, + "bytes": 553787, + "sha256": "0d0ae5bc4a51949159beea4455d198852087e7fd6b95e9d6730a6248700975bf", + "url": "https://web.archive.org/web/20250515183039id_/https://www.eisenhowerlibrary.gov/sites/default/files/file/nasa_Binder18.pdf", + "license": "public-domain-us-gov", + "license_evidence": "https://www.eisenhowerlibrary.gov/research/online-documents (\"All of these items are in the public domain\"); authored by US federal government employees in the course of duty (17 U.S.C. 105)", + "attribution": "Robert Piland (White House science advisory staff) / Dwight D. Eisenhower Presidential Library, NARA", + "ocr": true, + "challenge": "image-only typewritten scan (~150 dpi), declassification stamps and handwriting in margins, typed budget list with amounts", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/scan-nasa-notes.pdf" + }, + { + "id": "scan-oklahoma-address", + "file": "scan-oklahoma-address.pdf", + "category": "scanned", + "format": "pdf", + "title": "Text of the Address on \"Our Future Security\", Oklahoma City, November 13, 1957 (White House press release)", + "language": "en", + "pages": 7, + "bytes": 1941463, + "sha256": "f9da22937bda1617e919bb4a3b902551187bb484a07af254ccff22647ae81848", + "url": "https://web.archive.org/web/20250515183607id_/https://www.eisenhowerlibrary.gov/sites/default/files/research/online-documents/sputnik/11-13-57.pdf", + "license": "public-domain-us-gov", + "license_evidence": "https://www.eisenhowerlibrary.gov/research/online-documents (\"All of these items are in the public domain\"); authored by US federal government employees in the course of duty (17 U.S.C. 105)", + "attribution": "The White House (President Dwight D. Eisenhower) / Dwight D. Eisenhower Presidential Library, NARA", + "ocr": true, + "challenge": "image-only 7-page typeset/mimeographed press release on legal paper, uneven ink, library stamp overlapping text", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/scan-oklahoma-address.pdf" + }, + { + "id": "cjk-jp-stat-elderly", + "file": "cjk-jp-stat-elderly.pdf", + "category": "cjk", + "format": "pdf", + "title": "統計トピックスNo.142 統計からみた我が国の高齢者 Ⅰ 高齢者の人口", + "language": "ja", + "pages": 4, + "bytes": 347744, + "sha256": "5565b9c00fff2065023da95dd313a5f5b950d75b162669a167538152fcd4efe1", + "url": "https://web.archive.org/web/20241117224402id_/https://www.stat.go.jp/data/topics/pdf/topi142_01.pdf", + "license": "PDL-1.0", + "license_evidence": "https://www.stat.go.jp/info/riyou.html (統計局ホームページ利用規約: 公共データ利用規約（第1.0版）(PDL1.0) に準拠; 'クリエイティブ・コモンズ・ライセンスの表示4.0 国際 ... と互換性があり'; numeric data/simple tables not copyrightable)", + "attribution": "総務省統計局 (Statistics Bureau of Japan)", + "ocr": false, + "challenge": "letter-spaced justified Japanese (text layer has a space between most glyphs), full-width digits, three statistics tables incl. a two-level header table and a nested row-group table, charts with axis-label text", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/cjk-jp-stat-elderly.pdf" + }, + { + "id": "cjk-tw-archives", + "file": "cjk-tw-archives.pdf", + "category": "cjk", + "format": "pdf", + "title": "參考文獻格式 (reference citation format guide)", + "language": "zh-Hant", + "pages": 4, + "bytes": 323240, + "sha256": "b28e5e6d91923d27d6fa2d29b42018584e71b3a38ff04aa9acc638b0c11e1bd3", + "url": "https://www.archives.gov.tw/wSite/public/Attachment/0/f1715062099077.pdf", + "license": "CC-BY-4.0", + "license_evidence": "https://www.archives.gov.tw/tw/arctw/625-10226.html (政府網站資料開放宣告: all site materials released under CC BY 4.0, attribution required)", + "attribution": "國家發展委員會檔案管理局 (National Archives Administration, National Development Council, Taiwan)", + "ocr": false, + "challenge": "Traditional Chinese with ruled two-column tables whose cells hold multi-line mixed CJK/Latin text; second table continues across a page break; italic CJK", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/cjk-tw-archives.pdf" + }, + { + "id": "cjk-tw-dgbas-cpi", + "file": "cjk-tw-dgbas-cpi.pdf", + "category": "cjk", + "format": "pdf", + "title": "115 年 8 月物價變動概況 (DGBAS price statistics press release, Aug 2026)", + "language": "zh-Hant", + "pages": 6, + "bytes": 368900, + "sha256": "203a27ef3771ffc870f3694c860aff660b8cb06e5bf3e4351bfeba8d02074630", + "url": "https://ws.dgbas.gov.tw/001/Upload/463/relfile/10980/236675/news11508.pdf", + "license": "Taiwan-OGDL-1.0", + "license_evidence": "https://www.stat.gov.tw/cp.aspx?n=3164 (行政院主計總處 政府網站資料開放宣告: 採政府資料開放授權條款-第1版發布); release page https://www.stat.gov.tw/News_Content.aspx?n=2668&s=236675", + "attribution": "行政院主計總處 (Directorate-General of Budget, Accounting and Statistics, Executive Yuan, Taiwan)", + "ocr": false, + "challenge": "Traditional Chinese press release: two-level-header statistics table, body text wrapped around embedded charts whose labels interleave with prose, footnote circled numerals", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/cjk-tw-dgbas-cpi.pdf" + }, + { + "id": "slides-epa-1bp", + "file": "slides-epa-1bp.pptx", + "category": "slides", + "format": "pptx", + "title": "1-Bromopropane (1-BP) Proposed Rulemaking Under TSCA Section 6(a) - Public Webinar, August 28, 2024", + "language": "en", + "pages": 27, + "bytes": 294409, + "sha256": "c02b2620a068190818dddfbe7e15b083c21fd0f99f441f166762441a6734bf46", + "url": "https://web.archive.org/web/20241116234835id_/https://www.epa.gov/system/files/documents/2024-09/1-bp-public-webinar-slides_08.28.24.pptx", + "license": "public-domain-us-gov", + "license_evidence": "Work of US EPA staff (Office of Chemical Safety and Pollution Prevention), 17 U.S.C. 105; EPA site copyright note: https://www.epa.gov/web-policies-and-procedures/epa-disclaimers", + "attribution": "U.S. Environmental Protection Agency", + "ocr": false, + "challenge": "title placeholders come after the body shapes in XML order; 2 slide tables with multi-paragraph cells; zero-width spaces/NBSP; nested bullets", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/slides-epa-1bp.pptx" + }, + { + "id": "slides-markitdown-pptx", + "file": "slides-markitdown-pptx.pptx", + "category": "slides", + "format": "pptx", + "title": "markitdown test deck (AutoGen)", + "language": "en", + "pages": 6, + "bytes": 277515, + "sha256": "f0b9e5252aec3730c91f276a7c3b8f4a9893a7b66540f6c6af160a49855ca507", + "url": "https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.pptx", + "license": "MIT", + "license_evidence": "https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT, repository test file)", + "attribution": "Microsoft (markitdown contributors)", + "ocr": false, + "challenge": "slide table, chart, nested group shapes, alt text, UUID canaries", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/slides-markitdown-pptx.pptx" + }, + { + "id": "slides-nasa-clarreo", + "file": "slides-nasa-clarreo.pdf", + "category": "slides", + "format": "pdf", + "title": "CLARREO Pathfinder Mission Overview and Status (CEOS WGCV IVOS 2022)", + "language": "en", + "pages": 15, + "bytes": 4659017, + "sha256": "bb04eb9ea712c6960198b5153ae119ff58d3b5a4ce4d775af5299ba38c392a82", + "url": "https://ntrs.nasa.gov/api/citations/20220013266/downloads/Shea_etal_CEOS_WGCV_IVOS_2022_STRIVES.pdf", + "license": "public-domain-us-gov", + "license_evidence": "https://ntrs.nasa.gov/citations/20220013266 (NTRS copyright determination GOV_PUBLIC_USE_PERMITTED: 'Work of the US Gov. Public Use Permitted'; both authors are NASA LaRC civil servants)", + "attribution": "Yolanda L. Shea, Rajendra Bhatt / NASA Langley Research Center", + "ocr": false, + "challenge": "PowerPoint-to-PDF deck (Quartz); slide tables incl. an empty header cell; slide 10 table carries hidden text in its empty third column; labelled photos and diagrams; slides of logos only", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/slides-nasa-clarreo.pdf" + }, + { + "id": "slides-nist-pqc-update", + "file": "slides-nist-pqc-update.pdf", + "category": "slides", + "format": "pdf", + "title": "NIST Post-Quantum Cryptography Update (Dustin Moody, ISPAB Oct 2023)", + "language": "en", + "pages": 16, + "bytes": 2226669, + "sha256": "1c50a396a1eefc93ece22e43d6b389e6338c40685cb625bb29ff7d29453a5ee5", + "url": "https://web.archive.org/web/20241002031327id_/https://csrc.nist.gov/csrc/media/Presentations/2023/nist-post-quantum-cryptography-update/2a-Moody_NIST_PQC_2.pdf", + "license": "public-domain-us-gov", + "license_evidence": "Work of a NIST employee, 17 U.S.C. 105; https://www.nist.gov/oism/copyrights and https://csrc.nist.gov/presentations/2023/nist-post-quantum-cryptography-update", + "attribution": "Dustin Moody / NIST Computer Security Division", + "ocr": false, + "challenge": "all-caps titles, strikethrough cells in a table, 3 small slide tables, text rendered as images (timeline, memo scans), duplicated hidden title text on slide 5", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/slides-nist-pqc-update.pdf" + }, + { + "id": "csv-census-popest-2023", + "file": "csv-census-popest-2023.csv", + "category": "spreadsheet", + "format": "csv", + "title": "Annual Estimates of the Resident Population and Population Change, States, 2020-2023 (NST-EST2023-POPCHG2020_2023)", + "language": "en", + "pages": null, + "bytes": 11942, + "sha256": "c7f1d6d644007e129baaf1be509b83c914f4a1e167e3407af65207dd14d8a9e5", + "url": "https://web.archive.org/web/20240623113748id_/https://www2.census.gov/programs-surveys/popest/datasets/2020-2023/state/totals/NST-EST2023-POPCHG2020_2023.csv", + "license": "public-domain-us-gov", + "license_evidence": "US Census Bureau data product, a work of the US Government (17 U.S.C. 105); https://www.census.gov/about/policies/citation.html", + "attribution": "U.S. Census Bureau, Population Division", + "ocr": false, + "challenge": "31 columns wide; leading-zero codes (010, 00); 'X' placeholders; long decimals; negative numbers", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/csv-census-popest-2023.csv" + }, + { + "id": "csv-usgs-quakes-week", + "file": "csv-usgs-quakes-week.csv", + "category": "spreadsheet", + "format": "csv", + "title": "USGS earthquake feed: M4.5+ past 7 days, snapshot 2025-01-07", + "language": "en", + "pages": null, + "bytes": 24523, + "sha256": "707c6820e37b3a433ed4f46aec942bf8e1b18924e02025cdbf6348d098decade", + "url": "https://web.archive.org/web/20250107124642id_/https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_week.csv", + "license": "public-domain-us-gov", + "license_evidence": "https://www.usgs.gov/information-policies-and-instructions/copyrights-and-credits (USGS-authored information is in the US public domain)", + "attribution": "U.S. Geological Survey, Earthquake Hazards Program", + "ocr": false, + "challenge": "22 columns; quoted fields containing commas; ISO 8601 timestamps; empty cells; 130 data rows", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/csv-usgs-quakes-week.csv" + }, + { + "id": "sheet-ers-crop-insurance", + "file": "sheet-ers-crop-insurance.xlsx", + "category": "spreadsheet", + "format": "xlsx", + "title": "Federal crop insurance premiums and subsidies, 2000-25 (ERS chart data file 102257)", + "language": "en", + "pages": null, + "bytes": 19449, + "sha256": "edcffe753bec87bd69fe41c5da7bb5a71344169d8819da22cef9822d0a05ac50", + "url": "https://web.archive.org/web/20260629195302id_/https://www.ers.usda.gov/media/29251/102257-chart-data-file.xlsx?v=88166", + "license": "public-domain-us-gov", + "license_evidence": "USDA web policy (ERS follows it): 'Most information presented on the USDA Web site is considered public domain information' https://www.usda.gov/about-usda/policies-and-links ; https://www.ers.usda.gov/about-ers/policies-and-standards/website", + "attribution": "USDA Economic Research Service", + "ocr": false, + "challenge": "display formatting differs from stored values ('0.0000' number format on 9-digit floats); title/note/source rows inside the sheet; years stored as text", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/sheet-ers-crop-insurance.xlsx" + }, + { + "id": "sheet-markitdown-xlsx", + "file": "sheet-markitdown-xlsx.xlsx", + "category": "spreadsheet", + "format": "xlsx", + "title": "markitdown test workbook", + "language": "en", + "pages": null, + "bytes": 11562, + "sha256": "a867be6ece38a4224bcfe34312e5c296ed520a76a431e0454603ceb171fbd4af", + "url": "https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.xlsx", + "license": "MIT", + "license_evidence": "https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT, repository test file)", + "attribution": "Microsoft (markitdown contributors)", + "ocr": false, + "challenge": "two sheets, one sheet name is a truncated UUID, UUID canaries in cells", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/sheet-markitdown-xlsx.xlsx" + }, + { + "id": "docx-hhs-market-research", + "file": "docx-hhs-market-research.docx", + "category": "docx", + "format": "docx", + "title": "HHS Market Research Report Template (Tab D, final 1/18/2022)", + "language": "en", + "pages": null, + "bytes": 47893, + "sha256": "89e4f8e3b3ac964f04f85385310b97f70bd0ab33e741a4fe26b84c493c170e1e", + "url": "https://web.archive.org/web/20241008230234id_/https://www.cms.gov/files/document/tab-d-hhs-market-research-template-final-1182022.docx", + "license": "public-domain-us-gov", + "license_evidence": "Work of the US Department of Health and Human Services, published on cms.gov (17 U.S.C. 105); https://www.hhs.gov/web/policies-and-standards/index.html", + "attribution": "U.S. Department of Health and Human Services", + "ocr": false, + "challenge": "numbered section headings as plain paragraphs (no Heading styles); nested numbered lists; checkbox form fields; 4 blank template tables with empty cells; a cell split across paragraphs", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/docx-hhs-market-research.docx" + }, + { + "id": "docx-markitdown", + "file": "docx-markitdown.docx", + "category": "docx", + "format": "docx", + "title": "markitdown test document (AutoGen)", + "language": "en", + "pages": null, + "bytes": 135824, + "sha256": "ee1974633f3b1e8bb54201abce779a755cb1cd5259641a08cf7b1b9b36dec42b", + "url": "https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.docx", + "license": "MIT", + "license_evidence": "https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT, repository test file)", + "attribution": "Microsoft (markitdown contributors)", + "ocr": false, + "challenge": "table with no header row, embedded image with alt text, UUID canaries, a heading that is only a UUID", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/docx-markitdown.docx" + }, + { + "id": "docx-nsf-aaac-minutes", + "file": "docx-nsf-aaac-minutes.docx", + "category": "docx", + "format": "docx", + "title": "Minutes of the Meeting of the Astronomy and Astrophysics Advisory Committee (AAAC), 15 December 2023", + "language": "en", + "pages": null, + "bytes": 44443, + "sha256": "de2aa4a0731d343ddf71b1a2f27a500146ebc2a90df928193ae2a3bbdf2fc9a9", + "url": "https://web.archive.org/web/20260426103635id_/https://nsf-gov-resources.nsf.gov/attachments/308681/public/AAAC_Meeting_Minutes_Dec_15_2023.docx", + "license": "public-domain-us-gov", + "license_evidence": "Federal advisory committee minutes published by NSF (17 U.S.C. 105); NSF site policy: https://www.nsf.gov/policies/site-policies", + "attribution": "U.S. National Science Foundation (AAAC)", + "ocr": false, + "challenge": "title in a text box (mc:AlternateContent, duplicated in the fallback); TOC content control repeats the headings; repeated 'Q&A Session' headings; deeply nested bullet lists; layout-grid attendee table with spacer rows", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/docx-nsf-aaac-minutes.docx" + }, + { + "id": "epub-gutenberg-jekyll", + "file": "epub-gutenberg-jekyll.epub", + "category": "epub", + "format": "epub", + "title": "The Strange Case of Dr. Jekyll and Mr. Hyde (Project Gutenberg #43)", + "language": "en", + "pages": null, + "bytes": 305989, + "sha256": "b7b093591573ff4562dc799c3940be5ddab4e0ac686cdb062bb6853bb2c4d66e", + "url": "https://web.archive.org/web/20250115052559id_/https://www.gutenberg.org/cache/epub/43/pg43-images-3.epub", + "license": "public-domain", + "license_evidence": "EPUB content.opf Public domain in the USA.; https://www.gutenberg.org/ebooks/43 (Robert Louis Stevenson, 1886)", + "attribution": "Robert Louis Stevenson; Project Gutenberg", + "ocr": false, + "challenge": "10 chapters across spine files; chapter titles repeated in in-book contents, nav and NCX; PG header/license boilerplate; cover image", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/epub-gutenberg-jekyll.epub" + }, + { + "id": "epub-gutenberg-relativity", + "file": "epub-gutenberg-relativity.epub", + "category": "epub", + "format": "epub", + "title": "Relativity: The Special and General Theory (Project Gutenberg #5001)", + "language": "en", + "pages": null, + "bytes": 943168, + "sha256": "698baa25db272591503de1ed8df5a594595dc4dc1f6713fad1b6b4be97875c09", + "url": "https://web.archive.org/web/20241012103314id_/https://www.gutenberg.org/cache/epub/5001/pg5001-images-3.epub", + "license": "public-domain", + "license_evidence": "EPUB content.opf Public domain in the USA.; https://www.gutenberg.org/ebooks/5001 (Einstein, tr. R. W. Lawson, 1920)", + "attribution": "Albert Einstein, translated by Robert W. Lawson; Project Gutenberg", + "ocr": false, + "challenge": "math rendered as ~150 inline/display SVG images; TOC as a table; data table with two-level spanning header; colophon/corrections tables", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/epub-gutenberg-relativity.epub" + }, + { + "id": "epub-markitdown", + "file": "epub-markitdown.epub", + "category": "epub", + "format": "epub", + "title": "Test EPUB Document (markitdown test file)", + "language": "en", + "pages": null, + "bytes": 2677, + "sha256": "33e65f39f54bd24f74d5f333d24213a68c62ee2f2854d12f8c4c59b15a4031a7", + "url": "https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test.epub", + "license": "MIT", + "license_evidence": "https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE", + "attribution": "Microsoft (markitdown test suite)", + "ocr": false, + "challenge": "tiny EPUB3 with nav document in spine; inline strong/em, bullet list, blockquote", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/epub-markitdown.epub" + }, + { + "id": "html-bls-ooh-software-developers", + "file": "html-bls-ooh-software-developers.html", + "category": "html", + "format": "html", + "title": "Software Developers, Quality Assurance Analysts, and Testers - Occupational Outlook Handbook (BLS)", + "language": "en", + "pages": null, + "bytes": 125925, + "sha256": "bf71737fceeb5e1891815ec1881c20844365779e696eb1b6d52dd4de1f2ce34a", + "url": "https://web.archive.org/web/20260910003547id_/https://www.bls.gov/ooh/computer-and-information-technology/software-developers.htm", + "license": "public-domain-us-gov", + "license_evidence": "https://www.bls.gov/opub/copyright-information.htm (BLS material is in the public domain except photographs/graphics credited to other sources; this page's text credits none); U.S. Department of Labor work", + "attribution": "U.S. Bureau of Labor Statistics", + "ocr": false, + "challenge": "tabbed page: summary tab duplicates section text, hidden glossary pop-ups; header-less 2-column tables; quick-facts table with spanning title row; projections table with two-level header", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/html-bls-ooh-software-developers.html" + }, + { + "id": "html-markitdown-wikipedia", + "file": "html-markitdown-wikipedia.html", + "category": "html", + "format": "html", + "title": "Microsoft - Wikipedia (markitdown test_wikipedia.html)", + "language": "en", + "pages": null, + "bytes": 394444, + "sha256": "4c1f6c5ca1147455b8df8b19cd957844cad37d23d541c039af421a7634074ea6", + "url": "https://raw.githubusercontent.com/microsoft/markitdown/b8f79c57ebc0044be41323d89b2a45d3fda8460e/packages/markitdown/tests/test_files/test_wikipedia.html", + "license": "CC-BY-SA-4.0 (Wikipedia content in an MIT test file)", + "license_evidence": "https://github.com/microsoft/markitdown/blob/b8f79c57ebc0044be41323d89b2a45d3fda8460e/LICENSE (MIT repo); article text CC BY-SA 4.0 per https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License", + "attribution": "Wikipedia contributors (article 'Microsoft'); Microsoft markitdown", + "ocr": false, + "challenge": "full Vector-2022 skin page: sidebar nav/TOC, language list, infobox, navboxes around the article; financial wikitables with footnote markers", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/html-markitdown-wikipedia.html" + }, + { + "id": "html-wikipedia-roman-numerals", + "file": "html-wikipedia-roman-numerals.html", + "category": "html", + "format": "html", + "title": "Roman numerals - Wikipedia (revision 1374259684)", + "language": "en", + "pages": null, + "bytes": 895435, + "sha256": "4addcd13f130fbac818f8acbd57f2aa5d7aea9ad8a0f2687efa3cd629ee0a575", + "url": "https://web.archive.org/web/20260924023032id_/https://en.wikipedia.org/wiki/Roman_numerals", + "license": "CC-BY-SA-4.0", + "license_evidence": "https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License (footer of page states CC BY-SA 4.0)", + "attribution": "Wikipedia contributors (article 'Roman numerals', oldid 1374259684)", + "ocr": false, + "challenge": "Parsoid HTML with heavy chrome; tables with empty cells, fraction slashes, rare Unicode (Ↄ, 𐆕, Cyrillic); header-in-first-column table; footnote markers", + "mirror_url": "https://github.com/SylphxAI/anymd/releases/download/agentdocbench-corpus-v1/html-wikipedia-roman-numerals.html" + } + ] +} diff --git a/scripts/public_benchmarks/baseline.json b/scripts/public_benchmarks/baseline.json new file mode 100644 index 0000000..5db05da --- /dev/null +++ b/scripts/public_benchmarks/baseline.json @@ -0,0 +1,264 @@ +{ + "absolute_tolerance": 0.005, + "cases": [ + "agentdocbench/cjk-jp-stat-elderly", + "agentdocbench/cjk-tw-archives", + "agentdocbench/cjk-tw-dgbas-cpi", + "agentdocbench/csv-census-popest-2023", + "agentdocbench/csv-usgs-quakes-week", + "agentdocbench/docx-hhs-market-research", + "agentdocbench/docx-markitdown", + "agentdocbench/docx-nsf-aaac-minutes", + "agentdocbench/epub-gutenberg-jekyll", + "agentdocbench/epub-gutenberg-relativity", + "agentdocbench/epub-markitdown", + "agentdocbench/fed-h8", + "agentdocbench/fin-bls-realer", + "agentdocbench/fin-census-income", + "agentdocbench/form-1040es", + "agentdocbench/form-w4", + "agentdocbench/html-bls-ooh-software-developers", + "agentdocbench/html-markitdown-wikipedia", + "agentdocbench/html-wikipedia-roman-numerals", + "agentdocbench/invoice-borderless", + "agentdocbench/irs-w9", + "agentdocbench/paper-evsched", + "agentdocbench/paper-rankone", + "agentdocbench/paper-ulam", + "agentdocbench/report-crs-dod-approps", + "agentdocbench/report-plz", + "agentdocbench/scan-nasa-budget", + "agentdocbench/scan-nasa-notes", + "agentdocbench/scan-oklahoma-address", + "agentdocbench/sheet-ers-crop-insurance", + "agentdocbench/sheet-markitdown-xlsx", + "agentdocbench/slides-epa-1bp", + "agentdocbench/slides-markitdown-pptx", + "agentdocbench/slides-nasa-clarreo", + "agentdocbench/slides-nist-pqc-update", + "agentdocbench/twocol-bygpt5", + "agentdocbench/twocol-smile", + "agentdocbench/twocol-uscore", + "wcxb/0083/default", + "wcxb/0083/main_content", + "wcxb/0095/default", + "wcxb/0095/main_content", + "wcxb/0135/default", + "wcxb/0135/main_content", + "wcxb/0168/default", + "wcxb/0168/main_content", + "wcxb/0219/default", + "wcxb/0219/main_content", + "wcxb/0225/default", + "wcxb/0225/main_content", + "wcxb/0239/default", + "wcxb/0239/main_content", + "wcxb/0249/default", + "wcxb/0249/main_content", + "wcxb/0265/default", + "wcxb/0265/main_content", + "wcxb/0302/default", + "wcxb/0302/main_content", + "wcxb/0328/default", + "wcxb/0328/main_content", + "wcxb/0396/default", + "wcxb/0396/main_content", + "wcxb/0515/default", + "wcxb/0515/main_content", + "wcxb/0517/default", + "wcxb/0517/main_content", + "wcxb/0526/default", + "wcxb/0526/main_content", + "wcxb/0529/default", + "wcxb/0529/main_content", + "wcxb/0533/default", + "wcxb/0533/main_content", + "wcxb/0534/default", + "wcxb/0534/main_content", + "wcxb/0553/default", + "wcxb/0553/main_content", + "wcxb/0554/default", + "wcxb/0554/main_content", + "wcxb/0581/default", + "wcxb/0581/main_content", + "wcxb/0593/default", + "wcxb/0593/main_content", + "wcxb/0603/default", + "wcxb/0603/main_content", + "wcxb/0605/default", + "wcxb/0605/main_content", + "wcxb/0624/default", + "wcxb/0624/main_content", + "wcxb/0629/default", + "wcxb/0629/main_content", + "wcxb/0635/default", + "wcxb/0635/main_content", + "wcxb/0654/default", + "wcxb/0654/main_content", + "wcxb/0675/default", + "wcxb/0675/main_content", + "wcxb/0689/default", + "wcxb/0689/main_content", + "wcxb/0695/default", + "wcxb/0695/main_content", + "wcxb/0709/default", + "wcxb/0709/main_content", + "wcxb/1658/default", + "wcxb/1658/main_content", + "wcxb/1764/default", + "wcxb/1764/main_content", + "wcxb/2684/default", + "wcxb/2684/main_content", + "wcxb/2841/default", + "wcxb/2841/main_content", + "wcxb/2861/default", + "wcxb/2861/main_content", + "wcxb/2867/default", + "wcxb/2867/main_content", + "wcxb/2871/default", + "wcxb/2871/main_content", + "wcxb/2888/default", + "wcxb/2888/main_content", + "wcxb/2898/default", + "wcxb/2898/main_content", + "wcxb/2927/default", + "wcxb/2927/main_content", + "wcxb/2955/default", + "wcxb/2955/main_content", + "wcxb/2960/default", + "wcxb/2960/main_content", + "wcxb/2965/default", + "wcxb/2965/main_content", + "wcxb/2992/default", + "wcxb/2992/main_content", + "wcxb/3011/default", + "wcxb/3011/main_content", + "wcxb/3067/default", + "wcxb/3067/main_content", + "wcxb/3078/default", + "wcxb/3078/main_content", + "wcxb/3079/default", + "wcxb/3079/main_content", + "wcxb/4001/default", + "wcxb/4001/main_content", + "wcxb/4120/default", + "wcxb/4120/main_content", + "wcxb/4247/default", + "wcxb/4247/main_content", + "wcxb/4264/default", + "wcxb/4264/main_content", + "wcxb/4348/default", + "wcxb/4348/main_content", + "wcxb/4403/default", + "wcxb/4403/main_content", + "wcxb/4409/default", + "wcxb/4409/main_content", + "wcxb/4411/default", + "wcxb/4411/main_content", + "wcxb/4435/default", + "wcxb/4435/main_content", + "wcxb/4477/default", + "wcxb/4477/main_content", + "wcxb/4479/default", + "wcxb/4479/main_content", + "wcxb/4533/default", + "wcxb/4533/main_content", + "wcxb/4537/default", + "wcxb/4537/main_content", + "wcxb/4576/default", + "wcxb/4576/main_content", + "wcxb/4690/default", + "wcxb/4690/main_content", + "wcxb/4760/default", + "wcxb/4760/main_content", + "wcxb/4764/default", + "wcxb/4764/main_content", + "wcxb/4815/default", + "wcxb/4815/main_content", + "wcxb/4829/default", + "wcxb/4829/main_content", + "wcxb/4842/default", + "wcxb/4842/main_content" + ], + "contract": "9c558864ae0d463528ea92d088f55aed75f12c36f5345ca75f7e923c6f44066c", + "provenance": { + "binary_sha256": "390d54760c41a108b4ed4f94c6aa99b993bbf04d8a36e94e4ecb861e66e56918", + "build_profile": "release", + "command": [ + "/workspace/work/markkit-tools/bin/mira", + "run", + "--study-python", + "scripts/public_benchmarks/study.py", + "--max-concurrent", + "1", + "--max-retries", + "0", + "--timeout", + "300", + "--group-by", + "category", + "--format", + "json", + "--out", + "/workspace/markkit/target/public-benchmarks/report-pr/mira.json" + ], + "core_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "cpu_count": 5, + "machine": "x86_64", + "mira_version": "mira 0.5.0", + "pins": { + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "samples_per_type": 10, + "seed": "markkit-public-pilot-v1:", + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9" + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "python": "3.12.14", + "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "source_dirty": false, + "source_revision": "0e1a9df33e6f3ec0967be5ae7f6b8016a1f585ba", + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup" + }, + "quality": { + "agentdocbench/category/cjk": 0.749, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/financial": 0.423325, + "agentdocbench/category/form": 0.7486666666666667, + "agentdocbench/category/html": 1.0, + "agentdocbench/category/paper": 0.8231999999999999, + "agentdocbench/category/report": 0.87095, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/slides": 0.674775, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/two-column": 0.7465666666666667, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0, + "agentdocbench/format/pdf": 0.5835739130434783, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/no_ocr": 0.8204075757575757, + "agentdocbench/overall": 0.7520402777777778, + "wcxb/default/overall": 0.6770940834789322, + "wcxb/default/type/article": 0.8593022688301003, + "wcxb/default/type/collection": 0.5618264586879532, + "wcxb/default/type/documentation": 0.849698642479727, + "wcxb/default/type/forum": 0.4559187546618804, + "wcxb/default/type/listing": 0.6660601098285603, + "wcxb/default/type/product": 0.5220558838570278, + "wcxb/default/type/service": 0.8247964660072756, + "wcxb/main_content/overall": 0.4999264653091694, + "wcxb/main_content/type/article": 0.9687139672138977, + "wcxb/main_content/type/collection": 0.484765233729096, + "wcxb/main_content/type/documentation": 0.7157843169028515, + "wcxb/main_content/type/forum": 0.10889386123735083, + "wcxb/main_content/type/listing": 0.33261257051933946, + "wcxb/main_content/type/product": 0.4767832465487646, + "wcxb/main_content/type/service": 0.41193206101288565 + } +} diff --git a/scripts/public_benchmarks/common.py b/scripts/public_benchmarks/common.py new file mode 100644 index 0000000..9d5c997 --- /dev/null +++ b/scripts/public_benchmarks/common.py @@ -0,0 +1,131 @@ +"""Pinned public corpora, with bounded atomic downloads and verified caches. + +Decision: a benchmark input is immutable. A broken cache is repaired only from +its pinned source, never from the live document URL or an unverified fallback. +""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import tempfile +import urllib.request + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +CACHE = REPO / "target/public-benchmarks" +MAX_BYTES = 50 * 1024 * 1024 +PINS = json.loads((ROOT / "pins.json").read_text()) + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def git_sha1(data): + return hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest() + + +def safe_path(root, name): + path = PurePosixPath(name) + if path.is_absolute() or not path.parts or any(p in {".", ".."} for p in path.parts) or "\\" in name: + raise ValueError("invalid corpus path") + result = (root / name).resolve() + if not result.is_relative_to(root.resolve()): + raise ValueError("corpus path escapes cache") + return result + + +def atomic_write(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".download-") + try: + with os.fdopen(fd, "wb") as stream: + stream.write(data) + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + + +def download(url): + if not url.startswith(("https://raw.githubusercontent.com/", "https://github.com/SylphxAI/anymd/releases/download/")): + raise ValueError("source is not a pinned corpus host") + request = urllib.request.Request(url, headers={"User-Agent": "Markkit-public-benchmarks/1"}) + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise ValueError("corpus download exceeds 50 MiB") + return data + + +def fetch(path, url, expected, algorithm="sha256"): + digest = sha256 if algorithm == "sha256" else git_sha1 + if path.is_file() and path.stat().st_size <= MAX_BYTES and digest(path.read_bytes()) == expected: + return path + data = download(url) + if digest(data) != expected: + raise ValueError(f"checksum mismatch: {path.name}") + atomic_write(path, data) + return path + + +def parallel(fn, rows): + with ThreadPoolExecutor(max_workers=8) as pool: + return list(pool.map(fn, rows)) + + +def selected_wcxb(scope): + rows = json.loads((ROOT / "wcxb-corpus.json").read_text()) + if scope == "test": + return [r for r in rows if r["split"] == "test"] + # Upstream has 139 identical HTML files in dev and test. Protect test inputs + # by excluding those files from every development run, including the PR set. + dev = [r for r in rows if r["split"] == "dev" and not r["overlaps_test"]] + if scope == "dev": + return dev + if scope != "pr": + raise ValueError("scope must be pr, dev or test") + chosen = [] + for kind in sorted({r["category"] for r in dev}): + candidates = [r for r in dev if r["category"] == kind] + candidates.sort(key=lambda r: sha256((PINS["seed"] + r["id"]).encode())) + chosen.extend(candidates[:PINS["samples_per_type"]]) + return chosen + + +def contract(): + paths = [ROOT / n for n in ["pins.json", "wcxb-corpus.json", "agentdocbench-corpus.json", "agentdocbench-assets.json"]] + paths += sorted((ROOT / "vendor").rglob("*.py")) + paths += [ROOT / "study.py"] + return sha256(b"\n".join(str(p.relative_to(ROOT)).encode() + b":" + p.read_bytes() for p in paths)) + + +def selection_ids(scope): + result = {("wcxb", row["id"]) for row in selected_wcxb(scope)} + result.update(("agentdocbench", row["id"]) for row in json.loads((ROOT / "agentdocbench-corpus.json").read_text())["docs"]) + return result + + +def read_manifest(path): + manifest = json.loads(path.read_text()) + if manifest["contract"] != contract(): + raise ValueError("prepared corpus contract changed; run prepare again") + identifiers = [(r["suite"], r["id"]) for r in manifest["samples"]] + if len(identifiers) != len(set(identifiers)): + raise ValueError("duplicate corpus sample") + if set(identifiers) != selection_ids(manifest["scope"]): + raise ValueError("prepared corpus does not cover the complete pinned selection") + for row in manifest["samples"]: + for field in ["file", "truth"] + (["reference"] if "reference" in row else []): + if row.get(field) not in manifest["files"]: + raise ValueError("prepared sample has no verified file") + # Validate before the study advertises any cases. Missing/corrupt data can + # never become an unavailable target and disappear from the score denominator. + for name, expected in manifest["files"].items(): + file = safe_path(path.parent, name) + if file.stat().st_size > MAX_BYTES: + raise ValueError("prepared file exceeds 50 MiB") + data = file.read_bytes() + if sha256(data) != expected: + raise ValueError(f"prepared checksum mismatch: {name}") + return manifest diff --git a/scripts/public_benchmarks/install_mira.py b/scripts/public_benchmarks/install_mira.py new file mode 100644 index 0000000..4c48ef3 --- /dev/null +++ b/scripts/public_benchmarks/install_mira.py @@ -0,0 +1,30 @@ +"""Install the pinned, checksum-verified Mira Linux x86_64 host for CI.""" +import io +import platform +import tarfile +import urllib.request + +from common import CACHE, PINS, atomic_write, sha256 + + +def main(): + if platform.system() != "Linux" or platform.machine() != "x86_64": + raise SystemExit("install Mira 0.5.0 for your platform and pass --mira to run.py") + url = f"https://github.com/everruns/mira/releases/download/{PINS['mira']}/mira-x86_64-unknown-linux-gnu.tar.gz" + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read(10 * 1024 * 1024 + 1) + if sha256(data) != PINS["mira_linux_archive_sha256"]: + raise ValueError("Mira release checksum mismatch") + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + # Read exactly one member; never extract archive-controlled paths. + data = archive.extractfile("mira").read(10 * 1024 * 1024 + 1) + if len(data) > 10 * 1024 * 1024: + raise ValueError("Mira executable exceeds limit") + path = CACHE / "bin/mira" + atomic_write(path, data) + path.chmod(0o755) + print(path) + + +if __name__ == "__main__": + main() diff --git a/scripts/public_benchmarks/pins.json b/scripts/public_benchmarks/pins.json new file mode 100644 index 0000000..4dd6050 --- /dev/null +++ b/scripts/public_benchmarks/pins.json @@ -0,0 +1,9 @@ +{ + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9", + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "seed": "markkit-public-pilot-v1:", + "samples_per_type": 10 +} diff --git a/scripts/public_benchmarks/prepare.py b/scripts/public_benchmarks/prepare.py new file mode 100644 index 0000000..a6f8f0b --- /dev/null +++ b/scripts/public_benchmarks/prepare.py @@ -0,0 +1,65 @@ +"""Prepare pinned WCXB and AgentDocBench inputs; evaluation stays offline.""" +import argparse +import gzip +import io +import json +from pathlib import Path + +from common import CACHE, MAX_BYTES, PINS, ROOT, atomic_write, contract, fetch, parallel, safe_path, selected_wcxb, sha256 + + +def prepare(scope, destination): + destination = destination.resolve() + assets = json.loads((ROOT / "agentdocbench-assets.json").read_text()) + files = {} + + def wcxb(row): + base = f"https://raw.githubusercontent.com/Murrough-Foley/web-content-extraction-benchmark/{PINS['wcxb']}" + name = f"wcxb/{row['split']}/{row['id']}" + zipped = fetch(safe_path(destination, name + ".html.gz"), f"{base}/{row['split']}/html/{row['id']}.html.gz", row["html_sha1"], "git") + with gzip.GzipFile(fileobj=io.BytesIO(zipped.read_bytes())) as stream: + html = stream.read(MAX_BYTES + 1) + if len(html) > MAX_BYTES: + raise ValueError("decompressed HTML exceeds 50 MiB") + atomic_write(safe_path(destination, name + ".html"), html) + truth = fetch(safe_path(destination, name + ".json"), f"{base}/{row['split']}/ground-truth/{row['id']}.json", row["truth_sha1"], "git") + return {"suite": "wcxb", "id": row["id"], "category": row["category"], "format": "html", "split": row["split"], "ocr": False, + "full_text_reference": row["full_text_reference"], + "file": name + ".html", "truth": name + ".json"}, {name + ".html": sha256(html), name + ".json": sha256(truth.read_bytes())} + + def agentdoc(row): + name = "agentdocbench/" + row["file"] + fetch(safe_path(destination, name), row["mirror_url"], row["sha256"]) + digests = {name: row["sha256"]} + base = f"https://raw.githubusercontent.com/SylphxAI/anymd/{PINS['agentdocbench']}/bench/" + sample = {"suite": "agentdocbench", "id": row["id"], "category": row["category"], "format": row["format"], "ocr": row["ocr"], "file": name} + for field, source in [("truth", f"truth/{row['id']}.json"), ("reference", f"reference/{row['id']}.txt")]: + if source not in assets: + if field == "truth": + raise ValueError("document has no pinned ground truth") + continue + target = "agentdocbench/" + source + path = fetch(safe_path(destination, target), base + source, assets[source], "git") + sample[field] = target + digests[target] = sha256(path.read_bytes()) + return sample, digests + + prepared = parallel(wcxb, selected_wcxb(scope)) + prepared += parallel(agentdoc, json.loads((ROOT / "agentdocbench-corpus.json").read_text())["docs"]) + samples = [] + for sample, hashes in prepared: + samples.append(sample) + files.update(hashes) + manifest = {"scope": scope, "contract": contract(), "pins": PINS, "samples": samples, "files": files, + "excluded_dev_test_duplicates": 139} + path = destination / f"manifest-{scope}.json" + atomic_write(path, (json.dumps(manifest, indent=2) + "\n").encode()) + return path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scope", choices=["pr", "dev", "test"], default="pr") + parser.add_argument("--cache", type=Path, default=CACHE) + args = parser.parse_args() + print(prepare(args.scope, args.cache)) diff --git a/scripts/public_benchmarks/requirements.txt b/scripts/public_benchmarks/requirements.txt new file mode 100644 index 0000000..7d698ab --- /dev/null +++ b/scripts/public_benchmarks/requirements.txt @@ -0,0 +1,2 @@ +markdown-it-py==4.0.0 --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3 +mdurl==0.1.2 --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba diff --git a/scripts/public_benchmarks/run.py b/scripts/public_benchmarks/run.py new file mode 100644 index 0000000..79e7b85 --- /dev/null +++ b/scripts/public_benchmarks/run.py @@ -0,0 +1,205 @@ +"""Run public studies through Mira and gate canonical quality regressions. + +Decision: only the fixed development slice is a regression baseline. Full dev +runs must contain that entire slice; test runs enforce execution/coverage but +are report-only for quality, preserving the test split for milestone decisions. +Latency is reported, not gated across heterogeneous machines. +""" +import argparse +from collections import defaultdict +import json +import math +import os +from pathlib import Path +import platform +import shutil +import statistics +import subprocess +import sys + +from common import CACHE, REPO, ROOT, atomic_write, read_manifest, sha256 + + +def expected_cases(manifest): + result = {} + for row in manifest["samples"]: + modes = ["default", "main_content"] if row["suite"] == "wcxb" else ["default"] + for mode in modes: + key = f"{row['suite']}/{row['id']}" + (f"/{mode}" if row["suite"] == "wcxb" else "") + result[key] = row, mode + return result + + +def report_rows(report, manifest): + expected = expected_cases(manifest) + rows = [] + seen = set() + for case in report["cases"]: + suite = case["eval"].removeprefix("markkit_") + mode = case.get("params", {}).get("mode", "default") + key = f"{suite}/{case['sample']}" + (f"/{mode}" if suite == "wcxb" else "") + if key not in expected or key in seen or case["target"] != "markkit": + raise ValueError("unexpected or duplicate benchmark case") + seen.add(key) + transcript = case["transcript"] + scores = {} + for score in case["scores"]: + if score.get("na"): + continue + if not math.isfinite(score["value"]) or not 0 <= score["value"] <= 1: + raise ValueError("invalid quality metric") + scores[score["scorer"]] = score["value"] + required = "f1" if suite == "wcxb" else "document_score" + if scores.get("succeeded") != 1 or required not in scores or transcript.get("error"): + raise ValueError(f"failed conversion or absent required score: {key}") + row, mode = expected[key] + rows.append({"key": key, "suite": suite, "id": row["id"], "category": row["category"], "format": row["format"], + "ocr": row["ocr"], "mode": mode, "scores": {k: v for k, v in scores.items() if k != "succeeded"}, + "metrics": transcript.get("metrics", {})}) + if seen != set(expected): + raise ValueError(f"incomplete benchmark coverage: {len(seen)}/{len(expected)}") + return rows + + +def quality_vectors(rows): + vectors = {} + for mode in sorted({row["mode"] for row in rows if row["suite"] == "wcxb"}): + selected = [row for row in rows if row["suite"] == "wcxb" and row["mode"] == mode] + vectors[f"wcxb/{mode}/overall"] = statistics.mean(row["scores"]["f1"] for row in selected) + for kind in sorted({row["category"] for row in selected}): + vectors[f"wcxb/{mode}/type/{kind}"] = statistics.mean(row["scores"]["f1"] for row in selected if row["category"] == kind) + documents = [row for row in rows if row["suite"] == "agentdocbench"] + if documents: + categories = defaultdict(list) + no_ocr = defaultdict(list) + formats = defaultdict(list) + for row in documents: + score = row["scores"]["document_score"] + categories[row["category"]].append(score) + formats[row["format"]].append(score) + if not row["ocr"]: + no_ocr[row["category"]].append(score) + vectors["agentdocbench/overall"] = statistics.mean(statistics.mean(v) for v in categories.values()) + if no_ocr: + vectors["agentdocbench/no_ocr"] = statistics.mean(statistics.mean(v) for v in no_ocr.values()) + for kind, values in categories.items(): + vectors[f"agentdocbench/category/{kind}"] = statistics.mean(values) + for kind, values in formats.items(): + vectors[f"agentdocbench/format/{kind}"] = statistics.mean(values) + return vectors + + +def baseline_record(rows, contract): + return {"contract": contract, "cases": sorted(row["key"] for row in rows), "quality": quality_vectors(rows), "absolute_tolerance": 0.005} + + +def regressions(rows, baseline, contract): + if contract != baseline["contract"]: + return ["corpus/scorer contract changed; explicitly rebaseline the fixed PR sample"] + selected = [row for row in rows if row["key"] in baseline["cases"]] + if sorted(row["key"] for row in selected) != baseline["cases"]: + return ["run is missing baseline cases"] + current = quality_vectors(selected) + tolerance = baseline["absolute_tolerance"] + if (not math.isfinite(tolerance) or not 0 <= tolerance <= 0.005 or not baseline["quality"] or not baseline["cases"] + or not all(math.isfinite(value) and 0 <= value <= 1 for value in baseline["quality"].values())): + return ["invalid committed baseline policy"] + return [f"{key}: {current.get(key, -1):.6f} < baseline {value:.6f} minus {tolerance}" + for key, value in baseline["quality"].items() if current.get(key, -1) + tolerance + 1e-12 < value] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scope", choices=["pr", "dev", "test"], default="pr") + parser.add_argument("--cache", type=Path, default=CACHE) + parser.add_argument("--mira", default=shutil.which("mira") or str(CACHE / "bin/mira")) + parser.add_argument("--out", type=Path) + parser.add_argument("--write-baseline", action="store_true", help="Explicitly replace the committed PR baseline after review") + args = parser.parse_args() + if args.write_baseline and args.scope != "pr": + parser.error("only the fixed PR sample can become the regression baseline") + # Build here as well as in the convenience recipe: a caller cannot silently + # label an arbitrary/debug executable as a release benchmark. + subprocess.run(["cargo", "build", "--locked", "--release", "-p", "markkit-cli"], cwd=REPO, check=True) + metadata = json.loads(subprocess.check_output(["cargo", "metadata", "--format-version", "1", "--no-deps"], cwd=REPO)) + binary = Path(metadata["target_directory"]) / "release/markkit" + mira_version = subprocess.check_output([args.mira, "--version"], text=True).strip() + if mira_version != "mira 0.5.0": + raise ValueError("use the pinned Mira 0.5.0 host (install_mira.py)") + manifest_path = args.cache.resolve() / f"manifest-{args.scope}.json" + manifest = read_manifest(manifest_path) + if manifest["scope"] != args.scope: + raise ValueError("prepared selection does not match requested scope") + output = (args.out or args.cache / f"report-{args.scope}").resolve() + output.mkdir(parents=True, exist_ok=True) + provenance = {"source_revision": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), + "source_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=REPO, text=True)), + "build_profile": "release", "platform": platform.platform(), "machine": platform.machine(), + "cpu_count": os.cpu_count(), "python": platform.python_version(), + "binary_sha256": sha256(binary.read_bytes()), "pins": manifest["pins"], "mira_version": mira_version, + "rustc_version": subprocess.check_output(["rustc", "--version"], text=True).strip(), + "core_diff_sha256": sha256(subprocess.check_output(["git", "diff", "HEAD", "--", "crates/markkit", "crates/markkit-cli", "Cargo.lock"], cwd=REPO)), + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup"} + env = dict(os.environ, MARKKIT_BIN=str(binary), MARKKIT_PUBLIC_MANIFEST=str(manifest_path), + MARKKIT_PUBLIC_PROVENANCE=json.dumps(provenance)) + env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env.get("PATH", "") + command = [args.mira, "run", "--study-python", "scripts/public_benchmarks/study.py", "--max-concurrent", "1", + "--max-retries", "0", "--timeout", "300", "--group-by", "category", "--format", "json", "--out", str(output / "mira.json")] + provenance["command"] = command + # A failed new run must not upload a previous successful report from this + # output directory. Retain current provenance/notices even if Mira fails. + for name in ["mira.json", "summary.json", "summary.md"]: + (output / name).unlink(missing_ok=True) + atomic_write(output / "provenance.json", (json.dumps(provenance, indent=2) + "\n").encode()) + atomic_write(output / "manifest.json", manifest_path.read_bytes()) + for name in ["CORPUS-NOTICE", "agentdocbench-corpus.json"]: + atomic_write(output / name, (ROOT / name).read_bytes()) + atomic_write(output / "WCXB-NOTICE.txt", ("WCXB by Murrough Foley, CC-BY-4.0. Markkit outputs are conversions of the pinned HTML.\n" + f"https://github.com/Murrough-Foley/web-content-extraction-benchmark/tree/{manifest['pins']['wcxb']}\n" + "Source URLs and annotations are in the upstream ground-truth/.json files for the recorded split.\n" + + (ROOT / "vendor/wcxb/LICENSE").read_text()).encode()) + + def failed(reason): + atomic_write(output / "summary.json", (json.dumps({"status": "failed", "scope": args.scope, "reason": reason, "provenance": provenance}, indent=2) + "\n").encode()) + atomic_write(output / "summary.md", f"# Public benchmarks ({args.scope})\n\nExecution/coverage failed: {reason}\n".encode()) + + process = subprocess.run(command, cwd=REPO, env=env) + if process.returncode: + failed(f"Mira exited {process.returncode}") + raise SystemExit(process.returncode) + report = json.loads((output / "mira.json").read_text()) + try: + rows = report_rows(report, manifest) + except ValueError as error: + failed(str(error)) + raise + quality = quality_vectors(rows) + baseline_path = ROOT / "baseline.json" + if args.write_baseline: + baseline = {**baseline_record(rows, manifest["contract"]), "provenance": provenance} + atomic_write(baseline_path, (json.dumps(baseline, indent=2, sort_keys=True) + "\n").encode()) + failures = [] if args.scope == "test" else regressions(rows, json.loads(baseline_path.read_text()), manifest["contract"]) + summary = {"scope": args.scope, "case_count": len(rows), "quality": quality, "provenance": provenance, + "manifest_sha256": sha256(manifest_path.read_bytes()), "regressions": failures, + "test_quality_report_only": args.scope == "test", "cases": rows, + "wcxb_empty_references": [row["id"] for row in manifest["samples"] if row["suite"] == "wcxb" and not row["full_text_reference"]]} + atomic_write(output / "summary.json", (json.dumps(summary, indent=2, sort_keys=True) + "\n").encode()) + lines = [f"# Public benchmarks ({args.scope})", "", f"{len(rows)} cases; {len(failures)} quality regressions.", "", "| Metric | Score |", "|---|---:|"] + lines += [f"| {key} | {value:.4f} |" for key, value in sorted(quality.items())] + if summary["wcxb_empty_references"]: + lines += ["", f"WCXB empty/absent full-text references: {len(summary['wcxb_empty_references'])} pages. " + "Upstream empty-reference F1 behavior is preserved; these cases remain in the denominator and snippet scores still apply."] + lines += ["", "## Conversion timing", "", "Median of three fresh processes after one warm-up, sequential, startup included.", "", "| Suite / mode | Median per-document ms | Documents |", "|---|---:|---:|"] + for suite, mode in sorted({(row["suite"], row["mode"]) for row in rows}): + timings = [row["metrics"]["conversion_median_ms"] for row in rows if row["suite"] == suite and row["mode"] == mode] + lines.append(f"| {suite} / {mode} | {statistics.median(timings):.3f} | {len(timings)} |") + if failures: + lines += ["", "## Regressions", ""] + [f"- {failure}" for failure in failures] + atomic_write(output / "summary.md", ("\n".join(lines) + "\n").encode()) + print("\n".join(lines)) + if failures: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/public_benchmarks/study.py b/scripts/public_benchmarks/study.py new file mode 100644 index 0000000..13b15b4 --- /dev/null +++ b/scripts/public_benchmarks/study.py @@ -0,0 +1,162 @@ +"""Public quality studies; exact upstream scorers, Markkit CLI as subject. + +Quality scorers expose continuous values; pass means metric validity, not 100% +quality. The runner enforces baseline regressions after canonical aggregation. +Timing is the median of three fresh processes after one untimed warm-up; input +downloads, study/scorer work and report persistence are outside that boundary. +""" +from functools import lru_cache +from html.parser import HTMLParser +import importlib.util +import json +import os +from pathlib import Path +import shutil +import statistics +import subprocess +import sys +import time + +from common import CACHE, REPO, ROOT, read_manifest, safe_path, sha256 + +sys.path.insert(0, str(ROOT / "vendor")) +import mira + + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +WCXB = module("wcxb", ROOT / "vendor/wcxb/evaluate.py") +ADB = module("agentdocbench", ROOT / "vendor/agentdocbench/score.py") + + +class VisibleText(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + + def handle_data(self, value): + self.parts.append(value) + + def handle_starttag(self, tag, attrs): + if tag == "br": + self.parts.append("\n") + + def handle_endtag(self, tag): + if tag in {"p", "div", "li", "td", "th", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "pre"}: + self.parts.append("\n") + + +def plain(markdown): + from markdown_it import MarkdownIt + parser = VisibleText() + parser.feed(MarkdownIt("commonmark").enable("table").render(markdown)) + return "".join(parser.parts) + + +def values(row, markdown, cache): + truth = json.loads(safe_path(cache, row["truth"]).read_text()) + if row["suite"] == "wcxb": + gt = truth["ground_truth"] + text = plain(markdown) + # Match upstream load_ground_truth: empty/absent full-text labels remain + # in the denominator with the empty reference; snippets still apply. + p, r, f1 = WCXB.word_f1(text, gt.get("main_content", "") or "") + result = {"precision": p, "recall": r, "f1": f1} + if gt.get("with"): + result["with_recall"] = WCXB.snippet_check(text, gt["with"]) + if gt.get("without"): + result["boilerplate_exclusion"] = 1 - WCXB.snippet_check(text, gt["without"]) + return result + reference = safe_path(cache, row["reference"]).read_text() if row.get("reference") else "" + scores = ADB.score(row["id"], markdown, truth=truth, reference=reference) + result = {"document_score": scores["score"]} + if "sentences" in scores: + found, total = scores["sentences"] + result["sentences"] = found / total + if "text" in scores: + result["text_f1"] = scores["text"]["f1"] + if "order" in scores: + result["order"] = scores["order"] + if "tables" in scores: + result["table_cells_f1"] = scores["tables"]["cell_f1"] + return result + + +def extract(binary, path, mode): + command = [str(binary), "extract", "--file", str(path)] + if mode == "main_content": + command.append("--main-content") + elif mode != "default": + raise ValueError("unknown extraction mode") + durations = [] + output = None + for iteration in range(4): + start = time.perf_counter_ns() + try: + process = subprocess.run(command, capture_output=True, text=True, timeout=60) + except subprocess.TimeoutExpired: + return mira.transcript(error="conversion exceeded 60 seconds", error_kind="subject") + if process.returncode: + return mira.transcript(error=f"conversion exited {process.returncode}", error_kind="subject") + if output is not None and process.stdout != output: + return mira.transcript(error="non-deterministic conversion output", error_kind="subject") + output = process.stdout + if iteration: + durations.append(time.perf_counter_ns() - start) + median_ns = statistics.median(durations) + return mira.transcript(output, timing=mira.Timing(duration_ms=int(median_ns / 1_000_000)), + metrics={"conversion_median_ms": median_ns / 1_000_000, "conversion_min_ms": min(durations) / 1_000_000, + "conversion_max_ms": max(durations) / 1_000_000, "input_bytes": float(path.stat().st_size), + "output_bytes": float(len(output.encode())), "timed_processes": 3.0}) + + +def build_study(manifest_path, binary, provenance): + manifest = read_manifest(manifest_path) + cache = manifest_path.parent + if not binary.is_file() or not os.access(binary, os.X_OK): + raise ValueError("build Markkit CLI before running public benchmarks") + binary_hash = sha256(binary.read_bytes()) + study = mira.Study("markkit-public-benchmarks", version="1") + + @lru_cache(maxsize=8) + def cached_values(row_json, markdown): + return values(json.loads(row_json), markdown, cache) + + def scorer(name): + def score(sample, transcript): + row = sample.metadata["document"] + value = cached_values(json.dumps(row, sort_keys=True), transcript.final_response).get(name) + if value is None: + return mira.make_score(name, 0, False, "metric not applicable", na=True) + if transcript.error: + value = 0.0 + return mira.make_score(name, value, 0 <= value <= 1, "continuous quality; runner gates canonical baseline") + return mira.scorer(name, score) + + def subject(sample, cx): + row = sample.metadata["document"] + result = extract(binary, safe_path(cache, row["file"]), cx.param("mode", "default")) + result.metadata = {**provenance, "binary_sha256": binary_hash, "corpus_contract": manifest["contract"], "scope": manifest["scope"]} + return result + + for suite, names in [("wcxb", ["f1", "precision", "recall", "with_recall", "boilerplate_exclusion"]), + ("agentdocbench", ["document_score", "sentences", "text_f1", "order", "table_cells_f1"])]: + samples = [mira.Sample(row["id"], tags=[suite, row["category"], "ocr" if row["ocr"] else "no_ocr"], + metadata={"document": row, "category": row["category"], "format": row["format"], "ocr": row["ocr"]}) + for row in manifest["samples"] if row["suite"] == suite] + study.add(mira.Eval(name="markkit_" + suite, subject=subject, samples=samples, + targets=[mira.target("markkit")], scorers=[mira.succeeded()] + [scorer(name) for name in names], + axes=[mira.axis("mode", ["default", "main_content"])] if suite == "wcxb" else [])) + return study + + +if __name__ == "__main__": + manifest = Path(os.environ.get("MARKKIT_PUBLIC_MANIFEST", CACHE / "manifest-pr.json")).resolve() + binary = Path(os.environ.get("MARKKIT_BIN", REPO / "target/release/markkit")).resolve() + provenance = json.loads(os.environ.get("MARKKIT_PUBLIC_PROVENANCE", "{}")) + build_study(manifest, binary, provenance).serve() diff --git a/scripts/public_benchmarks/vendor/agentdocbench/LICENSE b/scripts/public_benchmarks/vendor/agentdocbench/LICENSE new file mode 100644 index 0000000..cf59ec5 --- /dev/null +++ b/scripts/public_benchmarks/vendor/agentdocbench/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 SylphxAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/scripts/public_benchmarks/vendor/agentdocbench/score.py b/scripts/public_benchmarks/vendor/agentdocbench/score.py new file mode 100644 index 0000000..86f7664 --- /dev/null +++ b/scripts/public_benchmarks/vendor/agentdocbench/score.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""AgentDocBench scoring: compare one Markdown output with one ground-truth file. + +The method is documented in bench/README.md. In short, every document yields +up to four scores in [0, 1]: + + sentences share of key sentences found verbatim (after normalization) + text_f1 bag-of-words F1 against the reference text (bench/reference/) + order longest in-order chain of the reading-order strings / count + tables cell-level F1 (precision only on tables marked complete) + +and the document score is their mean. Run it directly to rescore saved +outputs without converting again: + + python bench/score.py --outputs DIR --results bench/results/anymd.json +""" + +import argparse +import html +import json +import re +import unicodedata +from collections import Counter +from html.parser import HTMLParser +from pathlib import Path + +HERE = Path(__file__).resolve().parent + +CJK = "぀-ヿ㐀-䶿一-鿿豈-﫿가-힯　-〿＀-￯" +TOKEN = re.compile(rf"[{CJK}]|[^\W_{CJK}]+") +QUOTES = str.maketrans({"“": '"', "”": '"', "„": '"', "‟": '"', "‘": "'", "’": "'", "‚": "'", "‛": "'"}) + + +def normalize(text): + """Markdown/HTML-insensitive, whitespace-insensitive text for matching.""" + text = unicodedata.normalize("NFKC", text) + text = re.sub(r"", r" \1 ", text, flags=re.S) # comments are read by agents too + target = r"\((?:[^()\s]|\([^()\s]*\))*(?:\s+\"[^\"]*\")?\)" # (url) with one level of (parens) + text = re.sub(r"!\[([^\]]*)\]" + target, r"\1", text) # images -> alt text + text = re.sub(r"\[([^\]]*)\]" + target, r"\1", text) # links -> link text + text = re.sub(r"]*>", " ", text) # HTML tags + text = html.unescape(text) + text = re.sub(r"\\([\\`*_{}\[\]()#+\-.!|<>~])", r"\1", text) # Markdown escapes + text = re.sub(r"[#*_`|>~]", " ", text) + text = text.translate(QUOTES).replace("­", "") + return re.sub(r"\s+", " ", text).strip() + + +def squash(text): + return re.sub(r"\s+", "", normalize(text)).lower() + + +def tokens(text): + return TOKEN.findall(normalize(text).lower()) + + +# --- sentences --------------------------------------------------------------- + + +HAS_CJK = re.compile(rf"[{CJK}]") + + +def contains(flat, dense, needle): + """Needle in the normalized output. CJK text has no word spaces, and extractors + scatter spaces around its digits and brackets, so CJK needles ignore whitespace.""" + needle = normalize(needle) + if HAS_CJK.search(needle): + return re.sub(r"\s", "", needle) in dense + return needle in flat + + +def sentence_score(truth, flat, dense): + wanted = truth.get("sentences", []) + found = [s for s in wanted if contains(flat, dense, s)] + return len(found), len(wanted) + + +# --- bag-of-words F1 --------------------------------------------------------- + + +def text_f1(reference, markdown): + ref, out = Counter(tokens(reference)), Counter(tokens(markdown)) + overlap = sum((ref & out).values()) + if not overlap: + return 0.0, 0.0, 0.0 + precision = overlap / sum(out.values()) + recall = overlap / sum(ref.values()) + return precision, recall, 2 * precision * recall / (precision + recall) + + +# --- reading order ----------------------------------------------------------- + + +def occurrences(haystack, needle, limit=200): + spots, start = [], 0 + while len(spots) < limit: + found = haystack.find(needle, start) + if found < 0: + break + spots.append(found) + start = found + 1 + return spots + + +def order_score(truth, dense): + """Longest chain of order strings whose positions strictly increase, / n. + Positions are taken in the whitespace-free output: this measures order only.""" + needles = [re.sub(r"\s", "", normalize(s)) for s in truth.get("order", [])] + if not needles: + return None + best = [] # best[i] = list of (position, chain length ending here) + top = 0 + for i, needle in enumerate(needles): + here = [] + for pos in occurrences(dense, needle): + length = 1 + for j in range(i): + for prev_pos, prev_len in best[j]: + if prev_pos < pos and prev_len + 1 > length: + length = prev_len + 1 + here.append((pos, length)) + top = max(top, length) + best.append(here) + return top / len(needles) + + +# --- tables ------------------------------------------------------------------ + + +class _HtmlTables(HTMLParser): + def __init__(self): + super().__init__() + self.tables, self._rows, self._cell, self._depth = [], None, None, 0 + + def handle_starttag(self, tag, attrs): + if tag == "table": + self._depth += 1 + if self._depth == 1: + self._rows = [] + elif tag == "tr" and self._rows is not None: + self._rows.append([]) + elif tag in ("td", "th") and self._rows is not None: + self._cell = [] + elif tag == "br" and self._cell is not None: + self._cell.append(" ") + + def handle_endtag(self, tag): + if tag in ("td", "th") and self._cell is not None and self._rows: + self._rows[-1].append("".join(self._cell)) + self._cell = None + elif tag == "table": + self._depth -= 1 + if self._depth == 0 and self._rows is not None: + self.tables.append([r for r in self._rows if r]) + self._rows = None + + def handle_data(self, data): + if self._cell is not None: + self._cell.append(data) + + +def parse_tables(markdown): + """Pipe tables and HTML tables in the output, as lists of rows of squashed cells.""" + tables, current = [], [] + for line in markdown.splitlines(): + line = line.strip() + if line.startswith("|") and line.count("|") >= 2: + if re.fullmatch(r"\|[\s:|+-]*\|?", line): + continue + body = line[1:-1] if line.endswith("|") else line[1:] + current.append(re.split(r"(? places: + return f"{float(got.replace(',', '')):,.{places}f}".replace(",", "") == want.replace(",", "") + if not got.startswith(want) and want not in got: + return False + head, _, tail = got.partition(want) + if re.search(r"[^\W\d_]", want): + return (not head or bool(ROW_NUMBER.fullmatch(head))) and (not tail or bool(MARKER.fullmatch(tail))) + return not head and bool(LETTER_MARKER.fullmatch(tail)) + + +def lcs(a, b): + """Length of the longest common subsequence of two cell lists (cell_eq cells).""" + if not a or not b: + return 0 + prev = [0] * (len(b) + 1) + for x in a: + cur = [0] + for j, y in enumerate(b): + cur.append(prev[j] + 1 if cell_eq(x, y) else max(prev[j + 1], cur[j])) + prev = cur + return prev[-1] + + +def table_score(truth, markdown): + tables = truth.get("tables", []) + if not tables: + return None + out = parse_tables(markdown) + flat_rows = [(ti, row) for ti, table in enumerate(out) for row in table] + rows_total = rows_found = cells_total = cells_hit = 0 + prec_hit = prec_den = 0 + for table in tables: + used_tables, credit = set(), {} + for row in table["rows"]: + want = [squash(c) for c in row] + want = [c for c in want if c] + if not want: + continue + rows_total += 1 + cells_total += len(want) + best, best_row = 0, None + for index, (_, cells) in enumerate(flat_rows): + if len(cells) < best: + continue + hit = lcs(want, cells) + if hit > best: + best, best_row = hit, index + if hit == len(want): + break + cells_hit += best + if best == len(want): + rows_found += 1 + if best_row is not None and best * 2 >= len(want): + used_tables.add(flat_rows[best_row][0]) + credit[best_row] = max(credit.get(best_row, 0), best) + if table.get("complete"): + # an output row matched by several truth rows is credited once + prec_hit += sum(hit for index, hit in credit.items() if flat_rows[index][0] in used_tables) + prec_den += sum(len(r) for ti in used_tables for r in out[ti]) + recall = cells_hit / cells_total if cells_total else 0.0 + precision = prec_hit / prec_den if prec_den else (None if not any(t.get("complete") for t in tables) else 0.0) + if precision is None: + f1 = recall + else: + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + return { + "rows_found": rows_found, + "rows_total": rows_total, + "cells_hit": cells_hit, + "cells_total": cells_total, + "cell_recall": round(recall, 4), + "cell_precision": None if precision is None else round(precision, 4), + "cell_f1": round(f1, 4), + } + + +# --- per document ------------------------------------------------------------ + + +def load_truth(doc_id): + path = HERE / "truth" / f"{doc_id}.json" + return json.loads(path.read_text("utf-8")) if path.exists() else {} + + +def load_reference(doc_id): + path = HERE / "reference" / f"{doc_id}.txt" + return path.read_text("utf-8") if path.exists() else None + + +def score(doc_id, markdown, truth=None, reference=None): + truth = load_truth(doc_id) if truth is None else truth + reference = load_reference(doc_id) if reference is None else reference + flat = normalize(markdown) + dense = re.sub(r"\s", "", flat) + metrics, parts = {}, [] + found, total = sentence_score(truth, flat, dense) + if total: + metrics["sentences"] = [found, total] + parts.append(found / total) + if reference: + precision, recall, f1 = text_f1(reference, markdown) + metrics["text"] = {"precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)} + parts.append(f1) + order = order_score(truth, dense) + if order is not None: + metrics["order"] = round(order, 4) + parts.append(order) + tables = table_score(truth, markdown) + if tables is not None: + metrics["tables"] = tables + parts.append(tables["cell_f1"]) + metrics["score"] = round(sum(parts) / len(parts), 4) if parts else None + return metrics + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--outputs", required=True, help="directory of .md files saved by run.py") + parser.add_argument("--results", required=True, help="results JSON to rescore in place") + args = parser.parse_args() + data = json.loads(Path(args.results).read_text()) + for row in data["results"]: + if row.get("status") != "ok": + continue + md = Path(args.outputs) / f"{row['doc']}.md" + if md.exists(): + row["metrics"] = score(row["doc"], md.read_text("utf-8")) + Path(args.results).write_text(json.dumps(data, indent=1, ensure_ascii=False) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/public_benchmarks/vendor/mira/LICENSE b/scripts/public_benchmarks/vendor/mira/LICENSE new file mode 100644 index 0000000..4cec11c --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Everruns and Mira contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scripts/public_benchmarks/vendor/mira/__init__.py b/scripts/public_benchmarks/vendor/mira/__init__.py new file mode 100644 index 0000000..9a1f720 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/__init__.py @@ -0,0 +1,143 @@ +"""Mira eval SDK for Python. + +Author an eval *study* in Python and run it with the `mira` host CLI — no Rust +dependency, just the protocol (newline-delimited JSON over stdio). The wire +types in `mira._wire` are generated from the canonical JSON Schema under +`schema/v1/`, so they never drift from the Rust host. + + import mira + + study = mira.Study("my-evals", version="0.1.0") + + @study.eval( + samples=[mira.Sample("hi", prompt="Say hi and the answer to life.")], + targets=[mira.target("sim")], + scorers=[mira.succeeded(), mira.contains("42")], + ) + def greet(sample, cx): + return mira.transcript(f"Hi! The answer is 42. ({sample.text})") + + if __name__ == "__main__": + study.serve() +""" +from __future__ import annotations + +from typing import Any, Dict, Optional + +from . import _wire +from ._serve import ( + PROTOCOL_VERSION, + Eval, + Target, + RunCx, + Sample, + Study, + log, + target, + serve, +) +from ._wire import AxisInfo, Score, Timing, Trajectory, Transcript, Usage +from .trajectory import ATIF_VERSION, from_trajectory, parse_trajectory +from .scorers import ( + Scorer, + all_of, + any_of, + contains, + cost_within, + equals, + file_contains, + file_exists, + json_field_equals, + json_valid, + latency_within, + make_score, + matches_expected, + metric_at_least, + metric_within, + non_empty, + not_, + not_contains, + observation_contains, + output_tokens_within, + produced_modality, + regex, + scorer, + steps_within, + succeeded, + tokens_within, + tool_arg_matches, + tool_called, + tool_called_before, + tool_called_with, + tool_calls_within, + tool_not_called, + tools_used_exactly, + ttft_within, + turns_within, +) + + +def transcript(final_response: str = "", **kwargs: Any) -> Transcript: + """Convenience builder for a `Transcript` (subjects' return value).""" + return Transcript(final_response=final_response, **kwargs) + + +def axis(name: str, values) -> AxisInfo: + """Declare an extra matrix axis (crossed with the target matrix).""" + return AxisInfo(name=name, values=list(values)) + + +__all__ = [ + "ATIF_VERSION", + "PROTOCOL_VERSION", + "AxisInfo", + "Eval", + "Target", + "RunCx", + "Sample", + "Score", + "Scorer", + "Study", + "Timing", + "Trajectory", + "Transcript", + "Usage", + "axis", + "all_of", + "any_of", + "contains", + "cost_within", + "equals", + "file_contains", + "file_exists", + "from_trajectory", + "json_field_equals", + "json_valid", + "latency_within", + "log", + "make_score", + "matches_expected", + "metric_at_least", + "metric_within", + "non_empty", + "not_", + "not_contains", + "output_tokens_within", + "parse_trajectory", + "produced_modality", + "target", + "regex", + "scorer", + "serve", + "succeeded", + "tokens_within", + "tool_called", + "tool_called_before", + "tool_calls_within", + "tool_not_called", + "tools_used_exactly", + "ttft_within", + "turns_within", + "transcript", + "_wire", +] diff --git a/scripts/public_benchmarks/vendor/mira/_codec.py b/scripts/public_benchmarks/vendor/mira/_codec.py new file mode 100644 index 0000000..ff5dfe4 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/_codec.py @@ -0,0 +1,93 @@ +"""Generic JSON <-> dataclass codec for the generated wire types. + +Two contracts from docs/protocol.md drive this: +- **Ignore unknown fields** on decode (forward compatibility): a newer host may + send fields this build doesn't know. +- **Omit empty optionals** on encode, mirroring the Rust `skip_serializing_if`, + so a Python study's lines look like the Rust host's. +""" +from __future__ import annotations + +import dataclasses +import typing +from typing import Any, get_args, get_origin + +# Cache resolved type hints per class (get_type_hints is not cheap). +_HINTS: dict[type, dict[str, Any]] = {} + + +def _hints(cls: type) -> dict[str, Any]: + if cls not in _HINTS: + _HINTS[cls] = typing.get_type_hints(cls) + return _HINTS[cls] + + +def _wire_name(f: dataclasses.Field) -> str: + return f.metadata.get("wire", f.name) + + +def _is_empty(v: Any) -> bool: + return v is None or v == "" or v == [] or v == {} + + +def _is_default_instance(v: Any) -> bool: + """A nested dataclass still equal to its default construction (e.g. an + untouched `Usage()`/`Timing()`). Optional fields holding one are dropped on + encode, mirroring the Rust `skip_serializing_if` for such fields.""" + return dataclasses.is_dataclass(v) and not isinstance(v, type) and v == type(v)() + + +def to_dict(obj: Any) -> Any: + """Dataclass -> JSON-able dict, dropping empty non-required fields.""" + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + required = getattr(type(obj), "__required__", ()) + out: dict[str, Any] = {} + for f in dataclasses.fields(obj): + wire = _wire_name(f) + v = getattr(obj, f.name) + if wire in required or not (_is_empty(v) or _is_default_instance(v)): + out[wire] = to_dict(v) + return out + if isinstance(obj, list): + return [to_dict(v) for v in obj] + if isinstance(obj, dict): + return {k: to_dict(v) for k, v in obj.items()} + return obj + + +def from_dict(cls: type, data: Any) -> Any: + """JSON dict -> dataclass instance, ignoring unknown keys.""" + if not (dataclasses.is_dataclass(cls) and isinstance(data, dict)): + return data + hints = _hints(cls) + by_wire = {_wire_name(f): f for f in dataclasses.fields(cls)} + kwargs: dict[str, Any] = {} + for key, value in data.items(): + f = by_wire.get(key) + if f is None: # unknown field — forward compatibility + continue + kwargs[f.name] = _decode(hints[f.name], value) + return cls(**kwargs) + + +def _decode(ann: Any, value: Any) -> Any: + if value is None: + return None + origin = get_origin(ann) + if origin in (list, typing.List): + (item,) = get_args(ann) or (Any,) + return [_decode(item, v) for v in value] + if origin in (dict, typing.Dict): + args = get_args(ann) + vtype = args[1] if len(args) == 2 else Any + return {k: _decode(vtype, v) for k, v in value.items()} + if origin is typing.Union: # Optional[X] -> first non-None arg + for arg in get_args(ann): + if arg is not type(None): + return _decode(arg, value) + return value + if origin is typing.Literal: + return value + if dataclasses.is_dataclass(ann): + return from_dict(ann, value) + return value diff --git a/scripts/public_benchmarks/vendor/mira/_meta.py b/scripts/public_benchmarks/vendor/mira/_meta.py new file mode 100644 index 0000000..09af043 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/_meta.py @@ -0,0 +1,11 @@ +"""Protocol version, methods, and capability tokens — GENERATED, do not edit. + +Regenerate with `python3 codegen.py` from schema/v1/meta.json. CI runs +`codegen.py --check` to fail on drift. +""" + +PROTOCOL_VERSION = "1.1" +MIN_PROTOCOL_VERSION = "1.0" + +METHODS = ("initialize", "list", "list_samples", "run", "execute", "score", "cancel") +CAPABILITIES = ("axes", "events", "usage", "execute", "score", "trials", "cancel", "paginate", "trajectory") diff --git a/scripts/public_benchmarks/vendor/mira/_serve.py b/scripts/public_benchmarks/vendor/mira/_serve.py new file mode 100644 index 0000000..899fc10 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/_serve.py @@ -0,0 +1,356 @@ +"""The study side of the protocol: an eval registry plus the stdio serve loop. + +A `Study` answers `initialize`/`list`/`run`/`execute`/`score` over +newline-delimited JSON on stdio (see docs/protocol.md). stdout carries only +protocol JSON; logs go to stderr. +""" +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Sequence + +from . import _codec +from ._meta import PROTOCOL_VERSION +from ._wire import ( + AxisInfo, + CancelResult, + EvalInfo, + ExecuteResult, + InitializeResult, + ListResult, + ListSamplesResult, + TargetInfo, + RunResult, + SampleInfo, + Score, + Transcript, + TranscriptSummary, +) +from .scorers import Scorer, make_score +from .trajectory import ATIF_FORMAT, ATIF_VERSION, normalize as _normalize_trajectory + +# PROTOCOL_VERSION is imported from the generated `_meta` (above) — derived from +# schema/v1/meta.json, not hardcoded, so a version bump can't leave it stale. + +# JSON-RPC error codes for the structured `error` object (mirrors the Rust +# `protocol::codes`). All caller mistakes here are non-retryable. +_CODE_METHOD_NOT_FOUND = -32601 +_CODE_INVALID_PARAMS = -32602 +_CODE_INTERNAL_ERROR = -32603 + +# The protocol methods this SDK dispatches in `Study.handle`. Kept explicit so a +# test can assert it covers every method in the generated `_meta.METHODS` — a new +# protocol method then fails CI until the serve loop handles it. +HANDLED_METHODS = ( + "initialize", "list", "list_samples", "run", "execute", "score", "cancel", +) + +# Samples-per-page when paginating `list`. Small studies fit in one page (`list` +# enumerates every sample inline); a huge/lazy dataset is chunked across `list` + +# `list_samples` rather than enumerated in one giant line. Mirrors the Rust +# `DEFAULT_PAGE_SIZE`. +DEFAULT_PAGE_SIZE = 500 + + +# ----- authoring types -------------------------------------------------------- + +@dataclass +class Sample: + """One dataset row. `prompt` is convenience for a single input turn; `input` + holds multi-turn input. `text` joins them for the subject.""" + + id: str + prompt: Optional[str] = None + input: List[str] = field(default_factory=list) + tags: List[str] = field(default_factory=list) + expected: Optional[str] = None + files: Dict[str, str] = field(default_factory=dict) + metadata: Dict[str, Any] = field(default_factory=dict) + + @property + def text(self) -> str: + if self.prompt is not None: + return self.prompt + return "\n".join(self.input) + + +@dataclass +class Target: + label: str + provider: str = "" + available: bool = True + metadata: Dict[str, Any] = field(default_factory=dict) + + +def target( + label: str, + provider: str = "", + available: bool = True, + metadata: Optional[Dict[str, Any]] = None, +) -> Target: + return Target(label=label, provider=provider, available=available, + metadata=dict(metadata or {})) + + +@dataclass +class RunCx: + """Per-case context handed to a subject: the matrix target, turn budget, and + chosen axis values.""" + + target: str + provider: str = "" + max_turns: int = 0 + params: Dict[str, str] = field(default_factory=dict) + + def param(self, name: str, default: str = "") -> str: + return self.params.get(name, default) + + +Subject = Callable[[Sample, RunCx], Transcript] + + +@dataclass +class Eval: + name: str + subject: Subject + samples: Sequence[Sample] + targets: Sequence[Target] + scorers: Sequence[Scorer] = () + description: str = "" + axes: Sequence[AxisInfo] = () + max_turns: int = 0 + metadata: Dict[str, Any] = field(default_factory=dict) + + def info(self) -> EvalInfo: + return EvalInfo( + name=self.name, + description=self.description, + samples=[SampleInfo(id=s.id, tags=list(s.tags), metadata=dict(s.metadata)) + for s in self.samples], + scorers=[sc.name for sc in self.scorers], + targets=[TargetInfo(label=m.label, provider=m.provider, available=m.available, + metadata=dict(m.metadata)) + for m in self.targets], + axes=[AxisInfo(name=a.name, values=list(a.values)) for a in self.axes], + max_turns=self.max_turns, + metadata=dict(self.metadata), + ) + + def _sample(self, sid: str) -> Sample: + for s in self.samples: + if s.id == sid: + return s + raise ValueError(f"no such sample: {sid}") + + def _target(self, label: str) -> Target: + for m in self.targets: + if m.label == label: + return m + return Target(label=label) + + +# ----- scoring (mirrors crate::runner) ---------------------------------------- + +def _score_transcript(ev: Eval, sample: Sample, transcript: Transcript) -> List[Score]: + # Infra failure short-circuits to a single N/A, like score_transcript(). + if transcript.error is not None and transcript.error_kind == "infra": + return [make_score("infra", 0.0, False, transcript.error, na=True)] + return [sc.score(sample, transcript) for sc in ev.scorers] + + +def _verdict(scores: List[Score]) -> bool: + applicable = [s for s in scores if not s.na] + return bool(applicable) and all(s.pass_ for s in applicable) + + +def _aggregate(scores: List[Score]) -> float: + applicable = [s.value for s in scores if not s.na] + return sum(applicable) / len(applicable) if applicable else 0.0 + + +# ----- study + serve loop ----------------------------------------------------- + +class Study: + def __init__(self, name: str, version: Optional[str] = None, + page_size: Optional[int] = DEFAULT_PAGE_SIZE) -> None: + self.name = name + self.version = version + # Max samples per `list`/`list_samples` page. None (or <= 0) disables + # pagination: every sample is enumerated inline in `list`. + self.page_size = page_size if (page_size or 0) > 0 else None + self._evals: Dict[str, Eval] = {} + + def add(self, ev: Eval) -> None: + self._evals[ev.name] = ev + + def eval(self, **kw) -> Callable[[Subject], Subject]: + """Decorator: register a subject function as an eval.""" + def deco(fn: Subject) -> Subject: + self.add(Eval(name=kw.get("name", fn.__name__), subject=fn, + samples=kw["samples"], targets=kw["targets"], + scorers=kw.get("scorers", ()), description=kw.get("description", ""), + axes=kw.get("axes", ()), max_turns=kw.get("max_turns", 0), + metadata=kw.get("metadata", {}))) + return fn + return deco + + def _capabilities(self) -> List[str]: + caps = ["usage", "execute", "score", "paginate", "trajectory"] + if any(ev.axes for ev in self._evals.values()): + caps.insert(0, "axes") + return caps + + def _sample_page(self, ev: Eval, offset: int) -> tuple[List[SampleInfo], Optional[str]]: + """One page of `ev`'s samples from `offset`, plus the cursor for the page + after it (None once exhausted). With pagination off, one page holds them + all. Mirrors crate::study::Study::sample_page.""" + start = min(offset, len(ev.samples)) + end = len(ev.samples) if self.page_size is None \ + else min(start + self.page_size, len(ev.samples)) + page = [SampleInfo(id=s.id, tags=list(s.tags), metadata=dict(s.metadata)) + for s in ev.samples[start:end]] + nxt = str(end) if end < len(ev.samples) else None + return page, nxt + + def _eval_info(self, ev: Eval) -> EvalInfo: + """`info()` with only the first page of samples (and the cursor for more).""" + info = ev.info() + info.samples, info.next_cursor = self._sample_page(ev, 0) + return info + + def _list_samples(self, params: dict) -> ListSamplesResult: + ev = self._evals.get(params["eval"]) + if ev is None: + raise ValueError(f"no such eval: {params['eval']}") + cursor = params["cursor"] + try: + offset = int(cursor) + except (TypeError, ValueError): + raise ValueError(f"bad cursor: {cursor}") + samples, nxt = self._sample_page(ev, offset) + return ListSamplesResult(samples=samples, next_cursor=nxt) + + # --- method handlers --- + def _execute(self, params: dict) -> tuple[Transcript, bool]: + """Run one case's subject. Returns (transcript, skipped); an unavailable + target is skipped with an infra-error transcript (scored N/A, not failed).""" + ev = self._evals[params["eval"]] + sample = ev._sample(params["sample"]) + m = ev._target(params["target"]) + if not m.available: + return Transcript(error=f"target unavailable: {m.label}", error_kind="infra"), True + cx = RunCx(target=m.label, provider=m.provider, max_turns=ev.max_turns, + params=params.get("params", {})) + # Zero-burden trajectory contract: a subject may set only + # `transcript.trajectory`; the flat fields are projected here + # (fill-if-default — explicitly set fields win). + return _normalize_trajectory(ev.subject(sample, cx)), False + + def handle(self, method: str, params: dict) -> dict: + if method == "initialize": + return _codec.to_dict(InitializeResult( + protocol_version=PROTOCOL_VERSION, study=self.name, + evals=len(self._evals), study_version=self.version, + capabilities=self._capabilities(), + capability_params={ + # The trajectory representation this study emits (readers + # are more lenient — any ATIF-v1.x parses). + "trajectory": {"format": ATIF_FORMAT, + "version": ATIF_VERSION.removeprefix("ATIF-v")}, + })) + if method == "list": + return _codec.to_dict(ListResult( + evals=[self._eval_info(e) for e in self._evals.values()])) + if method == "list_samples": + return _codec.to_dict(self._list_samples(params)) + if method == "cancel": + # The serve loop is synchronous: it processes one request at a time, + # so there is never a concurrently in-flight run to abort. Cancel is + # therefore always a benign no-op (best-effort, like the protocol + # allows). Handled so the method isn't "unknown"; the `cancel` + # capability is left unadvertised since it can't do anything here. + return _codec.to_dict(CancelResult(cancelled=False)) + if method == "execute": + transcript, skipped = self._execute(params) + return _codec.to_dict(ExecuteResult( + eval=params["eval"], sample=params["sample"], target=params["target"], + params=params.get("params", {}), transcript=transcript, skipped=skipped)) + if method in ("run", "score"): + ev = self._evals[params["eval"]] + sample = ev._sample(params["sample"]) + if method == "score": + # Normalize on receipt: a replayed transcript may be + # trajectory-only; name-based scorers then see the projections. + transcript = _normalize_trajectory( + _codec.from_dict(Transcript, params["transcript"])) + skipped = False + else: + transcript, skipped = self._execute(params) + scores = _score_transcript(ev, sample, transcript) + return _codec.to_dict(RunResult( + eval=params["eval"], sample=params["sample"], target=params["target"], + params=params.get("params", {}), passed=_verdict(scores), + aggregate=_aggregate(scores), scores=scores, + transcript=_summary(transcript), skipped=skipped)) + raise ValueError(f"unknown method: {method}") + + def serve(self, stdin=None, stdout=None) -> None: + serve(self, stdin=stdin, stdout=stdout) + + +def _summary(t: Transcript) -> TranscriptSummary: + return TranscriptSummary( + final_response=t.final_response, iterations=t.iterations, + tool_calls_count=t.tool_calls_count, tool_calls=list(t.tool_calls), + usage=t.usage, timing=t.timing, metrics=dict(t.metrics), + metadata=dict(t.metadata), error=t.error, error_kind=t.error_kind) + + +def log(msg: str) -> None: + print(msg, file=sys.stderr, flush=True) + + +def _rpc_error(exc: Exception) -> dict: + """Build a JSON-RPC-shaped `error` object, classifying the failure so the host + can distinguish a caller mistake from an internal one (mirrors the Rust side). + All these are non-retryable, so `retryable` is left at its default `false`.""" + message = str(exc) + if message.startswith("unknown method"): + code = _CODE_METHOD_NOT_FOUND + elif isinstance(exc, (KeyError, ValueError)): + # Unknown eval/sample/target or a malformed request — the caller's mistake. + code = _CODE_INVALID_PARAMS + message = message.strip("'") if isinstance(exc, KeyError) else message + else: + code = _CODE_INTERNAL_ERROR + return {"code": code, "message": message} + + +def serve(study: Study, stdin=None, stdout=None) -> None: + """Drive `study` over newline-delimited JSON. One object per line in; one + Response/Notification per line out. Loops until stdin EOF.""" + rin = stdin or sys.stdin + out = stdout or sys.stdout + + def emit(obj: dict) -> None: + out.write(json.dumps(obj) + "\n") + out.flush() + + for line in rin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + emit({"method": "log", "params": {"message": "bad json"}}) + continue + rid = msg.get("id") + try: + result = study.handle(msg.get("method"), msg.get("params") or {}) + emit({"id": rid, "result": result}) + except Exception as exc: # report, don't crash the loop + emit({"id": rid, "error": _rpc_error(exc)}) + log(f"{study.name}: stdin closed, exiting") diff --git a/scripts/public_benchmarks/vendor/mira/_wire.py b/scripts/public_benchmarks/vendor/mira/_wire.py new file mode 100644 index 0000000..81998a9 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/_wire.py @@ -0,0 +1,354 @@ +"""Wire types for the Mira eval protocol — GENERATED, do not edit. + +Regenerate with `python3 codegen.py` from schema/v1/schema.json (the same +language-neutral contract the Rust host is generated from). CI runs +`codegen.py --check` to fail on drift. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Literal, Optional + +@dataclass +class Agent: + extra: Dict[str, Any] = field(default_factory=dict) + model_name: Optional[str] = None + name: str = "" + tool_definitions: List[Any] = field(default_factory=list) + version: str = "" + __required__ = ("name", "version") + +@dataclass +class AxisInfo: + name: str = "" + values: List[str] = field(default_factory=list) + __required__ = ("name", "values") + +@dataclass +class CancelParams: + id: int = 0 + __required__ = ("id",) + +@dataclass +class CancelResult: + cancelled: bool = False + __required__ = ("cancelled",) + +@dataclass +class ContentPart: + extra: Dict[str, Any] = field(default_factory=dict) + source: Optional["ImageSource"] = None + text: Optional[str] = None + type: str = "" + __required__ = ("type",) + +ErrorKind = Literal["subject", "infra"] + +@dataclass +class EvalInfo: + axes: List["AxisInfo"] = field(default_factory=list) + description: str = "" + max_turns: int = 0 + metadata: Dict[str, Any] = field(default_factory=dict) + name: str = "" + next_cursor: Optional[str] = None + samples: List["SampleInfo"] = field(default_factory=list) + scorers: List[str] = field(default_factory=list) + seed: Optional[int] = None + targets: List["TargetInfo"] = field(default_factory=list) + trials: int = 0 + __required__ = ("name", "samples", "scorers", "targets") + +@dataclass +class EventParams: + eval: str = "" + kind: str = "" + params: Dict[str, str] = field(default_factory=dict) + request_id: int = 0 + sample: str = "" + target: str = "" + text: Optional[str] = None + tool: Optional[str] = None + turn: Optional[int] = None + __required__ = ("eval", "kind", "sample", "target") + +@dataclass +class ExecuteResult: + eval: str = "" + params: Dict[str, str] = field(default_factory=dict) + sample: str = "" + seed: Optional[int] = None + skipped: bool = False + target: str = "" + transcript: "Transcript" = field(default_factory=lambda: Transcript()) + trial: int = 0 + trials: int = 0 + __required__ = ("eval", "sample", "target", "transcript") + +@dataclass +class FinalMetrics: + extra: Dict[str, Any] = field(default_factory=dict) + total_cached_tokens: Optional[int] = None + total_completion_tokens: Optional[int] = None + total_cost_usd: Optional[float] = None + total_prompt_tokens: Optional[int] = None + total_steps: Optional[int] = None + __required__ = () + +@dataclass +class ImageSource: + media_type: str = "" + path: str = "" + __required__ = ("media_type", "path") + +@dataclass +class InitializeResult: + capabilities: List[str] = field(default_factory=list) + capability_params: Dict[str, Any] = field(default_factory=dict) + evals: int = 0 + protocol_version: str = "" + study: str = "" + study_version: Optional[str] = None + __required__ = ("evals", "protocol_version", "study") + +@dataclass +class ListResult: + evals: List["EvalInfo"] = field(default_factory=list) + __required__ = ("evals",) + +@dataclass +class ListSamplesParams: + cursor: str = "" + eval: str = "" + __required__ = ("cursor", "eval") + +@dataclass +class ListSamplesResult: + next_cursor: Optional[str] = None + samples: List["SampleInfo"] = field(default_factory=list) + __required__ = ("samples",) + +@dataclass +class LogParams: + message: str = "" + request_id: int = 0 + __required__ = ("message",) + +@dataclass +class Notification: + method: str = "" + params: Any = None + __required__ = ("method",) + +@dataclass +class Observation: + results: List["ObservationResult"] = field(default_factory=list) + __required__ = ("results",) + +@dataclass +class ObservationResult: + content: Optional["StepContent"] = None + extra: Dict[str, Any] = field(default_factory=dict) + source_call_id: Optional[str] = None + subagent_trajectory_ref: List["SubagentTrajectoryRef"] = field(default_factory=list) + __required__ = () + +Part = Dict[str, Any] # tagged union: kind in (text, image, audio, file, json) + +@dataclass +class Request: + id: int = 0 + method: str = "" + params: Any = None + __required__ = ("id", "method") + +@dataclass +class Response: + error: Optional["RpcError"] = None + id: int = 0 + result: Any = None + __required__ = ("id",) + +@dataclass +class RpcError: + code: int = 0 + data: Any = None + message: str = "" + retryable: bool = False + __required__ = ("message",) + +@dataclass +class RunParams: + eval: str = "" + params: Dict[str, str] = field(default_factory=dict) + sample: str = "" + seed: Optional[int] = None + target: str = "" + trial: int = 0 + trials: int = 0 + __required__ = ("eval", "sample", "target") + +@dataclass +class RunResult: + aggregate: float = 0.0 + eval: str = "" + expected: Any = None + input: List[str] = field(default_factory=list) + params: Dict[str, str] = field(default_factory=dict) + passed: bool = False + sample: str = "" + scores: List["Score"] = field(default_factory=list) + seed: Optional[int] = None + skipped: bool = False + target: str = "" + transcript: "TranscriptSummary" = field(default_factory=lambda: TranscriptSummary()) + trial: int = 0 + trials: int = 0 + __required__ = ("aggregate", "eval", "passed", "sample", "scores", "target", "transcript") + +@dataclass +class SampleInfo: + id: str = "" + metadata: Dict[str, Any] = field(default_factory=dict) + tags: List[str] = field(default_factory=list) + __required__ = ("id",) + +@dataclass +class Score: + na: bool = False + pass_: bool = field(default=False, metadata={"wire": "pass"}) + reason: str = "" + scorer: str = "" + value: float = 0.0 + __required__ = ("pass", "reason", "scorer", "value") + +@dataclass +class ScoreParams: + eval: str = "" + params: Dict[str, str] = field(default_factory=dict) + sample: str = "" + seed: Optional[int] = None + target: str = "" + transcript: "Transcript" = field(default_factory=lambda: Transcript()) + trial: int = 0 + trials: int = 0 + __required__ = ("eval", "sample", "target", "transcript") + +Source = Dict[str, Any] # union of objects + +@dataclass +class Step: + extra: Dict[str, Any] = field(default_factory=dict) + is_copied_context: Optional[bool] = None + llm_call_count: Optional[int] = None + message: "StepContent" = None + metrics: Optional["StepMetrics"] = None + model_name: Optional[str] = None + observation: Optional["Observation"] = None + reasoning_content: Optional[str] = None + reasoning_effort: Any = None + source: str = "" + step_id: int = 0 + timestamp: Optional[str] = None + tool_calls: List["ToolCall"] = field(default_factory=list) + __required__ = ("source", "step_id") + +StepContent = Any # untagged union: str | List["ContentPart"] + +@dataclass +class StepMetrics: + cached_tokens: Optional[int] = None + completion_token_ids: Optional[List[int]] = None + completion_tokens: Optional[int] = None + cost_usd: Optional[float] = None + extra: Dict[str, Any] = field(default_factory=dict) + logprobs: Optional[List[float]] = None + prompt_token_ids: Optional[List[int]] = None + prompt_tokens: Optional[int] = None + __required__ = () + +@dataclass +class SubagentTrajectoryRef: + extra: Dict[str, Any] = field(default_factory=dict) + session_id: Optional[str] = None + trajectory_id: Optional[str] = None + trajectory_path: Optional[str] = None + __required__ = () + +@dataclass +class TargetInfo: + available: bool = False + label: str = "" + metadata: Dict[str, Any] = field(default_factory=dict) + provider: str = "" + __required__ = ("available", "label") + +@dataclass +class Timing: + duration_ms: int = 0 + time_to_first_token_ms: Optional[int] = None + __required__ = () + +@dataclass +class ToolCall: + arguments: Any = None + extra: Dict[str, Any] = field(default_factory=dict) + function_name: str = "" + tool_call_id: str = "" + __required__ = ("arguments", "function_name", "tool_call_id") + +@dataclass +class Trajectory: + agent: "Agent" = field(default_factory=lambda: Agent()) + continued_trajectory_ref: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + final_metrics: Optional["FinalMetrics"] = None + notes: Optional[str] = None + schema_version: str = "" + session_id: Optional[str] = None + steps: List["Step"] = field(default_factory=list) + subagent_trajectories: List["Trajectory"] = field(default_factory=list) + trajectory_id: Optional[str] = None + __required__ = ("agent", "schema_version", "steps") + +@dataclass +class Transcript: + error: Optional[str] = None + error_kind: "ErrorKind" = None + events: List[Any] = field(default_factory=list) + files: Dict[str, str] = field(default_factory=dict) + final_response: str = "" + iterations: int = 0 + metadata: Dict[str, Any] = field(default_factory=dict) + metrics: Dict[str, float] = field(default_factory=dict) + output: List["Part"] = field(default_factory=list) + timing: "Timing" = field(default_factory=lambda: Timing()) + tool_calls: List[str] = field(default_factory=list) + tool_calls_count: int = 0 + trajectory: Optional["Trajectory"] = None + usage: "Usage" = field(default_factory=lambda: Usage()) + __required__ = () + +@dataclass +class TranscriptSummary: + error: Optional[str] = None + error_kind: "ErrorKind" = None + final_response: str = "" + iterations: int = 0 + metadata: Dict[str, Any] = field(default_factory=dict) + metrics: Dict[str, float] = field(default_factory=dict) + output: List["Part"] = field(default_factory=list) + timing: "Timing" = field(default_factory=lambda: Timing()) + tool_calls: List[str] = field(default_factory=list) + tool_calls_count: int = 0 + usage: "Usage" = field(default_factory=lambda: Usage()) + __required__ = ("final_response", "iterations", "tool_calls_count", "usage") + +@dataclass +class Usage: + cache_read_tokens: int = 0 + cost_usd: float = 0.0 + input_tokens: int = 0 + output_tokens: int = 0 + reasoning_tokens: int = 0 + __required__ = ("cost_usd", "input_tokens", "output_tokens") diff --git a/scripts/public_benchmarks/vendor/mira/scorers.py b/scripts/public_benchmarks/vendor/mira/scorers.py new file mode 100644 index 0000000..9ff77b6 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/scorers.py @@ -0,0 +1,544 @@ +"""Built-in scorers and the `scorer(...)` escape hatch. + +A scorer maps `(sample, transcript) -> Score`. `value` is a continuous 0..1 +signal; `pass_` the boolean verdict; `na=True` means "couldn't evaluate" +(excluded from the case verdict and aggregate), mirroring `mira::Score`. + +PARITY — source of truth: `crates/mira-eval/src/scorer.rs`. +The Rust scorers are canonical; every function here is a hand-written mirror of +its Rust twin (same name, same verdict). Behaviour is pinned by the shared +vectors in `schema/v1/conformance/scorers.json` and verified by +`tests/test_scorer_parity.py`. Change a scorer in Rust → update the vectors → +mirror the change here and in the TypeScript SDK. `reason` strings are +human-facing and may differ across languages; the verdict (pass/value/na) must +not. The LLM-judge (`model_graded`) is deliberately not mirrored — it is not a +deterministic, portable scorer. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any, Callable, Iterator, List, Optional, Sequence, Tuple, Union + +from ._wire import Score, Trajectory +from .trajectory import _content_text + + +def make_score(name: str, value: float, passed: bool, reason: str, na: bool = False) -> Score: + return Score(scorer=name, value=value, pass_=passed, reason=reason, na=na) + + +@dataclass +class Scorer: + """A named `(sample, transcript) -> Score`.""" + + name: str + fn: Callable[["object", "object"], Score] + + def score(self, sample, transcript) -> Score: + return self.fn(sample, transcript) + + +def _passfail(name: str, ok: bool, yes: str, no: str) -> Score: + return make_score(name, 1.0 if ok else 0.0, ok, yes if ok else no) + + +def _tools_used(transcript) -> List[str]: + """Distinct tool names in first-seen order — mirrors `Transcript::tools_used`.""" + seen: List[str] = [] + for n in (transcript.tool_calls or []): + if n not in seen: + seen.append(n) + return seen + + +# ----- text scorers --------------------------------------------------------- + +def succeeded() -> Scorer: + def fn(_sample, transcript) -> Score: + ok = transcript.error is None + return make_score("succeeded", 1.0 if ok else 0.0, ok, + "no error" if ok else (transcript.error or "errored")) + return Scorer("succeeded", fn) + + +def contains(text: str) -> Scorer: + name = f'contains("{text}")' + + def fn(_sample, transcript) -> Score: + found = text in (transcript.final_response or "") + return make_score(name, 1.0 if found else 0.0, found, + f'found "{text}"' if found else f'missing "{text}"') + return Scorer(name, fn) + + +def not_contains(text: str) -> Scorer: + name = f'not_contains("{text}")' + + def fn(_sample, transcript) -> Score: + present = text in (transcript.final_response or "") + return _passfail(name, not present, f'absent "{text}"', f'unexpectedly found "{text}"') + return Scorer(name, fn) + + +def equals(target: str) -> Scorer: + """Trimmed, ASCII-case-insensitive match — mirrors Rust `equals`.""" + name = f'equals("{target}")' + + def fn(_sample, transcript) -> Score: + ok = (transcript.final_response or "").strip().lower() == target.strip().lower() + return _passfail(name, ok, "exact match", "mismatch") + return Scorer(name, fn) + + +def regex(pattern: str) -> Scorer: + name = f"regex({pattern!r})" + compiled = re.compile(pattern) + + def fn(_sample, transcript) -> Score: + ok = compiled.search(transcript.final_response or "") is not None + return _passfail(name, ok, "matched", "no match") + return Scorer(name, fn) + + +def matches_expected() -> Scorer: + """Trimmed, case-sensitive match against the sample's expected answer.""" + name = "matches_expected" + + def fn(sample, transcript) -> Score: + expected = getattr(sample, "expected", None) + if expected is None: + return make_score(name, 0.0, False, "sample has no string expected answer") + ok = (transcript.final_response or "").strip() == expected.strip() + return _passfail(name, ok, "matched expected", f"expected {expected!r}") + return Scorer(name, fn) + + +def non_empty() -> Scorer: + name = "non_empty" + + def fn(_sample, transcript) -> Score: + ok = bool((transcript.final_response or "").strip()) + return _passfail(name, ok, "non-empty response", "empty response") + return Scorer(name, fn) + + +# ----- file / workspace scorers --------------------------------------------- + +def file_exists(path: str) -> Scorer: + name = f"file_exists({path})" + + def fn(_sample, transcript) -> Score: + ok = path in (transcript.files or {}) + return _passfail(name, ok, f"{path} exists", f"no such file: {path}") + return Scorer(name, fn) + + +def file_contains(path: str, needle: str) -> Scorer: + name = f"file_contains({path}, {needle!r})" + + def fn(_sample, transcript) -> Score: + files = transcript.files or {} + if path not in files: + return make_score(name, 0.0, False, f"no such file: {path}") + ok = needle in files[path] + return _passfail(name, ok, f"{path} contains {needle!r}", f"{path} missing {needle!r}") + return Scorer(name, fn) + + +# ----- tool-call scorers ---------------------------------------------------- + +def tool_called(tool: str) -> Scorer: + name = f"tool_called({tool})" + + def fn(_sample, transcript) -> Score: + ok = tool in (transcript.tool_calls or []) + return _passfail(name, ok, f"{tool} was called", f"{tool} never called") + return Scorer(name, fn) + + +def tool_not_called(tool: str) -> Scorer: + name = f"tool_not_called({tool})" + + def fn(_sample, transcript) -> Score: + called = tool in (transcript.tool_calls or []) + return _passfail(name, not called, f"{tool} never called", f"{tool} was called") + return Scorer(name, fn) + + +def tool_calls_within(maximum: int) -> Scorer: + name = f"tool_calls_within({maximum})" + + def fn(_sample, transcript) -> Score: + n = transcript.tool_calls_count + return _passfail(name, n <= maximum, f"{n} <= {maximum}", f"{n} > {maximum}") + return Scorer(name, fn) + + +def turns_within(maximum: int) -> Scorer: + name = f"turns_within({maximum})" + + def fn(_sample, transcript) -> Score: + n = transcript.iterations + return _passfail(name, n <= maximum, f"{n} <= {maximum}", f"{n} > {maximum}") + return Scorer(name, fn) + + +def tools_used_exactly(tools: Sequence[str]) -> Scorer: + expected = sorted(set(tools)) + label = ",".join(expected) + name = f"tools_used_exactly([{label}])" + + def fn(_sample, transcript) -> Score: + used = sorted(_tools_used(transcript)) + ok = used == expected + return _passfail(name, ok, f"used exactly [{label}]", + f"expected [{label}], used [{','.join(used)}]") + return Scorer(name, fn) + + +def tool_called_before(first: str, second: str) -> Scorer: + name = f"tool_called_before({first}, {second})" + + def fn(_sample, transcript) -> Score: + calls = transcript.tool_calls or [] + fi = calls.index(first) if first in calls else None + si = calls.index(second) if second in calls else None + if fi is not None and si is not None: + return _passfail(name, fi < si, f"{first} before {second}", + f"{first} not before {second}") + return make_score(name, 0.0, False, f"both {first} and {second} must be called") + return Scorer(name, fn) + + +# ----- trajectory (ATIF) scorers ---------------------------------------------- +# These grade the structure of the ATIF trajectory (`Transcript.trajectory`, +# the protocol's primary trajectory contract) — tool arguments, correlated +# observations, step counts. A transcript without a trajectory FAILS with +# "subject reported no trajectory" (the ttft_within precedent: an unverifiable +# check fails, it isn't N/A — N/A is reserved for infra). + +_MISSING = object() + + +def _json_pointer(value: Any, pointer: str) -> Any: + """Resolve an RFC 6901 JSON Pointer against a JSON value; `_MISSING` when it + doesn't resolve (distinct from a resolved JSON null) — mirrors + `serde_json::Value::pointer`.""" + if pointer == "": + return value + if not pointer.startswith("/"): + return _MISSING + for token in pointer[1:].split("/"): + token = token.replace("~1", "/").replace("~0", "~") + if isinstance(value, dict): + if token not in value: + return _MISSING + value = value[token] + elif isinstance(value, list): + if not token.isdigit(): + return _MISSING + idx = int(token) + if idx >= len(value): + return _MISSING + value = value[idx] + else: + return _MISSING + return value + + +def _json_eq(a: Any, b: Any) -> bool: + """JSON value equality with Python's bool/int conflation removed + (`True != 1`, mirroring `serde_json::Value` equality).""" + if isinstance(a, bool) != isinstance(b, bool): + return False + if isinstance(a, dict) and isinstance(b, dict): + return a.keys() == b.keys() and all(_json_eq(v, b[k]) for k, v in a.items()) + if isinstance(a, list) and isinstance(b, list): + return len(a) == len(b) and all(_json_eq(x, y) for x, y in zip(a, b)) + if isinstance(a, (dict, list)) or isinstance(b, (dict, list)): + return False + return a == b + + +def _trajectory_invocations(trajectory: Trajectory) -> Iterator[Tuple[str, Any, Optional[Any]]]: + """(function_name, arguments, observation content) triples in step order — + the trajectory case of `Transcript::tool_invocations` (observation content + joined on `source_call_id`).""" + for step in trajectory.steps: + results = step.observation.results if step.observation is not None else [] + for call in step.tool_calls: + content = next( + (r.content for r in results if r.source_call_id == call.tool_call_id), + None, + ) + yield call.function_name, call.arguments, content + + +def tool_called_with(tool: str, pointer: str, expected: Any) -> Scorer: + """Passes when some invocation of `tool` has arguments whose value at + `pointer` (an RFC 6901 JSON Pointer) equals `expected` (JSON equality). + Requires the ATIF trajectory; a transcript without one fails.""" + name = f"tool_called_with({tool}, {pointer}, {json.dumps(expected)})" + + def fn(_sample, transcript) -> Score: + trajectory = transcript.trajectory + if trajectory is None: + return make_score(name, 0.0, False, "subject reported no trajectory") + for called, arguments, _content in _trajectory_invocations(trajectory): + if called != tool: + continue + v = _json_pointer(arguments, pointer) + if v is not _MISSING and _json_eq(v, expected): + return make_score(name, 1.0, True, f"{tool} called with {pointer} == {expected!r}") + return make_score(name, 0.0, False, f"no {tool} call with {pointer} == {expected!r}") + return Scorer(name, fn) + + +def tool_arg_matches(tool: str, pointer: str, pattern: str) -> Scorer: + """Passes when some invocation of `tool` has a **string** argument at + `pointer` matching the regex `pattern` — the regex variant of + `tool_called_with`. A non-string value at the pointer fails with a reason. + Requires the ATIF trajectory; a transcript without one fails.""" + name = f"tool_arg_matches({tool}, {pointer}, {pattern!r})" + compiled = re.compile(pattern) + + def fn(_sample, transcript) -> Score: + trajectory = transcript.trajectory + if trajectory is None: + return make_score(name, 0.0, False, "subject reported no trajectory") + non_string = False + for called, arguments, _content in _trajectory_invocations(trajectory): + if called != tool: + continue + v = _json_pointer(arguments, pointer) + if v is _MISSING: + continue + if isinstance(v, str): + if compiled.search(v): + return make_score( + name, 1.0, True, f"{tool} called with {pointer} matching {pattern!r}" + ) + else: + non_string = True + if non_string: + return make_score(name, 0.0, False, f"non-string value at {pointer} in {tool} arguments") + return make_score(name, 0.0, False, f"no {tool} call with {pointer} matching {pattern!r}") + return Scorer(name, fn) + + +def observation_contains(tool: str, needle: str) -> Scorer: + """Passes when the observation content correlated to some invocation of + `tool` (joined via `source_call_id`) contains `needle` as a substring + (multimodal content is graded on its text projection). Requires the ATIF + trajectory; a transcript without one fails.""" + name = f"observation_contains({tool}, {needle!r})" + + def fn(_sample, transcript) -> Score: + trajectory = transcript.trajectory + if trajectory is None: + return make_score(name, 0.0, False, "subject reported no trajectory") + for called, _arguments, content in _trajectory_invocations(trajectory): + if called == tool and content is not None and needle in _content_text(content): + return make_score(name, 1.0, True, f"{tool} observation contains {needle!r}") + return make_score(name, 0.0, False, f"no {tool} observation contains {needle!r}") + return Scorer(name, fn) + + +def steps_within(maximum: int) -> Scorer: + """Passes when the trajectory has at most `maximum` steps — the ATIF + step-count budget (distinct from `turns_within`, which counts + subject-reported iterations). Requires the ATIF trajectory; a transcript + without one fails.""" + name = f"steps_within({maximum})" + + def fn(_sample, transcript) -> Score: + trajectory = transcript.trajectory + if trajectory is None: + return make_score(name, 0.0, False, "subject reported no trajectory") + n = len(trajectory.steps) + return _passfail(name, n <= maximum, f"{n} <= {maximum}", f"{n} > {maximum}") + return Scorer(name, fn) + + +# ----- budget scorers ------------------------------------------------------- + +def cost_within(max_usd: float) -> Scorer: + name = f"cost_within(${max_usd})" + + def fn(_sample, transcript) -> Score: + c = transcript.usage.cost_usd + return _passfail(name, c <= max_usd, f"${c:.4f} <= ${max_usd}", f"${c:.4f} > ${max_usd}") + return Scorer(name, fn) + + +def tokens_within(maximum: int) -> Scorer: + name = f"tokens_within({maximum})" + + def fn(_sample, transcript) -> Score: + total = transcript.usage.input_tokens + transcript.usage.output_tokens + return _passfail(name, total <= maximum, f"{total} <= {maximum} tokens", + f"{total} > {maximum} tokens") + return Scorer(name, fn) + + +def output_tokens_within(maximum: int) -> Scorer: + name = f"output_tokens_within({maximum})" + + def fn(_sample, transcript) -> Score: + out = transcript.usage.output_tokens + return _passfail(name, out <= maximum, f"{out} <= {maximum}", f"{out} > {maximum}") + return Scorer(name, fn) + + +def latency_within(max_ms: int) -> Scorer: + name = f"latency_within({max_ms}ms)" + + def fn(_sample, transcript) -> Score: + ms = transcript.timing.duration_ms if transcript.timing else 0 + return _passfail(name, ms <= max_ms, f"{ms}ms <= {max_ms}ms", f"{ms}ms > {max_ms}ms") + return Scorer(name, fn) + + +def ttft_within(max_ms: int) -> Scorer: + name = f"ttft_within({max_ms}ms)" + + def fn(_sample, transcript) -> Score: + ms = transcript.timing.time_to_first_token_ms if transcript.timing else None + if ms is None: + return make_score(name, 0.0, False, "subject did not report TTFT") + return _passfail(name, ms <= max_ms, f"ttft {ms}ms <= {max_ms}ms", + f"ttft {ms}ms > {max_ms}ms") + return Scorer(name, fn) + + +# ----- custom (open-vocabulary) metric scorers ------------------------------ + +def metric_within(metric: str, maximum: float) -> Scorer: + name = f"metric_within({metric} <= {maximum})" + + def fn(_sample, transcript) -> Score: + v = (transcript.metrics or {}).get(metric) + if v is None: + return make_score(name, 0.0, False, f"subject did not report {metric}") + return _passfail(name, v <= maximum, f"{metric}={v} <= {maximum}", f"{metric}={v} > {maximum}") + return Scorer(name, fn) + + +def metric_at_least(metric: str, minimum: float) -> Scorer: + name = f"metric_at_least({metric} >= {minimum})" + + def fn(_sample, transcript) -> Score: + v = (transcript.metrics or {}).get(metric) + if v is None: + return make_score(name, 0.0, False, f"subject did not report {metric}") + return _passfail(name, v >= minimum, f"{metric}={v} >= {minimum}", f"{metric}={v} < {minimum}") + return Scorer(name, fn) + + +# ----- JSON output scorers -------------------------------------------------- + +def json_valid() -> Scorer: + name = "json_valid" + + def fn(_sample, transcript) -> Score: + try: + json.loads((transcript.final_response or "").strip()) + return make_score(name, 1.0, True, "valid JSON") + except (ValueError, TypeError) as e: + return make_score(name, 0.0, False, f"invalid JSON: {e}") + return Scorer(name, fn) + + +def json_field_equals(key: str, value: str) -> Scorer: + name = f"json_field_equals({key}={value!r})" + + def fn(_sample, transcript) -> Score: + try: + parsed = json.loads((transcript.final_response or "").strip()) + except (ValueError, TypeError): + return make_score(name, 0.0, False, f"no JSON field {key}") + if not isinstance(parsed, dict) or key not in parsed: + return make_score(name, 0.0, False, f"no JSON field {key}") + got = parsed[key] + if isinstance(got, str) and got == value: + return make_score(name, 1.0, True, f"{key} == {value!r}") + return make_score(name, 0.0, False, f"{key} is {got}, expected {value!r}") + return Scorer(name, fn) + + +# ----- multimodal output scorer --------------------------------------------- + +def produced_modality(modality: str) -> Scorer: + name = f"produced_modality({modality})" + + def fn(_sample, transcript) -> Score: + ok = any((p or {}).get("kind") == modality for p in (transcript.output or [])) + return _passfail(name, ok, f"produced a {modality} part", f"no {modality} part in output") + return Scorer(name, fn) + + +# ----- combinators ---------------------------------------------------------- + +def _glyph(s: Score) -> str: + return "–" if s.na else ("✓" if s.pass_ else "✗") + + +def _combine(name: str, scorers: Sequence[Scorer], require_all: bool) -> Scorer: + def fn(sample, transcript) -> Score: + values: List[tuple] = [] + reasons: List[str] = [] + for sc in scorers: + s = sc.score(sample, transcript) + reasons.append(f"{_glyph(s)}{s.scorer}") + if not s.na: + values.append((s.value, s.pass_)) + reason = ", ".join(reasons) + if not values: + return make_score(name, 0.0, False, reason, na=True) + if require_all: + passed = all(p for _, p in values) + value = sum(v for v, _ in values) / len(values) + else: + passed = any(p for _, p in values) + value = max(v for v, _ in values) + return make_score(name, value, passed, reason) + return Scorer(name, fn) + + +def all_of(name: str, scorers: Sequence[Scorer]) -> Scorer: + """Passes only if every inner scorer passes; value is their mean.""" + return _combine(name, scorers, require_all=True) + + +def any_of(name: str, scorers: Sequence[Scorer]) -> Scorer: + """Passes if any inner scorer passes; value is the max.""" + return _combine(name, scorers, require_all=False) + + +def not_(inner: Scorer) -> Scorer: + """Inverts a scorer; an N/A inner stays N/A (you can't invert 'unknown').""" + name = f"not({inner.name})" + + def fn(sample, transcript) -> Score: + s = inner.score(sample, transcript) + if s.na: + return make_score(f"not({s.scorer})", 0.0, False, f"inner N/A: {s.reason}", na=True) + return make_score(f"not({s.scorer})", 1.0 - s.value, not s.pass_, f"inverted: {s.reason}") + return Scorer(name, fn) + + +# ----- escape hatch --------------------------------------------------------- + +def scorer(name: str, fn: Callable[["object", "object"], Union[bool, Score]]) -> Scorer: + """Wrap an arbitrary predicate. The callable may return a bool (turned into a + pass/fail Score) or a fully-formed Score for graded/NA control. This is a + language-local escape hatch and intentionally has no cross-SDK parity.""" + + def wrapped(sample, transcript) -> Score: + out = fn(sample, transcript) + if isinstance(out, Score): + return out + return make_score(name, 1.0 if out else 0.0, bool(out), "ok" if out else "failed") + return Scorer(name, wrapped) diff --git a/scripts/public_benchmarks/vendor/mira/trajectory.py b/scripts/public_benchmarks/vendor/mira/trajectory.py new file mode 100644 index 0000000..9706ab0 --- /dev/null +++ b/scripts/public_benchmarks/vendor/mira/trajectory.py @@ -0,0 +1,148 @@ +"""ATIF trajectory helpers: parse, project, and build trajectory-only transcripts. + +`Transcript.trajectory` (generated from schema/v1/) is the protocol's **primary +structured trajectory contract** — an ATIF document (harbor RFC 0001). A study +may return a transcript that sets *only* `trajectory`: the serve loop projects +the flat fields (`final_response`, `tool_calls`, `tool_calls_count`, +`iterations`, `usage`) from it automatically, so the built-in name-based scorers +keep working with zero calls on the study author's part. `events` is optional +and fully independent of the trajectory. + +PARITY — source of truth: `crates/mira-eval/src/trajectory.rs`. +`project_into` here hand-mirrors `Trajectory::project_into` (fill-if-default, +never overwrite an explicitly set flat field), and `parse_trajectory` mirrors +`Trajectory::from_value` (accept any ATIF-v1.x, reject other prefixes with an +error). Behaviour is pinned by the shared vectors in +`schema/v1/conformance/trajectory.json`, run by +`tests/test_trajectory_conformance.py` (and the Rust + TypeScript twins). +""" +from __future__ import annotations + +from typing import Any, Optional + +from . import _codec +from ._wire import Trajectory, Transcript, Usage + +# The ATIF schema version this SDK emits. Parsing is more lenient: any +# ATIF-v1.x document is accepted — this is not a ceiling on what is read. +ATIF_VERSION = "ATIF-v1.7" +ATIF_FORMAT = "ATIF" + + +def is_supported_schema_version(version: str) -> bool: + """True when `version` names an ATIF v1 document this SDK can read.""" + return version == "ATIF-v1" or (isinstance(version, str) and version.startswith("ATIF-v1.")) + + +def parse_trajectory(data: Any) -> Trajectory: + """Parse one ATIF document (a JSON-decoded dict) into a `Trajectory`. + + Lenient within v1 (unknown fields are ignored by the codec); a non-v1 + `schema_version` raises `ValueError` — an error the serve loop can surface, + never a crash on untrusted agent output. + """ + if not isinstance(data, dict): + raise ValueError("invalid ATIF trajectory: not a JSON object") + trajectory = _codec.from_dict(Trajectory, data) + if not is_supported_schema_version(trajectory.schema_version): + raise ValueError( + f"unsupported ATIF schema_version {trajectory.schema_version!r}: " + f"this SDK reads ATIF-v1.x (emits {ATIF_VERSION})" + ) + return trajectory + + +def _content_text(content: Any) -> str: + """The text projection of a StepContent value: the string itself, or the + `text` parts joined by newlines (image parts skipped).""" + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join( + p["text"] for p in content + if isinstance(p, dict) and p.get("type") == "text" and isinstance(p.get("text"), str) + ) + return "" + + +def _final_agent_text(trajectory: Trajectory) -> Optional[str]: + for step in reversed(list(trajectory.steps)): + if step.source == "agent": + return _content_text(step.message) + return None + + +def _tool_call_names(trajectory: Trajectory) -> list: + return [call.function_name for step in trajectory.steps for call in step.tool_calls] + + +def _agent_iterations(trajectory: Trajectory) -> int: + return sum(1 for s in trajectory.steps if s.source == "agent" and s.llm_call_count != 0) + + +def _extra_u64(extra: Optional[dict], key: str) -> int: + v = (extra or {}).get(key) + return v if isinstance(v, int) and not isinstance(v, bool) and v >= 0 else 0 + + +def trajectory_usage(trajectory: Trajectory) -> Usage: + """The `Usage` projection: `final_metrics` when present, else the per-step + sum. `reasoning_tokens` is read from the corresponding `extra` map (ATIF has + no first-class slot for it).""" + fm = trajectory.final_metrics + if fm is not None: + return Usage( + input_tokens=fm.total_prompt_tokens or 0, + output_tokens=fm.total_completion_tokens or 0, + cache_read_tokens=fm.total_cached_tokens or 0, + reasoning_tokens=_extra_u64(fm.extra, "reasoning_tokens"), + cost_usd=fm.total_cost_usd or 0.0, + ) + usage = Usage() + for step in trajectory.steps: + m = step.metrics + if m is None: + continue + usage.input_tokens += m.prompt_tokens or 0 + usage.output_tokens += m.completion_tokens or 0 + usage.cache_read_tokens += m.cached_tokens or 0 + usage.reasoning_tokens += _extra_u64(m.extra, "reasoning_tokens") + usage.cost_usd += m.cost_usd or 0.0 + return usage + + +def project_into(trajectory: Trajectory, transcript: Transcript) -> None: + """Fill the transcript's flat fields from `trajectory` — fill-if-default, + so a flat field the study set explicitly is never overwritten. Mirrors + `Trajectory::project_into` (see the module docstring for the rules).""" + if not transcript.final_response: + text = _final_agent_text(trajectory) + if text is not None: + transcript.final_response = text + if not transcript.tool_calls: + transcript.tool_calls = _tool_call_names(trajectory) + if transcript.tool_calls_count == 0: + transcript.tool_calls_count = len(transcript.tool_calls) + if transcript.iterations == 0: + transcript.iterations = _agent_iterations(trajectory) + if transcript.usage is None or transcript.usage == Usage(): + transcript.usage = trajectory_usage(trajectory) + + +def from_trajectory(trajectory: Trajectory) -> Transcript: + """A transcript built from a trajectory alone — the zero-burden path: the + flat fields are projected automatically, and nothing else (not `events`) + is required of the study.""" + transcript = Transcript() + project_into(trajectory, transcript) + transcript.trajectory = trajectory + return transcript + + +def normalize(transcript: Transcript) -> Transcript: + """Project `transcript.trajectory` (if any) into flat fields still at their + defaults. The serve loop applies this to every produced or received + transcript, so studies never call it themselves.""" + if transcript.trajectory is not None: + project_into(transcript.trajectory, transcript) + return transcript diff --git a/scripts/public_benchmarks/vendor/wcxb/LICENSE b/scripts/public_benchmarks/vendor/wcxb/LICENSE new file mode 100644 index 0000000..2fb5073 --- /dev/null +++ b/scripts/public_benchmarks/vendor/wcxb/LICENSE @@ -0,0 +1,25 @@ +Creative Commons Attribution 4.0 International License (CC-BY-4.0) + +Copyright (c) 2026 Murrough Foley + +You are free to: + + Share - copy and redistribute the material in any medium or format for + any purpose, including commercial use. + + Adapt - remix, transform, and build upon the material for any purpose, + including commercial use. + +Under the following terms: + + Attribution - You must give appropriate credit, provide a link to the + license, and indicate if changes were made. You may do so in any + reasonable manner, but not in any way that suggests the licensor + endorses you or your use. + + No additional restrictions - You may not apply legal terms or + technological measures that legally restrict others from doing + anything the license permits. + +Full license text: https://creativecommons.org/licenses/by/4.0/legalcode + diff --git a/scripts/public_benchmarks/vendor/wcxb/evaluate.py b/scripts/public_benchmarks/vendor/wcxb/evaluate.py new file mode 100644 index 0000000..c697f4b --- /dev/null +++ b/scripts/public_benchmarks/vendor/wcxb/evaluate.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +""" +WCXB Evaluation Script + +Evaluate a web content extraction system against the WCEB benchmark. + +Usage: + # Evaluate with a custom extractor function + python evaluate.py --split dev --extractor trafilatura + + # Evaluate from a results JSON file + python evaluate.py --split dev --results results.json + + # Show per-page-type breakdown + python evaluate.py --split dev --results results.json --per-type +""" + +import argparse +import json +import re +import sys +from collections import Counter, defaultdict +from pathlib import Path + +WCEB_DIR = Path(__file__).parent + + +def tokenize(text: str) -> list[str]: + """Tokenize text into lowercase words.""" + if not text: + return [] + return re.findall(r'\w+', text.lower()) + + +def word_f1(predicted: str, reference: str) -> tuple[float, float, float]: + """Compute word-level precision, recall, and F1.""" + pred_tokens = tokenize(predicted) + ref_tokens = tokenize(reference) + + if not ref_tokens: + return (1.0, 1.0, 1.0) if not pred_tokens else (0.0, 0.0, 0.0) + if not pred_tokens: + return (0.0, 0.0, 0.0) + + pred_counts = Counter(pred_tokens) + ref_counts = Counter(ref_tokens) + overlap = sum((pred_counts & ref_counts).values()) + + precision = overlap / len(pred_tokens) + recall = overlap / len(ref_tokens) + f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 + + return (precision, recall, f1) + + +def snippet_check(text: str, snippets: list[str]) -> float: + """Check what fraction of snippets appear in text.""" + if not snippets: + return 1.0 + text_lower = text.lower() + found = sum(1 for s in snippets if s.lower() in text_lower) + return found / len(snippets) + + +def get_page_type(data: dict) -> str: + """Extract page type from GT file.""" + internal = data.get('_internal', {}) or {} + pt_obj = internal.get('page_type', {}) + if isinstance(pt_obj, dict): + pt = pt_obj.get('primary', 'article') + elif isinstance(pt_obj, str): + pt = pt_obj + else: + pt = 'article' + return 'collection' if pt == 'category' else pt + + +def load_ground_truth(split: str) -> dict: + """Load all ground truth files for a split.""" + gt_dir = WCEB_DIR / split / "ground-truth" + if not gt_dir.exists(): + print(f"Error: {gt_dir} does not exist") + sys.exit(1) + + gt_data = {} + for f in sorted(gt_dir.glob("*.json")): + with open(f) as fh: + data = json.load(fh) + gt = data.get('ground_truth', {}) + if not isinstance(gt, dict): + continue + gt_data[f.stem] = { + 'main_content': gt.get('main_content', '') or '', + 'with': gt.get('with', []) or [], + 'without': gt.get('without', []) or [], + 'title': gt.get('title', ''), + 'page_type': get_page_type(data), + } + return gt_data + + +def evaluate_results(gt_data: dict, predictions: dict, per_type: bool = False): + """Evaluate predictions against ground truth.""" + results = [] + type_results = defaultdict(list) + + for file_id, gt in sorted(gt_data.items()): + predicted = predictions.get(file_id, '') + reference = gt['main_content'] + + p, r, f1 = word_f1(predicted, reference) + with_rate = snippet_check(predicted, gt['with']) + without_rate = snippet_check(predicted, gt['without']) + + result = { + 'file_id': file_id, + 'page_type': gt['page_type'], + 'precision': p, + 'recall': r, + 'f1': f1, + 'with_rate': with_rate, + 'without_rate': without_rate, + } + results.append(result) + type_results[gt['page_type']].append(result) + + # Overall metrics + n = len(results) + avg_p = sum(r['precision'] for r in results) / n + avg_r = sum(r['recall'] for r in results) / n + avg_f1 = sum(r['f1'] for r in results) / n + avg_with = sum(r['with_rate'] for r in results) / n + avg_without = sum(r['without_rate'] for r in results) / n + + print(f"\nOverall ({n} pages):") + print(f" Precision: {avg_p:.4f}") + print(f" Recall: {avg_r:.4f}") + print(f" F1: {avg_f1:.4f}") + print(f" With snippets: {avg_with:.1%}") + print(f" Without snippets: {avg_without:.1%} (lower is better)") + + if per_type: + TYPE_ORDER = ['article', 'forum', 'product', 'collection', + 'listing', 'documentation', 'service'] + print(f"\nPer page type:") + print(f" {'Type':<16} {'N':>5} {'F1':>7} {'P':>7} {'R':>7}") + print(f" {'-'*44}") + for pt in TYPE_ORDER: + tr = type_results.get(pt, []) + if not tr: + continue + tf1 = sum(r['f1'] for r in tr) / len(tr) + tp = sum(r['precision'] for r in tr) / len(tr) + tr_val = sum(r['recall'] for r in tr) / len(tr) + print(f" {pt:<16} {len(tr):>5} {tf1:>7.3f} {tp:>7.3f} {tr_val:>7.3f}") + + return results + + +def main(): + parser = argparse.ArgumentParser( + description="Evaluate web content extraction against WCEB benchmark" + ) + parser.add_argument( + '--split', default='dev', choices=['dev', 'test'], + help="Which split to evaluate on (default: dev)" + ) + parser.add_argument( + '--results', type=str, + help="JSON file with predictions: {file_id: extracted_text, ...}" + ) + parser.add_argument( + '--per-type', action='store_true', + help="Show per-page-type breakdown" + ) + args = parser.parse_args() + + gt_data = load_ground_truth(args.split) + print(f"Loaded {len(gt_data)} ground truth files ({args.split} split)") + + if args.results: + with open(args.results) as f: + predictions = json.load(f) + print(f"Loaded {len(predictions)} predictions") + else: + print("No --results file provided. Run with --results ") + print("Expected format: {\"0001\": \"extracted text...\", \"0002\": \"...\"}") + sys.exit(1) + + evaluate_results(gt_data, predictions, per_type=args.per_type) + + +if __name__ == "__main__": + main() diff --git a/scripts/public_benchmarks/wcxb-corpus.json b/scripts/public_benchmarks/wcxb-corpus.json new file mode 100644 index 0000000..bc9d7f8 --- /dev/null +++ b/scripts/public_benchmarks/wcxb-corpus.json @@ -0,0 +1,2010 @@ +[ +{"id":"0001","category":"article","split":"dev","html_sha1":"79d42dadde3c204cd2fee21231d4f0e86beea343","truth_sha1":"46f87b05fc60f46328615c1a88fbd72ec2d9e9b3","overlaps_test":false,"full_text_reference":true}, +{"id":"0002","category":"article","split":"dev","html_sha1":"1dc508ba0b5f0aa8eacdd36d69e06c4fe0640e91","truth_sha1":"c52cada951f9fde27b8288fe99eb8ca3ae0b975b","overlaps_test":false,"full_text_reference":true}, +{"id":"0003","category":"article","split":"dev","html_sha1":"e22082d2b20ffaaabeae167b0a941ef1d2d3d347","truth_sha1":"8a95de398db62d4f2e54ef9054781e2d8b38bf60","overlaps_test":false,"full_text_reference":true}, +{"id":"0004","category":"article","split":"dev","html_sha1":"caa1095966c5fa649b846bb694cb7f4afd67651b","truth_sha1":"0be6f3cd4739b0b370a610e4e0cf78655da01f4c","overlaps_test":false,"full_text_reference":true}, +{"id":"0005","category":"article","split":"dev","html_sha1":"b7b3608fee6f16c4ad81b303fba61b9e18ac6f63","truth_sha1":"9c2e6cf1ddd9508e3f4c366cb09c9b211eba4362","overlaps_test":false,"full_text_reference":true}, +{"id":"0006","category":"article","split":"dev","html_sha1":"29e759dd16d0f9527cd287d35d18b6fbfb3eca78","truth_sha1":"d79fcbe0da4eaf8d3b16d04bb8adb54a9247441d","overlaps_test":false,"full_text_reference":true}, +{"id":"0007","category":"article","split":"dev","html_sha1":"c088f6ffb8482e6901d82c253a68d5e651cc2f54","truth_sha1":"1b9f28b545f8a0504afe231cb74d011434199241","overlaps_test":false,"full_text_reference":true}, +{"id":"0008","category":"article","split":"dev","html_sha1":"93ed3a4a5209a3772d0ab2cd34c4ae32b8553298","truth_sha1":"5d6f734556b9a177402236e13d91589462bfb7ea","overlaps_test":false,"full_text_reference":true}, +{"id":"0009","category":"article","split":"dev","html_sha1":"4146b7b1d490aca38269b10d8d20637dc099e338","truth_sha1":"8b72783c48dac178e2f0a011b7414c1eaebbd499","overlaps_test":false,"full_text_reference":true}, +{"id":"0010","category":"article","split":"dev","html_sha1":"8e4af27e562bed50fa1c83445cc841e828f9ee0f","truth_sha1":"a6ecf9cf0784b5614192e60c5fff5c6a6d75533f","overlaps_test":false,"full_text_reference":true}, +{"id":"0011","category":"article","split":"dev","html_sha1":"5aab1d60299c7e5c4cf853f5d0aed84166cf1a47","truth_sha1":"21c4d8c876c0f1fc4cefc64577dbad5d984c0afc","overlaps_test":false,"full_text_reference":true}, +{"id":"0012","category":"article","split":"dev","html_sha1":"b1d3998e29cbe9d6f2933edfa72abad5b0a878d1","truth_sha1":"3784a8825a225912a901a5bbefd00d8d821c0dac","overlaps_test":false,"full_text_reference":true}, +{"id":"0013","category":"article","split":"dev","html_sha1":"dc30391733a2408cba34ea67c5e6dcd347bf4626","truth_sha1":"4b07e03a2bcb05602d4cf514d35c6de075c9a952","overlaps_test":false,"full_text_reference":true}, +{"id":"0014","category":"article","split":"dev","html_sha1":"e98c88df792829b0455c360f4369923633c51f04","truth_sha1":"7f63337b0e89ba09ec72c9d9d9acb568c7e1da11","overlaps_test":false,"full_text_reference":true}, +{"id":"0015","category":"article","split":"dev","html_sha1":"82fffce8791a4fa9da7e8e7f24629912fb299288","truth_sha1":"13870ab2b9644fc442a7debc03d27329128c9b6d","overlaps_test":false,"full_text_reference":true}, +{"id":"0016","category":"article","split":"dev","html_sha1":"6c7d7c804a67b68fe3d9f7c3fd424b825b89183a","truth_sha1":"afcd2cfa4cfd39388b64f2bfe5d81e3c7a29c519","overlaps_test":false,"full_text_reference":true}, +{"id":"0017","category":"article","split":"dev","html_sha1":"6c31ff526ea7a6acbbbfce45bf8eb45e44fb3a7b","truth_sha1":"93a494032dfa2e77ef0d0c68ddd90c1918c94cfb","overlaps_test":false,"full_text_reference":true}, +{"id":"0018","category":"article","split":"dev","html_sha1":"60d6615157c8e0cc011aed83203711889122135a","truth_sha1":"4b0685f2cb2af76668f14cee00fe23964b6dc511","overlaps_test":false,"full_text_reference":true}, +{"id":"0019","category":"article","split":"dev","html_sha1":"72b0f25f59e4660406dbc2e710fc1dcf53fe94f8","truth_sha1":"f9443a730275e4f450e710242d934a4b0274467e","overlaps_test":false,"full_text_reference":true}, +{"id":"0020","category":"article","split":"dev","html_sha1":"5dd4dca5346152f8cfecd58fbcfdba7009366dd5","truth_sha1":"c38afbbce78bddddc54dca2bf9a3700e70a96919","overlaps_test":false,"full_text_reference":true}, +{"id":"0021","category":"service","split":"dev","html_sha1":"0868f9724bc77788e9eb4be4f99faffd7c90eccd","truth_sha1":"34e496fb562b2361eef1d14e9d78264d9241f49f","overlaps_test":false,"full_text_reference":true}, +{"id":"0022","category":"article","split":"dev","html_sha1":"69c0bf74b902f35ae9cc39074c8be11a2a35ca9e","truth_sha1":"eec3a9800d0245736335a9b1c3823c9205b3b6a1","overlaps_test":false,"full_text_reference":true}, +{"id":"0023","category":"article","split":"dev","html_sha1":"e35e36a746dc8f988bbd07c1fdf315d8f8c8530d","truth_sha1":"721e686ccd8ff47699a314c4172360369bdff68b","overlaps_test":false,"full_text_reference":true}, +{"id":"0024","category":"article","split":"dev","html_sha1":"eb0e5c8246fcc8483c38e131eb65711ee6ab24ad","truth_sha1":"b202e3e224bed90babc84810f833a20cc1deae25","overlaps_test":false,"full_text_reference":true}, +{"id":"0025","category":"service","split":"dev","html_sha1":"f89b28cf3154e584cf660f138ef92d112dd8c74a","truth_sha1":"15d44df96293939210e2abcaa0f201df71368403","overlaps_test":false,"full_text_reference":true}, +{"id":"0026","category":"article","split":"dev","html_sha1":"b4bd9de0735c737394e25a6dea62eee49a477db2","truth_sha1":"a4b32653e889d2798a23009183b2facaaa526acc","overlaps_test":false,"full_text_reference":true}, +{"id":"0027","category":"article","split":"dev","html_sha1":"790c9c6d8eb1c784a29aae4fa37d7dfd2adb6526","truth_sha1":"31ceedf92d5f3f68ce3970392c79c7d5065be264","overlaps_test":false,"full_text_reference":true}, +{"id":"0028","category":"service","split":"dev","html_sha1":"ec32df38900cbf28ce8909d332a9866e7ded458b","truth_sha1":"ee4377c04341943905903fcf7c34ce4aa3b760db","overlaps_test":false,"full_text_reference":true}, +{"id":"0029","category":"service","split":"dev","html_sha1":"1ae311970a866b758685605834f3d81aa0424f57","truth_sha1":"b50c9f331d1569dd5e53f46f08f13773b45691ab","overlaps_test":false,"full_text_reference":true}, +{"id":"0030","category":"article","split":"dev","html_sha1":"d6f5173a336d8fe25bd8ce538cbf307d0e8877b2","truth_sha1":"b57ff3ac2e038da7518a04559a2aae36984052a2","overlaps_test":false,"full_text_reference":true}, +{"id":"0031","category":"article","split":"dev","html_sha1":"3d4be91061a0f8c60c5bbdfe735767ee895d2290","truth_sha1":"499579f3151c0d1d7b3a68ca44a92efcbcf1f0a7","overlaps_test":false,"full_text_reference":true}, +{"id":"0032","category":"article","split":"dev","html_sha1":"514744a0d04f4fe42030ab1c8aad32663616d4ed","truth_sha1":"dee7dfead12db9e0e583e6c74765e331162af6b9","overlaps_test":false,"full_text_reference":true}, +{"id":"0033","category":"article","split":"dev","html_sha1":"62a175cbb10732497d878eb8a36ee7735c90914a","truth_sha1":"fc94954b8b990e2ec529fff337bf42bd34fedf6b","overlaps_test":false,"full_text_reference":true}, +{"id":"0034","category":"article","split":"dev","html_sha1":"8fe013660f798a7f993e1ece6563a2934043f532","truth_sha1":"98468ad93fdc90fd3a7cd4961161524a63488fa2","overlaps_test":false,"full_text_reference":true}, +{"id":"0035","category":"article","split":"dev","html_sha1":"43ee1ec7855cc8a3cedbf44a8251f51deebef41d","truth_sha1":"a102e92e5c91d167396609709f7c07e40385e874","overlaps_test":false,"full_text_reference":true}, +{"id":"0036","category":"article","split":"dev","html_sha1":"9635980cb176873c77b1a70b5009f2c920025112","truth_sha1":"7ce536c0fa33532449e34017163c191f2b49ce15","overlaps_test":false,"full_text_reference":true}, +{"id":"0037","category":"article","split":"dev","html_sha1":"7180040fd88796b9fe2287fe3706781c800bc45e","truth_sha1":"16883f05901cdb8c051567baa17fe79c7e7bb617","overlaps_test":false,"full_text_reference":true}, +{"id":"0038","category":"article","split":"dev","html_sha1":"01938e3ecb78fcc50824ed89be17c4e826ffcca9","truth_sha1":"ebfeb0b7b892fad9171dffa5d3fb79a69e8092db","overlaps_test":false,"full_text_reference":true}, +{"id":"0039","category":"article","split":"dev","html_sha1":"f9ab208e22f7842add3e44c41d3a63066a0735e3","truth_sha1":"df934083874cc94f64b1300a6120a72385f95fb3","overlaps_test":false,"full_text_reference":true}, +{"id":"0040","category":"article","split":"dev","html_sha1":"a00b8dbc5787f7d961a1a96e0fa8aaf8b7f718f0","truth_sha1":"8980fddfecc27ac50441edaac614e24a1eee5430","overlaps_test":false,"full_text_reference":true}, +{"id":"0041","category":"article","split":"dev","html_sha1":"57503d47593d29d2bd5081961a56c7fada232ff7","truth_sha1":"978fa9cfffd8e30728a09eb0bbaaf162401e2f06","overlaps_test":false,"full_text_reference":true}, +{"id":"0042","category":"article","split":"dev","html_sha1":"cd5727aedaf61afdfe1d05ce6c2982a6e8723b2c","truth_sha1":"05e2880a8d0aa62f36d5e14a3f6e2f7857969539","overlaps_test":false,"full_text_reference":true}, +{"id":"0043","category":"article","split":"dev","html_sha1":"3db7d0107389627d68b581e4f3aafd4e2b37872e","truth_sha1":"bb8858438f3d6232a15bb1805e0963eb2f10618e","overlaps_test":false,"full_text_reference":true}, +{"id":"0044","category":"article","split":"dev","html_sha1":"51d2bf6342c51fa11ab3661750e53b4a0173e934","truth_sha1":"1c3b7d245f68d1096aeabad1c1e5f852fcf31946","overlaps_test":false,"full_text_reference":true}, +{"id":"0045","category":"article","split":"dev","html_sha1":"2d8ecc0b21869c46bdea779ae7528035f2f17df8","truth_sha1":"9e659b4707b87b5ccdf85dd83814573d3608d63e","overlaps_test":false,"full_text_reference":true}, +{"id":"0046","category":"article","split":"dev","html_sha1":"0f16aa32fc8f7bc7465ba520e245c6c24f9fd5bd","truth_sha1":"2cc35c6bcbe5677d4dc91f2826aec7889c2f10f5","overlaps_test":false,"full_text_reference":true}, +{"id":"0047","category":"article","split":"dev","html_sha1":"02c24c645f2c54e8f30ee21191e3e15fb3c04ac6","truth_sha1":"b43d561004379a6531882a5b55f0e2bbe1f33895","overlaps_test":false,"full_text_reference":true}, +{"id":"0048","category":"article","split":"dev","html_sha1":"1d74f3c1e121a53b2d6b92b6e449d1aafcddd613","truth_sha1":"397239e0ab4741fa70647258ae55e01fa82b032b","overlaps_test":false,"full_text_reference":true}, +{"id":"0049","category":"article","split":"dev","html_sha1":"c9e55ecde97e60d9b846bdbe0343d08b099b9913","truth_sha1":"9b4cf864cf94f9dc473150fbdcc09f738b60efe5","overlaps_test":false,"full_text_reference":true}, +{"id":"0050","category":"article","split":"dev","html_sha1":"7f2301146b59778f5bfc956e635814b4fbcfbaf2","truth_sha1":"8ec2bbdbae4ed8f4346af0905de68c9e4a137afc","overlaps_test":false,"full_text_reference":true}, +{"id":"0051","category":"article","split":"dev","html_sha1":"24757ba6dd04396162bfb75f4abc70a68d336897","truth_sha1":"ee3220b6f43b9728dc39496a89e4dba3d7d22423","overlaps_test":false,"full_text_reference":true}, +{"id":"0052","category":"article","split":"dev","html_sha1":"4dc243f0573c42f3100db81ff75e512127a05cd6","truth_sha1":"b1e026b415ec5242b848e869a43150de0d4bc395","overlaps_test":false,"full_text_reference":true}, +{"id":"0053","category":"service","split":"dev","html_sha1":"fd6d7894f21769ec84bbda66808e826d4c6376bf","truth_sha1":"741d1d697e353f4ff9e88a012c90a55130280271","overlaps_test":false,"full_text_reference":true}, +{"id":"0054","category":"article","split":"dev","html_sha1":"5148847bfae849072c5124b2202b1f5b4978179c","truth_sha1":"d2450ab975752efdd915673a2666a581d0844878","overlaps_test":false,"full_text_reference":true}, +{"id":"0055","category":"article","split":"dev","html_sha1":"a2d998621b424d13c20e18f346d6a5e9d31c119a","truth_sha1":"9974d8bf72261323fa46e1880f4ee375ddb27096","overlaps_test":false,"full_text_reference":true}, +{"id":"0056","category":"article","split":"dev","html_sha1":"f4fcb2f3ed61a7e3d7e5b2feaa34c522490b2f8a","truth_sha1":"5984652b88db6b37628e67d276ac1d6d3ae04dc3","overlaps_test":false,"full_text_reference":true}, +{"id":"0057","category":"article","split":"dev","html_sha1":"3e9e0e94a9656f4e7f571ee131a44f1bb1dcaf91","truth_sha1":"c17c6a0426e4b3b2b0672043400482b151d9ebeb","overlaps_test":false,"full_text_reference":true}, +{"id":"0058","category":"article","split":"dev","html_sha1":"bcd9aeaa37de4b876920bb27f968794195f17a50","truth_sha1":"efe4f04a7b6d128ed3ebf0b9bc71a3a22ce8dea1","overlaps_test":false,"full_text_reference":true}, +{"id":"0059","category":"article","split":"dev","html_sha1":"f31c1a341af9c1dd27f870bd318bc10021edb477","truth_sha1":"af2cc7b1abf5704d87345074bcaa25bec00f4bae","overlaps_test":false,"full_text_reference":true}, +{"id":"0060","category":"article","split":"dev","html_sha1":"ea12c7bbd32eb83b75686e7e7baf2560fbbb1e65","truth_sha1":"025200f1aeeb39aba124536f25570b3bd6d86c50","overlaps_test":false,"full_text_reference":true}, +{"id":"0061","category":"article","split":"dev","html_sha1":"399ed6beef7de45eb5f820f0704a2eb0892a7117","truth_sha1":"ceec689931cfd44819c0ea3b9fcd45e14f7863f8","overlaps_test":false,"full_text_reference":true}, +{"id":"0062","category":"service","split":"dev","html_sha1":"596aa866785330fa94dbe7c03d604d191660217f","truth_sha1":"812308e744dcda4472ebe346d94be8db0a0c06c5","overlaps_test":false,"full_text_reference":true}, +{"id":"0063","category":"service","split":"dev","html_sha1":"eebef7adc6ea14b9fac66830a4e0ab4ca555fd11","truth_sha1":"94a8ce61b0139d901d3a3338e61174c90c08f46c","overlaps_test":false,"full_text_reference":true}, +{"id":"0064","category":"article","split":"dev","html_sha1":"5f7319a36041b4e5e4536d910bdb7e8f8b778845","truth_sha1":"dea98f28f960d8694fc72daac11e184bf8668380","overlaps_test":false,"full_text_reference":true}, +{"id":"0065","category":"article","split":"dev","html_sha1":"d857f12f7589b6c16437dac80f9d33a5a10452ad","truth_sha1":"6134677fa9f7a52ad966514f7f32998fbfc394a4","overlaps_test":false,"full_text_reference":true}, +{"id":"0066","category":"service","split":"dev","html_sha1":"80848b48d7b67e9d594230556cc6497f6c716d6f","truth_sha1":"87339589e329100f2f294fcce3e9a3d00a96450e","overlaps_test":false,"full_text_reference":true}, +{"id":"0067","category":"article","split":"dev","html_sha1":"edb697292e977c88a7c952b0f671b54e30972739","truth_sha1":"1261556da65d100327ecdffc0068f0fc06962a73","overlaps_test":false,"full_text_reference":true}, +{"id":"0068","category":"article","split":"dev","html_sha1":"9f385f3e61dee18a72a0c9af84c9a7fefce0be78","truth_sha1":"effaca0c7a71aa3961af825ce98f28880eec8c77","overlaps_test":false,"full_text_reference":true}, +{"id":"0069","category":"article","split":"dev","html_sha1":"e8904c802816d8609be9a2a7bfbe0f1228880015","truth_sha1":"27b4b1a939b2fcee5ae5f9c9528147ab32ae3803","overlaps_test":false,"full_text_reference":true}, +{"id":"0070","category":"article","split":"dev","html_sha1":"5996dda0155caa04b8ee0f7ef5eaeaa86c7505ce","truth_sha1":"29f8d3a64e3786384d83a43dcce5cb92bb29ffc6","overlaps_test":false,"full_text_reference":true}, +{"id":"0071","category":"article","split":"dev","html_sha1":"e0ff5fa303a391061d09bd88fb290204ca73b77f","truth_sha1":"acdfacbdbb611fe4bd74ad9b954bb90e0ac67b3e","overlaps_test":false,"full_text_reference":true}, +{"id":"0072","category":"article","split":"dev","html_sha1":"d96238edc009b6ccf04c907c01310bd846b21afc","truth_sha1":"380de6fd3721efc66f702745c280ade7e35eb7c1","overlaps_test":false,"full_text_reference":true}, +{"id":"0073","category":"article","split":"dev","html_sha1":"aea5058670c4f43b6b3a1c6283702dba616a93de","truth_sha1":"e2b47384908803cd52d42f9eadceae308128dafb","overlaps_test":false,"full_text_reference":true}, +{"id":"0074","category":"article","split":"dev","html_sha1":"02245d07c586abc23ec2d999ed1378336628feb1","truth_sha1":"2e9d29577be05a73c1a027dfb3c73130c4a9eb41","overlaps_test":false,"full_text_reference":true}, +{"id":"0075","category":"article","split":"dev","html_sha1":"c49f653ceb475a29e8f7790792918a267b3a4644","truth_sha1":"07d9550ca9467298db0f4ca7b6f6ed24f3a9fbba","overlaps_test":false,"full_text_reference":true}, +{"id":"0076","category":"listing","split":"dev","html_sha1":"4b2271c1b3667efd03492334647d47ccafbf93e7","truth_sha1":"baa3483df007c827c2359e633ed7218b2376be15","overlaps_test":false,"full_text_reference":true}, +{"id":"0077","category":"listing","split":"dev","html_sha1":"51844cab6e3886cd6e57de683dce99ff52aadb5f","truth_sha1":"3c070c43a72e5da25ef0a0b2f06b3835ebbce111","overlaps_test":false,"full_text_reference":true}, +{"id":"0078","category":"article","split":"dev","html_sha1":"73d4449bdd5500d13089e226c6ab1a2e29fa2c61","truth_sha1":"cbe5d7ba06e6089dba5f418bfd7f280955dc4d20","overlaps_test":false,"full_text_reference":true}, +{"id":"0079","category":"article","split":"dev","html_sha1":"b1f82089c58079201b9e57da5d2e93727d1f7ee2","truth_sha1":"579df71e73ac57400d52f6d80b3cc4cd54e70c30","overlaps_test":false,"full_text_reference":true}, +{"id":"0080","category":"article","split":"dev","html_sha1":"2accfb8eb63d9fe786146978e8156d7ef6f9b72f","truth_sha1":"b20a2c13b5b6e0076f5d5cb5788f5fff4337390b","overlaps_test":false,"full_text_reference":true}, +{"id":"0081","category":"article","split":"dev","html_sha1":"76e170949e77ef15a8ac5ca1ca56db0f1ab35668","truth_sha1":"ef9c9358705a32b4137158c55cc52369bfac03e9","overlaps_test":false,"full_text_reference":true}, +{"id":"0082","category":"listing","split":"dev","html_sha1":"f8819cb66b7d875af1e894cc93a870fb828ada55","truth_sha1":"1b05f82d28ef6495fe225fbeddd70ad988e01ae7","overlaps_test":false,"full_text_reference":true}, +{"id":"0083","category":"listing","split":"dev","html_sha1":"4049cdc5c8f69f0dc4f030f3f6fbe2e56c9e9fb0","truth_sha1":"2ef2707f67182537384d72dfd47076ac178365dd","overlaps_test":false,"full_text_reference":true}, +{"id":"0084","category":"service","split":"dev","html_sha1":"aeb4f034fdbdf009c9d9cdbbe15df466062b1f59","truth_sha1":"09cc2c791a33d2fdc45758add8591e7948ead07e","overlaps_test":false,"full_text_reference":true}, +{"id":"0085","category":"article","split":"dev","html_sha1":"5fabaa4f58afae02c71ef5914fd6e184e470f6f8","truth_sha1":"22ac167fd336b1e40c6069ca33ee01860528ea51","overlaps_test":false,"full_text_reference":true}, +{"id":"0086","category":"article","split":"dev","html_sha1":"6d52a69bd0f087d639b7356c64d46dd418f53b57","truth_sha1":"9ef9f99561b4b93f8a8ade03c7022d8c7996ee84","overlaps_test":false,"full_text_reference":true}, +{"id":"0087","category":"service","split":"dev","html_sha1":"f420ac4e5ab44e4b3e0640160b5a754fcf57c32f","truth_sha1":"270329a1952a3efea7a6c78f3c770720f5a5f993","overlaps_test":false,"full_text_reference":true}, +{"id":"0088","category":"service","split":"dev","html_sha1":"363de5c3d8008f4fa75ea15aa4855410160ad683","truth_sha1":"3ed9aae3840e47e4f2495b883eca8990dd382bac","overlaps_test":false,"full_text_reference":true}, +{"id":"0089","category":"article","split":"dev","html_sha1":"ad9d48936d3e4aa7582123a3a296c7a2564e9894","truth_sha1":"a34987574aa4c0509d13935aef3be594f82125d1","overlaps_test":false,"full_text_reference":true}, +{"id":"0090","category":"article","split":"dev","html_sha1":"f7bd5ff00ba95c78044b8f8f030e0d65abe3c1c3","truth_sha1":"f786062dbdb6259451bd4e44215c31dc57b5e3c1","overlaps_test":false,"full_text_reference":true}, +{"id":"0091","category":"article","split":"dev","html_sha1":"35cd2d7b02fb240187a2ff7599da93b19eb0dacd","truth_sha1":"0194ea3a5f6acf0c187251fcfacdadfa47e58b4b","overlaps_test":false,"full_text_reference":true}, +{"id":"0092","category":"article","split":"dev","html_sha1":"31c2a54a76d91cea11b8840982ddfbbb4c3782c3","truth_sha1":"afef357007566aaf8ce42029a2dd2317c7b2d28b","overlaps_test":false,"full_text_reference":true}, +{"id":"0093","category":"article","split":"dev","html_sha1":"6f7c9119272c1fb290d71d63c3947991b7a1ffd5","truth_sha1":"e3bfdaaec62427c048de5503c47807ecc507e0b2","overlaps_test":false,"full_text_reference":true}, +{"id":"0094","category":"article","split":"dev","html_sha1":"e2655a05251968f081f47b2642bd94153cf812df","truth_sha1":"e285b51de93a4f84a1cbeb447663bf91d14504b0","overlaps_test":false,"full_text_reference":true}, +{"id":"0095","category":"service","split":"dev","html_sha1":"81f3978ec2920d0acfb64b1aa0b6b9ffe4688b31","truth_sha1":"0d7187eded65dfb7b10b99c9519a884721925847","overlaps_test":false,"full_text_reference":true}, +{"id":"0096","category":"article","split":"dev","html_sha1":"be3647e836367236701fc65577c40f74fa2d0362","truth_sha1":"ff49c9bb2c41a3ababc51403d3d64c098008ca82","overlaps_test":false,"full_text_reference":true}, +{"id":"0097","category":"article","split":"dev","html_sha1":"0bedf9279753f8161547291efc6c4b92740ad00d","truth_sha1":"072c653f36353105a7309038ad37565346ad3fd0","overlaps_test":false,"full_text_reference":true}, +{"id":"0098","category":"article","split":"dev","html_sha1":"c1e588915d02269ae9cdac87dc51d6b86a050b31","truth_sha1":"6f4013da9b46ab0d54d6a29701f390b4ebe54310","overlaps_test":false,"full_text_reference":true}, +{"id":"0099","category":"article","split":"dev","html_sha1":"d576eba517ee628cc06e7377712b8eed4e095c07","truth_sha1":"f9be8c46e6a2861f859c385c6457b67f73e4fb6a","overlaps_test":false,"full_text_reference":true}, +{"id":"0100","category":"article","split":"dev","html_sha1":"763378b6551475c5bd636665bb5d727e3fe79bc8","truth_sha1":"62a0bac7aed24bb77113e927b79a67e4e0b479ce","overlaps_test":false,"full_text_reference":true}, +{"id":"0101","category":"article","split":"dev","html_sha1":"14dbc59cb28c8ac52aa593d06c02b7faaf9cda4d","truth_sha1":"462bbc354f15dffd06ea6c21186c96a0fd3daa93","overlaps_test":false,"full_text_reference":true}, +{"id":"0102","category":"service","split":"dev","html_sha1":"a1d3c734ee6109b13ce3a0e7d8efd8fb83e8bba1","truth_sha1":"cf43ef73f86dc6812267f0a4f35d9d087338d31e","overlaps_test":false,"full_text_reference":true}, +{"id":"0103","category":"article","split":"dev","html_sha1":"50a4bbcac694104ed52e48c96a9bc6481de275ef","truth_sha1":"7a7b96b8faad35bb661e607e8d2d19ff8f8719dd","overlaps_test":false,"full_text_reference":true}, +{"id":"0104","category":"article","split":"dev","html_sha1":"80f7e453b3c43e656d026ddc9aa72f8f013fdad3","truth_sha1":"1138b5b5476f73cc12bf4d87851a00814cd2f998","overlaps_test":false,"full_text_reference":true}, +{"id":"0105","category":"article","split":"dev","html_sha1":"e4fa0f4a0dd831a7e0d73e0cd92ceb8b4d11b643","truth_sha1":"5ad6b70445d94e41185f6a5c2cb10f7487dea327","overlaps_test":false,"full_text_reference":true}, +{"id":"0106","category":"article","split":"dev","html_sha1":"a343b204069cd1ce3c0d6064ec4c17da861f034e","truth_sha1":"5ee288376f2b0009d283dc07a3d3c145ebb7466f","overlaps_test":false,"full_text_reference":true}, +{"id":"0107","category":"article","split":"dev","html_sha1":"e4d27a28ab82d97b4c6968389f3608b697acff06","truth_sha1":"eda435698b81a4f9efb2a97ebf1887b307367815","overlaps_test":false,"full_text_reference":true}, +{"id":"0108","category":"article","split":"dev","html_sha1":"d9e7ce920bc93d38199edb77dcebdd392d585878","truth_sha1":"a75d6ee0a54fad111cbd3f460f1cedeefff87075","overlaps_test":false,"full_text_reference":true}, +{"id":"0109","category":"article","split":"dev","html_sha1":"d8f73c192ac6390825e3a31ef065e1e544908aa5","truth_sha1":"3999ae487aa2e4766dbed3ffe750bc792435e77b","overlaps_test":false,"full_text_reference":true}, +{"id":"0110","category":"article","split":"dev","html_sha1":"fe797a049d8d7bef4f242c60bd833c7a3cf9faaf","truth_sha1":"7f809a5abfe3d0ed52beb8e1faa4a11e191a5851","overlaps_test":false,"full_text_reference":true}, +{"id":"0111","category":"article","split":"dev","html_sha1":"9265cacfadf742b199ceda984df78139212296a0","truth_sha1":"dc4e811bca9a3ea9107eef055d3238d2518e6b10","overlaps_test":false,"full_text_reference":true}, +{"id":"0112","category":"service","split":"dev","html_sha1":"60bd90537153b7bbb143cc1ea77b1fc7847ee47e","truth_sha1":"a1901cd7b4754cabc3d7dc32f3021892fa845fae","overlaps_test":false,"full_text_reference":true}, +{"id":"0113","category":"article","split":"dev","html_sha1":"a87bfc23482a91682560adf6c148a8811c11d487","truth_sha1":"ed6c557c6c33509873f059bab48cefa23de005a9","overlaps_test":false,"full_text_reference":true}, +{"id":"0114","category":"article","split":"dev","html_sha1":"e1524d6cca92704d6e275cae87a70daabb5955c2","truth_sha1":"bac11bdea494aede7b8baf126d5d121c4e0c1c1e","overlaps_test":false,"full_text_reference":true}, +{"id":"0115","category":"article","split":"dev","html_sha1":"199d0bfa6ef1f99b4e37c7a82b2bca97609db123","truth_sha1":"69ad57531068600a23c7260e7d22eff5da8d4e21","overlaps_test":false,"full_text_reference":true}, +{"id":"0116","category":"article","split":"dev","html_sha1":"63e7a41b350bab50162902b0c00ffc3d866f41d3","truth_sha1":"d327b792e0f0d830ee178048f5e25bfb7db00c46","overlaps_test":false,"full_text_reference":true}, +{"id":"0117","category":"service","split":"dev","html_sha1":"a84d647038b53d37224b49f2f66aa69ef80abc6c","truth_sha1":"e3bf5bceb495124531bba620d14897f6e9772a89","overlaps_test":false,"full_text_reference":true}, +{"id":"0118","category":"article","split":"dev","html_sha1":"ea5004e85ce281624f731f5525bd4eb18d7fdcf9","truth_sha1":"562d23b1090d4b9d0f3d683487bdb713e62822d8","overlaps_test":false,"full_text_reference":true}, +{"id":"0119","category":"article","split":"dev","html_sha1":"5079506f6ba46b2f548afc968139ffa9f430fa19","truth_sha1":"727a6c39e35809b1686aa71e63ebb0b10d72aa98","overlaps_test":false,"full_text_reference":true}, +{"id":"0120","category":"article","split":"dev","html_sha1":"b2a435384052e5dce8a54b3783d17753bc7cd195","truth_sha1":"b518cae8eb5c358d8be02b134dc4dba3292f2a4c","overlaps_test":false,"full_text_reference":true}, +{"id":"0121","category":"article","split":"dev","html_sha1":"3b49fecd027eb362d825e37e44019d00b143a324","truth_sha1":"6ca7dd1e6a6e811fcd463b4c2535349e41f5b439","overlaps_test":false,"full_text_reference":true}, +{"id":"0122","category":"article","split":"dev","html_sha1":"c3e5b64ab6d333f56cb1e1a2de873065ac093cf2","truth_sha1":"c2fed9798935a7567f8d1c30a595fc5e4959d7da","overlaps_test":false,"full_text_reference":true}, +{"id":"0123","category":"article","split":"dev","html_sha1":"08cdb63a21a2a093cdc745324ee0da64074354e2","truth_sha1":"d0975e74dba70a4fb7ce0840e87da936d57d5b76","overlaps_test":false,"full_text_reference":true}, +{"id":"0124","category":"article","split":"dev","html_sha1":"ec6e019a998c11c6b8e29bd31c61cd68ea35623d","truth_sha1":"df0ab2dec08ee67a1d633ba5928b4806a7754072","overlaps_test":false,"full_text_reference":true}, +{"id":"0125","category":"article","split":"dev","html_sha1":"5627537c60d7918ef1af0f697c57461537a20401","truth_sha1":"4cf94017c6dc4174ae54fc7a66c55453104b1516","overlaps_test":false,"full_text_reference":true}, +{"id":"0126","category":"article","split":"dev","html_sha1":"f1c64aeaef813c8f7d072fb79f8fdb431abd95e2","truth_sha1":"6cb12c333285a22bce72452720b0a5def3136c24","overlaps_test":false,"full_text_reference":true}, +{"id":"0127","category":"article","split":"dev","html_sha1":"e1a09d0c2159c8dd0007404723b0713ef903cd59","truth_sha1":"2192a7fbd3e66a285cdf92e9d15f725f325a0539","overlaps_test":false,"full_text_reference":true}, +{"id":"0128","category":"article","split":"dev","html_sha1":"67943498955b8cb798921a8e700ce3dc038963e1","truth_sha1":"176d1815c114c133394aab798d01fe63ba7566b4","overlaps_test":false,"full_text_reference":true}, +{"id":"0130","category":"service","split":"dev","html_sha1":"2216c12c5077d894ef3a04f12e51ce4d8e29d94b","truth_sha1":"e6a6a8a1c1877d2ee83c1e31cebdb884152d8bc8","overlaps_test":false,"full_text_reference":true}, +{"id":"0132","category":"listing","split":"dev","html_sha1":"f4bb880e952e786b8032a118966905b906457927","truth_sha1":"5d1bd7cb3b787231a695d880b9b56a4e792a741c","overlaps_test":false,"full_text_reference":true}, +{"id":"0133","category":"article","split":"dev","html_sha1":"35b2771a1d4c6e4d888a53cd4a110ea76dbc1ffb","truth_sha1":"39307afdfab58233a06fbb0bd8618f11724efe9e","overlaps_test":false,"full_text_reference":true}, +{"id":"0134","category":"service","split":"dev","html_sha1":"17273b69890a0f4f269fe03a898b6d3c12e7ab12","truth_sha1":"3bb7c7896addb511509b195bbf3e6283740413ee","overlaps_test":false,"full_text_reference":true}, +{"id":"0135","category":"service","split":"dev","html_sha1":"88be763127d20ceb638778f1c8333e6f57bb499a","truth_sha1":"12db66451b8034ffe49c479651fc3a15c4910497","overlaps_test":false,"full_text_reference":true}, +{"id":"0136","category":"article","split":"dev","html_sha1":"8d2bbb5bd70168745d0e585c39212f0d52e00943","truth_sha1":"f3d9e36897a569c4fde5417d6d42647d535cba71","overlaps_test":false,"full_text_reference":true}, +{"id":"0137","category":"article","split":"dev","html_sha1":"6e40479841b5e3d80b2a8b2635026fff4b519f84","truth_sha1":"54e80af61cc5e9a77bccbacb4dc719afa1b9cf42","overlaps_test":false,"full_text_reference":true}, +{"id":"0138","category":"article","split":"dev","html_sha1":"0512bb880d8f7c843803a52458b48217db4d467e","truth_sha1":"97849d150b12895d23028db4157618c7633e8913","overlaps_test":false,"full_text_reference":true}, +{"id":"0139","category":"article","split":"dev","html_sha1":"12e3c3985afc34afd5f020b859d1a12260327e5b","truth_sha1":"d6c5ae46e1058d035636d00858c0ff6bbe8f2bac","overlaps_test":false,"full_text_reference":true}, +{"id":"0140","category":"article","split":"dev","html_sha1":"beb944b793973d1119343f38e073cc9957d58f6d","truth_sha1":"9b72dcc2563cf568caf38da0018ea36450389e72","overlaps_test":false,"full_text_reference":true}, +{"id":"0141","category":"article","split":"dev","html_sha1":"e60a1a25d0882d804d686a3e7cd362e0a2a4a1f4","truth_sha1":"6fb3abc48841d1c873fecde6c1799acf0fefe174","overlaps_test":false,"full_text_reference":true}, +{"id":"0142","category":"article","split":"dev","html_sha1":"1512946aa7eb4f7d5155ce6d5cc9ad4997aa4339","truth_sha1":"f78b0f65c0c700a19fe895e0b88861e032833556","overlaps_test":false,"full_text_reference":true}, +{"id":"0143","category":"article","split":"dev","html_sha1":"65640ab6c765a51ba641b05cf07ad6a4ac5f7dff","truth_sha1":"953202f7d7262291a9b562b6193a40c973f860e8","overlaps_test":false,"full_text_reference":true}, +{"id":"0144","category":"article","split":"dev","html_sha1":"4334081e84dd85cdb2db98c19a209b7d0edef46e","truth_sha1":"fba347c4b89d68abe6be4f1a18fcc125a5173e6b","overlaps_test":false,"full_text_reference":true}, +{"id":"0145","category":"article","split":"dev","html_sha1":"5a89c570b9180c56eddeedea3f9d6b023a6ae5e2","truth_sha1":"ce1962a115895bdc6640d313a95c3087531add29","overlaps_test":false,"full_text_reference":true}, +{"id":"0146","category":"article","split":"dev","html_sha1":"42069b77d26daa68014e68173248825d60357c42","truth_sha1":"dcc6d2260c5f828346573e7e25fb140fecaccbf9","overlaps_test":false,"full_text_reference":true}, +{"id":"0147","category":"listing","split":"dev","html_sha1":"c5502bddfa15d4400ef9b6d249a24491d73357d0","truth_sha1":"9ed2c10fa1f7e68b0e135a181372e1560fdfcae4","overlaps_test":false,"full_text_reference":true}, +{"id":"0148","category":"service","split":"dev","html_sha1":"948bd00f49a6a091590709406f67c54a0e34d377","truth_sha1":"5c2cd0af5b897abfedb3b777593e7e5e72cb7090","overlaps_test":false,"full_text_reference":true}, +{"id":"0149","category":"article","split":"dev","html_sha1":"7a861f57b7abb4d8ba42133b9e34f0d8d08c080a","truth_sha1":"e8232a1a4aa95260ead269e83cd2bed40b335597","overlaps_test":false,"full_text_reference":true}, +{"id":"0150","category":"article","split":"dev","html_sha1":"3973f874803dd1ba975444964ff430c416380b6f","truth_sha1":"8149ca8b6ae3e8f874c9ed74626edc5a3c8b6306","overlaps_test":false,"full_text_reference":true}, +{"id":"0151","category":"article","split":"dev","html_sha1":"a569165b4387edb89690fffcb8cdb0f90fee861d","truth_sha1":"e7c65ba4472105fcfa0a40e33c2869c8ca545ac8","overlaps_test":false,"full_text_reference":true}, +{"id":"0152","category":"service","split":"dev","html_sha1":"e7f8a77ea35f575495a947a543c60b44c3eedbee","truth_sha1":"7ae23e174eba9bfb35d3059841a151c133e733ee","overlaps_test":false,"full_text_reference":true}, +{"id":"0153","category":"article","split":"dev","html_sha1":"3b7a60bbe4ee4360d1f3e101efb9e5710635c74a","truth_sha1":"1962558948576c10571221af0a45b84d03c4983e","overlaps_test":false,"full_text_reference":true}, +{"id":"0154","category":"article","split":"dev","html_sha1":"57f98748d23cf7caed87ae80cd7fe0222b48d8a5","truth_sha1":"5c70788e0ca01a45c091bbe470c2f67c5a31961d","overlaps_test":false,"full_text_reference":true}, +{"id":"0155","category":"article","split":"dev","html_sha1":"c074cb07e15c1e057b4aadbf8c45a9555e19e70f","truth_sha1":"b6a38c42b62ebf8a10ae37dacc8c92c62c1e4b3b","overlaps_test":false,"full_text_reference":true}, +{"id":"0156","category":"article","split":"dev","html_sha1":"8ffd7f005634a035e519e9d38086ac2b81f04c78","truth_sha1":"dfe8642601417829436b9a8570550cca6ffd4670","overlaps_test":false,"full_text_reference":true}, +{"id":"0157","category":"article","split":"dev","html_sha1":"8a8b8221551d60f54b41cab82cc187bfc5a2a94d","truth_sha1":"bf6e857e390d84c4d6c2b3782473327586d5bbb6","overlaps_test":false,"full_text_reference":true}, +{"id":"0158","category":"article","split":"dev","html_sha1":"46fef9fa49078249c5985fa1edbfedbddac103e7","truth_sha1":"c70efd97c727bd5977f082d324d1bac0f2ab6ff9","overlaps_test":false,"full_text_reference":true}, +{"id":"0159","category":"article","split":"dev","html_sha1":"1cce4a4494c7bb81c00aa6b1fc7fbdf85ab8060e","truth_sha1":"d81ca755d0ec0b5b15e3424bfb2cbb58940111bf","overlaps_test":false,"full_text_reference":true}, +{"id":"0160","category":"article","split":"dev","html_sha1":"2a0bc401c970c22b67f95252fded70c1b749254d","truth_sha1":"7c4216e032438e4d9833078193e3a4eb99849616","overlaps_test":false,"full_text_reference":true}, +{"id":"0161","category":"article","split":"dev","html_sha1":"d47ece843a7465e44dcb356061e699f4919aaea7","truth_sha1":"74f4640ca9e253b0d4a23b737ed32dfb28bbb38d","overlaps_test":false,"full_text_reference":true}, +{"id":"0162","category":"article","split":"dev","html_sha1":"f210d126e831227d8f2165b0de726ed8bcf8e479","truth_sha1":"524f8d26c860e7621875468c2361c9fd738ae4ab","overlaps_test":false,"full_text_reference":true}, +{"id":"0163","category":"article","split":"dev","html_sha1":"b8a739083d9a99ec337989ab4bad45d1ffba9f46","truth_sha1":"6a148d9d7795fa13e9adcef8d90051a0fae2de32","overlaps_test":false,"full_text_reference":true}, +{"id":"0164","category":"article","split":"dev","html_sha1":"75c7c41f58303a226cbd34c626874df6a85d4b04","truth_sha1":"ea15cdd6ddc3fdb1e58996e6271f544d3b183334","overlaps_test":false,"full_text_reference":true}, +{"id":"0165","category":"article","split":"dev","html_sha1":"7c7836c1cd230a1837252fb0476e1751db567e43","truth_sha1":"ba64ae1c8d8760d3c6141ea7ff2176124efdd797","overlaps_test":false,"full_text_reference":true}, +{"id":"0166","category":"article","split":"dev","html_sha1":"ae61da245610e3456926689b66ee2e2bf9b55849","truth_sha1":"f687154223003ff4be1910dd32a26ae7cee3919f","overlaps_test":false,"full_text_reference":true}, +{"id":"0167","category":"article","split":"dev","html_sha1":"f436ec94bcf9d35923d88c774719cc6470a6eb02","truth_sha1":"14fe87c1dae86e87158eeac9d27b9bd3569f65b0","overlaps_test":false,"full_text_reference":true}, +{"id":"0168","category":"service","split":"dev","html_sha1":"61f8027ed032a077db5dc17e18c201659a7dfb3b","truth_sha1":"6dcf93767218b9fa1eb3a8582b91d198e14128af","overlaps_test":false,"full_text_reference":true}, +{"id":"0169","category":"service","split":"dev","html_sha1":"014c3375104d327532ac7a2db3ce96d93b389f6c","truth_sha1":"ab4729d3b63e1312891c8582d3a3504121fc1d1d","overlaps_test":false,"full_text_reference":true}, +{"id":"0170","category":"service","split":"dev","html_sha1":"47c93bb63c4b7177c8de005a99eb8bed25a4a465","truth_sha1":"65139a975648880d1e5a37c330beb8a2c3b760dd","overlaps_test":false,"full_text_reference":true}, +{"id":"0171","category":"article","split":"dev","html_sha1":"4b3c65b7ef5865487fc27e071b3d28e5c21fc430","truth_sha1":"cda8d02b366e346b911302bc2a5ddfbe1d7b19f2","overlaps_test":false,"full_text_reference":true}, +{"id":"0172","category":"forum","split":"dev","html_sha1":"19c3f99dc8ff8e3e6b75010d528ce088dcc29c30","truth_sha1":"80f40720308776ea959d55b2751061854f14f83e","overlaps_test":false,"full_text_reference":true}, +{"id":"0173","category":"article","split":"dev","html_sha1":"b030e0424654aad829f382e19f7b393097e4b99a","truth_sha1":"88404ca23acfe2b64bf5f27617d890ee7633ef5d","overlaps_test":false,"full_text_reference":true}, +{"id":"0174","category":"article","split":"dev","html_sha1":"76de1843430f268fe69ec45d54571f6a8e90620d","truth_sha1":"d6f3934800b792eca98b2b1a6178e50482f93e10","overlaps_test":false,"full_text_reference":true}, +{"id":"0175","category":"listing","split":"dev","html_sha1":"d7b8fcb3aeca691130ac7476943a9fcac9618d34","truth_sha1":"a34837c30f309212e4873d1721dc1cc86bebe302","overlaps_test":false,"full_text_reference":true}, +{"id":"0177","category":"article","split":"dev","html_sha1":"7a6af89d732168552f0f3958c5ce8df9b357de85","truth_sha1":"ec7a03f64a6c57d6b2b7b3309b2a84c9f14ff1de","overlaps_test":false,"full_text_reference":true}, +{"id":"0178","category":"article","split":"dev","html_sha1":"d8b8b624e3ec2dbb944976b8bf4b2bef9e4b46bf","truth_sha1":"7d0c08b5f4386bba781a457f9cb4acbf3bfd86e2","overlaps_test":false,"full_text_reference":true}, +{"id":"0179","category":"article","split":"dev","html_sha1":"155c9b54649d2de6d68c8d40aaea14f61f00b3ad","truth_sha1":"55a7490aeb3519c4dd2df7cee33d7b7f8981bbf8","overlaps_test":false,"full_text_reference":true}, +{"id":"0180","category":"article","split":"dev","html_sha1":"7ade2f7dfe4bec94c1acc41d2669f24b0fe5d4c0","truth_sha1":"5352ff0767b06a8f193393d0bf8a648099f270e4","overlaps_test":false,"full_text_reference":true}, +{"id":"0181","category":"service","split":"dev","html_sha1":"17f62f7374fb20dd4c98c3f4ef9a3c5fc29e7c85","truth_sha1":"a95484370cdb1dc5c3335e6a3b2f76e30dcdff2b","overlaps_test":false,"full_text_reference":true}, +{"id":"0182","category":"service","split":"dev","html_sha1":"00beb2385da5fbb70f01c515798c6d8ec035aa4a","truth_sha1":"854c8488aaf4d681055cd9d05d2afb348b971196","overlaps_test":false,"full_text_reference":true}, +{"id":"0183","category":"article","split":"dev","html_sha1":"bd017808f074bb2cf6212e1c61d1cb27ddeae6cf","truth_sha1":"33c8bc86edd09c9dd356d617c2f1aa6fab6f08e4","overlaps_test":false,"full_text_reference":true}, +{"id":"0184","category":"article","split":"dev","html_sha1":"189e3c48855721c50172473e31bc78c2f8fb1444","truth_sha1":"179d258111e79c778ededa79a6dcc9067c238e54","overlaps_test":false,"full_text_reference":true}, +{"id":"0185","category":"product","split":"dev","html_sha1":"8502b1f79c7937de21aa65ce51ba5f92a84c40bd","truth_sha1":"51d8f7de13b138cda1e3c76d83a55cd143046f2d","overlaps_test":false,"full_text_reference":true}, +{"id":"0186","category":"article","split":"dev","html_sha1":"eed64cd5a56b6836ac6a377d817476869c9887b7","truth_sha1":"29f15aa4a04a0208474d945255a63a0c07decadc","overlaps_test":false,"full_text_reference":true}, +{"id":"0187","category":"article","split":"dev","html_sha1":"a2240f1d320011a02ba02485f3b332b99cbbb723","truth_sha1":"358d65fbdb38bd28c35d4d11c9143e080e945f1e","overlaps_test":false,"full_text_reference":true}, +{"id":"0188","category":"article","split":"dev","html_sha1":"fc46b5f00743029d6deca11cf0a47907119153ab","truth_sha1":"8627c9d3083e741d9175aae77516ae3f5ea053a5","overlaps_test":false,"full_text_reference":true}, +{"id":"0189","category":"article","split":"dev","html_sha1":"cf5e06f7d64e7d571130c4858c7b7af709f25c48","truth_sha1":"a0e5ca930ee2c302d1d720ea2c7422a403207c84","overlaps_test":false,"full_text_reference":true}, +{"id":"0190","category":"article","split":"dev","html_sha1":"53ffbf9963e5d02789fb9278666e7b0676906776","truth_sha1":"604b2eda3f4ebf51f7c99e6923c3a3723762e2f2","overlaps_test":false,"full_text_reference":true}, +{"id":"0191","category":"article","split":"dev","html_sha1":"44ef815aae3d5fb11698bbc655178b1a8bd20711","truth_sha1":"01c1172e1c2b66c9be32d63d62555d5fb30d2852","overlaps_test":false,"full_text_reference":true}, +{"id":"0192","category":"article","split":"dev","html_sha1":"1356340a2ebd2db3acd71783be9bb682c92f57d3","truth_sha1":"0ec5cbb1c136c18ef6a8b5aea26e90f72bbe618a","overlaps_test":false,"full_text_reference":true}, +{"id":"0193","category":"listing","split":"dev","html_sha1":"45b4dab3b1e7b90408b5e2588b468cd22c411a17","truth_sha1":"33384e34bfd156b129797feb594eb7d603f64583","overlaps_test":false,"full_text_reference":true}, +{"id":"0194","category":"service","split":"dev","html_sha1":"bf708bb79ea19be719d2c537b3eaf791c179fbc7","truth_sha1":"2a641cf479f48d2b8f5f0a697f84b6add7df2e46","overlaps_test":false,"full_text_reference":true}, +{"id":"0195","category":"article","split":"dev","html_sha1":"c5037a862291ba5923ce80794567dabe9345d3c7","truth_sha1":"858687533f94668bbde75759630d61ff4ed90a1d","overlaps_test":false,"full_text_reference":true}, +{"id":"0196","category":"article","split":"dev","html_sha1":"598c768a3ddd28dc8d5ff50865ad1966dbe5ddf9","truth_sha1":"f0d5558f7c664410e0958e40938d8777cb225e92","overlaps_test":false,"full_text_reference":true}, +{"id":"0197","category":"article","split":"dev","html_sha1":"d7c78896c068a8c2d19ae26a9711361415844ddc","truth_sha1":"e5c73c8bbfcedfe9d0243e7a78686b9a24bacda8","overlaps_test":false,"full_text_reference":true}, +{"id":"0198","category":"article","split":"dev","html_sha1":"ef3996ff4f9c5dc24525e6eb98dd74c1b0846e10","truth_sha1":"31deac87590393ec17ce2b7fd7d29e221831f654","overlaps_test":false,"full_text_reference":true}, +{"id":"0199","category":"article","split":"dev","html_sha1":"889728550c6837e6c19a6468360ba4bfea060fe0","truth_sha1":"38f432bd0202150bcc4a16d573a0ac350cd53c20","overlaps_test":false,"full_text_reference":true}, +{"id":"0200","category":"article","split":"dev","html_sha1":"77601969a8a3a0ce8992482954fc4f368536ab75","truth_sha1":"be8bf6b444b22771b04d0f092030bf0cbefe7927","overlaps_test":false,"full_text_reference":true}, +{"id":"0202","category":"article","split":"dev","html_sha1":"ea9825a90060e97efbeff31d478abe73f23791bc","truth_sha1":"7943a915aa1253a4fae36fcfb8ca671cb61918c1","overlaps_test":false,"full_text_reference":true}, +{"id":"0203","category":"listing","split":"dev","html_sha1":"b60aa9947340f4dc2491d561989434df11bd649e","truth_sha1":"e75ec3165e14d56a4b8f943046ccfa36b5caa5e7","overlaps_test":false,"full_text_reference":true}, +{"id":"0204","category":"service","split":"dev","html_sha1":"5c8d0b51ee982b8e12c10a9ee660029427637009","truth_sha1":"0a30a2af61c8d7775f25de7705732103e159a70d","overlaps_test":false,"full_text_reference":true}, +{"id":"0205","category":"listing","split":"dev","html_sha1":"7bec04f3bc9d86f5bfd8c3839524119450e0aca8","truth_sha1":"772abcdb5fd49c040615dd456849c97c882e2a9a","overlaps_test":false,"full_text_reference":true}, +{"id":"0206","category":"article","split":"dev","html_sha1":"2a2b98094790c5c496a0de84be8edbb0eed4f006","truth_sha1":"5cd880fcceb9ec7698de897364308c04f0a51c8e","overlaps_test":false,"full_text_reference":true}, +{"id":"0207","category":"article","split":"dev","html_sha1":"d6d3f0bcc34b63e38c57bdb5f9ee02813f00a16f","truth_sha1":"10aab15319cb69ef7c74da0585e6d5db8aa05091","overlaps_test":false,"full_text_reference":true}, +{"id":"0208","category":"article","split":"dev","html_sha1":"4753e724f6cf765e348a01a9d21093312626f03d","truth_sha1":"8880317a6b9b973ae638d48c39624421547e87e2","overlaps_test":false,"full_text_reference":true}, +{"id":"0209","category":"article","split":"dev","html_sha1":"b1df5a422185289fc6a452968963eeffb37b781f","truth_sha1":"b288059306e93424092e620569e58dab4411d3d2","overlaps_test":false,"full_text_reference":true}, +{"id":"0210","category":"article","split":"dev","html_sha1":"e290532baa1d6a651a6df7e616168222636b1bfd","truth_sha1":"a1517157cae66a0fe1db849f2a0702a9ac879320","overlaps_test":false,"full_text_reference":true}, +{"id":"0211","category":"service","split":"dev","html_sha1":"becc33398a52b99a79b386d3acebed1d7652f104","truth_sha1":"b1c5d866835c66676bf8600eb8b1b474f2048a66","overlaps_test":false,"full_text_reference":true}, +{"id":"0212","category":"article","split":"dev","html_sha1":"ad4b594d883cbbd767756f0579a139a14e12342e","truth_sha1":"a7222f8d66b5a4578d32a20f559062b5fbb5cc25","overlaps_test":false,"full_text_reference":true}, +{"id":"0213","category":"article","split":"dev","html_sha1":"bc0e957973f84b8382252cc0d2f3e792245eed6d","truth_sha1":"a6ee5ca80d794b61074bd6c6931af7961344b752","overlaps_test":false,"full_text_reference":true}, +{"id":"0214","category":"article","split":"dev","html_sha1":"9819678fe81a40326a798cfce49c7c2531fcfa4e","truth_sha1":"3f138566db2cf637b85f2daf3a25c91a25464b5e","overlaps_test":false,"full_text_reference":true}, +{"id":"0215","category":"article","split":"dev","html_sha1":"658454ebebe7bb2e79a05e635d2127f462a06398","truth_sha1":"c0d6ec85d672e10f6ec118f7ded7c55cbf2dab7e","overlaps_test":false,"full_text_reference":true}, +{"id":"0216","category":"listing","split":"dev","html_sha1":"eac981949075927e81f4967575454123ffec779a","truth_sha1":"fbb3cd93e221eeb8ff24cbcc9f509fae37f2e3ba","overlaps_test":false,"full_text_reference":true}, +{"id":"0217","category":"article","split":"dev","html_sha1":"cf95d875d5433cf479144d39bd033ab3d3bd5e9a","truth_sha1":"cf9a58010e9289246a0840386f6c808d0f9b9d45","overlaps_test":false,"full_text_reference":true}, +{"id":"0218","category":"article","split":"dev","html_sha1":"27c90ef0270421ee39fdbb889bfdebd54295d083","truth_sha1":"3f0471b7ed0a47d0a8f791407b9392d66f57f4d9","overlaps_test":false,"full_text_reference":true}, +{"id":"0219","category":"article","split":"dev","html_sha1":"d822c7664b5d3c0dbba57dcf2026b8b1a1c5f43c","truth_sha1":"8716f16756f3af249eeaf838fb06a6134babe1cc","overlaps_test":false,"full_text_reference":true}, +{"id":"0220","category":"service","split":"dev","html_sha1":"b1b474c63c9ac04151fa2968aef5feb37f1b4c6a","truth_sha1":"eae8ae4d94a756f66e0a38b78b3e43f0751f4fb2","overlaps_test":false,"full_text_reference":true}, +{"id":"0221","category":"article","split":"dev","html_sha1":"720d4af950d7eb92e566ad2a07fcbf0f085aee5f","truth_sha1":"832abd16fd3773b25e96c66b13fff95dc4e51552","overlaps_test":false,"full_text_reference":true}, +{"id":"0222","category":"article","split":"dev","html_sha1":"d7792c7fd7b4ec7d5a641b74251bf4e10b496f33","truth_sha1":"31bdaa536fc5337e7c16c79dbfe67ace5d6d0e45","overlaps_test":false,"full_text_reference":true}, +{"id":"0223","category":"article","split":"dev","html_sha1":"5d0c697fafcccc56fc08963d2c49dc4fe42ad808","truth_sha1":"442dc113a2ed1ef70595f462973744c88a8ea2c5","overlaps_test":false,"full_text_reference":true}, +{"id":"0224","category":"article","split":"dev","html_sha1":"a31b4b77d55dc3ea0f93d9067945c75ab198811f","truth_sha1":"3b3414356c167bd41c774876b12b6e06804fab98","overlaps_test":false,"full_text_reference":true}, +{"id":"0225","category":"listing","split":"dev","html_sha1":"fb7422ec447dc81e31fec66118c021f9e8aef9d1","truth_sha1":"62d3c983b70ce4c96629b11917a6dfbe7fb314c3","overlaps_test":false,"full_text_reference":true}, +{"id":"0226","category":"article","split":"dev","html_sha1":"2dc7af5d06260814c9cb0fcdaddd25c268ba5b40","truth_sha1":"03e4fa8c32a20c1f925a1fafb08655a8e4a5fd90","overlaps_test":false,"full_text_reference":true}, +{"id":"0227","category":"article","split":"dev","html_sha1":"262fb3a8f38d33221e29abbcd773c54802e2cdb8","truth_sha1":"8ff9226b2ddbcb1413e367e7b204fc46438fe0ec","overlaps_test":false,"full_text_reference":true}, +{"id":"0228","category":"article","split":"dev","html_sha1":"2bdff3b8d31b94ece1554ed58e2c74340bc46022","truth_sha1":"6993056333af28d7f75df2c9d5049d0c64c5babb","overlaps_test":false,"full_text_reference":true}, +{"id":"0229","category":"article","split":"dev","html_sha1":"6dbe1c4aa4450b07b7c76de546a053b54d5638fb","truth_sha1":"7971cec42f89c7c9db94d983ebac04c7e7b9124d","overlaps_test":false,"full_text_reference":true}, +{"id":"0230","category":"article","split":"dev","html_sha1":"ef863a591befe02d133a5df2fb264b2dd2003fa8","truth_sha1":"9eb476f8b8f55ecb66d15dfb8d2cc3e5ac2f5399","overlaps_test":false,"full_text_reference":true}, +{"id":"0231","category":"article","split":"dev","html_sha1":"7776435c5b735db0095cca69dcb4714c9952ef50","truth_sha1":"618547148caa29d68ce3b639771cb16bbfe861ad","overlaps_test":false,"full_text_reference":true}, +{"id":"0232","category":"article","split":"dev","html_sha1":"8f5a99c733118a1bf49f1ea89a10e4beda2927b6","truth_sha1":"2d92abddd402571798f1119e19ee1d7d1b656572","overlaps_test":false,"full_text_reference":true}, +{"id":"0233","category":"article","split":"dev","html_sha1":"20ee9c39650c8d85bd55313b71fa0171131c1a21","truth_sha1":"b48cded153a7c464a56920911bd20b4070620755","overlaps_test":false,"full_text_reference":true}, +{"id":"0234","category":"article","split":"dev","html_sha1":"e90a87e07f6854c2fa0a188611323a9144d538ec","truth_sha1":"a63c1bfab1372526f783274d3945b5a23d966899","overlaps_test":false,"full_text_reference":true}, +{"id":"0235","category":"service","split":"dev","html_sha1":"854bd56e3a0cfb26116d8509c9e80041472894a6","truth_sha1":"8d126047c264f18a40ca1fe8a431272f0010f22b","overlaps_test":false,"full_text_reference":true}, +{"id":"0236","category":"article","split":"dev","html_sha1":"d78363711fef9e8ab39cd7a3c142bdfb1f1c11c8","truth_sha1":"39610cdc0c687faff504c4dbfef4d8508e0fc23e","overlaps_test":false,"full_text_reference":true}, +{"id":"0237","category":"article","split":"dev","html_sha1":"c323832cc86adf6b4a7a36d3f0828b1701f74bb1","truth_sha1":"4915f9715b2a38ed3cc6409180d57c46ca8d2f37","overlaps_test":false,"full_text_reference":true}, +{"id":"0238","category":"article","split":"dev","html_sha1":"d442b0519dc79c6b0d73f62efe6468492b9bfe61","truth_sha1":"57a0970e2830d21d682a2998adb15375c2db7345","overlaps_test":false,"full_text_reference":true}, +{"id":"0239","category":"documentation","split":"dev","html_sha1":"0cc3cdf5daad699a45999520c64feed895545e5d","truth_sha1":"2ec03ff127ef23e8c2ed5e550fd20b4e03e36d71","overlaps_test":false,"full_text_reference":true}, +{"id":"0240","category":"article","split":"dev","html_sha1":"6b77caf66310e56b2cc831cb5c11bc1b4d98507f","truth_sha1":"6ed64f70c7db7482ef11a47a214cf4fd99f32827","overlaps_test":false,"full_text_reference":true}, +{"id":"0241","category":"article","split":"dev","html_sha1":"b7e371fe4dfc6d3f97c13d3ef71382044ef464fd","truth_sha1":"cbf7122e404abbd67305bddbefb114d7ff4aa505","overlaps_test":false,"full_text_reference":true}, +{"id":"0242","category":"service","split":"dev","html_sha1":"604199f522abb42e372b61a054da2add672540c5","truth_sha1":"25dcf0796c3810f231b5c84779a9e1b33afcf8e5","overlaps_test":false,"full_text_reference":true}, +{"id":"0243","category":"article","split":"dev","html_sha1":"36f75bcfe689903d0ea2d719db5659d411003c71","truth_sha1":"a096ebebdab62376a166958d16592c7bd88ae85f","overlaps_test":false,"full_text_reference":true}, +{"id":"0244","category":"article","split":"dev","html_sha1":"eeb54d363d1e21ec69196b7732e2ddde5f377f11","truth_sha1":"ce2151e49f269fb2c69fd03d65a893b5788f3055","overlaps_test":false,"full_text_reference":true}, +{"id":"0245","category":"article","split":"dev","html_sha1":"b046e092b662712c300f3ac0c8e2759611761b89","truth_sha1":"e4a961ccbe10046e743bffa1c61262c1a990e8d2","overlaps_test":false,"full_text_reference":true}, +{"id":"0246","category":"article","split":"dev","html_sha1":"c0d187ae7534dbdf5b6c18ff99cd01fcf1c4f200","truth_sha1":"b093b373d29f2b62d14dee5aa26835bdd5f6961b","overlaps_test":false,"full_text_reference":true}, +{"id":"0247","category":"service","split":"dev","html_sha1":"49702d76080b51df391f8caf9175bf5ec2f4e48d","truth_sha1":"aeb046a04bb6d61d27ae9bccd69e4dcef342553b","overlaps_test":false,"full_text_reference":true}, +{"id":"0248","category":"service","split":"dev","html_sha1":"bddf32b4d67420fecbf8f815f22014c4aba443a2","truth_sha1":"2427424ca220c88d471f3e6c206f5eda3ba7cfd3","overlaps_test":false,"full_text_reference":true}, +{"id":"0249","category":"article","split":"dev","html_sha1":"456ab84a17cfc84635b7e62abfc47f63a029ce47","truth_sha1":"4f2189a5ef71257e8c1674ae58738702d80593df","overlaps_test":false,"full_text_reference":true}, +{"id":"0250","category":"article","split":"dev","html_sha1":"511b1e5ac0863028d55bdb08d862a66e15e8d23b","truth_sha1":"493271ea7ad598f3fc515faa9e1e79ad91a2b80b","overlaps_test":false,"full_text_reference":true}, +{"id":"0251","category":"article","split":"dev","html_sha1":"ace1c576875457ff86ced419c3e394a06ceb5780","truth_sha1":"4dd8e364b8d736804b3f160094595b06a7aefbd7","overlaps_test":false,"full_text_reference":true}, +{"id":"0252","category":"article","split":"dev","html_sha1":"d965f621762a0de9cc72aec97edf18599350227b","truth_sha1":"56d0f74a13544a3fbd27c2b960fa5bff18d42884","overlaps_test":false,"full_text_reference":true}, +{"id":"0253","category":"article","split":"dev","html_sha1":"71de135211f1b4b35b8269188f18ed4cf79dec9f","truth_sha1":"b8474ecfa36ba81e4645963c641b55d6bbe0bf9a","overlaps_test":false,"full_text_reference":true}, +{"id":"0254","category":"article","split":"dev","html_sha1":"a1379855b056557e32e24427c8401308c08855c2","truth_sha1":"bb1e569675f86560fac10af39c5159521e14ae8e","overlaps_test":false,"full_text_reference":true}, +{"id":"0255","category":"article","split":"dev","html_sha1":"3ea7255bbc2ca1b5a02d91e9abd06c5df8adf56c","truth_sha1":"5abffa6f82b4df4f70b8fd99edbce099a363ea2f","overlaps_test":false,"full_text_reference":true}, +{"id":"0256","category":"article","split":"dev","html_sha1":"29811caf761d09c98d5a1f3afaa0c16bb8aec55d","truth_sha1":"67923910aae8df9d23de03b5e9936d5b5295800d","overlaps_test":false,"full_text_reference":true}, +{"id":"0257","category":"article","split":"dev","html_sha1":"4cba8e7c00a539cfc605be929d2e9c45674a5234","truth_sha1":"b5bed00b9e811dace03ce427d9e2fe2d28ef56ef","overlaps_test":false,"full_text_reference":true}, +{"id":"0258","category":"service","split":"dev","html_sha1":"c11ac973b165ff5e7dea7455149ac4016d93274d","truth_sha1":"2d9e6f784b3151fa1ac23d250156aa20f8499610","overlaps_test":false,"full_text_reference":true}, +{"id":"0259","category":"listing","split":"dev","html_sha1":"2d4814d7b7a3ddfff772ac397f2e9a5992dc43f7","truth_sha1":"f8fbd354f783ca859af44e5882e39a56b69032d6","overlaps_test":false,"full_text_reference":true}, +{"id":"0260","category":"listing","split":"dev","html_sha1":"4c2c89482751146f463dba5b53e58d54f7e7c93c","truth_sha1":"6013de95f2859661eabc1da99d70a70afcee9623","overlaps_test":false,"full_text_reference":true}, +{"id":"0261","category":"article","split":"dev","html_sha1":"1beb6e83b10bbe31a2b367dbef53dd7f18c2ad10","truth_sha1":"c27df390320c51c93e29ee224ec72afc3f84590b","overlaps_test":false,"full_text_reference":true}, +{"id":"0262","category":"article","split":"dev","html_sha1":"d2196f927e332b2b8754fd26a9d19980280dcc1b","truth_sha1":"4aaf75cf6f2ef652516ee15e57f0c83001db8e64","overlaps_test":false,"full_text_reference":true}, +{"id":"0263","category":"article","split":"dev","html_sha1":"0b42cadeefef8acb9a88a2a6540fd634f2003932","truth_sha1":"1abfb599d3f69f269ea871262f62de4090a67ac3","overlaps_test":false,"full_text_reference":true}, +{"id":"0264","category":"listing","split":"dev","html_sha1":"f64cd0a82902d1459deefa3aff79591709f801ee","truth_sha1":"b15c1dc8733366b47fc1fef23465dd401254c61e","overlaps_test":false,"full_text_reference":true}, +{"id":"0265","category":"listing","split":"dev","html_sha1":"5d7bf0600d4be03f9c5c23756a345cd395122866","truth_sha1":"bda6fee034e0bab2d0387ac9ee50e2bb34e67a7a","overlaps_test":false,"full_text_reference":true}, +{"id":"0266","category":"service","split":"dev","html_sha1":"9b65bfa20695bad84e52c692d260683e24dba150","truth_sha1":"ac91ddb95878317274f22d6ec83026fe2a2d591d","overlaps_test":false,"full_text_reference":true}, +{"id":"0267","category":"article","split":"dev","html_sha1":"566a8f72962c8d9f384a7e39ad9bc67e42e7e781","truth_sha1":"ef4a4411f42bb10f6ac64e9a76eccd1fb8ab874b","overlaps_test":false,"full_text_reference":true}, +{"id":"0268","category":"article","split":"dev","html_sha1":"37542451fd966c1ad513ffd0e69bfd4c500e4b4d","truth_sha1":"7cc56a6aa100186f36a1ca926a572b57d8b65e53","overlaps_test":false,"full_text_reference":true}, +{"id":"0269","category":"article","split":"dev","html_sha1":"a3e599d2cda28c836dbcdee45d50f7d725e44320","truth_sha1":"ef97217d7415810a45d38dd6fe307ff81431247f","overlaps_test":false,"full_text_reference":true}, +{"id":"0270","category":"article","split":"dev","html_sha1":"ce583171581e33041a60a6cf809711e5ef91c906","truth_sha1":"dac99395a14bdcfd5f3a84507b51183d6a41c80e","overlaps_test":false,"full_text_reference":true}, +{"id":"0271","category":"article","split":"dev","html_sha1":"b4ac29b75047293fbda079ec2ccab60e60e20156","truth_sha1":"7c3c08f7bcbd3ee1108157a83fa3307dfe0e0d4e","overlaps_test":false,"full_text_reference":true}, +{"id":"0272","category":"article","split":"dev","html_sha1":"757cc9c06e4eb59abb8a9b7894be59814adccb0f","truth_sha1":"9b37262f29d275297e48aad35d8bfdf5c38de248","overlaps_test":false,"full_text_reference":true}, +{"id":"0273","category":"article","split":"dev","html_sha1":"76bca599a954d229766cdf124ddf05c52f398630","truth_sha1":"0429734674a5ee5383a02f191fc151052eb07cd5","overlaps_test":false,"full_text_reference":true}, +{"id":"0274","category":"listing","split":"dev","html_sha1":"fc0d9871e97bed6de40eabb9f96adc3734e0ae11","truth_sha1":"53988e1e68296ec8c2acdc7b8846a118c51e8da9","overlaps_test":false,"full_text_reference":true}, +{"id":"0275","category":"article","split":"dev","html_sha1":"bd3b756d2a626004aed326d5d0819d8a854e4672","truth_sha1":"734cee716cf7f3894385d67c4ae938187fa24318","overlaps_test":false,"full_text_reference":true}, +{"id":"0276","category":"article","split":"dev","html_sha1":"27aa16bc1f148189e3cbf8c3b5a45cc996e39b86","truth_sha1":"8f710da33688106fd06449a8d4e205181743ac27","overlaps_test":false,"full_text_reference":true}, +{"id":"0277","category":"article","split":"dev","html_sha1":"36dbd71c8bb775f7fd380abce8ee155633a52b68","truth_sha1":"448382a35489deec27e47dc0061629e9294c0797","overlaps_test":false,"full_text_reference":true}, +{"id":"0278","category":"article","split":"dev","html_sha1":"99c9f654448e3311e77729730e7cd5411d63d314","truth_sha1":"4755de411ff696dbc449ef7a68d471c95a72cea9","overlaps_test":false,"full_text_reference":true}, +{"id":"0279","category":"article","split":"dev","html_sha1":"53b6d6aa77a67544961947bbd41d1f7941e2d973","truth_sha1":"0fbf65aba866fe55542d45350131b9cd5035c880","overlaps_test":false,"full_text_reference":true}, +{"id":"0280","category":"article","split":"dev","html_sha1":"e8c17effe9eb6098eb74b0fbea2c124f342cfaca","truth_sha1":"c48a54a2705f839e5e89af414139efca81275447","overlaps_test":false,"full_text_reference":true}, +{"id":"0281","category":"article","split":"dev","html_sha1":"4e11db2cdaecf3867d537a7092467751af1ea638","truth_sha1":"488414e73aedec1f6737c1444b65239a6156d8a7","overlaps_test":false,"full_text_reference":true}, +{"id":"0282","category":"listing","split":"dev","html_sha1":"94a700a55b584f253b06799f13ab0ce94b35d835","truth_sha1":"19ae3c02be9696ba107c2d1ece8eb719fcd749b7","overlaps_test":false,"full_text_reference":true}, +{"id":"0283","category":"article","split":"dev","html_sha1":"469f100c066492c2359ed5b7fbb249ba369d4e74","truth_sha1":"3c0021ad328cf58a1036c51325c97803bc3f2147","overlaps_test":false,"full_text_reference":true}, +{"id":"0284","category":"article","split":"dev","html_sha1":"303c2889753ff701c2063f15b2356c7df829cce3","truth_sha1":"009927d6b2d062ff7650ee43af2fd445fc8bcb4f","overlaps_test":false,"full_text_reference":true}, +{"id":"0285","category":"article","split":"dev","html_sha1":"dd7f824804800f53244356b448323afd43993fba","truth_sha1":"10274b920e1c37492d83be91921d6b2a9e899b25","overlaps_test":false,"full_text_reference":true}, +{"id":"0286","category":"article","split":"dev","html_sha1":"8543ea6e683f8bd0c2652e34612df016739a7255","truth_sha1":"8db1b9fc378eae5dc4663d4696fc14aa9ade4380","overlaps_test":false,"full_text_reference":true}, +{"id":"0287","category":"article","split":"dev","html_sha1":"1a2c510b3694f5fd3446c47b33db77532843df4d","truth_sha1":"80249fa410a6f50ca524e77c9426165d6e1ab5d0","overlaps_test":false,"full_text_reference":true}, +{"id":"0288","category":"service","split":"dev","html_sha1":"ae64d39a98ea943dc7c314863fdd410eb553a826","truth_sha1":"289b45abcacff496a8d72c3e022cfcefc89a7d24","overlaps_test":false,"full_text_reference":true}, +{"id":"0289","category":"article","split":"dev","html_sha1":"0ecb85937fb0547fcea599d1c246de21285367d1","truth_sha1":"61d291d8b55c7b92197d1f36781b410703b24299","overlaps_test":false,"full_text_reference":true}, +{"id":"0290","category":"article","split":"dev","html_sha1":"08d00df420d88094faf1a77e953a699330203dc4","truth_sha1":"3b9f448d967f7740e0155236618ed0ee3f2e3545","overlaps_test":false,"full_text_reference":true}, +{"id":"0291","category":"article","split":"dev","html_sha1":"df36e0db7ad187ef2ab7089ed7e6b57003f60972","truth_sha1":"21f10743cfec193df6127beb403941f6e59df945","overlaps_test":false,"full_text_reference":true}, +{"id":"0292","category":"service","split":"dev","html_sha1":"4be32b27aeff29be73078fa6fb4ed27f7c0b5eb1","truth_sha1":"d61339890e99bdb643bcb7a24c8f74b94329dd47","overlaps_test":false,"full_text_reference":true}, +{"id":"0293","category":"article","split":"dev","html_sha1":"bb6c3ec5e188ce9ce916f51f4aa2e24bc4eb6683","truth_sha1":"f3c25721f6ca7e49b433dde22a3a8ab8df00a6a9","overlaps_test":false,"full_text_reference":true}, +{"id":"0294","category":"article","split":"dev","html_sha1":"28fb43100379b4e1cef2f66c4bd971de1c2e9715","truth_sha1":"a37ea9dc0d85d86d49667a404c7f5957df514a18","overlaps_test":false,"full_text_reference":true}, +{"id":"0295","category":"article","split":"dev","html_sha1":"ceeea6d1e80e60da28ba840c80ccbabb68f55fc4","truth_sha1":"93fda936966b33e5be3aa79be3b4270ad0692046","overlaps_test":false,"full_text_reference":true}, +{"id":"0296","category":"article","split":"dev","html_sha1":"82bb1742445b4cea57af7deda7f8a2e5fe36da48","truth_sha1":"475f0625172f82f0bf45f3b9445d162a43d62335","overlaps_test":false,"full_text_reference":true}, +{"id":"0297","category":"article","split":"dev","html_sha1":"e1d1c5b7ac978a389e1cb82b8ae36ae65c54abb9","truth_sha1":"0269ed0e5031b1af6443e05b31c442530641ed56","overlaps_test":false,"full_text_reference":true}, +{"id":"0298","category":"article","split":"dev","html_sha1":"f65224b59d6b2760aa9b64edb614025e3b8a5f30","truth_sha1":"eafbe6097a25a99170b25ed4b7a2edd875b148b1","overlaps_test":false,"full_text_reference":true}, +{"id":"0299","category":"article","split":"dev","html_sha1":"55738776e3c6fe9b3b204596159007451d90e699","truth_sha1":"3468904b6ca987c4ed84d8e99aeb58abd0c6f994","overlaps_test":false,"full_text_reference":true}, +{"id":"0300","category":"article","split":"dev","html_sha1":"249d83ed1af35df6a8c8edea986981128be61203","truth_sha1":"5b2f5d06e125185d5f008241fa6905adfbcfa08f","overlaps_test":false,"full_text_reference":true}, +{"id":"0301","category":"article","split":"dev","html_sha1":"834251df585f7afcd7dd13ddb7bf7270d07906fb","truth_sha1":"040f21c7d415644649685501f73bbfa2860f38ed","overlaps_test":false,"full_text_reference":true}, +{"id":"0302","category":"service","split":"dev","html_sha1":"6fa86e008875d1d1bac1438d991452e1501304c9","truth_sha1":"66a4b69e36dc50fe69daf5e7a8537d8b08e213e6","overlaps_test":false,"full_text_reference":true}, +{"id":"0303","category":"article","split":"dev","html_sha1":"d140055224853904bce3e2b5055a01f0744221fe","truth_sha1":"64b8b478cb1343d987e05506b65f65c074d8255f","overlaps_test":false,"full_text_reference":true}, +{"id":"0304","category":"article","split":"dev","html_sha1":"61d6e70121375c1764e7bc2706902024e82b49bd","truth_sha1":"12c7504a43a7c76170ff511391a10721048cc21e","overlaps_test":false,"full_text_reference":true}, +{"id":"0305","category":"article","split":"dev","html_sha1":"a6abb6fa6e04abaabd2fe5af6d72ddb01c998b1d","truth_sha1":"8cbb3dbc7e70e8ef9cc49d1db030b39d3111eec3","overlaps_test":false,"full_text_reference":true}, +{"id":"0306","category":"article","split":"dev","html_sha1":"6848e9275135bbb5db9a59314f6a19b1f53d5114","truth_sha1":"e49088733057ded99789f49c40f60569e5906d3f","overlaps_test":false,"full_text_reference":true}, +{"id":"0307","category":"article","split":"dev","html_sha1":"94124d4fe1af0fb0a6c563a57152619ae1deef10","truth_sha1":"aa3da46aa1833c87c4af29357300042dcab4534e","overlaps_test":false,"full_text_reference":true}, +{"id":"0308","category":"article","split":"dev","html_sha1":"c37b0c148de58acc3070c6a6890f75a241e2c705","truth_sha1":"b01d9b6db2409bb5e4bf2d3582c3677e699b2ebe","overlaps_test":false,"full_text_reference":true}, +{"id":"0309","category":"article","split":"dev","html_sha1":"56cee45adc91bd51a1af879c56393eb71ec92548","truth_sha1":"f8744321ff3a5c364541bb06718cbb8843a47132","overlaps_test":false,"full_text_reference":true}, +{"id":"0310","category":"article","split":"dev","html_sha1":"6b02a90b20da8058b62967c63a49239b57833fd4","truth_sha1":"95b4c868bc4a0e45b97c9494cecfa4b0b30e9f49","overlaps_test":false,"full_text_reference":true}, +{"id":"0311","category":"service","split":"dev","html_sha1":"9589c7a5178043829c5266d56a87a4053294dcb8","truth_sha1":"8bee00658985a8b919000ec0fd843e030cf2e732","overlaps_test":false,"full_text_reference":true}, +{"id":"0312","category":"article","split":"dev","html_sha1":"0435973090d081670d3c00f46458da8b9943f79f","truth_sha1":"eb1ae2fa2a61f466d51425e0dbaa412e3ff8f948","overlaps_test":false,"full_text_reference":true}, +{"id":"0313","category":"article","split":"dev","html_sha1":"d730f1faf85b7804ea0170713beb8e92c2a9837f","truth_sha1":"cf31307a0a922d4ac051889b30aba2acfdc1a3cd","overlaps_test":false,"full_text_reference":true}, +{"id":"0314","category":"article","split":"dev","html_sha1":"5e025cfbd7cf49dde5301579a674d275be44700e","truth_sha1":"d277d0e2cdf2378bb88277342f732e06ce101472","overlaps_test":false,"full_text_reference":true}, +{"id":"0315","category":"article","split":"dev","html_sha1":"4a3a39d61ddbe88113dc0fec6168a68de6dd026e","truth_sha1":"4e77185b1ce0f8e6a35022f959b507e768ce4423","overlaps_test":false,"full_text_reference":true}, +{"id":"0316","category":"article","split":"dev","html_sha1":"06de3e22237edade734bc9fd80a63e20f5bea67c","truth_sha1":"c9bd58084e160dae81f756894dc62fcd6d0117eb","overlaps_test":false,"full_text_reference":true}, +{"id":"0317","category":"article","split":"dev","html_sha1":"c7ddf6ada85005307f254a6889555802ba331d32","truth_sha1":"db5c39fd6daee97ed17a9834e7357dfd261715a8","overlaps_test":false,"full_text_reference":true}, +{"id":"0318","category":"article","split":"dev","html_sha1":"0b0455fe60457229224f58e13b2161ee761e2adb","truth_sha1":"174cb7d3327fa258f98ea9db449b93e70d5aaa11","overlaps_test":false,"full_text_reference":true}, +{"id":"0319","category":"article","split":"dev","html_sha1":"9cef1e0e551ef127928354a9aba65aa3955206bc","truth_sha1":"2e9b6efe2d274e3fc00e7b8e8a18fbbade308869","overlaps_test":false,"full_text_reference":true}, +{"id":"0320","category":"article","split":"dev","html_sha1":"4fdd1845407d31f3c0dc5703eaa30cbd1dac6e65","truth_sha1":"bd1d38540957cbdf15171a0d208ebd5e07db9a25","overlaps_test":false,"full_text_reference":true}, +{"id":"0321","category":"article","split":"dev","html_sha1":"0c25b22d5cf17165dc0ec9cd9a9f855c7affb210","truth_sha1":"07f5d9cb4ae222003d4d3f2621e6d6eec46753d3","overlaps_test":false,"full_text_reference":true}, +{"id":"0322","category":"article","split":"dev","html_sha1":"cff6ccd1a5c0a8ce329d966f4e2e276f40fdba02","truth_sha1":"6a4997057655bb027ff5e248295365af89a5a673","overlaps_test":false,"full_text_reference":true}, +{"id":"0323","category":"article","split":"dev","html_sha1":"7a4835996d17660ce271b7a679438f5f0a99f9b4","truth_sha1":"4fe16b1bda5348357eff7409903a446b950525f5","overlaps_test":false,"full_text_reference":true}, +{"id":"0324","category":"article","split":"dev","html_sha1":"33cb55b38a8a80984bbfc47a437fcee54ae88d4e","truth_sha1":"47699d0fed7d2cd6c6b5d29ab1fe5805b646f242","overlaps_test":false,"full_text_reference":true}, +{"id":"0325","category":"article","split":"dev","html_sha1":"00694ad707770a01f2f45558d81b4de4aa0ba1a4","truth_sha1":"6c1215a6599bb9c14e29a14df6fee64550af2c83","overlaps_test":false,"full_text_reference":true}, +{"id":"0326","category":"service","split":"dev","html_sha1":"99a571b192016e7efafc37e95a8a83d610c1cc8a","truth_sha1":"2dd7d55d5249d62c6b72f5cc4589ed4b9d9fcf0b","overlaps_test":false,"full_text_reference":true}, +{"id":"0327","category":"article","split":"dev","html_sha1":"290664690761b109f9b556be8ce69035d53fd209","truth_sha1":"5a4d7a935e107317e3652a08ddfae6feac280b8f","overlaps_test":false,"full_text_reference":true}, +{"id":"0328","category":"article","split":"dev","html_sha1":"d6b447e21f2d128be923a98ea3cc99e1970e2a3a","truth_sha1":"7bf2ce51994ab2e30924b173a24600a088a286ad","overlaps_test":false,"full_text_reference":true}, +{"id":"0329","category":"article","split":"dev","html_sha1":"101cefc09363378aaf1d50d7af2a5ab1c1f93331","truth_sha1":"11c425ab0f48c4383af0d34725f9d9866c332077","overlaps_test":false,"full_text_reference":true}, +{"id":"0330","category":"article","split":"dev","html_sha1":"9a3dd6cd0856493d47efd78a4ee4abff84ba2968","truth_sha1":"dd35fe7f5be77b6624a2072cb923cabe15ba90f7","overlaps_test":false,"full_text_reference":true}, +{"id":"0331","category":"article","split":"dev","html_sha1":"6b4d832cdac46f0811161938b69abede94effbc3","truth_sha1":"ee1609bb04be4ffbf8811ac257a14c2b101c3e37","overlaps_test":false,"full_text_reference":true}, +{"id":"0332","category":"article","split":"dev","html_sha1":"a19d651214d09b90dc3b5e62679a069bd251ec10","truth_sha1":"94a20b0a12413f8a724fa8df263040232f51f6e9","overlaps_test":false,"full_text_reference":true}, +{"id":"0333","category":"article","split":"dev","html_sha1":"7a6d0e57525999c5f5f1848993a61dcf8de123c4","truth_sha1":"f6ef22793ab5a65dd3db696927ec2429d7bede6c","overlaps_test":false,"full_text_reference":true}, +{"id":"0334","category":"article","split":"dev","html_sha1":"e3bc2eacd2f1e4b14c6e1ad7ff9be6dd4e1151e9","truth_sha1":"3c3b27991cfb4f12890a50dd432700c803231155","overlaps_test":false,"full_text_reference":true}, +{"id":"0335","category":"article","split":"dev","html_sha1":"9ab80cc8734e1c42ffa1780a77aa5d7e304e6e72","truth_sha1":"7fef04f324a843fb7a03ebd6fc2d3616391a26ea","overlaps_test":false,"full_text_reference":true}, +{"id":"0336","category":"article","split":"dev","html_sha1":"4d567bda1cd869ed181fdfdc881f61854c9858aa","truth_sha1":"05d349a59bbfb0e3f8c700a925100c2e4e0cba09","overlaps_test":false,"full_text_reference":true}, +{"id":"0337","category":"listing","split":"dev","html_sha1":"4a1cf40b32345bd872bcaef89acd7094f20d6bcc","truth_sha1":"394b25ee9c4f6a29a50b9e21f2cb7cf311dbbe50","overlaps_test":false,"full_text_reference":true}, +{"id":"0338","category":"service","split":"dev","html_sha1":"798af68a24e8e8e28c4928cd4773c3515c26508d","truth_sha1":"1ceadc1e94e6494048445ce6a991caf35c3d06a9","overlaps_test":false,"full_text_reference":true}, +{"id":"0339","category":"article","split":"dev","html_sha1":"2554454078b3bbc65d398340f5c205efe8ac6106","truth_sha1":"c0c6f4d2026ec4f79b9200d9b1ab43a5222937eb","overlaps_test":false,"full_text_reference":true}, +{"id":"0340","category":"service","split":"dev","html_sha1":"3a01cb34e7e86378de017ab21e1a2fac67b8acf4","truth_sha1":"d99258d3813085ff31147cdadbf258097bd12fb3","overlaps_test":false,"full_text_reference":true}, +{"id":"0341","category":"article","split":"dev","html_sha1":"7f335443aeb0fae6b99fe93d008ca84d2f57a8d0","truth_sha1":"82fe0c40d1f36a2b48bb0ac66bf0fb790ae0cb7c","overlaps_test":false,"full_text_reference":true}, +{"id":"0342","category":"article","split":"dev","html_sha1":"d72330bf307a0cceb004fa985deccfed6df29b72","truth_sha1":"52c9a35bb4b87be6e0dd27a55acbbfa81219134f","overlaps_test":false,"full_text_reference":true}, +{"id":"0343","category":"article","split":"dev","html_sha1":"4563c08e3c915b417c7f1f1fe6c4b5792f4f3eac","truth_sha1":"8def06446e38591ebcb8a698dea44c36d20203ff","overlaps_test":false,"full_text_reference":true}, +{"id":"0344","category":"article","split":"dev","html_sha1":"be8495d67bb252b542fa00d1d86f075c9687b36f","truth_sha1":"a6e169483fef5d51722b35449c67bbceada3b733","overlaps_test":false,"full_text_reference":true}, +{"id":"0345","category":"article","split":"dev","html_sha1":"27316bd587a526a9bbf2d02c8e782717af32aaf3","truth_sha1":"b1f051a97b0134a04bccb03410347a80084fa8ec","overlaps_test":false,"full_text_reference":true}, +{"id":"0346","category":"article","split":"dev","html_sha1":"85eee161b78d060ac000b8393140f4f68a7612f1","truth_sha1":"085278f6945ca95f9fd058ff8b8c771b0c2daa21","overlaps_test":false,"full_text_reference":true}, +{"id":"0347","category":"article","split":"dev","html_sha1":"3bdb74d7b56aa36f97af19a0a806c6cb5f010f57","truth_sha1":"8c42965f1ff293e4893e1ffbc5ff0db6f15f8475","overlaps_test":false,"full_text_reference":true}, +{"id":"0348","category":"article","split":"dev","html_sha1":"48e9d8b9b5f6b8d138ec4a7acbe3792a97857e0b","truth_sha1":"060f3b1a6d746fb45eeb63d29cc6b281ce60d494","overlaps_test":false,"full_text_reference":true}, +{"id":"0349","category":"documentation","split":"dev","html_sha1":"beedc12fa2ed3869ad58178e61ca3c33987a8d65","truth_sha1":"0ecea24ec5eb9e1ab492d958d9a525e85bc34370","overlaps_test":false,"full_text_reference":true}, +{"id":"0350","category":"article","split":"dev","html_sha1":"af5262b8325ae4d5f460a6d1a68e62d55f59fe96","truth_sha1":"1953363d37f0f82792e62536defed8367fbd03ac","overlaps_test":false,"full_text_reference":true}, +{"id":"0351","category":"article","split":"dev","html_sha1":"12ce75c788e21f4e0dac6c31a7a4acc388a4d8c5","truth_sha1":"3e2f820db1da9e6528764928813b23758771a144","overlaps_test":false,"full_text_reference":true}, +{"id":"0352","category":"article","split":"dev","html_sha1":"6774a36e99e33b8ca1ad4f3b06e34efcfc97ffb5","truth_sha1":"e29d3662bda2ceb37dc08b1934fb0ace399277eb","overlaps_test":false,"full_text_reference":true}, +{"id":"0353","category":"article","split":"dev","html_sha1":"73d2bb725bd735d632bdfc5729fae78b92f13aa7","truth_sha1":"3b0bd3f8f7a70aaf07a689e638f66d1b6acf1bdc","overlaps_test":false,"full_text_reference":true}, +{"id":"0354","category":"article","split":"dev","html_sha1":"4972bb43ffab4764877be4dfb247d965d2b763cc","truth_sha1":"5ee7f10447d2c6c459f1f3621f35b96dd8fe8717","overlaps_test":false,"full_text_reference":true}, +{"id":"0355","category":"article","split":"dev","html_sha1":"952a51b954791f088a79e2b8f3f7289742fa54f7","truth_sha1":"6b7946b675d995667ab0a87f1f69181aa6f76e5e","overlaps_test":false,"full_text_reference":true}, +{"id":"0356","category":"article","split":"dev","html_sha1":"5cfa750240da30619518e634b2875b5cba69d251","truth_sha1":"916fcc7ab96867053ff5b6b32de224f038d3027a","overlaps_test":false,"full_text_reference":true}, +{"id":"0357","category":"article","split":"dev","html_sha1":"44651d5650b43e1796377e5ee9ab4c159810d4b4","truth_sha1":"ac373cbb845f4f8ec1db9dd2455abf3b2cff97b7","overlaps_test":false,"full_text_reference":true}, +{"id":"0358","category":"article","split":"dev","html_sha1":"da01af2ea7564c17af659e043acdaa25fd4ea56f","truth_sha1":"36d18d20edd8f0d97ae06b91250f757ee85c90ce","overlaps_test":false,"full_text_reference":true}, +{"id":"0359","category":"article","split":"dev","html_sha1":"4fbdf0497f81348622362f810cc027d243c72cff","truth_sha1":"7c4945ada9439b906766efb17d32f608224873d3","overlaps_test":false,"full_text_reference":true}, +{"id":"0360","category":"forum","split":"dev","html_sha1":"fdb86cbba13dae7c345d549f87bfd2ce18d8bb84","truth_sha1":"c6bf47615d3f6182599ae21dffa96b4c31364c0a","overlaps_test":false,"full_text_reference":true}, +{"id":"0361","category":"article","split":"dev","html_sha1":"1f953be98dfc1c2ced87bd89eff759f9fb8b9075","truth_sha1":"5c8b847ccc72f5b52cfc76cbc6f9e283612e4cf2","overlaps_test":false,"full_text_reference":true}, +{"id":"0362","category":"article","split":"dev","html_sha1":"a104f58e56fcec31da535c056a9ce9bd702eb8f6","truth_sha1":"a3ceb16352990fa9dc28b65b260623fca1e7f8e4","overlaps_test":false,"full_text_reference":true}, +{"id":"0363","category":"article","split":"dev","html_sha1":"f2b4a5e7932db4b1e6167b38b65b80ce0322642c","truth_sha1":"a45019788b143e8380df2b7fb2a6c6cf39056b8c","overlaps_test":false,"full_text_reference":true}, +{"id":"0365","category":"article","split":"dev","html_sha1":"3844da2c4baea9fb71b1e777640eca56a3d59736","truth_sha1":"ef0a6674737ed2b6d6402650a57c9dc0e0f86757","overlaps_test":false,"full_text_reference":true}, +{"id":"0366","category":"article","split":"dev","html_sha1":"99f2299046719292a25c5ab500e87450000da63d","truth_sha1":"d71c6e1251a7e8db0376c6b96132a315f722c4b5","overlaps_test":false,"full_text_reference":true}, +{"id":"0367","category":"article","split":"dev","html_sha1":"4b6dff4d82db479e493d428f9b380092cf1059ca","truth_sha1":"3ac1bf49019c76f8cfb5a1068caa6d01ab29fe58","overlaps_test":false,"full_text_reference":true}, +{"id":"0368","category":"article","split":"dev","html_sha1":"c4313e385d780251ea59d57fca667b96b9f778d8","truth_sha1":"0610c29cf829c7a0165146d474840f24c2d68221","overlaps_test":false,"full_text_reference":true}, +{"id":"0369","category":"documentation","split":"dev","html_sha1":"e7979f26ef57a6201a3d82214c3f243d76aee595","truth_sha1":"eef2b2152eb8cc83222eaaf81e39c12657c26f58","overlaps_test":false,"full_text_reference":true}, +{"id":"0370","category":"article","split":"dev","html_sha1":"686721f3e5f8f30603aa5ddd9de49458eed211da","truth_sha1":"6a7a6dcf61715ce8f05177fc52ab7b6ee304a570","overlaps_test":false,"full_text_reference":true}, +{"id":"0371","category":"article","split":"dev","html_sha1":"d0d389294a33313e1a9b8e681cdf5df0ec43fd54","truth_sha1":"9d489e571d0d47f66a5083b969e8b47a9bed8fa7","overlaps_test":false,"full_text_reference":true}, +{"id":"0372","category":"article","split":"dev","html_sha1":"fdb63ac25582eea4ed672198208f0bce8c784c5c","truth_sha1":"fd97094a8180238e1a13bbd0cda1539f1d5f2d2d","overlaps_test":false,"full_text_reference":true}, +{"id":"0373","category":"article","split":"dev","html_sha1":"c7073f79cf1407a5af6e8a96c5d50cf64692aa65","truth_sha1":"450aa9f931a7d43439fbd1c820ad6347d42cb180","overlaps_test":false,"full_text_reference":true}, +{"id":"0374","category":"service","split":"dev","html_sha1":"c0e1531840ec8353a47d57d3165e0da16fdea018","truth_sha1":"7634023a8f9c6fa22427df1846dfda082e567bc1","overlaps_test":false,"full_text_reference":true}, +{"id":"0375","category":"article","split":"dev","html_sha1":"5e81e86e33302dd1bb659fb80fa383a3391cab1f","truth_sha1":"0be75b04256708407799608ed8138594d4fdfee3","overlaps_test":false,"full_text_reference":true}, +{"id":"0376","category":"service","split":"dev","html_sha1":"62f8a51875f2f7f6551d02d4b8f085943911387b","truth_sha1":"5bd8884f589d0de6cc967d212ed8fe32d89e54e2","overlaps_test":false,"full_text_reference":true}, +{"id":"0377","category":"service","split":"dev","html_sha1":"95f0027bc212e13efe2d62eafccc09b94b318cb4","truth_sha1":"3b832769b916939689b23f57304fb2a1195b6058","overlaps_test":false,"full_text_reference":true}, +{"id":"0378","category":"article","split":"dev","html_sha1":"fd7f25ae99c9877dd51155616191211a194e7676","truth_sha1":"53132c8ab22792e1be5007517e8d26efa18cbaae","overlaps_test":false,"full_text_reference":true}, +{"id":"0379","category":"article","split":"dev","html_sha1":"8c4f80fa5bbce2ce830b4711802ac0f49fd8d997","truth_sha1":"63d1c01b9eb254be43bcd4e6fd58f9090f455f0d","overlaps_test":false,"full_text_reference":true}, +{"id":"0380","category":"article","split":"dev","html_sha1":"b31861b06ca54d6bb047d35b2f689d40643b478c","truth_sha1":"2b93e79fef750eb54bffdbbcf3b4635acf33cb18","overlaps_test":false,"full_text_reference":true}, +{"id":"0381","category":"article","split":"dev","html_sha1":"6a3f284a7bda35e8baedaadb420717436ae4f862","truth_sha1":"c329f294e4df48d38d2b4cea4d7f5ff4f1036517","overlaps_test":false,"full_text_reference":true}, +{"id":"0382","category":"article","split":"dev","html_sha1":"d1103711f52f173fc6ed08f962b8a1bc1a62f729","truth_sha1":"053d31a6b0e75df74e5b033aeab515407273662b","overlaps_test":false,"full_text_reference":true}, +{"id":"0383","category":"article","split":"dev","html_sha1":"87d6f2b28740baf0b49b311c6f01f0952ec86397","truth_sha1":"e29375b269f7d06f7fb979eafdc93a3d53aaec88","overlaps_test":false,"full_text_reference":true}, +{"id":"0384","category":"article","split":"dev","html_sha1":"797efa8d2eca2f09339027d431211a930c868379","truth_sha1":"053f935992e766a827b43f3c01410819c91fb9d5","overlaps_test":false,"full_text_reference":true}, +{"id":"0385","category":"article","split":"dev","html_sha1":"ddc3c45b4904a46749b834263df1f5b0e79f4fa0","truth_sha1":"4a7de648fe269e238b2089509e78101892a9b940","overlaps_test":false,"full_text_reference":true}, +{"id":"0386","category":"article","split":"dev","html_sha1":"d531ef153aa99e2dc8edc59e3788c0d846964b4f","truth_sha1":"01c8a2cdc09f590ea8231ed6b748584972548632","overlaps_test":false,"full_text_reference":true}, +{"id":"0387","category":"article","split":"dev","html_sha1":"1a80353ad798373818ee598c966f6eef1b95c8b7","truth_sha1":"110de8ab55d34e2cee11c08dfcda4c772a9f889b","overlaps_test":false,"full_text_reference":true}, +{"id":"0388","category":"article","split":"dev","html_sha1":"f7ef3301a8b4633e376a8dbad5c73540c511af94","truth_sha1":"6d9ba1fe2bba045f4f1f5d02299b3a271d913a6e","overlaps_test":false,"full_text_reference":true}, +{"id":"0389","category":"article","split":"dev","html_sha1":"1f7488e5811e0426e26b7a559cae87eda8d859c0","truth_sha1":"3241b32e9cddc03acef26cb600def9f29d417462","overlaps_test":false,"full_text_reference":true}, +{"id":"0390","category":"article","split":"dev","html_sha1":"e007d611353eba348d526d5d3c455a5e2eda861a","truth_sha1":"1add5ea947706fcb6ef0cb0721ecc3839bf02540","overlaps_test":false,"full_text_reference":true}, +{"id":"0391","category":"article","split":"dev","html_sha1":"f241c5d5ad5ef62a659df3a42bb132ec7f39e663","truth_sha1":"cf72704ca5ab2e9cd7af47c59d07ae15e62129c4","overlaps_test":false,"full_text_reference":true}, +{"id":"0392","category":"article","split":"dev","html_sha1":"4908a4b68ded8a4d3f98f67dc180a62cf6bec25a","truth_sha1":"90038cbcf199514bb1545aaa80ade039e200be26","overlaps_test":false,"full_text_reference":true}, +{"id":"0393","category":"article","split":"dev","html_sha1":"49917b6a0a3ac01e57f39401c77f0408daae0e85","truth_sha1":"e31fe3539d1f6c37521abc8f078a06c31adeebc3","overlaps_test":false,"full_text_reference":true}, +{"id":"0394","category":"article","split":"dev","html_sha1":"a3746ea5f22ecaa963ed4549291efd878454fc57","truth_sha1":"76d8380979ff9bcddfd628668710a5813aba9a53","overlaps_test":false,"full_text_reference":true}, +{"id":"0395","category":"article","split":"dev","html_sha1":"7ede33dcb689c38877b3011b346d556beb5fdf35","truth_sha1":"b9aa28d6e3aa8b34fa33f401a02a28deb1eb2d18","overlaps_test":false,"full_text_reference":true}, +{"id":"0396","category":"article","split":"dev","html_sha1":"8b44631b62c175a5f52074e817f6b7a60528d000","truth_sha1":"b55a22f5ed3bb7a8aeb87252457ada7a9187237a","overlaps_test":false,"full_text_reference":true}, +{"id":"0397","category":"article","split":"dev","html_sha1":"17617d1311d847a2db35c80bd91c75ee4b448af4","truth_sha1":"eab0e515ed0a942fec99d38c02a6f19a1a1069ba","overlaps_test":false,"full_text_reference":true}, +{"id":"0398","category":"article","split":"dev","html_sha1":"1460f41b5c9a321c2b66a0a763d59f384c35b9e5","truth_sha1":"b3f2996ad864f58f1cd7a701692dda41325bcff3","overlaps_test":false,"full_text_reference":true}, +{"id":"0399","category":"article","split":"dev","html_sha1":"0b7d70844cdbd1efa6793f200a7cc0d71727db81","truth_sha1":"bb8d14f167a435616b7b17af2d8de322211fa5ee","overlaps_test":false,"full_text_reference":true}, +{"id":"0400","category":"article","split":"dev","html_sha1":"2538bf63737449a37aa5482e9e18c8d0fc934ceb","truth_sha1":"76987046230e5724859a5f381e6ebbb4ed53a30d","overlaps_test":false,"full_text_reference":true}, +{"id":"0401","category":"article","split":"dev","html_sha1":"2336d2cafc40820d656919c1d4ffe5464f051432","truth_sha1":"4e02e7307917b43aeac7420db7555cd3a6e3bb14","overlaps_test":false,"full_text_reference":true}, +{"id":"0402","category":"article","split":"dev","html_sha1":"f6e36561803d6a182a461f7c654141fe0ead957a","truth_sha1":"8e22618fcb2d06ad9187903c61174de7f80bf2de","overlaps_test":false,"full_text_reference":true}, +{"id":"0403","category":"forum","split":"dev","html_sha1":"08ae8556dfe0a81d411a583bc517bd98be5f13fa","truth_sha1":"bbadae01f95553613ba3f07dca195eb57e25b50c","overlaps_test":false,"full_text_reference":true}, +{"id":"0404","category":"article","split":"dev","html_sha1":"5139350472fa12e6b267acab72ded05e46c4a2b4","truth_sha1":"5a2a889e2c73e75b461d408920de526feb077351","overlaps_test":false,"full_text_reference":true}, +{"id":"0405","category":"article","split":"dev","html_sha1":"f1571873a3c03c4c24b5403e49e37b79d8db78c7","truth_sha1":"fd3b12c65e053d3cb749ba2a305c580eb2044d97","overlaps_test":false,"full_text_reference":true}, +{"id":"0406","category":"article","split":"dev","html_sha1":"a7e72f646e082b4c50b9be2881370c522315abd5","truth_sha1":"3b64b3f3352a02e5bd511096ec500110d2e13d88","overlaps_test":false,"full_text_reference":true}, +{"id":"0407","category":"article","split":"dev","html_sha1":"9762304a9d1a3b3a847f4a625d4aaf2e4666342b","truth_sha1":"0c94dabc135f366d963baba97f1ec1611afc983a","overlaps_test":false,"full_text_reference":true}, +{"id":"0408","category":"article","split":"dev","html_sha1":"c2f490515467c208432ef7778cf0a402393fbabc","truth_sha1":"80597f6053fa750f927ae2975bfa99105a43a16b","overlaps_test":false,"full_text_reference":true}, +{"id":"0410","category":"article","split":"dev","html_sha1":"bc0417ffe9048e887ca9c1d7a221423a009becdf","truth_sha1":"154cf039cbd3cf4856bb56a6eda1f442a772b2e4","overlaps_test":false,"full_text_reference":true}, +{"id":"0411","category":"article","split":"dev","html_sha1":"3238cad5e03564604c873919a2e050a90f78ea41","truth_sha1":"7621386b733d83b29d4fbb491715f8684d4146ea","overlaps_test":false,"full_text_reference":true}, +{"id":"0412","category":"article","split":"dev","html_sha1":"8572e63f7e4c5e46976ff383284256f9e57cc1dd","truth_sha1":"3411ae985b3f620fc771ebc6903fd448e3f7724b","overlaps_test":false,"full_text_reference":true}, +{"id":"0413","category":"article","split":"dev","html_sha1":"cf893c8dae6c7a96e8da2ee5c94d5bfb87c886fc","truth_sha1":"0fec46b505022ec0cf777a6944149b0cc4fdce54","overlaps_test":false,"full_text_reference":true}, +{"id":"0414","category":"article","split":"dev","html_sha1":"af7ebcc4657b9b7c6c5475b5f651be6899bd85c6","truth_sha1":"50d57e195944c18c6a6f14f14caf4a4efb96a824","overlaps_test":false,"full_text_reference":true}, +{"id":"0415","category":"article","split":"dev","html_sha1":"af5b32f89a9ee18ae0789b40e8ad97642e6bb371","truth_sha1":"c672f9243b05056de88fd9b50c85c4544a525a4e","overlaps_test":false,"full_text_reference":true}, +{"id":"0416","category":"article","split":"dev","html_sha1":"66201de3678c4f218d7a614e56fa4b97b5138d41","truth_sha1":"eb9bcd5b8c03dc380a4ae13cb749e72612dc7837","overlaps_test":false,"full_text_reference":true}, +{"id":"0417","category":"article","split":"dev","html_sha1":"66bfcad41a0ebb11b844360013d79e28d90388be","truth_sha1":"605fe47d55fcbe0c9f3b64398ac3066923bf547a","overlaps_test":false,"full_text_reference":true}, +{"id":"0418","category":"article","split":"dev","html_sha1":"52a8a0143f25084df0689dcd5da6e20e6e14c539","truth_sha1":"a12b1ec2e89cef222396da3ffe7d7d9bfbd7ed3f","overlaps_test":false,"full_text_reference":true}, +{"id":"0419","category":"article","split":"dev","html_sha1":"305abb6965c4e2896357f8c812c83b69fd14ab1b","truth_sha1":"f6dd43442c55a25f29f5522e3d17c1c642361589","overlaps_test":false,"full_text_reference":true}, +{"id":"0420","category":"article","split":"dev","html_sha1":"5d2d407f466fde10e9f953c616564fd4e3f558dc","truth_sha1":"44a5c9d7c30f1897c91d8ac2a1e9b3a1bcc5fbc3","overlaps_test":false,"full_text_reference":true}, +{"id":"0421","category":"article","split":"dev","html_sha1":"98b8ffeae912032f2cd1b287a73975941ee55020","truth_sha1":"710fa2873b8cfc517fe0c2fd53931251b53e5a7c","overlaps_test":false,"full_text_reference":true}, +{"id":"0422","category":"article","split":"dev","html_sha1":"d4eef34e4ad74d2b869abf795de8605dbde56b7e","truth_sha1":"ed3a348b757626e420975293013ac90017315323","overlaps_test":false,"full_text_reference":true}, +{"id":"0423","category":"documentation","split":"dev","html_sha1":"94d5b22f7e0bba1392b6976a5647cbe0cc0c1bee","truth_sha1":"8ac1011ce6587c9a198cf4797187760a640eb6d2","overlaps_test":false,"full_text_reference":true}, +{"id":"0424","category":"documentation","split":"dev","html_sha1":"38c4477173e8ded5116f347d7787441391cb0a13","truth_sha1":"fdfa0482db7441cb562983f3a2c93d868c11c7d4","overlaps_test":false,"full_text_reference":true}, +{"id":"0425","category":"article","split":"dev","html_sha1":"85c05438f5477eda5c2c4b7cebeeb02fa74a3f4e","truth_sha1":"d96bf17b8e48d5db6976202f01f25b98341ac9e5","overlaps_test":false,"full_text_reference":true}, +{"id":"0426","category":"article","split":"dev","html_sha1":"a3f3ae7ab5798627afa654971fa8a9876eeab804","truth_sha1":"5bf11565b47b90ffba1e6483e3ef8cfbbf57f344","overlaps_test":false,"full_text_reference":true}, +{"id":"0427","category":"article","split":"dev","html_sha1":"6d80f1e4a243fad2a36b0d609e6713ccc71262bc","truth_sha1":"24a8f0a91ab72357f6d653884598f12b2f9a7ef7","overlaps_test":false,"full_text_reference":true}, +{"id":"0428","category":"article","split":"dev","html_sha1":"0ec553fdd38d1cc6bc4a8864b8a2d8863657240a","truth_sha1":"3c3a09c729a2a3a34a2687e5fa1e0264c8da326d","overlaps_test":false,"full_text_reference":true}, +{"id":"0429","category":"article","split":"dev","html_sha1":"35be2cbba3c65a16ec720181e194083f123d3c46","truth_sha1":"175f003b31c7e3d9a61d40649a2a1a2e6d53537b","overlaps_test":false,"full_text_reference":true}, +{"id":"0430","category":"article","split":"dev","html_sha1":"a50275ed22abd6819c19f428937d50fe4e639743","truth_sha1":"548486de4c59f8c8ea5e937622ee977ee8084643","overlaps_test":false,"full_text_reference":true}, +{"id":"0431","category":"article","split":"dev","html_sha1":"542ad99f99888882d70f2ec1eb3a3ea98cbd7a64","truth_sha1":"df12ea5fb02226ec760c6667cbdc76528a9f181f","overlaps_test":false,"full_text_reference":true}, +{"id":"0432","category":"article","split":"dev","html_sha1":"1325d2e9cb64b77d32712c186d6e5db6343cd47d","truth_sha1":"b4ba81a7507b558170525c190357cc273294a5e5","overlaps_test":false,"full_text_reference":true}, +{"id":"0433","category":"article","split":"dev","html_sha1":"efba84c2ea9de931eed7861e1b28226ee9694e7f","truth_sha1":"cdb1e3c27797021d5badb63e5acba39fa3caa193","overlaps_test":false,"full_text_reference":true}, +{"id":"0434","category":"article","split":"dev","html_sha1":"350446d8591dde12d88e7f1c3dbad7eacc632c43","truth_sha1":"8b6c30930d218649133712b8521abe1fc12f7e71","overlaps_test":false,"full_text_reference":true}, +{"id":"0435","category":"article","split":"dev","html_sha1":"8eb69783d315e4b190d20b442c3fcc8f28e48bc8","truth_sha1":"70131a0b7cce7852fb7a4471416b534da13cdbe0","overlaps_test":false,"full_text_reference":true}, +{"id":"0436","category":"article","split":"dev","html_sha1":"df0cde8c10419a69f69ee96102ebcfcf7409807b","truth_sha1":"aa829b43af29e8f097021b984ea1732d5f49330f","overlaps_test":false,"full_text_reference":true}, +{"id":"0437","category":"article","split":"dev","html_sha1":"8690138d86641e3261538120d5449e69c8f896bc","truth_sha1":"8f9e64c542912d91d69aa78d457032d416934bde","overlaps_test":false,"full_text_reference":true}, +{"id":"0438","category":"article","split":"dev","html_sha1":"310ad3d667df619373f7fb1fa32444e98e40e008","truth_sha1":"fe5b493ea8dd26cbea79869ff598b73da838e422","overlaps_test":false,"full_text_reference":true}, +{"id":"0439","category":"article","split":"dev","html_sha1":"5adb5bd77661cafd9953324b2b19b3355f019cf9","truth_sha1":"5ffe26cc5161077749cff75145bd605bc9a98435","overlaps_test":false,"full_text_reference":true}, +{"id":"0440","category":"article","split":"dev","html_sha1":"bbd0ba692827d77432dc4e3b25c551fcb6c50e8d","truth_sha1":"8104693cdef7e2cc1a2a93e81c5a4ec87749ff31","overlaps_test":false,"full_text_reference":true}, +{"id":"0441","category":"article","split":"dev","html_sha1":"f4b06bd369f2611d9788d77c6914ae181f54b2f9","truth_sha1":"0d0c3f167ad826016adbc6771422c390ba1e71fd","overlaps_test":false,"full_text_reference":true}, +{"id":"0442","category":"article","split":"dev","html_sha1":"043d061bfcbc3211465b9eb0e6b35c2e78766250","truth_sha1":"8f29df02e06221f3ddb426277fe9d891eda07b06","overlaps_test":false,"full_text_reference":true}, +{"id":"0443","category":"article","split":"dev","html_sha1":"8e8d7cf7afe39038aa9ad6b7e013012c5fe82f5a","truth_sha1":"ff0fa112e1e9da3ff7a48f1ceb00373bf44c4658","overlaps_test":false,"full_text_reference":true}, +{"id":"0444","category":"article","split":"dev","html_sha1":"f5839979e2d27a9bb8c6e413a8b4ece2b074ad35","truth_sha1":"6f628e0fa30df491d907c4a92d63819599749365","overlaps_test":false,"full_text_reference":true}, +{"id":"0446","category":"article","split":"dev","html_sha1":"6f0f840839888e6caca32abb86665703a208e47c","truth_sha1":"541d270f2d60b75d6d7c6717ef87c54b734aaeda","overlaps_test":false,"full_text_reference":true}, +{"id":"0447","category":"article","split":"dev","html_sha1":"ee73e7b7f34f14f3d5d1c9f2278d1136f06373ae","truth_sha1":"80db3e9521a44d0871d0fb623b04d841cc68ef5a","overlaps_test":false,"full_text_reference":true}, +{"id":"0448","category":"article","split":"dev","html_sha1":"79cebbc354c28e3dbf21eb16409055aaf7f0fa77","truth_sha1":"8acc330f138f148c190153f684c2f5fb7de72c21","overlaps_test":false,"full_text_reference":true}, +{"id":"0449","category":"article","split":"dev","html_sha1":"2ddfe6b35ef3416cbbeb8f7def519d1e58cc0e9f","truth_sha1":"8e59c383d65acd8c5de8bdbfd76218e23f467107","overlaps_test":false,"full_text_reference":true}, +{"id":"0450","category":"article","split":"dev","html_sha1":"aef046dfa2052afe6b355256ca59c6962c51de7d","truth_sha1":"6b198e7f2cbe88c023d5d01be5684aee8a975b9f","overlaps_test":false,"full_text_reference":true}, +{"id":"0451","category":"article","split":"dev","html_sha1":"994fb6d1ab4ac7136e4689e58d721c9d51aeabf2","truth_sha1":"6d2addd6af4951737631e97eae850ea307b90df5","overlaps_test":false,"full_text_reference":true}, +{"id":"0452","category":"article","split":"dev","html_sha1":"c632a3904bff8157058b8be40d5ccd49bef9106d","truth_sha1":"756370a6a89e15526855850c31cf3a91e0c06e83","overlaps_test":false,"full_text_reference":true}, +{"id":"0453","category":"article","split":"dev","html_sha1":"1872f7f47c316b2d196863be724c9bf889b5cfd6","truth_sha1":"a4463ee15bdb353b4528d9ffe9e3091eccf3f9e4","overlaps_test":false,"full_text_reference":true}, +{"id":"0454","category":"article","split":"dev","html_sha1":"57bfeb5233cb7463c5c0eb51b9a6dc911e95fd08","truth_sha1":"22ad5956e137314cd8eb0360acbb02a93f1bc803","overlaps_test":false,"full_text_reference":true}, +{"id":"0455","category":"article","split":"dev","html_sha1":"546f5b356a8d79fe8ebd7f508492aeb9903cccc2","truth_sha1":"c90cbb622f2c3d8d290de4b59fdd2363305fa539","overlaps_test":false,"full_text_reference":true}, +{"id":"0456","category":"article","split":"dev","html_sha1":"a4b9765c25f98687dd86971d7e782a7eedd5bc32","truth_sha1":"0eba38d2b3ec2444b8b3016e440ce1f2d28eb084","overlaps_test":false,"full_text_reference":true}, +{"id":"0457","category":"article","split":"dev","html_sha1":"261c2591887a8606b072013f6916bf996c3c1427","truth_sha1":"7603469d0698b9e598ce1a2bb4bfac6d2bcda2c9","overlaps_test":false,"full_text_reference":true}, +{"id":"0458","category":"article","split":"dev","html_sha1":"6536c5c3896f1d15e0cfe9e6c6700c33e5ef2a01","truth_sha1":"d8f074fa0563cab0fa004f485c346d54fac7c719","overlaps_test":false,"full_text_reference":true}, +{"id":"0459","category":"article","split":"dev","html_sha1":"0caabea4e7e7a0d338f488ce1f537a9dfbd91a1c","truth_sha1":"b9e8286707b6546843945d7bf2b16d3a5d54fc15","overlaps_test":false,"full_text_reference":true}, +{"id":"0460","category":"article","split":"dev","html_sha1":"23e10e69818a46d287ec427fb178dd51c07b26e9","truth_sha1":"9a982af3d934ed92669c5e48d1433fcf63d49106","overlaps_test":false,"full_text_reference":true}, +{"id":"0461","category":"article","split":"dev","html_sha1":"b154ba8a608206e7b37b31e125a000f1f0b62c46","truth_sha1":"3600d9e20aba6175a96e2ec67cd81cea2a25ee43","overlaps_test":false,"full_text_reference":true}, +{"id":"0462","category":"article","split":"dev","html_sha1":"a300aedf79b6eb1949179e1d26d529fb1d0bbf50","truth_sha1":"31e089ee6d7e65fb8cfa8e789d84379c9c66782f","overlaps_test":false,"full_text_reference":true}, +{"id":"0463","category":"article","split":"dev","html_sha1":"9e388e2a5263f00a55da9af8713db67b0fd81b1b","truth_sha1":"0d9999297971b45e334160698f196c54e9a2ecd0","overlaps_test":false,"full_text_reference":true}, +{"id":"0464","category":"article","split":"dev","html_sha1":"21565bb6e1f4d14b9b9a1abea2108dc4d1a29c98","truth_sha1":"2dbbb0f0a97099a00e940ad08bfd7e23b7addc97","overlaps_test":false,"full_text_reference":true}, +{"id":"0465","category":"service","split":"dev","html_sha1":"8a4c13d6dc51225075bfab8aa00c6bc54c93ca32","truth_sha1":"ad5042f536b539970be6c378a81e20fed19e1fd8","overlaps_test":false,"full_text_reference":true}, +{"id":"0466","category":"article","split":"dev","html_sha1":"37f515d4d3094e8e029d65a7def0dd0a8203be1d","truth_sha1":"3f0ac2c03dac6464d9f54ef31b96f0a367b200f1","overlaps_test":false,"full_text_reference":true}, +{"id":"0467","category":"article","split":"dev","html_sha1":"53944afe11dbf59d1459a4fd84f24f5a1798c28e","truth_sha1":"4444ff3a8e54018c1ac9b8baa59840e7fd096e42","overlaps_test":false,"full_text_reference":true}, +{"id":"0468","category":"article","split":"dev","html_sha1":"f273fb6787d21df873042e9cc29eae7125c9fd82","truth_sha1":"da289886eb4aa6443cfdfbd5ac90988616c2685a","overlaps_test":false,"full_text_reference":true}, +{"id":"0469","category":"article","split":"dev","html_sha1":"9bc183c0f6de2eac73c7c5d8a0d17ade21791e65","truth_sha1":"0deb4a1daa27f056d29b8e407883c61686461ea3","overlaps_test":false,"full_text_reference":true}, +{"id":"0470","category":"listing","split":"dev","html_sha1":"a3383545d6a4a25f648609a657c104af7a5a6c84","truth_sha1":"5c7cff89a9109faf7b0f798e4c1c08fb5afd5fd7","overlaps_test":false,"full_text_reference":true}, +{"id":"0471","category":"article","split":"dev","html_sha1":"3a862d7c7d4aefe211ac39a02bf1631bfbd8e22c","truth_sha1":"414da215b0780f5e234777e9ceb66bacb8f388f5","overlaps_test":false,"full_text_reference":true}, +{"id":"0472","category":"article","split":"dev","html_sha1":"1e1c0685b2603e1f9dc00d9f4f5260d925837b6e","truth_sha1":"bcd174e84b5cbca3ea1548b4e6e813fec21c2605","overlaps_test":false,"full_text_reference":true}, +{"id":"0473","category":"article","split":"dev","html_sha1":"227e13741f1d58cd45350d388869c74eec3e1ea9","truth_sha1":"6415dbe61b9ea483a66fdde2788a0e46cd624440","overlaps_test":false,"full_text_reference":true}, +{"id":"0474","category":"article","split":"dev","html_sha1":"fe76bf5ff59fedb33c8f5912f7f1284b2e8403f7","truth_sha1":"7f28ebbcdbf4a58fd5c0b4cf52805eff571a3bc8","overlaps_test":false,"full_text_reference":true}, +{"id":"0475","category":"article","split":"dev","html_sha1":"8a9ac23b08fc34c0f128b76b9caca4c6a37856ba","truth_sha1":"9739ee9e16248529ed44506226144476d0219e9e","overlaps_test":false,"full_text_reference":true}, +{"id":"0476","category":"article","split":"dev","html_sha1":"4651548372eaa7ad953ac13b98d08f3e4af6f85f","truth_sha1":"b31e3c141928311c010400b31573e2f1625c9b1e","overlaps_test":false,"full_text_reference":true}, +{"id":"0477","category":"article","split":"dev","html_sha1":"8df6349d3d1e24ff5b4cdaaa38e275dfab58e80c","truth_sha1":"87aff4927a131ba91f91a9b86a9dbd58d0a30569","overlaps_test":false,"full_text_reference":true}, +{"id":"0478","category":"article","split":"dev","html_sha1":"37c7a92b153d46c3e8549ebb846986009f93afd1","truth_sha1":"45e51cc740c95c4dc63c7de9f0aaf27ce7638ad5","overlaps_test":false,"full_text_reference":true}, +{"id":"0479","category":"article","split":"dev","html_sha1":"63f618a808f2b03aa2f7b39c3d5d75ffb42e2843","truth_sha1":"9011f535c664a8cb1e21c1e0296a768a2f061b13","overlaps_test":false,"full_text_reference":true}, +{"id":"0480","category":"article","split":"dev","html_sha1":"f9f0ee80b28c6f1eb795d37167c5ff020f2f2bec","truth_sha1":"1991a9d2a127df5d62597850ca4758aa4e653697","overlaps_test":false,"full_text_reference":true}, +{"id":"0481","category":"article","split":"dev","html_sha1":"a1c282604a0139ecb0a8a6de582d74844a751239","truth_sha1":"37a025b730f10281a0ade70717b520e8f49ff5a1","overlaps_test":false,"full_text_reference":true}, +{"id":"0482","category":"article","split":"dev","html_sha1":"ab01f8f06521ca3e9e51767cf2bec2974aea9855","truth_sha1":"0200bec3573240b2bd606e92efa9089367b9f49a","overlaps_test":false,"full_text_reference":true}, +{"id":"0483","category":"article","split":"dev","html_sha1":"913f8bbdad088244a45e5bf8236afe8e6d55c127","truth_sha1":"8fd150d12f944d6977113f34dd85aa040df6c2a2","overlaps_test":false,"full_text_reference":true}, +{"id":"0484","category":"article","split":"dev","html_sha1":"022e2bf055ce763652b440f79e61c8caf2fbfef4","truth_sha1":"314f5379682d3fa707a72bb72248e314d393709a","overlaps_test":false,"full_text_reference":true}, +{"id":"0485","category":"article","split":"dev","html_sha1":"3827e24ed6fe6b761ecbed623ac407eea7757459","truth_sha1":"8941a7a58ce3d14577ca8c0aac7781db93cbbf36","overlaps_test":false,"full_text_reference":true}, +{"id":"0486","category":"article","split":"dev","html_sha1":"6b60dd72afc69afba1278018a0081fe8ac032821","truth_sha1":"027afbcba51ebc00fece0a72054f0fffee9e8254","overlaps_test":false,"full_text_reference":true}, +{"id":"0487","category":"article","split":"dev","html_sha1":"778e9c0e3f2b16f1fd1ef01bf025f4f223f3ca1e","truth_sha1":"b3fdc6a08a48464089fe74d72785ed4256707dcb","overlaps_test":false,"full_text_reference":true}, +{"id":"0488","category":"article","split":"dev","html_sha1":"7e43536446d8ae384579a26d4398a318b4c80417","truth_sha1":"04d8553dd3acf98789d443bc0401b57762c7b3a7","overlaps_test":false,"full_text_reference":true}, +{"id":"0489","category":"article","split":"dev","html_sha1":"45845d3b8a6b82dfc0c1370ba95fb9a09ac3911e","truth_sha1":"f924cded4947549b89f9e972bfbb775ecce4527c","overlaps_test":false,"full_text_reference":true}, +{"id":"0490","category":"article","split":"dev","html_sha1":"f7b779aabdcafce27bdfc475cdb4785c14f6e877","truth_sha1":"fc8388530f9c33eb73eb09e666ce33366ba2dc2c","overlaps_test":false,"full_text_reference":true}, +{"id":"0491","category":"article","split":"dev","html_sha1":"aa6df14dc713100d4461fa8537245c8ad984fa83","truth_sha1":"8aa0055b31871d6212f82b079bb7b029c8834db6","overlaps_test":false,"full_text_reference":true}, +{"id":"0493","category":"article","split":"dev","html_sha1":"8417224340c961f441146f0ba46e56138c8354a6","truth_sha1":"be5ab62c9d6b4f724937fdd11511e91c70f143ca","overlaps_test":false,"full_text_reference":true}, +{"id":"0494","category":"article","split":"dev","html_sha1":"d73b250a6f1a156f86eb3278a4325c3602709bbf","truth_sha1":"3b65b5e5e91417c91375d79ccca37e88118bfa43","overlaps_test":false,"full_text_reference":true}, +{"id":"0495","category":"article","split":"dev","html_sha1":"b472eb3524532a8f3f10da450e918d3fc40d0a2d","truth_sha1":"536c245cddad0a4935927026d14a9dd35aa0ba7e","overlaps_test":false,"full_text_reference":true}, +{"id":"0496","category":"article","split":"dev","html_sha1":"605ed314c1ecc4ef99f2b18bbdb896f5231de308","truth_sha1":"b89db95bd3010bdbd9b0dc46d0ec8482d558a386","overlaps_test":false,"full_text_reference":true}, +{"id":"0497","category":"documentation","split":"dev","html_sha1":"3b5adba6df27e0df9ff61ac83b49c9d33435c011","truth_sha1":"39921a38778496471c207dc6eab6777323101bf2","overlaps_test":false,"full_text_reference":true}, +{"id":"0498","category":"documentation","split":"dev","html_sha1":"4afc90b14b7070ab01e6965fb4a3e448148d96e2","truth_sha1":"54612db8fc18ff1f4efb6872fcfaab4017ed4d31","overlaps_test":false,"full_text_reference":true}, +{"id":"0499","category":"documentation","split":"dev","html_sha1":"e14752ca36a97dec89fffb8b14d1796f0d9fbd66","truth_sha1":"dbcd8394cffa4990e5ef11b3755f585bbd5f84bf","overlaps_test":false,"full_text_reference":true}, +{"id":"0500","category":"article","split":"dev","html_sha1":"75a8000b685ae45165bef3219adea92a7ce1d90a","truth_sha1":"572ddc4a9251e1cc3de01ae05d4a1cc2df939259","overlaps_test":false,"full_text_reference":true}, +{"id":"0501","category":"forum","split":"dev","html_sha1":"73b0b30e7bd777f2f332458f583d59e361c52ca8","truth_sha1":"56d0b28fb7429235cf4d793447bb6a81d4c30cd2","overlaps_test":false,"full_text_reference":true}, +{"id":"0502","category":"forum","split":"dev","html_sha1":"691bcf07026bddb2c0dad1c9b952ccbd232b2338","truth_sha1":"b46c0f537338a534d727d700d6970747dab01b01","overlaps_test":false,"full_text_reference":true}, +{"id":"0503","category":"forum","split":"dev","html_sha1":"8437095825f4dcbd493a40574fe215b0b5dec538","truth_sha1":"0110b300c35a1a1504976f9996297f89233ad0c3","overlaps_test":false,"full_text_reference":true}, +{"id":"0504","category":"forum","split":"dev","html_sha1":"201adb1a6250a3022b11c3a7bd66a4806c94e9c9","truth_sha1":"5e29f9c55e3107bc8e13c9ef20ed40199c372ff3","overlaps_test":false,"full_text_reference":true}, +{"id":"0505","category":"forum","split":"dev","html_sha1":"2f3237d845106efcbf689230c05fc8f1a48f0470","truth_sha1":"a2abbdaf0062dfd84b878386c4afc7aac0079a42","overlaps_test":false,"full_text_reference":true}, +{"id":"0506","category":"forum","split":"dev","html_sha1":"84602d6f25c0c50a8a5874faef10e130566f92e5","truth_sha1":"41e0ce6c10b8113f68609ecb061f403c5d9129c9","overlaps_test":false,"full_text_reference":true}, +{"id":"0507","category":"forum","split":"dev","html_sha1":"ec3865ecb44773be0e1ce776bb131d758dca5084","truth_sha1":"e2720984ec607716b8ed0a88dc734b8636d91ecb","overlaps_test":false,"full_text_reference":true}, +{"id":"0508","category":"forum","split":"dev","html_sha1":"378ee9ace6d8e71c2f00e23a481739e2e5e92464","truth_sha1":"174447c725717bf689d52096c2ae9e9ce9fdb3c8","overlaps_test":false,"full_text_reference":true}, +{"id":"0509","category":"forum","split":"dev","html_sha1":"820c63f7ac372d14eabc006ac2f8721940ebf286","truth_sha1":"5bc922ce0eb92fd025c9d5a7fbb71f47c4daed96","overlaps_test":false,"full_text_reference":true}, +{"id":"0510","category":"forum","split":"dev","html_sha1":"e2b4fbd3f48d5d3ea269dae0f51ca3fb07fc7258","truth_sha1":"29ab0b1302eb457cb3677485957c19b8fa9b5326","overlaps_test":false,"full_text_reference":true}, +{"id":"0511","category":"forum","split":"dev","html_sha1":"59d7f817cf18246f0216887695e34b820d8474d6","truth_sha1":"2cd6523c998f42a29c963a8fece4ffddaf9501b1","overlaps_test":false,"full_text_reference":true}, +{"id":"0512","category":"forum","split":"dev","html_sha1":"4773d87a8c2b0bd5cfd25aa9177c86d0ccbc1bd3","truth_sha1":"f5977d0260c71be28c662545a5eea86ae15082aa","overlaps_test":false,"full_text_reference":true}, +{"id":"0513","category":"forum","split":"dev","html_sha1":"e267317899b2b1d8b4c7dde37d343781b9e495b1","truth_sha1":"930fd2931ef89b5681f3fd22f200ac42b1187a00","overlaps_test":false,"full_text_reference":true}, +{"id":"0514","category":"forum","split":"dev","html_sha1":"9c5407ab69e1ca18a54ee1fe4b1783db297f0b0a","truth_sha1":"a6119c889bb8a80c89c73139e31d4594af71c296","overlaps_test":false,"full_text_reference":true}, +{"id":"0515","category":"forum","split":"dev","html_sha1":"d3b2176148668563f44219d040b0299687dba18e","truth_sha1":"7b511fd1a6fbbe7d4d8ec7bcb45259519ae21125","overlaps_test":false,"full_text_reference":true}, +{"id":"0516","category":"forum","split":"dev","html_sha1":"653822c2db75ac190f2e6e72dba069c0d975b7ff","truth_sha1":"06a370f7f9ccfe7efb5a5da5729c36a075074ad6","overlaps_test":false,"full_text_reference":true}, +{"id":"0517","category":"forum","split":"dev","html_sha1":"8a38c09a312c07aa72966e39c330c8a67ff41dc5","truth_sha1":"fda74619093a249ff189e28ae0ce2eeb01a79882","overlaps_test":false,"full_text_reference":true}, +{"id":"0518","category":"forum","split":"dev","html_sha1":"16fc42d98384af4d42f55b6cff6cb7b1ac3c52b9","truth_sha1":"a9ec4d2a5cd59dfc3cb22416636b31e1a0fb47b9","overlaps_test":false,"full_text_reference":true}, +{"id":"0519","category":"forum","split":"dev","html_sha1":"dd631123ce3e4e64b84eaf3a0c8ebc2809647fc0","truth_sha1":"aa1dd5b6052d221380900e88b7c9be03a24e294a","overlaps_test":false,"full_text_reference":true}, +{"id":"0520","category":"forum","split":"dev","html_sha1":"e69c0f87b6de35e39034423c22e148c68d74190f","truth_sha1":"6dff85dd0147f957b1b56b1de52557fff44dcfa0","overlaps_test":false,"full_text_reference":true}, +{"id":"0521","category":"forum","split":"dev","html_sha1":"67319b0fb9f4701a173948c272290602dd5cdf08","truth_sha1":"649b334ead3b051090d2472f85da290116856d2f","overlaps_test":false,"full_text_reference":true}, +{"id":"0522","category":"forum","split":"dev","html_sha1":"fd171cf5ee78617b35a337450665f35ace33dddc","truth_sha1":"d88586ff9e766184fa6e65541105792e63f4a49b","overlaps_test":false,"full_text_reference":true}, +{"id":"0523","category":"forum","split":"dev","html_sha1":"f32a6a6a1845e6a8adff6640718d168e2ca702d2","truth_sha1":"72e2694e0dc195cf4250e4efa22615b7e6c8015c","overlaps_test":false,"full_text_reference":true}, +{"id":"0524","category":"forum","split":"dev","html_sha1":"356ba6ac6ebbf3394fa1dad857bcfed96374d3ed","truth_sha1":"6ea8c871cf3ee45c6464e14025bb7cce2a864e70","overlaps_test":false,"full_text_reference":true}, +{"id":"0525","category":"forum","split":"dev","html_sha1":"d34c0c221930f23f6e1cda687a59a09c16899aa0","truth_sha1":"b4b070fb90178ac40a2cc7cd4b0863269cd66588","overlaps_test":false,"full_text_reference":true}, +{"id":"0526","category":"forum","split":"dev","html_sha1":"111bc7e4ba1fa75cbbee324bfd747933c688e7e1","truth_sha1":"db8811086748c822cedb1dd8d6deaa9a09a0f588","overlaps_test":false,"full_text_reference":true}, +{"id":"0527","category":"forum","split":"dev","html_sha1":"7ff75395cb9da38bd6e23244f33c14463f187eab","truth_sha1":"03df27d66c8bdc8f6ad9a6d512c20d2aac626b37","overlaps_test":false,"full_text_reference":true}, +{"id":"0528","category":"forum","split":"dev","html_sha1":"8a739187bf7cd773035dee5ac422b55b3b7e68fb","truth_sha1":"8530167380e89820a65f20cfaf22ed52085023e6","overlaps_test":false,"full_text_reference":true}, +{"id":"0529","category":"forum","split":"dev","html_sha1":"5237882babf4c213f891abc5c40434163efe3e1f","truth_sha1":"960cf243f67c46a85fde45769b6849e4a47a49d7","overlaps_test":false,"full_text_reference":true}, +{"id":"0530","category":"forum","split":"dev","html_sha1":"e19d400fd503c60ae469071ba22cef65bdd5bd64","truth_sha1":"77002e570d4507524b02e197510e8b0a5d07891a","overlaps_test":false,"full_text_reference":true}, +{"id":"0531","category":"forum","split":"dev","html_sha1":"939a0558a3cbbff8618d23c2a1e81b85bb3cb3f4","truth_sha1":"59e3a438de77b707f2449924e0e6891a08ba49b8","overlaps_test":false,"full_text_reference":true}, +{"id":"0532","category":"forum","split":"dev","html_sha1":"aa9d6ca4f00fe3e027e9ed18469cb62dad4d3574","truth_sha1":"30a4476aacc8f7b974f407539bb3f82fc94f5f31","overlaps_test":false,"full_text_reference":true}, +{"id":"0533","category":"forum","split":"dev","html_sha1":"72c0214a13fb6346a8dfdd0a8f35880475e51c28","truth_sha1":"87e6647c42139136f9c8105903abc69eec02e3bb","overlaps_test":false,"full_text_reference":true}, +{"id":"0534","category":"forum","split":"dev","html_sha1":"794d1b8322abf4b30fb25b07244f76d90eb80a9c","truth_sha1":"1891791dec2053a1cfce166f6fc96e76346c9501","overlaps_test":false,"full_text_reference":true}, +{"id":"0535","category":"forum","split":"dev","html_sha1":"32236922b3d6b4404beb7ff98128833486950eef","truth_sha1":"1bfc207a392a256565a6b3d9b4cbf50459f73d07","overlaps_test":false,"full_text_reference":true}, +{"id":"0536","category":"forum","split":"dev","html_sha1":"926129621998397cc2fd86528cb21d711ffb2ec5","truth_sha1":"7e5972ac1566a788b8a343a71f4b41223a2f2428","overlaps_test":false,"full_text_reference":true}, +{"id":"0537","category":"forum","split":"dev","html_sha1":"2c688dd7cd82e4785e51094967c7124cd314564c","truth_sha1":"e94f59751d2f2fb3d5f48af33a7b1b4002db7249","overlaps_test":false,"full_text_reference":true}, +{"id":"0538","category":"forum","split":"dev","html_sha1":"c2e77258cab7ab71669cb3ae1b500a90b7ce722e","truth_sha1":"71fc0f97cea0b7f88ade7ad4779c34fc4257a84c","overlaps_test":false,"full_text_reference":true}, +{"id":"0540","category":"forum","split":"dev","html_sha1":"0f794f4fe3dd305d2164cde944ff0c47c2bfa073","truth_sha1":"a5f0f1c396573c0f386b4726b6e05f7fa6dad77e","overlaps_test":false,"full_text_reference":true}, +{"id":"0541","category":"forum","split":"dev","html_sha1":"9acde5770a57dc1f875bc6712cebaa061f24c0ed","truth_sha1":"a3ca4ecf901954a21ca95bd0a95e373698e5f6b6","overlaps_test":false,"full_text_reference":true}, +{"id":"0543","category":"forum","split":"dev","html_sha1":"406bffa0d03f93faa33781a39b9fbf2813f56a69","truth_sha1":"1c3b2a9a418b293049a40b46c54dbe48ddf66c0a","overlaps_test":false,"full_text_reference":true}, +{"id":"0544","category":"forum","split":"dev","html_sha1":"8ef1568cbeed73d5e2d3a5b9c21435f72033dae1","truth_sha1":"92af6ea8ac41b8609fcc58a12cbc0be9b7f62d6c","overlaps_test":false,"full_text_reference":true}, +{"id":"0545","category":"forum","split":"dev","html_sha1":"cb8b9060d2f19659b9b5dc1c71b2aa284c302687","truth_sha1":"dec4ffd1ce32db21429e7825d607169619d19ffc","overlaps_test":false,"full_text_reference":true}, +{"id":"0546","category":"forum","split":"dev","html_sha1":"35f3e698f3b12b59ae6f03bd509c96e23e1eb304","truth_sha1":"2eaf836e3b42ba98dcd12343e4ff3eec22f7b598","overlaps_test":false,"full_text_reference":true}, +{"id":"0547","category":"forum","split":"dev","html_sha1":"c923feea3ac41c135e3635b57421b936c3279a8c","truth_sha1":"28d366aeb9f26a8975cfe997f37457631aef85d4","overlaps_test":false,"full_text_reference":true}, +{"id":"0548","category":"forum","split":"dev","html_sha1":"d104dc00faed6d5e4fa03751368198d0a414a968","truth_sha1":"05338a2093ba55984fd8d0fd3030c55ca9cd8aa3","overlaps_test":false,"full_text_reference":true}, +{"id":"0549","category":"forum","split":"dev","html_sha1":"8d8ad0aa9452e3ee1674886128cbd77c35052001","truth_sha1":"02c96824338a776940103960af0e27d51c4eae85","overlaps_test":false,"full_text_reference":true}, +{"id":"0550","category":"forum","split":"dev","html_sha1":"b43cbbcfd05374cb0efc4195bb9afd6f51f9d2f5","truth_sha1":"98a8b5f13e4f2859c4267f72721babc1912f8dca","overlaps_test":false,"full_text_reference":true}, +{"id":"0551","category":"forum","split":"dev","html_sha1":"6ec457398878fca35ef400799635445c11d1564b","truth_sha1":"3ad57ef404ffa018f1c0d3cd541f6f2f1a67875d","overlaps_test":false,"full_text_reference":true}, +{"id":"0552","category":"forum","split":"dev","html_sha1":"70e88f5d3bc6e1552d27e7db4f9176ba055c8ac0","truth_sha1":"5279cccd764b658a35e4d804d61512a3d9c6d14f","overlaps_test":false,"full_text_reference":true}, +{"id":"0553","category":"forum","split":"dev","html_sha1":"002901c044f71167f7f8351f96015630e6cc1c7e","truth_sha1":"a5a71998bf6eb89681b4b0877f76e3ee26d7c6c3","overlaps_test":false,"full_text_reference":true}, +{"id":"0554","category":"forum","split":"dev","html_sha1":"5be438ad07720a7035ca770ecc390804548aa1bd","truth_sha1":"e077cad115301749a88680bb1f09819e24048f8d","overlaps_test":false,"full_text_reference":true}, +{"id":"0555","category":"forum","split":"dev","html_sha1":"a3a39799e671b1cef20621aa0974138e71c20c96","truth_sha1":"d75a73a3a6ece4a687efa96ad704ff4d712b0dc1","overlaps_test":false,"full_text_reference":true}, +{"id":"0556","category":"forum","split":"dev","html_sha1":"f0813de1fd49a560a5998ef7f63cabf5ec70c555","truth_sha1":"7e7a8490e3757b9da3e98bd5ba6612294ae350d0","overlaps_test":false,"full_text_reference":true}, +{"id":"0557","category":"forum","split":"dev","html_sha1":"dcfce10cb316478eadcfb5a9d3152f8aaf82e2a6","truth_sha1":"72ae21c9fd437c5bd6494e27779ddbea58cadd42","overlaps_test":false,"full_text_reference":true}, +{"id":"0558","category":"forum","split":"dev","html_sha1":"095f67afed9437c26cb97feb1628f655fa197a83","truth_sha1":"0d13a04d5d1a1ae025ce382eb1fff4d265d86c5f","overlaps_test":false,"full_text_reference":true}, +{"id":"0559","category":"forum","split":"dev","html_sha1":"52b9122456bdf99cdde67cc41ac6b6b8486134ad","truth_sha1":"145cca6409d9bd541d060f4756d27622a715de20","overlaps_test":false,"full_text_reference":true}, +{"id":"0560","category":"forum","split":"dev","html_sha1":"3be6b8d8e9ac33bc9808367080ce09648d51ceb3","truth_sha1":"9aa8eb9fb013cc1b3eae12b8575ff6e2f4fb6230","overlaps_test":false,"full_text_reference":true}, +{"id":"0561","category":"forum","split":"dev","html_sha1":"c7611f54a06f2aff7f7da2202e4d03498e15c75e","truth_sha1":"7329acf489ee78de8aad7bedf9064bf10e324700","overlaps_test":false,"full_text_reference":true}, +{"id":"0562","category":"forum","split":"dev","html_sha1":"8cc4502b3e2f3e985553d403d38a78b5fa3d57e9","truth_sha1":"3b2e3924e38b7cae6b01fc6ecd9fd9931fd21850","overlaps_test":false,"full_text_reference":true}, +{"id":"0563","category":"forum","split":"dev","html_sha1":"48dbd94267e58276c2af9a2784c0b840e968c678","truth_sha1":"312b207d809b1fa842f3a03a62461a2dd2352a8d","overlaps_test":false,"full_text_reference":true}, +{"id":"0564","category":"forum","split":"dev","html_sha1":"7f902eb0fc4adf6aa4126508466688cdcb43da70","truth_sha1":"7f72ad76256aee69ab449ead701f7447ed24e0cd","overlaps_test":false,"full_text_reference":true}, +{"id":"0565","category":"forum","split":"dev","html_sha1":"e5677d7d4ca552fdc413cde30558b6338dabbfe7","truth_sha1":"57d4f038bbdd11d82fd0a54f17f8322b3eebde16","overlaps_test":false,"full_text_reference":true}, +{"id":"0566","category":"forum","split":"dev","html_sha1":"f3a6f01dcbf1224c49080befdf07b0c199fe9b30","truth_sha1":"77fb128b299d867e286c5d958c390e8e6e3e193f","overlaps_test":false,"full_text_reference":true}, +{"id":"0567","category":"forum","split":"dev","html_sha1":"e79d2c3553781d6ca06d36c385cd949060ebdcfa","truth_sha1":"a9d7e6e3e12c9290223b6a047a176a09156c31c2","overlaps_test":false,"full_text_reference":true}, +{"id":"0569","category":"forum","split":"dev","html_sha1":"b6e223ca23fff444de0750498bf0c9f59c2bb0d1","truth_sha1":"94dced1fa5aa1c2fdcb9167f1fce4e04ec57f177","overlaps_test":false,"full_text_reference":true}, +{"id":"0570","category":"forum","split":"dev","html_sha1":"df48e77548108f52612cb86a80e4e1abd2eec866","truth_sha1":"bababd8d39314a6ff3915d42e5c71a50afb01f2b","overlaps_test":false,"full_text_reference":true}, +{"id":"0573","category":"forum","split":"dev","html_sha1":"46c16efcf0418eca7f8a69257f598e5ba42d1221","truth_sha1":"61c083ec46e79714bf64cff4657e142dc7ab96c6","overlaps_test":false,"full_text_reference":true}, +{"id":"0574","category":"forum","split":"dev","html_sha1":"edb31eee6b2d055dfd0122ef8cbac08cf57c4a00","truth_sha1":"f69d5bc3a636b78975fa0aa637fe21913e0f8815","overlaps_test":false,"full_text_reference":true}, +{"id":"0575","category":"forum","split":"dev","html_sha1":"b15f2c98e06974f9310d39dc3c60f2bf65adb02a","truth_sha1":"35dcb73b10dc73a25432a618e304aea53f05b9f9","overlaps_test":false,"full_text_reference":true}, +{"id":"0576","category":"forum","split":"dev","html_sha1":"db1bc2299c7cafa5eaacd6d841eb3e2fa5f2acca","truth_sha1":"c637d3b21d5c129d80de02f4764c88b5efeafb85","overlaps_test":false,"full_text_reference":true}, +{"id":"0577","category":"forum","split":"dev","html_sha1":"501912703bd6c4574e8d809b6e85756e0b761576","truth_sha1":"4453e4aa4e3ebe8bf8c42e33eab22283c5218dc0","overlaps_test":false,"full_text_reference":true}, +{"id":"0578","category":"forum","split":"dev","html_sha1":"2abe28f37481de2ebe5d0c79f5892293e8c8c88f","truth_sha1":"7b4ddfa80f67df449d7073c698814efa8af7539c","overlaps_test":false,"full_text_reference":true}, +{"id":"0579","category":"forum","split":"dev","html_sha1":"01bda1d056936632e26c55310fc72f0b7aa67925","truth_sha1":"ff3deebeb236d0fc44b4fd09df4df3b09742ad1f","overlaps_test":false,"full_text_reference":true}, +{"id":"0580","category":"product","split":"dev","html_sha1":"d76be52b88c74b4495736c8cc879658635b272c6","truth_sha1":"5345c3bba02baf965c532109159da9255b25ce7f","overlaps_test":false,"full_text_reference":true}, +{"id":"0581","category":"product","split":"dev","html_sha1":"efc22e3412d81d682e9aa97db4ad68731d098ce0","truth_sha1":"3c8a2ab0dcf0d657203ac0c4c98092ba6ef623c4","overlaps_test":false,"full_text_reference":true}, +{"id":"0582","category":"product","split":"dev","html_sha1":"14ccfc9c98e1b75cbbef4882fc03b11fc4375109","truth_sha1":"6bd96f9113d562cefe1adec01af3407239e8bfe5","overlaps_test":false,"full_text_reference":true}, +{"id":"0583","category":"product","split":"dev","html_sha1":"af89101c349f456d2b9d4f5a5b41bdabbee925ba","truth_sha1":"9302a91be651429d4c1d7199da07f72f46effb3c","overlaps_test":false,"full_text_reference":true}, +{"id":"0584","category":"product","split":"dev","html_sha1":"1bb6816d8158bd6d3e582fbab441da91449f4573","truth_sha1":"695f692247358a5b80b4c6497e33a52947781269","overlaps_test":false,"full_text_reference":true}, +{"id":"0585","category":"product","split":"dev","html_sha1":"e872e43545c8f8141bdb842ec733b701f773c299","truth_sha1":"6fb21d4f7ddfaec56de80cd646120554b714fc54","overlaps_test":false,"full_text_reference":false}, +{"id":"0586","category":"product","split":"dev","html_sha1":"8efb4fe48c2762580ca83fa3771992b09fd00d5b","truth_sha1":"80b63e2cbf1523ba0b58435fe5ef8b15f16a2ee0","overlaps_test":false,"full_text_reference":true}, +{"id":"0587","category":"product","split":"dev","html_sha1":"9fbb1d32c7d78f027154db8910154157e1fb5e3b","truth_sha1":"1e34efcb6fc3c6c4560ac1e579ab0b981059086f","overlaps_test":false,"full_text_reference":true}, +{"id":"0588","category":"product","split":"dev","html_sha1":"750154564b3e977750fe5e598896d91924f4e086","truth_sha1":"4477059b82a74318419a64d81e925075d1f1f642","overlaps_test":false,"full_text_reference":true}, +{"id":"0589","category":"product","split":"dev","html_sha1":"db50726cecedf56828cf6a217c7f71ca3f222132","truth_sha1":"a47214f1911afcc7b65d9180bca83a52905c726e","overlaps_test":false,"full_text_reference":true}, +{"id":"0590","category":"product","split":"dev","html_sha1":"9baaf98d65f8bfddbe97058200c2c04e40558517","truth_sha1":"959cdbb4ff5a0d11cf97433ed94aba2ddf73548a","overlaps_test":false,"full_text_reference":true}, +{"id":"0591","category":"product","split":"dev","html_sha1":"6e30559699cb773a0546fe35f84acbf60774b1b0","truth_sha1":"98625dcef1ecba1d49a17b972acd80b8a5bf9c01","overlaps_test":false,"full_text_reference":true}, +{"id":"0592","category":"product","split":"dev","html_sha1":"b0b2c665afb5e5b75b6a6e28b01cf61576e6295f","truth_sha1":"23fc8b299e973ea616647337d5ace9650fbd3add","overlaps_test":false,"full_text_reference":true}, +{"id":"0593","category":"product","split":"dev","html_sha1":"1708aaaaedb55d94241ba1faf455d15687b87f39","truth_sha1":"e77b77467b898ff64bdf00a24d1a2bf163e67569","overlaps_test":false,"full_text_reference":true}, +{"id":"0594","category":"product","split":"dev","html_sha1":"38cff74c377570309e1c50fb1116add251503dd4","truth_sha1":"721cd1ae9ab13544da8143daabc7b1ab628cae80","overlaps_test":false,"full_text_reference":true}, +{"id":"0595","category":"product","split":"dev","html_sha1":"c64b7619ad4ed017b0bde64d51a442de8966f5bd","truth_sha1":"3767ee780528856808d815adb7019b6c0c8366e0","overlaps_test":false,"full_text_reference":true}, +{"id":"0596","category":"product","split":"dev","html_sha1":"8f59ea08edc1eca61c6f98b04df3ca14b2349fbf","truth_sha1":"b745b5294f7369a96eb576a0073dac534f9b8170","overlaps_test":false,"full_text_reference":true}, +{"id":"0597","category":"collection","split":"dev","html_sha1":"6bcd91cf7eea84926f6cbe77b20d26178ec58cb7","truth_sha1":"3f1368142b9735e537577a0e16fba7995ce6a135","overlaps_test":false,"full_text_reference":true}, +{"id":"0598","category":"product","split":"dev","html_sha1":"b8c328d6444d17e8e0174680c4180693a018fd68","truth_sha1":"7ee38016f092a834ee91520c3f10a96a7377a9c7","overlaps_test":false,"full_text_reference":true}, +{"id":"0599","category":"product","split":"dev","html_sha1":"5537daaf63266ae05d3b7967b1f180ad7bda9fcb","truth_sha1":"5b91daac74b24d3045c289b8ae1e3a2b72a62dfa","overlaps_test":false,"full_text_reference":true}, +{"id":"0600","category":"product","split":"dev","html_sha1":"bf64d8b2e73223920dfaa00e7fd9aae8a255d222","truth_sha1":"96b02314b6b63b18ac3224404545ae52bfe7e714","overlaps_test":false,"full_text_reference":true}, +{"id":"0601","category":"product","split":"dev","html_sha1":"2f556b6f44ce61849b10628c1735d54987b19aa4","truth_sha1":"b151d6a8576de17fc9a51dbce5c55a3ec87927e1","overlaps_test":false,"full_text_reference":true}, +{"id":"0602","category":"product","split":"dev","html_sha1":"dfd6a51b289ca269ea5e3d4a7b5f5844ccccd9a9","truth_sha1":"d53b8af8d22160df133f782eb8a6c7a5cb0c450e","overlaps_test":false,"full_text_reference":false}, +{"id":"0603","category":"product","split":"dev","html_sha1":"d5080413e39a47c678e9911fee9d686e5ea9edf7","truth_sha1":"e358437c393ea80912e62024db2295d6d203fae5","overlaps_test":false,"full_text_reference":true}, +{"id":"0604","category":"product","split":"dev","html_sha1":"6fa90a65a575b53e402f92348d75ca8ef4c44c70","truth_sha1":"a47cda06809d51ca7389edd4344389e5f1850f6b","overlaps_test":false,"full_text_reference":true}, +{"id":"0605","category":"product","split":"dev","html_sha1":"ee13de88af9262faa00c36e8b7fb3053e192dfca","truth_sha1":"2d839adfe0fa884da8b6a762e02a2a57fd8f9a63","overlaps_test":false,"full_text_reference":true}, +{"id":"0606","category":"product","split":"dev","html_sha1":"e50786bb06ba3e96df28ad313df5abcc664740ac","truth_sha1":"d118b33cf8e8c260f28d253c8226dfb88aa09070","overlaps_test":false,"full_text_reference":true}, +{"id":"0607","category":"product","split":"dev","html_sha1":"c705c485bb07065353397c737a912453e44f1f24","truth_sha1":"232aec9d23427a1e3b7cc20ab5b317f58445aab0","overlaps_test":false,"full_text_reference":true}, +{"id":"0608","category":"product","split":"dev","html_sha1":"906733710de11f1c93df2c5107f30a1ec4fbc4b8","truth_sha1":"dfc0565a04e749de8fd0ac9f8d747a646832a2ae","overlaps_test":false,"full_text_reference":true}, +{"id":"0609","category":"product","split":"dev","html_sha1":"ec6f3a5bfe2860ee03045159f5fd897035e6c3ee","truth_sha1":"fb5e829f2fab2571b1312bc61abbe06a8b3435c3","overlaps_test":false,"full_text_reference":true}, +{"id":"0610","category":"product","split":"dev","html_sha1":"19ca3e531864737102d27d9ca8bacaa6533dce6f","truth_sha1":"a9c5d9639754352a5467d999a506406c81141ec8","overlaps_test":false,"full_text_reference":true}, +{"id":"0611","category":"product","split":"dev","html_sha1":"d13f96bdcc129dc548490dcdec0d90c8a2281433","truth_sha1":"22d9528067fa12aea89730be6edea2ff9a189c3b","overlaps_test":false,"full_text_reference":true}, +{"id":"0612","category":"product","split":"dev","html_sha1":"0ed5118938af920101588e7870f5815702ed2396","truth_sha1":"105c8319efe6d6bba25fe0b7937dc298a6c5d17f","overlaps_test":false,"full_text_reference":true}, +{"id":"0613","category":"product","split":"dev","html_sha1":"82c9e298b6478cb78ade1dc7cfa859b017241e2a","truth_sha1":"1e8df2e37fbb3cc7d6232f99f842141b3d03e15f","overlaps_test":false,"full_text_reference":true}, +{"id":"0614","category":"product","split":"dev","html_sha1":"26d06fbac947b31f853b87a36339950a409909ef","truth_sha1":"8d13d7ee111f6bfda5952a77ba222096d93ecacd","overlaps_test":false,"full_text_reference":true}, +{"id":"0615","category":"product","split":"dev","html_sha1":"d9c2896260a293de402e5f50ff174d7f70b80e0f","truth_sha1":"3bb3b54165740c3f5cf57235ad61ca5773665e6a","overlaps_test":false,"full_text_reference":true}, +{"id":"0616","category":"product","split":"dev","html_sha1":"753d83f3b665088962bc084c7ca359d3c2375c87","truth_sha1":"e8c923e931a65ff3746c63d1bd1432f79c5b223a","overlaps_test":false,"full_text_reference":true}, +{"id":"0617","category":"product","split":"dev","html_sha1":"36acdeef915e5a994feb60536f62dbc4090774e9","truth_sha1":"ef22d2e3bc806a7c4490d9229ea483c350a90335","overlaps_test":false,"full_text_reference":true}, +{"id":"0618","category":"product","split":"dev","html_sha1":"687ddad5805c7a54357a155633b919074a24f383","truth_sha1":"de1fbf65237428da0b4948ceeae1e27c00131d06","overlaps_test":false,"full_text_reference":false}, +{"id":"0619","category":"product","split":"dev","html_sha1":"58c5220f9951552837c5d20722069cda0781efb7","truth_sha1":"23fdabdce9a84bed89bdfdb4c397aa413ab15319","overlaps_test":false,"full_text_reference":true}, +{"id":"0620","category":"product","split":"dev","html_sha1":"a2844503b1d997f8b7f90277bbc90f858340feb1","truth_sha1":"9de7af5ae306fc90cf02da8fc0fd4b398abe7aed","overlaps_test":false,"full_text_reference":true}, +{"id":"0621","category":"product","split":"dev","html_sha1":"aad1913f016e831a2d8d827657904706fdfa4c53","truth_sha1":"9cb7c627a03606404205532ad863d561d6701bcf","overlaps_test":false,"full_text_reference":true}, +{"id":"0622","category":"product","split":"dev","html_sha1":"231ed9c4af72c2a456dae7766f9ed1e979d1b6df","truth_sha1":"9c33d529a16d04899c12a4d7a911cd317b81d221","overlaps_test":false,"full_text_reference":true}, +{"id":"0624","category":"product","split":"dev","html_sha1":"c3f1fa507d9edfc345f0300b7e74efad64ed093e","truth_sha1":"46795aa113788d77e9772ae6a1365f8f613d7a98","overlaps_test":false,"full_text_reference":true}, +{"id":"0625","category":"product","split":"dev","html_sha1":"868fc0cd24f40c8f345c1e8c06581afac6ab3a39","truth_sha1":"be7e54a13ed5349d2e61e88bb9a779a2f818815f","overlaps_test":false,"full_text_reference":true}, +{"id":"0626","category":"product","split":"dev","html_sha1":"2b3e749b8ee10ff4702f9f52e76642e2ab33c3f6","truth_sha1":"f1a5751d6f539758c9631aa929cc9f59bc70fe3e","overlaps_test":false,"full_text_reference":true}, +{"id":"0627","category":"product","split":"dev","html_sha1":"7599cdf3d711bed8b26e43071603f722bdb50b59","truth_sha1":"bfda9753c642566183ee0c867f0abe4eed0bd81d","overlaps_test":false,"full_text_reference":true}, +{"id":"0628","category":"product","split":"dev","html_sha1":"89ef05eae967cc9741900a3b70085f167652176a","truth_sha1":"9e238f8c44a860340603ad5e09cd390ae79b0fb7","overlaps_test":false,"full_text_reference":true}, +{"id":"0629","category":"product","split":"dev","html_sha1":"7bc99038411e992db3f798cdbf1caaa9d0fbb8ab","truth_sha1":"ef3114fef00d12c829c6998fa626ca479436bed0","overlaps_test":false,"full_text_reference":true}, +{"id":"0631","category":"product","split":"dev","html_sha1":"296c2d14d54682550690f30bf2d30e58c0d4f787","truth_sha1":"a79b4e977da99a33a61a027aa8c4e3a4951dad6d","overlaps_test":false,"full_text_reference":true}, +{"id":"0632","category":"collection","split":"dev","html_sha1":"a77f2b1e2e8e0211965e26f441d2ec0fffa9314e","truth_sha1":"8032c1e5d9b4056c86c6ad94e7610b75ec6f7a43","overlaps_test":false,"full_text_reference":true}, +{"id":"0633","category":"collection","split":"dev","html_sha1":"64c0662eba1d273a3ac6f08b575fb43a99cea7e8","truth_sha1":"b8497f54e440ca2a6adea4a903b3bbc6849b4de6","overlaps_test":false,"full_text_reference":true}, +{"id":"0634","category":"collection","split":"dev","html_sha1":"23c2dc8cafe29e83577dbdf621bbe8be6cb8c82b","truth_sha1":"9cd2e52781f5507fedb5600cb0c8fdfc7933a51b","overlaps_test":false,"full_text_reference":true}, +{"id":"0635","category":"collection","split":"dev","html_sha1":"532e2673f188d9ec59e622dc9f5a81cf2aab2df7","truth_sha1":"d56f7f7f241d2595aa62f0379b941391dd5dc705","overlaps_test":false,"full_text_reference":true}, +{"id":"0637","category":"collection","split":"dev","html_sha1":"2776cd365449b522b75b1ccfdfd4b9cced75d670","truth_sha1":"82493ba096aadd944c86303d9a9c6af417c714bd","overlaps_test":false,"full_text_reference":true}, +{"id":"0638","category":"collection","split":"dev","html_sha1":"0d8e91740c572beff792f73ab17c66a62c03f64e","truth_sha1":"6485bf1050feb33fc69859d5d00197cd72c7efac","overlaps_test":false,"full_text_reference":true}, +{"id":"0639","category":"collection","split":"dev","html_sha1":"0744340810fa1a94126ed11e9e0fb2c3b810b5b9","truth_sha1":"5378f401f067d74eb3e47cdf349ecb8b371453fb","overlaps_test":false,"full_text_reference":true}, +{"id":"0640","category":"collection","split":"dev","html_sha1":"8bcf3be729a35591deb2fc6f311f5f575cc12c6c","truth_sha1":"8fc262ca722ca2f4e89df150bfd96eb8265581a3","overlaps_test":false,"full_text_reference":true}, +{"id":"0641","category":"collection","split":"dev","html_sha1":"c00663762e4cac6ead6d6a4db3a4865af87b5deb","truth_sha1":"7a969ce823cf7dc55982a96717144e1fa3dcfdde","overlaps_test":false,"full_text_reference":true}, +{"id":"0642","category":"collection","split":"dev","html_sha1":"929c0599c62a4fc9f0cf28d5ec09540da4f44724","truth_sha1":"898afcfc3d33854c49ac7908980850033a6132ba","overlaps_test":false,"full_text_reference":true}, +{"id":"0643","category":"collection","split":"dev","html_sha1":"78498cd0a93d9701e604bc75c7bc50c8c6d9e352","truth_sha1":"7ea0b4c04f5dc3688d938d9fcce6ebd7e5f56ee9","overlaps_test":false,"full_text_reference":true}, +{"id":"0644","category":"collection","split":"dev","html_sha1":"1b8c2bc20366b967ef458d0748742c63d65246e3","truth_sha1":"7062ee95de718f662e076d50b717742f2f3268fc","overlaps_test":false,"full_text_reference":true}, +{"id":"0645","category":"collection","split":"dev","html_sha1":"10db24ba6346012b162f4494a9bebddaf327e85c","truth_sha1":"56cf9ba052623e82091cafadc7c0e61b59292083","overlaps_test":false,"full_text_reference":true}, +{"id":"0646","category":"collection","split":"dev","html_sha1":"84a2824a68793779b06a648b4e51ee37b1cc48c1","truth_sha1":"891dcb4c84b91c6f26be8b09485aebdd718c6aa2","overlaps_test":false,"full_text_reference":true}, +{"id":"0647","category":"collection","split":"dev","html_sha1":"845dd9f40df947833fb663a7cf5846660a4ff373","truth_sha1":"be7935c6c1eb0663a81358ab59310974025c8f5d","overlaps_test":false,"full_text_reference":true}, +{"id":"0648","category":"collection","split":"dev","html_sha1":"eca6e1dd49a04cbde35c05caa9cbae3e192011bf","truth_sha1":"e4caaf2d0c4d942979cf651e4d6ca6ce2f1caa23","overlaps_test":false,"full_text_reference":true}, +{"id":"0649","category":"collection","split":"dev","html_sha1":"efad2fe162ebfd2ecfc4b970b24039278a585115","truth_sha1":"a355e2fee998eeec06ef142f07de089fdeb1177d","overlaps_test":false,"full_text_reference":true}, +{"id":"0650","category":"collection","split":"dev","html_sha1":"f3a64fb978d68dc5a12400d94e63e543dd5ca108","truth_sha1":"211848c7b3a9204231e6fca4f44e63f5a88e376d","overlaps_test":false,"full_text_reference":true}, +{"id":"0651","category":"collection","split":"dev","html_sha1":"2fb99ef1c22dbedd1efca8eda2c5d490a5e93a63","truth_sha1":"357c482b3812bc6a87b026c2bbdacf7dd5f2b96d","overlaps_test":false,"full_text_reference":true}, +{"id":"0652","category":"collection","split":"dev","html_sha1":"4a05bdd0869ef43d849aaa2f5911bf5ce570ed4b","truth_sha1":"013bbac137d3a294623ee267a5e4d8deb09685a6","overlaps_test":false,"full_text_reference":true}, +{"id":"0653","category":"collection","split":"dev","html_sha1":"2cfe5cf5cf350c11dac424d5ca926065588f68f6","truth_sha1":"d57bf113d99e4a1c3379569763728829f3835b38","overlaps_test":false,"full_text_reference":true}, +{"id":"0654","category":"collection","split":"dev","html_sha1":"5ecae38f82e3e84a387d57777af9d7c824d607c0","truth_sha1":"d87cea12d4f42871a0bcfb900c7a4c6db280377a","overlaps_test":false,"full_text_reference":true}, +{"id":"0655","category":"collection","split":"dev","html_sha1":"3b8679461d105860178453020f2b4e4891bdaf2d","truth_sha1":"8b14d70889a2af10a4c9329d2f77bf4024b9c986","overlaps_test":false,"full_text_reference":true}, +{"id":"0656","category":"collection","split":"dev","html_sha1":"8915dd8bb6e529f2da7d7cbbc4674d6241ff9c5d","truth_sha1":"11863d813feb307cab8a8dc651036e741a0138d5","overlaps_test":false,"full_text_reference":true}, +{"id":"0658","category":"collection","split":"dev","html_sha1":"2ea641cbcb96f5f68d5ee4561ffdaf6c352f8370","truth_sha1":"42d8d8a7fe91f660ef2a1a96cb36d30dd17c321d","overlaps_test":false,"full_text_reference":true}, +{"id":"0659","category":"collection","split":"dev","html_sha1":"b82bf2668cf3f75368e47a9a0b07ddb688cd8fae","truth_sha1":"833ee6ea042092d3fb86281d00284234a3533c98","overlaps_test":false,"full_text_reference":true}, +{"id":"0660","category":"collection","split":"dev","html_sha1":"1ccd80dcb75e5c60d40f24c451c3e202e76fe002","truth_sha1":"e2edbdc1d1d116db1b8eff018c995dbec4e45bf4","overlaps_test":false,"full_text_reference":true}, +{"id":"0661","category":"documentation","split":"dev","html_sha1":"42c597de21b8afd656d066c7ffc5c2d43d6884a8","truth_sha1":"a821bfea93bb09711b9916cb3f5faa004b145d5c","overlaps_test":false,"full_text_reference":true}, +{"id":"0662","category":"documentation","split":"dev","html_sha1":"e8ee4fa9c1a1cd21d48bbc57d4d1d8cffa2a50dd","truth_sha1":"5082f6affb6e2e6dc6a43b993cc5949db3fa29ff","overlaps_test":false,"full_text_reference":true}, +{"id":"0663","category":"documentation","split":"dev","html_sha1":"5048fc8c63cb15c7a0f0829427cf2b681ba13331","truth_sha1":"c5df7afd81b456a850bbe70cfde7a48e81dabcac","overlaps_test":false,"full_text_reference":true}, +{"id":"0664","category":"documentation","split":"dev","html_sha1":"12111ffd92652027b28f773c5cd8b5661afc0a5a","truth_sha1":"a24240ccc7b748fe13a55adf06f89a3c5c9aa184","overlaps_test":false,"full_text_reference":true}, +{"id":"0665","category":"documentation","split":"dev","html_sha1":"7423dfb79f148f6cffe5c5f294dd93047f7956ab","truth_sha1":"da4943e8e4f6e74b8ade8c16a66877cf5a57a00b","overlaps_test":false,"full_text_reference":true}, +{"id":"0666","category":"documentation","split":"dev","html_sha1":"cdf8d53206cfd583dbb0ab3817eeca0f1ba12226","truth_sha1":"34d202bddaec31eceee3d6c389027e0cf2bf56da","overlaps_test":false,"full_text_reference":true}, +{"id":"0667","category":"documentation","split":"dev","html_sha1":"03b305fa3a836087e3239dd39ca8f5613ab9735e","truth_sha1":"5fe612d3580fd79aef63f7b733fbcb1dade89712","overlaps_test":false,"full_text_reference":true}, +{"id":"0668","category":"documentation","split":"dev","html_sha1":"04d8a9e9578a703f117e7f883cb90bad3083969d","truth_sha1":"65e782b0aaa4c770746af40e8c7a74aa11527842","overlaps_test":false,"full_text_reference":true}, +{"id":"0669","category":"documentation","split":"dev","html_sha1":"febef575f6cca53bd6b9014b3e84f0e3dc75b601","truth_sha1":"c83c9a1f14f3b866f0864159f81a7ce50b050994","overlaps_test":false,"full_text_reference":true}, +{"id":"0670","category":"documentation","split":"dev","html_sha1":"cccd1708613c352ea8e47281f30648f12d1a6f8f","truth_sha1":"da88f7353e808c16dcf19523642651359bd03cb5","overlaps_test":false,"full_text_reference":true}, +{"id":"0671","category":"documentation","split":"dev","html_sha1":"8f4ca7c36971da9da3efe3bde78a66b4ac96e750","truth_sha1":"6cba9d49a1196222099e4735d49e0207390655db","overlaps_test":false,"full_text_reference":true}, +{"id":"0672","category":"documentation","split":"dev","html_sha1":"74f254d9a3fdbacd1f61b8fbad8f091446c90337","truth_sha1":"bf650961d737c4164c63066278bf4b9d3ea429ba","overlaps_test":false,"full_text_reference":true}, +{"id":"0673","category":"documentation","split":"dev","html_sha1":"ac6399d0852f34e0b7d7282d19b59783956303aa","truth_sha1":"f62db1711bf6b056cd6b4ba6ee1ca41ea6f04f31","overlaps_test":false,"full_text_reference":true}, +{"id":"0674","category":"documentation","split":"dev","html_sha1":"7d94240e749498bd0bcfd0381c55f3945d9cc385","truth_sha1":"5cd850fd3d99ef4dba984f7042a17b00825a2dfa","overlaps_test":false,"full_text_reference":true}, +{"id":"0675","category":"documentation","split":"dev","html_sha1":"80b21f8c1333fbc7ad0563f1ab8a13c3dfa76ec4","truth_sha1":"d036bc856eb6620efc2400eee12dfb2d9068c4c5","overlaps_test":false,"full_text_reference":true}, +{"id":"0676","category":"documentation","split":"dev","html_sha1":"cbcec30595837d2544a81bf54c8845c9918a35da","truth_sha1":"5aaf7dd2c9ff2fee7a8648facab469a71872cba1","overlaps_test":false,"full_text_reference":true}, +{"id":"0677","category":"documentation","split":"dev","html_sha1":"d3d4d26740801bf34468cc7f36e9612d373da269","truth_sha1":"3fef0fe6d62935bfc4c6f444c822b27f17ed9d04","overlaps_test":false,"full_text_reference":true}, +{"id":"0678","category":"documentation","split":"dev","html_sha1":"2e41455b668050488d155ba191256bce46ce8160","truth_sha1":"26ce31f1f2d69d8bbf5e3b4619d528a1ff16ef46","overlaps_test":false,"full_text_reference":true}, +{"id":"0679","category":"documentation","split":"dev","html_sha1":"dc57919fec04781f936c085499de935d8b60a288","truth_sha1":"56c02e44c17ae29fe4e09ae2f0b5a6c78e566935","overlaps_test":false,"full_text_reference":true}, +{"id":"0680","category":"documentation","split":"dev","html_sha1":"781ce18e521e05bffab1a656a5c67513b14bb3ad","truth_sha1":"355a84620019987ba63e2c8db3eba292484a5d0a","overlaps_test":false,"full_text_reference":true}, +{"id":"0681","category":"documentation","split":"dev","html_sha1":"e4455390b55bdf4eb1da66749a68f1eba26c4a9e","truth_sha1":"5eaa5c1565eaf594c65ab55ab5e32e97263fb9dd","overlaps_test":false,"full_text_reference":true}, +{"id":"0682","category":"documentation","split":"dev","html_sha1":"e89c1c7991c46d09c60f48e0e466d3e0e7226093","truth_sha1":"2392716753f63ef1654af3df33b5c54dcf0369f6","overlaps_test":false,"full_text_reference":true}, +{"id":"0683","category":"documentation","split":"dev","html_sha1":"d3e117134b9dfe5c08824ad2d7f0fb2eaf44b3fa","truth_sha1":"2455ecb947b87fbf626d4e51ef661d7bbb8a3e5a","overlaps_test":false,"full_text_reference":true}, +{"id":"0684","category":"documentation","split":"dev","html_sha1":"ef659aa5d871ceef7924d2dc7f06c649e2e827a7","truth_sha1":"0ba810012622fc40d1816b824ec81adfa50f0958","overlaps_test":false,"full_text_reference":true}, +{"id":"0685","category":"documentation","split":"dev","html_sha1":"f766e3123ac38a6561321a374b042d1b2087ec52","truth_sha1":"9199f48d1cae7fd6bc12f269c9d566e53599b253","overlaps_test":false,"full_text_reference":true}, +{"id":"0686","category":"documentation","split":"dev","html_sha1":"ce1a153e7f8d55d60df706aa101b003879682989","truth_sha1":"524cc4ae5731b373d5463b0a62eccb5a3fb0a697","overlaps_test":false,"full_text_reference":true}, +{"id":"0687","category":"documentation","split":"dev","html_sha1":"9872d801b5c898174ac62eca45d5ce8321253bf6","truth_sha1":"f3aba26d8a7ebabc12dca9b06fbdc7cf3b5075eb","overlaps_test":false,"full_text_reference":true}, +{"id":"0688","category":"documentation","split":"dev","html_sha1":"df5e4c0a9f943635046592600998540668944cb8","truth_sha1":"1db1135ac00f767efaadbd0bed64cefaa39ddac3","overlaps_test":false,"full_text_reference":true}, +{"id":"0689","category":"documentation","split":"dev","html_sha1":"d590a1ca3d585731473142e5d0331d845f8dddcd","truth_sha1":"96700b3193467dc1e3e317ddd9c17ee1b0fa6af1","overlaps_test":false,"full_text_reference":true}, +{"id":"0690","category":"service","split":"dev","html_sha1":"47b0a368aa2c37486670b2bc90dcfdbc4ca63e93","truth_sha1":"35aa02eb567b3e3c451bcb464f3c3a2e55be892d","overlaps_test":false,"full_text_reference":true}, +{"id":"0691","category":"service","split":"dev","html_sha1":"a401be20bd13ea89e7a88fcfaaaa4906f8ca7c5f","truth_sha1":"bc4b2b7403264fc23827ce86ebbb8cc0e5f3ee9a","overlaps_test":false,"full_text_reference":true}, +{"id":"0692","category":"service","split":"dev","html_sha1":"f9db96df4764481e2efb2354c01feda679ea1706","truth_sha1":"0ac80a3a31f803bb9afd4766ac6d572f16ec5d37","overlaps_test":false,"full_text_reference":true}, +{"id":"0693","category":"service","split":"dev","html_sha1":"c5244ad9d3211b189788213da21ea45a1d0623fe","truth_sha1":"8c6a60c2d7a2bc8ef346d7f25362c527b973dc08","overlaps_test":false,"full_text_reference":true}, +{"id":"0694","category":"service","split":"dev","html_sha1":"b6a556b3f9d73dbfbcf23755550969c60dad5245","truth_sha1":"9e185c903749eae4b5b6ca53760ff907d83f4d30","overlaps_test":false,"full_text_reference":true}, +{"id":"0695","category":"service","split":"dev","html_sha1":"3176db00bfbe91bbfd13302e7260e6f6f2871cfd","truth_sha1":"fcd790d1b98c7c0295fa4c3597fc20de401641bc","overlaps_test":false,"full_text_reference":true}, +{"id":"0696","category":"service","split":"dev","html_sha1":"393cd3ab2d54134f2e40b7a3b2d133e0497beac5","truth_sha1":"764a4044aaa2da795a6741729016e8b6a88ff650","overlaps_test":false,"full_text_reference":true}, +{"id":"0697","category":"service","split":"dev","html_sha1":"e3e0da2cc30d62cd65def7521778803542942ab4","truth_sha1":"e0ea99e110275ad531ac148af04d1eecace527a5","overlaps_test":false,"full_text_reference":true}, +{"id":"0698","category":"service","split":"dev","html_sha1":"dcc10894fb41a61354710f78422dab7783c9f432","truth_sha1":"673e1cdbffb80f120beb0f3960a2644a3c311853","overlaps_test":false,"full_text_reference":true}, +{"id":"0699","category":"service","split":"dev","html_sha1":"565fcd6a6be8203dd78ab9e74605062f8702ded4","truth_sha1":"02e40967e7152146a6065d27f1a73e0c49cc2837","overlaps_test":false,"full_text_reference":true}, +{"id":"0700","category":"service","split":"dev","html_sha1":"4a8767aca7e229b7aa87b0933fcdd48747c2bb78","truth_sha1":"989f0df9c5b241061d06a3d268938c1665d3dbdb","overlaps_test":false,"full_text_reference":true}, +{"id":"0701","category":"documentation","split":"dev","html_sha1":"1823cda28cfe1a89c6ed75fc940d4bde7e3f9a6a","truth_sha1":"9da8d9fde1586b82f92c029c81fc6e93429b20b9","overlaps_test":false,"full_text_reference":true}, +{"id":"0702","category":"service","split":"dev","html_sha1":"51a0d0a7cbda496ce7494359ca4d237acc5408b2","truth_sha1":"97054a1ac7f19d32a4721615dc98965ce62ff5cb","overlaps_test":false,"full_text_reference":true}, +{"id":"0703","category":"documentation","split":"dev","html_sha1":"3d4b690e4ff0df04793ff03b235db9929576c20c","truth_sha1":"957a14f9b62653a83b526b4dce33fd2ee6bc3534","overlaps_test":false,"full_text_reference":true}, +{"id":"0704","category":"service","split":"dev","html_sha1":"92f2097f49fde78fe902c6ded5a595aa098620f3","truth_sha1":"f33fe3b6d980d41e67c1625629de911f71152ff9","overlaps_test":false,"full_text_reference":true}, +{"id":"0706","category":"service","split":"dev","html_sha1":"942ab9d7ae12a2d6acf12a3bddf2b11ff51aeef6","truth_sha1":"1088b03c622aaae5daf5121caece1cb8172cb1a4","overlaps_test":false,"full_text_reference":true}, +{"id":"0707","category":"service","split":"dev","html_sha1":"7c00bdc682429e2d609435844dd10f29432194dd","truth_sha1":"636dac9f5c9cb31a4fa65adff3102a5e52fb6a1d","overlaps_test":false,"full_text_reference":true}, +{"id":"0708","category":"service","split":"dev","html_sha1":"50ee0402a4fd4e57302040ea405acfd9bb43e0c1","truth_sha1":"196b305daaf302dd62ed3084f921b7ba92cc1281","overlaps_test":false,"full_text_reference":true}, +{"id":"0709","category":"service","split":"dev","html_sha1":"b184740f8bc59a03a01c2becea3eaf0f7a9ba340","truth_sha1":"9ae289b2537bc3e829cddf86a00a1007d4944805","overlaps_test":false,"full_text_reference":true}, +{"id":"0710","category":"service","split":"dev","html_sha1":"9552c7fab2c02f406c9d51b70b278b7a5ee243f3","truth_sha1":"10cc33e853aea65bbc5a11a35aa9842112c3df38","overlaps_test":false,"full_text_reference":true}, +{"id":"0711","category":"service","split":"dev","html_sha1":"4aabec3607fac60135cb856d1b466965522bf399","truth_sha1":"0f5f05b8683613a0d3b80608ed1e8377d59ffd8a","overlaps_test":false,"full_text_reference":true}, +{"id":"0712","category":"service","split":"dev","html_sha1":"44b9510af0e053781597348225fa0c774c32d55c","truth_sha1":"0adf4e5bacd7add76872f6293bc8c29269eef0a5","overlaps_test":false,"full_text_reference":true}, +{"id":"0713","category":"service","split":"dev","html_sha1":"6606ae1aeb6e068864a1fcaa709281ddfc6d8c84","truth_sha1":"e8e4ff62baa2cb6f303b7dcf79564779b7b1f745","overlaps_test":false,"full_text_reference":true}, +{"id":"0714","category":"service","split":"dev","html_sha1":"16573cdd34d24a43222c13f808bd0bcd5a188de1","truth_sha1":"dfcf2578774542bef9c77b14c07c3e2aa8848a40","overlaps_test":false,"full_text_reference":true}, +{"id":"0715","category":"service","split":"dev","html_sha1":"af9e269e03d8ecc558314b28547e30f8efb5b3d1","truth_sha1":"c571f3d3539fde9a57a6c390635bb94541647f62","overlaps_test":false,"full_text_reference":true}, +{"id":"0716","category":"service","split":"dev","html_sha1":"5e666cd37a728ff013f1882fc0846cea6f31990e","truth_sha1":"0573961fdd0616438426dd97faf86d5412993db2","overlaps_test":false,"full_text_reference":true}, +{"id":"0717","category":"service","split":"dev","html_sha1":"147e7735ce09d8740df199f6aec10cdb00f2a66b","truth_sha1":"a82607b2f532caeecc1a3d3b8958d1148605b7bc","overlaps_test":false,"full_text_reference":true}, +{"id":"0718","category":"service","split":"dev","html_sha1":"ae00b6b649cf7fe2704d8856967dac43d7b03875","truth_sha1":"a807748af937352d773d2d34c77db18ae6e8022d","overlaps_test":false,"full_text_reference":true}, +{"id":"0719","category":"service","split":"dev","html_sha1":"fb8001691604739f5b8096b79058d87953dc39c7","truth_sha1":"bdf1d00146882194d88e7253098f8272908f3e5c","overlaps_test":false,"full_text_reference":true}, +{"id":"0729","category":"collection","split":"dev","html_sha1":"2e5a9316c92f1393208d4189723a0e5804d2476b","truth_sha1":"0e167d2117d0b6be60211d29c6a0fbadfba13cc6","overlaps_test":false,"full_text_reference":true}, +{"id":"0767","category":"listing","split":"dev","html_sha1":"29587ca02775cdae85e15c2195f3bc75e0827ac4","truth_sha1":"e5e634659d2465dcc23491b98b4c728b3c15e6df","overlaps_test":false,"full_text_reference":true}, +{"id":"0781","category":"article","split":"dev","html_sha1":"72572d09acadecee4c0d8823f38c0e37803f6b2e","truth_sha1":"57f926e24b3d68c97ea24b52b63d472725454959","overlaps_test":false,"full_text_reference":true}, +{"id":"0823","category":"listing","split":"dev","html_sha1":"bfe580b33f8387584cac767684916351aca5628a","truth_sha1":"437be7f15343187acf495690919d0d3ad26a3907","overlaps_test":false,"full_text_reference":true}, +{"id":"0847","category":"listing","split":"dev","html_sha1":"5eafbb74c527d1ab9371337f5bff637190add0ea","truth_sha1":"8c3a80768c4e9d368667f9dbde98adb1f5310e00","overlaps_test":false,"full_text_reference":true}, +{"id":"0898","category":"article","split":"dev","html_sha1":"bab508ba6c18e1f058dd876cbc691d298ed83d9a","truth_sha1":"ed47ec52caee428fc918f4785dfa7e0518d66344","overlaps_test":false,"full_text_reference":true}, +{"id":"0956","category":"collection","split":"dev","html_sha1":"2d522387a4a7e32fd3d193155af2561993bf5220","truth_sha1":"d5c92aed5bd93c3829c44704187a6cd0e4329806","overlaps_test":false,"full_text_reference":true}, +{"id":"1054","category":"service","split":"dev","html_sha1":"4c460c038aed80451b51cabe2d078148b6d8318a","truth_sha1":"335d224b16224a7a6f02841b17b50a6ab9afc636","overlaps_test":false,"full_text_reference":true}, +{"id":"1056","category":"service","split":"dev","html_sha1":"f7565c461414cf555dfc494f3c55434277cf19ae","truth_sha1":"032ac0cc55643562bb237b272116a090cec46d26","overlaps_test":false,"full_text_reference":true}, +{"id":"1066","category":"forum","split":"dev","html_sha1":"5e1537473394a0b58dc3e2183581c1b544774f13","truth_sha1":"7894713a7963905b3a564e0a125a15ba4d1c2293","overlaps_test":false,"full_text_reference":true}, +{"id":"1069","category":"forum","split":"dev","html_sha1":"1863eadeff6cbb901f51db7d1bdb7c67f0e02905","truth_sha1":"84b92c0e499f0ef4db938eca0603a202791087c9","overlaps_test":false,"full_text_reference":true}, +{"id":"1087","category":"service","split":"dev","html_sha1":"e0292ff2aa54606e86208d9ccd569bd9b0fcee1a","truth_sha1":"5fd49af33741bd9fd7ae7450f9320f897e7f9cc8","overlaps_test":false,"full_text_reference":true}, +{"id":"1126","category":"forum","split":"dev","html_sha1":"1caede297113dfd814648811b618dd24e77c4b54","truth_sha1":"d9584c00078f94cf9b2145bb1b9144e69cbd5539","overlaps_test":false,"full_text_reference":true}, +{"id":"1142","category":"product","split":"dev","html_sha1":"a5cc2b80d848a9dfafd0817261d6a3acc8d8a051","truth_sha1":"1cda7dffeeed81d3e350a51fde8047e7094eec26","overlaps_test":false,"full_text_reference":true}, +{"id":"1177","category":"forum","split":"dev","html_sha1":"1ee95a6c0cd18d76279ff0fbae8e042d8b81705a","truth_sha1":"b46621d7fac0a10282f9b467e43adb7822c3aab1","overlaps_test":false,"full_text_reference":true}, +{"id":"1222","category":"article","split":"dev","html_sha1":"c7a8336bc99d713fc8a29c64574cf7a55a39b561","truth_sha1":"21b6463e8bfcda865b519420b078e59726d1956b","overlaps_test":false,"full_text_reference":true}, +{"id":"1310","category":"collection","split":"dev","html_sha1":"65f352b24fae53a8cff66c1624872a007808224e","truth_sha1":"12b9769c493364d3130dc4eb1f9413efeafbb8d8","overlaps_test":false,"full_text_reference":true}, +{"id":"1350","category":"forum","split":"dev","html_sha1":"d7182ef28325280a5f5caab15228da580d010703","truth_sha1":"abd67d990b999dd2993bdc1b75f99ac493bac3c3","overlaps_test":false,"full_text_reference":true}, +{"id":"1371","category":"listing","split":"dev","html_sha1":"5ff0b4be440d832aba576439d1d69f0318c5eaae","truth_sha1":"76c2bdab65be0e3ef06a463f17114ba4f4dba450","overlaps_test":false,"full_text_reference":true}, +{"id":"1377","category":"product","split":"dev","html_sha1":"0c8617de080361fe632040f3214bec4c5e7b3249","truth_sha1":"9f897a9a975a64b4b0d134ec8fbd1744def9d68a","overlaps_test":false,"full_text_reference":true}, +{"id":"1396","category":"article","split":"dev","html_sha1":"ebd21f5de570cb2107e5634e7883a526a7a71963","truth_sha1":"9c5361dba3eac4c6a151ef4566c26f837c824db1","overlaps_test":false,"full_text_reference":true}, +{"id":"1405","category":"article","split":"dev","html_sha1":"8882445e3c46523e460e8a22b2591a8705f006f2","truth_sha1":"26d31561b0d52de840f87468a65cf6b3208216e5","overlaps_test":false,"full_text_reference":true}, +{"id":"1435","category":"collection","split":"dev","html_sha1":"011098b79421ba445874d5d3ef070eb5b3c20c3d","truth_sha1":"d62124da08925e0e742015708ef7fe35affc4879","overlaps_test":false,"full_text_reference":true}, +{"id":"1442","category":"documentation","split":"dev","html_sha1":"351cd76e8df768234db44fae6ef1e6d6eee782a8","truth_sha1":"807adfd3959427c336f985b76fd1cc8d60b9871b","overlaps_test":false,"full_text_reference":true}, +{"id":"1476","category":"article","split":"dev","html_sha1":"cda7d6bb9804f196e41fc64706614b6935516d37","truth_sha1":"8d33bc13116a9b3008a5d5d0ed8a01d3ed35771d","overlaps_test":false,"full_text_reference":true}, +{"id":"1520","category":"service","split":"dev","html_sha1":"3151d665889e324832cd33ad37426d65d44f62f2","truth_sha1":"0d61a18d1aa199035d47f96b67e69f25a7134ecf","overlaps_test":false,"full_text_reference":true}, +{"id":"1572","category":"listing","split":"dev","html_sha1":"a1981d082425ee9657412ba0782a339c7ff69038","truth_sha1":"08e2a7523943cec032738fd8b09039f713165614","overlaps_test":false,"full_text_reference":true}, +{"id":"1579","category":"article","split":"dev","html_sha1":"e34b85f13e1778bf36169b523007f18586b7321a","truth_sha1":"743849fc0df2163320dd4f6227d479f8d69bc085","overlaps_test":false,"full_text_reference":true}, +{"id":"1591","category":"forum","split":"dev","html_sha1":"96b1e73224f4e9d41ff4a6b1f136e9dc1fe7b3b8","truth_sha1":"417eb2f17b4ab3431f1e28f73487796575e1220c","overlaps_test":false,"full_text_reference":true}, +{"id":"1628","category":"article","split":"dev","html_sha1":"9219b2c4e79bc7ee2bc6fb868c5f5570145d9dee","truth_sha1":"c94a773b2a2b868b423272c6e4fc2a055d77a040","overlaps_test":false,"full_text_reference":true}, +{"id":"1647","category":"service","split":"dev","html_sha1":"293be562c6bac81863a036807f24a1260e3e62bf","truth_sha1":"f1118c2edf1d323a0119381163dcd13c8bd65910","overlaps_test":false,"full_text_reference":true}, +{"id":"1651","category":"article","split":"dev","html_sha1":"b9bba1e30239c1955f6cf8cf92308e5b6bc81596","truth_sha1":"d690d596337aebe0a737729a8588d6cd19d35e8e","overlaps_test":false,"full_text_reference":true}, +{"id":"1657","category":"article","split":"dev","html_sha1":"5f3edf059522913765c209b0a057b53160aa08f5","truth_sha1":"81f127b280b73dd0fbe2c1b448ecc201872d6d34","overlaps_test":false,"full_text_reference":true}, +{"id":"1658","category":"forum","split":"dev","html_sha1":"156cfeb7220edab4bfc7b4c60a867a3f848fd811","truth_sha1":"5120c9174c0d32275ebc1d3996c48a6e27cd75ff","overlaps_test":false,"full_text_reference":true}, +{"id":"1674","category":"forum","split":"dev","html_sha1":"db0c8add2884354b001b8ffac6a00f8cc14343e6","truth_sha1":"d91e788ac702bf9059776924b2624690932166fe","overlaps_test":false,"full_text_reference":true}, +{"id":"1708","category":"article","split":"dev","html_sha1":"d39d67d4bcbe4f16940eefefe62b799d89b0fbdd","truth_sha1":"d38ad895e5eb4c17a096587567b335789639b29e","overlaps_test":false,"full_text_reference":true}, +{"id":"1764","category":"forum","split":"dev","html_sha1":"0375020427e1efc86facaea77c4af59f6dafd8b2","truth_sha1":"cb6d6549974848218c16ad2ff00747331b8fb99a","overlaps_test":false,"full_text_reference":true}, +{"id":"1766","category":"documentation","split":"dev","html_sha1":"09ca838d13f8e51fe67c6e282f59e3ff6ee8fbf1","truth_sha1":"f66a73fb6f15ad2f8d2d39960c4dc8a7239f7e3b","overlaps_test":false,"full_text_reference":true}, +{"id":"1781","category":"article","split":"dev","html_sha1":"a648d392afb894a0eb8fcce4f423ca0b3ea3f2ba","truth_sha1":"1ddd6039d36680178f4874d3e9789a3a9277de1f","overlaps_test":false,"full_text_reference":true}, +{"id":"1788","category":"service","split":"dev","html_sha1":"528fe239400b937d7c8f3a6de61b604691d2d6c8","truth_sha1":"21834d7bb126d0b1e5db2f4b6d6bfb016088308d","overlaps_test":false,"full_text_reference":true}, +{"id":"1789","category":"article","split":"dev","html_sha1":"ae728136f33a6de232363513bb9541905d5d8aa7","truth_sha1":"7dae6c59b3d3ea2b0a0f29194bf157328018781c","overlaps_test":false,"full_text_reference":true}, +{"id":"1795","category":"listing","split":"dev","html_sha1":"750be38c802ea40f3d106a5c00dbb6d6a3d495ac","truth_sha1":"f8a29081248dfa831f5d8aecc499b0a1efa6dfe3","overlaps_test":false,"full_text_reference":true}, +{"id":"1806","category":"article","split":"dev","html_sha1":"73b0cb63a450128a7fc15a2f180c050a47c1bfe3","truth_sha1":"d66fd4e98d26a9fe7464f10624ff754290d3cf57","overlaps_test":false,"full_text_reference":true}, +{"id":"1821","category":"article","split":"dev","html_sha1":"0dccf1124fa0e53d9ad39b997d87238ea7f6144b","truth_sha1":"ec6d056d3273bf281c564b023080882480cb3a77","overlaps_test":false,"full_text_reference":true}, +{"id":"1837","category":"article","split":"dev","html_sha1":"9e6adc5e4206bb6ecb26c7f9e125689a0ff685e2","truth_sha1":"a834f8f81a99df4543f7725d561a8e9802f637a1","overlaps_test":false,"full_text_reference":true}, +{"id":"1974","category":"article","split":"dev","html_sha1":"ef2a4f3111b2ed9a3373b01046a34d019563d465","truth_sha1":"af1c5f4ebe78dd8ec05330e659b62d197f528663","overlaps_test":false,"full_text_reference":true}, +{"id":"1976","category":"forum","split":"dev","html_sha1":"8fbb9ec673f0769980db48273241078875f6b033","truth_sha1":"5aa7b8fcc525d7731cdd8fc30bf945c530565534","overlaps_test":false,"full_text_reference":true}, +{"id":"2018","category":"article","split":"dev","html_sha1":"60e6091b158104a384794d60a3b8e0c13a51ce7f","truth_sha1":"c1e63cdd1e3be6dcba725d813a9a37a4767d3635","overlaps_test":false,"full_text_reference":true}, +{"id":"2020","category":"article","split":"dev","html_sha1":"6df0fdd488de5a9533e1fc6339a1238bb9029ae7","truth_sha1":"3832b06f27a91f52575cfc11c849c792d9261482","overlaps_test":false,"full_text_reference":true}, +{"id":"2035","category":"article","split":"dev","html_sha1":"03320d348c63e17cf92010a4fdb8f1b9cade2b5f","truth_sha1":"ab062a17b81fc38deb6e9caf562a557390b6595e","overlaps_test":false,"full_text_reference":true}, +{"id":"2081","category":"forum","split":"dev","html_sha1":"fd84ca5792bec47000f79118ae58de2e24be9063","truth_sha1":"2e229e043dbf5eaae210db4ed38b795566b2b673","overlaps_test":false,"full_text_reference":true}, +{"id":"2102","category":"listing","split":"dev","html_sha1":"a6a13b45715b683b29770026812a522df8579361","truth_sha1":"8114cea51db02f42fe90acd0f51e24a7069bff04","overlaps_test":false,"full_text_reference":true}, +{"id":"2111","category":"service","split":"dev","html_sha1":"b83b69c57cab39e218dac87382fc2c8f5d63f54e","truth_sha1":"24cffc9006a87016f978a2be7f79ed4c4e50fcac","overlaps_test":false,"full_text_reference":true}, +{"id":"2157","category":"article","split":"dev","html_sha1":"e60a444265b427c35bc0722b7d444a3998f213ab","truth_sha1":"6a4d51830bcb8f80355b75cf3df5aa4e0ca868ed","overlaps_test":false,"full_text_reference":true}, +{"id":"2201","category":"forum","split":"dev","html_sha1":"e9cd097fdd3b8fe51fe2494b21a44a8616081c3f","truth_sha1":"95be4b156fff1721925157815ca6f707377a15b4","overlaps_test":false,"full_text_reference":true}, +{"id":"2218","category":"article","split":"dev","html_sha1":"2e77f78ef0c30311ad7c284847eb5e5371d37e1c","truth_sha1":"ee42534ec9760892681c84843b6462701ff710bd","overlaps_test":false,"full_text_reference":true}, +{"id":"2235","category":"article","split":"dev","html_sha1":"3c4751422508f9f6d0e6368be3e1b9dc2f9e448c","truth_sha1":"c604f711b1e293d50220dd6bf94816dab1647965","overlaps_test":false,"full_text_reference":true}, +{"id":"2269","category":"article","split":"dev","html_sha1":"119d1726afdda7068ba9cc0604b96fe640c99098","truth_sha1":"27828ea88e17179fe20e1bc74d59c447ddd46c07","overlaps_test":false,"full_text_reference":true}, +{"id":"2301","category":"article","split":"dev","html_sha1":"934aaf5f0e6ffdabd02ba96295321b02e4e6bb10","truth_sha1":"a5ef22ca55eb2eebf66db31737e67cbb5eeba86b","overlaps_test":false,"full_text_reference":true}, +{"id":"2362","category":"article","split":"dev","html_sha1":"0a13c760edf938a2fe9ee134bffbe43275a3e875","truth_sha1":"d1da359dfbb6a284f060554e66fde7c42070f1c3","overlaps_test":false,"full_text_reference":true}, +{"id":"2381","category":"article","split":"dev","html_sha1":"b81ddad715ca863e1fa872dda71d09e7416b6e75","truth_sha1":"f68b70c09c351788e3d061523ebb7443b5d7a21e","overlaps_test":false,"full_text_reference":true}, +{"id":"2393","category":"documentation","split":"dev","html_sha1":"8538a5ebfbd8e19591e77f04b477ed00bdd75726","truth_sha1":"82f6572661ab8f9362e869908ec7f9a114cfe507","overlaps_test":false,"full_text_reference":true}, +{"id":"2414","category":"article","split":"dev","html_sha1":"cfc8fdd695be78d0766fffa17754f117e6afaddf","truth_sha1":"1ea4e52095ce50412151f281b21be0ef699a69d2","overlaps_test":false,"full_text_reference":true}, +{"id":"2453","category":"article","split":"dev","html_sha1":"0d656b9175d50f617fd1715a84d6a0dfcea387fc","truth_sha1":"eddd98891d637e9e275764657048b18e56807957","overlaps_test":false,"full_text_reference":true}, +{"id":"2498","category":"article","split":"dev","html_sha1":"97f67cb3a53d46bd5caed01edaf793676aac09b0","truth_sha1":"a41301de48fcb733e8e525feb844be704f47ed53","overlaps_test":false,"full_text_reference":true}, +{"id":"2534","category":"article","split":"dev","html_sha1":"8ace3325169665594641f24d07913c62ca57c201","truth_sha1":"0b7c3e4fb85749cea12c54058c431dacb0fef225","overlaps_test":false,"full_text_reference":true}, +{"id":"2645","category":"article","split":"dev","html_sha1":"189612457dd358310aacce10cbde161c3305ce3b","truth_sha1":"8a016aea3a6aef968f1da311937428e0e3409203","overlaps_test":false,"full_text_reference":true}, +{"id":"2660","category":"documentation","split":"dev","html_sha1":"d8b4579da0579795df784403c79ff7862ee2d65e","truth_sha1":"0406aa20f631384ecb0d3b3b6822b25a7d48a708","overlaps_test":false,"full_text_reference":true}, +{"id":"2684","category":"article","split":"dev","html_sha1":"91905b40434f76b01f86a8ccbf0ba77fec37e77b","truth_sha1":"4e9532c55abed5041656cf168fa0f4fbc119563a","overlaps_test":false,"full_text_reference":true}, +{"id":"2722","category":"article","split":"dev","html_sha1":"e112cd62094bd1897bfe481c4ef382921a5a6042","truth_sha1":"93933ab36d334a4a10f3a8740f9a1cdb96194a79","overlaps_test":false,"full_text_reference":true}, +{"id":"2740","category":"article","split":"dev","html_sha1":"3c96fe16e9646a28b6ff08b13ed9b959cf7a6825","truth_sha1":"80857467fee91b539ce2fff81893fafe2b6ee955","overlaps_test":false,"full_text_reference":true}, +{"id":"2789","category":"article","split":"dev","html_sha1":"96d0908dc9197c6100caed73b67f35c77933f69f","truth_sha1":"583a2b18ecafb4af45e9f2dcb51cc5f923f0acf0","overlaps_test":false,"full_text_reference":true}, +{"id":"2797","category":"forum","split":"dev","html_sha1":"26e1b6ca0b58f78d0ffeefbd9b1b897354136502","truth_sha1":"93a9ebe4e3fc1789ea794536d748142935603dbd","overlaps_test":false,"full_text_reference":true}, +{"id":"2825","category":"article","split":"dev","html_sha1":"1e2e43188adc52c70ad8f36ba460c7b110e0c0be","truth_sha1":"7cb3a10801740e16db1805df7ba5a987cebb4832","overlaps_test":false,"full_text_reference":true}, +{"id":"2826","category":"forum","split":"dev","html_sha1":"ec9bb8e896eed0e29d851db48289b1085e28893d","truth_sha1":"2f214f637e033b1b8c42a4f68a3d4c1e83507d24","overlaps_test":false,"full_text_reference":true}, +{"id":"2827","category":"forum","split":"dev","html_sha1":"7e0ac1549656f35b8d9304f7b40b2fbdbee443b2","truth_sha1":"5977e8e2ba78eb67561bdb6ed01f0fbafaff1ab2","overlaps_test":false,"full_text_reference":true}, +{"id":"2828","category":"forum","split":"dev","html_sha1":"88a5e158f30706d4cb7cfc9efd003fb27308c36d","truth_sha1":"93cf5bdc74cef2a7715515f58a3cc2ef2f0770c8","overlaps_test":false,"full_text_reference":true}, +{"id":"2831","category":"service","split":"dev","html_sha1":"b768be14dfecb4a9084f4735002565818e0ff50f","truth_sha1":"24a30ad65919543e6869cf0b827619a2ec6cbe0d","overlaps_test":false,"full_text_reference":true}, +{"id":"2832","category":"service","split":"dev","html_sha1":"9b1ba74a791eb0e64b07c2c267417863bf0edd5e","truth_sha1":"6a7da78fe80c3f0ca6cb063f447c0f795e75aacd","overlaps_test":false,"full_text_reference":true}, +{"id":"2833","category":"service","split":"dev","html_sha1":"f693802262f3f7c0bf5e87e75dfebab45b712718","truth_sha1":"a6351deef64c7a47d9b810eed15fad862655f654","overlaps_test":false,"full_text_reference":true}, +{"id":"2834","category":"service","split":"dev","html_sha1":"3ac69592f493e6c9219746f9f612fcaf42f64815","truth_sha1":"8b3baff8dcac3ca9e8b3b1cf90264d96dda98ac8","overlaps_test":false,"full_text_reference":true}, +{"id":"2835","category":"service","split":"dev","html_sha1":"d0b16bc00103ad33676bccd5106800b37f49d11a","truth_sha1":"e079068ee0c37c36783cb56496bacb0ef6fc5830","overlaps_test":false,"full_text_reference":true}, +{"id":"2836","category":"service","split":"dev","html_sha1":"15fbb2453603d15549a31f3cc970a0d879f303aa","truth_sha1":"93fbb03784b4331c2d651f8c11e5af76aa3ad38a","overlaps_test":false,"full_text_reference":true}, +{"id":"2837","category":"service","split":"dev","html_sha1":"77e32d466d8c84106a99d608dcb510bbd6b7b91c","truth_sha1":"c9b3b177e71e1ea8f491141ab494482624719f14","overlaps_test":false,"full_text_reference":true}, +{"id":"2838","category":"service","split":"dev","html_sha1":"5455cc5863efc93985d63cfe7bf24ba9e94dd6d5","truth_sha1":"ab382ae85074d1ba0f550b4d1a548ed0be8b4908","overlaps_test":false,"full_text_reference":true}, +{"id":"2839","category":"service","split":"dev","html_sha1":"acd961088f69cd7861d0b60158212a2f2e127931","truth_sha1":"33297ae6efdd7bd478aad7b76d0fe19272dd9db2","overlaps_test":false,"full_text_reference":true}, +{"id":"2840","category":"service","split":"dev","html_sha1":"671e1f1a450713106eab7fe00ca938c3aacb38cf","truth_sha1":"8e6565e1dc789620ee7c96a6e74ea7d3dea241b4","overlaps_test":false,"full_text_reference":false}, +{"id":"2841","category":"service","split":"dev","html_sha1":"842f075d47d6bec684a19f87f1cf2e256b4b94f8","truth_sha1":"000deffa770d423b182602e8170c9e867613346d","overlaps_test":false,"full_text_reference":true}, +{"id":"2842","category":"service","split":"dev","html_sha1":"47583722c71f574c759c07f819fd46820b7fb1da","truth_sha1":"8b4a93cab04e75aaccaa58701e6508b346dfb103","overlaps_test":false,"full_text_reference":true}, +{"id":"2843","category":"service","split":"dev","html_sha1":"d8e4bcb9863912d7352249da5de3948f51e2f8d9","truth_sha1":"caa3c32f0652d79d49ea6e43fc6e8cdb401e0583","overlaps_test":false,"full_text_reference":true}, +{"id":"2844","category":"service","split":"dev","html_sha1":"a458b974e7273211100a2028c0971bd82a8942e7","truth_sha1":"ba3497905f5a27caa3ab2853eaa3134d7a50f545","overlaps_test":false,"full_text_reference":true}, +{"id":"2845","category":"service","split":"dev","html_sha1":"5364a3b9344b62b7caff3232ff90fbb208747709","truth_sha1":"b00fbf92f0443be96701742def21f0602e700d9e","overlaps_test":false,"full_text_reference":true}, +{"id":"2846","category":"service","split":"dev","html_sha1":"834b5a1887934619586b91bf51d4d50dba787167","truth_sha1":"e0e1c03c15ec05cbed1d0f513c68ef802bb273ae","overlaps_test":false,"full_text_reference":true}, +{"id":"2847","category":"service","split":"dev","html_sha1":"3664579e41b5ed98d541633f4b785844c40e2f9c","truth_sha1":"0887443559f091978f76dea5fc9a302d58ccad11","overlaps_test":false,"full_text_reference":true}, +{"id":"2848","category":"service","split":"dev","html_sha1":"868b562549f91d3bc9d286f70bcaa00aba5e7850","truth_sha1":"35c9cfc033e244b956c87b222b78d39802c3a963","overlaps_test":false,"full_text_reference":true}, +{"id":"2849","category":"service","split":"dev","html_sha1":"e6f5556129e8f1ddd85046aa5a13923b6e5c6b86","truth_sha1":"379d0b5cd472c703a014bc18ea6d1104a703f293","overlaps_test":false,"full_text_reference":true}, +{"id":"2850","category":"service","split":"dev","html_sha1":"23dddedd9c57cb624e6aec164849e0af94411017","truth_sha1":"109aa5bc521cd711cfb559c9c9aec94accaf4590","overlaps_test":false,"full_text_reference":true}, +{"id":"2851","category":"service","split":"dev","html_sha1":"af9479b6b4e470a98d4d0766ad291fe18fa57ba9","truth_sha1":"ee73e7c48b97f9c1105d1a4a22185de5dddd2d53","overlaps_test":false,"full_text_reference":true}, +{"id":"2852","category":"service","split":"dev","html_sha1":"7278aeabd883d712cbe13a11ba4765090e996c40","truth_sha1":"3178c7095b8981b27c2cc1cc5181fc2c9cf4fc4e","overlaps_test":false,"full_text_reference":true}, +{"id":"2853","category":"documentation","split":"dev","html_sha1":"d14e073896a1d88316721e1ad3df65c8929ad03d","truth_sha1":"f450906c9b229b168c44bd20964104ee66454bd9","overlaps_test":false,"full_text_reference":true}, +{"id":"2854","category":"documentation","split":"dev","html_sha1":"6a931b66246d661ba784edd4652cca61b3cd6d3d","truth_sha1":"e8ab203d76e91b9e1b70793155ade1a61baa9293","overlaps_test":false,"full_text_reference":true}, +{"id":"2855","category":"documentation","split":"dev","html_sha1":"f6132573d514ebd6b30a2e68fe478f563caefa24","truth_sha1":"27d823400b6774fedf207d1f273322db63b1ab24","overlaps_test":false,"full_text_reference":true}, +{"id":"2856","category":"documentation","split":"dev","html_sha1":"721b92e028d9d99ace440df6884a8512f7820ce4","truth_sha1":"76d70e7cba9e6a9ca7531dc8ea7da15c89b036dc","overlaps_test":false,"full_text_reference":true}, +{"id":"2857","category":"documentation","split":"dev","html_sha1":"47cd18da4749e33bd0b966abb594a0e2a6de7152","truth_sha1":"ee84170ca0eef1403ea3733f34bb686ed7afd65a","overlaps_test":false,"full_text_reference":true}, +{"id":"2858","category":"documentation","split":"dev","html_sha1":"56881fda1be7d0d98a01a90c225958714ed9431f","truth_sha1":"5b0de00e089ed9245152672b14cba25ff2f52f64","overlaps_test":false,"full_text_reference":true}, +{"id":"2859","category":"documentation","split":"dev","html_sha1":"411f8fa11a2752b18cbe2420d543ea903647750e","truth_sha1":"60d94a0f5b33411cb0c3ddc2c47abec5334a7f8f","overlaps_test":false,"full_text_reference":true}, +{"id":"2860","category":"documentation","split":"dev","html_sha1":"c6f60f951b134bbbaa7594a3b48058f96016e7db","truth_sha1":"95716a5c90dba738ea0a3d34db1f0935f83baccc","overlaps_test":false,"full_text_reference":true}, +{"id":"2861","category":"documentation","split":"dev","html_sha1":"3e1bbe5b013ed2ccfcc30224b711509c0b5645a8","truth_sha1":"921504c90976c0e7ee4313c067b01922dc9181d1","overlaps_test":false,"full_text_reference":true}, +{"id":"2862","category":"article","split":"dev","html_sha1":"b864ae6a44c90dd0f12a9107e7b3683256df41a0","truth_sha1":"72bb573d558c3d68ed638e4a9996890838e0e840","overlaps_test":false,"full_text_reference":true}, +{"id":"2863","category":"documentation","split":"dev","html_sha1":"8a8940ac1e4fac978071428e4a161e43b35c9142","truth_sha1":"4b11defa5fcf0d8fe22301ce3ea57c9c22fe0575","overlaps_test":false,"full_text_reference":true}, +{"id":"2864","category":"documentation","split":"dev","html_sha1":"7586763517b929987a9bfe1acbe3ec6811dc1492","truth_sha1":"6a078c4f3d349e101c8fa298a3c64cf387979284","overlaps_test":false,"full_text_reference":true}, +{"id":"2865","category":"documentation","split":"dev","html_sha1":"7b509b4a2f5a5ee614b64e6514d276ec7eaa506b","truth_sha1":"ad6764b7f9b4fe28f353e5f082644791cff12b08","overlaps_test":false,"full_text_reference":true}, +{"id":"2866","category":"documentation","split":"dev","html_sha1":"f77bd9c9d10b6044e38524853d8e5b8e7fa25b06","truth_sha1":"f801a3aca21538f549905584088bb138bb7b841c","overlaps_test":false,"full_text_reference":true}, +{"id":"2867","category":"documentation","split":"dev","html_sha1":"0839a715bcc43dc11b6e3eadf4e60d7a04b3bf1e","truth_sha1":"cbcdf18d34ea374b6454b68f256e037ac907ffad","overlaps_test":false,"full_text_reference":true}, +{"id":"2868","category":"documentation","split":"dev","html_sha1":"9052457ee7937e7f667c705280dcd115083c98b9","truth_sha1":"55b2f793e43359aa245dc01728dbb2c95e9f1cd7","overlaps_test":false,"full_text_reference":true}, +{"id":"2869","category":"documentation","split":"dev","html_sha1":"09736796ebfac9de0643708af11d7ca46980a6da","truth_sha1":"59dbc2822fd8422699b66408551e33e573351d5b","overlaps_test":false,"full_text_reference":true}, +{"id":"2870","category":"documentation","split":"dev","html_sha1":"2f2b1c116687dc3f0a71d1bc5638ab188caff1ac","truth_sha1":"e42abc14767d757b59a45623ed9342de83550ca8","overlaps_test":false,"full_text_reference":true}, +{"id":"2871","category":"documentation","split":"dev","html_sha1":"f626fd767e8032febc85779537dcb38c523adb26","truth_sha1":"911b674da0a5a885a99789e870deb4edb106c2f0","overlaps_test":false,"full_text_reference":true}, +{"id":"2872","category":"documentation","split":"dev","html_sha1":"fb9056791eeee4b32bd6ac207d90253b5a00acbb","truth_sha1":"806cc6e3b2c0a244b77e557d5230f897e9d0f1e6","overlaps_test":false,"full_text_reference":true}, +{"id":"2873","category":"documentation","split":"dev","html_sha1":"ae6037a401ff1116daa2bdf414cbd69f3b9883e4","truth_sha1":"32541cf39ba90202854bb8c23b400d0bd690892f","overlaps_test":false,"full_text_reference":true}, +{"id":"2874","category":"documentation","split":"dev","html_sha1":"b83edee99af7499c5ed3fc0998a5b29834c786eb","truth_sha1":"72355feef8395526b60bb976a4d1f41143ea2013","overlaps_test":false,"full_text_reference":true}, +{"id":"2875","category":"documentation","split":"dev","html_sha1":"a0171055cfebe26d67eeeeaa9e0e748524d312da","truth_sha1":"278786702859f08209ab5de2d0e800c04ff5e829","overlaps_test":false,"full_text_reference":true}, +{"id":"2876","category":"documentation","split":"dev","html_sha1":"0a506d163aec64ee9552dd3447dcce6af68fdbee","truth_sha1":"02d3077988c82b78e4a35eb95baafa27c926872a","overlaps_test":false,"full_text_reference":true}, +{"id":"2878","category":"documentation","split":"dev","html_sha1":"6f9e2b0eec46070c52a057527cb5edb6e72f9c04","truth_sha1":"5e1265f9138eb2d58da8a286612942f975faa4d7","overlaps_test":false,"full_text_reference":true}, +{"id":"2879","category":"documentation","split":"dev","html_sha1":"403c9bb373d8b2644f2d56fb99966437f61db12b","truth_sha1":"1c682a432f59b1cbe717e38f47bbfe6d11425170","overlaps_test":false,"full_text_reference":true}, +{"id":"2888","category":"product","split":"dev","html_sha1":"079acdcf59e6f2068b8d88106402c82cfda83316","truth_sha1":"633bace6d3329cac7e651c9f6a4c15834cda525e","overlaps_test":false,"full_text_reference":true}, +{"id":"2889","category":"product","split":"dev","html_sha1":"dd0e047e5824f4e0f848ea4ca29487ffbbae713f","truth_sha1":"70ad3c10f1c98f8f2abdbfaff53148b58157c3d8","overlaps_test":false,"full_text_reference":true}, +{"id":"2890","category":"product","split":"dev","html_sha1":"c0ba567202d5e73c588c6017b6cbb8eaf4180303","truth_sha1":"a0fdae752b227bbc6ba9e8b3cee135bf54cc4b42","overlaps_test":false,"full_text_reference":true}, +{"id":"2891","category":"article","split":"dev","html_sha1":"cf6f765d2d2341d1ac02a02471fafa8b502921dc","truth_sha1":"339aa3d4779b12e614d72fa019796b55885634c0","overlaps_test":false,"full_text_reference":true}, +{"id":"2893","category":"product","split":"dev","html_sha1":"3d15794bd713607140eca813ed5cdffed702c856","truth_sha1":"4cc8086e03bcb6a2a823628dac3a446896e14354","overlaps_test":false,"full_text_reference":true}, +{"id":"2894","category":"product","split":"dev","html_sha1":"781d0bedc41edca25a988429b6a2745219835910","truth_sha1":"2d5daf36b0b782f752525523cfba5febec8fbea9","overlaps_test":false,"full_text_reference":true}, +{"id":"2897","category":"collection","split":"dev","html_sha1":"2d0a9863dcac743e59dd99da81ca266852cd3c60","truth_sha1":"1cf615b5c140888c819494a46e76be2d40112182","overlaps_test":false,"full_text_reference":true}, +{"id":"2898","category":"product","split":"dev","html_sha1":"be81b1b8106a32e766ad9a2c4a97f6c10d5b70f3","truth_sha1":"aaf42587550e6047b0fb61ade143ef88317c52b5","overlaps_test":false,"full_text_reference":true}, +{"id":"2899","category":"listing","split":"dev","html_sha1":"f95415c743730f182d8a3a8b0ebdc5af840d1bd4","truth_sha1":"a9348d49f21a9125376d2647f616f904733c1e7c","overlaps_test":false,"full_text_reference":true}, +{"id":"2900","category":"listing","split":"dev","html_sha1":"fd1e848851ae016a4b94cee30db5e6836420a8c0","truth_sha1":"bcc143fa73da0806605374ffe294836cf107ae49","overlaps_test":false,"full_text_reference":true}, +{"id":"2901","category":"listing","split":"dev","html_sha1":"cba099067e7853732e1b0caa76e65ed552575598","truth_sha1":"e6e053c51d72c738fcfd71dc7a0573c119a9f031","overlaps_test":false,"full_text_reference":true}, +{"id":"2902","category":"listing","split":"dev","html_sha1":"8969a48a24a7e4d3ece8f7b40242704ff89d11e3","truth_sha1":"cc264d6d867d93a812000bb44ac7ee6f7ab1f02d","overlaps_test":false,"full_text_reference":true}, +{"id":"2903","category":"listing","split":"dev","html_sha1":"c67f87ea0be93b054730f13d33cbe2df06784641","truth_sha1":"b8a70b2141ebe9f0fbeb4195c0a5f8fbe5e63e99","overlaps_test":false,"full_text_reference":true}, +{"id":"2904","category":"listing","split":"dev","html_sha1":"4fb8c941a25d50f4e4f3b9364621011f9cea26a4","truth_sha1":"d6bd5e2da65d6391c780bad6153bc3ef16fed86f","overlaps_test":false,"full_text_reference":true}, +{"id":"2905","category":"listing","split":"dev","html_sha1":"67f816d2ee038af6664a01c171236085d8769abf","truth_sha1":"1b54a99096e87825fea3e5002d0828f42253f075","overlaps_test":false,"full_text_reference":true}, +{"id":"2906","category":"listing","split":"dev","html_sha1":"eb441c5992ed37ca47ea954c9fd1746af623fbfb","truth_sha1":"9290d7e27ca18aae4dbcb3149446660ae769dd70","overlaps_test":false,"full_text_reference":false}, +{"id":"2907","category":"listing","split":"dev","html_sha1":"4de407b947081d377e894d1412a3ddc75fdb9960","truth_sha1":"a9cf3d25fabe6598efde2ce522b6e654381f7237","overlaps_test":false,"full_text_reference":true}, +{"id":"2908","category":"listing","split":"dev","html_sha1":"27dbdba3ac2ea22b5865c3030d924312b874c591","truth_sha1":"0348d09dfa3e0fc2db275cd9565357f53f745d4f","overlaps_test":false,"full_text_reference":true}, +{"id":"2909","category":"listing","split":"dev","html_sha1":"8b1d351860b9532dfc9408c55e638b1210cd4ae1","truth_sha1":"95d32936018b1282a6832dd54db67aeb54804361","overlaps_test":false,"full_text_reference":true}, +{"id":"2910","category":"listing","split":"dev","html_sha1":"c3689b3251906e615bf8bfc4378278bff99729bd","truth_sha1":"c3092207c44ee968a7681cd59b24dd85b55da826","overlaps_test":false,"full_text_reference":true}, +{"id":"2911","category":"listing","split":"dev","html_sha1":"3eda079a5654eb15505b5677e3f60585c8246c8f","truth_sha1":"d4df7185f9821897bace6dcae0052fc815ed3d87","overlaps_test":false,"full_text_reference":true}, +{"id":"2913","category":"listing","split":"dev","html_sha1":"1acc2c94753c85b7ed3029d5fabcbf6fc58fd2e4","truth_sha1":"972e8c57008285530d5764f1f5ec4c318768e56b","overlaps_test":false,"full_text_reference":true}, +{"id":"2914","category":"listing","split":"dev","html_sha1":"ae6cf1b4f6e5aaa49b5aa2b08bf27c71df83c9e3","truth_sha1":"8be78cf1c7f6332087e6473aa5b9de127dd8a6db","overlaps_test":false,"full_text_reference":true}, +{"id":"2916","category":"listing","split":"dev","html_sha1":"ac10c57abaf6d908fa4e0ed472af789169d13b29","truth_sha1":"d685e9080cc87f8880df48e711b2efc7ca33e6aa","overlaps_test":false,"full_text_reference":true}, +{"id":"2917","category":"listing","split":"dev","html_sha1":"16fd58c6ffd19f326da02b5fbd2edfa3e7270f6a","truth_sha1":"46f052f36dbd06b717429da9109612d11b0181a1","overlaps_test":false,"full_text_reference":true}, +{"id":"2918","category":"listing","split":"dev","html_sha1":"de8635b242e9cc0ed20909b16a716ea0518c044f","truth_sha1":"b9384a3adf73ecf67726cc5ee7114c878301d708","overlaps_test":false,"full_text_reference":true}, +{"id":"2919","category":"listing","split":"dev","html_sha1":"0dfff743cb974b8e41754951d0a26f72c472041c","truth_sha1":"1988e5ea3efbc42f4c42788521b06ff8d76a47c6","overlaps_test":false,"full_text_reference":true}, +{"id":"2920","category":"collection","split":"dev","html_sha1":"543dc51ac7e50238d060870750ff06309fe81eee","truth_sha1":"b48e0f65e33f23ddd76cb683b6ee2f63f81b148b","overlaps_test":false,"full_text_reference":true}, +{"id":"2922","category":"collection","split":"dev","html_sha1":"26bceeadcd4192198e23c9d978486dd0c8c3fdf5","truth_sha1":"227d8e068d7beaec77f95ce33d613f38d564d0f6","overlaps_test":false,"full_text_reference":true}, +{"id":"2923","category":"collection","split":"dev","html_sha1":"11be3e1f80a7353103b8b96d3cadd08a25d0f4ee","truth_sha1":"abe07a68fad12a5aa4829018093c819231894d3c","overlaps_test":false,"full_text_reference":true}, +{"id":"2925","category":"collection","split":"dev","html_sha1":"13c7723563fa88a29a3ad442a4aa647fee9d6870","truth_sha1":"85e2d91f3c3db0e594044cff83fa8eea43e5d6be","overlaps_test":false,"full_text_reference":true}, +{"id":"2926","category":"collection","split":"dev","html_sha1":"dd74ed974b0d333ac2220f834d2dfeadfd5be824","truth_sha1":"760a01881d8af228125d594327e33f05cbf3dc37","overlaps_test":false,"full_text_reference":true}, +{"id":"2927","category":"collection","split":"dev","html_sha1":"9ac15eb6633550cf10e442ebb39f4c0415f4cc6b","truth_sha1":"a0f6f4fed6ef4dda52fd1d31600294adbf1d83d4","overlaps_test":false,"full_text_reference":true}, +{"id":"2934","category":"listing","split":"dev","html_sha1":"f882d5859d8455f5d397b1bc31b324d2d8edbfd9","truth_sha1":"f0b8c3118770f115a21af22dec25b7c7f629fbe2","overlaps_test":false,"full_text_reference":true}, +{"id":"2935","category":"collection","split":"dev","html_sha1":"d0875a82e15723b1c0415e23a3da91cf6afd35bd","truth_sha1":"d95a7afaca5c3062e141b228c901e6f5e5dcf2df","overlaps_test":false,"full_text_reference":true}, +{"id":"2936","category":"listing","split":"dev","html_sha1":"555f24a4d17bdcf8b80d666f47fca6c8055cffa6","truth_sha1":"6c7540afa4b2b109af6d421e06f3bda1e317cc27","overlaps_test":false,"full_text_reference":true}, +{"id":"2947","category":"product","split":"dev","html_sha1":"0c6d49ed04923124e1fdbf82ca57dbfb04bb5ace","truth_sha1":"d59956bf9d6ac499065a427cc3099dc266e5014d","overlaps_test":false,"full_text_reference":true}, +{"id":"2951","category":"collection","split":"dev","html_sha1":"454533b7b39ddbb98100572a34ddce98447b405e","truth_sha1":"98bb23f9d6f7005bb9e923d8e5daa2896437e25b","overlaps_test":false,"full_text_reference":true}, +{"id":"2952","category":"collection","split":"dev","html_sha1":"d3b14ce6303059cb959e6a0b7ccf0b3730f75669","truth_sha1":"7199757b74744778bfce6daddbe31aaaed20dbd8","overlaps_test":false,"full_text_reference":true}, +{"id":"2953","category":"collection","split":"dev","html_sha1":"b67eb4b2a4a26d835df366869863b97e48d5615b","truth_sha1":"f971ef39a4d23532a01206b7362276d12d64f7aa","overlaps_test":false,"full_text_reference":true}, +{"id":"2954","category":"collection","split":"dev","html_sha1":"6dfcc2affebc49cfe097e5c7e2085ed1bd386aaa","truth_sha1":"9cabd30cdbb117174fc0d10bfa5f3cdd53890aa9","overlaps_test":false,"full_text_reference":true}, +{"id":"2955","category":"collection","split":"dev","html_sha1":"df61eb62da172c8e089e2a2e5372e60823f0ce10","truth_sha1":"cd630ea25197569829842dbbe4b9320bdf68c287","overlaps_test":false,"full_text_reference":true}, +{"id":"2960","category":"collection","split":"dev","html_sha1":"05a69029bbfd6f4b28fc03b9630d3934da2de67c","truth_sha1":"ad5967c46b0e2771dea1e403eac047769d00589b","overlaps_test":false,"full_text_reference":true}, +{"id":"2961","category":"listing","split":"dev","html_sha1":"1b1cb1a3a3ec2e211ed8a7079089b5f3476148b7","truth_sha1":"c4ff547cb4446c7194dbb41d97e8694d57a47acf","overlaps_test":false,"full_text_reference":true}, +{"id":"2962","category":"listing","split":"dev","html_sha1":"27106e92b852edc5e3bf79d4c38291c530b83b69","truth_sha1":"ccad428fa62a9d7a673cbf543f81c8c883489f32","overlaps_test":false,"full_text_reference":true}, +{"id":"2965","category":"listing","split":"dev","html_sha1":"cb96393ce97d91b69078e2b4d414ad9fb898f583","truth_sha1":"ad0f18725f5f55c971645524b9d488313094f3a8","overlaps_test":false,"full_text_reference":true}, +{"id":"2966","category":"listing","split":"dev","html_sha1":"f18aa31691cf313d0d6fad330b6d49ba3fe6302a","truth_sha1":"f4c7d3dcc43f0a5d0edaf9776bbdaea9c5e4c5c7","overlaps_test":false,"full_text_reference":false}, +{"id":"2967","category":"listing","split":"dev","html_sha1":"cd490a1ecc46772e8165df73e2ffc113af00786a","truth_sha1":"825dc5859f99d0330b33b3b86bed1e8c1908a961","overlaps_test":false,"full_text_reference":true}, +{"id":"2968","category":"listing","split":"dev","html_sha1":"bfb60eee315c09a21fbb97b121d7063706e81846","truth_sha1":"eef19e2bfde5e63d19f0181e3d09d0b2c33838fe","overlaps_test":false,"full_text_reference":true}, +{"id":"2969","category":"listing","split":"dev","html_sha1":"a49e3e792691bde980fd8eee1fcd1213f36480d0","truth_sha1":"f171af4dc5875230ff6918785a9fe7f956900013","overlaps_test":false,"full_text_reference":true}, +{"id":"2970","category":"listing","split":"dev","html_sha1":"2a7a1ae2745ca509e98996e2bf14de12e9866fd5","truth_sha1":"439bf305bd95888b3378fc23cbdd4bb3ba7eac51","overlaps_test":false,"full_text_reference":true}, +{"id":"2971","category":"listing","split":"dev","html_sha1":"942757d0aa5cd7c948dcff163cdfe40038558b3e","truth_sha1":"f34b414e3a892ea212ff44ac52cf91e507ce9127","overlaps_test":false,"full_text_reference":true}, +{"id":"2972","category":"listing","split":"dev","html_sha1":"a38cb7a0d1ba673c73453a08c98aef4555763586","truth_sha1":"15196293b34c9aa4dd47a8a2ec6df85b3e3eb234","overlaps_test":false,"full_text_reference":true}, +{"id":"2973","category":"listing","split":"dev","html_sha1":"3e484e26b9203c0b10ee8743fb0c226c1759ac9e","truth_sha1":"729fd34285e7c86e4b2531469fc08730cdbd5f25","overlaps_test":false,"full_text_reference":true}, +{"id":"2974","category":"listing","split":"dev","html_sha1":"b412fa6eaae1e7e436fac606e1ccd2a6927bc18d","truth_sha1":"91ec71c5898a2e204af75775837f5023611291c8","overlaps_test":false,"full_text_reference":true}, +{"id":"2976","category":"article","split":"dev","html_sha1":"e865b97c31ed25f91641ee0fb21a26b7462da4c6","truth_sha1":"115208bc9169712b80e90eadd8ace027f503175c","overlaps_test":false,"full_text_reference":true}, +{"id":"2981","category":"product","split":"dev","html_sha1":"5d471cf601c0beada78930431cd7a6120eab1d84","truth_sha1":"8311d816ade6506bc77ba162ee7d26fdf679b100","overlaps_test":false,"full_text_reference":true}, +{"id":"2982","category":"product","split":"dev","html_sha1":"15e7cfa4c02913cfdad1ff639ae7bde169f7eda4","truth_sha1":"c789747366c70e2a0bd39a71a5ca67204145a27e","overlaps_test":false,"full_text_reference":true}, +{"id":"2985","category":"product","split":"dev","html_sha1":"c8d3f8dc27d608fb398b70b42cef59a98a24c5ac","truth_sha1":"bd6abbb2c9d75dc622007bbac85d662c69b89ab2","overlaps_test":false,"full_text_reference":true}, +{"id":"2990","category":"product","split":"dev","html_sha1":"445e26c188fc3971bf875a1e6605f5fb5b1dc43c","truth_sha1":"cbd7ee9eda2f41703b6846fecb5e9b9284bd13aa","overlaps_test":false,"full_text_reference":true}, +{"id":"2991","category":"product","split":"dev","html_sha1":"6504dad25c6719dafa63f48886e5116f055650b3","truth_sha1":"bb4ceb642eafd3f16b0fa20d44ef3e3ee3f6a5be","overlaps_test":false,"full_text_reference":true}, +{"id":"2992","category":"documentation","split":"dev","html_sha1":"ed367d80c67cedf6e789fdf1995d292abcf2d94e","truth_sha1":"0b4165dcf7448a7ed0c87d31fc6d3e19249d7faf","overlaps_test":false,"full_text_reference":true}, +{"id":"2993","category":"documentation","split":"dev","html_sha1":"1e20ff7d85223de2a1992e835f85c9d92e0cce29","truth_sha1":"47998877b7597ff23b8fdd341786c18bee802bf9","overlaps_test":false,"full_text_reference":true}, +{"id":"2994","category":"documentation","split":"dev","html_sha1":"be01873a3f08223b66bc47926d7a396b1c500d9b","truth_sha1":"a3f94dbcc16dd1738dddf0f315ba119d3d6d8b2b","overlaps_test":false,"full_text_reference":true}, +{"id":"2996","category":"documentation","split":"dev","html_sha1":"d47931ddbae8dc3dfd655aa221b4e60612e1d7d3","truth_sha1":"0dba8d31190803a28614cc1a7b62fa45c62f3a33","overlaps_test":false,"full_text_reference":true}, +{"id":"2997","category":"documentation","split":"dev","html_sha1":"51c6b9bdb6e0914cf74c81e8a7d8d373c5db5b42","truth_sha1":"6e2facc83b0420aa14b085f3ee758f140f13d715","overlaps_test":false,"full_text_reference":true}, +{"id":"2998","category":"documentation","split":"dev","html_sha1":"edce05d78d78a9f77dac144260f4fbac73dfc442","truth_sha1":"a9c486213a8bc2b9551e6edf6eb3b9d1ceabbdb2","overlaps_test":false,"full_text_reference":true}, +{"id":"2999","category":"documentation","split":"dev","html_sha1":"951f23fe98e43ee6ab77e06e474111f7dbc6aa7d","truth_sha1":"1c181817212b16f650d226ae7e38eaa295e0559f","overlaps_test":false,"full_text_reference":true}, +{"id":"3000","category":"collection","split":"dev","html_sha1":"35ae2221d3bd03e236dc709bc8c4148df9bb9377","truth_sha1":"96cb1ad9445a6b6d9eeaa886ed9c8f22c931bbe1","overlaps_test":false,"full_text_reference":true}, +{"id":"3001","category":"collection","split":"dev","html_sha1":"5501cc06e99116c2d0654bfc25c1339901d8e3b3","truth_sha1":"b3744447f6ab5b67f78a3238024c0134bd613d9d","overlaps_test":false,"full_text_reference":true}, +{"id":"3002","category":"collection","split":"dev","html_sha1":"24fcaf792d633544c8529db73a0fbad527216932","truth_sha1":"55a29ef557d83cd63ce411ad0beb98356be135a2","overlaps_test":false,"full_text_reference":true}, +{"id":"3003","category":"collection","split":"dev","html_sha1":"9eee0714f384ca1a2583fa12c587cb3bfd3367a3","truth_sha1":"d2839ebebf31913b0057e1d178d9374584022de3","overlaps_test":false,"full_text_reference":true}, +{"id":"3006","category":"collection","split":"dev","html_sha1":"be00c0c50cb315a9e35e5e0d72a39069c9385539","truth_sha1":"a000ae0079b9b5adffb4816036061272a8fd701f","overlaps_test":false,"full_text_reference":true}, +{"id":"3008","category":"collection","split":"dev","html_sha1":"afe31ab81d11696b47f1c69a76ca654b6a63ec4a","truth_sha1":"d1c6327a297ae7aa721f227763277adbd3186942","overlaps_test":false,"full_text_reference":false}, +{"id":"3010","category":"listing","split":"dev","html_sha1":"404cebe3d15da3d92acc5c5fde15f4ed675cfbda","truth_sha1":"3d784e8533cd14b9cbd5d79b798900c77f0696a0","overlaps_test":false,"full_text_reference":true}, +{"id":"3011","category":"listing","split":"dev","html_sha1":"470400cecc2eef5300db5f5873662b780304abb4","truth_sha1":"edc2860099ffe6328ac1689c2024649c5f37a943","overlaps_test":false,"full_text_reference":true}, +{"id":"3013","category":"listing","split":"dev","html_sha1":"edc4c76bc3fdfa424036812113823068fb15facf","truth_sha1":"332f2facd0dc854b19586a1577321f66cf4c2c1f","overlaps_test":false,"full_text_reference":true}, +{"id":"3014","category":"listing","split":"dev","html_sha1":"976a3512f1ec9f6483d7a6da5c4a3ce3f333a14a","truth_sha1":"0b2e0a2dbce6b0e27209709b0b84839432a5c15a","overlaps_test":false,"full_text_reference":true}, +{"id":"3015","category":"listing","split":"dev","html_sha1":"9fa1f338c3975934b2d57bae6eebaa6ab8a0b4d2","truth_sha1":"39d5069ba798ecbc362642f707e9ad3bb2305eb0","overlaps_test":false,"full_text_reference":true}, +{"id":"3016","category":"listing","split":"dev","html_sha1":"3ba2c807ef72e2b33c18f0a079dc2fa3b72bfb9d","truth_sha1":"e274ffabe1ff1caa6b8dda8b300e3b760cda7b68","overlaps_test":false,"full_text_reference":true}, +{"id":"3017","category":"product","split":"dev","html_sha1":"04fa85a9ff762d597422ae95dfb5017c8b43de51","truth_sha1":"4e87feef0ad3cf72798594ea650985b507a47e82","overlaps_test":false,"full_text_reference":true}, +{"id":"3019","category":"product","split":"dev","html_sha1":"a56d45f315b209e4c9a3747498f66b0a351d786d","truth_sha1":"aa3a1f4602001d933089a8678ae813b4d6005eec","overlaps_test":false,"full_text_reference":true}, +{"id":"3020","category":"product","split":"dev","html_sha1":"fe8580d4f044b5503802759e6a8bf67f1e4fdd61","truth_sha1":"29a80bd77e6c540ccfe6ec62751d5c64076a2f22","overlaps_test":false,"full_text_reference":true}, +{"id":"3021","category":"product","split":"dev","html_sha1":"ed914f0a6ff77078765aded1848db9b0bec67acf","truth_sha1":"0b5b7ba145141049a4ddc9b9f504f0363910022a","overlaps_test":false,"full_text_reference":true}, +{"id":"3022","category":"product","split":"dev","html_sha1":"54f438ebc801257e481fc1a9f17b444e679a2662","truth_sha1":"e7e1728fbe7187607c0ca98a9d14da178dff808e","overlaps_test":false,"full_text_reference":true}, +{"id":"3023","category":"product","split":"dev","html_sha1":"6dceda3b98fd100b0e60a640e1a64c27fffbceb0","truth_sha1":"4a6f96c0ef0d65e14384ff7601aa22aaaaaf5d95","overlaps_test":false,"full_text_reference":true}, +{"id":"3026","category":"product","split":"dev","html_sha1":"fa6650ed6f983cbdcd6ab6fc2373092ad78bd906","truth_sha1":"c76edffb7a78fe8ce2053f4104511e69e56b6cb8","overlaps_test":false,"full_text_reference":true}, +{"id":"3027","category":"collection","split":"dev","html_sha1":"be154a644f62549e1a109bd55ca63aa17cb02530","truth_sha1":"fd00348601292e4e0b22cb9c57cac5cce370b899","overlaps_test":false,"full_text_reference":true}, +{"id":"3028","category":"collection","split":"dev","html_sha1":"d58b53f5d1e2ad0aba19cb29fbe0735300df416e","truth_sha1":"dbdbdf6eb5873a2a5ca86a72b2ed65068c0eb33f","overlaps_test":false,"full_text_reference":true}, +{"id":"3029","category":"collection","split":"dev","html_sha1":"fef2b706c9e2d2a05859bec09436cfb22d12fa9d","truth_sha1":"27e75fb09c6d2ec41799645109ec2c81bfa6f4c8","overlaps_test":false,"full_text_reference":true}, +{"id":"3030","category":"listing","split":"dev","html_sha1":"3883cae3b6cd0d590e3a3d8147f2ea411ee6cdbb","truth_sha1":"782bf4c02236b70f9c398b801dc3fb356474dde1","overlaps_test":false,"full_text_reference":true}, +{"id":"3031","category":"listing","split":"dev","html_sha1":"d0b1bb9f0f021d87e0aaaac24493ce2325afdf21","truth_sha1":"757c2e3bc097b2602480ff6bd63d59dde3aeb496","overlaps_test":false,"full_text_reference":true}, +{"id":"3033","category":"service","split":"dev","html_sha1":"63998cf7e0817820d795c91ad3b417bd6ffb287b","truth_sha1":"05bd5d88b913a375f3c366b36154b910edc98733","overlaps_test":false,"full_text_reference":true}, +{"id":"3034","category":"service","split":"dev","html_sha1":"673e2252dfbd73e6ee01ae10a8a5682ba869f627","truth_sha1":"82e8860f2459a23721d9b33f15176e316b9ae6b3","overlaps_test":false,"full_text_reference":true}, +{"id":"3035","category":"article","split":"dev","html_sha1":"34f5ba9d622a0f09b0def589281741d0227daff4","truth_sha1":"e477edc7d4888fb6e49a6c3bb49f18234b6d5d32","overlaps_test":false,"full_text_reference":true}, +{"id":"3036","category":"article","split":"dev","html_sha1":"8db3d4380b763740380171a2bfd6bfcf0288ec8a","truth_sha1":"a7231121cc136dd61d2ff61bd2602ce295f41a17","overlaps_test":false,"full_text_reference":true}, +{"id":"3037","category":"documentation","split":"dev","html_sha1":"b18ab40e54a311618b96a43a19ae1ed0b58038a6","truth_sha1":"994003a61638d735be662ab686f764d081225d10","overlaps_test":false,"full_text_reference":true}, +{"id":"3039","category":"product","split":"dev","html_sha1":"10424e0e668bb275b704a8826033db041286fbfe","truth_sha1":"db4c0b0641b379240a9ca6bf1aa31da83c9eff7b","overlaps_test":false,"full_text_reference":true}, +{"id":"3043","category":"product","split":"dev","html_sha1":"88531f00a01c181136e83c753c33125319356bb7","truth_sha1":"0575a00223cfc4ea12969072c0a66bf9087469dd","overlaps_test":false,"full_text_reference":true}, +{"id":"3051","category":"collection","split":"dev","html_sha1":"218f327fbca8e7bdb9765efd95d2d41d93edd349","truth_sha1":"20dcebe877ffa3aee218e5ad994c83c5c5e427dd","overlaps_test":false,"full_text_reference":true}, +{"id":"3052","category":"product","split":"dev","html_sha1":"aa133a95cd8c7ce8098f24e9613cb1499ef80ad1","truth_sha1":"f849cdb60127414acdb45e05e65f8991405aa460","overlaps_test":false,"full_text_reference":true}, +{"id":"3054","category":"product","split":"dev","html_sha1":"2de1a1ffd241d49e7ea90b941fa27a65ac8f5b3a","truth_sha1":"d9bade47c41cbdf27b1436c13a40a1414f262054","overlaps_test":false,"full_text_reference":true}, +{"id":"3058","category":"product","split":"dev","html_sha1":"3cdba874a0f68390f21f72d295b7c16c90608f18","truth_sha1":"afc8c3c0b23046be1720903511c16142eac5e2ba","overlaps_test":false,"full_text_reference":true}, +{"id":"3060","category":"product","split":"dev","html_sha1":"4ec0d105b640efdcdb37945ddd70559af38a05b3","truth_sha1":"70b0914a2c75ab9e5018114901521c08415cd26e","overlaps_test":false,"full_text_reference":true}, +{"id":"3061","category":"product","split":"dev","html_sha1":"d0e5b9aa3d14b5b714bc7c2bf205b33c420e0fac","truth_sha1":"0d72c63fccbee4ab024acd71d777cfaa1ef50ecf","overlaps_test":false,"full_text_reference":true}, +{"id":"3062","category":"product","split":"dev","html_sha1":"6e8105dae62bac13f1a3f10ed159adf60d61d59e","truth_sha1":"76f355c48110c57eb65f70a1762c29eba0e63f59","overlaps_test":false,"full_text_reference":true}, +{"id":"3063","category":"product","split":"dev","html_sha1":"0eab9eb091d5b23209a648d721d96c7501a57163","truth_sha1":"16a904165b5a4bbbed8d8d58cd1d2c24ddb36c6f","overlaps_test":false,"full_text_reference":true}, +{"id":"3064","category":"collection","split":"dev","html_sha1":"d20051cebc60f5b083b18a2df48cec0110a79782","truth_sha1":"fbf48dafe9955c575a7979578ae12dddf91a0ae3","overlaps_test":false,"full_text_reference":true}, +{"id":"3065","category":"product","split":"dev","html_sha1":"d5489052c964ff6c8c3517f3de74f1fb9c6eaef9","truth_sha1":"b260bbea835eee9404e5f2793f11dcc2ce8865e8","overlaps_test":false,"full_text_reference":true}, +{"id":"3066","category":"product","split":"dev","html_sha1":"0fce8bffcadb2debb5e1053d7c872ab6d094f2d5","truth_sha1":"7eb4cb4ad699578245bfb533ef1dafdff0003da0","overlaps_test":false,"full_text_reference":true}, +{"id":"3067","category":"collection","split":"dev","html_sha1":"6e35682d3d562c3a862321f422fb0e4aebdb005d","truth_sha1":"7966cc5b87ee5969c75e9e35977fd74dcee560fb","overlaps_test":false,"full_text_reference":true}, +{"id":"3068","category":"collection","split":"dev","html_sha1":"75dbe5849894a104543f2c0e1371779dfe587844","truth_sha1":"d5512eced46feff8d701ff4aadba7005ad1c2f13","overlaps_test":false,"full_text_reference":true}, +{"id":"3069","category":"collection","split":"dev","html_sha1":"e6bd20f74bb7aa6efb289232b561c547ca99fb29","truth_sha1":"09ac95a5bfc9b602c2b0fc1b03a5ab2e7d43a5e9","overlaps_test":false,"full_text_reference":true}, +{"id":"3071","category":"product","split":"dev","html_sha1":"b896a096ad1d513e06cda46406310d6890a73de3","truth_sha1":"89908973bcb46f18dbc1d125a99e6814ad7cf2cc","overlaps_test":false,"full_text_reference":true}, +{"id":"3072","category":"product","split":"dev","html_sha1":"6a800dbfb59b1f0fd9843044ff4b4417da55ebcc","truth_sha1":"0222661e5824ca839712238d144e587ad0893cad","overlaps_test":false,"full_text_reference":true}, +{"id":"3074","category":"product","split":"dev","html_sha1":"a1f90ae0fd3411ac40b79e4806f147b91bdd0800","truth_sha1":"f287bb31e94f9c205d351e180032d8da96958f03","overlaps_test":false,"full_text_reference":true}, +{"id":"3075","category":"product","split":"dev","html_sha1":"63aac2bb4350df5350e5a367528eeb8bd6c35499","truth_sha1":"485591863f55a404fd52b47493d0735d3c6cd98e","overlaps_test":false,"full_text_reference":true}, +{"id":"3076","category":"product","split":"dev","html_sha1":"4d314babff154ef1cd9b0d3816034b7a5a6a6e36","truth_sha1":"7f6e18e4705dec8cdde5b5430c987019bd93f7b2","overlaps_test":false,"full_text_reference":true}, +{"id":"3078","category":"product","split":"dev","html_sha1":"be4a261ecc2c96718cc71201de38c6f27432ca3c","truth_sha1":"6a1bdb976c79e74b5fa9d48c89178c120bb87b96","overlaps_test":false,"full_text_reference":true}, +{"id":"3079","category":"product","split":"dev","html_sha1":"368c10c841dbd354e52df63c407304cd83a30e10","truth_sha1":"43a56ce3b0389fa03ee84cc944d8d9852db52988","overlaps_test":false,"full_text_reference":true}, +{"id":"4001","category":"collection","split":"dev","html_sha1":"91b26f4a371062345bee0a02f3b8ef89afb8f9e1","truth_sha1":"ec0d04faa4ba26bbb41b0f731b51f596b4f89787","overlaps_test":false,"full_text_reference":true}, +{"id":"4002","category":"article","split":"dev","html_sha1":"98de1eec9fc400f078841561bb775cf9f0b310cb","truth_sha1":"d4ad8b44bb4dbd8f5e445462f798fd846c93aee0","overlaps_test":false,"full_text_reference":true}, +{"id":"4007","category":"article","split":"dev","html_sha1":"1cc5e39f8ee6cec825004d82e51fed06f20f9dee","truth_sha1":"a2d34eacfd843e323f33ff9731ed642a3a3394c5","overlaps_test":false,"full_text_reference":true}, +{"id":"4008","category":"forum","split":"dev","html_sha1":"883ddb6457444dc5e22a0f0fece6072e97b9973c","truth_sha1":"08b268d451c07321f24a088b7cad41c4976bbfbf","overlaps_test":false,"full_text_reference":true}, +{"id":"4010","category":"article","split":"dev","html_sha1":"861e5cff956bbc9752e967a2e926cf2491a4cfb5","truth_sha1":"677803799dd7f7e6a09c3201bbf4653980218463","overlaps_test":false,"full_text_reference":true}, +{"id":"4011","category":"article","split":"dev","html_sha1":"3cc3866312491431cdf790a17b38ab0a71975cc9","truth_sha1":"6137df0f2cd65f80bbffb157a3e4fa7f9ae9089e","overlaps_test":true,"full_text_reference":true}, +{"id":"4012","category":"article","split":"dev","html_sha1":"7afda19e8ad4715c81f38db9c696f01d0b1e80a5","truth_sha1":"3a798436ee45d21394fcd672dbbf483eb138a161","overlaps_test":false,"full_text_reference":true}, +{"id":"4014","category":"article","split":"dev","html_sha1":"44040a2408ee72c75b171c5666b47bfa6d7d4e92","truth_sha1":"b99df31a174296d88f5f356a871cb8f2a56f59cb","overlaps_test":false,"full_text_reference":true}, +{"id":"4015","category":"collection","split":"dev","html_sha1":"0430017ba2fb12047dbf340f33deb69f15d158db","truth_sha1":"ae37b7b8033b752a73d5552fdcc51cd37f3aa148","overlaps_test":true,"full_text_reference":true}, +{"id":"4016","category":"article","split":"dev","html_sha1":"94719ded24fba972e6acff02b295a2ad8e16acad","truth_sha1":"7a2e9bc419e452ae85df47fb83dd5a02ec1502f0","overlaps_test":true,"full_text_reference":true}, +{"id":"4017","category":"product","split":"dev","html_sha1":"b4039cb8e1e4d5e10b3aaa24cf736249035f2ac2","truth_sha1":"fcdcdf75720324df1be1bc2471ecb7ec7b375aab","overlaps_test":true,"full_text_reference":true}, +{"id":"4020","category":"documentation","split":"dev","html_sha1":"fdc2a723ad27a0ff4803530ce2eaa25c94eb9547","truth_sha1":"3c94083a39bcbd97ff94fe470b47e136c9bb23f8","overlaps_test":false,"full_text_reference":true}, +{"id":"4021","category":"collection","split":"dev","html_sha1":"f29b1a512d1b3ac11334d864f8d2051be085b768","truth_sha1":"97c898b49613309489b2b12f0b552b4207090852","overlaps_test":true,"full_text_reference":true}, +{"id":"4022","category":"collection","split":"dev","html_sha1":"221f862e37263fa1bb7523f5893f82df276ba6ed","truth_sha1":"01a5188863f9cf6b541dd64b218602a5516699ea","overlaps_test":false,"full_text_reference":true}, +{"id":"4024","category":"product","split":"dev","html_sha1":"1130f912811bf15c3dce651c6ba453e6e5eebda4","truth_sha1":"df7f3f5041a018356f915ab894898129953b2160","overlaps_test":true,"full_text_reference":true}, +{"id":"4025","category":"article","split":"dev","html_sha1":"d5d2d0e2b7967d284940f117de9abec17888f854","truth_sha1":"621dff910ad5b7d8b8d7a90d33e33cb3af6a63db","overlaps_test":false,"full_text_reference":true}, +{"id":"4026","category":"product","split":"dev","html_sha1":"a19dbc6b50e39b5b3c1acbfc3cb631bd4cfa3013","truth_sha1":"17d686ea2b879d273bf260f2fd0d246bc5ddd449","overlaps_test":true,"full_text_reference":true}, +{"id":"4027","category":"article","split":"dev","html_sha1":"cd3704a6ac0b7b01349de1ab66544596653de3db","truth_sha1":"1137cab757cdab35d7b6b54744b49ecd1ded0d86","overlaps_test":false,"full_text_reference":true}, +{"id":"4028","category":"article","split":"dev","html_sha1":"8f316effc82e583fffac36a7ff099523e84c8b8e","truth_sha1":"ee44f8614ac39f9604dd285ae85a17ae1fe991c9","overlaps_test":true,"full_text_reference":true}, +{"id":"4029","category":"product","split":"dev","html_sha1":"d31d9b9ab653e294db399e60443a1020409e57b0","truth_sha1":"1e8524a8e95d17d6685c1da5eb80863abf263239","overlaps_test":true,"full_text_reference":true}, +{"id":"4030","category":"article","split":"dev","html_sha1":"d52a74ae9a25de48cc030041cb98ab586bc1fd69","truth_sha1":"011beeebbcb66b89ce45bda1e23dd76694bb55fa","overlaps_test":false,"full_text_reference":true}, +{"id":"4033","category":"listing","split":"dev","html_sha1":"14b1110bab204c5571a0223d5b6ca407eaff563b","truth_sha1":"06db28e43e386066c73f0928319ce7191a87c24f","overlaps_test":false,"full_text_reference":true}, +{"id":"4034","category":"article","split":"dev","html_sha1":"fe90e975d9757329ca5c3d7f59d892fed728b547","truth_sha1":"7588ee217f41c59f4d3eeb315a0c9b09af1074af","overlaps_test":false,"full_text_reference":true}, +{"id":"4035","category":"collection","split":"dev","html_sha1":"f7563d2e59858460a890d04c3ea592e06556b987","truth_sha1":"d8c51b6fccd4229d4bf71a99511a695306f564b0","overlaps_test":true,"full_text_reference":true}, +{"id":"4036","category":"article","split":"dev","html_sha1":"f7f10fc8adfd237f430f64f48409cfac0caf2621","truth_sha1":"08678f89ee7251bcebf8e93572ed594c81f3bbc0","overlaps_test":false,"full_text_reference":true}, +{"id":"4038","category":"article","split":"dev","html_sha1":"fc558132d1c8d538a897c4364fdbb93e9f398f00","truth_sha1":"869a12df0e4b555c450df636e29e1c2e2c7cfd08","overlaps_test":true,"full_text_reference":true}, +{"id":"4039","category":"article","split":"dev","html_sha1":"f9e356672db8d0660a6f328e41ed0706faddd509","truth_sha1":"b898827491bd82ae6f84397d89b35bfe1095f5e2","overlaps_test":false,"full_text_reference":true}, +{"id":"4040","category":"article","split":"dev","html_sha1":"5d4226620e86c841feaa01968cfe87920ca26aa4","truth_sha1":"ddb8465c0d774124f39357dfd56ab685c942de5b","overlaps_test":false,"full_text_reference":true}, +{"id":"4041","category":"article","split":"dev","html_sha1":"f8b58308c1a146ddc5cbb015b56ae0d5bc71971f","truth_sha1":"d1ef926ecb18d597594f62f40227fa7073a0e8ed","overlaps_test":false,"full_text_reference":true}, +{"id":"4042","category":"article","split":"dev","html_sha1":"63fd51ea5b489696a0699fdc0b0e64231100a51f","truth_sha1":"01813ff66dbc2c015bf5f05ad45fa13a5e266b14","overlaps_test":false,"full_text_reference":true}, +{"id":"4043","category":"article","split":"dev","html_sha1":"1432dbf6b2ac5c2280780428c0bb783818103990","truth_sha1":"8fbb453bbb9255cf6203d6bcbda1f8ec1e4de14a","overlaps_test":false,"full_text_reference":true}, +{"id":"4044","category":"article","split":"dev","html_sha1":"2dbb0676c52ef31fe02881aa4919ea421635bca2","truth_sha1":"27d2c64b2be2412ab870edff9ea95077d48618bb","overlaps_test":true,"full_text_reference":true}, +{"id":"4045","category":"article","split":"dev","html_sha1":"d0ecb2e1027691e81c904a5bdcbed5ddaef4a3e5","truth_sha1":"60a1c7a58fce886d070e6b9e93a268127ba52980","overlaps_test":false,"full_text_reference":true}, +{"id":"4048","category":"article","split":"dev","html_sha1":"205e858f1d2b3673a469cc5d779c51bfa950fdaa","truth_sha1":"6e61ca01fb8f333fe90408e0e00f27a4adad5499","overlaps_test":true,"full_text_reference":true}, +{"id":"4050","category":"product","split":"dev","html_sha1":"42a1c3bb31a5c585bd9cda2b6de0f484aafa4912","truth_sha1":"0304ca6abfc99b52018fb441746824d66bb9154d","overlaps_test":true,"full_text_reference":true}, +{"id":"4051","category":"article","split":"dev","html_sha1":"30752750e0db30b761001598847698cf98a1b927","truth_sha1":"062c6064ffb8daa308f0e2a1c174e8b2c806d9dc","overlaps_test":true,"full_text_reference":true}, +{"id":"4052","category":"article","split":"dev","html_sha1":"7a652278af49b9ff19ca055a379cdc62e6fba879","truth_sha1":"16271d4209670eff603cf4b24d362b6fbcf67fed","overlaps_test":true,"full_text_reference":true}, +{"id":"4053","category":"article","split":"dev","html_sha1":"93962fcd4ec26446fc59b895c26650997dac89ed","truth_sha1":"777b9af59945a4129172ef615e4f55b68bada2b1","overlaps_test":false,"full_text_reference":true}, +{"id":"4054","category":"listing","split":"dev","html_sha1":"a6e8a6b8cead44fe2c9e8bdac61e1b0748b14b3d","truth_sha1":"1f4b52dea118affb85de9782a16093d420e4a128","overlaps_test":true,"full_text_reference":true}, +{"id":"4055","category":"article","split":"dev","html_sha1":"5a390ed3ff9108788aa9f76b78a0e735592f513c","truth_sha1":"bde8da13a4dfccfb4a995d30c55b6b631bdfc289","overlaps_test":true,"full_text_reference":true}, +{"id":"4058","category":"article","split":"dev","html_sha1":"d1af588c7b69841969f47124919dbf7ef1561b8f","truth_sha1":"c5847bbf7d000ff375ea500c96982992c09c1010","overlaps_test":true,"full_text_reference":true}, +{"id":"4060","category":"article","split":"dev","html_sha1":"dbf66b51f03356b9e748db9b8647a5cd8832d87a","truth_sha1":"fba1d48cbca40e014868b5086069499318b46bf4","overlaps_test":true,"full_text_reference":true}, +{"id":"4061","category":"collection","split":"dev","html_sha1":"f1e574af1e49a33a9a6835360aa45aa9e483749d","truth_sha1":"aaf188a97cf2df0d088a80b44da6ee939fa1be2a","overlaps_test":true,"full_text_reference":true}, +{"id":"4062","category":"product","split":"dev","html_sha1":"2733864f053a320b942286339d8cce088fab39d4","truth_sha1":"b68b5bc5240978cc45195ac723f61d2d3ffbda1c","overlaps_test":true,"full_text_reference":true}, +{"id":"4063","category":"article","split":"dev","html_sha1":"2409e222e49ab6679a45ba7c9e6a210ca7284e38","truth_sha1":"fb41e63f83ffa3deb5c7fe2befedacfa52065b40","overlaps_test":true,"full_text_reference":true}, +{"id":"4064","category":"article","split":"dev","html_sha1":"62113c09e48b85927a14a3101b312dfd471e47eb","truth_sha1":"9e4d8718715d397c6c487e683376e3e24b2a64f7","overlaps_test":true,"full_text_reference":true}, +{"id":"4065","category":"collection","split":"dev","html_sha1":"c7523863fc60181b3ded67cf0de809fc2c0622b7","truth_sha1":"abd44bfc9e1ab4d7e58c23213c806666712d2cd7","overlaps_test":true,"full_text_reference":true}, +{"id":"4067","category":"article","split":"dev","html_sha1":"ef0e549eb608d835586992c3a6dc4837d34b8c18","truth_sha1":"ee498614ea154e0c97ead70641b79790e43f993d","overlaps_test":true,"full_text_reference":true}, +{"id":"4068","category":"article","split":"dev","html_sha1":"92589de9d723d38439cb027258ca4809eb10adef","truth_sha1":"a135f070bdd5419cecbafff54c4096fff15ce703","overlaps_test":false,"full_text_reference":true}, +{"id":"4069","category":"collection","split":"dev","html_sha1":"cc0f97e9a09d4597bdfdcfc7e4bd1fd86c421bf9","truth_sha1":"2808818e4a442327761dc762fabd532b2a7f78d1","overlaps_test":false,"full_text_reference":true}, +{"id":"4070","category":"article","split":"dev","html_sha1":"b4d61162af58cd65a024e50ba6d032fdaa297687","truth_sha1":"ace35a1dca5aa7852ad3507e34f96a5c9d95422a","overlaps_test":false,"full_text_reference":true}, +{"id":"4071","category":"collection","split":"dev","html_sha1":"d903c3de68ce6f0238aec225fe16ace9460a1845","truth_sha1":"826c94f11764477e6be1f950b12bf626284faa37","overlaps_test":true,"full_text_reference":true}, +{"id":"4072","category":"article","split":"dev","html_sha1":"457f27708e3d90eb9e38ad0a06901227cc4e4c82","truth_sha1":"31aa619e41b2623078775caf07281e97493abe6a","overlaps_test":true,"full_text_reference":true}, +{"id":"4074","category":"article","split":"dev","html_sha1":"fe711918c1fd6292bd74e330426695522275a7c9","truth_sha1":"a3159f7134fadb7b31a1a8e3f49ac813c8ad2b11","overlaps_test":false,"full_text_reference":true}, +{"id":"4076","category":"article","split":"dev","html_sha1":"4a2ae3bcad480b174c4317954d1f8b1991656dfe","truth_sha1":"a71a526b2c6b03dc2e97e0d75d91f1769106c762","overlaps_test":false,"full_text_reference":true}, +{"id":"4078","category":"article","split":"dev","html_sha1":"8f7e77e74faaefbbc9f015c55123aa2a77cc3de8","truth_sha1":"26a47acf60fc7df44dcd8d9dc3b3ebd960abefff","overlaps_test":false,"full_text_reference":true}, +{"id":"4079","category":"article","split":"dev","html_sha1":"102a0b39382c052cfc36a22693e6331aa56a0a58","truth_sha1":"afc4ddb955493e67a9fdd7bb2511d20f119e9003","overlaps_test":false,"full_text_reference":true}, +{"id":"4082","category":"article","split":"dev","html_sha1":"a067aa4334aa63b3299afe19142dfd3f9d1fac85","truth_sha1":"e41347fe3a70cd05bc7f7d42d708a8c4815a1335","overlaps_test":true,"full_text_reference":true}, +{"id":"4083","category":"article","split":"dev","html_sha1":"e41c3f147c7e48424fadbee7b2740c6d937792cc","truth_sha1":"2ee538dbc98e070189356facf03c3610548b5886","overlaps_test":false,"full_text_reference":true}, +{"id":"4084","category":"article","split":"dev","html_sha1":"3c4f4d59d4e9a475fbb0302f71bdf8bc42bd58ac","truth_sha1":"544f2f2fd1c03adff8f3f7d2217f98edfaf13718","overlaps_test":false,"full_text_reference":true}, +{"id":"4085","category":"article","split":"dev","html_sha1":"a429e3fc824156f1df49ed907eb28973ee15b7ce","truth_sha1":"80fa0cdc88eb3782d9529f493de7ba25b27731d6","overlaps_test":false,"full_text_reference":true}, +{"id":"4087","category":"forum","split":"dev","html_sha1":"b36d9ba96988eba980bf0a4aaf44277f1c514cb4","truth_sha1":"f70abd328244fbf50f1bd0c4e62ac542ba784da7","overlaps_test":false,"full_text_reference":true}, +{"id":"4088","category":"article","split":"dev","html_sha1":"b7101f6e3c31e4ebd6c6492253b6d68f14e1be25","truth_sha1":"ec1ceae1bb6f0f436d4dd49ed0880dd6624efdfe","overlaps_test":false,"full_text_reference":true}, +{"id":"4089","category":"forum","split":"dev","html_sha1":"a60758a0f8f07aef4763d0a7d6bbcfccb10335da","truth_sha1":"1dfdd3fce354bd2eb8997f30c47fff90ef7e0c51","overlaps_test":false,"full_text_reference":true}, +{"id":"4090","category":"article","split":"dev","html_sha1":"705f340920c177d97a05ebef37ed291acf19301c","truth_sha1":"f229cbe613c0208880a9539501413c7aefc85ee4","overlaps_test":false,"full_text_reference":true}, +{"id":"4091","category":"article","split":"dev","html_sha1":"92e1ce9fe61c00f506c48377fd4eea8ce936c669","truth_sha1":"036133e6078d75261acc985c4b40e667ce9e88e2","overlaps_test":false,"full_text_reference":true}, +{"id":"4092","category":"article","split":"dev","html_sha1":"b831288c8fd55f6080d0498940386f23b6ac59df","truth_sha1":"f0fb53581d7f5baee04efa684a442e6d83b1abe8","overlaps_test":true,"full_text_reference":true}, +{"id":"4095","category":"article","split":"dev","html_sha1":"e422938532348424a54b26eef77267045a4da8cf","truth_sha1":"d0213fba3d77d69b050df01cbd68f8895c471299","overlaps_test":false,"full_text_reference":true}, +{"id":"4096","category":"article","split":"dev","html_sha1":"68dc224f4d0f9871f6c9483544e52a3f6e091c74","truth_sha1":"3a05c14e23d88d9c83c59204c9edf9f18cb14041","overlaps_test":true,"full_text_reference":true}, +{"id":"4098","category":"product","split":"dev","html_sha1":"1f6d9b09ae49192f59223c68cd0d094994bc7672","truth_sha1":"867d4f9a17d1cef9030e186ade4420f69f88e1c6","overlaps_test":true,"full_text_reference":true}, +{"id":"4100","category":"collection","split":"dev","html_sha1":"b3543c0bfe4354a8c77851c202ef4459e4e9e192","truth_sha1":"5a4f4731ae3d727e3cacd632a1173082314f7fe6","overlaps_test":false,"full_text_reference":true}, +{"id":"4101","category":"article","split":"dev","html_sha1":"4bd8f44998cc7958b6effeae60c26ce6e6e663b5","truth_sha1":"7b7baf659fcdace22f81f14ff3fa3e6f4922a167","overlaps_test":false,"full_text_reference":true}, +{"id":"4102","category":"article","split":"dev","html_sha1":"22b7c173a04c3ccb0501070a0bdc7b8d987c552d","truth_sha1":"5dea688c893e9e1d6b7640919f6566f710f1ac32","overlaps_test":false,"full_text_reference":true}, +{"id":"4103","category":"article","split":"dev","html_sha1":"6bfefe01eaa7b7d5e7b4e6c4369bec44f27a7747","truth_sha1":"aacfef719642a8e7e2a52f38bf9e22b99ed74a58","overlaps_test":true,"full_text_reference":true}, +{"id":"4104","category":"article","split":"dev","html_sha1":"ec5b9d6cd627c550544955915e5d0ab29603dbd9","truth_sha1":"4dfb612dd8d4fab6ce9a32b5f8be6a78a2372b46","overlaps_test":false,"full_text_reference":true}, +{"id":"4105","category":"article","split":"dev","html_sha1":"3967a4d7621a5a40b4cd6fa33c7d61d9349f9903","truth_sha1":"5513fd7ceae219ce40f4a47013e0f0f0989ffde8","overlaps_test":false,"full_text_reference":true}, +{"id":"4106","category":"article","split":"dev","html_sha1":"8736fafdd7ee27edb70d9b6defbaf77abe10c9bf","truth_sha1":"a70d7966f62dc2ee99b839d25007b77d666448e4","overlaps_test":false,"full_text_reference":true}, +{"id":"4107","category":"product","split":"dev","html_sha1":"d1001328f88e40c6f1114adc2e3fcca08821838e","truth_sha1":"4e3cb44ab567fb1b4c650b981aff41dbc27b6077","overlaps_test":true,"full_text_reference":true}, +{"id":"4108","category":"article","split":"dev","html_sha1":"7ca2560831722aa2c611ed7ae5ac80baa8956a22","truth_sha1":"75c0a5e4cbacb394b4e518e0e6d9deffbec3f6a7","overlaps_test":true,"full_text_reference":true}, +{"id":"4110","category":"article","split":"dev","html_sha1":"7f65d9c2c8c4e2cd9b205c25e80eebd1912b3f8e","truth_sha1":"5c615b6717040ec1f7b3076ccc0e84a4e003ff22","overlaps_test":true,"full_text_reference":true}, +{"id":"4111","category":"article","split":"dev","html_sha1":"5212bdd1fc993cecfbe183f29b59078ed55b327e","truth_sha1":"ee04a51ef44231735e4417ef9f65bf9eaa7caa1f","overlaps_test":false,"full_text_reference":true}, +{"id":"4112","category":"listing","split":"dev","html_sha1":"1a26b3a554254a7ebe1d829b270a87bc6eae1486","truth_sha1":"559585f8c0031ff75680a3e4e4eea914f57aa81f","overlaps_test":false,"full_text_reference":true}, +{"id":"4113","category":"article","split":"dev","html_sha1":"95661831e61d5b5c1da2be83f620c97f933fb2ee","truth_sha1":"d0f8fdc06019b1786f925d32a692cb6b9700bfcd","overlaps_test":true,"full_text_reference":true}, +{"id":"4114","category":"article","split":"dev","html_sha1":"2370ab794371dc3ada18258ee381e706575ea49d","truth_sha1":"c82306c5453661be0a9c78f06066b146eb92372b","overlaps_test":true,"full_text_reference":true}, +{"id":"4115","category":"article","split":"dev","html_sha1":"1013ab0490c8721a43dd932261c1f6443746bad9","truth_sha1":"c5517a111f9fbaa4f56d4b79cf35076d603392ed","overlaps_test":false,"full_text_reference":true}, +{"id":"4116","category":"article","split":"dev","html_sha1":"13b9b1d2c348281983c9366489443eb114a30387","truth_sha1":"f1b2b9f444b58a8bd23b9109820a7d3ca1e2f094","overlaps_test":false,"full_text_reference":true}, +{"id":"4117","category":"article","split":"dev","html_sha1":"d405fa339ce5552476194e471633c7b6da4063e9","truth_sha1":"1c379d17b9dcad9c344d002c816c584e8ead2fd5","overlaps_test":true,"full_text_reference":true}, +{"id":"4119","category":"collection","split":"dev","html_sha1":"b0228c74b322891cec2ce218c85f5dca1c095f04","truth_sha1":"15e9fd1fa7de5dedee35db0910b000ea3783c263","overlaps_test":false,"full_text_reference":true}, +{"id":"4120","category":"listing","split":"dev","html_sha1":"01bee797a42b74cee87e1bfee004d6870fa0256a","truth_sha1":"467ce20cdc4ddd78d3b23122cee689ab888ab701","overlaps_test":false,"full_text_reference":true}, +{"id":"4121","category":"article","split":"dev","html_sha1":"5da834e743eb19d48322467ace4f6b5ddde42569","truth_sha1":"a426a434f02eb3acdfeb040ebae728d4e40fe3b0","overlaps_test":false,"full_text_reference":true}, +{"id":"4122","category":"listing","split":"dev","html_sha1":"f86a30fa4ba9a978ef20df04758eca34fa0facaf","truth_sha1":"404dc2ecfc7c72ddacaac0fc6322a2d083e72470","overlaps_test":false,"full_text_reference":true}, +{"id":"4123","category":"article","split":"dev","html_sha1":"7ed23e760d4ed7d4b86a0e05074fc2b0b708006c","truth_sha1":"d12e9983298290b25b6bde83d6b5a3845d98fc71","overlaps_test":false,"full_text_reference":true}, +{"id":"4124","category":"article","split":"dev","html_sha1":"8ef784092fb28df2f8e642a0dff7a1bade497570","truth_sha1":"261af6c32861d8af04bcec3d58a810b6d04cf9d4","overlaps_test":false,"full_text_reference":true}, +{"id":"4125","category":"article","split":"dev","html_sha1":"6d38a6288106a013901f6964509c8f9d156c87a9","truth_sha1":"7bac6873dc180e0b4c367767ca27bc7fdebd2a16","overlaps_test":false,"full_text_reference":true}, +{"id":"4126","category":"article","split":"dev","html_sha1":"306d361c7ba5b285ad995981487c821431135f85","truth_sha1":"4830ee38fdc3f51d660133188f38d4130bab5b09","overlaps_test":true,"full_text_reference":true}, +{"id":"4129","category":"article","split":"dev","html_sha1":"de085f11dc1b4a457492024fe0fd4aac2831048e","truth_sha1":"8fc47ad283557851747cfbf34e8f429566fc225d","overlaps_test":true,"full_text_reference":true}, +{"id":"4130","category":"article","split":"dev","html_sha1":"c6986cf151b94da31139900c5dd726c82a2a62fe","truth_sha1":"151859cad35f5a98d79331546426e9c51b9d789a","overlaps_test":false,"full_text_reference":true}, +{"id":"4132","category":"article","split":"dev","html_sha1":"8f520074dc0b7432778353cc695c38b5397c69a5","truth_sha1":"805768abf6822d63ace7b0ac4cb518e371724608","overlaps_test":true,"full_text_reference":true}, +{"id":"4135","category":"article","split":"dev","html_sha1":"ab908533d2fe05c499e5a9a8d5e38453bee2cfe4","truth_sha1":"1b1789d6fbc42bed824add35407a1b137448922b","overlaps_test":false,"full_text_reference":true}, +{"id":"4137","category":"collection","split":"dev","html_sha1":"e464f255b944ef4ff50cc6411b3fa3c03ae87b3e","truth_sha1":"e926a8a40fd48d1a21590b01e24eb6907d574e17","overlaps_test":false,"full_text_reference":true}, +{"id":"4138","category":"article","split":"dev","html_sha1":"73d73269ea5c930518a4988684335b5aba781b49","truth_sha1":"3f7111f3be246ce59bfbab947679759cb8ba660e","overlaps_test":false,"full_text_reference":true}, +{"id":"4139","category":"listing","split":"dev","html_sha1":"a7176c1e995995172ee5362f766b7eba34584a9b","truth_sha1":"7530f069c8b9448937bf8b706b6049456ce8713a","overlaps_test":false,"full_text_reference":true}, +{"id":"4142","category":"article","split":"dev","html_sha1":"e7b4e57c901531484a26220e73caefe5db0ef8af","truth_sha1":"4772133d315b248ec8540d8a3b0ac78bfa9ef63d","overlaps_test":true,"full_text_reference":true}, +{"id":"4145","category":"forum","split":"dev","html_sha1":"aa694f50f3d136d9c8b2c6fdafe47ee1687e23a3","truth_sha1":"5fc479a0d237fd54553a9be3d50edab15a8c7999","overlaps_test":true,"full_text_reference":true}, +{"id":"4146","category":"article","split":"dev","html_sha1":"db922d4e1b877ad360044c9be8318a51d10fd834","truth_sha1":"2f6f91b49d7bb4906e875b3bfa6e892f71f66fbf","overlaps_test":false,"full_text_reference":true}, +{"id":"4147","category":"forum","split":"dev","html_sha1":"8fa8ff8baf8b0e5010c6b1dc18b862730034df4a","truth_sha1":"1dbbf0b070a7601c80d4991614c52a52dc4caf4f","overlaps_test":false,"full_text_reference":true}, +{"id":"4148","category":"article","split":"dev","html_sha1":"006366438e86e4375ce893d580f35a9da5b29e8c","truth_sha1":"443052ce0900f48940c98814485c96ae7a550a45","overlaps_test":false,"full_text_reference":true}, +{"id":"4149","category":"collection","split":"dev","html_sha1":"5daa8e669be2f1191aa211daed252d32335c42ff","truth_sha1":"589cf756f89c4ce5b7a12d687ab12ff2ab215ff5","overlaps_test":false,"full_text_reference":true}, +{"id":"4150","category":"article","split":"dev","html_sha1":"673210c6114c16755c8f1d59880d0fe967f47a65","truth_sha1":"c98469daf7cc5338d8f9109d0e087fc165f9dca1","overlaps_test":true,"full_text_reference":true}, +{"id":"4151","category":"article","split":"dev","html_sha1":"e8100e259a01e20981a979ea1f0d2a9cea861225","truth_sha1":"04d3f85f540cada00ea346415dd7ae19239428d4","overlaps_test":false,"full_text_reference":true}, +{"id":"4155","category":"article","split":"dev","html_sha1":"0677fce7504d701a44d0b4f6bfebd3a6d1367ead","truth_sha1":"b8258c6178a6896a18d4f88741658b91671ef5c5","overlaps_test":false,"full_text_reference":true}, +{"id":"4158","category":"collection","split":"dev","html_sha1":"99ba3cf5654f26397481498cbf79c8d403037e1e","truth_sha1":"259e4b434b8187f44ea9fa1376fad00ce4b9a6a4","overlaps_test":false,"full_text_reference":true}, +{"id":"4159","category":"article","split":"dev","html_sha1":"fa16dedf5f79f954aa3356fae252d30c9b8c90cf","truth_sha1":"2f03553d8e2ce80a2b804e4be956f8e6510628a0","overlaps_test":false,"full_text_reference":true}, +{"id":"4160","category":"article","split":"dev","html_sha1":"83c0c90944c8777d9fedd321b8706fef9917884f","truth_sha1":"99f3f67e47c1c672c906cf915015407887064b27","overlaps_test":true,"full_text_reference":true}, +{"id":"4161","category":"article","split":"dev","html_sha1":"3ba59d0da1343768ea3f6d494d624c6a86c45841","truth_sha1":"707011ba2e00a12795a7d7b6364848b3c643fe69","overlaps_test":false,"full_text_reference":true}, +{"id":"4163","category":"article","split":"dev","html_sha1":"0afe4e4566aef0d4e89a0733820d5fdc77347b10","truth_sha1":"993e29295ba31a56a09c17bcce965de912dd4799","overlaps_test":false,"full_text_reference":true}, +{"id":"4164","category":"article","split":"dev","html_sha1":"ef2008332727901af681e11567ea3f75f2e52ab0","truth_sha1":"41c8a1ce96d9fefc3959077176ae9fac15d6f096","overlaps_test":false,"full_text_reference":true}, +{"id":"4166","category":"article","split":"dev","html_sha1":"eef473f962552ff70029aa284a0022a61757131f","truth_sha1":"b93a5be431f002cbcd75f53e4e94fafad44666fe","overlaps_test":false,"full_text_reference":true}, +{"id":"4168","category":"product","split":"dev","html_sha1":"efb83eae1e26680f8e1c828a4edec5d1a71bd8a2","truth_sha1":"b003196c373522dcf647646d86fffad614ee7800","overlaps_test":true,"full_text_reference":true}, +{"id":"4169","category":"product","split":"dev","html_sha1":"bcd5f4f2d8b0acec954879d72b5482a9c52011bc","truth_sha1":"835ad54f3c3a80b6a34a87bcc5f143d9da689a80","overlaps_test":true,"full_text_reference":true}, +{"id":"4170","category":"article","split":"dev","html_sha1":"1799f730a2be1da11661d1209b679e179c90a856","truth_sha1":"ba75768a7112bc9c8deb9697382e9eea3abb8386","overlaps_test":false,"full_text_reference":true}, +{"id":"4171","category":"article","split":"dev","html_sha1":"758dbde01507de09c06d5879c5db13132b0056cb","truth_sha1":"f75a09ac701c6242416e81b602fa01db0b37f76c","overlaps_test":true,"full_text_reference":true}, +{"id":"4172","category":"article","split":"dev","html_sha1":"fb6eeffb755912f0df9d0f13c6d0e22b90f4e71b","truth_sha1":"24925a9efc12f1268e28b3246e2dd1ac6987acd9","overlaps_test":false,"full_text_reference":true}, +{"id":"4174","category":"article","split":"dev","html_sha1":"8ff029807e506cc53ba7085eae393b2708417c76","truth_sha1":"940ba14047f448443bdd46168413fb2038e73469","overlaps_test":true,"full_text_reference":true}, +{"id":"4175","category":"article","split":"dev","html_sha1":"8c72752087bf0065116f993438abeec7dd84d0cd","truth_sha1":"6d2f34a0db7e142c3e9b797f54e731a63d80ab67","overlaps_test":false,"full_text_reference":true}, +{"id":"4177","category":"article","split":"dev","html_sha1":"5a8e630325c4351ae271202a8445d8044a0f9fa4","truth_sha1":"73ebe9a3317580ef667fe4b444d8c8ca06e5c5f5","overlaps_test":false,"full_text_reference":true}, +{"id":"4178","category":"article","split":"dev","html_sha1":"8057d057b07a9bbe71368cedca934d403566c8fd","truth_sha1":"8afdc482835890c55679d833f811d73e8f5531aa","overlaps_test":false,"full_text_reference":true}, +{"id":"4179","category":"article","split":"dev","html_sha1":"413c81bb15a1e4c462f1353dd352593daac70ec1","truth_sha1":"fd099d0a58a3d28f3c215b2bc6455d221247a478","overlaps_test":true,"full_text_reference":true}, +{"id":"4180","category":"article","split":"dev","html_sha1":"b84b1be15afb43197f491bd0f689f3f7a6def67d","truth_sha1":"39f3dcbfe41e92a71c23b98c9d047e103e2c6a37","overlaps_test":false,"full_text_reference":true}, +{"id":"4182","category":"article","split":"dev","html_sha1":"b84085ed242ec9175ca83036b34b7d12fea64ce7","truth_sha1":"8cf7e9f249c1075b78ad3dcd05db4ae9f8c568ab","overlaps_test":false,"full_text_reference":true}, +{"id":"4184","category":"article","split":"dev","html_sha1":"6588ca936fbbc2e39899efabf1d648e3af67056a","truth_sha1":"8f49a4385c57a63f5b0e7ae1a06c9bbfb97fc9a1","overlaps_test":false,"full_text_reference":true}, +{"id":"4185","category":"article","split":"dev","html_sha1":"c2d06f1841a58d2a22ed7f3b71a2451b08d389de","truth_sha1":"60af9e7efb2958ba9ab6b64975551da7c6becc31","overlaps_test":false,"full_text_reference":true}, +{"id":"4186","category":"article","split":"dev","html_sha1":"952ef1db68b21defa3b99da49118944896fc07c0","truth_sha1":"45e7446840d9407810cf7abb733a6fbd076ecdbe","overlaps_test":true,"full_text_reference":true}, +{"id":"4188","category":"service","split":"dev","html_sha1":"e3587d5a9bc55d34eff0c1960f725c235210184c","truth_sha1":"0d2f917b0440cf2e0aae33f03b8f54ec006a28e6","overlaps_test":false,"full_text_reference":true}, +{"id":"4190","category":"collection","split":"dev","html_sha1":"e419f23bf1f9cf23ad1bee6e7aba6259b8c28540","truth_sha1":"56f7e036e2ecf2fd0b1957e33f51f4bef76ac857","overlaps_test":true,"full_text_reference":true}, +{"id":"4191","category":"article","split":"dev","html_sha1":"6384ffc1a29ed05c60b11990087a8612cb8b37b8","truth_sha1":"62a45a3c578fe47410251fd062f63614a28d08ea","overlaps_test":false,"full_text_reference":true}, +{"id":"4192","category":"article","split":"dev","html_sha1":"02e294015f22bb30fd39d699c9852cd938e01eda","truth_sha1":"96f031b35a51e34d2c0547611a4414e4dc90e081","overlaps_test":false,"full_text_reference":true}, +{"id":"4193","category":"article","split":"dev","html_sha1":"b783d17c12e0d1d72e491d9a9ced74b2254bddeb","truth_sha1":"c1d1cc69c45e454b72a11bea18501db466004b53","overlaps_test":false,"full_text_reference":true}, +{"id":"4194","category":"article","split":"dev","html_sha1":"ffd388e0b0c4a7f277ac56b03ca9dd984e605ccd","truth_sha1":"72cd1337c13013bfe5e0d2ebfa8e5b94ab731456","overlaps_test":false,"full_text_reference":true}, +{"id":"4195","category":"product","split":"dev","html_sha1":"db1d2e4565a757981f89f251cd038ee3e8d962c9","truth_sha1":"1d97c42385e78d3be99e622adbc89a2f2bc10a1b","overlaps_test":true,"full_text_reference":true}, +{"id":"4200","category":"product","split":"dev","html_sha1":"22bf35b5516e85cdc4e4feb84ecb6375d5972758","truth_sha1":"5fa7f536f778d76cea2930c43c3855a9f8d189b9","overlaps_test":true,"full_text_reference":true}, +{"id":"4202","category":"article","split":"dev","html_sha1":"68467e84a3dc29f5eec72eff3459dbc664ee9fd7","truth_sha1":"deac49428060b0d8c9bf42e88f143a43222a7f26","overlaps_test":false,"full_text_reference":true}, +{"id":"4203","category":"collection","split":"dev","html_sha1":"650ff0f9a436acfc919d7dae7e018615b5af79ac","truth_sha1":"0138683ade92e8428b4e6c98e71ccb186d48202f","overlaps_test":false,"full_text_reference":true}, +{"id":"4204","category":"listing","split":"dev","html_sha1":"3ab3eb0a437f396ddc972efc75f34a8761c579af","truth_sha1":"1b11daefd293b26df8046479df57f2ed35747535","overlaps_test":false,"full_text_reference":true}, +{"id":"4206","category":"product","split":"dev","html_sha1":"2ac79112074d8a9dd61f0b71ca98a0657880f7cd","truth_sha1":"f1c7d17131fad11d4727f0f253b1316720cb4a6d","overlaps_test":true,"full_text_reference":true}, +{"id":"4207","category":"collection","split":"dev","html_sha1":"6456a0a790a3c6b48971b20d9bf1470cb00035d1","truth_sha1":"d5b71f964fad62609cae227e8c0badcfd455c575","overlaps_test":true,"full_text_reference":true}, +{"id":"4211","category":"product","split":"dev","html_sha1":"74057afd294e23985cb19ea80c9386af4d0b33fb","truth_sha1":"07fa8ae8bf8e31501707a3ac2d0829776b707036","overlaps_test":true,"full_text_reference":true}, +{"id":"4212","category":"product","split":"dev","html_sha1":"30d8e8b5302e87d453675902771b3324f43a17d8","truth_sha1":"f3236f0e952fef005952789afb302fdeb16754ab","overlaps_test":true,"full_text_reference":true}, +{"id":"4213","category":"product","split":"dev","html_sha1":"6c578f87a92392dbb0e551eeeff2a2cbe3d0f81e","truth_sha1":"3d57f112ec945b3cf55186cf9da74e5352e0c937","overlaps_test":true,"full_text_reference":true}, +{"id":"4218","category":"article","split":"dev","html_sha1":"3b6d2ede63973960edcaa52e510a30820b2aa57c","truth_sha1":"74dd3cd12b096990bddba438c69e8350f2837300","overlaps_test":false,"full_text_reference":true}, +{"id":"4219","category":"product","split":"dev","html_sha1":"c2ac9afbf6c8f3dca86d82e42f1f4c03356153da","truth_sha1":"d57eb50b8b29b83ad49da85d08eb62f670d14901","overlaps_test":true,"full_text_reference":true}, +{"id":"4221","category":"product","split":"dev","html_sha1":"98a0453a56132efac7f6d014d0a614a775f56147","truth_sha1":"5936c4a5d00c0a234c09f3ac5b1749d8517cdad1","overlaps_test":true,"full_text_reference":true}, +{"id":"4223","category":"article","split":"dev","html_sha1":"8aa755af3b74b1ce576a632e0ff704f26408aafe","truth_sha1":"11985ff90479c0c11abb76a5f7bbbb2940829d74","overlaps_test":false,"full_text_reference":true}, +{"id":"4224","category":"product","split":"dev","html_sha1":"a0fb4b27c67d38815b1e9070a13dbda8e69f46ac","truth_sha1":"06a951f38d4330cf44bf96572492d77b0b7c0301","overlaps_test":true,"full_text_reference":true}, +{"id":"4225","category":"product","split":"dev","html_sha1":"4b4b70f9413f19e6e7cb9159360d80c84582eaea","truth_sha1":"fc04e26d216016f034df377af5a75642ef66fae4","overlaps_test":true,"full_text_reference":true}, +{"id":"4232","category":"article","split":"dev","html_sha1":"1715a6b94c422983efad2cb292745761a9cc5151","truth_sha1":"ef984ee804201870ed33aded0921c07f4fec8d29","overlaps_test":false,"full_text_reference":true}, +{"id":"4233","category":"article","split":"dev","html_sha1":"2a1240523a4929f1a9f7e7082e8f20be125c05d1","truth_sha1":"bfe0d86c2f0dbf0a3637802f6a3327670a29c40f","overlaps_test":false,"full_text_reference":true}, +{"id":"4235","category":"article","split":"dev","html_sha1":"f032d7d00fb891fd39f50df98bf3568f871b1a8c","truth_sha1":"237a1977b0f0edf83c75b9bb79c25534c860084e","overlaps_test":false,"full_text_reference":true}, +{"id":"4239","category":"article","split":"dev","html_sha1":"b469c3531d72480462441a6efcd5ce54cbcf7763","truth_sha1":"5d992a105d89585138f5c5cd9f6aee2a75e06f74","overlaps_test":false,"full_text_reference":true}, +{"id":"4240","category":"article","split":"dev","html_sha1":"d00081e899804e42cf2e326cd591ed9c7d6ec9fb","truth_sha1":"b1978e6a0a84ced8f0da2d54b640a3ccd0014ee9","overlaps_test":false,"full_text_reference":true}, +{"id":"4241","category":"article","split":"dev","html_sha1":"00b7069a1d813c80e2e85977c55b6c503ae37333","truth_sha1":"c859ba2267c46aed0d3c242357ab0457b9cdd73c","overlaps_test":true,"full_text_reference":true}, +{"id":"4242","category":"service","split":"dev","html_sha1":"016fa6db91ca09986fd8539a0b5712093274505f","truth_sha1":"d045fe45b79d6f878bed0c006fe6371318aea10a","overlaps_test":false,"full_text_reference":true}, +{"id":"4243","category":"article","split":"dev","html_sha1":"5d92c94d4a53b5a42701f347d572ad60432d54ee","truth_sha1":"cc3413e40ce68a45c67b3352f30649ae9c563e71","overlaps_test":false,"full_text_reference":true}, +{"id":"4244","category":"article","split":"dev","html_sha1":"af9d9959639f8a7c1337d3887dc3c4d1a79a8d9a","truth_sha1":"1b98314f5fdb9a000aeeca800b3e34943162c997","overlaps_test":false,"full_text_reference":true}, +{"id":"4245","category":"article","split":"dev","html_sha1":"7d0a9d02736d1687e2f2a27b4cfa8fc5d47b94e1","truth_sha1":"b3d7b616ce97179a51d8ca746a889cbd0f7ddf5c","overlaps_test":false,"full_text_reference":true}, +{"id":"4246","category":"service","split":"dev","html_sha1":"b5c4993b51dc4a893ac034b122f4fc398fa231fe","truth_sha1":"0f7b1374b4f24a7cbdd0dfe09aca6cce441b77e5","overlaps_test":false,"full_text_reference":true}, +{"id":"4247","category":"listing","split":"dev","html_sha1":"e9ac0733e81d48f8a0d85737f90ee098bf326e3d","truth_sha1":"9610b54e4ecbd8edd750f2f1104ff739c2a4b51c","overlaps_test":false,"full_text_reference":true}, +{"id":"4249","category":"article","split":"dev","html_sha1":"e66bd84a72f59195c0462aaae7e53e22af2e7dbc","truth_sha1":"952d36ce07123a1a3d44c7beb19ecb5eef5dbfbf","overlaps_test":false,"full_text_reference":true}, +{"id":"4250","category":"service","split":"dev","html_sha1":"2d8069cec5127ddec92d2bd27d09636631b69887","truth_sha1":"630237a29702681cd023b85df259b0e00f2eb554","overlaps_test":false,"full_text_reference":true}, +{"id":"4251","category":"article","split":"dev","html_sha1":"ebfbd7fa838c27eb9131b00f1631b9dc746b6b6b","truth_sha1":"21c75057c2c27d2f212d76f5c82c432ccc0b7db6","overlaps_test":false,"full_text_reference":true}, +{"id":"4252","category":"article","split":"dev","html_sha1":"da277cd671c505a280062260767b6863806194a0","truth_sha1":"dd3fa5c663d60d4d7acb83eb8497a3cc1ce3b180","overlaps_test":false,"full_text_reference":true}, +{"id":"4253","category":"article","split":"dev","html_sha1":"a4e6851ffc1700550065d7c3f50fc6c66e5b8bb0","truth_sha1":"b49a0ca43a485ba29de5bed5d86e464e697ed985","overlaps_test":false,"full_text_reference":true}, +{"id":"4255","category":"article","split":"dev","html_sha1":"bde6763a534ff102908732e1301f1e984a5f5c62","truth_sha1":"3ca18f79271018200a78024489d95b4ec0c054ed","overlaps_test":false,"full_text_reference":true}, +{"id":"4256","category":"article","split":"dev","html_sha1":"27c22da64b591968d22ffb4593c4cfe216806a1a","truth_sha1":"b4c7281fa598cf14c782260f171cdd09bbfb587b","overlaps_test":false,"full_text_reference":true}, +{"id":"4257","category":"article","split":"dev","html_sha1":"c4abf42222c1c06033b2420a25e3c6342491b5c7","truth_sha1":"45969a42c708403d0ae3c6e97c5d631b9e31bd42","overlaps_test":false,"full_text_reference":true}, +{"id":"4258","category":"article","split":"dev","html_sha1":"cdf6dc0213a0816b0c65423b13275a9659208a4b","truth_sha1":"7fedd6224eb8730009ae2f2218ddc639ec7c1613","overlaps_test":false,"full_text_reference":true}, +{"id":"4259","category":"article","split":"dev","html_sha1":"0cddb12b1b2ffc91e9cd057bccd75269afb0eb3b","truth_sha1":"e5c30b66f0419e739ebfe3013f1013fee2504180","overlaps_test":false,"full_text_reference":true}, +{"id":"4260","category":"article","split":"dev","html_sha1":"dd00a99a18b0a69929b86e20351316c650b2dd56","truth_sha1":"b93ea1b92e204465a086e6fc5bd814a357e9b1d4","overlaps_test":false,"full_text_reference":true}, +{"id":"4261","category":"collection","split":"dev","html_sha1":"8324539720109f73603bbc38a6cf8b1fee38094f","truth_sha1":"759ef933d4db21e3a1ab97dd7c66cf20d8c5184c","overlaps_test":false,"full_text_reference":true}, +{"id":"4262","category":"article","split":"dev","html_sha1":"5ce5c5de607cb9af2d02afca32ec898c6d0ed22b","truth_sha1":"74de5de12b8be6337b955554f591cb7c23f28835","overlaps_test":false,"full_text_reference":true}, +{"id":"4263","category":"product","split":"dev","html_sha1":"65d6e8f5972aa3d9c919399fc8f070b58742c148","truth_sha1":"1151d9e95c45640d8bc1839a2fa6a01babe7cb89","overlaps_test":true,"full_text_reference":true}, +{"id":"4264","category":"collection","split":"dev","html_sha1":"be1346bac92ec73899913154113d9b5c64e65424","truth_sha1":"8d93c6e754c60399d256922e6bc99869e6d6cad7","overlaps_test":false,"full_text_reference":true}, +{"id":"4265","category":"article","split":"dev","html_sha1":"4acf6a39b0eb49e734720758105ca6be0e5bc6e7","truth_sha1":"9a4624c8e4ea3791ea4b90ff577449c8346d0a97","overlaps_test":false,"full_text_reference":true}, +{"id":"4266","category":"service","split":"dev","html_sha1":"a08d61e4d5da132c817696e01addefc0832b2974","truth_sha1":"0f60135c11e6f4371fa43b7cf459abf4ee03977a","overlaps_test":false,"full_text_reference":true}, +{"id":"4267","category":"article","split":"dev","html_sha1":"113f6c077769512ae9a21c9a40ea3aabb30c69ce","truth_sha1":"a515addaf34bde315e1d83d89c61a58e544e8c04","overlaps_test":false,"full_text_reference":true}, +{"id":"4268","category":"article","split":"dev","html_sha1":"7d49558a8c9e8f3068d0b1f5bb1f552f8224df52","truth_sha1":"fde9015f6df4ad0369d592489806c4e865be7159","overlaps_test":false,"full_text_reference":true}, +{"id":"4269","category":"service","split":"dev","html_sha1":"57fe4b76bbc64cc2ebec96d2a5eb4d6b2abea844","truth_sha1":"e37c0e6c982e6f9ad2a6e824afadb6c13f19969d","overlaps_test":false,"full_text_reference":true}, +{"id":"4271","category":"article","split":"dev","html_sha1":"f4e07d406e2e7b1648a505d1e7179d4a30b5274c","truth_sha1":"2e1fb99776c9b2696615ece5c48b4e02a55301d6","overlaps_test":false,"full_text_reference":true}, +{"id":"4272","category":"service","split":"dev","html_sha1":"201e1e3a6a3931de1a299865f70c6da465e1b0b9","truth_sha1":"1dd849eaf2213c545409cb62134ca07e100925be","overlaps_test":false,"full_text_reference":true}, +{"id":"4273","category":"service","split":"dev","html_sha1":"a6a941855e48ded7bcf0b2414df66e5f949637e2","truth_sha1":"690c068d1fa5a2f787c3e94f0ccfbc733034a310","overlaps_test":false,"full_text_reference":true}, +{"id":"4274","category":"article","split":"dev","html_sha1":"dd81851431bf8a0d7229bf635b8d8caa6fa6d955","truth_sha1":"fa340a81be2ad85542c5aa9c916bf6cd59ba2512","overlaps_test":false,"full_text_reference":true}, +{"id":"4277","category":"article","split":"dev","html_sha1":"8efae542b89d0aa3532b882a5ed3533c84b0681b","truth_sha1":"6a4c3453d40ea836776ff92962435de10cfe7069","overlaps_test":true,"full_text_reference":true}, +{"id":"4278","category":"forum","split":"dev","html_sha1":"e5d4b0d4b0ad1652ca0bd5b8c6afaae46d5d3427","truth_sha1":"71bacd95cc3215fad0d36d9e70392b99610f3210","overlaps_test":false,"full_text_reference":true}, +{"id":"4280","category":"collection","split":"dev","html_sha1":"16c8641a69d7a98a83acc2a9970e71518f1ab072","truth_sha1":"f3e921d69adc4805100e3af074cad1e7baeda105","overlaps_test":false,"full_text_reference":true}, +{"id":"4281","category":"service","split":"dev","html_sha1":"25b78c37e0f6429a7acbfab1897a5b4925efd44f","truth_sha1":"4b0af77d7777fe10d5c771aa2d2cdaa1f8bbec2a","overlaps_test":false,"full_text_reference":true}, +{"id":"4282","category":"service","split":"dev","html_sha1":"33d3917f2c97ac1a2a99c2613eb797d07e9730ee","truth_sha1":"4e2823b016aeeeb2770e8d5357bc71fe76b818fc","overlaps_test":false,"full_text_reference":true}, +{"id":"4284","category":"service","split":"dev","html_sha1":"943000f08d9aecf413b5841fdfe8085afeda928a","truth_sha1":"91b6ccb15d230713bdc5ec2db7ddff8e808f5afe","overlaps_test":false,"full_text_reference":true}, +{"id":"4286","category":"service","split":"dev","html_sha1":"bb6fde5b7ec40fe86ad7e70d71adb9fb22c3f220","truth_sha1":"68bf2bc2974079a2441b6d1296a8fb04b794f937","overlaps_test":false,"full_text_reference":true}, +{"id":"4287","category":"service","split":"dev","html_sha1":"2d75985f54ba422da3c091596ad23cc372c1146e","truth_sha1":"348b37e1261234139df806ecb7a52c1ac69f3989","overlaps_test":false,"full_text_reference":true}, +{"id":"4289","category":"service","split":"dev","html_sha1":"db5679274f855817b827f342b8bc6aa8a72060db","truth_sha1":"11660738f2f7ae1bbc957f65cb72ef56458881a1","overlaps_test":false,"full_text_reference":true}, +{"id":"4290","category":"service","split":"dev","html_sha1":"103702513c4a548aacac6ab9e57914126b43e35a","truth_sha1":"c30ee6b0b2a3002d5b93be086f314da9ae503e94","overlaps_test":false,"full_text_reference":true}, +{"id":"4291","category":"collection","split":"dev","html_sha1":"083cf3e4e9a635848c60d0ca5c32ed3270b98516","truth_sha1":"239f5559a534987a67e3b94e947b7716c4f1cf6c","overlaps_test":true,"full_text_reference":true}, +{"id":"4292","category":"service","split":"dev","html_sha1":"7dbef4697754a42fe1baa607aab14a4a888b056f","truth_sha1":"3fe81ea965509a6798e70ad5ea3a0af71b997f08","overlaps_test":false,"full_text_reference":true}, +{"id":"4293","category":"collection","split":"dev","html_sha1":"a1b91a6abc05d5cf828be7bcb5bcdf566f0b42c3","truth_sha1":"0e8b18615efb84e540ad7c1564cf5ee2fabb5bd3","overlaps_test":false,"full_text_reference":true}, +{"id":"4295","category":"article","split":"dev","html_sha1":"fc7cd53aa6a22ed28a5809b721a5dddf64920e6e","truth_sha1":"df528c86ed34075755ae2908932af222e73ea5f9","overlaps_test":false,"full_text_reference":true}, +{"id":"4296","category":"collection","split":"dev","html_sha1":"f4c378c13751e35276d9c79f8c5a5137583c1952","truth_sha1":"105d8346e4092e49e78c2a6a0b8fbdad6985b345","overlaps_test":false,"full_text_reference":true}, +{"id":"4297","category":"article","split":"dev","html_sha1":"ad38dfb08b9dfce9c9c7f4d88df8ef29d75db88f","truth_sha1":"26c3a7c2cb4d602f626f677aa36111064b6dfa8f","overlaps_test":false,"full_text_reference":true}, +{"id":"4300","category":"service","split":"dev","html_sha1":"ada8e77071de505f082edb68fd08a7ce0fe286e1","truth_sha1":"efede099f7128b7acdb2f82a1398386d94006801","overlaps_test":false,"full_text_reference":true}, +{"id":"4302","category":"service","split":"dev","html_sha1":"f07898b6d4ba5109957fa8d0c2c4efb7ec5b36c4","truth_sha1":"bc4159143e1b76de67cbc6eeffca5af29545a328","overlaps_test":false,"full_text_reference":true}, +{"id":"4303","category":"service","split":"dev","html_sha1":"5fa52e0ca7ed130c4fedaf5a69199349f21e2a9d","truth_sha1":"f8ef19c51561c1e16d5dd3bc59fc2b07425dac2c","overlaps_test":false,"full_text_reference":true}, +{"id":"4304","category":"article","split":"dev","html_sha1":"307ddb3e5b0f2b1da3889a313ded4876974c0eba","truth_sha1":"a664f3107c08b72a13f96264bb69d178d999425a","overlaps_test":false,"full_text_reference":true}, +{"id":"4305","category":"service","split":"dev","html_sha1":"fdfc0dbbc471bed5197b7a0994bdfabb743e44db","truth_sha1":"9c2ac79e5fa0a0ab45362f4dea4b1bb04c73f680","overlaps_test":false,"full_text_reference":true}, +{"id":"4306","category":"article","split":"dev","html_sha1":"da46c55f5db3fb8fe384cb844a46dbeca186e5a9","truth_sha1":"a1f83c182b542318e4acb34f5fc58a669e60691f","overlaps_test":false,"full_text_reference":true}, +{"id":"4309","category":"forum","split":"dev","html_sha1":"dd3efa4f8420ef5c521b58a3175cfbfbf2e9f63b","truth_sha1":"0998d923b5a4ef6ebd0713d7e550659f1aa91e0b","overlaps_test":false,"full_text_reference":true}, +{"id":"4310","category":"forum","split":"dev","html_sha1":"876c5817b107cc574b55b33e373dc22ca5544616","truth_sha1":"3d21987bf29906536ce98ff217e19c5ea473eb0f","overlaps_test":false,"full_text_reference":true}, +{"id":"4311","category":"forum","split":"dev","html_sha1":"1ac23bc541a65c6d70fd342c10d899c7e2605528","truth_sha1":"36bc277739799b8b057048e0f0438c1e9fd036b9","overlaps_test":false,"full_text_reference":true}, +{"id":"4312","category":"forum","split":"dev","html_sha1":"6cd54468bca5240abcdf89812b7033003fc13003","truth_sha1":"0f629321dcf3e0c387644729fb5f7e0f9919e0da","overlaps_test":false,"full_text_reference":true}, +{"id":"4314","category":"forum","split":"dev","html_sha1":"8af2b5a2a17bf1d8c47b14aadc4e1b6736436d2b","truth_sha1":"9afd6e9406fe6391e1aa62e084aa0c3317a1c494","overlaps_test":false,"full_text_reference":true}, +{"id":"4316","category":"forum","split":"dev","html_sha1":"65f8889b51caea212d6b9263fdf0cc54c23f664c","truth_sha1":"5f935645213d570ea07657bb410a2f0ae7ab86fa","overlaps_test":false,"full_text_reference":true}, +{"id":"4317","category":"forum","split":"dev","html_sha1":"009054499b086d023845503b07b25419dfb788f7","truth_sha1":"a6db4148e459c34dbc589fee076892273efb819e","overlaps_test":false,"full_text_reference":true}, +{"id":"4318","category":"forum","split":"dev","html_sha1":"14dbe5ac151838d7bde6dbec37cb3e32ace60579","truth_sha1":"cdfccf25781888037125d3284275fa75f0ac19a0","overlaps_test":false,"full_text_reference":true}, +{"id":"4320","category":"forum","split":"dev","html_sha1":"3d3e95deeb3f07d41a26d2119d37aceb94a3e494","truth_sha1":"439ba6a88f1f78b30d81a4e689af3e3ee88695e3","overlaps_test":false,"full_text_reference":true}, +{"id":"4326","category":"article","split":"dev","html_sha1":"2c50cb51f9b259ad576af261c6c2d0210b6c3b82","truth_sha1":"e7cf5f5fcf0dc31ae04a672feb6d4ab3e4d6dd4c","overlaps_test":false,"full_text_reference":true}, +{"id":"4329","category":"collection","split":"dev","html_sha1":"d7cf9e6e20167d555377a99ac4b2fec8dc831d92","truth_sha1":"7cd7c08ea322c7489a20a7238cd50810bf8ca152","overlaps_test":true,"full_text_reference":true}, +{"id":"4331","category":"listing","split":"dev","html_sha1":"32ec10fdfc0acd103b3029749c7dcd461ea1d980","truth_sha1":"b026bed22c34859886bbaa786f66bbad082239ad","overlaps_test":false,"full_text_reference":true}, +{"id":"4332","category":"listing","split":"dev","html_sha1":"d573969f7b6ebeffe82293bef422ed92fc171d16","truth_sha1":"4e915bb14eddbdfe073304f2c1836c2b0a30ba5b","overlaps_test":false,"full_text_reference":true}, +{"id":"4335","category":"collection","split":"dev","html_sha1":"74eaba140b2d6bc3245b6571f372f7a52dab00cf","truth_sha1":"b82b1f2e681bc6198c4858722ef5339bb1aa65b0","overlaps_test":true,"full_text_reference":true}, +{"id":"4339","category":"article","split":"dev","html_sha1":"76d50f276ab26171c2f9ee0f698d526153a05941","truth_sha1":"2642ff3988524b6657834219db92043fdcffdbf8","overlaps_test":false,"full_text_reference":true}, +{"id":"4340","category":"article","split":"dev","html_sha1":"f3ece2c4820fb6417bb23fe9698d3fdee2d83a3a","truth_sha1":"f1df6d1d85c478f0414cea5a81e6a7465d33e97d","overlaps_test":false,"full_text_reference":true}, +{"id":"4343","category":"listing","split":"dev","html_sha1":"bf1cd2ee52a78fca4ebfe09bd69e1a54a1851a8b","truth_sha1":"cff6fdbebca940c805eaaa9de26479d327194f40","overlaps_test":false,"full_text_reference":true}, +{"id":"4348","category":"listing","split":"dev","html_sha1":"2bbf412fa285f3b6724d6d680b0c7bb8f78d39f5","truth_sha1":"b3af81517629306c62cd5d18f3171c8229ecbbfc","overlaps_test":false,"full_text_reference":true}, +{"id":"4349","category":"article","split":"dev","html_sha1":"2162acf111e494e23107a25d4d52a43f04472a38","truth_sha1":"c1057421d84dad87db9509c1dfe280b099c74a38","overlaps_test":false,"full_text_reference":true}, +{"id":"4351","category":"collection","split":"dev","html_sha1":"c144e73ec273097d6924272ceb4ac6317b47631d","truth_sha1":"004bce01f7ab78d7f9c5389529c00a7e325d27d9","overlaps_test":true,"full_text_reference":true}, +{"id":"4354","category":"listing","split":"dev","html_sha1":"43eda13a580d60789f1c8540f7be55d49cae240e","truth_sha1":"9226df66c1a0fe13d1b0d28b2da25c1d332d15ea","overlaps_test":false,"full_text_reference":true}, +{"id":"4355","category":"listing","split":"dev","html_sha1":"9c2710926b815b2e6b8e0f9fff3afc83f7ebda8a","truth_sha1":"09b10e9b0bab897a04f42aade43d11f3a13bd687","overlaps_test":false,"full_text_reference":true}, +{"id":"4359","category":"listing","split":"dev","html_sha1":"5372bc5ba92c5d0f323625976c25285a9139df01","truth_sha1":"c91391e241730b692038fe700caf90850b81005b","overlaps_test":false,"full_text_reference":true}, +{"id":"4360","category":"collection","split":"dev","html_sha1":"3b9ee27b6200bc66689f723fa70addb4728d59a9","truth_sha1":"64553111227fec4ce89598cab333a0659652f9bf","overlaps_test":true,"full_text_reference":true}, +{"id":"4361","category":"listing","split":"dev","html_sha1":"947df1414799d07bb43c9e4b4aaeaaea65b4efb4","truth_sha1":"288af3afdc59f31846f0abf45fcf15a3469e65a4","overlaps_test":false,"full_text_reference":true}, +{"id":"4362","category":"article","split":"dev","html_sha1":"775ea6805a919eca8bb2a19e7f330e05c66e1842","truth_sha1":"6bb759d9efd3ffee34b5e32dfce178305bd7998c","overlaps_test":false,"full_text_reference":true}, +{"id":"4363","category":"article","split":"dev","html_sha1":"205d2fb8192d78c5f106ac385445c20e4cc3ba88","truth_sha1":"aa7f2b1b6e778cfa00fbc6f9d2a9d507cd063521","overlaps_test":false,"full_text_reference":true}, +{"id":"4365","category":"article","split":"dev","html_sha1":"28cac3d88f4c02e209fd9518aa764e725bbe8fd1","truth_sha1":"607bdd8758e210b5d0d1c632158e03f53fdc51c6","overlaps_test":false,"full_text_reference":true}, +{"id":"4366","category":"article","split":"dev","html_sha1":"f7e7b1749219cbc7500748bb2a1661e42c58a139","truth_sha1":"6bf1f56a5c4a5a81d0987e4deb8e9956fc953d6c","overlaps_test":false,"full_text_reference":true}, +{"id":"4367","category":"article","split":"dev","html_sha1":"6fd49ddc79621c09b609d168c69f1ec3681235a4","truth_sha1":"3b3140f90a70cb2f567dd61d2f0fa81e83cc6dce","overlaps_test":false,"full_text_reference":true}, +{"id":"4368","category":"article","split":"dev","html_sha1":"135984f859474753fddba71e527c9ce857b0b02b","truth_sha1":"339f2c15064de96449c704c06c15a998ec592808","overlaps_test":true,"full_text_reference":true}, +{"id":"4369","category":"collection","split":"dev","html_sha1":"dc0205223d753ad9971967cbcbfc000def434235","truth_sha1":"f29e5f791a036a645456aad675a0297fc1364109","overlaps_test":true,"full_text_reference":true}, +{"id":"4370","category":"article","split":"dev","html_sha1":"8ac7d8d0a1d9c1d0576639151127017bcb053217","truth_sha1":"9967cb3fe92a347423837eba7cdefd57317b81f6","overlaps_test":true,"full_text_reference":true}, +{"id":"4371","category":"article","split":"dev","html_sha1":"95c42d74778c4eef5f33a2271bf2f32fd0929c8d","truth_sha1":"c7d56c8b52cfaaf4bdec47606a53c81ec185adbb","overlaps_test":true,"full_text_reference":true}, +{"id":"4372","category":"article","split":"dev","html_sha1":"ebae1772e727b8fc4ce69844253cfa7df66f73f1","truth_sha1":"6b095cb949f63aa7a55c60be23fdf3760bd602d3","overlaps_test":true,"full_text_reference":true}, +{"id":"4373","category":"article","split":"dev","html_sha1":"c4edb0935b027719d4bc5e04e3e2e9d7e258ad8b","truth_sha1":"4f3775f6a0788072fe186b27636c70387826626b","overlaps_test":true,"full_text_reference":true}, +{"id":"4375","category":"collection","split":"dev","html_sha1":"18785b0ced678cbb82f395e0194ba6e2d13a74b0","truth_sha1":"ba42ab46f927542b47b08e08f62ea046c132ddf8","overlaps_test":true,"full_text_reference":true}, +{"id":"4376","category":"collection","split":"dev","html_sha1":"7a1907cf2d3a644c255a136d17e7821137885a04","truth_sha1":"c29630d74e7f161dbdae6bd698fcb94a8085a31b","overlaps_test":true,"full_text_reference":true}, +{"id":"4377","category":"article","split":"dev","html_sha1":"98c0029cb501c8a2d13f60cf5da5767e5e7886fa","truth_sha1":"8244df18068bd350575a48df6993a6f0b46c6327","overlaps_test":false,"full_text_reference":true}, +{"id":"4378","category":"listing","split":"dev","html_sha1":"41c7f3d48a34902a0dc1b5e0aba1492c61eb1ddd","truth_sha1":"a91f2c954fe22999b0ceefdfa73bc810fe2bb39f","overlaps_test":false,"full_text_reference":true}, +{"id":"4379","category":"collection","split":"dev","html_sha1":"13c0df4164da4a2069fbd8fa4bd63c982f00dff1","truth_sha1":"b40f4162226190983167d2fe72c0330a1d0781f5","overlaps_test":false,"full_text_reference":true}, +{"id":"4380","category":"listing","split":"dev","html_sha1":"b240c6f96df5e4404b4c9d4342b0d7f9a0187907","truth_sha1":"0214ce30db0f5fdb98c997833cb028a9cb13466c","overlaps_test":false,"full_text_reference":true}, +{"id":"4381","category":"article","split":"dev","html_sha1":"01bdd0060b64494731dcf5fcd40a91de84047594","truth_sha1":"833786d6273d623e985d9d8899e49dcf10f93609","overlaps_test":true,"full_text_reference":true}, +{"id":"4394","category":"collection","split":"dev","html_sha1":"68bf93c7e0d8e21d0f9a43696dc7693e30371646","truth_sha1":"38d7d5dc438a2bcfad8208227483fc904fc7c268","overlaps_test":true,"full_text_reference":true}, +{"id":"4397","category":"documentation","split":"dev","html_sha1":"bbe1f935cbd95613a75e14369c8181d38199e33d","truth_sha1":"518adce91d7237abe90e6e3e80a3d6feaab4bce2","overlaps_test":false,"full_text_reference":true}, +{"id":"4398","category":"documentation","split":"dev","html_sha1":"47fecd24e8d86ba2a924a4bf07cbcd155cc1e7ff","truth_sha1":"4e53888febe5784cc67cba1cf9374234f5fb8638","overlaps_test":false,"full_text_reference":true}, +{"id":"4400","category":"documentation","split":"dev","html_sha1":"38689f44b506767c1e0bf453ac7231f3d420cf15","truth_sha1":"db1ee46dc2e19f9190e79b3cde970b917cbffd4f","overlaps_test":true,"full_text_reference":true}, +{"id":"4401","category":"documentation","split":"dev","html_sha1":"4e4d1063a3d215e30f7daf97aaef5d28f2c32340","truth_sha1":"4f3cb48fe0e2d6da65f65eec0c18b2531c25517e","overlaps_test":false,"full_text_reference":true}, +{"id":"4402","category":"documentation","split":"dev","html_sha1":"833ecf1ed28c1f5621e48edb0c58a67c17859eee","truth_sha1":"28eacee73840872ed8326eabb09636a776bcbd55","overlaps_test":false,"full_text_reference":true}, +{"id":"4403","category":"documentation","split":"dev","html_sha1":"02c967f475116881e11caba52c798265ad262020","truth_sha1":"56546cca3c0fa3bfd4afc7553b47111a88c1b952","overlaps_test":false,"full_text_reference":true}, +{"id":"4407","category":"documentation","split":"dev","html_sha1":"bb42a224d09423314c7f9d1c406a2cbd73a77372","truth_sha1":"1bb92088e78feb19c57cd609fcca2135e23c0727","overlaps_test":false,"full_text_reference":true}, +{"id":"4408","category":"documentation","split":"dev","html_sha1":"45a9f6bbc6fba15e8bcca33d9c4edfb8e4b97fe8","truth_sha1":"ab315ad835dfaa46dc089478daa68a9b8df588ab","overlaps_test":true,"full_text_reference":true}, +{"id":"4409","category":"documentation","split":"dev","html_sha1":"9102b14d62882b1c0747c84183805db275617b27","truth_sha1":"d23734e8fd75fba280919f62a7ce29316cae01f3","overlaps_test":false,"full_text_reference":true}, +{"id":"4410","category":"collection","split":"dev","html_sha1":"eb6b4caec32a7cb7276762dc8053aeb1a9b07960","truth_sha1":"0b21e45364a89d79e89c6ea54ea198991bbf5ef9","overlaps_test":true,"full_text_reference":true}, +{"id":"4411","category":"documentation","split":"dev","html_sha1":"ee335442a30dcf332e1f3e22b85e3ec9f95511cf","truth_sha1":"7565e2e884de80013c55affbce0342c0f0edc936","overlaps_test":false,"full_text_reference":true}, +{"id":"4412","category":"documentation","split":"dev","html_sha1":"22ab4cac4ea78cd4141222be3eedfe79b26ea516","truth_sha1":"ab6c15d205f9be22ebb8725a920c7ef103c7cf69","overlaps_test":false,"full_text_reference":true}, +{"id":"4413","category":"article","split":"dev","html_sha1":"e3d95dcd9b5bad785b48ad45b1389c42747122f6","truth_sha1":"a431802aa4cb9ac2262724fdd6e2cabb40dbd57c","overlaps_test":false,"full_text_reference":true}, +{"id":"4414","category":"article","split":"dev","html_sha1":"b634083534ac305c72f7bcc7a7124c06d43229f3","truth_sha1":"ed03fa58c07625e50f52786b9d4500fb276fde28","overlaps_test":false,"full_text_reference":true}, +{"id":"4415","category":"documentation","split":"dev","html_sha1":"3dc5058be2ef9b95368774abbceb18ee0bb818b2","truth_sha1":"c390345ac5b9e908a9520cf535266254f5f54321","overlaps_test":false,"full_text_reference":true}, +{"id":"4416","category":"article","split":"dev","html_sha1":"8827f92c222061369505e21bcbeb8c8dcdd5c245","truth_sha1":"7befb845ac536efb4e9f8e42c6564b5574837058","overlaps_test":false,"full_text_reference":true}, +{"id":"4418","category":"collection","split":"dev","html_sha1":"ef5c40fbabd430fcdc2deb7fcd199941d3c08948","truth_sha1":"7ef07cf4f40ce03b45a9dd9ccd50f38f8b9296e6","overlaps_test":true,"full_text_reference":true}, +{"id":"4419","category":"collection","split":"dev","html_sha1":"c502bda65c953a14d49e47857bf578aec47d333a","truth_sha1":"533c9a3c814fdd48e3185bbd51ae2a61257ad6b0","overlaps_test":true,"full_text_reference":true}, +{"id":"4420","category":"article","split":"dev","html_sha1":"a1b43137029ba8cd27ffbde54de753bfbccdea19","truth_sha1":"c24412051378cb3decc8f51b31b28eecfc29001c","overlaps_test":false,"full_text_reference":true}, +{"id":"4427","category":"article","split":"dev","html_sha1":"9cd8552a05a4608c7181a56f617081e77d547892","truth_sha1":"d49208fca3aa78684910b105db651780887be21e","overlaps_test":false,"full_text_reference":true}, +{"id":"4430","category":"article","split":"dev","html_sha1":"ad076859d2e964033655a8e777fecffd55408c40","truth_sha1":"6be61d98143bb97151141cf04549bdc6184f9a8c","overlaps_test":false,"full_text_reference":true}, +{"id":"4431","category":"article","split":"dev","html_sha1":"c456a6efc37f09c021f47781d5bde356e9f6b866","truth_sha1":"609aa70c90a79f921b589c980deb4860b67d591a","overlaps_test":false,"full_text_reference":true}, +{"id":"4433","category":"article","split":"dev","html_sha1":"66e2ab77f01d2ebdcae468ff719c9a65e62e00da","truth_sha1":"86816e5d56c6c3d16c7625a3613c14c2dbde6a3b","overlaps_test":false,"full_text_reference":true}, +{"id":"4435","category":"collection","split":"dev","html_sha1":"45438501f10e448306dfa0b114fc01728cb0d803","truth_sha1":"3a77ebcc7aea170f37b2cb2b0e02207cb3a5475e","overlaps_test":false,"full_text_reference":true}, +{"id":"4437","category":"collection","split":"dev","html_sha1":"c53e3b055ea439df97f951c70d13edd0c03052a9","truth_sha1":"2b4902483f90a4d91dcdb605218e9c713f5a6aa1","overlaps_test":true,"full_text_reference":true}, +{"id":"4438","category":"collection","split":"dev","html_sha1":"3092ec59888e15ce27266a1c85fe888aeeed5622","truth_sha1":"17075e204c50fc31e023fd43eceb2a96c27331ec","overlaps_test":false,"full_text_reference":true}, +{"id":"4439","category":"article","split":"dev","html_sha1":"51ba65a5bfde10072feb7c5ade674648813d9870","truth_sha1":"fbd5335e25e7822a05b626c917b649d17a582151","overlaps_test":false,"full_text_reference":true}, +{"id":"4440","category":"article","split":"dev","html_sha1":"83ad29374b18e90caf9714a3ec5778e33fc146a6","truth_sha1":"dc7433fd41e0acce9ec3fa4638b07e0c048d0885","overlaps_test":true,"full_text_reference":true}, +{"id":"4441","category":"article","split":"dev","html_sha1":"cfd69079f065b61f5aaaaa2bdcab34c8fde601a9","truth_sha1":"9293df4a273a54c7ee0c0f6e06e0cd65c65431a3","overlaps_test":false,"full_text_reference":true}, +{"id":"4442","category":"article","split":"dev","html_sha1":"6301f83c0c0b671598d4572d2263af3610c7a160","truth_sha1":"15091f41e5a7c4c23c2f3276c6a89431b5b3cef0","overlaps_test":true,"full_text_reference":true}, +{"id":"4443","category":"article","split":"dev","html_sha1":"577d14e7e8b725aa3d7329e12df406d1448b236c","truth_sha1":"299761404598d7df4678ea417be9ecddcc194637","overlaps_test":true,"full_text_reference":true}, +{"id":"4444","category":"article","split":"dev","html_sha1":"cc66e65bf0da15b631d28684ac87a8c09f7637f4","truth_sha1":"3a48c4b0a4fe9e614e2ffad6745d1fc6dbb863cb","overlaps_test":false,"full_text_reference":true}, +{"id":"4445","category":"article","split":"dev","html_sha1":"5e2b60d8e5e3d6473194a4847575052f36d042fc","truth_sha1":"51c2f1042278ba37a66f6920b78201a9b2d484f2","overlaps_test":false,"full_text_reference":true}, +{"id":"4446","category":"article","split":"dev","html_sha1":"4cccc4e11dcd88d6d021bce8306222120e7cc5a2","truth_sha1":"0003e6ce61de1c596f4f59ae044b1afa68a79f77","overlaps_test":false,"full_text_reference":true}, +{"id":"4449","category":"article","split":"dev","html_sha1":"db941deef3075899521564e77808016714d8c9f3","truth_sha1":"fcedc33644885e309ff310d308e8a00c95b78246","overlaps_test":true,"full_text_reference":true}, +{"id":"4450","category":"article","split":"dev","html_sha1":"b92800b49213254ee1528420f42b51f4f6033465","truth_sha1":"e9e37c44166ffd760ad98f5d094ad5f5b35311d8","overlaps_test":false,"full_text_reference":true}, +{"id":"4451","category":"article","split":"dev","html_sha1":"a31e222da309b2799e9bb46b69638cc5351416eb","truth_sha1":"2ffd7fbd697a9ac948598b4a0b2c06159b62b09d","overlaps_test":false,"full_text_reference":true}, +{"id":"4452","category":"article","split":"dev","html_sha1":"8fba7c03c7bb8deeea905968700a62f07254a43d","truth_sha1":"6bfb53e192495d0d700276c38bc9205bebe01cbf","overlaps_test":false,"full_text_reference":true}, +{"id":"4454","category":"article","split":"dev","html_sha1":"55161c6295c21e4fc9fdc47e3dca0b5c279fe0b0","truth_sha1":"28f129f4bde56d1364a94145ed7913c7a2baa5c5","overlaps_test":false,"full_text_reference":true}, +{"id":"4456","category":"article","split":"dev","html_sha1":"9a56d50a0973996dd9df69570f7494436f32685c","truth_sha1":"81eec10a6a177c772688bf9e3d0eff6b749e27b4","overlaps_test":true,"full_text_reference":true}, +{"id":"4457","category":"article","split":"dev","html_sha1":"f733a8a821b2dcbc373f95127ce467a0e003f474","truth_sha1":"36c1c68e902ad0ddb16b0dddaa746c477439ad02","overlaps_test":false,"full_text_reference":true}, +{"id":"4458","category":"article","split":"dev","html_sha1":"db79c24501924d36cd154a8fadccd1f447e1732f","truth_sha1":"773efdb6f9d10323471251e8404a0e7a58ab264f","overlaps_test":false,"full_text_reference":true}, +{"id":"4459","category":"article","split":"dev","html_sha1":"620c17139ee1c9607103b020bbd17a0d54b44eee","truth_sha1":"3a6f22a0a5e90c598b3bcd9ba4d1e4acb4461c82","overlaps_test":false,"full_text_reference":true}, +{"id":"4460","category":"article","split":"dev","html_sha1":"634310645344fd313454e4686f409a3f0fe02136","truth_sha1":"fd8a3f3181aa4245ad5a584c9e5dad4a6ec88578","overlaps_test":false,"full_text_reference":true}, +{"id":"4461","category":"collection","split":"dev","html_sha1":"144dc1c86b7e483187be34ff440fc1b1a561ce0d","truth_sha1":"a717cefaac584e95ff33d47ac209fbd0d006277f","overlaps_test":true,"full_text_reference":true}, +{"id":"4462","category":"article","split":"dev","html_sha1":"442a85abcf5eee890a666fe913320895c4615ebb","truth_sha1":"97b8469d5f2a0f12d406c92d75156817838ea4d1","overlaps_test":false,"full_text_reference":true}, +{"id":"4464","category":"service","split":"dev","html_sha1":"9a76a06c335c3da82e436805b40283c270b12f19","truth_sha1":"00fee285e53c85ce04a80d6361cd63893e84b7ea","overlaps_test":false,"full_text_reference":true}, +{"id":"4465","category":"collection","split":"dev","html_sha1":"84ba1f47ed08f2fb843c0e8bb4a949e94c390734","truth_sha1":"cb9f6ea8a6fe350e89d75b4b08487d2ab9e859b1","overlaps_test":true,"full_text_reference":true}, +{"id":"4467","category":"service","split":"dev","html_sha1":"720143ab0e90f59463db00bfb4aea695d79ce6a1","truth_sha1":"ac12a71a414532cd8f2802bafc9203c80febda7d","overlaps_test":false,"full_text_reference":true}, +{"id":"4468","category":"collection","split":"dev","html_sha1":"33623eb0d15b1d5b61fbf8b026462ad06de82fd9","truth_sha1":"a3d23cc334aa5701681af64526c9ff9417a26ef2","overlaps_test":false,"full_text_reference":true}, +{"id":"4470","category":"collection","split":"dev","html_sha1":"930f3b94f5d5fc9ed3c0e171ea6ebfea955370df","truth_sha1":"7385d06d12e4b094e62b7bfc848826419b5dadf3","overlaps_test":false,"full_text_reference":true}, +{"id":"4471","category":"service","split":"dev","html_sha1":"bbd13bbf00c3d5a43230a5413693f7a1772244e8","truth_sha1":"40ee3025e34bf5b922c9cd2a7e7e64b81cfc4507","overlaps_test":false,"full_text_reference":true}, +{"id":"4472","category":"article","split":"dev","html_sha1":"021bd74b4b640b5a4a37614adc3b2496c7e600d1","truth_sha1":"2e1a0501e2baf2ddf59db60bcd108d52009cfc45","overlaps_test":false,"full_text_reference":true}, +{"id":"4474","category":"service","split":"dev","html_sha1":"470d8a12b8addf4c56d938faf94040d168ee701a","truth_sha1":"32c0feba686358a268af4e5ae9eb0085385d80ad","overlaps_test":false,"full_text_reference":true}, +{"id":"4475","category":"article","split":"dev","html_sha1":"6336d41488128bc4a47e61b7b1aea8ce26d04878","truth_sha1":"27ec00b267b3838e7166b76abeb58ab157cb9130","overlaps_test":false,"full_text_reference":true}, +{"id":"4476","category":"article","split":"dev","html_sha1":"bb03d888f3ee7e5ec8f985f84d0364aeb7cd4e17","truth_sha1":"def43fa0af182e025bb5e970ff28731c1ddfc0cc","overlaps_test":false,"full_text_reference":true}, +{"id":"4477","category":"service","split":"dev","html_sha1":"92df08f88b000b35aaadc81df46c6dfcb8835094","truth_sha1":"9589e2bc7588b9d6fc86e21954b7877ee194153c","overlaps_test":false,"full_text_reference":true}, +{"id":"4478","category":"article","split":"dev","html_sha1":"f943b60bf0ebdc7c57ebed7f5ece981235b706a6","truth_sha1":"99876d3a08597f6f82a8647515b63824718f744c","overlaps_test":true,"full_text_reference":true}, +{"id":"4479","category":"service","split":"dev","html_sha1":"8f618fe03feac8c4b01d7d90e3b3e9e61d454164","truth_sha1":"6ae2672afb0fd0718cb45e254c6b13a478e1cfba","overlaps_test":false,"full_text_reference":true}, +{"id":"4480","category":"service","split":"dev","html_sha1":"c132435817270439a20a6bd918b3ff1eeabe4e9e","truth_sha1":"4f1618dc2fefb51b5f016a7fa5ecc4087c2dbb8f","overlaps_test":false,"full_text_reference":true}, +{"id":"4485","category":"service","split":"dev","html_sha1":"687b1d096548b9b7c3b9cc486008538f2586fc5f","truth_sha1":"43b23a200ef8e638a803bd2c80abb77bc4a41d88","overlaps_test":false,"full_text_reference":true}, +{"id":"4486","category":"article","split":"dev","html_sha1":"9a0ae25e83d0f33676af7a26bc6a0957a3aac46b","truth_sha1":"40b241439d9370a5fee3eda2a8c28e494325d43f","overlaps_test":true,"full_text_reference":true}, +{"id":"4487","category":"article","split":"dev","html_sha1":"3a8a4505426571eee554124e0db133dcdbb7fb8d","truth_sha1":"d5608bc9de7b9427075fda40d1852738e2b19563","overlaps_test":true,"full_text_reference":true}, +{"id":"4492","category":"forum","split":"dev","html_sha1":"a9402ef3c35a9eeb31718ddf84eb6fc4621bb804","truth_sha1":"a80beb4c04e02d9a11d68ee5b13cebeea4ca588a","overlaps_test":false,"full_text_reference":true}, +{"id":"4495","category":"collection","split":"dev","html_sha1":"6d8cdbb44f81fb559d1a89579a9e0be2d5134b25","truth_sha1":"200f21f557ab0aeee36c3f7cbd6bcbe02ee5b7ed","overlaps_test":true,"full_text_reference":true}, +{"id":"4497","category":"article","split":"dev","html_sha1":"eda611c5b2bf12cf317d84181e32e054a25bfea5","truth_sha1":"fda3555fa7fe3e75d70146c7f0e6108391209497","overlaps_test":false,"full_text_reference":true}, +{"id":"4498","category":"article","split":"dev","html_sha1":"0d4941b0da350138c6c9f8c37f6a3009d94b83bf","truth_sha1":"f47a1cde58ff0b87f3e4a3f1acbd337181ceb1f2","overlaps_test":true,"full_text_reference":true}, +{"id":"4499","category":"article","split":"dev","html_sha1":"791daba1ea343b2ba79da795e0f505f09061a839","truth_sha1":"edab6183a9ad8ce2aa98842248f3979a5706ad42","overlaps_test":true,"full_text_reference":true}, +{"id":"4500","category":"article","split":"dev","html_sha1":"6a25f5e66d529d03adb75196cae05319e26be3f3","truth_sha1":"b89b270f3b8f83d8e5b521ea2cdb7ead910747e8","overlaps_test":false,"full_text_reference":true}, +{"id":"4503","category":"article","split":"dev","html_sha1":"16a8e0fca8dc9c877470460a0631b217d9e407de","truth_sha1":"56f39c26ab2e2cd6502cb30b72d892dd520e7721","overlaps_test":false,"full_text_reference":true}, +{"id":"4506","category":"article","split":"dev","html_sha1":"3a641a2776323e936dfde604fc1d26aa99354043","truth_sha1":"af058c4dbdf8ed0ed41fef71c480633e1c5bffe6","overlaps_test":false,"full_text_reference":true}, +{"id":"4508","category":"listing","split":"dev","html_sha1":"347b3f7fc5fc96d9ae0e29f91f27e016dd2357c9","truth_sha1":"1845642cc056dc218dfa4a70eea3776262fce3b1","overlaps_test":false,"full_text_reference":true}, +{"id":"4509","category":"collection","split":"dev","html_sha1":"b4c3ef1fb472f9a698e0473806522e65e7675762","truth_sha1":"5609d565385d58251c442cb35fba4b78583e91ab","overlaps_test":false,"full_text_reference":true}, +{"id":"4510","category":"collection","split":"dev","html_sha1":"67d60a4fcfc89e9975a8bd86311da1717c4c3e2d","truth_sha1":"9f150ed9cd62cf6a5cd2fd643f730c76cf8113d6","overlaps_test":false,"full_text_reference":true}, +{"id":"4511","category":"article","split":"dev","html_sha1":"bdfe4a98f1b835b752fdc36adec4e1edd633ab81","truth_sha1":"d77ab5d7e1a27e5ed766114fe22ce23644d474da","overlaps_test":false,"full_text_reference":true}, +{"id":"4512","category":"article","split":"dev","html_sha1":"33b497625a65d099132c2af939c6451402c754ef","truth_sha1":"c22b0f4d08fc5c971d8412c4a010ff10bc20207b","overlaps_test":false,"full_text_reference":true}, +{"id":"4515","category":"article","split":"dev","html_sha1":"32e340267b8cb2316cbbfc092b9932f8941fd588","truth_sha1":"8baff41198658b2c8c00c7bba3e75c531242cf5f","overlaps_test":false,"full_text_reference":true}, +{"id":"4517","category":"article","split":"dev","html_sha1":"98e6654c2e6b8638b7f9f2e91b71fa4af1ecd23a","truth_sha1":"7895c15b7917ee74eb502db0c0b21e180e16ecbd","overlaps_test":false,"full_text_reference":true}, +{"id":"4518","category":"collection","split":"dev","html_sha1":"def97cb87a11ad628edb34ec911f3147d8008937","truth_sha1":"2a6604f3e641703c1e9d45f22b9dc4744d604c7e","overlaps_test":false,"full_text_reference":true}, +{"id":"4519","category":"article","split":"dev","html_sha1":"ae74077d9a488d2c189fd3d42ba560f379e64edc","truth_sha1":"14e74efdfd40e0d47a6d2fa9ce8afb6defd66c44","overlaps_test":false,"full_text_reference":true}, +{"id":"4523","category":"article","split":"dev","html_sha1":"395d111ff30900819ce2e1b712af8aa69203eac0","truth_sha1":"43dff461d6fc47d2f987869ed562bfb15c766403","overlaps_test":false,"full_text_reference":true}, +{"id":"4524","category":"article","split":"dev","html_sha1":"d830acb4e4c41918ba2faf672ad2241d66c9621b","truth_sha1":"cb0367743266d20c4a211c427e37b455e7a34236","overlaps_test":false,"full_text_reference":true}, +{"id":"4526","category":"article","split":"dev","html_sha1":"44b2fc0a5651f4b9f7ff46fe1f8e8071287194c6","truth_sha1":"ccaef062bee003791ee97d844c87201176493345","overlaps_test":false,"full_text_reference":true}, +{"id":"4527","category":"article","split":"dev","html_sha1":"bdcf453ca75eea2acdfe9667c18a8b10cb384ec2","truth_sha1":"af38e6f4f31781f97fbcfc2255ee578b5e814d53","overlaps_test":false,"full_text_reference":true}, +{"id":"4530","category":"article","split":"dev","html_sha1":"a9942672a6bebeb125b8cd360e765ee744ebb378","truth_sha1":"9595ead9ad917ad582850f6564fb99c0f00a5b5b","overlaps_test":false,"full_text_reference":true}, +{"id":"4531","category":"article","split":"dev","html_sha1":"564a7186eff093c68005c4e48bcbb991d2041425","truth_sha1":"b05fe27ca54a312fe9ecc79610a0c2c2fcc075c9","overlaps_test":false,"full_text_reference":true}, +{"id":"4532","category":"listing","split":"dev","html_sha1":"4e4b3aac3b8556b0e1ec5c6af94b6263cd6485d4","truth_sha1":"6b996584d43a9fda80c2788022499d68c6c1a701","overlaps_test":false,"full_text_reference":true}, +{"id":"4533","category":"listing","split":"dev","html_sha1":"23173d338c7609f38fc9df3c58d8b8d21a44f2b1","truth_sha1":"0e06c631a57d7321fe0406c67f79ca921b35816a","overlaps_test":false,"full_text_reference":true}, +{"id":"4537","category":"article","split":"dev","html_sha1":"fd298584c83f2bf0eaaa252ef41fe5a5bb70d5e0","truth_sha1":"a9caf0efc457877ed159abd93cf9ec6d931e8f89","overlaps_test":false,"full_text_reference":true}, +{"id":"4538","category":"article","split":"dev","html_sha1":"13b3823e0e01f9b857290dc9da216f1b8dce6358","truth_sha1":"9443663d4d489befc66d2377e2ba50cc8289e29e","overlaps_test":true,"full_text_reference":true}, +{"id":"4540","category":"article","split":"dev","html_sha1":"b22cea113915e21750dc206e53b9649126b8d26c","truth_sha1":"e47480a2bfb6771d5120b560ffdb03c4236d2045","overlaps_test":false,"full_text_reference":true}, +{"id":"4544","category":"collection","split":"dev","html_sha1":"59b5064373595b7eafd126d7c8e22f28fb0939ab","truth_sha1":"6516f042e6214b489cf94ffc89f4b37951528c05","overlaps_test":false,"full_text_reference":true}, +{"id":"4547","category":"article","split":"dev","html_sha1":"421a8283f2da8f2ca12f72d84fa603afaf8c77f8","truth_sha1":"ae35730874504539f560b537ed2dc069054d9bb3","overlaps_test":true,"full_text_reference":true}, +{"id":"4548","category":"listing","split":"dev","html_sha1":"63538549fa218b9fd5f3bc03af03f53f4ab247f7","truth_sha1":"989abc1cc0e8f8342c02e03dd5163b42a73e530f","overlaps_test":false,"full_text_reference":true}, +{"id":"4549","category":"article","split":"dev","html_sha1":"dc34653baa6d4f4bba72d5ecfcf6a8ff809c3fbe","truth_sha1":"cd316afbe6903afeeac29e37c70a285caf431de0","overlaps_test":true,"full_text_reference":true}, +{"id":"4550","category":"service","split":"dev","html_sha1":"cba8cd8dc2f30e4cd683093019ba463c6ddb3348","truth_sha1":"34720ca44e900cef5ed6d393d7a4f1d469f3f820","overlaps_test":false,"full_text_reference":true}, +{"id":"4552","category":"service","split":"dev","html_sha1":"64d6157109c7e2aca9fbea32d0ff762bd6bf0da3","truth_sha1":"ac0b0dc9fc356389e041fd2e663cdf9ed0a033d8","overlaps_test":false,"full_text_reference":true}, +{"id":"4553","category":"service","split":"dev","html_sha1":"6f4deb995c2886b3f0641923b3ecb44e31d9eeb3","truth_sha1":"903b30fc1b75df2bbb8105e956f696e2a03c3786","overlaps_test":false,"full_text_reference":true}, +{"id":"4554","category":"service","split":"dev","html_sha1":"251ff5cbaac96d2ac7adad3f3544fd20b5d229ed","truth_sha1":"5774aadb018e0aa3316664f53523b867b2a58913","overlaps_test":false,"full_text_reference":true}, +{"id":"4555","category":"article","split":"dev","html_sha1":"60eb3526a860eb55425a394c5fa507a46cdc16dc","truth_sha1":"b206e74c1ec95e58268e44006ff37325be645fbc","overlaps_test":false,"full_text_reference":true}, +{"id":"4557","category":"article","split":"dev","html_sha1":"b85931a58293047941b97b8a56a08fbbef3d69cb","truth_sha1":"97981e4adf8ef90342b53f0366341623b12121ec","overlaps_test":false,"full_text_reference":true}, +{"id":"4558","category":"article","split":"dev","html_sha1":"8c7ee0350b49b99bba1f1e4bb7ec3a8039d95952","truth_sha1":"9f80dc3dce36deb4732885e38e5ab12ce4e79946","overlaps_test":false,"full_text_reference":true}, +{"id":"4560","category":"article","split":"dev","html_sha1":"2487c8b1aa67de953c576f97aebbabca3f6697b3","truth_sha1":"16c12507bf4e8c90889a985891530c1827c6150b","overlaps_test":false,"full_text_reference":true}, +{"id":"4561","category":"article","split":"dev","html_sha1":"458bf2a5bca23066b9b4fbabe1ced8d7bbcf368a","truth_sha1":"ceccd7eb0359bd0f42b9009b5ee97251c0360d76","overlaps_test":false,"full_text_reference":true}, +{"id":"4562","category":"article","split":"dev","html_sha1":"befc4141ca200af092642765498b4d911d369821","truth_sha1":"558b22390ef61f3067bdf112cc2eea5249d90fe9","overlaps_test":true,"full_text_reference":true}, +{"id":"4563","category":"article","split":"dev","html_sha1":"a2efc648ede6beb5a8f9a3e2a98308b3035e3a74","truth_sha1":"7b6e69d0f36f8e1b667854613c03a378e5dad136","overlaps_test":false,"full_text_reference":true}, +{"id":"4564","category":"forum","split":"dev","html_sha1":"2bb99225531b9da4a07f82f3f4179296648f70c1","truth_sha1":"6769c8c31156f206962f64d376bb1a0aecf73cc9","overlaps_test":false,"full_text_reference":true}, +{"id":"4565","category":"article","split":"dev","html_sha1":"da4d7f34cc1e315cf74e16cde0dbd2d1f3ee55de","truth_sha1":"224b12e581ccd1f516c7f363ff1e547ebd6bddf2","overlaps_test":false,"full_text_reference":true}, +{"id":"4570","category":"article","split":"dev","html_sha1":"f17a544603f9260dc3f9669bf499b4d41f8a003e","truth_sha1":"01af1e2ac195610fe07d22d204aacc0130f3a6bd","overlaps_test":true,"full_text_reference":true}, +{"id":"4572","category":"article","split":"dev","html_sha1":"c4b502e2f6141f4c4f752a5413655144ee2f383e","truth_sha1":"f8609632c475092018c78e6ee89dd6d9cfae0b3e","overlaps_test":false,"full_text_reference":true}, +{"id":"4574","category":"article","split":"dev","html_sha1":"8dc32db17b48bc090bdfa9507090eb039abf3b94","truth_sha1":"85677719b2a817db08807ebc8ddaae67a588e522","overlaps_test":false,"full_text_reference":true}, +{"id":"4576","category":"article","split":"dev","html_sha1":"2c5476b3bf888afa61797170e27bf7b1cccf2fb7","truth_sha1":"d30d00acac840feb0746dfb4b1f0aa0278f58a64","overlaps_test":false,"full_text_reference":true}, +{"id":"4577","category":"collection","split":"dev","html_sha1":"8d0c7a9c692f505369ca30181a7bd589e4fdc358","truth_sha1":"7779d1e14ddcf46cf5ba4b798b2ebac88e7ae819","overlaps_test":false,"full_text_reference":false}, +{"id":"4578","category":"article","split":"dev","html_sha1":"10aec6259faf5d14ae4c6845829cee12ecf9a8f5","truth_sha1":"de74214ab21a7eb903cb8b14f79d71b17f92a4d2","overlaps_test":false,"full_text_reference":true}, +{"id":"4579","category":"article","split":"dev","html_sha1":"85d69aacf6310ec298812ea54bc0bf3979e70ba9","truth_sha1":"5ccf1fd138fa2b7306a169face0d309a5336f70d","overlaps_test":true,"full_text_reference":true}, +{"id":"4580","category":"article","split":"dev","html_sha1":"60ef2dc8213a248dbb776199a20497e8c1616257","truth_sha1":"f8faf103c60bbc6e9820677557b9a1d2665b6242","overlaps_test":false,"full_text_reference":true}, +{"id":"4581","category":"article","split":"dev","html_sha1":"d0e6e7cb34de142ef881ef45019192c1e62117b6","truth_sha1":"ce00b0c356449307b0d63dc59d5ad400de5dce43","overlaps_test":false,"full_text_reference":true}, +{"id":"4582","category":"article","split":"dev","html_sha1":"c58b2af4ae19e8e223daf062741d6da3cd38134b","truth_sha1":"0c5c8a9da0b6e5d640c09dea59ae084b697c06e8","overlaps_test":false,"full_text_reference":true}, +{"id":"4583","category":"article","split":"dev","html_sha1":"e8fb2278e01e0eebe1bbd808f1bedbdf83c50d58","truth_sha1":"49f8b2f54a1a119d3596156a5772b1afdfd727a9","overlaps_test":false,"full_text_reference":true}, +{"id":"4584","category":"article","split":"dev","html_sha1":"4e4d1d406fe8f4a10b5d5259446cfe740597274a","truth_sha1":"ea8461d7215333b769a7946e8ef1aa9cef9c3a71","overlaps_test":false,"full_text_reference":true}, +{"id":"4587","category":"article","split":"dev","html_sha1":"bef2b03b4d8c3e95719fc6fa8d9d08e3c36560d9","truth_sha1":"fa6e248787450139926e3487f9b34f8ac96bf242","overlaps_test":true,"full_text_reference":true}, +{"id":"4588","category":"article","split":"dev","html_sha1":"d20cad7ddf1741d40cd8f0c412a28c26325d2e9d","truth_sha1":"b558c7aac4e778e9910ebf679777278f02702cec","overlaps_test":false,"full_text_reference":true}, +{"id":"4589","category":"article","split":"dev","html_sha1":"f784f6af5a5ab989563744b693431929f72905b9","truth_sha1":"dc78f857c3b20722d1cb5897cd8e8ad655fe5af3","overlaps_test":false,"full_text_reference":true}, +{"id":"4591","category":"article","split":"dev","html_sha1":"bf59cd241656c53c89e5b40cf84385728a6fc93d","truth_sha1":"537060be1784d6ff70c6f7fe8ec13dab52ec8cbc","overlaps_test":false,"full_text_reference":true}, +{"id":"4592","category":"article","split":"dev","html_sha1":"8295e543c922d208d876d494c5d4c414cf97f2a2","truth_sha1":"72a10742bcdf8e54817a390e386b291f08185400","overlaps_test":false,"full_text_reference":true}, +{"id":"4594","category":"article","split":"dev","html_sha1":"7505047e53010398eb56ad68ea97eff28b80f895","truth_sha1":"38969609b57948ac6eaafb2f95e3e26fb05d7aca","overlaps_test":false,"full_text_reference":true}, +{"id":"4596","category":"article","split":"dev","html_sha1":"8b460a809976f5c2cfb6df0157b78ac29d523654","truth_sha1":"577345fddbe585a56e2793d2ac74373fbed2d83f","overlaps_test":true,"full_text_reference":true}, +{"id":"4597","category":"article","split":"dev","html_sha1":"b73f00992f3c57af60412e801cc7b5e6fb909064","truth_sha1":"b583726be88752419503e93ea09bab643ff49a99","overlaps_test":false,"full_text_reference":true}, +{"id":"4598","category":"article","split":"dev","html_sha1":"354d7de0dae2345fc9beeed4fbf7e71818ae3049","truth_sha1":"977e5ca705c8ab114cf2819cf80b2aa310de1166","overlaps_test":true,"full_text_reference":true}, +{"id":"4599","category":"article","split":"dev","html_sha1":"87b2909a123e8288769a78490caa9a7d527d1871","truth_sha1":"10aa9efdafd3bc926c4b3af6a41d9cc7935182d4","overlaps_test":true,"full_text_reference":true}, +{"id":"4601","category":"article","split":"dev","html_sha1":"2f79502e9af8dbfe3b9dfd8f76d2489c6747aa9b","truth_sha1":"d0cde0db8ec3730a49914820086c3ad7d3367187","overlaps_test":false,"full_text_reference":true}, +{"id":"4603","category":"article","split":"dev","html_sha1":"04627206b92693165765b8e45b6385df5d5fb6f2","truth_sha1":"25649475fe32bc9acef1dfce3391ccae72be45ae","overlaps_test":false,"full_text_reference":true}, +{"id":"4604","category":"article","split":"dev","html_sha1":"fa1946f82b1106299b84e97c8c206bb74283f6bd","truth_sha1":"9f28ec62fc34c52dbeba965e0792462d48f7ea60","overlaps_test":false,"full_text_reference":true}, +{"id":"4605","category":"article","split":"dev","html_sha1":"08d7d2d19efd157c5bbecb43030722c89277eba8","truth_sha1":"bee4ac2ab4ef78ec8dee9c5904f19bca4befc19d","overlaps_test":false,"full_text_reference":true}, +{"id":"4606","category":"article","split":"dev","html_sha1":"aadd9da4c2c3d7b7ab6a06875b80ac1e7bba1438","truth_sha1":"5deee8efc38906e583c63f59fe7767e809f4618b","overlaps_test":false,"full_text_reference":true}, +{"id":"4607","category":"forum","split":"dev","html_sha1":"bf223e34204a1149c28485711e089cf7bab322d5","truth_sha1":"909251604ac5d0d05045520e985c75c029c726f2","overlaps_test":false,"full_text_reference":true}, +{"id":"4608","category":"article","split":"dev","html_sha1":"4ba170c6d3082d9dad948cb8ee9f31e78b362062","truth_sha1":"29e580ed2bf017d693dd620c0c772a54410c698b","overlaps_test":true,"full_text_reference":true}, +{"id":"4609","category":"listing","split":"dev","html_sha1":"a919b3ca1e93e960bd7e04ac4aad76b5fbc89651","truth_sha1":"6e216cacd15cad19a298354f8b7e307b535bbc44","overlaps_test":false,"full_text_reference":true}, +{"id":"4610","category":"article","split":"dev","html_sha1":"3c492c1ea63dfd1b7afd96979560cbc041dd6b63","truth_sha1":"577998c169d82dda8eabdcaff1f45bba372c4e5e","overlaps_test":false,"full_text_reference":true}, +{"id":"4611","category":"article","split":"dev","html_sha1":"a9ce471949d84b930b20aa943fb931b07dc053c4","truth_sha1":"352e6dee20603d6fb0329b7786779e947d8af200","overlaps_test":false,"full_text_reference":true}, +{"id":"4612","category":"article","split":"dev","html_sha1":"ae97afa777e9bddb663e4dd7c70da54f4f616494","truth_sha1":"53cff1733c41d48caf4abb58689de4f5b8034732","overlaps_test":false,"full_text_reference":true}, +{"id":"4613","category":"article","split":"dev","html_sha1":"6587129d763cc4c7ed3f5a27e917bedeb6beebb4","truth_sha1":"c3cb472a8ea10e57a721ed1120914b0cf8deae68","overlaps_test":false,"full_text_reference":true}, +{"id":"4614","category":"article","split":"dev","html_sha1":"fde0d445717b4e8fcab4b21259a7d340545169a8","truth_sha1":"67cffd39b591cd23e66790f6ac49a9de911f8b2d","overlaps_test":false,"full_text_reference":true}, +{"id":"4615","category":"article","split":"dev","html_sha1":"399a18a3fd22290c1105342006e4ac62ee9a1bcf","truth_sha1":"3787b02e2d78b758c853249f15ec585567716967","overlaps_test":false,"full_text_reference":true}, +{"id":"4617","category":"article","split":"dev","html_sha1":"0e6909000c1e4944864d7c43e6af330758688a63","truth_sha1":"51f1ed8837a08e847eaa01699ccf97b3a77dddb7","overlaps_test":false,"full_text_reference":true}, +{"id":"4618","category":"article","split":"dev","html_sha1":"8562f95fc2de7554620392db9168e9ea8d799747","truth_sha1":"da332187fd22d72f00e9b9ebc1427590f91985b3","overlaps_test":false,"full_text_reference":true}, +{"id":"4619","category":"article","split":"dev","html_sha1":"53798cc9434dd56426428fa46cfa469c61a293c3","truth_sha1":"3bb651515bb4cef196dee9794c29a98d251359c4","overlaps_test":false,"full_text_reference":true}, +{"id":"4620","category":"article","split":"dev","html_sha1":"9175ef35ae320d92629994f5e7016c9d00eeeae9","truth_sha1":"17c6f3c121c094a17a0948f5cdcf0a7fb4b0f5c3","overlaps_test":false,"full_text_reference":true}, +{"id":"4621","category":"article","split":"dev","html_sha1":"35d090a4b7c69bab3aecc037b9e65e263b0abfef","truth_sha1":"247dc836dc6a43e09c85a73243aa0aefda73ab41","overlaps_test":true,"full_text_reference":true}, +{"id":"4622","category":"article","split":"dev","html_sha1":"c9a1f6a07d4b3ea60574db4c0503e274b24da38f","truth_sha1":"e1c726729b0ef270b44c3c45de5221807868b071","overlaps_test":false,"full_text_reference":true}, +{"id":"4624","category":"article","split":"dev","html_sha1":"bcfac210db2193ceb930b4e24a7ba08669c19bff","truth_sha1":"7c37ea67ec508a6cbc119a88f49f2e30eea32831","overlaps_test":false,"full_text_reference":true}, +{"id":"4627","category":"listing","split":"dev","html_sha1":"73015dd2b6f51e239e7b502bb982d0c2e06b6fd6","truth_sha1":"774a3a2ff8340147b9648b000fe858a217a9c451","overlaps_test":false,"full_text_reference":true}, +{"id":"4628","category":"article","split":"dev","html_sha1":"986c78d56a8d5724c3a36582e829208ce9ca59e1","truth_sha1":"b61d40473e7155a0cdd231b7da6414988624fd5f","overlaps_test":false,"full_text_reference":true}, +{"id":"4629","category":"listing","split":"dev","html_sha1":"4d3f17d4128d37a7ace63efb556ce31f291b853a","truth_sha1":"c6593c199aabed8f9f5b6c75e8b145b3053fccc9","overlaps_test":false,"full_text_reference":true}, +{"id":"4632","category":"listing","split":"dev","html_sha1":"1e37f563e6297f043086f5db4deae8309863b5dd","truth_sha1":"ec32b63dcff559773df580d6ad0448f3ed7d4935","overlaps_test":false,"full_text_reference":true}, +{"id":"4633","category":"article","split":"dev","html_sha1":"5d496081011d5206e446e8cbc52f150ebb541f55","truth_sha1":"9def897157d4b5e99333007da0edec3c4da8ca23","overlaps_test":true,"full_text_reference":true}, +{"id":"4636","category":"listing","split":"dev","html_sha1":"e7fd206040e98dde099b674349e26e2e3af36720","truth_sha1":"2eb856867179ecb615ab2e5b1be51dbecc4c8408","overlaps_test":false,"full_text_reference":true}, +{"id":"4640","category":"article","split":"dev","html_sha1":"f8c534e56dc4a1b2f9b02dc76c246f6fa249450d","truth_sha1":"7899278669a204c906f2fdbe84ddc9fc5660673a","overlaps_test":false,"full_text_reference":true}, +{"id":"4642","category":"article","split":"dev","html_sha1":"4630394a60564d4c0a77883fc47ec42ae9cf1bbf","truth_sha1":"76faabd80e57e58122df937784c6d3994b5529a7","overlaps_test":false,"full_text_reference":true}, +{"id":"4643","category":"article","split":"dev","html_sha1":"22a9aeb316fd0ab288b8b0fb53dc7d899d7fedc8","truth_sha1":"1a18233c899e449b3b3c70a28b001aa9e6a2ca97","overlaps_test":true,"full_text_reference":true}, +{"id":"4644","category":"article","split":"dev","html_sha1":"bd9cb6e2e7b9863a5aaf447b8362ad04541357fd","truth_sha1":"89ffbface73b9f39e2f97b855384caf5953fd5e9","overlaps_test":true,"full_text_reference":true}, +{"id":"4645","category":"article","split":"dev","html_sha1":"9355b317fc80d9809fc010d552aeee53421ee030","truth_sha1":"864d3bdea1b9976b062a2d360c3ee5d62460bb1a","overlaps_test":true,"full_text_reference":true}, +{"id":"4646","category":"article","split":"dev","html_sha1":"12883ba5fc7425bec59b1fb09e4c5f2d4177f7fe","truth_sha1":"469160099fb85b8e71e04f5303f59c719a829ef3","overlaps_test":false,"full_text_reference":true}, +{"id":"4647","category":"article","split":"dev","html_sha1":"24f34835ff7565dae72ade6b7be04fe8eeb3a796","truth_sha1":"9a286f04b68400de4247837fb4ab53223421e98f","overlaps_test":false,"full_text_reference":true}, +{"id":"4649","category":"service","split":"dev","html_sha1":"47908c8712c2a2a490f7117b67978fa2122343b2","truth_sha1":"4b96cf7988692862ef2b03bd6174f6018a44ccb2","overlaps_test":false,"full_text_reference":true}, +{"id":"4650","category":"service","split":"dev","html_sha1":"09f24e2ac990bed1ac0aa8ab17c3b0262f809b70","truth_sha1":"786dc6ff19252bb671a889512ffeacaa0a952a6f","overlaps_test":false,"full_text_reference":true}, +{"id":"4654","category":"service","split":"dev","html_sha1":"a99bada6cac0e580374daa56b7cfd48a1e935a6d","truth_sha1":"f672e21d767f71d8986075cb11a7be21999658e6","overlaps_test":false,"full_text_reference":true}, +{"id":"4656","category":"article","split":"dev","html_sha1":"7aa411979c09ae89c30f48d2f76dd9a3e4f36305","truth_sha1":"7d424b9dd767e5a48171d4630a188d68542283b7","overlaps_test":false,"full_text_reference":true}, +{"id":"4657","category":"service","split":"dev","html_sha1":"6d3ab5ed3dfdd2288e633d8663e23398fce19d81","truth_sha1":"1c039514c3472640d99b66843b7ed4471d695d25","overlaps_test":false,"full_text_reference":true}, +{"id":"4659","category":"article","split":"dev","html_sha1":"452a6474376e6ca5d0715d1dd2a0b622b0fabc91","truth_sha1":"3d120a30715371672f95641c4b5f6b9aae2619f9","overlaps_test":false,"full_text_reference":true}, +{"id":"4660","category":"article","split":"dev","html_sha1":"f063d615c17418139c395bc9ca5c8119e0804e02","truth_sha1":"7c4906c20bc3f54bf7b01a98bc1a0c8013f4fe4c","overlaps_test":false,"full_text_reference":true}, +{"id":"4661","category":"article","split":"dev","html_sha1":"c15f0563b4a3b162f6ada4f4180ae32b9de24a10","truth_sha1":"e01fa2ab4e64f1bfc0849bd18475df6aa689a753","overlaps_test":false,"full_text_reference":true}, +{"id":"4662","category":"article","split":"dev","html_sha1":"70ea1011d4ff2bb89c3569973ddbe03d99415db1","truth_sha1":"82c2eed0da5b6674694efb744dd8ffa9970fb68b","overlaps_test":false,"full_text_reference":true}, +{"id":"4665","category":"article","split":"dev","html_sha1":"f072f28c63ea5144d8a4a774254cf4546e221f14","truth_sha1":"b276aa40e0cbf10a9279d3491915f299a4ad84f8","overlaps_test":false,"full_text_reference":true}, +{"id":"4666","category":"article","split":"dev","html_sha1":"2c04d316b22167e8a2bc11961bcf8a6be060d3fc","truth_sha1":"e2aedc9556e45ce3165ee6404b08c69432ffe9d5","overlaps_test":false,"full_text_reference":true}, +{"id":"4667","category":"article","split":"dev","html_sha1":"c8e5b3b602eae894d8e3c33eb49c9c38c277eb4c","truth_sha1":"b0a74e8d2ac0dc5515a82c6c47d6ef97a19a457d","overlaps_test":false,"full_text_reference":true}, +{"id":"4668","category":"article","split":"dev","html_sha1":"959d1d3c51236b27d0e01ba93aa87afe9c686f0d","truth_sha1":"9397cb0a3d7bf243fbe715937b2631f141df154b","overlaps_test":false,"full_text_reference":true}, +{"id":"4669","category":"article","split":"dev","html_sha1":"dc9ca49d48b3b934d65c61e1d605f92cf7685e2c","truth_sha1":"235740514f1301dc7f2b5f90bfd6703825629670","overlaps_test":true,"full_text_reference":true}, +{"id":"4672","category":"article","split":"dev","html_sha1":"2afc52f01c0b6b29f045214b289c5ab4ecf12f9b","truth_sha1":"ad2a594edd3bce47885ac3dcd96751a12574fc40","overlaps_test":true,"full_text_reference":true}, +{"id":"4673","category":"article","split":"dev","html_sha1":"60fa338037bea3fab035a32d98f9e9fd4798cdb0","truth_sha1":"837599376ae735aad46bc4f6a5d745d013d0af00","overlaps_test":false,"full_text_reference":true}, +{"id":"4674","category":"article","split":"dev","html_sha1":"0ade3038fb1c41a76e00d02fb3e04d0f4563adcc","truth_sha1":"fcdaed8c9cbe0386093411346c2bbca83cee8238","overlaps_test":false,"full_text_reference":true}, +{"id":"4677","category":"article","split":"dev","html_sha1":"1a46821c4538243c59739b00c5aea07ab0375a98","truth_sha1":"07fd4253f2bdc003112faacefd18e08382380f6e","overlaps_test":false,"full_text_reference":true}, +{"id":"4679","category":"article","split":"dev","html_sha1":"17de612a0004b2d348e1af68f5a97f82460c572b","truth_sha1":"32be04b26583a8014de7cc1125096c9d5cf54cd7","overlaps_test":true,"full_text_reference":true}, +{"id":"4680","category":"article","split":"dev","html_sha1":"ff7da0f8d610365ecb7f4b776251101f515dcb4d","truth_sha1":"94c52167304624f5d6a32801010316db6daa2458","overlaps_test":true,"full_text_reference":true}, +{"id":"4681","category":"article","split":"dev","html_sha1":"e85d3b917b47c1f5d7064932a162fdb90b788a7a","truth_sha1":"a00b77020e8bed8ea24a532458312358ba0ebc55","overlaps_test":false,"full_text_reference":false}, +{"id":"4682","category":"article","split":"dev","html_sha1":"f9686850049f654c754fca71f351b70b805fa05e","truth_sha1":"5e56c71be0ee606820bd5d1f033511df4d1618f3","overlaps_test":false,"full_text_reference":true}, +{"id":"4683","category":"article","split":"dev","html_sha1":"c9629e8f1e0d69362eb9bd5af0095d92c3a2aacf","truth_sha1":"774b50625239a8d0ee521408145e02df9e9d7f2d","overlaps_test":true,"full_text_reference":true}, +{"id":"4684","category":"article","split":"dev","html_sha1":"d1db74312f99d76d80a4d0f25b31e8baa5fce680","truth_sha1":"c17c74a7994c058e8edacebeb597503016d7dd53","overlaps_test":false,"full_text_reference":true}, +{"id":"4685","category":"article","split":"dev","html_sha1":"3475efb3d7008aa9ecc24eb116132491c5c0ef8b","truth_sha1":"854c6bb2d8e39ef46fcf76451dd4225348647c07","overlaps_test":false,"full_text_reference":true}, +{"id":"4688","category":"article","split":"dev","html_sha1":"15e58a97757a47cad260c304021c6d119aab2fb7","truth_sha1":"7e78963954a781386350cf95eb55f455b4264a63","overlaps_test":false,"full_text_reference":true}, +{"id":"4690","category":"article","split":"dev","html_sha1":"03f6a4b015d028873d8b40d17f4ef6807165a70b","truth_sha1":"457fc738e2a30c0abaaae30d5f9d0425c8e534ad","overlaps_test":false,"full_text_reference":true}, +{"id":"4691","category":"article","split":"dev","html_sha1":"f0297f8de37208006f1048653145bc4419e60675","truth_sha1":"743ecb6043fd2bf7d031c14ac8492037c71b6cda","overlaps_test":true,"full_text_reference":true}, +{"id":"4692","category":"article","split":"dev","html_sha1":"725bf8a7eb6b7e9aa1462b97926dcdcc351942e3","truth_sha1":"5adadb6aff9ac84aaf0325d51fc0969fdd10e5f3","overlaps_test":true,"full_text_reference":true}, +{"id":"4693","category":"article","split":"dev","html_sha1":"4aea4f29a388faa49d2b5cdc1c6dfa72807b6782","truth_sha1":"6af886cd8a3c425266c61eeff51bd57846cc8fb4","overlaps_test":false,"full_text_reference":true}, +{"id":"4694","category":"article","split":"dev","html_sha1":"0d02c7bffe3aff01d8b1bd405be448054a49fdda","truth_sha1":"3e07528705754586b7c163bca730f68f1f990a5c","overlaps_test":false,"full_text_reference":true}, +{"id":"4695","category":"article","split":"dev","html_sha1":"b9c718e937f78bccfb2968c9739657137255c681","truth_sha1":"a850eb2df2f1bf4fba516c335544d0c09fde2d27","overlaps_test":false,"full_text_reference":true}, +{"id":"4696","category":"article","split":"dev","html_sha1":"1cbf667b9246e13a5ea841291becef212e932393","truth_sha1":"c855f94ab188497b9b7883a24a5dfddd45f30bd1","overlaps_test":false,"full_text_reference":true}, +{"id":"4698","category":"article","split":"dev","html_sha1":"982ed637b45633522579217f741370d9587c0cc4","truth_sha1":"f32757d893f15e84e476f1f0a98c895a4c1641ca","overlaps_test":false,"full_text_reference":true}, +{"id":"4700","category":"article","split":"dev","html_sha1":"f0a0de0341312507784739887f0fea8680ae45e5","truth_sha1":"5a624198d96a4278940f39bfe7dd58aef5bf6fe9","overlaps_test":false,"full_text_reference":true}, +{"id":"4701","category":"article","split":"dev","html_sha1":"f198b6c210f817b7e2a9f59bf80ba999476e3cb9","truth_sha1":"d77058a5fde077b436c7238bd1710ea66074efc0","overlaps_test":false,"full_text_reference":true}, +{"id":"4702","category":"article","split":"dev","html_sha1":"125adbb0475ff08b442201e4fba25fad57a13524","truth_sha1":"69fd2121579e431d4111b25eaa46290e241aebd6","overlaps_test":true,"full_text_reference":true}, +{"id":"4703","category":"article","split":"dev","html_sha1":"b25ca64e105b8121d31fa869b674cbd6c89c4842","truth_sha1":"a0c8c7a96418a1b9e6e7588268ae3119a576eac7","overlaps_test":false,"full_text_reference":true}, +{"id":"4704","category":"article","split":"dev","html_sha1":"e8b33a78446ed2de47cc2148d459f21b94f92c5d","truth_sha1":"83e9c2766595c1ab0f863df0a25bc78067118e10","overlaps_test":false,"full_text_reference":true}, +{"id":"4705","category":"article","split":"dev","html_sha1":"756248410260b5c121cc206b62f74e2f8557eea4","truth_sha1":"382c90bb95ab6dd86a82057f02e5f1234c411f75","overlaps_test":false,"full_text_reference":true}, +{"id":"4706","category":"article","split":"dev","html_sha1":"7c535cc150d906c157cd8287799797e2bc5bda17","truth_sha1":"af5db892e1824be9871c30987d9d184051eb9d8b","overlaps_test":true,"full_text_reference":true}, +{"id":"4708","category":"article","split":"dev","html_sha1":"4eb7adbaffa0ee485911cd3cc6eb38a04c49fa8c","truth_sha1":"5c0d9ea39c2f3becc96392cb9b39f308b2ea527c","overlaps_test":false,"full_text_reference":true}, +{"id":"4709","category":"article","split":"dev","html_sha1":"969d8952b452f1904fd7134db0de111bdce6b90b","truth_sha1":"11050af1116fd8f6d0bd1f23bfe3153c16de46ee","overlaps_test":true,"full_text_reference":true}, +{"id":"4715","category":"article","split":"dev","html_sha1":"70ceb2c6ac71fa25041acc0bf5d6a95a0c3655b6","truth_sha1":"f87e6f6b321c4be2848b1eebc78605e9f0b7b129","overlaps_test":false,"full_text_reference":true}, +{"id":"4720","category":"product","split":"dev","html_sha1":"c3b264c894f0d4539c5da85252bf7d9498ec994c","truth_sha1":"811ba5ebc8a510b73a465caed8104d08eb7eba62","overlaps_test":false,"full_text_reference":false}, +{"id":"4721","category":"product","split":"dev","html_sha1":"9e588c92f5d5ba9c7496415a3c31f622b88e953d","truth_sha1":"68f686a7ad1759bade1465f1721d97cb416dea8f","overlaps_test":false,"full_text_reference":false}, +{"id":"4722","category":"product","split":"dev","html_sha1":"5edfd032c3bd8c3b368ecb64edee53a8392e3784","truth_sha1":"bcebc6843aeae25bdb178c91af77c80b2de6881a","overlaps_test":false,"full_text_reference":false}, +{"id":"4736","category":"collection","split":"dev","html_sha1":"45ffbc7c014f927b82fe12c74534d3b4bffb7582","truth_sha1":"75ca2ab5108e6441a0e626eba0b6c688023fcaa5","overlaps_test":false,"full_text_reference":true}, +{"id":"4750","category":"listing","split":"dev","html_sha1":"447e11bf46014a3351ffc546357e6ecb160d0ff3","truth_sha1":"dc9b6577b5cdaf1c75839d5ec66050c5addfc392","overlaps_test":false,"full_text_reference":false}, +{"id":"4757","category":"collection","split":"dev","html_sha1":"a965a46dc6a1151fdbc956a9b74e03e18c725e25","truth_sha1":"d6acc51569ced3bc7573318a533e9ce83fad8134","overlaps_test":true,"full_text_reference":true}, +{"id":"4758","category":"service","split":"dev","html_sha1":"c323c72cc446c20031d00a55e8ef048f03f6a13c","truth_sha1":"fcaa4786d9c327c03c95e8d48a7b8a8df161462c","overlaps_test":false,"full_text_reference":true}, +{"id":"4759","category":"article","split":"dev","html_sha1":"0ae40aade1ce0838f271e2f3781b233ffd11f124","truth_sha1":"c7da954b973476ea7708db0cd121d280de46f13b","overlaps_test":false,"full_text_reference":true}, +{"id":"4760","category":"collection","split":"dev","html_sha1":"902dfa59de8a28279b74a153265a3061e2799055","truth_sha1":"a937d1b52a9fefe62c5d99eab0cd9dddf194df86","overlaps_test":false,"full_text_reference":true}, +{"id":"4761","category":"article","split":"dev","html_sha1":"eb6389f8dc40617197865438fb8ee766a612e809","truth_sha1":"c7dcfcfa71dc42a614dab3c566320d63a8d7c0f7","overlaps_test":false,"full_text_reference":true}, +{"id":"4762","category":"collection","split":"dev","html_sha1":"8defc888f7a56feb922b63117dc2abe79caf918b","truth_sha1":"11a74be7886bb0b2820a3f6160deb3fdee345839","overlaps_test":true,"full_text_reference":true}, +{"id":"4763","category":"collection","split":"dev","html_sha1":"ce45b4585b969f633179e7861f51de6c09ae79c2","truth_sha1":"e2bacd64abcd63fbb3a2f0a25870ba853c473fd6","overlaps_test":false,"full_text_reference":false}, +{"id":"4764","category":"service","split":"dev","html_sha1":"2ac2b4f7b64ad6bd4cfa1926cb13987f2d718478","truth_sha1":"d07ed0d921d4c0e8adf6b5813a28fb35d73af431","overlaps_test":false,"full_text_reference":true}, +{"id":"4770","category":"collection","split":"dev","html_sha1":"a38b8083183c60289b652451d8a19c46760adfa8","truth_sha1":"95ad113debc61940b5368e0d9dd79ca42fe1aab2","overlaps_test":false,"full_text_reference":true}, +{"id":"4771","category":"product","split":"dev","html_sha1":"03e57e4735510fc79dab43c304c0999b9e508529","truth_sha1":"ac83ebe558d11e602b080b1b7aee658156921335","overlaps_test":true,"full_text_reference":true}, +{"id":"4780","category":"product","split":"dev","html_sha1":"847417a0443d77835a2eb882a76777f6ff59c203","truth_sha1":"80420791153f829fce29835f81ee68e03996c0c6","overlaps_test":true,"full_text_reference":true}, +{"id":"4781","category":"product","split":"dev","html_sha1":"1909ea301f594aa4ac875cfc7d69578cfd1556f2","truth_sha1":"c615b3f8901818d9a67201b44129dc2c06748a8e","overlaps_test":true,"full_text_reference":true}, +{"id":"4787","category":"article","split":"dev","html_sha1":"5f9568961a7f701e39d8ab4f84babadf6a34f479","truth_sha1":"8bf86a0ec042abcf9ca4bd215d7091529c722284","overlaps_test":true,"full_text_reference":true}, +{"id":"4788","category":"listing","split":"dev","html_sha1":"15ca2e25f576b5af9280713876372e5132c2a01e","truth_sha1":"e738a172d9debcc572f696457167163ef41936f6","overlaps_test":false,"full_text_reference":false}, +{"id":"4789","category":"product","split":"dev","html_sha1":"9d2045cb084f8c860b72b836bc0ee624942b74fd","truth_sha1":"100416d954f085b1a8655ae1d7058e545ff2292f","overlaps_test":true,"full_text_reference":true}, +{"id":"4795","category":"product","split":"dev","html_sha1":"30dc0b1a1c4df032abe74f85081903d01c3ba134","truth_sha1":"047be917b1d1fbd68b2833b49e12074b1c39a018","overlaps_test":true,"full_text_reference":true}, +{"id":"4796","category":"article","split":"dev","html_sha1":"210a057689a3ad976b1fa6e4ddfb48712eec8dfd","truth_sha1":"88efe1ccb8481e0a9adb7d8d7182b0abe28121f0","overlaps_test":false,"full_text_reference":true}, +{"id":"4799","category":"article","split":"dev","html_sha1":"c8efb65510404b864bd8d4677670b3b4d5920a37","truth_sha1":"430daedd6d6dc80458fe5bc54bd7b1540a22a8eb","overlaps_test":false,"full_text_reference":true}, +{"id":"4801","category":"article","split":"dev","html_sha1":"581985973af10ae205c39cb5c794a4aa5e13e17a","truth_sha1":"23d997d459079f7486764d32b7bb6efc882dd7da","overlaps_test":false,"full_text_reference":true}, +{"id":"4802","category":"forum","split":"dev","html_sha1":"3e48aba3377bf0582c3e12b285a511e6a9a09a46","truth_sha1":"13c42dd8437fa2969fed75b9f71d7810dcb0f579","overlaps_test":false,"full_text_reference":false}, +{"id":"4805","category":"collection","split":"dev","html_sha1":"19def8bbbbd517949d1d36f95e8cc6fbe9dadd58","truth_sha1":"df85fcca0f1e23296ea6b09449d04776b4116a08","overlaps_test":false,"full_text_reference":true}, +{"id":"4809","category":"article","split":"dev","html_sha1":"bddc6af55d6b62160290345322a8b1f1ac6f622c","truth_sha1":"d73dff3c2ceb30c5fce5f39b8bf7a29716fb4ce9","overlaps_test":false,"full_text_reference":true}, +{"id":"4810","category":"article","split":"dev","html_sha1":"39a25d0e14a63794dbd71db0513599d6cf1af364","truth_sha1":"0c6e06e10c7715be80fb069e27743fe5c069496d","overlaps_test":false,"full_text_reference":true}, +{"id":"4812","category":"service","split":"dev","html_sha1":"a9349f7fce0b6e27d98ed4fc62708fec1a4e1e7c","truth_sha1":"58cf15b3978e5fa7a9bbc9cb085352978e0c6183","overlaps_test":false,"full_text_reference":true}, +{"id":"4815","category":"article","split":"dev","html_sha1":"e01db48bff854188c0b10d5e0d239beb46013795","truth_sha1":"8dba6eb2ac7d75c55edaa73de43bfacf71190d67","overlaps_test":false,"full_text_reference":true}, +{"id":"4817","category":"article","split":"dev","html_sha1":"83ad21853ddebfd4686b2b87ba80d76427896f64","truth_sha1":"d1bbd75efdb4f995e5503dd4efb8d2e7a290ef09","overlaps_test":false,"full_text_reference":true}, +{"id":"4818","category":"listing","split":"dev","html_sha1":"3346f325d536e59c0263d01fd6bd827178232112","truth_sha1":"1a29260dfafb2715f576834d5651879cd15e89f1","overlaps_test":false,"full_text_reference":true}, +{"id":"4819","category":"article","split":"dev","html_sha1":"1e3beafad80e990ebe2dec9ace2312be917881e5","truth_sha1":"78e93b89b195b839bdf83d72091a88f70e7c0c04","overlaps_test":false,"full_text_reference":true}, +{"id":"4823","category":"article","split":"dev","html_sha1":"83dacc0fd3e397a91542ef760f12bbbdd0d41ccb","truth_sha1":"ea92327ca2920af91b2e87730f9eecb085d96147","overlaps_test":false,"full_text_reference":true}, +{"id":"4825","category":"article","split":"dev","html_sha1":"9273003d174a223f2343cffc3d6da15975591b82","truth_sha1":"4971ca6923210edcd1619b83a4e7a03af8abf691","overlaps_test":false,"full_text_reference":true}, +{"id":"4828","category":"article","split":"dev","html_sha1":"59b6119132f3b67bc6adba106376819f66b2b8d9","truth_sha1":"ccda98d6e8da45b1bf8a3ae33f269a680b22f7f4","overlaps_test":false,"full_text_reference":true}, +{"id":"4829","category":"listing","split":"dev","html_sha1":"eb2ae391541d29a07c98bd536189b09e299fe2b2","truth_sha1":"ec55c0ce545f22077e3c79542cdf12d22b9dbd1e","overlaps_test":false,"full_text_reference":false}, +{"id":"4830","category":"article","split":"dev","html_sha1":"af35b938df94857517ff0d625f18fa1f3652c966","truth_sha1":"35b338e0e2f767f7146a01b37fa2377af19bd64d","overlaps_test":true,"full_text_reference":true}, +{"id":"4833","category":"article","split":"dev","html_sha1":"f9945c66159ee9db72d986d872d0ed6461bd4633","truth_sha1":"37b66340dad0fdd060d2a2bfea4aca6cf4d59060","overlaps_test":true,"full_text_reference":true}, +{"id":"4835","category":"article","split":"dev","html_sha1":"4a8a98b65eec54a5ab0e6387a1c97651ec2ad3b4","truth_sha1":"734a85eca43a4e0d0e24b8162ed5039efda2b891","overlaps_test":false,"full_text_reference":true}, +{"id":"4838","category":"forum","split":"dev","html_sha1":"278a093047fcca27b20b95508e22e1742106c81b","truth_sha1":"32e543c856b7bda68029baf752f24811c436a811","overlaps_test":false,"full_text_reference":true}, +{"id":"4840","category":"article","split":"dev","html_sha1":"62c6ac0fac3eafe153dfb5a84908360577fccf2d","truth_sha1":"e46b9c97f3e01ca0d92641639be60acc14bda324","overlaps_test":false,"full_text_reference":true}, +{"id":"4841","category":"documentation","split":"dev","html_sha1":"f539aff84c5dbdb30ae0f99e2c45c8ab7a488e86","truth_sha1":"642132c061847edd94b0b182f2e89fbdd58a3a10","overlaps_test":false,"full_text_reference":true}, +{"id":"4842","category":"article","split":"dev","html_sha1":"d3fd5f263c613f506c06e4d76dcd7e6d5c7c1711","truth_sha1":"32ae4bb7aa8ce4e40f19c24c525a2b4145fc8f6d","overlaps_test":false,"full_text_reference":true}, +{"id":"4843","category":"documentation","split":"dev","html_sha1":"e30efa19b1bc935c09d1f2cad2ec4546e558c4db","truth_sha1":"5cd7fc3dfc33468a2ae7bd926aa2c3b6f44f3faf","overlaps_test":false,"full_text_reference":true}, +{"id":"4844","category":"article","split":"dev","html_sha1":"2de821561c8a39fca0a6435cea6b5d14f5e7b809","truth_sha1":"672a4b9890d669a15b30dea55c1a4bede34d00e9","overlaps_test":false,"full_text_reference":true}, +{"id":"4845","category":"article","split":"dev","html_sha1":"b2c9cfe6fba051f799aa5df5a69013be9dbb5ad0","truth_sha1":"8f6426238af408289b54a71472f008720b933c01","overlaps_test":false,"full_text_reference":true}, +{"id":"4846","category":"article","split":"dev","html_sha1":"9b05c43a28aea7c03224f328cccab88b12c67cfa","truth_sha1":"a2cdccf01ba3e04bf50d2b0844b3f4f41f7a16f8","overlaps_test":true,"full_text_reference":true}, +{"id":"4848","category":"service","split":"dev","html_sha1":"598712492bfab75a3fb3735bb956707868788f79","truth_sha1":"528db1e7f59ce43917a840fe84a54170a0f009f1","overlaps_test":false,"full_text_reference":true}, +{"id":"4850","category":"collection","split":"dev","html_sha1":"785504af80f091c8281155ecb44f7efc2ca1c33e","truth_sha1":"b745d029f40b4b2bcb98a4fcb4e365cc055761c9","overlaps_test":false,"full_text_reference":false}, +{"id":"4852","category":"collection","split":"dev","html_sha1":"d389b2b6eb4cbe94f7d8f83373aa6e93afea9135","truth_sha1":"51cda654193ed6669e3fc2162a61624886a2fda0","overlaps_test":false,"full_text_reference":true}, +{"id":"4854","category":"service","split":"dev","html_sha1":"589eb7d52eacd7745a0377d25cf1da74eaa8af00","truth_sha1":"4085fe327b289ea19535e054bc91a0b590a01eb6","overlaps_test":false,"full_text_reference":true}, +{"id":"4855","category":"service","split":"dev","html_sha1":"3942dddc3d9ea73d961b63cce1e38bbd0722b36e","truth_sha1":"701da169f21e99ba10187b0ce44b2542c0b6e6eb","overlaps_test":false,"full_text_reference":true}, +{"id":"4857","category":"article","split":"dev","html_sha1":"9c12f5d2a118dafa4365c47415edb39cb4810e48","truth_sha1":"d9b2a41a296836134921446ee3f163c989a3886c","overlaps_test":false,"full_text_reference":true}, +{"id":"4858","category":"article","split":"dev","html_sha1":"fbb907570240e74583cf0664fa4ed1a450237c45","truth_sha1":"a0c093cdd5e228a4779780d0b3c6a18d0f87f818","overlaps_test":false,"full_text_reference":true}, +{"id":"4859","category":"article","split":"dev","html_sha1":"b5813cfcfe0a8a397047199717e838f4bdffe558","truth_sha1":"39ffa9b9cf64b31ad3f12bee9d344268f4393176","overlaps_test":false,"full_text_reference":true}, +{"id":"4860","category":"article","split":"dev","html_sha1":"be3fd1eead0898b9c402406b2af246ff10ed188b","truth_sha1":"16b3c06efc786700d4782253cd924b3cc6c1090b","overlaps_test":false,"full_text_reference":true}, +{"id":"4861","category":"article","split":"dev","html_sha1":"94a326ed4a906561285661db8f0d42b994522f0e","truth_sha1":"edf085cde66563cccce0325b8fcf979596b64c38","overlaps_test":false,"full_text_reference":true}, +{"id":"4863","category":"article","split":"dev","html_sha1":"04ad064734726ae4b9e84d0f8c2950bce9181d92","truth_sha1":"98946bd06b3887435ca3a6b6e470dff0c471354f","overlaps_test":false,"full_text_reference":true}, +{"id":"4864","category":"article","split":"dev","html_sha1":"ea875971e1c4ca436c9cd905717e04770e9861cd","truth_sha1":"4c87259d13fdc78b1df5c6bcbb3b79ef861dc0e3","overlaps_test":false,"full_text_reference":true}, +{"id":"4866","category":"service","split":"dev","html_sha1":"4502873a0c19eb4deb3f75112253e8eaa4c2cd60","truth_sha1":"0ad40bd31f045db25e005e08ca651d0f81a0cb80","overlaps_test":false,"full_text_reference":true}, +{"id":"4868","category":"service","split":"dev","html_sha1":"fdd6da03e6e9bb0f62e2a7409d509c48a5157161","truth_sha1":"fd34f17deb00b945293b6625af1fca2aadc1a9f8","overlaps_test":false,"full_text_reference":false}, +{"id":"4869","category":"service","split":"dev","html_sha1":"4d71e3a7332f4ce257f3bdad0128cefb92e2c5f5","truth_sha1":"9c48e47401bc13d4232d895b45b88178e73b4117","overlaps_test":false,"full_text_reference":true}, +{"id":"4871","category":"service","split":"dev","html_sha1":"c9d1c1bdb8644e63c9cdf09a0815acd94c5a9b7d","truth_sha1":"4ba23a2ff08092a50efd72e09144e6ab1ce1a482","overlaps_test":false,"full_text_reference":false}, +{"id":"4873","category":"service","split":"dev","html_sha1":"84600e38cd5e23bbfd7cedf3869f7c35e65d702b","truth_sha1":"0209fb5fcacbdda98706a57916b6cf8a1700f2f3","overlaps_test":false,"full_text_reference":true}, +{"id":"4874","category":"service","split":"dev","html_sha1":"b7a4a4d7019a97303c3d565d4d3a2da9392f31c4","truth_sha1":"411bd81aa190cd3929e63545dc4d02ad32e6e1aa","overlaps_test":false,"full_text_reference":true}, +{"id":"4876","category":"service","split":"dev","html_sha1":"7a57b20c582bb5e7f159aadd9751e1d94ed3c9c7","truth_sha1":"7b712228ba0eef69374b1b909894a73ea46d4e45","overlaps_test":false,"full_text_reference":true}, +{"id":"4877","category":"article","split":"dev","html_sha1":"64485ad369298f0ef4edea5d9e5a305a3c9dacbd","truth_sha1":"2684e399cec3ab40912bec3d68a6eefc6851998d","overlaps_test":false,"full_text_reference":true}, +{"id":"4880","category":"article","split":"dev","html_sha1":"e6fc19168605d12cce6a0ddbd0b0253b5d7d791f","truth_sha1":"32f6a64ef06f60489a3beef8f3ba51c9003ec13a","overlaps_test":false,"full_text_reference":true}, +{"id":"4883","category":"article","split":"dev","html_sha1":"46f452be8c47a3e2cf5dc5c6f7218cb637c3b08a","truth_sha1":"2cb1dd9fdd15ad0da1f5009f2a7b8ed6ce4c2573","overlaps_test":false,"full_text_reference":true}, +{"id":"4884","category":"article","split":"dev","html_sha1":"bba57bda6774db241bf77ffcc03b91e4a2b415f9","truth_sha1":"c5fd66ab86ab38d929748f54408bfdd02b5b2e36","overlaps_test":false,"full_text_reference":true}, +{"id":"4891","category":"service","split":"dev","html_sha1":"f699d01afa4fcf4b31c2aa795d11ece6936022c8","truth_sha1":"15c62b8fd4812b59fe6748a349c2a9b65380e53d","overlaps_test":false,"full_text_reference":true}, +{"id":"4893","category":"article","split":"dev","html_sha1":"89cf1afcb6875d76447f7da406f9123d97436e90","truth_sha1":"713c01048fceb286cba92fe6b5f1b37af314a68e","overlaps_test":false,"full_text_reference":false}, +{"id":"4896","category":"service","split":"dev","html_sha1":"970d8a10ea7c6c612813fe6017789fe0b47c9dd5","truth_sha1":"8f85522ac1964c71b4ee729b4eff777784190149","overlaps_test":false,"full_text_reference":true}, +{"id":"4897","category":"product","split":"dev","html_sha1":"d855a66313952b6e76fa8e35bbed4e50bf259b79","truth_sha1":"a8324f7cfc5beb8bac11dd99fdff192de685784c","overlaps_test":true,"full_text_reference":true}, +{"id":"4899","category":"service","split":"dev","html_sha1":"787dd61cfd315aeb4af77845430f1306e31301e7","truth_sha1":"6027354adacb17eb1fe3e8c12aa4339b389cd165","overlaps_test":false,"full_text_reference":true}, +{"id":"4900","category":"service","split":"dev","html_sha1":"6a66fcbe3e12fc0dbbd8ae794d8a47eabec760f4","truth_sha1":"851bc50ed372701b0e4236c0be9b272149dc3a23","overlaps_test":false,"full_text_reference":true}, +{"id":"4901","category":"service","split":"dev","html_sha1":"a453da699983010ff28ddc951d35bb10be4ebe17","truth_sha1":"fe6d1a61927bd06796b268c79f8c7ea2356ed6a8","overlaps_test":false,"full_text_reference":true}, +{"id":"4903","category":"service","split":"dev","html_sha1":"d7d0077bed1e58474cd3ce93ada1143adca1ea11","truth_sha1":"5635c49f15b1e99fdbbcb7ead16ccb536a845a95","overlaps_test":false,"full_text_reference":true}, +{"id":"4905","category":"service","split":"dev","html_sha1":"932b4178897973d2207a4a2e0f1940c2a7ecfeb5","truth_sha1":"8a3ad009d976a4183d44483220a04049aa0bf4fb","overlaps_test":false,"full_text_reference":true}, +{"id":"4908","category":"service","split":"dev","html_sha1":"a06b3f8824e8361f664600cfa17b1de30e752705","truth_sha1":"9321d1215a01114aa1d4bac90e84f453bea75839","overlaps_test":false,"full_text_reference":true}, +{"id":"4911","category":"service","split":"dev","html_sha1":"ca2f23544b50ac8e9fad2704b2cfd4647fc7ccd0","truth_sha1":"f56d676a297d7eab7ba1de05dcd6ef63109e2827","overlaps_test":false,"full_text_reference":true}, +{"id":"4912","category":"article","split":"dev","html_sha1":"4db70f7a88142a2d41628d206a282ac476977340","truth_sha1":"e33e9dc06b1169b97311dddb223bf38ba15c100a","overlaps_test":false,"full_text_reference":true}, +{"id":"4913","category":"service","split":"dev","html_sha1":"01202b15e3a285ac8231d46c57cb24986bf24d77","truth_sha1":"8921794bb0d47c13a9d61d77f50b6ad589f46a6d","overlaps_test":false,"full_text_reference":true}, +{"id":"4914","category":"service","split":"dev","html_sha1":"80555ec337518b574b90ea9f0d734dd0907c9d51","truth_sha1":"8cc26e5b5f1c8236d7bda0b6e7cdad8303345c14","overlaps_test":false,"full_text_reference":true}, +{"id":"4917","category":"service","split":"dev","html_sha1":"28912f30ebad30a60bd87cd19efc570167381f57","truth_sha1":"1f23eb0ef467bdac6d2c8ed42143654c804a7fd0","overlaps_test":false,"full_text_reference":true}, +{"id":"4011","category":"article","split":"test","html_sha1":"3cc3866312491431cdf790a17b38ab0a71975cc9","truth_sha1":"85e2af6b235ac05cfdfde5a6db79326f6058028d","overlaps_test":false,"full_text_reference":true}, +{"id":"4015","category":"collection","split":"test","html_sha1":"0430017ba2fb12047dbf340f33deb69f15d158db","truth_sha1":"473c58991289a73902177a7919e1a8eb5aa3ea79","overlaps_test":false,"full_text_reference":true}, +{"id":"4016","category":"article","split":"test","html_sha1":"94719ded24fba972e6acff02b295a2ad8e16acad","truth_sha1":"59cafcafcde7b318988adf3809140ad3c9dfafbe","overlaps_test":false,"full_text_reference":true}, +{"id":"4017","category":"product","split":"test","html_sha1":"b4039cb8e1e4d5e10b3aaa24cf736249035f2ac2","truth_sha1":"e86228da95217974b9617627b664e45bd40377b0","overlaps_test":false,"full_text_reference":true}, +{"id":"4021","category":"collection","split":"test","html_sha1":"f29b1a512d1b3ac11334d864f8d2051be085b768","truth_sha1":"23b910bdaecf7395d178ae930b3bdccff5e6e704","overlaps_test":false,"full_text_reference":true}, +{"id":"4024","category":"product","split":"test","html_sha1":"1130f912811bf15c3dce651c6ba453e6e5eebda4","truth_sha1":"7d7cb76b05384cc4e536cb82d24c4aa9cefcbb3f","overlaps_test":false,"full_text_reference":true}, +{"id":"4026","category":"product","split":"test","html_sha1":"a19dbc6b50e39b5b3c1acbfc3cb631bd4cfa3013","truth_sha1":"7324a8eae2df505926c5df4014cef2638de86d36","overlaps_test":false,"full_text_reference":true}, +{"id":"4028","category":"article","split":"test","html_sha1":"8f316effc82e583fffac36a7ff099523e84c8b8e","truth_sha1":"1e24b05d826b26fec179d9f59c23e6e64668ed3e","overlaps_test":false,"full_text_reference":true}, +{"id":"4029","category":"product","split":"test","html_sha1":"d31d9b9ab653e294db399e60443a1020409e57b0","truth_sha1":"3a07c4a4ed539b8f3c95ae240e72f0ea388513a3","overlaps_test":false,"full_text_reference":true}, +{"id":"4035","category":"collection","split":"test","html_sha1":"f7563d2e59858460a890d04c3ea592e06556b987","truth_sha1":"0fdfe8b9fb00385822645340fd1fe070bd7cd938","overlaps_test":false,"full_text_reference":true}, +{"id":"4038","category":"article","split":"test","html_sha1":"fc558132d1c8d538a897c4364fdbb93e9f398f00","truth_sha1":"be7f30d30c2ea278c9d61b1aeff76085137b6463","overlaps_test":false,"full_text_reference":true}, +{"id":"4044","category":"article","split":"test","html_sha1":"2dbb0676c52ef31fe02881aa4919ea421635bca2","truth_sha1":"50ebbdecca10294810a6d66e26a2905942745873","overlaps_test":false,"full_text_reference":true}, +{"id":"4048","category":"article","split":"test","html_sha1":"205e858f1d2b3673a469cc5d779c51bfa950fdaa","truth_sha1":"d3a29b7befaef54aad947fe52eb3bab54393d44d","overlaps_test":false,"full_text_reference":true}, +{"id":"4050","category":"product","split":"test","html_sha1":"42a1c3bb31a5c585bd9cda2b6de0f484aafa4912","truth_sha1":"eb5ea2bfd06dd040c7a399336af0946d2dc44d9a","overlaps_test":false,"full_text_reference":true}, +{"id":"4051","category":"article","split":"test","html_sha1":"30752750e0db30b761001598847698cf98a1b927","truth_sha1":"d3c8bcf9ccd91f93bb85d19fff3b5b17faf82aaa","overlaps_test":false,"full_text_reference":true}, +{"id":"4052","category":"article","split":"test","html_sha1":"7a652278af49b9ff19ca055a379cdc62e6fba879","truth_sha1":"505fea3d8efd9b25c0d18dac5ee3eeeea7144488","overlaps_test":false,"full_text_reference":true}, +{"id":"4054","category":"listing","split":"test","html_sha1":"a6e8a6b8cead44fe2c9e8bdac61e1b0748b14b3d","truth_sha1":"e43b7da819b2a592d2fb0725617143390bcf122d","overlaps_test":false,"full_text_reference":true}, +{"id":"4055","category":"article","split":"test","html_sha1":"5a390ed3ff9108788aa9f76b78a0e735592f513c","truth_sha1":"e1263f905a023e53791bd399f12240d9bf0042c4","overlaps_test":false,"full_text_reference":true}, +{"id":"4058","category":"article","split":"test","html_sha1":"d1af588c7b69841969f47124919dbf7ef1561b8f","truth_sha1":"d2e33c2fce695eae703859be12b706d57da4a63e","overlaps_test":false,"full_text_reference":true}, +{"id":"4060","category":"article","split":"test","html_sha1":"dbf66b51f03356b9e748db9b8647a5cd8832d87a","truth_sha1":"5932a40eed3306e4946b79ae5b9f85404584e501","overlaps_test":false,"full_text_reference":true}, +{"id":"4061","category":"collection","split":"test","html_sha1":"f1e574af1e49a33a9a6835360aa45aa9e483749d","truth_sha1":"2ef887655df5bc90d11259380fbe177754d7b65f","overlaps_test":false,"full_text_reference":true}, +{"id":"4062","category":"product","split":"test","html_sha1":"2733864f053a320b942286339d8cce088fab39d4","truth_sha1":"e3cc2860dfe1d248ed817e6b36023bebc663cbac","overlaps_test":false,"full_text_reference":true}, +{"id":"4063","category":"article","split":"test","html_sha1":"2409e222e49ab6679a45ba7c9e6a210ca7284e38","truth_sha1":"c84a6af8d76560d9b65140e57c70b8b84e519718","overlaps_test":false,"full_text_reference":true}, +{"id":"4064","category":"article","split":"test","html_sha1":"62113c09e48b85927a14a3101b312dfd471e47eb","truth_sha1":"e5839315d74be625e3671d10de5149806eb26138","overlaps_test":false,"full_text_reference":true}, +{"id":"4065","category":"collection","split":"test","html_sha1":"c7523863fc60181b3ded67cf0de809fc2c0622b7","truth_sha1":"891e7fbe920e1f1f6fc5d111ab7d540c75c5ab61","overlaps_test":false,"full_text_reference":true}, +{"id":"4066","category":"listing","split":"test","html_sha1":"a52f23d8f3e8fb74999ef7ccb9397810b3317509","truth_sha1":"5960a096f56d1983b0aa3732263439969b07a2b7","overlaps_test":false,"full_text_reference":false}, +{"id":"4067","category":"article","split":"test","html_sha1":"ef0e549eb608d835586992c3a6dc4837d34b8c18","truth_sha1":"46afbcf433cb33a676a7711e056746e3f0af3bb2","overlaps_test":false,"full_text_reference":true}, +{"id":"4071","category":"listing","split":"test","html_sha1":"d903c3de68ce6f0238aec225fe16ace9460a1845","truth_sha1":"6301d51d4a79460db6fbf88a04dec51a86d2c7ce","overlaps_test":false,"full_text_reference":true}, +{"id":"4072","category":"article","split":"test","html_sha1":"457f27708e3d90eb9e38ad0a06901227cc4e4c82","truth_sha1":"0a91c36cf1ce4619ef8fd2dcbdf464198434f901","overlaps_test":false,"full_text_reference":true}, +{"id":"4082","category":"article","split":"test","html_sha1":"a067aa4334aa63b3299afe19142dfd3f9d1fac85","truth_sha1":"33575cc991681c8b71d9c6456c1d316c4b597930","overlaps_test":false,"full_text_reference":true}, +{"id":"4092","category":"article","split":"test","html_sha1":"b831288c8fd55f6080d0498940386f23b6ac59df","truth_sha1":"fdc5ca82baf5de1c2a38170d8b1d5104f81cc9a7","overlaps_test":false,"full_text_reference":true}, +{"id":"4096","category":"article","split":"test","html_sha1":"68dc224f4d0f9871f6c9483544e52a3f6e091c74","truth_sha1":"145a9d9551b609e314a349f67463c8a5b4374980","overlaps_test":false,"full_text_reference":true}, +{"id":"4098","category":"product","split":"test","html_sha1":"1f6d9b09ae49192f59223c68cd0d094994bc7672","truth_sha1":"92b06e4f45edf69d55bdcf5faa3a865a73b823b7","overlaps_test":false,"full_text_reference":true}, +{"id":"4103","category":"article","split":"test","html_sha1":"6bfefe01eaa7b7d5e7b4e6c4369bec44f27a7747","truth_sha1":"a5ff9af61ce3410836fd1df5dc39c4f46f29fa99","overlaps_test":false,"full_text_reference":true}, +{"id":"4107","category":"product","split":"test","html_sha1":"d1001328f88e40c6f1114adc2e3fcca08821838e","truth_sha1":"1dd9d11c6b871a14dbdb4bf5b5baf5edc92e18fe","overlaps_test":false,"full_text_reference":true}, +{"id":"4108","category":"article","split":"test","html_sha1":"7ca2560831722aa2c611ed7ae5ac80baa8956a22","truth_sha1":"5772817232459fd18e5c9d6ad6675d873e55883e","overlaps_test":false,"full_text_reference":true}, +{"id":"4110","category":"article","split":"test","html_sha1":"7f65d9c2c8c4e2cd9b205c25e80eebd1912b3f8e","truth_sha1":"ecf971951aa061ec8d8e52f14e8e05d47fce8cfd","overlaps_test":false,"full_text_reference":true}, +{"id":"4113","category":"article","split":"test","html_sha1":"95661831e61d5b5c1da2be83f620c97f933fb2ee","truth_sha1":"dfbd069d8b7f93415bf8b773d8cc601de54226f8","overlaps_test":false,"full_text_reference":true}, +{"id":"4114","category":"article","split":"test","html_sha1":"2370ab794371dc3ada18258ee381e706575ea49d","truth_sha1":"734cb52e1b356b4daf44db065a4275bb04f2257a","overlaps_test":false,"full_text_reference":true}, +{"id":"4117","category":"article","split":"test","html_sha1":"d405fa339ce5552476194e471633c7b6da4063e9","truth_sha1":"be4d35f23722e69f9ed15ccdeeea3330f393dcb5","overlaps_test":false,"full_text_reference":true}, +{"id":"4126","category":"article","split":"test","html_sha1":"306d361c7ba5b285ad995981487c821431135f85","truth_sha1":"ae7cdb3d68d3b03b9a12d66c3c081e2a9a91ffa6","overlaps_test":false,"full_text_reference":true}, +{"id":"4129","category":"article","split":"test","html_sha1":"de085f11dc1b4a457492024fe0fd4aac2831048e","truth_sha1":"b13c1b8b3f509f8acbd91619899bf04c2c001399","overlaps_test":false,"full_text_reference":true}, +{"id":"4132","category":"article","split":"test","html_sha1":"8f520074dc0b7432778353cc695c38b5397c69a5","truth_sha1":"d9229a668b9c3a14cc20bfc4473c133d9cb45175","overlaps_test":false,"full_text_reference":true}, +{"id":"4142","category":"article","split":"test","html_sha1":"e7b4e57c901531484a26220e73caefe5db0ef8af","truth_sha1":"68cc34d9557fab27767b639b0a09053e441da392","overlaps_test":false,"full_text_reference":true}, +{"id":"4145","category":"forum","split":"test","html_sha1":"aa694f50f3d136d9c8b2c6fdafe47ee1687e23a3","truth_sha1":"eb99bbc11c381d8b7dc5b4f012b3c9718b450998","overlaps_test":false,"full_text_reference":true}, +{"id":"4150","category":"article","split":"test","html_sha1":"673210c6114c16755c8f1d59880d0fe967f47a65","truth_sha1":"577ad640feeb5777946f22182fb631a2f95bd95b","overlaps_test":false,"full_text_reference":true}, +{"id":"4160","category":"article","split":"test","html_sha1":"83c0c90944c8777d9fedd321b8706fef9917884f","truth_sha1":"dfaaa7e299f306ab4b32e740de5c7c44153e2d88","overlaps_test":false,"full_text_reference":true}, +{"id":"4168","category":"product","split":"test","html_sha1":"efb83eae1e26680f8e1c828a4edec5d1a71bd8a2","truth_sha1":"442764e9581b2f2317c8f3d856dbc6fa8be3c9ee","overlaps_test":false,"full_text_reference":true}, +{"id":"4169","category":"product","split":"test","html_sha1":"bcd5f4f2d8b0acec954879d72b5482a9c52011bc","truth_sha1":"767f9c56a6bd072be39574fabdc0b9e6a589e845","overlaps_test":false,"full_text_reference":true}, +{"id":"4171","category":"article","split":"test","html_sha1":"758dbde01507de09c06d5879c5db13132b0056cb","truth_sha1":"57f905a4a4635b89f2813c1ccd26bee79950f896","overlaps_test":false,"full_text_reference":true}, +{"id":"4174","category":"article","split":"test","html_sha1":"8ff029807e506cc53ba7085eae393b2708417c76","truth_sha1":"51f851e4caac8c9189df319bb6811814e3d3333a","overlaps_test":false,"full_text_reference":true}, +{"id":"4179","category":"article","split":"test","html_sha1":"413c81bb15a1e4c462f1353dd352593daac70ec1","truth_sha1":"bf0a171e7ca8614d56d739fb17efa27fc91ec48f","overlaps_test":false,"full_text_reference":true}, +{"id":"4181","category":"product","split":"test","html_sha1":"f72db666479678f56b770d73ca43eb3fc2f812ee","truth_sha1":"a03525c971355440f1f788972599c292b772c880","overlaps_test":false,"full_text_reference":false}, +{"id":"4186","category":"article","split":"test","html_sha1":"952ef1db68b21defa3b99da49118944896fc07c0","truth_sha1":"77eca06858fc3b42e56432ca5d516a1b8cad3344","overlaps_test":false,"full_text_reference":true}, +{"id":"4190","category":"collection","split":"test","html_sha1":"e419f23bf1f9cf23ad1bee6e7aba6259b8c28540","truth_sha1":"029d06f4b608e639d8d155c99de1b0d9c5df24d8","overlaps_test":false,"full_text_reference":true}, +{"id":"4195","category":"product","split":"test","html_sha1":"db1d2e4565a757981f89f251cd038ee3e8d962c9","truth_sha1":"b4bbcf5d7b714b26639cdccf1606aee873ddf90f","overlaps_test":false,"full_text_reference":true}, +{"id":"4200","category":"product","split":"test","html_sha1":"22bf35b5516e85cdc4e4feb84ecb6375d5972758","truth_sha1":"13a0d2eb9d04dece230f62f7875c2413f42c8707","overlaps_test":false,"full_text_reference":true}, +{"id":"4206","category":"product","split":"test","html_sha1":"2ac79112074d8a9dd61f0b71ca98a0657880f7cd","truth_sha1":"ba315e84a012f303a5d6c00413f31dad5ab908d9","overlaps_test":false,"full_text_reference":true}, +{"id":"4207","category":"collection","split":"test","html_sha1":"6456a0a790a3c6b48971b20d9bf1470cb00035d1","truth_sha1":"b0939b47771226b44fc5f76f7a39b260c26682c7","overlaps_test":false,"full_text_reference":true}, +{"id":"4211","category":"product","split":"test","html_sha1":"74057afd294e23985cb19ea80c9386af4d0b33fb","truth_sha1":"89888d305582dadfcfd80405e055079fb0af0cbe","overlaps_test":false,"full_text_reference":true}, +{"id":"4212","category":"product","split":"test","html_sha1":"30d8e8b5302e87d453675902771b3324f43a17d8","truth_sha1":"b6e3854443708198479d0af27cffb2987a4e192a","overlaps_test":false,"full_text_reference":true}, +{"id":"4213","category":"product","split":"test","html_sha1":"6c578f87a92392dbb0e551eeeff2a2cbe3d0f81e","truth_sha1":"6ec6644dd339bf5e4f92c7b3a60a2cab1fb3866c","overlaps_test":false,"full_text_reference":true}, +{"id":"4219","category":"product","split":"test","html_sha1":"c2ac9afbf6c8f3dca86d82e42f1f4c03356153da","truth_sha1":"8fb7283cfb3f9307bb34e91a89b7375e9dc245c5","overlaps_test":false,"full_text_reference":true}, +{"id":"4221","category":"product","split":"test","html_sha1":"98a0453a56132efac7f6d014d0a614a775f56147","truth_sha1":"9b64958e5a9373274f0237ce0f061ac16255bf04","overlaps_test":false,"full_text_reference":true}, +{"id":"4224","category":"product","split":"test","html_sha1":"a0fb4b27c67d38815b1e9070a13dbda8e69f46ac","truth_sha1":"0a2545f7b293ebafb1ae28d7f2c6e4cfb651ced1","overlaps_test":false,"full_text_reference":true}, +{"id":"4225","category":"product","split":"test","html_sha1":"4b4b70f9413f19e6e7cb9159360d80c84582eaea","truth_sha1":"075d582750e97acb2fc044aa02f90597f8cc8b47","overlaps_test":false,"full_text_reference":true}, +{"id":"4241","category":"article","split":"test","html_sha1":"00b7069a1d813c80e2e85977c55b6c503ae37333","truth_sha1":"dda3ef9e8ac668dfd324af3bc769e4fe921f42fb","overlaps_test":false,"full_text_reference":true}, +{"id":"4263","category":"product","split":"test","html_sha1":"65d6e8f5972aa3d9c919399fc8f070b58742c148","truth_sha1":"473d31193d7209b5a96fb1e9bd7db7974b09c4e2","overlaps_test":false,"full_text_reference":true}, +{"id":"4277","category":"article","split":"test","html_sha1":"8efae542b89d0aa3532b882a5ed3533c84b0681b","truth_sha1":"9a4ee6fb6011a086cd4b7db21150047756331fca","overlaps_test":false,"full_text_reference":true}, +{"id":"4285","category":"collection","split":"test","html_sha1":"4b99225441675ffaf70982d82f2a445bbf86c466","truth_sha1":"61196b7e63412e40083aaf24ac33e671888aee81","overlaps_test":false,"full_text_reference":false}, +{"id":"4291","category":"collection","split":"test","html_sha1":"083cf3e4e9a635848c60d0ca5c32ed3270b98516","truth_sha1":"10bd135460ad7413e4aebeb19b1b4a4a867f2e82","overlaps_test":false,"full_text_reference":true}, +{"id":"4313","category":"collection","split":"test","html_sha1":"acfb33447bc23ff23b75184275f0a6d61f892c6b","truth_sha1":"dc1064a928fd111184544e82ff3b42476415104e","overlaps_test":false,"full_text_reference":true}, +{"id":"4329","category":"collection","split":"test","html_sha1":"d7cf9e6e20167d555377a99ac4b2fec8dc831d92","truth_sha1":"85b63c2d2fadeb53bb48f775d0cec9e544a4c1b2","overlaps_test":false,"full_text_reference":true}, +{"id":"4335","category":"collection","split":"test","html_sha1":"74eaba140b2d6bc3245b6571f372f7a52dab00cf","truth_sha1":"50b643a08ddb0da5af92fe20481cbfa052a81a48","overlaps_test":false,"full_text_reference":true}, +{"id":"4351","category":"collection","split":"test","html_sha1":"c144e73ec273097d6924272ceb4ac6317b47631d","truth_sha1":"bd979b3da2059770615dc735b02c4049e5a511d9","overlaps_test":false,"full_text_reference":true}, +{"id":"4352","category":"listing","split":"test","html_sha1":"150f9c2ef7267faff0d16d09b3b534aed66aca8c","truth_sha1":"6e904cc1216d5e10c7cbfb73fe47b11dca1d486e","overlaps_test":false,"full_text_reference":true}, +{"id":"4360","category":"collection","split":"test","html_sha1":"3b9ee27b6200bc66689f723fa70addb4728d59a9","truth_sha1":"14a1ff0b775c888f5095b09f283f9378888b5ec9","overlaps_test":false,"full_text_reference":true}, +{"id":"4368","category":"article","split":"test","html_sha1":"135984f859474753fddba71e527c9ce857b0b02b","truth_sha1":"b67ecf5c578ffbdd35dbf8e92ab35b2cc6cd56ce","overlaps_test":false,"full_text_reference":true}, +{"id":"4369","category":"collection","split":"test","html_sha1":"dc0205223d753ad9971967cbcbfc000def434235","truth_sha1":"a1585bddb123ef5e2811d7ce4bbcd43d049fa6b2","overlaps_test":false,"full_text_reference":true}, +{"id":"4370","category":"article","split":"test","html_sha1":"8ac7d8d0a1d9c1d0576639151127017bcb053217","truth_sha1":"77f3ed521b29c4f01068b43c7bce621e31b3e09f","overlaps_test":false,"full_text_reference":true}, +{"id":"4371","category":"article","split":"test","html_sha1":"95c42d74778c4eef5f33a2271bf2f32fd0929c8d","truth_sha1":"1ce9590d2368ed2e2791545b8491f7390ba03729","overlaps_test":false,"full_text_reference":true}, +{"id":"4372","category":"listing","split":"test","html_sha1":"ebae1772e727b8fc4ce69844253cfa7df66f73f1","truth_sha1":"c7bb40489c7f49f319c9ed2835b7b0e8d414d591","overlaps_test":false,"full_text_reference":true}, +{"id":"4373","category":"article","split":"test","html_sha1":"c4edb0935b027719d4bc5e04e3e2e9d7e258ad8b","truth_sha1":"dbb1af1a06e8ee5d3629ef1f9a3eeb6a3eb71a38","overlaps_test":false,"full_text_reference":true}, +{"id":"4375","category":"collection","split":"test","html_sha1":"18785b0ced678cbb82f395e0194ba6e2d13a74b0","truth_sha1":"3b17992d1fd88516a85bda484537cdbd4b5b1f7a","overlaps_test":false,"full_text_reference":true}, +{"id":"4376","category":"collection","split":"test","html_sha1":"7a1907cf2d3a644c255a136d17e7821137885a04","truth_sha1":"26a7a3a79faf0ac5ab189ca13557e4421c623e10","overlaps_test":false,"full_text_reference":true}, +{"id":"4381","category":"article","split":"test","html_sha1":"01bdd0060b64494731dcf5fcd40a91de84047594","truth_sha1":"90857597b57e0757e57e40675a35dd0b48122724","overlaps_test":false,"full_text_reference":true}, +{"id":"4394","category":"collection","split":"test","html_sha1":"68bf93c7e0d8e21d0f9a43696dc7693e30371646","truth_sha1":"1af2f72bc388e6825d76dd4a663a885116dbc72c","overlaps_test":false,"full_text_reference":true}, +{"id":"4396","category":"collection","split":"test","html_sha1":"3e7b425121ba6deef4954d5352f375d96cece8cd","truth_sha1":"90645324034b65e405a7d810bd2d2887b3d49924","overlaps_test":false,"full_text_reference":true}, +{"id":"4400","category":"documentation","split":"test","html_sha1":"38689f44b506767c1e0bf453ac7231f3d420cf15","truth_sha1":"a7e51a669d533291acc706c8826c7acb9ea7b1ab","overlaps_test":false,"full_text_reference":true}, +{"id":"4408","category":"documentation","split":"test","html_sha1":"45a9f6bbc6fba15e8bcca33d9c4edfb8e4b97fe8","truth_sha1":"880f7c51a3180fa94ca8090a96297953d054f02b","overlaps_test":false,"full_text_reference":true}, +{"id":"4410","category":"collection","split":"test","html_sha1":"eb6b4caec32a7cb7276762dc8053aeb1a9b07960","truth_sha1":"2843ef30891cc90c1d429a3a29130fc62380fcb4","overlaps_test":false,"full_text_reference":true}, +{"id":"4418","category":"collection","split":"test","html_sha1":"ef5c40fbabd430fcdc2deb7fcd199941d3c08948","truth_sha1":"b24b9ae30735d0bffdaf8f45ac55f5c7ea1f223e","overlaps_test":false,"full_text_reference":true}, +{"id":"4419","category":"collection","split":"test","html_sha1":"c502bda65c953a14d49e47857bf578aec47d333a","truth_sha1":"684158c8871b032740797f3007d445be7cb2d94a","overlaps_test":false,"full_text_reference":true}, +{"id":"4437","category":"collection","split":"test","html_sha1":"c53e3b055ea439df97f951c70d13edd0c03052a9","truth_sha1":"6f5582ccd59158fb56707bc524d61a8581a65fb7","overlaps_test":false,"full_text_reference":true}, +{"id":"4440","category":"article","split":"test","html_sha1":"83ad29374b18e90caf9714a3ec5778e33fc146a6","truth_sha1":"d14f47da13d994dff465802ad2a3da2e2dc2a5fe","overlaps_test":false,"full_text_reference":true}, +{"id":"4442","category":"article","split":"test","html_sha1":"6301f83c0c0b671598d4572d2263af3610c7a160","truth_sha1":"9357c841da486518cb60f7059a0a8b7efaaa8d7f","overlaps_test":false,"full_text_reference":true}, +{"id":"4443","category":"article","split":"test","html_sha1":"577d14e7e8b725aa3d7329e12df406d1448b236c","truth_sha1":"df780f3a5e2fbab758a9f30aa376acabe126bb7e","overlaps_test":false,"full_text_reference":true}, +{"id":"4449","category":"article","split":"test","html_sha1":"db941deef3075899521564e77808016714d8c9f3","truth_sha1":"eeb5303c69f5a7a0e93636b48b76d316eb69fed4","overlaps_test":false,"full_text_reference":true}, +{"id":"4456","category":"article","split":"test","html_sha1":"9a56d50a0973996dd9df69570f7494436f32685c","truth_sha1":"c81c8fde34bf3c8a7d6db957d7986ade07a3db4a","overlaps_test":false,"full_text_reference":true}, +{"id":"4461","category":"collection","split":"test","html_sha1":"144dc1c86b7e483187be34ff440fc1b1a561ce0d","truth_sha1":"f8513c8d8c5ce12780d645eb04526e4a229ef220","overlaps_test":false,"full_text_reference":true}, +{"id":"4465","category":"collection","split":"test","html_sha1":"84ba1f47ed08f2fb843c0e8bb4a949e94c390734","truth_sha1":"30e7be3c65e4fae46b621b009244e98e687338b4","overlaps_test":false,"full_text_reference":true}, +{"id":"4478","category":"article","split":"test","html_sha1":"f943b60bf0ebdc7c57ebed7f5ece981235b706a6","truth_sha1":"19731507f258677876a98d56293e69aed8c6ea0f","overlaps_test":false,"full_text_reference":true}, +{"id":"4482","category":"service","split":"test","html_sha1":"6c36c221c2e45e75f423ddd8a480401edaa43d79","truth_sha1":"d042cbcb7c4f97bba8d97779a2c82293693e4972","overlaps_test":false,"full_text_reference":false}, +{"id":"4486","category":"article","split":"test","html_sha1":"9a0ae25e83d0f33676af7a26bc6a0957a3aac46b","truth_sha1":"b3ee576f9d8e0021c2c694f1af514b035fe54e11","overlaps_test":false,"full_text_reference":true}, +{"id":"4487","category":"article","split":"test","html_sha1":"3a8a4505426571eee554124e0db133dcdbb7fb8d","truth_sha1":"5957dfd55a3cdd2e2cdf7199ef71bee546e1d9bf","overlaps_test":false,"full_text_reference":true}, +{"id":"4495","category":"collection","split":"test","html_sha1":"6d8cdbb44f81fb559d1a89579a9e0be2d5134b25","truth_sha1":"8582eb0f291dc47c12670f538a4f039854ecc869","overlaps_test":false,"full_text_reference":true}, +{"id":"4498","category":"article","split":"test","html_sha1":"0d4941b0da350138c6c9f8c37f6a3009d94b83bf","truth_sha1":"ea0d691c5e847e41085721e19463dc8a639c6226","overlaps_test":false,"full_text_reference":true}, +{"id":"4499","category":"article","split":"test","html_sha1":"791daba1ea343b2ba79da795e0f505f09061a839","truth_sha1":"9e235bf296caaac939518ae122846c70d3dd1dca","overlaps_test":false,"full_text_reference":true}, +{"id":"4538","category":"article","split":"test","html_sha1":"13b3823e0e01f9b857290dc9da216f1b8dce6358","truth_sha1":"1b0a82f2e82d1f9e54055f96c630996bb2b31d42","overlaps_test":false,"full_text_reference":true}, +{"id":"4542","category":"article","split":"test","html_sha1":"439b08e1dab43242784670adee5a2d4378bdb92d","truth_sha1":"804a6075c008fc67cab7fb55fd9fa656cd2dd087","overlaps_test":false,"full_text_reference":false}, +{"id":"4547","category":"service","split":"test","html_sha1":"421a8283f2da8f2ca12f72d84fa603afaf8c77f8","truth_sha1":"899435397752fe378a1051eda044d339d6cddc9a","overlaps_test":false,"full_text_reference":true}, +{"id":"4549","category":"service","split":"test","html_sha1":"dc34653baa6d4f4bba72d5ecfcf6a8ff809c3fbe","truth_sha1":"66de08566877a30bb29dbd39dfd4938cd4968e37","overlaps_test":false,"full_text_reference":true}, +{"id":"4562","category":"article","split":"test","html_sha1":"befc4141ca200af092642765498b4d911d369821","truth_sha1":"d654aae095bac9a4ea387c179b9ed8525da9e892","overlaps_test":false,"full_text_reference":true}, +{"id":"4570","category":"article","split":"test","html_sha1":"f17a544603f9260dc3f9669bf499b4d41f8a003e","truth_sha1":"77aeeab695d1c5ff322577938bad51ea50340292","overlaps_test":false,"full_text_reference":true}, +{"id":"4579","category":"article","split":"test","html_sha1":"85d69aacf6310ec298812ea54bc0bf3979e70ba9","truth_sha1":"38985715befe233014cff7bb78d9d91000dc86c6","overlaps_test":false,"full_text_reference":true}, +{"id":"4587","category":"article","split":"test","html_sha1":"bef2b03b4d8c3e95719fc6fa8d9d08e3c36560d9","truth_sha1":"b478277a2fb60e5bfb3b56dacedbc2b7ca0a4fd4","overlaps_test":false,"full_text_reference":true}, +{"id":"4596","category":"article","split":"test","html_sha1":"8b460a809976f5c2cfb6df0157b78ac29d523654","truth_sha1":"5d52b7f1c7738eea67e35c8efb95d4d6038a419d","overlaps_test":false,"full_text_reference":true}, +{"id":"4598","category":"article","split":"test","html_sha1":"354d7de0dae2345fc9beeed4fbf7e71818ae3049","truth_sha1":"d15ac7043f171e2a9ad735a911a26a763ec9251c","overlaps_test":false,"full_text_reference":true}, +{"id":"4599","category":"article","split":"test","html_sha1":"87b2909a123e8288769a78490caa9a7d527d1871","truth_sha1":"2a474ecafbeea9515443ae3685f5cfe8633080ad","overlaps_test":false,"full_text_reference":true}, +{"id":"4608","category":"article","split":"test","html_sha1":"4ba170c6d3082d9dad948cb8ee9f31e78b362062","truth_sha1":"087ffaa9628e25376ec329be8fdeb432051994a7","overlaps_test":false,"full_text_reference":true}, +{"id":"4621","category":"article","split":"test","html_sha1":"35d090a4b7c69bab3aecc037b9e65e263b0abfef","truth_sha1":"4d88a85edd4bff98206cc1386528e36f065cfd82","overlaps_test":false,"full_text_reference":true}, +{"id":"4633","category":"article","split":"test","html_sha1":"5d496081011d5206e446e8cbc52f150ebb541f55","truth_sha1":"14955383b4c92d8dc3acc5865ae0c12f4304ac5e","overlaps_test":false,"full_text_reference":true}, +{"id":"4643","category":"article","split":"test","html_sha1":"22a9aeb316fd0ab288b8b0fb53dc7d899d7fedc8","truth_sha1":"de11b68f31bb142c7855fc387518946c8d2d225f","overlaps_test":false,"full_text_reference":true}, +{"id":"4644","category":"article","split":"test","html_sha1":"bd9cb6e2e7b9863a5aaf447b8362ad04541357fd","truth_sha1":"8c70a5e745b41ac1c0866a553efc345deabfe1c0","overlaps_test":false,"full_text_reference":true}, +{"id":"4645","category":"article","split":"test","html_sha1":"9355b317fc80d9809fc010d552aeee53421ee030","truth_sha1":"b8a3c0bb3c916c6629f0baa54f436616c9167483","overlaps_test":false,"full_text_reference":true}, +{"id":"4669","category":"article","split":"test","html_sha1":"dc9ca49d48b3b934d65c61e1d605f92cf7685e2c","truth_sha1":"278b5249b759b962f28af62c40d8c0d9a4d53c8e","overlaps_test":false,"full_text_reference":true}, +{"id":"4672","category":"article","split":"test","html_sha1":"2afc52f01c0b6b29f045214b289c5ab4ecf12f9b","truth_sha1":"9e4befa15b675f9c484dbebe515eb3610585e9fe","overlaps_test":false,"full_text_reference":true}, +{"id":"4679","category":"article","split":"test","html_sha1":"17de612a0004b2d348e1af68f5a97f82460c572b","truth_sha1":"b4c6737182cee4874b8294be1b677fef05445b43","overlaps_test":false,"full_text_reference":true}, +{"id":"4680","category":"article","split":"test","html_sha1":"ff7da0f8d610365ecb7f4b776251101f515dcb4d","truth_sha1":"30db9447322d825c6e6f2a7c7dcdb1181215adf4","overlaps_test":false,"full_text_reference":true}, +{"id":"4683","category":"article","split":"test","html_sha1":"c9629e8f1e0d69362eb9bd5af0095d92c3a2aacf","truth_sha1":"1592325ef0345523359ff603b678e8cfd7657595","overlaps_test":false,"full_text_reference":true}, +{"id":"4691","category":"article","split":"test","html_sha1":"f0297f8de37208006f1048653145bc4419e60675","truth_sha1":"da347f526eebf56cb2b338d112743333fc31fedc","overlaps_test":false,"full_text_reference":true}, +{"id":"4692","category":"article","split":"test","html_sha1":"725bf8a7eb6b7e9aa1462b97926dcdcc351942e3","truth_sha1":"10b45651dfb27096ec927e0999e4b40a34776c23","overlaps_test":false,"full_text_reference":true}, +{"id":"4702","category":"article","split":"test","html_sha1":"125adbb0475ff08b442201e4fba25fad57a13524","truth_sha1":"18b42f035d95304b7fd927eccd2bed67f213fefb","overlaps_test":false,"full_text_reference":true}, +{"id":"4706","category":"article","split":"test","html_sha1":"7c535cc150d906c157cd8287799797e2bc5bda17","truth_sha1":"d515c94995d57449c3f0934420032a9d7d8a5036","overlaps_test":false,"full_text_reference":true}, +{"id":"4709","category":"article","split":"test","html_sha1":"969d8952b452f1904fd7134db0de111bdce6b90b","truth_sha1":"db2542b03bc21bcc45d9edc5490c798c504555cd","overlaps_test":false,"full_text_reference":true}, +{"id":"4757","category":"collection","split":"test","html_sha1":"a965a46dc6a1151fdbc956a9b74e03e18c725e25","truth_sha1":"0b49a79ee7e59ce9489bb8ae536f4d9bbfa89a55","overlaps_test":false,"full_text_reference":true}, +{"id":"4762","category":"collection","split":"test","html_sha1":"8defc888f7a56feb922b63117dc2abe79caf918b","truth_sha1":"20651a6e6a5ca8c17367bd8e857002ec40b943c2","overlaps_test":false,"full_text_reference":true}, +{"id":"4771","category":"product","split":"test","html_sha1":"03e57e4735510fc79dab43c304c0999b9e508529","truth_sha1":"32b1744365308cbeab8674381d9f98a910d9a421","overlaps_test":false,"full_text_reference":true}, +{"id":"4780","category":"product","split":"test","html_sha1":"847417a0443d77835a2eb882a76777f6ff59c203","truth_sha1":"348082f44bfce31941adb3b4d5b7dbb0c9a761d2","overlaps_test":false,"full_text_reference":true}, +{"id":"4781","category":"product","split":"test","html_sha1":"1909ea301f594aa4ac875cfc7d69578cfd1556f2","truth_sha1":"fda5b286ad3020a0f25608c4bc539c742d7201bb","overlaps_test":false,"full_text_reference":true}, +{"id":"4787","category":"article","split":"test","html_sha1":"5f9568961a7f701e39d8ab4f84babadf6a34f479","truth_sha1":"e3ecd1769f35dc2db8dd46d80662c2e4e9999b58","overlaps_test":false,"full_text_reference":true}, +{"id":"4789","category":"product","split":"test","html_sha1":"9d2045cb084f8c860b72b836bc0ee624942b74fd","truth_sha1":"6d4a46ec2fc35b65e12059566a3a669453d77289","overlaps_test":false,"full_text_reference":true}, +{"id":"4795","category":"product","split":"test","html_sha1":"30dc0b1a1c4df032abe74f85081903d01c3ba134","truth_sha1":"db78a862f87d57079fbaf43cadff5139d10b61be","overlaps_test":false,"full_text_reference":true}, +{"id":"4830","category":"article","split":"test","html_sha1":"af35b938df94857517ff0d625f18fa1f3652c966","truth_sha1":"26c14656d4960afbb1a4157b88f1274c6866c672","overlaps_test":false,"full_text_reference":true}, +{"id":"4833","category":"article","split":"test","html_sha1":"f9945c66159ee9db72d986d872d0ed6461bd4633","truth_sha1":"96cb1c1c0773d0c34473e285e1876250e93f2c6d","overlaps_test":false,"full_text_reference":true}, +{"id":"4846","category":"article","split":"test","html_sha1":"9b05c43a28aea7c03224f328cccab88b12c67cfa","truth_sha1":"eadbf5b5d2256c8df27db2ad479c82ea9374923d","overlaps_test":false,"full_text_reference":true}, +{"id":"4897","category":"service","split":"test","html_sha1":"d855a66313952b6e76fa8e35bbed4e50bf259b79","truth_sha1":"6eb96cf6ce9bb16ba9d043f4ab86bb129256adba","overlaps_test":false,"full_text_reference":true}, +{"id":"4919","category":"service","split":"test","html_sha1":"71026b855ed10a014f789d2c1f91b20638ebe10c","truth_sha1":"a6182b12d79d50b96bea39fd9f4226ff78479c7c","overlaps_test":false,"full_text_reference":true}, +{"id":"4920","category":"service","split":"test","html_sha1":"93dffcb6e565394b7a372f6db9a6fe5d659c79f3","truth_sha1":"6ee89399675efecd1b7908ac7bfec480a47f51d5","overlaps_test":false,"full_text_reference":true}, +{"id":"4922","category":"service","split":"test","html_sha1":"cae022bbdd9536bc67a6e00137193d2f8b7da0ba","truth_sha1":"06eeaf41c3e691c222b21a941a8f1c5c45c56f15","overlaps_test":false,"full_text_reference":false}, +{"id":"4923","category":"listing","split":"test","html_sha1":"4df10f295e4d979fec0bfd6d398c4e097262817d","truth_sha1":"1909632ee59d1fcb96f52a37b8ba310b506cc410","overlaps_test":false,"full_text_reference":true}, +{"id":"4925","category":"service","split":"test","html_sha1":"a7cb3edb67187e09c834d9c2c93ecf0191aae522","truth_sha1":"24beeab12528f39827d4ada28db7c0e91e5929a5","overlaps_test":false,"full_text_reference":true}, +{"id":"4926","category":"service","split":"test","html_sha1":"40ef2712c9d5f17e6a91cdd0c0aa9ed1ece53752","truth_sha1":"3f895fb039f7d9168fadee7ba8e6d25a2e7dc008","overlaps_test":false,"full_text_reference":true}, +{"id":"4928","category":"service","split":"test","html_sha1":"0bad03714adafc20fe9dd242a1340e41bab53717","truth_sha1":"6e59f32fa4e768f83471cc096c3a11414a228234","overlaps_test":false,"full_text_reference":true}, +{"id":"4930","category":"service","split":"test","html_sha1":"2836d6d5212cf5f95728b481d4e6666ece684ab2","truth_sha1":"5cda6d2c1f8ae18dc020f3b377c3bd6ad0a62161","overlaps_test":false,"full_text_reference":true}, +{"id":"4931","category":"service","split":"test","html_sha1":"9341a883419895ff1eb2fa4f6004ae3892586385","truth_sha1":"03572dcf4472d7e165825d72a565ebabc5c6207d","overlaps_test":false,"full_text_reference":true}, +{"id":"4933","category":"service","split":"test","html_sha1":"4da5a95bce7a45b97d34a3679fe17c6d9bd3bbeb","truth_sha1":"3262d1a634cf1a46aef4440b38078547a430cde6","overlaps_test":false,"full_text_reference":true}, +{"id":"4934","category":"service","split":"test","html_sha1":"9db058581bec8d0ce8c2cd7a1b902648bec17764","truth_sha1":"7443b3d01a52649d7bdafd3888f001c2b5b7ab1f","overlaps_test":false,"full_text_reference":true}, +{"id":"4936","category":"service","split":"test","html_sha1":"688bf802f3bb572992f888ef13f4cfa120ab43a3","truth_sha1":"b44c90095b327e303b963356984cd2cfc87181d6","overlaps_test":false,"full_text_reference":true}, +{"id":"4937","category":"service","split":"test","html_sha1":"aca1c554f7b25c49830e9144b01ea3285413dab7","truth_sha1":"fe46762b9f0b96f597e8c0bc3b1ce2bee31365af","overlaps_test":false,"full_text_reference":true}, +{"id":"4939","category":"service","split":"test","html_sha1":"217bc94801a7a0203fee6975845344220c1aa8cb","truth_sha1":"075d3cd99fe62951beaff855a4a0300934bc458a","overlaps_test":false,"full_text_reference":true}, +{"id":"4941","category":"service","split":"test","html_sha1":"b016134c6dd063c04e3eaf69209cd88ce0921c4e","truth_sha1":"8f2222126a3a76bb4ffaf3f7bc37539eee8f31b0","overlaps_test":false,"full_text_reference":true}, +{"id":"4943","category":"service","split":"test","html_sha1":"46e72bc1396e26fcd3862c766e25e2e693cd26fc","truth_sha1":"30572948f492d3c510a765b576f45d9f9101730a","overlaps_test":false,"full_text_reference":true}, +{"id":"4947","category":"service","split":"test","html_sha1":"fb32ba37723088e5ebd24809243eae0047747efa","truth_sha1":"2ba4db051b5ae9f96a83c9236b736500c12e8646","overlaps_test":false,"full_text_reference":true}, +{"id":"4948","category":"service","split":"test","html_sha1":"acc76d7fc76f2ea0d47633b463f714fe3e22bfe4","truth_sha1":"132fb4c0d5c1618aed8bd7b947fcfc9f319b3dce","overlaps_test":false,"full_text_reference":true}, +{"id":"4950","category":"service","split":"test","html_sha1":"02e2413d9fcdb8223550216e3ab81fc081624095","truth_sha1":"3fe765e624f1fafdabac14f206325bd4f6c5053e","overlaps_test":false,"full_text_reference":true}, +{"id":"4953","category":"service","split":"test","html_sha1":"c2661eb3b3a5dfbecc6da4d14d3de6010da8580a","truth_sha1":"15e5555faea3ec00f0fa4f19d808547768a9b476","overlaps_test":false,"full_text_reference":true}, +{"id":"4958","category":"service","split":"test","html_sha1":"62b14a96b3d623094d455e1abe63af5a25cb0056","truth_sha1":"db7301d46b2ec49226a61f25d310c8c71084d715","overlaps_test":false,"full_text_reference":true}, +{"id":"4959","category":"service","split":"test","html_sha1":"1231ebee4826802f91dcb4b1ed018bbdcd2caa5d","truth_sha1":"b98675019fa3620c701df1117142424c1add2caa","overlaps_test":false,"full_text_reference":true}, +{"id":"4963","category":"service","split":"test","html_sha1":"1b83923f17a01fed9b9d117621d03513d0954660","truth_sha1":"578ed7fa8c5d179a2e188aeab617e38c22af1407","overlaps_test":false,"full_text_reference":true}, +{"id":"4965","category":"service","split":"test","html_sha1":"0f3b837e0c030d2bde982716b2aa17aba060eb31","truth_sha1":"192fa78976ba2fa7afe12499f41faef212060c84","overlaps_test":false,"full_text_reference":true}, +{"id":"4966","category":"service","split":"test","html_sha1":"a98361e0babe6d276327513db8da55a4b01dd3b8","truth_sha1":"3c9dd71df5d31aef1098e88a0772329fa4180243","overlaps_test":false,"full_text_reference":true}, +{"id":"4971","category":"service","split":"test","html_sha1":"f00849b2ff500ebd13d5eb209944bab9e263c689","truth_sha1":"6d3ab8917a5babc32af2687e728d8e88322b7954","overlaps_test":false,"full_text_reference":true}, +{"id":"4973","category":"service","split":"test","html_sha1":"60b87ec9f66fe8abcccf044f73fe9b6ed8be1bba","truth_sha1":"8670343e4fa2388f60b227a27a6b51aa9cf79dd1","overlaps_test":false,"full_text_reference":true}, +{"id":"4974","category":"listing","split":"test","html_sha1":"bdbc0e2589561f117bd8842bbcf864a1f631ed61","truth_sha1":"def5f7627e45a64de4357f9ef482579f08d6adb3","overlaps_test":false,"full_text_reference":true}, +{"id":"4975","category":"article","split":"test","html_sha1":"7d41ac7d0e6d5fa5f47ebbe38397c19dacef0994","truth_sha1":"03c0b409df8dac9d889e6a7841bd08cf657aa6f1","overlaps_test":false,"full_text_reference":true}, +{"id":"4976","category":"article","split":"test","html_sha1":"7a77748e5792efd6b1db51334cae49e57a1f9500","truth_sha1":"2688c4f5a741de2d46fc18d63450383b0a5ad765","overlaps_test":false,"full_text_reference":true}, +{"id":"4978","category":"article","split":"test","html_sha1":"eeffed177cc4a337946f1bf9921c6f73f36daa83","truth_sha1":"24085ff7c85025a436c60af8f3bfc1023ae06cae","overlaps_test":false,"full_text_reference":true}, +{"id":"4979","category":"article","split":"test","html_sha1":"64a5dd2d07c0c24483edfc140bad498682679648","truth_sha1":"2b7c1da655569a031fee1dad198e6866bd1449cb","overlaps_test":false,"full_text_reference":true}, +{"id":"4982","category":"service","split":"test","html_sha1":"58848ccf02c780a3569dd9c14837890f1ad95d34","truth_sha1":"2239541ed07bb364099d8b588e77f4e47f314b51","overlaps_test":false,"full_text_reference":true}, +{"id":"4985","category":"service","split":"test","html_sha1":"1e440c06ba32e5b2834113956d194717792f5ed8","truth_sha1":"ba3939ca7aeb9f7785b5cef55c9f72ee2e971790","overlaps_test":false,"full_text_reference":true}, +{"id":"4986","category":"service","split":"test","html_sha1":"c42d5f474972c0c5c305d6ffb8077f218e69d901","truth_sha1":"5a31df626882878543b0e6cb965fe580588afa61","overlaps_test":false,"full_text_reference":true}, +{"id":"4987","category":"service","split":"test","html_sha1":"f91c9d60c888fc98eceed70a84a7050a31f072cb","truth_sha1":"f1ba827dd744c66754ab42257bbd34395675135b","overlaps_test":false,"full_text_reference":true}, +{"id":"4988","category":"service","split":"test","html_sha1":"0e6cb0a7fcec4899dabadb64d89e548294dc211a","truth_sha1":"430349e872baaf82a93086a63364bbc4b8207446","overlaps_test":false,"full_text_reference":true}, +{"id":"4992","category":"article","split":"test","html_sha1":"079338c8ad25a1f6a10d058525b015d1efdfbb7e","truth_sha1":"889d1aa6fad7fd38f3f348206f1fc26d0244057a","overlaps_test":false,"full_text_reference":true}, +{"id":"4993","category":"article","split":"test","html_sha1":"7fc279d0e7e41783e4ff85e706795fc984b952c5","truth_sha1":"ec73af27ded9dc9bff7f5447777d2b0940f047cb","overlaps_test":false,"full_text_reference":true}, +{"id":"4995","category":"article","split":"test","html_sha1":"8897f37f34c06fd9438f02be2ce0571783086536","truth_sha1":"931838a3495d65d01b039372b0a0b6c52a1c05e2","overlaps_test":false,"full_text_reference":true}, +{"id":"4997","category":"article","split":"test","html_sha1":"d1934d61444819fe6d5d8eeab298e9448062e613","truth_sha1":"c804ba8e17e9c84bef40da422c890ae3b47530a3","overlaps_test":false,"full_text_reference":true}, +{"id":"5001","category":"listing","split":"test","html_sha1":"06ef9a02f5d8a25f59b86b353f4c76e90b24ece6","truth_sha1":"f839acaeb79d7b620933fb81531a20052754e7fb","overlaps_test":false,"full_text_reference":true}, +{"id":"5003","category":"service","split":"test","html_sha1":"fe4cd365c38a7c0144d6117c9b56de20c58c5fa4","truth_sha1":"33236e52ee292312d5f89c0d830b477180d2772d","overlaps_test":false,"full_text_reference":true}, +{"id":"5004","category":"service","split":"test","html_sha1":"ce374618fa4b9f8bb2d2dae128249128f00e1198","truth_sha1":"30a54eff2d6ab896f3af750f6b070751f1a0b435","overlaps_test":false,"full_text_reference":true}, +{"id":"5005","category":"article","split":"test","html_sha1":"832ba2a3f8020a45896bbd4bc9f67cb3ad9a75ee","truth_sha1":"36088b73e51a6aee3e3c9b63ffad3c89193e4691","overlaps_test":false,"full_text_reference":true}, +{"id":"5006","category":"service","split":"test","html_sha1":"c80f56e052aa9f9c4380967766c5d8536f86d00a","truth_sha1":"2227c5e2e06b58a6db8dcf6376afbe300a91eaf2","overlaps_test":false,"full_text_reference":true}, +{"id":"5007","category":"forum","split":"test","html_sha1":"f289efe077dcb1d9624b0c2532ba152172d35b6e","truth_sha1":"d4715298d3d2bb229b3cf00ddd5945b67f5eb89c","overlaps_test":false,"full_text_reference":true}, +{"id":"5009","category":"forum","split":"test","html_sha1":"1edd3182dc310074c6256341a90ade11fe06167c","truth_sha1":"41f2e76d4ab7f5e11c61c6bc6f4f24e5f3e88b52","overlaps_test":false,"full_text_reference":true}, +{"id":"5013","category":"article","split":"test","html_sha1":"6752d30fa69bf2299bfeee1873f15fa558a65376","truth_sha1":"38865abbc4f2ba755a87b84e8a26a51c7249f86c","overlaps_test":false,"full_text_reference":true}, +{"id":"5019","category":"forum","split":"test","html_sha1":"b65451e6d6c1a8b6748c90f2ed34e5bb86ab9fdd","truth_sha1":"62155ec597c78c188c6e8a7ec1f7c92ccc9597bd","overlaps_test":false,"full_text_reference":true}, +{"id":"5022","category":"listing","split":"test","html_sha1":"5a831f41333020fba8dbe79602d50a101c20eda9","truth_sha1":"5bf31853d2b18fa1fc30bc268519d36b6d39a50a","overlaps_test":false,"full_text_reference":true}, +{"id":"5024","category":"listing","split":"test","html_sha1":"c14a0b946aa573e4546b58849900a1e9e913df9a","truth_sha1":"459565687fc801e0713676ef28fff89a591faf81","overlaps_test":false,"full_text_reference":true}, +{"id":"5028","category":"forum","split":"test","html_sha1":"3fb08b3467d54f23660a2d85044a5187a49b7df8","truth_sha1":"721f9fd0fe5473a42716c8a362dc6f54ac237692","overlaps_test":false,"full_text_reference":true}, +{"id":"5029","category":"forum","split":"test","html_sha1":"6f2b928444f74d09cb8ded2961a462d9d560a45f","truth_sha1":"2b7ef9dc639448c00f3bc47a6ea7ca72361bad50","overlaps_test":false,"full_text_reference":true}, +{"id":"5031","category":"article","split":"test","html_sha1":"4c4c9199c9df86cf83d42411a00646205676e5aa","truth_sha1":"1a44ba31adf62dcf4c375a6ab40ab4d627729075","overlaps_test":false,"full_text_reference":true}, +{"id":"5032","category":"forum","split":"test","html_sha1":"ee7265e1dd8852e3b226f0cd30a92eb762ae4e56","truth_sha1":"37a299549688f721e54a74d1a30995ea1ade21c5","overlaps_test":false,"full_text_reference":true}, +{"id":"5033","category":"forum","split":"test","html_sha1":"2b912d7041f57c146c1b5a073c7ae9f38c0780d2","truth_sha1":"8d9d121c9f043de122dc6b582c3d21a6fa4b2d29","overlaps_test":false,"full_text_reference":true}, +{"id":"5039","category":"forum","split":"test","html_sha1":"f769db4cce95d092bb10a94ebd0cd86caaece967","truth_sha1":"c252121448e21f65107e247b9c06dea044818a30","overlaps_test":false,"full_text_reference":true}, +{"id":"5044","category":"forum","split":"test","html_sha1":"a700df21e715977f2efa9ecdc03047a0bdfc671f","truth_sha1":"bc6f4a7521705be302210d7783beafccb1359bf4","overlaps_test":false,"full_text_reference":true}, +{"id":"5049","category":"forum","split":"test","html_sha1":"d28f8ec8a50ccf6ef035dd9eae3672e848bad129","truth_sha1":"3230320dfb20e54fff0c7282e6102fa9e01e88ff","overlaps_test":false,"full_text_reference":true}, +{"id":"5055","category":"forum","split":"test","html_sha1":"2f2671ba7992c38925cb2287aea5c1c2167c740a","truth_sha1":"3e39de68c3a091100017b11ccf76272ab334b994","overlaps_test":false,"full_text_reference":true}, +{"id":"5060","category":"forum","split":"test","html_sha1":"fa229c94c1904f8c42cf6381fd3341934c0821fb","truth_sha1":"10542499e5bc9e11c721237f3c2fa15fdb0c7018","overlaps_test":false,"full_text_reference":true}, +{"id":"5061","category":"forum","split":"test","html_sha1":"65156a652e9582bd4d9d76a8c51f99646e803000","truth_sha1":"2790785b884b6d1222f9f6f6c7e8c595cf35f9e5","overlaps_test":false,"full_text_reference":true}, +{"id":"5063","category":"forum","split":"test","html_sha1":"621a3f8c5b6091a5846704c8d2842bf9ad2e8b83","truth_sha1":"397a11e650490a74f49b5618ed5f7ce0e9558af5","overlaps_test":false,"full_text_reference":true}, +{"id":"5071","category":"forum","split":"test","html_sha1":"feab600d288e752545fc9440d70eae05c5ecccb5","truth_sha1":"b02490fccd6ff8610ed5bb16d3436046d9ce085c","overlaps_test":false,"full_text_reference":true}, +{"id":"5072","category":"forum","split":"test","html_sha1":"4589be7d1355ef8c53478a4a899d7934f80ce21c","truth_sha1":"3daa3eb1cba6f4c80d9a0e9db4c2e6cada46b270","overlaps_test":false,"full_text_reference":true}, +{"id":"5073","category":"forum","split":"test","html_sha1":"b4b9d23e6ccd850c004960e3ad4f2e2d3b8890cd","truth_sha1":"958d94aa753618a094f72e929cc34a3b8ebe64e5","overlaps_test":false,"full_text_reference":true}, +{"id":"5080","category":"forum","split":"test","html_sha1":"0e45df70ed14a21871f0b80d760321ad568db887","truth_sha1":"1723a9460cf7596fc40a9f0d1e185e7036f80861","overlaps_test":false,"full_text_reference":true}, +{"id":"5083","category":"article","split":"test","html_sha1":"14750b3807b0137e26c2c6c3e1887b4f57d64b0d","truth_sha1":"671d53ee8c8c8cfbb2c527f516acd176c1630942","overlaps_test":false,"full_text_reference":true}, +{"id":"5084","category":"forum","split":"test","html_sha1":"f8bd2775a6e6dcf30ddf64d9ccaae3e5de236cf6","truth_sha1":"f53327cbcf5327230b44698ddfa652939c403446","overlaps_test":false,"full_text_reference":true}, +{"id":"5086","category":"forum","split":"test","html_sha1":"6e98d6c3004ded92480796e9a1b7b1d885e4da11","truth_sha1":"20c3af2d778dbd00a7fb7b06398e5d4aaecf428e","overlaps_test":false,"full_text_reference":true}, +{"id":"5087","category":"listing","split":"test","html_sha1":"f84819b5bd4017a2bb0fc86763137602733c06de","truth_sha1":"51b1da8aa1fa5d7c2fb028135ac22e0aa48b0e2f","overlaps_test":false,"full_text_reference":true}, +{"id":"5089","category":"forum","split":"test","html_sha1":"6241ddb812da7747057e06c6ecd776834b609209","truth_sha1":"947b65eba88182f11c0a6405ddde26e908fd2156","overlaps_test":false,"full_text_reference":true}, +{"id":"5090","category":"listing","split":"test","html_sha1":"5f43ad37fbb2e1ec2a2fb21c01b4b147565ad3f0","truth_sha1":"95e2a9624926b1fce9064132d7de00559f49d0e4","overlaps_test":false,"full_text_reference":true}, +{"id":"5092","category":"forum","split":"test","html_sha1":"04e6e2bc7100d6f26e2f3ea5b212dd20914cb8a9","truth_sha1":"c3e2de3e40c49ed5181ea69565c79626e2fae4a9","overlaps_test":false,"full_text_reference":true}, +{"id":"5095","category":"forum","split":"test","html_sha1":"e501196a85585802a782d6cde54c6c52312ba3ce","truth_sha1":"7c0227d17fb79498d9ef344b189032ccf06a459c","overlaps_test":false,"full_text_reference":true}, +{"id":"5097","category":"listing","split":"test","html_sha1":"50edcf920b64419ee61030f33cbee80981142dbc","truth_sha1":"b30ad2199582e15d3f7c3462f65a2ba8636643ea","overlaps_test":false,"full_text_reference":true}, +{"id":"5101","category":"forum","split":"test","html_sha1":"8b9a08555604841a908dd81c4af0aabbb74d6326","truth_sha1":"37d9424e7c0c5faf9c56614622fdfac641579767","overlaps_test":false,"full_text_reference":true}, +{"id":"5103","category":"listing","split":"test","html_sha1":"e18b8886fb716bd25c1303a443f1c7971b41c6b7","truth_sha1":"41866db4a88fbf04778e29343c52009a2a82a4c2","overlaps_test":false,"full_text_reference":true}, +{"id":"5104","category":"forum","split":"test","html_sha1":"76852d64519b771be6d5ad12ca985f3d2c525ab2","truth_sha1":"06909e5e7bde79801bec3e710462074262c9c1c1","overlaps_test":false,"full_text_reference":true}, +{"id":"5105","category":"forum","split":"test","html_sha1":"2167361f17cc724626655e57f359af0fb24aaff1","truth_sha1":"8ce2cb6d27779296e65dbbe917a4659533b27967","overlaps_test":false,"full_text_reference":true}, +{"id":"5111","category":"article","split":"test","html_sha1":"0c56ff3f56c30b5349efb3c1c98445c674459a1d","truth_sha1":"d214a3bececc56c20cb9afd4ab0dc06f94301874","overlaps_test":false,"full_text_reference":true}, +{"id":"5115","category":"forum","split":"test","html_sha1":"663591ab8b52ae040c2b51dbe5048d842f846940","truth_sha1":"78a541547dfdd515d714a6cf24f629092f5eb947","overlaps_test":false,"full_text_reference":true}, +{"id":"5120","category":"forum","split":"test","html_sha1":"9339173ba3ea8512f4c21a4092a4722d9ab35d5e","truth_sha1":"18b10492af3a1161e729712caa92abc22c961fce","overlaps_test":false,"full_text_reference":true}, +{"id":"5121","category":"forum","split":"test","html_sha1":"bd067d06bbc2508cd92635d044aebce4c7600e41","truth_sha1":"0964e82cbaff884d0ce67da57c3fc5331b785f4e","overlaps_test":false,"full_text_reference":true}, +{"id":"5124","category":"forum","split":"test","html_sha1":"27cf40106bd85f5fe8cc9a3ecb1d46251bf3f531","truth_sha1":"10a791af6e2acf30cd26d086ef46c227e3bb2a0f","overlaps_test":false,"full_text_reference":true}, +{"id":"5127","category":"forum","split":"test","html_sha1":"7ff3ba30d1e228d086285108255780ed467537fd","truth_sha1":"684fbb81519c72bdf6ad8dc6b9f03919c5b5b383","overlaps_test":false,"full_text_reference":true}, +{"id":"5132","category":"forum","split":"test","html_sha1":"913f9c3e451c9b70d18b6326dd21ad12889b09c8","truth_sha1":"62420fb8f268401135d2d1e1e7fdd0889bfee0b9","overlaps_test":false,"full_text_reference":true}, +{"id":"5134","category":"listing","split":"test","html_sha1":"c4dc3c5ce54543a1b0ee65e5f346d492756254e0","truth_sha1":"988071c72f236145f9e5d6216bc6faa13bd99f62","overlaps_test":false,"full_text_reference":true}, +{"id":"5144","category":"forum","split":"test","html_sha1":"d6a9b05e980dc2c3ec549adf02bedc0f86a830e7","truth_sha1":"85a324441e23fc6771dc398fb846f8cfa7fdcf5f","overlaps_test":false,"full_text_reference":true}, +{"id":"5148","category":"forum","split":"test","html_sha1":"aa0302d99a1dcfde10a2cc31b56d5e43719c0186","truth_sha1":"1886e06e837e39aac399e1fb329a58fac6c9847d","overlaps_test":false,"full_text_reference":true}, +{"id":"5149","category":"article","split":"test","html_sha1":"64a67cfe81752c8c61c3db1710cdcc32dcee3263","truth_sha1":"728b3697ec3530d82eeb22b96f09b1599f641e5e","overlaps_test":false,"full_text_reference":true}, +{"id":"5151","category":"forum","split":"test","html_sha1":"732d7b2635e31105109d318e29c2bd21a04173d3","truth_sha1":"a0942580916ed9f1c211a028e1300add4b71d1c5","overlaps_test":false,"full_text_reference":true}, +{"id":"5152","category":"article","split":"test","html_sha1":"3e3bdb746657862745bc7c64102dddca3e41f1be","truth_sha1":"20d5752a0200874c474128c37445f2b4b3816f1f","overlaps_test":false,"full_text_reference":true}, +{"id":"5153","category":"forum","split":"test","html_sha1":"6f8dbb119d132400b4aca7b03cdd8a81467ddcca","truth_sha1":"d330ae5be922236a248c5d7d91949cb1f11965fc","overlaps_test":false,"full_text_reference":true}, +{"id":"5155","category":"forum","split":"test","html_sha1":"9889fdcb856cdf4b257fe4b10887b35b037adfc0","truth_sha1":"c17a3bd0cdfc80d5e9c284755aab0f38d510602f","overlaps_test":false,"full_text_reference":true}, +{"id":"5162","category":"article","split":"test","html_sha1":"bfc73fa71331393156625e39f48cdaa9a9e8629a","truth_sha1":"5038ce128cc8a2845729488dcafd4330017e4390","overlaps_test":false,"full_text_reference":true}, +{"id":"5163","category":"article","split":"test","html_sha1":"3c98746002fc38c86ad4a932764aad4d585001bd","truth_sha1":"c0a7767e5124ed030b5080b9a9ff5c5023c5b167","overlaps_test":false,"full_text_reference":true}, +{"id":"5164","category":"article","split":"test","html_sha1":"d383849f73835a6bddb14f11804638869c0a3768","truth_sha1":"ef76af555f78d220d9acd62716c8af9e2aaa77a7","overlaps_test":false,"full_text_reference":true}, +{"id":"5166","category":"service","split":"test","html_sha1":"a346b6107eaf1ee3a47d59b6149969c384e969cc","truth_sha1":"9136c7aa4200b08f0102703ee28338e16956f3bb","overlaps_test":false,"full_text_reference":true}, +{"id":"5167","category":"article","split":"test","html_sha1":"bbbf2269b043d0fa982458f2fa225786f9f07128","truth_sha1":"44f344a6bd524d198688c118e794b044bb52359c","overlaps_test":false,"full_text_reference":true}, +{"id":"5168","category":"article","split":"test","html_sha1":"4646db0aa3e5248987a01a7b93a369fead84bd48","truth_sha1":"87bbf59da56a749d6e1dc4be82cc418f10e753aa","overlaps_test":false,"full_text_reference":true}, +{"id":"5170","category":"article","split":"test","html_sha1":"78321e2ed7f851f769b5b0ab449331a9e1294be3","truth_sha1":"b5b62d67056d669d5f99e2578f76290a3b78c998","overlaps_test":false,"full_text_reference":true}, +{"id":"5171","category":"forum","split":"test","html_sha1":"75b043bdd5e84d691e9a7703e67bfe722a5570b3","truth_sha1":"99149b6a40d7d36f567f5783b9bf0b3a76cd49c5","overlaps_test":false,"full_text_reference":true}, +{"id":"5172","category":"forum","split":"test","html_sha1":"e23cd03de543c283db924a6a1b974145d776f50f","truth_sha1":"c05e0f1d6245a91a232a9342d9c6244a36ff69da","overlaps_test":false,"full_text_reference":true}, +{"id":"5175","category":"forum","split":"test","html_sha1":"b4e2114981deff45f7841480bd1a06b6edd63622","truth_sha1":"6be05058978b610370f9f97c73fc4e6d2f26b778","overlaps_test":false,"full_text_reference":true}, +{"id":"5176","category":"forum","split":"test","html_sha1":"87c53b4825fe1d6503b8d63aae68fc68afa02ae0","truth_sha1":"4d5265c1f6e082a78f4ec8700ab71d76491b0565","overlaps_test":false,"full_text_reference":true}, +{"id":"5177","category":"forum","split":"test","html_sha1":"da3768b366c9b3f96515310309b4faa558b62381","truth_sha1":"97ce466b106c66e579093e4a73cb889dbf570482","overlaps_test":false,"full_text_reference":true}, +{"id":"5178","category":"forum","split":"test","html_sha1":"508a7869e496bb12e25dc370cf59218f383a3268","truth_sha1":"a86dc65cfe9abe872008cf17d382ff2e5759a293","overlaps_test":false,"full_text_reference":true}, +{"id":"5179","category":"forum","split":"test","html_sha1":"19566f9b514eaa31d44e131bde77da8d3892ad18","truth_sha1":"f6c74070cf9a8b7d153b030f3f94519f994e1b79","overlaps_test":false,"full_text_reference":true}, +{"id":"5180","category":"forum","split":"test","html_sha1":"16b3e0be4dc42c15b2b6a3f4f47bf4b8b2ba8f8a","truth_sha1":"dc8fa600d177f9f9c7b008a58b9ddc7c6878d15a","overlaps_test":false,"full_text_reference":true}, +{"id":"5181","category":"forum","split":"test","html_sha1":"f60d5c73edaf512d59c21feb2bfffc4dff3e3f33","truth_sha1":"aaa7a41ae6817c0209c2d7f1eff7accac6d957eb","overlaps_test":false,"full_text_reference":true}, +{"id":"5182","category":"forum","split":"test","html_sha1":"d3f591222cadf70ce112fe762418035769e9ffa0","truth_sha1":"c494c552035d2b2a255792a5d388d116344cb562","overlaps_test":false,"full_text_reference":true}, +{"id":"5183","category":"forum","split":"test","html_sha1":"1475b2cd22b0378fbb7a2996e2fae1fce408a656","truth_sha1":"4c1b0af50e25f49b2d833361a37f4cdeaa0ba6d4","overlaps_test":false,"full_text_reference":true}, +{"id":"5189","category":"article","split":"test","html_sha1":"800e4b59e7e404074755663b58040b19be185fb5","truth_sha1":"c3b1bc20557c91c9c76591647fa42b52097c8fbb","overlaps_test":false,"full_text_reference":true}, +{"id":"5191","category":"forum","split":"test","html_sha1":"9faa8e0774f4cb50bf878eaaa75c733275916140","truth_sha1":"f369a54d961462baa70ac13b48056ba8e7d22e75","overlaps_test":false,"full_text_reference":true}, +{"id":"5195","category":"article","split":"test","html_sha1":"7b67706641e295bc7bd85f21979f4be674bf2b5e","truth_sha1":"b97f74495a67c3d2a2d42ec1bb5f774b5853e68f","overlaps_test":false,"full_text_reference":true}, +{"id":"5197","category":"listing","split":"test","html_sha1":"4f897ad2b922f9ca1a5effde48da45187ccf0e2f","truth_sha1":"07c297c05d628e8dc58008e96b58c3ba76e435f7","overlaps_test":false,"full_text_reference":true}, +{"id":"5199","category":"forum","split":"test","html_sha1":"0ec56fa50ca69a1c30e86b68e0ac1e56bf5ecbee","truth_sha1":"552425db208c5bbc794d6a0327e33bdb0cba2d53","overlaps_test":false,"full_text_reference":true}, +{"id":"5201","category":"article","split":"test","html_sha1":"dce7b4fe4122a6f319cdaea11d0b5275c8c61958","truth_sha1":"75d62c1abcf5dd1d04257bbed0cf09a9b995e2b7","overlaps_test":false,"full_text_reference":true}, +{"id":"5202","category":"collection","split":"test","html_sha1":"f1afd625e123c22075238abb34ea961f8bf9e1c6","truth_sha1":"f304802faedfddd5defa28811269b2aa01dc0528","overlaps_test":false,"full_text_reference":true}, +{"id":"5203","category":"article","split":"test","html_sha1":"d92fea339d33d9876e0357845435ebad116c026e","truth_sha1":"b325028af6fae704667ffaf2e8041460c301efac","overlaps_test":false,"full_text_reference":true}, +{"id":"5205","category":"article","split":"test","html_sha1":"35fc74d290f6cafd06789eb72e72d0a3db61981a","truth_sha1":"e1e06cabdb370bc74721b70f3799e56ea1629fd7","overlaps_test":false,"full_text_reference":true}, +{"id":"5208","category":"listing","split":"test","html_sha1":"56c7c7cbf603f261935f7417ea11dcc139eb6dc6","truth_sha1":"912c379033a4de5eebb9101496868fad44751622","overlaps_test":false,"full_text_reference":true}, +{"id":"5209","category":"article","split":"test","html_sha1":"860d4a0f9b20c2f1c795b4f950adf3e17c634425","truth_sha1":"ad38d01d8a9f697888141afc5abb794be3279fc2","overlaps_test":false,"full_text_reference":true}, +{"id":"5210","category":"article","split":"test","html_sha1":"81374be431163aef80c04b883fb6cc18a9b6e966","truth_sha1":"d34b25e42643816233723a5068cf43655c9f3d72","overlaps_test":false,"full_text_reference":true}, +{"id":"5215","category":"article","split":"test","html_sha1":"61f0b57e807034368e63de1e6ec4ee3a0a265555","truth_sha1":"f27d4f373f8793195e3d5d4519979dba002eb325","overlaps_test":false,"full_text_reference":true}, +{"id":"5216","category":"article","split":"test","html_sha1":"6a8cf83b24f026dade9aad7f688880c02180a82b","truth_sha1":"17566c6cc3f40b227a3eb4f2d433f168f9a21bc6","overlaps_test":false,"full_text_reference":true}, +{"id":"5218","category":"article","split":"test","html_sha1":"9e35a4684e0b60c4dec1f343122f256c6d1375e2","truth_sha1":"7807bbda017c6603da153cf0c0ce2d59b77fd0af","overlaps_test":false,"full_text_reference":true}, +{"id":"5225","category":"article","split":"test","html_sha1":"dd46cc381dbf9a03fd0112da83aaa2ef4cb73487","truth_sha1":"e977c2e558bed8de31240fe83c6762e0bd987f37","overlaps_test":false,"full_text_reference":true}, +{"id":"5233","category":"article","split":"test","html_sha1":"efaed4d865442ab10f4a0463cb5eab798a01c21e","truth_sha1":"d67e61e3625ede246935830956c19cb4ff267722","overlaps_test":false,"full_text_reference":true}, +{"id":"5234","category":"listing","split":"test","html_sha1":"1a970210895087e4afd0b8f2f9e626cce67a1e1b","truth_sha1":"983c9bd54aeb2fd85df4b7490f56a2376faa11f4","overlaps_test":false,"full_text_reference":true}, +{"id":"5235","category":"article","split":"test","html_sha1":"6f5a31e131355d0b7099474586c9305d2c2c1705","truth_sha1":"e6bbc4d44e6918746c9e2e057addef637e9f589c","overlaps_test":false,"full_text_reference":true}, +{"id":"5241","category":"listing","split":"test","html_sha1":"c2e0e86cc5997e0a19710f77319dd6cf9cadeef5","truth_sha1":"f31cac8618b541abf0c8c6094fc383fb6d7950db","overlaps_test":false,"full_text_reference":true}, +{"id":"5245","category":"listing","split":"test","html_sha1":"4e1127fb42eab1c3998074ba5b715df0e916b9e6","truth_sha1":"32b7ff5b9911b63c1160f2a01a29b8b7570fe40e","overlaps_test":false,"full_text_reference":true}, +{"id":"5247","category":"listing","split":"test","html_sha1":"25c560915cdf71f1d594eca891addba5d3f887bf","truth_sha1":"7158c1d5e1ce6b250333666094e46f53ce750a5d","overlaps_test":false,"full_text_reference":true}, +{"id":"5248","category":"listing","split":"test","html_sha1":"a9cab4ce398bb735acf1fc0b51fb5c2a9a7744a2","truth_sha1":"368f85dc9a81497262032409696c53b8c25e359c","overlaps_test":false,"full_text_reference":true}, +{"id":"5249","category":"documentation","split":"test","html_sha1":"179d6089c6098fea81cd5d1f4fb45d23fab87f14","truth_sha1":"7d966d05036be34cdf082fd651e29f6d6db9e481","overlaps_test":false,"full_text_reference":true}, +{"id":"5252","category":"article","split":"test","html_sha1":"ce5b3f49de66acabb8518ad258c20db919263f90","truth_sha1":"374cd64fc28a9842ab27413b33dd7e35d7168867","overlaps_test":false,"full_text_reference":true}, +{"id":"5253","category":"article","split":"test","html_sha1":"c6f483779df5240f5db8f6bc6f553a8e6b08dd9d","truth_sha1":"7ef325d6962555e174934b47deb34921b9bf4a17","overlaps_test":false,"full_text_reference":true}, +{"id":"5261","category":"listing","split":"test","html_sha1":"f4c3227859d47117389c846af2e3039364874a50","truth_sha1":"15d852fc4be65614dcf39338c7594da3570bc404","overlaps_test":false,"full_text_reference":true}, +{"id":"5263","category":"service","split":"test","html_sha1":"70e3bac1477733218e0f13f93a16e8c66c2b7e67","truth_sha1":"fa0c4557ef858691acc7ef4c131dacbefb29b9eb","overlaps_test":false,"full_text_reference":true}, +{"id":"5264","category":"service","split":"test","html_sha1":"9e107576582e061d792b4a15653ad0ef69eacbe5","truth_sha1":"c2aac52a05d9873022a99a55aa0a3d6c19254570","overlaps_test":false,"full_text_reference":true}, +{"id":"5266","category":"article","split":"test","html_sha1":"cc99e5c459cc3b4ef8558ae031922de98ab37398","truth_sha1":"91e9c5f278ac33e02656d979c5f68995ef760084","overlaps_test":false,"full_text_reference":true}, +{"id":"5268","category":"service","split":"test","html_sha1":"33a6b03831b23c05724e5876fee1968ee90d2c6d","truth_sha1":"33d18cac7e1143be0709e4d8bce09214fd11cbb4","overlaps_test":false,"full_text_reference":true}, +{"id":"5269","category":"service","split":"test","html_sha1":"d001b795c4b89294e83df1e8837257cff8e273b2","truth_sha1":"99d0f2818de31d8d0c9822a8ac7d5f739e02b2ea","overlaps_test":false,"full_text_reference":true}, +{"id":"5273","category":"listing","split":"test","html_sha1":"cb44ce2ae03dec259df56c66bd75950223fd9a93","truth_sha1":"fee47169511e406e68cb5f7c3dd6cd2ee6c0929f","overlaps_test":false,"full_text_reference":true}, +{"id":"5276","category":"listing","split":"test","html_sha1":"00d531e715a8cbf56d69747a6c7171b88ed0456c","truth_sha1":"d95d1f1e9d9f50f8038293e9b3e67836c7f349ac","overlaps_test":false,"full_text_reference":true}, +{"id":"5277","category":"listing","split":"test","html_sha1":"ee8a8daaeeac6a3e54278374502b37fbcfde73b9","truth_sha1":"6fb9f7fdc02d1e1021b1ed7944f815dac2f949aa","overlaps_test":false,"full_text_reference":true}, +{"id":"5285","category":"listing","split":"test","html_sha1":"869db573e7fdb962b748ce54b790fbba76c803e9","truth_sha1":"22045cd87522e00afcdaf3d854f87a82dbbb1be6","overlaps_test":false,"full_text_reference":true}, +{"id":"5286","category":"documentation","split":"test","html_sha1":"5243af09ec52e4097fe0f342792efa93db39c2a6","truth_sha1":"60dc14372994c414916a6e4ce2c307b078697880","overlaps_test":false,"full_text_reference":true}, +{"id":"5287","category":"documentation","split":"test","html_sha1":"0847bf1ab7456cfe5e610054ae45252eb8306ec6","truth_sha1":"9a872c5b08e26f17e6a8b7ae9a6b802e9908a1ac","overlaps_test":false,"full_text_reference":true}, +{"id":"5289","category":"article","split":"test","html_sha1":"098f8918e53f813e29d1cc1fec93fa9154d98c0b","truth_sha1":"5ca43e54473deabdfc25a1e6d0ead50c20b5fd1f","overlaps_test":false,"full_text_reference":true}, +{"id":"5290","category":"article","split":"test","html_sha1":"7cf88d04369897aadd423a2d61dac4cd73a14bff","truth_sha1":"c50c3d0e504ef5f29738a91eede9765318789053","overlaps_test":false,"full_text_reference":true}, +{"id":"5291","category":"article","split":"test","html_sha1":"0bb9968eab2cedc9d7e951b23e1ba3374cf1786e","truth_sha1":"1dbfc8cf9d9f998a1d50a8394f55e084b810f466","overlaps_test":false,"full_text_reference":true}, +{"id":"5292","category":"article","split":"test","html_sha1":"46c5d3cc8fdb9bbea385113edf7a699a73ad2544","truth_sha1":"aada168c78862ad3349971912ffe1c24a5746545","overlaps_test":false,"full_text_reference":true}, +{"id":"5294","category":"article","split":"test","html_sha1":"76d26251a5698bb52f4fe69d5801a0216fdf4fc8","truth_sha1":"fd252618816dc40fe8fa3819722e2735c9858720","overlaps_test":false,"full_text_reference":true}, +{"id":"5295","category":"listing","split":"test","html_sha1":"0b31a9b4a329bf946764d62371709927f8b4921e","truth_sha1":"d818b43bd1c6c412420ec068a77f9a6c750666b6","overlaps_test":false,"full_text_reference":true}, +{"id":"5300","category":"listing","split":"test","html_sha1":"4394e8190c0462a28b8e16aaf30a76f8a9efe003","truth_sha1":"4a35df87d39581b922c5aea672cf5c8f36c4d194","overlaps_test":false,"full_text_reference":true}, +{"id":"5301","category":"listing","split":"test","html_sha1":"2fbe33463c062d252a5c9ba60a940b04dd4f616e","truth_sha1":"94e439d24a9d51a07b2df2e051d0ad1c50fdd197","overlaps_test":false,"full_text_reference":true}, +{"id":"5302","category":"listing","split":"test","html_sha1":"b34e21994dc49ea912284780c02b59b80569c388","truth_sha1":"7af3730329e5417890f358f11df22306cdb567f7","overlaps_test":false,"full_text_reference":true}, +{"id":"5314","category":"listing","split":"test","html_sha1":"ab04b40bed24da3d7b9cffd74045a4c7d7bb6b01","truth_sha1":"eaa4b3038b78cc2a4ecf08c430ed92f759fffa9e","overlaps_test":false,"full_text_reference":true}, +{"id":"5316","category":"service","split":"test","html_sha1":"97ced78c1deb51a9021288b311b00f5bda664653","truth_sha1":"12f8f074f34b91e3e594e2192c267b9bd100a2e1","overlaps_test":false,"full_text_reference":true}, +{"id":"5317","category":"service","split":"test","html_sha1":"3ddf59ee45c766458b660aca7876383cd939e343","truth_sha1":"df1da2414e865413e731d93d462c4e8572cb71ad","overlaps_test":false,"full_text_reference":true}, +{"id":"5318","category":"service","split":"test","html_sha1":"ef018558d3f3b97fcc6095672a5776d7b501ae1e","truth_sha1":"8ad44ba7b05337675a870e71fb31a3aaaf529c1f","overlaps_test":false,"full_text_reference":true}, +{"id":"5324","category":"article","split":"test","html_sha1":"d28a15897224e8ee5d54726ed48d29b44b3ec492","truth_sha1":"0a1323e7940d0e88073504e3568a2c1e14742c69","overlaps_test":false,"full_text_reference":true}, +{"id":"5326","category":"article","split":"test","html_sha1":"e2da1d65777cfd390e7636c748280aa89a6004b4","truth_sha1":"a1f2281ea650f14e7b75e4e68e6c1317485d6e8f","overlaps_test":false,"full_text_reference":true}, +{"id":"5330","category":"article","split":"test","html_sha1":"ddb1f7ca3a8ae0012a20dc197dcdde8bba53a2f4","truth_sha1":"d89cb61f9f379d930fbab48897125fe0248b0c66","overlaps_test":false,"full_text_reference":true}, +{"id":"5335","category":"listing","split":"test","html_sha1":"74fbd8c643117e0c8eabb81fb11a06aebdc1cc22","truth_sha1":"80a420005bb03fee19d7c13dab8813d3d0bbe88b","overlaps_test":false,"full_text_reference":true}, +{"id":"5341","category":"service","split":"test","html_sha1":"2e3fe955703a78d20abb4b908dc3690c8939bb67","truth_sha1":"fdda62c19ddef5d14154c9aab863070c08ebd24b","overlaps_test":false,"full_text_reference":true}, +{"id":"5342","category":"service","split":"test","html_sha1":"5f3ccebca04b346a3694985b9fc0e4fcebad9f75","truth_sha1":"a25b6bc9892b8a54eab7b3f84b0f54463a3446a3","overlaps_test":false,"full_text_reference":true}, +{"id":"5343","category":"listing","split":"test","html_sha1":"068cf9734db01dc7dba137df4b3ba42848f3b26b","truth_sha1":"0edbca4d1fb54af217570840571fd8ac1edfbf0b","overlaps_test":false,"full_text_reference":true}, +{"id":"5345","category":"service","split":"test","html_sha1":"0ecf7180b9a58642554db537182ad95b2045b957","truth_sha1":"b8c791253f082a36fe99ddb45b53c14f4ab7ed83","overlaps_test":false,"full_text_reference":true}, +{"id":"5346","category":"service","split":"test","html_sha1":"07108f26c0a6d6a894ba406d89b5864a32f2c0ca","truth_sha1":"99014b228cdbf3d5ee2b53f266d65ce72c4cb7c6","overlaps_test":false,"full_text_reference":true}, +{"id":"5350","category":"article","split":"test","html_sha1":"77520394047a9c647bbcfd52b4b293812eb49b22","truth_sha1":"99c6f33e0a3619be6c5a94debda0ea348897d1a2","overlaps_test":false,"full_text_reference":true}, +{"id":"5354","category":"listing","split":"test","html_sha1":"a80006594871fbd323c6b11002fee895989efbb0","truth_sha1":"321e0e1f14f247570c8fa47ccb2d2740d83cba03","overlaps_test":false,"full_text_reference":true}, +{"id":"5356","category":"service","split":"test","html_sha1":"a1c5c04bd3a4e38f43cc51e82177f3257331659e","truth_sha1":"59b3ced8022eb2359f7442f393b246eeabd6ebf5","overlaps_test":false,"full_text_reference":true}, +{"id":"5357","category":"article","split":"test","html_sha1":"6873e3b23409bc47082a8f5c448bad6a141ec000","truth_sha1":"13463eefd95579de6c05ef6c3867b49cdb7caa0c","overlaps_test":false,"full_text_reference":true}, +{"id":"5364","category":"service","split":"test","html_sha1":"f26356f016ebd84c59e86f0b5e9380557dc31a4b","truth_sha1":"e2ceb3a150dee21f9ea68540c1ad767ed627dd87","overlaps_test":false,"full_text_reference":true}, +{"id":"5371","category":"documentation","split":"test","html_sha1":"33bb9116857692293d6357451db5992d486215af","truth_sha1":"1469af86399112a7ee286e91dfed9349fdb180a1","overlaps_test":false,"full_text_reference":true}, +{"id":"5372","category":"documentation","split":"test","html_sha1":"c7de9d8ff094df6e0a59364bf311d28f121e3bf4","truth_sha1":"cc22dbd08e79ba86d6e8a63f51fdc30cf73ac8ca","overlaps_test":false,"full_text_reference":true}, +{"id":"5373","category":"documentation","split":"test","html_sha1":"b983abdb81a6b96a4162a98ad65a2f230a348eda","truth_sha1":"231cf5083deeceb6cfba5d8153da62f20b3abd2a","overlaps_test":false,"full_text_reference":true}, +{"id":"5374","category":"documentation","split":"test","html_sha1":"5d17fe4eaf9ed312f0a874e9d15d8ec0a0e2f8f2","truth_sha1":"5813d81410038e7a83c20f0f27f9f111a51a0c66","overlaps_test":false,"full_text_reference":true}, +{"id":"5375","category":"documentation","split":"test","html_sha1":"1f514cdb81addf0b0054753448cdd2b0cb57555c","truth_sha1":"61cbc30e3fcc8ccce3d74f3f99deecde999c2b86","overlaps_test":false,"full_text_reference":true}, +{"id":"5379","category":"documentation","split":"test","html_sha1":"9010c4ad214e22ea1dc2a676c00cb8b5baa0b5b7","truth_sha1":"254a1719eadb447b6a814939cfe2dfba1657c529","overlaps_test":false,"full_text_reference":true}, +{"id":"5380","category":"documentation","split":"test","html_sha1":"e8c2f8826639caecd6fe5a344fad26adc973612d","truth_sha1":"bf93d413e8677eede920593d94630f9ccd3edc7a","overlaps_test":false,"full_text_reference":true}, +{"id":"5381","category":"documentation","split":"test","html_sha1":"c1895743f62f479e5a65bab3daf6a6ea0cd88dde","truth_sha1":"aab8432e8a2def43bbd0f7df128160f11de2cc71","overlaps_test":false,"full_text_reference":true}, +{"id":"5382","category":"documentation","split":"test","html_sha1":"7be4a600ff6e139f7c6ad9b1141c5f569b8e9187","truth_sha1":"b6600bf5cc80c8540ba28d51c35b7e0458615991","overlaps_test":false,"full_text_reference":true}, +{"id":"5384","category":"documentation","split":"test","html_sha1":"16bf19416937712c05392bd36d89bcbf5545bd1c","truth_sha1":"75b5ba7726d8dc4ff06835af828c67918ba62956","overlaps_test":false,"full_text_reference":true}, +{"id":"5385","category":"documentation","split":"test","html_sha1":"94f7bb69f6e558657a1f96820727c2738e4753f2","truth_sha1":"253187d45d796e9d4d142b3e3e408d6f4bd91757","overlaps_test":false,"full_text_reference":true}, +{"id":"5388","category":"documentation","split":"test","html_sha1":"d661a6f514b404687be6a72ea2d497375ebec54b","truth_sha1":"d31a4fb213115a3ef233b81389ca19d05d687a49","overlaps_test":false,"full_text_reference":true}, +{"id":"5390","category":"documentation","split":"test","html_sha1":"66bf390dfc205c80c428d3a3a4033f70452ed420","truth_sha1":"39dffcd2c33bfc112ed0f3dc32dbacf1f24b5f3d","overlaps_test":false,"full_text_reference":true}, +{"id":"5393","category":"documentation","split":"test","html_sha1":"9a89df652d1123cf6aee4cc5cee0377eab158536","truth_sha1":"54bdd5b12b06c934eb1afd090bb5e6a7972d3c3d","overlaps_test":false,"full_text_reference":true}, +{"id":"5394","category":"documentation","split":"test","html_sha1":"e5b25601b188fd58f8e3b3be6e6fd1150f5f53c8","truth_sha1":"16601f9cba30d200e8b041d17aa2c92b93fec5b3","overlaps_test":false,"full_text_reference":true}, +{"id":"5395","category":"documentation","split":"test","html_sha1":"b43214da205958743bef8d0c2f96f8184b2c15c1","truth_sha1":"36a80cc7a65a6ddef1cbd28c0dc16104d06e30e4","overlaps_test":false,"full_text_reference":true}, +{"id":"5406","category":"article","split":"test","html_sha1":"1fad251dac2fe0c2ad1beb40eaaeb906ed5d88bc","truth_sha1":"46836c479019ad455e138240bf7bc639b6695f7c","overlaps_test":false,"full_text_reference":true}, +{"id":"5407","category":"article","split":"test","html_sha1":"2c32bf26f24c25e53fc9ffb5bf368adc7b9b8560","truth_sha1":"1c44ba32647ce03a1d50f96920ef1ba6ee9da78b","overlaps_test":false,"full_text_reference":true}, +{"id":"5409","category":"documentation","split":"test","html_sha1":"0c2a6a20fca26f9fc7df585acbc24250651c0209","truth_sha1":"6286172a4b23ff77c2029ee1836785adda4bdde5","overlaps_test":false,"full_text_reference":true}, +{"id":"5410","category":"documentation","split":"test","html_sha1":"362ef552432397dc1ddbf2665000c44ee3b273c7","truth_sha1":"f15562e345e95ddb0244691ef39586540866fb32","overlaps_test":false,"full_text_reference":true}, +{"id":"5411","category":"documentation","split":"test","html_sha1":"06ae9eef8da2fc71e59a5658207c63ff6155ba50","truth_sha1":"41f90bad9c9b54166d285d6962f7cbadd293b2bd","overlaps_test":false,"full_text_reference":true}, +{"id":"5412","category":"documentation","split":"test","html_sha1":"ffe7b8970d022fb9807abf7760660d76c480ca4e","truth_sha1":"27dbcf2a81afdcb8ab6ce02ce27715905f5c0e29","overlaps_test":false,"full_text_reference":true}, +{"id":"5413","category":"article","split":"test","html_sha1":"e6cea41e2008233ce3f4b966ff61794f69a05758","truth_sha1":"404df2dd303b51afcf8e0fbb7ca553bf7d2f1694","overlaps_test":false,"full_text_reference":true}, +{"id":"5415","category":"documentation","split":"test","html_sha1":"ff33ec00ecdec8b122524affd1a461eb04481478","truth_sha1":"4c65df608ced1cc5ccbfdc893b8172e7cc04bbc1","overlaps_test":false,"full_text_reference":true}, +{"id":"5416","category":"article","split":"test","html_sha1":"0cc72523f23896f29ba7bc7b2654de125c92fbf4","truth_sha1":"5fb1677a3817be564f3e382b0dfed1b556238ac9","overlaps_test":false,"full_text_reference":true}, +{"id":"5417","category":"documentation","split":"test","html_sha1":"7e3152e631f6583856db6e68bb2784bd7a75ee32","truth_sha1":"6d540a5807c414cdc2a10b76c7cf235f0a867515","overlaps_test":false,"full_text_reference":true}, +{"id":"5418","category":"documentation","split":"test","html_sha1":"ffe9d57519dda5c3eb8cb0f288ff6e926ca7b62e","truth_sha1":"f290e1c76e84e50281cc3c1e20721567c798babc","overlaps_test":false,"full_text_reference":true}, +{"id":"5420","category":"article","split":"test","html_sha1":"dafe91c9b37e805c700f97b0eba07b65380b4641","truth_sha1":"bccad8885c8520defa51f2fac3b4dcc4f1ffc49f","overlaps_test":false,"full_text_reference":true}, +{"id":"5422","category":"article","split":"test","html_sha1":"390028aa0d8f846aa2b272f1624227087b564e6c","truth_sha1":"be61ca7d9fd89521baf3d801df5e68f977c2341c","overlaps_test":false,"full_text_reference":true}, +{"id":"5423","category":"article","split":"test","html_sha1":"9440298d99fbda1bcd927b0d091ad3a2337426e4","truth_sha1":"e3f4ffd78bac46f269bc8d5202bf5b9483d908c8","overlaps_test":false,"full_text_reference":true}, +{"id":"5426","category":"documentation","split":"test","html_sha1":"431ffe4d96c63e1fc4d3728314d9590587c55fed","truth_sha1":"91655f6103108370881c0b65d1dcc6175bb470da","overlaps_test":false,"full_text_reference":true}, +{"id":"5427","category":"documentation","split":"test","html_sha1":"34ea86d99c43eccd83cb4ea48513b05cd84e0284","truth_sha1":"cd8d9038b948619404c9b8e1986cfc3cfffab6a5","overlaps_test":false,"full_text_reference":true}, +{"id":"5428","category":"article","split":"test","html_sha1":"f83b89f931b8dca4a0a629922a0053cc4e92252e","truth_sha1":"dbe1550d6b8174474203bb5ce26490f196e56f7e","overlaps_test":false,"full_text_reference":true}, +{"id":"5430","category":"documentation","split":"test","html_sha1":"9a77d0b63f7cabf9ac4c83e6363a43f1efad8d43","truth_sha1":"20a4ea13d29530ae3c93d8be831f0a8cb37dd5d5","overlaps_test":false,"full_text_reference":true}, +{"id":"5431","category":"documentation","split":"test","html_sha1":"347136a9de56614d570a3dd3cd6d5b6bb1f72c51","truth_sha1":"8927403c8a6979975f97976c766b35152adcdfac","overlaps_test":false,"full_text_reference":true}, +{"id":"5433","category":"article","split":"test","html_sha1":"794b8983a62fcd486b11f65f0cdfe786845c9b52","truth_sha1":"487539fffce57ab697b682180066871c2552f6ed","overlaps_test":false,"full_text_reference":true}, +{"id":"5434","category":"article","split":"test","html_sha1":"1c246669f6034f79a4a1440285726db3b71411a6","truth_sha1":"8bfc66271e4b187d3da23d46f1791d1a02792c44","overlaps_test":false,"full_text_reference":true}, +{"id":"5435","category":"article","split":"test","html_sha1":"3d7ab5fb0fcc815717e19c5b1470bdd1dcc24f1a","truth_sha1":"81329c3c690bd0f75bcdda1ef0dbc29607fce66b","overlaps_test":false,"full_text_reference":true}, +{"id":"5436","category":"article","split":"test","html_sha1":"dade511c8a4a966c55364daaebc2807becb41584","truth_sha1":"ca25678523e2b99dcf8b395ea7ff85c015a71e5f","overlaps_test":false,"full_text_reference":true}, +{"id":"5437","category":"article","split":"test","html_sha1":"09c402a0ccc4c7239f6da709f0e843d848880cc0","truth_sha1":"a418cac32282988846d0242c794c3e044308715d","overlaps_test":false,"full_text_reference":true}, +{"id":"5438","category":"article","split":"test","html_sha1":"3181d856bd82ab1e4358a4224629d50c83d4449e","truth_sha1":"35595b179cea2b769c72ff012116cbcc58db40e3","overlaps_test":false,"full_text_reference":true}, +{"id":"5440","category":"article","split":"test","html_sha1":"9d58a120b05aa01c799c3e8127e41ada2b5fe020","truth_sha1":"e68be8c6d800507171977158dc73449e0760384d","overlaps_test":false,"full_text_reference":true}, +{"id":"5441","category":"article","split":"test","html_sha1":"75876cdb078c706a0cf7be91e7226e26c81658f8","truth_sha1":"c6980585aea1af7827beaa284f303b57e6a62406","overlaps_test":false,"full_text_reference":true}, +{"id":"5442","category":"article","split":"test","html_sha1":"24d9a3ca69f4f8a9b02b4454cedf58b3f9ae7bc3","truth_sha1":"cad09e5c507035c462493cf2cd21b39219667353","overlaps_test":false,"full_text_reference":true}, +{"id":"5444","category":"article","split":"test","html_sha1":"ccf9252671dd7cda27ac62d6c49e71f426636b6b","truth_sha1":"7515c204496b958c248bd4d4368d78fd2d732d26","overlaps_test":false,"full_text_reference":true}, +{"id":"5445","category":"documentation","split":"test","html_sha1":"1da42980daacc86b706b1194830970c344108c00","truth_sha1":"7718b401d6a65503b07cdcccb63e4ebb7ebed572","overlaps_test":false,"full_text_reference":true}, +{"id":"5446","category":"documentation","split":"test","html_sha1":"f451c0a4db0fe95c959a8d4d0d158b4aa7429262","truth_sha1":"3e8b48a61d0b4e04d955099ba7fda1e966b0c486","overlaps_test":false,"full_text_reference":true}, +{"id":"5447","category":"article","split":"test","html_sha1":"f529f2e23d3052adf21723f0a03111f28814affe","truth_sha1":"b59b78649bd5d127a2a14e71bf4c9a7156cfdc4c","overlaps_test":false,"full_text_reference":true}, +{"id":"5449","category":"documentation","split":"test","html_sha1":"a72e7899709112ce576ac41d308013a84ca5e9b8","truth_sha1":"16d7e5815b2ad730b1e6bf881569ec5d64aa2d5d","overlaps_test":false,"full_text_reference":true}, +{"id":"5452","category":"documentation","split":"test","html_sha1":"d0b5b72c5a084a0f9c0bf409c8e1b9c1896e6c41","truth_sha1":"7aed16d2e48f1ce66f7df6cd832b6efb8f0da261","overlaps_test":false,"full_text_reference":true}, +{"id":"5454","category":"documentation","split":"test","html_sha1":"7571be74b2a113ab8d91a576ddbd7a7c7d421a1c","truth_sha1":"d9662c329db84bfd4332bf761ff5220f0460afe3","overlaps_test":false,"full_text_reference":true}, +{"id":"5456","category":"documentation","split":"test","html_sha1":"d928136471ebe363ffc55888b334fc0ecdfc65c4","truth_sha1":"7db1e4e34904d62fed7873e2226f20ba08793cd0","overlaps_test":false,"full_text_reference":true}, +{"id":"5458","category":"documentation","split":"test","html_sha1":"c8d68450e4e188b25a14ad810787369f0a1796a6","truth_sha1":"f5b01c612ef04ca30303f52ebe92dfa4d7b93cd5","overlaps_test":false,"full_text_reference":true}, +{"id":"5461","category":"documentation","split":"test","html_sha1":"9ad491a9c09925831ca05e36f61226fb3b1051e8","truth_sha1":"1350c7d561050ebe6104d84d90fbf2eabc221c9c","overlaps_test":false,"full_text_reference":true}, +{"id":"5463","category":"article","split":"test","html_sha1":"00d0f5c5d866af89cf8d53a27f1f996e3a1414b8","truth_sha1":"d4b35dce95026a56e61c34d94a36b8b8160899df","overlaps_test":false,"full_text_reference":true}, +{"id":"5467","category":"documentation","split":"test","html_sha1":"ddce4f018e791687d6457491577a9c0557b86199","truth_sha1":"343bc762252bbb19f4042a7dd55cfe0521c3951f","overlaps_test":false,"full_text_reference":true}, +{"id":"5468","category":"article","split":"test","html_sha1":"68159bcb7b73ce7a3dc5962c401b7ae00d94b7fc","truth_sha1":"0359336d92fb3d301969a1682ce19566525a5d3b","overlaps_test":false,"full_text_reference":true}, +{"id":"5469","category":"documentation","split":"test","html_sha1":"56f47fd90f600041e86cde0e58a72ae785b17d7d","truth_sha1":"7b14b0ac9df2c6db2c5a3f4da43f32a0773751db","overlaps_test":false,"full_text_reference":true}, +{"id":"5475","category":"collection","split":"test","html_sha1":"09fca6237fee1adc047457d9c2fa970e035c99e7","truth_sha1":"f46621bdccbe668f8f76b4af4eafba34b8c0355f","overlaps_test":false,"full_text_reference":true}, +{"id":"5511","category":"collection","split":"test","html_sha1":"68b7ec5a4abb0a7f42cd5b3d05b9b90abed1eb4d","truth_sha1":"29bf1b736e639d9392b968c93ae97356183a7140","overlaps_test":false,"full_text_reference":true}, +{"id":"5512","category":"collection","split":"test","html_sha1":"a7eb2738b355be63f875e9fe0d0e45e96b3373ac","truth_sha1":"caf40a0f446a87a000e35b382dc5c488b20e6724","overlaps_test":false,"full_text_reference":true}, +{"id":"5521","category":"article","split":"test","html_sha1":"0cd596f1a33161727482b7f2a43065bcda6fb749","truth_sha1":"400e9443bea3c45748a7c5b23a25219507db5643","overlaps_test":false,"full_text_reference":true}, +{"id":"5530","category":"article","split":"test","html_sha1":"299fb27b0eccc035aa40c4148851079bc2116c9b","truth_sha1":"8a20ba34057b9f51f26955295ac3126813b921e4","overlaps_test":false,"full_text_reference":true}, +{"id":"5531","category":"article","split":"test","html_sha1":"7d8a5a727de631fca0053694aa5591cffe83fa49","truth_sha1":"279af028ff7b8b31c2ec9687d6f56369faf0e3ab","overlaps_test":false,"full_text_reference":true}, +{"id":"5532","category":"article","split":"test","html_sha1":"c57db832e9386f14b7c68a3bdc6bda1041f80e55","truth_sha1":"aa5c7fda296132176f6dca5529bd268de6942590","overlaps_test":false,"full_text_reference":true}, +{"id":"5533","category":"article","split":"test","html_sha1":"f7b57bb1f0c063c2ab4045d0249be0af48023006","truth_sha1":"ad7712f8c4d6be4eb71b3016d456e20b7c908f1f","overlaps_test":false,"full_text_reference":true}, +{"id":"5534","category":"article","split":"test","html_sha1":"e54edf7a3c1066bf63baa6acc4a5b8343266028a","truth_sha1":"a19f478b8037e1abc17d920d36f1df87a6c25475","overlaps_test":false,"full_text_reference":true}, +{"id":"5535","category":"article","split":"test","html_sha1":"df0cce7d4d5430b552be4386662530c78f3c4a4e","truth_sha1":"08a2b8fb469fa6dd7e1e45c3c5f0a201a1933d5a","overlaps_test":false,"full_text_reference":true}, +{"id":"5536","category":"article","split":"test","html_sha1":"32f308615a1c49209491150c045d98ca348e7b4b","truth_sha1":"d580ef41b3bd59e4d10bbe5fb866d0287c373b40","overlaps_test":false,"full_text_reference":true}, +{"id":"5537","category":"article","split":"test","html_sha1":"bc56b5323f5619f5cfbb595f95a517bfe8bd8903","truth_sha1":"d3f9d8e69b65ccc5dfb5658a52cfce7732374f9f","overlaps_test":false,"full_text_reference":true}, +{"id":"5538","category":"article","split":"test","html_sha1":"a027715cce7389a1e7ee5f7c4f9e106d74d00706","truth_sha1":"6e2f8ca6b6c13a19bf78ef7a76d30be079c80185","overlaps_test":false,"full_text_reference":true}, +{"id":"5539","category":"article","split":"test","html_sha1":"55128e2b52c1092441b442ad7f1cc73e6858f29e","truth_sha1":"889236248d6a360b8ff58ff9131447054253575d","overlaps_test":false,"full_text_reference":true}, +{"id":"5540","category":"article","split":"test","html_sha1":"504b39b8489f71f5945769a134a8eb87f4364675","truth_sha1":"c393abc527528543ca321d1d1de80076bdf10929","overlaps_test":false,"full_text_reference":true}, +{"id":"5541","category":"article","split":"test","html_sha1":"b0c716d86106d278503d86f0b089791ac9b6e9b0","truth_sha1":"39d73eed149385b3ac145b59997594c2ef2df756","overlaps_test":false,"full_text_reference":true}, +{"id":"5542","category":"article","split":"test","html_sha1":"9b92d08931a1ec7cf539156b4845eda07e4a8226","truth_sha1":"c4c0a8d9f781fd3fbdbb2e171bba478a91474ac7","overlaps_test":false,"full_text_reference":true}, +{"id":"5545","category":"article","split":"test","html_sha1":"5e3d1899e12840fb49a48fb2d665a0ff085c7486","truth_sha1":"f2ef35a7435c27998a93a5159599e8c9e790c21e","overlaps_test":false,"full_text_reference":true}, +{"id":"5546","category":"article","split":"test","html_sha1":"50c04408a9d55867d2a14bf3645d6b345f9a01f0","truth_sha1":"a6b11dae478a2ad4cb2a588fabad4146fd922535","overlaps_test":false,"full_text_reference":true}, +{"id":"5547","category":"article","split":"test","html_sha1":"32c9d2b1c5c1c43706a16b89710c18b7a47631db","truth_sha1":"a803513ff5ca1b6ad4313ecb07b1616d279f7a2e","overlaps_test":false,"full_text_reference":true}, +{"id":"5548","category":"article","split":"test","html_sha1":"5f657621b3e6b17c9885f200a71b4f0566c5fc2b","truth_sha1":"0eda4a8a03ec44aa23b227e5808c9d1e925614cb","overlaps_test":false,"full_text_reference":true}, +{"id":"5549","category":"article","split":"test","html_sha1":"052bf9a4539223d77845871f2ce65408ba404a9d","truth_sha1":"29222fa9469bd7db8c562142c8f07185258f6bab","overlaps_test":false,"full_text_reference":true}, +{"id":"5550","category":"article","split":"test","html_sha1":"6e7f609e80119bbca49f34f5e681f7602a507800","truth_sha1":"7bdf756cd81406d6f8c88f5a1d2b75c419c5e7b2","overlaps_test":false,"full_text_reference":true}, +{"id":"5551","category":"article","split":"test","html_sha1":"9dfc47557ed380f6d3b614652027f040168f096b","truth_sha1":"b3cf688011bc59443d6b1e74c010f570292071ed","overlaps_test":false,"full_text_reference":true}, +{"id":"5553","category":"article","split":"test","html_sha1":"634539aa8f0bc72b5920aee5f5a615c26643f54d","truth_sha1":"5339b1dc743e297722b6d916e895794a1e41f4ef","overlaps_test":false,"full_text_reference":true}, +{"id":"5554","category":"article","split":"test","html_sha1":"45eec45b4941360b01d645331ed7bbb8946b8d65","truth_sha1":"9e85ed7d86da298af6c80c33c182b3da2084be25","overlaps_test":false,"full_text_reference":true}, +{"id":"5555","category":"article","split":"test","html_sha1":"037c5d39081f6ac4fcbf6094a50e5b68c002f011","truth_sha1":"2a48dacbf3e0ad28927b0103315663096276f572","overlaps_test":false,"full_text_reference":true}, +{"id":"5559","category":"article","split":"test","html_sha1":"9e1bb983836f4f820f9206e2284918afd17bfebd","truth_sha1":"5e3fb98e83e710687ec3b6a87b3458bd90de5ce8","overlaps_test":false,"full_text_reference":true}, +{"id":"5562","category":"article","split":"test","html_sha1":"bd0cb5c7cb5243dd78b0080a6905f3a2329deaf9","truth_sha1":"67fa59f5314fab24bfe72a0c367bd41fdecf943d","overlaps_test":false,"full_text_reference":true}, +{"id":"5564","category":"article","split":"test","html_sha1":"6111416f3403841ded6d5c6e4f6e24c416828a47","truth_sha1":"3e7892ff49fc441c21f3ea987b0efb099b6633ff","overlaps_test":false,"full_text_reference":true}, +{"id":"5565","category":"article","split":"test","html_sha1":"4c57731e7b6dbbb23fb06695c6bcaf49dec8f7a8","truth_sha1":"90f868c44bac3db4aa7408770051c4d8084fefc2","overlaps_test":false,"full_text_reference":true}, +{"id":"5566","category":"article","split":"test","html_sha1":"fe1c3ba415c77504ed4f0e20bd27495ed88673ea","truth_sha1":"588603d287c4a1cf0be8b710512031b54811f0af","overlaps_test":false,"full_text_reference":true}, +{"id":"5570","category":"article","split":"test","html_sha1":"7b0f3e69a463bea2ec5b662dffdbe99efcac9e17","truth_sha1":"080def8572d2c8ba63037b061295e8b6ec08ae5e","overlaps_test":false,"full_text_reference":true}, +{"id":"5571","category":"collection","split":"test","html_sha1":"5efbd606957632608a4c0e4f0c8ab591a64e2970","truth_sha1":"38fa31fd38a5e074c8e004e0c05ccc259583e02e","overlaps_test":false,"full_text_reference":true}, +{"id":"5572","category":"listing","split":"test","html_sha1":"1c33671df6f7b627a0f9f943695b24475f138e9a","truth_sha1":"6287df802df1954eff25151b14a1a4e9a60baf98","overlaps_test":false,"full_text_reference":true}, +{"id":"5575","category":"listing","split":"test","html_sha1":"9f7d5b1bcc3abe96c8e2a779a3152ba45a706e51","truth_sha1":"79e0a4ab62e3646f067a357ae3df945baf2011d7","overlaps_test":false,"full_text_reference":true}, +{"id":"5576","category":"article","split":"test","html_sha1":"82fdbfb8de9a4ca7527580cc26d0469b3be2a73b","truth_sha1":"54f90ab8a8364f0501d93849cde3317def143b92","overlaps_test":false,"full_text_reference":true}, +{"id":"5577","category":"article","split":"test","html_sha1":"ba2c1fd3c947f16d9f6f943f7433d16c8e0a6a61","truth_sha1":"31b30dfb1e4a3f31148286cf8984610d36b23f04","overlaps_test":false,"full_text_reference":true}, +{"id":"5578","category":"article","split":"test","html_sha1":"e1833dbfff4b4c60beb417760da71c02ce6ecd54","truth_sha1":"f217e16d18ea5795b3a91da17a42cf88e99acbab","overlaps_test":false,"full_text_reference":true}, +{"id":"5579","category":"article","split":"test","html_sha1":"7d0cc77d09fff57a584c80444d5e70e4e0fc36b7","truth_sha1":"c364b56c7938157f4d8e6df51c2236f06bb1b3e7","overlaps_test":false,"full_text_reference":true}, +{"id":"5580","category":"article","split":"test","html_sha1":"7026fb8c514bf4567195d20794330db09c1cb7ab","truth_sha1":"35902788c99d21ea22a30f3d71309fa68b8b038c","overlaps_test":false,"full_text_reference":true}, +{"id":"5581","category":"article","split":"test","html_sha1":"02fe5dec616b01dcdb94aa519934e86c9d5747cf","truth_sha1":"39ff41bc83e2a923b4f94c8ad60ae3aab8d75359","overlaps_test":false,"full_text_reference":true}, +{"id":"5582","category":"article","split":"test","html_sha1":"7072862bd83350367288182aaf31401ad4bdb680","truth_sha1":"b7a15fe1c65980d6e5a4e86dfd73df02e87234ce","overlaps_test":false,"full_text_reference":true}, +{"id":"5583","category":"article","split":"test","html_sha1":"b87cf9156672812551917cf96b7c6d60254d44b2","truth_sha1":"624dd57e98392b1d30315a3a3e9a646907c23e8c","overlaps_test":false,"full_text_reference":true}, +{"id":"5584","category":"article","split":"test","html_sha1":"9279000e527bf08fe7fb602f4855a44309d87a99","truth_sha1":"5edfd86a392433ddd7c3eda10c23037582eca766","overlaps_test":false,"full_text_reference":true}, +{"id":"5585","category":"listing","split":"test","html_sha1":"01cb9252b87b9b3e2ffcdd242d5346107ea77a18","truth_sha1":"9a251174fb20a0dcde55c8c2c479ad65d878dd79","overlaps_test":false,"full_text_reference":true}, +{"id":"5587","category":"article","split":"test","html_sha1":"d99478ae4ff11fdb9f9bd4841a94668f7b6646af","truth_sha1":"35d644c9b3e906fa1ede0f9069f79023f2f5ec6a","overlaps_test":false,"full_text_reference":true}, +{"id":"5588","category":"article","split":"test","html_sha1":"35de7902957573af95c6ee8a0e98266c3a8bafda","truth_sha1":"2187fadc2d34ab36ee99fa2492c5edc7778eb210","overlaps_test":false,"full_text_reference":true}, +{"id":"5591","category":"article","split":"test","html_sha1":"5b8bb8c2d0ea91e878a38ef0d784a3b8e5733f8a","truth_sha1":"f1ba371a2aa29c7542aae88c72e23ee82ae40fb7","overlaps_test":false,"full_text_reference":true}, +{"id":"5592","category":"article","split":"test","html_sha1":"25420e62ffa29643a2f5580343480a017c817cc3","truth_sha1":"2ac832d77781d34a469879c4c3ebda34d253e8d0","overlaps_test":false,"full_text_reference":true}, +{"id":"5594","category":"article","split":"test","html_sha1":"1b3e165ef89927387a74f0f24754cbc00a2a17fe","truth_sha1":"7d25e4d81cbf2f60f3843eb07304a3fb195668f8","overlaps_test":false,"full_text_reference":true}, +{"id":"5595","category":"article","split":"test","html_sha1":"ebf910ec4e54d326b8219ee03f8fb6fc48a4f5ae","truth_sha1":"93257562a894b504c29345c01f4f35723fd2c975","overlaps_test":false,"full_text_reference":true}, +{"id":"5596","category":"article","split":"test","html_sha1":"35f2d5b69866a6c9091e553cf1dc4ec479f9afa8","truth_sha1":"f1c0731dac4a45b97fb83717dd050ea867ca83ab","overlaps_test":false,"full_text_reference":true}, +{"id":"5597","category":"article","split":"test","html_sha1":"8b044acfdaead6f61a233015de4b1fa711fa6671","truth_sha1":"6b42036c29fcb8ae9a05b44a967ade02f6fb0929","overlaps_test":false,"full_text_reference":true}, +{"id":"5598","category":"collection","split":"test","html_sha1":"916f642c6a370078fa18b79d102e3d5d842eded6","truth_sha1":"1f305a04f28e7c95e1e921f93b48d6a17d7a41db","overlaps_test":false,"full_text_reference":true}, +{"id":"5600","category":"listing","split":"test","html_sha1":"043140fbced889597146631fd7cdf72dfa59fb8f","truth_sha1":"f2babc7a0dcf665fb7090eb1d878b6c4f1d18531","overlaps_test":false,"full_text_reference":true}, +{"id":"5601","category":"article","split":"test","html_sha1":"df247d9a61e519e7d1e102cae118a7ec888bdf3d","truth_sha1":"c20b9f02fb79bd6db414ab57fdc92b9a79268410","overlaps_test":false,"full_text_reference":true}, +{"id":"5602","category":"article","split":"test","html_sha1":"4618c09e550e21bde30efe7976f1c2f2781297e9","truth_sha1":"36e7e3714432b94b280b1d81d96f3b2bff9f98bc","overlaps_test":false,"full_text_reference":true}, +{"id":"5603","category":"article","split":"test","html_sha1":"238a4fc17fa403f3c8c6b8337eae8f63afb05805","truth_sha1":"1ec6dd3943f4e16e89b9d0a2b70481b72e3578bf","overlaps_test":false,"full_text_reference":true}, +{"id":"5604","category":"listing","split":"test","html_sha1":"84a29a8c3bc15e1ec2f63e6fa693d918f74983b0","truth_sha1":"cc4292e84a80500adec3a8c6b5224a004447d8be","overlaps_test":false,"full_text_reference":true}, +{"id":"5606","category":"product","split":"test","html_sha1":"f3fbb80de45ba3a50ebea2550eb69833b4cdf54f","truth_sha1":"3f4c72a80bd831ff8f8186bd42618278f33837d0","overlaps_test":false,"full_text_reference":true}, +{"id":"5607","category":"article","split":"test","html_sha1":"44f475b1f710885d4863a241de04c5253f012387","truth_sha1":"1a18f0b48ec36835e4ff433373e84435e346096e","overlaps_test":false,"full_text_reference":true}, +{"id":"5608","category":"article","split":"test","html_sha1":"79a6dc71b015cd5aa41a972be2b70af9d78c51a5","truth_sha1":"2b62433bd1ee42acff0a8d7df56c5d321f5c4ea6","overlaps_test":false,"full_text_reference":true}, +{"id":"5609","category":"article","split":"test","html_sha1":"2c4443d6e0174ddb402ab4ff736f80ae98dfc8b8","truth_sha1":"8e54d5fe66893503ee029cbe9d5fb2f149c72a99","overlaps_test":false,"full_text_reference":true}, +{"id":"5610","category":"article","split":"test","html_sha1":"7d97622dfa74dfcffd2288847a34f27e62dc0360","truth_sha1":"3f64e307d18b0c5a95681ed707b3d89d3aaab411","overlaps_test":false,"full_text_reference":true}, +{"id":"5611","category":"article","split":"test","html_sha1":"b9447e5593e5b1171d653556cdc7ca8657a3257f","truth_sha1":"5760c61ed138871feb9bbb75d6e8dbfb4fe9d508","overlaps_test":false,"full_text_reference":true}, +{"id":"5612","category":"article","split":"test","html_sha1":"33fdb3260b5097e71113230c94a218f2cd48d0ab","truth_sha1":"213a9bbf7de5fb6a04b32f64c4a4ca3653bf9fe5","overlaps_test":false,"full_text_reference":true}, +{"id":"5613","category":"article","split":"test","html_sha1":"4d22ac4761358e3777e339ce8b305ef1e03aa2b9","truth_sha1":"28c556a8219f8e83f2e1cefac436362f533fbda4","overlaps_test":false,"full_text_reference":true}, +{"id":"5614","category":"article","split":"test","html_sha1":"e7715c9168eead63a77ef4d636ee837960109804","truth_sha1":"7162ef4ce41277c75d99432c56570f38bd5e9dd1","overlaps_test":false,"full_text_reference":true}, +{"id":"5615","category":"article","split":"test","html_sha1":"a5746a056c25ac11feb6a6bee328a7467636473d","truth_sha1":"88318368e54878dd8bcfff15ef30d64bcc9e8559","overlaps_test":false,"full_text_reference":true}, +{"id":"5617","category":"article","split":"test","html_sha1":"da244234634abcac34d8ffb2d1dfab8283edff27","truth_sha1":"a78e6e9fbea48be7daee812cdb7905f8297f1a5e","overlaps_test":false,"full_text_reference":true}, +{"id":"5618","category":"article","split":"test","html_sha1":"6f1a501c0945ca36b9775561e4ce0efc6176cfca","truth_sha1":"15ea69fa62ea8f0c66544ad31c3d352c85ac3f6f","overlaps_test":false,"full_text_reference":true}, +{"id":"5620","category":"article","split":"test","html_sha1":"2b3a6fa8f5398f80c6a3afa469c8df82e758b920","truth_sha1":"da5cf6e9341b40d33f5ff70d36a8344cd9cd1c9b","overlaps_test":false,"full_text_reference":true}, +{"id":"5621","category":"article","split":"test","html_sha1":"f0f7d454666f1ee8248cdcc862346735b209c67b","truth_sha1":"022d5e02b98a81f48769eb79f5c82d344e94e7c9","overlaps_test":false,"full_text_reference":true}, +{"id":"5623","category":"article","split":"test","html_sha1":"bbfe8f05a9be4662908e0a4377ff6d68f11eed5d","truth_sha1":"a46f3b0bc91390e80b41db27940fa5dc940ffa35","overlaps_test":false,"full_text_reference":true}, +{"id":"5624","category":"article","split":"test","html_sha1":"82f5003f8478ea0e34c89c8716d7a0062d39cdb3","truth_sha1":"14a7827e6ad6a742d24c59abd5b897b7f235592b","overlaps_test":false,"full_text_reference":true}, +{"id":"5626","category":"article","split":"test","html_sha1":"1cc382481583796c7c9208aed08031ea60c3bad5","truth_sha1":"b1053a3efd52a003727adfac4218579740b0f6d3","overlaps_test":false,"full_text_reference":true}, +{"id":"5627","category":"article","split":"test","html_sha1":"9054d040197c906d9477851fdd7762ac79c6f8a0","truth_sha1":"f1d40ab16fe71273f6cd70a0d55a2bece93b8ae0","overlaps_test":false,"full_text_reference":true}, +{"id":"5628","category":"article","split":"test","html_sha1":"5dde8d6a7041999669efc31e63225b3dd6f344d8","truth_sha1":"a0b3cd49bf9b8ea45b1bbad33be99191e46387e4","overlaps_test":false,"full_text_reference":true}, +{"id":"5629","category":"article","split":"test","html_sha1":"ee49abcedb1e06f126d499c53f9289574d582308","truth_sha1":"8fa3392ca86f134252612252403d9ef3fb76cebf","overlaps_test":false,"full_text_reference":true}, +{"id":"5631","category":"article","split":"test","html_sha1":"166d3f03ca82c719ec7085dd0261ab21ba46c496","truth_sha1":"fe4dc9e270212c3fcc7bb20365441758e616c54d","overlaps_test":false,"full_text_reference":true}, +{"id":"5634","category":"article","split":"test","html_sha1":"24fc14d6ef75306c7f5041b2407521f61d22cce1","truth_sha1":"721b5b416e990ae797eeca71dfec1480aa47bc89","overlaps_test":false,"full_text_reference":true}, +{"id":"5635","category":"article","split":"test","html_sha1":"285ab19b1a956e946b1c067c413d0537e6e2d13f","truth_sha1":"c4bef9d078add9a7cc1ba1cc8d956c1394148e22","overlaps_test":false,"full_text_reference":true}, +{"id":"5636","category":"article","split":"test","html_sha1":"c01b5fc7624efe5849d81d4c9b14f738ea7cd362","truth_sha1":"1fb641c043e3d32c9ac498630ac04adeada04774","overlaps_test":false,"full_text_reference":true}, +{"id":"5637","category":"service","split":"test","html_sha1":"72b01b227bbd17e6139ba034b7701e33900e5980","truth_sha1":"f57c1884cb58b1041141e900483f65bfd747c5a4","overlaps_test":false,"full_text_reference":true}, +{"id":"5638","category":"service","split":"test","html_sha1":"8900b5bbff014b8c0d1c960cf1061235f99e3f2e","truth_sha1":"b7e11a0a263d6e6ad2a6cc3c744b548337c89c69","overlaps_test":false,"full_text_reference":true}, +{"id":"5639","category":"article","split":"test","html_sha1":"269b07988aa498d93582740cfed60ed2fce6b0b0","truth_sha1":"33adc240576661bbe29ff6dbc8c025e53653da5d","overlaps_test":false,"full_text_reference":true}, +{"id":"5640","category":"article","split":"test","html_sha1":"cdeb0eff7d87b7ecb82c37ec6fad552ca8a22b30","truth_sha1":"28a1cdb43bdda3fafd51dcf55134ed02da6cc384","overlaps_test":false,"full_text_reference":true}, +{"id":"5642","category":"article","split":"test","html_sha1":"849efc3007a47adea81649b9e64a370cc96db6d1","truth_sha1":"328fd73d98e5879167be78cdb0512caaf13e0857","overlaps_test":false,"full_text_reference":true}, +{"id":"5643","category":"article","split":"test","html_sha1":"b0e19c79682123bcdd5ab991af15e148c5ad2ba4","truth_sha1":"cc204b098a2b34a1112d9ae0a7116afd5ec48fb9","overlaps_test":false,"full_text_reference":true}, +{"id":"5644","category":"article","split":"test","html_sha1":"d0f5238d517451f86eff0a7b91b1d53bcda63523","truth_sha1":"3869985b41f3af5104a5a00224d77a9b08bc2343","overlaps_test":false,"full_text_reference":true}, +{"id":"5645","category":"article","split":"test","html_sha1":"03b563b80c6305fb799a8d0682aa6da5e1b56c68","truth_sha1":"a2c377eaf6b6fe0567ae753d89c99feb93eccd1a","overlaps_test":false,"full_text_reference":true}, +{"id":"5646","category":"service","split":"test","html_sha1":"b8b37fd1637c4cc5ae5e5030b132d3f37cf62d3b","truth_sha1":"5441133e513f4c710e8fb78474ef85c6125de606","overlaps_test":false,"full_text_reference":true}, +{"id":"5647","category":"article","split":"test","html_sha1":"d64c09622bd9e242e5051b7613c588777e34fe84","truth_sha1":"654b506d2e5a565a257331ea497c94883546da18","overlaps_test":false,"full_text_reference":true}, +{"id":"5648","category":"article","split":"test","html_sha1":"94b4b2181a468efaecdb4635bd935e0f49cb3725","truth_sha1":"c20b75a70e7f19d4da34f097c1c9d96010690bb7","overlaps_test":false,"full_text_reference":true}, +{"id":"5649","category":"article","split":"test","html_sha1":"472dc396f0bb9c6cec4a1a711fe1e11f9f269d1e","truth_sha1":"bc775f025570228cec52cdb8a29b4013f7871bdb","overlaps_test":false,"full_text_reference":true}, +{"id":"5650","category":"article","split":"test","html_sha1":"6b30db0402e55c77cda7c60712b809a613a944a6","truth_sha1":"783fc38869fb3d17d533c383bb086a8e29ea24fd","overlaps_test":false,"full_text_reference":true}, +{"id":"5651","category":"article","split":"test","html_sha1":"567c20502a732bc632558052af4b301e6abc95c0","truth_sha1":"3b7be708e959a1916396aca9273e4bff65fff3b5","overlaps_test":false,"full_text_reference":true}, +{"id":"5652","category":"article","split":"test","html_sha1":"195202c6f5e6d3c73e6faea9a42aeb403160d866","truth_sha1":"91ed9f41afbf6b8f5730dcf99575bc3a7aff509d","overlaps_test":false,"full_text_reference":true}, +{"id":"5654","category":"article","split":"test","html_sha1":"e2f96b40a1e4c5beb758d16d9951172a2ea8efb4","truth_sha1":"40e94dd4a589122464afb909b16dfc77157710b2","overlaps_test":false,"full_text_reference":true}, +{"id":"5656","category":"article","split":"test","html_sha1":"264228aacdcbf07cc74cd28794e93b44f4dbbbe8","truth_sha1":"cd65e7aed7ed2f62e95b3fdce265c538c96952c4","overlaps_test":false,"full_text_reference":true}, +{"id":"5657","category":"service","split":"test","html_sha1":"5c39545ba70bdbfba9c4baf4e1c9e5bbdfbfdfed","truth_sha1":"2f6b66ddaaa983d67c3d2beee53b3d9899ddf57b","overlaps_test":false,"full_text_reference":true}, +{"id":"5659","category":"service","split":"test","html_sha1":"3298c7e4413512da77e6e92e345cbb85c8422f50","truth_sha1":"57752ec64d886194d7390f9b486752a0bf288e17","overlaps_test":false,"full_text_reference":true}, +{"id":"5662","category":"article","split":"test","html_sha1":"8f4a7455d23253b5d4c35919acc30aab1868c2ff","truth_sha1":"49512205765438786ab15c3ff645aaa902cc71d6","overlaps_test":false,"full_text_reference":true}, +{"id":"5663","category":"article","split":"test","html_sha1":"46a5b4593f203b356467d972e191ab4ff275f975","truth_sha1":"f5075fbbd994b0bb35238952078131e759055a51","overlaps_test":false,"full_text_reference":true}, +{"id":"5664","category":"article","split":"test","html_sha1":"0447b56c8f958b2363b6ea3a712d4cff0f3e61c0","truth_sha1":"ab9bee4f7d7cbb57d70b88d454a6a153bc1e4eea","overlaps_test":false,"full_text_reference":true}, +{"id":"5665","category":"article","split":"test","html_sha1":"3e46be7e5dfaee900eb9959bf509bf4da5b86ed7","truth_sha1":"01013db3db6cf7f458f3e850415dbeaf0778988f","overlaps_test":false,"full_text_reference":true}, +{"id":"5667","category":"article","split":"test","html_sha1":"d4892e88fcd5bdb90b97769366bdc7dcfa18f2a2","truth_sha1":"312ea9dc5d646ae52703bbd98afef061a18b1ca5","overlaps_test":false,"full_text_reference":true}, +{"id":"5668","category":"article","split":"test","html_sha1":"2f11cdc266d69780c9bb9fe1adc2affe82878598","truth_sha1":"e82e08be7d0fc24ab5806258188d8e2729a8f1de","overlaps_test":false,"full_text_reference":true}, +{"id":"5670","category":"article","split":"test","html_sha1":"71d17abb90416a3f7e60ff38c5d849df5b85da8f","truth_sha1":"56c968b3d5d3697c86424df4933201e8eee459fc","overlaps_test":false,"full_text_reference":true}, +{"id":"5671","category":"article","split":"test","html_sha1":"0a21ba573d11932d1b53979db65f82e7a32aefaa","truth_sha1":"31901d1a49e5af6d897c79577c9bcecde2348304","overlaps_test":false,"full_text_reference":true}, +{"id":"5672","category":"article","split":"test","html_sha1":"efde7e18311489f94587ad928c7c1d3b718df3b1","truth_sha1":"4503f3fb9b91446ea2fc2ee04f3a06bdb17dbc3b","overlaps_test":false,"full_text_reference":true}, +{"id":"5673","category":"article","split":"test","html_sha1":"ca64b22ef4cf94657a0bc0260061d445ed88fcd2","truth_sha1":"3f5f07191d63b1574b2c63ac4c1c0908bac41fc3","overlaps_test":false,"full_text_reference":true}, +{"id":"5674","category":"article","split":"test","html_sha1":"ad043c3ceee1f916ec9942d8670e2039975a1af2","truth_sha1":"467ce6622a072cfeb7adf41503fc569cda5fcbd5","overlaps_test":false,"full_text_reference":true}, +{"id":"5676","category":"article","split":"test","html_sha1":"09ad89054af9a8f10762aa99db7be9279a48407f","truth_sha1":"7e2f85e47d2224d421071df1898b1e4bf2d1028e","overlaps_test":false,"full_text_reference":true}, +{"id":"5677","category":"article","split":"test","html_sha1":"d5dcf655d5507bfee7c949efd7f3761bd5e9095b","truth_sha1":"5bd22fb5d62ab10751cd0095a9b7aa7a6ec6e834","overlaps_test":false,"full_text_reference":true}, +{"id":"5678","category":"article","split":"test","html_sha1":"a64d6b1e90976a1620c7abe5a61f87e8d68e8a4e","truth_sha1":"61713d6aeb88a618e2d703722ed1f803d3283abe","overlaps_test":false,"full_text_reference":true}, +{"id":"5682","category":"article","split":"test","html_sha1":"9d8967565a7b53ea0725cda7659306979ba6be04","truth_sha1":"0e73357b5fba1782cf37775d179a38f08282239e","overlaps_test":false,"full_text_reference":true}, +{"id":"5684","category":"article","split":"test","html_sha1":"9ca85bc3a25c9d19f8c97b8f7df1a0a4a834d78e","truth_sha1":"5125410fab7a2117db985a40e8aa0042176379d6","overlaps_test":false,"full_text_reference":true}, +{"id":"5685","category":"service","split":"test","html_sha1":"80531145b9679c368d95aa169d168dae2e7d010e","truth_sha1":"6c736443f08a1e59fee4fa2890299febeecc3a49","overlaps_test":false,"full_text_reference":true}, +{"id":"5687","category":"service","split":"test","html_sha1":"52f40016f1ac42ff20288933af080efe0e3dad2a","truth_sha1":"0cf1db699ab7633160657fd28c5cb94d5db443e3","overlaps_test":false,"full_text_reference":true}, +{"id":"5689","category":"article","split":"test","html_sha1":"bd472ee84576979b76ce8d246b6a091ced4a66e8","truth_sha1":"acb57f0ff9eb2792b64e2d089c7d1fa4456812e6","overlaps_test":false,"full_text_reference":true}, +{"id":"5690","category":"article","split":"test","html_sha1":"4b1b3dccdff61c9210a6490f108fb20c31bc9db9","truth_sha1":"faee8aa5e57abe02aab83506d9294f1d42312895","overlaps_test":false,"full_text_reference":true}, +{"id":"5691","category":"article","split":"test","html_sha1":"10b86cdd0168cbef09c4d7cf722ce5b49c09fc74","truth_sha1":"22e0825c06b8d368a82d8b60bbf34e118145bbb0","overlaps_test":false,"full_text_reference":true}, +{"id":"5693","category":"article","split":"test","html_sha1":"9fc2cacfb8298145154b6f019d07a513f2e8323c","truth_sha1":"b3056d6fa9758ff37ed728691bf6d8ec3d5a337a","overlaps_test":false,"full_text_reference":true}, +{"id":"5695","category":"article","split":"test","html_sha1":"e49f2ceeea498ae4ef0995d168c46cacd44bed09","truth_sha1":"ff4284c979dc1091b122bdfdc8db0a72c1f44bc5","overlaps_test":false,"full_text_reference":true}, +{"id":"5696","category":"article","split":"test","html_sha1":"898df7a6a233510d608f3b1d399c1e2a81e88386","truth_sha1":"f8ea03adc1acf7cbdf914d8e42c5190a18acd956","overlaps_test":false,"full_text_reference":true} +] diff --git a/scripts/tests/test_public_benchmarks.py b/scripts/tests/test_public_benchmarks.py new file mode 100644 index 0000000..17fcba4 --- /dev/null +++ b/scripts/tests/test_public_benchmarks.py @@ -0,0 +1,179 @@ +"""Public benchmark contracts: corpus integrity, denominator and regression gates.""" +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] / "public_benchmarks" +sys.path.insert(0, str(ROOT)) +import common +import run as public_run + + +class PublicBenchmarksTest(unittest.TestCase): + def test_split_selection_has_no_test_html_leakage(self): + pr = common.selected_wcxb("pr") + dev = common.selected_wcxb("dev") + test = common.selected_wcxb("test") + self.assertEqual((len(pr), len(dev), len(test)), (70, 1358, 511)) + self.assertEqual(len({r["category"] for r in pr}), 7) + self.assertTrue({r["html_sha1"] for r in dev}.isdisjoint({r["html_sha1"] for r in test})) + self.assertEqual(pr, common.selected_wcxb("pr")) + + def test_cache_mismatch_does_not_replace_good_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.html" + path.write_bytes(b"old") + with patch.object(common, "download", return_value=b"wrong"): + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + common.fetch(path, "https://raw.githubusercontent.com/pinned", common.sha256(b"expected")) + self.assertEqual(path.read_bytes(), b"old") + with patch.object(common, "download", return_value=b"expected"): + common.fetch(path, "https://raw.githubusercontent.com/pinned", common.sha256(b"expected")) + self.assertEqual(path.read_bytes(), b"expected") + + def test_git_blob_verification_and_verified_offline_cache(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "truth.json" + path.write_bytes(b"{}") + with patch.object(common, "download", side_effect=AssertionError("network forbidden")): + common.fetch(path, "https://raw.githubusercontent.com/pinned", common.git_sha1(b"{}"), "git") + + def test_cache_paths_reject_traversal_and_symlink_escape(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "cache" + root.mkdir() + (root / "escape").symlink_to(root.parent, target_is_directory=True) + for path in ["../escape", "/absolute", "a/../../outside", "a\\..\\outside", "escape/outside", "."]: + with self.subTest(path=path), self.assertRaises(ValueError): + common.safe_path(root, path) + + def test_download_rejects_live_or_unexpected_hosts(self): + for url in ["http://localhost/data", "file:///etc/passwd", "https://example.com/live"]: + with self.assertRaises(ValueError): + common.download(url) + + def test_prepared_manifest_rejects_changed_missing_and_corrupt_files(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest = {"scope": "pr", "contract": "fixed", "samples": [{"suite": "wcxb", "id": "a", "file": "a.html", "truth": "a.html"}], "files": {"a.html": common.sha256(b"wanted")}} + path = root / "manifest.json" + path.write_text(json.dumps(manifest)) + with patch.object(common, "contract", return_value="changed"), self.assertRaises(ValueError): + common.read_manifest(path) + with patch.object(common, "contract", return_value="fixed"), patch.object(common, "selection_ids", return_value={("wcxb", "a")}): + with self.assertRaises(FileNotFoundError): + common.read_manifest(path) + (root / "a.html").write_bytes(b"wrong") + with self.assertRaises(ValueError): + common.read_manifest(path) + (root / "a.html").write_bytes(b"wanted") + self.assertEqual(common.read_manifest(path), manifest) + manifest["samples"] = [] + path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "complete pinned selection"): + common.read_manifest(path) + + def test_category_macro_and_no_ocr_slice(self): + rows = [self.row(str(i), "a", 1) for i in range(3)] + [self.row("scan", "b", 0, ocr=True)] + vectors = public_run.quality_vectors(rows) + self.assertEqual(vectors["agentdocbench/overall"], .5) + self.assertEqual(vectors["agentdocbench/no_ocr"], 1) + self.assertEqual(vectors["agentdocbench/category/b"], 0) + + def test_gate_requires_every_baseline_case_and_contract(self): + rows = [self.row("a", "a", 1)] + baseline = public_run.baseline_record(rows, "contract") + self.assertEqual(public_run.regressions(rows, baseline, "contract"), []) + self.assertTrue(public_run.regressions([], baseline, "contract")) + self.assertTrue(public_run.regressions(rows, baseline, "changed")) + changed = [self.row("a", "a", .9)] + self.assertTrue(public_run.regressions(changed, baseline, "contract")) + self.assertEqual(public_run.regressions([self.row("a", "a", .996)], baseline, "contract"), []) + baseline["quality"]["agentdocbench/overall"] = float("nan") + self.assertTrue(public_run.regressions(rows, baseline, "contract")) + + def test_report_rejects_missing_duplicate_na_and_invalid_scores(self): + sample = {"suite": "agentdocbench", "id": "a", "category": "docx", "format": "docx", "ocr": False} + manifest = {"samples": [sample]} + case = {"eval": "markkit_agentdocbench", "sample": "a", "target": "markkit", "scores": [ + {"scorer": "succeeded", "value": 1}, {"scorer": "document_score", "value": 1}], + "transcript": {"final_response": "text"}} + self.assertEqual(len(public_run.report_rows({"cases": [case]}, manifest)), 1) + for cases in [[], [case, case]]: + with self.assertRaises(ValueError): + public_run.report_rows({"cases": cases}, manifest) + for score in [{"scorer": "document_score", "value": 0, "na": True}, + {"scorer": "document_score", "value": float("nan")}, + {"scorer": "document_score", "value": 2}]: + bad = json.loads(json.dumps(case)) + bad["scores"][1] = score + with self.assertRaises(ValueError): + public_run.report_rows({"cases": [bad]}, manifest) + + def test_failed_mira_run_replaces_stale_success_and_preserves_provenance(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + cache, output = root / "cache", root / "output" + cache.mkdir() + output.mkdir() + (root / "release").mkdir() + (root / "release/markkit").write_bytes(b"mock executable") + (cache / "manifest-pr.json").write_text("{}") + (output / "mira.json").write_text("stale success") + (output / "summary.json").write_text('{"quality": 1}') + + def command_output(command, **kwargs): + if command[:2] == ["cargo", "metadata"]: + return json.dumps({"target_directory": str(root)}).encode() + if command[0] == "mock-mira": + return "mira 0.5.0\n" + if command[:2] == ["git", "rev-parse"]: + return "commit\n" + if command[0] == "rustc": + return "rustc test\n" + return "" if kwargs.get("text") else b"" + + argv = ["run.py", "--cache", str(cache), "--out", str(output), "--mira", "mock-mira"] + manifest = {"scope": "pr", "contract": "fixed", "pins": {"wcxb": "fixed"}, "samples": []} + with patch.object(sys, "argv", argv), patch.object(public_run, "read_manifest", return_value=manifest), \ + patch.object(public_run.subprocess, "check_output", side_effect=command_output), \ + patch.object(public_run.subprocess, "run", side_effect=[SimpleNamespace(returncode=0), SimpleNamespace(returncode=1)]): + with self.assertRaises(SystemExit): + public_run.main() + summary = json.loads((output / "summary.json").read_text()) + self.assertEqual(summary["status"], "failed") + self.assertNotIn("quality", summary) + self.assertFalse((output / "mira.json").exists()) + self.assertTrue((output / "provenance.json").is_file()) + self.assertTrue((output / "CORPUS-NOTICE").is_file()) + + @unittest.skipUnless(importlib.util.find_spec("markdown_it"), "public benchmark environment installs markdown-it-py") + def test_wcxb_markdown_projection_excludes_link_destinations(self): + import study + value = study.plain('# Title\n\n[Linked words](https://noise.invalid/path).\n\n| a | b |\n|---|---|\n| 1 | 2 |') + self.assertEqual(study.WCXB.word_f1(value, 'Title Linked words. a b 1 2')[2], 1) + self.assertEqual(study.WCXB.word_f1('', 'required content')[2], 0) + + @unittest.skipUnless(importlib.util.find_spec("markdown_it"), "public benchmark environment installs markdown-it-py") + def test_missing_full_text_matches_upstream_without_dropping_snippets(self): + import study + with tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / "truth.json").write_text(json.dumps({"ground_truth": {"with": ["wanted"], "without": ["noise"]}})) + scores = study.values({"suite": "wcxb", "truth": "truth.json"}, "wanted", Path(tmp)) + self.assertEqual(scores["f1"], 0) + self.assertEqual(scores["with_recall"], 1) + self.assertEqual(scores["boilerplate_exclusion"], 1) + + @staticmethod + def row(key, category, value, ocr=False): + return {"key": "agentdocbench/" + key, "suite": "agentdocbench", "id": key, "mode": "default", "category": category, + "format": "pdf", "ocr": ocr, "scores": {"document_score": value}, "metrics": {}} + + +if __name__ == "__main__": + unittest.main()