From 79a4dedafe2cbd362c204b0dcc37970ed03468c0 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Sun, 19 Jul 2026 09:46:57 +0800 Subject: [PATCH] feat(ocr): add local PP-OCRv6 application capability --- .github/workflows/release.yml | 57 +- Cargo.lock | 739 +++++++++++++++++++++---- Cargo.toml | 6 + README.md | 66 ++- THIRD_PARTY_NOTICES.md | 54 ++ crates/ocr/Cargo.toml | 9 +- crates/ocr/README.md | 57 +- crates/ocr/skills/a3s-use-ocr/SKILL.md | 39 +- crates/ocr/src/assets.rs | 261 +++++++++ crates/ocr/src/cli.rs | 61 +- crates/ocr/src/client.rs | 650 ++++++---------------- crates/ocr/src/config.rs | 261 +++++++++ crates/ocr/src/engine.rs | 261 +++++++++ crates/ocr/src/install.rs | 671 ++++++++++++++++++++++ crates/ocr/src/lib.rs | 19 +- crates/ocr/src/mcp.rs | 10 +- crates/ocr/src/models.rs | 47 +- crates/ocr/src/postprocess.rs | 326 +++++++++++ crates/ocr/src/preprocess.rs | 192 +++++++ crates/ocr/tests/ppocr_v6_contract.rs | 19 + docs/architecture.md | 22 +- src/cli.rs | 62 ++- src/cli_tests.rs | 11 + src/ocr_builtin.rs | 6 +- 24 files changed, 3118 insertions(+), 788 deletions(-) create mode 100644 crates/ocr/src/assets.rs create mode 100644 crates/ocr/src/config.rs create mode 100644 crates/ocr/src/engine.rs create mode 100644 crates/ocr/src/install.rs create mode 100644 crates/ocr/src/postprocess.rs create mode 100644 crates/ocr/src/preprocess.rs create mode 100644 crates/ocr/tests/ppocr_v6_contract.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0116089b..899d5886 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,6 +87,9 @@ jobs: install -d "${stage}/skills" "${stage}/skill-data" "${stage}/office-skills" "${stage}/ocr-skills" "${stage}/dashboard" install -m 0755 "target/${{ matrix.target }}/release/a3s-use" "${stage}/a3s-use" install -m 0755 "target/${{ matrix.target }}/release/a3s-use-browser-driver" "${stage}/a3s-use-browser-driver" + A3S_USE_OCR_HOME="${stage}/ocr-models" \ + "${stage}/a3s-use" component install ocr --json > "${RUNNER_TEMP}/ocr-model-install.json" + rm -f "${stage}/ocr-models/.install.lock" cp -R crates/browser-driver/skills/. "${stage}/skills/" cp -R crates/browser-driver/skill-data/. "${stage}/skill-data/" cp -R crates/office/skills/. "${stage}/office-skills/" @@ -108,6 +111,11 @@ jobs: New-Item -ItemType Directory -Force -Path $stage | Out-Null Copy-Item "target/${{ matrix.target }}/release/a3s-use.exe" "$stage/a3s-use.exe" Copy-Item "target/${{ matrix.target }}/release/a3s-use-browser-driver.exe" "$stage/a3s-use-browser-driver.exe" + $env:A3S_USE_OCR_HOME = "$stage/ocr-models" + & "$stage/a3s-use.exe" component install ocr --json | Out-File "$env:RUNNER_TEMP/ocr-model-install.json" + if ($LASTEXITCODE -ne 0) { throw "Failed to install the pinned PP-OCRv6 release assets" } + Remove-Item Env:A3S_USE_OCR_HOME + Remove-Item "$stage/ocr-models/.install.lock" -ErrorAction SilentlyContinue Copy-Item -Recurse "crates/browser-driver/skills" "$stage/skills" Copy-Item -Recurse "crates/browser-driver/skill-data" "$stage/skill-data" Copy-Item -Recurse "crates/office/skills" "$stage/office-skills" @@ -131,6 +139,10 @@ jobs: test -f "${install_root}/skill-data/core/SKILL.md" test -f "${install_root}/office-skills/a3s-use-office/SKILL.md" test -f "${install_root}/ocr-skills/a3s-use-ocr/SKILL.md" + test -f "${install_root}/ocr-models/PP-OCRv6_small/det/inference.onnx" + test -f "${install_root}/ocr-models/PP-OCRv6_small/det/inference.yml" + test -f "${install_root}/ocr-models/PP-OCRv6_small/rec/inference.onnx" + test -f "${install_root}/ocr-models/PP-OCRv6_small/rec/inference.yml" test -f "${install_root}/dashboard/index.html" test -f "${install_root}/LICENSE-APACHE-2.0" test -f "${install_root}/UPSTREAM.md" @@ -151,7 +163,27 @@ jobs: import json, pathlib, sys value = json.loads(pathlib.Path(sys.argv[1]).read_text()) assert value["ok"] is True - assert value["data"]["readiness"] in {"ready", "missing", "broken", "unknown"} + assert value["data"]["readiness"] == "ready" + assert value["data"]["provider"] == "pp-ocr-v6" + assert value["data"]["engine"] == "onnx-runtime" + assert value["data"]["model"] == "PP-OCRv6_small" + assert value["data"]["sendsSourceOffDevice"] is False + PY + python3 - "${RUNNER_TEMP}/ocr-white.bmp" <<'PY' + import base64, pathlib, sys + pathlib.Path(sys.argv[1]).write_bytes(base64.b64decode( + "Qk06AAAAAAAAADYAAAAoAAAAAQAAAAEAAAABABgAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAA////AA==" + )) + PY + "${install_root}/a3s-use" ocr extract "${RUNNER_TEMP}/ocr-white.bmp" --json > "${RUNNER_TEMP}/ocr-extract.json" + python3 - "${RUNNER_TEMP}/ocr-extract.json" <<'PY' + import json, pathlib, sys + value = json.loads(pathlib.Path(sys.argv[1]).read_text()) + assert value["ok"] is True + assert value["data"]["provider"] == "pp-ocr-v6" + assert value["data"]["engine"] == "onnx-runtime" + assert value["data"]["model"] == "PP-OCRv6_small" + assert value["data"]["source"]["mediaType"] == "image/bmp" PY "${install_root}/a3s-use" capability snapshot --json > "${RUNNER_TEMP}/capabilities.json" python3 - "${RUNNER_TEMP}/capabilities.json" "${install_root}" <<'PY' @@ -206,6 +238,10 @@ jobs: "$root/skill-data/core/SKILL.md", "$root/office-skills/a3s-use-office/SKILL.md", "$root/ocr-skills/a3s-use-ocr/SKILL.md", + "$root/ocr-models/PP-OCRv6_small/det/inference.onnx", + "$root/ocr-models/PP-OCRv6_small/det/inference.yml", + "$root/ocr-models/PP-OCRv6_small/rec/inference.onnx", + "$root/ocr-models/PP-OCRv6_small/rec/inference.yml", "$root/dashboard/index.html", "$root/LICENSE-APACHE-2.0", "$root/UPSTREAM.md", @@ -222,7 +258,24 @@ jobs: throw "Packaged Office Skill smoke failed" } $ocr = (& "$root/a3s-use.exe" ocr doctor --json | ConvertFrom-Json) - if (-not $ocr.ok -or -not $ocr.data.readiness) { throw "Built-in OCR doctor smoke failed" } + if ( + -not $ocr.ok -or + $ocr.data.readiness -ne "ready" -or + $ocr.data.provider -ne "pp-ocr-v6" -or + $ocr.data.engine -ne "onnx-runtime" -or + $ocr.data.model -ne "PP-OCRv6_small" -or + $ocr.data.sendsSourceOffDevice + ) { throw "Packaged PP-OCRv6 doctor smoke failed" } + $whiteBmp = [Convert]::FromBase64String("Qk06AAAAAAAAADYAAAAoAAAAAQAAAAEAAAABABgAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAA////AA==") + [IO.File]::WriteAllBytes("$env:RUNNER_TEMP/ocr-white.bmp", $whiteBmp) + $ocrExtract = (& "$root/a3s-use.exe" ocr extract "$env:RUNNER_TEMP/ocr-white.bmp" --json | ConvertFrom-Json) + if ( + -not $ocrExtract.ok -or + $ocrExtract.data.provider -ne "pp-ocr-v6" -or + $ocrExtract.data.engine -ne "onnx-runtime" -or + $ocrExtract.data.model -ne "PP-OCRv6_small" -or + $ocrExtract.data.source.mediaType -ne "image/bmp" + ) { throw "Packaged PP-OCRv6 extraction smoke failed" } $capabilities = (& "$root/a3s-use.exe" capability snapshot --json | ConvertFrom-Json) $ocrCapability = $capabilities.data.registry.capabilities | Where-Object id -eq "use/ocr" diff --git a/Cargo.lock b/Cargo.lock index 80e0f846..37fa4d99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,15 +121,20 @@ name = "a3s-use-ocr" version = "0.1.1" dependencies = [ "a3s-use-core", - "axum", - "base64", "clap", + "clipper2", + "fs2", + "image", + "imageproc", + "ort", "reqwest", "rmcp", "schemars", "serde", "serde_json", + "serde_yaml", "sha2 0.10.9", + "tar", "tempfile", "tokio", "url", @@ -155,6 +160,22 @@ dependencies = [ "zip", ] +[[package]] +name = "ab_glyph" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" + [[package]] name = "adler2" version = "2.0.1" @@ -205,15 +226,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aligned" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" -dependencies = [ - "as-slice", -] - [[package]] name = "aligned-vec" version = "0.6.4" @@ -288,6 +300,15 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -314,15 +335,6 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" -[[package]] -name = "as-slice" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "async-attributes" version = "1.1.2" @@ -522,26 +534,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "av-scenechange" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" -dependencies = [ - "aligned", - "anyhow", - "arg_enum_proc_macro", - "arrayvec", - "log", - "num-rational", - "num-traits", - "pastey", - "rayon", - "thiserror 2.0.18", - "v_frame", - "y4m", -] - [[package]] name = "av1-grain" version = "0.2.5" @@ -623,12 +615,24 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit_field" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.0" @@ -637,12 +641,9 @@ checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "bitstream-io" -version = "4.10.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" -dependencies = [ - "no_std_io2", -] +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" [[package]] name = "block-buffer" @@ -677,9 +678,9 @@ dependencies = [ [[package]] name = "built" -version = "0.8.1" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" [[package]] name = "bumpalo" @@ -726,6 +727,16 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec 1.15.2", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -881,6 +892,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "clipper2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9c96871d5c50dd16c0f9df018f701fc62bd4f1b73cea8efe1985df6ab3d7e6" +dependencies = [ + "clipper2c-sys", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "clipper2c-sys" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6801d5a9a1d30e747025ec2afdc841d51665028481d61b5c719d173e88211ea3" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -908,6 +940,16 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1042,6 +1084,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1213,7 +1265,7 @@ dependencies = [ "num-complex", "pulp", "rayon-core", - "smallvec", + "smallvec 1.15.2", "zune-inflate", ] @@ -1223,12 +1275,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" -[[package]] -name = "fax" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" - [[package]] name = "fdeflate" version = "0.3.7" @@ -1238,6 +1284,16 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1260,6 +1316,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1447,9 +1518,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.14.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" dependencies = [ "color_quant", "weezl", @@ -1596,7 +1667,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "smallvec", + "smallvec 1.15.2", "tokio", "want", ] @@ -1701,7 +1772,7 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", - "smallvec", + "smallvec 1.15.2", "zerovec", ] @@ -1759,7 +1830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", - "smallvec", + "smallvec 1.15.2", "utf8_iter", ] @@ -1775,9 +1846,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.10" +version = "0.25.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" dependencies = [ "bytemuck", "byteorder-lite", @@ -1785,7 +1856,6 @@ dependencies = [ "exr", "gif", "image-webp", - "moxcms", "num-traits", "png", "qoi", @@ -1807,6 +1877,23 @@ dependencies = [ "quick-error", ] +[[package]] +name = "imageproc" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2393fb7808960751a52e8a154f67e7dd3f8a2ef9bd80d1553078a7b4e8ed3f0d" +dependencies = [ + "ab_glyph", + "approx", + "getrandom 0.2.17", + "image", + "itertools", + "nalgebra", + "num", + "rand 0.8.7", + "rand_distr", +] + [[package]] name = "imgref" version = "1.12.2" @@ -1857,9 +1944,9 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" -version = "0.14.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -1880,6 +1967,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + [[package]] name = "js-sys" version = "0.3.103" @@ -1985,6 +2078,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -2039,30 +2142,58 @@ dependencies = [ ] [[package]] -name = "moxcms" -version = "0.8.1" +name = "nalgebra" +version = "0.32.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" dependencies = [ + "approx", + "matrixmultiply", + "num-complex", + "num-rational", "num-traits", - "pxfm", + "simba", + "typenum", ] [[package]] -name = "new_debug_unreachable" -version = "1.0.6" +name = "native-tls" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] [[package]] -name = "no_std_io2" -version = "0.9.4" +name = "ndarray" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" dependencies = [ - "memchr", + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "8.0.0" @@ -2078,6 +2209,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -2124,6 +2269,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -2142,6 +2297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2162,12 +2318,89 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ort" +version = "2.0.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa7e49bd669d32d7bc2a15ec540a527e7764aec722a45467814005725bcd721" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec 2.0.0-alpha.10", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2aba9f5c7c479925205799216e7e5d07cc1d4fa76ea8058c60a9a30f6a4e890" +dependencies = [ + "flate2", + "pkg-config", + "sha2 0.10.9", + "tar", + "ureq", +] + +[[package]] +name = "owned_ttf_parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b" +dependencies = [ + "ttf-parser", +] + [[package]] name = "parking" version = "2.2.1" @@ -2181,10 +2414,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] -name = "pastey" -version = "0.1.1" +name = "pem-rfc7468" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] [[package]] name = "percent-encoding" @@ -2215,13 +2451,19 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "png" -version = "0.18.1" +version = "0.17.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crc32fast", "fdeflate", "flate2", @@ -2254,6 +2496,21 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2329,12 +2586,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" -[[package]] -name = "pxfm" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" - [[package]] name = "qoi" version = "0.4.1" @@ -2512,6 +2763,16 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.7", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -2523,15 +2784,13 @@ dependencies = [ [[package]] name = "rav1e" -version = "0.8.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" dependencies = [ - "aligned-vec", "arbitrary", "arg_enum_proc_macro", "arrayvec", - "av-scenechange", "av1-grain", "bitstream-io", "built", @@ -2546,21 +2805,23 @@ dependencies = [ "noop_proc_macro", "num-derive", "num-traits", + "once_cell", "paste", "profiling", - "rand 0.9.5", - "rand_chacha 0.9.0", + "rand 0.8.7", + "rand_chacha 0.3.1", "simd_helpers", - "thiserror 2.0.18", + "system-deps", + "thiserror 1.0.69", "v_frame", "wasm-bindgen", ] [[package]] name = "ravif" -version = "0.13.0" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +checksum = "5825c26fddd16ab9f515930d49028a630efec172e903483c94796cfe31893e6b" dependencies = [ "avif-serialize", "imgref", @@ -2577,9 +2838,15 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags", + "bitflags 2.13.0", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -2818,7 +3085,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2831,7 +3098,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.12.1", @@ -2885,6 +3152,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2894,6 +3170,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "1.2.1" @@ -2920,6 +3205,29 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -2991,6 +3299,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3003,6 +3320,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.7" @@ -3052,6 +3382,19 @@ dependencies = [ "libc", ] +[[package]] +name = "simba" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061507c94fc6ab4ba1c9a0305018408e312e17c041eb63bef8aa726fa33aceae" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", + "wide", +] + [[package]] name = "simd-adler32" version = "0.3.9" @@ -3085,6 +3428,12 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "smallvec" +version = "2.0.0-alpha.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d44cfb396c3caf6fbfd0ab422af02631b69ddd96d2eff0b0f0724f9024051b" + [[package]] name = "socket2" version = "0.6.5" @@ -3095,6 +3444,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "sse-stream" version = "0.2.4" @@ -3168,6 +3528,36 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.27.0" @@ -3223,16 +3613,13 @@ dependencies = [ [[package]] name = "tiff" -version = "0.11.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" dependencies = [ - "fax", "flate2", - "half", - "quick-error", + "jpeg-decoder", "weezl", - "zune-jpeg", ] [[package]] @@ -3367,6 +3754,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -3389,7 +3810,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.0", "bytes", "futures-util", "http", @@ -3451,6 +3872,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "tungstenite" version = "0.23.0" @@ -3517,12 +3944,48 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -3548,6 +4011,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3588,6 +4057,18 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd4ec1eb1d240636e354a30110a1dfcb37047169a4d9bd6d9d3469df574b5c4" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + [[package]] name = "version_check" version = "0.9.5" @@ -3716,6 +4197,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -3764,6 +4254,16 @@ dependencies = [ "winsafe", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4011,6 +4511,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" @@ -4040,10 +4549,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "y4m" -version = "0.8.0" +name = "xattr" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] [[package]] name = "yoke" @@ -4185,9 +4698,9 @@ dependencies = [ [[package]] name = "zune-core" -version = "0.5.1" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" [[package]] name = "zune-inflate" @@ -4200,9 +4713,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.5.15" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index 8c17bb9f..614fdd9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,13 @@ async-trait = "0.1" axum = "0.8" base64 = "0.22" clap = { version = "4", features = ["derive"] } +clipper2 = { version = "=0.5.3", default-features = false } fs2 = "0.4" futures-util = "0.3" getrandom = "0.3" +image = { version = "=0.25.5", default-features = false, features = ["bmp", "gif", "jpeg", "png", "tiff", "webp"] } +imageproc = { version = "=0.25.0", default-features = false } +ort = { version = "=2.0.0-rc.10", default-features = false, features = ["copy-dylibs", "download-binaries", "ndarray", "std"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream"] } quick-xml = "0.38" regex = "1" @@ -33,7 +37,9 @@ rmcp = { version = "=0.8.5", default-features = false, features = ["base64", "cl schemars = "1.2" serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_yaml = "0.9" sha2 = "0.10" +tar = "0.4" thiserror = "2" tempfile = "3" tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "process", "sync", "time"] } diff --git a/README.md b/README.md index 5b4916fa..41e69e45 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,9 @@ a3s use mcp serve office-compat # Legacy alias: a3s use mcp serve office -# Built-in OCR; provider readiness remains explicit. +# Built-in local PP-OCRv6. a3s use ocr doctor --json -a3s use ocr extract ./scan.png --language eng --json +a3s use ocr extract ./scan.png --json a3s use mcp serve ocr ``` @@ -137,9 +137,9 @@ Every domain argument accepted by `a3s use ...` can also be passed directly to safe Word, Spreadsheet, Presentation, native MCP, and compatibility workflows - **External Domains**: Install process-isolated packages that expose any useful combination of CLI, MCP, and Skill surfaces -- **First-Party OCR Domain**: Extract text and bounded layout evidence with - a local Tesseract provider or an explicitly configured vision endpoint, - without silently installing a provider or hiding remote image transfer +- **First-Party OCR Domain**: Run pinned PP-OCRv6 detection and recognition + models locally through ONNX Runtime, with source digests and bounded layout + evidence - **Hot-Plug Discovery**: Publish immutable generation/revision snapshots so a resident host can add, replace, or remove live capabilities without restarting - **Content-Bound Skills**: Project an absolute package path and lowercase @@ -160,7 +160,7 @@ Every domain argument accepted by `a3s use ...` can also be passed directly to | Browser | Built in | Full Browser vocabulary | A3S Use standard MCP server | Six packaged Browser Skills | A3S Use | | Office | Built in | Stable Office vocabulary | Typed native preview plus OfficeCLI compatibility server | Packaged `a3s-use-office` Skill | A3S Use native engine; OfficeCLI compatibility in 0.1.x | | Box | Reserved built-in route | Native A3S Box vocabulary | — | — | Umbrella A3S CLI | -| OCR | Built in | Doctor and typed image extraction | `ocr_doctor` and `ocr_extract` | One provider-safe OCR Skill | A3S Use process and explicitly configured provider | +| OCR | Built in | Doctor and typed image extraction | `ocr_doctor` and `ocr_extract` | One local PP-OCRv6 Skill | A3S Use process with ONNX Runtime | | External domain | Installed extension | Optional native executable | Optional standard MCP server | Optional `SKILL.md` | Extension package plus A3S Use lifecycle | The Box route is component-backed. The umbrella CLI resolves its authoritative @@ -175,7 +175,7 @@ Default features are `browser`, `office`, `ocr`, `extensions`, and `mcp`. | --- | --- | | `browser` | Typed Browser library, stateless rendering, and full Browser driver delegation | | `office` | Typed Office contracts, native OOXML read engine, and temporary OfficeCLI compatibility | -| `ocr` | Built-in typed OCR CLI/MCP with local Tesseract and explicit vision providers | +| `ocr` | Built-in typed PP-OCRv6 CLI/MCP with local ONNX inference | | `extensions` | ACL manifests, package receipts, hot-plug registry, and external CLI/MCP/Skill routes | | `mcp` | Standard MCP servers plus the managed Browser Streamable HTTP lifecycle | | `lightpanda` | Explicit opt-in Lightpanda provider support in addition to Chrome | @@ -192,7 +192,7 @@ A compiled command surface is not proof that its provider is installed. Use | `a3s-use-browser-driver` | Complete interactive Browser CLI, MCP tools, Skills, Dashboard, and compatibility runtime | | `a3s-use-office` | Native OOXML foundation, typed Office operations, and compatibility lifecycle | | `a3s-use-extension` | A3S ACL manifest model, package registry, leases, and native surface descriptors | -| `a3s-use-ocr` | Typed local/vision OCR providers, CLI, MCP tools, and release-packaged Skill assets | +| `a3s-use-ocr` | Local PP-OCRv6 engine, CLI, MCP tools, pinned models, and release-packaged Skill assets | | `a3s-use` | Facade library, standalone CLI host, capability projection, and MCP entry points | ## Quick Start @@ -212,9 +212,9 @@ a3s use doctor --json Prebuilt archives are also published on [GitHub Releases](https://github.com/A3S-Lab/Use/releases). A complete archive contains `a3s-use`, its sibling `a3s-use-browser-driver`, Browser Skills, the -first-party Office and OCR Skills, the Dashboard, and license/provenance -notices. Keep those packaged assets together; installing only the facade binary -does not provide the complete Browser, Office, and OCR Skill surfaces. +first-party Office Skill, the Dashboard, and license/provenance notices. Keep +those packaged assets together; installing only the facade binary does not +provide the complete Browser and Office Skill surfaces. Build all binaries from source with: @@ -1610,27 +1610,33 @@ release packages its `a3s-use-ocr` Skill and exposes `ocr_doctor` plus `ocr_extract` over standard MCP, so a resident A3S Code session receives `mcp__use_ocr__*` without installing a separate extension. -OCR never installs a provider silently. `auto` prefers an explicitly configured -or discoverable Tesseract executable. Vision OCR is enabled only when its model -and endpoint configuration are present; non-loopback endpoints require HTTPS -and an API key, and the diagnostic discloses that the complete source image -leaves the device. Supported inputs are bounded local PNG, JPEG, WebP, GIF, +OCR has one backend: the pinned `PP-OCRv6_small` detection and recognition +models running locally through ONNX Runtime. Release archives package those +models; `a3s install use/ocr` explicitly installs or repairs the same pinned +bundle when needed. Supported inputs are bounded local PNG, JPEG, WebP, GIF, BMP, and TIFF files. The result binds the canonical source path, media type, -byte length, and SHA-256 alongside text and any available -confidence/bounding-box evidence. +byte length, and SHA-256 alongside text, recognition/detection confidence, +polygons, and bounding boxes. + +The pipeline decodes and normalizes the image, runs +`PP-OCRv6_small_det`, applies DB post-processing and reading-order sorting, +perspective-rectifies and rotates text crops, runs batched +`PP-OCRv6_small_rec`, and applies CTC decoding. It does not require Python or +PaddlePaddle, call a remote OCR API, or transfer source bytes off the device. ```bash a3s use ocr doctor --json -a3s use ocr extract ./scan.png --language eng --json +a3s use ocr extract ./scan.png --json a3s use mcp serve ocr ``` -A3S Code may first-use install the verified parent Use release. OCR provider -selection remains explicit, and remote vision extraction still escalates to -the parent TUI before source bytes leave the device. +A3S Code may first-use install the verified parent Use release. A missing or +damaged managed model bundle is repaired explicitly with +`a3s install use/ocr`; the Code `use` worker never installs it implicitly. + +See the [OCR crate](crates/ocr/README.md) for model resolution, the inference +workflow, and input boundaries. -See the [OCR crate](crates/ocr/README.md) for configuration and provider -boundaries. ## External Extensions External Use domains stay behind process boundaries. A package contains an @@ -1760,7 +1766,7 @@ crash, and in-flight calls retain the exact package generation they accepted. ┌──────────┬──────────┬──────────┬──────────────┐ │ │ │ │ │ Browser Office OCR extension registry - typed + driver OOXML local/vision CLI / MCP / Skill + typed + driver OOXML PP-OCRv6 ONNX CLI / MCP / Skill + 0.1 compat │ │ │ │ └──────── capability snapshot/watch ───────────► A3S Code @@ -1773,11 +1779,11 @@ crash, and in-flight calls retain the exact package generation they accepted. The dependency arrows are intentional. Search links only the Browser contract, so rendering does not require `a3s-use`, MCP, or a resident process. Office is an in-process typed engine with a temporary 0.1.x compatibility process. OCR -uses an explicitly present local Tesseract executable or an explicitly -configured vision provider; it never installs either silently. External -domains retain their process boundaries. A3S Code consumes the read-only -projection and connects standard MCP/Skill surfaces; bounded provider -installation requests still require the parent TUI's authority. +runs the pinned PP-OCRv6 models locally through ONNX Runtime; model installation +is an explicit component operation. External domains retain their process +boundaries. A3S Code consumes the read-only projection and connects standard +MCP/Skill surfaces; bounded component installation requests still require the +parent TUI's authority. Source is split between the facade under `src/` and focused workspace crates under `crates/`. See [Architecture](docs/architecture.md) for package leases, diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 596fd092..8040153b 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -11,3 +11,57 @@ license text is distributed as `LICENSE-APACHE-2.0`, and detailed provenance is distributed as `UPSTREAM.md`. Upstream repository: + +## PaddlePaddle/PaddleOCR PP-OCRv6 Models + +A3S Use release archives redistribute the official +`PP-OCRv6_small_det` and `PP-OCRv6_small_rec` ONNX inference model bundles +published by PaddlePaddle/PaddleOCR. The installer pins the upstream archive +URLs, byte sizes, and SHA-256 digests and does not modify the model weights. + +PaddleOCR is licensed under the Apache License, Version 2.0. + +Upstream repository: + +Model collection: + +## Microsoft ONNX Runtime + +`a3s-use-ocr` executes the models with Microsoft ONNX Runtime 1.22.0, obtained +through the pinned `ort`/`ort-sys` Rust dependencies. ONNX Runtime is licensed +under the MIT License. + +Copyright (c) Microsoft Corporation. + +Upstream repository: + +## pykeio/ort + +The `ort` and `ort-sys` Rust crates, version `2.0.0-rc.10`, provide the native +ONNX Runtime bindings and build integration. They are available under the MIT +License or the Apache License, Version 2.0. + +Upstream repository: + +## image-rs/imageproc + +`a3s-use-ocr` uses `imageproc` version `0.25.0` for geometric image +transformations. `imageproc` is licensed under the MIT License. + +Copyright (c) 2015 PistonDevelopers. + +Upstream repository: + +## clipper2 + +`a3s-use-ocr` uses the `clipper2` Rust crate version `0.5.3` and +`clipper2c-sys` version `0.1.6` for bounded polygon offsetting during DB +post-processing. The Rust crates are available under the MIT License or the +Apache License, Version 2.0. Their bundled Clipper2 C/C++ implementation is +licensed under the Boost Software License, Version 1.0. + +Upstream repositories: + +- +- +- diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml index 4a5f58aa..dac1f616 100644 --- a/crates/ocr/Cargo.toml +++ b/crates/ocr/Cargo.toml @@ -18,17 +18,22 @@ path = "src/main.rs" [dependencies] a3s-use-core = { version = "0.1.1", path = "../core" } -base64.workspace = true clap.workspace = true +clipper2.workspace = true +fs2.workspace = true +image.workspace = true +imageproc.workspace = true +ort.workspace = true reqwest = { workspace = true, features = ["json"] } rmcp.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +serde_yaml.workspace = true sha2.workspace = true +tar.workspace = true tokio.workspace = true url.workspace = true [dev-dependencies] -axum.workspace = true tempfile.workspace = true diff --git a/crates/ocr/README.md b/crates/ocr/README.md index d61b4096..a3b6e624 100644 --- a/crates/ocr/README.md +++ b/crates/ocr/README.md @@ -2,33 +2,50 @@ `a3s-use-ocr` implements the first-party built-in OCR domain for A3S Use. A3S Code receives it as `mcp__use_ocr__*` through the release-matched Use registry, -without a separate extension install. It exposes the same typed extraction -through a native CLI and standard stdio MCP, and does not silently install an -OCR provider. +without installing a separate extension. The native CLI and standard stdio MCP +share one local PP-OCRv6 implementation. -Provider selection is explicit: +There is one OCR provider: -- `A3S_OCR_PROVIDER=auto|tesseract|vision` -- `A3S_OCR_TESSERACT_EXECUTABLE=/absolute/path/to/tesseract` -- `A3S_OCR_VISION_MODEL=` -- `A3S_OCR_VISION_BASE_URL=https://provider.example/v1/` -- `A3S_OCR_VISION_API_KEY=` -- `A3S_OCR_TIMEOUT_MS=60000` +- provider: `pp-ocr-v6` +- engine: `onnx-runtime` +- model bundle: `PP-OCRv6_small` -`auto` prefers a configured or discoverable local Tesseract executable. It uses -the vision provider only when the vision environment is configured. Remote -vision endpoints require HTTPS and an API key; loopback HTTP is allowed for a -local provider. +The release packages the pinned detection and recognition models. If the model +bundle is absent or damaged, install or repair it explicitly: -Build and exercise the domain through the Use facade: +```bash +a3s install use/ocr +a3s install use/ocr --force +``` + +`A3S_OCR_MODEL_DIR` can point development builds at an explicit model bundle. +`A3S_USE_OCR_HOME` overrides the managed model root for packaging, tests, or an +isolated installation. Neither setting selects another OCR backend. + +## Workflow + +For each bounded local image, the native engine: + +1. decodes the image and applies PP-OCRv6 BGR normalization; +2. runs `PP-OCRv6_small_det` through ONNX Runtime; +3. applies DB post-processing, polygon unclipping, and reading-order sorting; +4. perspective-rectifies each text polygon and rotates tall crops; +5. runs batched `PP-OCRv6_small_rec` inference; and +6. applies CTC decoding and returns text, recognition/detection confidence, + polygons, bounding boxes, and the source SHA-256. + +All inference stays in the local `a3s-use` process. It does not require Python +or PaddlePaddle, does not call an OCR API, and does not transfer image bytes off +the device. + +## Commands ```bash a3s use ocr doctor --json -a3s use ocr extract ./scan.png --language eng --json +a3s use ocr extract ./scan.png --json a3s use mcp serve ocr ``` -The A3S Use release packages the OCR Skill beside the facade binary. A3S Code -can first-use install that verified release and hot-plug the built-in route. -Provider setup remains explicit: local Tesseract never sends source bytes -off-device, while a configured remote vision provider requires parent HITL. +Supported inputs are bounded local PNG, JPEG, WebP, GIF, BMP, and TIFF files. +URLs and PDF rasterization are outside this crate. diff --git a/crates/ocr/skills/a3s-use-ocr/SKILL.md b/crates/ocr/skills/a3s-use-ocr/SKILL.md index 9ca0950b..0a876eaf 100644 --- a/crates/ocr/skills/a3s-use-ocr/SKILL.md +++ b/crates/ocr/skills/a3s-use-ocr/SKILL.md @@ -1,36 +1,37 @@ --- name: a3s-use-ocr -description: Extract text and layout evidence from local image files through the built-in A3S Use OCR domain. Use when an agent needs optical character recognition for a PNG, JPEG, WebP, GIF, BMP, or TIFF image and must preserve the source digest, provider disclosure, confidence, and bounding-box evidence. +description: Extract text and layout evidence from local image files through the built-in A3S Use PP-OCRv6 domain. Use when an agent needs optical character recognition for a PNG, JPEG, WebP, GIF, BMP, or TIFF image and must preserve the source digest, confidence, polygon, and bounding-box evidence. --- # A3S Use OCR Use the host-provided A3S Use surface. In an A3S Code `use` worker, call `mcp__use_ocr__ocr_doctor` and `mcp__use_ocr__ocr_extract` directly. The host -owns the MCP process; do not run a shell command, install a provider, or read the +owns the MCP process; do not run a shell command, install models, or read the file through another tool. ## Workflow 1. Call `mcp__use_ocr__ocr_doctor`. -2. Confirm which provider is ready and whether `sendsSourceOffDevice` is true. +2. Confirm that `pp-ocr-v6`, `onnx-runtime`, and `PP-OCRv6_small` are ready. 3. Call `mcp__use_ocr__ocr_extract` with the exact local image path from the - task. Supply language identifiers only when known. -4. Preserve the returned source path, media type, size, and SHA-256 in the - result. Treat text, confidence, and bounding boxes as OCR evidence, not as a - verified transcription. - -The local Tesseract provider does not send the image over the network. The -vision provider sends the complete source image and prompt to its disclosed -endpoint. Do not use a non-loopback vision provider unless the user has -authorized that data transfer. Never install, repair, or switch providers from -inside the `use` worker. + task. +4. Preserve the returned source path, media type, size, and SHA-256. Treat the + decoded text, recognition/detection confidence, polygons, and bounding boxes + as OCR evidence rather than verified source text. + +The engine runs detection, reading-order sorting, perspective crop correction, +tall-crop rotation, recognition, and CTC decoding locally. It does not require +Python or PaddlePaddle and never sends the source image off the device. If the +doctor reports missing or damaged models, return its typed error and explicit +`a3s install use/ocr` suggestion to the parent; never install or repair models +from inside the `use` worker. In a CLI-only host, equivalent commands are: ```bash a3s use ocr doctor --json -a3s use ocr extract "$IMAGE" --language eng --json +a3s use ocr extract "$IMAGE" --json ``` `a3s-use-ocr` accepts the same arguments when invoked as a standalone @@ -40,9 +41,7 @@ development binary. - Only bounded local image files are accepted. URLs and PDF rasterization are outside this domain. -- Keep the default prompt for faithful transcription. A custom vision prompt - must remain an extraction instruction; do not ask the provider to interpret - unrelated content. -- Never report vision output as calibrated confidence or layout evidence. -- Do not silently fall back from a requested provider. Report typed provider, - source, and response errors to the parent agent. +- Do not ask OCR to interpret unrelated content or present OCR output as + verified source text. +- Do not hide empty results, warnings, model readiness failures, or source + digest evidence from the parent agent. diff --git a/crates/ocr/src/assets.rs b/crates/ocr/src/assets.rs new file mode 100644 index 00000000..e60e23fa --- /dev/null +++ b/crates/ocr/src/assets.rs @@ -0,0 +1,261 @@ +use std::path::{Path, PathBuf}; + +use a3s_use_core::{UseError, UseResult}; +use serde::{Deserialize, Serialize}; + +use crate::config::{load_detection, load_recognition, MODEL_FAMILY}; + +pub(crate) const RECEIPT_FILE: &str = ".a3s-ppocr-v6.json"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OcrInstallSource { + Environment, + Packaged, + Managed, + Missing, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OcrRuntimeStatus { + pub available: bool, + pub source: OcrInstallSource, + pub model: String, + pub model_dir: Option, + pub managed_root: Option, + pub detail: String, +} + +#[derive(Debug, Clone)] +pub(crate) struct ModelAssets { + pub(crate) root: PathBuf, + pub(crate) detection_model: PathBuf, + pub(crate) detection_config: PathBuf, + pub(crate) recognition_model: PathBuf, + pub(crate) recognition_config: PathBuf, + pub(crate) source: OcrInstallSource, +} + +pub fn ocr_status() -> OcrRuntimeStatus { + let managed_root = managed_root().ok(); + match resolve_model_assets() { + Ok(assets) => OcrRuntimeStatus { + available: true, + source: assets.source, + model: MODEL_FAMILY.to_string(), + model_dir: Some(assets.root), + managed_root, + detail: "ready".to_string(), + }, + Err(error) => OcrRuntimeStatus { + available: false, + source: error + .details + .get("source") + .and_then(serde_json::Value::as_str) + .map(source_from_name) + .unwrap_or(OcrInstallSource::Missing), + model: MODEL_FAMILY.to_string(), + model_dir: error + .details + .get("modelDir") + .and_then(serde_json::Value::as_str) + .map(PathBuf::from), + managed_root, + detail: error.message, + }, + } +} + +pub(crate) fn resolve_model_assets() -> UseResult { + if let Some(path) = std::env::var_os("A3S_OCR_MODEL_DIR") + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + { + let path = absolute(path)?; + return validate_assets(&path, OcrInstallSource::Environment); + } + + let managed = managed_model_dir()?; + if path_exists(&managed)? { + return validate_assets(&managed, OcrInstallSource::Managed); + } + + if let Ok(executable) = std::env::current_exe() { + if let Some(parent) = executable.parent() { + let packaged = parent.join("ocr-models").join(MODEL_FAMILY); + if path_exists(&packaged)? { + return validate_assets(&packaged, OcrInstallSource::Packaged); + } + } + } + + Err(UseError::new( + "use.ocr.model_missing", + format!("The local {MODEL_FAMILY} model bundle is not installed."), + ) + .with_suggestion("Run 'a3s install use/ocr'.") + .with_detail("source", "missing") + .with_detail("modelDir", managed.display().to_string())) +} + +pub(crate) fn validate_assets(root: &Path, source: OcrInstallSource) -> UseResult { + let root = std::fs::canonicalize(root).map_err(|error| { + model_error( + source, + root, + format!( + "Failed to resolve the {MODEL_FAMILY} model directory '{}': {error}", + root.display() + ), + ) + })?; + let detection_model = checked_file(&root, "det/inference.onnx", 256 * 1024 * 1024, source)?; + let detection_config = checked_file(&root, "det/inference.yml", 2 * 1024 * 1024, source)?; + let recognition_model = checked_file(&root, "rec/inference.onnx", 256 * 1024 * 1024, source)?; + let recognition_config = checked_file(&root, "rec/inference.yml", 2 * 1024 * 1024, source)?; + + load_detection(&detection_config)?; + load_recognition(&recognition_config)?; + + Ok(ModelAssets { + root, + detection_model, + detection_config, + recognition_model, + recognition_config, + source, + }) +} + +pub(crate) fn managed_root() -> UseResult { + if let Some(value) = std::env::var_os("A3S_USE_OCR_HOME") { + return absolute(PathBuf::from(value)); + } + if let Some(value) = std::env::var_os("A3S_DATA_HOME") { + return Ok(absolute(PathBuf::from(value))?.join("use/ocr")); + } + if let Some(value) = std::env::var_os("XDG_DATA_HOME") { + return Ok(absolute(PathBuf::from(value))?.join("a3s/use/ocr")); + } + if let Some(home) = std::env::var_os("HOME").map(PathBuf::from) { + return Ok(absolute(home)?.join(".local/share/a3s/use/ocr")); + } + #[cfg(windows)] + if let Some(value) = std::env::var_os("LOCALAPPDATA") { + return Ok(absolute(PathBuf::from(value))?.join("a3s/use/ocr")); + } + Err(UseError::new( + "use.ocr.data_home_missing", + "Cannot determine the A3S Use OCR data directory.", + )) +} + +pub(crate) fn managed_model_dir() -> UseResult { + Ok(managed_root()?.join(MODEL_FAMILY)) +} + +fn checked_file( + root: &Path, + relative: &str, + max_bytes: u64, + source: OcrInstallSource, +) -> UseResult { + let path = root.join(relative); + let canonical = std::fs::canonicalize(&path).map_err(|error| { + model_error( + source, + root, + format!( + "Required {MODEL_FAMILY} asset '{}' is unreadable: {error}", + path.display() + ), + ) + })?; + if !canonical.starts_with(root) { + return Err(model_error( + source, + root, + format!( + "Required {MODEL_FAMILY} asset '{}' escapes its model directory.", + path.display() + ), + )); + } + let metadata = std::fs::metadata(&canonical).map_err(|error| { + model_error( + source, + root, + format!( + "Failed to inspect {MODEL_FAMILY} asset '{}': {error}", + canonical.display() + ), + ) + })?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > max_bytes { + return Err(model_error( + source, + root, + format!( + "{MODEL_FAMILY} asset '{}' must be a non-empty regular file no larger than {max_bytes} bytes.", + canonical.display() + ), + )); + } + Ok(canonical) +} + +fn path_exists(path: &Path) -> UseResult { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(UseError::new( + "use.ocr.model_unreadable", + format!( + "Failed to inspect OCR model path '{}': {error}", + path.display() + ), + )), + } +} + +fn model_error(source: OcrInstallSource, root: &Path, message: impl Into) -> UseError { + UseError::new("use.ocr.model_invalid", message) + .with_suggestion("Run 'a3s install use/ocr --force' to restore the pinned PP-OCRv6 bundle.") + .with_detail("source", source_name(source)) + .with_detail("modelDir", root.display().to_string()) +} + +fn source_name(source: OcrInstallSource) -> &'static str { + match source { + OcrInstallSource::Environment => "environment", + OcrInstallSource::Packaged => "packaged", + OcrInstallSource::Managed => "managed", + OcrInstallSource::Missing => "missing", + } +} + +fn source_from_name(value: &str) -> OcrInstallSource { + match value { + "environment" => OcrInstallSource::Environment, + "packaged" => OcrInstallSource::Packaged, + "managed" => OcrInstallSource::Managed, + _ => OcrInstallSource::Missing, + } +} + +fn absolute(path: PathBuf) -> UseResult { + if path.is_absolute() { + Ok(path) + } else { + std::env::current_dir() + .map(|directory| directory.join(path)) + .map_err(|error| { + UseError::new( + "use.ocr.path_resolution_failed", + format!("Failed to resolve OCR data path: {error}"), + ) + }) + } +} diff --git a/crates/ocr/src/cli.rs b/crates/ocr/src/cli.rs index ea76bcda..f5e3e690 100644 --- a/crates/ocr/src/cli.rs +++ b/crates/ocr/src/cli.rs @@ -2,10 +2,10 @@ use std::path::PathBuf; use a3s_use_core::{UseError, UseResult}; use clap::error::ErrorKind; -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; use serde::Serialize; -use crate::{OcrClient, OcrMcpServer, OcrProviderKind, OcrRequest}; +use crate::{OcrClient, OcrMcpServer, OcrRequest}; #[derive(Debug)] pub struct CommandOutput { @@ -75,24 +75,10 @@ struct Cli { #[derive(Debug, Subcommand)] enum Command { - /// Inspect provider readiness without reading an image. + /// Inspect local PP-OCRv6 readiness without reading an image. Doctor, - /// Extract text and available layout evidence from one local image. - Extract { - path: PathBuf, - /// OCR language identifier; may be repeated. - #[arg(long = "language")] - languages: Vec, - /// Tesseract page segmentation mode from 0 through 13. - #[arg(long = "psm")] - page_segmentation_mode: Option, - /// Override the configured OCR provider for this call. - #[arg(long, value_enum)] - provider: Option, - /// Vision-only extraction instruction. - #[arg(long)] - prompt: Option, - }, + /// Extract text and layout evidence from one local image. + Extract { path: PathBuf }, /// Run an extension protocol surface. Serve { /// Serve standard MCP over stdin/stdout. @@ -101,23 +87,6 @@ enum Command { }, } -#[derive(Debug, Clone, Copy, ValueEnum)] -enum ProviderArg { - Auto, - Tesseract, - Vision, -} - -impl From for OcrProviderKind { - fn from(value: ProviderArg) -> Self { - match value { - ProviderArg::Auto => Self::Auto, - ProviderArg::Tesseract => Self::Tesseract, - ProviderArg::Vision => Self::Vision, - } - } -} - pub async fn run(args: Vec) -> UseResult { let mut argv = vec!["a3s-use-ocr".to_string()]; argv.extend(args); @@ -148,23 +117,9 @@ pub async fn run(args: Vec) -> UseResult { let client = OcrClient::from_env()?; match cli.command { Command::Doctor => CommandOutput::data(client.diagnostic()), - Command::Extract { - path, - languages, - page_segmentation_mode, - provider, - prompt, - } => CommandOutput::data( - client - .extract(OcrRequest { - path, - languages, - page_segmentation_mode, - provider: provider.map(Into::into), - prompt, - }) - .await?, - ), + Command::Extract { path } => { + CommandOutput::data(client.extract(OcrRequest { path }).await?) + } Command::Serve { .. } => Err(UseError::new( "use.ocr.command_invalid", "OCR MCP command dispatch reached an invalid state.", diff --git a/crates/ocr/src/client.rs b/crates/ocr/src/client.rs index 21fb3494..9dc3e33c 100644 --- a/crates/ocr/src/client.rs +++ b/crates/ocr/src/client.rs @@ -1,216 +1,178 @@ -use std::collections::BTreeMap; -use std::path::Path; -#[cfg(all(test, unix))] -use std::path::PathBuf; -use std::process::Stdio; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; -use a3s_use_core::{Artifact, UseError, UseResult}; -use base64::Engine; +use a3s_use_core::{Artifact, Readiness, UseError, UseResult}; use sha2::{Digest, Sha256}; use tokio::io::AsyncReadExt; -use tokio::process::Command; -use crate::models::{OcrBlock, OcrBoundingBox, OcrProviderKind, OcrRequest, OcrResult}; -use crate::provider::{Provider, ProviderConfig}; -use crate::OcrDiagnostic; +use crate::assets::{ocr_status, resolve_model_assets, OcrInstallSource}; +use crate::config::MODEL_FAMILY; +use crate::engine::{EngineBlock, PpOcrV6Engine}; +use crate::models::{ + OcrBlock, OcrBoundingBox, OcrDiagnostic, OcrPoint, OcrProviderKind, OcrRequest, OcrResult, +}; +use crate::preprocess::decode_image; const MAX_INPUT_BYTES: u64 = 32 * 1024 * 1024; -const MAX_PROVIDER_OUTPUT_BYTES: usize = 8 * 1024 * 1024; -const DEFAULT_VISION_PROMPT: &str = "Transcribe all visible text in reading order. Preserve line breaks and meaningful spacing. Return only the transcription; do not summarize, translate, or wrap it in Markdown."; +const ENGINE_NAME: &str = "onnx-runtime"; #[derive(Clone)] pub struct OcrClient { - providers: ProviderConfig, - http: reqwest::Client, + loaded: Arc>>, +} + +struct LoadedEngine { + model_dir: PathBuf, + engine: PpOcrV6Engine, } impl OcrClient { pub fn from_env() -> UseResult { - Self::from_provider_config(ProviderConfig::from_env()?) - } - - fn from_provider_config(providers: ProviderConfig) -> UseResult { - let http = reqwest::Client::builder() - .user_agent(concat!("a3s-use-ocr/", env!("CARGO_PKG_VERSION"))) - .build() - .map_err(|error| { - UseError::new( - "use.ocr.client_failed", - format!("Failed to initialize the OCR HTTP client: {error}"), - ) - })?; - Ok(Self { providers, http }) - } - - #[cfg(all(test, unix))] - pub(crate) fn with_tesseract(executable: PathBuf) -> UseResult { - Self::from_provider_config(ProviderConfig::tesseract(executable)) + Ok(Self { + loaded: Arc::new(Mutex::new(None)), + }) } pub fn diagnostic(&self) -> OcrDiagnostic { - self.providers.diagnostic() + let status = ocr_status(); + let (readiness, suggestions) = if status.available { + (Readiness::Ready, Vec::new()) + } else if status.source == OcrInstallSource::Missing { + ( + Readiness::Missing, + vec![ + "Run 'a3s install use/ocr' to install the pinned local model bundle." + .to_string(), + ], + ) + } else { + ( + Readiness::Broken, + vec![ + "Run 'a3s install use/ocr --force' to restore the pinned local model bundle." + .to_string(), + ], + ) + }; + OcrDiagnostic { + readiness, + provider: Some(OcrProviderKind::PpOcrV6), + engine: Some(ENGINE_NAME.to_string()), + model: Some(status.model), + model_dir: status.model_dir, + sends_source_off_device: false, + message: if status.available { + "Local PP-OCRv6 detection and recognition models are ready.".to_string() + } else { + status.detail + }, + suggestions, + } } pub async fn extract(&self, request: OcrRequest) -> UseResult { - validate_request(&request)?; let source = read_source(&request.path).await?; - let provider = self - .providers - .resolve(request.provider.unwrap_or(OcrProviderKind::Auto))?; - let languages = if request.languages.is_empty() { - vec!["eng".to_string()] - } else { - request.languages.clone() - }; - - let (text, blocks, warnings) = match &provider { - Provider::Tesseract { - executable, - timeout, - } => { - let output = run_tesseract( - executable, - &source.artifact.path, - &languages, - request.page_segmentation_mode, - *timeout, + let loaded = Arc::clone(&self.loaded); + tokio::task::spawn_blocking(move || { + let image = decode_image(&source.bytes)?; + let assets = resolve_model_assets()?; + let mut loaded = loaded.lock().map_err(|_| { + UseError::new( + "use.ocr.runtime_failed", + "The local PP-OCRv6 engine lock is poisoned.", ) - .await?; - let (text, blocks) = parse_tesseract_tsv(&output)?; - (text, blocks, Vec::new()) + })?; + let should_load = loaded + .as_ref() + .map(|loaded| loaded.model_dir != assets.root) + .unwrap_or(true); + if should_load { + *loaded = Some(LoadedEngine { + model_dir: assets.root.clone(), + engine: PpOcrV6Engine::load(&assets)?, + }); } - Provider::Vision { - endpoint, - api_key, - model, - timeout, - } => { - let text = self - .run_vision( - endpoint, - api_key.as_deref(), - model, - &source, - request.prompt.as_deref(), - *timeout, - ) - .await?; - let blocks = (!text.is_empty()) - .then(|| OcrBlock { - page: 1, - text: text.clone(), - confidence: None, - bounding_box: None, - }) - .into_iter() - .collect(); - ( - text, - blocks, - vec![ - "The vision provider does not return calibrated OCR confidence or bounding boxes." - .to_string(), - ], + let engine = loaded.as_mut().ok_or_else(|| { + UseError::new( + "use.ocr.runtime_failed", + "The local PP-OCRv6 engine failed to initialize.", ) - } - }; - - Ok(OcrResult { - provider: provider.kind(), - source: source.artifact, - languages, - text, - blocks, - warnings, + })?; + let blocks = engine.engine.extract(&image)?; + build_result(source.artifact, blocks) }) - } - - async fn run_vision( - &self, - endpoint: &url::Url, - api_key: Option<&str>, - model: &str, - source: &SourceImage, - prompt: Option<&str>, - timeout: std::time::Duration, - ) -> UseResult { - let encoded = base64::engine::general_purpose::STANDARD.encode(&source.bytes); - let data_url = format!("data:{};base64,{encoded}", source.artifact.media_type); - let prompt = prompt - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(DEFAULT_VISION_PROMPT); - let body = serde_json::json!({ - "model": model, - "temperature": 0, - "messages": [{ - "role": "user", - "content": [ - { "type": "text", "text": prompt }, - { - "type": "image_url", - "image_url": { - "url": data_url, - "detail": "high" - } - } - ] - }] - }); - let mut request = self - .http - .post(endpoint.clone()) - .timeout(timeout) - .json(&body); - if let Some(api_key) = api_key { - request = request.bearer_auth(api_key); - } - let response = request.send().await.map_err(|error| { - UseError::new( - "use.ocr.vision_request_failed", - format!("The vision OCR request failed: {error}"), - ) - .with_detail("endpoint", redacted_endpoint(endpoint)) - })?; - let status = response.status(); - let bytes = response.bytes().await.map_err(|error| { - UseError::new( - "use.ocr.vision_response_invalid", - format!("Failed to read the vision OCR response: {error}"), - ) - })?; - if bytes.len() > MAX_PROVIDER_OUTPUT_BYTES { - return Err(UseError::new( - "use.ocr.output_too_large", - "The vision OCR provider response exceeded 8 MiB.", - )); - } - if !status.is_success() { - let message = String::from_utf8_lossy(&bytes); - return Err(UseError::new( - "use.ocr.vision_request_failed", - format!( - "The vision OCR provider returned HTTP {status}: {}", - bounded_text(&message, 1024) - ), - ) - .with_detail("status", u64::from(status.as_u16()))); - } - let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|error| { - UseError::new( - "use.ocr.vision_response_invalid", - format!("The vision OCR provider returned invalid JSON: {error}"), - ) - })?; - let content = value.pointer("/choices/0/message/content").ok_or_else(|| { + .await + .map_err(|error| { UseError::new( - "use.ocr.vision_response_invalid", - "The vision OCR response did not contain choices[0].message.content.", + "use.ocr.runtime_failed", + format!("The local PP-OCRv6 inference task failed: {error}"), ) - })?; - let text = vision_content_text(content)?; - Ok(text.trim().to_string()) + })? + } +} + +fn build_result(source: Artifact, blocks: Vec) -> UseResult { + let blocks = blocks + .into_iter() + .map(|block| { + let [first, second, third, fourth] = block.polygon; + let polygon = [ + ocr_point(first)?, + ocr_point(second)?, + ocr_point(third)?, + ocr_point(fourth)?, + ]; + let min_x = polygon.iter().map(|point| point.x).min().unwrap_or(0); + let max_x = polygon.iter().map(|point| point.x).max().unwrap_or(0); + let min_y = polygon.iter().map(|point| point.y).min().unwrap_or(0); + let max_y = polygon.iter().map(|point| point.y).max().unwrap_or(0); + Ok(OcrBlock { + page: 1, + text: block.text, + confidence: block.confidence, + detection_confidence: block.detection_confidence, + polygon, + bounding_box: OcrBoundingBox { + x: min_x, + y: min_y, + width: max_x.saturating_sub(min_x), + height: max_y.saturating_sub(min_y), + }, + }) + }) + .collect::>>()?; + let text = blocks + .iter() + .filter(|block| !block.text.trim().is_empty()) + .map(|block| block.text.as_str()) + .collect::>() + .join("\n"); + Ok(OcrResult { + provider: OcrProviderKind::PpOcrV6, + engine: ENGINE_NAME.to_string(), + model: MODEL_FAMILY.to_string(), + source, + text, + blocks, + warnings: Vec::new(), + }) +} + +fn ocr_point(point: imageproc::point::Point) -> UseResult { + Ok(OcrPoint { + x: finite_coordinate(point.x)?, + y: finite_coordinate(point.y)?, + }) +} + +fn finite_coordinate(value: f32) -> UseResult { + if !value.is_finite() || value < 0.0 || value > u32::MAX as f32 { + return Err(UseError::new( + "use.ocr.provider_output_invalid", + "PP-OCRv6 returned an invalid polygon coordinate.", + )); } + Ok(value.round() as u32) } struct SourceImage { @@ -303,247 +265,6 @@ async fn read_source(path: &Path) -> UseResult { }) } -fn validate_request(request: &OcrRequest) -> UseResult<()> { - if request.languages.len() > 16 { - return Err(UseError::new( - "use.ocr.languages_invalid", - "At most 16 OCR language identifiers may be requested.", - )); - } - for language in &request.languages { - if language.is_empty() - || language.len() > 32 - || !language - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) - { - return Err(UseError::new( - "use.ocr.languages_invalid", - format!("OCR language identifier '{language}' is invalid."), - )); - } - } - if request.page_segmentation_mode.is_some_and(|mode| mode > 13) { - return Err(UseError::new( - "use.ocr.page_segmentation_invalid", - "Tesseract page segmentation mode must be from 0 through 13.", - )); - } - if request - .prompt - .as_ref() - .is_some_and(|prompt| prompt.len() > 8 * 1024) - { - return Err(UseError::new( - "use.ocr.prompt_too_large", - "The vision OCR prompt must not exceed 8192 bytes.", - )); - } - Ok(()) -} - -async fn run_tesseract( - executable: &Path, - source: &Path, - languages: &[String], - page_segmentation_mode: Option, - timeout: std::time::Duration, -) -> UseResult> { - let mut command = Command::new(executable); - command - .arg(source) - .arg("stdout") - .arg("-l") - .arg(languages.join("+")) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true); - if let Some(mode) = page_segmentation_mode { - command.arg("--psm").arg(mode.to_string()); - } - command.arg("tsv"); - - let output = tokio::time::timeout(timeout, command.output()) - .await - .map_err(|_| { - UseError::new( - "use.ocr.provider_timeout", - format!( - "Tesseract exceeded the {} ms OCR timeout.", - timeout.as_millis() - ), - ) - })? - .map_err(|error| { - UseError::new( - "use.ocr.provider_failed", - format!( - "Failed to launch Tesseract executable '{}': {error}", - executable.display() - ), - ) - })?; - if output.stdout.len() > MAX_PROVIDER_OUTPUT_BYTES - || output.stderr.len() > MAX_PROVIDER_OUTPUT_BYTES - { - return Err(UseError::new( - "use.ocr.output_too_large", - "Tesseract output exceeded 8 MiB.", - )); - } - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(UseError::new( - "use.ocr.provider_failed", - format!( - "Tesseract exited with {}: {}", - output.status, - bounded_text(&stderr, 2048) - ), - )); - } - Ok(output.stdout) -} - -#[derive(Default)] -struct LineAccumulator { - page: u32, - words: Vec, - confidence_sum: f32, - confidence_count: usize, - left: u32, - top: u32, - right: u32, - bottom: u32, -} - -fn parse_tesseract_tsv(output: &[u8]) -> UseResult<(String, Vec)> { - let output = std::str::from_utf8(output).map_err(|error| { - UseError::new( - "use.ocr.provider_output_invalid", - format!("Tesseract TSV output was not UTF-8: {error}"), - ) - })?; - let mut lines = BTreeMap::<(u32, u32, u32, u32), LineAccumulator>::new(); - for (index, row) in output.lines().enumerate() { - if index == 0 && row.starts_with("level\t") { - continue; - } - if row.trim().is_empty() { - continue; - } - let columns = row.splitn(12, '\t').collect::>(); - if columns.len() != 12 { - return Err(UseError::new( - "use.ocr.provider_output_invalid", - format!( - "Tesseract TSV row {} did not contain 12 columns.", - index + 1 - ), - )); - } - let level = parse_u32(columns[0], index)?; - if level != 5 || columns[11].trim().is_empty() { - continue; - } - let page = parse_u32(columns[1], index)?; - let block = parse_u32(columns[2], index)?; - let paragraph = parse_u32(columns[3], index)?; - let line = parse_u32(columns[4], index)?; - let left = parse_u32(columns[6], index)?; - let top = parse_u32(columns[7], index)?; - let width = parse_u32(columns[8], index)?; - let height = parse_u32(columns[9], index)?; - let confidence = columns[10] - .parse::() - .ok() - .filter(|value| *value >= 0.0); - let entry = lines - .entry((page, block, paragraph, line)) - .or_insert_with(|| LineAccumulator { - page, - left, - top, - right: left.saturating_add(width), - bottom: top.saturating_add(height), - ..LineAccumulator::default() - }); - entry.words.push(columns[11].trim().to_string()); - if let Some(confidence) = confidence { - entry.confidence_sum += confidence; - entry.confidence_count += 1; - } - entry.left = entry.left.min(left); - entry.top = entry.top.min(top); - entry.right = entry.right.max(left.saturating_add(width)); - entry.bottom = entry.bottom.max(top.saturating_add(height)); - } - let blocks = lines - .into_values() - .filter_map(|line| { - let text = line.words.join(" "); - (!text.is_empty()).then(|| OcrBlock { - page: line.page, - text, - confidence: (line.confidence_count > 0) - .then(|| line.confidence_sum / line.confidence_count as f32), - bounding_box: Some(OcrBoundingBox { - x: line.left, - y: line.top, - width: line.right.saturating_sub(line.left), - height: line.bottom.saturating_sub(line.top), - }), - }) - }) - .collect::>(); - let text = blocks - .iter() - .map(|block| block.text.as_str()) - .collect::>() - .join("\n"); - Ok((text, blocks)) -} - -fn parse_u32(value: &str, row: usize) -> UseResult { - value.parse::().map_err(|_| { - UseError::new( - "use.ocr.provider_output_invalid", - format!( - "Tesseract TSV row {} contained an invalid integer.", - row + 1 - ), - ) - }) -} - -fn vision_content_text(content: &serde_json::Value) -> UseResult { - if let Some(text) = content.as_str() { - return Ok(text.to_string()); - } - let Some(parts) = content.as_array() else { - return Err(UseError::new( - "use.ocr.vision_response_invalid", - "Vision OCR message content was neither text nor a text-part array.", - )); - }; - let text = parts - .iter() - .filter_map(|part| { - part.get("text") - .and_then(serde_json::Value::as_str) - .or_else(|| part.as_str()) - }) - .collect::>() - .join(""); - if text.is_empty() { - return Err(UseError::new( - "use.ocr.vision_response_invalid", - "Vision OCR message content did not contain text.", - )); - } - Ok(text) -} - fn detect_image_type(bytes: &[u8]) -> Option<&'static str> { if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { Some("image/png") @@ -562,46 +283,10 @@ fn detect_image_type(bytes: &[u8]) -> Option<&'static str> { } } -fn bounded_text(value: &str, max: usize) -> String { - let mut text = value.chars().take(max).collect::(); - if value.chars().count() > max { - text.push('…'); - } - text -} - -fn redacted_endpoint(endpoint: &url::Url) -> String { - let mut redacted = endpoint.clone(); - redacted.set_query(None); - redacted.set_fragment(None); - redacted.to_string() -} - #[cfg(test)] mod tests { use super::*; - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - - #[test] - fn parses_tesseract_words_into_ordered_lines() { - let tsv = b"level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t10\t95.0\tHello\n5\t1\t1\t1\t1\t2\t45\t20\t35\t10\t85.0\tworld\n5\t1\t1\t1\t2\t1\t10\t40\t20\t10\t90.0\tNext\n"; - let (text, blocks) = parse_tesseract_tsv(tsv).unwrap(); - assert_eq!(text, "Hello world\nNext"); - assert_eq!(blocks.len(), 2); - assert_eq!(blocks[0].confidence, Some(90.0)); - assert_eq!( - blocks[0].bounding_box, - Some(OcrBoundingBox { - x: 10, - y: 20, - width: 70, - height: 10, - }) - ); - } - #[test] fn detects_supported_image_signatures() { assert_eq!( @@ -612,38 +297,11 @@ mod tests { assert_eq!(detect_image_type(b"not an image"), None); } - #[cfg(unix)] - #[tokio::test] - async fn local_provider_extracts_a_real_bounded_source_through_its_process_boundary() { - let temp = tempfile::tempdir().unwrap(); - let executable = temp.path().join("tesseract-fixture"); - std::fs::write( - &executable, - "#!/bin/sh\nprintf 'level\\tpage_num\\tblock_num\\tpar_num\\tline_num\\tword_num\\tleft\\ttop\\twidth\\theight\\tconf\\ttext\\n5\\t1\\t1\\t1\\t1\\t1\\t2\\t3\\t20\\t8\\t98.0\\tA3S\\n5\\t1\\t1\\t1\\t1\\t2\\t24\\t3\\t30\\t8\\t96.0\\tUse\\n'\n", - ) - .unwrap(); - let mut permissions = std::fs::metadata(&executable).unwrap().permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&executable, permissions).unwrap(); - let image = temp.path().join("scan.png"); - std::fs::write(&image, b"\x89PNG\r\n\x1a\nfixture").unwrap(); - - let result = OcrClient::with_tesseract(executable) - .unwrap() - .extract(OcrRequest { - path: image, - languages: vec!["eng".to_string()], - page_segmentation_mode: Some(6), - provider: Some(OcrProviderKind::Tesseract), - prompt: None, - }) - .await - .unwrap(); - - assert_eq!(result.provider, OcrProviderKind::Tesseract); - assert_eq!(result.text, "A3S Use"); - assert_eq!(result.blocks.len(), 1); - assert_eq!(result.source.media_type, "image/png"); - assert_eq!(result.source.sha256.len(), 64); + #[test] + fn diagnostic_never_discloses_an_off_device_provider() { + let diagnostic = OcrClient::from_env().unwrap().diagnostic(); + assert_eq!(diagnostic.provider, Some(OcrProviderKind::PpOcrV6)); + assert!(!diagnostic.sends_source_off_device); + assert_eq!(diagnostic.engine.as_deref(), Some(ENGINE_NAME)); } } diff --git a/crates/ocr/src/config.rs b/crates/ocr/src/config.rs new file mode 100644 index 00000000..9395ed07 --- /dev/null +++ b/crates/ocr/src/config.rs @@ -0,0 +1,261 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use a3s_use_core::{UseError, UseResult}; +use serde::Deserialize; +use serde_yaml::Value; + +pub(crate) const MODEL_FAMILY: &str = "PP-OCRv6_small"; +pub(crate) const DETECTION_MODEL: &str = "PP-OCRv6_small_det"; +pub(crate) const RECOGNITION_MODEL: &str = "PP-OCRv6_small_rec"; + +#[derive(Debug, Clone)] +pub(crate) struct DetectionConfig { + pub(crate) scale: f32, + pub(crate) mean: [f32; 3], + pub(crate) std: [f32; 3], + pub(crate) threshold: f32, + pub(crate) box_threshold: f32, + pub(crate) max_candidates: usize, + pub(crate) unclip_ratio: f32, +} + +#[derive(Debug, Clone)] +pub(crate) struct RecognitionConfig { + pub(crate) channels: usize, + pub(crate) height: usize, + pub(crate) default_width: usize, + pub(crate) characters: Vec, +} + +#[derive(Debug, Deserialize)] +struct RawConfig { + #[serde(rename = "Global")] + global: RawGlobal, + #[serde(rename = "PreProcess")] + pre_process: RawPreProcess, + #[serde(rename = "PostProcess")] + post_process: RawPostProcess, +} + +#[derive(Debug, Deserialize)] +struct RawGlobal { + model_name: String, +} + +#[derive(Debug, Deserialize)] +struct RawPreProcess { + transform_ops: Vec>, +} + +#[derive(Debug, Deserialize)] +struct RawPostProcess { + #[serde(default)] + name: String, + #[serde(default = "default_threshold")] + thresh: f32, + #[serde(default = "default_box_threshold")] + box_thresh: f32, + #[serde(default = "default_max_candidates")] + max_candidates: usize, + #[serde(default = "default_unclip_ratio")] + unclip_ratio: f32, + #[serde(default)] + character_dict: Vec, +} + +pub(crate) fn load_detection(path: &Path) -> UseResult { + let raw = load(path)?; + if raw.global.model_name != DETECTION_MODEL { + return Err(config_error(format!( + "Expected detection model '{DETECTION_MODEL}', found '{}'.", + raw.global.model_name + ))); + } + if raw.post_process.name != "DBPostProcess" { + return Err(config_error(format!( + "Expected DBPostProcess, found '{}'.", + raw.post_process.name + ))); + } + let normalize = transform(&raw.pre_process.transform_ops, "NormalizeImage") + .ok_or_else(|| config_error("Detection config has no NormalizeImage transform."))?; + let scale = normalize + .get("scale") + .and_then(parse_scale) + .unwrap_or(1.0 / 255.0); + let mean = float_triplet(normalize.get("mean"), [0.485, 0.456, 0.406])?; + let std = float_triplet(normalize.get("std"), [0.229, 0.224, 0.225])?; + if std.iter().any(|value| *value <= 0.0) { + return Err(config_error( + "Detection normalization standard deviations must be positive.", + )); + } + Ok(DetectionConfig { + scale, + mean, + std, + threshold: raw.post_process.thresh, + box_threshold: raw.post_process.box_thresh, + max_candidates: raw.post_process.max_candidates.min(10_000), + unclip_ratio: raw.post_process.unclip_ratio, + }) +} + +pub(crate) fn load_recognition(path: &Path) -> UseResult { + let raw = load(path)?; + if raw.global.model_name != RECOGNITION_MODEL { + return Err(config_error(format!( + "Expected recognition model '{RECOGNITION_MODEL}', found '{}'.", + raw.global.model_name + ))); + } + if raw.post_process.name != "CTCLabelDecode" { + return Err(config_error(format!( + "Expected CTCLabelDecode, found '{}'.", + raw.post_process.name + ))); + } + if raw.post_process.character_dict.is_empty() || raw.post_process.character_dict.len() > 100_000 + { + return Err(config_error( + "Recognition character dictionary is empty or unreasonably large.", + )); + } + let resize = transform(&raw.pre_process.transform_ops, "RecResizeImg") + .ok_or_else(|| config_error("Recognition config has no RecResizeImg transform."))?; + let shape = resize + .get("image_shape") + .and_then(Value::as_sequence) + .ok_or_else(|| config_error("RecResizeImg.image_shape must be an integer triplet."))?; + if shape.len() != 3 { + return Err(config_error( + "RecResizeImg.image_shape must contain channels, height, and width.", + )); + } + let dimensions = shape + .iter() + .map(|value| value.as_u64().and_then(|value| usize::try_from(value).ok())) + .collect::>>() + .ok_or_else(|| config_error("RecResizeImg.image_shape contains an invalid dimension."))?; + if dimensions[0] != 3 + || !(16..=256).contains(&dimensions[1]) + || !(32..=4096).contains(&dimensions[2]) + { + return Err(config_error(format!( + "Unsupported PP-OCRv6 recognition input shape {:?}.", + dimensions + ))); + } + Ok(RecognitionConfig { + channels: dimensions[0], + height: dimensions[1], + default_width: dimensions[2], + characters: raw.post_process.character_dict, + }) +} + +fn load(path: &Path) -> UseResult { + let metadata = std::fs::metadata(path).map_err(|error| { + config_error(format!( + "Failed to inspect PP-OCRv6 config '{}': {error}", + path.display() + )) + })?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > 2 * 1024 * 1024 { + return Err(config_error(format!( + "PP-OCRv6 config '{}' must be a non-empty regular file no larger than 2 MiB.", + path.display() + ))); + } + let text = std::fs::read_to_string(path).map_err(|error| { + config_error(format!( + "Failed to read PP-OCRv6 config '{}': {error}", + path.display() + )) + })?; + serde_yaml::from_str(&text).map_err(|error| { + config_error(format!( + "Failed to parse PP-OCRv6 config '{}': {error}", + path.display() + )) + }) +} + +fn transform<'a>( + transforms: &'a [BTreeMap], + name: &str, +) -> Option<&'a serde_yaml::Mapping> { + transforms + .iter() + .find_map(|transform| transform.get(name)) + .and_then(Value::as_mapping) +} + +fn float_triplet(value: Option<&Value>, default: [f32; 3]) -> UseResult<[f32; 3]> { + let Some(values) = value.and_then(Value::as_sequence) else { + return Ok(default); + }; + if values.len() != 3 { + return Err(config_error( + "Detection normalization mean and std must contain three values.", + )); + } + let mut output = [0.0_f32; 3]; + for (index, value) in values.iter().enumerate() { + output[index] = yaml_f32(value) + .ok_or_else(|| config_error("Detection normalization contains a non-number."))?; + } + Ok(output) +} + +fn parse_scale(value: &Value) -> Option { + if let Some(value) = yaml_f32(value) { + return Some(value); + } + let value = value.as_str()?.trim(); + if let Some((numerator, denominator)) = value.split_once('/') { + let numerator = numerator.trim_matches('.').parse::().ok()?; + let denominator = denominator.trim_matches('.').parse::().ok()?; + return (denominator != 0.0).then_some(numerator / denominator); + } + value.parse().ok() +} + +fn yaml_f32(value: &Value) -> Option { + value + .as_f64() + .map(|value| value as f32) + .filter(|value| value.is_finite()) +} + +fn default_threshold() -> f32 { + 0.3 +} + +fn default_box_threshold() -> f32 { + 0.6 +} + +fn default_max_candidates() -> usize { + 1_000 +} + +fn default_unclip_ratio() -> f32 { + 1.5 +} + +fn config_error(message: impl Into) -> UseError { + UseError::new("use.ocr.model_config_invalid", message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_fractional_detection_scale() { + let value = Value::String("1./255.".to_string()); + assert!((parse_scale(&value).unwrap() - 1.0 / 255.0).abs() < f32::EPSILON); + } +} diff --git a/crates/ocr/src/engine.rs b/crates/ocr/src/engine.rs new file mode 100644 index 00000000..4c72f1fa --- /dev/null +++ b/crates/ocr/src/engine.rs @@ -0,0 +1,261 @@ +use std::path::Path; + +use a3s_use_core::{UseError, UseResult}; +use image::{imageops, ImageBuffer, Rgb, RgbImage}; +use imageproc::geometric_transformations::{warp_into, Interpolation, Projection}; +use imageproc::point::Point; +use ort::session::builder::GraphOptimizationLevel; +use ort::session::Session; +use ort::value::TensorRef; + +use crate::assets::ModelAssets; +use crate::config::{load_detection, load_recognition, DetectionConfig, RecognitionConfig}; +use crate::postprocess::{decode_ctc, detection_boxes, Detection}; +use crate::preprocess::{detection_input, recognition_input}; + +const RECOGNITION_BATCH_SIZE: usize = 8; +const MAX_CROP_PIXELS: u64 = 64 * 1024 * 1024; + +#[derive(Debug, Clone)] +pub(crate) struct EngineBlock { + pub(crate) polygon: [Point; 4], + pub(crate) detection_confidence: f32, + pub(crate) text: String, + pub(crate) confidence: f32, +} + +pub(crate) struct PpOcrV6Engine { + detection: Session, + recognition: Session, + detection_config: DetectionConfig, + recognition_config: RecognitionConfig, +} + +impl PpOcrV6Engine { + pub(crate) fn load(assets: &ModelAssets) -> UseResult { + let detection_config = load_detection(&assets.detection_config)?; + let recognition_config = load_recognition(&assets.recognition_config)?; + let detection = load_session(&assets.detection_model, "detection")?; + let recognition = load_session(&assets.recognition_model, "recognition")?; + Ok(Self { + detection, + recognition, + detection_config, + recognition_config, + }) + } + + pub(crate) fn extract(&mut self, image: &RgbImage) -> UseResult> { + let input = detection_input(image, &self.detection_config)?; + let (shape, output) = + run_session(&mut self.detection, &input.data, input.shape, "detection")?; + let detections = detection_boxes( + &output, + &shape, + input.original_width, + input.original_height, + &self.detection_config, + )?; + if detections.is_empty() { + return Ok(Vec::new()); + } + + let crops = detections + .iter() + .map(|detection| perspective_crop(image, detection)) + .collect::>>()?; + let mut blocks = Vec::with_capacity(detections.len()); + for (detection_batch, crop_batch) in detections + .chunks(RECOGNITION_BATCH_SIZE) + .zip(crops.chunks(RECOGNITION_BATCH_SIZE)) + { + let input = recognition_input(crop_batch, &self.recognition_config)?; + let (shape, output) = run_session( + &mut self.recognition, + &input.data, + input.shape, + "recognition", + )?; + if shape.len() != 3 || shape[0] != detection_batch.len() { + return Err(engine_error( + "use.ocr.provider_output_invalid", + format!( + "PP-OCRv6 recognition output shape must be [N, T, C] for N={}, found {shape:?}.", + detection_batch.len() + ), + )); + } + let item_len = shape[1].checked_mul(shape[2]).ok_or_else(|| { + engine_error( + "use.ocr.provider_output_invalid", + "PP-OCRv6 recognition output dimensions overflowed.", + ) + })?; + if output.len() != detection_batch.len().saturating_mul(item_len) { + return Err(engine_error( + "use.ocr.provider_output_invalid", + "PP-OCRv6 recognition output length does not match its batch shape.", + )); + } + for (index, detection) in detection_batch.iter().enumerate() { + let start = index * item_len; + let recognition = decode_ctc( + &output[start..start + item_len], + &[1, shape[1], shape[2]], + &self.recognition_config, + )?; + blocks.push(EngineBlock { + polygon: detection.polygon, + detection_confidence: detection.confidence, + text: recognition.text, + confidence: recognition.confidence, + }); + } + } + Ok(blocks) + } +} + +fn load_session(path: &Path, role: &str) -> UseResult { + let session = Session::builder() + .map_err(|error| runtime_error(role, "create an ONNX Runtime session", error))? + .with_optimization_level(GraphOptimizationLevel::Level3) + .map_err(|error| runtime_error(role, "configure graph optimization", error))? + .commit_from_file(path) + .map_err(|error| runtime_error(role, "load the ONNX model", error))?; + if session.inputs.len() != 1 || session.outputs.len() != 1 { + return Err(engine_error( + "use.ocr.model_invalid", + format!( + "PP-OCRv6 {role} model must expose exactly one input and one output; found {} inputs and {} outputs.", + session.inputs.len(), + session.outputs.len() + ), + )); + } + Ok(session) +} + +fn run_session( + session: &mut Session, + data: &[f32], + shape: [usize; 4], + role: &str, +) -> UseResult<(Vec, Vec)> { + let expected = shape + .iter() + .try_fold(1_usize, |total, dimension| total.checked_mul(*dimension)); + if expected != Some(data.len()) { + return Err(engine_error( + "use.ocr.provider_input_invalid", + format!("PP-OCRv6 {role} tensor length does not match its shape."), + )); + } + let input = TensorRef::from_array_view((shape, data)) + .map_err(|error| runtime_error(role, "create an ONNX Runtime input tensor", error))?; + let outputs = session + .run(ort::inputs![input]) + .map_err(|error| runtime_error(role, "run ONNX inference", error))?; + if outputs.len() != 1 { + return Err(engine_error( + "use.ocr.provider_output_invalid", + format!( + "PP-OCRv6 {role} inference returned {} outputs instead of one.", + outputs.len() + ), + )); + } + let output = outputs.values().next().ok_or_else(|| { + engine_error( + "use.ocr.provider_output_invalid", + format!("PP-OCRv6 {role} inference returned no output tensor."), + ) + })?; + let (output_shape, output_data) = output + .try_extract_tensor::() + .map_err(|error| runtime_error(role, "read the ONNX output tensor", error))?; + let output_shape = output_shape + .iter() + .map(|dimension| { + usize::try_from(*dimension).map_err(|_| { + engine_error( + "use.ocr.provider_output_invalid", + format!("PP-OCRv6 {role} output contains an invalid dimension {dimension}."), + ) + }) + }) + .collect::>>()?; + if output_data.iter().any(|value| !value.is_finite()) { + return Err(engine_error( + "use.ocr.provider_output_invalid", + format!("PP-OCRv6 {role} output contains a non-finite value."), + )); + } + Ok((output_shape, output_data.to_vec())) +} + +fn perspective_crop(image: &RgbImage, detection: &Detection) -> UseResult { + let polygon = detection.polygon; + let width = distance(polygon[0], polygon[1]) + .max(distance(polygon[2], polygon[3])) + .round() + .max(1.0) as u32; + let height = distance(polygon[0], polygon[3]) + .max(distance(polygon[1], polygon[2])) + .round() + .max(1.0) as u32; + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .ok_or_else(|| crop_error("PP-OCRv6 text crop dimensions overflowed."))?; + if pixels > MAX_CROP_PIXELS { + return Err(crop_error( + "PP-OCRv6 text crop exceeds the 64 megapixel safety limit.", + )); + } + + let source = polygon.map(|point| (point.x, point.y)); + let destination = [ + (0.0, 0.0), + (width.saturating_sub(1) as f32, 0.0), + ( + width.saturating_sub(1) as f32, + height.saturating_sub(1) as f32, + ), + (0.0, height.saturating_sub(1) as f32), + ]; + let projection = Projection::from_control_points(source, destination).ok_or_else(|| { + crop_error("PP-OCRv6 detected a degenerate text polygon that cannot be rectified.") + })?; + let mut crop = ImageBuffer::new(width, height); + warp_into( + image, + &projection, + Interpolation::Bicubic, + Rgb([255, 255, 255]), + &mut crop, + ); + if f64::from(height) / f64::from(width) >= 1.5 { + Ok(imageops::rotate270(&crop)) + } else { + Ok(crop) + } +} + +fn distance(left: Point, right: Point) -> f32 { + (left.x - right.x).hypot(left.y - right.y) +} + +fn runtime_error(role: &str, action: &str, error: impl std::fmt::Display) -> UseError { + engine_error( + "use.ocr.runtime_failed", + format!("Failed to {action} for PP-OCRv6 {role}: {error}"), + ) +} + +fn crop_error(message: impl Into) -> UseError { + engine_error("use.ocr.crop_invalid", message) +} + +fn engine_error(code: &str, message: impl Into) -> UseError { + UseError::new(code, message) +} diff --git a/crates/ocr/src/install.rs b/crates/ocr/src/install.rs new file mode 100644 index 00000000..91031170 --- /dev/null +++ b/crates/ocr/src/install.rs @@ -0,0 +1,671 @@ +use std::fs::OpenOptions; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use a3s_use_core::{UseError, UseResult}; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::io::AsyncWriteExt; + +use crate::assets::{ + managed_model_dir, managed_root, ocr_status, validate_assets, OcrInstallSource, + OcrRuntimeStatus, RECEIPT_FILE, +}; +use crate::config::MODEL_FAMILY; + +const INSTALL_LOCK: &str = ".install.lock"; +const STAGE_PREFIX: &str = ".stage-"; +const BACKUP_PREFIX: &str = ".backup-"; +const DOWNLOAD_HOST: &str = "paddle-model-ecology.bj.bcebos.com"; +const MAX_ARCHIVE_BYTES: u64 = 256 * 1024 * 1024; + +const DETECTION_ARCHIVE: PinnedArchive = PinnedArchive { + role: "det", + directory: "PP-OCRv6_small_det_onnx_infer", + url: "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv6_small_det_onnx_infer.tar", + bytes: 9_891_840, + sha256: "d218f6fbf0f1c23d2161bd6ac7f5eaa6104fa89955c09290497e31008e2618e4", +}; +const RECOGNITION_ARCHIVE: PinnedArchive = PinnedArchive { + role: "rec", + directory: "PP-OCRv6_small_rec_onnx_infer", + url: "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv6_small_rec_onnx_infer.tar", + bytes: 21_319_680, + sha256: "d267ab077a44a0eedb1ea8f8c542d263f211de8e9d7a029bf9fcfff7e5a88fb1", +}; + +#[derive(Debug, Clone, Copy)] +struct PinnedArchive { + role: &'static str, + directory: &'static str, + url: &'static str, + bytes: u64, + sha256: &'static str, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct InstallReceipt { + schema_version: u32, + provider: String, + model: String, + detection_url: String, + detection_sha256: String, + recognition_url: String, + recognition_sha256: String, +} + +struct InstallLock { + _file: std::fs::File, +} + +struct Downloaded { + bytes: u64, + sha256: String, +} + +pub async fn install_ppocr_v6(force: bool) -> UseResult { + let current = ocr_status(); + if !force && current.available { + return Ok(current); + } + + let root = managed_root()?; + let _lock = acquire_lock(&root).await?; + cleanup_stale(&root).await?; + + let current = ocr_status(); + if !force && current.available { + return Ok(current); + } + + let stage = create_stage(&root).await?; + let install_result = install_into(&stage).await; + if install_result.is_err() { + let _ = tokio::fs::remove_dir_all(&stage).await; + } + install_result?; + + let target = managed_model_dir()?; + activate(&stage, &target).await?; + validate_assets(&target, OcrInstallSource::Managed)?; + + let status = ocr_status(); + if status.available { + Ok(status) + } else { + Err(ocr_error( + "use.ocr.install_failed", + "PP-OCRv6 installation completed without a usable model bundle.", + )) + } +} + +pub async fn repair_ppocr_v6() -> UseResult { + let status = ocr_status(); + if status.available { + Ok(status) + } else { + install_ppocr_v6(true).await + } +} + +pub async fn uninstall_managed_ppocr_v6() -> UseResult { + let root = managed_root()?; + let _lock = acquire_lock(&root).await?; + let target = managed_model_dir()?; + if !owned_install(&target) { + return Ok(false); + } + tokio::fs::remove_dir_all(&target).await.map_err(|error| { + ocr_error( + "use.ocr.uninstall_failed", + format!( + "Failed to remove managed PP-OCRv6 bundle '{}': {error}", + target.display() + ), + ) + })?; + Ok(true) +} + +async fn install_into(stage: &Path) -> UseResult<()> { + let client = download_client()?; + for archive in [DETECTION_ARCHIVE, RECOGNITION_ARCHIVE] { + let archive_path = stage.join(format!("{}.tar", archive.role)); + let downloaded = download(&client, archive.url, &archive_path).await?; + if downloaded.bytes != archive.bytes || downloaded.sha256 != archive.sha256 { + return Err(ocr_error( + "use.ocr.integrity_mismatch", + format!( + "{} archive integrity mismatch: expected {} bytes and {}, got {} bytes and {}.", + archive.directory, + archive.bytes, + archive.sha256, + downloaded.bytes, + downloaded.sha256 + ), + )); + } + let archive_path_for_task = archive_path.clone(); + let destination = stage.join(archive.role); + tokio::task::spawn_blocking(move || { + extract_archive(&archive_path_for_task, &destination, archive) + }) + .await + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!("PP-OCRv6 archive extraction task failed: {error}"), + ) + })??; + tokio::fs::remove_file(&archive_path) + .await + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to remove staged archive '{}': {error}", + archive_path.display() + ), + ) + })?; + } + write_receipt(stage).await?; + validate_assets(stage, OcrInstallSource::Managed)?; + Ok(()) +} + +fn download_client() -> UseResult { + let redirects = reqwest::redirect::Policy::custom(|attempt| { + let approved = attempt.previous().len() < 5 + && attempt.url().scheme() == "https" + && attempt.url().host_str() == Some(DOWNLOAD_HOST); + if approved { + attempt.follow() + } else { + attempt.error("PP-OCRv6 download redirected to an unapproved host") + } + }); + reqwest::Client::builder() + .user_agent(concat!("a3s-use-ocr/", env!("CARGO_PKG_VERSION"))) + .redirect(redirects) + .timeout(std::time::Duration::from_secs(300)) + .build() + .map_err(|error| { + ocr_error( + "use.ocr.download_failed", + format!("Failed to create PP-OCRv6 download client: {error}"), + ) + }) +} + +async fn download( + client: &reqwest::Client, + value: &str, + destination: &Path, +) -> UseResult { + let url = reqwest::Url::parse(value).map_err(|error| { + ocr_error( + "use.ocr.download_source_invalid", + format!("Invalid PP-OCRv6 download URL: {error}"), + ) + })?; + if url.scheme() != "https" || url.host_str() != Some(DOWNLOAD_HOST) { + return Err(ocr_error( + "use.ocr.download_source_invalid", + "PP-OCRv6 download source is not the pinned official HTTPS host.", + )); + } + let mut response = client + .get(url) + .send() + .await + .map_err(|error| { + ocr_error( + "use.ocr.download_failed", + format!("Failed to download PP-OCRv6: {error}"), + ) + })? + .error_for_status() + .map_err(|error| { + ocr_error( + "use.ocr.download_failed", + format!("PP-OCRv6 download failed: {error}"), + ) + })?; + if response + .content_length() + .is_some_and(|length| length > MAX_ARCHIVE_BYTES) + { + return Err(ocr_error( + "use.ocr.download_too_large", + "PP-OCRv6 archive exceeds the 256 MiB limit.", + )); + } + let mut file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(destination) + .await + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to create PP-OCRv6 download '{}': {error}", + destination.display() + ), + ) + })?; + let mut hasher = Sha256::new(); + let mut total = 0_u64; + while let Some(chunk) = response.chunk().await.map_err(|error| { + ocr_error( + "use.ocr.download_failed", + format!("Failed to read PP-OCRv6 download: {error}"), + ) + })? { + total = total + .checked_add(chunk.len() as u64) + .ok_or_else(|| ocr_error("use.ocr.download_too_large", "Download size overflowed."))?; + if total > MAX_ARCHIVE_BYTES { + return Err(ocr_error( + "use.ocr.download_too_large", + "PP-OCRv6 archive exceeds the 256 MiB limit.", + )); + } + hasher.update(&chunk); + file.write_all(&chunk).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to write PP-OCRv6 download '{}': {error}", + destination.display() + ), + ) + })?; + } + file.flush().await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to flush PP-OCRv6 download '{}': {error}", + destination.display() + ), + ) + })?; + file.sync_all().await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to sync PP-OCRv6 download '{}': {error}", + destination.display() + ), + ) + })?; + Ok(Downloaded { + bytes: total, + sha256: format!("{:x}", hasher.finalize()), + }) +} + +fn extract_archive(archive_path: &Path, destination: &Path, spec: PinnedArchive) -> UseResult<()> { + std::fs::create_dir(destination).map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to create PP-OCRv6 model directory '{}': {error}", + destination.display() + ), + ) + })?; + let file = std::fs::File::open(archive_path).map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to open PP-OCRv6 archive '{}': {error}", + archive_path.display() + ), + ) + })?; + let mut archive = tar::Archive::new(file); + let mut extracted = [false; 2]; + for entry in archive.entries().map_err(archive_error)? { + let entry = entry.map_err(archive_error)?; + let path = entry.path().map_err(archive_error)?; + let components = path.components().collect::>(); + if components.len() == 1 + && matches!(components[0], Component::Normal(value) if value == spec.directory) + && entry.header().entry_type().is_dir() + { + continue; + } + if components.len() != 2 + || !matches!(components[0], Component::Normal(value) if value == spec.directory) + || !entry.header().entry_type().is_file() + { + return Err(ocr_error( + "use.ocr.archive_invalid", + format!( + "PP-OCRv6 archive contains an unexpected entry '{}'.", + path.display() + ), + )); + } + let name = match components[1] { + Component::Normal(name) if name == "inference.onnx" => { + extracted[0] = true; + "inference.onnx" + } + Component::Normal(name) if name == "inference.yml" => { + extracted[1] = true; + "inference.yml" + } + _ => { + return Err(ocr_error( + "use.ocr.archive_invalid", + format!( + "PP-OCRv6 archive contains an unexpected entry '{}'.", + path.display() + ), + )) + } + }; + let max = if name.ends_with(".onnx") { + 256 * 1024 * 1024 + } else { + 2 * 1024 * 1024 + }; + if entry.size() == 0 || entry.size() > max { + return Err(ocr_error( + "use.ocr.archive_invalid", + format!("PP-OCRv6 archive entry '{name}' has an invalid size."), + )); + } + let expected_size = entry.size(); + let output_path = destination.join(name); + let mut output = OpenOptions::new() + .create_new(true) + .write(true) + .open(&output_path) + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to create PP-OCRv6 asset '{}': {error}", + output_path.display() + ), + ) + })?; + let copied = std::io::copy(&mut entry.take(max + 1), &mut output).map_err(archive_error)?; + if copied != expected_size { + return Err(ocr_error( + "use.ocr.archive_invalid", + format!("PP-OCRv6 archive entry '{name}' was truncated."), + )); + } + output.flush().map_err(archive_error)?; + output.sync_all().map_err(archive_error)?; + } + if !extracted.into_iter().all(|present| present) { + return Err(ocr_error( + "use.ocr.archive_invalid", + "PP-OCRv6 archive is missing inference.onnx or inference.yml.", + )); + } + Ok(()) +} + +async fn write_receipt(stage: &Path) -> UseResult<()> { + let receipt = InstallReceipt { + schema_version: 1, + provider: "pp-ocr-v6".to_string(), + model: MODEL_FAMILY.to_string(), + detection_url: DETECTION_ARCHIVE.url.to_string(), + detection_sha256: DETECTION_ARCHIVE.sha256.to_string(), + recognition_url: RECOGNITION_ARCHIVE.url.to_string(), + recognition_sha256: RECOGNITION_ARCHIVE.sha256.to_string(), + }; + let bytes = serde_json::to_vec_pretty(&receipt).map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!("Failed to encode PP-OCRv6 install receipt: {error}"), + ) + })?; + let path = stage.join(RECEIPT_FILE); + let mut file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&path) + .await + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to create PP-OCRv6 receipt '{}': {error}", + path.display() + ), + ) + })?; + file.write_all(&bytes).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to write PP-OCRv6 receipt '{}': {error}", + path.display() + ), + ) + })?; + file.sync_all().await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to sync PP-OCRv6 receipt '{}': {error}", + path.display() + ), + ) + }) +} + +async fn acquire_lock(root: &Path) -> UseResult { + tokio::fs::create_dir_all(root).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to create OCR data root '{}': {error}", + root.display() + ), + ) + })?; + let path = root.join(INSTALL_LOCK); + tokio::task::spawn_blocking(move || { + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&path) + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to open OCR install lock '{}': {error}", + path.display() + ), + ) + })?; + file.lock_exclusive().map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to acquire OCR install lock '{}': {error}", + path.display() + ), + ) + })?; + Ok(InstallLock { _file: file }) + }) + .await + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!("OCR install lock task failed: {error}"), + ) + })? +} + +async fn create_stage(root: &Path) -> UseResult { + static NEXT_STAGE: AtomicU64 = AtomicU64::new(1); + for _ in 0..32 { + let path = root.join(format!( + "{STAGE_PREFIX}{}-{}", + std::process::id(), + NEXT_STAGE.fetch_add(1, Ordering::Relaxed) + )); + match tokio::fs::create_dir(&path).await { + Ok(()) => return Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(ocr_error( + "use.ocr.install_failed", + format!( + "Failed to create OCR staging directory '{}': {error}", + path.display() + ), + )) + } + } + } + Err(ocr_error( + "use.ocr.install_failed", + "Failed to allocate a unique OCR staging directory.", + )) +} + +async fn cleanup_stale(root: &Path) -> UseResult<()> { + let mut entries = tokio::fs::read_dir(root).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to inspect OCR data root '{}': {error}", + root.display() + ), + ) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to inspect OCR data root '{}': {error}", + root.display() + ), + ) + })? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + let is_owned_backup = name.starts_with(BACKUP_PREFIX) && owned_install(&entry.path()); + if name.starts_with(STAGE_PREFIX) || is_owned_backup { + tokio::fs::remove_dir_all(entry.path()) + .await + .map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!("Failed to remove stale OCR staging directory: {error}"), + ) + })?; + } + } + Ok(()) +} + +async fn activate(stage: &Path, target: &Path) -> UseResult<()> { + static NEXT_BACKUP: AtomicU64 = AtomicU64::new(1); + let parent = target.parent().ok_or_else(|| { + ocr_error( + "use.ocr.install_failed", + "OCR install target has no parent directory.", + ) + })?; + let backup = parent.join(format!( + "{BACKUP_PREFIX}{}-{}", + std::process::id(), + NEXT_BACKUP.fetch_add(1, Ordering::Relaxed) + )); + let had_target = tokio::fs::try_exists(target).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to inspect OCR install target '{}': {error}", + target.display() + ), + ) + })?; + if had_target { + if !owned_install(target) { + return Err(ocr_error( + "use.ocr.install_target_unowned", + format!( + "Refusing to replace unowned OCR model directory '{}'.", + target.display() + ), + )); + } + tokio::fs::rename(target, &backup).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Failed to stage existing OCR install '{}': {error}", + target.display() + ), + ) + })?; + } + if let Err(error) = tokio::fs::rename(stage, target).await { + if had_target { + let _ = tokio::fs::rename(&backup, target).await; + } + return Err(ocr_error( + "use.ocr.install_failed", + format!( + "Failed to activate OCR install '{}': {error}", + target.display() + ), + )); + } + if had_target { + tokio::fs::remove_dir_all(&backup).await.map_err(|error| { + ocr_error( + "use.ocr.install_failed", + format!( + "Activated OCR but failed to remove backup '{}': {error}", + backup.display() + ), + ) + })?; + } + Ok(()) +} + +fn owned_install(path: &Path) -> bool { + let Ok(bytes) = std::fs::read(path.join(RECEIPT_FILE)) else { + return false; + }; + serde_json::from_slice::(&bytes).is_ok_and(|receipt| { + receipt.schema_version == 1 + && receipt.provider == "pp-ocr-v6" + && receipt.model == MODEL_FAMILY + }) +} + +fn archive_error(error: impl std::fmt::Display) -> UseError { + ocr_error( + "use.ocr.archive_invalid", + format!("Failed to extract PP-OCRv6 archive: {error}"), + ) +} + +fn ocr_error(code: &str, message: impl Into) -> UseError { + UseError::new(code, message) +} diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index a1bd1740..57486889 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -1,18 +1,27 @@ //! Typed optical character recognition for A3S Use. //! //! OCR is a first-party built-in Use domain and remains process-isolated from -//! A3S Code through its standard MCP server. The crate supports a local -//! Tesseract executable and an explicitly configured OpenAI-compatible vision -//! endpoint without silently installing either provider. +//! A3S Code through its standard MCP server. Detection and recognition run +//! locally with the pinned PP-OCRv6_small ONNX models. There is no alternate +//! OCR provider or off-device fallback. +mod assets; pub mod cli; mod client; +mod config; +mod engine; +mod install; pub mod mcp; mod models; -mod provider; +mod postprocess; +mod preprocess; +pub use assets::{ocr_status, OcrInstallSource, OcrRuntimeStatus}; pub use client::OcrClient; +pub use install::{install_ppocr_v6, repair_ppocr_v6, uninstall_managed_ppocr_v6}; pub use mcp::OcrMcpServer; -pub use models::{OcrBlock, OcrBoundingBox, OcrDiagnostic, OcrProviderKind, OcrRequest, OcrResult}; +pub use models::{ + OcrBlock, OcrBoundingBox, OcrDiagnostic, OcrPoint, OcrProviderKind, OcrRequest, OcrResult, +}; pub use a3s_use_core::{Artifact, Readiness, UseError, UseResult}; diff --git a/crates/ocr/src/mcp.rs b/crates/ocr/src/mcp.rs index f2de4ada..1a3db29f 100644 --- a/crates/ocr/src/mcp.rs +++ b/crates/ocr/src/mcp.rs @@ -43,7 +43,7 @@ impl OcrMcpServer { impl OcrMcpServer { #[tool( name = "ocr_doctor", - description = "Inspect OCR provider readiness without reading an image or making a network request", + description = "Inspect local PP-OCRv6 model readiness without reading an image or making a network request", output_schema = rmcp::handler::server::tool::cached_schema_for_type::(), annotations( read_only_hint = true, @@ -58,13 +58,13 @@ impl OcrMcpServer { #[tool( name = "ocr_extract", - description = "Extract text and available layout evidence from one bounded local image; the configured vision provider may send source bytes to its disclosed endpoint", + description = "Extract text, polygons, bounding boxes, and confidence from one bounded local image with PP-OCRv6; source bytes remain on this device", output_schema = rmcp::handler::server::tool::cached_schema_for_type::(), annotations( read_only_hint = true, destructive_hint = false, idempotent_hint = true, - open_world_hint = true + open_world_hint = false ) )] async fn ocr_extract( @@ -88,7 +88,7 @@ impl ServerHandler for OcrMcpServer { website_url: Some("https://github.com/A3S-Lab/Use".to_string()), }, instructions: Some( - "Call ocr_doctor first. Use ocr_extract only for a local image path supplied by the task. A vision provider may send the complete image to its configured endpoint; do not use it without the user's authority. Preserve the source SHA-256 and distinguish OCR text from verified source text." + "Call ocr_doctor first. Use ocr_extract only for a local image path supplied by the task. PP-OCRv6 detection and recognition run locally through ONNX Runtime and never send source bytes off device. Preserve the source SHA-256 and distinguish OCR text from verified source text." .to_string(), ), ..Default::default() @@ -164,7 +164,7 @@ mod tests { .annotations .as_ref() .and_then(|annotations| annotations.open_world_hint), - Some(true) + Some(false) ); } } diff --git a/crates/ocr/src/models.rs b/crates/ocr/src/models.rs index 6cbf6bae..29b41198 100644 --- a/crates/ocr/src/models.rs +++ b/crates/ocr/src/models.rs @@ -6,9 +6,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(rename_all = "kebab-case")] pub enum OcrProviderKind { - Auto, - Tesseract, - Vision, + PpOcrV6, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] @@ -16,23 +14,16 @@ pub enum OcrProviderKind { pub struct OcrRequest { #[schemars(description = "Local PNG, JPEG, WebP, GIF, BMP, or TIFF image path")] pub path: PathBuf, - #[serde(default)] - #[schemars( - description = "OCR language identifiers; Tesseract values are joined with '+', for example ['eng', 'chi_sim']" - )] - pub languages: Vec, - #[serde(default)] - #[schemars(description = "Optional Tesseract page segmentation mode from 0 through 13")] - pub page_segmentation_mode: Option, - #[serde(default)] - #[schemars(description = "Override the configured provider for this call")] - pub provider: Option, - #[serde(default)] - #[schemars(description = "Optional extraction instruction used only by the vision provider")] - pub prompt: Option, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct OcrPoint { + pub x: u32, + pub y: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(rename_all = "camelCase")] pub struct OcrBoundingBox { pub x: u32, @@ -46,19 +37,23 @@ pub struct OcrBoundingBox { pub struct OcrBlock { pub page: u32, pub text: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub confidence: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub bounding_box: Option, + #[schemars(description = "PP-OCRv6 text recognition confidence from 0 through 1")] + pub confidence: f32, + #[schemars(description = "PP-OCRv6 DB text detection confidence from 0 through 1")] + pub detection_confidence: f32, + #[schemars(description = "Four PP-OCRv6 polygon vertices in source-image coordinates")] + pub polygon: [OcrPoint; 4], + pub bounding_box: OcrBoundingBox, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(rename_all = "camelCase")] pub struct OcrResult { pub provider: OcrProviderKind, + pub engine: String, + pub model: String, #[schemars(with = "OcrArtifactSchema")] pub source: Artifact, - pub languages: Vec, pub text: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub blocks: Vec, @@ -74,11 +69,11 @@ pub struct OcrDiagnostic { #[serde(skip_serializing_if = "Option::is_none")] pub provider: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub executable: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub endpoint: Option, + pub engine: Option, #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub model_dir: Option, pub sends_source_off_device: bool, pub message: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] diff --git a/crates/ocr/src/postprocess.rs b/crates/ocr/src/postprocess.rs new file mode 100644 index 00000000..6104ae94 --- /dev/null +++ b/crates/ocr/src/postprocess.rs @@ -0,0 +1,326 @@ +use a3s_use_core::{UseError, UseResult}; +use clipper2::{Centi, EndType, JoinType}; +use image::{GrayImage, Luma}; +use imageproc::contours::find_contours; +use imageproc::geometry::{contour_area, min_area_rect}; +use imageproc::point::Point; + +use crate::config::{DetectionConfig, RecognitionConfig}; + +#[derive(Debug, Clone)] +pub(crate) struct Detection { + pub(crate) polygon: [Point; 4], + pub(crate) confidence: f32, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Recognition { + pub(crate) text: String, + pub(crate) confidence: f32, +} + +pub(crate) fn detection_boxes( + output: &[f32], + shape: &[usize], + original_width: u32, + original_height: u32, + config: &DetectionConfig, +) -> UseResult> { + if shape.len() != 4 || shape[0] != 1 || shape[1] != 1 { + return Err(output_error(format!( + "PP-OCRv6 detection output shape must be [1, 1, H, W], found {shape:?}." + ))); + } + let height = shape[2]; + let width = shape[3]; + let map_len = height + .checked_mul(width) + .ok_or_else(|| output_error("PP-OCRv6 detection output dimensions overflowed."))?; + if width == 0 || height == 0 || output.len() != map_len { + return Err(output_error( + "PP-OCRv6 detection output length does not match its shape.", + )); + } + let width_u32 = u32::try_from(width) + .map_err(|_| output_error("PP-OCRv6 detection output width is too large."))?; + let height_u32 = u32::try_from(height) + .map_err(|_| output_error("PP-OCRv6 detection output height is too large."))?; + let mask = GrayImage::from_fn(width_u32, height_u32, |x, y| { + let index = y as usize * width + x as usize; + Luma([if output[index] > config.threshold { + 255 + } else { + 0 + }]) + }); + + let mut detections = Vec::new(); + for contour in find_contours::(&mask) + .into_iter() + .take(config.max_candidates) + { + if contour.points.len() < 3 { + continue; + } + let mini = order_points(min_area_rect(&contour.points)); + if minimum_side(&mini) < 3.0 { + continue; + } + let score = box_score(output, width, height, &mini); + if score < config.box_threshold { + continue; + } + let area = contour_area(&mini); + let perimeter = polygon_perimeter(&mini); + if !area.is_finite() || !perimeter.is_finite() || perimeter <= f64::EPSILON { + continue; + } + let distance = area * f64::from(config.unclip_ratio) / perimeter; + let path = mini + .iter() + .map(|point| (f64::from(point.x), f64::from(point.y))) + .collect::>(); + let inflated: Vec> = + clipper2::inflate::(path, distance, JoinType::Round, EndType::Polygon, 2.0) + .into(); + if inflated.len() != 1 || inflated[0].len() < 3 { + continue; + } + let inflated = inflated[0] + .iter() + .filter(|(x, y)| x.is_finite() && y.is_finite()) + .map(|(x, y)| { + Point::new( + x.round().clamp(f64::from(i32::MIN), f64::from(i32::MAX)) as i32, + y.round().clamp(f64::from(i32::MIN), f64::from(i32::MAX)) as i32, + ) + }) + .collect::>(); + if inflated.len() < 3 { + continue; + } + let expanded = order_points(min_area_rect(&inflated)); + if minimum_side(&expanded) < 5.0 { + continue; + } + let polygon = expanded.map(|point| { + Point::new( + (point.x as f32 / width as f32 * original_width as f32) + .round() + .clamp(0.0, original_width.saturating_sub(1) as f32), + (point.y as f32 / height as f32 * original_height as f32) + .round() + .clamp(0.0, original_height.saturating_sub(1) as f32), + ) + }); + detections.push(Detection { + polygon, + confidence: score.clamp(0.0, 1.0), + }); + } + sort_reading_order(&mut detections); + Ok(detections) +} + +pub(crate) fn decode_ctc( + output: &[f32], + shape: &[usize], + config: &RecognitionConfig, +) -> UseResult { + if shape.len() != 3 || shape[0] != 1 || shape[1] == 0 || shape[2] == 0 { + return Err(output_error(format!( + "PP-OCRv6 recognition output shape must be [1, T, C], found {shape:?}." + ))); + } + let timesteps = shape[1]; + let classes = shape[2]; + let expected_classes = config.characters.len() + 2; + if classes != expected_classes { + return Err(output_error(format!( + "PP-OCRv6 recognition class count is {classes}, but the model dictionary requires {expected_classes}." + ))); + } + if output.len() != timesteps.saturating_mul(classes) { + return Err(output_error( + "PP-OCRv6 recognition output length does not match its shape.", + )); + } + + let mut text = String::new(); + let mut confidence = 0.0_f32; + let mut selected = 0_usize; + let mut previous = usize::MAX; + for timestep in 0..timesteps { + let row = &output[timestep * classes..(timestep + 1) * classes]; + let (index, score) = row + .iter() + .copied() + .enumerate() + .max_by(|left, right| left.1.total_cmp(&right.1)) + .ok_or_else(|| output_error("PP-OCRv6 recognition output row is empty."))?; + if index != 0 && index != previous { + if index == config.characters.len() + 1 { + text.push(' '); + } else if let Some(character) = config.characters.get(index - 1) { + text.push_str(character); + } + confidence += score; + selected += 1; + } + previous = index; + } + Ok(Recognition { + text, + confidence: if selected == 0 { + 0.0 + } else { + (confidence / selected as f32).clamp(0.0, 1.0) + }, + }) +} + +fn box_score(output: &[f32], width: usize, height: usize, polygon: &[Point; 4]) -> f32 { + let min_x = polygon + .iter() + .map(|point| point.x) + .min() + .unwrap_or(0) + .clamp(0, width.saturating_sub(1) as i32) as usize; + let max_x = polygon + .iter() + .map(|point| point.x) + .max() + .unwrap_or(0) + .clamp(0, width.saturating_sub(1) as i32) as usize; + let min_y = polygon + .iter() + .map(|point| point.y) + .min() + .unwrap_or(0) + .clamp(0, height.saturating_sub(1) as i32) as usize; + let max_y = polygon + .iter() + .map(|point| point.y) + .max() + .unwrap_or(0) + .clamp(0, height.saturating_sub(1) as i32) as usize; + let polygon = polygon.map(|point| Point::new(point.x as f32, point.y as f32)); + let mut sum = 0.0_f32; + let mut count = 0_usize; + for y in min_y..=max_y { + for x in min_x..=max_x { + if point_in_convex_polygon(Point::new(x as f32 + 0.5, y as f32 + 0.5), &polygon) { + sum += output[y * width + x]; + count += 1; + } + } + } + if count == 0 { + 0.0 + } else { + sum / count as f32 + } +} + +fn point_in_convex_polygon(point: Point, polygon: &[Point; 4]) -> bool { + let mut sign = 0_i8; + for index in 0..4 { + let start = polygon[index]; + let end = polygon[(index + 1) % 4]; + let cross = + (end.x - start.x) * (point.y - start.y) - (end.y - start.y) * (point.x - start.x); + if cross.abs() <= f32::EPSILON { + continue; + } + let current = if cross > 0.0 { 1 } else { -1 }; + if sign != 0 && sign != current { + return false; + } + sign = current; + } + true +} + +fn minimum_side(points: &[Point; 4]) -> f64 { + (0..4) + .map(|index| distance(points[index], points[(index + 1) % 4])) + .fold(f64::INFINITY, f64::min) +} + +fn polygon_perimeter(points: &[Point; 4]) -> f64 { + (0..4) + .map(|index| distance(points[index], points[(index + 1) % 4])) + .sum() +} + +fn distance(left: Point, right: Point) -> f64 { + let x = f64::from(left.x - right.x); + let y = f64::from(left.y - right.y); + x.hypot(y) +} + +fn order_points(mut points: [Point; 4]) -> [Point; 4] { + points.sort_by(|left, right| left.x.cmp(&right.x).then(left.y.cmp(&right.y))); + let (top_left, bottom_left) = if points[0].y <= points[1].y { + (points[0], points[1]) + } else { + (points[1], points[0]) + }; + let (top_right, bottom_right) = if points[2].y <= points[3].y { + (points[2], points[3]) + } else { + (points[3], points[2]) + }; + [top_left, top_right, bottom_right, bottom_left] +} + +fn sort_reading_order(detections: &mut [Detection]) { + detections.sort_by(|left, right| { + left.polygon[0] + .y + .total_cmp(&right.polygon[0].y) + .then_with(|| left.polygon[0].x.total_cmp(&right.polygon[0].x)) + }); + for index in 1..detections.len() { + let mut cursor = index; + while cursor > 0 { + let current = detections[cursor].polygon[0]; + let previous = detections[cursor - 1].polygon[0]; + if (current.y - previous.y).abs() < 10.0 && current.x < previous.x { + detections.swap(cursor, cursor - 1); + cursor -= 1; + } else { + break; + } + } + } +} + +fn output_error(message: impl Into) -> UseError { + UseError::new("use.ocr.provider_output_invalid", message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ctc_decoder_removes_blanks_and_repeated_classes() { + let config = RecognitionConfig { + channels: 3, + height: 48, + default_width: 320, + characters: vec!["A".to_string(), "B".to_string()], + }; + let output = [ + 0.9, 0.1, 0.0, 0.0, // blank + 0.1, 0.8, 0.1, 0.0, // A + 0.9, 0.1, 0.0, 0.0, // blank + 0.1, 0.8, 0.1, 0.0, // A + 0.1, 0.1, 0.8, 0.0, // B + ]; + let result = decode_ctc(&output, &[1, 5, 4], &config).unwrap(); + assert_eq!(result.text, "AAB"); + assert!((result.confidence - 0.8).abs() < f32::EPSILON); + } +} diff --git a/crates/ocr/src/preprocess.rs b/crates/ocr/src/preprocess.rs new file mode 100644 index 00000000..4cdcd9f7 --- /dev/null +++ b/crates/ocr/src/preprocess.rs @@ -0,0 +1,192 @@ +use std::io::Cursor; + +use a3s_use_core::{UseError, UseResult}; +use image::imageops::FilterType; +use image::{DynamicImage, ImageReader, Limits, RgbImage}; + +use crate::config::{DetectionConfig, RecognitionConfig}; + +const MAX_IMAGE_SIDE: u32 = 16_384; +const MAX_DECODED_BYTES: u64 = 256 * 1024 * 1024; +const DETECTION_MIN_SIDE: u32 = 736; +const DETECTION_MAX_SIDE: u32 = 4_000; +const RECOGNITION_MAX_WIDTH: u32 = 3_200; + +pub(crate) struct DetectionInput { + pub(crate) data: Vec, + pub(crate) shape: [usize; 4], + pub(crate) original_width: u32, + pub(crate) original_height: u32, +} + +pub(crate) struct RecognitionInput { + pub(crate) data: Vec, + pub(crate) shape: [usize; 4], +} + +pub(crate) fn decode_image(bytes: &[u8]) -> UseResult { + let cursor = Cursor::new(bytes); + let mut reader = ImageReader::new(cursor) + .with_guessed_format() + .map_err(|error| image_error(format!("Failed to detect OCR image format: {error}")))?; + let mut limits = Limits::default(); + limits.max_image_width = Some(MAX_IMAGE_SIDE); + limits.max_image_height = Some(MAX_IMAGE_SIDE); + limits.max_alloc = Some(MAX_DECODED_BYTES); + reader.limits(limits); + let image = reader + .decode() + .map_err(|error| image_error(format!("Failed to decode OCR image: {error}")))?; + let width = image.width(); + let height = image.height(); + if width == 0 + || height == 0 + || u64::from(width) + .checked_mul(u64::from(height)) + .and_then(|pixels| pixels.checked_mul(4)) + .is_none_or(|bytes| bytes > MAX_DECODED_BYTES) + { + return Err(image_error( + "Decoded OCR image dimensions exceed the 256 MiB pixel limit.", + )); + } + Ok(image.to_rgb8()) +} + +pub(crate) fn detection_input( + image: &RgbImage, + config: &DetectionConfig, +) -> UseResult { + let original_width = image.width(); + let original_height = image.height(); + let (width, height) = detection_dimensions(original_width, original_height)?; + let resized = if width == original_width && height == original_height { + image.clone() + } else { + DynamicImage::ImageRgb8(image.clone()) + .resize_exact(width, height, FilterType::Triangle) + .to_rgb8() + }; + let plane = usize::try_from(u64::from(width) * u64::from(height)) + .map_err(|_| image_error("Detection tensor dimensions overflowed."))?; + let mut data = vec![0.0_f32; plane * 3]; + for (index, pixel) in resized.pixels().enumerate() { + let channels = [pixel[2], pixel[1], pixel[0]]; + for channel in 0..3 { + data[channel * plane + index] = (f32::from(channels[channel]) * config.scale + - config.mean[channel]) + / config.std[channel]; + } + } + Ok(DetectionInput { + data, + shape: [1, 3, height as usize, width as usize], + original_width, + original_height, + }) +} + +pub(crate) fn recognition_input( + images: &[RgbImage], + config: &RecognitionConfig, +) -> UseResult { + if images.is_empty() || images.len() > 8 { + return Err(image_error( + "PP-OCRv6 recognition batches must contain from 1 through 8 text crops.", + )); + } + if images + .iter() + .any(|image| image.width() == 0 || image.height() == 0) + { + return Err(image_error("PP-OCRv6 text crop has zero width or height.")); + } + let model_height = u32::try_from(config.height) + .map_err(|_| image_error("Recognition model height is invalid."))?; + let default_width = u32::try_from(config.default_width) + .map_err(|_| image_error("Recognition model width is invalid."))?; + let resized_widths = images + .iter() + .map(|image| { + ((f64::from(model_height) * f64::from(image.width()) / f64::from(image.height())).ceil() + as u32) + .clamp(1, RECOGNITION_MAX_WIDTH) + }) + .collect::>(); + let widest = resized_widths.iter().copied().max().unwrap_or(1); + let canvas_width = default_width.max(widest).min(RECOGNITION_MAX_WIDTH); + let target_plane = usize::try_from(u64::from(canvas_width) * u64::from(model_height)) + .map_err(|_| image_error("Recognition tensor dimensions overflowed."))?; + let batch_stride = config + .channels + .checked_mul(target_plane) + .ok_or_else(|| image_error("Recognition tensor dimensions overflowed."))?; + let mut data = vec![0.0_f32; images.len() * batch_stride]; + for (batch, (image, resized_width)) in images.iter().zip(resized_widths).enumerate() { + let resized = DynamicImage::ImageRgb8(image.clone()) + .resize_exact(resized_width, model_height, FilterType::Triangle) + .to_rgb8(); + for y in 0..model_height { + for x in 0..resized_width { + let pixel = resized.get_pixel(x, y); + let target = y as usize * canvas_width as usize + x as usize; + let channels = [pixel[2], pixel[1], pixel[0]]; + for channel in 0..config.channels { + data[batch * batch_stride + channel * target_plane + target] = + f32::from(channels[channel]) / 127.5 - 1.0; + } + } + } + } + Ok(RecognitionInput { + data, + shape: [ + images.len(), + config.channels, + config.height, + canvas_width as usize, + ], + }) +} + +fn detection_dimensions(width: u32, height: u32) -> UseResult<(u32, u32)> { + if width == 0 || height == 0 { + return Err(image_error("OCR image has zero width or height.")); + } + let mut ratio = 1.0_f64; + let min_side = width.min(height); + let max_side = width.max(height); + if min_side < DETECTION_MIN_SIDE { + ratio = f64::from(DETECTION_MIN_SIDE) / f64::from(min_side); + } + if f64::from(max_side) * ratio > f64::from(DETECTION_MAX_SIDE) { + ratio = f64::from(DETECTION_MAX_SIDE) / f64::from(max_side); + } + let resized_width = round_stride(f64::from(width) * ratio, 32); + let resized_height = round_stride(f64::from(height) * ratio, 32); + Ok((resized_width, resized_height)) +} + +fn round_stride(value: f64, stride: u32) -> u32 { + let rounded = (value / f64::from(stride)).round_ties_even() as u32 * stride; + rounded.max(stride) +} + +fn image_error(message: impl Into) -> UseError { + UseError::new("use.ocr.image_invalid", message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detection_dimensions_are_bounded_stride_multiples() { + assert_eq!(detection_dimensions(10, 20).unwrap(), (736, 1_472)); + assert_eq!(detection_dimensions(4_000, 1_000).unwrap(), (4_000, 992)); + assert_eq!( + detection_dimensions(20_000, 10_000).unwrap(), + (4_000, 1_984) + ); + } +} diff --git a/crates/ocr/tests/ppocr_v6_contract.rs b/crates/ocr/tests/ppocr_v6_contract.rs new file mode 100644 index 00000000..8f27ccd2 --- /dev/null +++ b/crates/ocr/tests/ppocr_v6_contract.rs @@ -0,0 +1,19 @@ +use std::path::PathBuf; + +use a3s_use_ocr::{OcrProviderKind, OcrRequest}; + +#[test] +fn public_contract_names_only_pp_ocr_v6() { + assert_eq!( + serde_json::to_value(OcrProviderKind::PpOcrV6).unwrap(), + serde_json::json!("pp-ocr-v6") + ); + + let request = OcrRequest { + path: PathBuf::from("scan.png"), + }; + assert_eq!( + serde_json::to_value(request).unwrap(), + serde_json::json!({ "path": "scan.png" }) + ); +} diff --git a/docs/architecture.md b/docs/architecture.md index c165d7b5..92d7603d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -41,10 +41,10 @@ does not define JSON-RPC methods or convert surfaces implicitly. Use build. The release packages its content-bound Skill and exposes the native CLI plus standard stdio MCP without a separate extension install. The process accepts bounded local image files and binds every result to the canonical -source digest. It uses only an explicitly present Tesseract executable or -configured vision endpoint; it never installs a provider silently. The vision -diagnostic discloses off-device image transfer, and its MCP extraction tool -carries conservative open-world annotations. +source digest. It runs the pinned `PP-OCRv6_small` detection and recognition +models locally through ONNX Runtime, without Python, PaddlePaddle, a remote OCR +endpoint, or an alternate backend. Model installation and repair are explicit +`use/ocr` component operations. Both MCP tools are closed-world and read-only. ## Hot-plug registry @@ -89,8 +89,8 @@ compatibility provider. For resident hosts, `use/office` targets the built-in `office-native` MCP server and is ready independently of OfficeCLI. A discovered OfficeCLI provider is projected separately as `use/office-compat`, targeting the standard compatibility server without carrying the native Skill. The -`use/ocr` route targets `ocr-native`; provider readiness remains explicit and -never triggers a silent Tesseract or vision-provider install. +`use/ocr` route targets `ocr-native`; model readiness remains explicit and +never triggers a silent install. The projection contains content-bound Skill references and an MCP launch target, never executable extension code or a generic action payload. Consumers still @@ -495,11 +495,11 @@ Implemented: 15. A packaged first-party `a3s-use-office` Skill with progressive Word/Spreadsheet/Presentation/MCP references, bounded local discovery, release-archive smoke checks, and content-bound capability projection. -16. A first-party built-in OCR route with typed provider diagnostics, bounded - image admission, source SHA-256 evidence, local Tesseract and explicit - vision adapters, standard MCP annotations/output schemas, and a - release-packaged content-bound Skill that projects to `mcp__use_ocr__*` in - A3S Code. +16. A first-party built-in OCR route with typed PP-OCRv6 diagnostics, bounded + image admission, source SHA-256 evidence, native ONNX Runtime detection and + recognition, standard closed-world MCP annotations/output schemas, pinned + release models, and a content-bound Skill that projects to + `mcp__use_ocr__*` in A3S Code. Next: diff --git a/src/cli.rs b/src/cli.rs index 23bad00f..e0875d02 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -119,7 +119,7 @@ fn help() -> CommandOutput { " a3s-use office native get|query|view|watch|raw|raw-set|dump|merge|validate|create|add|add-part|set|sort|remove|move|copy|swap|insert-rows|delete-rows|insert-columns|delete-columns|rename-sheet|move-sheet|copy-sheet|batch [args] [--json]\n", " a3s-use office \n", " a3s-use ocr doctor [--json]\n", - " a3s-use ocr extract [--language ] [--provider ] [--json]\n", + " a3s-use ocr extract [--json]\n", " a3s-use extension list|inspect|doctor [args] [--json]\n", " a3s-use extension enable [--json]\n", " a3s-use extension disable [--timeout-ms ] [--json]\n", @@ -416,6 +416,36 @@ async fn component_install(args: &[String]) -> UseResult { )); } } + if matches!(id, "ocr" | "use/ocr") { + #[cfg(feature = "ocr")] + { + if option_argument(args, "--from")?.is_some() { + return Err(usage_error("--from is valid only for external extensions")); + } + let force = args.iter().any(|argument| argument == "--force"); + let previous = a3s_use_ocr::ocr_status(); + let status = a3s_use_ocr::install_ppocr_v6(force).await?; + let changed = force + || !previous.available + || previous.model_dir != status.model_dir + || previous.source != status.source; + let diagnostic = ocr_diagnostic(); + return Ok(CommandOutput::success( + format!( + "Local PP-OCRv6 model bundle is ready at {}.", + status.model_dir.as_ref().map_or_else( + || "an unknown path".to_string(), + |path| path.display().to_string() + ) + ), + serde_json::json!({ + "component": component_value(id, &diagnostic), + "changed": changed, + "runtime": status + }), + )); + } + } if let Some(diagnostic) = builtin_diagnostic(id) { if option_argument(args, "--from")?.is_some() { return Err(usage_error("--from is valid only for external extensions")); @@ -513,6 +543,24 @@ async fn component_uninstall(id: &str) -> UseResult { )); } } + if matches!(id, "ocr" | "use/ocr") { + #[cfg(feature = "ocr")] + { + let changed = a3s_use_ocr::uninstall_managed_ppocr_v6().await?; + return Ok(CommandOutput::success( + if changed { + "Removed A3S-managed PP-OCRv6 model files." + } else { + "No A3S-managed PP-OCRv6 model files are installed." + }, + serde_json::json!({ + "component": id, + "changed": changed, + "builtInCommandPreserved": true + }), + )); + } + } if matches!( id, "browser" | "use/browser" | "office" | "use/office" | "ocr" | "use/ocr" @@ -896,6 +944,8 @@ fn builtin_presence(id: &str) -> &'static str { ), #[cfg(feature = "office")] "office" | "use/office" => office_presence(a3s_use_office::office_status().source), + #[cfg(feature = "ocr")] + "ocr" | "use/ocr" => ocr_presence(a3s_use_ocr::ocr_status().source), _ => "external", } } @@ -921,6 +971,16 @@ fn office_presence(source: a3s_use_office::OfficeInstallSource) -> &'static str } } +#[cfg(feature = "ocr")] +fn ocr_presence(source: a3s_use_ocr::OcrInstallSource) -> &'static str { + match source { + a3s_use_ocr::OcrInstallSource::Environment => "external", + a3s_use_ocr::OcrInstallSource::Packaged => "packaged", + a3s_use_ocr::OcrInstallSource::Managed => "managed", + a3s_use_ocr::OcrInstallSource::Missing => "missing", + } +} + fn builtin_diagnostic(id: &str) -> Option { match id { "browser" | "use/browser" => Some(browser_diagnostic()), diff --git a/src/cli_tests.rs b/src/cli_tests.rs index e60610cd..23ff377b 100644 --- a/src/cli_tests.rs +++ b/src/cli_tests.rs @@ -157,3 +157,14 @@ fn office_component_presence_preserves_runtime_ownership() { assert_eq!(office_presence(OfficeInstallSource::Managed), "managed"); assert_eq!(office_presence(OfficeInstallSource::Missing), "missing"); } + +#[cfg(feature = "ocr")] +#[test] +fn ocr_component_presence_preserves_model_ownership() { + use a3s_use_ocr::OcrInstallSource; + + assert_eq!(ocr_presence(OcrInstallSource::Environment), "external"); + assert_eq!(ocr_presence(OcrInstallSource::Packaged), "packaged"); + assert_eq!(ocr_presence(OcrInstallSource::Managed), "managed"); + assert_eq!(ocr_presence(OcrInstallSource::Missing), "missing"); +} diff --git a/src/ocr_builtin.rs b/src/ocr_builtin.rs index 127004c2..0e983c18 100644 --- a/src/ocr_builtin.rs +++ b/src/ocr_builtin.rs @@ -14,7 +14,7 @@ pub(crate) fn diagnostic() -> DomainDiagnostic { readiness: diagnostic.readiness, provider: diagnostic.provider.map(provider_name).map(str::to_string), version: None, - path: diagnostic.executable, + path: diagnostic.model_dir, message: diagnostic.message, suggestions: diagnostic.suggestions, } @@ -65,9 +65,7 @@ pub(crate) async fn primary_skill_surface() -> Option<(PathBuf, PathBuf)> { fn provider_name(provider: OcrProviderKind) -> &'static str { match provider { - OcrProviderKind::Auto => "auto", - OcrProviderKind::Tesseract => "tesseract", - OcrProviderKind::Vision => "vision", + OcrProviderKind::PpOcrV6 => "pp-ocr-v6", } }