diff --git a/.gitignore b/.gitignore index 15981d7..c66c14c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,11 +45,30 @@ pnpm-debug.log* # Workspace scratch (internal, not shipped) .workspace/scratch/ .workspace/reports/ +.workspace/shots/ +.workspace/exoquill-design/ +.workspace/*.mjs +# Private / unrelated files that land in the workspace (invoices, etc.) +.workspace/*.pdf # Design exploration: keep the brand HTML + mockups, drop the artifact cruft .workspace/**/*.zip .workspace/design-exploration/support.js .workspace/design-exploration/.thumbnail +# Local test artifacts / caches (not shipped) +*.wav +__pycache__/ + +# Stray spec from an unrelated project (not part of ExoQuill) +/read.md + # Local AI runtimes + models (bundled as resources for release, not in git) runtimes/ + +# Experimental TTS sidecars: local venvs, the Zonos source clone, and the +# user's private reference-voice clips — all local, none committed. +.venv-xtts/ +.venv-zonos/ +.zonos-src/ +zonos-voices/ diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8e4fa..fde4be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2026-06-22 + ### Added -- Repository foundation: GPL-3.0 license, project docs, `.gitignore`. -- Product/technical decisions (`docs/decisions.md`) and v0.1 roadmap (`docs/roadmap.md`). -- Source product specification under `.workspace/specs/`. +- **OCR** (Tesseract): file picker, drag & drop, clipboard paste, a selectable + result overlay, and desktop region capture (Ctrl+Alt+O snipping tool). +- **Formatting**: deterministic cleanup with a before/after preview (D6) and an + optional local-LLM "prepare for speech" pass. +- **Dictation**: cpal capture + VAD with live, word-by-word streaming through a + persistent whisper-server (ghost text, multiple insertion modes). +- **Read-aloud**: bundled Piper TTS with a streaming sentence queue + prefetch; + optional multilingual sidecars **XTTS-v2** and **Zonos-v0.1**, backend-selectable, + with per-voice tuning and WAV export (D2/D10). +- **Model manager**: a three-tier catalog (bundled / download / gated) with + install / delete and license info (D9); read-only on-device provider info (D5). +- **Notes management** (D12): scopes (Active / Archived / Trash), a pinned group, + soft-delete with undo toasts, multi-select bulk actions, and sort. +- **Edit history** (D12): content-hash-deduped snapshots (`note_versions`) and a + diff-timeline overlay with operation badges and non-destructive version restore. +- Bilingual **DE/EN** UI via `lib/i18n.ts`, and the Direction B "Local AI Utility" + skin in light + dark. +- TTS backend roadmap, incl. Chatterbox Multilingual as the MIT high-quality slot (D11). + +### Changed + +- Consolidated into a single toolbar (the former top bar was removed); branding + moved into the sidebar. -[Unreleased]: https://github.com/Exoridus/exoquill/commits/main +[Unreleased]: https://github.com/Exoridus/exoquill/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/Exoridus/exoquill/releases/tag/v0.2.0 diff --git a/Cargo.lock b/Cargo.lock index 5f33fb9..79ead7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -251,6 +251,28 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "base64" version = "0.21.7" @@ -494,6 +516,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex 2.0.1", ] @@ -580,6 +604,15 @@ dependencies = [ "libloading 0.8.9", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "combine" version = "4.6.7" @@ -1172,17 +1205,18 @@ dependencies = [ [[package]] name = "exoquill-ai" -version = "0.1.0" +version = "0.2.0" dependencies = [ "exoquill-capture", "exoquill-core", "reqwest", "serde", + "serde_json", ] [[package]] name = "exoquill-audio" -version = "0.1.0" +version = "0.2.0" dependencies = [ "cpal", "ort", @@ -1191,7 +1225,7 @@ dependencies = [ [[package]] name = "exoquill-capture" -version = "0.1.0" +version = "0.2.0" dependencies = [ "image", "xcap", @@ -1199,7 +1233,7 @@ dependencies = [ [[package]] name = "exoquill-core" -version = "0.1.0" +version = "0.2.0" dependencies = [ "chrono", "serde", @@ -1208,7 +1242,7 @@ dependencies = [ [[package]] name = "exoquill-db" -version = "0.1.0" +version = "0.2.0" dependencies = [ "exoquill-core", "rusqlite", @@ -1217,7 +1251,7 @@ dependencies = [ [[package]] name = "exoquill-desktop" -version = "0.1.0" +version = "0.2.0" dependencies = [ "base64 0.22.1", "exoquill-ai", @@ -1225,6 +1259,7 @@ dependencies = [ "exoquill-capture", "exoquill-core", "exoquill-db", + "reqwest", "serde", "serde_json", "tauri", @@ -1335,6 +1370,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.32" @@ -1570,8 +1611,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1581,9 +1624,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1909,6 +1954,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -2259,6 +2319,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.102" @@ -2486,6 +2556,12 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "mach2" version = "0.6.0" @@ -3123,6 +3199,12 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" @@ -3515,6 +3597,62 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -3661,16 +3799,22 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "mime_guess", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -3706,6 +3850,20 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rsqlite-vfs" version = "0.1.1" @@ -3772,6 +3930,81 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -3787,6 +4020,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3850,6 +4092,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.36.1" @@ -4222,6 +4487,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "swift-rs" version = "1.0.7" @@ -4803,6 +5074,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -5128,6 +5409,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -5415,6 +5702,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web_atoms" version = "0.2.5" @@ -5471,6 +5768,15 @@ dependencies = [ "system-deps 6.2.2", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webview2-com" version = "0.38.2" @@ -5749,6 +6055,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -6296,6 +6611,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index 81a1596..cf13048 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,14 +8,20 @@ members = [ ] [workspace.package] -version = "0.1.0" +version = "0.2.0" edition = "2021" license = "GPL-3.0-only" authors = ["ExoQuill contributors"] repository = "https://github.com/Exoridus/exoquill" [profile.release] -opt-level = "s" +opt-level = 3 lto = true codegen-units = 1 strip = true + +# Optimize dependencies even in dev builds (heavy C/DSP deps: whisper.cpp/ggml, +# resampling, ONNX Runtime) so dictation/read-aloud feel fast while developing, +# without slowing incremental rebuilds of our own crates. +[profile.dev.package."*"] +opt-level = 2 diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 0b3145b..165569a 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@exoquill/desktop", "private": true, - "version": "0.1.0", + "version": "0.2.0", "type": "module", "scripts": { "dev": "vite", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 17096d9..9eed71f 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -22,13 +22,19 @@ tauri-plugin-opener = "2" tauri-plugin-dialog = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" -exoquill-db = { version = "0.1.0", path = "../../../crates/exoquill-db" } -exoquill-ai = { version = "0.1.0", path = "../../../crates/exoquill-ai" } -exoquill-audio = { version = "0.1.0", path = "../../../crates/exoquill-audio" } -exoquill-capture = { version = "0.1.0", path = "../../../crates/exoquill-capture" } +exoquill-db = { version = "0.2.0", path = "../../../crates/exoquill-db" } +exoquill-ai = { version = "0.2.0", path = "../../../crates/exoquill-ai" } +exoquill-audio = { version = "0.2.0", path = "../../../crates/exoquill-audio" } +exoquill-capture = { version = "0.2.0", path = "../../../crates/exoquill-capture" } tauri-plugin-global-shortcut = "2.3.2" # Inline the region screenshot as a data: URL for the selection overlay. base64 = "0.22" +# Download on-demand models (HuggingFace, https) for the model manager. rustls +# avoids native-tls/openssl on Windows; blocking is fine inside async commands. +reqwest = { version = "0.13", default-features = false, features = [ + "blocking", + "rustls", +] } [features] # Neural VAD for dictation (Silero ONNX). Off by default: needs onnxruntime + the diff --git a/apps/desktop/src-tauri/models.json b/apps/desktop/src-tauri/models.json new file mode 100644 index 0000000..e204cb4 --- /dev/null +++ b/apps/desktop/src-tauri/models.json @@ -0,0 +1,76 @@ +{ + "version": 1, + "_comment": "Model catalog for the in-app manager. tier: bundled (ships in the installer, redistributable) | download (free, fetched on demand) | gated (restrictive license, needs acceptance). Licenses are verified in docs/decisions.md; 'TBD' = pending per-asset check (decisions D2).", + "models": [ + { + "id": "voice-de_DE-thorsten-high", + "provider": "piper", + "kind": "voice", + "displayName": "Thorsten — Deutsch (high)", + "language": "de_DE", + "license": "CC0-1.0", + "commercialOk": true, + "tier": "bundled", + "files": [ + { "url": "https://huggingface.co/rhasspy/piper-voices/resolve/main/de/de_DE/thorsten/high/de_DE-thorsten-high.onnx", "relPath": "piper-voices/de_DE-thorsten-high.onnx" }, + { "url": "https://huggingface.co/rhasspy/piper-voices/resolve/main/de/de_DE/thorsten/high/de_DE-thorsten-high.onnx.json", "relPath": "piper-voices/de_DE-thorsten-high.onnx.json" } + ] + }, + { + "id": "voice-en_GB-cori-high", + "provider": "piper", + "kind": "voice", + "displayName": "Cori — English GB (high)", + "language": "en_GB", + "license": "Public Domain (LibriVox)", + "commercialOk": true, + "tier": "download", + "notes": "Public Domain — bündelbar; aktuell als Download, um den Installer schlank zu halten.", + "files": [ + { "url": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_GB/cori/high/en_GB-cori-high.onnx", "relPath": "piper-voices/en_GB-cori-high.onnx" }, + { "url": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_GB/cori/high/en_GB-cori-high.onnx.json", "relPath": "piper-voices/en_GB-cori-high.onnx.json" } + ] + }, + { + "id": "voice-en_US-ryan-high", + "provider": "piper", + "kind": "voice", + "displayName": "Ryan — English US (high)", + "language": "en_US", + "license": "CC-BY-NC-SA-4.0", + "commercialOk": false, + "tier": "gated", + "notes": "Nicht-kommerziell (CC BY-NC-SA 4.0). Nur Download mit Zustimmung, NICHT bündelbar, keine kommerzielle Nutzung.", + "files": [ + { "url": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/ryan/high/en_US-ryan-high.onnx", "relPath": "piper-voices/en_US-ryan-high.onnx" }, + { "url": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/ryan/high/en_US-ryan-high.onnx.json", "relPath": "piper-voices/en_US-ryan-high.onnx.json" } + ] + }, + { + "id": "tts-xtts-v2", + "provider": "xtts", + "kind": "runtime", + "displayName": "XTTS-v2 — multilingual (experimentell)", + "language": "multi", + "license": "CPML (non-commercial)", + "commercialOk": false, + "tier": "gated", + "setup": "scripts/setup-xtts.ps1", + "notes": "Nicht-kommerzielle Modell-Lizenz (CPML). Wird nicht gebündelt; lokal per Setup-Skript installiert. ~1,75 GB Modell + Python/PyTorch. Alle ~58 Sprecher inklusive.", + "files": [] + }, + { + "id": "tts-zonos-v0_1", + "provider": "zonos", + "kind": "runtime", + "displayName": "Zonos-v0.1 — multilingual (experimentell)", + "language": "multi", + "license": "Apache-2.0", + "commercialOk": true, + "tier": "download", + "setup": "scripts/setup-zonos.ps1", + "notes": "Apache-2.0 (kommerziell ok, anders als XTTS). Wird nicht gebündelt; lokal per Setup-Skript installiert. Benötigt eine CUDA-GPU. Stimmen via Voice-Cloning aus eigenen Referenz-WAVs (zonos-voices/).", + "files": [] + } + ] +} diff --git a/apps/desktop/src-tauri/src/dictation.rs b/apps/desktop/src-tauri/src/dictation.rs index 4ddce10..41cb67e 100644 --- a/apps/desktop/src-tauri/src/dictation.rs +++ b/apps/desktop/src-tauri/src/dictation.rs @@ -56,7 +56,7 @@ pub struct DictationController { } /// Begin streaming dictation into the active note. No-op if already running. -#[tauri::command] +#[tauri::command(async)] #[allow(clippy::too_many_arguments)] pub fn start_dictation( state: State, @@ -104,7 +104,7 @@ pub fn start_dictation( } /// Stop the current dictation session, flushing any trailing utterance. -#[tauri::command] +#[tauri::command(async)] pub fn stop_dictation(state: State) -> Result<(), String> { let controller = state.dictation.lock().map_err(|e| e.to_string())?.take(); if let Some(controller) = controller { @@ -116,7 +116,7 @@ pub fn stop_dictation(state: State) -> Result<(), String> { /// The available dictation sources: microphones plus output devices that can be /// captured via WASAPI loopback (to dictate from system audio). -#[tauri::command] +#[tauri::command(async)] pub fn list_capture_sources() -> Vec { let mut sources: Vec = exoquill_audio::list_input_devices() .into_iter() diff --git a/apps/desktop/src-tauri/src/jobs.rs b/apps/desktop/src-tauri/src/jobs.rs index 9ca7fd8..c41d686 100644 --- a/apps/desktop/src-tauri/src/jobs.rs +++ b/apps/desktop/src-tauri/src/jobs.rs @@ -4,13 +4,13 @@ use std::sync::Arc; use base64::Engine; -use exoquill_ai::formatter::FormatRequest; +use exoquill_ai::formatter::{FormatRequest, FormatterProvider}; use exoquill_ai::ocr::{OcrLayout, OcrRequest}; use exoquill_ai::provider::{Health, Provider}; -use exoquill_ai::tts::{TtsRequest, TtsResponse}; +use exoquill_ai::tts::{TextToSpeechProvider, TtsRequest, TtsVoice}; use exoquill_core::note::{NewNoteEvent, NoteUpdate}; use exoquill_core::{CancelToken, Event, EventSink, Job}; -use tauri::{AppHandle, Emitter, State}; +use tauri::{AppHandle, Emitter, Manager, State}; use crate::notes::AppState; @@ -32,10 +32,95 @@ impl EventSink for TauriEventSink { } } +/// Run a blocking command body on the dedicated blocking thread pool. This keeps +/// it off the UI thread (so the webview never freezes) AND off the Tokio worker +/// pool — provider calls use `reqwest::blocking`, whose internal runtime panics +/// if dropped inside an async/Tokio context. The closure gets `&AppState`, +/// resolved from the handle on the blocking thread. +pub(crate) async fn off_thread(app: AppHandle, f: F) -> Result +where + T: Send + 'static, + F: FnOnce(&AppState) -> Result + Send + 'static, +{ + tauri::async_runtime::spawn_blocking(move || f(app.state::().inner())) + .await + .map_err(|e| format!("background task failed: {e}"))? +} + +/// The formatter for a request: the persistent llama-server (started on first +/// use, model resident → fast for chunked formatting), else the per-call +/// fallback in [`AppState`]. A server-start failure falls through silently. +fn ensure_formatter(state: &AppState) -> Arc { + if let Some((binary, model)) = state.llama_server_paths.clone() { + let mut slot = match state.llama_server.lock() { + Ok(slot) => slot, + Err(poisoned) => poisoned.into_inner(), + }; + if slot.is_none() { + if let Ok(server) = exoquill_ai::LlamaServer::start(&binary, &model) { + *slot = Some(server); + } + } + if let Some(server) = slot.as_ref() { + if let Ok(client) = server.client() { + return Arc::new(client) as Arc; + } + } + } + Arc::clone(&state.formatter) +} + +/// The TTS provider for a request, honoring the backend the UI picked. +/// `provider` is the voice's backend (`"piper"` | `"xtts"` | `"zonos"`); `None` +/// keeps the legacy auto behavior (prefer a warm XTTS sidecar, else Piper). +/// +/// - `"piper"` → the bundled Piper provider, always available. +/// - `"zonos"` → the auto-spawned Zonos sidecar (Apache-2.0, GPU) once warmed up. +/// - `"xtts"` / auto → the auto-spawned XTTS sidecar once it has warmed up +/// (multilingual, better with technical terms). +/// +/// A sidecar that's still warming up falls back to Piper so playback works rather +/// than failing. Returns `None` only when there's no local TTS at all (UI → +/// system speech). +fn tts_for(state: &AppState, provider: Option<&str>) -> Option> { + // An EXPLICITLY chosen sidecar backend never falls back to Piper: that would + // play (and cache) the wrong — Piper — voice while the sidecar is still + // warming. Return `None` instead so the UI knows it isn't ready yet. + match provider { + Some("piper") => state.tts.clone(), + Some("zonos") => state + .zonos_server + .lock() + .ok() + .and_then(|slot| slot.as_ref().and_then(|server| server.client())) + .map(|client| Arc::new(client) as Arc), + Some("xtts") => { + state.xtts_paths.as_ref()?; + state + .xtts_server + .lock() + .ok() + .and_then(|slot| slot.as_ref().and_then(|server| server.client())) + .map(|client| Arc::new(client) as Arc) + } + // Auto (no explicit backend): prefer a warm XTTS sidecar, else Piper. + _ => { + if state.xtts_paths.is_some() { + if let Ok(slot) = state.xtts_server.lock() { + if let Some(client) = slot.as_ref().and_then(|server| server.client()) { + return Some(Arc::new(client) as Arc); + } + } + } + state.tts.clone() + } + } +} + /// Quick-format the whole note via the formatter provider, as an async job. /// Returns the job id immediately; the result is persisted and announced via /// a `job_updated` event. -#[tauri::command] +#[tauri::command(async)] pub fn format_note(state: State, note_id: String) -> Result { let note = { let db = state.db.lock().map_err(|e| e.to_string())?; @@ -45,7 +130,7 @@ pub fn format_note(state: State, note_id: String) -> Result, note_id: String) -> Result, id: String) { state.jobs.cancel(&id); } -#[tauri::command] +#[tauri::command(async)] pub fn list_jobs(state: State) -> Vec { state.jobs.jobs() } /// Run OCR on an image and append the recognized text to the note, as an async /// job. Returns the job id; the result is persisted and announced via an event. -#[tauri::command] +#[tauri::command(async)] pub fn run_ocr( state: State, note_id: String, @@ -170,7 +255,7 @@ pub fn run_ocr( /// the selectable OCR overlay. Synchronous and does not touch any note — the UI /// decides what to insert. Returns boxes only with the real Tesseract provider; /// the mock returns text alone. -#[tauri::command] +#[tauri::command(async)] pub fn ocr_image(state: State, image_bytes: Vec) -> Result { let request = OcrRequest { image_bytes, @@ -206,7 +291,7 @@ fn png_data_url(bytes: &[u8]) -> String { /// Hand the frozen screenshot to the selection overlay so it can display the /// monitor it covers. -#[tauri::command] +#[tauri::command(async)] pub fn get_region_capture(state: State) -> Result { let guard = state.region_capture.lock().map_err(|e| e.to_string())?; let shot = guard.as_ref().ok_or("no region capture in progress")?; @@ -219,7 +304,7 @@ pub fn get_region_capture(state: State) -> Result, x: f64, @@ -252,7 +337,7 @@ pub fn ocr_region( } /// Discard an in-progress region capture (the overlay was cancelled). -#[tauri::command] +#[tauri::command(async)] pub fn cancel_region_ocr(state: State) -> Result<(), String> { *state.region_capture.lock().map_err(|e| e.to_string())? = None; Ok(()) @@ -262,47 +347,382 @@ pub fn cancel_region_ocr(state: State) -> Result<(), String> { /// directly. Synchronous: selections are short and the result must land back at /// the exact cursor position. Whole-note formatting uses the job queue instead. #[tauri::command] -pub fn format_text( - state: State, +pub async fn format_text( + app: AppHandle, text: String, instruction: Option, ) -> Result { - let operation = if instruction.is_some() { - "custom_format" - } else { - "quick_format" + off_thread(app, move |state| { + let operation = if instruction.is_some() { + "custom_format" + } else { + "quick_format" + }; + let request = FormatRequest { + text, + source: "manual".into(), + language_mode: "de_en_terms".into(), + operation: operation.into(), + instruction, + custom_terms: Vec::new(), + }; + let response = ensure_formatter(state) + .run(request, &CancelToken::new()) + .map_err(|e| e.to_string())?; + Ok(response.formatted_text) + }) + .await +} + +/// Instruction for the read-aloud "prepare for speech" pass. Lives here so the +/// whole speech-prep call is one IPC round-trip per chunk (text in, prose out). +/// Rewrites a screen-oriented note (tables, lists, code, links) into a linear, +/// spoken commentary — what a person would actually read aloud. +const SPEECH_INSTRUCTION: &str = "Schreibe den folgenden Text in eine Vorlese-Fassung um: \ +einen zusammenhängenden, gut hörbaren Fließtext, wie ihn ein Mensch flüssig vorlesen würde, \ +nicht wie eine technische Notiz. Wandle Tabellen in gesprochene Sätze um — lies jede Zeile als \ +ganzen Satz, niemals als Spalten, Striche oder senkrechte Striche. Löse Aufzählungen und Listen \ +in Fließtext auf, etwa „die wichtigsten Punkte sind A, B und C\". Entferne Codeblöcke, \ +Dateipfade, URLs und Markdown-Zeichen und nenne ihren Inhalt nur knapp in Worten, wenn er \ +wichtig ist. Schreibe Abkürzungen beim ersten Vorkommen aus und erkläre sie kurz, etwa „OCR, \ +also Texterkennung\". Gib schwierigen Fachbegriffen einen kurzen erklärenden Halbsatz. Verwende \ +kurze bis mittlere Sätze in kleinen Absätzen mit natürlichen Übergängen. Schreibe linear, ohne \ +Verweise wie „siehe oben\" oder „in der Tabelle\" — beim Hören gibt es kein Oben oder Unten. \ +Erfinde keine neuen Inhalte und lass nichts Wesentliches weg. Gib nur den reinen gesprochenen \ +Fließtext zurück, ohne Überschriften und ohne Markdown."; + +/// Kick off a sidecar backend's warm-up in a background thread. Idempotent: a +/// no-op when the backend is already warm, already warming, or not configured. +/// Shared by [`warm_tts`] (fire-and-forget) and [`ensure_tts_ready`] (which then +/// waits for it to finish). +fn warm_backend(state: &AppState, app: &AppHandle, provider: &str) { + use std::sync::atomic::Ordering; + match provider { + "xtts" => { + let Some((python, script)) = state.xtts_paths.clone() else { + return; + }; + if state + .xtts_server + .lock() + .map(|s| s.is_some()) + .unwrap_or(false) + { + return; // already warm + } + if state.xtts_warming.swap(true, Ordering::SeqCst) { + return; // already starting + } + let handle = app.clone(); + std::thread::spawn(move || { + let server = exoquill_ai::XttsServer::start(python, script).ok(); + if let Some(state) = handle.try_state::() { + if let (Some(server), Ok(mut slot)) = (server, state.xtts_server.lock()) { + *slot = Some(server); + } + state.xtts_warming.store(false, Ordering::SeqCst); + } + }); + } + "zonos" => { + let Some((python, script, voices)) = state.zonos_paths.clone() else { + return; + }; + if state + .zonos_server + .lock() + .map(|s| s.is_some()) + .unwrap_or(false) + { + return; + } + if state.zonos_warming.swap(true, Ordering::SeqCst) { + return; + } + let handle = app.clone(); + std::thread::spawn(move || { + let server = exoquill_ai::ZonosServer::start(python, script, voices).ok(); + if let Some(state) = handle.try_state::() { + if let (Some(server), Ok(mut slot)) = (server, state.zonos_server.lock()) { + *slot = Some(server); + } + state.zonos_warming.store(false, Ordering::SeqCst); + } + }); + } + _ => {} + } +} + +/// Warm up a TTS backend's sidecar in the background (idempotent). The UI calls +/// this when a backend is selected, so only the *active* backend ever loads — +/// never both at launch, which is what froze the UI. Piper needs no warm-up. +/// Returns immediately; synthesis falls back to Piper until the sidecar is ready. +#[tauri::command(async)] +pub fn warm_tts(state: State, app: AppHandle, provider: String) { + warm_backend(&state, &app, &provider); +} + +/// Block until `provider`'s sidecar is warm (model loaded), starting its warm-up +/// if needed. Piper (and any unknown backend, which falls back to Piper) is +/// always ready at once. Returns `Ok` when synthesis can run, or `Err` on an +/// unconfigured backend, a warm-up failure, or timeout. The read-aloud UI calls +/// this when it finds the chosen voice cold, then retries the read automatically +/// once ready — so the user never has to click play a second time. +#[tauri::command(async)] +pub fn ensure_tts_ready( + state: State, + app: AppHandle, + provider: String, +) -> Result<(), String> { + use std::sync::atomic::Ordering; + use std::time::{Duration, Instant}; + + let st = state.inner(); + let warm = || match provider.as_str() { + "xtts" => st.xtts_server.lock().map(|s| s.is_some()).unwrap_or(false), + "zonos" => st.zonos_server.lock().map(|s| s.is_some()).unwrap_or(false), + _ => true, // Piper / unknown → ready (synthesis falls back to Piper). }; - let request = FormatRequest { - text, - source: "manual".into(), - language_mode: "de_en_terms".into(), - operation: operation.into(), - instruction, - custom_terms: Vec::new(), + let warming = || match provider.as_str() { + "xtts" => st.xtts_warming.load(Ordering::SeqCst), + "zonos" => st.zonos_warming.load(Ordering::SeqCst), + _ => false, }; - let response = state - .formatter - .run(request, &CancelToken::new()) - .map_err(|e| e.to_string())?; - Ok(response.formatted_text) + let configured = match provider.as_str() { + "xtts" => st.xtts_paths.is_some(), + "zonos" => st.zonos_paths.is_some(), + _ => true, + }; + + if warm() { + return Ok(()); + } + if !configured { + return Err(format!("{provider}-Backend ist nicht eingerichtet")); + } + + warm_backend(st, &app, &provider); + + // Poll the shared state until the slot fills (success) or the warm-up thread + // finishes without filling it (failure), bounded by a generous timeout — the + // first ever run may download model weights. + let deadline = Instant::now() + Duration::from_secs(600); + loop { + std::thread::sleep(Duration::from_millis(300)); + if warm() { + return Ok(()); + } + if !warming() { + // Warm-up finished but the slot is empty → it failed. Re-check once to + // dodge the race where the slot is set just after the flag clears. + if warm() { + return Ok(()); + } + return Err(format!("{provider}-Sidecar konnte nicht geladen werden")); + } + if Instant::now() >= deadline { + return Err(format!("{provider}-Sidecar wurde nicht rechtzeitig bereit")); + } + } +} + +/// Begin a read-aloud session: install a fresh cancel token. A subsequent +/// [`cancel_read`] trips it, which stops the streaming speech-prep generation +/// mid-flight (rather than letting the running chunk finish in the background). +#[tauri::command(async)] +pub fn begin_read(state: State) { + if let Ok(mut slot) = state.read_cancel.lock() { + *slot = CancelToken::new(); + } +} + +/// Cancel the in-progress read-aloud speech-prep (trips the session token). The +/// running `prepare_speech` chunk observes it between streamed tokens and bails. +#[tauri::command(async)] +pub fn cancel_read(state: State) { + if let Ok(slot) = state.read_cancel.lock() { + slot.cancel(); + } +} + +/// Rewrite one chunk of a note into clean, speakable prose for read-aloud, under +/// the current read session's cancel token. Synchronous (runs off the main +/// thread) and streamed inside the provider, so a cancel takes effect promptly. +#[tauri::command] +pub async fn prepare_speech(app: AppHandle, text: String) -> Result { + off_thread(app, move |state| { + let cancel = state + .read_cancel + .lock() + .map(|t| t.clone()) + .unwrap_or_default(); + if cancel.is_cancelled() { + return Err("cancelled".into()); + } + let request = FormatRequest { + text, + source: "manual".into(), + language_mode: "de_en_terms".into(), + operation: "speech_prep".into(), + instruction: Some(SPEECH_INSTRUCTION.to_string()), + custom_terms: Vec::new(), + }; + let response = ensure_formatter(state) + .run(request, &cancel) + .map_err(|e| e.to_string())?; + Ok(response.formatted_text) + }) + .await +} + +/// Synthesized audio for the webview: 16-bit little-endian mono PCM, base64'd. +/// Far cheaper over IPC than a `Vec` JSON array (~3× smaller and no +/// number-array parse), which matters because read-aloud calls this per sentence. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TtsAudio { + pub pcm: String, + pub sample_rate: u32, +} + +/// Wrap raw 16-bit little-endian mono PCM in a RIFF/WAVE container at +/// `sample_rate`. Mirrors the frontend's old `encodeWav`, now that the file is +/// written natively (the webview can't trigger a real download in WebView2). +fn wav_from_pcm(pcm: &[u8], sample_rate: u32) -> Vec { + let data_len = pcm.len() as u32; + let mut buf = Vec::with_capacity(44 + pcm.len()); + buf.extend_from_slice(b"RIFF"); + buf.extend_from_slice(&(36 + data_len).to_le_bytes()); + buf.extend_from_slice(b"WAVE"); + buf.extend_from_slice(b"fmt "); + buf.extend_from_slice(&16u32.to_le_bytes()); // fmt chunk size + buf.extend_from_slice(&1u16.to_le_bytes()); // format: PCM + buf.extend_from_slice(&1u16.to_le_bytes()); // channels: mono + buf.extend_from_slice(&sample_rate.to_le_bytes()); + buf.extend_from_slice(&(sample_rate * 2).to_le_bytes()); // byte rate (mono, 2 B/sample) + buf.extend_from_slice(&2u16.to_le_bytes()); // block align + buf.extend_from_slice(&16u16.to_le_bytes()); // bits per sample + buf.extend_from_slice(b"data"); + buf.extend_from_slice(&data_len.to_le_bytes()); + buf.extend_from_slice(pcm); + buf +} + +/// Concatenate the read-aloud segments (base64 16-bit LE mono PCM) into one WAV +/// and write it to a file the user picks (native save dialog). Returns the saved +/// path, or `None` if the user cancelled. The webview can't trigger a real +/// download in WebView2, so — like [`crate::notes::export_note`] — the file is +/// written natively here. +#[tauri::command(async)] +pub fn export_audio( + app: AppHandle, + segments: Vec, + sample_rate: u32, + suggested_name: String, +) -> Result, String> { + use tauri_plugin_dialog::DialogExt; + + let engine = base64::engine::general_purpose::STANDARD; + let mut pcm: Vec = Vec::new(); + for seg in &segments { + pcm.extend_from_slice( + &engine + .decode(seg) + .map_err(|e| format!("decode audio: {e}"))?, + ); + } + if pcm.is_empty() { + return Err("no audio to export".into()); + } + + let Some(path) = app + .dialog() + .file() + .add_filter("WAV-Audio", &["wav"]) + .set_file_name(&suggested_name) + .blocking_save_file() + else { + return Ok(None); // user cancelled + }; + let path = path.into_path().map_err(|e| e.to_string())?; + std::fs::write(&path, wav_from_pcm(&pcm, sample_rate)).map_err(|e| e.to_string())?; + Ok(Some(path.to_string_lossy().into_owned())) } /// Synthesize speech for `text` with the local TTS provider, returning the PCM -/// samples for the frontend to play. Errors when no local TTS is available so -/// the UI can fall back to system speech. +/// for the frontend to play. `voice_id` picks the voice; an unknown or `None` +/// value falls back to the provider's default voice. Errors when no local TTS is +/// available so the UI can fall back to system speech. #[tauri::command] -pub fn tts_speak(state: State, text: String) -> Result { - let tts = state +#[allow(clippy::too_many_arguments)] +pub async fn tts_speak( + app: AppHandle, + text: String, + voice_id: Option, + provider: Option, + speed: Option, + expressiveness: Option, + cadence: Option, + sentence_silence: Option, + intonation: Option, + brightness: Option, + emotion: Option>, +) -> Result { + off_thread(app, move |state| { + let tts = tts_for(state, provider.as_deref()) + .ok_or_else(|| "no local TTS provider".to_string())?; + let request = TtsRequest { + text, + voice_id: voice_id + .filter(|v| !v.is_empty()) + .or_else(|| tts.default_voice()) + .unwrap_or_default(), + speed: speed.unwrap_or(1.0), + expressiveness, + cadence, + sentence_silence, + intonation, + brightness, + emotion, + }; + let response = tts + .run(request, &CancelToken::new()) + .map_err(|e| e.to_string())?; + + // f32 [-1,1] → 16-bit LE PCM → base64. + let mut bytes = Vec::with_capacity(response.samples.len() * 2); + for &s in &response.samples { + let v = (s.clamp(-1.0, 1.0) * 32767.0) as i16; + bytes.extend_from_slice(&v.to_le_bytes()); + } + Ok(TtsAudio { + pcm: base64::engine::general_purpose::STANDARD.encode(&bytes), + sample_rate: response.sample_rate, + }) + }) + .await +} + +/// The voices the local TTS offers, across every available backend, so the UI +/// can let the user switch backend and voice. Piper's bundled voices come first +/// (always available); the XTTS voices follow when the sidecar is configured +/// (listed statically, even before it has warmed up). Each voice carries its +/// `provider`, which the UI passes back to [`tts_speak`]. +#[tauri::command(async)] +pub fn list_tts_voices(state: State) -> Vec { + let mut voices = state .tts .as_ref() - .ok_or_else(|| "no local TTS provider".to_string())?; - let request = TtsRequest { - text, - voice_id: "de".into(), - speed: 1.0, - }; - tts.run(request, &CancelToken::new()) - .map_err(|e| e.to_string()) + .map(|tts| tts.voices()) + .unwrap_or_default(); + if state.xtts_paths.is_some() { + voices.extend(exoquill_ai::XttsTts::voices_static()); + } + if let Some((_, _, voices_dir)) = &state.zonos_paths { + voices.extend(exoquill_ai::ZonosTts::voices_in_dir(voices_dir)); + } + voices } /// Read-only summary of the provider behind an AI capability, for the settings / @@ -345,23 +765,27 @@ fn describe(feature: &str, provider: &P) -> ModelInfo { /// List the resolved AI providers with license + status for the settings view. #[tauri::command] -pub fn list_model_info(state: State) -> Vec { - let mut out = vec![ - describe("stt", state.stt.as_ref()), - describe("ocr", state.ocr.as_ref()), - describe("formatter", state.formatter.as_ref()), - ]; - match state.tts.as_ref() { - Some(tts) => out.push(describe("tts", tts.as_ref())), - None => out.push(ModelInfo { - feature: "tts".into(), - provider_id: "tts.system".into(), - display_name: "System speech (fallback)".into(), - version: "-".into(), - status: "fallback".into(), - runtime_license: "OS".into(), - source: None, - }), - } - out +pub async fn list_model_info(app: AppHandle) -> Vec { + off_thread(app, move |state| -> Result, String> { + let mut out = vec![ + describe("stt", state.stt.as_ref()), + describe("ocr", state.ocr.as_ref()), + describe("formatter", state.formatter.as_ref()), + ]; + match state.tts.as_ref() { + Some(tts) => out.push(describe("tts", tts.as_ref())), + None => out.push(ModelInfo { + feature: "tts".into(), + provider_id: "tts.system".into(), + display_name: "System speech (fallback)".into(), + version: "-".into(), + status: "fallback".into(), + runtime_license: "OS".into(), + source: None, + }), + } + Ok(out) + }) + .await + .unwrap_or_default() } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index ad86036..d49cd69 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1,5 +1,6 @@ mod dictation; mod jobs; +mod models; mod notes; mod tray; @@ -12,7 +13,7 @@ use exoquill_ai::ocr::OcrProvider; use exoquill_ai::provider::{Health, Provider}; use exoquill_ai::stt::SpeechToTextProvider; use exoquill_ai::tts::TextToSpeechProvider; -use exoquill_ai::{LlamaFormatter, PiperTts, TesseractOcr, WhisperStt}; +use exoquill_ai::{LlamaFormatter, PiperTts, TesseractOcr, WhisperStt, XttsTts}; use exoquill_core::{EventSink, JobQueue}; use exoquill_db::Database; use jobs::TauriEventSink; @@ -75,6 +76,32 @@ fn resolve_formatter_provider(app: &App) -> Arc { } } +/// Resolve the persistent llama-server binary + model. `llama-server.exe` sits +/// next to `llama-completion.exe` (bundled together) and shares the same Qwen +/// GGUF as the per-call formatter. `None` if either is missing — formatting then +/// runs via the per-call fallback in [`AppState`]. +fn resolve_llama_server_paths(app: &App) -> Option<(PathBuf, PathBuf)> { + let resources = app.path().resource_dir().ok(); + let cli = std::env::var("EXOQUILL_LLAMA") + .map(PathBuf::from) + .ok() + .or_else(|| { + resources + .as_ref() + .map(|d| d.join("llama/llama-completion.exe")) + })?; + let server = cli.with_file_name("llama-server.exe"); + let model = std::env::var("EXOQUILL_FORMATTER_MODEL") + .map(PathBuf::from) + .ok() + .or_else(|| { + resources + .as_ref() + .map(|d| d.join("models/qwen2.5-1.5b-instruct-q4_k_m.gguf")) + })?; + (server.exists() && model.exists()).then_some((server, model)) +} + /// Pick the STT provider: real whisper.cpp + ggml model when reachable, else /// the mock (placeholder transcript). Paths come from env vars (dev) or the /// bundled resource dir (release). @@ -170,26 +197,78 @@ fn resolve_silero_model_path(app: &App) -> Option { } /// Pick the TTS provider: real Piper when reachable, else `None` (the UI then -/// falls back to the webview's system speech synthesis). +/// falls back to the webview's system speech synthesis). Every `*.onnx` in the +/// voices directory becomes a selectable voice. `EXOQUILL_PIPER_VOICE` still +/// names the default voice (dev); its parent directory is scanned for the rest. +/// For release the bundled `piper-voices/` resource dir is scanned instead, with +/// `de_DE-thorsten-medium` as the default. fn resolve_tts_provider(app: &App) -> Option> { + // Experimental: prefer the XTTS-v2 sidecar when its URL is set and reachable + // (multilingual DE/EN; non-commercial weights — test only, never bundled). + // Falls through to Piper otherwise. Start it with scripts/xtts-server.py. + if let Ok(url) = std::env::var("EXOQUILL_XTTS_URL") { + if let Some(xtts) = XttsTts::connect(url) { + return Some(Arc::new(xtts) as Arc); + } + } + let resources = app.path().resource_dir().ok(); let binary = std::env::var("EXOQUILL_PIPER") .map(PathBuf::from) .ok() .or_else(|| resources.as_ref().map(|d| d.join("piper/piper.exe")))?; - let model = std::env::var("EXOQUILL_PIPER_VOICE") + let default_voice = std::env::var("EXOQUILL_PIPER_VOICE") .map(PathBuf::from) - .ok() - .or_else(|| { - resources - .as_ref() - .map(|d| d.join("piper-voices/de_DE-thorsten-medium.onnx")) - })?; - let piper = PiperTts::new(binary, model, 22_050); + .ok(); + let voices_dir = default_voice + .as_ref() + .and_then(|p| p.parent().map(PathBuf::from)) + .or_else(|| resources.as_ref().map(|d| d.join("piper-voices")))?; + let default_id = default_voice + .as_ref() + .and_then(|p| p.file_stem()) + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "de_DE-thorsten-medium".to_string()); + let piper = PiperTts::discover(binary, voices_dir, default_id); matches!(piper.health_check(), Health::Ready) .then(|| Arc::new(piper) as Arc) } +/// Resolve the auto-spawn paths for the experimental XTTS sidecar: the venv +/// Python + `xtts-server.py`, from `EXOQUILL_XTTS_PYTHON` / `EXOQUILL_XTTS_SCRIPT` +/// (set by dev.ps1). `None` when not configured, or when `EXOQUILL_XTTS_URL` +/// points at an already-running server (that takes precedence). The XTTS weights +/// are non-commercial, so this is opt-in and never part of a bundled release. +fn resolve_xtts_paths(_app: &App) -> Option<(PathBuf, PathBuf)> { + if std::env::var_os("EXOQUILL_XTTS_URL").is_some() { + return None; + } + let python = std::env::var("EXOQUILL_XTTS_PYTHON") + .map(PathBuf::from) + .ok()?; + let script = std::env::var("EXOQUILL_XTTS_SCRIPT") + .map(PathBuf::from) + .ok()?; + (python.exists() && script.exists()).then_some((python, script)) +} + +/// Python + `zonos-server.py` + a reference-voice folder, from +/// `EXOQUILL_ZONOS_PYTHON` / `EXOQUILL_ZONOS_SCRIPT` / `EXOQUILL_ZONOS_VOICES` +/// (set by dev.ps1). `None` when not configured. Zonos weights are Apache-2.0, +/// but it needs a CUDA GPU, so it's opt-in via the env vars. +fn resolve_zonos_paths(_app: &App) -> Option<(PathBuf, PathBuf, PathBuf)> { + let python = std::env::var("EXOQUILL_ZONOS_PYTHON") + .map(PathBuf::from) + .ok()?; + let script = std::env::var("EXOQUILL_ZONOS_SCRIPT") + .map(PathBuf::from) + .ok()?; + let voices = std::env::var("EXOQUILL_ZONOS_VOICES") + .map(PathBuf::from) + .ok()?; + (python.exists() && script.exists() && voices.exists()).then_some((python, script, voices)) +} + /// Open the region-OCR selection overlay: freeze the monitor under the cursor, /// stash the screenshot in state, and show a borderless, always-on-top window /// covering that monitor where the user drags a rectangle (snipping-tool style). @@ -270,24 +349,41 @@ pub fn run() { let ocr = resolve_ocr_provider(app); let formatter = resolve_formatter_provider(app); + let llama_server_paths = resolve_llama_server_paths(app); let stt = resolve_stt_provider(app); let whisper_server_paths = resolve_whisper_server_paths(app); let tts = resolve_tts_provider(app); + let xtts_paths = resolve_xtts_paths(app); + let zonos_paths = resolve_zonos_paths(app); app.manage(AppState { db: Arc::new(Mutex::new(db)), jobs, formatter, + llama_server_paths, + llama_server: Mutex::new(None), ocr, stt, whisper_server_paths, whisper_server: Mutex::new(None), tts, + xtts_paths, + xtts_server: Mutex::new(None), + xtts_warming: std::sync::atomic::AtomicBool::new(false), + zonos_paths, + zonos_server: Mutex::new(None), + zonos_warming: std::sync::atomic::AtomicBool::new(false), + read_cancel: Mutex::new(exoquill_core::CancelToken::new()), dictation: Mutex::new(None), region_capture: Mutex::new(None), #[cfg(feature = "silero")] silero_model_path: resolve_silero_model_path(app), }); + // TTS sidecars are NOT started here. Loading both XTTS and Zonos at + // launch froze the UI (two heavy Python/CUDA model loads at once). + // The UI calls `warm_tts(provider)` for the active backend instead, so + // only one loads, on demand, at below-normal priority. + tray::setup_tray(app)?; app.global_shortcut() .register(tray::quick_note_shortcut())?; @@ -302,10 +398,17 @@ pub fn run() { notes::get_note, notes::update_note, notes::delete_note, + notes::restore_note, + notes::set_archived, + notes::hard_delete_note, + notes::purge_trash, notes::list_notes, notes::search_notes, notes::resolve_target_note, notes::list_note_events, + notes::snapshot_note_version, + notes::list_note_history, + notes::restore_note_version, notes::export_note, jobs::format_note, jobs::cancel_job, @@ -316,8 +419,18 @@ pub fn run() { jobs::ocr_region, jobs::cancel_region_ocr, jobs::format_text, + jobs::begin_read, + jobs::prepare_speech, + jobs::cancel_read, jobs::tts_speak, + jobs::export_audio, + jobs::warm_tts, + jobs::ensure_tts_ready, + jobs::list_tts_voices, jobs::list_model_info, + models::list_catalog, + models::install_model, + models::delete_model, dictation::start_dictation, dictation::stop_dictation, dictation::list_capture_sources, @@ -332,6 +445,15 @@ pub fn run() { if let Ok(mut server) = state.whisper_server.lock() { let _ = server.take(); } + if let Ok(mut server) = state.llama_server.lock() { + let _ = server.take(); + } + if let Ok(mut server) = state.xtts_server.lock() { + let _ = server.take(); + } + if let Ok(mut server) = state.zonos_server.lock() { + let _ = server.take(); + } } } }); diff --git a/apps/desktop/src-tauri/src/models.rs b/apps/desktop/src-tauri/src/models.rs new file mode 100644 index 0000000..a7c3110 --- /dev/null +++ b/apps/desktop/src-tauri/src/models.rs @@ -0,0 +1,230 @@ +//! On-demand model catalog + install/delete — the model manager backend. +//! +//! The catalog (`models.json`, embedded) lists TTS assets in three tiers: +//! `bundled` (ships in the installer, redistributable), `download` (free, fetched +//! on demand), and `gated` (restrictive license, e.g. XTTS-v2 CPML — installed +//! locally via a setup script, never bundled). See docs/decisions.md for the +//! licensing rationale + the verified per-asset matrix. +//! +//! Files download to a writable models root (`EXOQUILL_MODELS_ROOT`, else the +//! app-data dir) under each file's `relPath`, which mirrors the layout the +//! providers resolve from (e.g. `piper-voices/…`). Newly downloaded voices are +//! picked up on the next app start. + +use std::fs; +use std::io::{Read, Write}; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Emitter, Manager}; + +const CATALOG_JSON: &str = include_str!("../models.json"); + +#[derive(Deserialize)] +struct Catalog { + models: Vec, +} + +#[derive(Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +struct ModelEntry { + id: String, + provider: String, + kind: String, + display_name: String, + language: String, + license: String, + commercial_ok: bool, + tier: String, + #[serde(default)] + files: Vec, + #[serde(default)] + setup: Option, + #[serde(default)] + notes: Option, +} + +#[derive(Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +struct ModelFile { + url: String, + rel_path: String, +} + +/// A catalog entry plus its on-disk status, for the manager UI. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CatalogItem { + id: String, + provider: String, + kind: String, + display_name: String, + language: String, + license: String, + commercial_ok: bool, + tier: String, + setup: Option, + notes: Option, + installed: bool, + installed_bytes: u64, +} + +/// Per-file download progress, emitted on the `model_progress` event. +#[derive(Serialize, Clone)] +#[serde(rename_all = "camelCase")] +struct ModelProgress { + id: String, + file: String, + downloaded: u64, + total: u64, +} + +fn catalog() -> Catalog { + serde_json::from_str(CATALOG_JSON).expect("embedded models.json is valid") +} + +/// The writable root downloaded models land in. `EXOQUILL_MODELS_ROOT` (dev → +/// `runtimes/`, where the providers look) or the app-data dir in release. +fn models_root(app: &AppHandle) -> PathBuf { + if let Ok(p) = std::env::var("EXOQUILL_MODELS_ROOT") { + return PathBuf::from(p); + } + app.path() + .app_data_dir() + .map(|d| d.join("models")) + .unwrap_or_else(|_| PathBuf::from("models")) +} + +/// Whether an entry is installed + the bytes it occupies. File entries check the +/// files under the models root; the XTTS runtime is detected by its env path. +fn entry_status(app: &AppHandle, entry: &ModelEntry) -> (bool, u64) { + if entry.files.is_empty() { + if entry.provider == "xtts" { + let ok = std::env::var("EXOQUILL_XTTS_PYTHON") + .map(|p| PathBuf::from(p).exists()) + .unwrap_or(false); + return (ok, 0); + } + return (false, 0); + } + let root = models_root(app); + let mut bytes = 0u64; + let mut all = true; + for f in &entry.files { + match fs::metadata(root.join(&f.rel_path)) { + Ok(m) => bytes += m.len(), + Err(_) => all = false, + } + } + (all, bytes) +} + +/// The installable model catalog with on-disk status, for the manager window. +#[tauri::command(async)] +pub fn list_catalog(app: AppHandle) -> Vec { + catalog() + .models + .into_iter() + .map(|e| { + let (installed, installed_bytes) = entry_status(&app, &e); + CatalogItem { + id: e.id, + provider: e.provider, + kind: e.kind, + display_name: e.display_name, + language: e.language, + license: e.license, + commercial_ok: e.commercial_ok, + tier: e.tier, + setup: e.setup, + notes: e.notes, + installed, + installed_bytes, + } + }) + .collect() +} + +/// Download a catalog entry's files to the models root, streaming with progress +/// (`model_progress` events). Each file goes to a `.part` then atomically renamed +/// so a crash never leaves a half file looking complete. Async so the download +/// never blocks the UI thread. Gated/setup-only entries return guidance instead. +#[tauri::command(async)] +pub fn install_model(app: AppHandle, id: String) -> Result<(), String> { + let entry = catalog() + .models + .into_iter() + .find(|e| e.id == id) + .ok_or_else(|| format!("unbekanntes Modell: {id}"))?; + if entry.files.is_empty() { + return Err(match entry.setup { + Some(s) => format!("Dieses Modell wird per Setup-Skript installiert: {s}"), + None => "Dieses Modell lässt sich nicht herunterladen.".into(), + }); + } + + let root = models_root(&app); + let client = reqwest::blocking::Client::builder() + .build() + .map_err(|e| format!("HTTP-Client: {e}"))?; + + for f in &entry.files { + let dest = root.join(&f.rel_path); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).map_err(|e| format!("Ordner anlegen: {e}"))?; + } + let mut resp = client + .get(&f.url) + .send() + .map_err(|e| format!("Download {}: {e}", f.url))?; + if !resp.status().is_success() { + return Err(format!("Download {} → HTTP {}", f.url, resp.status())); + } + let total = resp.content_length().unwrap_or(0); + let tmp = dest.with_extension("part"); + let mut out = fs::File::create(&tmp).map_err(|e| format!("Datei anlegen: {e}"))?; + let mut buf = vec![0u8; 1 << 16]; + let mut downloaded = 0u64; + loop { + let n = resp.read(&mut buf).map_err(|e| format!("Lesen: {e}"))?; + if n == 0 { + break; + } + out.write_all(&buf[..n]) + .map_err(|e| format!("Schreiben: {e}"))?; + downloaded += n as u64; + let _ = app.emit( + "model_progress", + ModelProgress { + id: id.clone(), + file: f.rel_path.clone(), + downloaded, + total, + }, + ); + } + out.flush().ok(); + drop(out); + fs::rename(&tmp, &dest).map_err(|e| format!("Abschließen: {e}"))?; + } + Ok(()) +} + +/// Delete a downloaded entry's files, freeing the disk. Bundled/gated entries +/// with no downloadable files are a no-op. +#[tauri::command(async)] +pub fn delete_model(app: AppHandle, id: String) -> Result<(), String> { + let entry = catalog() + .models + .into_iter() + .find(|e| e.id == id) + .ok_or_else(|| format!("unbekanntes Modell: {id}"))?; + let root = models_root(&app); + for f in &entry.files { + let path = root.join(&f.rel_path); + if path.exists() { + fs::remove_file(&path).map_err(|e| format!("Löschen {}: {e}", f.rel_path))?; + } + } + Ok(()) +} diff --git a/apps/desktop/src-tauri/src/notes.rs b/apps/desktop/src-tauri/src/notes.rs index 02c1fa3..7ceddf7 100644 --- a/apps/desktop/src-tauri/src/notes.rs +++ b/apps/desktop/src-tauri/src/notes.rs @@ -8,8 +8,11 @@ use exoquill_ai::formatter::FormatterProvider; use exoquill_ai::ocr::OcrProvider; use exoquill_ai::stt::SpeechToTextProvider; use exoquill_ai::tts::TextToSpeechProvider; -use exoquill_core::note::{NewNote, Note, NoteEvent, NoteSource, NoteUpdate}; -use exoquill_core::JobQueue; +use exoquill_core::note::{ + NewNote, NewNoteVersion, Note, NoteEvent, NoteScope, NoteSort, NoteSource, NoteUpdate, + NoteVersion, +}; +use exoquill_core::{CancelToken, JobQueue}; use exoquill_db::Database; use tauri::{AppHandle, State}; use tauri_plugin_dialog::DialogExt; @@ -19,7 +22,15 @@ pub struct AppState { /// Shared so job tasks can persist results from the worker thread. pub db: Arc>, pub jobs: JobQueue, + /// Per-call llama.cpp (`llama-cli`) when reachable, otherwise the mock. Used + /// as the formatter fallback when the persistent server can't start. pub formatter: Arc, + /// `(llama-server.exe, model)` paths for the persistent formatter server, or + /// `None` when the runtime/model isn't available. Resolved once at setup. + pub llama_server_paths: Option<(PathBuf, PathBuf)>, + /// The persistent llama-server, started lazily on first format and kept alive + /// (model resident) so chunked formatting is fast. Dropping it kills it. + pub llama_server: Mutex>, pub ocr: Arc, /// Per-call Whisper (`whisper-cli`) when reachable, otherwise the mock. Used /// as the dictation fallback when the persistent server can't start. @@ -32,7 +43,33 @@ pub struct AppState { /// Dropping it kills the server. pub whisper_server: Mutex>, /// `None` when no local TTS is available; the UI falls back to system speech. + /// This is the Piper (or external-URL XTTS) provider resolved at setup; it's + /// the fallback when the auto-spawned XTTS sidecar isn't running. pub tts: Option>, + /// `(python, xtts-server.py)` paths to auto-spawn the XTTS sidecar, or `None` + /// when not configured (then TTS uses `tts` above). Experimental / dev. + pub xtts_paths: Option<(PathBuf, PathBuf)>, + /// The XTTS sidecar, warmed up on demand (when the UI selects the XTTS + /// backend) and kept alive. Dropping it kills the Python process. Not started + /// at launch — that's what froze the UI when two sidecars loaded at once. + pub xtts_server: Mutex>, + /// Guards against starting two XTTS sidecars when `warm_tts` is called twice + /// before the first finishes loading. + pub xtts_warming: std::sync::atomic::AtomicBool, + /// `(python, zonos-server.py, voices_dir)` to spawn the Zonos sidecar, or + /// `None` when not configured. `voices_dir` holds the reference `.wav` clips + /// (one per voice). Apache-2.0 weights, but needs a CUDA GPU. + pub zonos_paths: Option<(PathBuf, PathBuf, PathBuf)>, + /// The Zonos sidecar, warmed up on demand (when the UI selects Zonos) and + /// kept alive. Dropping it kills the Python process. + pub zonos_server: Mutex>, + /// Guards against starting two Zonos sidecars concurrently (see above). + pub zonos_warming: std::sync::atomic::AtomicBool, + /// Cancellation for the in-progress read-aloud speech-prep pass. `begin_read` + /// installs a fresh token, `cancel_read` trips it, and each `prepare_speech` + /// chunk runs under it so a cancel stops the streaming llama generation + /// mid-flight instead of letting the chunk run to completion. + pub read_cancel: Mutex, /// The active dictation session, if capturing. Guarded so start/stop and the /// worker never race on it. pub dictation: Mutex>, @@ -47,7 +84,7 @@ pub struct AppState { type CommandResult = Result; -#[tauri::command] +#[tauri::command(async)] pub fn create_note( state: State, content_markdown: String, @@ -64,13 +101,13 @@ pub fn create_note( .map_err(|e| e.to_string()) } -#[tauri::command] +#[tauri::command(async)] pub fn get_note(state: State, id: String) -> CommandResult> { let db = state.db.lock().map_err(|e| e.to_string())?; db.get_note(&id).map_err(|e| e.to_string()) } -#[tauri::command] +#[tauri::command(async)] pub fn update_note( state: State, id: String, @@ -80,25 +117,99 @@ pub fn update_note( db.update_note(&id, update).map_err(|e| e.to_string()) } -#[tauri::command] +/// Move a note to the trash (soft-delete). Returns `true` if a live note moved. +#[tauri::command(async)] pub fn delete_note(state: State, id: String) -> CommandResult { let db = state.db.lock().map_err(|e| e.to_string())?; db.delete_note(&id).map_err(|e| e.to_string()) } -#[tauri::command] -pub fn list_notes(state: State) -> CommandResult> { +/// Restore a trashed note back to Active. +#[tauri::command(async)] +pub fn restore_note(state: State, id: String) -> CommandResult { let db = state.db.lock().map_err(|e| e.to_string())?; - db.list_notes().map_err(|e| e.to_string()) + db.restore_note(&id).map_err(|e| e.to_string()) } -#[tauri::command] -pub fn search_notes(state: State, query: String) -> CommandResult> { +/// Archive or un-archive a live note. +#[tauri::command(async)] +pub fn set_archived(state: State, id: String, archived: bool) -> CommandResult { let db = state.db.lock().map_err(|e| e.to_string())?; - db.search_notes(&query).map_err(|e| e.to_string()) + db.set_archived(&id, archived).map_err(|e| e.to_string()) } -#[tauri::command] +/// Permanently delete a note (and its events + versions). No undo. +#[tauri::command(async)] +pub fn hard_delete_note(state: State, id: String) -> CommandResult { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.hard_delete_note(&id).map_err(|e| e.to_string()) +} + +/// Permanently delete trashed notes older than `before` (an RFC-3339 timestamp, +/// e.g. now − 30 days; the frontend computes the cutoff). Returns the count. +#[tauri::command(async)] +pub fn purge_trash(state: State, before: String) -> CommandResult { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.purge_trash(&before).map_err(|e| e.to_string()) +} + +#[tauri::command(async)] +pub fn list_notes( + state: State, + scope: Option, + sort: Option, +) -> CommandResult> { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.list_notes(scope.unwrap_or_default(), sort.unwrap_or_default()) + .map_err(|e| e.to_string()) +} + +#[tauri::command(async)] +pub fn search_notes( + state: State, + query: String, + scope: Option, +) -> CommandResult> { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.search_notes(&query, scope.unwrap_or_default()) + .map_err(|e| e.to_string()) +} + +/// Record a content snapshot for the edit history (deduped by content hash, so +/// no-op saves add nothing). Returns the stored version, or `None` if deduped. +#[tauri::command(async)] +pub fn snapshot_note_version( + state: State, + version: NewNoteVersion, +) -> CommandResult> { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.insert_version(version).map_err(|e| e.to_string()) +} + +/// A note's edit-history versions (diff timeline), most recent first. +#[tauri::command(async)] +pub fn list_note_history( + state: State, + note_id: String, +) -> CommandResult> { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.list_versions(¬e_id).map_err(|e| e.to_string()) +} + +/// Restore a stored version's content into the note as a new, undoable change +/// (non-destructive). Returns the updated note, or `None` if it's gone. +#[tauri::command(async)] +pub fn restore_note_version( + state: State, + note_id: String, + version_id: String, +) -> CommandResult> { + let db = state.db.lock().map_err(|e| e.to_string())?; + db.restore_version(¬e_id, &version_id) + .map_err(|e| e.to_string()) +} + +#[tauri::command(async)] pub fn resolve_target_note(state: State, active: Option) -> CommandResult { let db = state.db.lock().map_err(|e| e.to_string())?; db.resolve_target_note(active.as_deref()) @@ -107,7 +218,7 @@ pub fn resolve_target_note(state: State, active: Option) -> Co /// The recorded events for a note (formatting/OCR history + undo safety net), /// most recent first. -#[tauri::command] +#[tauri::command(async)] pub fn list_note_events(state: State, note_id: String) -> CommandResult> { let db = state.db.lock().map_err(|e| e.to_string())?; db.list_events(¬e_id).map_err(|e| e.to_string()) @@ -116,7 +227,7 @@ pub fn list_note_events(state: State, note_id: String) -> CommandResul /// Export a note's Markdown to a file the user picks (native save dialog). /// Returns the saved path, or `None` if the user cancelled. Notes are stored as /// Markdown, so this writes `content_markdown` verbatim. -#[tauri::command] +#[tauri::command(async)] pub fn export_note( state: State, app: AppHandle, diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index ed04d88..61606ac 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "ExoQuill", - "version": "0.1.0", + "version": "0.2.0", "identifier": "de.codexo.exoquill", "build": { "beforeDevCommand": "pnpm dev", diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 8c275f1..6479620 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -1,6 +1,14 @@ import { type Editor as TiptapEditor } from "@tiptap/react"; import { listen } from "@tauri-apps/api/event"; -import { type ChangeEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + type ChangeEvent, + type MouseEvent as ReactMouseEvent, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { ActionBar } from "./components/ActionBar"; import { @@ -12,63 +20,136 @@ import { selectionText, separatorBefore, } from "./components/Editor"; +import { HistoryOverlay } from "./components/HistoryOverlay"; +import { ModelManager } from "./components/ModelManager"; import { OcrOverlay } from "./components/OcrOverlay"; -import { PlusIcon } from "./components/icons"; +import { ReadAloudSettings, TTS_DEFAULTS } from "./components/ReadAloudSettings"; +import { ArchiveIcon, PlusIcon, RestoreIcon, TrashIcon } from "./components/icons"; import { Sidebar } from "./components/Sidebar"; import { Statusbar } from "./components/Statusbar"; -import { Topbar } from "./components/Topbar"; +import { ToastStack, useToasts } from "./components/Toasts"; import { useTheme } from "./hooks/useTheme"; import * as api from "./lib/api"; -import { stopPlayback } from "./lib/audio"; +import { getLang, translate, useI18n } from "./lib/i18n"; +import type { I18n, TranslationKey } from "./lib/i18n"; +import { decodePcm, playSamples, stopPlayback } from "./lib/audio"; +import { chunkMarkdown, cleanDictation } from "./lib/format"; import { startDictation, stopDictation, subscribeDictation } from "./lib/dictation"; -import { readAloud, type ReadAloudHandle } from "./lib/readaloud"; +import { plainSource, preparedSource, readAloud, type ReadAloudHandle } from "./lib/readaloud"; import { stopSpeaking } from "./lib/speech"; +import { ZONOS_EMOTIONS } from "./lib/tts"; import type { BackendEvent, CaptureSource, + CatalogItem, ModelInfo, + ModelProgress, Note, - NoteEvent, + NoteScope, + NoteSort, NoteUpdate, + NoteVersion, OcrLayout, RegionOcr, + TtsTuning, + TtsResponse, + TtsVoice, } from "./lib/types"; import "./styles/app.css"; -function sortNotes(notes: Note[]): Note[] { +/** Sort a note list to match the backend ordering: pinned first, then by the + * chosen key (so optimistic local updates keep the same order as a reload). */ +function sortNotes(notes: Note[], sort: NoteSort): Note[] { return [...notes].sort((a, b) => { if (a.pinned !== b.pinned) return a.pinned ? -1 : 1; + if (sort === "title") return a.title.localeCompare(b.title, undefined, { sensitivity: "base" }); + if (sort === "created") return b.createdAt.localeCompare(a.createdAt); return b.updatedAt.localeCompare(a.updatedAt); }); } -function noteMeta(note: Note): string { +function noteMeta(note: Note, t: I18n["t"]): string { const text = note.contentMarkdown.trim(); const words = text ? text.split(/\s+/).length : 0; - return `${text ? "DRAFT" : "EMPTY"} · ${words} WORDS`; + return `${text ? t("meta.draft") : t("meta.empty")} · ${t("meta.words", { count: words })}`; } +// Display labels for the TTS backends behind the toolbar's backend picker. +const BACKEND_LABELS: Record = { piper: "Piper", xtts: "XTTS", zonos: "Zonos" }; +const backendLabel = (provider: string) => BACKEND_LABELS[provider] ?? provider; + export default function App() { + const { t, lang, setLang } = useI18n(); const [theme, toggleTheme] = useTheme(); const [notes, setNotes] = useState([]); const [activeId, setActiveId] = useState(null); const [query, setQuery] = useState(""); + // The sidebar scope (Active / Archived / Trash) and sort order (persisted). + const [scope, setScope] = useState("active"); + const [sort, setSort] = useState( + () => (localStorage.getItem("notes-sort") as NoteSort) || "modified", + ); + // Multi-selected note ids (non-empty → the sidebar is in selection mode). + const [selected, setSelected] = useState>(() => new Set()); + // Undo-toast queue for reversible/destructive note actions. + const toasts = useToasts(); + // Set when a rename was requested, to focus + select the title input once. + const [pendingTitleFocus, setPendingTitleFocus] = useState(false); const [saved, setSaved] = useState(true); const [formatting, setFormatting] = useState(false); const [reading, setReading] = useState(false); + // Whether audio has actually started (vs. still preparing/buffering). Drives + // the toolbar: cancel-only while preparing, pause/stop once speaking. + const [speaking, setSpeaking] = useState(false); + // Whether read-aloud is currently paused (suspended AudioContext). + const [paused, setPaused] = useState(false); const [dictating, setDictating] = useState(false); const [micLevel, setMicLevel] = useState(0); const [dictationError, setDictationError] = useState(null); const [sources, setSources] = useState([]); // The chosen dictation source; `null` is the system default microphone. const [source, setSource] = useState(null); + // The read-aloud voices the local TTS offers, and the chosen one (persisted). + // Empty `voices` means no local TTS — read-aloud uses system speech instead. + const [voices, setVoices] = useState([]); + const [voiceId, setVoiceId] = useState(() => localStorage.getItem("tts-voice") ?? ""); + // Read-aloud synthesis knobs (persisted), and whether the settings dialog is open. + const [tuning, setTuning] = useState(() => { + try { + return { ...TTS_DEFAULTS, ...JSON.parse(localStorage.getItem("tts-tuning") ?? "{}") }; + } catch { + return { ...TTS_DEFAULTS }; + } + }); + const [showVoiceSettings, setShowVoiceSettings] = useState(false); + // Chunked-format progress (null = idle), and whether to run an LLM "prepare for + // speech" pass before read-aloud (persisted). + const [formatProgress, setFormatProgress] = useState<{ done: number; total: number } | null>( + null, + ); + // Progress of the Zonos prebuffer synthesis (whole-text-then-play), null = idle. + const [synthProgress, setSynthProgress] = useState<{ done: number; total: number } | null>(null); + // Shows a brief "voice still loading" hint when a read starts before the chosen + // sidecar voice is warm (instead of playing a wrong/robot voice). + const [voiceLoading, setVoiceLoading] = useState(false); + // Whether the settings dialog's one-sentence preview is currently rendering. + const [previewing, setPreviewing] = useState(false); + const [speechPrep, setSpeechPrep] = useState( + () => localStorage.getItem("tts-speech-prep") === "1", + ); // The open OCR result overlay (pasted/picked image), and whether OCR is running. const [ocr, setOcr] = useState<{ url: string; layout: OcrLayout } | null>(null); const [ocrBusy, setOcrBusy] = useState(false); - // The open note-event history overlay (null = closed). - const [history, setHistory] = useState(null); - // The open on-device models overlay (null = closed). + // The open edit-history overlay's versions (null = closed). + const [history, setHistory] = useState(null); + // The on-device provider summaries (shown inside the model manager). const [models, setModels] = useState(null); + // The model manager window: open flag, catalog, live download progress, and + // the entry currently installing. + const [showModels, setShowModels] = useState(false); + const [catalog, setCatalog] = useState([]); + const [modelProgress, setModelProgress] = useState>({}); + const [modelBusy, setModelBusy] = useState(null); // The open formatting preview (original vs formatted), with its apply action. const [preview, setPreview] = useState<{ original: string; @@ -81,6 +162,9 @@ export default function App() { const editorRef = useRef(null); const fileInputRef = useRef(null); + const titleInputRef = useRef(null); + // The note id last clicked in the sidebar (anchor for Shift-range selection). + const lastClickedId = useRef(null); // Guards against re-entering the async start/stop (e.g. a double-click). const dictationBusy = useRef(false); // Where dictated text lands, decided when dictation starts: "replace" the @@ -95,26 +179,75 @@ export default function App() { const utterance = useRef<{ from: number; prefix: string; committed: string } | null>(null); // The running read-aloud queue, if any (so it can be stopped). const readHandle = useRef(null); + // Cache of speech-prep results, keyed by raw chunk text. The LLM rewrite is + // backend-independent, so switching voice/backend (or re-reading the same note) + // reuses already-prepared chunks instead of running the LLM again. + const prepCache = useRef>(new Map()); + // Cache of synthesized audio, keyed by voice+tuning+text. Same voice & text → + // replay the stored audio instead of re-synthesizing. `lastAudioKey` is the most + // recently rendered key, which the Export-Audio button saves to a WAV. + const audioCache = useRef>(new Map()); + const [lastAudioKey, setLastAudioKey] = useState(null); - const activeNote = useMemo( - () => notes.find((n) => n.id === activeId) ?? null, - [notes, activeId], - ); + // The open note stays available even when the sidebar shows another scope + // (e.g. after archiving it, or while viewing Trash): keep the last-seen copy. + const activeNoteCache = useRef(null); + const activeNote = useMemo(() => { + const found = notes.find((n) => n.id === activeId) ?? null; + if (found) activeNoteCache.current = found; + if (found) return found; + return activeNoteCache.current?.id === activeId ? activeNoteCache.current : null; + }, [notes, activeId]); - const load = useCallback(async (q: string) => { - const list = q.trim() ? await api.searchNotes(q) : await api.listNotes(); - const sorted = sortNotes(list); - setNotes(sorted); - setActiveId((cur) => (cur && sorted.some((n) => n.id === cur) ? cur : sorted[0]?.id ?? null)); + const load = useCallback(async (q: string, sc: NoteScope, so: NoteSort) => { + const list = q.trim() ? await api.searchNotes(q, sc) : await api.listNotes(sc, so); + setNotes(sortNotes(list, so)); + // Open the first note only on the very first Active view with nothing open; + // otherwise keep the current selection (it may now live in another scope). + setActiveId((cur) => cur ?? (sc === "active" ? list[0]?.id ?? null : null)); }, []); + // Refs so stable action callbacks can reload with the latest view params. const queryRef = useRef(query); queryRef.current = query; + const scopeRef = useRef(scope); + scopeRef.current = scope; + const sortRef = useRef(sort); + sortRef.current = sort; + const notesRef = useRef(notes); + notesRef.current = notes; + const activeIdRef = useRef(activeId); + activeIdRef.current = activeId; + + const reload = useCallback( + () => load(queryRef.current, scopeRef.current, sortRef.current), + [load], + ); useEffect(() => { - const t = window.setTimeout(() => void load(query), query ? 200 : 0); - return () => clearTimeout(t); - }, [query, load]); + const handle = window.setTimeout(() => void load(query, scope, sort), query ? 200 : 0); + return () => clearTimeout(handle); + }, [query, scope, sort, load]); + + // Persist the sort choice. + useEffect(() => { + localStorage.setItem("notes-sort", sort); + }, [sort]); + + // Switching scope clears any multi-selection (it's scope-specific). + const changeScope = useCallback((next: NoteScope) => { + setScope(next); + setSelected(new Set()); + }, []); + + // Focus + select the title input once after a "rename" request. + useEffect(() => { + if (pendingTitleFocus && titleInputRef.current) { + titleInputRef.current.focus(); + titleInputRef.current.select(); + setPendingTitleFocus(false); + } + }, [pendingTitleFocus, activeNote]); // Stop any read-aloud when switching notes. useEffect(() => { @@ -125,6 +258,19 @@ export default function App() { setReading(false); }, [activeId]); + // On leaving a note, snapshot its current content as a manual history baseline + // (deduped in the backend, so an unchanged note adds nothing). + useEffect(() => { + const leaving = activeId; + return () => { + if (!leaving) return; + const note = notesRef.current.find((n) => n.id === leaving); + if (note && note.contentMarkdown.trim()) { + void api.snapshotNoteVersion({ noteId: leaving, contentMarkdown: note.contentMarkdown }); + } + }; + }, [activeId]); + // Debounced autosave: optimistic local update now, persist after a pause. const saveTimer = useRef(null); const pending = useRef({}); @@ -138,7 +284,7 @@ export default function App() { pending.current = {}; const updated = await api.updateNote(id, update); if (updated) { - setNotes((prev) => sortNotes(prev.map((n) => (n.id === id ? updated : n)))); + setNotes((prev) => sortNotes(prev.map((n) => (n.id === id ? updated : n)), sortRef.current)); } setSaved(true); }, 450); @@ -176,16 +322,16 @@ export default function App() { if (terminal) { if (job.jobType === "format") setFormatting(false); if (job.status === "failed" && job.error) console.error("Job failed:", job.error); - void load(queryRef.current).then(() => setReloadKey((k) => k + 1)); + void reload().then(() => setReloadKey((k) => k + 1)); } } else if (payload.type === "notes_changed") { - void load(queryRef.current); + void reload(); } }); return () => { void unlisten.then((fn) => fn()); }; - }, [load]); + }, [reload]); const handleEditorReady = useCallback((editor: TiptapEditor) => { editorRef.current = editor; @@ -194,73 +340,460 @@ export default function App() { const newNote = useCallback(async () => { const note = await api.createNote(""); setQuery(""); - setNotes((prev) => sortNotes([note, ...prev.filter((n) => n.id !== note.id)])); + setScope("active"); // a new note belongs to the Active view + setSelected(new Set()); + setNotes((prev) => sortNotes([note, ...prev.filter((n) => n.id !== note.id)], sortRef.current)); + setActiveId(note.id); + }, []); + + // Drop a note from the visible list (it left the current scope) and, if it was + // the open one, pick a neighbour so the editor doesn't go blank unexpectedly. + const removeFromList = useCallback((id: string) => { + setNotes((prev) => { + if (activeIdRef.current === id) { + const idx = prev.findIndex((n) => n.id === id); + const fallback = prev[idx + 1] ?? prev[idx - 1] ?? null; + setActiveId(fallback?.id ?? null); + } + return prev.filter((n) => n.id !== id); + }); + }, []); + + // --- Sidebar selection (click = open; Ctrl/⌘ or Shift = multi-select) --- + const toggleSelect = useCallback((id: string) => { + setSelected((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + lastClickedId.current = id; + }, []); + + const handleSelect = useCallback( + (id: string, e: ReactMouseEvent) => { + if (e.shiftKey && lastClickedId.current) { + const ids = notesRef.current.map((n) => n.id); + const a = ids.indexOf(lastClickedId.current); + const b = ids.indexOf(id); + if (a !== -1 && b !== -1) { + const [lo, hi] = a < b ? [a, b] : [b, a]; + setSelected((prev) => { + const next = new Set(prev); + for (let i = lo; i <= hi; i++) next.add(ids[i]); + return next; + }); + } + return; + } + if (e.ctrlKey || e.metaKey || selected.size > 0) { + toggleSelect(id); + return; + } + setActiveId(id); + lastClickedId.current = id; + }, + [selected.size, toggleSelect], + ); + + // --- Per-note actions (optimistic, with undo toasts) --- + const handlePin = useCallback( + async (note: Note) => { + const pinned = !note.pinned; + setNotes((prev) => + sortNotes( + prev.map((n) => (n.id === note.id ? { ...n, pinned } : n)), + sortRef.current, + ), + ); + await api.updateNote(note.id, { pinned }); + void reload(); + }, + [reload], + ); + + const handleArchive = useCallback( + async (note: Note) => { + removeFromList(note.id); + await api.setArchived(note.id, true); + void reload(); + toasts.push(t("toast.archived"), { + icon: , + actionLabel: t("toast.undo"), + onAction: async () => { + await api.setArchived(note.id, false); + void reload(); + }, + }); + }, + [removeFromList, reload, toasts, t], + ); + + const handleTrash = useCallback( + async (note: Note) => { + removeFromList(note.id); + await api.deleteNote(note.id); + void reload(); + toasts.push(t("toast.trashed"), { + icon: , + actionLabel: t("toast.undo"), + onAction: async () => { + await api.restoreNote(note.id); + void reload(); + }, + }); + }, + [removeFromList, reload, toasts, t], + ); + + // Restore from Archive (un-archive) or Trash (un-delete), with undo. + const handleRestore = useCallback( + async (note: Note) => { + removeFromList(note.id); + const fromTrash = !!note.deletedAt; + if (fromTrash) await api.restoreNote(note.id); + else await api.setArchived(note.id, false); + void reload(); + toasts.push(t("toast.restored"), { + icon: , + actionLabel: t("toast.undo"), + onAction: async () => { + if (fromTrash) await api.deleteNote(note.id); + else await api.setArchived(note.id, true); + void reload(); + }, + }); + }, + [removeFromList, reload, toasts, t], + ); + + const handleDeleteForever = useCallback( + async (note: Note) => { + removeFromList(note.id); + await api.hardDeleteNote(note.id); + void reload(); + toasts.push(t("toast.deletedForever")); + }, + [removeFromList, reload, toasts, t], + ); + + const handleEmptyTrash = useCallback(async () => { + setNotes([]); + // Everything currently trashed was deleted before "now". + await api.purgeTrash(new Date().toISOString()); + void reload(); + toasts.push(t("toast.deletedForever")); + }, [reload, toasts, t]); + + const handleDuplicate = useCallback( + async (note: Note) => { + const copy = await api.createNote(note.contentMarkdown); + void reload(); + setActiveId(copy.id); + }, + [reload], + ); + + const handleRename = useCallback((note: Note) => { setActiveId(note.id); + setPendingTitleFocus(true); }, []); - const deleteActive = useCallback(async () => { - if (!activeNote) return; - const id = activeNote.id; - const idx = notes.findIndex((n) => n.id === id); - const fallback = notes[idx + 1] ?? notes[idx - 1] ?? null; - await api.deleteNote(id); - setNotes((prev) => prev.filter((n) => n.id !== id)); - setActiveId(fallback?.id ?? null); - }, [activeNote, notes]); + const handleExportNote = useCallback( + (note: Note) => { + void flushSave().then(() => api.exportNote(note.id)); + }, + [flushSave], + ); + + // Delete the open note (action-bar trash) → move to Trash with undo. + const deleteActive = useCallback(() => { + if (activeNote) void handleTrash(activeNote); + }, [activeNote, handleTrash]); + + // --- Bulk actions over the current selection --- + const handleBulkPin = useCallback(async () => { + const ids = [...selected]; + setSelected(new Set()); + await Promise.all(ids.map((id) => api.updateNote(id, { pinned: true }))); + void reload(); + }, [selected, reload]); + + const handleBulkArchive = useCallback(async () => { + const ids = [...selected]; + setSelected(new Set()); + setNotes((prev) => prev.filter((n) => !ids.includes(n.id))); + await Promise.all(ids.map((id) => api.setArchived(id, true))); + void reload(); + toasts.push(t("toast.archivedMany", { count: ids.length }), { + icon: , + actionLabel: t("toast.undo"), + onAction: async () => { + await Promise.all(ids.map((id) => api.setArchived(id, false))); + void reload(); + }, + }); + }, [selected, reload, toasts, t]); + + const handleBulkTrash = useCallback(async () => { + const ids = [...selected]; + setSelected(new Set()); + setNotes((prev) => prev.filter((n) => !ids.includes(n.id))); + if (activeIdRef.current && ids.includes(activeIdRef.current)) setActiveId(null); + await Promise.all(ids.map((id) => api.deleteNote(id))); + void reload(); + toasts.push(t("toast.trashedMany", { count: ids.length }), { + icon: , + actionLabel: t("toast.undo"), + onAction: async () => { + await Promise.all(ids.map((id) => api.restoreNote(id))); + void reload(); + }, + }); + }, [selected, reload, toasts, t]); + + const handleBulkExport = useCallback(async () => { + const ids = [...selected]; + setSelected(new Set()); + await flushSave(); + // Sequential native save dialogs (WebView2 can't batch downloads). + for (const id of ids) await api.exportNote(id); + }, [selected, flushSave]); + + // --- Edit history --- + const openHistory = useCallback(() => { + if (activeIdRef.current) void api.listNoteHistory(activeIdRef.current).then(setHistory); + }, []); + + const restoreVersion = useCallback( + async (versionId: string) => { + const id = activeIdRef.current; + if (!id) return; + await api.restoreNoteVersion(id, versionId); + setHistory(null); + void reload(); + setReloadKey((k) => k + 1); + toasts.push(t("toast.versionRestored")); + }, + [reload, toasts, t], + ); // Format the selection, or the whole note when nothing is selected, then open // a preview (original vs formatted) so the change is applied only on confirm // (D6 — replace + undo, now with a look-before-you-leap step). - const formatActive = useCallback(async () => { + const formatActive = useCallback(() => { const editor = editorRef.current; if (!editor) return; const selection = selectionText(editor); const isSelection = !!selection.trim(); const original = isSelection ? selection : editorMarkdown(editor); if (!original.trim()) return; - setFormatting(true); - try { - const formatted = await api.formatText(original); - setPreview({ - original, - formatted, - onApply: async () => { - if (isSelection) { - replaceSelection(editor, formatted); - return; + // Deterministic, instant cleanup — no LLM, so no garbage output and no hang. + const formatted = cleanDictation(original); + setPreview({ + original, + formatted, + onApply: async () => { + if (isSelection) { + replaceSelection(editor, formatted); + if (activeId) { + void api.snapshotNoteVersion({ + noteId: activeId, + contentMarkdown: editorMarkdown(editor), + source: "op", + op: "format", + }); } - if (!activeId) return; - await api.updateNote(activeId, { contentMarkdown: formatted }); - await load(queryRef.current); - setReloadKey((k) => k + 1); - }, - }); - } catch (err) { - console.error("format failed:", err); - } finally { - setFormatting(false); - } - }, [activeId, load]); + return; + } + if (!activeId) return; + await api.updateNote(activeId, { contentMarkdown: formatted }); + void api.snapshotNoteVersion({ + noteId: activeId, + contentMarkdown: formatted, + source: "op", + op: "format", + }); + await reload(); + setReloadKey((k) => k + 1); + }, + }); + }, [activeId, reload]); // Read the selection, or the whole note, aloud — toggling stop. Streams via the // read-aloud queue (sentence chunks, local Piper TTS with prefetch, system // speech fallback) so playback starts on the first segment, not the whole note. - const readActive = useCallback(() => { - if (reading) { + const readActive = useCallback(async (retry = false) => { + // `retry` is the automatic restart after a cold sidecar warmed up — it skips + // the stop toggle (the previous attempt already tore itself down) and only + // ever fires once, so a still-failing voice can't loop. + if (!retry && reading) { readHandle.current?.stop(); readHandle.current = null; + void api.cancelRead(); // stop any in-flight speech-prep generation setReading(false); + setSpeaking(false); + setPaused(false); + setFormatProgress(null); + setSynthProgress(null); return; } const selection = selectionText(editorRef.current); - const text = selection.trim() ? selection : activeNote?.contentMarkdown ?? ""; - if (!text.trim()) return; + const raw = selection.trim() ? selection : activeNote?.contentMarkdown ?? ""; + if (!raw.trim()) return; setReading(true); - readHandle.current = readAloud(text, api.ttsSpeak, () => { - readHandle.current = null; - setReading(false); - }); - }, [reading, activeNote]); + setSpeaking(false); + setPaused(false); + setVoiceLoading(false); + // Route synthesis to the chosen voice's backend (Piper / XTTS). + const provider = voices.find((v) => v.id === voiceId)?.provider; + const speak = (chunk: string) => api.ttsSpeak(chunk, voiceId || undefined, provider, tuning); + // With speech-prep on, rewrite the note for speech chunk-by-chunk and feed + // each prepared chunk straight into synthesis — playback starts on the first + // chunk instead of waiting for the whole note (pipelined). `beginRead` arms a + // cancel token so the Abbrechen button can stop generation mid-chunk. + // Otherwise speak the raw text directly. + // Audio cache: same voice + FULL tuning + text (+ speech-prep flag) → replay + // the stored audio instead of re-synthesizing. The key must include every + // tuning knob (speed, Piper's expressiveness/cadence/sentence-silence, Zonos' + // intonation/brightness/emotion) — changing any of them has to invalidate the + // cache. Survives backend switches. + const tuneKey = `${tuning.speed ?? ""}/${tuning.expressiveness ?? ""}/${tuning.cadence ?? ""}/${tuning.sentenceSilence ?? ""}/${tuning.intonation ?? ""}/${tuning.brightness ?? ""}/${tuning.emotion ?? ""}`; + const cacheKey = `${provider ?? ""}|${voiceId}|${tuneKey}|${speechPrep ? 1 : 0}|${raw}`; + const cachedAudios = audioCache.current.get(cacheKey); + if (cachedAudios) setLastAudioKey(cacheKey); + + const source = cachedAudios + ? plainSource("") // unused on a cache hit + : speechPrep + ? (() => { + const chunks = chunkMarkdown(raw); + setFormatProgress({ done: 0, total: chunks.length }); + void api.beginRead(); + return preparedSource( + chunks, + async (chunk) => { + const cached = prepCache.current.get(chunk); + if (cached !== undefined) return cached; + const formatted = await api.prepareSpeech(chunk); + prepCache.current.set(chunk, formatted); // only on success (cancel throws) + return formatted; + }, + (done, total) => setFormatProgress(done >= total ? null : { done, total }), + ); + })() + : plainSource(raw); + // Zonos is slower than real time, so streaming stutters between sentences. + // For it, synthesize the whole text up front (with progress) and play gapless. + const prebuffer = provider === "zonos"; + readHandle.current = readAloud( + source, + speak, + () => { + readHandle.current = null; + setReading(false); + setSpeaking(false); + setPaused(false); + setFormatProgress(null); + setSynthProgress(null); + }, + raw, + { + onPlaybackStart: () => setSpeaking(true), + prebuffer, + onPrepare: (done, total) => + setSynthProgress(done >= total ? null : { done, total }), + cachedAudios, + onAudio: (audios) => { + if (audios.length) { + audioCache.current.set(cacheKey, audios); + setLastAudioKey(cacheKey); + } + }, + onUnavailable: () => { + // Chosen sidecar voice isn't warm yet. Keep the "loading" hint up, wait + // for the warm-up to finish, then restart the read automatically — no + // second click. The `retry` guard means a still-cold voice gives up + // after one auto-retry instead of looping. + readHandle.current = null; + setReading(false); + setSpeaking(false); + setFormatProgress(null); + setSynthProgress(null); + if (!provider || retry) { + setVoiceLoading(false); + if (provider) void api.warmTts(provider); // best-effort for next time + return; + } + setVoiceLoading(true); + void (async () => { + let ready = false; + try { + await api.ensureTtsReady(provider); + ready = true; + } catch { + ready = false; // warm-up failed/timed out — drop the hint, stay put + } + setVoiceLoading(false); + if (ready) void readActiveRef.current(true); // warm now → auto-start + })(); + }, + }, + ); + }, [reading, activeNote, voiceId, voices, tuning, speechPrep]); + + // Always points at the latest `readActive`, so the warm-up handler inside it can + // restart the read once the cold sidecar is ready (it can't reference itself). + const readActiveRef = useRef(readActive); + readActiveRef.current = readActive; + + // Export the most recently rendered read-aloud audio to a WAV file via a native + // save dialog (WebView2 can't trigger a browser download). + const exportAudio = useCallback(() => { + const audios = lastAudioKey ? audioCache.current.get(lastAudioKey) : null; + if (!audios || !audios.length) return; + const name = `${(activeNote?.title || translate(getLang(), "readaloud.exportName")).replace(/[^\w.-]+/g, "_")}.wav`; + void api.exportAudio( + audios.map((a) => a.pcm), + audios[0].sampleRate, + name, + ); + }, [lastAudioKey, activeNote]); + + // Pause / resume the running read-aloud (suspends the audio; the queue waits). + const togglePauseRead = useCallback(() => { + const handle = readHandle.current; + if (!handle) return; + if (paused) { + handle.resume(); + setPaused(false); + } else { + handle.pause(); + setPaused(true); + } + }, [paused]); + + // One-sentence preview of the current voice + tuning for the settings dialog — + // a single synthesis (one slice), played at once, for quick A/B tuning. + const previewVoice = useCallback(async () => { + if (previewing) return; + const provider = voices.find((v) => v.id === voiceId)?.provider; + setPreviewing(true); + try { + const audio = await api.ttsSpeak( + translate(getLang(), "readaloud.sample"), + voiceId || undefined, + provider, + tuning, + ); + stopPlayback(); + playSamples(decodePcm(audio.pcm), audio.sampleRate); + } catch (err) { + console.error("preview failed:", err); + } finally { + setPreviewing(false); + } + }, [previewing, voiceId, voices, tuning]); // Toggle microphone dictation. Captured audio is segmented in the webview and // transcribed locally by Whisper; each finalized segment is inserted at the @@ -288,7 +821,7 @@ export default function App() { let language = activeNote?.languageMode; if (!activeId) { const note = await api.createNote("", "dictation"); - setNotes((prev) => sortNotes([note, ...prev.filter((n) => n.id !== note.id)])); + setNotes((prev) => sortNotes([note, ...prev.filter((n) => n.id !== note.id)], sortRef.current)); setActiveId(note.id); language = note.languageMode; } @@ -307,6 +840,88 @@ export default function App() { void api.listCaptureSources().then(setSources).catch(() => setSources([])); }, []); + // Load the read-aloud voices once; if nothing is stored (or the stored voice + // is gone), fall back to the first available voice. + useEffect(() => { + void api + .listTtsVoices() + .then((vs) => { + setVoices(vs); + // Keep a valid stored voice; otherwise default to XTTS (fast on GPU, + // fluent streaming) and fall back to the first voice. + const stored = localStorage.getItem("tts-voice") ?? ""; + const active = vs.some((v) => v.id === stored) + ? stored + : (vs.find((v) => v.provider === "xtts") ?? vs[0])?.id ?? ""; + setVoiceId(active); + // Warm only the active backend's sidecar — never both at launch. + const provider = vs.find((v) => v.id === active)?.provider; + if (provider) void api.warmTts(provider); + }) + .catch(() => setVoices([])); + }, []); + + // Persist the read-aloud tuning knobs. + useEffect(() => { + localStorage.setItem("tts-tuning", JSON.stringify(tuning)); + }, [tuning]); + + // Live model-download progress for the manager. + useEffect(() => { + const unlisten = listen("model_progress", (e) => { + setModelProgress((p) => ({ ...p, [e.payload.id]: e.payload })); + }); + return () => void unlisten.then((f) => f()); + }, []); + + // Open the model manager: load the catalog + active-provider summaries. + const openModels = useCallback(() => { + setShowModels(true); + void api.listCatalog().then(setCatalog).catch(() => setCatalog([])); + void api.listModelInfo().then(setModels).catch(() => setModels([])); + }, []); + + // Install a catalog entry (license-gate restrictive ones first), then refresh. + const handleInstallModel = useCallback(async (item: CatalogItem) => { + if ( + !item.commercialOk && + !window.confirm( + translate(getLang(), "models.confirmNonCommercial", { + name: item.displayName, + license: item.license, + }), + ) + ) { + return; + } + setModelBusy(item.id); + try { + await api.installModel(item.id); + setCatalog(await api.listCatalog()); + } catch (err) { + console.error("install failed:", err); + window.alert(translate(getLang(), "models.installFailed", { error: String(err) })); + } finally { + setModelBusy(null); + setModelProgress((p) => { + const rest = { ...p }; + delete rest[item.id]; + return rest; + }); + } + }, []); + + const handleDeleteModel = useCallback(async (item: CatalogItem) => { + if (!window.confirm(translate(getLang(), "models.confirmDelete", { name: item.displayName }))) + return; + try { + await api.deleteModel(item.id); + setCatalog(await api.listCatalog()); + } catch (err) { + console.error("delete failed:", err); + } + }, []); + // Subscribe once to the backend's live dictation events: insert each // transcript chunk at the cursor, drive the level meter, surface errors, and // track recording state. Stop capture if the app unmounts mid-dictation. @@ -408,7 +1023,7 @@ export default function App() { }); }); const error = listen("region-ocr-error", ({ payload }) => - setDictationError(`Region-OCR fehlgeschlagen: ${payload}`), + setDictationError(translate(getLang(), "ocr.regionFailed", { error: payload })), ); return () => { void result.then((fn) => fn()); @@ -438,7 +1053,7 @@ export default function App() { }); } catch (err) { URL.revokeObjectURL(url); - setDictationError(`Texterkennung fehlgeschlagen: ${String(err)}`); + setDictationError(translate(getLang(), "ocr.failed", { error: String(err) })); } finally { setOcrBusy(false); } @@ -454,9 +1069,15 @@ export default function App() { const editor = editorRef.current; if (activeId && editor) { insertAtCursor(editor, trimmed); + void api.snapshotNoteVersion({ + noteId: activeId, + contentMarkdown: editorMarkdown(editor), + source: "op", + op: "ocr", + }); } else { const note = await api.createNote(trimmed, "ocr"); - setNotes((prev) => sortNotes([note, ...prev.filter((n) => n.id !== note.id)])); + setNotes((prev) => sortNotes([note, ...prev.filter((n) => n.id !== note.id)], sortRef.current)); setActiveId(note.id); } }, @@ -497,42 +1118,58 @@ export default function App() { onChange={handleOcrFile} style={{ display: "none" }} /> - void api.listModelInfo().then(setModels)} - />
void newNote()} + selected={selected} + onClearSelection={() => setSelected(new Set())} + onPin={(n) => void handlePin(n)} + onRename={handleRename} + onDuplicate={(n) => void handleDuplicate(n)} + onArchive={(n) => void handleArchive(n)} + onExport={handleExportNote} + onTrash={(n) => void handleTrash(n)} + onRestore={(n) => void handleRestore(n)} + onDeleteForever={(n) => void handleDeleteForever(n)} + onBulkPin={() => void handleBulkPin()} + onBulkArchive={() => void handleBulkArchive()} + onBulkExport={() => void handleBulkExport()} + onBulkTrash={() => void handleBulkTrash()} + onEmptyTrash={() => void handleEmptyTrash()} />
+ void toggleDictation()} + onOcr={triggerOcr} + onFormat={() => void formatActive()} + onRead={() => void readActive()} + onExport={() => activeId && void flushSave().then(() => api.exportNote(activeId))} + onHistory={openHistory} + onDelete={() => void deleteActive()} + dictating={dictating} + formatting={formatting} + reading={reading} + hasNote={!!activeNote} + theme={theme} + onToggleTheme={toggleTheme} + onShowModels={openModels} + lang={lang} + onToggleLang={() => setLang(lang === "de" ? "en" : "de")} + /> {activeNote ? ( <> - void toggleDictation()} - onOcr={triggerOcr} - onFormat={() => void formatActive()} - onRead={() => void readActive()} - onExport={() => - activeId && void flushSave().then(() => api.exportNote(activeId)) - } - onHistory={() => - activeId && void api.listNoteEvents(activeId).then(setHistory) - } - onDelete={() => void deleteActive()} - dictating={dictating} - formatting={formatting} - reading={reading} - /> {!dictating && sources.length > 0 && (
- Diktat-Quelle + {t("dictation.source")}
)} + {!dictating && + voices.length > 0 && + (() => { + const backends = [...new Set(voices.map((v) => v.provider))]; + const backend = voices.find((v) => v.id === voiceId)?.provider ?? backends[0] ?? ""; + const backendVoices = voices.filter((v) => v.provider === backend); + return ( +
+ {t("readaloud.label")} + {backends.length > 1 && ( + + )} + + {backend === "zonos" && ( + + )} + {lastAudioKey && ( + + )} + +
+ ); + })()} {dictating && (
- Aufnahme läuft… + {t("dictation.recording")}
)} + {formatProgress && !reading && ( +
+ + + {t("format.progress", { + done: formatProgress.done, + total: formatProgress.total, + })} + +
+ )} + {reading && !speaking && ( +
+ + + {synthProgress + ? t("readaloud.prepAudio", { + done: synthProgress.done, + total: synthProgress.total, + }) + : formatProgress + ? t("readaloud.prepText", { + done: formatProgress.done, + total: formatProgress.total, + }) + : t("readaloud.preparing")} + + +
+ )} + {reading && speaking && ( +
+ + + {paused ? t("readaloud.paused") : t("readaloud.reading")} + {formatProgress + ? t("readaloud.prepSuffix", { + done: formatProgress.done, + total: formatProgress.total, + }) + : ""} + + + +
+ )} + {voiceLoading && !reading && ( +
+ + {t("readaloud.voiceLoading")} +
+ )}
patchActive({ title: e.target.value })} /> -
{noteMeta(activeNote)}
+
{noteMeta(activeNote, t)}
) : (
-
No note selected
-
Start with a note, dictate something, or paste a screenshot.
+
{t("empty.title")}
+
{t("empty.desc")}
)} @@ -600,9 +1381,10 @@ export default function App() { {dictationError}
)} - {ocrBusy &&
Texterkennung läuft…
} + {ocrBusy &&
{t("ocr.busy")}
} {ocr && ( void insertOcrText(text)} @@ -610,85 +1392,77 @@ export default function App() { /> )} {history && ( -
setHistory(null)}> -
e.stopPropagation()}> -
- Verlauf - -
- {history.length === 0 ? ( -

Noch keine Ereignisse.

- ) : ( -
    - {history.map((ev) => ( -
  • - {ev.operation ?? ev.sourceType} - - {ev.providerId && ( - {ev.providerId} - )} -
    • - ))} -
- )} -
-
+ void restoreVersion(versionId)} + onClose={() => setHistory(null)} + /> )} - {models && ( -
setModels(null)}> -
e.stopPropagation()}> -
- On-Device-Modelle - -
-
    - {models.map((m) => ( -
  • - {m.feature} - {m.displayName} - - {m.status} - - {m.runtimeLicense} -
    • - ))} -
-
-
+ {showModels && ( + setShowModels(false)} + /> )} + {showVoiceSettings && + (() => { + const activeProvider = voices.find((v) => v.id === voiceId)?.provider ?? "piper"; + return ( + { + setSpeechPrep(v); + localStorage.setItem("tts-speech-prep", v ? "1" : "0"); + }} + onPreview={() => void previewVoice()} + previewing={previewing} + onClose={() => setShowVoiceSettings(false)} + /> + ); + })()} {preview && (
setPreview(null)}>
e.stopPropagation()} role="dialog" - aria-label="Formatting preview" + aria-label={t("preview.aria")} >
- Formatierung – Vorschau -
- Original + {t("preview.original")} 
{preview.original}
- Formatiert + {t("preview.formatted")} 
{preview.formatted}
)} +
); } diff --git a/apps/desktop/src/components/ActionBar.tsx b/apps/desktop/src/components/ActionBar.tsx index ed05e14..c537ed6 100644 --- a/apps/desktop/src/components/ActionBar.tsx +++ b/apps/desktop/src/components/ActionBar.tsx @@ -1,4 +1,16 @@ -import { DictateIcon, FormatIcon, OcrIcon, ReadIcon, TrashIcon } from "./icons"; +import type { Lang } from "../lib/i18n"; +import { useI18n } from "../lib/i18n"; +import type { Theme } from "../hooks/useTheme"; +import { + DictateIcon, + FormatIcon, + GlobeIcon, + MoonIcon, + OcrIcon, + ReadIcon, + SunIcon, + TrashIcon, +} from "./icons"; interface Props { onDictate: () => void; @@ -11,11 +23,20 @@ interface Props { dictating: boolean; formatting: boolean; reading: boolean; + /** Whether a note is open; note-scoped actions are disabled without one. */ + hasNote: boolean; + // Global controls (moved here from the former top bar). + theme: Theme; + onToggleTheme: () => void; + onShowModels: () => void; + lang: Lang; + onToggleLang: () => void; } /** - * The four core actions plus delete. Dictate toggles microphone capture; the - * audio is transcribed locally by Whisper (PR 5). OCR, Format and Read are wired. + * The per-note actions on the left (dictate / OCR / format / read / export / + * history / delete) and the global controls on the right (models, language, + * theme). This is the app's only toolbar — the former top bar was folded in. */ export function ActionBar({ onDictate, @@ -28,47 +49,94 @@ export function ActionBar({ dictating, formatting, reading, + hasNote, + theme, + onToggleTheme, + onShowModels, + lang, + onToggleLang, }: Props) { + const { t } = useI18n(); return (
- - - + + + + - MD - + + +
diff --git a/apps/desktop/src/components/Editor.tsx b/apps/desktop/src/components/Editor.tsx index 6a93551..84fc225 100644 --- a/apps/desktop/src/components/Editor.tsx +++ b/apps/desktop/src/components/Editor.tsx @@ -4,6 +4,7 @@ import StarterKit from "@tiptap/starter-kit"; import { useEffect } from "react"; import { Markdown } from "tiptap-markdown"; +import { translate, useI18n } from "../lib/i18n"; import { DictationGhost } from "./dictationGhost"; interface Props { @@ -15,11 +16,12 @@ interface Props { } export function Editor({ initialMarkdown, onChange, onReady }: Props) { + const { lang } = useI18n(); const editor = useEditor({ extensions: [ StarterKit, Markdown.configure({ html: false, transformPastedText: true }), - Placeholder.configure({ placeholder: "Start writing, or capture something…" }), + Placeholder.configure({ placeholder: translate(lang, "editor.placeholder") }), DictationGhost, ], content: initialMarkdown, @@ -36,6 +38,18 @@ export function Editor({ initialMarkdown, onChange, onReady }: Props) { if (editor && onReady) onReady(editor); }, [editor, onReady]); + // Keep the placeholder in the active language without remounting the editor: + // update the Placeholder extension's option and nudge ProseMirror to recompute + // its decorations. Defensive about the extension internals. + useEffect(() => { + if (!editor) return; + const ext = editor.extensionManager.extensions.find((e) => e.name === "placeholder"); + if (ext) { + (ext.options as { placeholder: string }).placeholder = translate(lang, "editor.placeholder"); + editor.view.dispatch(editor.state.tr); + } + }, [editor, lang]); + return ; } diff --git a/apps/desktop/src/components/HistoryOverlay.tsx b/apps/desktop/src/components/HistoryOverlay.tsx new file mode 100644 index 0000000..7456f3e --- /dev/null +++ b/apps/desktop/src/components/HistoryOverlay.tsx @@ -0,0 +1,160 @@ +// Edit-history overlay (design "Bereich 1", wireframe F): a version timeline on +// the left (current state + stored snapshots with op badges and +x/−y word +// counts) and a word-level diff of the selected version against the current +// content on the right, with a non-destructive "restore this version" action. + +import { useMemo, useState } from "react"; + +import { relativeTime } from "../lib/datetime"; +import { diffStats, diffWords } from "../lib/diff"; +import { useI18n } from "../lib/i18n"; +import type { TranslationKey } from "../lib/i18n"; +import type { NoteVersion } from "../lib/types"; +import { ClockIcon, HistoryIcon, RestoreIcon } from "./icons"; + +interface Props { + versions: NoteVersion[]; + currentContent: string; + noteTitle: string; + onRestore: (versionId: string) => void; + onClose: () => void; +} + +/** The i18n key for a version's operation badge. */ +function opKey(v: NoteVersion): TranslationKey { + if (v.source === "manual") return "history.op.manual"; + switch (v.op) { + case "format": + return "history.op.format"; + case "ocr": + return "history.op.ocr"; + case "dictation": + return "history.op.dictation"; + case "restore": + return "history.op.restore"; + default: + return "history.op.snapshot"; + } +} + +export function HistoryOverlay({ versions, currentContent, noteTitle, onRestore, onClose }: Props) { + const { t } = useI18n(); + // The selected version (defaults to the most recent). `null` = the current + // state node is selected → nothing to diff/restore. + const [selectedId, setSelectedId] = useState(versions[0]?.id ?? null); + const selected = versions.find((v) => v.id === selectedId) ?? null; + + const segments = useMemo( + () => (selected ? diffWords(selected.contentMarkdown, currentContent) : []), + [selected, currentContent], + ); + + return ( +
+
e.stopPropagation()} + > + {/* Timeline */} +
+
+
+ + {t("history.title")} +
+
+ {t("history.changesOnly")} · {t("history.versions", { count: versions.length })} +
+
+
+ + {versions.map((v, i) => { + const prev = versions[i + 1]?.contentMarkdown ?? ""; + const { added, removed } = diffStats(prev, v.contentMarkdown); + const isLast = i === versions.length - 1; + return ( + + ); + })} +
+
+ + {/* Diff view */} +
+
+ {t("history.compare")} + + {t("history.thisVersion")} + → + {t("history.currentState")} + + {t("history.readOnly")} +
+
+
{noteTitle}
+ {selected ? ( +

+ {segments.map((seg, i) => ( + + {seg.text} + + ))} +

+ ) : ( +

{t("history.selectVersion")}

+ )} +
+
+ + + + {t("history.restoreHint")} + +
+
+
+
+ ); +} diff --git a/apps/desktop/src/components/ModelManager.tsx b/apps/desktop/src/components/ModelManager.tsx new file mode 100644 index 0000000..fed7653 --- /dev/null +++ b/apps/desktop/src/components/ModelManager.tsx @@ -0,0 +1,138 @@ +// Model manager window: install / delete the catalog of TTS voices + runtimes, +// with per-entry license + tier badges and download progress. Restrictive +// entries (non-commercial) are gated behind a confirm in the parent handler. + +import { useI18n } from "../lib/i18n"; +import type { TranslationKey } from "../lib/i18n"; +import type { CatalogItem, ModelInfo, ModelProgress } from "../lib/types"; + +interface Props { + items: CatalogItem[]; + providers: ModelInfo[]; + /** Live download progress keyed by model id. */ + progress: Record; + /** The id currently installing (disables its buttons), or null. */ + busyId: string | null; + onInstall: (item: CatalogItem) => void; + onDelete: (item: CatalogItem) => void; + onClose: () => void; +} + +const TIER_KEY: Record = { + bundled: "models.tier.bundled", + download: "models.tier.download", + gated: "models.tier.gated", +}; + +function fmtBytes(n: number): string { + if (!n) return ""; + const mb = n / (1024 * 1024); + return mb >= 1024 ? `${(mb / 1024).toFixed(1)} GB` : `${Math.round(mb)} MB`; +} + +export function ModelManager({ + items, + providers, + progress, + busyId, + onInstall, + onDelete, + onClose, +}: Props) { + const { t } = useI18n(); + return ( +
+
e.stopPropagation()} + > +
+ {t("models.title")} + +
+ +
    + {items.map((m) => { + const p = progress[m.id]; + const pct = p && p.total ? Math.round((p.downloaded / p.total) * 100) : null; + const busy = busyId === m.id; + return ( +
  • +
    + {m.displayName} + + {m.language} + + {TIER_KEY[m.tier] ? t(TIER_KEY[m.tier]) : m.tier} + + {m.license} + {!m.commercialOk && ( + {t("models.nonCommercial")} + )} + {m.installed && m.installedBytes > 0 && ( + {fmtBytes(m.installedBytes)} + )} + + {m.notes && {m.notes}} + {busy && ( + + + + {pct != null ? `${pct}%` : t("models.loading")} + + + )} +
    +
    + {m.setup && !m.installed ? ( + + {t("models.setup")} {m.setup} + + ) : m.installed ? ( + <> + {t("models.installed")} + {m.tier !== "bundled" && !m.setup && ( + + )} + + ) : ( + + )} +
    +
    • + ); + })} +
+ + {providers.length > 0 && ( + <> +

{t("models.activeProviders")}

+
    + {providers.map((pr) => ( +
  • + + {pr.feature}: {pr.displayName} + + + + {pr.status} + + {pr.runtimeLicense} + +
    • + ))} +
+ + )} +
+
+ ); +} diff --git a/apps/desktop/src/components/OcrOverlay.tsx b/apps/desktop/src/components/OcrOverlay.tsx index 7e51220..6de5a94 100644 --- a/apps/desktop/src/components/OcrOverlay.tsx +++ b/apps/desktop/src/components/OcrOverlay.tsx @@ -1,5 +1,6 @@ -import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; +import { useI18n } from "../lib/i18n"; import type { OcrLayout } from "../lib/types"; interface Props { @@ -11,22 +12,31 @@ interface Props { onClose: () => void; } -/** The current selection within the overlay, or the full layout text if none. */ -function selectionOrAll(layout: OcrLayout): string { - const selection = window.getSelection()?.toString().trim(); - return selection || layout.text; -} - /** * A Snipping-Tool-style OCR result: the image with a transparent, selectable * text layer positioned per recognized word. Select and copy (Ctrl+C) or push - * the text into the note; with no selection the whole recognized text is used. + * the text into the note; with no selection (or "Select all") the whole + * recognized text is used. + * + * The word boxes are absolutely positioned, so a raw `Selection.toString()` + * mashes them together without spaces. To make selection actually useful, the + * selected text is reconstructed from the underlying OCR words (`layout.words`), + * re-inserting spaces between words and newlines between rows — and a full + * selection falls back to the layout-preserving `layout.text`. */ export function OcrOverlay({ imageUrl, layout, onInsert, onClose }: Props) { + const { t } = useI18n(); const imgRef = useRef(null); + const layerRef = useRef(null); + const plainRef = useRef(null); + // One DOM node per recognized word, so we can map a DOM selection back to the + // OCR words it covers. + const wordRefs = useRef<(HTMLSpanElement | null)[]>([]); // Rendered pixels per OCR pixel, so the word boxes line up with the image. const [scale, setScale] = useState(0); + const hasBoxes = layout.words.length > 0 && layout.width > 0 && scale > 0; + useLayoutEffect(() => { const img = imgRef.current; if (!img || !layout.width) return; @@ -37,73 +47,121 @@ export function OcrOverlay({ imageUrl, layout, onInsert, onClose }: Props) { return () => observer.disconnect(); }, [layout.width]); - // Ctrl+C with no selection copies the whole text; Escape closes. + // The current selection as clean text: reconstructed from the covered OCR + // words (spaces between words, newlines between rows), or the full layout text + // when nothing — or everything — is selected. + const gatherText = useCallback((): string => { + const sel = window.getSelection(); + if (!hasBoxes) { + const s = sel?.toString().trim(); + return s || layout.text; + } + if (!sel || sel.isCollapsed || sel.rangeCount === 0) return layout.text; + const picked: number[] = []; + wordRefs.current.forEach((el, i) => { + if (el && sel.containsNode(el, true)) picked.push(i); + }); + if (picked.length === 0) return layout.text; + if (picked.length === layout.words.length) return layout.text; // full → layout text + + let out = ""; + let prev: OcrLayout["words"][number] | null = null; + for (const i of picked) { + const w = layout.words[i]; + if (prev) out += w.y > prev.y + prev.height * 0.6 ? "\n" : " "; + out += w.text; + prev = w; + } + return out; + }, [hasBoxes, layout]); + + // Select the whole recognized text (the word layer, or the plain fallback). + const selectAll = useCallback(() => { + const node = hasBoxes ? layerRef.current : plainRef.current; + const sel = window.getSelection(); + if (!node || !sel) return; + const range = document.createRange(); + range.selectNodeContents(node); + sel.removeAllRanges(); + sel.addRange(range); + }, [hasBoxes]); + + // Escape closes; Ctrl+A selects all; Ctrl+C copies the reconstructed text. useEffect(() => { const onKey = (e: KeyboardEvent) => { if (e.key === "Escape") { onClose(); return; } - if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "c") { - if (!window.getSelection()?.toString()) { - e.preventDefault(); - void navigator.clipboard.writeText(layout.text); - } + const mod = e.ctrlKey || e.metaKey; + if (mod && e.key.toLowerCase() === "a") { + e.preventDefault(); + selectAll(); + } else if (mod && e.key.toLowerCase() === "c") { + e.preventDefault(); + void navigator.clipboard.writeText(gatherText()); } }; window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); - }, [layout.text, onClose]); + }, [onClose, selectAll, gatherText]); // preventDefault on mousedown keeps the text selection from being cleared when // a toolbar button takes focus, so the buttons can act on the selection. const keepSelection = (e: React.MouseEvent) => e.preventDefault(); - const hasBoxes = layout.words.length > 0 && layout.width > 0 && scale > 0; return (
- Texterkennung + {t("ocr.title")} - +
-
- Text markieren und kopieren oder „In Notiz“ übernehmen — ohne Auswahl wird der gesamte - erkannte Text genutzt. -
+
{t("ocr.hint")}
Erkanntes Bild {hasBoxes && ( -
+
{layout.words.map((word, i) => ( { + wordRefs.current[i] = el; + }} className="ocr-overlay__word" style={{ left: word.x * scale, @@ -119,7 +177,11 @@ export function OcrOverlay({ imageUrl, layout, onInsert, onClose }: Props) {
)}
- {!hasBoxes && 
{layout.text}
} + {!hasBoxes && ( + 
+              {layout.text}
+
+ )}
diff --git a/apps/desktop/src/components/ReadAloudSettings.tsx b/apps/desktop/src/components/ReadAloudSettings.tsx new file mode 100644 index 0000000..e895bcd --- /dev/null +++ b/apps/desktop/src/components/ReadAloudSettings.tsx @@ -0,0 +1,205 @@ +// Read-aloud synthesis settings dialog. Speed applies to every backend; the +// other knobs are backend-specific and shown only for their backend: Piper gets +// expressiveness/cadence/sentence-silence, Zonos gets intonation/brightness. A +// "Probe" button synthesizes one short sentence with the current settings for +// quick A/B tuning. + +import { useI18n } from "../lib/i18n"; +import type { TranslationKey } from "../lib/i18n"; +import type { TtsTuning } from "../lib/types"; + +/** Model defaults, mirrored from the Rust providers (Piper CLI defaults; Zonos + * sidecar defaults for intonation/brightness; "neutral" emotion). */ +export const TTS_DEFAULTS: Required = { + speed: 1.0, + expressiveness: 0.667, + cadence: 0.8, + sentenceSilence: 0.2, + intonation: 42, + brightness: 22050, + emotion: "neutral", +}; + +/** The numeric tuning knobs — the ones a slider can drive (excludes `emotion`, + * which is a preset key shown as a dropdown next to the voice picker). */ +type SliderKey = { + [K in keyof TtsTuning]-?: NonNullable extends number ? K : never; +}[keyof TtsTuning]; + +interface SliderDef { + key: SliderKey; + labelKey: TranslationKey; + hintKey: TranslationKey; + min: number; + max: number; + step: number; + /** Render the live value (defaults to two decimals). */ + format?: (value: number) => string; + /** Only meaningful for Piper; hidden for the other backends. */ + piperOnly?: boolean; + /** Only meaningful for Zonos; hidden for the other backends. */ + zonosOnly?: boolean; +} + +const SLIDERS: SliderDef[] = [ + { + key: "speed", + labelKey: "slider.speed", + hintKey: "slider.speed.hint", + min: 0.5, + max: 1.5, + step: 0.05, + }, + { + key: "expressiveness", + labelKey: "slider.expressiveness", + hintKey: "slider.expressiveness.hint", + min: 0, + max: 1, + step: 0.05, + piperOnly: true, + }, + { + key: "cadence", + labelKey: "slider.cadence", + hintKey: "slider.cadence.hint", + min: 0, + max: 1.5, + step: 0.05, + piperOnly: true, + }, + { + key: "sentenceSilence", + labelKey: "slider.sentenceSilence", + hintKey: "slider.sentenceSilence.hint", + min: 0, + max: 1, + step: 0.05, + piperOnly: true, + }, + { + key: "intonation", + labelKey: "slider.intonation", + hintKey: "slider.intonation.hint", + min: 0, + max: 100, + step: 1, + format: (v) => v.toFixed(0), + zonosOnly: true, + }, + { + key: "brightness", + labelKey: "slider.brightness", + hintKey: "slider.brightness.hint", + min: 12000, + max: 22050, + step: 50, + format: (v) => `${(v / 1000).toFixed(1)} kHz`, + zonosOnly: true, + }, +]; + +interface Props { + /** Label of the active backend (Piper / XTTS / Zonos), shown in the header. */ + backendLabel: string; + /** Whether the active voice is a Piper voice (shows the Piper-only knobs). */ + isPiper: boolean; + /** Whether the active voice is a Zonos voice (shows the Zonos-only knobs). */ + isZonos: boolean; + tuning: TtsTuning; + onChange: (tuning: TtsTuning) => void; + /** Whether to run an LLM "prepare for speech" pass before reading. */ + speechPrep: boolean; + onSpeechPrepChange: (value: boolean) => void; + /** Synthesize + play one short test sentence with the current settings. */ + onPreview: () => void; + /** Whether a preview is currently rendering (disables the button). */ + previewing: boolean; + onClose: () => void; +} + +export function ReadAloudSettings({ + backendLabel, + isPiper, + isZonos, + tuning, + onChange, + speechPrep, + onSpeechPrepChange, + onPreview, + previewing, + onClose, +}: Props) { + const { t } = useI18n(); + const sliders = SLIDERS.filter( + (s) => (!s.piperOnly || isPiper) && (!s.zonosOnly || isZonos), + ); + return ( +
+
e.stopPropagation()} + > +
+ {t("raSettings.title", { backend: backendLabel })} + +
+
+ {sliders.map((s) => { + const value = tuning[s.key] ?? TTS_DEFAULTS[s.key]; + return ( + + ); + })} + {!isPiper && !isZonos && ( +

+ {t("raSettings.onlySpeed", { backend: backendLabel })} +

+ )} +
+ +
+ + +
+
+
+ ); +} diff --git a/apps/desktop/src/components/RegionOverlay.tsx b/apps/desktop/src/components/RegionOverlay.tsx index 01ccaf9..732c2b7 100644 --- a/apps/desktop/src/components/RegionOverlay.tsx +++ b/apps/desktop/src/components/RegionOverlay.tsx @@ -10,6 +10,7 @@ import { getCurrentWindow } from "@tauri-apps/api/window"; import { useCallback, useEffect, useRef, useState } from "react"; import { cancelRegionOcr, getRegionCapture, ocrRegion } from "../lib/api"; +import { useI18n } from "../lib/i18n"; interface Rect { x: number; @@ -28,6 +29,7 @@ function rectFrom(ax: number, ay: number, bx: number, by: number): Rect { } export function RegionOverlay() { + const { t } = useI18n(); const [imageUrl, setImageUrl] = useState(null); const [rect, setRect] = useState(null); const [busy, setBusy] = useState(false); @@ -153,7 +155,7 @@ export function RegionOverlay() { pointerEvents: "none", }} > - Bereich aufziehen · Esc bricht ab + {t("region.hint")}
)} diff --git a/apps/desktop/src/components/Sidebar.tsx b/apps/desktop/src/components/Sidebar.tsx index 868f09a..c4808ab 100644 --- a/apps/desktop/src/components/Sidebar.tsx +++ b/apps/desktop/src/components/Sidebar.tsx @@ -1,53 +1,592 @@ -import type { Note } from "../lib/types"; -import { PlusIcon, SearchIcon } from "./icons"; +// Notes sidebar (design "Bereich 1"): scope tabs (Active / Archived / Trash), +// a pinned group, sort, per-note hover + context-menu actions, multi-select with +// a bulk action bar, and the archive/trash views with restore / delete-forever. + +import { type MouseEvent, useEffect, useRef, useState } from "react"; + +import { useI18n } from "../lib/i18n"; +import { relativeTime, daysUntilPurge } from "../lib/datetime"; +import type { Note, NoteScope, NoteSort } from "../lib/types"; +import { + ArchiveIcon, + CheckIcon, + ChevronDownIcon, + DuplicateIcon, + ExportIcon, + PinIcon, + PlusIcon, + RenameIcon, + RestoreIcon, + SearchIcon, + TrashIcon, +} from "./icons"; +import { LogoMark } from "./Logo"; interface Props { notes: Note[]; + scope: NoteScope; + onScopeChange: (scope: NoteScope) => void; + sort: NoteSort; + onSortChange: (sort: NoteSort) => void; activeId: string | null; query: string; onQueryChange: (q: string) => void; - onSelect: (id: string) => void; + /** Open a note (plain click). The event carries Ctrl/⌘/Shift for selection. */ + onSelect: (id: string, e: MouseEvent) => void; onNewNote: () => void; + /** Currently multi-selected note ids. Non-empty → selection mode. */ + selected: Set; + onClearSelection: () => void; + // Per-note actions. + onPin: (note: Note) => void; + onRename: (note: Note) => void; + onDuplicate: (note: Note) => void; + onArchive: (note: Note) => void; + onExport: (note: Note) => void; + onTrash: (note: Note) => void; + onRestore: (note: Note) => void; + onDeleteForever: (note: Note) => void; + // Bulk actions (operate on `selected`). + onBulkPin: () => void; + onBulkArchive: () => void; + onBulkExport: () => void; + onBulkTrash: () => void; + onEmptyTrash: () => void; } -function preview(note: Note): string { +const SCOPES: NoteScope[] = ["active", "archived", "trash"]; +const SORTS: NoteSort[] = ["modified", "created", "title"]; + +function preview(note: Note, emptyLabel: string): string { const lines = note.contentMarkdown .split("\n") .map((line) => line.replace(/^#+\s*/, "").replace(/[*`_>]/g, "").trim()); - // The first meaningful line is usually the title; show the next one. - return lines.slice(1).find(Boolean) ?? lines.find(Boolean) ?? "Empty note"; + return lines.slice(1).find(Boolean) ?? lines.find(Boolean) ?? emptyLabel; } -export function Sidebar({ notes, activeId, query, onQueryChange, onSelect, onNewNote }: Props) { +export function Sidebar(props: Props) { + const { notes, scope, sort, activeId, query, selected, onSelect } = props; + const { t } = useI18n(); + const emptyLabel = t("note.emptyPreview"); + const selectionMode = selected.size > 0; + + // The open context menu: which note + screen position. + const [menu, setMenu] = useState<{ note: Note; x: number; y: number } | null>(null); + // Inline sort dropdown open state. + const [sortOpen, setSortOpen] = useState(false); + + const openMenu = (note: Note, e: MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + setMenu({ note, x: e.clientX, y: e.clientY }); + }; + + const pinned = notes.filter((n) => n.pinned); + const rest = notes.filter((n) => !n.pinned); + + const countLabel = + scope === "archived" + ? t("sidebar.archivedCount", { count: notes.length }) + : scope === "trash" + ? t("sidebar.trashCount", { count: notes.length }) + : t("sidebar.count", { count: notes.length }); + return ( ); } + +interface RowProps { + note: Note; + active: boolean; + selected: boolean; + selectionMode: boolean; + emptyLabel: string; + onClick: (e: MouseEvent) => void; + onPin: () => void; + onMenu: (e: MouseEvent) => void; +} + +function NoteRow({ + note, + active, + selected, + selectionMode, + emptyLabel, + onClick, + onPin, + onMenu, +}: RowProps) { + return ( +
+ {selectionMode && ( + + {selected && } + + )} +
+
{note.title}
+
{preview(note, emptyLabel)}
+
+ {!selectionMode && ( +
+ + +
+ )} + {note.pinned && selectionMode && ( + + )} +
+ ); +} + +function ArchiveRow({ + note, + onRestore, + onTrash, + onClick, + selected, + selectionMode, +}: { + note: Note; + onRestore: () => void; + onTrash: () => void; + onClick: (e: MouseEvent) => void; + selected: boolean; + selectionMode: boolean; +}) { + const { t } = useI18n(); + return ( +
+
+ {selectionMode ? ( + + {selected && } + + ) : ( + + )} + {note.title} +
+
{t("archive.archivedAgo", { when: relativeTime(note.updatedAt, t) })}
+
+ + +
+
+ ); +} + +function TrashRow({ + note, + onRestore, + onDeleteForever, +}: { + note: Note; + onRestore: () => void; + onDeleteForever: () => void; +}) { + const { t } = useI18n(); + const days = note.deletedAt ? daysUntilPurge(note.deletedAt) : 0; + return ( +
+
{note.title}
+
+ {note.deletedAt && relativeTime(note.deletedAt, t) + ? t("trash.deletedAgo", { when: relativeTime(note.deletedAt, t) }) + : ""} + {" · "} + {t("trash.daysLeft", { count: days })} +
+
+ + +
+
+ ); +} + +function BulkBar({ + count, + scope, + onCancel, + onPin, + onArchive, + onExport, + onTrash, +}: { + count: number; + scope: NoteScope; + onCancel: () => void; + onPin: () => void; + onArchive: () => void; + onExport: () => void; + onTrash: () => void; +}) { + const { t } = useI18n(); + return ( +
+
+ {t("select.count", { count })} + +
+
+ {scope === "active" && ( + + )} + {scope === "active" && ( + + )} + + +
+
+ ); +} + +function NoteContextMenu({ + note, + x, + y, + onClose, + onPin, + onRename, + onDuplicate, + onArchive, + onExport, + onTrash, +}: { + note: Note; + x: number; + y: number; + onClose: () => void; + onPin: (n: Note) => void; + onRename: (n: Note) => void; + onDuplicate: (n: Note) => void; + onArchive: (n: Note) => void; + onExport: (n: Note) => void; + onTrash: (n: Note) => void; +}) { + const ref = useRef(null); + // Keep the menu on-screen: clamp to the viewport once we know its size. + const [pos, setPos] = useState({ x, y }); + useEffect(() => { + const el = ref.current; + if (!el) return; + const r = el.getBoundingClientRect(); + setPos({ + x: Math.min(x, window.innerWidth - r.width - 8), + y: Math.min(y, window.innerHeight - r.height - 8), + }); + }, [x, y]); + useEffect(() => { + const onKey = (e: KeyboardEvent) => e.key === "Escape" && onClose(); + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, [onClose]); + + const { t } = useI18n(); + const run = (fn: () => void) => () => { + fn(); + onClose(); + }; + + return ( + <> +
e.preventDefault()} /> +
+ + + + + +
+ +
+ + ); +} diff --git a/apps/desktop/src/components/Statusbar.tsx b/apps/desktop/src/components/Statusbar.tsx index 7c9a379..a63dd1a 100644 --- a/apps/desktop/src/components/Statusbar.tsx +++ b/apps/desktop/src/components/Statusbar.tsx @@ -1,3 +1,4 @@ +import { useI18n } from "../lib/i18n"; import type { Note } from "../lib/types"; interface Props { @@ -11,12 +12,15 @@ function countWords(markdown: string): number { } export function Statusbar({ note, saved }: Props) { + const { t } = useI18n(); return (
de-DE · en-US - {saved ? "SAVED" : "SAVING…"} + {saved ? t("status.saved") : t("status.saving")} LOCAL · ON-DEVICE - {note ? countWords(note.contentMarkdown) : 0} WORDS + + {t("status.words", { count: note ? countWords(note.contentMarkdown) : 0 })} +
); } diff --git a/apps/desktop/src/components/Toasts.tsx b/apps/desktop/src/components/Toasts.tsx new file mode 100644 index 0000000..42851aa --- /dev/null +++ b/apps/desktop/src/components/Toasts.tsx @@ -0,0 +1,130 @@ +// Undo-toast queue — the feedback channel for reversible/destructive actions +// (trash, archive, restore, bulk ops), replacing modal confirm()/alert(). Each +// toast auto-dismisses after a few seconds; an action toast carries an "Undo" +// button and can also be triggered with Ctrl/⌘+Z while not typing in the editor. + +import { type ReactNode, useCallback, useEffect, useRef, useState } from "react"; + +export interface Toast { + id: number; + message: string; + /** Label for the action button (e.g. "Rückgängig"); omit for a plain toast. */ + actionLabel?: string; + /** Run on click / Ctrl+Z; its presence makes the toast an "undo" toast. */ + onAction?: () => void; + icon?: ReactNode; +} + +export interface ToastOptions { + actionLabel?: string; + onAction?: () => void; + icon?: ReactNode; + /** Auto-dismiss delay in ms (default 6000). */ + duration?: number; +} + +export interface ToastApi { + toasts: Toast[]; + push: (message: string, opts?: ToastOptions) => number; + dismiss: (id: number) => void; + /** Run a toast's action and dismiss it (the "Undo" button handler). */ + runAction: (toast: Toast) => void; +} + +/** Toast queue with auto-dismiss timers and a window-level undo shortcut. */ +export function useToasts(): ToastApi { + const [toasts, setToasts] = useState([]); + const timers = useRef>(new Map()); + const idRef = useRef(0); + + const dismiss = useCallback((id: number) => { + setToasts((ts) => ts.filter((t) => t.id !== id)); + const timer = timers.current.get(id); + if (timer) { + clearTimeout(timer); + timers.current.delete(id); + } + }, []); + + const push = useCallback( + (message: string, opts: ToastOptions = {}) => { + const id = (idRef.current += 1); + setToasts((ts) => [ + ...ts, + { id, message, actionLabel: opts.actionLabel, onAction: opts.onAction, icon: opts.icon }, + ]); + const timer = window.setTimeout(() => dismiss(id), opts.duration ?? 6000); + timers.current.set(id, timer); + return id; + }, + [dismiss], + ); + + const runAction = useCallback( + (toast: Toast) => { + toast.onAction?.(); + dismiss(toast.id); + }, + [dismiss], + ); + + // Ctrl/⌘+Z triggers the newest undoable toast — but only when the user isn't + // typing in the editor or an input, where the same chord means text-undo. + const toastsRef = useRef(toasts); + toastsRef.current = toasts; + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if (!(e.ctrlKey || e.metaKey) || e.shiftKey || e.key.toLowerCase() !== "z") return; + const target = e.target as HTMLElement | null; + if ( + target && + (target.isContentEditable || /^(input|textarea|select)$/i.test(target.tagName)) + ) { + return; + } + const undoable = [...toastsRef.current].reverse().find((t) => t.onAction); + if (!undoable) return; + e.preventDefault(); + runAction(undoable); + }; + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, [runAction]); + + // Clear pending timers on unmount. + useEffect(() => { + const map = timers.current; + return () => map.forEach((t) => clearTimeout(t)); + }, []); + + return { toasts, push, dismiss, runAction }; +} + +interface StackProps { + toasts: Toast[]; + onAction: (toast: Toast) => void; + onDismiss: (id: number) => void; +} + +/** Bottom-center stack of toasts. */ +export function ToastStack({ toasts, onAction, onDismiss }: StackProps) { + if (toasts.length === 0) return null; + return ( +
+ {toasts.map((t) => ( +
+ {t.icon && {t.icon}} + {t.message} + {t.onAction && ( + + )} + +
+ ))} +
+ ); +} diff --git a/apps/desktop/src/components/Topbar.tsx b/apps/desktop/src/components/Topbar.tsx deleted file mode 100644 index 39d66f0..0000000 --- a/apps/desktop/src/components/Topbar.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import type { Theme } from "../hooks/useTheme"; -import { LogoMark } from "./Logo"; -import { MoonIcon, SunIcon } from "./icons"; - -interface Props { - theme: Theme; - onToggleTheme: () => void; - onShowModels: () => void; -} - -export function Topbar({ theme, onToggleTheme, onShowModels }: Props) { - return ( -
- - exoquill -
- - - ON-DEVICE - - - -
- ); -} diff --git a/apps/desktop/src/components/icons.tsx b/apps/desktop/src/components/icons.tsx index 33a7a9b..9e9518a 100644 --- a/apps/desktop/src/components/icons.tsx +++ b/apps/desktop/src/components/icons.tsx @@ -87,3 +87,91 @@ export const MoonIcon = (p: Props) => ( ); + +export const GlobeIcon = (p: Props) => ( + + + + + +); + +// Pin (quill nib). Pass `fill="currentColor"` for the filled/pinned state. +export const PinIcon = (p: Props) => ( + + + +); + +export const ArchiveIcon = (p: Props) => ( + + + + + +); + +// Counter-clockwise arrow — restore from archive/trash, and version restore. +export const RestoreIcon = (p: Props) => ( + + + + +); + +export const DuplicateIcon = (p: Props) => ( + + + + +); + +export const RenameIcon = (p: Props) => ( + + + + +); + +// Arrow down to a line — export/download. +export const ExportIcon = (p: Props) => ( + + + + + +); + +export const CheckIcon = (p: Props) => ( + + + +); + +// Clock with a rewind arrow — edit history. +export const HistoryIcon = (p: Props) => ( + + + + + +); + +export const ClockIcon = (p: Props) => ( + + + + +); + +export const ChevronDownIcon = (p: Props) => ( + + + +); + +export const ChevronRightIcon = (p: Props) => ( + + + +); diff --git a/apps/desktop/src/lib/api.ts b/apps/desktop/src/lib/api.ts index 8db278e..6fb9ef3 100644 --- a/apps/desktop/src/lib/api.ts +++ b/apps/desktop/src/lib/api.ts @@ -1,22 +1,32 @@ // Typed wrappers around the Tauri notes commands (see src-tauri/src/notes.rs). import { invoke } from "@tauri-apps/api/core"; +import { emotionVector } from "./tts"; import type { CaptureSource, + CatalogItem, Job, ModelInfo, + NewNoteVersion, Note, NoteEvent, + NoteScope, + NoteSort, NoteSource, NoteUpdate, + NoteVersion, OcrLayout, RegionCapture, RegionOcr, TtsResponse, + TtsTuning, + TtsVoice, } from "./types"; -export function listNotes(): Promise { - return invoke("list_notes"); +/** List notes in a scope (default "active"), ordered by `sort` (default + * "modified"); pinned notes always come first. */ +export function listNotes(scope: NoteScope = "active", sort: NoteSort = "modified"): Promise { + return invoke("list_notes", { scope, sort }); } export function getNote(id: string): Promise { @@ -35,12 +45,34 @@ export function updateNote(id: string, update: NoteUpdate): Promise return invoke("update_note", { id, update }); } +/** Move a note to the trash (soft-delete). */ export function deleteNote(id: string): Promise { return invoke("delete_note", { id }); } -export function searchNotes(query: string): Promise { - return invoke("search_notes", { query }); +/** Restore a trashed note back to Active. */ +export function restoreNote(id: string): Promise { + return invoke("restore_note", { id }); +} + +/** Archive or un-archive a live note. */ +export function setArchived(id: string, archived: boolean): Promise { + return invoke("set_archived", { id, archived }); +} + +/** Permanently delete a note (and its events + versions). No undo. */ +export function hardDeleteNote(id: string): Promise { + return invoke("hard_delete_note", { id }); +} + +/** Permanently delete trashed notes deleted before `before` (RFC-3339). Returns + * the count removed. The caller computes the cutoff (e.g. now − 30 days). */ +export function purgeTrash(before: string): Promise { + return invoke("purge_trash", { before }); +} + +export function searchNotes(query: string, scope: NoteScope = "active"): Promise { + return invoke("search_notes", { query, scope }); } /** The recorded events for a note (format/OCR history), most recent first. */ @@ -48,6 +80,22 @@ export function listNoteEvents(noteId: string): Promise { return invoke("list_note_events", { noteId }); } +/** Record a content snapshot for the edit history (deduped by content hash). + * Resolves to the stored version, or `null` if it was a no-op duplicate. */ +export function snapshotNoteVersion(version: NewNoteVersion): Promise { + return invoke("snapshot_note_version", { version }); +} + +/** A note's edit-history versions (diff timeline), most recent first. */ +export function listNoteHistory(noteId: string): Promise { + return invoke("list_note_history", { noteId }); +} + +/** Restore a stored version's content into the note as a new, undoable change. */ +export function restoreNoteVersion(noteId: string, versionId: string): Promise { + return invoke("restore_note_version", { noteId, versionId }); +} + /** Export a note's Markdown via a native save dialog. Resolves to the saved * path, or `null` if the user cancelled. */ export function exportNote(id: string): Promise { @@ -83,6 +131,24 @@ export function formatText(text: string, instruction?: string): Promise return invoke("format_text", { text, instruction: instruction ?? null }); } +/** Begin a read-aloud session: installs a fresh cancel token so a later + * `cancelRead` can stop the speech-prep generation mid-flight. */ +export function beginRead(): Promise { + return invoke("begin_read"); +} + +/** Cancel the in-progress read-aloud speech-prep (stops the streaming llama + * generation promptly instead of letting the current chunk run to completion). */ +export function cancelRead(): Promise { + return invoke("cancel_read"); +} + +/** Rewrite one chunk of a note into clean, speakable prose for read-aloud, under + * the current read session's cancel token. */ +export function prepareSpeech(text: string): Promise { + return invoke("prepare_speech", { text }); +} + /** The frozen screenshot for the region-OCR overlay to display (PNG data URL). */ export function getRegionCapture(): Promise { return invoke("get_region_capture"); @@ -139,12 +205,79 @@ export function listCaptureSources(): Promise { return invoke("list_capture_sources"); } -/** Synthesize speech via the local TTS provider; rejects if none is available. */ -export function ttsSpeak(text: string): Promise { - return invoke("tts_speak", { text }); +/** Synthesize speech via the local TTS provider; rejects if none is available. + * `voiceId` picks a voice (see `listTtsVoices`); `provider` routes to that + * voice's backend (`"piper"` | `"xtts"`); `tuning` overrides the synthesis + * knobs. Omitted values fall back to the provider/model defaults. */ +export function ttsSpeak( + text: string, + voiceId?: string, + provider?: string, + tuning: TtsTuning = {}, +): Promise { + return invoke("tts_speak", { + text, + voiceId: voiceId ?? null, + provider: provider ?? null, + speed: tuning.speed ?? null, + expressiveness: tuning.expressiveness ?? null, + cadence: tuning.cadence ?? null, + sentenceSilence: tuning.sentenceSilence ?? null, + intonation: tuning.intonation ?? null, + brightness: tuning.brightness ?? null, + emotion: emotionVector(tuning.emotion) ?? null, + }); +} + +/** Save the read-aloud audio to a WAV file via a native save dialog. `segments` + * are the base64 PCM slices (`TtsResponse.pcm`), joined under one RIFF header at + * `sampleRate`. Resolves to the saved path, or `null` if the user cancelled. + * WebView2 can't trigger a browser download, so the file is written natively + * (like `exportNote`). */ +export function exportAudio( + segments: string[], + sampleRate: number, + suggestedName: string, +): Promise { + return invoke("export_audio", { segments, sampleRate, suggestedName }); +} + +/** The read-aloud voices the local TTS provider offers (empty when none). */ +export function listTtsVoices(): Promise { + return invoke("list_tts_voices"); +} + +/** Warm up a TTS backend's sidecar in the background (idempotent). Call when a + * backend becomes active so only it loads — never both at launch. Returns at + * once; synthesis falls back to Piper until the sidecar is ready. */ +export function warmTts(provider: string): Promise { + return invoke("warm_tts", { provider }); +} + +/** Resolve once `provider`'s sidecar is warm (model loaded), starting its warm-up + * if needed; rejects on an unconfigured backend, warm-up failure, or timeout. + * Piper resolves at once. Lets read-aloud wait for a cold voice and then start + * automatically, instead of asking the user to click play again. */ +export function ensureTtsReady(provider: string): Promise { + return invoke("ensure_tts_ready", { provider }); } /** The resolved on-device AI providers with license + status (settings view). */ export function listModelInfo(): Promise { return invoke("list_model_info"); } + +/** The installable model/voice catalog with on-disk status (model manager). */ +export function listCatalog(): Promise { + return invoke("list_catalog"); +} + +/** Download + install a catalog entry's files; emits `model_progress` events. */ +export function installModel(id: string): Promise { + return invoke("install_model", { id }); +} + +/** Delete a downloaded entry's files, freeing disk. */ +export function deleteModel(id: string): Promise { + return invoke("delete_model", { id }); +} diff --git a/apps/desktop/src/lib/audio.ts b/apps/desktop/src/lib/audio.ts index c70200a..7b22638 100644 --- a/apps/desktop/src/lib/audio.ts +++ b/apps/desktop/src/lib/audio.ts @@ -1,6 +1,7 @@ -// Plays raw PCM samples (from the local Piper TTS provider) via the Web Audio -// API. Used for read-aloud when a real TTS provider is available; otherwise the -// app falls back to the system speech synthesis in speech.ts. +// Plays raw PCM (from the local TTS provider) via the Web Audio API. Used for +// read-aloud when a real TTS provider is available; otherwise the app falls back +// to the system speech synthesis in speech.ts. Pause/resume suspend the shared +// AudioContext so the read-aloud queue naturally waits. let ctx: AudioContext | null = null; let currentSource: AudioBufferSourceNode | null = null; @@ -10,7 +11,21 @@ function context(): AudioContext { return ctx; } -export function playSamples(samples: number[], sampleRate: number, onEnd?: () => void): void { +/** Decode base64 16-bit little-endian mono PCM into Web-Audio float samples. + * Cheap compared to parsing a JSON number array — the IPC sends this per + * sentence during read-aloud. */ +export function decodePcm(b64: string): Float32Array { + if (!b64) return new Float32Array(0); + const bin = atob(b64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + const i16 = new Int16Array(bytes.buffer, 0, bytes.length >> 1); + const out = new Float32Array(i16.length); + for (let i = 0; i < i16.length; i++) out[i] = i16[i] / 32768; + return out; +} + +export function playSamples(samples: Float32Array, sampleRate: number, onEnd?: () => void): void { stopPlayback(); if (samples.length === 0) { onEnd?.(); @@ -18,7 +33,7 @@ export function playSamples(samples: number[], sampleRate: number, onEnd?: () => } const audioCtx = context(); const buffer = audioCtx.createBuffer(1, samples.length, sampleRate); - buffer.copyToChannel(Float32Array.from(samples), 0); + buffer.copyToChannel(samples, 0); const source = audioCtx.createBufferSource(); source.buffer = buffer; source.connect(audioCtx.destination); @@ -28,7 +43,7 @@ export function playSamples(samples: number[], sampleRate: number, onEnd?: () => }; source.start(); currentSource = source; - void audioCtx.resume(); + void audioCtx.resume(); // also un-suspends after a pause } export function stopPlayback(): void { @@ -42,3 +57,12 @@ export function stopPlayback(): void { currentSource = null; } } + +/** Pause playback (and the queue, which awaits the current segment's end). */ +export function pausePlayback(): void { + void ctx?.suspend(); +} + +export function resumePlayback(): void { + void ctx?.resume(); +} diff --git a/apps/desktop/src/lib/datetime.ts b/apps/desktop/src/lib/datetime.ts new file mode 100644 index 0000000..80a8f40 --- /dev/null +++ b/apps/desktop/src/lib/datetime.ts @@ -0,0 +1,37 @@ +// Human-relative timestamps for the sidebar / history, localized via i18n. + +import type { I18n } from "./i18n"; + +/** Relative time ("gerade eben", "vor 5 Min", "gestern", "vor 3 Tagen") for a + * stored RFC-3339 timestamp. Falls back to the locale date for anything older + * than a week. */ +export function relativeTime(iso: string, t: I18n["t"]): string { + const then = new Date(iso).getTime(); + if (Number.isNaN(then)) return ""; + const min = Math.floor((Date.now() - then) / 60_000); + if (min < 1) return t("time.justNow"); + if (min < 60) return t("time.minutesAgo", { count: min }); + const hours = Math.floor(min / 60); + if (hours < 24) return t("time.hoursAgo", { count: hours }); + const days = Math.floor(hours / 24); + if (days === 1) return t("time.yesterday"); + if (days < 7) return t("time.daysAgo", { count: days }); + return new Date(iso).toLocaleDateString(); +} + +/** Whole days remaining until a note trashed at `deletedAt` is purged, given a + * `retentionDays` window. Clamped to ≥ 0. */ +export function daysUntilPurge(deletedAt: string, retentionDays = TRASH_RETENTION_DAYS): number { + const deleted = new Date(deletedAt).getTime(); + if (Number.isNaN(deleted)) return retentionDays; + const elapsedDays = (Date.now() - deleted) / 86_400_000; + return Math.max(0, Math.ceil(retentionDays - elapsedDays)); +} + +/** How long trashed notes are kept before the purge cleanup removes them. */ +export const TRASH_RETENTION_DAYS = 30; + +/** The RFC-3339 cutoff for `purgeTrash`: notes trashed before now − retention. */ +export function purgeCutoff(retentionDays = TRASH_RETENTION_DAYS): string { + return new Date(Date.now() - retentionDays * 86_400_000).toISOString(); +} diff --git a/apps/desktop/src/lib/diff.ts b/apps/desktop/src/lib/diff.ts new file mode 100644 index 0000000..7bfbdf0 --- /dev/null +++ b/apps/desktop/src/lib/diff.ts @@ -0,0 +1,70 @@ +// Word-level diff for the edit-history view — a small, dependency-free LCS +// (no npm), fine for note-sized inputs. + +/** A diff segment: text that's unchanged, added, or removed. */ +export interface DiffSegment { + type: "equal" | "add" | "remove"; + text: string; +} + +// Split into words plus the whitespace runs between them, so concatenating the +// tokens back together is lossless (the diff can be rendered inline verbatim). +function tokenize(s: string): string[] { + return s.match(/\s+|\S+/g) ?? []; +} + +/** Word-level diff between `before` and `after` via a longest-common-subsequence + * table. Consecutive same-type tokens are coalesced into one segment. */ +export function diffWords(before: string, after: string): DiffSegment[] { + const a = tokenize(before); + const b = tokenize(after); + const n = a.length; + const m = b.length; + + // lcs[i][j] = length of the LCS of a[i..] and b[j..]. + const lcs: number[][] = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0)); + for (let i = n - 1; i >= 0; i--) { + for (let j = m - 1; j >= 0; j--) { + lcs[i][j] = a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]); + } + } + + const segments: DiffSegment[] = []; + const push = (type: DiffSegment["type"], text: string) => { + const last = segments[segments.length - 1]; + if (last && last.type === type) last.text += text; + else segments.push({ type, text }); + }; + + let i = 0; + let j = 0; + while (i < n && j < m) { + if (a[i] === b[j]) { + push("equal", a[i]); + i++; + j++; + } else if (lcs[i + 1][j] >= lcs[i][j + 1]) { + push("remove", a[i]); + i++; + } else { + push("add", b[j]); + j++; + } + } + while (i < n) push("remove", a[i++]); + while (j < m) push("add", b[j++]); + return segments; +} + +/** Added/removed word counts between two strings (for "+x / −y Wörter"). */ +export function diffStats(before: string, after: string): { added: number; removed: number } { + let added = 0; + let removed = 0; + for (const seg of diffWords(before, after)) { + if (seg.type === "equal") continue; + const words = (seg.text.match(/\S+/g) ?? []).length; + if (seg.type === "add") added += words; + else removed += words; + } + return { added, removed }; +} diff --git a/apps/desktop/src/lib/format.ts b/apps/desktop/src/lib/format.ts new file mode 100644 index 0000000..8fedbf9 --- /dev/null +++ b/apps/desktop/src/lib/format.ts @@ -0,0 +1,84 @@ +// Split long Markdown into chunks the formatter can handle in one pass. The +// per-call model has a bounded context/output, so a whole long note must be +// formatted in pieces (and reassembled) instead of one giant — truncating — +// request. Chunks never split a fenced code block or a table mid-way; blocks are +// packed up to a character budget. + +/** Group lines into blocks separated by blank lines, keeping fenced code blocks + * (``` … ```) whole even when they contain blank lines. */ +function splitBlocks(text: string): string[] { + const blocks: string[] = []; + let buf: string[] = []; + let inFence = false; + const flush = () => { + if (buf.length) { + blocks.push(buf.join("\n")); + buf = []; + } + }; + for (const line of text.split("\n")) { + if (/^\s*```/.test(line)) { + inFence = !inFence; + buf.push(line); + continue; + } + if (!inFence && line.trim() === "") { + flush(); + } else { + buf.push(line); + } + } + flush(); + return blocks; +} + +/** + * Pack Markdown blocks into chunks no larger than `budget` characters (a single + * oversized block — e.g. a huge table — is emitted alone rather than split). The + * default budget leaves room for the system prompt + the model's reply within a + * 4k-token context. + */ +export function chunkMarkdown(text: string, budget = 3500): string[] { + const chunks: string[] = []; + let current = ""; + for (const block of splitBlocks(text)) { + if (current && current.length + block.length + 2 > budget) { + chunks.push(current); + current = ""; + } + current = current ? `${current}\n\n${block}` : block; + } + if (current) chunks.push(current); + return chunks; +} + +/** + * Clean up dictated/raw text deterministically — no model, so it can never + * hallucinate, loop, or hang (unlike the small LLM, which did all three). It does + * the safe, predictable things: applies spoken paragraph commands, drops filler + * words, normalizes whitespace and the spacing before punctuation, and + * capitalizes sentence starts. It deliberately does NOT invent sentence + * boundaries or rephrase — that's the job of the punctuation model (next step); + * here it relies on punctuation already present (Whisper adds it during dictation). + */ +export function cleanDictation(text: string): string { + let out = text + // Spoken paragraph commands → real breaks. + .replace(/\b(neuer absatz|neue zeile)\b[ .,]*/gi, "\n\n") + // Drop filler words ("äh", "ähm", "öhm", "ehm"). Can't use `\b` here — it's + // ASCII-only, so it doesn't see a word boundary before "ä"; match the + // surrounding boundary chars explicitly instead. + .replace(/(^|[\s.,!?;:])(äh+|ähm+|öhm+|ehm+)(?=[\s.,!?;:]|$)/gi, "$1") + // Collapse runs of spaces/tabs. + .replace(/[ \t]{2,}/g, " ") + // No space before sentence punctuation. + .replace(/ +([.,!?;:])/g, "$1") + // Tidy whitespace around line breaks; at most one blank line. + .replace(/[ \t]*\n[ \t]*/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); + // Capitalize the first letter at the start of the text, after a sentence end, + // and after a line break. `\p{Ll}` covers German lowercase incl. umlauts. + out = out.replace(/(^|[.!?]\s|\n+)(\p{Ll})/gu, (_m, sep, ch) => sep + ch.toUpperCase()); + return out; +} diff --git a/apps/desktop/src/lib/i18n.ts b/apps/desktop/src/lib/i18n.ts new file mode 100644 index 0000000..9267441 --- /dev/null +++ b/apps/desktop/src/lib/i18n.ts @@ -0,0 +1,537 @@ +// Lightweight bilingual (German / English) UI localization. +// +// No provider is needed: the active language lives in a module-level store that +// components subscribe to via `useI18n()` (a `useSyncExternalStore` hook), so a +// language switch re-renders every consumer. The choice is persisted in +// localStorage and defaults to the browser language (German-first). + +import { useSyncExternalStore } from "react"; + +export type Lang = "de" | "en"; + +const STORAGE_KEY = "exoquill-lang"; + +// German strings are the source of truth; `TranslationKey` is derived from them, +// and the English table is type-checked to cover exactly the same keys. +const de = { + // -- shared -- + "common.close": "Schließen", + "common.cancel": "Abbrechen", + "common.apply": "Übernehmen", + "common.delete": "Löschen", + "common.reset": "Zurücksetzen", + + // -- top toolbar (global controls) -- + "toolbar.models": "Modelle", + "toolbar.models.title": "On-Device-Modelle & Lizenzen", + "toolbar.theme.title": "Theme wechseln", + "toolbar.theme.aria": "Hell-/Dunkel-Theme umschalten", + "toolbar.lang.title": "Sprache / Language", + "toolbar.lang.aria": "Sprache wechseln", + + // -- action bar -- + "action.dictate": "Diktieren", + "action.stop": "Stopp", + "action.dictate.title": "In diese Notiz diktieren", + "action.dictate.stopTitle": "Diktat stoppen", + "action.ocr": "OCR", + "action.ocr.title": "Ein Bild per Texterkennung in diese Notiz einlesen", + "action.format": "Formatieren", + "action.formatting": "Formatiere…", + "action.format.title": "Auswahl – oder die gesamte Notiz – formatieren", + "action.read": "Vorlesen", + "action.read.title": "Auswahl oder Notiz vorlesen", + "action.export": "Export", + "action.export.title": "Diese Notiz als Markdown exportieren", + "action.history": "Verlauf", + "action.history.title": "Verlauf dieser Notiz anzeigen", + "action.delete.title": "Notiz löschen", + + // -- sidebar -- + "sidebar.search": "Notizen durchsuchen", + "sidebar.count": "{count} NOTIZEN", + "sidebar.archivedCount": "{count} ARCHIVIERT", + "sidebar.trashCount": "{count} ELEMENTE", + "sidebar.newNote": "Neue Notiz", + "note.emptyPreview": "Leere Notiz", + + // -- sidebar scopes + sort + groups -- + "scope.active": "Aktiv", + "scope.archived": "Archiviert", + "scope.trash": "Papierkorb", + "scope.archivedEmpty": "Keine archivierten Notizen.", + "scope.trashEmpty": "Der Papierkorb ist leer.", + "scope.activeEmpty": "Noch keine Notizen.", + "sort.aria": "Sortierung", + "sort.modified": "Zuletzt geändert", + "sort.created": "Erstellt", + "sort.title": "Titel", + "group.pinned": "ANGEHEFTET", + "group.allNotes": "ALLE NOTIZEN", + + // -- note actions (context menu, hover, bulk bar) -- + "noteAction.menu": "Aktionen", + "noteAction.pin": "Anheften", + "noteAction.unpin": "Loslösen", + "noteAction.rename": "Umbenennen", + "noteAction.duplicate": "Duplizieren", + "noteAction.archive": "Archivieren", + "noteAction.export": "Exportieren", + "noteAction.toTrash": "In den Papierkorb", + "noteAction.restore": "Wiederherstellen", + "noteAction.deleteForever": "Endgültig löschen", + "trash.emptyTrash": "Papierkorb leeren", + "trash.deletedAgo": "Gelöscht {when}", + "trash.daysLeft": "noch {count} Tage", + "trash.retention": "Papierkorb-Einträge werden nach 30 Tagen entfernt.", + "archive.archivedAgo": "Archiviert {when}", + + // -- multi-select bulk bar -- + "select.count": "{count} ausgewählt", + "select.cancel": "Abbrechen", + + // -- undo toasts -- + "toast.undo": "Rückgängig", + "toast.trashed": "Notiz in den Papierkorb verschoben", + "toast.trashedMany": "{count} Notizen in den Papierkorb verschoben", + "toast.archived": "Notiz archiviert", + "toast.archivedMany": "{count} Notizen archiviert", + "toast.restored": "Notiz wiederhergestellt", + "toast.deletedForever": "Notiz endgültig gelöscht", + "toast.versionRestored": "Version wiederhergestellt", + + // -- relative time -- + "time.justNow": "gerade eben", + "time.minutesAgo": "vor {count} Min", + "time.hoursAgo": "vor {count} Std", + "time.yesterday": "gestern", + "time.daysAgo": "vor {count} Tagen", + + // -- editor + meta -- + "editor.placeholder": "Schreib los oder erfasse etwas…", + "editor.titlePlaceholder": "Unbenannte Notiz", + "meta.draft": "ENTWURF", + "meta.empty": "LEER", + "meta.words": "{count} WÖRTER", + + // -- empty state -- + "empty.title": "Keine Notiz ausgewählt", + "empty.desc": "Beginne mit einer Notiz, diktiere etwas oder füge einen Screenshot ein.", + + // -- dictation -- + "dictation.source": "Diktat-Quelle", + "dictation.defaultMic": "Standard-Mikrofon", + "dictation.recording": "Aufnahme läuft…", + + // -- read-aloud toolbar -- + "readaloud.label": "Vorlesen", + "readaloud.backend.aria": "Vorlese-Backend", + "readaloud.backend.title": "Sprachmodell / Backend", + "readaloud.voice.aria": "Vorlese-Stimme", + "readaloud.emotion.aria": "Vorlese-Stimmung", + "readaloud.emotion.title": "Stimmung / Emotion (Zonos)", + "readaloud.exportAudio.aria": "Audio exportieren", + "readaloud.exportAudio.title": "Vorgelesenes Audio als WAV speichern", + "readaloud.settings.aria": "Weitere Vorlese-Einstellungen", + + // -- read-aloud status bars -- + "readaloud.prepAudio": "Bereite Audio vor … {done}/{total}", + "readaloud.prepText": "Bereite Text vor … {done}/{total}", + "readaloud.preparing": "Wird vorbereitet …", + "readaloud.paused": "Pausiert", + "readaloud.reading": "Liest vor…", + "readaloud.prepSuffix": " · bereitet {done}/{total} auf", + "readaloud.resume": "▶ Fortsetzen", + "readaloud.pause": "⏸ Pause", + "readaloud.stop": "⏹ Stopp", + "readaloud.cancel": "✕ Abbrechen", + "readaloud.voiceLoading": + "Stimme lädt noch … das Vorlesen startet automatisch, sobald sie bereit ist.", + "readaloud.sample": + "Dies ist eine kurze Hörprobe der aktuellen Stimme und der gewählten Einstellungen.", + "readaloud.exportName": "vorlesen", + + // -- formatting -- + "format.progress": "Formatiere … {done}/{total}", + + // -- OCR -- + "ocr.busy": "Texterkennung läuft…", + "ocr.failed": "Texterkennung fehlgeschlagen: {error}", + "ocr.regionFailed": "Bereichs-OCR fehlgeschlagen: {error}", + "ocr.title": "Texterkennung", + "ocr.insert": "In Notiz", + "ocr.insert.title": "Auswahl – oder den gesamten Text – in die Notiz übernehmen", + "ocr.copy": "Kopieren", + "ocr.copy.title": "Auswahl – oder den gesamten Text – kopieren (Strg+C)", + "ocr.selectAll": "Alles auswählen", + "ocr.selectAll.title": "Den gesamten erkannten Text markieren (Strg+A)", + "ocr.hint": + "Text markieren und kopieren oder „In Notiz“ übernehmen — ohne Auswahl wird der gesamte erkannte Text genutzt.", + "ocr.imageAlt": "Erkanntes Bild", + + // -- edit-history (diff) overlay -- + "history.title": "Verlauf", + "history.empty": "Noch keine Ereignisse.", + "history.changesOnly": "NUR ÄNDERUNGEN", + "history.versions": "{count} VERSIONEN", + "history.current": "Aktueller Stand", + "history.words": "Wörter", + "history.compare": "VERGLEICH", + "history.thisVersion": "Diese Version", + "history.currentState": "Aktuell", + "history.readOnly": "NUR-LESEN", + "history.restoreVersion": "Diese Version wiederherstellen", + "history.restoreHint": "Schreibt als neue, undobare Änderung – nicht destruktiv", + "history.selectVersion": "Wähle links eine Version, um den Diff zu sehen.", + "history.op.format": "FORMATIERT", + "history.op.manual": "MANUELL", + "history.op.ocr": "OCR", + "history.op.dictation": "DIKTAT", + "history.op.restore": "WIEDERHERGESTELLT", + "history.op.snapshot": "SNAPSHOT", + + // -- formatting preview -- + "preview.title": "Formatierung – Vorschau", + "preview.aria": "Formatierungs-Vorschau", + "preview.original": "Original", + "preview.formatted": "Formatiert", + + // -- status bar -- + "status.saved": "GESPEICHERT", + "status.saving": "SPEICHERT…", + "status.words": "{count} WÖRTER", + + // -- model manager -- + "models.title": "Modelle verwalten", + "models.tier.bundled": "gebündelt", + "models.tier.download": "Download", + "models.tier.gated": "Lizenz nötig", + "models.nonCommercial": "nicht-kommerziell", + "models.loading": "lädt…", + "models.setup": "Setup:", + "models.installed": "installiert ✓", + "models.install": "Installieren", + "models.activeProviders": "Aktive Provider", + "models.confirmNonCommercial": + "„{name}“ steht unter {license} (nicht für kommerzielle Nutzung). Trotzdem installieren?", + "models.confirmDelete": "„{name}“ löschen?", + "models.installFailed": "Installation fehlgeschlagen: {error}", + + // -- read-aloud settings dialog -- + "raSettings.title": "Vorlese-Einstellungen · {backend}", + "raSettings.aria": "Vorlese-Einstellungen", + "raSettings.onlySpeed": + "{backend} nutzt nur das Tempo; weitere Klangregler gibt es nur für Piper und Zonos.", + "raSettings.speechPrep": "Für Sprache aufbereiten (LLM)", + "raSettings.speechPrepHint": + "Schreibt Text vor dem Vorlesen in flüssige Sätze um — bessere Qualität, aber Vorlauf.", + "raSettings.previewing": "▶ Probe läuft …", + "raSettings.preview": "▶ Probe abspielen", + "slider.speed": "Tempo", + "slider.speed.hint": "Sprechgeschwindigkeit", + "slider.expressiveness": "Ausdruck", + "slider.expressiveness.hint": "Klangvariation (noise_scale)", + "slider.cadence": "Rhythmus", + "slider.cadence.hint": "Längenvariation (noise_w)", + "slider.sentenceSilence": "Satzpause", + "slider.sentenceSilence.hint": "Sekunden Stille nach jedem Satz", + "slider.intonation": "Intonation", + "slider.intonation.hint": "Lebhaftigkeit der Betonung (monoton ↔ lebhaft)", + "slider.brightness": "Klangfarbe", + "slider.brightness.hint": "Höhenanteil (wärmer ↔ brillanter)", + + // -- Zonos emotion presets -- + "emotion.neutral": "Neutral", + "emotion.happy": "Fröhlich", + "emotion.lively": "Lebhaft", + "emotion.surprised": "Überrascht", + "emotion.calm": "Ruhig", + "emotion.sad": "Traurig", + "emotion.fearful": "Ängstlich", + "emotion.angry": "Wütend", + "emotion.disgust": "Angewidert", + + // -- region OCR overlay -- + "region.hint": "Bereich aufziehen · Esc bricht ab", +} as const; + +export type TranslationKey = keyof typeof de; + +const en: Record = { + "common.close": "Close", + "common.cancel": "Cancel", + "common.apply": "Apply", + "common.delete": "Delete", + "common.reset": "Reset", + + "toolbar.models": "Models", + "toolbar.models.title": "On-device models & licenses", + "toolbar.theme.title": "Toggle theme", + "toolbar.theme.aria": "Toggle light/dark theme", + "toolbar.lang.title": "Sprache / Language", + "toolbar.lang.aria": "Switch language", + + "action.dictate": "Dictate", + "action.stop": "Stop", + "action.dictate.title": "Dictate into this note", + "action.dictate.stopTitle": "Stop dictation", + "action.ocr": "OCR", + "action.ocr.title": "OCR an image into this note", + "action.format": "Format", + "action.formatting": "Formatting…", + "action.format.title": "Format the selection, or the whole note", + "action.read": "Read", + "action.read.title": "Read the selection or note aloud", + "action.export": "Export", + "action.export.title": "Export this note as Markdown", + "action.history": "History", + "action.history.title": "Show this note's event history", + "action.delete.title": "Delete note", + + "sidebar.search": "Search notes", + "sidebar.count": "{count} NOTES", + "sidebar.archivedCount": "{count} ARCHIVED", + "sidebar.trashCount": "{count} ITEMS", + "sidebar.newNote": "New note", + "note.emptyPreview": "Empty note", + + "scope.active": "Active", + "scope.archived": "Archived", + "scope.trash": "Trash", + "scope.archivedEmpty": "No archived notes.", + "scope.trashEmpty": "Trash is empty.", + "scope.activeEmpty": "No notes yet.", + "sort.aria": "Sort order", + "sort.modified": "Last modified", + "sort.created": "Created", + "sort.title": "Title", + "group.pinned": "PINNED", + "group.allNotes": "ALL NOTES", + + "noteAction.menu": "Actions", + "noteAction.pin": "Pin", + "noteAction.unpin": "Unpin", + "noteAction.rename": "Rename", + "noteAction.duplicate": "Duplicate", + "noteAction.archive": "Archive", + "noteAction.export": "Export", + "noteAction.toTrash": "Move to Trash", + "noteAction.restore": "Restore", + "noteAction.deleteForever": "Delete forever", + "trash.emptyTrash": "Empty Trash", + "trash.deletedAgo": "Deleted {when}", + "trash.daysLeft": "{count} days left", + "trash.retention": "Trash items are removed after 30 days.", + "archive.archivedAgo": "Archived {when}", + + "select.count": "{count} selected", + "select.cancel": "Cancel", + + "toast.undo": "Undo", + "toast.trashed": "Note moved to Trash", + "toast.trashedMany": "{count} notes moved to Trash", + "toast.archived": "Note archived", + "toast.archivedMany": "{count} notes archived", + "toast.restored": "Note restored", + "toast.deletedForever": "Note permanently deleted", + "toast.versionRestored": "Version restored", + + "time.justNow": "just now", + "time.minutesAgo": "{count} min ago", + "time.hoursAgo": "{count} h ago", + "time.yesterday": "yesterday", + "time.daysAgo": "{count} days ago", + + "editor.placeholder": "Start writing, or capture something…", + "editor.titlePlaceholder": "Untitled note", + "meta.draft": "DRAFT", + "meta.empty": "EMPTY", + "meta.words": "{count} WORDS", + + "empty.title": "No note selected", + "empty.desc": "Start with a note, dictate something, or paste a screenshot.", + + "dictation.source": "Dictation source", + "dictation.defaultMic": "Default microphone", + "dictation.recording": "Recording…", + + "readaloud.label": "Read aloud", + "readaloud.backend.aria": "Read-aloud backend", + "readaloud.backend.title": "Voice model / backend", + "readaloud.voice.aria": "Read-aloud voice", + "readaloud.emotion.aria": "Read-aloud mood", + "readaloud.emotion.title": "Mood / emotion (Zonos)", + "readaloud.exportAudio.aria": "Export audio", + "readaloud.exportAudio.title": "Save the spoken audio as WAV", + "readaloud.settings.aria": "More read-aloud settings", + + "readaloud.prepAudio": "Preparing audio … {done}/{total}", + "readaloud.prepText": "Preparing text … {done}/{total}", + "readaloud.preparing": "Preparing …", + "readaloud.paused": "Paused", + "readaloud.reading": "Reading aloud…", + "readaloud.prepSuffix": " · preparing {done}/{total}", + "readaloud.resume": "▶ Resume", + "readaloud.pause": "⏸ Pause", + "readaloud.stop": "⏹ Stop", + "readaloud.cancel": "✕ Cancel", + "readaloud.voiceLoading": + "Voice still loading … read-aloud will start automatically once it's ready.", + "readaloud.sample": + "This is a short audio sample of the current voice and the chosen settings.", + "readaloud.exportName": "read-aloud", + + "format.progress": "Formatting … {done}/{total}", + + "ocr.busy": "Recognizing text…", + "ocr.failed": "Text recognition failed: {error}", + "ocr.regionFailed": "Region OCR failed: {error}", + "ocr.title": "Text recognition", + "ocr.insert": "To note", + "ocr.insert.title": "Insert the selection – or all the text – into the note", + "ocr.copy": "Copy", + "ocr.copy.title": "Copy the selection – or all the text – (Ctrl+C)", + "ocr.selectAll": "Select all", + "ocr.selectAll.title": "Select all recognized text (Ctrl+A)", + "ocr.hint": + "Select and copy text, or send it to the note — with no selection the whole recognized text is used.", + "ocr.imageAlt": "Recognized image", + + "history.title": "History", + "history.empty": "No events yet.", + "history.changesOnly": "CHANGES ONLY", + "history.versions": "{count} VERSIONS", + "history.current": "Current", + "history.words": "words", + "history.compare": "COMPARE", + "history.thisVersion": "This version", + "history.currentState": "Current", + "history.readOnly": "READ-ONLY", + "history.restoreVersion": "Restore this version", + "history.restoreHint": "Writes as a new, undoable change — non-destructive", + "history.selectVersion": "Pick a version on the left to see its diff.", + "history.op.format": "FORMATTED", + "history.op.manual": "MANUAL", + "history.op.ocr": "OCR", + "history.op.dictation": "DICTATION", + "history.op.restore": "RESTORED", + "history.op.snapshot": "SNAPSHOT", + + "preview.title": "Formatting – preview", + "preview.aria": "Formatting preview", + "preview.original": "Original", + "preview.formatted": "Formatted", + + "status.saved": "SAVED", + "status.saving": "SAVING…", + "status.words": "{count} WORDS", + + "models.title": "Manage models", + "models.tier.bundled": "bundled", + "models.tier.download": "Download", + "models.tier.gated": "License required", + "models.nonCommercial": "non-commercial", + "models.loading": "loading…", + "models.setup": "Setup:", + "models.installed": "installed ✓", + "models.install": "Install", + "models.activeProviders": "Active providers", + "models.confirmNonCommercial": + "“{name}” is licensed under {license} (not for commercial use). Install anyway?", + "models.confirmDelete": "Delete “{name}”?", + "models.installFailed": "Installation failed: {error}", + + "raSettings.title": "Read-aloud settings · {backend}", + "raSettings.aria": "Read-aloud settings", + "raSettings.onlySpeed": + "{backend} uses speed only; the other sound controls exist for Piper and Zonos.", + "raSettings.speechPrep": "Prepare for speech (LLM)", + "raSettings.speechPrepHint": + "Rewrites the text into fluent sentences before reading — better quality, but a short delay.", + "raSettings.previewing": "▶ Sample playing …", + "raSettings.preview": "▶ Play sample", + "slider.speed": "Speed", + "slider.speed.hint": "Speaking rate", + "slider.expressiveness": "Expressiveness", + "slider.expressiveness.hint": "Timbre variation (noise_scale)", + "slider.cadence": "Cadence", + "slider.cadence.hint": "Length variation (noise_w)", + "slider.sentenceSilence": "Sentence pause", + "slider.sentenceSilence.hint": "Seconds of silence after each sentence", + "slider.intonation": "Intonation", + "slider.intonation.hint": "Liveliness of emphasis (monotone ↔ lively)", + "slider.brightness": "Brightness", + "slider.brightness.hint": "Treble share (warmer ↔ brighter)", + + "emotion.neutral": "Neutral", + "emotion.happy": "Happy", + "emotion.lively": "Lively", + "emotion.surprised": "Surprised", + "emotion.calm": "Calm", + "emotion.sad": "Sad", + "emotion.fearful": "Fearful", + "emotion.angry": "Angry", + "emotion.disgust": "Disgusted", + + "region.hint": "Drag a region · Esc cancels", +}; + +const tables: Record> = { de, en }; + +function detect(): Lang { + const stored = localStorage.getItem(STORAGE_KEY); + if (stored === "de" || stored === "en") return stored; + return navigator.language?.toLowerCase().startsWith("de") ? "de" : "en"; +} + +let current: Lang = detect(); +document.documentElement.lang = current; + +const listeners = new Set<() => void>(); + +function subscribe(cb: () => void): () => void { + listeners.add(cb); + return () => listeners.delete(cb); +} + +export function getLang(): Lang { + return current; +} + +export function setLang(lang: Lang): void { + if (lang === current) return; + current = lang; + localStorage.setItem(STORAGE_KEY, lang); + document.documentElement.lang = lang; + listeners.forEach((l) => l()); +} + +type Vars = Record; + +/** Translate `key` in `lang`, interpolating `{name}`-style placeholders. Falls + * back to the German string, then the raw key. */ +export function translate(lang: Lang, key: TranslationKey, vars?: Vars): string { + let s: string = tables[lang][key] ?? de[key] ?? key; + if (vars) { + for (const [k, v] of Object.entries(vars)) { + s = s.split(`{${k}}`).join(String(v)); + } + } + return s; +} + +export interface I18n { + lang: Lang; + setLang: (lang: Lang) => void; + t: (key: TranslationKey, vars?: Vars) => string; +} + +/** Subscribe a component to the active language. Re-renders on a switch. */ +export function useI18n(): I18n { + const lang = useSyncExternalStore(subscribe, getLang, getLang); + return { + lang, + setLang, + t: (key, vars) => translate(lang, key, vars), + }; +} diff --git a/apps/desktop/src/lib/readaloud.ts b/apps/desktop/src/lib/readaloud.ts index 97854eb..2f607b6 100644 --- a/apps/desktop/src/lib/readaloud.ts +++ b/apps/desktop/src/lib/readaloud.ts @@ -4,30 +4,122 @@ // Piper TTS (`ttsSpeak`) when available; falls back to the webview's system // speech (speech.ts) if Piper isn't there or the first segment fails. -import { playSamples, stopPlayback } from "./audio"; +import { decodePcm, pausePlayback, playSamples, resumePlayback, stopPlayback } from "./audio"; import { speak, stopSpeaking } from "./speech"; import type { TtsResponse } from "./types"; -/** A running read-aloud session; call `stop` to cancel it. */ +/** A running read-aloud session: `stop` cancels it; `pause`/`resume` suspend and + * continue playback (the queue waits while paused). */ export interface ReadAloudHandle { stop: () => void; + pause: () => void; + resume: () => void; } -/** Split Markdown into speakable chunks: drop fenced code, flatten list/heading/ - * quote markers and emphasis, reduce links to their label, then split into - * sentence-sized pieces so synthesis can start quickly and stay bounded. */ +/** Strip the leading block marker from a line: ATX heading (`#`), blockquote + * (`>`), and unordered/ordered list bullets. */ +function stripLineMarker(line: string): string { + return line + .replace(/^\s*#{1,6}\s+/, "") + .replace(/^\s*>+\s?/, "") + .replace(/^\s*([*\-+]|\d+[.)])\s+/, "") + .trim(); +} + +/** A thematic break (`---`, `***`, `___`) — speak nothing for it. */ +function isThematicBreak(line: string): boolean { + return /^\s*([-*_])\s*(\1\s*){2,}$/.test(line); +} + +/** A Markdown table's separator row (`| --- | :--: |`) — carries no words. */ +function isTableSeparator(line: string): boolean { + return /^[\s|:-]+$/.test(line) && line.includes("-") && line.includes("|"); +} + +/** Reduce inline Markdown to spoken words: links/images to their label, bare + * URLs to their host, and drop emphasis / inline-code marks. */ +function cleanInline(text: string): string { + return text + .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") // links / images → label + .replace(/\bhttps?:\/\/(?:www\.)?([^\s/]+)\S*/gi, "$1") // bare URL → host + .replace(/[*_`~]/g, "") // emphasis / inline-code marks + .replace(/\s+/g, " ") + .trim(); +} + +/** Reduce file paths and package names to a speakable tail: `src/core/Foo.ts` → + * `Foo`, `@codexo/exojs-tilemap` → `exojs-tilemap`. Reading every slash and + * extension aloud is noise; the last segment carries the meaning. */ +function reducePaths(text: string): string { + return text + .replace(/[^\s]*\/[^\s]*/g, (token) => token.split("/").filter(Boolean).pop() ?? token) + .replace(/\.(tsx?|jsx?|rs|py|json|jsonc|md|exe|onnx|gguf|toml|css|html?)\b/gi, ""); +} + +/** Map meaningful symbols to spoken words (so a table's ✅/❌ isn't lost), then + * drop arrows and bullets. */ +function mapSymbols(text: string): string { + return text + .replace(/✅/g, " ja ") + .replace(/❌/g, " nein ") + .replace(/⚠️?/g, " Achtung ") + .replace(/[×✕✖]/g, " mal ") + .replace(/[→⟶➜➔⇒]/g, " ") // arrows → a pause + .replace(/[↔⇄⟷]/g, " ") + .replace(/[•◦▪‣·]/g, " "); // bullets / middots +} + +/** Normalize already-flattened text for speech: reduce paths, map/strip symbols + * a TTS would mispronounce. Sentence punctuation (`.,!?;:`) is kept — Piper uses + * it for pauses/intonation, not literal speech. */ +function speechNormalize(text: string): string { + return mapSymbols(reducePaths(text)) + .replace(/[|#<>~^*_=`{}[\]]/g, " ") // residual markdown / structural symbols + .replace(/\s+([.,!?;:])/g, "$1") // tidy space left before punctuation + .replace(/\s+/g, " ") + .trim(); +} + +/** Turn a Markdown table into spoken text. The first row is the header; each data + * row is read as `Header: value, Header: value` so a cell keeps its meaning, and + * rows are separate sentences. A header-only table just reads its cells. */ +function tableToSpeech(lines: string[]): string { + const splitRow = (line: string): string[] => + line + .replace(/^\s*\|/, "") + .replace(/\|\s*$/, "") + .split("|") + .map((cell) => cleanInline(cell)); + const rows = lines.filter((l) => l.includes("|") && !isTableSeparator(l)).map(splitRow); + if (rows.length === 0) return ""; + const [header, ...body] = rows; + if (body.length === 0) return header.filter(Boolean).join(", "); + return body + .map((cells) => + cells + .map((cell, i) => (header[i] && cell ? `${header[i]}: ${cell}` : cell)) + .filter(Boolean) + .join(", "), + ) + .filter(Boolean) + .join(". "); +} + +/** Split Markdown into speakable chunks: drop fenced code, turn tables into + * spoken rows (with column labels), flatten list/heading/quote markers and + * emphasis, reduce links/paths, map/strip symbols a TTS would mispronounce, then + * split into sentence-sized pieces so synthesis can start quickly and stay + * bounded. */ export function splitForSpeech(markdown: string): string[] { const noCode = markdown.replace(/```[\s\S]*?```/g, " "); const chunks: string[] = []; - for (const paragraph of noCode.split(/\n{2,}/)) { - const clean = paragraph - .split("\n") - .map((line) => line.replace(/^\s*([#>]+|[*\-+]|\d+\.)\s*/, "").trim()) - .join(" ") - .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") // links / images → label - .replace(/[*_`~]/g, "") // emphasis / inline-code marks - .replace(/\s+/g, " ") - .trim(); + for (const block of noCode.split(/\n{2,}/)) { + const lines = block.split("\n"); + const isTable = lines.some(isTableSeparator) && lines.filter((l) => l.includes("|")).length >= 2; + const flattened = isTable + ? tableToSpeech(lines) + : cleanInline(lines.map(stripLineMarker).filter((l) => l && !isThematicBreak(l)).join(" ")); + const clean = speechNormalize(flattened); if (!clean) continue; for (const sentence of clean.match(/[^.!?]+[.!?]*\s*/g) ?? [clean]) { const text = sentence.trim(); @@ -41,60 +133,249 @@ export function splitForSpeech(markdown: string): string[] { function playToEnd(audio: TtsResponse, onStopRef: { resolve: (() => void) | null }): Promise { return new Promise((resolve) => { onStopRef.resolve = resolve; - playSamples(audio.samples, audio.sampleRate, () => { + playSamples(decodePcm(audio.pcm), audio.sampleRate, () => { onStopRef.resolve = null; resolve(); }); }); } +/** A one-shot, re-armable notification used to hand control between the producer + * and consumer of the synthesis buffer (no value, just "something changed"). */ +function makeSignal(): { wait: () => Promise; fire: () => void } { + let resolve: () => void = () => {}; + let promise = new Promise((r) => (resolve = r)); + return { + wait: () => promise, + fire: () => { + resolve(); + promise = new Promise((r) => (resolve = r)); + }, + }; +} + +/** How many segments to synthesize ahead of playback. A deeper buffer hides + * per-segment synthesis latency (e.g. XTTS, slower than real time) so short + * sentences after a long one don't drain it into an audible gap. */ +const LOOKAHEAD = 3; + +/** A sentence source that yields every speakable sentence of `markdown` up + * front — no LLM pass, so synthesis can start immediately. */ +export async function* plainSource(markdown: string): AsyncGenerator { + for (const sentence of splitForSpeech(markdown)) yield sentence; +} + +/** A sentence source that rewrites the note for speech through `prepare` (an LLM + * pass). All chunks are kicked off at once — the llama-server's parallel slots + * batch them, so preparation runs concurrently instead of one chunk after the + * next. Sentences are still yielded in order (playback needs order); a chunk + * whose preparation fails falls back to its raw text. `onProgress(done, total)` + * counts completions (any order). `prepare` may be a cache so an already-prepared + * chunk returns instantly. */ +export async function* preparedSource( + chunks: string[], + prepare: (chunk: string) => Promise, + onProgress?: (done: number, total: number) => void, +): AsyncGenerator { + let done = 0; + const pending = chunks.map((chunk) => + prepare(chunk) + .then((formatted) => formatted || chunk) + .catch(() => chunk) + .finally(() => onProgress?.(++done, chunks.length)), + ); + try { + for (const chunkResult of pending) { + const formatted = await chunkResult; + for (const sentence of splitForSpeech(formatted)) yield sentence; + } + } finally { + onProgress?.(chunks.length, chunks.length); + } +} + /** - * Start reading `markdown` aloud. Returns a handle whose `stop()` cancels - * playback and any further synthesis. `onDone` fires when the queue finishes or - * the fallback ends (not when stopped). + * Start reading the sentences from `source` aloud. Returns a handle whose + * `stop()` cancels playback and any further synthesis. `onDone` fires when the + * queue finishes or the system-speech fallback ends (not when stopped). + * + * Pipeline: a producer pulls sentences from `source` and synthesizes them up to + * `LOOKAHEAD` ahead into a bounded buffer; a consumer plays them in order. This + * keeps a primed buffer so variable-length segments don't open audible gaps, and + * lets a slow `source` (the LLM speech-prep pass) overlap with playback. + * + * `fallbackText` is read with the webview's system speech when no local TTS is + * available (the first synthesis fails). `onPlaybackStart` fires once, when the + * first audio actually begins — letting the UI distinguish the prepare/buffer + * phase (cancel only) from playback (pause/stop). */ +export interface ReadAloudOptions { + /** Fires once when the first audio actually begins (UI: prepare → playback). */ + onPlaybackStart?: () => void; + /** Synthesize the WHOLE text up front, then play it gapless — for backends + * slower than real time (Zonos), where streaming opens audible gaps between + * sentences. Off (streaming) for fast backends (Piper, XTTS). */ + prebuffer?: boolean; + /** Progress of the prebuffer synthesis pass (`done`/`total` segments). */ + onPrepare?: (done: number, total: number) => void; + /** Pre-synthesized audio to play directly — skips synthesis entirely. Used when + * the same voice+text was already rendered (cache hit). */ + cachedAudios?: TtsResponse[]; + /** Delivers the full set of synthesized segments once *generation* finishes + * (not on stop), so the caller can cache them and offer an audio export. Fires + * as soon as synthesis is done — before playback drains — and only for a + * complete read, so the cached audio is never partial. */ + onAudio?: (audios: TtsResponse[]) => void; + /** Called instead of the system-speech fallback when the first synthesis fails + * — e.g. the chosen sidecar voice isn't warm yet. Lets the UI show a "voice + * loading" hint and play nothing, rather than a jarring robot voice. */ + onUnavailable?: () => void; +} + export function readAloud( - markdown: string, + source: AsyncIterable, ttsSpeak: (text: string) => Promise, onDone: () => void, + fallbackText: string, + opts: ReadAloudOptions = {}, ): ReadAloudHandle { - const chunks = splitForSpeech(markdown); - let stopped = false; + const { onPlaybackStart, prebuffer, onPrepare, cachedAudios, onAudio, onUnavailable } = opts; const playState: { resolve: (() => void) | null } = { resolve: null }; + const buffer: (TtsResponse | null)[] = []; + const hasItem = makeSignal(); + const hasSpace = makeSignal(); + let producerDone = false; + let cancelled = false; // stop producing/consuming (user stop OR fallback) + let userStopped = false; // the user pressed stop — suppress onDone const stop = () => { - stopped = true; + userStopped = true; + cancelled = true; stopPlayback(); stopSpeaking(); - playState.resolve?.(); // unblock a chunk we're awaiting + playState.resolve?.(); // unblock a segment we're awaiting playState.resolve = null; + hasItem.fire(); // unblock a waiting consumer + hasSpace.fire(); // unblock a waiting producer }; - void (async () => { - if (chunks.length === 0) { - onDone(); - return; - } - let current: TtsResponse; + // Cache hit: play already-synthesized audio directly, no TTS at all. + if (cachedAudios) { + void (async () => { + onPlaybackStart?.(); + for (const audio of cachedAudios) { + if (cancelled) return; + await playToEnd(audio, playState); + if (cancelled) return; + } + if (!userStopped) onDone(); + })(); + return { stop, pause: pausePlayback, resume: resumePlayback }; + } + + // Prebuffer mode: synthesize everything first (with progress), then play it + // back gapless. For real-time-slower backends, this trades an up-front wait for + // a smooth read instead of a sentence-by-sentence stutter. + if (prebuffer) { + void (async () => { + const sentences: string[] = []; + for await (const s of source) { + if (cancelled) return; + sentences.push(s); + } + const audios: (TtsResponse | null)[] = []; + for (let i = 0; i < sentences.length; i++) { + if (cancelled) return; + onPrepare?.(i, sentences.length); + const audio = await ttsSpeak(sentences[i]).catch(() => null); + if (i === 0 && audio === null) { + // First synthesis failed: voice not warm / unavailable. + cancelled = true; + if (onUnavailable) { + onUnavailable(); + return; + } + onPlaybackStart?.(); + speak(fallbackText, { onEnd: onDone }); + return; + } + audios.push(audio); + } + onPrepare?.(sentences.length, sentences.length); + if (cancelled) return; + const finalAudios = audios.filter((a): a is TtsResponse => a !== null); + onAudio?.(finalAudios); // cache as soon as synthesis is done + onPlaybackStart?.(); + for (const audio of finalAudios) { + if (cancelled) return; + await playToEnd(audio, playState); + if (cancelled) return; + } + if (!userStopped) onDone(); + })(); + return { stop, pause: pausePlayback, resume: resumePlayback }; + } + + // Producer: synthesize sentences from `source`, keeping up to LOOKAHEAD ready + // segments primed in `buffer`. A failed synthesis is buffered as `null`. Every + // successful segment is also collected so the full audio can be cached/exported + // the moment *generation* finishes — without waiting for playback to drain (so + // the save button shows up as soon as synthesis is done, like the prebuffer + // path, rather than only after the whole note has been read out). + const produce = async () => { + const produced: TtsResponse[] = []; try { - current = await ttsSpeak(chunks[0]); - } catch { - // Piper unavailable: read the whole thing with system speech instead. - if (!stopped) speak(chunks.join(" "), { onEnd: onDone }); - return; + for await (const sentence of source) { + if (cancelled) return; + while (buffer.length >= LOOKAHEAD && !cancelled) await hasSpace.wait(); + if (cancelled) return; + const audio = await ttsSpeak(sentence).catch(() => null); + if (audio) produced.push(audio); + buffer.push(audio); + hasItem.fire(); + } + } finally { + producerDone = true; + hasItem.fire(); + // Full generation finished (not stopped): hand over the complete set for + // caching + export. Only fires when every segment was synthesized, so the + // cached audio is never a partial read. + if (!cancelled && produced.length) onAudio?.(produced); } - for (let i = 0; i < chunks.length && !stopped; i++) { - // Synthesize the next segment while the current one plays. - const next = - i + 1 < chunks.length ? ttsSpeak(chunks[i + 1]).catch(() => null) : Promise.resolve(null); - await playToEnd(current, playState); - if (stopped) return; - const ready = await next; - if (!ready) break; - current = ready; + }; + + // Consumer: play buffered segments in order, making room for the producer. + void (async () => { + void produce(); + let first = true; + for (;;) { + while (buffer.length === 0 && !producerDone && !cancelled) await hasItem.wait(); + if (cancelled) return; + if (buffer.length === 0) break; // producer done and buffer drained + const audio = buffer.shift()!; + hasSpace.fire(); + if (audio === null) { + if (first) { + // First-segment failure: voice not warm / unavailable. + cancelled = true; // stop the producer + hasSpace.fire(); + if (onUnavailable) { + onUnavailable(); + return; + } + onPlaybackStart?.(); + speak(fallbackText, { onEnd: onDone }); + return; + } + break; // mid-stream failure → just end + } + if (first) onPlaybackStart?.(); + first = false; + await playToEnd(audio, playState); + if (cancelled) return; } - if (!stopped) onDone(); + if (!userStopped) onDone(); })(); - return { stop }; + return { stop, pause: pausePlayback, resume: resumePlayback }; } diff --git a/apps/desktop/src/lib/tts.ts b/apps/desktop/src/lib/tts.ts new file mode 100644 index 0000000..3e97528 --- /dev/null +++ b/apps/desktop/src/lib/tts.ts @@ -0,0 +1,39 @@ +// Zonos read-aloud emotion presets. Zonos conditions synthesis on an 8-value +// emotion vector [happiness, sadness, disgust, fear, surprise, anger, other, +// neutral] — each ~a probability, summing to ~1. These presets map a friendly +// mood label to a plausible vector; "neutral" carries no vector, so Zonos falls +// back to its own balanced default (same as before this feature existed). Only +// Zonos reads emotion; Piper/XTTS ignore it. + +export interface ZonosEmotion { + /** Stable key persisted in the tuning + sent over IPC as the preset id. */ + key: string; + label: string; + /** Leading glyph for the picker (matches the 🗣 voice style). */ + icon: string; + /** The 8-value Zonos emotion vector, or undefined to leave Zonos' default. */ + vector?: number[]; +} + +// The full set of Zonos emotion dimensions (the six Ekman emotions + neutral), +// each preset making its target emotion dominant. Vector order is +// [happiness, sadness, disgust, fear, surprise, anger, other, neutral] and each +// sums to ~1. "neutral" carries no vector (Zonos uses its own default). Two +// softened blends ("ruhig"/"lebhaft") round out the read-aloud-friendly moods. +export const ZONOS_EMOTIONS: ZonosEmotion[] = [ + { key: "neutral", label: "Neutral", icon: "😐" }, + { key: "happy", label: "Fröhlich", icon: "😊", vector: [0.8, 0.02, 0.02, 0.02, 0.04, 0.02, 0.04, 0.04] }, + { key: "lively", label: "Lebhaft", icon: "✨", vector: [0.45, 0.02, 0.02, 0.03, 0.35, 0.03, 0.05, 0.05] }, + { key: "surprised", label: "Überrascht", icon: "😲", vector: [0.1, 0.02, 0.02, 0.04, 0.7, 0.02, 0.04, 0.06] }, + { key: "calm", label: "Ruhig", icon: "🧘", vector: [0.15, 0.08, 0.02, 0.02, 0.02, 0.02, 0.19, 0.5] }, + { key: "sad", label: "Traurig", icon: "😢", vector: [0.02, 0.8, 0.02, 0.04, 0.02, 0.02, 0.04, 0.04] }, + { key: "fearful", label: "Ängstlich", icon: "😨", vector: [0.02, 0.04, 0.02, 0.8, 0.04, 0.02, 0.02, 0.04] }, + { key: "angry", label: "Wütend", icon: "😠", vector: [0.02, 0.04, 0.04, 0.02, 0.02, 0.8, 0.02, 0.04] }, + { key: "disgust", label: "Angewidert", icon: "🤢", vector: [0.02, 0.04, 0.8, 0.02, 0.02, 0.04, 0.02, 0.04] }, +]; + +/** The emotion vector for a preset key, or undefined for "neutral"/unknown — in + * which case Zonos uses its own default vector. */ +export function emotionVector(key: string | undefined): number[] | undefined { + return ZONOS_EMOTIONS.find((e) => e.key === key)?.vector; +} diff --git a/apps/desktop/src/lib/types.ts b/apps/desktop/src/lib/types.ts index c53cbc9..5cf4fb4 100644 --- a/apps/desktop/src/lib/types.ts +++ b/apps/desktop/src/lib/types.ts @@ -5,6 +5,8 @@ export type NoteSource = "manual" | "dictation" | "ocr"; export interface Note { id: string; title: string; + /** True while the title auto-follows the content (the user hasn't named it). */ + titleAuto: boolean; contentMarkdown: string; createdAt: string; updatedAt: string; @@ -24,6 +26,36 @@ export interface NoteUpdate { lastCursorPosition?: number; } +/** Which slice of notes a listing returns (the sidebar's scope tabs). */ +export type NoteScope = "active" | "archived" | "trash"; + +/** Sort order for a note listing, applied within the pinned/un-pinned split. */ +export type NoteSort = "modified" | "created" | "title"; + +/** A stored content snapshot for the edit-history diff timeline. `source` is + * "manual" (a typing-pause snapshot) or "op" (written by an operation); `op` + * names that operation (e.g. "format", "ocr", "dictation", "restore"). */ +export interface NoteVersion { + id: string; + noteId: string; + createdAt: string; + contentMarkdown: string; + contentHash: string; + source: string; + op: string | null; + providerId: string | null; +} + +/** Input for recording a NoteVersion (id/createdAt/hash filled in by the DB). */ +export interface NewNoteVersion { + noteId: string; + contentMarkdown: string; + /** "manual" | "op"; defaults to "manual" when omitted. */ + source?: string; + op?: string; + providerId?: string; +} + /** A recorded note event (format/OCR history + undo safety net). */ export interface NoteEvent { id: string; @@ -38,6 +70,32 @@ export interface NoteEvent { createdAt: string; } +/** An installable model/voice in the catalog, with its on-disk status. */ +export interface CatalogItem { + id: string; + provider: string; + kind: string; + displayName: string; + language: string; + license: string; + commercialOk: boolean; + /** "bundled" | "download" | "gated". */ + tier: string; + /** Setup-script path for runtimes that aren't a plain file download (XTTS). */ + setup: string | null; + notes: string | null; + installed: boolean; + installedBytes: number; +} + +/** Download progress for a model file (the `model_progress` backend event). */ +export interface ModelProgress { + id: string; + file: string; + downloaded: number; + total: number; +} + /** Read-only summary of the provider behind an AI capability (settings/about). */ export interface ModelInfo { feature: string; @@ -64,10 +122,43 @@ export interface Job { } export interface TtsResponse { - samples: number[]; + /** Base64 of 16-bit little-endian mono PCM (decoded in audio.ts). */ + pcm: string; sampleRate: number; } +/** A selectable read-aloud voice offered by the local TTS provider. */ +export interface TtsVoice { + id: string; + displayName: string; + language: string; + quality: string; + /** Synthesis backend this voice belongs to (`"piper"` | `"xtts"`); passed + * back to `ttsSpeak` so the request is routed to the right provider. */ + provider: string; +} + +/** Read-aloud synthesis knobs. `speed` applies to every backend; the rest are + * backend-specific (a provider applies only the ones it understands). Omitted + * fields fall back to model defaults. */ +export interface TtsTuning { + /** Speaking rate; 1.0 = normal, >1 faster. */ + speed?: number; + /** Piper expressiveness / timbre variation (noise_scale). */ + expressiveness?: number; + /** Piper cadence variability (noise_w). */ + cadence?: number; + /** Seconds of silence after each sentence (Piper). */ + sentenceSilence?: number; + /** Zonos intonation liveliness (pitch_std); low monotone, high lively. */ + intonation?: number; + /** Zonos synthesis frequency ceiling in Hz (fmax); lower warmer, higher brighter. */ + brightness?: number; + /** Zonos emotion preset key (see `lib/tts.ts`); resolved to an 8-value vector + * before synthesis. `"neutral"`/omitted leaves Zonos' own default. */ + emotion?: string; +} + /** A dictation source: a microphone, or an output device captured via loopback. */ export interface CaptureSource { name: string; diff --git a/apps/desktop/src/styles/app.css b/apps/desktop/src/styles/app.css index 350b788..aadb9ff 100644 --- a/apps/desktop/src/styles/app.css +++ b/apps/desktop/src/styles/app.css @@ -5,41 +5,7 @@ background: var(--bg); } -/* ---- Top bar ---- */ -.topbar { - flex: none; - height: 40px; - display: flex; - align-items: center; - gap: 10px; - padding: 0 14px; - background: var(--surface); - border-bottom: 1px solid var(--border); -} -.topbar__wordmark { - font-weight: 600; - font-size: 14px; - color: var(--ink); - letter-spacing: -0.01em; -} -.topbar__spacer { - margin-left: auto; -} -.on-device { - display: flex; - align-items: center; - gap: 6px; - font-family: var(--font-mono); - font-size: 10px; - letter-spacing: 0.04em; - color: var(--accent); -} -.on-device__dot { - width: 6px; - height: 6px; - border-radius: 50%; - background: var(--accent); -} +/* ---- Icon buttons (toolbar) ---- */ .icon-btn { display: flex; align-items: center; @@ -49,10 +15,26 @@ border-radius: var(--radius); color: var(--text-secondary); } -.icon-btn:hover { +.icon-btn:hover:not(:disabled) { background: var(--bg-sidebar); color: var(--ink); } +.icon-btn:disabled { + opacity: 0.4; + cursor: not-allowed; +} +/* Language switch: globe glyph + the active language code. */ +.icon-btn--lang { + width: auto; + gap: 4px; + padding: 0 7px; +} +.icon-btn__tag { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 600; + letter-spacing: 0.04em; +} /* ---- Logo mark ---- */ .logo-mark { @@ -85,11 +67,23 @@ flex-direction: column; min-height: 0; } +.sidebar__brand { + display: flex; + align-items: center; + gap: 8px; + padding: 12px 13px 4px; +} +.sidebar__wordmark { + font-weight: 600; + font-size: 14px; + color: var(--ink); + letter-spacing: -0.01em; +} .sidebar__search { display: flex; align-items: center; gap: 7px; - margin: 10px 10px 8px; + margin: 8px 10px 8px; padding: 7px 9px; background: var(--surface); border: 1px solid var(--border); @@ -117,6 +111,9 @@ padding: 0 8px; } .note-item { + display: flex; + align-items: flex-start; + gap: 8px; padding: 8px 9px; margin-bottom: 3px; border-radius: var(--radius); @@ -132,6 +129,14 @@ border-color: var(--active-note-border); border-left-color: var(--accent); } +.note-item.selected { + background: var(--surface); + border-color: var(--accent-border); +} +.note-item__body { + flex: 1; + min-width: 0; +} .note-item__title { font-weight: 600; font-size: 12.5px; @@ -152,6 +157,63 @@ overflow: hidden; text-overflow: ellipsis; } +/* Per-note actions: the pin shows when pinned or on hover, the menu on hover. + (Steering each child's opacity, not the parent's, so a pinned-but-unhovered + row still shows its pin.) */ +.note-item__actions { + display: flex; + align-items: center; + gap: 2px; + flex: none; +} +.note-item__pin, +.note-item__menu { + opacity: 0; + transition: opacity 0.12s; +} +.note-item:hover .note-item__pin, +.note-item:hover .note-item__menu, +.note-item__pin.pinned { + opacity: 1; +} +.note-item__pin, +.note-item__menu { + display: flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + border-radius: var(--radius-sm); + color: var(--text-muted); +} +.note-item__pin:hover, +.note-item__menu:hover { + background: var(--panel2); + color: var(--ink); +} +.note-item__pin.pinned { + color: var(--pin); +} +.note-item__dots { + font-size: 15px; + line-height: 1; +} +/* Multi-select checkbox. */ +.note-check { + flex: none; + display: flex; + align-items: center; + justify-content: center; + width: 15px; + height: 15px; + margin-top: 1px; + border-radius: var(--radius-sm); + border: 1.5px solid var(--border-strong); +} +.note-check.checked { + background: var(--accent); + border-color: var(--accent); +} .sidebar__footer { padding: 9px; } @@ -218,11 +280,15 @@ border-color: var(--accent-hover); color: var(--accent-contrast); } -.actionbar__badge { +/* Pushes the global controls (models / language / theme / delete) to the right. */ +.actionbar__spacer { margin-left: auto; - font-family: var(--font-mono); - font-size: 10px; - color: var(--text-muted); +} +.actionbar__divider { + width: 1px; + align-self: stretch; + margin: 2px 2px; + background: var(--border); } /* ---- Editor surface ---- */ @@ -525,6 +591,212 @@ font-family: var(--font-mono); font-size: 11px; } +/* Backend picker sits left of the voice picker; keep it compact, not stretched. */ +.dictation-source__select--backend { + flex: 0 0 auto; + max-width: 110px; +} +.dictation-source__select--emotion { + flex: 0 0 auto; + max-width: 130px; +} +/* Compact read-aloud speed slider in the toolbar. */ +.tts-speed { + flex: 0 1 110px; + min-width: 64px; + accent-color: var(--accent, #6a9fb5); +} +.tts-speed__value { + min-width: 38px; + text-align: right; + color: var(--text); + font-variant-numeric: tabular-nums; +} + +/* ---- Read-aloud settings dialog ---- */ +.tts-settings { + display: flex; + flex-direction: column; + gap: 16px; +} +.tts-setting { + display: flex; + flex-direction: column; + gap: 4px; +} +.tts-setting__label { + display: flex; + justify-content: space-between; + font-family: var(--font-mono); + font-size: 12px; + color: var(--text); +} +.tts-setting__value { + color: var(--text-muted); +} +.tts-setting input[type="range"] { + width: 100%; + accent-color: var(--accent, #6a9fb5); +} +.tts-setting__hint { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-muted); +} +.tts-prep { + display: flex; + align-items: flex-start; + gap: 8px; + margin-top: 16px; + font-family: var(--font-mono); + font-size: 12px; + color: var(--text); + cursor: pointer; +} +.tts-prep input[type="checkbox"] { + margin-top: 2px; + accent-color: var(--accent, #6a9fb5); +} +.tts-prep .tts-setting__hint { + display: block; + margin-top: 2px; +} +.tts-settings__actions { + margin-top: 16px; + display: flex; + justify-content: flex-end; +} +.tts-reset { + padding: 5px 12px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--surface); + color: var(--text); + font-family: var(--font-mono); + font-size: 11px; + cursor: pointer; +} +.tts-reset:hover { + background: var(--surface-raised); +} + +/* ---- Model manager ---- */ +.model-mgr { + width: min(640px, 92vw); +} +.model-mgr__subhead { + margin: 18px 0 8px; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--text-muted); +} +.model-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 8px; +} +.model-row { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 12px; + padding: 8px 10px; + border: 1px solid var(--border); + border-radius: 8px; +} +.model-row__main { + display: flex; + flex-direction: column; + gap: 4px; + min-width: 0; +} +.model-row__name { + font-size: 13px; + color: var(--text); +} +.model-row__meta { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; +} +.model-badge { + padding: 1px 6px; + border-radius: 4px; + background: var(--surface-raised); + border: 1px solid var(--border); + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-secondary); +} +.model-badge--bundled { + border-color: #3a7; + color: #6c9; +} +.model-badge--download { + border-color: #69c; + color: #8ab; +} +.model-badge--gated, +.model-badge--nc { + border-color: #c66; + color: #d88; +} +.model-row__size { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-muted); +} +.model-row__notes { + font-size: 11px; + color: var(--text-muted); +} +.model-row__progress { + position: relative; + height: 6px; + width: 220px; + max-width: 100%; + margin-top: 4px; + border-radius: 3px; + background: var(--surface-raised); + overflow: hidden; +} +.model-row__bar { + position: absolute; + inset: 0 auto 0 0; + background: var(--accent, #6a9fb5); + transition: width 0.2s linear; +} +.model-row__pct { + position: absolute; + right: 4px; + top: -14px; + font-family: var(--font-mono); + font-size: 9px; + color: var(--text-muted); +} +.model-row__actions { + display: flex; + align-items: center; + gap: 8px; + flex-shrink: 0; +} +.model-row__ok { + font-family: var(--font-mono); + font-size: 11px; + color: #6c9; +} +.model-row__hint { + font-size: 10px; + color: var(--text-muted); +} +.model-row--provider { + align-items: center; +} /* ---- OCR result overlay ---- */ .ocr-busy { @@ -626,3 +898,596 @@ color: var(--text); user-select: text; } + +/* ============================================================ + Notes management (Bereich 1): scopes, sort, pin group, + context menu, multi-select, archive/trash, toasts, history + ============================================================ */ + +/* ---- Scope tabs ---- */ +.scope-tabs { + display: flex; + gap: 3px; + padding: 0 10px 8px; +} +.scope-tab { + flex: 1; + padding: 6px 4px; + border: 1px solid var(--border); + border-radius: var(--radius); + font-size: 11px; + font-weight: 500; + color: var(--text-secondary); + text-align: center; + transition: background 0.12s, color 0.12s, border-color 0.12s; +} +.scope-tab:hover:not(.active) { + border-color: var(--border-strong); + color: var(--ink); +} +.scope-tab.active { + font-weight: 600; + border-color: transparent; +} +.scope-tab--active.active { + background: var(--accent); + color: var(--accent-contrast); +} +.scope-tab--archived.active { + background: var(--ink); + color: var(--bg); +} +.scope-tab--trash.active { + background: var(--danger); + color: #fff; +} + +/* ---- Count + sort / empty-trash row ---- */ +.sidebar__meta { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 13px 6px; +} +.sidebar__meta .sidebar__count { + padding: 0; +} +.sidebar__danger-link { + font-size: 10.5px; + font-weight: 600; + color: var(--danger); +} +.sidebar__danger-link:hover { + text-decoration: underline; +} + +/* ---- Sort dropdown ---- */ +.sort-control { + position: relative; +} +.sort-control__btn { + display: flex; + align-items: center; + gap: 4px; + font-family: var(--font-mono); + font-size: 9px; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--text-secondary); +} +.sort-control__btn:hover { + color: var(--ink); +} +.popover-scrim { + position: fixed; + inset: 0; + z-index: 40; +} +.sort-menu { + position: absolute; + right: 0; + top: calc(100% + 4px); + z-index: 41; + min-width: 150px; + padding: 4px; + background: var(--surface); + border: 1px solid var(--border-strong); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-pop); +} +.sort-menu__item { + display: block; + width: 100%; + padding: 7px 9px; + border-radius: var(--radius-sm); + font-size: 12.5px; + color: var(--text); + text-align: left; +} +.sort-menu__item:hover { + background: var(--panel2); +} +.sort-menu__item.active { + color: var(--accent-soft-text); + font-weight: 600; +} + +/* ---- Group headers (pinned / all notes) ---- */ +.note-group { + display: flex; + align-items: center; + gap: 5px; + padding: 6px 5px 4px; + font-family: var(--font-mono); + font-size: 9px; + letter-spacing: 0.08em; + color: var(--text-faint); +} +.note-group--pin { + color: var(--pin); +} +.note-list__empty { + padding: 24px 14px; + font-size: 12px; + color: var(--text-muted); + text-align: center; +} + +/* ---- Archive rows ---- */ +.archive-row, +.trash-row { + padding: 9px; + margin-bottom: 5px; + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); +} +.archive-row { + cursor: pointer; +} +.archive-row.selected { + border-color: var(--accent-border); +} +.archive-row__head { + display: flex; + align-items: center; + gap: 7px; +} +.archive-row__title, +.trash-row__title { + font-weight: 600; + font-size: 12.5px; + color: var(--ink); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.archive-row__meta, +.trash-row__meta { + display: flex; + align-items: center; + gap: 6px; + margin: 4px 0 9px; + font-size: 11px; + color: var(--text-faint); +} +.archive-row__head + .archive-row__meta { + margin-left: 20px; +} + +/* ---- Row action buttons (restore / delete) ---- */ +.row-actions { + display: flex; + gap: 6px; +} +.row-action { + display: flex; + align-items: center; + justify-content: center; + gap: 5px; + padding: 6px 9px; + border: 1px solid var(--border); + border-radius: var(--radius); + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); +} +.row-action--restore { + flex: 1; + color: var(--accent-soft-text); + border-color: var(--accent-border); + background: var(--accent-soft-bg); +} +.row-action--danger { + flex: 1; + color: var(--danger); + border-color: var(--danger-border); +} +.row-action--danger:hover, +.row-action--danger-icon:hover { + background: var(--danger-soft); +} +.row-action--danger-icon { + color: var(--danger); +} + +/* ---- Trash rows ---- */ +.trash-row__meta { + margin-left: 0; +} + +/* ---- Bulk action bar (multi-select) ---- */ +.bulk-bar { + margin: 0 0 6px; +} +.bulk-bar__head { + display: flex; + align-items: center; + justify-content: space-between; + padding: 9px 13px; + background: var(--accent); + color: var(--accent-contrast); +} +.bulk-bar__count { + font-size: 12.5px; + font-weight: 600; +} +.bulk-bar__cancel { + font-size: 11px; + color: var(--accent-contrast); + opacity: 0.85; +} +.bulk-bar__cancel:hover { + opacity: 1; +} +.bulk-bar__actions { + display: flex; + gap: 3px; + padding: 8px 9px; + border-bottom: 1px solid var(--border); +} +.bulk-action { + flex: 1; + display: flex; + flex-direction: column; + align-items: center; + gap: 3px; + padding: 4px 2px; + font-size: 10px; + color: var(--text-secondary); + border-radius: var(--radius-sm); +} +.bulk-action:hover { + background: var(--panel2); + color: var(--ink); +} +.bulk-action--danger { + color: var(--danger); +} + +/* ---- Context menu ---- */ +.context-menu { + position: fixed; + z-index: 60; + min-width: 190px; + padding: 5px; + background: var(--surface); + border: 1px solid var(--border-strong); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-pop); +} +.context-menu__item { + display: flex; + align-items: center; + gap: 9px; + width: 100%; + padding: 7px 9px; + border-radius: var(--radius-sm); + font-size: 12.5px; + color: var(--text); + text-align: left; +} +.context-menu__item:hover { + background: var(--panel2); +} +.context-menu__item--danger { + color: var(--danger); +} +.context-menu__item--danger:hover { + background: var(--danger-soft); +} +.context-menu__divider { + height: 1px; + margin: 4px 6px; + background: var(--border); +} + +/* ---- Undo toasts ---- */ +.toast-stack { + position: fixed; + bottom: 18px; + left: 50%; + transform: translateX(-50%); + z-index: 80; + display: flex; + flex-direction: column; + gap: 8px; + width: min(420px, 90vw); +} +.toast { + display: flex; + align-items: center; + gap: 10px; + padding: 11px 13px; + background: var(--ink); + color: var(--bg); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-pop); + animation: toast-in 0.18s ease-out; +} +@keyframes toast-in { + from { + opacity: 0; + transform: translateY(8px); + } +} +.toast__icon { + display: flex; + opacity: 0.8; +} +.toast__msg { + flex: 1; + font-size: 12.5px; +} +.toast__action { + font-size: 12px; + font-weight: 700; + color: var(--accent); +} +.toast__close { + font-size: 15px; + line-height: 1; + color: var(--bg); + opacity: 0.5; +} +.toast__close:hover { + opacity: 0.9; +} + +/* ---- Edit-history (diff) overlay ---- */ +.diff-overlay { + display: flex; + width: min(960px, 94vw); + height: min(620px, 86vh); + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-window); + overflow: hidden; +} +.diff-timeline { + width: 250px; + flex: none; + display: flex; + flex-direction: column; + background: var(--bg-sidebar); + border-right: 1px solid var(--border); +} +.diff-timeline__head { + padding: 13px 14px 10px; + border-bottom: 1px solid var(--border); +} +.diff-timeline__title { + display: flex; + align-items: center; + gap: 7px; + font-weight: 600; + font-size: 14px; + color: var(--ink); +} +.diff-timeline__sub { + margin-top: 5px; + font-family: var(--font-mono); + font-size: 9px; + letter-spacing: 0.06em; + color: var(--accent-soft-text); +} +.diff-timeline__list { + flex: 1; + overflow-y: auto; + padding: 8px; +} +.diff-node { + display: flex; + gap: 9px; + width: 100%; + padding: 7px 6px; + border: 1px solid transparent; + border-radius: var(--radius); + text-align: left; +} +.diff-node:hover { + background: color-mix(in srgb, var(--surface) 60%, transparent); +} +.diff-node.active { + background: var(--surface); + border-color: var(--accent-border); +} +.diff-node__rail { + display: flex; + flex-direction: column; + align-items: center; + flex: none; + padding-top: 2px; +} +.diff-node__dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--border-strong); +} +.diff-node__dot--current { + width: 9px; + height: 9px; + background: var(--accent); + box-shadow: 0 0 0 3px var(--accent-soft-bg); +} +.diff-node__line { + width: 1.5px; + flex: 1; + margin-top: 3px; + background: var(--border); +} +.diff-node__body { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} +.diff-node__current { + font-weight: 600; + font-size: 12px; + color: var(--ink); +} +.diff-node__stats { + font-size: 11.5px; + color: var(--text-secondary); +} +.diff-node__time { + font-family: var(--font-mono); + font-size: 9px; + color: var(--text-faint); +} +.op-badge { + align-self: flex-start; + font-family: var(--font-mono); + font-size: 8.5px; + font-weight: 600; + padding: 1px 5px; + border-radius: 3px; +} +.op-badge--op { + color: var(--accent-soft-text); + background: var(--accent-soft-bg); +} +.op-badge--manual { + color: var(--text-secondary); + background: var(--panel2); + border: 1px solid var(--border); +} + +/* Diff view (right pane) */ +.diff-view { + flex: 1; + display: flex; + flex-direction: column; + min-width: 0; +} +.diff-view__head { + display: flex; + align-items: center; + gap: 10px; + padding: 11px 16px; + border-bottom: 1px solid var(--border); +} +.diff-view__compare { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-faint); +} +.diff-view__versions { + display: flex; + align-items: center; + gap: 6px; + font-family: var(--font-mono); + font-size: 10px; + font-weight: 600; +} +.diff-view__versions .muted { + color: var(--text-secondary); +} +.diff-view__versions .accent { + color: var(--accent); +} +.diff-view__arrow { + color: var(--text-faint); +} +.diff-view__readonly { + margin-left: auto; + font-family: var(--font-mono); + font-size: 9px; + color: var(--text-faint); + background: var(--bg-sidebar); + border: 1px solid var(--border); + border-radius: 4px; + padding: 3px 7px; +} +.diff-view__body { + flex: 1; + overflow-y: auto; + padding: 16px; + font-size: 13px; + line-height: 1.7; +} +.diff-view__title { + font-weight: 600; + font-size: 15px; + color: var(--ink); + margin-bottom: 12px; +} +.diff-text { + margin: 0; + white-space: pre-wrap; + word-break: break-word; + color: var(--text); +} +.diff-seg--add { + background: var(--accent-soft-bg); + color: var(--accent-soft-text); + border-radius: 2px; + padding: 0 2px; +} +.diff-seg--remove { + background: var(--danger-soft); + color: var(--danger-ink); + text-decoration: line-through; + border-radius: 2px; + padding: 0 2px; +} +.diff-view__hint { + color: var(--text-muted); +} +.diff-view__actions { + display: flex; + align-items: center; + gap: 10px; + padding: 11px 16px; + border-top: 1px solid var(--border); +} +.diff-restore { + display: flex; + align-items: center; + gap: 6px; + padding: 8px 12px; + background: var(--accent); + color: var(--accent-contrast); + border-radius: var(--radius); + font-size: 12.5px; + font-weight: 600; +} +.diff-restore:disabled { + opacity: 0.5; + cursor: not-allowed; +} +.diff-view__resthint { + display: flex; + align-items: center; + gap: 5px; + margin-left: auto; + max-width: 180px; + font-family: var(--font-mono); + font-size: 9px; + color: var(--text-faint); + line-height: 1.4; + text-align: right; +} diff --git a/apps/desktop/src/styles/theme.css b/apps/desktop/src/styles/theme.css index 7392071..56e9e9b 100644 --- a/apps/desktop/src/styles/theme.css +++ b/apps/desktop/src/styles/theme.css @@ -22,6 +22,20 @@ --accent-contrast: #ffffff; --accent-soft-bg: #e1f1e8; --accent-soft-text: #157a4b; + --accent-border: #9fd3bb; + + /* Trash / destructive (Papierkorb), and pin (Amber — distinct from the green + action accent). Used by scope tabs, trash actions, and the pin group. */ + --danger: #be4334; + --danger-ink: #9c3526; + --danger-soft: #f7e7e4; + --danger-border: #e3b6ae; + --pin: #b7791f; + --pin-soft: #f6ecd8; + + /* Raised panel fill behind muted badges (e.g. the MANUAL history op badge). */ + --panel2: #eef0f1; + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); --active-note-bg: #ffffff; --active-note-border: #d6e7dd; diff --git a/crates/exoquill-ai/Cargo.toml b/crates/exoquill-ai/Cargo.toml index 9693387..edbe5e9 100644 --- a/crates/exoquill-ai/Cargo.toml +++ b/crates/exoquill-ai/Cargo.toml @@ -8,9 +8,11 @@ authors.workspace = true repository.workspace = true [dependencies] -exoquill-capture = { version = "0.1.0", path = "../exoquill-capture" } -exoquill-core = { version = "0.1.0", path = "../exoquill-core" } +exoquill-capture = { version = "0.2.0", path = "../exoquill-capture" } +exoquill-core = { version = "0.2.0", path = "../exoquill-core" } serde = { version = "1.0.228", features = ["derive"] } +# Parse Piper voice configs (`*.onnx.json`) for the per-voice sample rate. +serde_json = "1" # Talks to the persistent whisper-server over localhost HTTP (multipart upload of # WAV to /inference). Blocking is fine: the dictation worker is a plain thread. reqwest = { version = "0.13", default-features = false, features = [ diff --git a/crates/exoquill-ai/src/lib.rs b/crates/exoquill-ai/src/lib.rs index 628fdd0..943a919 100644 --- a/crates/exoquill-ai/src/lib.rs +++ b/crates/exoquill-ai/src/lib.rs @@ -7,6 +7,7 @@ pub mod formatter; pub mod llama; +pub mod llama_server; pub mod mock; pub mod ocr; pub mod piper; @@ -17,12 +18,17 @@ pub mod tts; pub mod vad; pub mod whisper; pub mod whisper_server; +pub mod xtts; +pub mod zonos; pub use llama::LlamaFormatter; +pub use llama_server::{LlamaServer, LlamaServerFormatter}; pub use piper::PiperTts; pub use tesseract::TesseractOcr; pub use whisper::WhisperStt; pub use whisper_server::{WhisperServer, WhisperServerStt}; +pub use xtts::{XttsServer, XttsTts}; +pub use zonos::{ZonosServer, ZonosTts}; pub use provider::{ CancelToken, Capability, Health, LicenseInfo, ModelRequirement, Provider, ProviderError, diff --git a/crates/exoquill-ai/src/llama.rs b/crates/exoquill-ai/src/llama.rs index 988da93..224aa4d 100644 --- a/crates/exoquill-ai/src/llama.rs +++ b/crates/exoquill-ai/src/llama.rs @@ -13,7 +13,8 @@ use crate::provider::{ ProviderResult, }; -const DEFAULT_SYSTEM: &str = "Du bist ein präziser Text-Formatierer für Diktate und OCR-Text. \ +pub(crate) const DEFAULT_SYSTEM: &str = + "Du bist ein präziser Text-Formatierer für Diktate und OCR-Text. \ Korrigiere Rechtschreibung, Zeichensetzung und offensichtliche Erkennungsfehler und verbessere \ die Lesbarkeit mit sauberem Markdown. Erfinde keine neuen Inhalte, bewahre Bedeutung und \ Fachbegriffe (Produkt- und Bibliotheksnamen). Gib ausschließlich den formatierten Text zurück."; diff --git a/crates/exoquill-ai/src/llama_server.rs b/crates/exoquill-ai/src/llama_server.rs new file mode 100644 index 0000000..8827b28 --- /dev/null +++ b/crates/exoquill-ai/src/llama_server.rs @@ -0,0 +1,351 @@ +//! Persistent llama.cpp server provider for fast, repeated formatting. +//! +//! Unlike [`crate::llama::LlamaFormatter`] (which spawns `llama-cli` per call and +//! reloads the ~1.5B model every time, seconds of latency), this keeps a single +//! `llama-server` child alive with the model resident, so each format is just an +//! HTTP POST to the OpenAI-compatible `/v1/chat/completions` endpoint — cheap +//! enough to run repeatedly when a long note is formatted in chunks. Still an +//! isolated process (decisions D8), now persistent instead of per-call. + +use std::io::{BufRead, BufReader}; +use std::net::TcpListener; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +use crate::formatter::{FormatRequest, FormatResponse, FormatterProvider}; +use crate::llama::DEFAULT_SYSTEM; +use crate::provider::{ + below_normal_priority, CancelToken, Capability, Health, LicenseInfo, ModelRequirement, + Provider, ProviderError, ProviderResult, +}; + +/// A running `llama-server` child + the localhost URL it serves. Dropping it +/// kills the server. Use [`LlamaServer::client`] to talk to it. +pub struct LlamaServer { + child: Child, + base_url: String, +} + +impl LlamaServer { + /// Spawn `llama-server` for `model` and wait until the model is loaded. + /// `-ngl 999` offloads to the GPU when the bundled build is CUDA (ignored on + /// a CPU build); `-c 4096` is ample for chunk-sized formatting requests. + /// + /// On a CPU build the inference is the heavy part, and left unchecked it + /// saturates every core and starves the webview (the UI freezes). Two guards + /// keep the app responsive: we cap the worker threads (leaving cores for the + /// UI/OS, overridable via `EXOQUILL_LLAMA_THREADS`) and, on Windows, drop the + /// child to below-normal priority so the scheduler favors the foreground UI. + pub fn start(binary: impl Into, model: impl Into) -> ProviderResult { + let binary = binary.into(); + let model = model.into(); + let threads = llama_threads(); + // Parallel slots: with `-np N`, llama.cpp serves N requests concurrently + // via continuous batching, so chunked formatting / speech-prep can run its + // chunks in parallel instead of one after another. `-c` is the *total* + // context, split across slots, so it scales with the slot count (4096 per + // slot). More slots = more KV-cache VRAM; tune via `EXOQUILL_LLAMA_*`. + let parallel = llama_parallel(); + let context = 4096 * parallel; + let port = free_port()?; + let base_url = format!("http://127.0.0.1:{port}"); + + let mut command = Command::new(&binary); + command + .arg("-m") + .arg(&model) + .arg("--host") + .arg("127.0.0.1") + .arg("--port") + .arg(port.to_string()) + .arg("-t") + .arg(threads.to_string()) + .arg("-np") + .arg(parallel.to_string()) + .arg("-c") + .arg(context.to_string()) + .arg("-ngl") + .arg("999") + .stdout(Stdio::null()) + .stderr(Stdio::null()); + below_normal_priority(&mut command); + + let child = command + .spawn() + .map_err(|e| ProviderError::Runtime(format!("spawn llama-server: {e}")))?; + + let server = Self { child, base_url }; + server.wait_ready(Duration::from_secs(60))?; + Ok(server) + } + + /// Poll `/health` until the model is loaded (returns 200) or `timeout` passes. + fn wait_ready(&self, timeout: Duration) -> ProviderResult<()> { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .map_err(|e| ProviderError::Runtime(format!("http client: {e}")))?; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Ok(resp) = client.get(format!("{}/health", self.base_url)).send() { + if resp.status().is_success() { + return Ok(()); + } + } + std::thread::sleep(Duration::from_millis(250)); + } + Err(ProviderError::Runtime( + "llama-server did not become ready in time".into(), + )) + } + + /// A formatter client bound to this server (own HTTP client; wrap in `Arc`). + pub fn client(&self) -> ProviderResult { + LlamaServerFormatter::new(self.base_url.clone()) + } +} + +impl Drop for LlamaServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Worker-thread count for llama-server: `EXOQUILL_LLAMA_THREADS` if set, else +/// the logical CPUs minus one (≥1) so the UI/OS keep a core during CPU inference. +fn llama_threads() -> u32 { + if let Some(n) = std::env::var("EXOQUILL_LLAMA_THREADS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| *n > 0) + { + return n; + } + let cores = std::thread::available_parallelism() + .map(|n| n.get() as u32) + .unwrap_or(4); + cores.saturating_sub(2).max(2) +} + +/// Parallel request slots for llama-server: `EXOQUILL_LLAMA_PARALLEL` if set, +/// else 4 — enough to batch a handful of format / speech-prep chunks at once +/// without an outsized KV-cache. Each slot adds context (VRAM); lower it if the +/// GPU is tight (it shares VRAM with the TTS sidecars). +fn llama_parallel() -> u32 { + std::env::var("EXOQUILL_LLAMA_PARALLEL") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(4) +} + +/// Reserve a free localhost TCP port by binding to :0 and reading it back. The +/// port is released on drop; llama-server re-binds it (a tiny race we accept). +fn free_port() -> ProviderResult { + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|e| ProviderError::Runtime(format!("reserve port: {e}")))?; + let port = listener + .local_addr() + .map_err(|e| ProviderError::Runtime(format!("read port: {e}")))? + .port(); + Ok(port) +} + +/// Formatting against a running [`LlamaServer`]'s `/v1/chat/completions` endpoint. +/// The server applies the model's chat template and stops at EOS, so output isn't +/// truncated the way a fixed `-n` cap truncates the per-call CLI. +pub struct LlamaServerFormatter { + base_url: String, + client: reqwest::blocking::Client, +} + +#[derive(Serialize)] +struct ChatMessage<'a> { + role: &'a str, + content: &'a str, +} + +#[derive(Serialize)] +struct ChatRequest<'a> { + messages: Vec>, + temperature: f32, + top_p: f32, + max_tokens: u32, + stream: bool, +} + +/// One streamed SSE delta from `/v1/chat/completions` (`stream: true`). +#[derive(Deserialize)] +struct StreamChunk { + choices: Vec, +} + +#[derive(Deserialize)] +struct StreamChoice { + delta: StreamDelta, +} + +#[derive(Deserialize)] +struct StreamDelta { + content: Option, +} + +impl LlamaServerFormatter { + fn new(base_url: String) -> ProviderResult { + // A generous total timeout: a long completion may stream for minutes on + // CPU, so a tight cap would abort mid-generation. The cancel token is the + // real "stop" — this just bounds a wedged server. (`reqwest::blocking` + // here has no per-read timeout, so we can't time out on inter-token gaps.) + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(600)) + .build() + .map_err(|e| ProviderError::Runtime(format!("http client: {e}")))?; + Ok(Self { base_url, client }) + } + + fn system_prompt(request: &FormatRequest) -> String { + // Read-aloud prep is a rewrite task, not light formatting — use the + // instruction as the whole system prompt so the model isn't anchored to + // the conservative "Diktat/OCR formatter" frame and actually recomposes + // tables/lists/code into spoken prose. + if request.operation == "speech_prep" { + if let Some(instruction) = &request.instruction { + if !instruction.trim().is_empty() { + return instruction.clone(); + } + } + } + match &request.instruction { + Some(instruction) if !instruction.trim().is_empty() => { + format!("{DEFAULT_SYSTEM}\n\nZusätzliche Anweisung: {instruction}") + } + _ => DEFAULT_SYSTEM.to_string(), + } + } +} + +impl Provider for LlamaServerFormatter { + fn id(&self) -> &str { + "formatter.llama_server" + } + fn display_name(&self) -> &str { + "llama.cpp Formatter (server)" + } + fn version(&self) -> &str { + "1" + } + fn capabilities(&self) -> Vec { + Vec::new() + } + fn required_models(&self) -> Vec { + vec![ModelRequirement { + model_id: "formatter.qwen".into(), + feature: "formatter".into(), + required: true, + }] + } + fn license_info(&self) -> LicenseInfo { + LicenseInfo { + runtime_license: "MIT".into(), + source: Some("ggml-org/llama.cpp".into()), + } + } + fn health_check(&self) -> Health { + match self.client.get(format!("{}/health", self.base_url)).send() { + Ok(resp) if resp.status().is_success() => Health::Ready, + Ok(_) => Health::Unavailable { + reason: "llama-server loading".into(), + }, + Err(e) => Health::Unavailable { + reason: format!("llama-server unreachable: {e}"), + }, + } + } +} + +impl FormatterProvider for LlamaServerFormatter { + fn run(&self, request: FormatRequest, cancel: &CancelToken) -> ProviderResult { + if cancel.is_cancelled() { + return Err(ProviderError::Cancelled); + } + if request.text.trim().is_empty() { + return Ok(FormatResponse { + formatted_text: String::new(), + warnings: Vec::new(), + changed_meaning_risk: "low".into(), + }); + } + + let system = Self::system_prompt(&request); + let body = ChatRequest { + messages: vec![ + ChatMessage { + role: "system", + content: &system, + }, + ChatMessage { + role: "user", + content: request.text.trim(), + }, + ], + temperature: 0.3, + top_p: 0.9, + max_tokens: 2048, + stream: true, + }; + + // Stream the completion (SSE) so we can observe `cancel` between tokens + // and bail mid-generation: returning early drops the response, which + // closes the connection and makes llama-server stop generating — the + // freed CPU is what lets a "cancel" actually take effect, instead of the + // request running to completion in the background. + let response = self + .client + .post(format!("{}/v1/chat/completions", self.base_url)) + .json(&body) + .send() + .map_err(|e| ProviderError::Runtime(format!("llama-server request: {e}")))?; + if !response.status().is_success() { + return Err(ProviderError::Runtime(format!( + "llama-server returned {}", + response.status() + ))); + } + + let reader = BufReader::new(response); + let mut formatted = String::new(); + for line in reader.lines() { + if cancel.is_cancelled() { + return Err(ProviderError::Cancelled); + } + let line = + line.map_err(|e| ProviderError::Runtime(format!("llama-server stream: {e}")))?; + let Some(data) = line.strip_prefix("data:").map(str::trim) else { + continue; + }; + if data == "[DONE]" { + break; + } + if let Ok(chunk) = serde_json::from_str::(data) { + if let Some(content) = chunk + .choices + .into_iter() + .next() + .and_then(|c| c.delta.content) + { + formatted.push_str(&content); + } + } + } + + Ok(FormatResponse { + formatted_text: formatted.trim().to_string(), + warnings: Vec::new(), + changed_meaning_risk: "low".into(), + }) + } +} diff --git a/crates/exoquill-ai/src/mock.rs b/crates/exoquill-ai/src/mock.rs index 8a34916..c038804 100644 --- a/crates/exoquill-ai/src/mock.rs +++ b/crates/exoquill-ai/src/mock.rs @@ -212,11 +212,7 @@ mod tests { fn tts_emits_samples() { let out = MockTextToSpeech .run( - TtsRequest { - text: "hallo".into(), - voice_id: "de-calm".into(), - speed: 1.0, - }, + TtsRequest::new("hallo", "de-calm", 1.0), &CancelToken::new(), ) .unwrap(); diff --git a/crates/exoquill-ai/src/piper.rs b/crates/exoquill-ai/src/piper.rs index d58ea34..5a7bdf7 100644 --- a/crates/exoquill-ai/src/piper.rs +++ b/crates/exoquill-ai/src/piper.rs @@ -2,32 +2,113 @@ //! //! Text is written to Piper over stdin; raw 16-bit mono PCM comes back on //! stdout (`--output-raw`) and is normalized to `f32` samples for playback. +//! +//! A single `PiperTts` instance fronts *all* voices found in a directory: every +//! `*.onnx` next to its `*.onnx.json` config is one selectable voice. The voice +//! is chosen per request via `TtsRequest::voice_id` (the model file stem); the +//! sample rate is read from each voice's config since it varies by quality +//! (`x_low`/`low` are 16 kHz, `medium`/`high` are 22.05 kHz). use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use crate::provider::{ CancelToken, Capability, Health, LicenseInfo, ModelRequirement, Provider, ProviderError, ProviderResult, }; -use crate::tts::{TextToSpeechProvider, TtsRequest, TtsResponse}; +use crate::tts::{TextToSpeechProvider, TtsRequest, TtsResponse, TtsVoice}; -/// Neural TTS via a bundled Piper executable + ONNX voice. -pub struct PiperTts { - binary: PathBuf, +/// A discovered Piper voice: its model file plus metadata for the picker. +#[derive(Debug, Clone)] +struct Voice { + /// Model file stem, e.g. `de_DE-thorsten-medium`. This is the `voice_id`. + id: String, + language: String, + quality: String, model: PathBuf, sample_rate: u32, } +/// Neural TTS via a bundled Piper executable + one or more ONNX voices. +pub struct PiperTts { + binary: PathBuf, + voices: Vec, + /// Voice id used when a request names an unknown/empty voice. Falls back to + /// the first discovered voice if this id isn't present. + default_id: String, +} + impl PiperTts { + /// Build a provider for a single voice file (the sample rate is taken as + /// given). Kept for callers/tests that point at one explicit `.onnx`. pub fn new(binary: impl Into, model: impl Into, sample_rate: u32) -> Self { + let model = model.into(); + let id = file_stem(&model); + let (language, quality) = split_voice_id(&id); + let default_id = id.clone(); + Self { + binary: binary.into(), + voices: vec![Voice { + id, + language, + quality, + model, + sample_rate, + }], + default_id, + } + } + + /// Build a provider by scanning `voices_dir` for every `*.onnx` voice. Each + /// voice's sample rate is read from its sibling `*.onnx.json` (default 22050 + /// if absent/unreadable). `default_id` (a file stem) selects the voice used + /// for requests that name an unknown voice; if it isn't found, the first + /// discovered voice wins. Voices are sorted by id for a stable picker order. + pub fn discover( + binary: impl Into, + voices_dir: impl AsRef, + default_id: impl Into, + ) -> Self { + let mut voices = Vec::new(); + if let Ok(entries) = std::fs::read_dir(voices_dir.as_ref()) { + for entry in entries.flatten() { + let model = entry.path(); + if model.extension().and_then(|e| e.to_str()) != Some("onnx") { + continue; + } + let id = file_stem(&model); + if id.is_empty() { + continue; + } + let (language, quality) = split_voice_id(&id); + let sample_rate = read_sample_rate(&model).unwrap_or(22_050); + voices.push(Voice { + id, + language, + quality, + model, + sample_rate, + }); + } + } + voices.sort_by(|a, b| a.id.cmp(&b.id)); Self { binary: binary.into(), - model: model.into(), - sample_rate, + voices, + default_id: default_id.into(), } } + + /// Pick the voice for a request: the named voice, else the configured + /// default, else the first discovered voice. + fn select(&self, voice_id: &str) -> Option<&Voice> { + self.voices + .iter() + .find(|v| v.id == voice_id) + .or_else(|| self.voices.iter().find(|v| v.id == self.default_id)) + .or_else(|| self.voices.first()) + } } impl Provider for PiperTts { @@ -57,7 +138,7 @@ impl Provider for PiperTts { } } fn health_check(&self) -> Health { - if !self.model.exists() { + if self.voices.is_empty() { Health::MissingModel { model_id: "tts.piper.de".into(), } @@ -76,18 +157,41 @@ impl TextToSpeechProvider for PiperTts { if cancel.is_cancelled() { return Err(ProviderError::Cancelled); } + let voice = self + .select(&request.voice_id) + .ok_or_else(|| ProviderError::Runtime("no piper voice available".into()))?; + let text = request.text.trim(); if text.is_empty() { return Ok(TtsResponse { samples: Vec::new(), - sample_rate: self.sample_rate, + sample_rate: voice.sample_rate, }); } - let mut child = Command::new(&self.binary) - .arg("--model") - .arg(&self.model) - .arg("--output-raw") + let mut command = Command::new(&self.binary); + command.arg("--model").arg(&voice.model).arg("--output_raw"); + // Speaking rate: Piper's length_scale is phoneme *duration*, so a faster + // voice is a smaller scale — invert. Omit at 1.0 to keep the default. + if request.speed > 0.0 && (request.speed - 1.0).abs() > f32::EPSILON { + command + .arg("--length_scale") + .arg(format!("{:.3}", 1.0 / request.speed)); + } + if let Some(noise_scale) = request.expressiveness { + command + .arg("--noise_scale") + .arg(format!("{noise_scale:.3}")); + } + if let Some(noise_w) = request.cadence { + command.arg("--noise_w").arg(format!("{noise_w:.3}")); + } + if let Some(silence) = request.sentence_silence { + command + .arg("--sentence_silence") + .arg(format!("{silence:.3}")); + } + let mut child = command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) @@ -119,7 +223,125 @@ impl TextToSpeechProvider for PiperTts { .collect(); Ok(TtsResponse { samples, - sample_rate: self.sample_rate, + sample_rate: voice.sample_rate, }) } + + fn voices(&self) -> Vec { + self.voices + .iter() + .map(|v| TtsVoice { + id: v.id.clone(), + display_name: display_name(v), + language: v.language.clone(), + quality: v.quality.clone(), + provider: "piper".into(), + }) + .collect() + } + + fn default_voice(&self) -> Option { + self.select("").map(|v| v.id.clone()) + } +} + +/// File stem ("de_DE-thorsten-medium" for ".../de_DE-thorsten-medium.onnx"). +fn file_stem(model: &Path) -> String { + model + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Split a Piper voice id `--` into (language, quality). +/// The name may contain `-`; the first segment is the language and the last is +/// the quality. Ids that don't fit yield empty parts (the id still selects). +fn split_voice_id(id: &str) -> (String, String) { + let segs: Vec<&str> = id.split('-').collect(); + match segs.as_slice() { + [lang, _name @ .., qual] if !_name.is_empty() => (lang.to_string(), qual.to_string()), + [lang, qual] => (lang.to_string(), qual.to_string()), + _ => (String::new(), String::new()), + } +} + +/// The voice "name" segment(s) between language and quality, title-cased for the +/// picker (`thorsten_emotional` → `Thorsten Emotional`). +fn voice_name(id: &str) -> String { + let segs: Vec<&str> = id.split('-').collect(); + let name = if segs.len() >= 3 { + segs[1..segs.len() - 1].join("-") + } else { + segs.first().copied().unwrap_or(id).to_string() + }; + name.split(['_', '-']) + .filter(|w| !w.is_empty()) + .map(|w| { + let mut chars = w.chars(); + match chars.next() { + Some(first) => first.to_uppercase().chain(chars).collect::(), + None => String::new(), + } + }) + .collect::>() + .join(" ") +} + +/// Build the picker label, e.g. `Thorsten — de_DE (medium)`. +fn display_name(v: &Voice) -> String { + let name = voice_name(&v.id); + match (v.language.is_empty(), v.quality.is_empty()) { + (false, false) => format!("{name} — {} ({})", v.language, v.quality), + (false, true) => format!("{name} — {}", v.language), + (true, false) => format!("{name} ({})", v.quality), + (true, true) => name, + } +} + +/// Read `audio.sample_rate` from a voice's sibling `*.onnx.json` config. +fn read_sample_rate(model: &Path) -> Option { + let config = PathBuf::from(format!("{}.json", model.to_string_lossy())); + let text = std::fs::read_to_string(config).ok()?; + let json: serde_json::Value = serde_json::from_str(&text).ok()?; + json.get("audio")? + .get("sample_rate")? + .as_u64() + .map(|r| r as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_three_part_id() { + assert_eq!( + split_voice_id("de_DE-thorsten-medium"), + ("de_DE".into(), "medium".into()) + ); + } + + #[test] + fn splits_name_with_underscore() { + assert_eq!( + split_voice_id("de_DE-thorsten_emotional-medium"), + ("de_DE".into(), "medium".into()) + ); + assert_eq!( + voice_name("de_DE-thorsten_emotional-medium"), + "Thorsten Emotional" + ); + } + + #[test] + fn builds_label() { + let v = Voice { + id: "en_US-amy-medium".into(), + language: "en_US".into(), + quality: "medium".into(), + model: PathBuf::new(), + sample_rate: 22_050, + }; + assert_eq!(display_name(&v), "Amy — en_US (medium)"); + } } diff --git a/crates/exoquill-ai/src/provider.rs b/crates/exoquill-ai/src/provider.rs index 024d06c..4a899cc 100644 --- a/crates/exoquill-ai/src/provider.rs +++ b/crates/exoquill-ai/src/provider.rs @@ -7,6 +7,7 @@ //! processes (decisions D8). use std::fmt; +use std::process::Command; use serde::{Deserialize, Serialize}; @@ -14,6 +15,21 @@ use serde::{Deserialize, Serialize}; // one type. Re-exported here for ergonomic `crate::provider::CancelToken` use. pub use exoquill_core::CancelToken; +/// On Windows, start a child process at below-normal priority so heavy CPU work +/// (LLM inference, neural-TTS warm-up/synthesis) yields to the foreground UI and +/// doesn't freeze the webview. No-op on other platforms. Shared by the spawned +/// runtimes (llama-server, XTTS, Zonos). +pub(crate) fn below_normal_priority(command: &mut Command) { + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + const BELOW_NORMAL_PRIORITY_CLASS: u32 = 0x0000_4000; + command.creation_flags(BELOW_NORMAL_PRIORITY_CLASS); + } + #[cfg(not(windows))] + let _ = command; +} + /// A capability a provider advertises to the UI and scheduler. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Capability { diff --git a/crates/exoquill-ai/src/tts.rs b/crates/exoquill-ai/src/tts.rs index b745b4f..76c79cd 100644 --- a/crates/exoquill-ai/src/tts.rs +++ b/crates/exoquill-ai/src/tts.rs @@ -4,13 +4,50 @@ use serde::{Deserialize, Serialize}; use crate::provider::{CancelToken, Provider, ProviderResult}; -/// A segment of text to synthesize. +/// A segment of text to synthesize, plus the tuning knobs the UI exposes. The +/// `Option` fields fall back to each model's defaults when `None`; a provider +/// applies only the ones it understands (XTTS uses `speed` alone). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TtsRequest { pub text: String, pub voice_id: String, + /// Speaking rate; 1.0 = normal, >1 faster. Piper maps this to `length_scale` + /// (= 1/speed); XTTS passes it through as its `speed`. pub speed: f32, + /// Piper `noise_scale` — expressiveness / timbre variation (default 0.667). + pub expressiveness: Option, + /// Piper `noise_w` — phoneme-duration (cadence) variability (default 0.8). + pub cadence: Option, + /// Seconds of silence after each sentence (Piper, default 0.2). + pub sentence_silence: Option, + /// Zonos `pitch_std` — intonation liveliness; low is monotone, high is lively + /// (Zonos default 20, our sidecar default 42). + pub intonation: Option, + /// Zonos `fmax` — synthesis frequency ceiling in Hz; lower is warmer/duller, + /// higher is brighter (default 22050, the native rate's ceiling). + pub brightness: Option, + /// Zonos `emotion` — the 8-value conditioning vector [happiness, sadness, + /// disgust, fear, surprise, anger, other, neutral]; `None` leaves Zonos' own + /// default. The UI resolves a mood preset to this vector. + pub emotion: Option>, +} + +impl TtsRequest { + /// A request with model defaults for every knob (just text + voice + speed). + pub fn new(text: impl Into, voice_id: impl Into, speed: f32) -> Self { + Self { + text: text.into(), + voice_id: voice_id.into(), + speed, + expressiveness: None, + cadence: None, + sentence_silence: None, + intonation: None, + brightness: None, + emotion: None, + } + } } /// Synthesized audio. @@ -22,8 +59,77 @@ pub struct TtsResponse { pub sample_rate: u32, } +/// A selectable voice the provider offers (settings/read-aloud voice picker). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TtsVoice { + /// Stable identifier passed back in `TtsRequest::voice_id` (e.g. the Piper + /// model file stem, `de_DE-thorsten-medium`). + pub id: String, + /// Human-readable label for the picker (e.g. `Thorsten — de_DE (medium)`). + pub display_name: String, + /// BCP-47-ish language tag (`de_DE`, `en_US`), best-effort from the id. + pub language: String, + /// Quality tier (`x_low` | `low` | `medium` | `high`), best-effort. + pub quality: String, + /// Synthesis backend that offers this voice (`"piper"` | `"xtts"`). Lets the + /// UI group voices by backend and route a request to the right provider. + pub provider: String, +} + +/// Pick the optimal synthesis language for `text` — German vs. English, the two +/// languages this app targets. Umlauts/ß are a hard German signal; otherwise a +/// stop-word vote decides, defaulting to German (the app's primary language). +/// Run per segment by the multilingual providers (XTTS, Zonos), so a German note +/// with English quotes reads each part optimally without the user picking. +pub(crate) fn detect_language(text: &str) -> &'static str { + let lower = text.to_lowercase(); + if lower.chars().any(|c| matches!(c, 'ä' | 'ö' | 'ü' | 'ß')) { + return "de"; + } + const DE: &[&str] = &[ + "der", "die", "das", "und", "ist", "nicht", "ein", "eine", "ich", "wir", "mit", "den", + "dem", "von", "zu", "sich", "auch", "wird", "werden", "oder", "aber", "sind", "im", "des", + "wie", "noch", "auf", "es", "an", "als", + ]; + const EN: &[&str] = &[ + "the", "is", "are", "and", "to", "of", "in", "that", "with", "for", "this", "you", "it", + "on", "be", "as", "at", "or", "an", "we", "not", "but", "by", "from", "can", "will", "has", + "was", "have", "they", + ]; + let mut de = 0usize; + let mut en = 0usize; + for word in lower + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + { + if DE.contains(&word) { + de += 1; + } + if EN.contains(&word) { + en += 1; + } + } + if en > de { + "en" + } else { + "de" + } +} + /// Synthesizes speech from text segments (the read-aloud queue calls this per /// sentence, product spec §13.5). pub trait TextToSpeechProvider: Provider { fn run(&self, request: TtsRequest, cancel: &CancelToken) -> ProviderResult; + + /// The voices this provider offers, for the picker. Empty when the provider + /// has no notion of selectable voices. + fn voices(&self) -> Vec { + Vec::new() + } + + /// The id of the voice used when a request names an unknown (or empty) voice. + fn default_voice(&self) -> Option { + None + } } diff --git a/crates/exoquill-ai/src/xtts.rs b/crates/exoquill-ai/src/xtts.rs new file mode 100644 index 0000000..0b8ddea --- /dev/null +++ b/crates/exoquill-ai/src/xtts.rs @@ -0,0 +1,323 @@ +//! Experimental XTTS-v2 text-to-speech provider (Coqui), via a Python sidecar. +//! +//! Unlike Piper (single-language espeak phonemes), XTTS-v2 is multilingual and +//! handles mixed DE/EN + technical terms far better. It's too heavy to run as a +//! native sidecar, so a small Python HTTP server (`scripts/xtts-server.py`) +//! loads the model once and synthesizes on `POST /tts`; this provider is a thin +//! blocking client, mirroring [`crate::whisper_server`]. +//! +//! TEST ONLY: the XTTS-v2 *weights* ship under the non-commercial Coqui Public +//! Model License (CPML). The library (the `coqui-tts` fork) is MPL-2.0 and fine, +//! but the weights must not be redistributed in ExoQuill's GPL build. Enable by +//! running the sidecar and setting `EXOQUILL_XTTS_URL`; otherwise Piper is used. + +use std::net::TcpListener; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use serde::Serialize; + +use crate::provider::{ + below_normal_priority, CancelToken, Capability, Health, LicenseInfo, ModelRequirement, + Provider, ProviderError, ProviderResult, +}; +use crate::tts::{detect_language, TextToSpeechProvider, TtsRequest, TtsResponse, TtsVoice}; + +/// XTTS-v2 output is fixed at 24 kHz mono. +const SAMPLE_RATE: u32 = 24_000; + +/// XTTS-v2's full set of built-in studio speakers. Each is one selectable voice; +/// the speaking language isn't part of the voice — it's auto-detected per segment +/// (see [`detect_language`]) so a speaker reads German or English text optimally +/// without the user picking, and without a de/en duplicate per speaker. +const SPEAKERS: &[&str] = &[ + "Claribel Dervla", + "Daisy Studious", + "Gracie Wise", + "Tammie Ema", + "Alison Dietlinde", + "Ana Florence", + "Annmarie Nele", + "Asya Anara", + "Brenda Stern", + "Gitta Nikolina", + "Henriette Usha", + "Sofia Hellen", + "Tammy Grit", + "Tanja Adelina", + "Vjollca Johnnie", + "Andrew Chipper", + "Badr Odhiambo", + "Dionisio Schuyler", + "Royston Min", + "Viktor Eka", + "Abrahan Mack", + "Adde Michal", + "Baldur Sanjin", + "Craig Gutsy", + "Damien Black", + "Gilberto Mathias", + "Ilkin Urabena", + "Kazuhiko Atallah", + "Ludvig Milivoj", + "Suad Qasim", + "Torcull Diarmuid", + "Viktor Menelaos", + "Zacharie Aimilios", + "Nova Hogarth", + "Maja Ruoho", + "Uta Obando", + "Lidiya Szekeres", + "Chandra MacFarland", + "Szofi Granger", + "Camilla Holmström", + "Lilya Stainthorpe", + "Zofija Kendrick", + "Narelle Moon", + "Barbora MacLean", + "Alexandra Hisakawa", + "Alma María", + "Rosemary Okafor", + "Ige Behringer", + "Filip Traverse", + "Damjan Chapman", + "Wulf Carlevaro", + "Aaron Dreschner", + "Kumar Dahl", + "Eugenio Mataracı", + "Ferran Simen", + "Xavier Hayasaka", + "Luis Moray", + "Marcos Rudaski", +]; + +/// A running XTTS-v2 Python sidecar child + the localhost URL it serves. Dropping +/// it kills the sidecar. Mirrors [`crate::whisper_server::WhisperServer`] — the +/// app spawns this so XTTS "just works" without the user starting the script. +pub struct XttsServer { + child: Child, + base_url: String, +} + +impl XttsServer { + /// Spawn `python script --port P` and wait until the model is loaded (the + /// sidecar only answers `GET /` once `TTS(...)` finished loading). The model + /// is normally cached, so loading is seconds; the first ever run downloads + /// ~1.8 GB, hence the generous timeout. + pub fn start(python: impl Into, script: impl Into) -> ProviderResult { + let port = free_port()?; + let base_url = format!("http://127.0.0.1:{port}"); + let mut command = Command::new(python.into()); + command + .arg(script.into()) + .arg("--port") + .arg(port.to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + below_normal_priority(&mut command); + let child = command + .spawn() + .map_err(|e| ProviderError::Runtime(format!("spawn xtts sidecar: {e}")))?; + let server = Self { child, base_url }; + server.wait_ready(Duration::from_secs(600))?; + Ok(server) + } + + fn wait_ready(&self, timeout: Duration) -> ProviderResult<()> { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .map_err(|e| ProviderError::Runtime(format!("http client: {e}")))?; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Ok(resp) = client.get(&self.base_url).send() { + if resp.status().is_success() { + return Ok(()); + } + } + std::thread::sleep(Duration::from_millis(500)); + } + Err(ProviderError::Runtime( + "xtts sidecar did not become ready in time".into(), + )) + } + + /// A TTS client bound to this sidecar (own HTTP client; wrap in `Arc`). + pub fn client(&self) -> Option { + XttsTts::connect(self.base_url.clone()) + } +} + +impl Drop for XttsServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Reserve a free localhost TCP port by binding to :0 and reading it back. +fn free_port() -> ProviderResult { + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|e| ProviderError::Runtime(format!("reserve port: {e}")))?; + let port = listener + .local_addr() + .map_err(|e| ProviderError::Runtime(format!("read port: {e}")))? + .port(); + Ok(port) +} + +/// Thin client for a running XTTS-v2 sidecar. +pub struct XttsTts { + base_url: String, + client: reqwest::blocking::Client, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct TtsBody<'a> { + text: &'a str, + language: &'a str, + speaker: &'a str, + speed: f32, +} + +impl XttsTts { + /// Connect to a sidecar at `base_url`; `None` if it isn't reachable. The + /// synthesis client gets a generous timeout (XTTS is slow on CPU); a short + /// probe decides reachability. + pub fn connect(base_url: impl Into) -> Option { + let base_url = base_url.into(); + let probe = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .ok()?; + probe.get(&base_url).send().ok()?; + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .ok()?; + Some(Self { base_url, client }) + } + + /// The selectable voices (one per studio speaker), without needing a live + /// server — lets the picker populate before the sidecar has finished loading. + /// Language is auto-detected at synthesis time, so it's not part of the id. + pub fn voices_static() -> Vec { + SPEAKERS + .iter() + .map(|speaker| TtsVoice { + id: (*speaker).to_string(), + display_name: (*speaker).to_string(), + language: "auto".into(), + quality: "xtts".into(), + provider: "xtts".into(), + }) + .collect() + } + + /// The speaker for a voice id. Accepts a bare speaker name and, for backward + /// compatibility, the old `"|"` form (the language is ignored + /// now — it's auto-detected). Falls back to the first speaker. + fn parse_speaker(voice_id: &str) -> &str { + let speaker = voice_id.split('|').next().unwrap_or("").trim(); + if speaker.is_empty() { + SPEAKERS[0] + } else { + speaker + } + } +} + +impl Provider for XttsTts { + fn id(&self) -> &str { + "tts.xtts" + } + fn display_name(&self) -> &str { + "XTTS-v2 (experimental)" + } + fn version(&self) -> &str { + "2" + } + fn capabilities(&self) -> Vec { + Vec::new() + } + fn required_models(&self) -> Vec { + vec![ModelRequirement { + model_id: "tts.xtts_v2".into(), + feature: "tts".into(), + required: true, + }] + } + fn license_info(&self) -> LicenseInfo { + LicenseInfo { + runtime_license: "CPML (non-commercial)".into(), + source: Some("coqui/XTTS-v2".into()), + } + } + fn health_check(&self) -> Health { + match self.client.get(&self.base_url).send() { + Ok(_) => Health::Ready, + Err(e) => Health::Unavailable { + reason: format!("xtts sidecar unreachable: {e}"), + }, + } + } +} + +impl TextToSpeechProvider for XttsTts { + fn run(&self, request: TtsRequest, cancel: &CancelToken) -> ProviderResult { + if cancel.is_cancelled() { + return Err(ProviderError::Cancelled); + } + let text = request.text.trim(); + if text.is_empty() { + return Ok(TtsResponse { + samples: Vec::new(), + sample_rate: SAMPLE_RATE, + }); + } + let speaker = Self::parse_speaker(&request.voice_id); + let language = detect_language(text); + let body = TtsBody { + text, + language, + speaker, + speed: request.speed, + }; + + let response = self + .client + .post(format!("{}/tts", self.base_url)) + .json(&body) + .send() + .map_err(|e| ProviderError::Runtime(format!("xtts request: {e}")))?; + if !response.status().is_success() { + return Err(ProviderError::Runtime(format!( + "xtts sidecar returned {}", + response.status() + ))); + } + let bytes = response + .bytes() + .map_err(|e| ProviderError::Runtime(format!("xtts read: {e}")))?; + + // Raw 16-bit little-endian mono PCM → normalized f32 (same as Piper). + let samples = bytes + .chunks_exact(2) + .map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / 32768.0) + .collect(); + Ok(TtsResponse { + samples, + sample_rate: SAMPLE_RATE, + }) + } + + fn voices(&self) -> Vec { + Self::voices_static() + } + + fn default_voice(&self) -> Option { + Some(SPEAKERS[0].to_string()) + } +} diff --git a/crates/exoquill-ai/src/zonos.rs b/crates/exoquill-ai/src/zonos.rs new file mode 100644 index 0000000..dcc5b3a --- /dev/null +++ b/crates/exoquill-ai/src/zonos.rs @@ -0,0 +1,294 @@ +//! Zonos-v0.1 text-to-speech provider (Zyphra), via a Python sidecar. +//! +//! Like XTTS, Zonos is multilingual (incl. German) and far better with mixed +//! DE/EN technical terms than Piper. Unlike XTTS it has no fixed studio speakers: +//! it *clones* a voice from a 10–30 s reference clip. So a voice here is a `.wav` +//! in a reference folder (mirrors how `PiperTts` treats a folder of `.onnx`), +//! and the sidecar embeds that clip to synthesize. It's too heavy to run as a +//! native sidecar, so a small Python HTTP server (`scripts/zonos-server.py`) +//! loads the model once and synthesizes on `POST /tts`; this is a thin blocking +//! client, mirroring [`crate::xtts`]. +//! +//! Unlike XTTS, the Zonos *weights* are Apache-2.0 — fine to redistribute — but +//! the model needs a CUDA GPU to be usable (CPU is far too slow). Enable by +//! pointing `EXOQUILL_ZONOS_*` at the venv/script/voice folder; otherwise the +//! other TTS providers are used. + +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use serde::Serialize; + +use crate::provider::{ + below_normal_priority, CancelToken, Capability, Health, LicenseInfo, ModelRequirement, + Provider, ProviderError, ProviderResult, +}; +use crate::tts::{detect_language, TextToSpeechProvider, TtsRequest, TtsResponse, TtsVoice}; + +/// Zonos output is fixed at 44.1 kHz mono (its DAC autoencoder's native rate). +const SAMPLE_RATE: u32 = 44_100; + +/// A running Zonos Python sidecar child + the localhost URL it serves. Dropping +/// it kills the sidecar. Mirrors [`crate::xtts::XttsServer`]. +pub struct ZonosServer { + child: Child, + base_url: String, +} + +impl ZonosServer { + /// Spawn `python script --port P --voices DIR` and wait until the model is + /// loaded (the sidecar only answers `GET /` once the model is ready). The + /// model is cached after the first run, so loading is seconds; the first ever + /// run downloads the weights, hence the generous timeout. + pub fn start( + python: impl Into, + script: impl Into, + voices_dir: impl Into, + ) -> ProviderResult { + let port = free_port()?; + let base_url = format!("http://127.0.0.1:{port}"); + let mut command = Command::new(python.into()); + command + .arg(script.into()) + .arg("--port") + .arg(port.to_string()) + .arg("--voices") + .arg(voices_dir.into()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + below_normal_priority(&mut command); + let child = command + .spawn() + .map_err(|e| ProviderError::Runtime(format!("spawn zonos sidecar: {e}")))?; + let server = Self { child, base_url }; + server.wait_ready(Duration::from_secs(600))?; + Ok(server) + } + + fn wait_ready(&self, timeout: Duration) -> ProviderResult<()> { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .map_err(|e| ProviderError::Runtime(format!("http client: {e}")))?; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Ok(resp) = client.get(&self.base_url).send() { + if resp.status().is_success() { + return Ok(()); + } + } + std::thread::sleep(Duration::from_millis(500)); + } + Err(ProviderError::Runtime( + "zonos sidecar did not become ready in time".into(), + )) + } + + /// A TTS client bound to this sidecar (own HTTP client; wrap in `Arc`). + pub fn client(&self) -> Option { + ZonosTts::connect(self.base_url.clone()) + } +} + +impl Drop for ZonosServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Reserve a free localhost TCP port by binding to :0 and reading it back. +fn free_port() -> ProviderResult { + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|e| ProviderError::Runtime(format!("reserve port: {e}")))?; + let port = listener + .local_addr() + .map_err(|e| ProviderError::Runtime(format!("read port: {e}")))? + .port(); + Ok(port) +} + +/// Thin client for a running Zonos sidecar. +pub struct ZonosTts { + base_url: String, + client: reqwest::blocking::Client, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct TtsBody<'a> { + text: &'a str, + language: &'a str, + speaker: &'a str, + speed: f32, + /// `pitch_std` — intonation liveliness; `None` lets the sidecar use its + /// default. Maps from [`TtsRequest::intonation`]. + #[serde(skip_serializing_if = "Option::is_none")] + pitch: Option, + /// `fmax` — frequency ceiling (Hz); `None` keeps the sidecar default of + /// 22050. Maps from [`TtsRequest::brightness`]. + #[serde(skip_serializing_if = "Option::is_none")] + fmax: Option, + /// `emotion` — the 8-value conditioning vector; `None` lets the sidecar leave + /// Zonos' default. Maps from [`TtsRequest::emotion`]. + #[serde(skip_serializing_if = "Option::is_none")] + emotion: Option<&'a [f32]>, +} + +impl ZonosTts { + /// Connect to a sidecar at `base_url`; `None` if it isn't reachable. The + /// synthesis client gets a generous timeout (neural TTS isn't instant); a + /// short probe decides reachability. + pub fn connect(base_url: impl Into) -> Option { + let base_url = base_url.into(); + let probe = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .ok()?; + probe.get(&base_url).send().ok()?; + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .ok()?; + Some(Self { base_url, client }) + } + + /// The selectable voices — one per `.wav` reference clip in `dir` (the file + /// stem is the voice id), without needing a live server, so the picker can + /// populate before the sidecar has finished loading. Empty when the folder is + /// missing or has no clips. Mirrors how `PiperTts` enumerates a voice folder. + pub fn voices_in_dir(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut voices: Vec = entries + .flatten() + .filter_map(|e| { + let path = e.path(); + let is_wav = path + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| x.eq_ignore_ascii_case("wav")); + if !is_wav { + return None; + } + let stem = path.file_stem()?.to_string_lossy().into_owned(); + Some(TtsVoice { + id: stem.clone(), + display_name: stem.replace(['_', '-'], " "), + language: "auto".into(), + quality: "zonos".into(), + provider: "zonos".into(), + }) + }) + .collect(); + voices.sort_by(|a, b| a.display_name.cmp(&b.display_name)); + voices + } + + /// The reference clip (voice id) for a request; empty falls back to the + /// sidecar's own default (it picks the first clip it loaded). + fn parse_speaker(voice_id: &str) -> &str { + voice_id.trim() + } +} + +impl Provider for ZonosTts { + fn id(&self) -> &str { + "tts.zonos" + } + fn display_name(&self) -> &str { + "Zonos-v0.1" + } + fn version(&self) -> &str { + "0.1" + } + fn capabilities(&self) -> Vec { + Vec::new() + } + fn required_models(&self) -> Vec { + vec![ModelRequirement { + model_id: "tts.zonos_v0_1".into(), + feature: "tts".into(), + required: true, + }] + } + fn license_info(&self) -> LicenseInfo { + LicenseInfo { + runtime_license: "Apache-2.0".into(), + source: Some("Zyphra/Zonos-v0.1".into()), + } + } + fn health_check(&self) -> Health { + match self.client.get(&self.base_url).send() { + Ok(_) => Health::Ready, + Err(e) => Health::Unavailable { + reason: format!("zonos sidecar unreachable: {e}"), + }, + } + } +} + +impl TextToSpeechProvider for ZonosTts { + fn run(&self, request: TtsRequest, cancel: &CancelToken) -> ProviderResult { + if cancel.is_cancelled() { + return Err(ProviderError::Cancelled); + } + let text = request.text.trim(); + if text.is_empty() { + return Ok(TtsResponse { + samples: Vec::new(), + sample_rate: SAMPLE_RATE, + }); + } + let speaker = Self::parse_speaker(&request.voice_id); + let language = detect_language(text); + let body = TtsBody { + text, + language, + speaker, + speed: request.speed, + pitch: request.intonation, + fmax: request.brightness, + emotion: request.emotion.as_deref(), + }; + + let response = self + .client + .post(format!("{}/tts", self.base_url)) + .json(&body) + .send() + .map_err(|e| ProviderError::Runtime(format!("zonos request: {e}")))?; + if !response.status().is_success() { + return Err(ProviderError::Runtime(format!( + "zonos sidecar returned {}", + response.status() + ))); + } + let bytes = response + .bytes() + .map_err(|e| ProviderError::Runtime(format!("zonos read: {e}")))?; + + // Raw 16-bit little-endian mono PCM → normalized f32 (same as Piper/XTTS). + let samples = bytes + .chunks_exact(2) + .map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / 32768.0) + .collect(); + Ok(TtsResponse { + samples, + sample_rate: SAMPLE_RATE, + }) + } + + fn voices(&self) -> Vec { + // The live client doesn't own the folder; voices are listed statically by + // the command layer via `voices_in_dir`. Empty here is fine. + Vec::new() + } + + fn default_voice(&self) -> Option { + None + } +} diff --git a/crates/exoquill-audio/src/capture.rs b/crates/exoquill-audio/src/capture.rs index cf843e5..b0550c8 100644 --- a/crates/exoquill-audio/src/capture.rs +++ b/crates/exoquill-audio/src/capture.rs @@ -120,10 +120,14 @@ where let channels = channels as usize; let mut downmixer = Downmixer::new(channels); let mut agc = AutoGain::new(); + // Reused across callbacks so the realtime audio thread doesn't reallocate the + // interleaved→f32 conversion buffer every buffer (~10–20 ms). + let mut interleaved: Vec = Vec::new(); device.build_input_stream::( *config, move |data: &[T], _| { - let interleaved: Vec = data.iter().copied().map(|s| f32::from_sample(s)).collect(); + interleaved.clear(); + interleaved.extend(data.iter().copied().map(f32::from_sample)); let mut mono = downmixer.mix(&interleaved); match gain { None => agc.process(&mut mono), diff --git a/crates/exoquill-core/src/note.rs b/crates/exoquill-core/src/note.rs index 65b7289..369aafc 100644 --- a/crates/exoquill-core/src/note.rs +++ b/crates/exoquill-core/src/note.rs @@ -20,12 +20,61 @@ pub enum NoteSource { Ocr, } +/// Which slice of notes a listing returns. Backs the sidebar's scope tabs: +/// `Active` (live, un-archived) is the default; `Archived` and `Trash` are the +/// other two views. A note is in exactly one scope at a time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum NoteScope { + #[default] + Active, + Archived, + Trash, +} + +impl NoteScope { + /// The SQL predicate selecting this scope's rows. A fixed string per variant + /// (no user input), so it's safe to interpolate into a query. + pub fn predicate(self) -> &'static str { + match self { + NoteScope::Active => "deleted_at IS NULL AND archived = 0", + NoteScope::Archived => "deleted_at IS NULL AND archived = 1", + NoteScope::Trash => "deleted_at IS NOT NULL", + } + } +} + +/// Sort order for a note listing, applied within the pinned/un-pinned split. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum NoteSort { + #[default] + Modified, + Created, + Title, +} + +impl NoteSort { + /// The SQL `ORDER BY` term for this sort (a fixed string per variant). + pub fn order_by(self) -> &'static str { + match self { + NoteSort::Modified => "updated_at DESC", + NoteSort::Created => "created_at DESC", + NoteSort::Title => "title COLLATE NOCASE ASC", + } + } +} + /// A note as persisted and sent to the frontend (camelCase over the IPC bridge). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Note { pub id: String, pub title: String, + /// `true` while the title is auto-derived from the content (the user hasn't + /// edited it). Cleared the moment the user types a title; restored when they + /// clear it again. Drives the auto-title regeneration in the persistence layer. + pub title_auto: bool, pub content_markdown: String, pub created_at: String, pub updated_at: String, @@ -80,6 +129,38 @@ pub struct NewNoteEvent { pub model_version: Option, } +/// A stored snapshot of a note's content for the edit-history diff timeline +/// (sent to the frontend as camelCase). Snapshots are deduplicated by +/// `content_hash`, so a no-op save adds nothing. `source` is `"manual"` (a +/// typing-pause snapshot) or `"op"` (written by an operation — format/OCR/ +/// dictation); `op` names that operation when `source == "op"`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NoteVersion { + pub id: String, + pub note_id: String, + pub created_at: String, + pub content_markdown: String, + pub content_hash: String, + pub source: String, + pub op: Option, + pub provider_id: Option, +} + +/// Input for recording a [`NoteVersion`]; `id`, `created_at`, and `content_hash` +/// are filled in by the persistence layer. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewNoteVersion { + pub note_id: String, + pub content_markdown: String, + /// `"manual"` | `"op"`; defaults to `"manual"` when omitted by the caller. + #[serde(default)] + pub source: Option, + pub op: Option, + pub provider_id: Option, +} + /// Partial update for a note. Only `Some` fields are written; `updated_at` is /// always bumped by the persistence layer. #[derive(Debug, Clone, Default, Deserialize)] diff --git a/crates/exoquill-db/Cargo.toml b/crates/exoquill-db/Cargo.toml index d24670b..d4b3ff3 100644 --- a/crates/exoquill-db/Cargo.toml +++ b/crates/exoquill-db/Cargo.toml @@ -8,6 +8,6 @@ authors.workspace = true repository.workspace = true [dependencies] -exoquill-core = { version = "0.1.0", path = "../exoquill-core" } +exoquill-core = { version = "0.2.0", path = "../exoquill-core" } rusqlite = { version = "0.40.1", features = ["bundled"] } serde_json = "1.0.150" diff --git a/crates/exoquill-db/src/lib.rs b/crates/exoquill-db/src/lib.rs index fa545eb..d5dc449 100644 --- a/crates/exoquill-db/src/lib.rs +++ b/crates/exoquill-db/src/lib.rs @@ -5,19 +5,20 @@ use exoquill_core::clock::{now_rfc3339, title_timestamp}; use exoquill_core::note::{ - generate_title, new_note_id, NewNote, NewNoteEvent, Note, NoteEvent, NoteUpdate, - DEFAULT_LANGUAGE_MODE, + generate_title, new_note_id, NewNote, NewNoteEvent, NewNoteVersion, Note, NoteEvent, NoteScope, + NoteSort, NoteSource, NoteUpdate, NoteVersion, DEFAULT_LANGUAGE_MODE, }; use rusqlite::{params, Connection, OptionalExtension, Result, Row}; /// Schema version stamped into `PRAGMA user_version`. Bump when the schema /// changes and add a migration step in [`Database::migrate`]. -const SCHEMA_VERSION: i64 = 1; +const SCHEMA_VERSION: i64 = 3; const SCHEMA: &str = r#" CREATE TABLE IF NOT EXISTS notes ( id TEXT PRIMARY KEY, title TEXT NOT NULL, + title_auto INTEGER NOT NULL DEFAULT 1, content_markdown TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, @@ -48,6 +49,20 @@ CREATE TABLE IF NOT EXISTS note_events ( CREATE INDEX IF NOT EXISTS idx_note_events_note_id ON note_events (note_id); +CREATE TABLE IF NOT EXISTS note_versions ( + id TEXT PRIMARY KEY, + note_id TEXT NOT NULL, + created_at TEXT NOT NULL, + content_md TEXT NOT NULL, + content_hash TEXT NOT NULL, + source TEXT NOT NULL, + op TEXT, + provider_id TEXT, + FOREIGN KEY (note_id) REFERENCES notes (id) +); + +CREATE INDEX IF NOT EXISTS idx_note_versions_note_id ON note_versions (note_id); + CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value_json TEXT NOT NULL, @@ -74,6 +89,12 @@ impl Database { fn from_connection(conn: Connection) -> Result { conn.pragma_update(None, "foreign_keys", "ON")?; + // WAL + NORMAL sync drastically cut write latency for the frequent + // autosave writes (one per ~450 ms while typing) while staying durable + // across app crashes (only the last in-flight txn risks an OS/power loss). + // No-op on the in-memory test DB. Best-effort: never fail open on these. + let _ = conn.pragma_update(None, "journal_mode", "WAL"); + let _ = conn.pragma_update(None, "synchronous", "NORMAL"); let db = Self { conn }; db.migrate()?; Ok(db) @@ -81,17 +102,48 @@ impl Database { fn migrate(&self) -> Result<()> { self.conn.execute_batch(SCHEMA)?; + // v2: `title_auto` tracks whether a note's title is still auto-derived. + // A fresh DB already has it (it's in SCHEMA); only older DBs need the + // column added. Existing notes keep their titles (auto only for the ones + // that clearly were never named). + if !self.column_exists("notes", "title_auto")? { + self.conn.execute_batch( + "ALTER TABLE notes ADD COLUMN title_auto INTEGER NOT NULL DEFAULT 1; \ + UPDATE notes SET title_auto = \ + CASE WHEN title = '' OR title = 'Untitled Note' THEN 1 ELSE 0 END;", + )?; + } + // v3: `note_versions` (edit-history snapshots) is created by the SCHEMA + // batch above (CREATE TABLE IF NOT EXISTS), so older DBs pick it up on + // open with no extra step — the version bump just records the change. self.conn .pragma_update(None, "user_version", SCHEMA_VERSION)?; Ok(()) } + /// Whether `table` has a column named `column` (used to make migrations + /// idempotent regardless of the DB's prior schema version). + fn column_exists(&self, table: &str, column: &str) -> Result { + let mut stmt = self.conn.prepare(&format!("PRAGMA table_info({table})"))?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let name: String = row.get(1)?; + if name == column { + return Ok(true); + } + } + Ok(false) + } + /// Create a note. When `new.title` is `None`, the title is auto-derived. pub fn create_note(&self, new: NewNote) -> Result { let now = now_rfc3339(); let language_mode = new .language_mode .unwrap_or_else(|| DEFAULT_LANGUAGE_MODE.to_string()); + // An explicit title is the user's; a derived one stays auto-tracked so it + // keeps following the content until the user names the note. + let title_auto = new.title.is_none(); let title = new.title.unwrap_or_else(|| { generate_title(&new.content_markdown, new.source, &title_timestamp()) }); @@ -99,6 +151,7 @@ impl Database { let note = Note { id: new_note_id(), title, + title_auto, content_markdown: new.content_markdown, created_at: now.clone(), updated_at: now, @@ -111,11 +164,12 @@ impl Database { self.conn.execute( "INSERT INTO notes \ - (id, title, content_markdown, created_at, updated_at, pinned, archived, deleted_at, language_mode, last_cursor_position) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + (id, title, title_auto, content_markdown, created_at, updated_at, pinned, archived, deleted_at, language_mode, last_cursor_position) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", params![ note.id, note.title, + note.title_auto, note.content_markdown, note.created_at, note.updated_at, @@ -132,11 +186,8 @@ impl Database { /// Fetch a note by id, including soft-deleted ones. pub fn get_note(&self, id: &str) -> Result> { self.conn - .query_row( - "SELECT * FROM notes WHERE id = ?1", - params![id], - row_to_note, - ) + .prepare_cached("SELECT * FROM notes WHERE id = ?1")? + .query_row(params![id], row_to_note) .optional() } @@ -151,6 +202,9 @@ impl Database { } if let Some(v) = update.title { + // Naming the note pins the title; clearing it hands control back to + // the auto-derivation below. + note.title_auto = v.trim().is_empty(); note.title = v; } if let Some(v) = update.content_markdown { @@ -168,14 +222,26 @@ impl Database { if let Some(v) = update.last_cursor_position { note.last_cursor_position = v; } + // Keep an un-named note's title in sync with its content (the first + // meaningful line), so dictation/OCR/typing all surface a useful title. + if note.title_auto { + note.title = generate_title( + ¬e.content_markdown, + NoteSource::Manual, + &title_timestamp(), + ); + } note.updated_at = now_rfc3339(); - self.conn.execute( - "UPDATE notes SET title = ?2, content_markdown = ?3, pinned = ?4, archived = ?5, \ - language_mode = ?6, last_cursor_position = ?7, updated_at = ?8 WHERE id = ?1", + self.conn.prepare_cached( + "UPDATE notes SET title = ?2, title_auto = ?3, content_markdown = ?4, pinned = ?5, \ + archived = ?6, language_mode = ?7, last_cursor_position = ?8, updated_at = ?9 \ + WHERE id = ?1", + )?.execute( params![ note.id, note.title, + note.title_auto, note.content_markdown, note.pinned, note.archived, @@ -187,7 +253,7 @@ impl Database { Ok(Some(note)) } - /// Soft-delete a note. Returns `true` if a live note was deleted. + /// Soft-delete (trash) a note. Returns `true` if a live note was trashed. pub fn delete_note(&self, id: &str) -> Result { let now = now_rfc3339(); let affected = self.conn.execute( @@ -197,26 +263,89 @@ impl Database { Ok(affected > 0) } - /// List all live notes, pinned first, then most recently updated. - pub fn list_notes(&self) -> Result> { - let mut stmt = self.conn.prepare( - "SELECT * FROM notes WHERE deleted_at IS NULL \ - ORDER BY pinned DESC, updated_at DESC", + /// Restore a trashed note (clears `deleted_at`). Returns `true` if a trashed + /// note was restored. + pub fn restore_note(&self, id: &str) -> Result { + let now = now_rfc3339(); + let affected = self.conn.execute( + "UPDATE notes SET deleted_at = NULL, updated_at = ?2 \ + WHERE id = ?1 AND deleted_at IS NOT NULL", + params![id, now], )?; + Ok(affected > 0) + } + + /// Archive or un-archive a live note. Returns `true` if a live note changed. + pub fn set_archived(&self, id: &str, archived: bool) -> Result { + let now = now_rfc3339(); + let affected = self.conn.execute( + "UPDATE notes SET archived = ?2, updated_at = ?3 \ + WHERE id = ?1 AND deleted_at IS NULL", + params![id, archived, now], + )?; + Ok(affected > 0) + } + + /// Permanently delete a note and its dependent rows (events + versions). + /// Returns `true` if a note row was removed. + pub fn hard_delete_note(&self, id: &str) -> Result { + self.conn + .execute("DELETE FROM note_versions WHERE note_id = ?1", params![id])?; + self.conn + .execute("DELETE FROM note_events WHERE note_id = ?1", params![id])?; + let affected = self + .conn + .execute("DELETE FROM notes WHERE id = ?1", params![id])?; + Ok(affected > 0) + } + + /// Permanently delete trashed notes whose `deleted_at` is older than the + /// given RFC-3339 cutoff (the trash-retention cleanup). Returns the count + /// removed. RFC-3339 timestamps sort lexicographically, so a string compare + /// is a valid time compare. + pub fn purge_trash(&self, cutoff_rfc3339: &str) -> Result { + let select = "SELECT id FROM notes WHERE deleted_at IS NOT NULL AND deleted_at < ?1"; + self.conn.execute( + &format!("DELETE FROM note_versions WHERE note_id IN ({select})"), + params![cutoff_rfc3339], + )?; + self.conn.execute( + &format!("DELETE FROM note_events WHERE note_id IN ({select})"), + params![cutoff_rfc3339], + )?; + let affected = self.conn.execute( + "DELETE FROM notes WHERE deleted_at IS NOT NULL AND deleted_at < ?1", + params![cutoff_rfc3339], + )?; + Ok(affected) + } + + /// List notes in `scope` (active / archived / trash), pinned first, then by + /// `sort`. The caller's UI groups the pinned ones; we just keep them on top. + pub fn list_notes(&self, scope: NoteScope, sort: NoteSort) -> Result> { + let sql = format!( + "SELECT * FROM notes WHERE {} ORDER BY pinned DESC, {}", + scope.predicate(), + sort.order_by(), + ); + let mut stmt = self.conn.prepare(&sql)?; let notes = stmt .query_map([], row_to_note)? .collect::>>()?; Ok(notes) } - /// Basic case-insensitive search over title and content of live notes. - pub fn search_notes(&self, query: &str) -> Result> { + /// Case-insensitive search over title and content within `scope`, pinned + /// first then most recently updated. + pub fn search_notes(&self, query: &str, scope: NoteScope) -> Result> { let pattern = format!("%{}%", escape_like(query)); - let mut stmt = self.conn.prepare( - "SELECT * FROM notes WHERE deleted_at IS NULL \ + let sql = format!( + "SELECT * FROM notes WHERE {} \ AND (title LIKE ?1 ESCAPE '\\' OR content_markdown LIKE ?1 ESCAPE '\\') \ ORDER BY pinned DESC, updated_at DESC", - )?; + scope.predicate(), + ); + let mut stmt = self.conn.prepare(&sql)?; let notes = stmt .query_map(params![pattern], row_to_note)? .collect::>>()?; @@ -286,6 +415,99 @@ impl Database { Ok(events) } + /// Record a content snapshot for the edit history, unless it's identical to + /// the note's latest stored version (dedup by content hash → "only on real + /// changes"). Returns the stored version, or `None` when it was a no-op + /// duplicate. `source` defaults to `"manual"`. + pub fn insert_version(&self, new: NewNoteVersion) -> Result> { + let hash = content_hash(&new.content_markdown); + let latest: Option = self + .conn + .query_row( + "SELECT content_hash FROM note_versions WHERE note_id = ?1 \ + ORDER BY created_at DESC, rowid DESC LIMIT 1", + params![new.note_id], + |r| r.get(0), + ) + .optional()?; + if latest.as_deref() == Some(hash.as_str()) { + return Ok(None); + } + let version = NoteVersion { + id: new_note_id(), + note_id: new.note_id, + created_at: now_rfc3339(), + content_markdown: new.content_markdown, + content_hash: hash, + source: new.source.unwrap_or_else(|| "manual".to_string()), + op: new.op, + provider_id: new.provider_id, + }; + self.conn.execute( + "INSERT INTO note_versions \ + (id, note_id, created_at, content_md, content_hash, source, op, provider_id) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + version.id, + version.note_id, + version.created_at, + version.content_markdown, + version.content_hash, + version.source, + version.op, + version.provider_id, + ], + )?; + Ok(Some(version)) + } + + /// A note's stored content snapshots, most recent first. + pub fn list_versions(&self, note_id: &str) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT id, note_id, created_at, content_md, content_hash, source, op, provider_id \ + FROM note_versions WHERE note_id = ?1 ORDER BY created_at DESC, rowid DESC", + )?; + let versions = stmt + .query_map(params![note_id], row_to_version)? + .collect::>>()?; + Ok(versions) + } + + /// Restore a stored version's content into the (live) note as a new, undoable + /// change — non-destructive: the prior content is preserved as history and a + /// fresh `restore` version is recorded. Returns the updated note, or `None` + /// if the version or note is gone (or the note is trashed). + pub fn restore_version(&self, note_id: &str, version_id: &str) -> Result> { + let content: Option = self + .conn + .query_row( + "SELECT content_md FROM note_versions WHERE id = ?1 AND note_id = ?2", + params![version_id, note_id], + |r| r.get(0), + ) + .optional()?; + let Some(content) = content else { + return Ok(None); + }; + let updated = self.update_note( + note_id, + NoteUpdate { + content_markdown: Some(content.clone()), + ..Default::default() + }, + )?; + if updated.is_some() { + self.insert_version(NewNoteVersion { + note_id: note_id.to_string(), + content_markdown: content, + source: Some("op".to_string()), + op: Some("restore".to_string()), + provider_id: None, + })?; + } + Ok(updated) + } + /// Read a setting value (raw JSON string). pub fn get_setting(&self, key: &str) -> Result> { self.conn @@ -312,6 +534,7 @@ fn row_to_note(row: &Row) -> Result { Ok(Note { id: row.get("id")?, title: row.get("title")?, + title_auto: row.get("title_auto")?, content_markdown: row.get("content_markdown")?, created_at: row.get("created_at")?, updated_at: row.get("updated_at")?, @@ -338,6 +561,31 @@ fn row_to_event(row: &Row) -> Result { }) } +fn row_to_version(row: &Row) -> Result { + Ok(NoteVersion { + id: row.get("id")?, + note_id: row.get("note_id")?, + created_at: row.get("created_at")?, + content_markdown: row.get("content_md")?, + content_hash: row.get("content_hash")?, + source: row.get("source")?, + op: row.get("op")?, + provider_id: row.get("provider_id")?, + }) +} + +/// Stable 64-bit FNV-1a hash of `s`, hex-encoded — used to dedup identical note +/// snapshots (a no-op save hashes the same as the previous version). Stable +/// across runs so the dedup survives restarts. +fn content_hash(s: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for b in s.as_bytes() { + hash ^= u64::from(*b); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("{hash:016x}") +} + /// Escape LIKE wildcards in a user query so they match literally. fn escape_like(query: &str) -> String { query @@ -401,12 +649,76 @@ mod tests { assert!(updated.updated_at >= created.updated_at); } + #[test] + fn auto_title_follows_content_until_named() { + let db = Database::open_in_memory().unwrap(); + let n = db.create_note(note("Erste Zeile\nmehr")).unwrap(); + assert!(n.title_auto); + assert_eq!(n.title, "Erste Zeile"); + + // Editing the content re-derives the title while it's still auto. + let n = db + .update_note( + &n.id, + NoteUpdate { + content_markdown: Some("# Neue Überschrift\nText".into()), + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + assert!(n.title_auto); + assert_eq!(n.title, "Neue Überschrift"); + + // Naming the note pins the title; later content edits leave it alone. + let n = db + .update_note( + &n.id, + NoteUpdate { + title: Some("Mein Titel".into()), + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + assert!(!n.title_auto); + let n = db + .update_note( + &n.id, + NoteUpdate { + content_markdown: Some("Komplett anderer Inhalt".into()), + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + assert!(!n.title_auto); + assert_eq!(n.title, "Mein Titel"); + + // Clearing the title hands control back to the auto-derivation. + let n = db + .update_note( + &n.id, + NoteUpdate { + title: Some(" ".into()), + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + assert!(n.title_auto); + assert_eq!(n.title, "Komplett anderer Inhalt"); + } + #[test] fn delete_is_soft_and_hides_from_list() { let db = Database::open_in_memory().unwrap(); let created = db.create_note(note("bye")).unwrap(); assert!(db.delete_note(&created.id).unwrap()); - assert!(db.list_notes().unwrap().is_empty()); + assert!(db + .list_notes(NoteScope::Active, NoteSort::Modified) + .unwrap() + .is_empty()); // Second delete is a no-op. assert!(!db.delete_note(&created.id).unwrap()); // Row still exists (soft delete) but updates are rejected. @@ -423,7 +735,12 @@ mod tests { db.insert_at("a", "2026-06-19T10:00:00.000Z", false); db.insert_at("b", "2026-06-19T12:00:00.000Z", false); db.insert_at("c", "2026-06-19T09:00:00.000Z", true); - let ids: Vec = db.list_notes().unwrap().into_iter().map(|n| n.id).collect(); + let ids: Vec = db + .list_notes(NoteScope::Active, NoteSort::Modified) + .unwrap() + .into_iter() + .map(|n| n.id) + .collect(); assert_eq!(ids, vec!["c", "b", "a"]); } @@ -432,9 +749,153 @@ mod tests { let db = Database::open_in_memory().unwrap(); db.create_note(note("Rust und WebGPU")).unwrap(); db.create_note(note("Einkaufsliste")).unwrap(); - assert_eq!(db.search_notes("webgpu").unwrap().len(), 1); + assert_eq!( + db.search_notes("webgpu", NoteScope::Active).unwrap().len(), + 1 + ); // Wildcards are treated literally, not as SQL LIKE patterns. - assert_eq!(db.search_notes("%").unwrap().len(), 0); + assert_eq!(db.search_notes("%", NoteScope::Active).unwrap().len(), 0); + } + + #[test] + fn scopes_partition_active_archived_trash() { + let db = Database::open_in_memory().unwrap(); + let active = db.create_note(note("active one")).unwrap(); + let arch = db.create_note(note("archived one")).unwrap(); + let trash = db.create_note(note("trashed one")).unwrap(); + assert!(db.set_archived(&arch.id, true).unwrap()); + assert!(db.delete_note(&trash.id).unwrap()); + + let ids = |scope| { + db.list_notes(scope, NoteSort::Modified) + .unwrap() + .into_iter() + .map(|n| n.id) + .collect::>() + }; + assert_eq!(ids(NoteScope::Active), vec![active.id.clone()]); + assert_eq!(ids(NoteScope::Archived), vec![arch.id.clone()]); + assert_eq!(ids(NoteScope::Trash), vec![trash.id.clone()]); + + // Search is scoped too: a term in the archived note isn't found in Active. + assert!(db + .search_notes("archived", NoteScope::Active) + .unwrap() + .is_empty()); + assert_eq!( + db.search_notes("archived", NoteScope::Archived) + .unwrap() + .len(), + 1 + ); + } + + #[test] + fn sort_by_title_is_case_insensitive() { + let db = Database::open_in_memory().unwrap(); + db.create_note(note("banana")).unwrap(); + db.create_note(note("Apple")).unwrap(); + let titles: Vec = db + .list_notes(NoteScope::Active, NoteSort::Title) + .unwrap() + .into_iter() + .map(|n| n.title) + .collect(); + assert_eq!(titles, vec!["Apple", "banana"]); + } + + #[test] + fn restore_brings_a_trashed_note_back_to_active() { + let db = Database::open_in_memory().unwrap(); + let n = db.create_note(note("oops")).unwrap(); + db.delete_note(&n.id).unwrap(); + assert!(db.restore_note(&n.id).unwrap()); + assert_eq!( + db.list_notes(NoteScope::Active, NoteSort::Modified) + .unwrap() + .len(), + 1 + ); + // Restoring a live note is a no-op. + assert!(!db.restore_note(&n.id).unwrap()); + } + + #[test] + fn hard_delete_removes_the_row_and_dependents() { + let db = Database::open_in_memory().unwrap(); + let n = db.create_note(note("gone")).unwrap(); + db.insert_version(NewNoteVersion { + note_id: n.id.clone(), + content_markdown: "gone".into(), + ..Default::default() + }) + .unwrap(); + assert!(db.hard_delete_note(&n.id).unwrap()); + assert!(db.get_note(&n.id).unwrap().is_none()); + assert!(db.list_versions(&n.id).unwrap().is_empty()); + assert!(!db.hard_delete_note(&n.id).unwrap()); + } + + #[test] + fn purge_trash_removes_only_old_trashed_notes() { + let db = Database::open_in_memory().unwrap(); + // A trashed note with an old deleted_at, and a live one. + let old = db.create_note(note("old")).unwrap(); + let live = db.create_note(note("live")).unwrap(); + db.conn + .execute( + "UPDATE notes SET deleted_at = '2020-01-01T00:00:00.000Z' WHERE id = ?1", + params![old.id], + ) + .unwrap(); + let purged = db.purge_trash("2026-01-01T00:00:00.000Z").unwrap(); + assert_eq!(purged, 1); + assert!(db.get_note(&old.id).unwrap().is_none()); + assert!(db.get_note(&live.id).unwrap().is_some()); + } + + #[test] + fn versions_dedup_and_restore() { + let db = Database::open_in_memory().unwrap(); + let n = db.create_note(note("v1")).unwrap(); + assert!(db + .insert_version(NewNoteVersion { + note_id: n.id.clone(), + content_markdown: "v1".into(), + ..Default::default() + }) + .unwrap() + .is_some()); + // Identical content is deduped (no-op save adds nothing). + assert!(db + .insert_version(NewNoteVersion { + note_id: n.id.clone(), + content_markdown: "v1".into(), + ..Default::default() + }) + .unwrap() + .is_none()); + let v1 = db.list_versions(&n.id).unwrap()[0].clone(); + // A second, different snapshot is stored. + db.update_note( + &n.id, + NoteUpdate { + content_markdown: Some("v2".into()), + ..Default::default() + }, + ) + .unwrap(); + db.insert_version(NewNoteVersion { + note_id: n.id.clone(), + content_markdown: "v2".into(), + ..Default::default() + }) + .unwrap(); + assert_eq!(db.list_versions(&n.id).unwrap().len(), 2); + // Restoring v1 writes its content back and records a fresh version. + let restored = db.restore_version(&n.id, &v1.id).unwrap().unwrap(); + assert_eq!(restored.content_markdown, "v1"); + assert_eq!(db.list_versions(&n.id).unwrap().len(), 3); } #[test] diff --git a/docs/decisions.md b/docs/decisions.md index 36c39bc..1105c3c 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -17,10 +17,14 @@ document wins. | D2 | TTS provider | **Piper**, bundled, behind a provider interface | | D3 | Editor | **TipTap** + `tiptap-markdown` | | D4 | Screen-region OCR | **v0.2** (out of v0.1) | -| D5 | Model delivery | **All models bundled**, no model manager in v0.1 | +| D5 | Model delivery | All models bundled, no model manager in v0.1 — **revised by D9** | | D6 | Formatting UX | **Replace + Undo everywhere**, no preview/diff in v0.1 | | D7 | External insertion | **No** — note-internal only | | D8 | Runtime isolation | AI runtimes run as **isolated processes** (stability, not license)| +| D9 | Model acquisition | **3-tier** (bundle / download / gated) + in-app **model manager**| +| D10| Multilingual TTS | **XTTS** + **Zonos** as opt-in sidecars, **backend-selectable** in the UI | +| D11| TTS backend roadmap | Ranked backend lineup; **Chatterbox Multilingual** adopted as the MIT high-quality slot | +| D12| Notes management & history | Scopes (active/archived/trash), pin group, soft-delete + undo toasts, multi-select, diff history | --- @@ -124,3 +128,202 @@ processes**, not in-process libraries. **stability** requirement: a crashing inference process must not take down the app (cf. PLAN.md §24.6 "Provider crashed → Restart provider"). This is an architecture rule for PR 2 onward. + +## D9 — Model acquisition: three tiers + in-app model manager + +**Revises D5.** D5 ("all models bundled, no manager") doesn't survive contact with the real +license landscape (below) or with heavy/optional models like XTTS-v2. We separate the **app** +(GPL-3.0, bundled) from the **models** (each with its own license), and acquire each model by +its license class. + +**Decision.** Every model/voice is one of three tiers, surfaced in an in-app **model manager** +(install / use / delete per entry, with a license gate for restrictive ones): + +- **bundled** — ships in the installer. Only assets that are redistributable **and** + GPL-3.0-compatible **and** commercial-OK. +- **download** — free + clean, but fetched on demand (keeps the installer slim). Same license + freedoms as bundled; the only difference is delivery. +- **gated** — restrictive license (e.g. non-commercial). Never bundled; on-demand download + only, behind an explicit license-acceptance step; hidden entirely in a commercial build via + a build flag. + +**Mechanism.** A shipped manifest (`apps/desktop/src-tauri/models.json`) is the single source +of truth: per entry `id, provider, kind, language, license, commercialOk, tier, files[] (url + +relPath), setup?, notes?`. Backend commands `list_catalog` / `install_model` (streaming +download to a writable models root, `model_progress` events, `.part`→rename) / `delete_model`. +The app-level `license` rule (D1) still governs only runtime **code**; model **weights** are +data with their own terms (the manager enforces them). + +**Verified license matrix** (read from canonical LICENSE files / HF MODEL_CARDs, 2026-06-20): + +| Asset | License | Commercial | Redistribute | GPL-3.0 bundle | Tier | +|---|---|---|---|---|---| +| Piper runtime (`piper1-gpl`) | GPL-3.0 | yes | yes | yes | **bundle** | +| Voice de_DE-thorsten-high | CC0 | yes | yes | yes | **bundle** | +| Voice en_GB-cori-high | Public Domain (LibriVox) | yes | yes | yes | **bundle** (currently download) | +| Voice en_US-lessac-high | CSTR Blizzard 2013 (research) | **no** | **no** | **no** | **EXCLUDE** | +| Voice en_US-ryan-high | CC BY-NC-SA 4.0 | **no** | yes (NC+SA) | **no** | **gated** | +| whisper.cpp runtime | MIT | yes | yes | yes | **bundle** | +| Whisper ggml-large-v3-turbo | MIT | yes | yes | yes | **bundle** (or download, ~1.6 GB) | +| llama.cpp runtime | MIT | yes | yes | yes | **bundle** | +| Qwen2.5-**1.5B**-Instruct | Apache-2.0 | yes | yes | yes | **bundle** (1.5B only!) | +| Tesseract engine | Apache-2.0 | yes | yes | yes | **bundle** | +| tessdata deu/eng (best/fast) | Apache-2.0 | yes | yes | yes | **bundle** | +| Silero VAD | MIT | yes | yes | yes | **bundle** (verify weights provenance) | +| ONNX Runtime | MIT | yes | yes | yes | **bundle** | +| coqui-tts library (idiap fork) | MPL-2.0 | yes | yes | yes | **bundle** (code only) | +| XTTS-v2 **weights** | CPML 1.0 | **no** | restricted | **no** | **gated** (never bundle) | +| Zonos-v0.1 **weights** | Apache-2.0 | yes | yes | yes | **download** (needs CUDA GPU) | + +**Consequences / open items.** +- **Three blockers, all model/data side:** `lessac` (research-only, **exclude entirely** — not + even download), `ryan` (NC → gated only), XTTS-v2 weights (CPML NC → gated only). +- **The en_US bundle slot needs a clean voice:** both vetted en_US candidates are unusable; + pick a CC0/Public-Domain/permissive en_US Piper voice before shipping a default English voice. +- **Qwen size trap:** 1.5B is Apache-2.0 (clean); 3B/72B use the restrictive Qwen License — do + not swap up without re-checking. +- **Attribution:** ship a `THIRD-PARTY-LICENSES` file (MIT notices, Apache-2.0 LICENSE+NOTICE + for Tesseract/tessdata/Qwen, MPL-2.0 source availability for any modified files). +- **Provenance check:** confirm the Silero `.onnx` came from the official MIT repo, not an old + CC-BY-NC snapshot. +- XTTS install is special (Python + PyTorch + ~1.75 GB weights, all 58 speakers in one 7.4 MB + embeddings file) — modeled as a `gated` runtime entry installed via `scripts/setup-xtts.ps1`, + not a plain file download. A packaged self-contained sidecar is a later improvement. + +*Informational, not legal advice — confirm the lessac/ryan exclusions and the CPML gate with +counsel before a commercial release.* + +## D10 — Multilingual TTS backends: XTTS + Zonos, backend-selectable + +- Status: **accepted** · Date: 2026-06-21 · Extends D2 + +**Context.** Piper (D2) is the bundled default — the only reliable high-quality German +voice — but it's single-language (espeak phonemes) and weak on mixed DE/EN technical +terms, which are common in these notes. We want optional **multilingual neural** voices +without giving up the on-device principle, and the UI now exposes a **backend picker** +(Piper / XTTS / Zonos) plus voice + speed in the toolbar; the rest of the knobs stay in +the settings overlay. + +**Decision.** Offer multilingual TTS as **opt-in local sidecars** behind the existing +`TextToSpeechProvider` interface, each auto-spawned and warmed in the background, each +listing its voices into one merged picker (every voice carries its `provider`, which the +`tts_speak` command routes on). Two are wired: + +- **XTTS-v2** — `gated` (CPML, non-commercial weights). All ~58 studio speakers; one voice + per speaker, language auto-detected per segment (no de/en duplicates). Test-only. +- **Zonos-v0.1** — `download` (**Apache-2.0** weights → commercial-OK, the key advantage + over XTTS). Voices are **cloned** from reference `.wav` clips in a folder (like Piper + enumerates a voice folder); language auto-detected per segment. Needs a **CUDA GPU** + (CPU is unusably slow); depends on eSpeak NG (bundled via `espeakng-loader`); output is + 44.1 kHz. Installed via `scripts/setup-zonos.ps1` + `scripts/zonos-server.py`. + +**Candidates evaluated (for "try a better TTS than XTTS").** + +| Engine | Weights license | Local? | German | Verdict | +|---|---|---|---|---| +| **Zonos-v0.1** | **Apache-2.0** | yes (GPU) | yes | **Adopted** — only option clean for a commercial GPL build | +| Cartesia Sonic | proprietary, no open weights | cloud API only (on-prem = enterprise) | yes | **Rejected** — breaks the on-device/offline principle | +| Fish Speech / OpenAudio S1 | CC-BY-NC-SA (S2 reportedly MIT, in flux) | yes (GPU) | yes | **Deferred** — same NC problem as XTTS; test-only at best | + +**Rationale.** Zonos is the only one of the three whose weights are permissive enough to +ship in a commercial GPL-3.0 build, so it's the strategic successor to XTTS for the +multilingual slot. Cartesia is cloud-only (privacy + offline regression). Fish Speech is +local and good but non-commercial, i.e. no better than XTTS on the blocking constraint. + +**Open items.** +- Zonos Python sidecar is **unverified end-to-end** (no CUDA/Zonos test env at integration + time) — confirm `make_cond_dict`/`generate`/`autoencoder.decode` shapes and the 44.1 kHz + rate on first real run. +- A packaged self-contained sidecar (no manual venv) is a later improvement, as for XTTS. +- Bundled reference voices: ship a clean, license-clear default clip if Zonos becomes a + first-class (non-experimental) backend. + +## D11 — TTS backend roadmap: ranked lineup + Chatterbox Multilingual + +- Status: **accepted** · Date: 2026-06-22 · Extends D2 / D10 + +**Context.** D2 set Piper as the bundled German default; D10 added XTTS + Zonos as opt-in +multilingual sidecars behind `TextToSpeechProvider`. The neural-TTS field moved fast in early +2026 (Chatterbox Multilingual, Qwen3-TTS, ZONOS2, CosyVoice 3), and several new options are +**permissively licensed** — which matters because a commercial GPL-3.0 build can only *bundle* +weights that are redistributable **and** commercial-OK (D9). This decision records the target +lineup and ranks each backend by role, so future TTS work has one ordered list to pull from. + +**Decision.** Adopt the ranked lineup below. The architecture is unchanged: every backend is a +`TextToSpeechProvider`, heavy neural models run as an auto-spawned local sidecar (Python HTTP +server + thin blocking Rust client, like `xtts`/`zonos`), and each lists its voices into the one +merged, backend-routed picker. New backends are added incrementally — **Chatterbox Multilingual +is the next one to wire** (the MIT high-quality slot, the cleanest successor to XTTS/Zonos). + +| Rank | Role | Model(s) | License (verified) | Tier | Why / status | +|---|---|---|---|---|---| +| 1 | **Safe default** | **Piper + good German voice** (Thorsten) | GPL-3.0 runtime / CC0 voice | **bundle** | Fast, robust, local, minimal fuss; the proven D2 default. **Wired.** | +| 2 | **Modern fast default / Fast Mode** | **Kokoro-82M** | Apache-2.0 | download | Tiny, permissive, more natural than classic mini-TTS. **Blocker:** no *official* German (community voice only) — must pass a German quality bar before it can be a default (see D2). | +| 3 | **Best optional high-quality backend** | **Chatterbox Multilingual (v3)** | **MIT** | download (GPU) | 23+ languages incl. German, voice cloning, emotion, realistic product integration. MIT → first multilingual option that is *redistributable + commercial-OK*. **Caveat:** embeds a Resemble "Perth" neural watermark in every output by default, no documented opt-out — disclose this for an offline/privacy tool. **Adopted, next to wire.** | +| 4 | **Power backend for GPU users** | **Qwen3-TTS** (0.6B / 1.7B) | Apache-2.0 (open weights, not API-only) | download (GPU) | German among 10 languages, streaming, voice design. Open weights on HF (~2.5 / 4.5 GB); Windows-native path exists. | +| 5 | **Experimental premium GPU backend** | **ZONOS2** | Apache-2.0 (HF model card; "MIT" in some write-ups is wrong) | download (GPU) | Strong cloning, 8B-MoE / 900M active. **Hard caveat:** *Linux-only (x86_64) + CUDA*, ~20× slower than realtime on an 8 GB consumer GPU → only viable via WSL2 / strong GPU; unusable for live read-aloud on typical hardware. Keep behind the existing Zonos slot until a Windows path exists. | +| 6 | **Server / research backend** | **CosyVoice 3** | Apache-2.0 (lineage) | download (GPU) | Strong, multilingual, ~150 ms class, but higher integration/deployment cost; German support unconfirmed. Lower priority. | +| 7 | **Support only, do not bundle** | **F5-TTS, XTTS-v2, Fish Speech 1.5** | NC / custom (XTTS = CPML) | **gated** | Technically relevant but non-commercial/custom licenses → never a clean bundle core. XTTS is already wired as a `gated` test-only sidecar (D10). | + +**Rationale.** Chatterbox is the strategic pick for the high-quality multilingual slot: **MIT** +beats both XTTS (CPML, non-commercial → gated) and Zonos (Apache but GPU/Linux-bound), it covers +German, and it slots into the existing sidecar pattern with no new architecture. Qwen3-TTS and +CosyVoice 3 are also permissive and worth supporting for GPU users, but rank below Chatterbox on +maturity/integration cost. ZONOS2 is permissive but platform-blocked today. + +**Open items.** +- **Chatterbox watermark:** confirm whether the Perth watermark can be disabled or must be + disclosed; decide how the UI/About screen surfaces "all generated audio is watermarked." +- **Kokoro German:** validate the community German voice against the German-first quality bar + before promoting Kokoro to the Fast-Mode default (D2 still says "too risky" until then). +- **GPU realism:** ranks 3–6 all want a GPU; document the CPU-fallback story (Piper stays the + no-GPU default) and per-backend hardware notes in the model manager. +- **Catalog entries:** add Chatterbox (and later Qwen3-TTS) to `models.json` as `download` + runtime entries with a `setup` script, mirroring the Zonos entry; never bundle weights until + installer-size + watermark questions are resolved. + +*Informational, not legal advice — re-verify each weight license (and the Chatterbox watermark +terms) against its canonical LICENSE/model card before a commercial release.* + +## D12 — Notes management & edit history + +- Status: **accepted** · Date: 2026-06-22 · Implements design "Bereich 1 — Notizverwaltung & Historie" + +**Context.** The design exploration settled on **Direction B "Local AI Utility"** (the green +on-device accent + dense IBM Plex typography — already the tokens in `theme.css`). Its first +worked-through area, *Bereich 1*, specifies a real note-management layer: scopes, pinning, +soft-delete, multi-select, and a diff-based edit history. + +**Decision.** Implement it as designed, on the existing schema where possible: + +- **Scopes.** Sidebar tabs **Active / Archived / Trash**; `list_notes(scope, sort)` / + `search_notes(q, scope)` filter by scope. `active = deleted_at IS NULL AND archived = 0`, + `archived = archived = 1`, `trash = deleted_at IS NOT NULL`. Sort by modified / created / title; + pinned always first (the UI renders the pin group only in Active). +- **Pinning.** A pinned group on top; toggle via hover icon or context menu; the pin uses its own + **amber** colour, distinct from the green action accent. +- **Soft-delete + undo, no modal confirms.** `delete_note` trashes (sets `deleted_at`); + `restore_note` un-trashes; `hard_delete_note` and `purge_trash(before)` are the permanent ops + (30-day retention cutoff computed client-side). Every reversible action shows an **undo toast** + (6 s, also Ctrl/⌘+Z outside the editor); permanent deletes show a plain toast. +- **Multi-select.** Ctrl/⌘- and Shift-click select; a bulk action bar runs pin / archive / export / + trash as one undoable batch. +- **Edit history (diff).** New `note_versions` table (`content_md`, `content_hash`, `source` + manual|op, `op`, `provider_id`); snapshots are **deduped by content hash** (no-op saves add + nothing). Snapshots are written on note-switch (manual baseline) and after format/OCR (op). A + timeline overlay shows versions with op badges + word deltas and a **word-level diff** (own LCS, + no npm) of the selected version against the current content; `restore_note_version` writes the + old content back as a new, non-destructive, undoable version. + +**Schema.** `SCHEMA_VERSION` 2 → 3; `note_versions` is created idempotently (`CREATE TABLE IF NOT +EXISTS`), so older DBs pick it up on open with no data migration. + +**Verification.** DB/core unit tests cover scope partitioning, sort, restore, hard-delete, purge, +and version dedup/restore; the Tauri crate type-checks; a headless visual pass (chromium, mocked +`invoke`) confirmed every scope + interaction state against the wireframes in both themes +(screenshots under `.workspace/shots/`). + +**Open items.** Manual snapshots are currently taken on note-switch only (not on a timed typing +pause); a dictation-stop snapshot isn't wired yet. Bulk export opens one native save dialog per +note (WebView2 can't batch downloads). Remaining design areas (Bereich 2–4: action handling, +settings, …) are not yet in scope. diff --git a/docs/design-prompt.md b/docs/design-prompt.md new file mode 100644 index 0000000..2c4c118 --- /dev/null +++ b/docs/design-prompt.md @@ -0,0 +1,218 @@ +# Design-/Feature-Auftrag für ExoQuill (neue Claude-Session) + +> Diesen Text in einer **neuen Chat-Session** derselben Projektmappe einfügen. +> Er ist als kompletter Auftrag formuliert und enthält den nötigen Projektkontext. + +--- + +## Rolle & Kontext + +Du arbeitest an **ExoQuill**, einer datenschutzfreundlichen, **vollständig on-device** +Notiz-App. Alles läuft lokal, ohne Cloud: Diktat (Whisper), Texterkennung +(Tesseract), Formatierung (llama.cpp) und Vorlesen (Piper/XTTS/Zonos). + +**Tech-Stack** +- **Desktop-Shell:** Tauri v2 (Windows-first), Lizenz **GPL-3.0-only**. +- **Frontend:** React 18 + TypeScript + Vite, Editor auf Basis von **Tiptap** + (Markdown als Speicherformat). Styles in handgeschriebenem CSS mit + CSS-Variablen (Light/Dark-Theme). +- **Backend:** Rust-Workspace. + - `crates/exoquill-core` – Domänenmodell (`note.rs`), Jobs, Events, Clock. + - `crates/exoquill-db` – SQLite-Persistenz (Notizen, `note_events`, `settings`). + - `crates/exoquill-ai` – KI-Provider (STT/OCR/Formatter/TTS) + Sidecars. + - `crates/exoquill-audio`, `crates/exoquill-capture` – Audio/Screen-Capture. + - `apps/desktop/src-tauri` – Tauri-Commands (`notes.rs`, `jobs.rs`, `models.rs`, + `dictation.rs`, `lib.rs`). + +**Arbeitsteilung:** Normalerweise besitzt der Nutzer UI/Branding, Claude die +Internals. **Für diesen Auftrag bekommst du ausdrücklich volle gestalterische +Freiheit** für UI, UX, QoL und Features – Branding-Grundelemente (Wortmarke +„exoquill“, grüner Akzent, Logo-Mark) bitte respektieren bzw. nur behutsam +weiterentwickeln. + +**Sprache:** Mit dem Nutzer immer auf **Deutsch** kommunizieren (Code/Identifier +im Original). Vollständige deutsche Orthografie inkl. Umlaute/ß. + +--- + +## Was gerade frisch umgebaut wurde (NICHT rückgängig machen) + +Diese Änderungen sind die Ausgangsbasis – darauf aufbauen: + +1. **i18n (DE/EN):** `apps/desktop/src/lib/i18n.ts` ist ein leichtgewichtiges + Lokalisierungssystem (`useI18n()`-Hook + `translate()`), persistiert die Sprache + in `localStorage`, Default nach Browser-Sprache. **Jeder neue UI-Text muss über + diese Tabellen (de/en) laufen** – keine hartkodierten Strings mehr. +2. **Eine einzige Toolbar:** Die frühere oberste Leiste (`Topbar`) wurde entfernt. + Modelle-Button, Sprach-Umschalter (Globus + DE/EN) und Theme-Toggle leben jetzt + rechts in `components/ActionBar.tsx`; links die notizbezogenen Aktionen. Die + Wortmarke sitzt jetzt im Sidebar-Kopf (`components/Sidebar.tsx`). +3. **Auto-Titel:** Notiztitel folgen automatisch dem Inhalt (erste sinnvolle + Zeile), solange der Nutzer den Titel nicht selbst gesetzt hat. Backend: + `Note.title_auto` (in `exoquill-core/note.rs` + `exoquill-db`), Regeln in + `Database::update_note`. +4. **OCR-Auswahl gefixt:** `components/OcrOverlay.tsx` rekonstruiert markierten Text + aus den Wort-Boxen (Leerzeichen/Zeilenumbrüche) und bietet „Alles auswählen“ + (Strg+A). + +**Vorhandene Systeme zum Wiederverwenden:** +- Theme: `hooks/useTheme.ts` (`data-theme`-Attribut, persistiert). +- Job-Queue + Event-Bus: `backend-event` (`job_updated`, `notes_changed`), + `model_progress`. Schwere Aktionen laufen als Jobs (siehe `jobs.rs`). +- Verlauf: Tabelle `note_events` speichert pro Operation `raw_text`, + `processed_text`, `operation`, `provider_id`, … – Command `list_note_events`. +- Soft-Delete: `Note.deleted_at` existiert bereits (aktuell kein UI dafür). +- Pin/Archiv: `Note.pinned` und `Note.archived` existieren in Schema + `NoteUpdate` + (Pin wird sortiert; Archiv hat noch **kein** UI und wird im List-Query noch nicht + gefiltert). +- API-Wrapper: `apps/desktop/src/lib/api.ts` (typed `invoke`-Wrapper). + +--- + +## Auftrag – vier Bereiche + +> Liefere **echte Implementierung** (Komponenten, CSS, ggf. neue Tauri-Commands + +> Rust-Logik + Tests), nicht nur Mockups. Arbeite inkrementell und halte die +> bestehenden IPC-Verträge stabil bzw. erweitere sie sauber (serde camelCase, +> Typen in `lib/types.ts` spiegeln). Alles bleibt offline/on-device. + +### 1. Notiz-Verwaltung & Editier-Historie + +Gestalte das **Verwalten, Bearbeiten, Löschen, Archivieren, Pinnen** von Notizen +neu und vollständig: + +- **Pinnen:** Toggle in Sidebar-Item und/oder Kontextmenü; visuell klar (Pin-Icon, + Gruppierung „Angeheftet“). Backend kann `pinned` schon. +- **Archivieren:** UI zum Archivieren/Wiederherstellen + eigene Ansicht/Filter + („Aktiv | Archiviert | Papierkorb“). **Backend-Arbeit nötig:** `list_notes`/ + `search_notes` filtern Archiv aktuell nicht – Query/Command anpassen + (z. B. Parameter `scope`). +- **Löschen:** Soft-Delete (`deleted_at`) existiert. Baue einen echten + **Papierkorb** mit Wiederherstellen + endgültigem Löschen (neuer Command für + Hard-Delete + ggf. Aufräum-Routine). Bestätigungen mit Undo-Toast statt nur + `window.confirm`. +- **Bearbeiten/Organisation:** Mehrfachauswahl, Sortierung (zuletzt geändert / + erstellt / Titel), evtl. Tags/Ordner als optionales Feature (Schema-Erweiterung + mit Migration, Muster in `exoquill-db` beachten: `SCHEMA_VERSION` + `migrate()`). +- **Kontextmenü** pro Notiz (Rechtsklick) mit allen Aktionen. + +**Editier-Historie „nur bei Diffs“, mit Sprung + Diff gegen aktuellen Stand:** + +- Heute existiert nur ein metadaten-orientierter Verlauf (`note_events`) mit + `raw_text`/`processed_text` für **Operationen** (Format/OCR/Diktat). Manuelles + Tippen ist nicht versioniert. +- Entwirf eine echte **Versions-/Diff-Historie**: + - **Entscheide & dokumentiere** den Ansatz: entweder (a) auf `note_events` + aufbauen (Snapshots je Operation) oder (b) eine **Snapshot-/Versionstabelle** + ergänzen (z. B. periodische/abgegrenzte Inhalts-Snapshots), oder eine + Kombination. Empfehlung beim Nutzer einholen, wenn größere Schema-Änderung. + - **„nur bei Diffs“:** Im Verlauf nur Einträge zeigen, bei denen sich der Inhalt + tatsächlich geändert hat (kein Rauschen durch No-Op-Saves). + - **Diff-Ansicht:** Gegenüberstellung früher ↔ aktuell (zeilen-/wortweise, + Markdown-bewusst). Es gibt bereits ein Vorschau-Panel + (`.preview-cols`/`.preview-col` in `styles/app.css`) als Stilreferenz. + - **Springen / Wiederherstellen:** Zu einer Version springen (read-only Vorschau) + und „diese Version wiederherstellen“ (schreibt als neue, undobare Änderung – + nicht destruktiv). + +### 2. Aktions-Handling für ganze Notiz **und** Auswahl + +Gestalte das Zusammenspiel von **OCR, Format, Export, Import, Diktat (Dictate), +Vorlesen (Read)** neu – jeweils sauber **für die gesamte Notiz** *und* **für eine +Auswahl innerhalb der Notiz**. + +- **Status heute:** + - Format & Read berücksichtigen bereits eine Auswahl (`selectionText`), sonst + ganze Notiz. + - OCR fügt am Cursor ein bzw. legt neue Notiz an. + - Diktat: ersetzt Auswahl / fügt am Cursor ein. + - **Export** existiert nur für die **ganze** Notiz als Markdown + (`export_note`), nicht für Auswahl. + - **Import** existiert **noch gar nicht** → neuer Tauri-Command nötig + (Datei-Dialog, `.md`/`.txt` → neue Notiz oder am Cursor einfügen; ggf. + Mehrfach-Import). +- **Ziel:** Ein **einheitliches, entdeckbares Interaktionsmodell**: + - **Auswahl-Bubble-Menü** (Tiptap BubbleMenu o. Ä.), das bei Markierung + erscheint und kontextbezogen Format/Read/Export-Auswahl/Diktat-ersetzt/„als + neue Notiz“ anbietet. + - Klare Trennung „wirkt auf Auswahl“ vs. „wirkt auf ganze Notiz“ (Label/Tooltip/ + State), damit nie unklar ist, was passiert. + - **Export der Auswahl** (Markdown/Plaintext) ergänzen; Export-Formate erweitern + (z. B. `.txt`, evtl. `.pdf`/`.html` – nur wenn offline gut machbar). + - Fortschritts-/Abbrechen-UX für lange Läufe vereinheitlichen (heute mehrere + `dictation-bar`-Varianten in `App.tsx`). + - Tastenkürzel für alle Aktionen, konsistent dokumentiert. + +### 3. Neues Settings-Fenster + +Ein vollwertiges, gut strukturiertes **Einstellungs-Fenster** (Dialog oder eigene +Route), das **alles** bündelt. Heute verstreut: Modell-Manager +(`components/ModelManager.tsx`), Vorlese-Einstellungen +(`components/ReadAloudSettings.tsx`), Theme-Toggle, Sprach-Umschalter. + +Inhalte (als Tabs/Sektionen): + +- **Modelle & Runtimes:** Katalog installieren/löschen, Download-Fortschritt, + Lizenz-/Tier-Badges (vorhanden via `list_catalog`/`install_model`/`delete_model`, + `model_progress`), aktive Provider + Status/Health (`list_model_info`). Pfade, + Speicherort, belegter Speicher, „nicht-kommerziell“-Gates. +- **Vorlesen/Audio:** Backend-/Stimmen-Wahl, Tuning (vorhanden in + `ReadAloudSettings`), Sprach-Aufbereitung (LLM) erklären. +- **Diktat:** Quelle/Loopback, Sprache (`languageMode`), VAD-/Gain-Optionen + (siehe `startDictation`-Optionen in `api.ts`). +- **Darstellung:** Theme (Light/Dark/System), Sprache (DE/EN), Editor-Optionen + (Schriftgröße/Breite), Akzentfarbe. +- **About:** Version, Lizenz (GPL-3.0), verwendete Open-Source-Modelle/Runtimes + + deren Lizenzen, Links/Credits. +- **Updates:** Update-Seite/Feature (Tauri-Updater prüfen; falls noch nicht + konfiguriert: UI + Backend-Anbindung entwerfen, mind. „nach Updates suchen“, + Versionsanzeige, Changelog-Hook). Offline-tauglich/optional halten. + +Persistenz: Es gibt bereits eine `settings`-Tabelle (`get_setting`/`set_setting`, +JSON-Werte) – nutze sie für serverseitig relevante Settings; reine UI-Settings +dürfen in `localStorage` bleiben (Konsistenz mit bestehendem Code, z. B. +`tts-voice`, `tts-tuning`, `exoquill-theme`, `exoquill-lang`). + +### 4. Freie Quality-of-Life- & UX-/Performance-Verbesserungen + +Volle kreative Freiheit für sinnvolle Verbesserungen, z. B.: + +- Command-Palette (Strg+K), globale Shortcuts-Übersicht, bessere Suche + (Treffer-Hervorhebung, Filter). +- Editor-QoL: Slash-Commands, bessere Markdown-Toolbar/BubbleMenu, Outline, + Wortzähler/Lesezeit, Auto-Save-Indikator-Feinschliff. +- Performance: virtualisierte Notizliste bei vielen Notizen, Diff-Berechnung + effizient, Editor-Remounts vermeiden (`reloadKey`-Muster in `App.tsx` ansehen). +- A11y: Fokus-Management, Tastaturbedienung, ARIA, Kontraste. +- Onboarding/Empty-States, Toaster/Benachrichtigungen statt blockierender + `window.confirm`/`alert`. + +Größere Architektur-/Schema-Eingriffe oder neue Abhängigkeiten vorher kurz mit dem +Nutzer abstimmen. + +--- + +## Leitplanken (Definition of Done) + +- **Offline/On-device** bleibt Pflicht – keine externen Netzwerk-Calls für + Kernfunktionen. +- **Zweisprachig:** alle neuen Texte über `lib/i18n.ts` (de + en gepflegt). +- **Theme-fähig:** CSS über bestehende Variablen, Light **und** Dark testen. +- **IPC sauber:** neue Commands in `src-tauri` registrieren (`lib.rs` + `invoke_handler`), Typen in `lib/types.ts` spiegeln (serde `camelCase`). +- **DB-Migrationen** über `SCHEMA_VERSION` + idempotentes `migrate()` (Muster in + `exoquill-db/src/lib.rs`), bestehende Daten nicht zerstören. +- **Tauri-Threading:** schwere IPC-Commands als sync `fn` (laufen off-main-thread) + bzw. über die Job-Queue – nicht `async` blockierend. +- **Tests** für Backend-Logik (siehe vorhandene `#[cfg(test)]`-Module) und + Typecheck grün: `cd apps/desktop && npx tsc --noEmit`, `cargo test --workspace`. +- Inkrementell vorgehen, Zwischenstände erklären, Empfehlungen aktiv geben. + +## Erste Schritte (Vorschlag) + +1. Repo-Kontext sichten: `docs/decisions.md`, `docs/roadmap.md`, `App.tsx`, + `components/`, `exoquill-db/src/lib.rs`, `exoquill-core/src/note.rs`, + `src-tauri/src/{notes,jobs,models}.rs`. +2. Kurzes Umsetzungskonzept je Bereich vorschlagen (inkl. nötiger Schema-/IPC- + Änderungen) und offene Design-Entscheidungen mit dem Nutzer klären. +3. Bereich für Bereich umsetzen, mit Light/Dark- und DE/EN-Check. diff --git a/package.json b/package.json index 0698366..041205d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "exoquill", - "version": "0.1.0", + "version": "0.2.0", "private": true, "description": "Local-first desktop app for dictation, OCR, formatting and read-aloud.", "license": "GPL-3.0-only", diff --git a/scripts/dev.ps1 b/scripts/dev.ps1 index ef95bf6..b2c4af5 100644 --- a/scripts/dev.ps1 +++ b/scripts/dev.ps1 @@ -10,8 +10,22 @@ $env:PATH = "$env:USERPROFILE\.cargo\bin;$env:PATH" $env:EXOQUILL_TESSDATA = Join-Path $runtimes "tessdata" $env:EXOQUILL_LLAMA = Join-Path $runtimes "llama\llama-completion.exe" $env:EXOQUILL_FORMATTER_MODEL = Join-Path $runtimes "models\qwen2.5-1.5b-instruct-q4_k_m.gguf" +# Where the in-app model manager downloads/looks for on-demand models (dev: the +# same runtimes/ tree the providers resolve from). +$env:EXOQUILL_MODELS_ROOT = $runtimes $env:EXOQUILL_PIPER = Join-Path $runtimes "piper\piper.exe" -$env:EXOQUILL_PIPER_VOICE = Join-Path $runtimes "piper-voices\de_DE-thorsten-medium.onnx" +$env:EXOQUILL_PIPER_VOICE = Join-Path $runtimes "piper-voices\de_DE-thorsten-high.onnx" +# Auto-start the experimental XTTS-v2 sidecar → XTTS becomes the default voice +# (Piper stays the fallback until it warms up / if it fails). Comment these two +# out to use Piper only. Setup once with scripts/setup-xtts.ps1. +$env:EXOQUILL_XTTS_PYTHON = Join-Path $root ".venv-xtts\Scripts\python.exe" +$env:EXOQUILL_XTTS_SCRIPT = Join-Path $root "scripts\xtts-server.py" +# Auto-start the experimental Zonos-v0.1 sidecar (Apache-2.0 weights, CUDA GPU). +# It only activates once scripts/setup-zonos.ps1 has run and the voices folder +# holds reference .wav clips; until then these paths don't exist and it's skipped. +$env:EXOQUILL_ZONOS_PYTHON = Join-Path $root ".venv-zonos\Scripts\python.exe" +$env:EXOQUILL_ZONOS_SCRIPT = Join-Path $root "scripts\zonos-server.py" +$env:EXOQUILL_ZONOS_VOICES = Join-Path $root "zonos-voices" $env:EXOQUILL_WHISPER = Join-Path $runtimes "whisper\whisper-cli.exe" $env:EXOQUILL_WHISPER_MODEL = Join-Path $runtimes "models\ggml-large-v3-turbo-q5_0.bin" # Optional Silero neural VAD (only used in a `--features silero` build); harmless diff --git a/scripts/fetch-piper-voices.ps1 b/scripts/fetch-piper-voices.ps1 new file mode 100644 index 0000000..4fb6685 --- /dev/null +++ b/scripts/fetch-piper-voices.ps1 @@ -0,0 +1,68 @@ +# Fetches a curated set of high-quality (`high` tier) Piper TTS voices into +# runtimes/piper-voices/, where dev.ps1 points ExoQuill. Each voice is an +# `.onnx` model plus its `.onnx.json` config (the config carries the sample rate +# the provider reads per voice). Like the other AI assets these are bundled as +# Tauri resources for release and are not in git. +# +# pwsh scripts/fetch-piper-voices.ps1 # missing voices only +# pwsh scripts/fetch-piper-voices.ps1 -Force # re-download everything +# pwsh scripts/fetch-piper-voices.ps1 -Prune # also delete non-curated voices +# +# Only `high` models are used here (audiobook-grade). German has just one high +# voice (Thorsten); the others are English. The provider discovers whatever +# lands in the folder; -Prune removes anything not listed below. Voices come from +# the rhasspy/piper-voices repo on Hugging Face. +# +# Licensing (verify before bundling — decisions D2 open item): the Thorsten +# dataset is CC0; the English voices derive from other corpora with their own +# terms. Each voice's MODEL_CARD on Hugging Face states its license. + +param( + [switch]$Force, + [switch]$Prune +) + +$ErrorActionPreference = "Stop" +$root = Split-Path $PSScriptRoot -Parent +$voicesDir = Join-Path $root "runtimes\piper-voices" +New-Item -ItemType Directory -Force -Path $voicesDir | Out-Null + +$base = "https://huggingface.co/rhasspy/piper-voices/resolve/main" + +# lang = HF top-level family dir; code-name-quality form the file stem + sub-path. +$voices = @( + @{ Lang = "de"; Code = "de_DE"; Name = "thorsten"; Quality = "high" }, + @{ Lang = "en"; Code = "en_US"; Name = "lessac"; Quality = "high" }, + @{ Lang = "en"; Code = "en_US"; Name = "ryan"; Quality = "high" }, + @{ Lang = "en"; Code = "en_GB"; Name = "cori"; Quality = "high" } +) + +$keep = $voices | ForEach-Object { "$($_.Code)-$($_.Name)-$($_.Quality)" } + +if ($Prune) { + Get-ChildItem -Path $voicesDir -Filter "*.onnx" | ForEach-Object { + if ($keep -notcontains $_.BaseName) { + Write-Host "Pruning $($_.Name) ..." + Remove-Item $_.FullName -Force + $json = "$($_.FullName).json" + if (Test-Path $json) { Remove-Item $json -Force } + } + } +} + +foreach ($v in $voices) { + $stem = "$($v.Code)-$($v.Name)-$($v.Quality)" + $dir = "$($v.Lang)/$($v.Code)/$($v.Name)/$($v.Quality)" + foreach ($ext in @("onnx", "onnx.json")) { + $dest = Join-Path $voicesDir "$stem.$ext" + if ($Force -or -not (Test-Path $dest)) { + $url = "$base/$dir/$stem.$ext" + Write-Host "Downloading $stem.$ext ..." + Invoke-WebRequest -Uri $url -OutFile $dest + } else { + Write-Host "Already present: $stem.$ext" + } + } +} + +Write-Host "Done. $($voices.Count) high-quality voices in $voicesDir. Restart the app to pick them up." diff --git a/scripts/setup-xtts.ps1 b/scripts/setup-xtts.ps1 new file mode 100644 index 0000000..bb3d85a --- /dev/null +++ b/scripts/setup-xtts.ps1 @@ -0,0 +1,57 @@ +# Sets up a local Python venv with Coqui XTTS-v2 for the EXPERIMENTAL XTTS TTS +# sidecar (scripts/xtts-server.py). Test-only: the XTTS-v2 weights are +# non-commercial (CPML) and must not ship in ExoQuill's GPL build. The library +# (the maintained `coqui-tts` fork) is MPL-2.0. +# +# pwsh scripts/setup-xtts.ps1 # CUDA wheels (default cu128) +# pwsh scripts/setup-xtts.ps1 -Cuda cpu # CPU-only torch (slow) +# +# Pick -Cuda to match your GPU: cu128 for Blackwell (RTX 50xx, sm_120) — older +# cu124 wheels lack sm_120 kernels and fail at inference; cu124/cu121 for 40xx +# and earlier. Requires Python 3.12 (coqui-tts has no 3.13 wheels yet). Then +# start the sidecar +# and point ExoQuill at it: +# .\.venv-xtts\Scripts\python.exe scripts\xtts-server.py --port 8020 +# $env:EXOQUILL_XTTS_URL = "http://127.0.0.1:8020"; pnpm dev +# +# The first synthesis downloads the model (~1.8 GB) into the Coqui cache and +# accepts the CPML via COQUI_TOS_AGREED=1 (set by the server). + +param( + [string]$Cuda = "cu128" +) + +$ErrorActionPreference = "Stop" +$root = Split-Path $PSScriptRoot -Parent +$venv = Join-Path $root ".venv-xtts" +$py = Join-Path $venv "Scripts\python.exe" + +if (-not (Test-Path $py)) { + Write-Host "Creating venv at $venv (Python 3.12) ..." + # Prefer an explicit 3.12 via the py launcher; fall back to `python`. + if (Get-Command py -ErrorAction SilentlyContinue) { + py -3.12 -m venv $venv + } else { + python -m venv $venv + } +} + +& $py -m pip install --upgrade pip wheel + +# PyTorch (+ torchaudio, required by XTTS) from the index matching your GPU +# (cpu | cu121 | cu124 | cu128). Both from the same index to match ABIs. Pinned +# to 2.7/2.8: new enough for Blackwell sm_120 kernels (added in 2.7), but below +# 2.9 — from 2.9 coqui-tts also demands torchcodec (fragile on Windows). +& $py -m pip install "torch>=2.7,<2.9" "torchaudio>=2.7,<2.9" --index-url "https://download.pytorch.org/whl/$Cuda" + +# Coqui TTS — maintained fork (idiap), MPL-2.0; pulls in XTTS-v2 support + numpy. +& $py -m pip install coqui-tts numpy +# coqui-tts needs transformers>=4.57, but transformers 5.x dropped a symbol it +# imports (isin_mps_friendly). Pin to the last 4.x line, which has both. +& $py -m pip install "transformers>=4.57,<5" + +Write-Host "" +Write-Host "Done. Start the sidecar with:" +Write-Host " $py scripts\xtts-server.py --port 8020" +Write-Host "Then, in another shell:" +Write-Host ' $env:EXOQUILL_XTTS_URL = "http://127.0.0.1:8020"; pnpm dev' diff --git a/scripts/setup-zonos.ps1 b/scripts/setup-zonos.ps1 new file mode 100644 index 0000000..d264604 --- /dev/null +++ b/scripts/setup-zonos.ps1 @@ -0,0 +1,68 @@ +# Sets up a local Python venv with Zyphra Zonos-v0.1 for the EXPERIMENTAL Zonos +# TTS sidecar (scripts/zonos-server.py). Unlike XTTS, the Zonos weights are +# Apache-2.0 (fine to redistribute), but the model needs a CUDA GPU to be usable. +# +# pwsh scripts/setup-zonos.ps1 # CUDA wheels (default cu128) +# pwsh scripts/setup-zonos.ps1 -Cuda cu124 # RTX 30xx/40xx +# +# Pick -Cuda to match your GPU: cu128 (default) covers Blackwell (RTX 50xx, +# sm_120) and is backward-compatible down to sm_70; cu124 for RTX 30xx/40xx if +# you prefer. cu128 needs torch >= 2.7 (sm_120 kernels landed in 2.7). Requires +# Python 3.12 and git. Then create a voices folder with one or more 10-30 s +# reference .wav clips (each file = one voice), start the sidecar, and point +# ExoQuill at it via dev.ps1 (EXOQUILL_ZONOS_*). +# +# Zonos uses eSpeak NG for phonemization; we install `espeakng-loader`, which +# bundles the shared library so no system-wide eSpeak NG install is needed. The +# first synthesis downloads the model weights into the Hugging Face cache. + +param( + [string]$Cuda = "cu128" +) + +$ErrorActionPreference = "Stop" +$root = Split-Path $PSScriptRoot -Parent +$venv = Join-Path $root ".venv-zonos" +$py = Join-Path $venv "Scripts\python.exe" +$src = Join-Path $root ".zonos-src" +$voices = Join-Path $root "zonos-voices" + +if (-not (Test-Path $py)) { + Write-Host "Creating venv at $venv (Python 3.12) ..." + if (Get-Command py -ErrorAction SilentlyContinue) { + py -3.12 -m venv $venv + } else { + python -m venv $venv + } +} + +& $py -m pip install --upgrade pip wheel + +# PyTorch (+ torchaudio, used to load reference clips) from the index matching +# your GPU. Pinned to 2.7/2.8: new enough for Blackwell sm_120 kernels (cu128), +# but below 2.9. Both from the same index to match ABIs. +& $py -m pip install "torch>=2.7,<2.9" "torchaudio>=2.7,<2.9" --index-url "https://download.pytorch.org/whl/$Cuda" + +# Zonos from a git *clone* installed editable (-e). A plain `pip install git+...` +# builds a wheel that drops the `zonos/backbone` subpackage (ModuleNotFoundError: +# zonos.backbone at runtime); an editable install links the source tree directly, +# so all subpackages resolve. Transformer variant only — no mamba-ssm needed. +if (-not (Test-Path (Join-Path $src ".git"))) { + Write-Host "Cloning Zonos into $src ..." + git clone --depth 1 https://github.com/Zyphra/Zonos.git $src +} +& $py -m pip install -e $src +# Bundled eSpeak NG library so phonemizer works without a system install. +& $py -m pip install espeakng-loader numpy + +# A default voices folder so the sidecar has something to offer on first run. +if (-not (Test-Path $voices)) { + New-Item -ItemType Directory -Path $voices | Out-Null +} + +Write-Host "" +Write-Host "Done. Add one or more 10-30s reference .wav clips to:" +Write-Host " $voices" +Write-Host "Then start the sidecar with:" +Write-Host " $py scripts\zonos-server.py --port 8021 --voices $voices" +Write-Host "Or let ExoQuill auto-start it: the EXOQUILL_ZONOS_* lines in scripts\dev.ps1." diff --git a/scripts/xtts-server.py b/scripts/xtts-server.py new file mode 100644 index 0000000..e351dd4 --- /dev/null +++ b/scripts/xtts-server.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +r"""Minimal XTTS-v2 HTTP sidecar for ExoQuill (EXPERIMENTAL, test-only). + +Loads Coqui XTTS-v2 once and serves synthesis over localhost HTTP, mirroring the +whisper-server pattern. Endpoints: + GET / -> 200 "ok" (health check) + GET /speakers -> JSON list of names (built-in studio speakers) + POST /tts -> raw int16 mono PCM @ 24 kHz + body: {"text": str, "language": "de"|"en"|..., + "speaker": str, "speed": float (optional)} + +The XTTS-v2 *weights* are non-commercial (CPML); the library (coqui-tts fork) is +MPL-2.0. Run only for local testing — do not bundle the weights in a release. + +Setup: pwsh scripts/setup-xtts.ps1 +Run: .\.venv-xtts\Scripts\python.exe scripts\xtts-server.py --port 8020 +Use: $env:EXOQUILL_XTTS_URL = "http://127.0.0.1:8020"; pnpm dev +""" + +import argparse +import json +import os +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import numpy as np + +# Accept the CPML non-interactively so the first run can download the weights. +os.environ.setdefault("COQUI_TOS_AGREED", "1") + +from TTS.api import TTS # noqa: E402 (import after the env var is set) + +MODEL = "tts_models/multilingual/multi-dataset/xtts_v2" + + +def load_model(): + import torch + + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"[xtts] loading {MODEL} on {device} (first run downloads ~1.8 GB) ...", flush=True) + tts = TTS(MODEL).to(device) + + # Built-in speaker names vary by version; try the known attributes. + speakers = [] + manager = getattr(tts.synthesizer.tts_model, "speaker_manager", None) + for attr in ("speakers", "name_to_id"): + table = getattr(manager, attr, None) + if table: + speakers = list(table.keys()) + break + print(f"[xtts] ready. {len(speakers)} speakers: {', '.join(speakers) or '(none)'}", flush=True) + return tts, speakers + + +def make_handler(tts, speakers): + default_speaker = speakers[0] if speakers else None + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): # keep the console quiet + pass + + def _send(self, code, body, ctype): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def do_GET(self): + if self.path.startswith("/speakers"): + self._send(200, json.dumps(speakers).encode(), "application/json") + else: + self._send(200, b"ok", "text/plain") + + def do_POST(self): + try: + length = int(self.headers.get("Content-Length", 0)) + req = json.loads(self.rfile.read(length) or b"{}") + text = (req.get("text") or "").strip() + language = req.get("language") or "de" + speaker = req.get("speaker") or default_speaker + speed = float(req.get("speed") or 1.0) + if not text: + self._send(200, b"", "application/octet-stream") + return + try: + wav = tts.tts(text=text, speaker=speaker, language=language, speed=speed) + except TypeError: + # Older builds don't accept `speed`. + wav = tts.tts(text=text, speaker=speaker, language=language) + pcm = np.clip(np.asarray(wav, dtype=np.float32), -1.0, 1.0) + pcm = (pcm * 32767.0).astype(" 200 "ok" (health check, only once ready) + GET /voices -> JSON list of voice ids (reference clip stems) + POST /tts -> raw int16 mono PCM @ 44.1 kHz + body: {"text": str, "language": "de"|"en"|..., + "speaker": str (voice id), "speed": float (optional), + "pitch": float (optional, pitch_std intonation), + "fmax": float (optional, Hz frequency ceiling), + "emotion": [float]*8 (optional, emotion vector)} + +The Zonos *weights* are Apache-2.0 (fine to redistribute), but the model needs a +CUDA GPU to be usable. Zonos also depends on eSpeak NG for phonemization — it +must be installed and discoverable (see scripts/setup-zonos.ps1). + +Setup: pwsh scripts/setup-zonos.ps1 +Run: .\.venv-zonos\Scripts\python.exe scripts\zonos-server.py --port 8021 --voices .\zonos-voices +""" + +import argparse +import glob +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import numpy as np + +# Help phonemizer (Zonos' text frontend) find eSpeak NG on Windows without a +# system-wide install: the `espeakng-loader` package ships the shared library. +# phonemizer finds the .dll via PHONEMIZER_ESPEAK_LIBRARY; espeak-ng finds its +# voice data via the ESPEAK_DATA_PATH env var, pointed at the directory that +# *contains* `espeak-ng-data`. Without this the bundled DLL reports a build-time +# data path that doesn't exist here ("phontab: No such file or directory"), which +# crashes synthesis. Best-effort — if absent, a system eSpeak NG is used. +try: + import espeakng_loader + + os.environ["PHONEMIZER_ESPEAK_LIBRARY"] = espeakng_loader.get_library_path() + os.environ["ESPEAK_DATA_PATH"] = os.path.dirname(espeakng_loader.get_data_path()) +except Exception: + pass + +# Default model — the transformer variant (broadest GPU compatibility; the hybrid +# variant needs mamba-ssm kernels that are painful on Windows). Override with +# EXOQUILL_ZONOS_MODEL. +MODEL = os.environ.get("EXOQUILL_ZONOS_MODEL", "Zyphra/Zonos-v0.1-transformer") + +# torch.compile makes generation several times faster but needs an MSVC compiler +# (cl.exe) on the PATH (Inductor codegen). Off by default — eager mode is robust +# and needs no toolchain. Set EXOQUILL_ZONOS_COMPILE=1 and launch from a VS dev +# environment (vcvars on the PATH) to enable it. +USE_COMPILE = os.environ.get("EXOQUILL_ZONOS_COMPILE") == "1" + +# eSpeak language codes Zonos expects, mapped from our short de/en tags. +LANGUAGE_MAP = {"de": "de", "en": "en-us"} + +# Zonos' default speaking rate (phonemes/sec); our `speed` scales it. +BASE_SPEAKING_RATE = 15.0 + + +def normalize_loudness(wav, target_rms=0.12, peak_ceiling=0.99): + """Scale a clip to a target RMS so per-sentence volume stays consistent + (Zonos' output level drifts between generations). Capped below `peak_ceiling` + to avoid clipping; near-silent clips are left untouched. ~0.12 RMS ≈ -18 dBFS, + a comfortable speech level.""" + if wav.size == 0: + return wav + rms = float(np.sqrt(np.mean(np.square(wav)))) + if rms < 1e-5: + return wav + gain = target_rms / rms + peak = float(np.max(np.abs(wav))) + if peak * gain > peak_ceiling: + gain = peak_ceiling / max(peak, 1e-5) + return wav * gain + + +def load_model(voices_dir): + import torch + import torchaudio + from zonos.model import Zonos + from zonos.speaker_cloning import SpeakerEmbeddingLDA + + device = "cuda" if torch.cuda.is_available() else "cpu" + if device != "cuda": + print("[zonos] WARNING: no CUDA GPU found — Zonos on CPU is far too slow.", flush=True) + print(f"[zonos] loading {MODEL} on {device} ...", flush=True) + model = Zonos.from_pretrained(MODEL, device=device) + + # Run speaker cloning on CPU: Zonos builds the embedding net under + # `with torch.device(cuda)`, which leaves some buffers on CPU and raises + # "tensors on cuda:0 and cpu" during embedding. CPU keeps every tensor on one + # device; it's a one-time step at startup (not in the synthesis hot path), and + # we move just the small resulting embedding back to the GPU for generation. + model.spk_clone_model = SpeakerEmbeddingLDA(device="cpu") + + # Embed every reference clip once: stem -> speaker embedding (on the GPU). + speakers = {} + for path in sorted(glob.glob(os.path.join(voices_dir, "*.wav"))): + stem = os.path.splitext(os.path.basename(path))[0] + try: + wav, sr = torchaudio.load(path) + speakers[stem] = model.make_speaker_embedding(wav, sr).to(device) + except Exception as e: # skip an unreadable clip, keep the rest + print(f"[zonos] skip {stem}: {e}", flush=True) + names = ", ".join(speakers) or "(none — add .wav clips to the voices folder)" + print(f"[zonos] ready. {len(speakers)} voices: {names}", flush=True) + return model, speakers + + +def make_handler(model, speakers): + import torch + from zonos.conditioning import make_cond_dict + + default_speaker = next(iter(speakers), None) + sample_rate = int(model.autoencoder.sampling_rate) + # Serialize generation: one GPU model, not safe for concurrent generate(). + lock = threading.Lock() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): # keep the console quiet + pass + + def _send(self, code, body, ctype): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def do_GET(self): + if self.path.startswith("/voices"): + self._send(200, json.dumps(list(speakers)).encode(), "application/json") + else: + self._send(200, b"ok", "text/plain") + + def do_POST(self): + try: + length = int(self.headers.get("Content-Length", 0)) + req = json.loads(self.rfile.read(length) or b"{}") + text = (req.get("text") or "").strip() + language = LANGUAGE_MAP.get(req.get("language") or "de", "de") + speaker_id = req.get("speaker") or default_speaker + speed = float(req.get("speed") or 1.0) + if not text or speaker_id not in speakers: + self._send(200, b"", "application/octet-stream") + return + rate = max(5.0, min(30.0, BASE_SPEAKING_RATE * speed)) + # Per-request intonation (pitch_std) and brightness (fmax), each + # falling back to the read-aloud-tuned default when the client omits + # it. Zonos' own pitch_std=20 sounds monotone; 40-45 gives a lively- + # but-not-crazy intonation. fmax is the synthesis frequency ceiling; + # 22050 suits 44.1 kHz clones, lower sounds warmer/duller. + pitch_std = float( + req.get("pitch") + if req.get("pitch") is not None + else os.environ.get("EXOQUILL_ZONOS_PITCH", "42") + ) + fmax = float(req.get("fmax") if req.get("fmax") is not None else 22050.0) + pitch_std = max(0.0, min(400.0, pitch_std)) + fmax = max(0.0, min(24000.0, fmax)) + cond_kwargs = dict( + text=text, + speaker=speakers[speaker_id], + language=language, + speaking_rate=rate, + pitch_std=pitch_std, + fmax=fmax, + ) + # Optional emotion conditioning: an 8-value vector [happiness, + # sadness, disgust, fear, surprise, anger, other, neutral]. Some + # Zonos builds list "emotion" in the default unconditional_keys + # (which would silently ignore it), so when a vector is given we + # also pin unconditional_keys to the quality keys, leaving emotion + # conditioned. Omitted → Zonos uses its own default vector. + emotion = req.get("emotion") + if isinstance(emotion, list) and len(emotion) == 8: + cond_kwargs["emotion"] = [float(x) for x in emotion] + cond_kwargs["unconditional_keys"] = ["vqscore_8", "dnsmos_ovrl"] + with lock: + try: + cond = make_cond_dict(**cond_kwargs) + except TypeError: + # Older/newer Zonos signature without these kwargs — drop + # the optional ones and synthesize without emotion. + cond_kwargs.pop("emotion", None) + cond_kwargs.pop("unconditional_keys", None) + cond = make_cond_dict(**cond_kwargs) + codes = model.generate( + model.prepare_conditioning(cond), + disable_torch_compile=not USE_COMPILE, + ) + audio = model.autoencoder.decode(codes).cpu().detach() + wav = np.asarray(audio, dtype=np.float32).reshape(-1) + wav = normalize_loudness(wav) + pcm = np.clip(wav, -1.0, 1.0) + pcm = (pcm * 32767.0).astype("