diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 744496b..6c4ea20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,10 @@ jobs: with: node-version: '26.5.1' cache: npm + - run: sudo apt-get update && sudo apt-get install -y ffmpeg - run: npm ci - run: npm run typecheck - run: npm test + - run: python3 scripts/test_local_whisper.py - run: npx playwright install --with-deps chromium - run: npm run test:e2e diff --git a/README.md b/README.md index e7dd305..31d8741 100644 --- a/README.md +++ b/README.md @@ -115,3 +115,23 @@ screenshots. For sensitive reports, follow [SECURITY.md](SECURITY.md). ## License Agentflow is licensed under the [MIT License](LICENSE). + +## Fully local transcription + +Select **Local Whisper** under Speech & settings → Listening. It runs speech recognition on your Mac with no API key or 9Router. Speaking still uses the independently selected TTS provider, and Codex/Claude retain their own model routing; local transcription does not make an agent conversation fully offline. + +Install `ffmpeg`, Python 3, and the pinned `whisper-server` runtime described in [local runtime setup](docs/local-whisper-runtime.md). Then explicitly prepare and start the engine: + +```sh +agentflow local-whisper setup +agentflow local-whisper start +agentflow local-whisper status +``` + +The default URL is `http://127.0.0.1:17864`. To use an existing engine, enter its loopback HTTP origin in **Local Whisper URL**. The model setting is a label for the server's loaded model; changing it does not load a different model. Models are never downloaded while recording or during login. `agentflow local-whisper stop` only stops the engine owned by Agentflow. The helper accepts `import-model PATH --sha256 HASH` for an explicitly trusted local model; use `--help` for options. Agentflow's helper state lives beneath `$AGENTFLOW_HOME/local-whisper` (default `~/.config/agentflow/local-whisper`). + +Browser WebM/MP4/WAV recordings are converted in memory by ffmpeg to 16 kHz mono PCM WAV, with a 90-second duration limit, 20 MB input limit and 15-second conversion deadline. Set `AGENTFLOW_FFMPEG` to an absolute executable path if needed. No conversion files are retained. The local endpoint cannot contain remote hosts, credentials, paths, queries or redirects, and its transport bypasses global HTTP proxies. Silent/near-silent recordings do not reach Whisper. Very quiet speech may need a closer microphone. + +Each recording captures its listening settings before microphone permission is requested. Canceling or changing sessions discards late transcription and cannot submit it to the agent. Canceling closes local transport and terminates conversion, but whisper-server may continue inference internally; this is not a claim that HTTP disconnect preempts GPU computation. Inference has a 60-second timeout; check `agentflow local-whisper status`, the server's loaded model and the configured URL after engine errors. + +Tests and TDD evidence: [local transcription validation](docs/local-whisper-validation.md). Real microphone permission, audible playback and sleep/wake still require device acceptance. diff --git a/docs/local-whisper-qa.json b/docs/local-whisper-qa.json new file mode 100644 index 0000000..88d6042 --- /dev/null +++ b/docs/local-whisper-qa.json @@ -0,0 +1,292 @@ +{ + "date": "2026-10-15", + "engine": "whisper.cpp 1.9.1", + "model": "base.en", + "model_sha256": "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002", + "hardware": "Apple M3, 24 GiB unified memory", + "isolation": "macOS sandbox-exec denied external network for both server and clients; loopback allowed. External hostname and 1.1.1.1 curl probes failed.", + "benchmark": { + "description": "Raw engine synthetic TTS fixtures, not human microphone accuracy", + "clean_wer": 0.005988023952095809, + "clean_warm_p95_seconds": 0.138, + "results": [ + { + "fixture": "clean1", + "duration_seconds": 10, + "elapsed_seconds": 0.116, + "reference": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "word_errors": 0, + "reference_words": 13 + }, + { + "fixture": "clean2", + "duration_seconds": 10, + "elapsed_seconds": 0.12, + "reference": "The weather is clear today. I will walk to the store and buy some coffee before lunch.", + "text": "The weather is clear today. I will walk to the store and buy some coffee before lunch.", + "word_errors": 0, + "reference_words": 17 + }, + { + "fixture": "clean3", + "duration_seconds": 10, + "elapsed_seconds": 0.13, + "reference": "We finished the first draft yesterday. The next step is to review the changes and prepare the final report.", + "text": "We finished the first draft yesterday. The next step is to review the changes and prepare the final report.", + "word_errors": 0, + "reference_words": 19 + }, + { + "fixture": "clean4", + "duration_seconds": 10, + "elapsed_seconds": 0.132, + "reference": "Could you check the schedule and tell me whether we have enough time to finish this task today?", + "text": "Could you check the schedule and tell me whether we have enough time to finish this task today?", + "word_errors": 0, + "reference_words": 18 + }, + { + "fixture": "clean5", + "duration_seconds": 10, + "elapsed_seconds": 0.128, + "reference": "This recording is a test of local speech recognition. No internet connection should be needed to understand these words.", + "text": "This recording is a test of local speech recognition. No internet connection should be needed to understand these words.", + "word_errors": 0, + "reference_words": 19 + }, + { + "fixture": "clean6", + "duration_seconds": 10, + "elapsed_seconds": 0.135, + "reference": "After breakfast we will take the train into the city and meet our friends near the museum.", + "text": "After breakfast we will take the train into the city and meet our friends near the museum.", + "word_errors": 0, + "reference_words": 17 + }, + { + "fixture": "clean7", + "duration_seconds": 10, + "elapsed_seconds": 0.125, + "reference": "Remember to close the window before leaving the house. The forecast says it will rain later this afternoon.", + "text": "Remember to close the window before leaving the house. The forecast says it will rain later this afternoon.", + "word_errors": 0, + "reference_words": 18 + }, + { + "fixture": "clean8", + "duration_seconds": 10, + "elapsed_seconds": 0.128, + "reference": "Please save a copy of the document and put it in the folder for next week.", + "text": "Please save a copy of the document and put it in the folder for next week.", + "word_errors": 0, + "reference_words": 16 + }, + { + "fixture": "clean9", + "duration_seconds": 10, + "elapsed_seconds": 0.126, + "reference": "The package arrived this morning. Everything was packed carefully and nothing appears to be missing.", + "text": "The package arrived this morning, everything was packed carefully, and nothing appears to be missing.", + "word_errors": 0, + "reference_words": 15 + }, + { + "fixture": "clean10", + "duration_seconds": 10, + "elapsed_seconds": 0.12, + "reference": "Let us start with the simplest example and make sure it works before adding any more features.", + "text": "Let us start with the simplest example and make sure it works before adding any more features.", + "word_errors": 0, + "reference_words": 17 + }, + { + "fixture": "clean11", + "duration_seconds": 10, + "elapsed_seconds": 0.13, + "reference": "I would like a table for four people on Saturday evening, preferably beside a window.", + "text": "I would like a table for four people on Saturday evening, preferably beside a window.", + "word_errors": 0, + "reference_words": 15 + }, + { + "fixture": "clean12", + "duration_seconds": 10, + "elapsed_seconds": 0.123, + "reference": "Thank you for your help with the project. Your comments made the instructions much easier to follow.", + "text": "Thank you for your help with the project. Your comments made the instructions much easier to follow.", + "word_errors": 0, + "reference_words": 17 + }, + { + "fixture": "clean13", + "duration_seconds": 10, + "elapsed_seconds": 0.131, + "reference": "Turn left at the next street and continue for two blocks. The library is on the right.", + "text": "Turn left at the next street and continue for two blocks. The library is on the right.", + "word_errors": 0, + "reference_words": 17 + }, + { + "fixture": "clean14", + "duration_seconds": 10, + "elapsed_seconds": 0.112, + "reference": "We should test the new settings before the meeting starts so that everyone can hear clearly.", + "text": "We should test the new settings before the meeting starts so that everyone can hear clearly.", + "word_errors": 0, + "reference_words": 16 + }, + { + "fixture": "clean15", + "duration_seconds": 10, + "elapsed_seconds": 0.12, + "reference": "The team agreed to fix the current problems before beginning work on the next version.", + "text": "The team agreed to fix the current problems before beginning work on the next version.", + "word_errors": 0, + "reference_words": 15 + }, + { + "fixture": "clean16", + "duration_seconds": 10, + "elapsed_seconds": 0.12, + "reference": "There are several ways to solve this problem. Let us compare them carefully and choose one.", + "text": "There are several ways to solve this problem. Let us compare them carefully and choose one.", + "word_errors": 0, + "reference_words": 16 + }, + { + "fixture": "clean17", + "duration_seconds": 10, + "elapsed_seconds": 0.132, + "reference": "I left my notebook on the kitchen table. Please bring it with you when you come over.", + "text": "I left my notebook on the kitchen table. Please bring it with you when you come over.", + "word_errors": 0, + "reference_words": 17 + }, + { + "fixture": "clean18", + "duration_seconds": 10, + "elapsed_seconds": 0.138, + "reference": "The train leaves at nine tomorrow morning. We should arrive at the station at least twenty minutes early.", + "text": "The train leaves at 9 tomorrow morning. We should arrive at the station at least 20 minutes early.", + "word_errors": 2, + "reference_words": 18 + }, + { + "fixture": "clean19", + "duration_seconds": 10, + "elapsed_seconds": 0.128, + "reference": "Put the blue box on the shelf and leave the smaller green box beside the door.", + "text": "Put the blue box on the shelf and leave the smaller green box beside the door.", + "word_errors": 0, + "reference_words": 16 + }, + { + "fixture": "clean20", + "duration_seconds": 10, + "elapsed_seconds": 0.136, + "reference": "The lights are working again. We can finish cleaning the room and move the chairs back into place.", + "text": "The lights are working again. We can finish cleaning the room and move the chairs back into place.", + "word_errors": 0, + "reference_words": 18 + }, + { + "fixture": "technical", + "duration_seconds": 10, + "elapsed_seconds": 0.15, + "reference": "Run the unit tests, review the pull request, and restart the local Whisper server. Check the TypeScript compiler and the Rust build.", + "text": "Run the unit tests, review the pull request, and restart the local whisper server. Check the TypeScript compiler and the REST build.", + "word_errors": 1, + "reference_words": 22 + } + ] + }, + "agentflow_adapter": [ + { + "name": "clean1.wav", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.407278791 + }, + { + "name": "clean1.webm", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.13789725 + }, + { + "name": "clean1.mp4", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.12846329200000003 + }, + { + "name": "noise.wav", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.12652508299999998 + }, + { + "name": "silence.wav", + "text": "", + "elapsed_seconds": 0.02996124999999995 + } + ], + "managed_runtime": { + "two_engines_simultaneously": true, + "idempotent_start": true, + "stop_isolated_in_both_directions": true, + "restart_without_external_network": true, + "state_paths_with_spaces": true, + "observed_start_seconds": [ + 0.441, + 7.837, + 0.335 + ] + }, + "murmur_adapter": { + "clean_wav_expected_transcript": true, + "digital_silence_empty": true, + "varied_120_second_audio_transcribed": true + }, + "agentflow_limits": { + "varied_90_second_audio_transcribed": true, + "seconds": 1.099, + "120_second_audio_rejected": true + }, + "memory": { + "server_rss_kib_snapshot": 311136, + "note": "Single ps RSS snapshot after workload; not peak usage and does not fully account for Metal/unified GPU allocations." + }, + "limitations": [ + "Synthetic macOS say voice only, not a representative human/accent/microphone corpus.", + "Raw engine hallucinated you on digital silence; app guards returned empty. Noisy silence may still hallucinate.", + "Technical fixture misrecognized Rust as REST.", + "Aborting HTTP discards output but may not immediately stop engine CPU/GPU work.", + "Native microphone, actual hotkeys/focus/paste, sleep/wake and real browser device acceptance remain manual gates.", + "Native-agent reasoning and TTS are independent; this validates offline STT, not fully offline Codex/Claude conversations." + ], + "agentflow_adapter_final_transport": [ + { + "name": "clean1.wav", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.161127833 + }, + { + "name": "clean1.webm", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.127854666 + }, + { + "name": "clean1.mp4", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.125231583 + }, + { + "name": "noise.wav", + "text": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "elapsed_seconds": 0.12394433300000003 + }, + { + "name": "silence.wav", + "text": "", + "elapsed_seconds": 0.026634292000000073 + } + ] +} diff --git a/docs/local-whisper-qa.md b/docs/local-whisper-qa.md new file mode 100644 index 0000000..9fda819 --- /dev/null +++ b/docs/local-whisper-qa.md @@ -0,0 +1,68 @@ +# Local Whisper QA — 2026-10-15 + +This evidence covers the new local STT adapters and managed runtime. No personal +recordings or native agent sessions were used. Detailed synthetic references, +transcripts, timings and limitations are in [the JSON report](local-whisper-qa.json). + +## Environment and results + +- Apple M3, 24 GiB unified memory; whisper.cpp 1.9.1 with Metal; pinned base.en + SHA-256 `a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002`. +- Both server and test clients ran under the included macOS sandbox profile, + which denies external networking and permits loopback. External hostname and + numeric-IP probes failed; local inference succeeded. +- Twenty synthetic clean-English utterances, each padded to exactly 10 seconds: + normalized WER **0.60%**; raw-engine warm p95 **0.138 seconds** across 19 calls + after the first request. These meet the proposed 10%/3-second gates for this + narrow corpus only. Audio generation/conversion is outside raw inference timing. +- Technical fixture: one error, “Rust” → “REST”. This is not a guarantee of + human microphone, accent, vocabulary, or noisy-room accuracy. +- Murmur's compiled adapter transcribed clean WAV and varied 120-second speech; + digital silence returned empty. Its capture tests separately verify draining, + resampling and completion through the local adapter. +- Agentflow's compiled adapter transcribed WAV, WebM, MP4 and noisy WAV locally; + silence returned empty. Observed short-input adapter times were 0.128–0.407s, + including local format conversion. After the direct-HTTP transport hardening, + repeated speech checks passed in 0.124–0.161s and silence in 0.027s. + A 90-second fixture transcribed in 1.099s; + 120 seconds failed with the documented 90-second limit. +- Two independently managed engines ran simultaneously. Repeated start was + idempotent. Stopping either preserved the other; offline restart passed. + State directories contained spaces. Startup ranged 0.335–7.837s in these runs. +- Server RSS snapshot: 311,136 KiB. This is not a peak or full Metal memory metric. + +## Reproduce the raw-engine benchmark + +Install runtime/model explicitly using the runtime guide first. Start an engine +with networking restricted (use a free port and your model path): + +```sh +sandbox-exec -f scripts/local-whisper-offline.sb whisper-server \ + --host 127.0.0.1 --port 17865 -m /absolute/path/to/model.bin +``` + +In another terminal: + +```sh +sandbox-exec -f scripts/local-whisper-offline.sb python3 \ + scripts/benchmark-local-whisper.py --url http://127.0.0.1:17865 \ + --output /tmp/local-whisper-benchmark.json +``` + +The script uses macOS `say` and `ffmpeg`, reads committed synthetic reference +text, generates temporary audio, and deletes that audio when it finishes. It +never downloads a model or submits anything to a native agent. This benchmark +tests the raw engine; the app-specific adapter tests remain necessary. + +## Remaining gates and limits + +Physical microphone/permission checks, real Control hold/release and focused +app insertion, sleep/wake, and browser hardware acceptance have **not** been +performed. Keep those QA issue checkboxes open. Automated capture, browser and +session fixtures do not prove physical-device behavior. + +Raw Whisper produced “you” on digital silence. Both adapters prevent this for +their silence thresholds; noisy silence can still hallucinate. Aborting an HTTP +request prevents insertion/submission but does not promise immediate engine +compute interruption. Local STT does not make separately configured TTS or +native Codex/Claude reasoning local. diff --git a/docs/local-whisper-runtime.md b/docs/local-whisper-runtime.md new file mode 100644 index 0000000..1d7d967 --- /dev/null +++ b/docs/local-whisper-runtime.md @@ -0,0 +1,32 @@ +# Local Whisper development runtime + +Requires Python 3.9+ and a separately installed `whisper-server`. Tested with whisper.cpp **1.9.1** on Apple Silicon. Install deliberately with `brew install whisper-cpp` (Homebrew may supply a newer version), or build upstream v1.9.1 and pass `start --binary '/path with spaces/whisper-server'`. The helper never installs executables or implicitly downloads models. No Python packages are required. + +From the repository root, substituting `agentflow` for `murmur` where appropriate: + +```sh +python3 scripts/local-whisper.py --app murmur setup +python3 scripts/local-whisper.py --app murmur start +python3 scripts/local-whisper.py --app murmur status +python3 scripts/local-whisper.py --app murmur stop +``` + +`setup` explicitly downloads base.en (~148 MB), verifies its SHA-256, then atomically selects it. Alternatively, import an existing model: + +```sh +python3 scripts/local-whisper.py --app murmur import-model '/path/to/ggml-base.en.bin' --sha256 a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002 +``` + +Custom models require their known SHA-256. Runtime start rechecks integrity offline. Corrupt or interrupted imports preserve the previous model selection; verified immutable digest-named files are selected through an atomic symlink replacement. Old verified models remain on disk for manual removal while stopped. Stop before changing the selected model. State defaults to `~/Library/Application Support//local-whisper`; override via `--state-dir PATH` before the subcommand. State is private to the current user. + +Murmur defaults to `http://127.0.0.1:17863`; Agentflow uses `http://127.0.0.1:17864`. `start --port N --timeout 30` overrides the port and readiness deadline (maximum 300 seconds). Configure the app's local transcription endpoint to match. Start is idempotent for an already-owned runtime, retaining its original configuration; stop before changing binary or port. + +A private authenticated loopback supervisor owns the child process. Its random bearer token is handed over through a private stdin pipe with a five-second deadline and strict frame-size/format validation; it never appears in process arguments or public CLI JSON. Stop contacts this supervisor and terminates only its own child, never a PID from a stale state file. Runtime output is discarded to avoid storing transcripts in logs. Readiness and control ignore HTTP proxy variables and reject redirects. A taken port reports an error rather than adopting or stopping another server. To use an external server, configure its loopback endpoint in the app and manage that process yourself; helper stop will not affect it. These are development CLI controls, not a bundled production installer or automatic app-exit hook. App and helper lifecycle remain explicit. + +Setup requires internet. Start, status, stop and transcription can run offline after model installation. Local transcription does not make unrelated cloud LLM/TTS features local. + +Pinned model source: [ggerganov/whisper.cpp revision 5359861c739e955e79d9a303bcbc70fb988958b1](https://huggingface.co/ggerganov/whisper.cpp/tree/5359861c739e955e79d9a303bcbc70fb988958b1), file `ggml-base.en.bin`. SHA and size (147,964,211 bytes) verified from the [Hugging Face model API](https://huggingface.co/api/models/ggerganov/whisper.cpp/revision/5359861c739e955e79d9a303bcbc70fb988958b1?blobs=true). Runtime source: [whisper.cpp v1.9.1](https://github.com/ggml-org/whisper.cpp/tree/v1.9.1). + +Tests: `python3 -m unittest discover -s scripts -p 'test_local_whisper.py'` (9 passed locally in 3.8 seconds). Initial four acceptance tests failed before implementation; all nine now pass. Coverage includes verified imports, corrupt/interrupted replacement preservation, corrupted model startup, missing model, authenticated idempotent lifecycle, paths with spaces, external port collision, stale records, exit-zero/unready engines, readiness timeout cleanup, rejected redirects, and bounded secret handoff. The lifecycle test reads the actual process list and verifies that the secret is absent while authenticated status and stop still work. Real installed 1.9.1/base.en import → start → ready → stop was also exercised. Audio accuracy and end-to-end app insertion/submission require separate app QA. + +CI timing: the successful-engine fixture allows a 30-second readiness deadline and 60 seconds for each CLI subprocess, accommodating cold macOS runner startup. The deliberate unready-engine case retains its 0.5-second readiness deadline; readiness, process ownership, port collision, and timeout cleanup assertions are unchanged. The process-list token assertion failed before the private-pipe fix and passed afterward. diff --git a/docs/local-whisper-validation.md b/docs/local-whisper-validation.md new file mode 100644 index 0000000..41a4592 --- /dev/null +++ b/docs/local-whisper-validation.md @@ -0,0 +1,38 @@ +# Local transcription implementation evidence + +Scope: GitHub #21, #22, with real engine/performance QA tracked by #23. Local TTS (#24) is a future follow-up. + +## Observed red → green cycles + +- `npm run build && node --test dist/test/local-whisper.test.js`: initial four behavior tests failed with `Invalid provider` / `Unknown speech provider`. After adding provider contracts, settings persistence, local WAV conversion and `/inference`, all four passed. +- `npm run test:e2e -- --grep 'local listening'`: failed waiting for the absent Local Whisper URL field. Added local input selection/URL, kept local out of TTS/key controls, and passed. +- `npm run test:e2e -- --grep 'snapshots listening'`: failed because the recording had no settings snapshot header. Added the capture-time snapshot and server validation; the test passed and proved a canceled late result did not dispatch a native turn. +- Oversized local transcript response regression failed `Missing expected rejection`; streaming response bound (64 KiB) made it pass. +- Global-fetch proxy isolation regression failed `Speech provider is unreachable or timed out` with a deliberately replaced global transport; a direct loopback HTTP agent made it pass. +- `node --test dist/test/build.test.js`: local setup CLI help failed because it printed generic Agentflow help. CLI delegation and packaging the Python helper made it pass. + +Additional passing integration contracts cover real loopback HTTP multipart audio, redirect refusal, no Authorization header, malformed audio, silence without inference, cancellation after response, persisted STT independence, no local Keychain lookup, and server use of a captured local provider while saved settings still select cloud STT. Existing cloud and native-session tests remain required. + +The Python runtime's own red/green evidence and pinning details are in [runtime setup](local-whisper-runtime.md). Its lifecycle tests use temporary directories and disposable helper processes. Browser tests use fake microphones, fake speech responses and synthetic native sessions; they do not prove microphone quality or a fully offline native agent. + +- Independent review reproduced an uncaught HTTP 204 Response-constructor error; empty-body statuses now become recoverable transcript errors. +- Independent review reproduced a stale native submission when starting another recording during inference; each new recording now aborts its predecessor and increments the generation. Browser regression passed. +- Browser capture stops at 89 seconds to leave a flush margin beneath the converter's strict 90-second decoded limit. + +- Follow-up review reproduced HTTP 700 crashing the process (Response RangeError); response construction now destroys the stream and rejects safely on malformed metadata. +- Out-of-order microphone permission success reproduced a live track leak after End; acquired streams remain local until generation validation. A stale permission rejection also stopped/submitted the current recording; the outer error path now ignores superseded generations. Both browser tests were observed red before the fixes and green afterward. + +Final local checks: 53 Node tests, 14 Playwright browser tests and 8 Python lifecycle tests passed; typecheck and build passed. The manually exercised microphone and audible playback gates remain outstanding. + +## Reproduce + +```sh +npm ci +npm run typecheck +npm test +python3 scripts/test_local_whisper.py +npx playwright install chromium +npm run test:e2e +``` + +Node local speech tests require ffmpeg on PATH; CI installs it explicitly. No model or provider credential is required for deterministic tests. Real-engine audio quality/performance and network-isolated QA evidence is separate from these contracts and belongs to #23; do not infer those results from mocked provider tests. diff --git a/public/app.ts b/public/app.ts index e5808be..8b242db 100644 --- a/public/app.ts +++ b/public/app.ts @@ -56,6 +56,7 @@ const names = { openai: "OpenAI", elevenlabs: "ElevenLabs", "9router": "9Router", + "local-whisper": "Local Whisper", }; let token: string, settings: PublicSettings, @@ -86,13 +87,15 @@ interface ApiOptions { body?: unknown; raw?: boolean; signal?: AbortSignal; + sttSnapshot?: string; } async function api>( path: string, decode: (value: unknown) => T, - { method = "GET", body, raw = false, signal }: ApiOptions = {}, + { method = "GET", body, raw = false, signal, sttSnapshot }: ApiOptions = {}, ): Promise { const headers: Record = { "x-agentflow-token": token }; + if (sttSnapshot) headers["x-agentflow-stt"] = sttSnapshot; if (raw && !(body instanceof Blob)) throw Error("Expected audio blob"); if (body !== undefined) headers["Content-Type"] = raw @@ -459,11 +462,17 @@ async function listen() { return; } call = true; - const g = generation; + aborter?.abort(); + const g = ++generation; + const sttSnapshot = JSON.stringify({ + stt: settings.stt, + routerUrl: settings.routerUrl, + }); aborter = new AbortController(); + const recordingAborter = aborter; notice(""); try { - stream = await navigator.mediaDevices.getUserMedia({ + const acquiredStream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, @@ -471,9 +480,10 @@ async function listen() { }, }); if (g !== generation) { - stream.getTracks().forEach((t) => t.stop()); + acquiredStream.getTracks().forEach((t) => t.stop()); return; } + stream = acquiredStream; const chunks: Blob[] = []; const mime = ["audio/webm;codecs=opus", "audio/mp4", "audio/webm"].find( (x) => MediaRecorder.isTypeSupported(x), @@ -485,6 +495,7 @@ async function listen() { if (e.data.size) chunks.push(e.data); }; recorder.onstop = async () => { + if (g !== generation) return; const blob = new Blob(chunks, { type: recorder.mimeType }); releaseMic(); if (g !== generation) return; @@ -498,7 +509,8 @@ async function listen() { method: "POST", body: blob, raw: true, - signal: aborter.signal, + signal: recordingAborter.signal, + sttSnapshot, }); if (g !== generation) return; if (!d.text.trim()) { @@ -512,7 +524,7 @@ async function listen() { $("message").value = d.text; await send(d.text); } catch (e) { - report(e); + if (g === generation) report(e); } }; recorder.start(200); @@ -536,7 +548,7 @@ async function listen() { const finish = gate.sample(rms, performance.now()); if ( (finish && $("handsfree").checked) || - performance.now() - started > 90000 + performance.now() - started > 89000 ) { recorder.stop(); return; @@ -545,6 +557,7 @@ async function listen() { } tick(); } catch (e) { + if (g !== generation) return; call = false; releaseMic(); report( @@ -669,6 +682,7 @@ function paintSettings() { for (const side of ["stt", "tts"] as const) { $(`${side}Provider`).replaceChildren(); for (const [id, name] of Object.entries(names)) { + if (side === "tts" && id === "local-whisper") continue; const o = new Option(name, id); $(`${side}Provider`).add(o); } @@ -678,6 +692,7 @@ function paintSettings() { $("ttsVoice").value = settings.tts.voice || ""; $("ttsSpeed").value = String(settings.tts.speed ?? 1); $("routerUrl").value = settings.routerUrl; + $("localWhisperUrl").value = settings.stt.baseUrl || "http://127.0.0.1:17864"; $("sttLabel").textContent = names[settings.stt.provider] + " listens"; $("ttsLabel").textContent = (settings.tts.provider === "deepgram" && deepgramVoice(settings.tts.model) @@ -688,6 +703,7 @@ function paintSettings() { paintVoicePicker(); $("keyRows").replaceChildren(); for (const [id, name] of Object.entries(names)) { + if (id === "local-whisper") continue; const row = document.createElement("div"); row.className = "keyrow"; const label = document.createElement("span"); @@ -829,7 +845,11 @@ $("settingsForm").onsubmit = async (e) => { settings = await api("/settings", parsePublicSettings, { method: "PUT", body: { - stt: { provider: $("sttProvider").value, model: $("sttModel").value }, + stt: { + provider: $("sttProvider").value, + model: $("sttModel").value, + baseUrl: $("localWhisperUrl").value, + }, tts: { provider: $("ttsProvider").value, model: $("ttsModel").value, diff --git a/public/contracts.ts b/public/contracts.ts index 68b5c2f..e800db5 100644 --- a/public/contracts.ts +++ b/public/contracts.ts @@ -1,5 +1,6 @@ export type AgentKind = "codex" | "claude"; -export type Provider = "deepgram" | "openai" | "elevenlabs" | "9router"; +export type Provider = + "deepgram" | "openai" | "elevenlabs" | "9router" | "local-whisper"; export type SpeechSide = "stt" | "tts"; export interface SpeechConfig { provider: Provider; @@ -105,7 +106,9 @@ export function errorCode(error: unknown): string | number | undefined { export function isProvider(value: unknown): value is Provider { return ( typeof value === "string" && - ["deepgram", "openai", "elevenlabs", "9router"].includes(value) + ["deepgram", "openai", "elevenlabs", "9router", "local-whisper"].includes( + value, + ) ); } export function isAgent(value: unknown): value is AgentKind { @@ -137,9 +140,14 @@ export function parseSettings(value: unknown): SettingsValue { return { provider: c.provider, model: string(c.model), + ...(c.provider === "local-whisper" + ? { baseUrl: localWhisperUrl(c.baseUrl) } + : {}), ...(typeof c.voice === "string" ? { voice: c.voice } : {}), }; } + if (record(r.tts).provider === "local-whisper") + throw Error("Local Whisper supports transcription only"); return { stt: side(r.stt), tts: { ...side(r.tts), speed: playbackSpeed(record(r.tts).speed) }, @@ -283,12 +291,14 @@ export function parsePublicSettings(value: unknown): PublicSettings { openai: boolean(cfg.openai), elevenlabs: boolean(cfg.elevenlabs), "9router": boolean(cfg["9router"]), + "local-whisper": true, }, defaults: { deepgram: defaults("deepgram"), openai: defaults("openai"), elevenlabs: defaults("elevenlabs"), "9router": defaults("9router"), + "local-whisper": { stt: "base.en", tts: "", voice: "" }, }, }; } @@ -325,3 +335,29 @@ export function parseRouterModels(value: unknown) { }; }); } + +export function localWhisperUrl( + value: unknown = "http://127.0.0.1:17864", +): string { + const message = + "Local Whisper requires a loopback HTTP URL without credentials, paths or queries"; + let u: URL; + try { + u = new URL(String(value)); + } catch { + throw Error(message); + } + if ( + u.protocol !== "http:" || + !["127.0.0.1", "[::1]", "localhost"].includes(u.hostname) || + u.username || + u.password || + u.pathname !== "/" || + u.search || + u.hash + ) + throw Error(message); + // Avoid DNS resolution even for localhost. + if (u.hostname === "localhost") u.hostname = "127.0.0.1"; + return u.origin; +} diff --git a/public/dom.ts b/public/dom.ts index c21180e..dc66215 100644 --- a/public/dom.ts +++ b/public/dom.ts @@ -51,6 +51,7 @@ interface Elements { ttsVoice: HTMLInputElement; ttsSpeed: HTMLSelectElement; routerUrl: HTMLInputElement; + localWhisperUrl: HTMLInputElement; keyRows: HTMLElement; settingsNotice: HTMLElement; routerModels: HTMLButtonElement; diff --git a/public/index.html b/public/index.html index 0210049..684c1c8 100644 --- a/public/index.html +++ b/public/index.html @@ -3,5 +3,5 @@
YOUR WORK, OUT LOUD

Pick up the conversation.

Select a session, then start talking.
CODEX + CLAUDE CODE

Ready when you are

Your session. Your context. Now with a voice.

Deepgram listens/Deepgram speaks
No session selected
-
MAKE IT YOURS

Speech & settings

Choose who listens and who speaks. Keys stay in your Mac’s Keychain.

Listening · speech to text
Speaking · text to speech

Provider keys

+
MAKE IT YOURS

Speech & settings

Choose who listens and who speaks. Keys stay in your Mac’s Keychain.

Listening · speech to text
Speaking · text to speech

Local Whisper transcribes on this Mac with no API key. Set up and start the engine with agentflow local-whisper setup and agentflow local-whisper start. The model field labels the model loaded by that server; it does not switch server models. Speaking and agent reasoning use their independently configured providers.

Provider keys

New conversation

Uses your existing agent login. You can return to this conversation from the terminal.

diff --git a/scripts/benchmark-local-whisper.py b/scripts/benchmark-local-whisper.py new file mode 100644 index 0000000..9eac8b9 --- /dev/null +++ b/scripts/benchmark-local-whisper.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Opt-in synthetic whisper.cpp benchmark; requires macOS say and ffmpeg, no downloads. + +Run with external networking denied after installing/starting the engine. +This tests the raw engine; application adapter and microphone tests are separate. +""" +import argparse +import json +import math +from pathlib import Path +import re +import subprocess +import tempfile +import time +import urllib.request +from urllib.parse import urlsplit + + +def words(text): + return re.findall(r"[a-z0-9]+", text.lower()) + + +def distance(expected, actual): + row = list(range(len(actual) + 1)) + for i, left in enumerate(expected, 1): + current = [i] + for j, right in enumerate(actual, 1): + current.append(min(current[-1] + 1, row[j] + 1, row[j - 1] + (left != right))) + row = current + return row[-1] + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + raise ValueError("Benchmark redirects prohibited") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + url = urlsplit(args.url) + if url.scheme != "http" or url.hostname not in ("127.0.0.1", "::1") or url.username or url.password or url.path not in ("", "/") or url.query or url.fragment: + parser.error("Use a literal loopback HTTP engine URL") + corpus = json.loads(Path(__file__).with_name("local-whisper-corpus.json").read_text()) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + results = [] + with tempfile.TemporaryDirectory(prefix="local-speech-benchmark-") as temp: + directory = Path(temp) + for name, reference in corpus.items(): + aiff, wav = directory / "fixture.aiff", directory / "fixture.wav" + subprocess.run(["/usr/bin/say", "-r", "145", "-o", str(aiff), reference], check=True) + subprocess.run(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", str(aiff), "-ar", "16000", "-ac", "1", "-af", "apad", "-t", "10", "-c:a", "pcm_s16le", str(wav)], check=True) + boundary = "local-speech-benchmark-fixture" + body = (f'--{boundary}\r\nContent-Disposition: form-data; name="response_format"\r\n\r\njson\r\n--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="fixture.wav"\r\nContent-Type: audio/wav\r\n\r\n'.encode() + wav.read_bytes() + f'\r\n--{boundary}--\r\n'.encode()) + request = urllib.request.Request(args.url.rstrip("/") + "/inference", data=body, headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}) + start = time.monotonic() + with opener.open(request, timeout=180) as response: + transcript = json.load(response)["text"].strip() + elapsed = time.monotonic() - start + results.append({"fixture": name, "duration_seconds": 10, "elapsed_seconds": round(elapsed, 3), "reference": reference, "text": transcript, "word_errors": distance(words(reference), words(transcript)), "reference_words": len(words(reference))}) + print(name, round(elapsed, 3), flush=True) + clean = [row for row in results if row["fixture"].startswith("clean")] + warm = sorted(row["elapsed_seconds"] for row in clean[1:]) + report = {"description": "Raw engine synthetic TTS fixtures, not human microphone accuracy", "clean_wer": sum(row["word_errors"] for row in clean) / sum(row["reference_words"] for row in clean), "clean_warm_p95_seconds": warm[math.ceil(len(warm) * .95) - 1], "results": results} + args.output.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({key: value for key, value in report.items() if key != "results"})) + + +if __name__ == "__main__": + main() diff --git a/scripts/build-assets.ts b/scripts/build-assets.ts index 5621559..0c51c8b 100644 --- a/scripts/build-assets.ts +++ b/scripts/build-assets.ts @@ -4,3 +4,8 @@ await mkdir(resolve("dist/public"), { recursive: true }); for (const name of ["index.html", "style.css"]) await copyFile(resolve("public", name), resolve("dist/public", name)); await chmod(resolve("dist/src/cli.js"), 0o755); + +await copyFile( + resolve("scripts/local-whisper.py"), + resolve("dist/scripts/local-whisper.py"), +); diff --git a/scripts/local-whisper-corpus.json b/scripts/local-whisper-corpus.json new file mode 100644 index 0000000..d20f4e7 --- /dev/null +++ b/scripts/local-whisper-corpus.json @@ -0,0 +1,23 @@ +{ + "clean1": "Please move the meeting to Thursday morning and send everyone an updated invitation.", + "clean2": "The weather is clear today. I will walk to the store and buy some coffee before lunch.", + "clean3": "We finished the first draft yesterday. The next step is to review the changes and prepare the final report.", + "clean4": "Could you check the schedule and tell me whether we have enough time to finish this task today?", + "clean5": "This recording is a test of local speech recognition. No internet connection should be needed to understand these words.", + "clean6": "After breakfast we will take the train into the city and meet our friends near the museum.", + "clean7": "Remember to close the window before leaving the house. The forecast says it will rain later this afternoon.", + "clean8": "Please save a copy of the document and put it in the folder for next week.", + "clean9": "The package arrived this morning. Everything was packed carefully and nothing appears to be missing.", + "clean10": "Let us start with the simplest example and make sure it works before adding any more features.", + "clean11": "I would like a table for four people on Saturday evening, preferably beside a window.", + "clean12": "Thank you for your help with the project. Your comments made the instructions much easier to follow.", + "clean13": "Turn left at the next street and continue for two blocks. The library is on the right.", + "clean14": "We should test the new settings before the meeting starts so that everyone can hear clearly.", + "clean15": "The team agreed to fix the current problems before beginning work on the next version.", + "clean16": "There are several ways to solve this problem. Let us compare them carefully and choose one.", + "clean17": "I left my notebook on the kitchen table. Please bring it with you when you come over.", + "clean18": "The train leaves at nine tomorrow morning. We should arrive at the station at least twenty minutes early.", + "clean19": "Put the blue box on the shelf and leave the smaller green box beside the door.", + "clean20": "The lights are working again. We can finish cleaning the room and move the chairs back into place.", + "technical": "Run the unit tests, review the pull request, and restart the local Whisper server. Check the TypeScript compiler and the Rust build." +} diff --git a/scripts/local-whisper-offline.sb b/scripts/local-whisper-offline.sb new file mode 100644 index 0000000..8cff53c --- /dev/null +++ b/scripts/local-whisper-offline.sb @@ -0,0 +1,6 @@ +(version 1) +(allow default) +(deny network*) +(allow network-inbound (local ip "localhost:*")) +(allow network-outbound (remote ip "localhost:*")) +(allow network-bind (local ip "localhost:*")) diff --git a/scripts/local-whisper.py b/scripts/local-whisper.py new file mode 100644 index 0000000..8e60c46 --- /dev/null +++ b/scripts/local-whisper.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Explicit, offline-after-setup whisper.cpp development runtime (Python 3.9+).""" + +import argparse +import contextlib +import fcntl +import hashlib +import http.server +import json +import os +from pathlib import Path +import secrets +import select +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import urllib.request + +REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1" +MODEL_SHA256 = "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002" +MODEL_URL = ( + f"https://huggingface.co/ggerganov/whisper.cpp/resolve/{REVISION}/ggml-base.en.bin" +) +PORTS = {"murmur": 17863, "agentflow": 17864} + + +def emit(value): + print(json.dumps(value), flush=True) + + +def write_json(path, value): + temp = path.with_suffix(".tmp") + temp.write_text(json.dumps(value)) + temp.chmod(0o600) + temp.replace(path) + + +def read_json(path): + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + return {} + + +def digest(path): + with path.open("rb") as source: + h = hashlib.sha256() + for chunk in iter(lambda: source.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def install(state, source, expected): + if len(expected) != 64 or any(c not in "0123456789abcdef" for c in expected): + raise ValueError("sha256 must be 64 lowercase hexadecimal characters") + fd, name = tempfile.mkstemp(prefix="model-", suffix=".partial", dir=state) + try: + with os.fdopen(fd, "wb") as target: + shutil.copyfileobj(source, target) + target.flush() + os.fsync(target.fileno()) + if digest(Path(name)) != expected: + raise ValueError("model checksum mismatch; existing model preserved") + models = state / "models" + models.mkdir(exist_ok=True, mode=0o700) + os.replace(name, models / (expected + ".bin")) + link = state / "model.next" + link.unlink(missing_ok=True) + link.symlink_to(Path("models") / (expected + ".bin")) + os.replace(link, state / "model.bin") + finally: + with contextlib.suppress(FileNotFoundError): + os.unlink(name) + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("local runtime redirects are forbidden") + + +def request_local(url, token=None, timeout=0.5): + # Ignore proxy environment: runtime control and readiness must stay on loopback. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + request = urllib.request.Request( + url, headers={"Authorization": "Bearer " + token} if token else {} + ) + with opener.open(request, timeout=timeout) as response: + return json.load(response) + + +def control(record, action): + try: + result = request_local( + f"http://127.0.0.1:{int(record['control_port'])}/{action}", record["token"] + ) + return result if result.get("token") == record["token"] else None + except (OSError, ValueError, KeyError): + return None + + +def ready(port): + try: + return request_local(f"http://127.0.0.1:{port}/health").get("status") == "ok" + except (OSError, ValueError): + return False + + +def read_supervisor_token(fd, timeout=5): + """Read a single bounded secret frame from the parent's private pipe.""" + deadline = time.monotonic() + timeout + frame = bytearray() + while len(frame) <= 65: + remaining = deadline - time.monotonic() + if remaining <= 0 or not select.select([fd], [], [], remaining)[0]: + raise ValueError("supervisor secret handoff timed out") + chunk = os.read(fd, 66 - len(frame)) + if not chunk: + raise ValueError("supervisor secret handoff was incomplete") + frame.extend(chunk) + if b"\n" in frame: + break + if ( + len(frame) != 65 + or frame[-1:] != b"\n" + or any(byte not in b"0123456789abcdef" for byte in frame[:-1]) + ): + raise ValueError("invalid supervisor secret handoff") + return frame[:-1].decode("ascii") + + +def supervise(state, binary, port, token, timeout): + child = None + stopping = False + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + nonlocal stopping + if self.headers.get("Authorization") != "Bearer " + token: + self.send_error(403) + return + if self.path == "/stop": + stopping = True + elif self.path != "/status": + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write( + json.dumps( + { + "token": token, + "ready": bool(child and child.poll() is None and ready(port)), + "url": f"http://127.0.0.1:{port}", + } + ).encode() + ) + + def log_message(self, *args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + server.timeout = 0.2 + record = {"control_port": server.server_port, "token": token, "port": port} + try: + child = subprocess.Popen( + [ + binary, + "--host", + "127.0.0.1", + "--port", + str(port), + "-m", + str(state / "model.bin"), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + write_json(state / "runtime.json", record) + deadline = time.monotonic() + timeout + became_ready = False + while not stopping and child.poll() is None: + became_ready = became_ready or ready(port) + if not became_ready and time.monotonic() >= deadline: + break + server.handle_request() + finally: + write_json( + state / "lifecycle.json", + {"reason": "stopped" if stopping else "engine_exited_or_unready"}, + ) + if child and child.poll() is None: + child.terminate() + try: + child.wait(timeout=5) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + server.server_close() + if read_json(state / "runtime.json").get("token") == token: + (state / "runtime.json").unlink(missing_ok=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--app", choices=PORTS, required=True) + parser.add_argument("--state-dir", type=Path) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser( + "setup", help="explicitly download and verify pinned base.en model" + ) + imp = commands.add_parser("import-model") + imp.add_argument("path", type=Path) + imp.add_argument("--sha256", required=True) + start = commands.add_parser("start") + start.add_argument("--binary", default="whisper-server") + start.add_argument("--port", type=int) + start.add_argument("--timeout", type=float, default=30) + commands.add_parser("status") + commands.add_parser("stop") + internal = commands.add_parser("_supervise", help=argparse.SUPPRESS) + internal.add_argument("binary") + internal.add_argument("port", type=int) + internal.add_argument("timeout", type=float) + args = parser.parse_args() + state = ( + ( + args.state_dir + or Path.home() + / "Library" + / "Application Support" + / args.app + / "local-whisper" + ) + .expanduser() + .resolve() + ) + state.mkdir(parents=True, exist_ok=True, mode=0o700) + state.chmod(0o700) + if args.command == "_supervise": + token = read_supervisor_token(sys.stdin.fileno()) + sys.stdin.close() + supervise(state, args.binary, args.port, token, args.timeout) + return + with (state / "lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + record = read_json(state / "runtime.json") + if args.command == "status": + result = control(record, "status") + model = state / "model.bin" + reason = read_json(state / "lifecycle.json").get("reason", "stopped") + if not model.is_file(): + reason = "missing_model" + elif digest(model) != model.resolve().stem: + reason = "corrupt_model" + elif not shutil.which("whisper-server"): + reason = "missing_binary" + public = {k: v for k, v in (result or {}).items() if k != "token"} + public.update( + { + "owned": bool(result), + "ready": bool(result and result["ready"]), + "reason": ("ready" if result["ready"] else "loading") + if result + else reason, + "tested_runtime": "whisper.cpp 1.9.1", + "model": str(model), + } + ) + emit(public) + elif args.command == "stop": + result = control(record, "stop") + deadline = time.monotonic() + 7 + while ( + result + and (state / "runtime.json").exists() + and time.monotonic() < deadline + ): + time.sleep(0.05) + if result and (state / "runtime.json").exists(): + raise ValueError("owned runtime did not stop within deadline") + emit({"ready": False, "stopped": bool(result)}) + elif args.command in ("setup", "import-model"): + if control(record, "status"): + raise ValueError("stop owned runtime before replacing model") + if args.command == "setup": + with urllib.request.urlopen(MODEL_URL, timeout=60) as source: + install(state, source, MODEL_SHA256) + else: + with args.path.open("rb") as source: + install(state, source, args.sha256) + emit( + { + "model": str(state / "model.bin"), + "sha256": digest(state / "model.bin"), + } + ) + elif args.command == "start": + existing = control(record, "status") + if existing: + emit({k: v for k, v in existing.items() if k != "token"}) + return + model = state / "model.bin" + expected = model.resolve().stem + if not model.is_file() or digest(model) != expected: + raise ValueError( + "missing or corrupt model; run setup or import-model explicitly" + ) + binary = shutil.which(args.binary) + if not binary: + raise ValueError( + "whisper-server missing; install with brew install whisper-cpp, or pass --binary PATH (tested 1.9.1)" + ) + port = args.port or PORTS[args.app] + if not 1 <= port <= 65535 or not 0 < args.timeout <= 300: + raise ValueError( + "port must be 1..65535 and timeout must be >0 and <=300 seconds" + ) + with socket.socket() as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", port)) + except OSError: + write_json(state / "lifecycle.json", {"reason": "port_conflict"}) + raise ValueError( + "port already in use; choose --port or use external server mode" + ) from None + token = secrets.token_hex(32) + supervisor = subprocess.Popen( + [ + sys.executable, + str(Path(__file__).resolve()), + "--app", + args.app, + "--state-dir", + str(state), + "_supervise", + binary, + str(port), + str(args.timeout), + ], + start_new_session=True, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + supervisor.stdin.write((token + "\n").encode("ascii")) + supervisor.stdin.close() + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline and supervisor.poll() is None: + record = read_json(state / "runtime.json") + result = ( + control(record, "status") if record.get("token") == token else None + ) + if result and result["ready"]: + emit({k: v for k, v in result.items() if k != "token"}) + return + time.sleep(0.1) + if record.get("token") == token: + control(record, "stop") + with contextlib.suppress(subprocess.TimeoutExpired): + supervisor.wait(timeout=7) + raise ValueError( + "local engine failed readiness deadline; verify model and whisper-server binary" + ) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError) as error: + emit({"error": str(error)}) + sys.exit(1) diff --git a/scripts/test_local_whisper.py b/scripts/test_local_whisper.py new file mode 100644 index 0000000..76d06be --- /dev/null +++ b/scripts/test_local_whisper.py @@ -0,0 +1,208 @@ +import hashlib, json, pathlib, socket, subprocess, sys, tempfile, unittest +import importlib.util, io, http.server, threading, os + +spec = importlib.util.spec_from_file_location( + "runtime", pathlib.Path(__file__).with_name("local-whisper.py") +) +runtime = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runtime) +SCRIPT = pathlib.Path(__file__).with_name("local-whisper.py") + + +class RuntimeTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="local whisper test ") + self.root = pathlib.Path(self.tmp.name) + self.model = self.root / "source model.bin" + self.model.write_bytes(b"fixture model") + self.digest = hashlib.sha256(self.model.read_bytes()).hexdigest() + + def tearDown(self): + self.run_cli("stop") + self.tmp.cleanup() + + def run_cli(self, *args): + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--app", + "murmur", + "--state-dir", + str(self.root / "state folder"), + *args, + ], + capture_output=True, + text=True, + timeout=60, + ) + + def install(self): + r = self.run_cli("import-model", str(self.model), "--sha256", self.digest) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def test_interrupted_import_preserves_selection(self): + self.install() + state = self.root / "state folder" + before = (state / "model.bin").read_bytes() + + class Broken(io.BytesIO): + def read(self, *args): + raise OSError("interrupted download") + + with self.assertRaises(OSError): + runtime.install(state, Broken(b"new"), "0" * 64) + self.assertEqual((state / "model.bin").read_bytes(), before) + self.assertFalse(list(state.glob("*.partial"))) + + def test_redirect_never_follows_or_sends_token_to_destination(self): + calls = [] + + class Destination(http.server.BaseHTTPRequestHandler): + def do_GET(self): + calls.append(self.headers) + self.send_response(200) + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, *args): + pass + + target = http.server.HTTPServer(("127.0.0.1", 0), Destination) + + class Redirect(Destination): + def do_GET(self): + self.send_response(302) + self.send_header("Location", f"http://127.0.0.1:{target.server_port}/") + self.end_headers() + + redirect = http.server.HTTPServer(("127.0.0.1", 0), Redirect) + for server in (target, redirect): + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with self.assertRaises(ValueError): + runtime.request_local( + f"http://127.0.0.1:{redirect.server_port}/", "private-token" + ) + self.assertEqual(calls, []) + finally: + for server in (target, redirect): + server.shutdown() + server.server_close() + + def test_bad_import_preserves_model(self): + self.install() + before = (self.root / "state folder/model.bin").read_bytes() + r = self.run_cli("import-model", str(self.model), "--sha256", "0" * 64) + self.assertNotEqual(r.returncode, 0) + self.assertEqual((self.root / "state folder/model.bin").read_bytes(), before) + + def test_missing_model_never_starts(self): + r = self.run_cli("start") + self.assertNotEqual(r.returncode, 0) + self.assertIn("model", r.stdout) + + def test_corruption_detected_at_start(self): + self.install() + (self.root / "state folder/model.bin").write_bytes(b"corrupt") + r = self.run_cli("start") + self.assertNotEqual(r.returncode, 0) + self.assertIn("corrupt", r.stdout) + + def test_startup_exit_zero_and_timeout_fail_without_runtime(self): + self.install() + for body in ("pass", "import time;time.sleep(60)"): + fake = self.root / "unready engine" + fake.write_text("#!" + sys.executable + "\n" + body + "\n") + fake.chmod(0o755) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + r = self.run_cli( + "start", "--binary", str(fake), "--port", str(port), "--timeout", "0.5" + ) + self.assertNotEqual(r.returncode, 0, r.stdout) + self.assertFalse(json.loads(self.run_cli("status").stdout)["ready"]) + self.assertFalse((self.root / "state folder/runtime.json").exists()) + + def test_supervisor_secret_handoff_is_bounded_and_validated(self): + for frame in (b"bad\n", b"g" * 64 + b"\n", b"a" * 64, b"a" * 66): + with self.subTest(length=len(frame)): + reader, writer = os.pipe() + try: + os.write(writer, frame) + os.close(writer) + with self.assertRaises(ValueError): + runtime.read_supervisor_token(reader, timeout=0.1) + finally: + os.close(reader) + reader, writer = os.pipe() + try: + with self.assertRaisesRegex(ValueError, "timed out"): + runtime.read_supervisor_token(reader, timeout=0.05) + finally: + os.close(reader) + os.close(writer) + + def test_stale_record_cannot_kill_unrelated_process(self): + state = self.root / "state folder" + state.mkdir() + (state / "runtime.json").write_text( + json.dumps({"port": 1, "token": "stale", "pid": 1}) + ) + r = self.run_cli("stop") + self.assertEqual(r.returncode, 0, r.stdout) + + def test_lifecycle_path_spaces_and_port_conflict(self): + self.install() + fake = self.root / "fake whisper server" + fake.write_text( + "#!" + + sys.executable + + '\nimport argparse,http.server\np=argparse.ArgumentParser();p.add_argument("--port",type=int);a,_=p.parse_known_args()\nclass H(http.server.BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200);self.end_headers();self.wfile.write(b"{\\"status\\":\\"ok\\"}")\n def log_message(self,*a):pass\nhttp.server.HTTPServer(("127.0.0.1",a.port),H).serve_forever()\n' + ) + fake.chmod(0o755) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + args = ("start", "--binary", str(fake), "--port", str(port), "--timeout", "30") + r = self.run_cli(*args) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + first = json.loads(r.stdout) + self.assertTrue(first["ready"]) + token = json.loads((self.root / "state folder/runtime.json").read_text())[ + "token" + ] + commands = subprocess.check_output(["ps", "-axo", "command="], text=True) + self.assertFalse( + token in commands, "supervisor secret must not appear in process arguments" + ) + self.assertNotIn("token", first) + self.assertEqual(self.run_cli(*args).returncode, 0) + self.assertEqual( + json.loads((self.root / "state folder/runtime.json").read_text())["token"], + token, + ) + self.assertTrue(json.loads(self.run_cli("status").stdout)["ready"]) + self.assertEqual(self.run_cli("stop").returncode, 0) + self.assertFalse(json.loads(self.run_cli("status").stdout)["ready"]) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen() + port = s.getsockname()[1] + args = ( + "start", + "--binary", + str(fake), + "--port", + str(port), + "--timeout", + "30", + ) + r = self.run_cli(*args) + self.assertNotEqual(r.returncode, 0) + self.assertIn("port", r.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/cli.ts b/src/cli.ts index d0c6f8f..dd0fcb5 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,6 +9,7 @@ import { } from "../public/contracts.js"; import { object } from "./native-protocol.js"; import WebSocket from "ws"; +import { spawn } from "node:child_process"; import { readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -285,6 +286,31 @@ async function mcp() { } async function main() { const [command, ...args] = process.argv.slice(2); + if (command === "local-whisper") { + const script = fileURLToPath( + new URL("../scripts/local-whisper.py", import.meta.url), + ); + const code = await new Promise((resolve, reject) => { + const child = spawn( + "python3", + [ + script, + "--app", + "agentflow", + "--state-dir", + join(appHome, "local-whisper"), + ...args, + ], + { stdio: "inherit" }, + ); + child.on("error", () => + reject(Error("Local Whisper setup requires Python 3 on PATH")), + ); + child.on("close", (code) => resolve(code ?? 1)); + }); + process.exitCode = code; + return; + } if (command === "service") { console.log(await service(args[0] || "status")); return; @@ -334,7 +360,7 @@ async function main() { return; } console.log( - 'Agentflow\n start | open | status\n run codex|claude [--resume ID]\n speak [--speed 0.75–2] --text "Hello" (or pipe text through stdin)\n service install|status|restart|stop|uninstall\n mcp', + 'Agentflow\n start | open | status\n run codex|claude [--resume ID]\n speak [--speed 0.75–2] --text "Hello" (or pipe text through stdin)\n service install|status|restart|stop|uninstall\n local-whisper setup|import-model|start|status|stop [--help]\n mcp', ); } main().catch((e) => { diff --git a/src/local-audio.ts b/src/local-audio.ts new file mode 100644 index 0000000..ff1e7cc --- /dev/null +++ b/src/local-audio.ts @@ -0,0 +1,94 @@ +import { spawn } from "node:child_process"; + +// In-memory conversion: no recordings or converter diagnostics are retained. +export async function normalizeLocalAudio( + audio: Buffer, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const pcm = await new Promise((resolve, reject) => { + const child = spawn( + process.env.AGENTFLOW_FFMPEG || "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-protocol_whitelist", + "pipe", + "-i", + "pipe:0", + "-t", + "91", + "-vn", + "-ar", + "16000", + "-ac", + "1", + "-f", + "s16le", + "pipe:1", + ], + { stdio: ["pipe", "pipe", "ignore"] }, + ); + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + const fail = (message: string) => { + if (settled) return; + settled = true; + child.kill("SIGKILL"); + reject(Error(message)); + }; + const abort = () => fail("Local transcription canceled"); + const timer = setTimeout( + () => fail("Audio conversion timed out. Check ffmpeg and retry."), + 15000, + ); + signal?.addEventListener("abort", abort, { once: true }); + child.on("error", () => + fail( + "Local audio conversion requires ffmpeg on PATH (or AGENTFLOW_FFMPEG).", + ), + ); + child.stdin.on("error", () => {}); + child.stdout.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > 90 * 32000) + fail("Recording exceeds the 90-second local transcription limit"); + else chunks.push(chunk); + }); + child.on("close", (code) => { + clearTimeout(timer); + signal?.removeEventListener("abort", abort); + if (settled) return; + settled = true; + if (code || !size) + reject( + Error("Could not decode recording. Install ffmpeg and record again."), + ); + else resolve(Buffer.concat(chunks)); + }); + child.stdin.end(audio); + if (signal?.aborted) abort(); + }); + signal?.throwIfAborted(); + let energy = 0; + for (let i = 0; i + 1 < pcm.length; i += 2) + energy += (pcm.readInt16LE(i) / 32768) ** 2; + if (Math.sqrt(energy / (pcm.length / 2)) < 0.001) return null; + const header = Buffer.alloc(44); + header.write("RIFF"); + header.writeUInt32LE(36 + pcm.length, 4); + header.write("WAVEfmt ", 8); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(1, 22); + header.writeUInt32LE(16000, 24); + header.writeUInt32LE(32000, 28); + header.writeUInt16LE(2, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36); + header.writeUInt32LE(pcm.length, 40); + return Buffer.concat([header, pcm]); +} diff --git a/src/local-http.ts b/src/local-http.ts new file mode 100644 index 0000000..3e62ba4 --- /dev/null +++ b/src/local-http.ts @@ -0,0 +1,46 @@ +import { request, Agent } from "node:http"; +import { Readable } from "node:stream"; +import type { Fetcher } from "./speech.js"; +// A dedicated agent deliberately ignores the global fetch dispatcher and proxy env. +const direct = new Agent({ proxyEnv: {} }); +export const localFetch: Fetcher = async (url, options) => { + const encoded = new Request(url, options); + const body = Buffer.from(await encoded.arrayBuffer()); + options.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const req = request( + url, + { + method: "POST", + agent: direct, + headers: Object.fromEntries(encoded.headers), + signal: options.signal ?? undefined, + }, + (res) => { + try { + const headers = new Headers(); + for (const [key, value] of Object.entries(res.headers)) + if (value !== undefined) + headers.set(key, Array.isArray(value) ? value.join(", ") : value); + if ([204, 205, 304].includes(res.statusCode || 0)) res.resume(); + resolve( + new Response( + [204, 205, 304].includes(res.statusCode || 0) + ? null + : (Readable.toWeb(res) as ReadableStream), + { + status: res.statusCode || 502, + headers, + }, + ), + ); + } catch (error) { + res.destroy(); + reject(error); + } + }, + ); + req.on("error", reject); + req.end(body); + }); +}; diff --git a/src/server.ts b/src/server.ts index 87fceac..9f3ad3a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -8,6 +8,7 @@ import { errorCode, playbackSpeed, parseTerminalMessage, + parseSettings, type KeyStorage, } from "../public/contracts.js"; import type { TerminalSession } from "./terminal.js"; @@ -301,7 +302,30 @@ export async function createApp({ return reply(res, { ok: true }); } if (req.method === "POST" && url.pathname === "/api/transcribe") { - const c = await settings.speech("stt"); + let c = await settings.speech("stt"); + const snapshot = req.headers["x-agentflow-stt"]; + if (snapshot !== undefined) { + try { + if (typeof snapshot !== "string" || snapshot.length > 4096) + throw Error(); + const parsed = parseSettings({ + ...record(JSON.parse(snapshot)), + tts: settings.value.tts, + }); + c = { + ...parsed.stt, + baseUrl: + parsed.stt.provider === "local-whisper" + ? parsed.stt.baseUrl! + : parsed.routerUrl, + }; + } catch { + throw failure( + "invalid_configuration", + "Invalid recording speech settings. Start a new recording.", + ); + } + } const audio = await body(req, 20 * 1024 * 1024); const ac = new AbortController(); res.on("close", () => ac.abort()); @@ -309,7 +333,7 @@ export async function createApp({ res, await transcribe( c, - await store.get(c.provider), + c.provider === "local-whisper" ? "" : await store.get(c.provider), audio, req.headers["content-type"] || "audio/webm", ac.signal, @@ -332,7 +356,7 @@ export async function createApp({ res.on("close", () => ac.abort()); const r = await synthesize( c, - await store.get(c.provider), + c.provider === "local-whisper" ? "" : await store.get(c.provider), string(b.text), ac.signal, ); diff --git a/src/speech.ts b/src/speech.ts index 5406d26..5e2635c 100644 --- a/src/speech.ts +++ b/src/speech.ts @@ -1,4 +1,6 @@ -import { record } from "../public/contracts.js"; +import { localFetch } from "./local-http.js"; +import { normalizeLocalAudio } from "./local-audio.js"; +import { record, localWhisperUrl } from "../public/contracts.js"; import type { SpeechConfig, Provider } from "../public/contracts.js"; export type Fetcher = (url: string, options: RequestInit) => Promise; import { deepgramVoice } from "../public/deepgram-voices.js"; @@ -7,6 +9,7 @@ export const providers: Provider[] = [ "openai", "elevenlabs", "9router", + "local-whisper", ]; export const defaults = { deepgram: { stt: "nova-3", tts: "aura-2-helena-en", voice: "" }, @@ -16,12 +19,18 @@ export const defaults = { voice: "coral", }, elevenlabs: { stt: "scribe_v2", tts: "eleven_flash_v2_5", voice: "" }, + "local-whisper": { stt: "base.en", tts: "", voice: "" }, "9router": { stt: "", tts: "", voice: "" }, }; export function failure(code: string, message: string, status = 400) { return Object.assign(new Error(message), { code, status }); } function config(c: SpeechConfig, key: string) { + if (c.provider === "local-whisper") + return { + base: localWhisperUrl(c.baseUrl), + headers: {} as Record, + }; if (!providers.includes(c.provider)) throw failure("invalid_provider", "Unknown speech provider"); if (!key && c.provider !== "9router") @@ -112,15 +121,28 @@ export async function transcribe( audio: Buffer, mime = "audio/webm", signal?: AbortSignal, - fetcher: Fetcher = fetch, + fetcher?: Fetcher, ) { + fetcher ??= c.provider === "local-whisper" ? localFetch : fetch; const { base, headers } = config(c, key); if (!audio?.length || audio.length > 20 * 1024 * 1024) throw failure("invalid_audio", "Audio must be between 1 byte and 20 MB"); if (!/^audio\/(webm|wav|x-wav|mpeg|mp3|mp4|ogg|flac)(;.*)?$/.test(mime)) throw failure("invalid_audio", "Unsupported audio format"); let url: string, body: BodyInit; - if (c.provider === "deepgram") { + if (c.provider === "local-whisper") { + const normalized = await normalizeLocalAudio(audio, signal); + if (!normalized) return { text: "" }; + url = `${base}/inference`; + body = new FormData(); + body.append( + "file", + new Blob([new Uint8Array(normalized)], { type: "audio/wav" }), + "recording.wav", + ); + body.append("response_format", "json"); + body.append("temperature", "0"); + } else if (c.provider === "deepgram") { url = `${base}/v1/listen?model=${encodeURIComponent(c.model)}&smart_format=true`; headers["Content-Type"] = mime; body = new Uint8Array(audio); @@ -140,9 +162,43 @@ export async function transcribe( signal, fetcher, ); + let payload: unknown; + if (c.provider === "local-whisper") { + const reader = r.body?.getReader(); + if (!reader) + throw failure( + "malformed_response", + "Local Whisper returned no transcript", + 502, + ); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + signal?.throwIfAborted(); + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > 65536) + throw failure( + "malformed_response", + "Local Whisper transcript response is too large", + 502, + ); + chunks.push(value); + } + payload = Buffer.concat(chunks).toString("utf8"); + } finally { + await reader.cancel().catch(() => {}); + } + } let data: Record; try { - data = record(await r.json()); + data = record( + c.provider === "local-whisper" + ? JSON.parse(String(payload)) + : await r.json(), + ); } catch { throw failure( "malformed_response", @@ -168,6 +224,7 @@ export async function transcribe( "Speech provider returned no valid transcript", 502, ); + signal?.throwIfAborted(); return { text: text.trim() }; } export async function synthesize( @@ -177,6 +234,11 @@ export async function synthesize( signal?: AbortSignal, fetcher: Fetcher = fetch, ) { + if (c.provider === "local-whisper") + throw failure( + "invalid_provider", + "Local Whisper supports transcription only", + ); const { base, headers } = config(c, key); if (typeof text !== "string" || !text.trim() || text.length > 12000) throw failure( diff --git a/src/state.ts b/src/state.ts index 787a23c..4afa800 100644 --- a/src/state.ts +++ b/src/state.ts @@ -1,5 +1,6 @@ import { record, + localWhisperUrl, isProvider, parseSettings, playbackSpeed, @@ -44,7 +45,8 @@ export class Settings { } async public(): Promise { const configured = {} as Record; - for (const p of providers) configured[p] = !!(await this.store.get(p)); + for (const p of providers) + configured[p] = p === "local-whisper" || !!(await this.store.get(p)); return { ...structuredClone(this.value), configured, defaults }; } async update(value: unknown) { @@ -57,7 +59,15 @@ export class Settings { throw failure("invalid_provider", "Invalid provider"); if (typeof c.model !== "string" || c.model.length > 200) throw failure("invalid_model", "Invalid model"); + if (side === "tts" && c.provider === "local-whisper") + throw failure( + "invalid_provider", + "Local Whisper supports transcription only", + ); next[side] = { + ...(c.provider === "local-whisper" + ? { baseUrl: localWhisperUrl(c.baseUrl) } + : {}), provider: c.provider, model: c.model.trim(), voice: typeof c.voice === "string" ? c.voice.slice(0, 200) : "", @@ -84,7 +94,10 @@ export class Settings { async speech(side: SpeechSide) { return { ...structuredClone(this.value[side]), - baseUrl: this.value.routerUrl, + baseUrl: + this.value[side].provider === "local-whisper" + ? localWhisperUrl(this.value[side].baseUrl) + : this.value.routerUrl, }; } } diff --git a/src/store.ts b/src/store.ts index 67b99da..d0a7554 100644 --- a/src/store.ts +++ b/src/store.ts @@ -27,6 +27,7 @@ const envNames = { openai: "OPENAI_API_KEY", elevenlabs: "ELEVENLABS_API_KEY", "9router": "NINEROUTER_KEY", + "local-whisper": "", }; export class KeyStore { cache: Map; @@ -34,6 +35,7 @@ export class KeyStore { this.cache = new Map(); } async get(provider: Provider) { + if (provider === "local-whisper") return ""; if (this.cache.has(provider)) return this.cache.get(provider)!; let value = ""; try { diff --git a/test/browser/dashboard.spec.ts b/test/browser/dashboard.spec.ts index fb87ffa..eb9e10c 100644 --- a/test/browser/dashboard.spec.ts +++ b/test/browser/dashboard.spec.ts @@ -16,6 +16,7 @@ const settings: PublicSettings = { openai: false, elevenlabs: false, "9router": false, + "local-whisper": true, }, defaults: { deepgram: { stt: "nova-3", tts: "aura-2-thalia-en", voice: "" }, @@ -25,6 +26,7 @@ const settings: PublicSettings = { voice: "coral", }, elevenlabs: { stt: "scribe_v2", tts: "eleven_flash_v2_5", voice: "" }, + "local-whisper": { stt: "base.en", tts: "", voice: "" }, "9router": { stt: "", tts: "", voice: "" }, }, }; @@ -382,3 +384,166 @@ test("voice preview is blocked during an active microphone conversation", async ); expect(calls).toBe(0); }); +test("local listening persists independently and is unavailable as TTS or API key", async ({ + page, +}) => { + await mock(page); + await page.goto("/"); + await page.getByRole("button", { name: /Native sync test/ }).click(); + await page.locator("#settingsButton").click(); + await page.locator("#sttProvider").selectOption("local-whisper"); + await page.getByLabel("Local Whisper URL").fill("http://127.0.0.1:17864"); + await expect( + page.locator('#ttsProvider option[value="local-whisper"]'), + ).toHaveCount(0); + await expect(page.getByLabel("Local Whisper API key")).toHaveCount(0); + await page.getByRole("button", { name: "Save preferences" }).click(); + await expect(page.locator("#sttLabel")).toHaveText("Local Whisper listens"); + await expect(page.locator("#ttsLabel")).toContainText("Deepgram"); + await page.reload(); + await expect(page.locator("#sessionId")).toHaveText(id); + await page.locator("#settingsButton").click(); + await expect(page.getByLabel("Local Whisper URL")).toHaveValue( + "http://127.0.0.1:17864", + ); +}); +test("recording snapshots listening settings and canceled inference cannot submit a turn", async ({ + page, +}) => { + const getTurn = await mock(page); + let snapshot: unknown; + let release: () => void = () => {}; + await page.route("**/api/transcribe", async (route) => { + snapshot = route.request().headers()["x-agentflow-stt"]; + await new Promise((r) => (release = r)); + await route.fulfill({ json: { text: "late transcript" } }).catch(() => {}); + }); + await page.goto("/"); + await page.getByRole("button", { name: /Native sync test/ }).click(); + await page.locator("#handsfree").uncheck(); + await page.locator("#talk").click(); + await expect(page.locator("#state")).toHaveText("Listening"); + await page.waitForTimeout(300); + await page.locator("#talk").click(); + await expect.poll(() => snapshot).toBeTruthy(); + expect(JSON.parse(String(snapshot)).stt.provider).toBe("deepgram"); + await page.locator("#end").click(); + release(); + await page.waitForTimeout(200); + expect(getTurn()).toBeUndefined(); +}); +test("starting another recording invalidates the previous pending transcript", async ({ + page, +}) => { + const getTurn = await mock(page); + let release: () => void = () => {}; + let received = false; + await page.route("**/api/transcribe", async (route) => { + received = true; + await new Promise((r) => (release = r)); + await route.fulfill({ json: { text: "superseded" } }).catch(() => {}); + }); + await page.goto("/"); + await page.getByRole("button", { name: /Native sync test/ }).click(); + await page.locator("#handsfree").uncheck(); + await page.locator("#talk").click(); + await expect(page.locator("#state")).toHaveText("Listening"); + await page.waitForTimeout(300); + await page.locator("#talk").click(); + await expect.poll(() => received).toBe(true); + await page.locator("#talk").click(); + await expect(page.locator("#state")).toHaveText("Listening"); + release(); + await page.waitForTimeout(200); + expect(getTurn()).toBeUndefined(); + await page.locator("#end").click(); +}); +test("out-of-order microphone permission results release every acquired track", async ({ + page, +}) => { + await page.addInitScript(() => { + const original = navigator.mediaDevices.getUserMedia.bind( + navigator.mediaDevices, + ); + const microphoneRequests: { stream: MediaStream; release: () => void }[] = + []; + Object.assign(window, { microphoneRequests }); + navigator.mediaDevices.getUserMedia = async (constraints) => { + const stream = await original(constraints); + await new Promise((release) => + microphoneRequests.push({ stream, release }), + ); + return stream; + }; + }); + await mock(page); + await page.goto("/"); + await page.getByRole("button", { name: /Native sync test/ }).click(); + await page.locator("#handsfree").uncheck(); + await page.locator("#talk").click(); + const count = () => + page.evaluate( + () => + (window as unknown as { microphoneRequests: unknown[] }) + .microphoneRequests.length, + ); + await expect.poll(count).toBe(1); + await page.locator("#talk").click(); + await expect.poll(count).toBe(2); + await page.evaluate(() => + ( + window as unknown as { microphoneRequests: { release: () => void }[] } + ).microphoneRequests[1]!.release(), + ); + await expect(page.locator("#state")).toHaveText("Listening"); + await page.evaluate(() => + ( + window as unknown as { microphoneRequests: { release: () => void }[] } + ).microphoneRequests[0]!.release(), + ); + await page.locator("#end").click(); + await expect + .poll(() => + page.evaluate(() => + ( + window as unknown as { microphoneRequests: { stream: MediaStream }[] } + ).microphoneRequests.flatMap((r) => + r.stream.getTracks().map((t) => t.readyState), + ), + ), + ) + .toEqual(["ended", "ended"]); +}); +test("stale microphone permission rejection cannot stop the current recording", async ({ + page, +}) => { + await page.addInitScript(() => { + const original = navigator.mediaDevices.getUserMedia.bind( + navigator.mediaDevices, + ); + let requests = 0; + navigator.mediaDevices.getUserMedia = async (constraints) => { + if (++requests === 1) + return new Promise((_, reject) => + Object.assign(window, { + rejectOldMicrophone: () => reject(Error("Old permission denied")), + }), + ); + return original(constraints); + }; + }); + await mock(page); + await page.goto("/"); + await page.getByRole("button", { name: /Native sync test/ }).click(); + await page.locator("#handsfree").uncheck(); + await page.locator("#talk").click(); + await page.locator("#talk").click(); + await expect(page.locator("#state")).toHaveText("Listening"); + await page.evaluate(() => + ( + window as unknown as { rejectOldMicrophone: () => void } + ).rejectOldMicrophone(), + ); + await expect(page.locator("#state")).toHaveText("Listening"); + await page.locator("#end").click(); +}); diff --git a/test/build.test.ts b/test/build.test.ts index b00990b..a089901 100644 --- a/test/build.test.ts +++ b/test/build.test.ts @@ -15,3 +15,12 @@ test("built CLI and browser assets run without TypeScript at runtime", async () const pkg = JSON.parse(await readFile("package.json", "utf8")); assert.equal(pkg.bin.agentflow, "./dist/src/cli.js"); }); +test("built local-whisper CLI exposes setup help without starting services", async () => { + const { stdout } = await promisify(execFile)(process.execPath, [ + "dist/src/cli.js", + "local-whisper", + "--help", + ]); + assert.match(stdout, /import-model/); + assert.match(stdout, /setup/); +}); diff --git a/test/local-whisper.test.ts b/test/local-whisper.test.ts new file mode 100644 index 0000000..c4b430c --- /dev/null +++ b/test/local-whisper.test.ts @@ -0,0 +1,292 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Settings } from "../src/state.js"; +import { transcribe, synthesize } from "../src/speech.js"; +import type { SpeechConfig } from "../public/contracts.js"; +const local = { + provider: "local-whisper", + model: "base.en", + baseUrl: "http://127.0.0.1:17864", +} as unknown as SpeechConfig; +function wav() { + const b = Buffer.alloc(44 + 3200); + b.write("RIFF"); + b.writeUInt32LE(b.length - 8, 4); + b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); + b.writeUInt16LE(1, 20); + b.writeUInt16LE(1, 22); + b.writeUInt32LE(16000, 24); + b.writeUInt32LE(32000, 28); + b.writeUInt16LE(2, 32); + b.writeUInt16LE(16, 34); + b.write("data", 36); + b.writeUInt32LE(3200, 40); + for (let i = 44; i < b.length; i += 2) b.writeInt16LE(1000, i); + return b; +} +test("local STT persists independently without consulting its Keychain", async () => { + const dir = await mkdtemp(join(tmpdir(), "af-local-")); + try { + const store = { + get: async (p: string) => { + assert.notEqual(p, "local-whisper"); + return ""; + }, + }; + const s = new Settings(store, dir); + const tts = structuredClone(s.value.tts); + await s.update({ stt: local }); + const reloaded = new Settings(store, dir); + await reloaded.init(); + assert.deepEqual(await reloaded.speech("stt"), { ...local, voice: "" }); + assert.deepEqual(reloaded.value.tts, tts); + assert.equal( + (await reloaded.public()).configured[ + "local-whisper" as keyof Awaited< + ReturnType + >["configured"] + ], + true, + ); + await assert.rejects(() => s.update({ tts: local }), /transcription/i); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); +test("local Whisper posts normalized WAV to inference without credentials or redirects", async () => { + const result = await transcribe( + local, + "DO_NOT_SEND", + wav(), + "audio/wav", + undefined, + async (url, options) => { + assert.equal(url, local.baseUrl + "/inference"); + assert.equal(options.redirect, "error"); + assert.equal(new Headers(options.headers).get("Authorization"), null); + assert.ok(options.body instanceof FormData); + const f = options.body.get("file") as File; + assert.equal(f.type, "audio/wav"); + assert.equal( + Buffer.from(await f.arrayBuffer()).toString("ascii", 0, 4), + "RIFF", + ); + assert.equal(options.body.get("response_format"), "json"); + return Response.json({ text: " hello locally " }); + }, + ); + assert.equal(result.text, "hello locally"); +}); +test("local endpoint rejects remote hosts, credentials, paths and queries before transport", async () => { + for (const baseUrl of [ + "https://example.com", + "http://localhost.evil", + "http://a:b@127.0.0.1", + "http://127.0.0.1/path", + "http://127.0.0.1?key=secret", + ]) + await assert.rejects( + () => transcribe({ ...local, baseUrl }, "", wav(), "audio/wav"), + /loopback/i, + ); +}); +test("local Whisper cannot be used for speech output", async () => { + await assert.rejects(() => synthesize(local, "", "hello"), /transcription/i); +}); +test("silence bypasses inference, broken recordings fail recoverably, and canceled work never returns text", async () => { + const silent = wav(); + silent.fill(0, 44); + let requests = 0; + const fetcher = async () => { + requests++; + return Response.json({ text: "hallucination" }); + }; + assert.equal( + (await transcribe(local, "", silent, "audio/wav", undefined, fetcher)).text, + "", + ); + assert.equal(requests, 0); + await assert.rejects( + () => + transcribe( + local, + "", + Buffer.from("broken"), + "audio/webm", + undefined, + fetcher, + ), + /decode recording/i, + ); + assert.equal(requests, 0); + const c = new AbortController(); + await assert.rejects(() => + transcribe(local, "", wav(), "audio/wav", c.signal, async () => { + c.abort(); + return Response.json({ text: "late" }); + }), + ); +}); +test("local response is bounded and canceling during response cannot yield a transcript", async () => { + await assert.rejects( + () => + transcribe(local, "", wav(), "audio/wav", undefined, async () => + Response.json({ text: "x".repeat(70000) }), + ), + /too large/i, + ); +}); +test("real local HTTP transport sends WAV without secrets and refuses redirects", async () => { + const { createServer } = await import("node:http"); + let count = 0; + const server = createServer(async (req, res) => { + count++; + assert.equal(req.headers.authorization, undefined); + assert.equal(req.url, "/inference"); + let bytes = 0; + for await (const chunk of req) bytes += chunk.length; + assert.ok(bytes > 3200); + res.writeHead(302, { location: "http://127.0.0.1:1/forbidden" }); + res.end(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + try { + await assert.rejects( + () => + transcribe( + { ...local, baseUrl: `http://127.0.0.1:${address.port}` }, + "SECRET", + wav(), + "audio/wav", + ), + /unreachable|HTTP 302/, + ); + assert.equal(count, 1); + } finally { + await new Promise((r) => server.close(() => r())); + } +}); +test("local production transport bypasses global fetch proxy configuration", async () => { + const { createServer } = await import("node:http"); + const server = createServer(async (req, res) => { + for await (const chunk of req) { + } + res.setHeader("content-type", "application/json"); + res.end('{"text":"direct local"}'); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const a = server.address(); + assert.ok(a && typeof a !== "string"); + const saved = globalThis.fetch; + try { + globalThis.fetch = async () => { + throw Error("global proxy must not see recording"); + }; + assert.equal( + ( + await transcribe( + { ...local, baseUrl: `http://127.0.0.1:${a.port}` }, + "", + wav(), + "audio/wav", + ) + ).text, + "direct local", + ); + } finally { + globalThis.fetch = saved; + await new Promise((r) => server.close(() => r())); + } +}); +test("HTTP transcription uses captured local selection even after settings change without reading a key", async () => { + const { createApp } = await import("../src/server.js"); + const dir = await mkdtemp(join(tmpdir(), "af-snapshot-")); + let reads = 0; + const app = await createApp({ + port: 0, + dir, + store: { + get: async () => { + reads++; + throw Error("no key lookup"); + }, + }, + agents: { init: async () => {}, close() {} }, + }); + try { + const { token } = await (await fetch(app.url + "/api/bootstrap")).json(); + const silent = wav(); + silent.fill(0, 44); + const r = await fetch(app.url + "/api/transcribe", { + method: "POST", + headers: { + "x-agentflow-token": token, + "content-type": "audio/wav", + "x-agentflow-stt": JSON.stringify({ + stt: local, + routerUrl: "http://127.0.0.1:20128", + }), + }, + body: silent, + }); + assert.equal(r.status, 200); + assert.deepEqual(await r.json(), { text: "" }); + assert.equal(reads, 0); + } finally { + await app.close(); + await rm(dir, { recursive: true, force: true }); + } +}); +test("empty HTTP statuses become recoverable transcript errors instead of crashing the service", async () => { + const { createServer } = await import("node:http"); + const server = createServer(async (req, res) => { + for await (const chunk of req) { + } + res.writeHead(204); + res.end(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const a = server.address(); + assert.ok(a && typeof a !== "string"); + try { + await assert.rejects( + () => + transcribe( + { ...local, baseUrl: `http://127.0.0.1:${a.port}` }, + "", + wav(), + "audio/wav", + ), + /no transcript/, + ); + } finally { + await new Promise((r) => server.close(() => r())); + } +}); +test("invalid HTTP status rejects in the caller instead of crashing the process", async () => { + const { execFile } = await import("node:child_process"); + const { promisify } = await import("node:util"); + const { stdout } = await promisify(execFile)( + process.execPath, + [ + "--input-type=module", + "-e", + ` + import {createServer} from 'node:http';import {localFetch} from './dist/src/local-http.js'; + const server=createServer(async(req,res)=>{for await(const chunk of req){}res.writeHead(700);res.end('bad');}); + await new Promise(r=>server.listen(0,'127.0.0.1',r)); + try { await localFetch('http://127.0.0.1:'+server.address().port,{method:'POST',body:'audio'});throw Error('must reject'); } + catch(e) { if(e.message==='must reject') throw e; console.log('recovered'); } + finally {server.closeAllConnections();await new Promise(r=>server.close(r));} + `, + ], + { timeout: 5000 }, + ); + assert.match(stdout, /recovered/); +});