Add managed local Kokoro voice output and offline playback

.github/workflows/ci.yml

@@ -20,5 +20,6 @@ jobs:
       - run: npm run typecheck
       - run: npm test
       - run: python3 scripts/test_local_whisper.py
+      - run: python3 scripts/test_local_kokoro.py
       - run: npx playwright install --with-deps chromium
       - run: npm run test:e2e

README.md

@@ -135,3 +135,7 @@ Browser WebM/MP4/WAV recordings are converted in memory by ffmpeg to 16 kHz mono
 Each recording captures its listening settings before microphone permission is requested. Canceling or changing sessions discards late transcription and cannot submit it to the agent. Canceling closes local transport and terminates conversion, but whisper-server may continue inference internally; this is not a claim that HTTP disconnect preempts GPU computation. Inference has a 60-second timeout; check `agentflow local-whisper status`, the server's loaded model and the configured URL after engine errors.
 
 Tests and TDD evidence: [local transcription validation](docs/local-whisper-validation.md). Real microphone permission, audible playback and sleep/wake still require device acceptance.
+
+### Local spoken replies
+
+Use [Local Kokoro](docs/local-kokoro.md) for keyless, offline voice output, independently of local Whisper input. Explicit setup installs the verified model and CPU runtime; select Local Kokoro in Speaking settings. Native agent reasoning remains independently configured.

docs/local-kokoro-validation.md

@@ -0,0 +1,42 @@
+# Local Kokoro acceptance evidence
+
+Issue: [#24](https://github.com/nolanmak/Agentflow/issues/24). Reference QA: 2026-10-15, Apple M3, 24 GiB RAM, macOS; Node 26.5.1, Python 3.12 for real inference. See [setup and measured performance](local-kokoro.md).
+
+| Criterion | Evidence |
+| --- | --- |
+| AC1 pinning/offline | Hash-locked `kokoro-requirements.txt`; model/voices SHA-256 in `local-kokoro.py`; real engine and app executed under sandbox denying non-loopback outbound traffic |
+| AC2 independent/keyless | `local-kokoro.test.ts` persistence, rejection of STT and remote origins, direct HTTP proxy/redirect and no-key-lookup tests; browser Local Kokoro settings test |
+| AC3 lifecycle | `test_local_kokoro.py` asset validation, atomic install, explicit missing-model failure, owned control stop, invalid-origin rejection; setup/start/status/stop exercised with real engine |
+| AC4 playback | Browser ordered retry test and exact-text chunking test; real `/api/synthesize` returned PCM WAV; CLI and MCP successfully executed native playback offline at saved 1.25× speed |
+| AC5 interruption/retry | Browser pending-synthesis and delayed-replay tests; real Chromium WAV playback pause measured 1.3–2.8 ms (250 ms bound); no new agent turn on Replay; real HTTP stalled-response abort/recovery and runtime disconnect tests |
+| AC6 failure boundaries | Runtime tests cover invalid model/voice, empty/oversized text, Unicode payloads, output bounds and deadlines; adapter tests cover malformed/oversized audio, cancellation and HTTP failures; lifecycle tests cover asset integrity, occupied ports, idempotency and startup recovery; full deterministic suites |
+| AC7 real-engine measurements | Offline cold 1.975 s, warm 1.714–1.741 s for 5.735 s audio, RSS about754 MiB; full model selected after int8 CPU operator incompatibility; real CLI/MCP smoke succeeded |
+| AC8 delivery | PR/CI and merge recorded in issue; task worktree cleaned after merge |
+
+## Actual test-first evidence
+
+- Adapter settings initially failed `Invalid/Unknown provider`; minimal provider/config support made them pass.
+- Malformed WAV regression initially failed with `Missing expected rejection`; validation now rejects it.
+- HTTP synthesis initially returned500 because the endpoint read a credential; selected local output now bypasses key lookup.
+- Runtime first tests failed because `local-kokoro.py` did not exist. Disconnect tests later failed for missing synthesis helper before its implementation.
+- Browser local output test failed because the provider option count was0, then passed after UI wiring.
+- CLI help test failed because `local-kokoro --help` printed generic help without `state-dir`, then passed after lifecycle dispatch/build packaging.
+- Voice-test interruption failed because Interrupt was disabled during synthesis. It now remains available and late results cannot play.
+- Delayed replay overwrote Interrupted with Your turn. A generation guard fixes the reproduced failure.
+- Speech failure was overwritten by Your turn and later queued sentences kept running. The regression now retains the error, stops the queue, and retries only on explicit Replay in original sentence order.
+- Long reply chunking test failed before the helper existed; chunks now concatenate to the complete original input without splitting surrogate pairs.
+- Real `afplay` rejected WAV saved as `.mp3` (`AudioFileOpen dta?`); the filename regression failed before container-aware extension handling. Same bytes as `.wav` and the real CLI/MCP path then succeeded.
+
+## Commands
+
+- `npm run typecheck`
+- `npm test`:65 tests passed
+- `npm run test:e2e`:19 tests passed, including real browser audio-element interruption
+- `python3 scripts/test_local_kokoro.py`:15 tests passed
+- `python3 scripts/test_local_whisper.py`:9 tests passed
+- `node scripts/live-local-kokoro.mjs --play` under the sandbox profile in the setup guide: real app synthesis, CLI and MCP playback passed
+- `git diff --check`
+
+The browser tests use synthetic audio and fake native sessions, while real-engine checks use synthetic text and disposable application settings. No personal recordings, native histories or cloud credentials are fixtures. Native audio-player success is observed, but speaker quality, microphone/echo behavior and sleep/wake are not human-verified here. General device QA remains tracked in #23.
+
+- Unicode request regression first failed HTTP413 for12000 escaped characters; the byte limit now accommodates the complete permitted Unicode text while retaining the character bound.

docs/local-kokoro.md

@@ -0,0 +1,55 @@
+# Local Kokoro speech output
+
+Kokoro speaks replies on your computer. It is independent of local Whisper input and the native agent's reasoning provider. No cloud key is read for local synthesis, and failure never falls back to a cloud voice.
+
+## Setup
+
+Install Python 3.12 or 3.13 (on macOS, `brew install python@3.12`), then run from a built Agentflow checkout:
+
+```sh
+npm ci
+npm run build
+node dist/src/cli.js local-kokoro setup --python python3.12
+node dist/src/cli.js local-kokoro start
+node dist/src/cli.js local-kokoro status
+```
+
+Setup explicitly downloads the pinned Python runtime dependencies, model and voices. It verifies SHA-256 checksums; inference needs no external network. The full model is roughly 311 MB, plus voice data and the Python environment. Default state is `~/.config/agentflow/local-kokoro` (`AGENTFLOW_HOME` overrides the parent directory). Use `--state-dir PATH` before the lifecycle subcommand for isolated state. A stopped runtime stays stopped until explicitly started; Agentflow startup does not download models or capture a microphone.
+
+In Speech & settings choose **Local Kokoro** for Speaking. Model: `kokoro`. Voice: `af_heart` (American English); other installed American/British voices can be listed at `http://127.0.0.1:17865/v1/audio/voices`. Save preferences and use **Test voice**. Playback speed applies in the browser or native player, independently of synthesis. The default endpoint is `http://127.0.0.1:17865`; only explicit loopback HTTP origins are accepted. The browser talks to Agentflow, which connects directly without proxy routing or redirects.
+
+```sh
+node dist/src/cli.js speak --text "Local speech is ready."
+node dist/src/cli.js local-kokoro stop
+```
+
+The MCP `speak` tool uses the same local provider and macOS playback path. Changing voice output does not change transcription, agent routing, or session identity. This integration currently exposes English voices, not custom voice cloning.
+
+## Failure and interruption
+
+Missing/corrupt assets require explicit `setup`; no download occurs during a speech request. `start` checks assets and reports an occupied port without stopping the other process. `stop` authenticates ownership through a private control token, not a guessed PID.
+
+Native replies are spoken in sentence order. Long unpunctuated text is split without discarding its tail. Interrupt immediately pauses browser playback, aborts the HTTP request, and invalidates late audio. The runtime finishes its current bounded CPU chunk, then detects disconnect and skips subsequent chunks. While busy it rejects concurrent work instead of queuing stale speech. Each request has a synthesis deadline and bounded audio output. If output fails, the error remains visible and remaining queued sentences stop; Replay retries speech without another agent turn.
+
+## Runtime and licenses
+
+Pinned `kokoro-onnx` 0.6.1 (MIT), ONNX Runtime 1.23.2 and NumPy 2.3.5 run on CPU. Kokoro v1.0 weights are Apache-2.0. The phonemizer/eSpeak dependency stack includes GPL-family components; the complete runtime is not Apache-only. Setup installs dependencies in a separate virtual environment; see package licenses before redistributing a bundled environment.
+
+The model and voice checksums are in `scripts/local-kokoro.py`; all Python package hashes are in `scripts/kokoro-requirements.txt`. The FP32 model is intentional: the tested int8 variant lacked a supported ARM CPU operator.
+
+Sources: [Kokoro model](https://huggingface.co/hexgrad/Kokoro-82M), [ONNX implementation](https://github.com/thewh1teagle/kokoro-onnx).
+
+## Validation
+
+Reference: Apple M3 Mac, 24 GiB RAM, CPU inference. Under a macOS sandbox denying external outbound networking, a 94-character synthetic sentence produced 5.735 seconds of valid mono 24 kHz PCM audio. Cold synthesis: 1.975 seconds. Three warm runs: 1.741, 1.714, 1.734 seconds. Runtime RSS after four runs: about 754 MiB. Readiness including asset verification took about 0.77 seconds. These are measurements on this device, not minimum hardware guarantees or voice-quality scores.
+
+Chromium playback of synthetic WAV on the reference Mac paused in about 3 ms after an interrupt; the test enforces 250 ms and verifies no additional agent turn during Replay. Automated device playback is distinct from a person confirming speaker quality. Microphone, echo, sleep/wake and human listening checks remain part of the broader device QA issue, not claimed here.
+
+To reproduce the opt-in real-engine app check after starting Kokoro, run from the repository root:
+
+```sh
+npm run build
+sandbox-exec -p '(version 1)(allow default)(deny network-outbound)(allow network-outbound (remote ip "localhost:*"))' node scripts/live-local-kokoro.mjs --play
+```
+
+This uses disposable application settings and synthetic text. `--play` exercises CLI and MCP through the real Mac audio player; omit it to validate audio without playback. Start the runtime under the same sandbox profile to also enforce offline model execution. The reference run returned HTTP 200 with 82,060 bytes of PCM WAV, CLI `Spoken.`, and MCP `Speech played on the local Mac.` No cloud credentials were read. This confirms successful audio-device execution, not a subjective listening assessment.

public/app.ts

@@ -10,7 +10,7 @@ import {
   parseRouterModels,
 } from "./contracts.js";
 import { deepgramVoices, deepgramVoice } from "./deepgram-voices.js";
-import { sentences, VoiceGate } from "./voice.js";
+import { sentences, speechChunks, VoiceGate } from "./voice.js";
 import { element as $ } from "./dom.js";
 import {
   errorMessage,
@@ -57,6 +57,7 @@ const names = {
   elevenlabs: "ElevenLabs",
   "9router": "9Router",
   "local-whisper": "Local Whisper",
+  "local-kokoro": "Local Kokoro",
 };
 let token: string,
   settings: PublicSettings,
@@ -134,8 +135,9 @@ function state(label: string, detail: string, kind = "idle") {
   $("state").textContent = label;
   $("hint").textContent = detail;
   $("orb").dataset.state = kind;
-  $("end").disabled = !call && !currentJob;
-  $("interrupt").disabled = !call && !currentJob && !audio;
+  $("end").disabled = !call && !currentJob && kind !== "speaking";
+  $("interrupt").disabled =
+    !call && !currentJob && !audio && kind !== "speaking";
   $("talk").textContent = recording
     ? "■ Send recording"
     : call
@@ -259,11 +261,13 @@ async function select(s: SessionSummary) {
   }
 }
 function queueSpeech(text: string, g: number) {
-  const spoken = text;
+  for (const chunk of speechChunks(text)) queueSpeechChunk(chunk, g);
+}
+function queueSpeechChunk(spoken: string, g: number) {
   if (!spoken) return;
   speechQueue = speechQueue
     .then(async () => {
-      if (g !== generation) return;
+      if (g !== generation || lastSpeechError) return;
       state(
         "Speaking",
         selected?.agent === "claude"
@@ -273,7 +277,7 @@ function queueSpeech(text: string, g: number) {
       );
       const blob = await api("/synthesize", parseAudio, {
         method: "POST",
-        body: { text: spoken.slice(0, 12000) },
+        body: { text: spoken },
         signal: aborter.signal,
       });
       if (g !== generation) return;
@@ -393,7 +397,7 @@ async function send(text: string) {
       $("approval").hidden = true;
     }
   }
-  if (g !== generation) return;
+  if (g !== generation || lastSpeechError) return;
   state("Your turn", "Continue the conversation, or return to your terminal.");
   if (call && $("handsfree").checked && !lastSpeechError) await listen();
   refresh().catch(() => {});
@@ -682,7 +686,11 @@ function paintSettings() {
   for (const side of ["stt", "tts"] as const) {
     $(`${side}Provider`).replaceChildren();
     for (const [id, name] of Object.entries(names)) {
-      if (side === "tts" && id === "local-whisper") continue;
+      if (
+        (side === "tts" && id === "local-whisper") ||
+        (side === "stt" && id === "local-kokoro")
+      )
+        continue;
       const o = new Option(name, id);
       $(`${side}Provider`).add(o);
     }
@@ -693,6 +701,7 @@ function paintSettings() {
   $("ttsSpeed").value = String(settings.tts.speed ?? 1);
   $("routerUrl").value = settings.routerUrl;
   $("localWhisperUrl").value = settings.stt.baseUrl || "http://127.0.0.1:17864";
+  $("localKokoroUrl").value = settings.tts.baseUrl || "http://127.0.0.1:17865";
   $("sttLabel").textContent = names[settings.stt.provider] + " listens";
   $("ttsLabel").textContent =
     (settings.tts.provider === "deepgram" && deepgramVoice(settings.tts.model)
@@ -703,7 +712,7 @@ function paintSettings() {
   paintVoicePicker();
   $("keyRows").replaceChildren();
   for (const [id, name] of Object.entries(names)) {
-    if (id === "local-whisper") continue;
+    if (id === "local-whisper" || id === "local-kokoro") continue;
     const row = document.createElement("div");
     row.className = "keyrow";
     const label = document.createElement("span");
@@ -759,12 +768,13 @@ async function testVoice() {
     await stop();
     lastSpeechError = false;
     aborter = new AbortController();
+    const g = generation;
     queueSpeech(
       "Agentflow is ready. This is your selected voice. You can talk to your Codex and Claude Code sessions here.",
       generation,
     );
     await speechQueue;
-    if (lastSpeechError) return;
+    if (lastSpeechError || g !== generation) return;
     state(
       "Voice test finished",
       "If you heard that, your speech output is connected.",
@@ -784,9 +794,12 @@ $("replay").onclick = async () => {
   if (!lastReply) return;
   await stop();
   aborter = new AbortController();
-  for (const s of sentences(lastReply, true).ready) queueSpeech(s, generation);
+  const g = generation;
+  lastSpeechError = false;
+  for (const s of sentences(lastReply, true).ready) queueSpeech(s, g);
   await speechQueue;
-  state("Your turn", "Replay complete.");
+  if (g === generation && !lastSpeechError)
+    state("Your turn", "Replay complete.");
 };
 $("composer").onsubmit = async (e) => {
   e.preventDefault();
@@ -854,6 +867,7 @@ $("settingsForm").onsubmit = async (e) => {
           provider: $("ttsProvider").value,
           model: $("ttsModel").value,
           voice: $("ttsVoice").value,
+          baseUrl: $("localKokoroUrl").value,
           speed: Number($("ttsSpeed").value),
         },
         routerUrl: $("routerUrl").value,

public/contracts.ts

@@ -1,6 +1,11 @@
 export type AgentKind = "codex" | "claude";
 export type Provider =
-  "deepgram" | "openai" | "elevenlabs" | "9router" | "local-whisper";
+  | "deepgram"
+  | "openai"
+  | "elevenlabs"
+  | "9router"
+  | "local-whisper"
+  | "local-kokoro";
 export type SpeechSide = "stt" | "tts";
 export interface SpeechConfig {
   provider: Provider;
@@ -106,9 +111,14 @@ export function errorCode(error: unknown): string | number | undefined {
 export function isProvider(value: unknown): value is Provider {
   return (
     typeof value === "string" &&
-    ["deepgram", "openai", "elevenlabs", "9router", "local-whisper"].includes(
-      value,
-    )
+    [
+      "deepgram",
+      "openai",
+      "elevenlabs",
+      "9router",
+      "local-whisper",
+      "local-kokoro",
+    ].includes(value)
   );
 }
 export function isAgent(value: unknown): value is AgentKind {
@@ -142,12 +152,16 @@ export function parseSettings(value: unknown): SettingsValue {
       model: string(c.model),
       ...(c.provider === "local-whisper"
         ? { baseUrl: localWhisperUrl(c.baseUrl) }
-        : {}),
+        : c.provider === "local-kokoro"
+          ? { baseUrl: localKokoroUrl(c.baseUrl) }
+          : {}),
       ...(typeof c.voice === "string" ? { voice: c.voice } : {}),
     };
   }
   if (record(r.tts).provider === "local-whisper")
     throw Error("Local Whisper supports transcription only");
+  if (record(r.stt).provider === "local-kokoro")
+    throw Error("Local Kokoro supports speech output only");
   return {
     stt: side(r.stt),
     tts: { ...side(r.tts), speed: playbackSpeed(record(r.tts).speed) },
@@ -292,13 +306,15 @@ export function parsePublicSettings(value: unknown): PublicSettings {
       elevenlabs: boolean(cfg.elevenlabs),
       "9router": boolean(cfg["9router"]),
       "local-whisper": true,
+      "local-kokoro": true,
     },
     defaults: {
       deepgram: defaults("deepgram"),
       openai: defaults("openai"),
       elevenlabs: defaults("elevenlabs"),
       "9router": defaults("9router"),
       "local-whisper": { stt: "base.en", tts: "", voice: "" },
+      "local-kokoro": { stt: "", tts: "kokoro", voice: "af_heart" },
     },
   };
 }
@@ -361,3 +377,15 @@ export function localWhisperUrl(
   if (u.hostname === "localhost") u.hostname = "127.0.0.1";
   return u.origin;
 }
+
+export function localKokoroUrl(
+  value: unknown = "http://127.0.0.1:17865",
+): string {
+  try {
+    return localWhisperUrl(value);
+  } catch {
+    throw Error(
+      "Local Kokoro requires a loopback HTTP URL without credentials, paths or queries",
+    );
+  }
+}

public/dom.ts

@@ -52,6 +52,7 @@ interface Elements {
   ttsSpeed: HTMLSelectElement;
   routerUrl: HTMLInputElement;
   localWhisperUrl: HTMLInputElement;
+  localKokoroUrl: HTMLInputElement;
   keyRows: HTMLElement;
   settingsNotice: HTMLElement;
   routerModels: HTMLButtonElement;