From c25ad552c529ce11bf1a5bbd0ac393b9b206da0f Mon Sep 17 00:00:00 2001 From: witbrock Date: Thu, 10 Sep 2026 18:48:42 -0400 Subject: [PATCH] Repair contextual dictation and mobile speech lifecycle (JVNAUTOSCI-888) --- docs/design_index.md | 1 + docs/engineering/speech_io.md | 111 +++++++++ .../integrations/internal_mcp/orchestrator.py | 4 +- src/backend/languagemodels/model_defaults.py | 10 + src/backend/server/routes/speech_routes.py | 101 +++++++- .../services/model_registry_service.py | 9 +- .../services/speech_transcription_service.py | 137 +++++++++++ .../durable/model_selection_workflow.py | 4 +- .../web/von_interface/static/js/chatTab.js | 150 ++++-------- .../web/von_interface/static/js/dictation.js | 227 ++++++++++++++++++ .../von_interface/static/js/settingsPage.js | 11 +- .../web/von_interface/static/js/speech.js | 38 ++- .../web/von_interface/static/styles.css | 21 ++ .../web/von_interface/templates/chat_tab.html | 8 + .../von_interface/templates/settings_tab.html | 1 + tests/backend/test_model_registry_service.py | 9 + ...t_orchestrator_model_fallback_execution.py | 20 ++ tests/backend/test_speech_transcription.py | 146 +++++++++++ .../test_workflow_model_selection_workflow.py | 1 + tests/frontend/dictation.test.js | 109 +++++++++ tests/frontend/speech.test.js | 21 +- 21 files changed, 1014 insertions(+), 125 deletions(-) create mode 100644 docs/engineering/speech_io.md create mode 100644 src/backend/services/speech_transcription_service.py create mode 100644 src/frontend/web/von_interface/static/js/dictation.js create mode 100644 tests/backend/test_speech_transcription.py create mode 100644 tests/frontend/dictation.test.js diff --git a/docs/design_index.md b/docs/design_index.md index 05692ab92..feeb6272f 100644 --- a/docs/design_index.md +++ b/docs/design_index.md @@ -95,6 +95,7 @@ live. | Evaluation, benchmarks, or research-sensitive architecture | [agent evaluation and research uptake](engineering/agent_evaluation_and_research_uptake.md) | | Minimal imposition, elicitation, or write policy | [minimal-imposition principle](engineering/minimal_imposition_design_principle.md) | | Frontend or browser acceptance | [Frontend browser validation](engineering/frontend_browser_user_view_validation.md), at the validation tier justified by the claim | +| Dictation, audio transcription or mobile speech playback | [Speech I/O](engineering/speech_io.md); current conversational delivery plan remains in its linked Jira issue | | Live user-visible behaviour or telemetry diagnosis | Applicable sections of the [real-path replay and telemetry loop](engineering/real_path_server_replay_and_telemetry_loop.md), using the risk tier in `AGENTS.md` | | Substantial Jira task definition or pull-request merge boundary | The decision discipline in [`AGENTS.md`](../AGENTS.md#5-capability-slice-planning), the practical [Jira guidance](engineering/operational_engineering_guide.md#53-jira) and [merge-decision procedure](engineering/operational_engineering_guide.md#62-make-new-evidence-change-a-decision), and the [pull-request template](../.github/PULL_REQUEST_TEMPLATE.md) | | Practical repository operation | [operational engineering guide](engineering/operational_engineering_guide.md), [authority-alignment scan](engineering/maintaining_global_design_constraints_and_authority_alignment_with_coding_agents.md) | diff --git a/docs/engineering/speech_io.md b/docs/engineering/speech_io.md new file mode 100644 index 000000000..f6770b27e --- /dev/null +++ b/docs/engineering/speech_io.md @@ -0,0 +1,111 @@ +# Recorded dictation and speech playback + +- **Kind:** Bounded implementation and operating contract +- **Reviewed:** 10 September 2026 +- **Owner / review trigger:** Speech workstream; review on audio-provider, + authentication, browser capture or presenter changes. +- **Current decisions:** [JVNAUTOSCI-888](https://naoinstitute.atlassian.net/browse/JVNAUTOSCI-888) + owns the recorded-dictation repair; + [JVNAUTOSCI-812](https://naoinstitute.atlassian.net/browse/JVNAUTOSCI-812) + owns the current conversational speech plan. Streaming voice is planned, + not provided by the block-dictation implementation. + +## User flow + +Select **Dictate**, speak, then **Finish dictation**. The transcript is inserted +at the original selection if the draft is unchanged; otherwise it is appended +to the edited draft. It remains editable. **Send Prompt** during recording +finishes transcription first and only sends on success. **Cancel dictation** +releases the microphone and prevents late permission or transcription results +from changing the draft. A failed upload retains audio in the active page and +offers **Retry recording** without another microphone request. + +Changing conversation or organisation cancels the capture. Leaving the page +releases it; backgrounding finishes a recording. Raw audio is not saved by Von. +Retry audio is lost when the page is closed or the capture is cancelled. + +**More actions → Dictation** also offers browser recognition explicitly. Its +support and contextual biasing vary by browser. Recognition results are treated +as complete, revisable snapshots so replayed final results do not accumulate +duplicates. Interim text is shown in the status; only final text enters the draft. + +## Runtime configuration + +The server needs its existing configured OpenAI key and an enabled audio model +in the authenticated user's or organisation's existing model pool. Supported +transcription models, in selection order, are `gpt-transcribe`, +`gpt-4o-transcribe` and `gpt-4o-mini-transcribe`. Optional +`VON_TRANSCRIPTION_MODEL` selects exactly one of these; it must still be enabled +in the scoped model pool. Add it as an additional model, not the primary +conversational model. Audio models are excluded from automatic chat fallback +and selection candidates. This does not change the main conversational model. +No credentials or enabled-model settings are changed by this feature. + +`GET /api/speech/capabilities` reports actor-specific availability. The composer +refreshes it on initialisation, focus and preference changes. A configured key +and enabled model do not prove provider entitlement or live recognition quality; +provider rejection is a visible retryable failure. + +`POST /api/speech/transcribe` accepts multipart `audio`, `context`, JSON +`vocabulary` and optional BCP 47 `language`. It derives identity and organisation +from existing trusted context, including the owned window-session header. Legacy +identity headers alone cannot authorise an external audio request. No request +field selects another actor's history or credentials. + +Audio is bounded to 24 MB, below the provider's 25 MB limit. Multipart parsing +also has a request bound. The recorder negotiates MP4 or WebM and stops near the +size limit; formats are validated again on the server. The provider HTTP request +has a 120-second transport bound to limit occupied workers, with no automatic +SDK retry. A failed request can be retried from retained browser audio. This is +an initial operational bound, not a speech-turn duration or a latency claim; +revisit it against actual successful recording/transcription durations. + +Context is the recent visible conversation and current draft (up to 6,000 +characters), plus up to 80 visible literal concept names. The service does not +scan Vontology or fetch private conversations. Context/keywords are recognition +hints, not required output or tool instructions. Transcription uses one audio +model call and no semantic rewriting pass. `gpt-transcribe` receives plural +`languages`/`keywords`; the GPT-4o audio models receive the compatible prompt +and singular language parameter. + +## Playback repair + +Starting dictation stops current narration. Browser speech continues to use +the existing presenter/spoken channel and playback telemetry. The long-utterance +pause/resume workaround is restricted to desktop Chromium; it must not run on +Android or iOS/iPadOS. Mobile playback is invoked synchronously with the user +gesture, and playback errors offer a visible retry. Automatic narration can +still be blocked by device autoplay policy; tap **Speak** in that case. + +## Evidence boundary + +Baseline `59cbeb2f` had browser-only dictation, no vocabulary context, +append-only final handling and console-only input errors. The DGX conversation **Von Task Smoke Test**, session +`9b49b86d-26f1-49ac-9027-58b8864ca7e5`, supplies exact source evidence: +user entries 2 and 4 contain repeated progressively longer prefixes; entry 8 +reports correct desktop behaviour; entry 10 begins by replaying entry 8 before +new dictation. The native issue is +`#V#task_agent_8fbedbe81f4951431e8ff417abf402f8`. +The accumulation bug reproduces this class of output, but raw audio and browser +recognition events were not retained, so independent acoustic hallucination or +physical-device behaviour cannot be reconstructed from transcript text alone. + +Targeted tests cover MP4/WebM forwarding, contextual language/vocabulary, +authenticated session binding, model denial, invalid uploads and safe provider +errors; browser lifecycle tests cover retained-audio retry, permission/cancel +races, conversation fences, revised recognition snapshots and preservation of +edits. Speech tests cover mobile gesture timing and exclusion of pause/resume. + +On 10 September 2026, Chrome at desktop and 390 × 844 viewports exercised the +actual `chat_tab.html` composer and `dictation.js` controller at +`http://127.0.0.1:5018/` using a temporary simulated microphone/provider fixture. +Failure → edit draft → retry produced one transcript while preserving the edit; +cancel preserved the draft; the mobile composer had no horizontal overflow. +This is layout/lifecycle evidence, not acoustic, physical-device, provider-quality +or live DGX acceptance. The DGX transcript was read through its own canonical +history service after temporarily reconnecting this Mac's existing Tailscale +profile. Its observed pre-repair runtime was `12ae94c0`. Runtime deployment and +physical iOS/Android behaviour require separate verification receipts. + +Provider reference: [OpenAI file transcription](https://developers.openai.com/api/docs/guides/speech-to-text). +Browser reference: [MDN SpeechRecognition](https://developer.mozilla.org/en-US/docs/Web/API/SpeechRecognition). diff --git a/src/backend/integrations/internal_mcp/orchestrator.py b/src/backend/integrations/internal_mcp/orchestrator.py index 432ce4c20..78c720599 100644 --- a/src/backend/integrations/internal_mcp/orchestrator.py +++ b/src/backend/integrations/internal_mcp/orchestrator.py @@ -14310,7 +14310,9 @@ def _stage_model_candidates( str(entry.get("model") or "").strip() or None ) host = str(entry.get("host") or "").strip() or None - if not provider or not model: + from ...languagemodels.model_defaults import is_transcription_model + + if not provider or not model or is_transcription_model(provider, model): continue model_parameters = ( normalise_model_parameters_for_storage( diff --git a/src/backend/languagemodels/model_defaults.py b/src/backend/languagemodels/model_defaults.py index 252077cea..0ac7e3470 100644 --- a/src/backend/languagemodels/model_defaults.py +++ b/src/backend/languagemodels/model_defaults.py @@ -8,6 +8,16 @@ import os +# Audio/transcriptions models do not implement the chat/tool interface. They +# may be authorised in the shared model pool without becoming chat fallbacks. +TRANSCRIPTION_MODELS = ("gpt-transcribe", "gpt-4o-transcribe", "gpt-4o-mini-transcribe") + + +def is_transcription_model(provider: object, model: object) -> bool: + return str(provider or "").strip().lower() == "openai" and str( + model or "" + ).strip().removeprefix("openai:") in TRANSCRIPTION_MODELS + # --------------------------------------------------------------------------- # Ollama # --------------------------------------------------------------------------- diff --git a/src/backend/server/routes/speech_routes.py b/src/backend/server/routes/speech_routes.py index b1e116655..c1dbff429 100644 --- a/src/backend/server/routes/speech_routes.py +++ b/src/backend/server/routes/speech_routes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from datetime import datetime, timezone from typing import Any, Dict, Optional @@ -10,10 +11,98 @@ update_llm_debug_data_for_request_id, ) - speech_bp = Blueprint("speech_bp", __name__, url_prefix="/api/speech") +def _speech_actor(): + from ...security.access_control import ( + get_effective_organisation_concept_id, + get_effective_user_concept_id_with_source, + ) + + actor, source = get_effective_user_concept_id_with_source() + if not actor or source not in { + "authenticated_session", + "authenticated_session_derived", + "trusted_in_process_actor", + }: + return None, None + return actor, get_effective_organisation_concept_id() + + +@speech_bp.route("/capabilities", methods=["GET"]) +def speech_capabilities(): + from ...services.speech_transcription_service import ( + SpeechUnavailable, + transcription_capability, + ) + + actor, organisation = _speech_actor() + if not actor: + return jsonify(error="authentication_required"), 401 + try: + data = transcription_capability(actor, organisation) + except SpeechUnavailable as exc: + data = {"available": False, "reason": str(exc)} + except Exception: + data = { + "available": False, + "reason": "Speech configuration is temporarily unavailable.", + } + response = jsonify(transcription=data) + response.headers["Cache-Control"] = "private, no-store" + return response + + +@speech_bp.route("/transcribe", methods=["POST"]) +def transcribe_speech(): + from ...languagemodels.llm_interface import ModelExecutionEligibilityError + from ...services.speech_transcription_service import ( + MAX_AUDIO_BYTES, + SpeechUnavailable, + transcribe_audio, + ) + + actor, organisation = _speech_actor() + if not actor: + return jsonify(error="authentication_required"), 401 + # Bound multipart parsing as well as the file read; audio is never retained. + request.max_content_length = MAX_AUDIO_BYTES + 65536 + uploaded = request.files.get("audio") + if uploaded is None: + return jsonify(error="missing_audio", message="No recording was received."), 400 + try: + vocabulary = json.loads(request.form.get("vocabulary", "[]")) + result = transcribe_audio( + actor=actor, + organisation=organisation, + audio=uploaded.read(MAX_AUDIO_BYTES + 1), + mime_type=uploaded.mimetype or "", + context=request.form.get("context", ""), + vocabulary=vocabulary, + language=request.form.get("language", ""), + ) + except (ValueError, json.JSONDecodeError) as exc: + return jsonify(error="invalid_audio_request", message=str(exc)), 400 + except (SpeechUnavailable, ModelExecutionEligibilityError) as exc: + return jsonify(error="speech_unavailable", message=str(exc)), 503 + except Exception as exc: + # Provider exceptions can contain credentials, URLs and submitted text. + current_app.logger.warning( + "Speech transcription failed (%s)", type(exc).__name__ + ) + return ( + jsonify( + error="transcription_failed", + message="Transcription failed. Your recording is available to retry.", + ), + 502, + ) + response = jsonify(success=True, **result) + response.headers["Cache-Control"] = "private, no-store" + return response + + def _coerce_float(value: Any) -> Optional[float]: if isinstance(value, bool): return None @@ -70,8 +159,10 @@ def _normalise_timestamp(value: Any) -> Optional[str]: return value if isinstance(value, (int, float)): try: - return datetime.fromtimestamp(float(value) / 1000.0, tz=timezone.utc).isoformat().replace( - "+00:00", "Z" + return ( + datetime.fromtimestamp(float(value) / 1000.0, tz=timezone.utc) + .isoformat() + .replace("+00:00", "Z") ) except Exception: return None @@ -87,7 +178,9 @@ def _sanitise_playback(payload: Dict[str, Any]) -> Dict[str, Any]: settings = settings if isinstance(settings, dict) else {} return { - "request_id": _coerce_text(payload.get("request_id") or playback.get("request_id")), + "request_id": _coerce_text( + payload.get("request_id") or playback.get("request_id") + ), "turn_id": _coerce_text(payload.get("turn_id")), "conversation_id": _coerce_text(payload.get("conversation_id")), "started_at": _normalise_timestamp(playback.get("started_at")), diff --git a/src/backend/services/model_registry_service.py b/src/backend/services/model_registry_service.py index f4cd429e1..9bf9f6331 100644 --- a/src/backend/services/model_registry_service.py +++ b/src/backend/services/model_registry_service.py @@ -2,18 +2,19 @@ from __future__ import annotations -from datetime import datetime, timezone import hashlib import json import logging import os -from pathlib import Path import tempfile import threading import time +from datetime import datetime, timezone +from pathlib import Path from typing import Any, Mapping, Optional, Sequence from urllib.parse import parse_qsl, unquote, urlsplit +from ..languagemodels.model_defaults import is_transcription_model from .settings_service import resolve_enabled_llm_settings, resolve_llm_setting logger = logging.getLogger(__name__) @@ -1280,7 +1281,7 @@ def _build_registry_from_settings() -> Mapping[str, Any]: user_concept_id = None org_concept_id = None try: - from flask import session, has_request_context + from flask import has_request_context, session if has_request_context(): user_concept_id = session.get("user_concept_id") @@ -1298,6 +1299,8 @@ def _build_registry_from_settings() -> Mapping[str, Any]: continue provider = entry.get("provider") model = entry.get("model") + if is_transcription_model(provider, model): + continue locality = "external" if isinstance(provider, str) and provider.lower() == "ollama": locality = "local" diff --git a/src/backend/services/speech_transcription_service.py b/src/backend/services/speech_transcription_service.py new file mode 100644 index 000000000..c6aa29246 --- /dev/null +++ b/src/backend/services/speech_transcription_service.py @@ -0,0 +1,137 @@ +"""Bounded audio transport. Context is supplied from the visible composer; +it is transcription evidence, never an instruction or authority to use tools. +No recording or context is persisted by this service. +""" + +from __future__ import annotations + +import os +import re +import time + +from ..languagemodels.llm_interface import assert_model_execution_allowed +from ..languagemodels.model_defaults import TRANSCRIPTION_MODELS +from .settings_service import get_openai_env_var, resolve_enabled_llm_settings + +MAX_AUDIO_BYTES = 24_000_000 +AUDIO_TYPES = { + "audio/webm": "webm", + "audio/mp4": "mp4", + "audio/x-m4a": "m4a", + "audio/mpeg": "mp3", + "audio/wav": "wav", + "audio/x-wav": "wav", +} + + +class SpeechUnavailable(Exception): + pass + + +def transcription_model(actor: str, organisation: str | None) -> str: + enabled = resolve_enabled_llm_settings( + user_concept_id=actor, org_concept_id=organisation + ) + models = { + str(e.get("model", "")).removeprefix("openai:") + for e in enabled + if isinstance(e, dict) and e.get("provider") == "openai" + } + preferred = os.environ.get("VON_TRANSCRIPTION_MODEL", "").strip() + candidates = (preferred,) if preferred else TRANSCRIPTION_MODELS + for model in candidates: + if model in TRANSCRIPTION_MODELS and model in models: + return model + raise SpeechUnavailable( + "Enable an OpenAI transcription model in Settings: " + + ", ".join(TRANSCRIPTION_MODELS) + + "." + ) + + +def transcription_capability(actor: str, organisation: str | None) -> dict: + model = transcription_model(actor, organisation) + if not os.environ.get(get_openai_env_var() or "OPENAI_API_KEY"): + raise SpeechUnavailable("The server's configured OpenAI key is unavailable.") + return { + "available": True, + "provider": "openai", + "model": model, + "max_audio_bytes": MAX_AUDIO_BYTES, + "mime_types": list(AUDIO_TYPES), + } + + +def normalise_context(context, vocabulary) -> tuple[str, list[str]]: + context = context[-6000:] if isinstance(context, str) else "" + terms = [] + for value in vocabulary[:80] if isinstance(vocabulary, list) else []: + if not isinstance(value, str): + continue + term = re.sub(r"[<>\r\n]", " ", value).strip()[:100] + if term and term.casefold() not in {t.casefold() for t in terms}: + terms.append(term) + return context, terms + + +def transcribe_audio( + *, actor, organisation, audio, mime_type, context="", vocabulary=None, language="" +): + import openai + + mime_type = mime_type.split(";", 1)[0].lower().strip() + if mime_type not in AUDIO_TYPES: + raise ValueError("Unsupported recording format. Use MP4, WebM, MP3 or WAV.") + if not audio or len(audio) > MAX_AUDIO_BYTES: + raise ValueError("Recording is empty or exceeds the 24 MB upload limit.") + capability = transcription_capability(actor, organisation) + model = capability["model"] + assert_model_execution_allowed( + provider="openai", + model=model, + user_concept_id=actor, + org_concept_id=organisation, + allow_ambient_actor_scope=False, + ) + context, terms = normalise_context(context, vocabulary) + # Only visible context and literal spellings; no corrective LLM pass that + # could silently rewrite the user's intended message. + prompt = "\n".join(filter(None, [", ".join(terms), context]))[:8000] + options = { + "model": model, + "file": ("dictation." + AUDIO_TYPES[mime_type], audio, mime_type), + "response_format": "json", + "prompt": prompt, + } + language = language.strip() if isinstance(language, str) else "" + if language and not re.fullmatch(r"[a-zA-Z]{2,3}(?:-[a-zA-Z0-9]{2,8})*", language): + raise ValueError( + "Use a language tag such as en-NZ, mi or zh-CN, or leave it empty." + ) + if model == "gpt-transcribe": + options["extra_body"] = { + "keywords": terms, + **({"languages": [language]} if language else {}), + } + elif language: + options["language"] = language.split("-", 1)[0].lower() + start = time.monotonic() + # Explicit transport bound protects occupied HTTP workers; no client timer + # discards a usable result. Users can cancel or retry the retained recording. + with openai.OpenAI( + api_key=os.environ[get_openai_env_var() or "OPENAI_API_KEY"], + timeout=120, + max_retries=0, + ) as client: + result = client.audio.transcriptions.create(**options) + text = result.text.strip() + return { + "text": text, + "provider": "openai", + "model": model, + "language_hint": language or None, + "context_chars": len(context), + "vocabulary_count": len(terms), + "audio_bytes": len(audio), + "elapsed_ms": round((time.monotonic() - start) * 1000), + } diff --git a/src/backend/workflows/durable/model_selection_workflow.py b/src/backend/workflows/durable/model_selection_workflow.py index e38d3c3e6..4e19c649e 100644 --- a/src/backend/workflows/durable/model_selection_workflow.py +++ b/src/backend/workflows/durable/model_selection_workflow.py @@ -129,7 +129,9 @@ def _enabled_model_pool( provider = _context_string(entry.get("provider")) model = _context_string(entry.get("model")) host = _context_string(entry.get("host")) - if not provider or not model: + from ...languagemodels.model_defaults import is_transcription_model + + if not provider or not model or is_transcription_model(provider, model): continue key = (provider.lower(), model, host) if key in seen: diff --git a/src/frontend/web/von_interface/static/js/chatTab.js b/src/frontend/web/von_interface/static/js/chatTab.js index 194dc8073..9fd4b108c 100644 --- a/src/frontend/web/von_interface/static/js/chatTab.js +++ b/src/frontend/web/von_interface/static/js/chatTab.js @@ -1,3 +1,4 @@ +import { createDictationController } from './dictation.js'; import { renderImageAttachments, renderImageComposer, uploadConversationImage, imagesBlocked, takeImages, restoreImages, descriptorsForIds } from "./utils/conversationImages.js"; // Chat Tab Module import { annotateTurn, fetchWithTimeout, getJsonDetailed, getUserContext, getWindowSessionId, postJson, WINDOW_SESSION_HEADER } from './apiService.js'; @@ -26,10 +27,8 @@ import { isAnnotationEnabled } from './featureFlags.js'; import { detectMarkdown, renderMarkdownViaServer } from './markdownUtils.js'; import { getSpeechSynthesisVoices, - isSpeechRecognitionSupported, isTextToSpeechSupported, speakText, - startSpeechRecognition, stopSpeaking } from './speech.js'; import { getPreferredLanguage, selectBestNameForContext, selectShortestNameForContext } from './utils/nameSelection.js'; @@ -24604,8 +24603,8 @@ const CHAT_STT_LANGUAGE_STORAGE_KEY = 'chatSttLanguage'; const CHAT_STT_CONTINUOUS_STORAGE_KEY = 'chatSttContinuous'; const CHAT_STT_INTERIM_RESULTS_STORAGE_KEY = 'chatSttInterimResults'; -let activeDictation = null; -let dictationState = null; +let dictationController = null; +let dictationSendPending = false; let activeTtsTurnId = null; let activeTtsButton = null; @@ -25000,6 +24999,7 @@ function toggleSpeakTurn(turnId, text, button) { }); } catch (err) { console.warn('[chatTab] TTS failed:', err); + showToast('Speech playback failed. Tap Speak to retry.', 'error'); clearActiveTtsUi(); return; } @@ -25026,22 +25026,36 @@ function toggleSpeakTurn(turnId, text, button) { } }; utterance.onend = () => finish('ended'); - utterance.onerror = () => finish('error'); + utterance.onerror = (event) => { + if (activeTtsTurnId === turnId && !['canceled', 'interrupted'].includes(event?.error)) { + showToast('Speech playback failed. Tap Speak to retry; your response remains in the conversation.', 'error'); + } + finish('error'); + }; } catch (_) { // Ignore. } } function stopDictation() { - if (!activeDictation) { - return; - } - - try { - activeDictation.stop?.(); - } catch (_) { - // Ignore. - } + dictationController?.cancel(); +} + +function getDictationContext() { + const prompt = getPromptInputElement()?.value || ''; + const history = getConversationTranscriptTurnsSnapshot().slice(-4) + .map(turn => turn.message || turn.text || '').join('\n').slice(-4500); + const vocabulary = ['Von', 'Vontology']; + document.querySelectorAll('#scrollableField [data-full-concept-id], .prompt-input-overlay [data-full-concept-id]') + .forEach(element => { + const name = element.querySelector('.vontology-cartouche-name')?.textContent || element.textContent; + if (name?.trim()) vocabulary.push(name.trim()); + }); + return { + key: `${getCurrentUserConceptId() || ''}:${activeChatSessionId || ''}:${getSessionScopedNamespace() || ''}`, + text: `${history}\n${prompt}`.slice(-6000), vocabulary: [...new Set(vocabulary)].slice(0, 80), + ...getChatSpeechSettings().stt + }; } // Persist sender/message pairs for transcript exports @@ -28694,6 +28708,7 @@ async function renameChatSession(sessionId, sessionName) { } async function switchToChatSession(sessionId, options = {}) { + stopDictation(); const sid = String(sessionId || '').trim(); if (!sid) { return { ok: false, error: 'session_id required' }; @@ -33544,6 +33559,7 @@ function detachChatWorkForOrganisationSwitch() { } function startOrganisationSwitchForChatTab(detail = {}) { + stopDictation(); const requestedSwitchId = normaliseOrganisationSwitchId(detail); if (requestedSwitchId && pendingChatOrganisationSwitchId === requestedSwitchId) { return chatOrganisationGeneration; @@ -34243,89 +34259,20 @@ export function initializeChatTab() { }); } - // Dictation (STT): optional browser capability. - if (dictateButton) { - if (!isSpeechRecognitionSupported()) { - dictateButton.disabled = true; - dictateButton.title = 'Dictation is not supported in this browser.'; - } else { - dictateButton.addEventListener('click', () => { - if (activeDictation) { - stopDictation(); - return; - } - - if (!promptInput) { - return; - } - - const baseText = String(promptInput.value || ''); - dictationState = { - baseText, - finalText: '', - interimText: '' - }; - - dictateButton.textContent = 'Stop dictation'; - dictateButton.classList.add('active-dictation'); - - try { - const settings = getChatSpeechSettings(); - activeDictation = startSpeechRecognition({ - language: settings.stt.language, - continuous: settings.stt.continuous, - interimResults: settings.stt.interimResults, - onResult: ({ finalText, interimText }) => { - if (!dictationState) { - return; - } - - if (finalText) { - dictationState.finalText = [dictationState.finalText, finalText] - .map(t => String(t || '').trim()) - .filter(Boolean) - .join(' '); - } - - dictationState.interimText = String(interimText || '').trim(); - - const baseText = String(dictationState.baseText || ''); - const dictatedText = [dictationState.finalText, dictationState.interimText] - .map(t => String(t || '').trim()) - .filter(Boolean) - // Normalise spaces/tabs, but preserve newlines. - .join(' ') - .replace(/[ \t]+/g, ' '); - - if (!dictatedText) { - setPromptComposerValue(baseText, { promptInput }); - } else { - const needsSpacer = baseText.length > 0 && !/[ \t\n]$/.test(baseText); - setPromptComposerValue( - `${baseText}${needsSpacer ? ' ' : ''}${dictatedText}`, - { promptInput } - ); - } - }, - onError: (event) => { - console.warn('[chatTab] Dictation error:', event); - }, - onEnd: () => { - activeDictation = null; - dictationState = null; - dictateButton.textContent = 'Dictate'; - dictateButton.classList.remove('active-dictation'); - } - }); - } catch (err) { - console.warn('[chatTab] Unable to start dictation:', err); - activeDictation = null; - dictationState = null; - dictateButton.textContent = 'Dictate'; - dictateButton.classList.remove('active-dictation'); - } - }); - } + // Shared mobile/desktop recording lifecycle; browser recognition is explicit. + dictationController?.dispose(); + if (dictateButton && promptInput) { + dictationController = createDictationController({ + input: promptInput, button: dictateButton, + status: document.getElementById('dictationStatus'), + cancelButton: document.getElementById('cancelDictationButton'), + retryButton: document.getElementById('retryDictationButton'), + engineSelect: document.getElementById('dictationEngineSelect'), + getContext: getDictationContext, + fetchImpl: (url, options = {}) => fetch(url, { ...options, headers: buildChatFetchHeaders(options.headers) }), + setValue: value => setPromptComposerValue(value, { promptInput }), + onStart: () => { stopSpeaking(); clearActiveTtsUi(); } + }); } // Add event listeners @@ -37213,6 +37160,14 @@ async function handleSendPrompt(options = {}) { && !hasPromptOverride && !selectedQueueEntry ); + if (directComposerSubmission && dictationController?.getState() !== 'idle' && dictationController) { + if (dictationSendPending) return; + const scopeBefore = getDictationContext().key; + dictationSendPending = true; + try { + if (!await dictationController.finish() || scopeBefore !== getDictationContext().key) return; + } finally { dictationSendPending = false; } + } if (directComposerSubmission && newChatCreationInFlight) { const createdSession = await newChatCreationInFlight; if (!createdSession || pendingChatOrganisationSwitchId !== null) { @@ -38124,6 +38079,7 @@ async function handleSendPrompt(options = {}) { } async function handleResetContext() { + stopDictation(); try { const response = await fetch('/von/reset', { method: 'POST', diff --git a/src/frontend/web/von_interface/static/js/dictation.js b/src/frontend/web/von_interface/static/js/dictation.js new file mode 100644 index 000000000..4f7c9d473 --- /dev/null +++ b/src/frontend/web/von_interface/static/js/dictation.js @@ -0,0 +1,227 @@ +import { isSpeechRecognitionSupported, startSpeechRecognition, stopSpeaking } from './speech.js'; + +const RECORDING_TYPES = ['audio/webm;codecs=opus', 'audio/mp4', 'audio/webm']; + +export function recordingMimeType(root = globalThis) { + return RECORDING_TYPES.find(type => root.MediaRecorder?.isTypeSupported?.(type)) || ''; +} + +export function supportsAudioRecording(root = globalThis) { + return root.isSecureContext !== false && typeof root.MediaRecorder === 'function' + && typeof root.navigator?.mediaDevices?.getUserMedia === 'function'; +} + +export function insertDictation(value, text, selection = null) { + if (!text) return value; + const start = selection?.start ?? value.length; + const end = selection?.end ?? start; + const before = value.slice(0, start); + const after = value.slice(end); + return `${before}${before && !/\s$/.test(before) ? ' ' : ''}${text}${after && !/^\s|^[,.;:!?]/.test(after) ? ' ' : ''}${after}`; +} + +function microphoneError(error) { + if (['NotAllowedError', 'not-allowed', 'service-not-allowed'].includes(error?.name || error?.error)) { + return 'Microphone access was denied. Allow microphone access in your browser and try again.'; + } + if (error?.name === 'NotFoundError') return 'No microphone is available.'; + if (error?.error === 'no-speech') return 'No speech was detected. Try again.'; + return error?.message || 'Speech recognition stopped. Try recorded audio or type your message.'; +} + +// One controller per composer. A capture belongs to the conversation and actor +// that started it. Cancelling fences permission, recognition and fetch callbacks. +export function createDictationController({ input, button, status, cancelButton, retryButton, + engineSelect, getContext, setValue, onStart = stopSpeaking, fetchImpl = (...args) => fetch(...args), + root = globalThis }) { + let current = null; + let capability = null; + let state = 'idle'; + let disposed = false; + + function render(next, message = '') { + state = next; + const busy = ['requesting', 'recording', 'transcribing'].includes(next); + button.textContent = next === 'recording' ? 'Finish dictation' : next === 'transcribing' ? 'Transcribing…' : next === 'requesting' ? 'Opening microphone…' : 'Dictate'; + button.disabled = next === 'transcribing' || next === 'requesting'; + button.setAttribute('aria-pressed', String(next === 'recording')); + button.classList.toggle('active-dictation', busy); + status.textContent = message; + cancelButton.hidden = !current; + retryButton.hidden = !(next === 'error' && current?.blob); + engineSelect.disabled = !!current; + } + + function valid(capture) { + if (!disposed && capture === current && capture.key !== getContext().key) cancel(); + return !disposed && capture === current; + } + + function release(capture) { + capture.stream?.getTracks().forEach(track => track.stop()); + } + + function cancel() { + const capture = current; + current = null; + if (capture) { + capture.abort?.abort(); + capture.recognition?.abort(); + if (capture.recorder?.state === 'recording') capture.recorder.stop(); + release(capture); + capture.resolve?.(false); + } + render('idle'); + } + + function fail(capture, message) { + if (!valid(capture)) return; + release(capture); + render('error', message); + capture.resolve?.(false); + } + + function commit(capture, text) { + if (!valid(capture)) return; + const value = String(input.value || ''); + const selection = value === capture.base ? capture.selection : null; + setValue(insertDictation(value, text.trim(), selection)); + current = null; + render('idle', text.trim() ? 'Dictation added. Edit it or send when ready.' : 'No speech was detected. Try again.'); + capture.resolve?.(!!text.trim()); + } + + async function transcribe(capture) { + if (!valid(capture)) return; + render('transcribing', 'Transcribing with conversation vocabulary…'); + capture.abort = new AbortController(); + const form = new FormData(); + const mime = capture.blob.type.split(';')[0]; + form.append('audio', capture.blob, mime === 'audio/mp4' ? 'dictation.mp4' : 'dictation.webm'); + form.append('language', capture.context.language || ''); + form.append('context', capture.context.text || ''); + form.append('vocabulary', JSON.stringify(capture.context.vocabulary || [])); + try { + const response = await fetchImpl('/api/speech/transcribe', { + method: 'POST', body: form, credentials: 'same-origin', signal: capture.abort.signal + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.message || 'Transcription failed. Retry your recording.'); + if (typeof data.text !== 'string') throw new Error('No transcript was returned. Retry your recording.'); + commit(capture, data.text); + } catch (error) { + if (valid(capture)) fail(capture, error.message || 'Connection lost. Retry your recording.'); + } + } + + async function start() { + if (current) cancel(); + const context = getContext(); + const capture = { key: context.key, context, base: input.value, + selection: { start: input.selectionStart, end: input.selectionEnd }, chunks: [], bytes: 0 }; + capture.completion = new Promise(resolve => { capture.resolve = resolve; }); + current = capture; + onStart(); + render('requesting', 'Allow microphone access to dictate.'); + try { + if (engineSelect.value === 'browser') { + if (!isSpeechRecognitionSupported()) throw new Error('Browser recognition is unavailable. Choose recorded audio.'); + capture.recognition = startSpeechRecognition({ + language: context.language, continuous: context.continuous ?? false, interimResults: context.interimResults ?? true, + vocabulary: context.vocabulary, + onResult: ({ finalText, interimText }) => { + if (!valid(capture)) return; + capture.finalText = finalText; + status.textContent = [finalText, interimText].filter(Boolean).join(' ') || 'Listening…'; + }, + onError: error => { capture.failed = true; fail(capture, microphoneError(error)); }, + onEnd: () => { if (!capture.failed) commit(capture, capture.finalText || ''); } + }); + } else { + if (!supportsAudioRecording(root)) throw new Error('Recording requires HTTPS and a browser with microphone recording support.'); + if (!capability?.available) throw new Error(capability?.reason || 'Recorded transcription is unavailable. Try browser recognition or check Speech settings.'); + capture.stream = await root.navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true }, video: false }); + if (!valid(capture)) { release(capture); return; } + const mimeType = recordingMimeType(root); + capture.recorder = new root.MediaRecorder(capture.stream, mimeType ? { mimeType } : undefined); + capture.recorder.ondataavailable = event => { + if (!valid(capture) || !event.data?.size) return; + capture.chunks.push(event.data); + capture.bytes += event.data.size; + if (capture.bytes >= capability.max_audio_bytes * 0.9 && capture.recorder.state === 'recording') { + status.textContent = 'Finishing this recording before the upload size limit.'; + capture.recorder.stop(); + } + }; + capture.recorder.onerror = error => { + capture.failed = true; + fail(capture, microphoneError(error)); + }; + capture.recorder.onstop = () => { + release(capture); + if (!valid(capture) || capture.failed) return; + capture.blob = new Blob(capture.chunks, { type: capture.recorder.mimeType || mimeType }); + capture.chunks = []; + if (!capture.blob.size) { fail(capture, 'No audio was recorded. Try again.'); return; } + void transcribe(capture); + }; + capture.stream.getTracks().forEach(track => track.addEventListener?.('ended', () => { + if (valid(capture) && capture.recorder.state === 'recording') capture.recorder.stop(); + })); + capture.recorder.start(1000); + } + if (valid(capture) && state !== 'error') render('recording', 'Listening. Select Finish dictation when you are ready.'); + } catch (error) { + fail(capture, microphoneError(error)); + } + } + + async function finish() { + if (!current) return true; + if (state === 'error') return false; + const capture = current; + if (capture.recorder?.state === 'recording') capture.recorder.stop(); + capture.recognition?.stop(); + if (state === 'requesting') { cancel(); return false; } + return capture.completion; + } + + async function refreshCapabilities() { + try { + const response = await fetchImpl('/api/speech/capabilities', { credentials: 'same-origin', cache: 'no-store' }); + const data = await response.json(); + capability = response.ok ? data.transcription : { available: false, reason: 'Sign in to use recorded transcription.' }; + } catch (_) { + capability = { available: false, reason: 'Unable to check recorded transcription. Try again when connected.' }; + } + if (!disposed && !current) { + render('idle', capability?.available + ? 'Recorded audio and visible conversation context are sent to OpenAI for transcription. Audio is not saved by Von.' + : capability?.reason || 'Recorded transcription is unavailable.'); + } + } + + const click = () => { if (state === 'recording') void finish(); else void start(); }; + const retry = () => { if (current?.blob) { + current.completion = new Promise(resolve => { current.resolve = resolve; }); + void transcribe(current); + } }; + const visibility = () => { if (root.document?.hidden && current) void finish(); }; + button.addEventListener('click', click); + cancelButton.addEventListener('click', cancel); + retryButton.addEventListener('click', retry); + root.addEventListener?.('pagehide', cancel); + root.addEventListener?.('focus', refreshCapabilities); + root.addEventListener?.('von-preferences-changed', refreshCapabilities); + root.document?.addEventListener('visibilitychange', visibility); + render('idle'); + void refreshCapabilities(); + return { cancel, finish, start, refreshCapabilities, getState: () => state, + dispose: () => { cancel(); disposed = true; + button.removeEventListener('click', click); cancelButton.removeEventListener('click', cancel); + retryButton.removeEventListener('click', retry); root.removeEventListener?.('pagehide', cancel); + root.removeEventListener?.('focus', refreshCapabilities); + root.removeEventListener?.('von-preferences-changed', refreshCapabilities); + root.document?.removeEventListener('visibilitychange', visibility); + } }; +} diff --git a/src/frontend/web/von_interface/static/js/settingsPage.js b/src/frontend/web/von_interface/static/js/settingsPage.js index 81e009404..9d48ffe0b 100644 --- a/src/frontend/web/von_interface/static/js/settingsPage.js +++ b/src/frontend/web/von_interface/static/js/settingsPage.js @@ -1,3 +1,4 @@ +import { supportsAudioRecording } from './dictation.js'; import { ensureUniqueWindowSessionId, getWindowSessionId, postJson, WINDOW_SESSION_HEADER } from './apiService.js'; import { clearBackgroundTaskHistory, @@ -3932,15 +3933,7 @@ function setupSpeechSettingsSection() { const sttSupported = isSpeechRecognitionSupported(); if (supportNote) { - if (ttsSupported && sttSupported) { - supportNote.textContent = 'Text-to-speech and dictation are available in this browser.'; - } else if (ttsSupported) { - supportNote.textContent = 'Dictation is not supported in this browser.'; - } else if (sttSupported) { - supportNote.textContent = 'Text-to-speech is not supported in this browser.'; - } else { - supportNote.textContent = 'Speech is not supported in this browser.'; - } + supportNote.textContent = `${supportsAudioRecording() ? 'Recorded dictation is supported; enable an OpenAI transcription model (gpt-transcribe, gpt-4o-transcribe or gpt-4o-mini-transcribe) in your model pool.' : 'Recorded dictation needs HTTPS and microphone recording support.'} Browser recognition: ${sttSupported ? 'available with limited context support' : 'unavailable'}. Spoken playback: ${ttsSupported ? 'available' : 'unavailable'}.`; } if (!ttsSupported) { diff --git a/src/frontend/web/von_interface/static/js/speech.js b/src/frontend/web/von_interface/static/js/speech.js index f97b4a9c4..a5ff84b35 100644 --- a/src/frontend/web/von_interface/static/js/speech.js +++ b/src/frontend/web/von_interface/static/js/speech.js @@ -43,6 +43,11 @@ function clearKeepaliveInterval(root) { function startKeepaliveInterval(root) { clearKeepaliveInterval(root); + // Mobile pause/resume can terminate or restart speech. Restrict the + // keepalive workaround to the affected desktop Chromium implementation. + const ua = root?.navigator?.userAgent || ''; + if (!/Chrome\//.test(ua) || /Android|Mobile|iPhone|iPad|iPod/.test(ua) + || (root?.navigator?.maxTouchPoints > 1 && /Macintosh/.test(ua))) return; if (!root || typeof root.setInterval !== 'function') { return; @@ -79,6 +84,7 @@ export function startSpeechRecognition(options = {}) { language, continuous = true, interimResults = true, + vocabulary = [], onResult, onError, onEnd @@ -92,6 +98,12 @@ export function startSpeechRecognition(options = {}) { const recognition = new SpeechRecognitionCtor(); recognition.continuous = !!continuous; recognition.interimResults = !!interimResults; + const Phrase = globalThis.SpeechRecognitionPhrase; + if ('phrases' in recognition && typeof Phrase === 'function') { + try { + recognition.phrases = vocabulary.slice(0, 80).map(term => new Phrase(String(term), 3)); + } catch (_) { /* Optional contextual biasing must not break capture. */ } + } if (language) { recognition.lang = String(language); @@ -103,10 +115,11 @@ export function startSpeechRecognition(options = {}) { } try { + // Each event is a revisable snapshot, including replayed finals. let finalText = ''; let interimText = ''; - for (let i = event.resultIndex; i < event.results.length; i += 1) { + for (let i = 0; i < event.results.length; i += 1) { const result = event.results[i]; if (!result || !result[0]) { continue; @@ -114,9 +127,9 @@ export function startSpeechRecognition(options = {}) { const transcript = String(result[0].transcript || ''); if (result.isFinal) { - finalText += transcript; + finalText += ` ${transcript}`; } else { - interimText += transcript; + interimText += ` ${transcript}`; } } @@ -125,7 +138,7 @@ export function startSpeechRecognition(options = {}) { interimText: interimText.trim(), event }); - } catch (err) { + } catch (_err) { // Best-effort: swallow and keep recognition alive. } }; @@ -251,8 +264,13 @@ export function speakText(text, options = {}) { // Attach event listeners to clear keepalive when speech ends or errors. const cleanupKeepalive = () => clearKeepaliveInterval(root); - utterance.onend = cleanupKeepalive; - utterance.onerror = cleanupKeepalive; + if (typeof utterance.addEventListener === 'function') { + utterance.addEventListener('end', cleanupKeepalive); + utterance.addEventListener('error', cleanupKeepalive); + } else { + utterance.onend = cleanupKeepalive; + utterance.onerror = cleanupKeepalive; + } // Prefer immediate playback (and avoid queueing multiple long utterances). try { @@ -266,7 +284,9 @@ export function speakText(text, options = {}) { const settleMs = typeof options.cancelSettleMs === 'number' ? options.cancelSettleMs - : DEFAULT_CANCEL_SETTLE_MS; + : (/Android|Mobile|iPhone|iPad|iPod/.test(root.navigator?.userAgent || '') + || (root.navigator?.maxTouchPoints > 1 && /Macintosh/.test(root.navigator?.userAgent || ''))) + ? 0 : DEFAULT_CANCEL_SETTLE_MS; const doSpeak = () => { pendingSpeakTimerId = null; @@ -274,8 +294,8 @@ export function speakText(text, options = {}) { root.speechSynthesis.speak(utterance); // Start the Chrome keepalive workaround after speech begins. startKeepaliveInterval(root); - } catch (_) { - // Ignore. + } catch (error) { + utterance.onerror?.({ error: 'synthesis-failed', message: error?.message }); } }; diff --git a/src/frontend/web/von_interface/static/styles.css b/src/frontend/web/von_interface/static/styles.css index d28ee6ac5..845c090a3 100644 --- a/src/frontend/web/von_interface/static/styles.css +++ b/src/frontend/web/von_interface/static/styles.css @@ -11483,10 +11483,31 @@ body[data-active-tab="settingsTab"] .tab-content-area { .chat-composer-actions, .chat-composer-primary-actions { display: flex; + flex-wrap: wrap; align-items: center; gap: 8px; } +#cancelDictationButton[hidden], +#retryDictationButton[hidden] { + display: none; +} + +#dictateButton, +#cancelDictationButton, +#retryDictationButton { + min-height: 44px; +} + +#dictationStatus { + margin: 8px 0 0; + overflow-wrap: anywhere; +} + +#dictationEngineSelect { + max-width: 100%; +} + .chat-composer-actions { justify-content: space-between; } diff --git a/src/frontend/web/von_interface/templates/chat_tab.html b/src/frontend/web/von_interface/templates/chat_tab.html index f22eb94cf..ecde9d433 100644 --- a/src/frontend/web/von_interface/templates/chat_tab.html +++ b/src/frontend/web/von_interface/templates/chat_tab.html @@ -246,11 +246,18 @@

Dictate + +
More actions
+ + @@ -281,6 +288,7 @@

diff --git a/src/frontend/web/von_interface/templates/settings_tab.html b/src/frontend/web/von_interface/templates/settings_tab.html index bf7f368e6..3397dd80a 100644 --- a/src/frontend/web/von_interface/templates/settings_tab.html +++ b/src/frontend/web/von_interface/templates/settings_tab.html @@ -217,6 +217,7 @@

Text-to-speech

Dictation

+

Recorded audio uses conversation vocabulary. The browser controls below apply only to browser recognition.

diff --git a/tests/backend/test_model_registry_service.py b/tests/backend/test_model_registry_service.py index 39c88c3bc..f3473aff9 100644 --- a/tests/backend/test_model_registry_service.py +++ b/tests/backend/test_model_registry_service.py @@ -646,3 +646,12 @@ def test_runtime_parameter_policy_rejects_values_outside_graph_allowed_set( ) is None ) + + +def test_settings_chat_registry_excludes_audio_only_models(monkeypatch): + import src.backend.services.model_registry_service as mod + monkeypatch.setattr(mod, "resolve_enabled_llm_settings", lambda **kwargs: [ + {"provider": "openai", "model": "gpt-5.5"}, + {"provider": "openai", "model": "gpt-transcribe"}, + ]) + assert [entry["model_id"] for entry in mod._build_registry_from_settings()["models"]] == ["gpt-5.5"] diff --git a/tests/backend/test_orchestrator_model_fallback_execution.py b/tests/backend/test_orchestrator_model_fallback_execution.py index 1bb59720f..ac432202c 100644 --- a/tests/backend/test_orchestrator_model_fallback_execution.py +++ b/tests/backend/test_orchestrator_model_fallback_execution.py @@ -5222,3 +5222,23 @@ def _orchestrator_with_concepts() -> InternalMCPChatOrchestrator: state, telemetry = orchestrator._load_workflow_model_policy(None) assert telemetry["policy_source"] == "json" assert telemetry["graph_completeness"] == "graph_absent" + + +def test_transcription_models_do_not_become_chat_fallbacks(monkeypatch): + monkeypatch.setattr( + "src.backend.services.settings_service.resolve_enabled_llm_settings", + lambda **kwargs: [ + {"provider": "openai", "model": "gpt-5.5"}, + {"provider": "openai", "model": "gpt-transcribe"}, + {"provider": "openai", "model": "gpt-4o-transcribe"}, + {"provider": "openai", "model": "gpt-4o-mini-transcribe"}, + ], + ) + candidates = _bare_orchestrator()._stage_model_candidates( + stage="planner", default_model="gpt-5.5", default_provider="openai", + policy_state=_WorkflowModelPolicyState( + enabled=False, policy=None, policy_id=None, predicate_id=None, errors=(), + ), + ) + assert candidates + assert all(candidate.model == "gpt-5.5" for candidate in candidates) diff --git a/tests/backend/test_speech_transcription.py b/tests/backend/test_speech_transcription.py new file mode 100644 index 000000000..1c5df38d4 --- /dev/null +++ b/tests/backend/test_speech_transcription.py @@ -0,0 +1,146 @@ +import io +from unittest.mock import MagicMock + +import pytest +from flask import Flask + +from src.backend.server.routes import speech_routes as routes +from src.backend.services import speech_transcription_service as speech + + +@pytest.fixture +def provider(monkeypatch): + monkeypatch.setattr( + speech, + "resolve_enabled_llm_settings", + lambda **kw: [{"provider": "openai", "model": "gpt-transcribe"}], + ) + monkeypatch.setattr(speech, "get_openai_env_var", lambda: "TEST_SPEECH_KEY") + monkeypatch.setenv("TEST_SPEECH_KEY", "test-only-key") + monkeypatch.delenv("VON_TRANSCRIPTION_MODEL", raising=False) + gate = MagicMock() + monkeypatch.setattr(speech, "assert_model_execution_allowed", gate) + client = MagicMock() + client.__enter__.return_value = client + client.audio.transcriptions.create.return_value.text = "Von uses Vontology." + import openai + + monkeypatch.setattr(openai, "OpenAI", MagicMock(return_value=client)) + return client, gate + + +@pytest.mark.parametrize( + "mime,extension", [("audio/mp4", "mp4"), ("audio/webm;codecs=opus", "webm")] +) +def test_recorded_audio_carries_vocabulary_and_language(provider, mime, extension): + client, gate = provider + result = speech.transcribe_audio( + actor="#V#speaker", + organisation="#V#org", + audio=b"fixture", + mime_type=mime, + context="Discuss Vontology.", + vocabulary=["Vontology", "Wikidata", "wikidata"], + language="en-NZ", + ) + args = client.audio.transcriptions.create.call_args.kwargs + assert args["file"][0] == "dictation." + extension + assert args["extra_body"] == { + "keywords": ["Vontology", "Wikidata"], + "languages": ["en-NZ"], + } + assert "Discuss Vontology" in args["prompt"] + assert result["text"] == "Von uses Vontology." + gate.assert_called_once_with( + provider="openai", + model="gpt-transcribe", + user_concept_id="#V#speaker", + org_concept_id="#V#org", + allow_ambient_actor_scope=False, + ) + + +def test_model_gate_prevents_audio_request(provider): + client, gate = provider + gate.side_effect = PermissionError("model denied") + with pytest.raises(PermissionError): + speech.transcribe_audio( + actor="#V#a", organisation=None, audio=b"fixture", mime_type="audio/mp4" + ) + client.audio.transcriptions.create.assert_not_called() + + +def test_non_transcription_and_sol_models_cannot_be_selected(provider, monkeypatch): + monkeypatch.setattr( + speech, + "resolve_enabled_llm_settings", + lambda **kw: [{"provider": "openai", "model": "gpt-5.6-sol"}], + ) + monkeypatch.setenv("VON_TRANSCRIPTION_MODEL", "gpt-5.6-sol") + with pytest.raises(speech.SpeechUnavailable): + speech.transcription_capability("#V#a", None) + + +def test_context_is_bounded_literal_vocabulary(): + context, terms = speech.normalise_context( + "x" * 10000, ["\r\nnext", "a" * 200] * 100 + ) + assert len(context) == 6000 + assert terms == ["term next", "a" * 100] + + +@pytest.fixture +def app(): + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="test-only-session") + app.register_blueprint(routes.speech_bp) + return app + + +def test_normal_authenticated_session_and_forged_payload_identity(app, provider): + with app.test_client() as client: + with client.session_transaction() as session: + session["user_concept_id"] = "#V#speaker" + response = client.post( + "/api/speech/transcribe", + data={ + "audio": (io.BytesIO(b"fixture"), "clip.mp4", "audio/mp4"), + "user_concept_id": "#V#other", + }, + ) + assert response.status_code == 200 + assert provider[1].call_args.kwargs["user_concept_id"] == "#V#speaker" + assert response.headers["Cache-Control"] == "private, no-store" + + +def test_unauthenticated_upload_is_denied(app, provider): + response = app.test_client().post( + "/api/speech/transcribe", data={"user_concept_id": "#V#speaker"} + ) + assert response.status_code == 401 + provider[0].audio.transcriptions.create.assert_not_called() + + +def test_provider_failure_is_safe_and_retryable(app, provider): + provider[0].audio.transcriptions.create.side_effect = RuntimeError( + "secret-provider-token" + ) + with app.test_client() as client: + with client.session_transaction() as session: + session["user_concept_id"] = "#V#speaker" + response = client.post( + "/api/speech/transcribe", + data={"audio": (io.BytesIO(b"fixture"), "clip.mp4", "audio/mp4")}, + ) + assert response.status_code == 502 + assert b"secret-provider-token" not in response.data + assert "retry" in response.json["message"] + + +@pytest.mark.parametrize("data,mime", [(b"", "audio/mp4"), (b"a", "text/plain")]) +def test_bad_recording_never_reaches_provider(provider, data, mime): + with pytest.raises(ValueError): + speech.transcribe_audio( + actor="#V#a", organisation=None, audio=data, mime_type=mime + ) + provider[0].audio.transcriptions.create.assert_not_called() diff --git a/tests/backend/test_workflow_model_selection_workflow.py b/tests/backend/test_workflow_model_selection_workflow.py index 45afaa105..f372c4391 100644 --- a/tests/backend/test_workflow_model_selection_workflow.py +++ b/tests/backend/test_workflow_model_selection_workflow.py @@ -54,6 +54,7 @@ def test_model_selection_action_returns_enabled_pool_and_workflow_policy_choice( "src.backend.services.settings_service.resolve_enabled_llm_settings", lambda **_kwargs: [ {"provider": "openai", "model": "gpt-5.2-chat-latest", "scope": "user"}, + {"provider": "openai", "model": "gpt-transcribe", "scope": "user"}, {"provider": "ollama", "model": "granite3.3:2b", "scope": "user"}, ], ) diff --git a/tests/frontend/dictation.test.js b/tests/frontend/dictation.test.js new file mode 100644 index 000000000..fa1dda5d3 --- /dev/null +++ b/tests/frontend/dictation.test.js @@ -0,0 +1,109 @@ +/** @jest-environment jsdom */ +const { createDictationController, insertDictation, recordingMimeType } = require('../../src/frontend/web/von_interface/static/js/dictation.js'); +const flush = async () => { for (let i = 0; i < 12; i++) await Promise.resolve(); }; +const deferred = () => { let resolve; const promise = new Promise(r => { resolve = r; }); return { resolve, promise }; }; + +describe('recorded dictation lifecycle', () => { + let controller, recorder, stopTrack, getUserMedia, fetchImpl, input, context, root, pending, buttons; + beforeEach(async () => { + recorder = undefined; + document.body.innerHTML = '

'; + input = document.querySelector('textarea'); + input.value = 'Existing draft'; input.selectionStart = input.selectionEnd = input.value.length; + context = { key: 'actor:conversation', text: 'Vontology and Wikidata', vocabulary: ['Vontology', 'Wikidata'], language: 'en-NZ' }; + stopTrack = jest.fn(); + getUserMedia = jest.fn(async () => ({ getTracks: () => [{ stop: stopTrack, addEventListener: jest.fn() }] })); + class Recorder { + static isTypeSupported(type) { return type === 'audio/mp4'; } + constructor(stream, options) { recorder = this; this.mimeType = options.mimeType; this.state = 'inactive'; } + start() { this.state = 'recording'; } + stop() { + this.state = 'inactive'; + this.ondataavailable({ data: new Blob(['recorded-audio'], { type: this.mimeType }) }); + this.onstop(); + } + } + root = { navigator: { mediaDevices: { getUserMedia } }, MediaRecorder: Recorder, isSecureContext: true }; + pending = deferred(); + fetchImpl = jest.fn(async url => url.endsWith('capabilities') + ? { ok: true, json: async () => ({ transcription: { available: true, max_audio_bytes: 24 * 1024 * 1024 } }) } + : pending.promise); + buttons = [...document.querySelectorAll('button')]; + controller = createDictationController({ input, button: buttons[0], status: document.querySelector('p'), cancelButton: buttons[1], retryButton: buttons[2], engineSelect: document.querySelector('select'), getContext: () => context, setValue: value => { input.value = value; }, onStart: jest.fn(), fetchImpl, root }); + await flush(); + }); + afterEach(() => controller.dispose()); + test('negotiates Safari MP4 and preserves edits made during transcription', async () => { + await controller.start(); + const finish = controller.finish(); + input.value = 'Edited while waiting'; + pending.resolve({ ok: true, json: async () => ({ text: 'Vontology works.' }) }); + expect(await finish).toBe(true); + expect(input.value).toBe('Edited while waiting Vontology works.'); + expect(stopTrack).toHaveBeenCalled(); + const form = fetchImpl.mock.calls[1][1].body; + expect(form.get('audio').type).toBe('audio/mp4'); + expect(form.get('vocabulary')).toContain('Wikidata'); + expect(form.get('language')).toBe('en-NZ'); + }); + test('cancel before permission returns releases late stream without recording', async () => { + const permission = deferred(); getUserMedia.mockReturnValue(permission.promise); + const start = controller.start(); + controller.cancel(); + permission.resolve({ getTracks: () => [{ stop: stopTrack }] }); + await start; + expect(stopTrack).toHaveBeenCalled(); expect(recorder).toBeUndefined(); + expect(controller.getState()).toBe('idle'); + }); + test('late result after cancellation cannot modify draft', async () => { + await controller.start(); const finish = controller.finish(); controller.cancel(); + pending.resolve({ ok: true, json: async () => ({ text: 'late transcript' }) }); + await flush(); expect(await finish).toBe(false); expect(input.value).toBe('Existing draft'); + }); + test('conversation change fences transcription result', async () => { + await controller.start(); void controller.finish(); + context = { ...context, key: 'another conversation' }; + pending.resolve({ ok: true, json: async () => ({ text: 'wrong conversation' }) }); + await flush(); expect(input.value).toBe('Existing draft'); + }); + test('failed upload retains audio and retry does not reopen microphone', async () => { + await controller.start(); const finish = controller.finish(); + pending.resolve({ ok: false, json: async () => ({ message: 'Network error. Retry.' }) }); + expect(await finish).toBe(false); expect(controller.getState()).toBe('error'); + expect(buttons[2].hidden).toBe(false); expect(input.value).toBe('Existing draft'); + pending = deferred(); buttons[2].click(); + pending.resolve({ ok: true, json: async () => ({ text: 'Recovered' }) }); + await flush(); expect(input.value).toBe('Existing draft Recovered'); + expect(getUserMedia).toHaveBeenCalledTimes(1); + }); + test('permission denial has an actionable visible error', async () => { + getUserMedia.mockRejectedValue({ name: 'NotAllowedError' }); + await controller.start(); + expect(document.querySelector('p').textContent).toContain('Microphone access was denied'); + expect(await controller.finish()).toBe(false); + }); + test('inserts at selected range only when original draft is unchanged', () => { + expect(insertDictation('Hello old world', 'new', { start: 6, end: 9 })).toBe('Hello new world'); + expect(recordingMimeType(root)).toBe('audio/mp4'); + }); +}); + +describe('browser recognition snapshots', () => { + let recognition; + afterEach(() => { delete global.SpeechRecognition; }); + test('replayed final results are a complete snapshot, not duplicate deltas', () => { + global.SpeechRecognition = class { constructor() { recognition = this; } start() {} }; + const onResult = jest.fn(); + require('../../src/frontend/web/von_interface/static/js/speech.js').startSpeechRecognition({ onResult }); + const final = text => Object.assign([{ transcript: text }], { isFinal: true }); + recognition.onresult({ resultIndex: 0, results: [final('Von')] }); + recognition.onresult({ resultIndex: 0, results: [final('Von'), final('Vontology')] }); + recognition.onresult({ resultIndex: 1, results: [final('Von'), final('Vontology')] }); + expect(onResult.mock.calls.map(call => call[0].finalText)).toEqual(['Von', 'Von Vontology', 'Von Vontology']); + // DGX report: progressively longer prefixes were appended repeatedly. + for (const text of ['it', 'it seems', 'it seems that', 'it seems that the speech']) { + recognition.onresult({ resultIndex: 0, results: [final(text)] }); + } + expect(onResult.mock.calls.at(-1)[0].finalText).toBe('it seems that the speech'); + }); +}); diff --git a/tests/frontend/speech.test.js b/tests/frontend/speech.test.js index 66d97965c..c8cb1d126 100644 --- a/tests/frontend/speech.test.js +++ b/tests/frontend/speech.test.js @@ -2,13 +2,32 @@ const speechModulePath = '../../src/frontend/web/von_interface/static/js/speech.js'; -const { getSpeechSynthesisVoices, speakText } = require(speechModulePath); +const { getSpeechSynthesisVoices, speakText, stopSpeaking } = require(speechModulePath); describe('speech TTS utilities', () => { afterEach(() => { + stopSpeaking(); delete global.speechSynthesis; delete global.SpeechSynthesisUtterance; jest.restoreAllMocks(); + jest.useRealTimers(); + }); + + test.each([ + ['Android Chrome/130 Mobile', 1], + ['iPhone Mobile Safari', 1], + ['Macintosh Safari', 5] + ])('mobile %s speaks in the gesture and never runs Chromium pause/resume', (userAgent, maxTouchPoints) => { + jest.useFakeTimers(); + jest.spyOn(navigator, 'userAgent', 'get').mockReturnValue(userAgent); + Object.defineProperty(navigator, 'maxTouchPoints', { configurable: true, get: () => maxTouchPoints }); + global.speechSynthesis = { cancel: jest.fn(), speak: jest.fn(), getVoices: () => [], speaking: true, paused: false, pause: jest.fn(), resume: jest.fn() }; + global.SpeechSynthesisUtterance = function (text) { this.text = text; }; + speakText('A longer conversational reply.'); + expect(global.speechSynthesis.speak).toHaveBeenCalledTimes(1); + jest.advanceTimersByTime(30000); + expect(global.speechSynthesis.pause).not.toHaveBeenCalled(); + expect(global.speechSynthesis.resume).not.toHaveBeenCalled(); }); test('getSpeechSynthesisVoices returns empty array when unsupported', () => {