From 1992be3d35e90b79b17f105bf16d225b41fef22b Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:25:06 +0200 Subject: [PATCH 1/5] fix(voice): bounded recognition lifecycle; keep interim speech Owner hand test (Chrome 155, on-device): English works, Chinese shows "Listening..." and nothing appears, with no error. The hook made every failure silent: it claimed to listen before audio capture, ignored interim results, went quietly idle on an empty end and had no deadline. - ready after pack install (no auto-open); starting until audiostart; stopping while awaiting finals. - interimResults on: an ephemeral per-index preview; finals appended once; unfinalized speech goes to the answer box on end, stop timeout or a non-terminal error (never sent automatically). - empty end, nomatch or no-speech -> noText; a foreign aborted -> interrupted (mic stays available); self-aborts are suppressed. - 8 s startup deadline, 10 s stop grace, no overlapping sessions, timers cleared on every terminal path. - processLocally fail-closed checks unchanged; no cloud fallback. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/voice/useVoiceInput.ts | 130 ++++++++--- tests/unit/useVoiceInput.device.test.tsx | 269 ++++++++++++++++++++++- 2 files changed, 362 insertions(+), 37 deletions(-) diff --git a/src/lib/voice/useVoiceInput.ts b/src/lib/voice/useVoiceInput.ts index 48ef3b0..0e9d05c 100644 --- a/src/lib/voice/useVoiceInput.ts +++ b/src/lib/voice/useVoiceInput.ts @@ -14,11 +14,14 @@ import { MAX_CLIP_SECONDS, toVoiceClip } from './wavClip'; export type VoiceState = | { kind: 'idle' } + | { kind: 'ready' } + | { kind: 'starting' } + | { kind: 'stopping' } | { kind: 'recording'; seconds: number } | { kind: 'listening' } | { kind: 'transcribing' } | { kind: 'preparing' } - | { kind: 'error'; reason: 'denied' | 'failed' | 'unsupported' | 'limited' | 'unavailable' | 'deviceUnavailable' }; + | { kind: 'error'; reason: 'denied' | 'failed' | 'unsupported' | 'limited' | 'unavailable' | 'deviceUnavailable' | 'noText' | 'interrupted' }; /** Shared by local availability, installation and recognition. * Chrome 155 reports all six as available or downloadable for local processing (checked 2026-10-10). @@ -35,8 +38,12 @@ type SpeechRecognitionLike = { onresult: ((event: { resultIndex: number; results: ArrayLike<{ isFinal: boolean; 0: { transcript: string } }> }) => void) | null; onerror: ((event: { error: string }) => void) | null; onend: (() => void) | null; + onaudiostart: (() => void) | null; + onspeechstart: (() => void) | null; + onnomatch: (() => void) | null; start(): void; stop(): void; + abort(): void; }; type DeviceAvailability = 'available' | 'downloadable' | 'downloading' | 'unavailable'; @@ -85,6 +92,10 @@ export function useVoiceInput(options: { const { mode, language, studyId, researcherPreview, participantSessionHandle, onText } = options; const deviceSessionKey = `oi:device-voice-unavailable:${researcherPreview ? 'preview' : participantSessionHandle ?? 'session'}:${studyId ?? ''}`; const [state, setState] = useState({ kind: 'idle' }); + const [preview, setPreview] = useState(''); + const [speechTag, setSpeechTag] = useState(SPEECH_TAGS[language]); + const speechCleanup = useRef<(() => void) | null>(null); + const speechStop = useRef<(() => void) | null>(null); const [supported, setSupported] = useState(false); const recorder = useRef(null); const recognition = useRef(null); @@ -130,9 +141,7 @@ export function useVoiceInput(options: { } return () => { generation.current += 1; - const session = recognition.current; - recognition.current = null; - session?.stop(); + speechCleanup.current?.(); }; }, [mode, language, deviceSessionKey]); @@ -143,9 +152,7 @@ export function useVoiceInput(options: { deviceAvailability.current = 'unavailable'; setSupported(false); setState({ kind: 'error', reason: 'deviceUnavailable' }); - const session = recognition.current; - recognition.current = null; - session?.stop(); + speechCleanup.current?.(); }, [deviceSessionKey]); const clearTimer = () => { @@ -158,7 +165,7 @@ export function useVoiceInput(options: { return () => { mounted.current = false; clearTimer(); - recognition.current?.stop(); + speechCleanup.current?.(); if (recorder.current?.state === 'recording') recorder.current.stop(); recorder.current?.stream.getTracks().forEach((track) => track.stop()); }; @@ -222,6 +229,7 @@ export function useVoiceInput(options: { }, [transcribe]); const startListening = useCallback(() => { + if (recognition.current) return; const Recognition = speechRecognitionClass(); if (!Recognition) { if (mode === 'device') disableDevice(); @@ -250,35 +258,107 @@ export function useVoiceInput(options: { } session.lang = SPEECH_TAGS[language]; session.continuous = true; - session.interimResults = false; + session.interimResults = true; + const committed = new Set(); + const interim = new Map(); + let finalCount = 0; + let stopping = false; + let startupTimer: ReturnType | undefined; + let stopTimer: ReturnType | undefined; + const active = () => mounted.current && recognition.current === session; + const previewText = () => [...interim.entries()].sort(([a], [b]) => a - b).map(([, text]) => text).filter(Boolean).join(' '); + const release = (abort: boolean) => { + clearTimeout(startupTimer); + clearTimeout(stopTimer); + recognition.current = null; + speechCleanup.current = null; + speechStop.current = null; + interim.clear(); + if (mounted.current) setPreview(''); + // Invalidate before abort: browsers can emit error/end synchronously. + if (abort) { + try { session.abort(); } catch { /* Already ended. */ } + } + }; + // Unfinalized speech still goes to the answer box for review (some local + // recognizers emit interim text without a final). + const keepInterim = () => { + const text = previewText(); + if (text) { + onTextRef.current(text); + finalCount += 1; + } + }; + const finish = (abort = false) => { + if (!active()) return; + keepInterim(); + release(abort); + setState(finalCount ? { kind: 'idle' } : { kind: 'error', reason: 'noText' }); + }; + session.onaudiostart = () => { + if (!active()) return; + clearTimeout(startupTimer); + if (!stopping) setState({ kind: 'listening' }); + }; + session.onnomatch = () => { /* The end handler decides whether any text was captured. */ }; session.onresult = (event) => { - if (!mounted.current || recognition.current !== session) return; + if (!active()) return; + for (const index of interim.keys()) { + if (index >= event.results.length) interim.delete(index); + } for (let index = event.resultIndex; index < event.results.length; index += 1) { const result = event.results[index]; - if (result.isFinal && result[0].transcript.trim()) onTextRef.current(result[0].transcript.trim()); + const text = result[0].transcript.trim(); + if (result.isFinal) { + interim.delete(index); + if (!committed.has(index)) { + committed.add(index); + if (text) { + onTextRef.current(text); + finalCount += 1; + } + } + } else if (!committed.has(index)) interim.set(index, text); } + setPreview(previewText()); }; session.onerror = (event) => { - if (!mounted.current || recognition.current !== session) return; + if (!active()) return; if (mode === 'device' && ['language-not-supported', 'service-not-allowed'].includes(event.error)) { disableDevice(); return; } - setState({ kind: 'error', reason: event.error === 'not-allowed' || event.error === 'service-not-allowed' ? 'denied' : 'failed' }); - }; - session.onend = () => { - if (recognition.current !== session) return; - recognition.current = null; - if (mounted.current) setState((current) => (current.kind === 'listening' ? { kind: 'idle' } : current)); + keepInterim(); + release(true); + setState({ kind: 'error', reason: event.error === 'aborted' ? 'interrupted' + : event.error === 'no-speech' ? 'noText' + : event.error === 'not-allowed' || event.error === 'service-not-allowed' ? 'denied' : 'failed' }); }; + session.onend = () => finish(); recognition.current = session; - setState({ kind: 'listening' }); + speechCleanup.current = () => release(true); + speechStop.current = () => { + if (!active() || stopping) return; + stopping = true; + clearTimeout(startupTimer); + setState({ kind: 'stopping' }); + stopTimer = setTimeout(() => finish(true), 10_000); + try { session.stop(); } catch { finish(true); } + }; + setPreview(''); + setSpeechTag(session.lang); + setState({ kind: 'starting' }); + startupTimer = setTimeout(() => { + if (!active()) return; + release(true); + setState({ kind: 'error', reason: 'failed' }); + }, 8_000); try { session.start(); } catch { if (mode === 'device') disableDevice(); else { - recognition.current = null; + release(true); setState({ kind: 'error', reason: 'failed' }); } } @@ -302,7 +382,7 @@ export function useVoiceInput(options: { deviceAvailability.current = 'available'; // Do not open the microphone unexpectedly after a slow download. // The participant can keep typing/sending, then press the mic when ready. - setState({ kind: 'idle' }); + setState({ kind: 'ready' }); } } catch { if (mounted.current && generation.current === currentGeneration) disableDevice(); @@ -316,11 +396,11 @@ export function useVoiceInput(options: { recorder.current?.stop(); return; } - if (state.kind === 'listening') { - recognition.current?.stop(); + if (state.kind === 'starting' || state.kind === 'listening') { + speechStop.current?.(); return; } - if (state.kind === 'transcribing' || preparing.current || !supported) return; + if (state.kind === 'stopping' || state.kind === 'transcribing' || preparing.current || !supported) return; if (mode === 'installation') void startRecording(); else if (mode === 'browser') startListening(); else if (mode === 'device' && !deviceDisabled.current) { @@ -329,5 +409,5 @@ export function useVoiceInput(options: { } }, [mode, startListening, startRecording, prepareDevice, state.kind, supported]); - return { enabled: mode === 'installation' || mode === 'browser' || mode === 'device', supported, state, toggle }; + return { enabled: mode === 'installation' || mode === 'browser' || mode === 'device', supported, state, preview, speechTag, toggle }; } diff --git a/tests/unit/useVoiceInput.device.test.tsx b/tests/unit/useVoiceInput.device.test.tsx index 6095e57..6380069 100644 --- a/tests/unit/useVoiceInput.device.test.tsx +++ b/tests/unit/useVoiceInput.device.test.tsx @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { act, renderHook, waitFor } from '@testing-library/react'; +import { act, cleanup, renderHook, waitFor } from '@testing-library/react'; import { SPEECH_TAGS, useVoiceInput } from '@/lib/voice/useVoiceInput'; import type { InterviewLanguage } from '@/lib/i18n/languages'; @@ -10,16 +10,23 @@ class Recognition { lang = ''; processLocally = false; continuous = false; - interimResults = true; + interimResults = false; onresult: ((event: unknown) => void) | null = null; onerror: ((event: { error: string }) => void) | null = null; onend: (() => void) | null = null; + onaudiostart: (() => void) | null = null; + onspeechstart: (() => void) | null = null; + onnomatch: (() => void) | null = null; start = vi.fn(() => { // Every started instance must satisfy the local-only contract, including retries. expect(this.processLocally).toBe(true); Recognition.sessions.push(this); }); - stop = vi.fn(() => this.onend?.()); + stop = vi.fn(); + abort = vi.fn(); + result(results: [string, boolean][], resultIndex = 0) { + this.onresult?.({ resultIndex, results: results.map(([transcript, isFinal]) => ({ isFinal, 0: { transcript } })) }); + } } const onText = vi.fn(); let sessionNumber = 0; @@ -39,7 +46,7 @@ beforeEach(() => { vi.stubGlobal('webkitSpeechRecognition', undefined); vi.stubGlobal('fetch', vi.fn()); }); -afterEach(() => vi.unstubAllGlobals()); +afterEach(() => { cleanup(); vi.useRealTimers(); vi.unstubAllGlobals(); }); describe('device-only speech input', () => { it.each(['available', 'downloadable', 'downloading'])('offers the mic only after %s is resolved', async availability => { @@ -78,7 +85,7 @@ describe('device-only speech input', () => { expect(Recognition.install).toHaveBeenCalledExactlyOnceWith({ langs: ['en-US'], processLocally: true }); expect(Recognition.sessions).toHaveLength(0); await act(async () => resolve(true)); - expect(result.current.state.kind).toBe('idle'); + expect(result.current.state.kind).toBe('ready'); expect(Recognition.sessions).toHaveLength(0); act(() => result.current.toggle()); expect(Recognition.sessions).toHaveLength(1); @@ -214,21 +221,259 @@ describe('device-only speech input', () => { }); }); +describe('shared speech lifecycle', () => { + async function start() { + const hook = mount(); + await act(async () => {}); + expect(hook.result.current.supported).toBe(true); + act(() => hook.result.current.toggle()); + return { ...hook, session: Recognition.sessions[0] }; + } + + it('starts once, and waits for audio rather than speech before reporting listening', async () => { + const { result, session } = await start(); + expect(result.current.state).toEqual({ kind: 'starting' }); + expect(session.interimResults).toBe(true); + act(() => session.onspeechstart?.()); + expect(result.current.state.kind).toBe('starting'); + act(() => session.onaudiostart?.()); + expect(result.current.state).toEqual({ kind: 'listening' }); + }); + + it('does not open two sessions for two presses before React renders', async () => { + const { result } = mount(); + await act(async () => {}); + act(() => { result.current.toggle(); result.current.toggle(); }); + expect(Recognition.sessions).toHaveLength(1); + expect(result.current.state.kind).toBe('starting'); + }); + + it('aborts a startup after eight seconds, suppresses its aborted event, and permits retry', async () => { + vi.useFakeTimers(); + const { result, session } = await start(); + session.abort.mockImplementation(() => { + session.onerror?.({ error: 'aborted' }); + session.onend?.(); + }); + act(() => vi.advanceTimersByTime(7_999)); + expect(result.current.state.kind).toBe('starting'); + act(() => vi.advanceTimersByTime(1)); + expect(session.abort).toHaveBeenCalledTimes(1); + expect(result.current.state).toEqual({ kind: 'error', reason: 'failed' }); + expect(result.current.supported).toBe(true); + expect(vi.getTimerCount()).toBe(0); + act(() => result.current.toggle()); + expect(Recognition.sessions).toHaveLength(2); + act(() => { session.onaudiostart?.(); session.result([['stale', true]]); session.onend?.(); }); + expect(result.current.state.kind).toBe('starting'); + expect(onText).not.toHaveBeenCalled(); + }); + + it('clears the startup timer and disables device mode when start throws', async () => { + vi.useFakeTimers(); + class CannotStart extends Recognition { + start = vi.fn(() => { throw new Error('cannot start'); }); + } + vi.stubGlobal('SpeechRecognition', CannotStart); + const { result } = mount(); + await act(async () => {}); + act(() => result.current.toggle()); + expect(result.current.state).toEqual({ kind: 'error', reason: 'deviceUnavailable' }); + expect(result.current.supported).toBe(false); + expect(vi.getTimerCount()).toBe(0); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('clears the startup deadline when audio begins', async () => { + vi.useFakeTimers(); + const { result, session } = await start(); + act(() => session.onaudiostart?.()); + act(() => vi.advanceTimersByTime(8_000)); + expect(result.current.state.kind).toBe('listening'); + expect(session.abort).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + + it('replaces interim text per index, removes retracted results, and commits each final once', async () => { + const { result, session } = await start(); + act(() => session.result([[' first ', false], ['draft', false]])); + expect(result.current.preview).toBe('first draft'); + expect(onText).not.toHaveBeenCalled(); + act(() => session.result([['revised', false]])); + expect(result.current.preview).toBe('revised'); + act(() => session.result([['finished', true], ['next', false]])); + act(() => session.result([['finished', true], ['new next', false]], 1)); + act(() => session.result([['finished', true], ['new next', false]])); + expect(result.current.preview).toBe('new next'); + expect(onText).toHaveBeenCalledExactlyOnceWith('finished'); + act(() => session.result([['finished', true], ['second', true]], 1)); + expect(result.current.preview).toBe(''); + expect(onText.mock.calls).toEqual([['finished'], ['second']]); + act(() => session.onend?.()); + expect(result.current.state.kind).toBe('idle'); + }); + + it('accepts a late final while stopping and clears the grace timer on end', async () => { + vi.useFakeTimers(); + const { result, session } = await start(); + act(() => session.onaudiostart?.()); + act(() => result.current.toggle()); + expect(result.current.state.kind).toBe('stopping'); + expect(session.stop).toHaveBeenCalledTimes(1); + act(() => result.current.toggle()); + expect(session.stop).toHaveBeenCalledTimes(1); + expect(Recognition.sessions).toHaveLength(1); + act(() => session.result([['late final', true]])); + expect(onText).toHaveBeenCalledExactlyOnceWith('late final'); + act(() => session.onend?.()); + expect(result.current.state.kind).toBe('idle'); + expect(vi.getTimerCount()).toBe(0); + expect(session.abort).not.toHaveBeenCalled(); + }); + + it('commits interim-only speech once on end, and drops any late events', async () => { + const { result, session } = await start(); + act(() => session.result([['你好', false], ['世界', false]])); + expect(onText).not.toHaveBeenCalled(); + act(() => { session.onend?.(); session.onend?.(); session.result([['late', true]]); }); + expect(onText).toHaveBeenCalledExactlyOnceWith('你好 世界'); + expect(result.current.preview).toBe(''); + expect(result.current.state.kind).toBe('idle'); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('keeps an unfinished last phrase after a committed final, once', async () => { + const { result, session } = await start(); + act(() => session.result([['final', true], ['unfinished', false]])); + act(() => session.onend?.()); + expect(onText.mock.calls).toEqual([['final'], ['unfinished']]); + expect(result.current.preview).toBe(''); + expect(result.current.state.kind).toBe('idle'); + }); + + it.each(['empty', 'whitespace', 'nomatch'])('reports noText after %s and releases the session', async outcome => { + vi.useFakeTimers(); + const { result, session } = await start(); + act(() => { + if (outcome === 'whitespace') session.result([[' ', false]]); + if (outcome === 'nomatch') session.onnomatch?.(); + session.onend?.(); + }); + expect(result.current.state).toEqual({ kind: 'error', reason: 'noText' }); + expect(result.current.supported).toBe(true); + expect(onText).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + + it('reports foreign aborted as interrupted without disabling the mic, even without end', async () => { + vi.useFakeTimers(); + const { result, session } = await start(); + act(() => session.result([['kept draft', false]])); + act(() => session.onerror?.({ error: 'aborted' })); + expect(result.current.state).toEqual({ kind: 'error', reason: 'interrupted' }); + expect(result.current.supported).toBe(true); + expect(result.current.preview).toBe(''); + expect(vi.getTimerCount()).toBe(0); + act(() => result.current.toggle()); + expect(Recognition.sessions).toHaveLength(2); + expect(onText).toHaveBeenCalledExactlyOnceWith('kept draft'); + }); + + it('reports no-speech as noText', async () => { + const { result, session } = await start(); + act(() => session.onerror?.({ error: 'no-speech' })); + expect(result.current.state).toEqual({ kind: 'error', reason: 'noText' }); + expect(onText).not.toHaveBeenCalled(); + }); + + it.each([false, true])('bounds stop during startup with a ten-second grace, interim=%s', async hasInterim => { + vi.useFakeTimers(); + const { result, session } = await start(); + session.abort.mockImplementation(() => { + session.onerror?.({ error: 'aborted' }); + session.onend?.(); + }); + if (hasInterim) act(() => session.result([['recovered', false]])); + act(() => result.current.toggle()); + act(() => session.onaudiostart?.()); + act(() => vi.advanceTimersByTime(9_999)); + expect(result.current.state.kind).toBe('stopping'); + expect(session.abort).not.toHaveBeenCalled(); + act(() => vi.advanceTimersByTime(1)); + expect(session.abort).toHaveBeenCalledTimes(1); + expect(result.current.state).toEqual(hasInterim ? { kind: 'idle' } : { kind: 'error', reason: 'noText' }); + expect(result.current.preview).toBe(''); + expect(onText.mock.calls).toEqual(hasInterim ? [['recovered']] : []); + expect(vi.getTimerCount()).toBe(0); + act(() => result.current.toggle()); + expect(Recognition.sessions).toHaveLength(2); + }); + + it.each(['not-allowed', 'network', 'language-not-supported'])('clears timers and preview after %s without waiting for end', async error => { + vi.useFakeTimers(); + const { result, session } = await start(); + act(() => session.result([['draft', false]])); + act(() => result.current.toggle()); + act(() => session.onerror?.({ error })); + expect(result.current.state.kind).toBe('error'); + expect(result.current.preview).toBe(''); + expect(vi.getTimerCount()).toBe(0); + act(() => session.onend?.()); + expect(result.current.state.kind).toBe('error'); + // A terminal device failure drops the draft; other errors keep it for review. + if (error === 'language-not-supported') expect(onText).not.toHaveBeenCalled(); + else expect(onText).toHaveBeenCalledExactlyOnceWith('draft'); + }); + + it.each(['starting', 'stopping'])('aborts and clears timers on unmount while %s without committing', async phase => { + vi.useFakeTimers(); + const { result, session, unmount } = await start(); + act(() => session.result([['discard', false]])); + if (phase === 'stopping') act(() => result.current.toggle()); + unmount(); + expect(session.abort).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + act(() => { session.onerror?.({ error: 'aborted' }); session.onend?.(); }); + expect(onText).not.toHaveBeenCalled(); + }); + + it('cleans up a device session on language change and ignores its late events', async () => { + vi.useFakeTimers(); + const { result, rerender } = renderHook(({ language }) => useVoiceInput(options(language)), { initialProps: { language: 'en' as InterviewLanguage } }); + await act(async () => {}); + act(() => result.current.toggle()); + const session = Recognition.sessions[0]; + act(() => session.result([['discard', false]])); + rerender({ language: 'zh' }); + await act(async () => {}); + expect(session.abort).toHaveBeenCalledTimes(1); + expect(result.current.preview).toBe(''); + expect(vi.getTimerCount()).toBe(0); + act(() => result.current.toggle()); + expect(Recognition.sessions[1].lang).toBe('zh-CN'); + act(() => { session.result([['stale', true]]); session.onend?.(); }); + expect(result.current.state.kind).toBe('starting'); + expect(onText).not.toHaveBeenCalled(); + }); +}); + describe('language changes outside device mode', () => { - it('keeps browser dictation listening when the language prop changes', async () => { - const stop = vi.fn(); - class CloudRecognition { - lang = ''; continuous = false; interimResults = true; - onresult = null; onerror = null; onend = null; - start = vi.fn(); stop = stop; + it('keeps browser dictation and its speech tag when the language prop changes', async () => { + class CloudRecognition extends Recognition { + start = vi.fn(() => { Recognition.sessions.push(this); }); } vi.stubGlobal('SpeechRecognition', CloudRecognition); const { result, rerender } = renderHook(({ language }) => useVoiceInput({ ...options(language), mode: 'browser' }), { initialProps: { language: 'en' as InterviewLanguage } }); await waitFor(() => expect(result.current.supported).toBe(true)); act(() => result.current.toggle()); + const session = Recognition.sessions[0]; + expect(session.processLocally).toBe(false); + act(() => session.onaudiostart?.()); expect(result.current.state).toEqual({ kind: 'listening' }); rerender({ language: 'ja' }); - expect(stop).not.toHaveBeenCalled(); + expect(session.stop).not.toHaveBeenCalled(); + expect(session.abort).not.toHaveBeenCalled(); expect(result.current.state).toEqual({ kind: 'listening' }); + expect(result.current.speechTag).toBe('en-US'); }); }); From 3470160407b79e9108654752cc6adbd4c0941159 Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:25:06 +0200 Subject: [PATCH 2/5] fix(interview): show voice readiness, start-up and interim text in six languages New statuses (ready, starting, noText, interrupted) in all six dictionaries; a muted live preview tagged with the recognition language; Send and Cmd/Ctrl+Enter held while starting/listening/stopping and released on every terminal path. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/InterviewChat.tsx | 21 ++++-- src/lib/i18n/messages/en.ts | 8 ++ src/lib/i18n/messages/es.ts | 4 + src/lib/i18n/messages/fr.ts | 4 + src/lib/i18n/messages/ja.ts | 4 + src/lib/i18n/messages/ko.ts | 4 + src/lib/i18n/messages/zh.ts | 4 + tests/unit/InterviewChat.voice.test.tsx | 98 ++++++++++++++++++++++++- 8 files changed, 137 insertions(+), 10 deletions(-) diff --git a/src/components/InterviewChat.tsx b/src/components/InterviewChat.tsx index 5d76196..7261f31 100644 --- a/src/components/InterviewChat.tsx +++ b/src/components/InterviewChat.tsx @@ -69,7 +69,9 @@ const InterviewChat: React.FC = () => { // Text is added to the answer box for the participant to check; never sent. onText: (text) => setInput((current) => (current.trimEnd() ? `${current.trimEnd()} ${text}` : text)), }); - const voiceBusy = voice.state.kind === 'recording' || voice.state.kind === 'listening' || voice.state.kind === 'transcribing'; + const voiceListening = voice.state.kind === 'starting' || voice.state.kind === 'listening'; + const voicePreviewVisible = voiceListening || voice.state.kind === 'stopping'; + const voiceBusy = voice.state.kind === 'recording' || voicePreviewVisible || voice.state.kind === 'transcribing'; const messagesEndRef = useRef(null); const textareaRef = useRef(null); @@ -268,7 +270,7 @@ const InterviewChat: React.FC = () => { // Cmd/Ctrl+Enter sends; the Send button is the other way to send. if (e.key === 'Enter' && (e.metaKey || e.ctrlKey)) { e.preventDefault(); - if (!isAiThinking && input.trim()) { + if (!isAiThinking && !voiceBusy && input.trim()) { void handleSend(); } } @@ -451,11 +453,11 @@ const InterviewChat: React.FC = () => { type="button" variant="quiet" onClick={voice.toggle} - disabled={isAiThinking || voice.state.kind === 'transcribing' || voice.state.kind === 'preparing'} - aria-pressed={voice.state.kind === 'recording' || voice.state.kind === 'listening'} + disabled={isAiThinking || voice.state.kind === 'transcribing' || voice.state.kind === 'preparing' || voice.state.kind === 'stopping'} + aria-pressed={voice.state.kind === 'recording' || voiceListening} className="min-h-11 w-full sm:w-auto" > - {voice.state.kind === 'recording' || voice.state.kind === 'listening' ? m.voice.stop : m.voice.start} + {voice.state.kind === 'recording' || voiceListening ? m.voice.stop : m.voice.start} {sendButton} @@ -465,13 +467,18 @@ const InterviewChat: React.FC = () => {

{voice.state.kind === 'recording' ? m.voice.recording(`${Math.floor(voice.state.seconds / 60)}:${String(voice.state.seconds % 60).padStart(2, '0')}`) - : voice.state.kind === 'listening' ? m.voice.listening + : voice.state.kind === 'ready' ? m.voice.ready + : voice.state.kind === 'starting' ? m.voice.starting + : voice.state.kind === 'listening' || voice.state.kind === 'stopping' ? m.voice.listening : voice.state.kind === 'transcribing' ? m.voice.transcribing : voice.state.kind === 'preparing' ? m.voice.preparing : m.voice[voice.state.reason]}

)} - {voice.enabled && voice.state.kind === 'idle' && input.trim() !== '' && ( + {voice.enabled && voicePreviewVisible && voice.preview && ( +

{voice.preview}

+ )} + {voice.enabled && (voice.state.kind === 'idle' || voice.state.kind === 'ready') && input.trim() !== '' && (

{m.voice.review}

)}

{m.sendShortcut}

diff --git a/src/lib/i18n/messages/en.ts b/src/lib/i18n/messages/en.ts index 55a1982..0e4d3a4 100644 --- a/src/lib/i18n/messages/en.ts +++ b/src/lib/i18n/messages/en.ts @@ -90,6 +90,10 @@ export interface Messages { recording: (elapsed: string) => string; listening: string; transcribing: string; + ready: string; + starting: string; + noText: string; + interrupted: string; preparing: string; deviceUnavailable: string; review: string; @@ -218,6 +222,10 @@ export const en: Messages = { send: 'Send', sendShortcut: '⌘/Ctrl + Enter to send', voice: { + ready: 'Voice input is ready. Press the microphone to speak.', + starting: 'Starting the microphone…', + noText: 'No text was captured. Please try again or type your answer.', + interrupted: 'Voice input stopped unexpectedly. Please try again or type your answer.', preparing: 'Preparing speech on this device…', deviceUnavailable: 'Voice input is unavailable on this device. Please type your answer.', start: 'Start voice input', diff --git a/src/lib/i18n/messages/es.ts b/src/lib/i18n/messages/es.ts index 8f9d9b7..7489158 100644 --- a/src/lib/i18n/messages/es.ts +++ b/src/lib/i18n/messages/es.ts @@ -97,6 +97,10 @@ export const es: Messages = { send: 'Enviar', sendShortcut: '⌘/Ctrl + Intro para enviar', voice: { + ready: 'La entrada de voz está lista. Pulsa el micrófono para hablar.', + starting: 'Activando el micrófono…', + noText: 'No se ha reconocido ningún texto. Inténtalo de nuevo o escribe tu respuesta.', + interrupted: 'La entrada de voz se ha detenido de forma inesperada. Inténtalo de nuevo o escribe tu respuesta.', preparing: 'Preparando la entrada de voz en este dispositivo…', deviceUnavailable: 'La entrada de voz no está disponible en este dispositivo. Escribe tu respuesta.', start: 'Iniciar la entrada de voz', diff --git a/src/lib/i18n/messages/fr.ts b/src/lib/i18n/messages/fr.ts index e65cd3c..287dd74 100644 --- a/src/lib/i18n/messages/fr.ts +++ b/src/lib/i18n/messages/fr.ts @@ -97,6 +97,10 @@ export const fr: Messages = { send: 'Envoyer', sendShortcut: '⌘/Ctrl + Entrée pour envoyer', voice: { + ready: 'La saisie vocale est prête. Appuyez sur le micro pour parler.', + starting: 'Démarrage du micro…', + noText: 'Aucun texte n’a été reconnu. Veuillez réessayer ou saisir votre réponse.', + interrupted: 'La saisie vocale s’est arrêtée de manière inattendue. Veuillez réessayer ou saisir votre réponse.', preparing: 'Préparation de la saisie vocale sur cet appareil…', deviceUnavailable: 'La saisie vocale n’est pas disponible sur cet appareil. Veuillez saisir votre réponse.', start: 'Démarrer la saisie vocale', diff --git a/src/lib/i18n/messages/ja.ts b/src/lib/i18n/messages/ja.ts index d09e53b..2972319 100644 --- a/src/lib/i18n/messages/ja.ts +++ b/src/lib/i18n/messages/ja.ts @@ -97,6 +97,10 @@ export const ja: Messages = { send: '送信', sendShortcut: '⌘/Ctrl + Enter で送信', voice: { + ready: '音声入力の準備ができました。マイクボタンを押して話してください。', + starting: 'マイクを起動しています…', + noText: '文字を認識できませんでした。もう一度お試しいただくか、回答を入力してください。', + interrupted: '音声入力が予期せず停止しました。もう一度お試しいただくか、回答を入力してください。', preparing: 'このデバイスで音声入力を準備しています…', deviceUnavailable: 'このデバイスでは音声入力を利用できません。回答を入力してください。', start: '音声入力を開始', diff --git a/src/lib/i18n/messages/ko.ts b/src/lib/i18n/messages/ko.ts index b5ee8ac..7d3b73a 100644 --- a/src/lib/i18n/messages/ko.ts +++ b/src/lib/i18n/messages/ko.ts @@ -97,6 +97,10 @@ export const ko: Messages = { send: '보내기', sendShortcut: '⌘/Ctrl + Enter로 보내기', voice: { + ready: '음성 입력이 준비되었습니다. 마이크 버튼을 누르고 말씀해 주세요.', + starting: '마이크를 켜는 중…', + noText: '인식된 텍스트가 없습니다. 다시 시도하시거나 응답을 입력해 주세요.', + interrupted: '음성 입력이 예기치 않게 중지되었습니다. 다시 시도하시거나 응답을 입력해 주세요.', preparing: '이 기기에서 음성 입력을 준비하고 있습니다…', deviceUnavailable: '이 기기에서는 음성 입력을 사용할 수 없습니다. 응답을 입력해 주세요.', start: '음성 입력 시작', diff --git a/src/lib/i18n/messages/zh.ts b/src/lib/i18n/messages/zh.ts index 08720b0..b874f31 100644 --- a/src/lib/i18n/messages/zh.ts +++ b/src/lib/i18n/messages/zh.ts @@ -97,6 +97,10 @@ export const zh: Messages = { send: '发送', sendShortcut: '按 ⌘/Ctrl + Enter 发送', voice: { + ready: '语音输入已准备好。请按麦克风按钮开始说话。', + starting: '正在启动麦克风……', + noText: '没有识别到文字。请重试，或直接输入您的回答。', + interrupted: '语音输入意外中断。请重试，或直接输入您的回答。', preparing: '正在这台设备上准备语音输入…', deviceUnavailable: '这台设备无法使用语音输入。请直接输入您的回答。', start: '开始语音输入', diff --git a/tests/unit/InterviewChat.voice.test.tsx b/tests/unit/InterviewChat.voice.test.tsx index d58627f..955fa97 100644 --- a/tests/unit/InterviewChat.voice.test.tsx +++ b/tests/unit/InterviewChat.voice.test.tsx @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'; import { makeStudyConfig } from '../fixtures/models'; -import { useStore } from '@/store'; +import { RESEARCH_STORE_KEY, useStore } from '@/store'; import type { VoiceInputMode } from '@/types'; const interviewApiMock = vi.hoisted(() => ({ @@ -25,8 +25,12 @@ class FakeRecognition { onresult: ((event: unknown) => void) | null = null; onerror: ((event: { error: string }) => void) | null = null; onend: (() => void) | null = null; + onaudiostart: (() => void) | null = null; + onspeechstart: (() => void) | null = null; + onnomatch: (() => void) | null = null; start = vi.fn(() => { FakeRecognition.last = this; }); - stop = vi.fn(() => this.onend?.()); + stop = vi.fn(); + abort = vi.fn(); } function seed(voiceInput?: VoiceInputMode, extra: Parameters[0] = {}) { @@ -49,6 +53,8 @@ beforeEach(() => { }); afterEach(() => { + cleanup(); + vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -72,11 +78,79 @@ describe('participant voice input', () => { FakeRecognition.last?.onresult?.({ resultIndex: 0, results: [{ isFinal: true, 0: { transcript: ' 研究者です ' } }] }); }); fireEvent.click(screen.getByRole('button', { name: '録音を停止' })); + expect(screen.getByRole('button', { name: '送信' })).toBeDisabled(); + act(() => FakeRecognition.last?.onend?.()); await waitFor(() => expect(textarea.value).toBe('私は 研究者です')); + expect(screen.getByRole('button', { name: '送信' })).toBeEnabled(); expect(screen.getByText('送信する前に文字をご確認ください。')).toBeInTheDocument(); expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); }); + it('shows a polite speech-tagged preview without storing or sending it, then commits interim-only text for review', async () => { + seed('browser', { interviewLanguages: ['zh'] }); + useStore.setState({ participantLanguage: 'zh' }); + render(); + const textarea = screen.getByLabelText('您的回答'); + const storedBefore = sessionStorage.getItem(RESEARCH_STORE_KEY); + expect(storedBefore).not.toBeNull(); + const historyBefore = useStore.getState().interviewHistory; + fireEvent.click(await screen.findByRole('button', { name: '开始语音输入' })); + expect(screen.getByRole('status')).toHaveTextContent('正在启动麦克风……'); + expect(screen.getByRole('button', { name: '停止录音' })).toHaveAttribute('aria-pressed', 'true'); + fireEvent.change(textarea, { target: { value: '手动输入' } }); + expect(textarea).toBeEnabled(); + expect(screen.getByRole('button', { name: '发送' })).toBeDisabled(); + fireEvent.keyDown(textarea, { key: 'Enter', ctrlKey: true }); + expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); + act(() => FakeRecognition.last?.onaudiostart?.()); + fireEvent.keyDown(textarea, { key: 'Enter', metaKey: true }); + expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); + expect(screen.getByRole('status')).toHaveTextContent('正在聆听……'); + act(() => FakeRecognition.last?.onresult?.({ resultIndex: 0, results: [{ isFinal: false, 0: { transcript: '临时文字' } }] })); + const preview = screen.getByText('临时文字'); + expect(preview).toHaveAttribute('lang', 'zh-CN'); + expect(preview).toHaveAttribute('aria-live', 'polite'); + expect(preview).toHaveClass('text-ink-500'); + expect(textarea).toHaveValue('手动输入'); + expect(sessionStorage.getItem(RESEARCH_STORE_KEY)).toBe(storedBefore); + expect(useStore.getState().interviewHistory).toBe(historyBefore); + fireEvent.click(screen.getByRole('button', { name: '停止录音' })); + fireEvent.keyDown(textarea, { key: 'Enter', ctrlKey: true }); + expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); + expect(screen.getByText('临时文字')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: '发送' })).toBeDisabled(); + act(() => FakeRecognition.last?.onend?.()); + expect(textarea).toHaveValue('手动输入 临时文字'); + expect(screen.queryByText('临时文字')).not.toBeInTheDocument(); + expect(screen.getByText('发送前请检查文字。')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: '发送' })).toBeEnabled(); + expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); + }); + + it.each(['end', 'aborted', 'startup', 'grace'])('releases Send and preserves typing after %s', async terminal => { + vi.useFakeTimers(); + seed('browser'); + render(); + fireEvent.change(screen.getByLabelText('Your response'), { target: { value: 'typed answer' } }); + fireEvent.click(screen.getByRole('button', { name: 'Start voice input' })); + expect(screen.getByRole('button', { name: 'Send' })).toBeDisabled(); + act(() => { + if (terminal === 'end') FakeRecognition.last?.onend?.(); + if (terminal === 'aborted') FakeRecognition.last?.onerror?.({ error: 'aborted' }); + if (terminal === 'startup') vi.advanceTimersByTime(8_000); + }); + if (terminal === 'grace') { + fireEvent.click(screen.getByRole('button', { name: 'Stop recording' })); + act(() => vi.advanceTimersByTime(10_000)); + } + expect(screen.getByRole('alert')).toHaveTextContent(terminal === 'aborted' ? 'Voice input stopped unexpectedly' + : terminal === 'startup' ? 'Your recording could not be turned into text' : 'No text was captured'); + expect(screen.getByLabelText('Your response')).toHaveValue('typed answer'); + expect(screen.getByRole('button', { name: 'Send' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Start voice input' })).toBeEnabled(); + expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); + }); + it('a blocked microphone says so and leaves typing available', async () => { seed('browser'); render(); @@ -132,6 +206,24 @@ describe('device mic and consent', () => { expect(screen.getByLabelText('あなたの回答')).toHaveValue('入力できます'); }); + it('announces ready after installation without opening the microphone or blocking Send', async () => { + sessionStorage.clear(); + FakeRecognition.available.mockResolvedValue('downloadable'); + FakeRecognition.install.mockResolvedValue(true); + seed('device', { id: 'ready-study', interviewLanguages: ['fr'] }); + useStore.setState({ participantLanguage: 'fr' }); + render(); + fireEvent.change(screen.getByLabelText('Votre réponse'), { target: { value: 'Ma réponse' } }); + fireEvent.click(await screen.findByRole('button', { name: 'Démarrer la saisie vocale' })); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent('La saisie vocale est prête. Appuyez sur le micro pour parler.')); + expect(FakeRecognition.last).toBeNull(); + expect(screen.getByRole('button', { name: 'Envoyer' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Démarrer la saisie vocale' })).toHaveAttribute('aria-pressed', 'false'); + fireEvent.click(screen.getByRole('button', { name: 'Démarrer la saisie vocale' })); + expect(FakeRecognition.last?.processLocally).toBe(true); + expect(screen.getByRole('status')).toHaveTextContent('Démarrage du micro…'); + }); + it.each([ ['ja', /録音はこのコンピューターの外に出ないとしています/], ['zh', /浏览器会在这台电脑上将您的语音转为文字/], From 52c07e6f3e4ec44b8e818b5e022271639d5544b6 Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:32:45 +0200 Subject: [PATCH 3/5] fix(voice): review fixes: CJK joining, stopping label, quieter preview Kimi review of the lifecycle change (no blockers): - zh/ja dictated text is joined without spaces, both interim fragments and successive finals appended to the answer (the latter predates this branch). - While stopping, the mic button keeps "Stop" and aria-pressed, disabled. - The live preview is no longer an aria-live region (it re-announced the whole growing text on every result). - A terminal device error keeps already-recognized interim text, like every other error path. - zh "preparing" uses the CJK ellipsis like the other zh voice strings; unused onspeechstart removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/InterviewChat.tsx | 10 ++++----- src/lib/i18n/messages/zh.ts | 2 +- src/lib/voice/useVoiceInput.ts | 9 ++++++-- tests/unit/InterviewChat.voice.test.tsx | 10 +++++---- tests/unit/useVoiceInput.device.test.tsx | 28 +++++++++++++++++------- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/components/InterviewChat.tsx b/src/components/InterviewChat.tsx index 7261f31..6e240e3 100644 --- a/src/components/InterviewChat.tsx +++ b/src/components/InterviewChat.tsx @@ -13,7 +13,7 @@ import { Button, Turn } from '@/components/ui'; import NoSessionNotice from '@/components/NoSessionNotice'; import { useParticipantLanguage } from '@/lib/i18n/useParticipantLanguage'; import { hasLanguageSetting } from '@/lib/i18n/languages'; -import { useVoiceInput } from '@/lib/voice/useVoiceInput'; +import { speechJoiner, useVoiceInput } from '@/lib/voice/useVoiceInput'; // Primitive props keep completed turns from reparsing Markdown when the // composer changes or another message arrives. @@ -67,7 +67,7 @@ const InterviewChat: React.FC = () => { researcherPreview: viewMode === 'preview', participantSessionHandle, // Text is added to the answer box for the participant to check; never sent. - onText: (text) => setInput((current) => (current.trimEnd() ? `${current.trimEnd()} ${text}` : text)), + onText: (text) => setInput((current) => (current.trimEnd() ? `${current.trimEnd()}${speechJoiner(shownLanguage)}${text}` : text)), }); const voiceListening = voice.state.kind === 'starting' || voice.state.kind === 'listening'; const voicePreviewVisible = voiceListening || voice.state.kind === 'stopping'; @@ -454,10 +454,10 @@ const InterviewChat: React.FC = () => { variant="quiet" onClick={voice.toggle} disabled={isAiThinking || voice.state.kind === 'transcribing' || voice.state.kind === 'preparing' || voice.state.kind === 'stopping'} - aria-pressed={voice.state.kind === 'recording' || voiceListening} + aria-pressed={voice.state.kind === 'recording' || voicePreviewVisible} className="min-h-11 w-full sm:w-auto" > - {voice.state.kind === 'recording' || voiceListening ? m.voice.stop : m.voice.start} + {voice.state.kind === 'recording' || voicePreviewVisible ? m.voice.stop : m.voice.start} {sendButton} @@ -476,7 +476,7 @@ const InterviewChat: React.FC = () => {

)} {voice.enabled && voicePreviewVisible && voice.preview && ( -

{voice.preview}

+

{voice.preview}

)} {voice.enabled && (voice.state.kind === 'idle' || voice.state.kind === 'ready') && input.trim() !== '' && (

{m.voice.review}

diff --git a/src/lib/i18n/messages/zh.ts b/src/lib/i18n/messages/zh.ts index b874f31..8f8b1d0 100644 --- a/src/lib/i18n/messages/zh.ts +++ b/src/lib/i18n/messages/zh.ts @@ -101,7 +101,7 @@ export const zh: Messages = { starting: '正在启动麦克风……', noText: '没有识别到文字。请重试，或直接输入您的回答。', interrupted: '语音输入意外中断。请重试，或直接输入您的回答。', - preparing: '正在这台设备上准备语音输入…', + preparing: '正在这台设备上准备语音输入……', deviceUnavailable: '这台设备无法使用语音输入。请直接输入您的回答。', start: '开始语音输入', stop: '停止录音', diff --git a/src/lib/voice/useVoiceInput.ts b/src/lib/voice/useVoiceInput.ts index 0e9d05c..4ae7309 100644 --- a/src/lib/voice/useVoiceInput.ts +++ b/src/lib/voice/useVoiceInput.ts @@ -30,6 +30,11 @@ export const SPEECH_TAGS: Record = { en: 'en-US', zh: 'zh-CN', fr: 'fr-FR', ja: 'ja-JP', ko: 'ko-KR', es: 'es-ES', }; +/** Chinese and Japanese text is not space-separated. */ +export function speechJoiner(language: InterviewLanguage): string { + return language === 'zh' || language === 'ja' ? '' : ' '; +} + type SpeechRecognitionLike = { lang: string; processLocally?: boolean; @@ -39,7 +44,6 @@ type SpeechRecognitionLike = { onerror: ((event: { error: string }) => void) | null; onend: (() => void) | null; onaudiostart: (() => void) | null; - onspeechstart: (() => void) | null; onnomatch: (() => void) | null; start(): void; stop(): void; @@ -266,7 +270,7 @@ export function useVoiceInput(options: { let startupTimer: ReturnType | undefined; let stopTimer: ReturnType | undefined; const active = () => mounted.current && recognition.current === session; - const previewText = () => [...interim.entries()].sort(([a], [b]) => a - b).map(([, text]) => text).filter(Boolean).join(' '); + const previewText = () => [...interim.entries()].sort(([a], [b]) => a - b).map(([, text]) => text).filter(Boolean).join(speechJoiner(language)); const release = (abort: boolean) => { clearTimeout(startupTimer); clearTimeout(stopTimer); @@ -325,6 +329,7 @@ export function useVoiceInput(options: { session.onerror = (event) => { if (!active()) return; if (mode === 'device' && ['language-not-supported', 'service-not-allowed'].includes(event.error)) { + keepInterim(); disableDevice(); return; } diff --git a/tests/unit/InterviewChat.voice.test.tsx b/tests/unit/InterviewChat.voice.test.tsx index 955fa97..fbe044f 100644 --- a/tests/unit/InterviewChat.voice.test.tsx +++ b/tests/unit/InterviewChat.voice.test.tsx @@ -26,7 +26,6 @@ class FakeRecognition { onerror: ((event: { error: string }) => void) | null = null; onend: (() => void) | null = null; onaudiostart: (() => void) | null = null; - onspeechstart: (() => void) | null = null; onnomatch: (() => void) | null = null; start = vi.fn(() => { FakeRecognition.last = this; }); stop = vi.fn(); @@ -80,7 +79,7 @@ describe('participant voice input', () => { fireEvent.click(screen.getByRole('button', { name: '録音を停止' })); expect(screen.getByRole('button', { name: '送信' })).toBeDisabled(); act(() => FakeRecognition.last?.onend?.()); - await waitFor(() => expect(textarea.value).toBe('私は 研究者です')); + await waitFor(() => expect(textarea.value).toBe('私は研究者です')); expect(screen.getByRole('button', { name: '送信' })).toBeEnabled(); expect(screen.getByText('送信する前に文字をご確認ください。')).toBeInTheDocument(); expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); @@ -109,18 +108,21 @@ describe('participant voice input', () => { act(() => FakeRecognition.last?.onresult?.({ resultIndex: 0, results: [{ isFinal: false, 0: { transcript: '临时文字' } }] })); const preview = screen.getByText('临时文字'); expect(preview).toHaveAttribute('lang', 'zh-CN'); - expect(preview).toHaveAttribute('aria-live', 'polite'); + expect(preview).not.toHaveAttribute('aria-live'); expect(preview).toHaveClass('text-ink-500'); expect(textarea).toHaveValue('手动输入'); expect(sessionStorage.getItem(RESEARCH_STORE_KEY)).toBe(storedBefore); expect(useStore.getState().interviewHistory).toBe(historyBefore); fireEvent.click(screen.getByRole('button', { name: '停止录音' })); + // While Chrome finalizes, the button still reads Stop (pressed) but is disabled. + expect(screen.getByRole('button', { name: '停止录音' })).toHaveAttribute('aria-pressed', 'true'); + expect(screen.getByRole('button', { name: '停止录音' })).toBeDisabled(); fireEvent.keyDown(textarea, { key: 'Enter', ctrlKey: true }); expect(interviewApiMock.generateInterviewResponse).not.toHaveBeenCalled(); expect(screen.getByText('临时文字')).toBeInTheDocument(); expect(screen.getByRole('button', { name: '发送' })).toBeDisabled(); act(() => FakeRecognition.last?.onend?.()); - expect(textarea).toHaveValue('手动输入 临时文字'); + expect(textarea).toHaveValue('手动输入临时文字'); expect(screen.queryByText('临时文字')).not.toBeInTheDocument(); expect(screen.getByText('发送前请检查文字。')).toBeInTheDocument(); expect(screen.getByRole('button', { name: '发送' })).toBeEnabled(); diff --git a/tests/unit/useVoiceInput.device.test.tsx b/tests/unit/useVoiceInput.device.test.tsx index 6380069..073142f 100644 --- a/tests/unit/useVoiceInput.device.test.tsx +++ b/tests/unit/useVoiceInput.device.test.tsx @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { act, cleanup, renderHook, waitFor } from '@testing-library/react'; -import { SPEECH_TAGS, useVoiceInput } from '@/lib/voice/useVoiceInput'; +import { SPEECH_TAGS, speechJoiner, useVoiceInput } from '@/lib/voice/useVoiceInput'; import type { InterviewLanguage } from '@/lib/i18n/languages'; class Recognition { @@ -15,7 +15,6 @@ class Recognition { onerror: ((event: { error: string }) => void) | null = null; onend: (() => void) | null = null; onaudiostart: (() => void) | null = null; - onspeechstart: (() => void) | null = null; onnomatch: (() => void) | null = null; start = vi.fn(() => { // Every started instance must satisfy the local-only contract, including retries. @@ -221,6 +220,22 @@ describe('device-only speech input', () => { }); }); +describe('speechJoiner', () => { + it('joins Chinese and Japanese without spaces, other languages with one', () => { + expect(['zh', 'ja', 'ko', 'en', 'fr', 'es'].map(l => speechJoiner(l as InterviewLanguage))).toEqual(['', '', ' ', ' ', ' ', ' ']); + }); + + it('commits zh interim fragments without a space', async () => { + const { result } = renderHook(() => useVoiceInput(options('zh'))); + await act(async () => {}); + act(() => result.current.toggle()); + const session = Recognition.sessions[0]; + act(() => session.result([['你好', false], ['世界', false]])); + act(() => session.onend?.()); + expect(onText).toHaveBeenCalledExactlyOnceWith('你好世界'); + }); +}); + describe('shared speech lifecycle', () => { async function start() { const hook = mount(); @@ -230,12 +245,10 @@ describe('shared speech lifecycle', () => { return { ...hook, session: Recognition.sessions[0] }; } - it('starts once, and waits for audio rather than speech before reporting listening', async () => { + it('starts once, and waits for audio before reporting listening', async () => { const { result, session } = await start(); expect(result.current.state).toEqual({ kind: 'starting' }); expect(session.interimResults).toBe(true); - act(() => session.onspeechstart?.()); - expect(result.current.state.kind).toBe('starting'); act(() => session.onaudiostart?.()); expect(result.current.state).toEqual({ kind: 'listening' }); }); @@ -336,6 +349,7 @@ describe('shared speech lifecycle', () => { act(() => session.result([['你好', false], ['世界', false]])); expect(onText).not.toHaveBeenCalled(); act(() => { session.onend?.(); session.onend?.(); session.result([['late', true]]); }); + // The test hook speaks English; see the joiner test for zh/ja. expect(onText).toHaveBeenCalledExactlyOnceWith('你好 世界'); expect(result.current.preview).toBe(''); expect(result.current.state.kind).toBe('idle'); @@ -420,9 +434,7 @@ describe('shared speech lifecycle', () => { expect(vi.getTimerCount()).toBe(0); act(() => session.onend?.()); expect(result.current.state.kind).toBe('error'); - // A terminal device failure drops the draft; other errors keep it for review. - if (error === 'language-not-supported') expect(onText).not.toHaveBeenCalled(); - else expect(onText).toHaveBeenCalledExactlyOnceWith('draft'); + expect(onText).toHaveBeenCalledExactlyOnceWith('draft'); }); it.each(['starting', 'stopping'])('aborts and clears timers on unmount while %s without committing', async phase => { From bd0f6689f7437835bf48f2643f6a9ff3513a4989 Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:32:45 +0200 Subject: [PATCH 4/5] release: 5.4.2 Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 4 ++-- docs/releases/v5.4.2.md | 33 +++++++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 docs/releases/v5.4.2.md diff --git a/README.md b/README.md index e9bca64..17d25d3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ OpenInterviewer is an open-source platform for adaptive, AI-assisted qualitative interviews. Researchers configure a study, share an opaque participant link, and review transcripts and synthesis in a dashboard. An AI interviewer conducts each interview, and the transcript is saved before any analysis runs. -The latest release is 5.4.1, which fixes how Chinese and Japanese dashes display. Version 5.4.0 added on-device voice input and analysis files in the study export. Read the [5.4.1](docs/releases/v5.4.1.md) and [5.4.0](docs/releases/v5.4.0.md) release notes, or [all release notes](docs/releases/). +The latest release is 5.4.2, which makes voice input show what it is doing and never fail silently. Version 5.4.1 fixed how Chinese and Japanese dashes display, and 5.4.0 added on-device voice input and analysis files in the study export. Read the [5.4.2](docs/releases/v5.4.2.md), [5.4.1](docs/releases/v5.4.1.md) and [5.4.0](docs/releases/v5.4.0.md) release notes, or [all release notes](docs/releases/). The project's own instance is [open-interview.linxule.com](https://open-interview.linxule.com). It is a standalone installation on Cloudflare, with an EU-jurisdiction workspace and provider calls through its own Cloudflare AI Gateway. The former address, `openinterviewer.vercel.app`, redirects there. @@ -133,7 +133,7 @@ For contributors: - [Development and verification](docs/development.md): checks, test lanes, live-provider smoke test, releases, provider API and model contract, and project structure - [Contributing guide](CONTRIBUTING.md) and [agent and architecture guide](AGENTS.md) -- [Release notes](docs/releases/), latest [5.4.1](docs/releases/v5.4.1.md) +- [Release notes](docs/releases/), latest [5.4.2](docs/releases/v5.4.2.md) - [Translation review, October 2026](docs/translations/REVIEW-2026-10.md) ## Acknowledgments diff --git a/docs/releases/v5.4.2.md b/docs/releases/v5.4.2.md new file mode 100644 index 0000000..4245b53 --- /dev/null +++ b/docs/releases/v5.4.2.md @@ -0,0 +1,33 @@ +# OpenInterviewer 5.4.2 + +Voice input now tells participants what is happening and never leaves them +talking into silence. This mainly affects on-device voice input (added in +5.4.0), where Chrome's recognizer could record without returning any text or +error. + +## What changed + +- **Ready.** After Chrome downloads its speech pack, the participant sees + "Voice input is ready. Press the microphone to speak." The microphone still + never opens on its own. +- **Starting.** "Starting the microphone…" shows until the browser is actually + capturing sound. "Listening…" then means it really is. +- **Live text.** Words appear under the answer box as they are recognized. They + stay on the participant's screen and are not saved or sent. +- **Unfinished speech is kept.** If the browser recognizes speech but never + marks it finished, that text goes into the answer box when the participant + stops. As before, nothing is sent until the participant presses Send. +- **No more silent failures.** If no text was recognized, or the browser stopped + listening unexpectedly, the participant is told and can try again or type. + A stuck recognizer is released after a few seconds, so Send never stays + disabled. +- **Chinese and Japanese.** Dictated text is no longer joined with spaces. +- The new messages are in all six interview languages. + +On-device voice input still runs only in the browser on the participant's +computer. Nothing about where speech is processed has changed. + +## Upgrade and rollback + +A code-only release, with no schema, configuration or binding changes. Rolling +back to 5.4.1 or 5.4.0 is safe. diff --git a/package-lock.json b/package-lock.json index d62aff8..9084915 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "openinterviewer", - "version": "5.4.1", + "version": "5.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openinterviewer", - "version": "5.4.1", + "version": "5.4.2", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.131.0", diff --git a/package.json b/package.json index b11bdf6..ce09f05 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openinterviewer", - "version": "5.4.1", + "version": "5.4.2", "description": "Open-source platform for adaptive, AI-assisted qualitative interviews.", "private": true, "license": "MIT", From b41bc02e04c7ddf46cb7734bc66503c15a134d2f Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:34:27 +0200 Subject: [PATCH 5/5] fix(voice): start the start-up deadline only once mic access is granted Chrome captures no audio while its "Allow microphone" prompt is open, so a participant who read it for more than 8 s got a false "could not be turned into text" on first use (side-agent review). The deadline now waits for navigator.permissions to report microphone 'granted' (immediately, or via onchange); without the Permissions API there is no deadline and Stop still releases the session. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/voice/useVoiceInput.ts | 29 +++++++++++++++++++---- tests/unit/InterviewChat.voice.test.tsx | 2 ++ tests/unit/useVoiceInput.device.test.tsx | 30 ++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/lib/voice/useVoiceInput.ts b/src/lib/voice/useVoiceInput.ts index 4ae7309..2651140 100644 --- a/src/lib/voice/useVoiceInput.ts +++ b/src/lib/voice/useVoiceInput.ts @@ -269,11 +269,14 @@ export function useVoiceInput(options: { let stopping = false; let startupTimer: ReturnType | undefined; let stopTimer: ReturnType | undefined; + let audioStarted = false; + let permission: PermissionStatus | null = null; const active = () => mounted.current && recognition.current === session; const previewText = () => [...interim.entries()].sort(([a], [b]) => a - b).map(([, text]) => text).filter(Boolean).join(speechJoiner(language)); const release = (abort: boolean) => { clearTimeout(startupTimer); clearTimeout(stopTimer); + if (permission) permission.onchange = null; recognition.current = null; speechCleanup.current = null; speechStop.current = null; @@ -301,6 +304,7 @@ export function useVoiceInput(options: { }; session.onaudiostart = () => { if (!active()) return; + audioStarted = true; clearTimeout(startupTimer); if (!stopping) setState({ kind: 'listening' }); }; @@ -353,11 +357,14 @@ export function useVoiceInput(options: { setPreview(''); setSpeechTag(session.lang); setState({ kind: 'starting' }); - startupTimer = setTimeout(() => { - if (!active()) return; - release(true); - setState({ kind: 'error', reason: 'failed' }); - }, 8_000); + const armStartupDeadline = () => { + if (!active() || stopping || audioStarted || startupTimer !== undefined) return; + startupTimer = setTimeout(() => { + if (!active()) return; + release(true); + setState({ kind: 'error', reason: 'failed' }); + }, 8_000); + }; try { session.start(); } catch { @@ -366,7 +373,19 @@ export function useVoiceInput(options: { release(true); setState({ kind: 'error', reason: 'failed' }); } + return; } + // Capture waits for the browser's permission prompt, so the start-up + // deadline runs only once microphone access is granted. Without the + // Permissions API there is no deadline; Stop still releases the session. + try { + navigator.permissions?.query({ name: 'microphone' as PermissionName }).then((status) => { + if (!active()) return; + permission = status; + if (status.state === 'granted') armStartupDeadline(); + else status.onchange = () => { if (status.state === 'granted') armStartupDeadline(); }; + }, () => { /* Unsupported permission name. */ }); + } catch { /* Unsupported permission name. */ } }, [language, mode, disableDevice]); const prepareDevice = useCallback(async () => { diff --git a/tests/unit/InterviewChat.voice.test.tsx b/tests/unit/InterviewChat.voice.test.tsx index fbe044f..2cf064a 100644 --- a/tests/unit/InterviewChat.voice.test.tsx +++ b/tests/unit/InterviewChat.voice.test.tsx @@ -46,6 +46,7 @@ function seed(voiceInput?: VoiceInputMode, extra: Parameters { + Object.defineProperty(navigator, 'permissions', { configurable: true, value: { query: vi.fn(async () => ({ state: 'granted', onchange: null })) } }); vi.clearAllMocks(); FakeRecognition.last = null; vi.stubGlobal('webkitSpeechRecognition', FakeRecognition); @@ -135,6 +136,7 @@ describe('participant voice input', () => { render(); fireEvent.change(screen.getByLabelText('Your response'), { target: { value: 'typed answer' } }); fireEvent.click(screen.getByRole('button', { name: 'Start voice input' })); + await act(async () => {}); // permission query expect(screen.getByRole('button', { name: 'Send' })).toBeDisabled(); act(() => { if (terminal === 'end') FakeRecognition.last?.onend?.(); diff --git a/tests/unit/useVoiceInput.device.test.tsx b/tests/unit/useVoiceInput.device.test.tsx index 073142f..bf8c577 100644 --- a/tests/unit/useVoiceInput.device.test.tsx +++ b/tests/unit/useVoiceInput.device.test.tsx @@ -34,6 +34,7 @@ function options(language: InterviewLanguage = 'en') { participantSessionHandle: `device-session-${sessionNumber}`, onText }; } const mount = () => renderHook(() => useVoiceInput(options())); +let micPermission: { state: PermissionState; onchange: (() => void) | null }; beforeEach(() => { sessionNumber += 1; @@ -44,6 +45,8 @@ beforeEach(() => { vi.stubGlobal('SpeechRecognition', Recognition); vi.stubGlobal('webkitSpeechRecognition', undefined); vi.stubGlobal('fetch', vi.fn()); + micPermission = { state: 'granted', onchange: null }; + Object.defineProperty(navigator, 'permissions', { configurable: true, value: { query: vi.fn(async () => micPermission) } }); }); afterEach(() => { cleanup(); vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -242,6 +245,7 @@ describe('shared speech lifecycle', () => { await act(async () => {}); expect(hook.result.current.supported).toBe(true); act(() => hook.result.current.toggle()); + await act(async () => {}); // permission query return { ...hook, session: Recognition.sessions[0] }; } @@ -282,6 +286,32 @@ describe('shared speech lifecycle', () => { expect(onText).not.toHaveBeenCalled(); }); + it('waits for the permission prompt before starting the deadline', async () => { + vi.useFakeTimers(); + micPermission = { state: 'prompt', onchange: null }; + const { result } = await start(); + act(() => vi.advanceTimersByTime(30_000)); + expect(result.current.state.kind).toBe('starting'); + expect(vi.getTimerCount()).toBe(0); + micPermission.state = 'granted'; + act(() => micPermission.onchange?.()); + act(() => vi.advanceTimersByTime(7_999)); + expect(result.current.state.kind).toBe('starting'); + act(() => vi.advanceTimersByTime(1)); + expect(result.current.state).toEqual({ kind: 'error', reason: 'failed' }); + expect(micPermission.onchange).toBeNull(); + }); + + it('has no start-up deadline without the Permissions API', async () => { + vi.useFakeTimers(); + Object.defineProperty(navigator, 'permissions', { configurable: true, value: undefined }); + const { result } = await start(); + act(() => vi.advanceTimersByTime(30_000)); + expect(result.current.state.kind).toBe('starting'); + act(() => result.current.toggle()); + expect(result.current.state.kind).toBe('stopping'); + }); + it('clears the startup timer and disables device mode when start throws', async () => { vi.useFakeTimers(); class CannotStart extends Recognition {