From 749bd5245d1c69f767a76c92df275cdb764434b6 Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:30:48 +0200 Subject: [PATCH 1/6] feat(voice): accept revision-bound device mode across study configuration 'device' is accepted wherever 'browser' is: validation, drafts, hosted rules and follow-up copies. /api/transcribe still accepts only 'installation'. Changing the mode advances the study revision (shared contract scenario). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/researcherStudyDraft.ts | 2 +- src/lib/studyConfigValidation.ts | 2 +- src/types.ts | 3 ++- tests/contract/workspaceStoreScenarios.ts | 27 ++++++++++++++++++++ tests/unit/api.followup.provenance.test.ts | 7 +++++ tests/unit/api.studies.cloudflare.test.ts | 16 ++++++++++++ tests/unit/api.study.createOwnership.test.ts | 17 ++++++++++-- tests/unit/studyConfigValidation.test.ts | 23 +++++++++++++++++ tests/unit/voiceInput.test.ts | 2 +- 9 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/lib/researcherStudyDraft.ts b/src/lib/researcherStudyDraft.ts index 56ab348..874309f 100644 --- a/src/lib/researcherStudyDraft.ts +++ b/src/lib/researcherStudyDraft.ts @@ -48,7 +48,7 @@ export function copyStudyConfiguration(config: Partial): StudyConfi thankYouText: config.thankYouText ?? '', interviewerInstructions: config.interviewerInstructions ?? '', ...draftLanguageMembers(config), - ...(config.voiceInput === 'installation' || config.voiceInput === 'browser' ? { voiceInput: config.voiceInput } : {}), + ...(config.voiceInput === 'installation' || config.voiceInput === 'browser' || config.voiceInput === 'device' ? { voiceInput: config.voiceInput } : {}), }; } diff --git a/src/lib/studyConfigValidation.ts b/src/lib/studyConfigValidation.ts index e2d0dfa..83b8d94 100644 --- a/src/lib/studyConfigValidation.ts +++ b/src/lib/studyConfigValidation.ts @@ -272,7 +272,7 @@ export function validateStudyConfig(value: unknown): ValidationResult { } const languages = validateInterviewLanguages(value); if (!languages.ok) return languages; - if (value.voiceInput !== undefined && !['off', 'installation', 'browser'].includes(value.voiceInput as string)) { + if (value.voiceInput !== undefined && !['off', 'installation', 'browser', 'device'].includes(value.voiceInput as string)) { return { ok: false, error: 'Invalid voice input setting' }; } if (value.linksEnabled !== undefined && typeof value.linksEnabled !== 'boolean') { diff --git a/src/types.ts b/src/types.ts index 00ffd83..bc3da98 100644 --- a/src/types.ts +++ b/src/types.ts @@ -122,9 +122,10 @@ export type ProviderCommitment = 'fixed' | 'may-change'; * Participant voice input. 'installation': recordings are transcribed by this * installation's Cloudflare account (Workers AI) and not stored. 'browser': * the browser's own speech service (Google in Chrome, Apple in Safari). + * 'device': the browser's on-device recognition, with no server fallback. * Absent or 'off': no microphone. The consent notice names the processor. */ -export type VoiceInputMode = 'off' | 'installation' | 'browser'; +export type VoiceInputMode = 'off' | 'installation' | 'browser' | 'device'; export interface StudyConfig { id: string; diff --git a/tests/contract/workspaceStoreScenarios.ts b/tests/contract/workspaceStoreScenarios.ts index 006660e..cccefbf 100644 --- a/tests/contract/workspaceStoreScenarios.ts +++ b/tests/contract/workspaceStoreScenarios.ts @@ -460,6 +460,33 @@ export function defineWorkspaceStoreContract(label: string, harness: WorkspaceSt }); }); + it('device voice is revision-bound: an unchanged save is stable and switching modes fences old consent and completion', async () => { + const store = await harness.createStore(); + const study = await createStudy(store); + const { link } = await createLink(store, study); + const participant = await consentedParticipant(store, study); + const input = completion(study, participant, link.id); + const updated = expectStatus(await store.replaceStudyConfig({ + studyId: study.id, expectedRevision: 1, + config: { ...study.config, voiceInput: 'device' }, now: Date.now(), + }), 'updated').study; + expect(updated.revision).toBe(2); + expect(updated.config.voiceInput).toBe('device'); + expect((await store.verifyConsent({ participantSessionId: participant.sessionId, + studyId: study.id, studyRevision: updated.revision, + consentText: updated.config.consentText, now: Date.now() })).status).toBe('mismatch'); + expect((await store.persistCompletedInterview(input)).status).toBe('revision-stale'); + const unchanged = expectStatus(await store.replaceStudyConfig({ + studyId: study.id, expectedRevision: 2, config: updated.config, now: Date.now(), + }), 'updated').study; + expect(unchanged.revision).toBe(2); + const switched = expectStatus(await store.replaceStudyConfig({ + studyId: study.id, expectedRevision: 2, + config: { ...updated.config, voiceInput: 'browser' }, now: Date.now(), + }), 'updated').study; + expect(switched.revision).toBe(3); + }); + it('ST-01: replacing the config advances the revision and a stale expected revision conflicts', async () => { const store = await harness.createStore(); const study = await createStudy(store); diff --git a/tests/unit/api.followup.provenance.test.ts b/tests/unit/api.followup.provenance.test.ts index 11ea77c..19a7ff1 100644 --- a/tests/unit/api.followup.provenance.test.ts +++ b/tests/unit/api.followup.provenance.test.ts @@ -126,6 +126,13 @@ afterEach(() => { }); describe('follow-up synthesis provenance', () => { + it('copies device voice from the canonical parent into a generated follow-up', async () => { + parentStudy.config.voiceInput = 'device'; + const response = await POST(request(), { params: Promise.resolve({ id: parentStudy.id }) }); + expect(response.status).toBe(200); + expect((await response.json()).followUpConfig.voiceInput).toBe('device'); + }); + async function scopedAggregate() { const sources = [ makeStoredInterview({ id: 'interview-a', studyId: parentStudy.id, studyRevision: 2, synthesis: {} as never }), diff --git a/tests/unit/api.studies.cloudflare.test.ts b/tests/unit/api.studies.cloudflare.test.ts index e2d10f8..eb507c9 100644 --- a/tests/unit/api.studies.cloudflare.test.ts +++ b/tests/unit/api.studies.cloudflare.test.ts @@ -429,6 +429,22 @@ describe('/api/studies/[id] on Cloudflare (ST-01, ST-03)', () => { expect(store.replaceStudyConfig).not.toHaveBeenCalled(); }); + it('PUT carries device mode through validation into a revision-guarded replacement', async () => { + const study = studyAt(5); + store.readiness.mockResolvedValue(ready()); + store.getStudy.mockResolvedValue({ status: 'found', study }); + store.replaceStudyConfig.mockResolvedValue({ status: 'updated', study: { + ...study, revision: 6, config: { ...study.config, voiceInput: 'device' }, + } }); + const response = await updateStudy(jsonRequest('http://localhost', 'PUT', { + config: { voiceInput: 'device' }, expectedRevision: 5, + }), params); + expect(response.status).toBe(200); + expect(store.replaceStudyConfig).toHaveBeenCalledWith(expect.objectContaining({ + expectedRevision: 5, config: expect.objectContaining({ voiceInput: 'device' }), + })); + }); + it('ST-03: PUT replaces the config against the loaded revision and keeps the soft lock', async () => { const study = studyAt(5, { interviewCount: 2 }); store.readiness.mockResolvedValue(ready()); diff --git a/tests/unit/api.study.createOwnership.test.ts b/tests/unit/api.study.createOwnership.test.ts index 2c23377..a9ffd22 100644 --- a/tests/unit/api.study.createOwnership.test.ts +++ b/tests/unit/api.study.createOwnership.test.ts @@ -85,13 +85,13 @@ function v2CreateOperation(studyId: string, researcherId: string) { }; } -const request = () => new Request('http://localhost/api/studies', { +const request = (voiceInput?: 'browser' | 'device' | 'installation') => new Request('http://localhost/api/studies', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Idempotency-Key': IDEMPOTENCY_KEY, }, - body: JSON.stringify({ config: makeStudyConfig() }), + body: JSON.stringify({ config: makeStudyConfig({ ...(voiceInput ? { voiceInput } : {}) }) }), }); beforeEach(() => { @@ -182,6 +182,19 @@ beforeEach(() => { }); describe('hosted study creation ownership saga', () => { + it.each(['browser', 'device'] as const)('hosted creation accepts %s without server transcription capability', async voiceInput => { + modeMock.isHostedMode.mockReturnValue(true); + const response = await POST(request(voiceInput)); + expect(response.status).toBe(200); + expect(kvMock.createStudyAtomic.mock.calls[0][0].config.voiceInput).toBe(voiceInput); + }); + + it('hosted creation still refuses installation transcription', async () => { + modeMock.isHostedMode.mockReturnValue(true); + expect((await POST(request('installation'))).status).toBe(400); + expect(kvMock.createStudyAtomic).not.toHaveBeenCalled(); + }); + it('does not create researcher data when the durable operation cannot begin', async () => { modeMock.isHostedMode.mockReturnValue(true); platformMock.beginCreateStudyOperationV2.mockResolvedValue({ status: 'unavailable' }); diff --git a/tests/unit/studyConfigValidation.test.ts b/tests/unit/studyConfigValidation.test.ts index 6d50a25..4503963 100644 --- a/tests/unit/studyConfigValidation.test.ts +++ b/tests/unit/studyConfigValidation.test.ts @@ -8,6 +8,8 @@ import { } from '@/lib/studyConfigValidation'; import { CONSENT_TEXT_PLACEHOLDER_ERROR } from '@/lib/consentText'; import { THANK_YOU_TEXT_PLACEHOLDER_ERROR } from '@/lib/thankYouText'; +import { copyStudyConfiguration } from '@/lib/researcherStudyDraft'; +import { studyConfigEqual } from '@/lib/studyConfigEquality'; import { makeStudyConfig } from '../fixtures/models'; describe('validateStudyConfig', () => { @@ -362,3 +364,24 @@ describe('interviewerInstructions (Slice Q)', () => { .toMatchObject({ ok: true, config: { [field]: 'Existing text.' } }); }); }); + + +describe('voice input configuration', () => { + it.each(['off', 'installation', 'browser', 'device'] as const)('accepts %s on canonical, create and update paths', voiceInput => { + const config = makeStudyConfig({ voiceInput }); + expect(validateStudyConfig(config)).toMatchObject({ ok: true, config: { voiceInput } }); + expect(validateStudyConfigForCreate(config, { id: config.id, createdAt: config.createdAt })).toMatchObject({ ok: true, config: { voiceInput } }); + expect(validateStudyConfigUpdate(makeStudyConfig(), { voiceInput }, undefined)).toMatchObject({ ok: true, config: { voiceInput } }); + }); + it.each(['cloud', '', null, true, {}, []])('fails closed for an unknown voice input %j', voiceInput => { + expect(validateStudyConfig({ ...makeStudyConfig(), voiceInput })).toEqual({ ok: false, error: 'Invalid voice input setting' }); + expect(validateStudyConfigUpdate(makeStudyConfig(), { voiceInput }, undefined)).toMatchObject({ ok: false }); + }); + it('preserves device mode across researcher draft copies and treats changing it as a protocol change', () => { + const config = makeStudyConfig({ voiceInput: 'device' }); + expect(copyStudyConfiguration(config).voiceInput).toBe('device'); + expect(studyConfigEqual(config, { ...config })).toBe(true); + expect(studyConfigEqual(config, { ...config, voiceInput: 'browser' })).toBe(false); + expect(studyConfigEqual(config, { ...config, voiceInput: 'off' })).toBe(false); + }); +}); diff --git a/tests/unit/voiceInput.test.ts b/tests/unit/voiceInput.test.ts index 4825612..29bd71e 100644 --- a/tests/unit/voiceInput.test.ts +++ b/tests/unit/voiceInput.test.ts @@ -211,7 +211,7 @@ describe('POST /api/transcribe', () => { it('refuses a study that does not offer transcription by this installation', async () => { withAiBinding(); - for (const voiceInput of [undefined, 'off', 'browser'] as const) { + for (const voiceInput of [undefined, 'off', 'browser', 'device'] as const) { kvMock.getStudy.mockResolvedValue(makeStoredStudy({ id: 'study-a', config: { ...voiceConfig, voiceInput } })); const res = await transcribePOST(transcribeRequest()); expect(res.status).toBe(403); From 0571ea09e3d47675588177e7d01cb62aba12ea77 Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:30:48 +0200 Subject: [PATCH 2/6] feat(voice): on-device dictation with Chrome local recognition, never the cloud Device mode gates the mic on SpeechRecognition.available({processLocally}), installs the language pack on first press while typing stays available, and starts only recognizers that have the processLocally attribute and accept true. language-not-supported and service-not-allowed hide the mic for the session. There is no fallback to cloud dictation or server transcription. Consent presents locality as the browser's promise: the browser turns speech into text on this computer and says the recording stays there; the study receives only the text the participant sends. zh-CN verified as accepted by Chrome 155 (available() for all six tags). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/Consent.tsx | 2 +- src/components/InterviewChat.tsx | 3 +- .../studySetup/VoiceInputSection.tsx | 5 + src/lib/export/transcriptsMarkdown.ts | 1 + src/lib/i18n/messages/en.ts | 7 +- src/lib/i18n/messages/es.ts | 3 + src/lib/i18n/messages/fr.ts | 3 + src/lib/i18n/messages/ja.ts | 3 + src/lib/i18n/messages/ko.ts | 3 + src/lib/i18n/messages/zh.ts | 3 + src/lib/voice/useVoiceInput.ts | 164 +++++++++++-- tests/e2e/device-voice.spec.ts | 43 ++++ tests/unit/InterviewChat.voice.test.tsx | 35 +++ tests/unit/VoiceInputSection.test.tsx | 18 ++ tests/unit/api.export.markdown.test.ts | 9 + tests/unit/useVoiceInput.device.test.tsx | 215 ++++++++++++++++++ 16 files changed, 498 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/device-voice.spec.ts create mode 100644 tests/unit/VoiceInputSection.test.tsx create mode 100644 tests/unit/useVoiceInput.device.test.tsx diff --git a/src/components/Consent.tsx b/src/components/Consent.tsx index 6275c36..144ffcc 100644 --- a/src/components/Consent.tsx +++ b/src/components/Consent.tsx @@ -217,7 +217,7 @@ const Consent: React.FC = () => { {providerDisclosure} {providerCommitmentNotice ? <>{' '}{providerCommitmentNotice} : null} - {studyConfig.voiceInput === 'installation' || studyConfig.voiceInput === 'browser' + {studyConfig.voiceInput === 'installation' || studyConfig.voiceInput === 'browser' || studyConfig.voiceInput === 'device' ? <>{' '}{m.voice[studyConfig.voiceInput]} : null} {' '} diff --git a/src/components/InterviewChat.tsx b/src/components/InterviewChat.tsx index 944aaf4..5d76196 100644 --- a/src/components/InterviewChat.tsx +++ b/src/components/InterviewChat.tsx @@ -451,7 +451,7 @@ const InterviewChat: React.FC = () => { type="button" variant="quiet" onClick={voice.toggle} - disabled={isAiThinking || voice.state.kind === 'transcribing'} + disabled={isAiThinking || voice.state.kind === 'transcribing' || voice.state.kind === 'preparing'} aria-pressed={voice.state.kind === 'recording' || voice.state.kind === 'listening'} className="min-h-11 w-full sm:w-auto" > @@ -467,6 +467,7 @@ const InterviewChat: React.FC = () => { ? m.voice.recording(`${Math.floor(voice.state.seconds / 60)}:${String(voice.state.seconds % 60).padStart(2, '0')}`) : voice.state.kind === 'listening' ? m.voice.listening : voice.state.kind === 'transcribing' ? m.voice.transcribing + : voice.state.kind === 'preparing' ? m.voice.preparing : m.voice[voice.state.reason]}

)} diff --git a/src/components/studySetup/VoiceInputSection.tsx b/src/components/studySetup/VoiceInputSection.tsx index f5a07c5..3fe814f 100644 --- a/src/components/studySetup/VoiceInputSection.tsx +++ b/src/components/studySetup/VoiceInputSection.tsx @@ -18,6 +18,11 @@ const OPTIONS: { value: VoiceInputMode; label: string; detail: string }[] = [ label: 'Transcribed by this installation (Cloudflare Workers AI)', detail: 'Recordings of up to one minute go to Cloudflare, which already hosts the study, and are turned into text by Whisper. OpenInterviewer does not keep them; Cloudflare’s terms rule out training on them but state no retention period. Works in current browsers on phones and computers. Cloudflare bills about $0.0005 per audio minute after a free daily allowance.', }, + { + value: 'device', + label: 'On the participant’s computer (desktop Chrome only)', + detail: 'Your participant’s browser turns speech into text on their computer and says the audio stays there; the study receives only the text they send. Chrome makes that promise; this app cannot check it. The first use may download a speech pack (about 60 MB). Participants on phones, Safari and Firefox will not see the microphone and will type instead.', + }, { value: 'browser', label: 'The browser’s own dictation', diff --git a/src/lib/export/transcriptsMarkdown.ts b/src/lib/export/transcriptsMarkdown.ts index 0bd52ba..37609e1 100644 --- a/src/lib/export/transcriptsMarkdown.ts +++ b/src/lib/export/transcriptsMarkdown.ts @@ -88,6 +88,7 @@ export function voiceInputLine(interview: StoredInterview): string | null { switch (interview.collectionConfig?.voiceInput) { case 'installation': return 'Voice input offered: speech turned into text by Cloudflare Workers AI; the participant could edit the text before sending.'; case 'browser': return 'Voice input offered: speech turned into text by the participant\'s browser speech service; the participant could edit the text before sending.'; + case 'device': return 'Voice input offered: the browser turns speech into text on the participant’s computer and says the recording stays there; the study receives only the text the participant sends. The browser may first download a speech pack (about 60 MB); the participant could edit the text before sending.'; default: return null; } } diff --git a/src/lib/i18n/messages/en.ts b/src/lib/i18n/messages/en.ts index 5e65251..55a1982 100644 --- a/src/lib/i18n/messages/en.ts +++ b/src/lib/i18n/messages/en.ts @@ -62,7 +62,7 @@ export interface Messages { mayChange: string; }; /** Shown when the study offers voice input; names who turns speech into text. */ - voice: { installation: string; browser: string }; + voice: { installation: string; browser: string; device: string }; }; interview: { phases: Record; @@ -90,6 +90,8 @@ export interface Messages { recording: (elapsed: string) => string; listening: string; transcribing: string; + preparing: string; + deviceUnavailable: string; review: string; denied: string; failed: string; @@ -184,6 +186,7 @@ export const en: Messages = { mayChange: 'The researcher may later analyze your responses with a different AI provider or model.', }, voice: { + device: 'If you use the microphone, your browser turns your speech into text on this computer and says the recording stays there. This study receives only the text you choose to send, never the recording. The first time, your browser may download a speech pack (about 60 MB).', installation: 'If you use the microphone, your recording is sent to Cloudflare to be turned into text by Cloudflare Workers AI. This study does not keep the recording, and you can edit the text before sending it.', browser: 'If you use the microphone, your browser\'s speech service turns your speech into text: in Chrome this is Google, in Safari Apple, under their own terms. You can edit the text before sending it.', }, @@ -215,6 +218,8 @@ export const en: Messages = { send: 'Send', sendShortcut: '⌘/Ctrl + Enter to send', voice: { + preparing: 'Preparing speech on this device…', + deviceUnavailable: 'Voice input is unavailable on this device. Please type your answer.', start: 'Start voice input', stop: 'Stop recording', recording: (elapsed) => `Recording ${elapsed} (up to 1:00)`, diff --git a/src/lib/i18n/messages/es.ts b/src/lib/i18n/messages/es.ts index e8bd21c..6762020 100644 --- a/src/lib/i18n/messages/es.ts +++ b/src/lib/i18n/messages/es.ts @@ -65,6 +65,7 @@ export const es: Messages = { mayChange: 'La persona responsable del estudio puede analizar más adelante tus respuestas con otro proveedor o modelo de IA.', }, voice: { + device: 'Si usa el micrófono, su navegador convierte su voz en texto en este ordenador e indica que la grabación no sale de él. Este estudio solo recibe el texto que usted decida enviar, nunca la grabación. La primera vez, su navegador puede descargar un paquete de voz (unos 60 MB).', installation: 'Si usas el micrófono, tu grabación se envía a Cloudflare para que Cloudflare Workers AI la convierta en texto. Este estudio no guarda la grabación y puedes editar el texto antes de enviarlo.', browser: 'Si usas el micrófono, el servicio de voz de tu navegador convierte tu voz en texto: en Chrome lo hace Google y en Safari, Apple, según sus propias condiciones. Puedes editar el texto antes de enviarlo.', }, @@ -96,6 +97,8 @@ export const es: Messages = { send: 'Enviar', sendShortcut: '⌘/Ctrl + Intro para enviar', voice: { + preparing: 'Preparando la entrada de voz en este dispositivo…', + deviceUnavailable: 'La entrada de voz no está disponible en este dispositivo. Escriba su respuesta.', start: 'Iniciar la entrada de voz', stop: 'Detener la grabación', recording: (elapsed) => `Grabando ${elapsed} (máximo 1:00)`, diff --git a/src/lib/i18n/messages/fr.ts b/src/lib/i18n/messages/fr.ts index 868859e..e65cd3c 100644 --- a/src/lib/i18n/messages/fr.ts +++ b/src/lib/i18n/messages/fr.ts @@ -65,6 +65,7 @@ export const fr: Messages = { mayChange: 'La personne responsable de l’étude peut analyser ultérieurement vos réponses avec un autre fournisseur ou modèle d’IA.', }, voice: { + device: 'Si vous utilisez le microphone, votre navigateur transcrit votre voix en texte sur cet ordinateur et indique que l’enregistrement y reste. Cette étude ne reçoit que le texte que vous choisissez d’envoyer, jamais l’enregistrement. Lors de la première utilisation, votre navigateur peut télécharger un pack vocal (environ 60 Mo).', installation: 'Si vous utilisez le microphone, votre enregistrement est envoyé à Cloudflare pour être transcrit en texte par Cloudflare Workers AI. Cette étude ne conserve pas l’enregistrement, et vous pouvez modifier le texte avant de l’envoyer.', browser: 'Si vous utilisez le microphone, le service vocal de votre navigateur transcrit votre voix en texte : Google dans Chrome, Apple dans Safari, selon leurs propres conditions. Vous pouvez modifier le texte avant de l’envoyer.', }, @@ -96,6 +97,8 @@ export const fr: Messages = { send: 'Envoyer', sendShortcut: '⌘/Ctrl + Entrée pour envoyer', voice: { + preparing: 'Préparation de la saisie vocale sur cet appareil…', + deviceUnavailable: 'La saisie vocale n’est pas disponible sur cet appareil. Veuillez saisir votre réponse.', start: 'Démarrer la saisie vocale', stop: 'Arrêter l’enregistrement', recording: (elapsed) => `Enregistrement ${elapsed} (1:00 maximum)`, diff --git a/src/lib/i18n/messages/ja.ts b/src/lib/i18n/messages/ja.ts index bbddc7c..89e6c61 100644 --- a/src/lib/i18n/messages/ja.ts +++ b/src/lib/i18n/messages/ja.ts @@ -65,6 +65,7 @@ export const ja: Messages = { mayChange: '研究者は今後、別の AI プロバイダーやモデルを使ってあなたの回答を分析する場合があります。', }, voice: { + device: 'マイクを使う場合、ブラウザーがこのコンピューター上で音声を文字に変換し、録音はこのコンピューターの外に出ないとしています。この調査が受け取るのは、あなたが送信を選んだ文字だけで、録音は受け取りません。初回は、ブラウザーが音声パック（約60 MB）をダウンロードすることがあります。', installation: 'マイクを使用すると、録音は Cloudflare に送信され、Cloudflare Workers AI によって文字に変換されます。この研究では録音を保管せず、送信前に文字を編集できます。', browser: 'マイクを使用すると、ブラウザの音声サービスが音声を文字に変換します。Chrome では Google、Safari では Apple が、それぞれの規約に基づいて処理します。送信前に文字を編集できます。', }, @@ -96,6 +97,8 @@ export const ja: Messages = { send: '送信', sendShortcut: '⌘/Ctrl + Enter で送信', voice: { + preparing: 'このデバイスで音声入力を準備しています…', + deviceUnavailable: 'このデバイスでは音声入力を利用できません。回答を入力してください。', start: '音声入力を開始', stop: '録音を停止', recording: (elapsed) => `録音中 ${elapsed}（最長 1:00）`, diff --git a/src/lib/i18n/messages/ko.ts b/src/lib/i18n/messages/ko.ts index b5205a3..17e9824 100644 --- a/src/lib/i18n/messages/ko.ts +++ b/src/lib/i18n/messages/ko.ts @@ -65,6 +65,7 @@ export const ko: Messages = { mayChange: '연구자는 나중에 다른 AI 제공업체나 모델로 응답을 분석할 수 있습니다.', }, voice: { + device: '마이크를 사용하면 브라우저가 이 컴퓨터에서 음성을 텍스트로 변환하며, 녹음은 이 컴퓨터를 벗어나지 않는다고 안내합니다. 이 연구는 보내기로 선택한 텍스트만 받으며 녹음은 받지 않습니다. 처음 사용할 때 브라우저가 음성 팩(약 60 MB)을 다운로드할 수 있습니다.', installation: '마이크를 사용하면 녹음이 Cloudflare로 전송되어 Cloudflare Workers AI가 텍스트로 변환합니다. 이 연구는 녹음을 보관하지 않으며, 보내기 전에 텍스트를 수정할 수 있습니다.', browser: '마이크를 사용하면 브라우저의 음성 서비스가 말을 텍스트로 변환합니다. Chrome에서는 Google, Safari에서는 Apple이 각자의 약관에 따라 처리합니다. 보내기 전에 텍스트를 수정할 수 있습니다.', }, @@ -96,6 +97,8 @@ export const ko: Messages = { send: '보내기', sendShortcut: '⌘/Ctrl + Enter로 보내기', voice: { + preparing: '이 기기에서 음성 입력을 준비하고 있습니다…', + deviceUnavailable: '이 기기에서는 음성 입력을 사용할 수 없습니다. 답변을 입력해 주세요.', start: '음성 입력 시작', stop: '녹음 중지', recording: (elapsed) => `녹음 중 ${elapsed} (최대 1:00)`, diff --git a/src/lib/i18n/messages/zh.ts b/src/lib/i18n/messages/zh.ts index 63a1291..3cf4d0a 100644 --- a/src/lib/i18n/messages/zh.ts +++ b/src/lib/i18n/messages/zh.ts @@ -65,6 +65,7 @@ export const zh: Messages = { mayChange: '研究人员之后可能会使用其他 AI 服务商或模型分析您的回答。', }, voice: { + device: '如果您使用麦克风，浏览器会在这台电脑上将您的语音转为文字，并表示录音不会离开这台电脑。本研究只会收到您选择发送的文字，不会收到录音。首次使用时，浏览器可能需要下载语音包（约 60 MB）。', installation: '如果您使用麦克风，您的录音将发送至 Cloudflare，由 Cloudflare Workers AI 转换为文字。本研究不会保留录音，发送前您可以编辑文字。', browser: '如果您使用麦克风，您的浏览器的语音服务会将语音转换为文字：在 Chrome 中由 Google 提供，在 Safari 中由 Apple 提供，并适用其各自的条款。发送前您可以编辑文字。', }, @@ -96,6 +97,8 @@ export const zh: Messages = { send: '发送', sendShortcut: '按 ⌘/Ctrl + Enter 发送', voice: { + preparing: '正在为这台设备准备语音输入…', + deviceUnavailable: '这台设备无法使用语音输入。请键入您的回答。', start: '开始语音输入', stop: '停止录音', recording: (elapsed) => `正在录音 ${elapsed}（最长 1:00）`, diff --git a/src/lib/voice/useVoiceInput.ts b/src/lib/voice/useVoiceInput.ts index 0ed0974..a4b9c51 100644 --- a/src/lib/voice/useVoiceInput.ts +++ b/src/lib/voice/useVoiceInput.ts @@ -3,8 +3,8 @@ // Participant voice input (study setting voiceInput). 'installation' records // a clip of at most 60 seconds, converts it to 16 kHz WAV in the browser and // sends it to /api/transcribe (Workers AI); 'browser' uses the browser's own -// SpeechRecognition. Either way the text lands in the answer box for the -// participant to check before sending; nothing is sent automatically. +// SpeechRecognition; device mode requires local processing with no fallback. +// The text lands in the answer box to check; nothing is sent automatically. import { useCallback, useEffect, useRef, useState } from 'react'; import type { VoiceInputMode } from '@/types'; @@ -17,15 +17,19 @@ export type VoiceState = | { kind: 'recording'; seconds: number } | { kind: 'listening' } | { kind: 'transcribing' } - | { kind: 'error'; reason: 'denied' | 'failed' | 'unsupported' | 'limited' | 'unavailable' }; + | { kind: 'preparing' } + | { kind: 'error'; reason: 'denied' | 'failed' | 'unsupported' | 'limited' | 'unavailable' | 'deviceUnavailable' }; -/** Regional tags the browsers' speech services expect. */ -const SPEECH_TAGS: Record = { +/** Shared by local availability, installation and recognition. + * zh-CN is provisional: manually check Chrome against cmn-Hans-CN before release. + */ +export const SPEECH_TAGS: Record = { en: 'en-US', zh: 'zh-CN', fr: 'fr-FR', ja: 'ja-JP', ko: 'ko-KR', es: 'es-ES', }; type SpeechRecognitionLike = { lang: string; + processLocally?: boolean; continuous: boolean; interimResults: boolean; onresult: ((event: { resultIndex: number; results: ArrayLike<{ isFinal: boolean; 0: { transcript: string } }> }) => void) | null; @@ -35,11 +39,19 @@ type SpeechRecognitionLike = { stop(): void; }; -function speechRecognitionClass(): (new () => SpeechRecognitionLike) | null { +type DeviceAvailability = 'available' | 'downloadable' | 'downloading' | 'unavailable'; +type LocalSpeechOptions = { langs: string[]; processLocally: true }; +type SpeechRecognitionClass = { + new (): SpeechRecognitionLike; + available?: (options: LocalSpeechOptions) => Promise; + install?: (options: LocalSpeechOptions) => Promise; +}; + +function speechRecognitionClass(): SpeechRecognitionClass | null { if (typeof window === 'undefined') return null; const candidate = (window as unknown as Record).SpeechRecognition ?? (window as unknown as Record).webkitSpeechRecognition; - return typeof candidate === 'function' ? candidate as new () => SpeechRecognitionLike : null; + return typeof candidate === 'function' ? candidate as SpeechRecognitionClass : null; } export function voiceInputSupported(mode: VoiceInputMode | undefined): boolean { @@ -53,6 +65,15 @@ export function voiceInputSupported(mode: VoiceInputMode | undefined): boolean { return false; } +// Remember terminal device failures across interview remounts in this tab. +// Only a capability decision and non-secret session selector, never audio/text. +// Memory retains it when sessionStorage is unavailable. +const disabledDeviceSessions = new Set(); +function deviceSessionDisabled(key: string): boolean { + if (disabledDeviceSessions.has(key)) return true; + try { return sessionStorage.getItem(key) === '1'; } catch { return false; } +} + export function useVoiceInput(options: { mode: VoiceInputMode | undefined; language: InterviewLanguage; @@ -62,6 +83,7 @@ export function useVoiceInput(options: { onText: (text: string) => void; }) { const { mode, language, studyId, researcherPreview, participantSessionHandle, onText } = options; + const deviceSessionKey = `oi:device-voice-unavailable:${researcherPreview ? 'preview' : participantSessionHandle ?? 'session'}:${studyId ?? ''}`; const [state, setState] = useState({ kind: 'idle' }); const [supported, setSupported] = useState(false); const recorder = useRef(null); @@ -69,10 +91,55 @@ export function useVoiceInput(options: { const timer = useRef | null>(null); const mounted = useRef(true); const onTextRef = useRef(onText); + const deviceAvailability = useRef('unavailable'); + const deviceDisabled = useRef(false); + const preparing = useRef(false); + const generation = useRef(0); useEffect(() => { onTextRef.current = onText; }, [onText]); // Feature detection runs after mount so the server and first client render agree. - useEffect(() => { setSupported(voiceInputSupported(mode)); }, [mode]); + useEffect(() => { + const currentGeneration = ++generation.current; + deviceDisabled.current = deviceSessionDisabled(deviceSessionKey); + setState(mode === 'device' && deviceDisabled.current ? { kind: 'error', reason: 'deviceUnavailable' } : { kind: 'idle' }); + preparing.current = false; + deviceAvailability.current = 'unavailable'; + setSupported(voiceInputSupported(mode)); + if (mode === 'device' && !deviceDisabled.current) { + const Recognition = speechRecognitionClass(); + if (typeof Recognition?.available === 'function' && typeof Recognition.install === 'function') { + // A missing/rejected API is not permission to use remote dictation. + void (async () => { + try { + const availability = await Recognition.available!({ langs: [SPEECH_TAGS[language]], processLocally: true }); + if (generation.current !== currentGeneration) return; + deviceAvailability.current = availability; + setSupported(['available', 'downloadable', 'downloading'].includes(availability)); + } catch { + if (generation.current === currentGeneration) setSupported(false); + } + })(); + } + } + return () => { + generation.current += 1; + const session = recognition.current; + recognition.current = null; + session?.stop(); + }; + }, [mode, language, deviceSessionKey]); + + const disableDevice = useCallback(() => { + deviceDisabled.current = true; + disabledDeviceSessions.add(deviceSessionKey); + try { sessionStorage.setItem(deviceSessionKey, '1'); } catch { /* In-memory fallback. */ } + deviceAvailability.current = 'unavailable'; + setSupported(false); + setState({ kind: 'error', reason: 'deviceUnavailable' }); + const session = recognition.current; + recognition.current = null; + session?.stop(); + }, [deviceSessionKey]); const clearTimer = () => { if (timer.current) clearInterval(timer.current); @@ -150,31 +217,92 @@ export function useVoiceInput(options: { const startListening = useCallback(() => { const Recognition = speechRecognitionClass(); if (!Recognition) { - setState({ kind: 'error', reason: 'unsupported' }); + if (mode === 'device') disableDevice(); + else setState({ kind: 'error', reason: 'unsupported' }); + return; + } + let session: SpeechRecognitionLike; + try { session = new Recognition(); } catch { + if (mode === 'device') disableDevice(); + else setState({ kind: 'error', reason: 'failed' }); return; } - const session = new Recognition(); + if (mode === 'device') { + // Never start a device recognizer without the browser's local-only flag. + // A rejected/ignored setter fails closed too. + try { + // A browser without the attribute would accept the assignment as a plain + // property and recognize in the cloud, so require it to exist first. + if (!('processLocally' in session)) throw new Error('Local recognition unsupported'); + session.processLocally = true; + if (session.processLocally !== true) throw new Error('Local recognition unavailable'); + } catch { + disableDevice(); + return; + } + } session.lang = SPEECH_TAGS[language]; session.continuous = true; session.interimResults = false; session.onresult = (event) => { + if (!mounted.current || recognition.current !== session) return; for (let index = event.resultIndex; index < event.results.length; index += 1) { const result = event.results[index]; if (result.isFinal && result[0].transcript.trim()) onTextRef.current(result[0].transcript.trim()); } }; session.onerror = (event) => { - if (!mounted.current) return; + if (!mounted.current || recognition.current !== session) return; + if (mode === 'device' && ['language-not-supported', 'service-not-allowed'].includes(event.error)) { + disableDevice(); + return; + } setState({ kind: 'error', reason: event.error === 'not-allowed' || event.error === 'service-not-allowed' ? 'denied' : 'failed' }); }; session.onend = () => { + if (recognition.current !== session) return; recognition.current = null; if (mounted.current) setState((current) => (current.kind === 'listening' ? { kind: 'idle' } : current)); }; recognition.current = session; - session.start(); setState({ kind: 'listening' }); - }, [language]); + try { + session.start(); + } catch { + if (mode === 'device') disableDevice(); + else { + recognition.current = null; + setState({ kind: 'error', reason: 'failed' }); + } + } + }, [language, mode, disableDevice]); + + const prepareDevice = useCallback(async () => { + if (preparing.current || deviceDisabled.current) return; + const Recognition = speechRecognitionClass(); + if (typeof Recognition?.install !== 'function') { + disableDevice(); + return; + } + preparing.current = true; + const currentGeneration = generation.current; + setState({ kind: 'preparing' }); + try { + const installed = await Recognition.install({ langs: [SPEECH_TAGS[language]], processLocally: true }); + if (!mounted.current || generation.current !== currentGeneration) return; + if (!installed) disableDevice(); + else { + deviceAvailability.current = 'available'; + // Do not open the microphone unexpectedly after a slow download. + // The participant can keep typing/sending, then press the mic when ready. + setState({ kind: 'idle' }); + } + } catch { + if (mounted.current && generation.current === currentGeneration) disableDevice(); + } finally { + if (generation.current === currentGeneration) preparing.current = false; + } + }, [language, disableDevice]); const toggle = useCallback(() => { if (state.kind === 'recording') { @@ -185,10 +313,14 @@ export function useVoiceInput(options: { recognition.current?.stop(); return; } - if (state.kind === 'transcribing') return; + if (state.kind === 'transcribing' || preparing.current || !supported) return; if (mode === 'installation') void startRecording(); else if (mode === 'browser') startListening(); - }, [mode, startListening, startRecording, state.kind]); + else if (mode === 'device' && !deviceDisabled.current) { + if (deviceAvailability.current === 'available') startListening(); + else if (deviceAvailability.current === 'downloadable' || deviceAvailability.current === 'downloading') void prepareDevice(); + } + }, [mode, startListening, startRecording, prepareDevice, state.kind, supported]); - return { enabled: mode === 'installation' || mode === 'browser', supported, state, toggle }; + return { enabled: mode === 'installation' || mode === 'browser' || mode === 'device', supported, state, toggle }; } diff --git a/tests/e2e/device-voice.spec.ts b/tests/e2e/device-voice.spec.ts new file mode 100644 index 0000000..73e2038 --- /dev/null +++ b/tests/e2e/device-voice.spec.ts @@ -0,0 +1,43 @@ +import { test, expect, GREETING, ANSWER } from './workflow-fixture'; + +// Synthetic external provider only; study save, consent and interview handlers are real. +for (const mode of ['off', 'device'] as const) { + test(`${mode} voice: a Chromium without the local API hides the mic and allows a typed interview`, async ({ page, workflow }) => { + await page.goto('/login'); + await page.getByLabel('Password').fill('synthetic-e2e-admin-password'); + await page.getByRole('button', { name: 'Login', exact: true }).click(); + await expect(page).toHaveURL(/\/studies$/); + await page.goto('/setup'); + await page.getByLabel('Study Name *', { exact: true }).fill(`Synthetic ${mode} voice study`); + await page.getByLabel('Research Question *', { exact: true }).fill('How do people resume research?'); + await page.getByPlaceholder('Question 1...', { exact: true }).fill('How do you return to a saved document?'); + await page.getByRole('radio', { name: /OpenAI/ }).check(); + if (mode === 'device') await page.getByRole('radio', { name: /On the participant’s computer/ }).check(); + await page.getByRole('button', { name: 'Save Study', exact: true }).click(); + await expect(page).toHaveURL(/\/studies\/[0-9a-f-]+$/); + await page.getByRole('tab', { name: 'Study settings' }).click(); + await page.getByRole('button', { name: 'Generate New Link' }).click(); + const linkInput = page.locator('input[readonly]'); + await expect(linkInput).toHaveValue(/\/p\/[A-Za-z0-9_-]+$/); + const context = await workflow.participantContext(); + await context.addInitScript(() => { + Object.defineProperty(window, 'SpeechRecognition', { configurable: true, value: undefined }); + Object.defineProperty(window, 'webkitSpeechRecognition', { configurable: true, value: undefined }); + }); + const participant = await context.newPage(); + const transcriptionRequests: string[] = []; + participant.on('request', request => { + if (new URL(request.url()).pathname === '/api/transcribe') transcriptionRequests.push(request.url()); + }); + await participant.goto(new URL(await linkInput.inputValue()).pathname); + if (mode === 'device') await expect(participant.getByText(/your browser turns your speech into text on this computer/)).toBeVisible(); + await participant.getByRole('button', { name: 'I consent — begin the interview' }).click(); + await expect(participant.getByText(GREETING, { exact: true })).toBeVisible(); + await expect(participant.getByRole('button', { name: 'Start voice input' })).toHaveCount(0); + await participant.getByLabel('Your response').fill(ANSWER); + await participant.getByRole('button', { name: 'Send', exact: true }).click(); + await expect(participant.getByRole('heading', { name: /conversation complete/ })).toBeVisible(); + expect(transcriptionRequests).toEqual([]); + await context.close(); + }); +} diff --git a/tests/unit/InterviewChat.voice.test.tsx b/tests/unit/InterviewChat.voice.test.tsx index 313952f..d58627f 100644 --- a/tests/unit/InterviewChat.voice.test.tsx +++ b/tests/unit/InterviewChat.voice.test.tsx @@ -16,6 +16,9 @@ import Consent from '@/components/Consent'; class FakeRecognition { static last: FakeRecognition | null = null; + static available = vi.fn(); + static install = vi.fn(); + processLocally = false; lang = ''; continuous = false; interimResults = true; @@ -94,6 +97,7 @@ describe('consent notice for voice input', () => { it.each([ ['installation', /recording is sent to Cloudflare to be turned into text by Cloudflare Workers AI/], ['browser', /in Chrome this is Google, in Safari Apple/], + ['device', /your browser turns your speech into text on this computer/], ] as const)('names who turns speech into text: %s', (mode, text) => { useStore.setState(useStore.getInitialState(), true); useStore.getState().beginParticipantSession(makeStudyConfig({ id: 'study-v', voiceInput: mode }), 'participant-handle-v-123456'); @@ -108,3 +112,34 @@ describe('consent notice for voice input', () => { expect(screen.queryByText(/microphone/i)).not.toBeInTheDocument(); }); }); + + +describe('device mic and consent', () => { + it('keeps typing and Send available during preparation, then shows a localized failure without a mic', async () => { + FakeRecognition.available.mockResolvedValue('downloadable'); + let finish!: (value: boolean) => void; + FakeRecognition.install.mockReturnValue(new Promise(resolve => { finish = resolve; })); + seed('device', { interviewLanguages: ['ja'] }); + useStore.setState({ participantLanguage: 'ja' }); + render(); + fireEvent.click(await screen.findByRole('button', { name: '音声入力を開始' })); + expect(screen.getByRole('status')).toHaveTextContent('このデバイスで音声入力を準備しています…'); + fireEvent.change(screen.getByLabelText('あなたの回答'), { target: { value: '入力できます' } }); + expect(screen.getByRole('button', { name: '送信' })).toBeEnabled(); + await act(async () => finish(false)); + expect(screen.queryByRole('button', { name: '音声入力を開始' })).not.toBeInTheDocument(); + expect(screen.getByRole('alert')).toHaveTextContent('このデバイスでは音声入力を利用できません'); + expect(screen.getByLabelText('あなたの回答')).toHaveValue('入力できます'); + }); + + it.each([ + ['ja', /録音はこのコンピューターの外に出ないとしています/], + ['zh', /浏览器会在这台电脑上将您的语音转为文字/], + ] as const)('renders the device consent notice in %s', (language, text) => { + useStore.setState(useStore.getInitialState(), true); + useStore.getState().beginParticipantSession(makeStudyConfig({ id: 'study-v', voiceInput: 'device', interviewLanguages: [language] }), 'participant-handle-v-123456'); + render(); + expect(screen.getByText(text)).toHaveTextContent(/60 MB/); + expect(screen.queryByText(/Cloudflare Workers AI/)).not.toBeInTheDocument(); + }); +}); diff --git a/tests/unit/VoiceInputSection.test.tsx b/tests/unit/VoiceInputSection.test.tsx new file mode 100644 index 0000000..7ea74cb --- /dev/null +++ b/tests/unit/VoiceInputSection.test.tsx @@ -0,0 +1,18 @@ +import { describe, expect, it, vi } from 'vitest'; +import { fireEvent, render, screen } from '@testing-library/react'; +import { VoiceInputSection } from '@/components/studySetup/VoiceInputSection'; +import type { StudyDraft } from '@/components/studySetup/useStudyDraft'; + +describe('device voice setup', () => { + it.each([true, false])('offers device mode without transcription capability (hosted=%s)', hosted => { + const setVoiceInput = vi.fn(); + const draft = { voiceInput: 'off', setVoiceInput } as unknown as StudyDraft; + render(); + const device = screen.getByRole('radio', { name: /On the participant’s computer/ }); + expect(device).toBeEnabled(); + expect(screen.getByRole('radio', { name: /Transcribed by this installation/ })).toBeDisabled(); + fireEvent.click(device); + expect(setVoiceInput).toHaveBeenCalledWith('device'); + expect(screen.getByText(/Participants on phones, Safari and Firefox/)).toHaveTextContent('says the audio stays there'); + }); +}); diff --git a/tests/unit/api.export.markdown.test.ts b/tests/unit/api.export.markdown.test.ts index 314bd28..6506371 100644 --- a/tests/unit/api.export.markdown.test.ts +++ b/tests/unit/api.export.markdown.test.ts @@ -54,6 +54,15 @@ const study = makeStoredStudy({ id: 'study-a', config: { ...makeStoredStudy().co const exportedAt = new Date(Date.UTC(2026, 9, 8, 12, 0, 0)); describe('transcripts Markdown builder', () => { + it('describes device voice from the saved collection revision, not the current mode', () => { + const markdown = buildTranscriptsMarkdown(study, [interview({ + collectionConfig: { ...study.config, voiceInput: 'device' }, + })], exportedAt); + expect(markdown).toContain('the browser turns speech into text on the participant’s computer'); + expect(markdown).toContain('says the recording stays there; the study receives only the text the participant sends'); + expect(markdown).not.toContain('speech turned into text by Cloudflare Workers AI'); + }); + it('quotes every transcript line so content cannot forge headings, speakers or the closing marker', () => { const markdown = buildTranscriptsMarkdown(study, [interview()], exportedAt); const lines = markdown.split('\n'); diff --git a/tests/unit/useVoiceInput.device.test.tsx b/tests/unit/useVoiceInput.device.test.tsx new file mode 100644 index 0000000..1c5fb56 --- /dev/null +++ b/tests/unit/useVoiceInput.device.test.tsx @@ -0,0 +1,215 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { act, renderHook, waitFor } from '@testing-library/react'; +import { SPEECH_TAGS, useVoiceInput } from '@/lib/voice/useVoiceInput'; +import type { InterviewLanguage } from '@/lib/i18n/languages'; + +class Recognition { + static available = vi.fn(); + static install = vi.fn(); + static sessions: Recognition[] = []; + lang = ''; + processLocally = false; + continuous = false; + interimResults = true; + onresult: ((event: unknown) => void) | null = null; + onerror: ((event: { error: string }) => void) | null = null; + onend: (() => void) | null = null; + start = vi.fn(() => { + // Every started instance must satisfy the local-only contract, including retries. + expect(this.processLocally).toBe(true); + Recognition.sessions.push(this); + }); + stop = vi.fn(() => this.onend?.()); +} +const onText = vi.fn(); +let sessionNumber = 0; +function options(language: InterviewLanguage = 'en') { + return { mode: 'device' as const, language, studyId: 'study-device', researcherPreview: false, + participantSessionHandle: `device-session-${sessionNumber}`, onText }; +} +const mount = () => renderHook(() => useVoiceInput(options())); + +beforeEach(() => { + sessionNumber += 1; + sessionStorage.clear(); + Recognition.sessions = []; + Recognition.available.mockResolvedValue('available'); + Recognition.install.mockResolvedValue(true); + vi.stubGlobal('SpeechRecognition', Recognition); + vi.stubGlobal('webkitSpeechRecognition', undefined); + vi.stubGlobal('fetch', vi.fn()); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe('device-only speech input', () => { + it.each(['available', 'downloadable', 'downloading'])('offers the mic only after %s is resolved', async availability => { + let resolve!: (value: string) => void; + Recognition.available.mockReturnValue(new Promise(r => { resolve = r; })); + const { result } = mount(); + expect(result.current.supported).toBe(false); + await act(async () => resolve(availability)); + expect(result.current.supported).toBe(true); + expect(Recognition.available).toHaveBeenCalledWith({ langs: ['en-US'], processLocally: true }); + expect(Recognition.sessions).toHaveLength(0); + }); + + it.each(['unavailable', 'missing', 'missing-static', 'missing-install', 'rejected'])('hides the mic for %s and never starts any fallback', async availability => { + if (availability === 'missing') vi.stubGlobal('SpeechRecognition', undefined); + else if (availability === 'missing-static') vi.stubGlobal('SpeechRecognition', class {}); + else if (availability === 'missing-install') vi.stubGlobal('SpeechRecognition', class { static available = Recognition.available; }); + else if (availability === 'rejected') Recognition.available.mockRejectedValue(new Error('policy')); + else Recognition.available.mockResolvedValue(availability); + const { result } = mount(); + await act(async () => {}); + act(() => result.current.toggle()); + expect(result.current.supported).toBe(false); + expect(Recognition.sessions).toHaveLength(0); + expect(fetch).not.toHaveBeenCalled(); + }); + + it.each(['downloadable', 'downloading'])('prepares a %s pack once, then requires a fresh press to record locally', async availability => { + Recognition.available.mockResolvedValue(availability); + let resolve!: (value: boolean) => void; + Recognition.install.mockReturnValue(new Promise(r => { resolve = r; })); + const { result } = mount(); + await waitFor(() => expect(result.current.supported).toBe(true)); + act(() => { result.current.toggle(); result.current.toggle(); }); + expect(result.current.state.kind).toBe('preparing'); + expect(Recognition.install).toHaveBeenCalledExactlyOnceWith({ langs: ['en-US'], processLocally: true }); + expect(Recognition.sessions).toHaveLength(0); + await act(async () => resolve(true)); + expect(result.current.state.kind).toBe('idle'); + expect(Recognition.sessions).toHaveLength(0); + act(() => result.current.toggle()); + expect(Recognition.sessions).toHaveLength(1); + expect(Recognition.sessions[0].processLocally).toBe(true); + expect(fetch).not.toHaveBeenCalled(); + }); + + it.each(['false', 'rejected'])('disables local speech after install %s', async outcome => { + Recognition.available.mockResolvedValue('downloadable'); + if (outcome === 'false') Recognition.install.mockResolvedValue(false); + else Recognition.install.mockRejectedValue(new Error('download failed')); + const { result } = mount(); + await waitFor(() => expect(result.current.supported).toBe(true)); + await act(async () => result.current.toggle()); + expect(result.current.supported).toBe(false); + expect(result.current.state).toEqual({ kind: 'error', reason: 'deviceUnavailable' }); + act(() => result.current.toggle()); + expect(Recognition.install).toHaveBeenCalledTimes(1); + expect(Recognition.sessions).toHaveLength(0); + expect(fetch).not.toHaveBeenCalled(); + }); + + it.each(['language-not-supported', 'service-not-allowed'])('hides the mic for the session after %s, dropping late results', async error => { + const { result, rerender } = renderHook(({ language }) => useVoiceInput(options(language)), { initialProps: { language: 'en' as InterviewLanguage } }); + await waitFor(() => expect(result.current.supported).toBe(true)); + act(() => result.current.toggle()); + const session = Recognition.sessions[0]; + act(() => session.onerror?.({ error })); + expect(result.current.supported).toBe(false); + expect(result.current.state).toEqual({ kind: 'error', reason: 'deviceUnavailable' }); + act(() => session.onresult?.({ resultIndex: 0, results: [{ isFinal: true, 0: { transcript: 'late' } }] })); + expect(onText).not.toHaveBeenCalled(); + rerender({ language: 'ja' }); + act(() => result.current.toggle()); + expect(result.current.supported).toBe(false); + expect(Recognition.sessions).toHaveLength(1); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('keeps a terminal failure hidden after remount, but not in a new participant session', async () => { + const first = mount(); + await waitFor(() => expect(first.result.current.supported).toBe(true)); + act(() => first.result.current.toggle()); + act(() => Recognition.sessions[0].onerror?.({ error: 'service-not-allowed' })); + first.unmount(); + const second = mount(); + await act(async () => {}); + expect(second.result.current.supported).toBe(false); + expect(second.result.current.state).toEqual({ kind: 'error', reason: 'deviceUnavailable' }); + second.unmount(); + sessionNumber += 1; + const third = mount(); + await waitFor(() => expect(third.result.current.supported).toBe(true)); + }); + + it.each(Object.entries(SPEECH_TAGS))('uses the one language table for %s availability, installation and recognition', async (language, tag) => { + Recognition.available.mockResolvedValue('downloadable'); + const { result } = renderHook(() => useVoiceInput(options(language as InterviewLanguage))); + await waitFor(() => expect(result.current.supported).toBe(true)); + await act(async () => result.current.toggle()); + act(() => result.current.toggle()); + expect(Recognition.available).toHaveBeenCalledWith({ langs: [tag], processLocally: true }); + expect(Recognition.install).toHaveBeenCalledWith({ langs: [tag], processLocally: true }); + expect(Recognition.sessions[0]).toMatchObject({ lang: tag, processLocally: true }); + }); + + it('never starts with processLocally false, including a second recording and a generic error retry', async () => { + const { result } = mount(); + await waitFor(() => expect(result.current.supported).toBe(true)); + for (let i = 0; i < 3; i++) { + act(() => result.current.toggle()); + const session = Recognition.sessions[i]; + act(() => { + if (i === 1) session.onerror?.({ error: 'network' }); + session.onend?.(); + }); + } + expect(Recognition.sessions).toHaveLength(3); + expect(Recognition.sessions.every(s => s.processLocally === true)).toBe(true); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('never starts a recognizer that refuses the local-only flag', async () => { + class RefusesLocal extends Recognition { + constructor() { + super(); + Object.defineProperty(this, 'processLocally', { get: () => false, set: () => {} }); + } + } + vi.stubGlobal('SpeechRecognition', RefusesLocal); + const { result } = mount(); + await waitFor(() => expect(result.current.supported).toBe(true)); + act(() => result.current.toggle()); + expect(result.current.supported).toBe(false); + expect(result.current.state).toEqual({ kind: 'error', reason: 'deviceUnavailable' }); + expect(Recognition.sessions).toHaveLength(0); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('never starts a recognizer that lacks the processLocally attribute (an older browser would ignore it and use the cloud)', async () => { + class NoLocalAttribute extends Recognition { + constructor() { + super(); + delete (this as { processLocally?: boolean }).processLocally; + } + } + vi.stubGlobal('SpeechRecognition', NoLocalAttribute); + const { result } = mount(); + await waitFor(() => expect(result.current.supported).toBe(true)); + act(() => result.current.toggle()); + expect(result.current.state).toEqual({ kind: 'error', reason: 'deviceUnavailable' }); + expect(Recognition.sessions).toHaveLength(0); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('ignores stale availability and install completions after a language change or unmount', async () => { + let available!: (value: string) => void; + Recognition.available.mockReturnValueOnce(new Promise(r => { available = r; })); + const { result, rerender, unmount } = renderHook(({ language }) => useVoiceInput(options(language)), { initialProps: { language: 'en' as InterviewLanguage } }); + Recognition.available.mockResolvedValue('unavailable'); + rerender({ language: 'ja' }); + await act(async () => available('available')); + expect(result.current.supported).toBe(false); + Recognition.available.mockResolvedValue('downloadable'); + rerender({ language: 'fr' }); + await waitFor(() => expect(result.current.supported).toBe(true)); + let install!: (value: boolean) => void; + Recognition.install.mockReturnValueOnce(new Promise(r => { install = r; })); + act(() => result.current.toggle()); + unmount(); + await act(async () => install(true)); + expect(Recognition.sessions).toHaveLength(0); + }); +}); From 84635ca05b349ed26395f4882fac6c825bc91c6b Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:30:48 +0200 Subject: [PATCH 3/6] docs(voice): device mode, verification and the 5.3 rollback refusal Co-Authored-By: Claude Opus 5.5 (1M context) --- ACKNOWLEDGMENTS.md | 9 +++ AGENTS.md | 3 +- README.md | 5 +- .../cloudflare-migration/RUNBOOK.md | 21 +++++ docs/releases/v5.4.0.md | 78 +++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 docs/releases/v5.4.0.md diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 529c4f5..421ca74 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -4,6 +4,15 @@ OpenInterviewer grows through contributions, working forks and feedback from people conducting real research. Thank you for sharing what works, what is missing and how you have adapted the project. +## Version 5.4.0 (draft): device voice + +Device voice continues the participant-microphone idea inspired by Yinghua +([@8888oukaouka-spec](https://github.com/8888oukaouka-spec)) and her +[openinterviewerver02](https://github.com/8888oukaouka-spec/openinterviewerver02) +fork, credited under 5.1.0 below. The local-only mode is a new implementation +against OpenInterviewer’s configuration and consent contracts, not code authored +by her or cherry-picked from the fork. + ## Version 5.3.0: projects Thank you to [@8888oukaouka-spec](https://github.com/8888oukaouka-spec) for the diff --git a/AGENTS.md b/AGENTS.md index 79ca1ad..60f6f78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,7 @@ Hosted study create/delete is a durable cross-database operation. Preserve the o - Server-recorded consent: `src/lib/participantConsent.ts` - Consent coverage of the provider transport (Cloudflare; disclosed transport vs the current route): `src/lib/transportDisclosure.ts` - The consent notice's provider commitment (`fixed` or `may-change`) and the retry check that keeps it: `src/lib/providerCommitment.ts` -- Participant voice input: `src/lib/voice/` (browser recording to 16 kHz WAV, browser dictation) and `src/lib/transcription/workersAi.ts` with `src/app/api/transcribe/route.ts` (Workers AI) +- Participant voice input: `src/lib/voice/` (browser recording to 16 kHz WAV, browser dictation, local-only device dictation) and `src/lib/transcription/workersAi.ts` with `src/app/api/transcribe/route.ts` (Workers AI) - Interview languages and participant text: `src/lib/i18n/` (`languages.ts` is pure and shared with the Durable Object; `messages/en.ts` is the source every translation must match) - Canonical study loading: `src/lib/canonicalStudy.ts` - Save validation and deferred analysis: `src/lib/interviewSubmission.ts`, `src/lib/interviewAnalysis.ts`, `src/lib/analysisState.ts` @@ -113,6 +113,7 @@ On Cloudflare the participant sequence is the same, with analysis queued by the - The non-secret participant session selector must accompany participant API calls so parallel tabs remain isolated. - Consent is a server record bound to participant session, study revision, and consent hash. Client Zustand state alone is not consent authority. - Voice transcription runs only for a study whose `voiceInput` is `installation` (the consent notice names Cloudflare), after the same consent verification and participant admission as an interview turn (`transcribe` operation). It calls Workers AI once, never through AI Gateway, and never stores or logs the audio or text. +- Voice modes are `off`, `installation`, `browser` and `device`. Device needs no server capability and follows browser mode’s hosted rules. Its mic is gated by local pack availability; every recognition uses `processLocally = true`, never a remote fallback. Terminal device failures hide the mic for the participant session. `SPEECH_TAGS` in `src/lib/voice/useVoiceInput.ts` is the single speech-tag table (Chinese needs manual Chrome verification). - In a study with interview languages, the consent hash is of the chosen language's consent text. Greeting, interview and save name that language and are verified against the same hash, so a switched language fails consent; the saved record's `interviewLanguage` must match it (rechecked by the Durable Object completion). A study without the setting sends and records no language. - Researcher preview may call the real provider but must not persist or increment study results. - Study revision, link status, ownership, consent, rate limits, and storage uncertainty fail closed. diff --git a/README.md b/README.md index 5d1cf99..205a6b8 100644 --- a/README.md +++ b/README.md @@ -438,9 +438,12 @@ Studies saved before this setting existed keep their old notice, and their inter - **Transcribed by this installation**: the browser records up to one minute, converts it to 16 kHz WAV and sends it to `/api/transcribe`, which passes it to Cloudflare Workers AI (`@cf/openai/whisper-large-v3-turbo`) once. OpenInterviewer does not store or log the recording or its text. Cloudflare's Workers AI terms say customer content is not used to train models; they do not state a retention period, so the consent line promises only that the study does not keep the recording. Works in current Chrome, Edge, Firefox and Safari, on phones and computers. On Cloudflare the installation's `AI` binding is used directly, never through AI Gateway; on Node set `CLOUDFLARE_WORKERS_AI_ACCOUNT_ID` and `CLOUDFLARE_WORKERS_AI_TOKEN` (an API token with Workers AI permissions). Cloudflare charges about $0.0005 per audio minute after its free daily allowance of 10,000 Neurons (roughly 200 audio minutes). Each request is admitted like an interview turn (40 clips per session per hour; 1,500 per study per day). The model's voice-activity filter drops silence and steady noise, but like any speech model it can occasionally turn a non-speech sound into a stock phrase (in staging tests, once, the Japanese for “thanks for watching”). The participant's check before sending is the safeguard, so the text is never sent on its own. - **The browser's own dictation**: no setup. Chrome sends the audio to Google and Safari to Apple, under their terms; Firefox does not support it. Use it only where your ethics approval allows these processors. +- **On the participant’s computer (desktop Chrome only)** (`device`): the browser turns speech into text locally with `processLocally = true`, never falling back to a speech service or installation transcription. Available in desktop Chrome 139+ where the selected language pack is supported; phones, Safari, Firefox and unsupported devices show no microphone and participants type instead. The browser may first download a speech pack (about 60 MB). Participants can keep typing and sending during preparation, then press the mic again when ready. This is the browser’s local-processing promise, not something the page can independently verify. No server credentials, bindings or readiness capability are needed. - **Off** (the default). -Transcription by the installation is not offered on the hosted service, which refuses to save it; browser dictation works there. On-device transcription is being evaluated; the open models available today do not cover all six interview languages. +Transcription by the installation is not offered on the hosted service, which refuses to save it; browser dictation and device voice work there. The six speech tags are centralized in `SPEECH_TAGS` in `src/lib/voice/useVoiceInput.ts`; the Chinese tag `zh-CN` is provisional pending a manual Chrome check against `cmn-Hans-CN`. New device consent/status translations are drafts pending language review. See the [draft v5.4.0 notes](docs/releases/v5.4.0.md). + +A 5.3 build rejects saved `voiceInput: device` configurations, rather than silently changing modes: participant access and link creation are refused (409); canonical-study validation also fails closed (503). Unchanged saves return 400 `Invalid voice input setting`. Before rollback, stop collection and explicitly change affected studies to **Off** on 5.4, which advances their revision; distribute new links before resuming. Saved interviews retain their original device-mode collection configuration, which 5.3 does not fully understand—prefer rolling forward if those records need processing or export. Changing study configuration advances its revision and invalidates links and participant sessions issued for the previous revision. Generate and distribute a new link after a consequential edit. Pausing/resuming collection and unchanged saves do not advance the revision. diff --git a/docs/operations/cloudflare-migration/RUNBOOK.md b/docs/operations/cloudflare-migration/RUNBOOK.md index e23db59..f759760 100644 --- a/docs/operations/cloudflare-migration/RUNBOOK.md +++ b/docs/operations/cloudflare-migration/RUNBOOK.md @@ -37,6 +37,27 @@ A lost response is resolved by reading `status`, never by repeating a transition **Unknown outcomes.** A maintenance transition, activation, restore, import chunk or import finalize may have committed when the Worker answers `OUTCOME_UNKNOWN` or a 5xx other than a definite refusal, when a 2xx reply lacks the expected result (for example a body that is not JSON), or when no complete reply arrives: the connection fails or is reset, the body is cut off, or nothing arrives within 120 s (`no reply: `, with `detail.network`). The CLI then exits 1 with `detail.outcome: "unknown"`. It does not resend the request, except an import chunk or finalize, which is idempotent and is sent up to three times in all. It reads `status` once and puts it in `detail.status`. If that read fails too, `detail.status.unavailable` gives the reason and the message says `status could not be read either`. In that case run `status` before anything else, and repeat it until it answers. Compare it with what you asked for: a changed state or version, or a changed activated epoch, means the request ran. For an import, re-run the same command: accepted chunks come back as duplicates, and a finalized import answers `finalized` again. +### Voice input modes and rollback + +A study offers `off`, `installation`, `browser` or `device`. Only `installation` +uses the Workers AI binding. `browser` uses the browser's speech service; +`device` requires desktop Chrome's local recognition and no server capability, +credential or gateway. An absent local API or unavailable pack hides the mic; +participants type. A first-use pack download (about 60 MB) does not block typing. +If preparation or local recognition is unavailable, there is no remote fallback. + +Before reverting 5.4 to 5.3, stop collection and explicitly save affected studies +with voice input **Off** while still on 5.4. That changes the revision and +invalidates prior participant authority; issue new links before resuming. +A 5.3 validator rejects `device` with `Invalid voice input setting`: unchanged +saves return 400, participant context and link generation return 409, and the +canonical loader's independent check returns 503. It does not silently disable +voice or send audio elsewhere. Existing data is not deleted. Saved interview +collection configurations can still contain `device`; old exports cannot describe +that mode and operations validating those configurations can refuse them. Prefer +rolling forward to 5.4 if these records need processing. This does not alter any +existing schema rollback restrictions. See the [draft release notes](../../releases/v5.4.0.md). + ### Sign-in lockout Sign-in is limited on Cloudflare: 10 failed attempts per client per 15 minutes, and 200 across all clients per hour. Each client is its IPv4 address, or its /64 for IPv6, so a lockout covers every address in that /64. Each window opens with its first counted attempt and is never extended. Over a limit, `/api/auth` answers 429 with `Retry-After`, and the CLI exits 1 with "retry after N seconds". A client-level lockout clears within 15 minutes. A global lockout means many failures from many clients; it blocks every sign-in, including the correct password, for up to one hour. Sign-in works in every maintenance state and under epoch, identity and bootstrap holds. Only an unsupported schema or unavailable workspace storage refuses it (503). diff --git a/docs/releases/v5.4.0.md b/docs/releases/v5.4.0.md new file mode 100644 index 0000000..3449b91 --- /dev/null +++ b/docs/releases/v5.4.0.md @@ -0,0 +1,78 @@ +# OpenInterviewer 5.4.0 — draft + +## Device voice input + +A fourth study voice mode, **On the participant’s computer (desktop Chrome +only)** (`device`), lets the participant's browser turn speech into text locally. +Like the other voice modes, text appears in the answer box for review and is +never sent automatically. Inspired by Yinghua's +([@8888oukaouka-spec](https://github.com/8888oukaouka-spec)) participant-microphone +idea in her fork, credited in [5.1.0](v5.1.0.md) and +[ACKNOWLEDGMENTS.md](../../ACKNOWLEDGMENTS.md); this local-only implementation is +not authored by her or imported from the fork. + +- Desktop Chrome 139+ exposes local speech recognition. The microphone appears + only if the selected language's local pack is available, downloadable or + downloading. Phones, Safari, Firefox stable and unsupported devices show no + microphone; participants type instead. +- The first press can ask the browser to install a speech pack (about 60 MB). + A localized preparation note appears; typing and Send remain available. Once + ready, press the microphone again to start (no surprise recording after a + slow download). +- Every recognizer started in device mode has `processLocally = true`. There is + no fallback to browser cloud dictation or installation transcription. + `language-not-supported` and `service-not-allowed` disable the mic for that + participant session, with a short localized note. +- Consent presents locality as the browser's promise, not ours: the browser + turns speech into text on this computer and says the recording stays there; + the study receives only the text the participant sends. OpenInterviewer cannot + verify what the browser does internally. The submitted **text** still + follows the study's usual AI interview and storage terms. +- Works with standalone and hosted studies without server transcription + credentials, capability or readiness changes. `/api/transcribe` still accepts + only `installation`. Changing voice mode advances the study revision and + invalidates older participant authority. Markdown exports describe the mode + offered at collection, not whether the microphone was used. + +## Upgrade and rollback + +No dependency, database schema, environment variable or binding changes. + +**Do not roll back a collecting device-mode study to 5.3.** The actual 5.3 +validator at `f3eb006` was exercised against `voiceInput: 'device'`: canonical +validation, creation and an unchanged update return `Invalid voice input +setting`. An explicit update to `off` is accepted. The older code paths map +these failures to: + +- 400 on creation or an unchanged configuration save. +- 409 from participant-context resolution (review/resave required) and link + generation (`STUDY_REQUIRES_RESAVE`). +- 503 if the canonical-study loader independently encounters the invalid config. + +The refusal occurs before participant provider use or persistence. Nothing is +silently converted to cloud dictation or deleted. Before rollback, stop +collection and save affected studies as **Off** on 5.4; this advances revision, +so issue new links before resuming. Historical interviews retain `device` in +`collectionConfig`; 5.3's Markdown export omits the device voice line, and +operations that validate historical configurations can refuse those records. +Rolling forward is preferable if they need processing or accurate export. +Existing schema downgrade restrictions still apply. + +## Verification still required before release + +Unit mocks verify the local-only flag, feature detection, preparation, failures, +language mapping, consent and server/config boundaries. They cannot prove real +browser locality or language-pack support. + +- Manually verify all six languages in desktop Chrome, including a fresh pack + install, offline recognition with an installed pack, denied microphone access, + and continued typing during download. Check unavailable browsers and a 375px + setup/participant layout. +- Confirm the Chinese BCP-47 tag: `zh-CN` is provisional; `cmn-Hans-CN` is the + other candidate. Change only the exported `SPEECH_TAGS` table in + `src/lib/voice/useVoiceInput.ts` after that check. +- Have the five new translations reviewed. They are drafts, not part of the + earlier translations' review receipt. +- Run the workers/Redis shared revision contract and browser lanes listed in + `CHANGES-device-voice.md`. No deployment or real browser verification was done + in the restricted implementation sandbox. From b369358057dc0b71f9c98e56737f1c085cd39cc7 Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:30:58 +0200 Subject: [PATCH 4/6] docs(voice): record the Chrome check of the six speech tags Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- docs/releases/v5.4.0.md | 5 ++--- src/lib/voice/useVoiceInput.ts | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 205a6b8..84cc969 100644 --- a/README.md +++ b/README.md @@ -441,7 +441,7 @@ Studies saved before this setting existed keep their old notice, and their inter - **On the participant’s computer (desktop Chrome only)** (`device`): the browser turns speech into text locally with `processLocally = true`, never falling back to a speech service or installation transcription. Available in desktop Chrome 139+ where the selected language pack is supported; phones, Safari, Firefox and unsupported devices show no microphone and participants type instead. The browser may first download a speech pack (about 60 MB). Participants can keep typing and sending during preparation, then press the mic again when ready. This is the browser’s local-processing promise, not something the page can independently verify. No server credentials, bindings or readiness capability are needed. - **Off** (the default). -Transcription by the installation is not offered on the hosted service, which refuses to save it; browser dictation and device voice work there. The six speech tags are centralized in `SPEECH_TAGS` in `src/lib/voice/useVoiceInput.ts`; the Chinese tag `zh-CN` is provisional pending a manual Chrome check against `cmn-Hans-CN`. New device consent/status translations are drafts pending language review. See the [draft v5.4.0 notes](docs/releases/v5.4.0.md). +Transcription by the installation is not offered on the hosted service, which refuses to save it; browser dictation and device voice work there. The six speech tags are centralized in `SPEECH_TAGS` in `src/lib/voice/useVoiceInput.ts`; desktop Chrome 155 accepts all six (including `zh-CN`) for local processing. New device consent/status translations are drafts pending language review. See the [draft v5.4.0 notes](docs/releases/v5.4.0.md). A 5.3 build rejects saved `voiceInput: device` configurations, rather than silently changing modes: participant access and link creation are refused (409); canonical-study validation also fails closed (503). Unchanged saves return 400 `Invalid voice input setting`. Before rollback, stop collection and explicitly change affected studies to **Off** on 5.4, which advances their revision; distribute new links before resuming. Saved interviews retain their original device-mode collection configuration, which 5.3 does not fully understand—prefer rolling forward if those records need processing or export. diff --git a/docs/releases/v5.4.0.md b/docs/releases/v5.4.0.md index 3449b91..83027a7 100644 --- a/docs/releases/v5.4.0.md +++ b/docs/releases/v5.4.0.md @@ -68,9 +68,8 @@ browser locality or language-pack support. install, offline recognition with an installed pack, denied microphone access, and continued typing during download. Check unavailable browsers and a 375px setup/participant layout. -- Confirm the Chinese BCP-47 tag: `zh-CN` is provisional; `cmn-Hans-CN` is the - other candidate. Change only the exported `SPEECH_TAGS` table in - `src/lib/voice/useVoiceInput.ts` after that check. +- Chinese tag: desktop Chrome 155 reports `zh-CN` (and the five other tags) as + downloadable for local processing; `SPEECH_TAGS` keeps `zh-CN`. - Have the five new translations reviewed. They are drafts, not part of the earlier translations' review receipt. - Run the workers/Redis shared revision contract and browser lanes listed in diff --git a/src/lib/voice/useVoiceInput.ts b/src/lib/voice/useVoiceInput.ts index a4b9c51..4bc173f 100644 --- a/src/lib/voice/useVoiceInput.ts +++ b/src/lib/voice/useVoiceInput.ts @@ -21,7 +21,7 @@ export type VoiceState = | { kind: 'error'; reason: 'denied' | 'failed' | 'unsupported' | 'limited' | 'unavailable' | 'deviceUnavailable' }; /** Shared by local availability, installation and recognition. - * zh-CN is provisional: manually check Chrome against cmn-Hans-CN before release. + * Chrome 155 reports all six as available or downloadable for local processing (checked 2026-10-10). */ export const SPEECH_TAGS: Record = { en: 'en-US', zh: 'zh-CN', fr: 'fr-FR', ja: 'ja-JP', ko: 'ko-KR', es: 'es-ES', From 98851cae836f3b3f646918972dd3c314151f21ef Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:43:19 +0200 Subject: [PATCH 5/6] =?UTF-8?q?fix(voice):=20language=20changes=20no=20lon?= =?UTF-8?q?ger=20reset=20installation/browser=20input;=20Spanish=20t=C3=BA?= =?UTF-8?q?=20register?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device availability effect re-ran for every mode on a language change, resetting state and stopping an active browser dictation. Device detection is now its own effect that returns early outside device mode (regression test). The Spanish device lines used usted; the file uses tú. Both from the Kimi review. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/i18n/messages/es.ts | 4 ++-- src/lib/voice/useVoiceInput.ts | 13 ++++++++++--- tests/unit/useVoiceInput.device.test.tsx | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/lib/i18n/messages/es.ts b/src/lib/i18n/messages/es.ts index 6762020..8f9d9b7 100644 --- a/src/lib/i18n/messages/es.ts +++ b/src/lib/i18n/messages/es.ts @@ -65,7 +65,7 @@ export const es: Messages = { mayChange: 'La persona responsable del estudio puede analizar más adelante tus respuestas con otro proveedor o modelo de IA.', }, voice: { - device: 'Si usa el micrófono, su navegador convierte su voz en texto en este ordenador e indica que la grabación no sale de él. Este estudio solo recibe el texto que usted decida enviar, nunca la grabación. La primera vez, su navegador puede descargar un paquete de voz (unos 60 MB).', + device: 'Si usas el micrófono, tu navegador convierte tu voz en texto en este ordenador e indica que la grabación no sale de él. Este estudio solo recibe el texto que decidas enviar, nunca la grabación. La primera vez, tu navegador puede descargar un paquete de voz (unos 60 MB).', installation: 'Si usas el micrófono, tu grabación se envía a Cloudflare para que Cloudflare Workers AI la convierta en texto. Este estudio no guarda la grabación y puedes editar el texto antes de enviarlo.', browser: 'Si usas el micrófono, el servicio de voz de tu navegador convierte tu voz en texto: en Chrome lo hace Google y en Safari, Apple, según sus propias condiciones. Puedes editar el texto antes de enviarlo.', }, @@ -98,7 +98,7 @@ export const es: Messages = { sendShortcut: '⌘/Ctrl + Intro para enviar', voice: { preparing: 'Preparando la entrada de voz en este dispositivo…', - deviceUnavailable: 'La entrada de voz no está disponible en este dispositivo. Escriba su respuesta.', + deviceUnavailable: 'La entrada de voz no está disponible en este dispositivo. Escribe tu respuesta.', start: 'Iniciar la entrada de voz', stop: 'Detener la grabación', recording: (elapsed) => `Grabando ${elapsed} (máximo 1:00)`, diff --git a/src/lib/voice/useVoiceInput.ts b/src/lib/voice/useVoiceInput.ts index 4bc173f..48ef3b0 100644 --- a/src/lib/voice/useVoiceInput.ts +++ b/src/lib/voice/useVoiceInput.ts @@ -99,13 +99,20 @@ export function useVoiceInput(options: { // Feature detection runs after mount so the server and first client render agree. useEffect(() => { + if (mode !== 'device') setSupported(voiceInputSupported(mode)); + }, [mode]); + + // Device mode re-checks local availability per language. Kept separate so a + // language change never resets or stops installation/browser voice input. + useEffect(() => { + if (mode !== 'device') return; const currentGeneration = ++generation.current; deviceDisabled.current = deviceSessionDisabled(deviceSessionKey); - setState(mode === 'device' && deviceDisabled.current ? { kind: 'error', reason: 'deviceUnavailable' } : { kind: 'idle' }); + setState(deviceDisabled.current ? { kind: 'error', reason: 'deviceUnavailable' } : { kind: 'idle' }); preparing.current = false; deviceAvailability.current = 'unavailable'; - setSupported(voiceInputSupported(mode)); - if (mode === 'device' && !deviceDisabled.current) { + setSupported(false); + if (!deviceDisabled.current) { const Recognition = speechRecognitionClass(); if (typeof Recognition?.available === 'function' && typeof Recognition.install === 'function') { // A missing/rejected API is not permission to use remote dictation. diff --git a/tests/unit/useVoiceInput.device.test.tsx b/tests/unit/useVoiceInput.device.test.tsx index 1c5fb56..6095e57 100644 --- a/tests/unit/useVoiceInput.device.test.tsx +++ b/tests/unit/useVoiceInput.device.test.tsx @@ -213,3 +213,22 @@ describe('device-only speech input', () => { expect(Recognition.sessions).toHaveLength(0); }); }); + +describe('language changes outside device mode', () => { + it('keeps browser dictation listening when the language prop changes', async () => { + const stop = vi.fn(); + class CloudRecognition { + lang = ''; continuous = false; interimResults = true; + onresult = null; onerror = null; onend = null; + start = vi.fn(); stop = stop; + } + vi.stubGlobal('SpeechRecognition', CloudRecognition); + const { result, rerender } = renderHook(({ language }) => useVoiceInput({ ...options(language), mode: 'browser' }), { initialProps: { language: 'en' as InterviewLanguage } }); + await waitFor(() => expect(result.current.supported).toBe(true)); + act(() => result.current.toggle()); + expect(result.current.state).toEqual({ kind: 'listening' }); + rerender({ language: 'ja' }); + expect(stop).not.toHaveBeenCalled(); + expect(result.current.state).toEqual({ kind: 'listening' }); + }); +}); From 34f80073305b2828b46f27df9151db613203e69f Mon Sep 17 00:00:00 2001 From: Xule Lin <43122877+linxule@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:17:43 +0200 Subject: [PATCH 6/6] i18n(voice): apply the three-model review of the device strings Gemini, Kimi and Grok reviewed the five device-voice translations as the 5.1 strings were. No promise was strengthened or weakened; fixes are term and register consistency in ja, zh and ko. Decisions recorded in docs/translations/REVIEW-2026-10.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- docs/releases/v5.4.0.md | 4 ++-- docs/translations/REVIEW-2026-10.md | 18 ++++++++++++++++++ src/lib/i18n/messages/ja.ts | 2 +- src/lib/i18n/messages/ko.ts | 4 ++-- src/lib/i18n/messages/zh.ts | 4 ++-- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 84cc969..302f293 100644 --- a/README.md +++ b/README.md @@ -441,7 +441,7 @@ Studies saved before this setting existed keep their old notice, and their inter - **On the participant’s computer (desktop Chrome only)** (`device`): the browser turns speech into text locally with `processLocally = true`, never falling back to a speech service or installation transcription. Available in desktop Chrome 139+ where the selected language pack is supported; phones, Safari, Firefox and unsupported devices show no microphone and participants type instead. The browser may first download a speech pack (about 60 MB). Participants can keep typing and sending during preparation, then press the mic again when ready. This is the browser’s local-processing promise, not something the page can independently verify. No server credentials, bindings or readiness capability are needed. - **Off** (the default). -Transcription by the installation is not offered on the hosted service, which refuses to save it; browser dictation and device voice work there. The six speech tags are centralized in `SPEECH_TAGS` in `src/lib/voice/useVoiceInput.ts`; desktop Chrome 155 accepts all six (including `zh-CN`) for local processing. New device consent/status translations are drafts pending language review. See the [draft v5.4.0 notes](docs/releases/v5.4.0.md). +Transcription by the installation is not offered on the hosted service, which refuses to save it; browser dictation and device voice work there. The six speech tags are centralized in `SPEECH_TAGS` in `src/lib/voice/useVoiceInput.ts`; desktop Chrome 155 accepts all six (including `zh-CN`) for local processing. Its translations were reviewed like the others (`docs/translations/REVIEW-2026-10.md`). See the [draft v5.4.0 notes](docs/releases/v5.4.0.md). A 5.3 build rejects saved `voiceInput: device` configurations, rather than silently changing modes: participant access and link creation are refused (409); canonical-study validation also fails closed (503). Unchanged saves return 400 `Invalid voice input setting`. Before rollback, stop collection and explicitly change affected studies to **Off** on 5.4, which advances their revision; distribute new links before resuming. Saved interviews retain their original device-mode collection configuration, which 5.3 does not fully understand—prefer rolling forward if those records need processing or export. diff --git a/docs/releases/v5.4.0.md b/docs/releases/v5.4.0.md index 83027a7..427781f 100644 --- a/docs/releases/v5.4.0.md +++ b/docs/releases/v5.4.0.md @@ -70,8 +70,8 @@ browser locality or language-pack support. setup/participant layout. - Chinese tag: desktop Chrome 155 reports `zh-CN` (and the five other tags) as downloadable for local processing; `SPEECH_TAGS` keeps `zh-CN`. -- Have the five new translations reviewed. They are drafts, not part of the - earlier translations' review receipt. +- The five new translations were reviewed by Gemini, Kimi and Grok; decisions are in + `docs/translations/REVIEW-2026-10.md`. - Run the workers/Redis shared revision contract and browser lanes listed in `CHANGES-device-voice.md`. No deployment or real browser verification was done in the restricted implementation sandbox. diff --git a/docs/translations/REVIEW-2026-10.md b/docs/translations/REVIEW-2026-10.md index 6002ea9..4ce7f49 100644 --- a/docs/translations/REVIEW-2026-10.md +++ b/docs/translations/REVIEW-2026-10.md @@ -55,3 +55,21 @@ Before release, the installation notice was also corrected in every language aft ## Process for later changes Change `en.ts` first. Any change to `consent.transport`, `consent.commitment` or `consent.controller` needs every translation updated and reviewed again, and a new entry in this folder. `tests/unit/interviewLanguages.test.ts` checks that every language has every message, with no empty or placeholder text, and that each notice names the provider, model and Cloudflare exactly as given. + +## On-device voice strings, 5.4.0 (2026-10-10) + +New strings: `consent.voice.device` (part of the consent notice), `interview.voice.preparing` and `interview.voice.deviceUnavailable`. They were drafted in zh, fr, ja, ko and es with AI assistance (Codex, then Claude rewrote the consent line so that locality is the browser's claim, not the study's). Kimi K3 checked the Spanish register in code review. Then the same three reviewers as above (Gemini 3.8 Flash, Kimi K3, Grok 4.6, through cc-fleet) each got the English source, all five translations, the neighbouring voice strings for register, and the same legal-text instructions. + +No reviewer found a promise added, removed, weakened or strengthened. All five keep the locality claim with the browser (浏览器…并表示, indique que, としています, -다고 안내합니다, indica que). + +| Reviewer | Language | Key | Severity | Finding | Decision | +| --- | --- | --- | --- | --- | --- | +| Kimi K3 (code review) | es | `consent.voice.device`, `interview.voice.deviceUnavailable` | minor | Formal usted; the rest of the file uses tú. | Accepted. | +| Gemini, Kimi, Grok | ja | `consent.voice.device` | major | この調査 (survey) instead of the established この研究; ブラウザー vs ブラウザ; あなた unlike the surrounding subject-less strings; マイクを使う場合 vs the established マイクを使用すると. | Accepted (Gemini's wording). | +| Gemini, Kimi | zh | `interview.voice.deviceUnavailable` | minor | 请键入 differs from the existing 请直接输入 for the same English sentence. | Accepted: 请直接输入您的回答。 | +| Grok | zh | `interview.voice.preparing` | minor | 为这台设备准备 reads as preparing *for* the device, losing "on this device". | Accepted: 正在这台设备上准备语音输入… | +| Gemini | ko | `consent.voice.device` | minor | The chooser of "the text you choose to send" is left implicit; the UI addresses the participant as 참여자님. | Accepted: 참여자님이 보내기로 선택한 텍스트 | +| Gemini, Grok (Kimi preferred the opposite direction) | ko | `interview.voice.deviceUnavailable` | minor | 답변 vs the existing line's 응답 for the same English. | Accepted: 응답, matching the existing string. | +| Kimi | es | `consent.unavailable` | blocker | Reported missing. | Rejected: present (es.ts line 50); the reviewer's excerpt omitted it. | + +fr: no issues from any reviewer. As above, these are AI-reviewed translations, not certified ones. diff --git a/src/lib/i18n/messages/ja.ts b/src/lib/i18n/messages/ja.ts index 89e6c61..d09e53b 100644 --- a/src/lib/i18n/messages/ja.ts +++ b/src/lib/i18n/messages/ja.ts @@ -65,7 +65,7 @@ export const ja: Messages = { mayChange: '研究者は今後、別の AI プロバイダーやモデルを使ってあなたの回答を分析する場合があります。', }, voice: { - device: 'マイクを使う場合、ブラウザーがこのコンピューター上で音声を文字に変換し、録音はこのコンピューターの外に出ないとしています。この調査が受け取るのは、あなたが送信を選んだ文字だけで、録音は受け取りません。初回は、ブラウザーが音声パック（約60 MB）をダウンロードすることがあります。', + device: 'マイクを使用すると、ブラウザがこのコンピューター上で音声を文字に変換し、録音はこのコンピューターの外に出ないとしています。この研究が受け取るのは、送信を選択した文字だけで、録音は受け取りません。初回は、ブラウザが音声パック（約60 MB）をダウンロードすることがあります。', installation: 'マイクを使用すると、録音は Cloudflare に送信され、Cloudflare Workers AI によって文字に変換されます。この研究では録音を保管せず、送信前に文字を編集できます。', browser: 'マイクを使用すると、ブラウザの音声サービスが音声を文字に変換します。Chrome では Google、Safari では Apple が、それぞれの規約に基づいて処理します。送信前に文字を編集できます。', }, diff --git a/src/lib/i18n/messages/ko.ts b/src/lib/i18n/messages/ko.ts index 17e9824..b5ee8ac 100644 --- a/src/lib/i18n/messages/ko.ts +++ b/src/lib/i18n/messages/ko.ts @@ -65,7 +65,7 @@ export const ko: Messages = { mayChange: '연구자는 나중에 다른 AI 제공업체나 모델로 응답을 분석할 수 있습니다.', }, voice: { - device: '마이크를 사용하면 브라우저가 이 컴퓨터에서 음성을 텍스트로 변환하며, 녹음은 이 컴퓨터를 벗어나지 않는다고 안내합니다. 이 연구는 보내기로 선택한 텍스트만 받으며 녹음은 받지 않습니다. 처음 사용할 때 브라우저가 음성 팩(약 60 MB)을 다운로드할 수 있습니다.', + device: '마이크를 사용하면 브라우저가 이 컴퓨터에서 음성을 텍스트로 변환하며, 녹음은 이 컴퓨터를 벗어나지 않는다고 안내합니다. 이 연구는 참여자님이 보내기로 선택한 텍스트만 받으며 녹음은 받지 않습니다. 처음 사용할 때 브라우저가 음성 팩(약 60 MB)을 다운로드할 수 있습니다.', installation: '마이크를 사용하면 녹음이 Cloudflare로 전송되어 Cloudflare Workers AI가 텍스트로 변환합니다. 이 연구는 녹음을 보관하지 않으며, 보내기 전에 텍스트를 수정할 수 있습니다.', browser: '마이크를 사용하면 브라우저의 음성 서비스가 말을 텍스트로 변환합니다. Chrome에서는 Google, Safari에서는 Apple이 각자의 약관에 따라 처리합니다. 보내기 전에 텍스트를 수정할 수 있습니다.', }, @@ -98,7 +98,7 @@ export const ko: Messages = { sendShortcut: '⌘/Ctrl + Enter로 보내기', voice: { preparing: '이 기기에서 음성 입력을 준비하고 있습니다…', - deviceUnavailable: '이 기기에서는 음성 입력을 사용할 수 없습니다. 답변을 입력해 주세요.', + deviceUnavailable: '이 기기에서는 음성 입력을 사용할 수 없습니다. 응답을 입력해 주세요.', start: '음성 입력 시작', stop: '녹음 중지', recording: (elapsed) => `녹음 중 ${elapsed} (최대 1:00)`, diff --git a/src/lib/i18n/messages/zh.ts b/src/lib/i18n/messages/zh.ts index 3cf4d0a..08720b0 100644 --- a/src/lib/i18n/messages/zh.ts +++ b/src/lib/i18n/messages/zh.ts @@ -97,8 +97,8 @@ export const zh: Messages = { send: '发送', sendShortcut: '按 ⌘/Ctrl + Enter 发送', voice: { - preparing: '正在为这台设备准备语音输入…', - deviceUnavailable: '这台设备无法使用语音输入。请键入您的回答。', + preparing: '正在这台设备上准备语音输入…', + deviceUnavailable: '这台设备无法使用语音输入。请直接输入您的回答。', start: '开始语音输入', stop: '停止录音', recording: (elapsed) => `正在录音 ${elapsed}（最长 1:00）`,