From 77af609da3b0a87fbedb7f8fa72364ed8ea438c7 Mon Sep 17 00:00:00 2001 From: ekko <152005280+EKKOLearnAI@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:44:32 +0800 Subject: [PATCH] add cross-platform PCM voice capture (#2084) --- .../2026-07-15-vad-realtime-voice-capture.md | 10 + packages/client/src/api/hermes/stt.ts | 11 +- .../hermes/chat/RealtimeVoiceStage.vue | 190 +++++++-- .../hermes/settings/VoiceSettings.vue | 12 +- .../src/composables/usePcmStreamRecorder.ts | 368 ++++++++++++++++++ packages/client/src/i18n/locales/de.ts | 2 +- packages/client/src/i18n/locales/en.ts | 1 + packages/client/src/i18n/locales/es.ts | 2 +- packages/client/src/i18n/locales/fr.ts | 2 +- packages/client/src/i18n/locales/ja.ts | 2 +- packages/client/src/i18n/locales/ko.ts | 2 +- packages/client/src/i18n/locales/pt.ts | 2 +- packages/client/src/i18n/locales/ru.ts | 2 +- packages/client/src/i18n/locales/zh-TW.ts | 1 + packages/client/src/i18n/locales/zh.ts | 1 + packages/desktop/electron-builder.yml | 2 + packages/desktop/src/main/index.ts | 41 +- .../realtime-voice-playback-queue.test.ts | 272 ++++++++++++- tests/client/stt-settings-api.test.ts | 23 ++ tests/client/stt-settings-ui.test.ts | 44 ++- tests/client/use-pcm-stream-recorder.test.ts | 199 ++++++++++ 21 files changed, 1124 insertions(+), 65 deletions(-) create mode 100644 docs/chat-chain-changes/2026-07-15-vad-realtime-voice-capture.md create mode 100644 packages/client/src/composables/usePcmStreamRecorder.ts create mode 100644 tests/client/use-pcm-stream-recorder.test.ts diff --git a/docs/chat-chain-changes/2026-07-15-vad-realtime-voice-capture.md b/docs/chat-chain-changes/2026-07-15-vad-realtime-voice-capture.md new file mode 100644 index 00000000..37a689a3 --- /dev/null +++ b/docs/chat-chain-changes/2026-07-15-vad-realtime-voice-capture.md @@ -0,0 +1,10 @@ +--- +date: 2026-07-15 +pr: pending +feature: VAD realtime voice capture +impact: Mobile and Electron realtime voice now capture 16 kHz PCM/WAV with voice-activity-driven segments, while desktop browsers retain Web Speech recognition. +--- + +Mobile browsers and the macOS, Windows, and Linux desktop shells use the configured backend STT provider for dynamically segmented WAV transcription. Capture warms up before requiring sustained voice activity, natural pauses produce incremental captions, pure silence and provider no-speech responses are discarded, and 1.5 seconds of silence commits the complete turn on every platform. Manual microphone stops remain paused instead of immediately restarting, the submitted transcript remains visible while the agent thinks, and only replies from the current voice turn enter the speech playback queue. + +The STT settings test also records a single PCM/WAV sample instead of using platform-specific MediaRecorder output. This avoids Windows WebM/Opus and macOS MP4/AAC compatibility failures plus Electron's unavailable browser speech network service without changing the existing desktop-browser path. Desktop permission handling now requests microphone access through Electron on macOS and permits the main renderer's audio request on Windows and Linux; packaged macOS builds include the required microphone usage description. diff --git a/packages/client/src/api/hermes/stt.ts b/packages/client/src/api/hermes/stt.ts index 983c2cde..eb90c856 100644 --- a/packages/client/src/api/hermes/stt.ts +++ b/packages/client/src/api/hermes/stt.ts @@ -22,7 +22,7 @@ export async function transcribeSpeech(req: TranscribeSpeechRequest): Promise void } -const SILENCE_COMMIT_MS = 2_000 +const SILENCE_COMMIT_MS = 1_500 +const BACKEND_STREAM_VOICE_LEVEL = 0.035 +const BACKEND_STREAM_END_SILENCE_MS = 700 +const BACKEND_STREAM_MIN_SEGMENT_MS = 1_800 +const BACKEND_STREAM_MAX_SEGMENT_MS = 8_000 const MAX_CONCURRENT_TTS_SYNTHESIS = 5 const emit = defineEmits<{ @@ -54,13 +60,24 @@ const micRecorder = useMicRecorder({ recordingFailed: t('chat.voiceInput.microphoneRecordingFailed'), }, }) +const pcmStreamRecorder = usePcmStreamRecorder({ + minSegmentDurationMs: BACKEND_STREAM_MIN_SEGMENT_MS, + speechEndSilenceMs: BACKEND_STREAM_END_SILENCE_MS, + maxSegmentDurationMs: BACKEND_STREAM_MAX_SEGMENT_MS, + voiceActivityThreshold: BACKEND_STREAM_VOICE_LEVEL, + onChunk: queueBackendStreamChunk, + messages: { + unsupported: t('chat.voiceInput.microphoneUnsupported'), + recordingFailed: t('chat.voiceInput.microphoneRecordingFailed'), + }, +}) const mode = ref('idle') const sessionActive = ref(false) const submittedTranscript = ref('') +const backendStreamTranscript = ref('') const errorMessage = ref('') const waitingForResponse = ref(false) -const manualCapture = ref(false) const responseStartedAt = ref(0) const audioLevel = ref(0) const segmentAudioPlaying = ref(false) @@ -86,8 +103,11 @@ let speechQueueRunning = false let activeSynthesisCount = 0 let activeBackendSetting: SttProviderSettingsResponse | null = null let backendSettingPromise: Promise | null = null -let activeCaptureMode: 'browser' | 'backend' | null = null +let activeCaptureMode: 'browser' | 'backend-stream' | null = null let handlingRecognitionFailure = false +let backendStreamGeneration = 0 +let backendStreamQueue: Promise = Promise.resolve() +let backendStreamFailure: unknown = null const speechQueueIdleWaiters = new Set<() => void>() const synthesisControllers = new Set() const synthesisJobs: SpeechSynthesisJob[] = [] @@ -99,6 +119,7 @@ const currentTranscript = computed(() => { const liveTranscript = normalizeText([ browserRecognition.transcript.value, browserRecognition.partialTranscript.value, + backendStreamTranscript.value, ].filter(Boolean).join(' ')) return liveTranscript || submittedTranscript.value }) @@ -122,9 +143,6 @@ const statusLabel = computed(() => t(`realtimeVoice.status.${mode.value}`, { agent: agentDisplayName.value, })) const statusHint = computed(() => { - if (mode.value === 'listening' && manualCapture.value) { - return t('realtimeVoice.hint.listeningManual') - } return t(`realtimeVoice.hint.${mode.value}`) }) const sessionTitle = computed(() => chatStore.activeSession?.title?.trim() || t('realtimeVoice.untitledSession')) @@ -133,7 +151,10 @@ const displayCaption = computed(() => { if (mode.value === 'speaking' && activeSpeechSegment.value) { return activeSpeechSegment.value.subtitleText } - if ((mode.value === 'listening' || mode.value === 'processing') && currentTranscript.value) { + if ( + (mode.value === 'listening' || mode.value === 'processing' || mode.value === 'thinking') + && currentTranscript.value + ) { return currentTranscript.value } return statusHint.value @@ -206,6 +227,10 @@ function browserCaptureContinuous() { return !isMobileDevice() } +function shouldUseBackendStream(setting: SttProviderSettingsResponse | null) { + return Boolean(setting && (isDesktopShell() || isMobileDevice())) +} + function clearTimers() { if (silenceTimer) clearTimeout(silenceTimer) if (restartTimer) clearTimeout(restartTimer) @@ -433,10 +458,72 @@ async function loadActiveBackendSetting() { } function setError(cause: unknown) { + if (isNoSpeechError(cause)) { + errorMessage.value = '' + if (mode.value !== 'listening') mode.value = 'idle' + return + } errorMessage.value = cause instanceof Error ? cause.message : String(cause) mode.value = 'error' } +function isNoSpeechError(cause: unknown) { + const message = cause instanceof Error ? cause.message : String(cause) + return /no[_ -]?speech|no speech detected/i.test(message) +} + +function resetBackendStream() { + backendStreamGeneration += 1 + backendStreamQueue = Promise.resolve() + backendStreamFailure = null + backendStreamTranscript.value = '' +} + +function queueBackendStreamChunk(audio: Blob) { + const setting = activeBackendSetting + const generation = backendStreamGeneration + if (!setting || audio.size <= 44) return backendStreamQueue + + const settings = setting.settings + backendStreamQueue = backendStreamQueue.then(async () => { + if (generation !== backendStreamGeneration) return + try { + const result = await transcribeSpeech({ + audio, + provider: setting.provider, + language: typeof settings.language === 'string' ? settings.language : undefined, + prompt: typeof settings.prompt === 'string' ? settings.prompt : undefined, + }) + if (generation !== backendStreamGeneration) return + const text = normalizeText(result.text) + if (text) { + backendStreamTranscript.value = normalizeText(`${backendStreamTranscript.value} ${text}`) + } + } catch (cause) { + if (generation !== backendStreamGeneration || isNoSpeechError(cause)) return + backendStreamFailure = cause + throw cause + } + }).catch((cause) => { + if (generation !== backendStreamGeneration || isNoSpeechError(cause)) return + backendStreamFailure = cause + pcmStreamRecorder.cancel() + activeCaptureMode = null + stopVisualizer() + setError(cause) + }) + + return backendStreamQueue +} + +async function startBackendStreamCapture() { + resetBackendStream() + activeCaptureMode = 'backend-stream' + await pcmStreamRecorder.start() + mode.value = 'listening' + void startVisualizer(pcmStreamRecorder.stream.value) +} + async function startCapture() { clearTimers() if (!chatStore.activeSessionId) { @@ -452,18 +539,24 @@ async function startCapture() { browserRecognition.clearError() errorMessage.value = '' submittedTranscript.value = '' + backendStreamTranscript.value = '' activeSpeechSegment.value = null responseStartedAt.value = 0 sessionActive.value = true activeBackendSetting = await loadActiveBackendSetting() - manualCapture.value = Boolean(activeBackendSetting && isMobileDevice()) - activeCaptureMode = manualCapture.value ? 'backend' : 'browser' + + if ((isDesktopShell() || isMobileDevice()) && !activeBackendSetting) { + activeCaptureMode = null + setError(t('realtimeVoice.backendStreamRequired')) + return + } + + const backendStream = shouldUseBackendStream(activeBackendSetting) + activeCaptureMode = backendStream ? 'backend-stream' : 'browser' try { - if (manualCapture.value) { - await micRecorder.start() - mode.value = 'listening' - void startVisualizer(micRecorder.stream.value) + if (backendStream) { + await startBackendStreamCapture() return } @@ -524,6 +617,14 @@ async function transcribeBackendCapture(audio: Blob, setting: SttProviderSetting }) await submitTranscript(result.text) } catch (cause) { + if (isNoSpeechError(cause)) { + errorMessage.value = '' + activeCaptureMode = null + stopVisualizer() + mode.value = 'idle' + scheduleRestart() + return + } setError(cause) } } @@ -570,20 +671,35 @@ async function handleRecognitionFailure() { } } -async function stopCapture() { +async function stopCapture(manual = false) { if (mode.value !== 'listening') return clearTimers() mode.value = 'processing' - if (activeCaptureMode === 'backend' && activeBackendSetting) { + if (activeCaptureMode === 'backend-stream' && activeBackendSetting) { try { - const audio = await micRecorder.stop() + const finalChunk = await pcmStreamRecorder.stop() stopVisualizer() + if (finalChunk) queueBackendStreamChunk(finalChunk) + await backendStreamQueue + if (backendStreamFailure) return activeCaptureMode = null - await transcribeBackendCapture(audio, activeBackendSetting) + if (!backendStreamTranscript.value) { + mode.value = manual ? 'paused' : 'idle' + if (!manual) scheduleRestart() + return + } + await submitTranscript(backendStreamTranscript.value) } catch (cause) { + if (isNoSpeechError(cause)) { + activeCaptureMode = null + stopVisualizer() + mode.value = manual ? 'paused' : 'idle' + if (!manual) scheduleRestart() + return + } activeCaptureMode = null - micRecorder.cancel() + pcmStreamRecorder.cancel() stopVisualizer() setError(cause) } @@ -608,6 +724,11 @@ async function stopCapture() { } activeCaptureMode = null stopVisualizer() + if (!normalizeText(transcript)) { + mode.value = manual ? 'paused' : 'idle' + if (!manual) scheduleRestart() + return + } await submitTranscript(transcript) } @@ -615,6 +736,8 @@ async function stopActiveTurn() { clearTimers() browserRecognition.cancel() micRecorder.cancel() + pcmStreamRecorder.cancel() + backendStreamGeneration += 1 activeCaptureMode = null stopVisualizer() waitingForResponse.value = false @@ -626,7 +749,7 @@ async function stopActiveTurn() { async function toggleCapture() { if (mode.value === 'listening') { - await stopCapture() + await stopCapture(true) return } if (mode.value === 'thinking' || mode.value === 'speaking') { @@ -638,10 +761,6 @@ async function toggleCapture() { function scheduleRestart(delay = 420) { if (!sessionActive.value) return - if (manualCapture.value) { - mode.value = 'idle' - return - } if (restartTimer) clearTimeout(restartTimer) restartTimer = setTimeout(() => { restartTimer = null @@ -674,7 +793,7 @@ function scheduleSilenceCommit() { if (silenceTimer) clearTimeout(silenceTimer) silenceTimer = setTimeout(() => { silenceTimer = null - if (mode.value === 'listening') void stopCapture() + if (mode.value === 'listening') void stopCapture(false) }, SILENCE_COMMIT_MS) } @@ -845,8 +964,9 @@ function processResponseMessages() { } function finishResponseIfReady() { + if (!waitingForResponse.value) return processResponseMessages() - if (!waitingForResponse.value || chatStore.isStreaming || responseFinalizing) return + if (chatStore.isStreaming || responseFinalizing) return const responseMessages = chatStore.messages.slice(responseStartMessageIndex) const response = [...responseMessages].reverse().find(message => message.role === 'assistant' && Boolean(message.content.trim()), @@ -887,6 +1007,8 @@ function closeStage() { clearTimers() browserRecognition.cancel() micRecorder.cancel() + pcmStreamRecorder.cancel() + backendStreamGeneration += 1 activeCaptureMode = null stopVisualizer() speech.stop(true) @@ -894,7 +1016,16 @@ function closeStage() { } watch(currentTranscript, (value) => { - if (mode.value !== 'listening' || !value) return + if (mode.value !== 'listening' || activeCaptureMode === 'backend-stream' || !value) return + scheduleSilenceCommit() +}) +watch(() => pcmStreamRecorder.level.value, (value) => { + if ( + mode.value !== 'listening' + || activeCaptureMode !== 'backend-stream' + || !pcmStreamRecorder.hasSpeech.value + || value < BACKEND_STREAM_VOICE_LEVEL + ) return scheduleSilenceCommit() }) watch(() => browserRecognition.error.value, () => { @@ -917,8 +1048,7 @@ onMounted(() => { void loadActiveBackendSetting().then((setting) => { if (!sessionActive.value) return activeBackendSetting = setting - manualCapture.value = Boolean(setting && isMobileDevice()) - if (!manualCapture.value) void startCapture() + void startCapture() }) }, 180) }) @@ -933,6 +1063,8 @@ onBeforeUnmount(() => { clearTimers() browserRecognition.cancel() micRecorder.cancel() + pcmStreamRecorder.cancel() + backendStreamGeneration += 1 activeCaptureMode = null stopVisualizer() speech.stop(true) diff --git a/packages/client/src/components/hermes/settings/VoiceSettings.vue b/packages/client/src/components/hermes/settings/VoiceSettings.vue index 08110461..19295d42 100644 --- a/packages/client/src/components/hermes/settings/VoiceSettings.vue +++ b/packages/client/src/components/hermes/settings/VoiceSettings.vue @@ -1,9 +1,9 @@