add cross-platform PCM voice capture (#2084)

This commit is contained in:
ekko
2026-07-15 15:44:32 +08:00
committed by GitHub
parent 0dabbda394
commit 77af609da3
21 changed files with 1124 additions and 65 deletions
@@ -0,0 +1,10 @@
---
date: 2026-07-15
pr: pending
feature: VAD realtime voice capture
impact: Mobile and Electron realtime voice now capture 16 kHz PCM/WAV with voice-activity-driven segments, while desktop browsers retain Web Speech recognition.
---
Mobile browsers and the macOS, Windows, and Linux desktop shells use the configured backend STT provider for dynamically segmented WAV transcription. Capture warms up before requiring sustained voice activity, natural pauses produce incremental captions, pure silence and provider no-speech responses are discarded, and 1.5 seconds of silence commits the complete turn on every platform. Manual microphone stops remain paused instead of immediately restarting, the submitted transcript remains visible while the agent thinks, and only replies from the current voice turn enter the speech playback queue.
The STT settings test also records a single PCM/WAV sample instead of using platform-specific MediaRecorder output. This avoids Windows WebM/Opus and macOS MP4/AAC compatibility failures plus Electron's unavailable browser speech network service without changing the existing desktop-browser path. Desktop permission handling now requests microphone access through Electron on macOS and permits the main renderer's audio request on Windows and Linux; packaged macOS builds include the required microphone usage description.
+10 -1
View File
@@ -22,7 +22,7 @@ export async function transcribeSpeech(req: TranscribeSpeechRequest): Promise<Tr
}
const formData = new FormData()
formData.append('audio', req.audio, 'speech.webm')
formData.append('audio', req.audio, speechFileName(req.audio.type))
formData.append('provider', req.provider)
if (typeof req.language === 'string' && req.language) {
@@ -38,3 +38,12 @@ export async function transcribeSpeech(req: TranscribeSpeechRequest): Promise<Tr
body: formData,
})
}
function speechFileName(mimeType: string) {
const normalized = mimeType.split(';')[0]?.trim().toLowerCase()
if (normalized === 'audio/wav' || normalized === 'audio/x-wav' || normalized === 'audio/wave') return 'speech.wav'
if (normalized === 'audio/mpeg' || normalized === 'audio/mp3') return 'speech.mp3'
if (normalized === 'audio/mp4' || normalized === 'audio/x-m4a') return 'speech.m4a'
if (normalized === 'audio/ogg') return 'speech.ogg'
return 'speech.webm'
}
@@ -7,13 +7,15 @@ import { fetchSttSettings, type SttProviderSettingsResponse } from '@/api/hermes
import { synthesizeSpeech } from '@/api/hermes/tts'
import { useBrowserSpeechRecognition } from '@/composables/useBrowserSpeechRecognition'
import { useMicRecorder } from '@/composables/useMicRecorder'
import { usePcmStreamRecorder } from '@/composables/usePcmStreamRecorder'
import { useGlobalSpeech } from '@/composables/useSpeech'
import { useSttSettings } from '@/composables/useSttSettings'
import { useVoiceSettings } from '@/composables/useVoiceSettings'
import { useChatStore, type Message } from '@/stores/hermes/chat'
import { isDesktopShell } from '@/utils/desktop-bridge'
import { hzToEdgePitch, speedToEdgeRate } from '@/utils/ttsHelpers'
type VoiceStageMode = 'idle' | 'listening' | 'processing' | 'thinking' | 'speaking' | 'error'
type VoiceStageMode = 'idle' | 'paused' | 'listening' | 'processing' | 'thinking' | 'speaking' | 'error'
type PreparedSpeech =
| { ok: true; audio: Blob }
| { ok: false; error: unknown }
@@ -29,7 +31,11 @@ type SpeechSynthesisJob = {
text: string
resolve: (result: PreparedSpeech) => void
}
const SILENCE_COMMIT_MS = 2_000
const SILENCE_COMMIT_MS = 1_500
const BACKEND_STREAM_VOICE_LEVEL = 0.035
const BACKEND_STREAM_END_SILENCE_MS = 700
const BACKEND_STREAM_MIN_SEGMENT_MS = 1_800
const BACKEND_STREAM_MAX_SEGMENT_MS = 8_000
const MAX_CONCURRENT_TTS_SYNTHESIS = 5
const emit = defineEmits<{
@@ -54,13 +60,24 @@ const micRecorder = useMicRecorder({
recordingFailed: t('chat.voiceInput.microphoneRecordingFailed'),
},
})
const pcmStreamRecorder = usePcmStreamRecorder({
minSegmentDurationMs: BACKEND_STREAM_MIN_SEGMENT_MS,
speechEndSilenceMs: BACKEND_STREAM_END_SILENCE_MS,
maxSegmentDurationMs: BACKEND_STREAM_MAX_SEGMENT_MS,
voiceActivityThreshold: BACKEND_STREAM_VOICE_LEVEL,
onChunk: queueBackendStreamChunk,
messages: {
unsupported: t('chat.voiceInput.microphoneUnsupported'),
recordingFailed: t('chat.voiceInput.microphoneRecordingFailed'),
},
})
const mode = ref<VoiceStageMode>('idle')
const sessionActive = ref(false)
const submittedTranscript = ref('')
const backendStreamTranscript = ref('')
const errorMessage = ref('')
const waitingForResponse = ref(false)
const manualCapture = ref(false)
const responseStartedAt = ref(0)
const audioLevel = ref(0)
const segmentAudioPlaying = ref(false)
@@ -86,8 +103,11 @@ let speechQueueRunning = false
let activeSynthesisCount = 0
let activeBackendSetting: SttProviderSettingsResponse | null = null
let backendSettingPromise: Promise<SttProviderSettingsResponse | null> | null = null
let activeCaptureMode: 'browser' | 'backend' | null = null
let activeCaptureMode: 'browser' | 'backend-stream' | null = null
let handlingRecognitionFailure = false
let backendStreamGeneration = 0
let backendStreamQueue: Promise<void> = Promise.resolve()
let backendStreamFailure: unknown = null
const speechQueueIdleWaiters = new Set<() => void>()
const synthesisControllers = new Set<AbortController>()
const synthesisJobs: SpeechSynthesisJob[] = []
@@ -99,6 +119,7 @@ const currentTranscript = computed(() => {
const liveTranscript = normalizeText([
browserRecognition.transcript.value,
browserRecognition.partialTranscript.value,
backendStreamTranscript.value,
].filter(Boolean).join(' '))
return liveTranscript || submittedTranscript.value
})
@@ -122,9 +143,6 @@ const statusLabel = computed(() => t(`realtimeVoice.status.${mode.value}`, {
agent: agentDisplayName.value,
}))
const statusHint = computed(() => {
if (mode.value === 'listening' && manualCapture.value) {
return t('realtimeVoice.hint.listeningManual')
}
return t(`realtimeVoice.hint.${mode.value}`)
})
const sessionTitle = computed(() => chatStore.activeSession?.title?.trim() || t('realtimeVoice.untitledSession'))
@@ -133,7 +151,10 @@ const displayCaption = computed(() => {
if (mode.value === 'speaking' && activeSpeechSegment.value) {
return activeSpeechSegment.value.subtitleText
}
if ((mode.value === 'listening' || mode.value === 'processing') && currentTranscript.value) {
if (
(mode.value === 'listening' || mode.value === 'processing' || mode.value === 'thinking')
&& currentTranscript.value
) {
return currentTranscript.value
}
return statusHint.value
@@ -206,6 +227,10 @@ function browserCaptureContinuous() {
return !isMobileDevice()
}
function shouldUseBackendStream(setting: SttProviderSettingsResponse | null) {
return Boolean(setting && (isDesktopShell() || isMobileDevice()))
}
function clearTimers() {
if (silenceTimer) clearTimeout(silenceTimer)
if (restartTimer) clearTimeout(restartTimer)
@@ -433,10 +458,72 @@ async function loadActiveBackendSetting() {
}
function setError(cause: unknown) {
if (isNoSpeechError(cause)) {
errorMessage.value = ''
if (mode.value !== 'listening') mode.value = 'idle'
return
}
errorMessage.value = cause instanceof Error ? cause.message : String(cause)
mode.value = 'error'
}
function isNoSpeechError(cause: unknown) {
const message = cause instanceof Error ? cause.message : String(cause)
return /no[_ -]?speech|no speech detected/i.test(message)
}
function resetBackendStream() {
backendStreamGeneration += 1
backendStreamQueue = Promise.resolve()
backendStreamFailure = null
backendStreamTranscript.value = ''
}
function queueBackendStreamChunk(audio: Blob) {
const setting = activeBackendSetting
const generation = backendStreamGeneration
if (!setting || audio.size <= 44) return backendStreamQueue
const settings = setting.settings
backendStreamQueue = backendStreamQueue.then(async () => {
if (generation !== backendStreamGeneration) return
try {
const result = await transcribeSpeech({
audio,
provider: setting.provider,
language: typeof settings.language === 'string' ? settings.language : undefined,
prompt: typeof settings.prompt === 'string' ? settings.prompt : undefined,
})
if (generation !== backendStreamGeneration) return
const text = normalizeText(result.text)
if (text) {
backendStreamTranscript.value = normalizeText(`${backendStreamTranscript.value} ${text}`)
}
} catch (cause) {
if (generation !== backendStreamGeneration || isNoSpeechError(cause)) return
backendStreamFailure = cause
throw cause
}
}).catch((cause) => {
if (generation !== backendStreamGeneration || isNoSpeechError(cause)) return
backendStreamFailure = cause
pcmStreamRecorder.cancel()
activeCaptureMode = null
stopVisualizer()
setError(cause)
})
return backendStreamQueue
}
async function startBackendStreamCapture() {
resetBackendStream()
activeCaptureMode = 'backend-stream'
await pcmStreamRecorder.start()
mode.value = 'listening'
void startVisualizer(pcmStreamRecorder.stream.value)
}
async function startCapture() {
clearTimers()
if (!chatStore.activeSessionId) {
@@ -452,18 +539,24 @@ async function startCapture() {
browserRecognition.clearError()
errorMessage.value = ''
submittedTranscript.value = ''
backendStreamTranscript.value = ''
activeSpeechSegment.value = null
responseStartedAt.value = 0
sessionActive.value = true
activeBackendSetting = await loadActiveBackendSetting()
manualCapture.value = Boolean(activeBackendSetting && isMobileDevice())
activeCaptureMode = manualCapture.value ? 'backend' : 'browser'
if ((isDesktopShell() || isMobileDevice()) && !activeBackendSetting) {
activeCaptureMode = null
setError(t('realtimeVoice.backendStreamRequired'))
return
}
const backendStream = shouldUseBackendStream(activeBackendSetting)
activeCaptureMode = backendStream ? 'backend-stream' : 'browser'
try {
if (manualCapture.value) {
await micRecorder.start()
mode.value = 'listening'
void startVisualizer(micRecorder.stream.value)
if (backendStream) {
await startBackendStreamCapture()
return
}
@@ -524,6 +617,14 @@ async function transcribeBackendCapture(audio: Blob, setting: SttProviderSetting
})
await submitTranscript(result.text)
} catch (cause) {
if (isNoSpeechError(cause)) {
errorMessage.value = ''
activeCaptureMode = null
stopVisualizer()
mode.value = 'idle'
scheduleRestart()
return
}
setError(cause)
}
}
@@ -570,20 +671,35 @@ async function handleRecognitionFailure() {
}
}
async function stopCapture() {
async function stopCapture(manual = false) {
if (mode.value !== 'listening') return
clearTimers()
mode.value = 'processing'
if (activeCaptureMode === 'backend' && activeBackendSetting) {
if (activeCaptureMode === 'backend-stream' && activeBackendSetting) {
try {
const audio = await micRecorder.stop()
const finalChunk = await pcmStreamRecorder.stop()
stopVisualizer()
if (finalChunk) queueBackendStreamChunk(finalChunk)
await backendStreamQueue
if (backendStreamFailure) return
activeCaptureMode = null
await transcribeBackendCapture(audio, activeBackendSetting)
if (!backendStreamTranscript.value) {
mode.value = manual ? 'paused' : 'idle'
if (!manual) scheduleRestart()
return
}
await submitTranscript(backendStreamTranscript.value)
} catch (cause) {
if (isNoSpeechError(cause)) {
activeCaptureMode = null
stopVisualizer()
mode.value = manual ? 'paused' : 'idle'
if (!manual) scheduleRestart()
return
}
activeCaptureMode = null
micRecorder.cancel()
pcmStreamRecorder.cancel()
stopVisualizer()
setError(cause)
}
@@ -608,6 +724,11 @@ async function stopCapture() {
}
activeCaptureMode = null
stopVisualizer()
if (!normalizeText(transcript)) {
mode.value = manual ? 'paused' : 'idle'
if (!manual) scheduleRestart()
return
}
await submitTranscript(transcript)
}
@@ -615,6 +736,8 @@ async function stopActiveTurn() {
clearTimers()
browserRecognition.cancel()
micRecorder.cancel()
pcmStreamRecorder.cancel()
backendStreamGeneration += 1
activeCaptureMode = null
stopVisualizer()
waitingForResponse.value = false
@@ -626,7 +749,7 @@ async function stopActiveTurn() {
async function toggleCapture() {
if (mode.value === 'listening') {
await stopCapture()
await stopCapture(true)
return
}
if (mode.value === 'thinking' || mode.value === 'speaking') {
@@ -638,10 +761,6 @@ async function toggleCapture() {
function scheduleRestart(delay = 420) {
if (!sessionActive.value) return
if (manualCapture.value) {
mode.value = 'idle'
return
}
if (restartTimer) clearTimeout(restartTimer)
restartTimer = setTimeout(() => {
restartTimer = null
@@ -674,7 +793,7 @@ function scheduleSilenceCommit() {
if (silenceTimer) clearTimeout(silenceTimer)
silenceTimer = setTimeout(() => {
silenceTimer = null
if (mode.value === 'listening') void stopCapture()
if (mode.value === 'listening') void stopCapture(false)
}, SILENCE_COMMIT_MS)
}
@@ -845,8 +964,9 @@ function processResponseMessages() {
}
function finishResponseIfReady() {
if (!waitingForResponse.value) return
processResponseMessages()
if (!waitingForResponse.value || chatStore.isStreaming || responseFinalizing) return
if (chatStore.isStreaming || responseFinalizing) return
const responseMessages = chatStore.messages.slice(responseStartMessageIndex)
const response = [...responseMessages].reverse().find(message =>
message.role === 'assistant' && Boolean(message.content.trim()),
@@ -887,6 +1007,8 @@ function closeStage() {
clearTimers()
browserRecognition.cancel()
micRecorder.cancel()
pcmStreamRecorder.cancel()
backendStreamGeneration += 1
activeCaptureMode = null
stopVisualizer()
speech.stop(true)
@@ -894,7 +1016,16 @@ function closeStage() {
}
watch(currentTranscript, (value) => {
if (mode.value !== 'listening' || !value) return
if (mode.value !== 'listening' || activeCaptureMode === 'backend-stream' || !value) return
scheduleSilenceCommit()
})
watch(() => pcmStreamRecorder.level.value, (value) => {
if (
mode.value !== 'listening'
|| activeCaptureMode !== 'backend-stream'
|| !pcmStreamRecorder.hasSpeech.value
|| value < BACKEND_STREAM_VOICE_LEVEL
) return
scheduleSilenceCommit()
})
watch(() => browserRecognition.error.value, () => {
@@ -917,8 +1048,7 @@ onMounted(() => {
void loadActiveBackendSetting().then((setting) => {
if (!sessionActive.value) return
activeBackendSetting = setting
manualCapture.value = Boolean(setting && isMobileDevice())
if (!manualCapture.value) void startCapture()
void startCapture()
})
}, 180)
})
@@ -933,6 +1063,8 @@ onBeforeUnmount(() => {
clearTimers()
browserRecognition.cancel()
micRecorder.cancel()
pcmStreamRecorder.cancel()
backendStreamGeneration += 1
activeCaptureMode = null
stopVisualizer()
speech.stop(true)
@@ -1,9 +1,9 @@
<script setup lang="ts">
import { computed, onMounted, ref } from 'vue'
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
import { NButton, NInput, NSelect, useMessage } from 'naive-ui'
import { useI18n } from 'vue-i18n'
import { useSpeech, type MimoTtsOptions, type OpenaiTtsOptions } from '@/composables/useSpeech'
import { useMicRecorder } from '@/composables/useMicRecorder'
import { usePcmStreamRecorder } from '@/composables/usePcmStreamRecorder'
import { transcribeSpeech } from '@/api/hermes/stt'
import { useVoiceApiConnections } from '@/composables/useVoiceApiConnections'
import { useVoiceSettings } from '@/composables/useVoiceSettings'
@@ -31,7 +31,7 @@ const showAddModal = ref(false)
const addModalKind = ref<VoiceApiKind>('tts')
const showConfigurator = ref(false)
const editingConnection = ref<VoiceApiConnection | null>(null)
const sttRecorder = useMicRecorder({ maxDurationMs: 30_000 })
const sttRecorder = usePcmStreamRecorder({ voiceActivityThreshold: 0.02 })
const cardTestStates = ref<Record<string, VoiceApiCardTestState>>({})
const activeTtsDescription = computed(() => voiceApi.activeTtsConnection.value?.label || t('settings.voice.noneSelected'))
@@ -41,6 +41,10 @@ onMounted(async () => {
await voiceApi.refresh()
})
onBeforeUnmount(() => {
sttRecorder.cancel()
})
function openAddModal(kind: VoiceApiKind) {
addModalKind.value = kind
showAddModal.value = true
@@ -203,7 +207,7 @@ async function handleSttTest(connection: VoiceApiConnection) {
setCardTestState(connection.id, 'loading', t('settings.voice.transcribing'))
try {
const audio = await sttRecorder.stop()
if (!audio.size) {
if (!audio?.size) {
setCardTestState(connection.id, 'error', t('settings.voice.sttEmptyAudio'))
return
}
@@ -0,0 +1,368 @@
import { computed, ref, shallowRef } from 'vue'
export type PcmStreamRecorderStatus = 'idle' | 'requesting' | 'recording' | 'error'
export interface PcmStreamRecorderOptions {
minSegmentDurationMs?: number
maxSegmentDurationMs?: number
speechEndSilenceMs?: number
preRollMs?: number
targetSampleRate?: number
voiceActivityThreshold?: number
constraints?: MediaStreamConstraints
onChunk?: (audio: Blob) => void
messages?: {
unsupported?: string
recordingFailed?: string
}
}
const DEFAULT_MIN_SEGMENT_DURATION_MS = 700
const DEFAULT_MAX_SEGMENT_DURATION_MS = 8_000
const DEFAULT_SPEECH_END_SILENCE_MS = 700
const DEFAULT_PRE_ROLL_MS = 250
const DEFAULT_TARGET_SAMPLE_RATE = 16_000
const DEFAULT_VOICE_ACTIVITY_THRESHOLD = 0.035
const MIN_FINAL_CHUNK_MS = 350
const MIN_VOICE_ACTIVITY_MS = 350
const STREAM_WARMUP_MS = 500
const SUPPORT_ERROR_MESSAGE = 'Streaming microphone capture is not supported in this browser.'
export function encodePcmWav(
input: Float32Array,
sourceSampleRate: number,
targetSampleRate = DEFAULT_TARGET_SAMPLE_RATE,
): Blob {
const samples = normalizeSpeechLevel(resampleLinear(input, sourceSampleRate, targetSampleRate))
const buffer = new ArrayBuffer(44 + samples.length * 2)
const view = new DataView(buffer)
writeAscii(view, 0, 'RIFF')
view.setUint32(4, 36 + samples.length * 2, true)
writeAscii(view, 8, 'WAVE')
writeAscii(view, 12, 'fmt ')
view.setUint32(16, 16, true)
view.setUint16(20, 1, true)
view.setUint16(22, 1, true)
view.setUint32(24, targetSampleRate, true)
view.setUint32(28, targetSampleRate * 2, true)
view.setUint16(32, 2, true)
view.setUint16(34, 16, true)
writeAscii(view, 36, 'data')
view.setUint32(40, samples.length * 2, true)
for (let index = 0; index < samples.length; index += 1) {
const sample = Math.max(-1, Math.min(1, samples[index] || 0))
view.setInt16(44 + index * 2, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true)
}
return new Blob([buffer], { type: 'audio/wav' })
}
export function usePcmStreamRecorder(options: PcmStreamRecorderOptions = {}) {
const status = ref<PcmStreamRecorderStatus>('idle')
const error = ref<Error | null>(null)
const level = ref(0)
const hasSpeech = ref(false)
const stream = shallowRef<MediaStream | null>(null)
const isRecording = computed(() => status.value === 'recording')
let context: AudioContext | null = null
let source: MediaStreamAudioSourceNode | null = null
let processor: ScriptProcessorNode | null = null
let activeStream: MediaStream | null = null
let sampleRate = DEFAULT_TARGET_SAMPLE_RATE
let bufferedSamples: Float32Array[] = []
let bufferedSampleCount = 0
let bufferedSampleOffset = 0
let bufferedHasVoice = false
let voicedSampleCount = 0
let trailingSilenceSamples = 0
let capturedSampleCount = 0
let consecutiveVoiceSampleCount = 0
let sessionToken = 0
function setError(cause: unknown) {
const normalized = cause instanceof Error
? cause
: new Error(options.messages?.recordingFailed || String(cause))
error.value = normalized
status.value = 'error'
return normalized
}
function stopTracks(mediaStream: MediaStream | null) {
for (const track of mediaStream?.getTracks() || []) track.stop()
}
function clearAudioGraph() {
if (processor) processor.onaudioprocess = null
try { processor?.disconnect() } catch { /* already disconnected */ }
try { source?.disconnect() } catch { /* already disconnected */ }
processor = null
source = null
}
async function releaseResources() {
clearAudioGraph()
const closingContext = context
context = null
stopTracks(activeStream)
activeStream = null
stream.value = null
level.value = 0
if (closingContext && closingContext.state !== 'closed') {
await closingContext.close().catch(() => undefined)
}
}
function resetBuffer() {
bufferedSamples = []
bufferedSampleCount = 0
bufferedSampleOffset = 0
bufferedHasVoice = false
voicedSampleCount = 0
trailingSilenceSamples = 0
}
function resetCaptureState() {
resetBuffer()
capturedSampleCount = 0
consecutiveVoiceSampleCount = 0
hasSpeech.value = false
}
function takeSamples(count: number) {
const output = new Float32Array(count)
let outputOffset = 0
while (outputOffset < count && bufferedSamples.length > 0) {
const first = bufferedSamples[0]
const available = first.length - bufferedSampleOffset
const take = Math.min(available, count - outputOffset)
output.set(first.subarray(bufferedSampleOffset, bufferedSampleOffset + take), outputOffset)
outputOffset += take
bufferedSampleOffset += take
bufferedSampleCount -= take
if (bufferedSampleOffset >= first.length) {
bufferedSamples.shift()
bufferedSampleOffset = 0
}
}
return output
}
function emitReadySegment() {
const minSamples = sampleRate * Math.max(250, options.minSegmentDurationMs ?? DEFAULT_MIN_SEGMENT_DURATION_MS) / 1_000
const maxSamples = sampleRate * Math.max(1_000, options.maxSegmentDurationMs ?? DEFAULT_MAX_SEGMENT_DURATION_MS) / 1_000
const endSilenceSamples = sampleRate * Math.max(200, options.speechEndSilenceMs ?? DEFAULT_SPEECH_END_SILENCE_MS) / 1_000
const preRollSamples = sampleRate * Math.max(0, options.preRollMs ?? DEFAULT_PRE_ROLL_MS) / 1_000
const minVoiceSamples = sampleRate * MIN_VOICE_ACTIVITY_MS / 1_000
if (!bufferedHasVoice) {
// Single-shot capture (the settings STT test) keeps the whole recording;
// startup trimming is only useful for continuous realtime segmentation.
if (!options.onChunk) return
if (bufferedSampleCount > preRollSamples) takeSamples(bufferedSampleCount - Math.floor(preRollSamples))
return
}
// Without a chunk consumer this acts as a single-shot WAV recorder and
// keeps the complete utterance buffered until stop() is called.
if (!options.onChunk) return
const reachedNaturalPause = bufferedSampleCount >= minSamples
&& voicedSampleCount >= minVoiceSamples
&& trailingSilenceSamples >= endSilenceSamples
if (!reachedNaturalPause && bufferedSampleCount < maxSamples) return
if (voicedSampleCount < minVoiceSamples) {
resetBuffer()
return
}
const segment = takeSamples(bufferedSampleCount)
options.onChunk(encodePcmWav(segment, sampleRate, options.targetSampleRate))
bufferedHasVoice = false
voicedSampleCount = 0
trailingSilenceSamples = 0
}
function appendAudio(event: AudioProcessingEvent) {
const input = event.inputBuffer
if (!input.length || !input.numberOfChannels) return
const mono = new Float32Array(input.length)
let squareSum = 0
for (let channelIndex = 0; channelIndex < input.numberOfChannels; channelIndex += 1) {
const channel = input.getChannelData(channelIndex)
for (let sampleIndex = 0; sampleIndex < input.length; sampleIndex += 1) {
mono[sampleIndex] += channel[sampleIndex] / input.numberOfChannels
}
}
for (let index = 0; index < mono.length; index += 1) squareSum += mono[index] * mono[index]
const rms = Math.sqrt(squareSum / mono.length)
const nextLevel = Math.min(1, rms * 5)
level.value += (nextLevel - level.value) * 0.45
capturedSampleCount += mono.length
const warmupSamples = options.onChunk ? sampleRate * STREAM_WARMUP_MS / 1_000 : 0
const aboveVoiceThreshold = capturedSampleCount > warmupSamples
&& nextLevel >= (options.voiceActivityThreshold ?? DEFAULT_VOICE_ACTIVITY_THRESHOLD)
if (aboveVoiceThreshold) {
consecutiveVoiceSampleCount += mono.length
voicedSampleCount += mono.length
if (consecutiveVoiceSampleCount >= sampleRate * MIN_VOICE_ACTIVITY_MS / 1_000) {
hasSpeech.value = true
bufferedHasVoice = true
}
if (bufferedHasVoice) trailingSilenceSamples = 0
} else {
consecutiveVoiceSampleCount = 0
if (!bufferedHasVoice) voicedSampleCount = 0
else trailingSilenceSamples += mono.length
}
bufferedSamples.push(mono)
bufferedSampleCount += mono.length
emitReadySegment()
}
async function start() {
if (status.value === 'requesting' || status.value === 'recording') return
const AudioContextConstructor = window.AudioContext
if (!navigator.mediaDevices?.getUserMedia || !AudioContextConstructor) {
throw setError(new Error(options.messages?.unsupported || SUPPORT_ERROR_MESSAGE))
}
const token = ++sessionToken
status.value = 'requesting'
error.value = null
resetCaptureState()
try {
const mediaStream = await navigator.mediaDevices.getUserMedia(options.constraints ?? {
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
})
if (token !== sessionToken) {
stopTracks(mediaStream)
return
}
const audioContext = new AudioContextConstructor()
const inputSource = audioContext.createMediaStreamSource(mediaStream)
const scriptProcessor = audioContext.createScriptProcessor(4_096, 1, 1)
scriptProcessor.onaudioprocess = appendAudio
inputSource.connect(scriptProcessor)
// ScriptProcessor output buffers are silent unless explicitly written.
// Connecting directly keeps Chromium from optimizing away a zero-gain
// branch while still preventing microphone monitoring/echo.
scriptProcessor.connect(audioContext.destination)
context = audioContext
source = inputSource
processor = scriptProcessor
activeStream = mediaStream
stream.value = mediaStream
sampleRate = audioContext.sampleRate
await audioContext.resume()
if (token !== sessionToken) {
scriptProcessor.onaudioprocess = null
inputSource.disconnect()
scriptProcessor.disconnect()
stopTracks(mediaStream)
await audioContext.close().catch(() => undefined)
return
}
status.value = 'recording'
} catch (cause) {
if (token !== sessionToken) return
await releaseResources()
throw setError(cause)
}
}
async function stop() {
sessionToken += 1
const remainingDurationMs = bufferedSampleCount / sampleRate * 1_000
const finalChunk = remainingDurationMs >= MIN_FINAL_CHUNK_MS
&& bufferedHasVoice
&& voicedSampleCount >= sampleRate * MIN_VOICE_ACTIVITY_MS / 1_000
? encodePcmWav(takeSamples(bufferedSampleCount), sampleRate, options.targetSampleRate)
: null
resetCaptureState()
await releaseResources()
status.value = 'idle'
error.value = null
return finalChunk
}
function cancel() {
sessionToken += 1
resetCaptureState()
void releaseResources()
status.value = 'idle'
error.value = null
}
return {
status,
error,
level,
hasSpeech,
stream,
isRecording,
start,
stop,
cancel,
}
}
function resampleLinear(input: Float32Array, sourceRate: number, targetRate: number) {
if (!input.length || sourceRate === targetRate) return input
const length = Math.max(1, Math.round(input.length * targetRate / sourceRate))
const output = new Float32Array(length)
const ratio = sourceRate / targetRate
for (let index = 0; index < length; index += 1) {
const position = index * ratio
const left = Math.floor(position)
const right = Math.min(input.length - 1, left + 1)
const weight = position - left
output[index] = input[left] * (1 - weight) + input[right] * weight
}
return output
}
function normalizeSpeechLevel(input: Float32Array) {
if (!input.length) return input
let peak = 0
let squareSum = 0
for (const sample of input) {
const absolute = Math.abs(sample)
peak = Math.max(peak, absolute)
squareSum += sample * sample
}
const rms = Math.sqrt(squareSum / input.length)
if (peak <= 0 || rms <= 0) return input
const gain = Math.min(8, 0.92 / peak, 0.1 / rms)
if (gain <= 1.05) return input
const output = new Float32Array(input.length)
for (let index = 0; index < input.length; index += 1) {
output[index] = Math.max(-1, Math.min(1, input[index] * gain))
}
return output
}
function writeAscii(view: DataView, offset: number, value: string) {
for (let index = 0; index < value.length; index += 1) {
view.setUint8(offset + index, value.charCodeAt(index))
}
}
+1 -1
View File
@@ -484,7 +484,7 @@ export default {
realtimeVoice: {
title: 'Echtzeit-Sprache', mode: 'Sprachmodus', open: 'Echtzeit-Sprache öffnen', back: 'Zurück zum Chat', connected: 'Sprachverbindung bereit',
turnMode: 'Echtzeit-Vorschau · rundenbasiert', browserStt: 'Browser', browserTts: 'Browser', untitledSession: 'Unbenannte Unterhaltung',
sessionMissing: 'Die Unterhaltung ist nicht verfügbar.', networkUnavailableNoFallback: 'Die Browser-Spracherkennung ist offline und es ist kein nutzbares Ersatz-STT konfiguriert.', liveTranscript: 'Live-Transkript', transcriptPlaceholder: 'Sprachkern antippen und zu sprechen beginnen',
sessionMissing: 'Die Unterhaltung ist nicht verfügbar.', backendStreamRequired: 'Echtzeit-Sprache auf Mobilgeräten und dem Desktop erfordert einen konfigurierten serverseitigen Spracherkennungsdienst.', networkUnavailableNoFallback: 'Die Browser-Spracherkennung ist offline und es ist kein nutzbares Ersatz-STT konfiguriert.', liveTranscript: 'Live-Transkript', transcriptPlaceholder: 'Sprachkern antippen und zu sprechen beginnen',
recentTurns: 'Letzte Beiträge', you: 'Du', mute: 'Ausgabe stummschalten', unmute: 'Ton einschalten', finishTurn: 'Beitrag beenden', speak: 'Sprechen beginnen', end: 'Beenden',
status: { idle: 'Bereit', paused: 'Mikrofon aus', listening: 'Hört zu', processing: 'Transkribiert', thinking: '{agent} denkt nach', speaking: '{agent} spricht', error: 'Sprachverbindung unterbrochen' },
hint: { idle: 'Mitte antippen, um einen Sprachbeitrag zu starten', paused: 'Schalte das Mikrofon ein, wenn du bereit bist', listening: 'Zum Senden kurz pausieren oder zum Beenden tippen', listeningManual: 'Aufnahme läuft. Erneut in die Mitte tippen, um zu stoppen und zu transkribieren', processing: 'Deine Sprache wird in Text umgewandelt', thinking: 'Deine Nachricht wurde an die Unterhaltung gesendet', speaking: 'Du kannst jederzeit unterbrechen', error: 'Mitte antippen, um es erneut zu versuchen' },
+1
View File
@@ -492,6 +492,7 @@ export default {
browserTts: 'Browser',
untitledSession: 'Untitled conversation',
sessionMissing: 'The conversation is unavailable.',
backendStreamRequired: 'Realtime voice on mobile and desktop requires a configured backend speech-to-text provider.',
networkUnavailableNoFallback: 'Browser speech recognition is offline and no usable fallback STT is currently configured.',
liveTranscript: 'Live transcript',
transcriptPlaceholder: 'Tap the voice core and start speaking',
+1 -1
View File
@@ -484,7 +484,7 @@ export default {
realtimeVoice: {
title: 'Voz en tiempo real', mode: 'Modo de voz', open: 'Abrir voz en tiempo real', back: 'Volver al chat', connected: 'Enlace de voz listo',
turnMode: 'Vista previa en tiempo real · por turnos', browserStt: 'Navegador', browserTts: 'Navegador', untitledSession: 'Conversación sin título',
sessionMissing: 'La conversación no está disponible.', networkUnavailableNoFallback: 'El reconocimiento de voz del navegador no tiene conexión y no hay un STT alternativo utilizable configurado.', liveTranscript: 'Transcripción en directo', transcriptPlaceholder: 'Toca el núcleo de voz y empieza a hablar',
sessionMissing: 'La conversación no está disponible.', backendStreamRequired: 'La voz en tiempo real en móviles y escritorio requiere un proveedor de transcripción de servidor configurado.', networkUnavailableNoFallback: 'El reconocimiento de voz del navegador no tiene conexión y no hay un STT alternativo utilizable configurado.', liveTranscript: 'Transcripción en directo', transcriptPlaceholder: 'Toca el núcleo de voz y empieza a hablar',
recentTurns: 'Turnos recientes', you: 'Tú', mute: 'Silenciar salida', unmute: 'Activar sonido', finishTurn: 'Finalizar turno', speak: 'Empezar a hablar', end: 'Finalizar',
status: { idle: 'Listo', paused: 'Micrófono apagado', listening: 'Escuchando', processing: 'Transcribiendo', thinking: '{agent} está pensando', speaking: '{agent} está hablando', error: 'Enlace de voz interrumpido' },
hint: { idle: 'Toca el centro para iniciar un turno de voz', paused: 'Activa el micrófono cuando estés listo', listening: 'Haz una pausa para enviar o toca para finalizar', listeningManual: 'Grabando. Toca el centro otra vez para detener y transcribir', processing: 'Convirtiendo tu voz en texto', thinking: 'Tu mensaje se envió a la conversación actual', speaking: 'Puedes interrumpir en cualquier momento', error: 'Toca el centro para intentarlo de nuevo' },
+1 -1
View File
@@ -484,7 +484,7 @@ export default {
realtimeVoice: {
title: 'Voix en temps réel', mode: 'Mode vocal', open: 'Ouvrir la voix en temps réel', back: 'Retour au chat', connected: 'Liaison vocale prête',
turnMode: 'Aperçu temps réel · mode tour par tour', browserStt: 'Navigateur', browserTts: 'Navigateur', untitledSession: 'Conversation sans titre',
sessionMissing: 'La conversation est indisponible.', networkUnavailableNoFallback: 'La reconnaissance vocale du navigateur est hors ligne et aucun STT de secours utilisable n est configuré.', liveTranscript: 'Transcription en direct', transcriptPlaceholder: 'Touchez le noyau vocal et commencez à parler',
sessionMissing: 'La conversation est indisponible.', backendStreamRequired: 'La voix temps réel sur mobile et ordinateur nécessite un service de transcription côté serveur configuré.', networkUnavailableNoFallback: 'La reconnaissance vocale du navigateur est hors ligne et aucun STT de secours utilisable n est configuré.', liveTranscript: 'Transcription en direct', transcriptPlaceholder: 'Touchez le noyau vocal et commencez à parler',
recentTurns: 'Échanges récents', you: 'Vous', mute: 'Couper le son', unmute: 'Rétablir le son', finishTurn: 'Terminer le tour', speak: 'Commencer à parler', end: 'Terminer',
status: { idle: 'Prêt', paused: 'Microphone coupé', listening: 'Écoute', processing: 'Transcription', thinking: '{agent} réfléchit', speaking: '{agent} parle', error: 'Liaison vocale interrompue' },
hint: { idle: 'Touchez le centre pour commencer un tour vocal', paused: 'Réactivez le microphone quand vous êtes prêt', listening: 'Faites une pause pour envoyer ou touchez pour terminer', listeningManual: 'Enregistrement en cours. Touchez à nouveau le centre pour arrêter et transcrire', processing: 'Conversion de votre voix en texte', thinking: 'Votre message a été envoyé à la conversation', speaking: 'Vous pouvez interrompre à tout moment', error: 'Touchez le centre pour réessayer' },
+1 -1
View File
@@ -484,7 +484,7 @@ export default {
realtimeVoice: {
title: 'リアルタイム音声', mode: '音声モード', open: 'リアルタイム音声を開く', back: 'チャットに戻る', connected: '音声リンク準備完了',
turnMode: 'リアルタイムプレビュー · ターン方式', browserStt: 'ブラウザ', browserTts: 'ブラウザ', untitledSession: '無題の会話',
sessionMissing: '会話を利用できません。', networkUnavailableNoFallback: 'ブラウザ音声認識がオフラインで、利用可能な予備 STT も設定されていません。', liveTranscript: 'リアルタイム文字起こし', transcriptPlaceholder: '音声コアをタップして話し始めてください',
sessionMissing: '会話を利用できません。', backendStreamRequired: 'モバイルとデスクトップのリアルタイム音声には、設定済みのバックエンド音声認識サービスが必要です。', networkUnavailableNoFallback: 'ブラウザ音声認識がオフラインで、利用可能な予備 STT も設定されていません。', liveTranscript: 'リアルタイム文字起こし', transcriptPlaceholder: '音声コアをタップして話し始めてください',
recentTurns: '最近の会話', you: 'あなた', mute: '出力をミュート', unmute: 'ミュート解除', finishTurn: '発話を終了', speak: '話し始める', end: '終了',
status: { idle: '準備完了', paused: 'マイクはオフです', listening: '聞き取り中', processing: '文字起こし中', thinking: '{agent} が考えています', speaking: '{agent} が話しています', error: '音声リンクが中断しました' },
hint: { idle: '中央をタップして音声ターンを開始', paused: '準備ができたらマイクをオンにしてください', listening: '自然に間を置くと送信、またはタップで終了', listeningManual: '録音中です。中央をもう一度タップして停止し、文字起こしします', processing: '音声をテキストに変換しています', thinking: '現在の会話にメッセージを送信しました', speaking: 'いつでも中断できます', error: '中央をタップして再試行' },
+1 -1
View File
@@ -484,7 +484,7 @@ export default {
realtimeVoice: {
title: '실시간 음성', mode: '음성 모드', open: '실시간 음성 열기', back: '채팅으로 돌아가기', connected: '음성 연결 준비됨',
turnMode: '실시간 미리보기 · 턴 방식', browserStt: '브라우저', browserTts: '브라우저', untitledSession: '제목 없는 대화',
sessionMissing: '대화를 사용할 수 없습니다.', networkUnavailableNoFallback: '브라우저 음성 인식이 오프라인이며 사용 가능한 대체 STT가 설정되지 않았습니다.', liveTranscript: '실시간 텍스트', transcriptPlaceholder: '음성 코어를 누르고 말해 보세요',
sessionMissing: '대화를 사용할 수 없습니다.', backendStreamRequired: '모바일과 데스크톱 실시간 음성을 사용하려면 백엔드 음성 인식 서비스를 먼저 설정해야 합니다.', networkUnavailableNoFallback: '브라우저 음성 인식이 오프라인이며 사용 가능한 대체 STT가 설정되지 않았습니다.', liveTranscript: '실시간 텍스트', transcriptPlaceholder: '음성 코어를 누르고 말해 보세요',
recentTurns: '최근 대화', you: '나', mute: '출력 음소거', unmute: '음소거 해제', finishTurn: '말하기 완료', speak: '말하기 시작', end: '종료',
status: { idle: '준비됨', paused: '마이크 꺼짐', listening: '듣는 중', processing: '변환 중', thinking: '{agent}가 생각 중', speaking: '{agent}가 말하는 중', error: '음성 연결이 중단됨' },
hint: { idle: '중앙을 눌러 음성 턴을 시작하세요', paused: '준비되면 마이크를 다시 켜세요', listening: '자연스럽게 멈추면 전송되며 눌러서 끝낼 수도 있습니다', listeningManual: '녹음 중입니다. 중앙을 다시 눌러 중지하고 텍스트로 변환하세요', processing: '음성을 텍스트로 변환하고 있습니다', thinking: '현재 대화로 메시지를 보냈습니다', speaking: '언제든지 응답을 중단할 수 있습니다', error: '중앙을 눌러 다시 시도하세요' },
+1 -1
View File
@@ -484,7 +484,7 @@ export default {
realtimeVoice: {
title: 'Voz em tempo real', mode: 'Modo de voz', open: 'Abrir voz em tempo real', back: 'Voltar ao chat', connected: 'Conexão de voz pronta',
turnMode: 'Prévia em tempo real · por turnos', browserStt: 'Navegador', browserTts: 'Navegador', untitledSession: 'Conversa sem título',
sessionMissing: 'A conversa não está disponível.', networkUnavailableNoFallback: 'O reconhecimento de voz do navegador está offline e não há um STT alternativo utilizável configurado.', liveTranscript: 'Transcrição ao vivo', transcriptPlaceholder: 'Toque no núcleo de voz e comece a falar',
sessionMissing: 'A conversa não está disponível.', backendStreamRequired: 'A voz em tempo real no celular e no desktop requer um serviço de transcrição no servidor configurado.', networkUnavailableNoFallback: 'O reconhecimento de voz do navegador está offline e não há um STT alternativo utilizável configurado.', liveTranscript: 'Transcrição ao vivo', transcriptPlaceholder: 'Toque no núcleo de voz e comece a falar',
recentTurns: 'Turnos recentes', you: 'Você', mute: 'Silenciar saída', unmute: 'Ativar som', finishTurn: 'Concluir turno', speak: 'Começar a falar', end: 'Encerrar',
status: { idle: 'Pronto', paused: 'Microfone desligado', listening: 'Ouvindo', processing: 'Transcrevendo', thinking: '{agent} está pensando', speaking: '{agent} está falando', error: 'Conexão de voz interrompida' },
hint: { idle: 'Toque no centro para iniciar um turno de voz', paused: 'Ligue o microfone quando estiver pronto', listening: 'Faça uma pausa para enviar ou toque para concluir', listeningManual: 'Gravando. Toque novamente no centro para parar e transcrever', processing: 'Convertendo sua voz em texto', thinking: 'Sua mensagem foi enviada à conversa atual', speaking: 'Você pode interromper a qualquer momento', error: 'Toque no centro para tentar novamente' },
+1 -1
View File
@@ -411,7 +411,7 @@ export default {
realtimeVoice: {
title: 'Голос в реальном времени', mode: 'Голосовой режим', open: 'Открыть голосовой режим', back: 'Вернуться в чат', connected: 'Голосовая связь готова',
turnMode: 'Предпросмотр · по очереди', browserStt: 'Браузер', browserTts: 'Браузер', untitledSession: 'Разговор без названия',
sessionMissing: 'Разговор недоступен.', networkUnavailableNoFallback: 'Распознавание речи браузера недоступно по сети, и резервный STT не настроен.', liveTranscript: 'Живая расшифровка', transcriptPlaceholder: 'Нажмите на голосовое ядро и начните говорить',
sessionMissing: 'Разговор недоступен.', backendStreamRequired: 'Для голосового режима на мобильных устройствах и компьютере требуется настроенный серверный сервис распознавания речи.', networkUnavailableNoFallback: 'Распознавание речи браузера недоступно по сети, и резервный STT не настроен.', liveTranscript: 'Живая расшифровка', transcriptPlaceholder: 'Нажмите на голосовое ядро и начните говорить',
recentTurns: 'Последние реплики', you: 'Вы', mute: 'Выключить звук', unmute: 'Включить звук', finishTurn: 'Закончить реплику', speak: 'Начать говорить', end: 'Завершить',
status: { idle: 'Готово', paused: 'Микрофон выключен', listening: 'Слушаю', processing: 'Распознавание', thinking: '{agent} думает', speaking: '{agent} говорит', error: 'Голосовая связь прервана' },
hint: { idle: 'Нажмите в центре, чтобы начать голосовую реплику', paused: 'Включите микрофон, когда будете готовы', listening: 'Сделайте паузу для отправки или нажмите для завершения', listeningManual: 'Идет запись. Нажмите в центре еще раз, чтобы остановить и распознать речь', processing: 'Преобразую речь в текст', thinking: 'Сообщение отправлено в текущий разговор', speaking: 'Ответ можно прервать в любое время', error: 'Нажмите в центре, чтобы повторить' },
@@ -492,6 +492,7 @@ export default {
browserTts: '瀏覽器',
untitledSession: '未命名對話',
sessionMissing: '目前對話無法使用。',
backendStreamRequired: '手機端和桌面端即時語音需要先設定後端語音轉寫服務。',
networkUnavailableNoFallback: '瀏覽器語音辨識網路無法使用,且目前沒有設定可用的備用 STT。',
liveTranscript: '即時轉寫',
transcriptPlaceholder: '輕觸語音核心,然後開始說話',
+1
View File
@@ -492,6 +492,7 @@ export default {
browserTts: '浏览器',
untitledSession: '未命名对话',
sessionMissing: '当前对话不可用。',
backendStreamRequired: '手机端和桌面端实时语音需要先配置后端语音转写服务。',
networkUnavailableNoFallback: '浏览器语音识别网络不可用,且当前没有配置可用的备用 STT。',
liveTranscript: '实时转写',
transcriptPlaceholder: '轻触语音核心,然后开始说话',
+2
View File
@@ -61,6 +61,8 @@ mac:
hardenedRuntime: true
gatekeeperAssess: false
notarize: true
extendInfo:
NSMicrophoneUsageDescription: "Hermes Studio uses the microphone for voice conversations and speech transcription."
artifactName: "Hermes.Studio-${version}-${arch}.${ext}"
win:
+40 -1
View File
@@ -1,4 +1,4 @@
import { app, BrowserWindow, Menu, Tray, shell, ipcMain, nativeImage, Notification, screen, dialog, type MessageBoxOptions } from 'electron'
import { app, BrowserWindow, Menu, Tray, shell, ipcMain, nativeImage, Notification, screen, dialog, session, systemPreferences, type MessageBoxOptions } from 'electron'
import { existsSync } from 'node:fs'
import { join } from 'node:path'
import { startWebUiServer, stopWebUiServer, getToken } from './webui-server'
@@ -462,6 +462,44 @@ function createWindow() {
updateTrayMenu()
}
function installMicrophonePermissionHandler() {
session.defaultSession.setPermissionRequestHandler((webContents, permission, callback, details) => {
const isMainRenderer = !!mainWindow
&& !mainWindow.isDestroyed()
&& webContents === mainWindow.webContents
if (permission !== 'media') {
callback(isMainRenderer)
return
}
const mediaTypes = ('mediaTypes' in details ? details.mediaTypes : undefined) ?? []
const requestsAudio = mediaTypes.length ? mediaTypes.includes('audio') : true
if (!isMainRenderer || !requestsAudio) {
callback(false)
return
}
if (process.platform !== 'darwin') {
callback(true)
return
}
const status = systemPreferences.getMediaAccessStatus('microphone')
if (status === 'granted') {
callback(true)
return
}
if (status === 'denied' || status === 'restricted') {
callback(false)
return
}
void systemPreferences.askForMediaAccess('microphone')
.then(granted => callback(granted))
.catch(() => callback(false))
})
}
function splashHtml(label = t('desktop.startingLocalServices')): string {
const startingLabel = escapeHtml(label)
const html = `<!doctype html><html><head><meta charset="utf-8"><title>Hermes Studio</title>
@@ -805,6 +843,7 @@ function runDesktopApp() {
// visual clutter. macOS keeps a menu (system requirement) but Electron's
// default is fine there.
if (process.platform !== 'darwin') Menu.setApplicationMenu(null)
installMicrophonePermissionHandler()
if (app.isPackaged) {
installHermesStudioCliShim({
nodePath: bundledNode(),
@@ -15,6 +15,8 @@ const testState = vi.hoisted(() => ({
recognitionStopResult: '',
browserRecognition: null as any,
recorder: null as any,
pcmRecorder: null as any,
pcmOnChunk: null as null | ((audio: Blob) => void),
sttSettingsResponse: { providers: [], activeProvider: 'browser' } as any,
transcribeSpeech: vi.fn(),
}))
@@ -73,6 +75,32 @@ vi.mock('@/composables/useMicRecorder', async () => {
}
})
vi.mock('@/composables/usePcmStreamRecorder', async () => {
const { ref, shallowRef } = await import('vue')
return {
usePcmStreamRecorder: (options: { onChunk?: (audio: Blob) => void }) => {
testState.pcmOnChunk = options.onChunk || null
return (testState.pcmRecorder = {
status: ref('idle'),
error: ref<Error | null>(null),
level: ref(0),
hasSpeech: ref(false),
stream: shallowRef(null),
start: vi.fn().mockImplementation(async () => {
testState.pcmRecorder.status.value = 'recording'
}),
stop: vi.fn().mockImplementation(async () => {
testState.pcmRecorder.status.value = 'idle'
return new Blob([new Uint8Array(256)], { type: 'audio/wav' })
}),
cancel: vi.fn().mockImplementation(() => {
testState.pcmRecorder.status.value = 'idle'
}),
})
},
}
})
vi.mock('@/api/hermes/stt-settings', () => ({
fetchSttSettings: vi.fn(async () => testState.sttSettingsResponse),
}))
@@ -194,6 +222,12 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
testState.recognitionStopResult = ''
testState.browserRecognition = null
testState.recorder = null
testState.pcmRecorder = null
testState.pcmOnChunk = null
testState.store.messages.splice(0)
testState.store.isStreaming = true
testState.store.sendMessage.mockReset()
testState.store.stopStreaming.mockReset()
testState.sttSettingsResponse = { providers: [], activeProvider: 'browser' }
testState.transcribeSpeech.mockReset()
testState.transcribeSpeech.mockResolvedValue({
@@ -203,6 +237,10 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
durationMs: 10,
})
vi.stubGlobal('Audio', MockAudio)
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: undefined,
})
vi.stubGlobal('URL', {
createObjectURL: vi.fn((blob: Blob) => `blob:voice-${blob.size}-${Math.random()}`),
revokeObjectURL: vi.fn(),
@@ -216,10 +254,17 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
})
it('prepares at most five segments concurrently but always plays and captions FIFO', async () => {
testState.recognitionStopResult = '播放队列测试'
const wrapper = mount(RealtimeVoiceStage)
expect(wrapper.find('.voice-stage__controls').exists()).toBe(false)
expect(wrapper.find('.voice-stage__back').exists()).toBe(true)
expect(wrapper.get('.voice-stage__identity span').text()).toBe('Codex')
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
await settle()
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
await settle()
testState.store.messages.push({
id: 'assistant-stream',
role: 'assistant',
@@ -265,6 +310,36 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
wrapper.unmount()
})
it('never plays assistant messages that existed before the current voice turn', async () => {
testState.store.isStreaming = false
testState.store.messages.push(
{
id: 'historical-assistant-1',
role: 'assistant',
content: '第一条历史回复。',
timestamp: Date.now() - 2_000,
isStreaming: false,
},
{
id: 'historical-assistant-2',
role: 'assistant',
content: '第二条历史回复。',
timestamp: Date.now() - 1_000,
isStreaming: false,
},
)
const wrapper = mount(RealtimeVoiceStage)
await settle()
testState.store.messages[0].content = '第一条历史回复。 '
await settle()
expect(testState.requests).toHaveLength(0)
expect(testState.audioInstances).toHaveLength(0)
wrapper.unmount()
})
it('stops the active model turn when the animation is clicked while thinking', async () => {
const wrapper = mount(RealtimeVoiceStage)
testState.recognitionStopResult = '执行一个任务'
@@ -339,7 +414,7 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
},
)
it('uses explicit manual backend recording on mobile even when desktop-site mode hides the mobile UA', async () => {
it('uses continuous backend WAV streaming on mobile even when desktop-site mode hides the mobile UA', async () => {
vi.useFakeTimers()
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/138 Safari/537.36',
@@ -361,17 +436,23 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
await vi.advanceTimersByTimeAsync(180)
await settle()
expect(wrapper.classes()).toContain('voice-stage--idle')
expect(wrapper.classes()).toContain('voice-stage--listening')
expect(testState.recorder.start).not.toHaveBeenCalled()
expect(testState.browserRecognition.start).not.toHaveBeenCalled()
expect(testState.pcmRecorder.start).toHaveBeenCalledTimes(1)
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('realtimeVoice.hint.listening')
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
testState.transcribeSpeech.mockResolvedValueOnce({
text: '实时字幕第一段',
provider: 'openai',
model: 'gpt-4o-transcribe',
durationMs: 10,
})
testState.pcmOnChunk?.(new Blob([new Uint8Array(256)], { type: 'audio/wav' }))
await settle()
expect(wrapper.classes()).toContain('voice-stage--listening')
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('realtimeVoice.hint.listeningManual')
expect(testState.recorder.start).toHaveBeenCalledTimes(1)
expect(testState.browserRecognition.start).not.toHaveBeenCalled()
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('实时字幕第一段')
testState.pcmRecorder.stop.mockResolvedValueOnce(null)
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
await settle()
@@ -380,12 +461,170 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
provider: 'openai',
language: 'zh-CN',
prompt: '中英混合',
audio: expect.objectContaining({ type: 'audio/wav' }),
}))
expect(testState.store.sendMessage).toHaveBeenCalledWith('备用识别文本')
expect(testState.store.sendMessage).toHaveBeenCalledWith('实时字幕第一段')
wrapper.unmount()
})
it('keeps restart-based browser capture on mobile when no backend STT is configured', async () => {
it('uses backend WAV streaming in Electron instead of browser speech or Opus recording', async () => {
vi.useFakeTimers()
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: { isDesktop: true },
})
testState.sttSettingsResponse = {
activeProvider: 'openai',
providers: [{
provider: 'openai',
settings: { language: 'zh-CN' },
secrets: { apiKey: '[stored]' },
updatedAt: Date.now(),
}],
}
const wrapper = mount(RealtimeVoiceStage)
await vi.advanceTimersByTimeAsync(180)
await settle()
expect(wrapper.classes()).toContain('voice-stage--listening')
expect(testState.pcmRecorder.start).toHaveBeenCalledOnce()
expect(testState.browserRecognition.start).not.toHaveBeenCalled()
expect(testState.recorder.start).not.toHaveBeenCalled()
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
await settle()
expect(testState.transcribeSpeech).toHaveBeenCalledWith(expect.objectContaining({
provider: 'openai',
audio: expect.objectContaining({ type: 'audio/wav' }),
}))
expect(testState.store.sendMessage).toHaveBeenCalledWith('备用识别文本')
expect(wrapper.classes()).toContain('voice-stage--thinking')
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('备用识别文本')
wrapper.unmount()
})
it('ignores backend no-speech chunks in Electron and keeps listening', async () => {
vi.useFakeTimers()
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: { isDesktop: true },
})
testState.sttSettingsResponse = {
activeProvider: 'doubao',
providers: [{
provider: 'doubao',
settings: { language: 'zh-CN' },
secrets: { apiKey: '[stored]' },
updatedAt: Date.now(),
}],
}
testState.transcribeSpeech.mockRejectedValueOnce(
new Error('API Error 400: No speech detected in the audio'),
)
const wrapper = mount(RealtimeVoiceStage)
await vi.advanceTimersByTimeAsync(180)
await settle()
testState.pcmOnChunk?.(new Blob([new Uint8Array(256)], { type: 'audio/wav' }))
await settle()
expect(wrapper.classes()).toContain('voice-stage--listening')
expect(wrapper.classes()).not.toContain('voice-stage--error')
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('realtimeVoice.hint.listening')
expect(testState.pcmRecorder.cancel).not.toHaveBeenCalled()
testState.transcribeSpeech.mockResolvedValueOnce({
text: '下一段可以识别',
provider: 'doubao',
model: 'volc.seedasr.auc',
durationMs: 10,
})
testState.pcmOnChunk?.(new Blob([new Uint8Array(256)], { type: 'audio/wav' }))
await settle()
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('下一段可以识别')
wrapper.unmount()
})
it('does not leave listening mode for unconfirmed Electron startup noise', async () => {
vi.useFakeTimers()
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: { isDesktop: true },
})
testState.sttSettingsResponse = {
activeProvider: 'doubao',
providers: [{
provider: 'doubao',
settings: { language: 'zh-CN' },
secrets: { apiKey: '[stored]' },
updatedAt: Date.now(),
}],
}
const wrapper = mount(RealtimeVoiceStage)
await vi.advanceTimersByTimeAsync(180)
await settle()
testState.pcmRecorder.level.value = 0.2
await settle()
await vi.advanceTimersByTimeAsync(2_500)
await settle()
expect(testState.pcmRecorder.stop).not.toHaveBeenCalled()
expect(testState.transcribeSpeech).not.toHaveBeenCalled()
expect(wrapper.classes()).toContain('voice-stage--listening')
expect(wrapper.classes()).not.toContain('voice-stage--processing')
expect(wrapper.classes()).not.toContain('voice-stage--error')
testState.pcmRecorder.stop.mockResolvedValueOnce(null)
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
await settle()
await vi.advanceTimersByTimeAsync(1_000)
await settle()
expect(wrapper.classes()).toContain('voice-stage--paused')
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('realtimeVoice.hint.paused')
expect(testState.pcmRecorder.start).toHaveBeenCalledTimes(1)
wrapper.unmount()
})
it('commits confirmed Electron speech after 1.5 seconds of silence', async () => {
vi.useFakeTimers()
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: { isDesktop: true },
})
testState.sttSettingsResponse = {
activeProvider: 'doubao',
providers: [{
provider: 'doubao',
settings: { language: 'zh-CN' },
secrets: { apiKey: '[stored]' },
updatedAt: Date.now(),
}],
}
const wrapper = mount(RealtimeVoiceStage)
await vi.advanceTimersByTimeAsync(180)
await settle()
testState.pcmRecorder.hasSpeech.value = true
testState.pcmRecorder.level.value = 0.2
await settle()
await vi.advanceTimersByTimeAsync(1_499)
expect(testState.pcmRecorder.stop).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1)
await settle()
expect(testState.pcmRecorder.stop).toHaveBeenCalledOnce()
expect(testState.store.sendMessage).toHaveBeenCalledWith('备用识别文本')
expect(wrapper.classes()).toContain('voice-stage--thinking')
wrapper.unmount()
})
it('requires backend STT on mobile instead of falling back to browser recognition', async () => {
vi.useFakeTimers()
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Linux; Android 15) AppleWebKit/537.36 Chrome/138 Mobile Safari/537.36',
@@ -398,17 +637,10 @@ describe('RealtimeVoiceStage prepared playback queue', () => {
await vi.advanceTimersByTimeAsync(180)
await settle()
expect(testState.browserRecognition.start).toHaveBeenCalledWith({
language: 'zh-CN',
continuous: false,
})
testState.recognitionStopResult = '移动端问题'
await wrapper.get('[data-testid="realtime-voice-toggle"]').trigger('click')
await settle()
expect(testState.store.sendMessage).toHaveBeenCalledWith('移动端问题')
expect(testState.browserRecognition.start).toHaveBeenCalledTimes(1)
expect(wrapper.classes()).toContain('voice-stage--error')
expect(wrapper.get('[data-testid="realtime-voice-caption"]').text()).toBe('realtimeVoice.backendStreamRequired')
expect(testState.browserRecognition.start).not.toHaveBeenCalled()
expect(testState.pcmRecorder.start).not.toHaveBeenCalled()
wrapper.unmount()
})
+23
View File
@@ -350,6 +350,29 @@ describe('stt api wrappers', () => {
expect(serialized).not.toContain('127.0.0.1:8000')
})
it('uploads PCM stream fragments with a WAV filename', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
status: 200,
json: () => Promise.resolve({
text: 'streamed transcript',
provider: 'openai',
model: 'gpt-4o-transcribe',
durationMs: 10,
}),
})
await transcribeSpeech({
audio: new Blob([new Uint8Array(256)], { type: 'audio/wav' }),
provider: 'openai',
})
const [, init] = mockFetch.mock.calls[0] as [string, RequestInit]
const upload = (init.body as FormData).get('audio') as File
expect(upload.name).toBe('speech.wav')
expect(upload.type).toBe('audio/wav')
})
it('does not persist raw STT API keys to localStorage when composable state changes', async () => {
const { clearSttSettingsAuthState, useSttSettings } = await import('../../packages/client/src/composables/useSttSettings')
+41 -3
View File
@@ -382,9 +382,12 @@ vi.mock('@/api/hermes/stt', () => ({
transcribeSpeech: mockTranscribeSpeech,
}))
vi.mock('@/composables/useMicRecorder', () => ({
useMicRecorder: () => ({
state: mockMicRecorderState,
vi.mock('@/composables/usePcmStreamRecorder', () => ({
usePcmStreamRecorder: () => ({
status: ref(mockMicRecorderState.value.status),
error: ref(null),
level: ref(0),
stream: ref(null),
isRecording: ref(mockMicRecorderState.value.status === 'recording'),
start: mockMicStart,
stop: mockMicStop,
@@ -1053,6 +1056,41 @@ describe('VoiceSettings STT UI', () => {
expect(mockOpenaiPlay).not.toHaveBeenCalled()
})
it('records the STT card test as PCM WAV before transcription', async () => {
const wav = new Blob([new Uint8Array(256)], { type: 'audio/wav' })
mockMicStart.mockResolvedValue(undefined)
mockMicStop.mockResolvedValue(wav)
mockTranscribeSpeech.mockResolvedValue({
text: 'WAV transcription succeeded',
provider: 'openai',
model: 'gpt-4o-transcribe',
durationMs: 10,
})
mockFetchSttSettings.mockResolvedValue({
activeProvider: 'openai',
providers: [{
provider: 'openai',
settings: { model: 'gpt-4o-transcribe' },
secrets: { apiKey: '[stored]' },
updatedAt: 12,
}],
})
const wrapper = await mountComponent()
await flushPromises()
const testButton = wrapper.get('[data-testid="voice-card-test-stt-openai"]')
await testButton.trigger('click')
await flushPromises()
expect(mockMicStart).toHaveBeenCalledOnce()
await testButton.trigger('click')
await flushPromises()
expect(mockTranscribeSpeech).toHaveBeenCalledWith({ audio: wav, provider: 'openai' })
expect(wrapper.text()).toContain('WAV transcription succeeded')
})
it('shows missing-key and inline testing/error states on provider cards', async () => {
let resolvePlay!: () => void
mockOpenaiPlay.mockReturnValue(new Promise<void>(resolve => { resolvePlay = resolve }))
@@ -0,0 +1,199 @@
// @vitest-environment jsdom
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { encodePcmWav, usePcmStreamRecorder } from '../../packages/client/src/composables/usePcmStreamRecorder'
class FakeScriptProcessor {
onaudioprocess: ((event: AudioProcessingEvent) => void) | null = null
connect = vi.fn()
disconnect = vi.fn()
emit(samples: Float32Array) {
this.onaudioprocess?.({
inputBuffer: {
length: samples.length,
numberOfChannels: 1,
getChannelData: () => samples,
},
} as unknown as AudioProcessingEvent)
}
}
class FakeAudioContext {
static instances: FakeAudioContext[] = []
readonly sampleRate = 48_000
readonly destination = {} as AudioDestinationNode
readonly processor = new FakeScriptProcessor()
readonly source = { connect: vi.fn(), disconnect: vi.fn() }
readonly gain = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }
state: AudioContextState = 'running'
createMediaStreamSource = vi.fn(() => this.source)
createScriptProcessor = vi.fn(() => this.processor)
createGain = vi.fn(() => this.gain)
resume = vi.fn().mockResolvedValue(undefined)
close = vi.fn(async () => { this.state = 'closed' })
constructor() {
FakeAudioContext.instances.push(this)
}
}
function mockStream() {
const track = { stop: vi.fn() }
return {
stream: { getTracks: () => [track] } as unknown as MediaStream,
track,
}
}
describe('usePcmStreamRecorder', () => {
const getUserMedia = vi.fn()
beforeEach(() => {
FakeAudioContext.instances = []
vi.stubGlobal('AudioContext', FakeAudioContext)
Object.defineProperty(navigator, 'mediaDevices', {
configurable: true,
value: { getUserMedia },
})
})
afterEach(() => {
vi.clearAllMocks()
vi.unstubAllGlobals()
})
it('encodes mono 16-bit PCM with a valid 16 kHz WAV header', async () => {
const audio = encodePcmWav(new Float32Array(48_000).fill(0.25), 48_000, 16_000)
const view = new DataView(await readBlob(audio))
expect(audio.type).toBe('audio/wav')
expect(String.fromCharCode(...new Uint8Array(view.buffer, 0, 4))).toBe('RIFF')
expect(String.fromCharCode(...new Uint8Array(view.buffer, 8, 4))).toBe('WAVE')
expect(view.getUint16(20, true)).toBe(1)
expect(view.getUint16(22, true)).toBe(1)
expect(view.getUint32(24, true)).toBe(16_000)
expect(view.getUint16(34, true)).toBe(16)
expect(audio.size).toBe(32_044)
})
it('emits a WAV segment at a natural pause without using MediaRecorder or Opus', async () => {
const { stream, track } = mockStream()
getUserMedia.mockResolvedValue(stream)
const onChunk = vi.fn()
const recorder = usePcmStreamRecorder({
minSegmentDurationMs: 300,
speechEndSilenceMs: 200,
voiceActivityThreshold: 0.035,
onChunk,
})
await recorder.start()
const context = FakeAudioContext.instances[0]
for (let index = 0; index < 12; index += 1) {
context.processor.emit(new Float32Array(4_096).fill(0.1))
}
for (let index = 0; index < 3; index += 1) {
context.processor.emit(new Float32Array(4_096))
}
expect(onChunk).toHaveBeenCalledOnce()
expect(onChunk.mock.calls[0][0]).toMatchObject({ type: 'audio/wav' })
expect(onChunk.mock.calls[0][0].size).toBeGreaterThan(44)
expect(recorder.level.value).toBeGreaterThan(0)
expect(recorder.hasSpeech.value).toBe(true)
await expect(recorder.stop()).resolves.toBeNull()
expect(track.stop).toHaveBeenCalledOnce()
expect(context.close).toHaveBeenCalledOnce()
expect(recorder.status.value).toBe('idle')
})
it('flushes a final WAV fragment when capture stops between chunks', async () => {
const { stream } = mockStream()
getUserMedia.mockResolvedValue(stream)
const recorder = usePcmStreamRecorder({ maxSegmentDurationMs: 3_000, voiceActivityThreshold: 0.035 })
await recorder.start()
const context = FakeAudioContext.instances[0]
for (let index = 0; index < 5; index += 1) {
context.processor.emit(new Float32Array(4_096).fill(0.1))
}
const finalChunk = await recorder.stop()
expect(finalChunk).toMatchObject({ type: 'audio/wav' })
expect(finalChunk?.size).toBeGreaterThan(44)
})
it('drops pure silence instead of sending it to STT', async () => {
const { stream } = mockStream()
getUserMedia.mockResolvedValue(stream)
const onChunk = vi.fn()
const recorder = usePcmStreamRecorder({
maxSegmentDurationMs: 1_000,
voiceActivityThreshold: 0.035,
onChunk,
})
await recorder.start()
const context = FakeAudioContext.instances[0]
for (let index = 0; index < 16; index += 1) {
context.processor.emit(new Float32Array(4_096))
}
expect(onChunk).not.toHaveBeenCalled()
await expect(recorder.stop()).resolves.toBeNull()
})
it('does not confirm speech from microphone startup noise', async () => {
const { stream } = mockStream()
getUserMedia.mockResolvedValue(stream)
const onChunk = vi.fn()
const recorder = usePcmStreamRecorder({
voiceActivityThreshold: 0.035,
onChunk,
})
await recorder.start()
const context = FakeAudioContext.instances[0]
for (let index = 0; index < 5; index += 1) {
context.processor.emit(new Float32Array(4_096).fill(0.1))
}
for (let index = 0; index < 12; index += 1) {
context.processor.emit(new Float32Array(4_096))
}
expect(recorder.hasSpeech.value).toBe(false)
expect(onChunk).not.toHaveBeenCalled()
await expect(recorder.stop()).resolves.toBeNull()
})
it('uses the maximum segment duration only as a continuous-speech safeguard', async () => {
const { stream } = mockStream()
getUserMedia.mockResolvedValue(stream)
const onChunk = vi.fn()
const recorder = usePcmStreamRecorder({
maxSegmentDurationMs: 1_000,
voiceActivityThreshold: 0.035,
onChunk,
})
await recorder.start()
const context = FakeAudioContext.instances[0]
for (let index = 0; index < 20; index += 1) {
context.processor.emit(new Float32Array(4_096).fill(0.1))
}
expect(onChunk).toHaveBeenCalledOnce()
expect(recorder.status.value).toBe('recording')
recorder.cancel()
})
})
function readBlob(blob: Blob) {
return new Promise<ArrayBuffer>((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => resolve(reader.result as ArrayBuffer)
reader.onerror = () => reject(reader.error)
reader.readAsArrayBuffer(blob)
})
}