From 072d2000f35a9f7b96342fa9bb28f925a92e7b4c Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Sat, 9 May 2026 04:53:47 +0900 Subject: [PATCH] refac --- backend/open_webui/routers/audio.py | 5 + .../chat/MessageInput/CallOverlay.svelte | 155 +++++++++++++++--- 2 files changed, 133 insertions(+), 27 deletions(-) diff --git a/backend/open_webui/routers/audio.py b/backend/open_webui/routers/audio.py index e4b89299da..72ba2f6d03 100644 --- a/backend/open_webui/routers/audio.py +++ b/backend/open_webui/routers/audio.py @@ -1166,6 +1166,11 @@ def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None else: if is_audio_conversion_required(file_path): file_path = convert_audio_to_mp3(file_path) + if not file_path: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Audio conversion failed. The audio file may be corrupted or empty.', + ) try: file_path = compress_audio(file_path) diff --git a/src/lib/components/chat/MessageInput/CallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay.svelte index 58ac2a6663..008845c67a 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay.svelte +++ b/src/lib/components/chat/MessageInput/CallOverlay.svelte @@ -32,6 +32,7 @@ let confirmed = false; let interrupted = false; let assistantSpeaking = false; + let muted = false; let emoji = null; let camera = false; @@ -155,6 +156,10 @@ const transcribeHandler = async (audioBlob) => { // Create a blob from the audio chunks + if (!audioBlob || audioBlob.size < 100) { + console.log('Audio blob too small or empty, skipping transcription'); + return; + } await tick(); const file = blobToFile(audioBlob, 'recording.wav'); @@ -236,6 +241,11 @@ } }); } + + if (audioStream) { + // hardware track muting disabled to prevent backend translation errors with malformed WebM files + } + mediaRecorder = new MediaRecorder(audioStream); mediaRecorder.onstart = () => { @@ -310,8 +320,8 @@ return; } - if (assistantSpeaking && !($settings?.voiceInterruption ?? false)) { - // Mute the audio if the assistant is speaking + if (muted || (assistantSpeaking && !($settings?.voiceInterruption ?? false))) { + // Suppress mic input when muted or when assistant is speaking without interruption enabled analyser.maxDecibels = 0; analyser.minDecibels = -1; } else { @@ -325,6 +335,10 @@ // Calculate RMS level from time domain data rmsLevel = calculateRMS(timeDomainData); + if (muted || (assistantSpeaking && !($settings?.voiceInterruption ?? false))) { + rmsLevel = 0; + } + // Check if initial speech/noise has started const hasSound = domainData.some((value) => value > 0); if (hasSound) { @@ -627,6 +641,43 @@ chatStreaming = false; }; + const toggleMute = () => { + muted = !muted; + if (muted && hasStartedSpeaking) { + // Abort the ongoing recording so it doesn't accidentally send a partial sentence + hasStartedSpeaking = false; + confirmed = false; + audioChunks = []; + if (mediaRecorder && mediaRecorder.state === 'recording') { + mediaRecorder.stop(); + } + } + }; + + let wasAssistantSpeaking = false; + $: { + if (assistantSpeaking && !wasAssistantSpeaking) { + wasAssistantSpeaking = true; + } else if (!assistantSpeaking && wasAssistantSpeaking) { + wasAssistantSpeaking = false; + // Auto unmute when AI finishes speaking + if (muted) { + muted = false; + } + } + } + + const handleKeydown = (e: KeyboardEvent) => { + // Only handle M key when not typing in an input/textarea + if (e.key === 'm' || e.key === 'M') { + const target = e.target as HTMLElement; + if (target.tagName !== 'INPUT' && target.tagName !== 'TEXTAREA' && !target.isContentEditable) { + e.preventDefault(); + toggleMute(); + } + } + }; + onMount(async () => { const setWakeLock = async () => { try { @@ -664,6 +715,8 @@ eventTarget.addEventListener('chat', chatEventHandler); eventTarget.addEventListener('chat:finish', chatFinishHandler); + document.addEventListener('keydown', handleKeydown); + return async () => { await stopAllAudio(); @@ -673,6 +726,8 @@ eventTarget.removeEventListener('chat', chatEventHandler); eventTarget.removeEventListener('chat:finish', chatFinishHandler); + document.removeEventListener('keydown', handleKeydown); + audioAbortController.abort(); await tick(); @@ -692,6 +747,9 @@ eventTarget.removeEventListener('chat:start', chatStartHandler); eventTarget.removeEventListener('chat', chatEventHandler); eventTarget.removeEventListener('chat:finish', chatFinishHandler); + + document.removeEventListener('keydown', handleKeydown); + audioAbortController.abort(); await tick(); @@ -887,8 +945,30 @@ {/if} -
-
+
+ + +
{#if camera} -
-
-
- -
+ + -