mirror of
https://github.com/open-webui/open-webui.git
synced 2026-07-20 00:35:35 -05:00
1401 lines
53 KiB
Python
1401 lines
53 KiB
Python
"""Audio router — TTS speech synthesis and STT transcription endpoints."""
|
|
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import html
|
|
import io
|
|
import json
|
|
import logging
|
|
import mimetypes
|
|
import os
|
|
import uuid
|
|
from fnmatch import fnmatch
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import aiofiles
|
|
import aiohttp
|
|
from fastapi import (
|
|
APIRouter,
|
|
Depends,
|
|
File,
|
|
Form,
|
|
HTTPException,
|
|
Request,
|
|
UploadFile,
|
|
status,
|
|
)
|
|
from fastapi.responses import FileResponse
|
|
from pydantic import BaseModel
|
|
from pydub import AudioSegment
|
|
from pydub.silence import split_on_silence
|
|
from pydub.utils import mediainfo
|
|
|
|
from open_webui.config import (
|
|
CACHE_DIR,
|
|
ELEVENLABS_API_BASE_URL,
|
|
WHISPER_COMPUTE_TYPE,
|
|
WHISPER_LANGUAGE,
|
|
WHISPER_MODEL_AUTO_UPDATE,
|
|
WHISPER_MODEL_DIR,
|
|
WHISPER_MULTILINGUAL,
|
|
WHISPER_VAD_FILTER,
|
|
)
|
|
from open_webui.constants import ERROR_MESSAGES
|
|
from open_webui.env import (
|
|
AIOHTTP_CLIENT_SESSION_SSL,
|
|
AIOHTTP_CLIENT_TIMEOUT,
|
|
AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST,
|
|
BYPASS_PYDUB_PREPROCESSING,
|
|
DEVICE_TYPE,
|
|
ENABLE_FORWARD_USER_INFO_HEADERS,
|
|
ENV,
|
|
)
|
|
from open_webui.utils.access_control import has_permission
|
|
from open_webui.utils.auth import get_admin_user, get_verified_user
|
|
from open_webui.utils.headers import include_user_info_headers
|
|
from open_webui.utils.misc import strict_match_mime_type
|
|
from open_webui.utils.session_pool import get_session
|
|
|
|
log = logging.getLogger(__name__)
|
|
router = APIRouter()
|
|
|
|
# --- Constants ---
|
|
|
|
MAX_FILE_SIZE_MB: int = 20
|
|
MAX_FILE_SIZE: int = MAX_FILE_SIZE_MB * 1024 * 1024
|
|
AZURE_MAX_FILE_SIZE_MB: int = 200
|
|
AZURE_MAX_FILE_SIZE: int = AZURE_MAX_FILE_SIZE_MB * 1024 * 1024
|
|
|
|
SPEECH_CACHE_DIR = CACHE_DIR / 'audio' / 'speech'
|
|
SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def is_audio_conversion_required(file_path):
|
|
"""
|
|
Check if the given audio file needs conversion to mp3.
|
|
"""
|
|
SUPPORTED_FORMATS = {'flac', 'm4a', 'mp3', 'mp4', 'mpeg', 'wav', 'webm'}
|
|
|
|
if not os.path.isfile(file_path):
|
|
log.error(f'File not found: {file_path}')
|
|
return False
|
|
|
|
try:
|
|
info = mediainfo(file_path)
|
|
codec_name = info.get('codec_name', '').lower()
|
|
codec_type = info.get('codec_type', '').lower()
|
|
codec_tag_string = info.get('codec_tag_string', '').lower()
|
|
|
|
if codec_name == 'aac' and codec_type == 'audio' and codec_tag_string == 'mp4a':
|
|
# File is AAC/mp4a audio, recommend mp3 conversion
|
|
return True
|
|
|
|
# If the codec name is in the supported formats
|
|
if codec_name in SUPPORTED_FORMATS:
|
|
return False
|
|
|
|
return True
|
|
except Exception as e:
|
|
log.error(f'Error getting audio format: {e}')
|
|
return False
|
|
|
|
|
|
def convert_audio_to_mp3(file_path):
|
|
"""Convert audio file to mp3 format."""
|
|
try:
|
|
output_path = os.path.splitext(file_path)[0] + '.mp3'
|
|
audio = AudioSegment.from_file(file_path)
|
|
audio.export(output_path, format='mp3')
|
|
log.info(f'Converted {file_path} to {output_path}')
|
|
return output_path
|
|
except Exception as e:
|
|
log.error(f'Error converting audio file: {e}')
|
|
return None
|
|
|
|
|
|
def transcode_audio_to_mp3(audio_data: bytes, content_type_header: str, output_path: str) -> bool:
|
|
"""
|
|
Transcode audio bytes to MP3 if the Content-Type indicates a non-MP3 format.
|
|
|
|
Handles raw PCM audio (e.g. Gemini-TTS via OpenRouter/LiteLLM) by parsing
|
|
optional rate/channels from the Content-Type params, defaulting to 24kHz,
|
|
16-bit, mono. For other non-MP3 formats, uses pydub auto-detection.
|
|
|
|
Returns True if transcoding was performed, False if the data is already MP3.
|
|
Respects BYPASS_PYDUB_PREPROCESSING — when set, writes raw bytes and logs a warning.
|
|
"""
|
|
mime_type = content_type_header.split(';')[0].strip().lower()
|
|
|
|
if mime_type in ('audio/mpeg', 'audio/mp3'):
|
|
return False
|
|
|
|
if BYPASS_PYDUB_PREPROCESSING:
|
|
log.warning(
|
|
f'TTS returned {mime_type} but BYPASS_PYDUB_PREPROCESSING is set; writing raw audio without transcoding'
|
|
)
|
|
return False
|
|
|
|
if mime_type in ('audio/pcm', 'audio/l16', 'audio/raw'):
|
|
# Parse optional rate/channels from Content-Type params,
|
|
# default: 24kHz, 16-bit, mono (standard for Gemini TTS).
|
|
ct_params = {}
|
|
for part in content_type_header.split(';')[1:]:
|
|
key_val = part.strip().split('=')
|
|
if len(key_val) == 2:
|
|
ct_params[key_val[0].strip().lower()] = key_val[1].strip()
|
|
|
|
sample_rate = int(ct_params.get('rate', 24000))
|
|
channels = int(ct_params.get('channels', 1))
|
|
|
|
audio_segment = AudioSegment.from_raw(
|
|
io.BytesIO(audio_data),
|
|
sample_width=2,
|
|
frame_rate=sample_rate,
|
|
channels=channels,
|
|
)
|
|
else:
|
|
audio_segment = AudioSegment.from_file(io.BytesIO(audio_data))
|
|
|
|
audio_segment.export(str(output_path), format='mp3')
|
|
log.info(f'Transcoded {mime_type} audio to MP3: {output_path}')
|
|
return True
|
|
|
|
|
|
def set_faster_whisper_model(model: str, auto_update: bool = False):
|
|
whisper_model = None
|
|
if model:
|
|
from faster_whisper import WhisperModel
|
|
|
|
faster_whisper_kwargs = {
|
|
'model_size_or_path': model,
|
|
'device': DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == 'cuda' else 'cpu',
|
|
'compute_type': WHISPER_COMPUTE_TYPE,
|
|
'download_root': WHISPER_MODEL_DIR,
|
|
'local_files_only': not auto_update,
|
|
}
|
|
|
|
try:
|
|
whisper_model = WhisperModel(**faster_whisper_kwargs)
|
|
except Exception:
|
|
log.warning('WhisperModel initialization failed, attempting download with local_files_only=False')
|
|
faster_whisper_kwargs['local_files_only'] = False
|
|
whisper_model = WhisperModel(**faster_whisper_kwargs)
|
|
return whisper_model
|
|
|
|
|
|
class TTSConfigForm(BaseModel):
|
|
OPENAI_API_BASE_URL: str
|
|
OPENAI_API_KEY: str
|
|
OPENAI_PARAMS: Optional[dict] = None
|
|
API_KEY: str
|
|
ENGINE: str
|
|
MODEL: str
|
|
VOICE: str
|
|
SPLIT_ON: str
|
|
AZURE_SPEECH_REGION: str
|
|
AZURE_SPEECH_BASE_URL: str
|
|
AZURE_SPEECH_OUTPUT_FORMAT: str
|
|
MISTRAL_API_KEY: str
|
|
MISTRAL_API_BASE_URL: str
|
|
|
|
|
|
class STTConfigForm(BaseModel):
|
|
OPENAI_API_BASE_URL: str
|
|
OPENAI_API_KEY: str
|
|
ENGINE: str
|
|
MODEL: str
|
|
SUPPORTED_CONTENT_TYPES: list[str] = []
|
|
ALLOWED_EXTENSIONS: list[str] = []
|
|
WHISPER_MODEL: str
|
|
DEEPGRAM_API_KEY: str
|
|
AZURE_API_KEY: str
|
|
AZURE_REGION: str
|
|
AZURE_LOCALES: str
|
|
AZURE_BASE_URL: str
|
|
AZURE_MAX_SPEAKERS: str
|
|
MISTRAL_API_KEY: str
|
|
MISTRAL_API_BASE_URL: str
|
|
MISTRAL_USE_CHAT_COMPLETIONS: bool
|
|
|
|
|
|
class AudioConfigUpdateForm(BaseModel):
|
|
tts: TTSConfigForm
|
|
stt: STTConfigForm
|
|
|
|
|
|
@router.get('/config')
|
|
async def get_audio_config(request: Request, user=Depends(get_admin_user)):
|
|
return {
|
|
'tts': {
|
|
'OPENAI_API_BASE_URL': request.app.state.config.TTS_OPENAI_API_BASE_URL,
|
|
'OPENAI_API_KEY': request.app.state.config.TTS_OPENAI_API_KEY,
|
|
'OPENAI_PARAMS': request.app.state.config.TTS_OPENAI_PARAMS,
|
|
'API_KEY': request.app.state.config.TTS_API_KEY,
|
|
'ENGINE': request.app.state.config.TTS_ENGINE,
|
|
'MODEL': request.app.state.config.TTS_MODEL,
|
|
'VOICE': request.app.state.config.TTS_VOICE,
|
|
'SPLIT_ON': request.app.state.config.TTS_SPLIT_ON,
|
|
'AZURE_SPEECH_REGION': request.app.state.config.TTS_AZURE_SPEECH_REGION,
|
|
'AZURE_SPEECH_BASE_URL': request.app.state.config.TTS_AZURE_SPEECH_BASE_URL,
|
|
'AZURE_SPEECH_OUTPUT_FORMAT': request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
|
|
'MISTRAL_API_KEY': request.app.state.config.TTS_MISTRAL_API_KEY,
|
|
'MISTRAL_API_BASE_URL': request.app.state.config.TTS_MISTRAL_API_BASE_URL,
|
|
},
|
|
'stt': {
|
|
'OPENAI_API_BASE_URL': request.app.state.config.STT_OPENAI_API_BASE_URL,
|
|
'OPENAI_API_KEY': request.app.state.config.STT_OPENAI_API_KEY,
|
|
'ENGINE': request.app.state.config.STT_ENGINE,
|
|
'MODEL': request.app.state.config.STT_MODEL,
|
|
'SUPPORTED_CONTENT_TYPES': request.app.state.config.STT_SUPPORTED_CONTENT_TYPES,
|
|
'ALLOWED_EXTENSIONS': request.app.state.config.STT_ALLOWED_EXTENSIONS,
|
|
'WHISPER_MODEL': request.app.state.config.WHISPER_MODEL,
|
|
'DEEPGRAM_API_KEY': request.app.state.config.DEEPGRAM_API_KEY,
|
|
'AZURE_API_KEY': request.app.state.config.AUDIO_STT_AZURE_API_KEY,
|
|
'AZURE_REGION': request.app.state.config.AUDIO_STT_AZURE_REGION,
|
|
'AZURE_LOCALES': request.app.state.config.AUDIO_STT_AZURE_LOCALES,
|
|
'AZURE_BASE_URL': request.app.state.config.AUDIO_STT_AZURE_BASE_URL,
|
|
'AZURE_MAX_SPEAKERS': request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS,
|
|
'MISTRAL_API_KEY': request.app.state.config.AUDIO_STT_MISTRAL_API_KEY,
|
|
'MISTRAL_API_BASE_URL': request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL,
|
|
'MISTRAL_USE_CHAT_COMPLETIONS': request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS,
|
|
},
|
|
}
|
|
|
|
|
|
@router.post('/config/update')
|
|
async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)):
|
|
# TTS settings
|
|
request.app.state.config.TTS_OPENAI_API_BASE_URL = form_data.tts.OPENAI_API_BASE_URL
|
|
request.app.state.config.TTS_OPENAI_API_KEY = form_data.tts.OPENAI_API_KEY
|
|
request.app.state.config.TTS_OPENAI_PARAMS = form_data.tts.OPENAI_PARAMS
|
|
request.app.state.config.TTS_API_KEY = form_data.tts.API_KEY
|
|
request.app.state.config.TTS_ENGINE = form_data.tts.ENGINE
|
|
request.app.state.config.TTS_MODEL = form_data.tts.MODEL
|
|
request.app.state.config.TTS_VOICE = form_data.tts.VOICE
|
|
request.app.state.config.TTS_SPLIT_ON = form_data.tts.SPLIT_ON
|
|
request.app.state.config.TTS_AZURE_SPEECH_REGION = form_data.tts.AZURE_SPEECH_REGION
|
|
request.app.state.config.TTS_AZURE_SPEECH_BASE_URL = form_data.tts.AZURE_SPEECH_BASE_URL
|
|
request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = form_data.tts.AZURE_SPEECH_OUTPUT_FORMAT
|
|
request.app.state.config.TTS_MISTRAL_API_KEY = form_data.tts.MISTRAL_API_KEY
|
|
request.app.state.config.TTS_MISTRAL_API_BASE_URL = form_data.tts.MISTRAL_API_BASE_URL
|
|
|
|
# STT settings
|
|
request.app.state.config.STT_OPENAI_API_BASE_URL = form_data.stt.OPENAI_API_BASE_URL
|
|
request.app.state.config.STT_OPENAI_API_KEY = form_data.stt.OPENAI_API_KEY
|
|
request.app.state.config.STT_ENGINE = form_data.stt.ENGINE
|
|
request.app.state.config.STT_MODEL = form_data.stt.MODEL
|
|
request.app.state.config.STT_SUPPORTED_CONTENT_TYPES = form_data.stt.SUPPORTED_CONTENT_TYPES
|
|
request.app.state.config.STT_ALLOWED_EXTENSIONS = form_data.stt.ALLOWED_EXTENSIONS
|
|
request.app.state.config.WHISPER_MODEL = form_data.stt.WHISPER_MODEL
|
|
request.app.state.config.DEEPGRAM_API_KEY = form_data.stt.DEEPGRAM_API_KEY
|
|
request.app.state.config.AUDIO_STT_AZURE_API_KEY = form_data.stt.AZURE_API_KEY
|
|
request.app.state.config.AUDIO_STT_AZURE_REGION = form_data.stt.AZURE_REGION
|
|
request.app.state.config.AUDIO_STT_AZURE_LOCALES = form_data.stt.AZURE_LOCALES
|
|
request.app.state.config.AUDIO_STT_AZURE_BASE_URL = form_data.stt.AZURE_BASE_URL
|
|
request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS = form_data.stt.AZURE_MAX_SPEAKERS
|
|
request.app.state.config.AUDIO_STT_MISTRAL_API_KEY = form_data.stt.MISTRAL_API_KEY
|
|
request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL = form_data.stt.MISTRAL_API_BASE_URL
|
|
request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS = form_data.stt.MISTRAL_USE_CHAT_COMPLETIONS
|
|
|
|
if request.app.state.config.STT_ENGINE == '':
|
|
request.app.state.faster_whisper_model = set_faster_whisper_model(
|
|
form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE
|
|
)
|
|
else:
|
|
request.app.state.faster_whisper_model = None
|
|
|
|
return {
|
|
'tts': {
|
|
'ENGINE': request.app.state.config.TTS_ENGINE,
|
|
'MODEL': request.app.state.config.TTS_MODEL,
|
|
'VOICE': request.app.state.config.TTS_VOICE,
|
|
'OPENAI_API_BASE_URL': request.app.state.config.TTS_OPENAI_API_BASE_URL,
|
|
'OPENAI_API_KEY': request.app.state.config.TTS_OPENAI_API_KEY,
|
|
'OPENAI_PARAMS': request.app.state.config.TTS_OPENAI_PARAMS,
|
|
'API_KEY': request.app.state.config.TTS_API_KEY,
|
|
'SPLIT_ON': request.app.state.config.TTS_SPLIT_ON,
|
|
'AZURE_SPEECH_REGION': request.app.state.config.TTS_AZURE_SPEECH_REGION,
|
|
'AZURE_SPEECH_BASE_URL': request.app.state.config.TTS_AZURE_SPEECH_BASE_URL,
|
|
'AZURE_SPEECH_OUTPUT_FORMAT': request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
|
|
'MISTRAL_API_KEY': request.app.state.config.TTS_MISTRAL_API_KEY,
|
|
'MISTRAL_API_BASE_URL': request.app.state.config.TTS_MISTRAL_API_BASE_URL,
|
|
},
|
|
'stt': {
|
|
'OPENAI_API_BASE_URL': request.app.state.config.STT_OPENAI_API_BASE_URL,
|
|
'OPENAI_API_KEY': request.app.state.config.STT_OPENAI_API_KEY,
|
|
'ENGINE': request.app.state.config.STT_ENGINE,
|
|
'MODEL': request.app.state.config.STT_MODEL,
|
|
'SUPPORTED_CONTENT_TYPES': request.app.state.config.STT_SUPPORTED_CONTENT_TYPES,
|
|
'ALLOWED_EXTENSIONS': request.app.state.config.STT_ALLOWED_EXTENSIONS,
|
|
'WHISPER_MODEL': request.app.state.config.WHISPER_MODEL,
|
|
'DEEPGRAM_API_KEY': request.app.state.config.DEEPGRAM_API_KEY,
|
|
'AZURE_API_KEY': request.app.state.config.AUDIO_STT_AZURE_API_KEY,
|
|
'AZURE_REGION': request.app.state.config.AUDIO_STT_AZURE_REGION,
|
|
'AZURE_LOCALES': request.app.state.config.AUDIO_STT_AZURE_LOCALES,
|
|
'AZURE_BASE_URL': request.app.state.config.AUDIO_STT_AZURE_BASE_URL,
|
|
'AZURE_MAX_SPEAKERS': request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS,
|
|
'MISTRAL_API_KEY': request.app.state.config.AUDIO_STT_MISTRAL_API_KEY,
|
|
'MISTRAL_API_BASE_URL': request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL,
|
|
'MISTRAL_USE_CHAT_COMPLETIONS': request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS,
|
|
},
|
|
}
|
|
|
|
|
|
def load_speech_pipeline(request):
|
|
from datasets import load_dataset
|
|
from transformers import pipeline
|
|
|
|
if request.app.state.speech_synthesiser is None:
|
|
request.app.state.speech_synthesiser = pipeline('text-to-speech', 'microsoft/speecht5_tts')
|
|
|
|
if request.app.state.speech_speaker_embeddings_dataset is None:
|
|
request.app.state.speech_speaker_embeddings_dataset = load_dataset(
|
|
'Matthijs/cmu-arctic-xvectors', split='validation'
|
|
)
|
|
|
|
|
|
async def _raise_tts_error(exc: Exception, r=None) -> None:
|
|
"""Raise a standardised HTTPException from a TTS provider failure."""
|
|
code = r.status if r is not None else 500
|
|
detail = 'Open WebUI: Server Connection Error'
|
|
if r is not None:
|
|
try:
|
|
res = await r.json()
|
|
if 'error' in res:
|
|
msg = res['error']
|
|
detail = f'External: {msg.get("message", msg) if isinstance(msg, dict) else msg}'
|
|
elif 'message' in res:
|
|
detail = f'External: {res["message"]}'
|
|
except Exception:
|
|
detail = f'External: {exc}'
|
|
raise HTTPException(status_code=code, detail=detail)
|
|
|
|
|
|
async def _write_tts_cache(
|
|
file_path: Path,
|
|
audio: bytes,
|
|
body_path: Path,
|
|
payload: dict,
|
|
) -> None:
|
|
"""Persist audio + request metadata to the speech cache."""
|
|
async with aiofiles.open(file_path, 'wb') as f:
|
|
await f.write(audio)
|
|
async with aiofiles.open(body_path, 'w') as f:
|
|
await f.write(json.dumps(payload))
|
|
|
|
|
|
async def _tts_openai(request, payload, file_path, file_body_path, user):
|
|
"""Generate speech via an OpenAI-compatible TTS endpoint."""
|
|
payload['model'] = request.app.state.config.TTS_MODEL
|
|
if not payload.get('voice'):
|
|
payload['voice'] = request.app.state.config.TTS_VOICE
|
|
payload = {**payload, **(request.app.state.config.TTS_OPENAI_PARAMS or {})}
|
|
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': f'Bearer {request.app.state.config.TTS_OPENAI_API_KEY}',
|
|
}
|
|
if ENABLE_FORWARD_USER_INFO_HEADERS:
|
|
headers = include_user_info_headers(headers, user)
|
|
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
r = await session.post(
|
|
url=f'{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/speech',
|
|
json=payload,
|
|
headers=headers,
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
r.raise_for_status()
|
|
|
|
audio_data = await r.read()
|
|
content_type = r.headers.get('Content-Type', 'audio/mpeg')
|
|
|
|
if not await asyncio.to_thread(transcode_audio_to_mp3, audio_data, content_type, file_path):
|
|
async with aiofiles.open(file_path, 'wb') as f:
|
|
await f.write(audio_data)
|
|
|
|
async with aiofiles.open(file_body_path, 'w') as f:
|
|
await f.write(json.dumps(payload))
|
|
|
|
return FileResponse(file_path)
|
|
except Exception as exc:
|
|
log.exception(exc)
|
|
await _raise_tts_error(exc, r)
|
|
|
|
|
|
async def _tts_elevenlabs(request, payload, file_path, file_body_path, user):
|
|
"""Generate speech via the ElevenLabs TTS API."""
|
|
voice_id = payload.get('voice', '')
|
|
if voice_id not in await get_available_voices(request):
|
|
raise HTTPException(status_code=400, detail='Invalid voice id')
|
|
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
async with session.post(
|
|
f'{ELEVENLABS_API_BASE_URL}/v1/text-to-speech/{voice_id}',
|
|
json={
|
|
'text': payload['input'],
|
|
'model_id': request.app.state.config.TTS_MODEL,
|
|
'voice_settings': {'stability': 0.5, 'similarity_boost': 0.5},
|
|
},
|
|
headers={
|
|
'Accept': 'audio/mpeg',
|
|
'Content-Type': 'application/json',
|
|
'xi-api-key': request.app.state.config.TTS_API_KEY,
|
|
},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
) as r:
|
|
r.raise_for_status()
|
|
await _write_tts_cache(file_path, await r.read(), file_body_path, payload)
|
|
return FileResponse(file_path)
|
|
except Exception as exc:
|
|
log.exception(exc)
|
|
await _raise_tts_error(exc, r)
|
|
|
|
|
|
async def _tts_azure(request, payload, file_path, file_body_path, user):
|
|
"""Generate speech via Azure Cognitive Services TTS."""
|
|
az_region = request.app.state.config.TTS_AZURE_SPEECH_REGION or 'eastus'
|
|
az_base = request.app.state.config.TTS_AZURE_SPEECH_BASE_URL
|
|
language = payload.get('voice') or request.app.state.config.TTS_VOICE
|
|
locale = '-'.join(language.split('-')[:2])
|
|
output_format = request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT
|
|
|
|
ssml = (
|
|
f'<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="{locale}">'
|
|
f'<voice name="{language}">{html.escape(payload["input"])}</voice>'
|
|
f'</speak>'
|
|
)
|
|
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
async with session.post(
|
|
(az_base or f'https://{az_region}.tts.speech.microsoft.com') + '/cognitiveservices/v1',
|
|
headers={
|
|
'Ocp-Apim-Subscription-Key': request.app.state.config.TTS_API_KEY,
|
|
'Content-Type': 'application/ssml+xml',
|
|
'X-Microsoft-OutputFormat': output_format,
|
|
},
|
|
data=ssml,
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
) as r:
|
|
r.raise_for_status()
|
|
await _write_tts_cache(file_path, await r.read(), file_body_path, payload)
|
|
return FileResponse(file_path)
|
|
except Exception as exc:
|
|
log.exception(exc)
|
|
await _raise_tts_error(exc, r)
|
|
|
|
|
|
async def _tts_transformers(request, payload, file_path, file_body_path, user):
|
|
"""Generate speech via the local HuggingFace SpeechT5 pipeline (thread-offloaded)."""
|
|
import soundfile as sf
|
|
import torch
|
|
|
|
load_speech_pipeline(request)
|
|
|
|
embeddings = request.app.state.speech_speaker_embeddings_dataset
|
|
model_name = request.app.state.config.TTS_MODEL
|
|
|
|
idx = 6799
|
|
try:
|
|
idx = embeddings['filename'].index(model_name)
|
|
except (ValueError, KeyError):
|
|
log.debug(f'Speaker embedding not found for {model_name}, using default index {idx}')
|
|
|
|
def _run_pipeline():
|
|
speaker_embedding = torch.tensor(embeddings[idx]['xvector']).unsqueeze(0)
|
|
wav = request.app.state.speech_synthesiser(
|
|
payload['input'], # raw text to synthesize
|
|
forward_params={
|
|
'speaker_embeddings': speaker_embedding,
|
|
},
|
|
)
|
|
sf.write(str(file_path), wav['audio'], samplerate=wav['sampling_rate'])
|
|
|
|
await asyncio.to_thread(_run_pipeline)
|
|
|
|
# Audio file already written by sf.write; just persist the request metadata.
|
|
async with aiofiles.open(file_body_path, 'w') as f:
|
|
await f.write(json.dumps(payload))
|
|
return FileResponse(file_path)
|
|
|
|
|
|
async def _tts_mistral(request, payload, file_path, file_body_path, user):
|
|
"""Generate speech via the Mistral TTS API."""
|
|
api_key = request.app.state.config.TTS_MISTRAL_API_KEY
|
|
api_base_url = request.app.state.config.TTS_MISTRAL_API_BASE_URL or 'https://api.mistral.ai/v1'
|
|
|
|
if not api_key:
|
|
raise HTTPException(status_code=400, detail='Mistral API key is required for Mistral TTS')
|
|
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
r = await session.post(
|
|
url=f'{api_base_url}/audio/speech',
|
|
json={
|
|
'input': payload.get('input', ''), # text to synthesize
|
|
'model': request.app.state.config.TTS_MODEL or 'voxtral-mini-tts-2603',
|
|
'voice_id': payload.get('voice', ''),
|
|
'response_format': 'mp3',
|
|
},
|
|
headers={
|
|
'Content-Type': 'application/json',
|
|
'Authorization': f'Bearer {api_key}',
|
|
},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
r.raise_for_status()
|
|
|
|
res = await r.json()
|
|
audio_b64 = res.get('audio_data', '')
|
|
if not audio_b64:
|
|
raise ValueError('No audio_data in Mistral TTS response')
|
|
|
|
await _write_tts_cache(file_path, base64.b64decode(audio_b64), file_body_path, payload)
|
|
return FileResponse(file_path)
|
|
except Exception as exc:
|
|
log.exception(exc)
|
|
await _raise_tts_error(exc, r)
|
|
|
|
|
|
# Dispatcher map: engine name -> handler
|
|
_TTS_ENGINES = {
|
|
'openai': _tts_openai,
|
|
'elevenlabs': _tts_elevenlabs,
|
|
'azure': _tts_azure,
|
|
'transformers': _tts_transformers,
|
|
'mistral': _tts_mistral,
|
|
}
|
|
|
|
|
|
@router.post('/speech')
|
|
async def speech(request: Request, user=Depends(get_verified_user)):
|
|
engine = request.app.state.config.TTS_ENGINE
|
|
if engine == '':
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
|
)
|
|
|
|
if user.role != 'admin' and not await has_permission(
|
|
user.id, 'chat.tts', request.app.state.config.USER_PERMISSIONS
|
|
):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
|
|
)
|
|
|
|
body = await request.body()
|
|
name = hashlib.sha256(
|
|
body + str(engine).encode('utf-8') + str(request.app.state.config.TTS_MODEL).encode('utf-8')
|
|
).hexdigest()
|
|
|
|
file_path = SPEECH_CACHE_DIR.joinpath(f'{name}.mp3')
|
|
file_body_path = SPEECH_CACHE_DIR.joinpath(f'{name}.json')
|
|
|
|
# Return cached result if available
|
|
if file_path.is_file():
|
|
return FileResponse(file_path)
|
|
|
|
try:
|
|
payload = json.loads(body)
|
|
except Exception as exc:
|
|
log.exception(exc)
|
|
raise HTTPException(status_code=400, detail='Invalid JSON payload')
|
|
|
|
handler = _TTS_ENGINES.get(engine)
|
|
if handler is None:
|
|
raise HTTPException(status_code=400, detail=f'Unsupported TTS engine: {engine}')
|
|
|
|
return await handler(request, payload, file_path, file_body_path, user)
|
|
|
|
|
|
async def _transcribe_whisper(request, file_path, languages, file_dir, id):
|
|
if request.app.state.faster_whisper_model is None:
|
|
request.app.state.faster_whisper_model = set_faster_whisper_model(request.app.state.config.WHISPER_MODEL)
|
|
|
|
model = request.app.state.faster_whisper_model
|
|
|
|
def _run():
|
|
segments, info = model.transcribe(
|
|
file_path,
|
|
beam_size=5,
|
|
vad_filter=WHISPER_VAD_FILTER,
|
|
language=languages[0],
|
|
multilingual=WHISPER_MULTILINGUAL,
|
|
)
|
|
log.info("Detected language '%s' with probability %f" % (info.language, info.language_probability))
|
|
return ''.join([segment.text for segment in list(segments)])
|
|
|
|
transcript = await asyncio.to_thread(_run)
|
|
data = {'text': transcript.strip()}
|
|
|
|
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
|
|
await f.write(json.dumps(data))
|
|
|
|
log.debug(data)
|
|
return data
|
|
|
|
|
|
async def _transcribe_openai(request, file_path, filename, languages, file_dir, id, user=None):
|
|
"""Transcribe audio via an OpenAI-compatible STT endpoint."""
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
for language in languages:
|
|
payload = {'model': request.app.state.config.STT_MODEL}
|
|
if language:
|
|
payload['language'] = language
|
|
|
|
headers = {'Authorization': f'Bearer {request.app.state.config.STT_OPENAI_API_KEY}'}
|
|
if user and ENABLE_FORWARD_USER_INFO_HEADERS:
|
|
headers = include_user_info_headers(headers, user)
|
|
|
|
form_data = aiohttp.FormData()
|
|
for key, value in payload.items():
|
|
form_data.add_field(key, str(value))
|
|
form_data.add_field('file', open(file_path, 'rb'), filename=filename)
|
|
|
|
r = await session.post(
|
|
url=f'{request.app.state.config.STT_OPENAI_API_BASE_URL}/audio/transcriptions',
|
|
headers=headers,
|
|
data=form_data,
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
if r.status == 200:
|
|
break
|
|
|
|
r.raise_for_status()
|
|
data = await r.json()
|
|
|
|
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
|
|
await f.write(json.dumps(data))
|
|
return data
|
|
except Exception as e:
|
|
log.exception(e)
|
|
detail = None
|
|
if r is not None:
|
|
try:
|
|
res = await r.json()
|
|
if 'error' in res:
|
|
detail = f'External: {res["error"].get("message", "")}'
|
|
except Exception:
|
|
detail = f'External: {e}'
|
|
raise Exception(detail if detail else 'Open WebUI: Server Connection Error')
|
|
|
|
|
|
async def _transcribe_deepgram(request, file_path, languages, file_dir, id):
|
|
"""Transcribe audio via the Deepgram listen API with language fallback."""
|
|
content_type = mimetypes.guess_type(file_path)[0] or 'audio/wav'
|
|
|
|
async with aiofiles.open(file_path, 'rb') as f:
|
|
audio_bytes = await f.read()
|
|
|
|
api_key = request.app.state.config.DEEPGRAM_API_KEY
|
|
stt_model = request.app.state.config.STT_MODEL
|
|
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
for lang in languages:
|
|
query: dict = {'smart_format': 'true'}
|
|
if stt_model:
|
|
query['model'] = stt_model
|
|
if lang:
|
|
query['language'] = lang
|
|
|
|
r = await session.post(
|
|
'https://api.deepgram.com/v1/listen',
|
|
headers={'Authorization': f'Token {api_key}', 'Content-Type': content_type},
|
|
params=query,
|
|
data=audio_bytes,
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
if r.status == 200:
|
|
break
|
|
|
|
r.raise_for_status()
|
|
body = await r.json()
|
|
|
|
# Parse the Deepgram response structure
|
|
try:
|
|
transcript = body['results']['channels'][0]['alternatives'][0].get('transcript', '').strip()
|
|
except (KeyError, IndexError) as exc:
|
|
log.error(f'Malformed Deepgram response: {exc}')
|
|
raise Exception('Failed to parse Deepgram response') from exc
|
|
|
|
data = {'text': transcript}
|
|
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
|
|
await f.write(json.dumps(data))
|
|
return data
|
|
|
|
except Exception as e:
|
|
log.exception(e)
|
|
detail = 'Open WebUI: Server Connection Error'
|
|
if r is not None:
|
|
try:
|
|
res = await r.json()
|
|
msg = (
|
|
res.get('error', {}).get('message', '')
|
|
if isinstance(res.get('error'), dict)
|
|
else str(res.get('error', ''))
|
|
)
|
|
if msg:
|
|
detail = f'External: {msg}'
|
|
except Exception:
|
|
detail = f'External: {e}'
|
|
raise Exception(detail)
|
|
|
|
|
|
async def _transcribe_azure(request, file_path, filename, file_dir, id):
|
|
"""Transcribe audio via Azure Cognitive Services batch transcription."""
|
|
if not os.path.isfile(file_path):
|
|
raise HTTPException(status_code=400, detail='Audio file not found')
|
|
|
|
audio_size = os.path.getsize(file_path)
|
|
if audio_size > AZURE_MAX_FILE_SIZE:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f'File size ({audio_size // (1024 * 1024)}MB) exceeds Azure limit of {AZURE_MAX_FILE_SIZE_MB}MB',
|
|
)
|
|
|
|
api_key = request.app.state.config.AUDIO_STT_AZURE_API_KEY
|
|
region = request.app.state.config.AUDIO_STT_AZURE_REGION or 'eastus'
|
|
locale_str = request.app.state.config.AUDIO_STT_AZURE_LOCALES
|
|
base_url = request.app.state.config.AUDIO_STT_AZURE_BASE_URL
|
|
max_speakers = request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS or 3
|
|
|
|
# Default to a broad set of locales when none are configured
|
|
if len(locale_str) < 2:
|
|
locale_str = ','.join(
|
|
[
|
|
'en-US',
|
|
'es-ES',
|
|
'es-MX',
|
|
'fr-FR',
|
|
'hi-IN',
|
|
'it-IT',
|
|
'de-DE',
|
|
'en-GB',
|
|
'en-IN',
|
|
'ja-JP',
|
|
'ko-KR',
|
|
'pt-BR',
|
|
'zh-CN',
|
|
]
|
|
)
|
|
|
|
if not api_key or not region:
|
|
raise HTTPException(status_code=400, detail='Azure API key and region are required for Azure STT')
|
|
|
|
# Build the transcription definition payload
|
|
definition = json.dumps(
|
|
{'locales': locale_str.split(','), 'diarization': {'maxSpeakers': max_speakers, 'enabled': True}}
|
|
if locale_str
|
|
else {}
|
|
)
|
|
endpoint = (
|
|
base_url or f'https://{region}.api.cognitive.microsoft.com'
|
|
) + '/speechtotext/transcriptions:transcribe?api-version=2024-11-15'
|
|
|
|
form_data = aiohttp.FormData()
|
|
form_data.add_field('definition', definition)
|
|
form_data.add_field('audio', open(file_path, 'rb'), filename=filename)
|
|
|
|
r = None
|
|
try:
|
|
session = await get_session()
|
|
r = await session.post(
|
|
url=endpoint,
|
|
data=form_data,
|
|
headers={'Ocp-Apim-Subscription-Key': api_key},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
r.raise_for_status()
|
|
response = await r.json()
|
|
|
|
if not response.get('combinedPhrases'):
|
|
raise ValueError('No transcription found in response')
|
|
|
|
transcript = response['combinedPhrases'][0].get('text', '').strip()
|
|
if not transcript:
|
|
raise ValueError('Empty transcript in response')
|
|
|
|
data = {'text': transcript}
|
|
|
|
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
|
|
await f.write(json.dumps(data))
|
|
|
|
log.debug(data)
|
|
return data
|
|
|
|
except (KeyError, IndexError, ValueError) as e:
|
|
log.exception('Error parsing Azure response')
|
|
raise HTTPException(status_code=500, detail=f'Failed to parse Azure response: {str(e)}')
|
|
except aiohttp.ClientResponseError as e:
|
|
log.exception(e)
|
|
detail = None
|
|
try:
|
|
if r is not None and r.status != 200:
|
|
res = await r.json()
|
|
if 'code' in res and 'message' in res:
|
|
azure_code = res.get('innerError', {}).get('code', res['code'])
|
|
user_facing_codes = {
|
|
'EmptyAudioFile',
|
|
'AudioLengthLimitExceeded',
|
|
'NoLanguageIdentified',
|
|
'MultipleLanguagesIdentified',
|
|
}
|
|
if azure_code in user_facing_codes:
|
|
detail = res['message']
|
|
else:
|
|
log.error(f'Azure STT error [{azure_code}]: {res["message"]}')
|
|
detail = 'An error occurred during transcription.'
|
|
elif 'error' in res:
|
|
detail = f'External: {res["error"].get("message", "")}'
|
|
except Exception:
|
|
detail = f'External: {e}'
|
|
raise HTTPException(
|
|
status_code=e.status if e.status else 500,
|
|
detail=detail if detail else 'Open WebUI: Server Connection Error',
|
|
)
|
|
|
|
|
|
async def transcription_handler(request, file_path, metadata, user=None):
|
|
filename = os.path.basename(file_path)
|
|
file_dir = os.path.dirname(file_path)
|
|
id = filename.split('.')[0]
|
|
|
|
metadata = metadata or {}
|
|
|
|
languages = [
|
|
metadata.get('language', None) if not WHISPER_LANGUAGE else WHISPER_LANGUAGE,
|
|
None, # Always fallback to None in case transcription fails
|
|
]
|
|
|
|
if request.app.state.config.STT_ENGINE == '':
|
|
return await _transcribe_whisper(request, file_path, languages, file_dir, id)
|
|
elif request.app.state.config.STT_ENGINE == 'openai':
|
|
return await _transcribe_openai(request, file_path, filename, languages, file_dir, id, user)
|
|
elif request.app.state.config.STT_ENGINE == 'deepgram':
|
|
return await _transcribe_deepgram(request, file_path, languages, file_dir, id)
|
|
elif request.app.state.config.STT_ENGINE == 'azure':
|
|
return await _transcribe_azure(request, file_path, filename, file_dir, id)
|
|
|
|
elif request.app.state.config.STT_ENGINE == 'mistral':
|
|
return await _transcribe_mistral(request, file_path, filename, metadata, file_dir, id)
|
|
|
|
|
|
async def _transcribe_mistral(request, file_path, filename, metadata, file_dir, id):
|
|
"""Transcribe audio via the Mistral STT API."""
|
|
if not os.path.isfile(file_path):
|
|
raise HTTPException(status_code=400, detail='Audio file not found')
|
|
|
|
file_size = os.path.getsize(file_path)
|
|
if file_size > MAX_FILE_SIZE:
|
|
raise HTTPException(status_code=400, detail=f'File size exceeds limit of {MAX_FILE_SIZE_MB}MB')
|
|
|
|
api_key = request.app.state.config.AUDIO_STT_MISTRAL_API_KEY
|
|
api_base_url = request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL or 'https://api.mistral.ai/v1'
|
|
use_chat_completions = request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS
|
|
|
|
if not api_key:
|
|
raise HTTPException(status_code=400, detail='Mistral API key is required for Mistral STT')
|
|
|
|
r = None
|
|
try:
|
|
model = request.app.state.config.STT_MODEL or 'voxtral-mini-latest'
|
|
log.info(
|
|
f'Mistral STT - model: {model}, method: {"chat_completions" if use_chat_completions else "transcriptions"}'
|
|
)
|
|
|
|
session = await get_session()
|
|
if use_chat_completions:
|
|
audio_file_to_use = file_path
|
|
if is_audio_conversion_required(file_path):
|
|
log.debug('Converting audio to mp3 for chat completions API')
|
|
converted_path = await asyncio.to_thread(convert_audio_to_mp3, file_path)
|
|
if converted_path:
|
|
audio_file_to_use = converted_path
|
|
else:
|
|
log.error('Audio conversion failed')
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail='Audio conversion failed. Chat completions API requires mp3 or wav format.',
|
|
)
|
|
|
|
async with aiofiles.open(audio_file_to_use, 'rb') as audio_file:
|
|
raw = await audio_file.read()
|
|
audio_base64 = {
|
|
'data': base64.b64encode(raw).decode('utf-8'),
|
|
'format': mimetypes.guess_extension(mimetypes.guess_type(audio_file_to_use)[0]).lstrip('.'),
|
|
}
|
|
|
|
language = metadata.get('language', None) if metadata else None
|
|
text_instruction = (
|
|
f'Transcribe this audio exactly as spoken in {language}. Do not translate it.'
|
|
if language
|
|
else 'Transcribe this audio exactly as spoken in its original language. Do not translate it to another language.'
|
|
)
|
|
|
|
payload = {
|
|
'model': model,
|
|
'messages': [
|
|
{
|
|
'role': 'user',
|
|
'content': [
|
|
{'type': 'input_audio', 'input_audio': audio_base64},
|
|
{'type': 'text', 'text': text_instruction},
|
|
],
|
|
}
|
|
],
|
|
}
|
|
|
|
r = await session.post(
|
|
url=f'{api_base_url}/chat/completions',
|
|
json=payload,
|
|
headers={'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
r.raise_for_status()
|
|
response = await r.json()
|
|
|
|
transcript = response.get('choices', [{}])[0].get('message', {}).get('content', '').strip()
|
|
if not transcript:
|
|
raise ValueError('Empty transcript in response')
|
|
data = {'text': transcript}
|
|
|
|
else:
|
|
mime_type, _ = mimetypes.guess_type(file_path)
|
|
if not mime_type:
|
|
mime_type = 'audio/webm'
|
|
|
|
form_data = aiohttp.FormData()
|
|
form_data.add_field('model', model)
|
|
|
|
language = metadata.get('language', None) if metadata else None
|
|
if language:
|
|
form_data.add_field('language', language)
|
|
|
|
form_data.add_field('file', open(file_path, 'rb'), filename=filename, content_type=mime_type)
|
|
|
|
r = await session.post(
|
|
url=f'{api_base_url}/audio/transcriptions',
|
|
data=form_data,
|
|
headers={'Authorization': f'Bearer {api_key}'},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
)
|
|
r.raise_for_status()
|
|
response = await r.json()
|
|
|
|
transcript = response.get('text', '').strip()
|
|
if not transcript:
|
|
raise ValueError('Empty transcript in response')
|
|
data = {'text': transcript}
|
|
|
|
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
|
|
await f.write(json.dumps(data))
|
|
|
|
log.debug(data)
|
|
return data
|
|
|
|
except ValueError as e:
|
|
log.exception('Error parsing Mistral response')
|
|
raise HTTPException(status_code=500, detail=f'Failed to parse Mistral response: {str(e)}')
|
|
except aiohttp.ClientResponseError as e:
|
|
log.exception(e)
|
|
detail = None
|
|
try:
|
|
if r is not None and r.status != 200:
|
|
res = await r.json()
|
|
if 'error' in res:
|
|
detail = f'External: {res["error"].get("message", "")}'
|
|
else:
|
|
detail = f'External: {await r.text()}'
|
|
except Exception:
|
|
detail = f'External: {e}'
|
|
raise HTTPException(
|
|
status_code=e.status if e.status else 500,
|
|
detail=detail if detail else 'Open WebUI: Server Connection Error',
|
|
)
|
|
|
|
|
|
async def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None, user=None):
|
|
log.info(f'transcribe: {file_path} {metadata}')
|
|
|
|
if BYPASS_PYDUB_PREPROCESSING:
|
|
log.info('Bypassing pydub preprocessing (BYPASS_PYDUB_PREPROCESSING=true)')
|
|
chunk_paths = [file_path]
|
|
else:
|
|
if is_audio_conversion_required(file_path):
|
|
file_path = await asyncio.to_thread(convert_audio_to_mp3, file_path)
|
|
if not file_path:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail='Audio conversion failed. The audio file may be corrupted or empty.',
|
|
)
|
|
|
|
try:
|
|
file_path = await asyncio.to_thread(compress_audio, file_path)
|
|
except Exception as e:
|
|
log.exception(e)
|
|
|
|
# Always produce a list of chunk paths (could be one entry if small)
|
|
try:
|
|
chunk_paths = await asyncio.to_thread(split_audio, file_path, MAX_FILE_SIZE)
|
|
print(f'Chunk paths: {chunk_paths}')
|
|
except Exception as e:
|
|
log.exception(e)
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=ERROR_MESSAGES.DEFAULT(e),
|
|
)
|
|
|
|
results = []
|
|
try:
|
|
tasks = [transcription_handler(request, chunk_path, metadata, user) for chunk_path in chunk_paths]
|
|
for coro in asyncio.as_completed(tasks):
|
|
try:
|
|
results.append(await coro)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as transcribe_exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f'Error transcribing chunk: {transcribe_exc}',
|
|
)
|
|
finally:
|
|
# Clean up only the temporary chunks, never the original file
|
|
for chunk_path in chunk_paths:
|
|
if chunk_path != file_path and os.path.isfile(chunk_path):
|
|
try:
|
|
os.remove(chunk_path)
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
'text': ' '.join([result['text'] for result in results]),
|
|
}
|
|
|
|
|
|
def compress_audio(file_path):
|
|
if os.path.getsize(file_path) > MAX_FILE_SIZE:
|
|
id = os.path.splitext(os.path.basename(file_path))[0] # Handles names with multiple dots
|
|
file_dir = os.path.dirname(file_path)
|
|
|
|
audio = AudioSegment.from_file(file_path)
|
|
audio = audio.set_frame_rate(16000).set_channels(1) # Compress audio
|
|
|
|
compressed_path = os.path.join(file_dir, f'{id}_compressed.mp3')
|
|
audio.export(compressed_path, format='mp3', bitrate='32k')
|
|
# log.debug(f"Compressed audio to {compressed_path}") # Uncomment if log is defined
|
|
|
|
return compressed_path
|
|
else:
|
|
return file_path
|
|
|
|
|
|
def split_audio(file_path, max_bytes, format='mp3', bitrate='32k'):
|
|
"""
|
|
Splits audio into chunks not exceeding max_bytes.
|
|
Returns a list of chunk file paths. If audio fits, returns list with original path.
|
|
"""
|
|
file_size = os.path.getsize(file_path)
|
|
if file_size <= max_bytes:
|
|
return [file_path] # Nothing to split
|
|
|
|
audio = AudioSegment.from_file(file_path)
|
|
duration_ms = len(audio)
|
|
orig_size = file_size
|
|
|
|
approx_chunk_ms = max(int(duration_ms * (max_bytes / orig_size)) - 1000, 1000)
|
|
chunks = []
|
|
start = 0
|
|
i = 0
|
|
|
|
base, _ = os.path.splitext(file_path)
|
|
|
|
while start < duration_ms:
|
|
end = min(start + approx_chunk_ms, duration_ms)
|
|
chunk = audio[start:end]
|
|
chunk_path = f'{base}_chunk_{i}.{format}'
|
|
chunk.export(chunk_path, format=format, bitrate=bitrate)
|
|
|
|
# Reduce chunk duration if still too large
|
|
while os.path.getsize(chunk_path) > max_bytes and (end - start) > 5000:
|
|
end = start + ((end - start) // 2)
|
|
chunk = audio[start:end]
|
|
chunk.export(chunk_path, format=format, bitrate=bitrate)
|
|
|
|
if os.path.getsize(chunk_path) > max_bytes:
|
|
os.remove(chunk_path)
|
|
raise Exception('Audio chunk cannot be reduced below max file size.')
|
|
|
|
chunks.append(chunk_path)
|
|
start = end
|
|
i += 1
|
|
|
|
return chunks
|
|
|
|
|
|
@router.post('/transcriptions')
|
|
async def transcription(
|
|
request: Request,
|
|
file: UploadFile = File(...),
|
|
language: Optional[str] = Form(None),
|
|
user=Depends(get_verified_user),
|
|
):
|
|
if user.role != 'admin' and not await has_permission(
|
|
user.id, 'chat.stt', request.app.state.config.USER_PERMISSIONS
|
|
):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
|
|
)
|
|
log.info(f'file.content_type: {file.content_type}')
|
|
stt_supported_content_types = getattr(request.app.state.config, 'STT_SUPPORTED_CONTENT_TYPES', [])
|
|
|
|
if not strict_match_mime_type(stt_supported_content_types, file.content_type):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=ERROR_MESSAGES.FILE_NOT_SUPPORTED,
|
|
)
|
|
|
|
try:
|
|
safe_name = os.path.basename(file.filename) if file.filename else ''
|
|
ext = safe_name.rsplit('.', 1)[-1].lower() if '.' in safe_name else ''
|
|
|
|
allowed_extensions = getattr(request.app.state.config, 'STT_ALLOWED_EXTENSIONS', [])
|
|
if allowed_extensions and ext not in allowed_extensions:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail='Invalid audio file extension',
|
|
)
|
|
|
|
id = uuid.uuid4()
|
|
|
|
filename = f'{id}.{ext}'
|
|
contents = await file.read()
|
|
|
|
file_dir = os.path.join(CACHE_DIR, 'audio', 'transcriptions')
|
|
os.makedirs(file_dir, exist_ok=True)
|
|
file_path = os.path.join(file_dir, filename)
|
|
|
|
# Defense-in-depth: ensure resolved path stays within intended directory
|
|
if not os.path.realpath(file_path).startswith(os.path.realpath(file_dir)):
|
|
raise ValueError('Invalid file path detected')
|
|
|
|
with open(file_path, 'wb') as f:
|
|
f.write(contents)
|
|
|
|
try:
|
|
metadata = None
|
|
|
|
if language:
|
|
metadata = {'language': language}
|
|
|
|
result = await transcribe(request, file_path, metadata, user)
|
|
|
|
return {
|
|
**result,
|
|
'filename': os.path.basename(file_path),
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
log.exception(e)
|
|
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail='Transcription failed.',
|
|
)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
log.exception(e)
|
|
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail='Transcription failed.',
|
|
)
|
|
|
|
|
|
async def get_available_models(request: Request) -> list[dict]:
|
|
"""Return the list of available TTS models for the configured engine."""
|
|
available_models = []
|
|
engine = request.app.state.config.TTS_ENGINE
|
|
_timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST)
|
|
|
|
if engine == 'openai':
|
|
base_url = request.app.state.config.TTS_OPENAI_API_BASE_URL
|
|
if not base_url.startswith('https://api.openai.com'):
|
|
session = await get_session()
|
|
try:
|
|
async with session.get(
|
|
f'{base_url}/audio/models',
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
data = await resp.json()
|
|
available_models = data.get('models', [])
|
|
except Exception as e:
|
|
log.debug(f'/audio/models not available, trying /models fallback: {e}')
|
|
try:
|
|
async with session.get(
|
|
f'{base_url}/models',
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
data = await resp.json()
|
|
available_models = data.get('data', data.get('models', []))
|
|
except Exception as e2:
|
|
log.error(f'Error fetching models from custom endpoint: {e2}')
|
|
available_models = [{'id': 'tts-1'}, {'id': 'tts-1-hd'}]
|
|
else:
|
|
available_models = [{'id': 'tts-1'}, {'id': 'tts-1-hd'}]
|
|
|
|
elif engine == 'elevenlabs':
|
|
try:
|
|
session = await get_session()
|
|
async with session.get(
|
|
f'{ELEVENLABS_API_BASE_URL}/v1/models',
|
|
headers={
|
|
'xi-api-key': request.app.state.config.TTS_API_KEY,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
models = await resp.json()
|
|
available_models = [{'name': m['name'], 'id': m['model_id']} for m in models]
|
|
except Exception as e:
|
|
log.error(f'Error fetching models: {e}')
|
|
|
|
elif engine == 'mistral':
|
|
available_models = [{'id': 'voxtral-mini-tts-2603'}]
|
|
|
|
return available_models
|
|
|
|
|
|
@router.get('/models')
|
|
async def get_models(request: Request, user=Depends(get_verified_user)):
|
|
return {'models': await get_available_models(request)}
|
|
|
|
|
|
_OPENAI_DEFAULT_VOICES = {
|
|
'alloy': 'alloy',
|
|
'echo': 'echo',
|
|
'fable': 'fable',
|
|
'onyx': 'onyx',
|
|
'nova': 'nova',
|
|
'shimmer': 'shimmer',
|
|
}
|
|
|
|
|
|
async def get_available_voices(request) -> dict:
|
|
"""Return ``{voice_id: voice_name}`` for the configured TTS engine."""
|
|
engine = request.app.state.config.TTS_ENGINE
|
|
_timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST)
|
|
|
|
if engine == 'openai':
|
|
base_url = request.app.state.config.TTS_OPENAI_API_BASE_URL
|
|
if not base_url.startswith('https://api.openai.com'):
|
|
try:
|
|
session = await get_session()
|
|
async with session.get(
|
|
f'{base_url}/audio/voices',
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
data = await resp.json()
|
|
return {v['id']: v['name'] for v in data.get('voices', [])}
|
|
except Exception as e:
|
|
log.error(f'Error fetching voices from custom endpoint: {e}')
|
|
return dict(_OPENAI_DEFAULT_VOICES)
|
|
return dict(_OPENAI_DEFAULT_VOICES)
|
|
|
|
if engine == 'elevenlabs':
|
|
try:
|
|
session = await get_session()
|
|
async with session.get(
|
|
f'{ELEVENLABS_API_BASE_URL}/v1/voices',
|
|
headers={
|
|
'xi-api-key': request.app.state.config.TTS_API_KEY,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
voices_data = await resp.json()
|
|
return {v['voice_id']: v['name'] for v in voices_data.get('voices', [])}
|
|
except Exception as e:
|
|
log.error(f'Error fetching ElevenLabs voices: {e}')
|
|
return {}
|
|
|
|
if engine == 'azure':
|
|
try:
|
|
region = request.app.state.config.TTS_AZURE_SPEECH_REGION
|
|
base_url = request.app.state.config.TTS_AZURE_SPEECH_BASE_URL
|
|
url = (base_url or f'https://{region}.tts.speech.microsoft.com') + '/cognitiveservices/voices/list'
|
|
|
|
session = await get_session()
|
|
async with session.get(
|
|
url,
|
|
headers={'Ocp-Apim-Subscription-Key': request.app.state.config.TTS_API_KEY},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
voices = await resp.json()
|
|
return {v['ShortName']: f'{v["DisplayName"]} ({v["ShortName"]})' for v in voices}
|
|
except Exception as e:
|
|
log.error(f'Error fetching Azure voices: {e}')
|
|
return {}
|
|
|
|
if engine == 'mistral':
|
|
api_key = request.app.state.config.TTS_MISTRAL_API_KEY
|
|
api_base_url = request.app.state.config.TTS_MISTRAL_API_BASE_URL or 'https://api.mistral.ai/v1'
|
|
if api_key:
|
|
try:
|
|
session = await get_session()
|
|
async with session.get(
|
|
f'{api_base_url}/audio/voices',
|
|
headers={'Authorization': f'Bearer {api_key}'},
|
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
|
timeout=_timeout,
|
|
) as resp:
|
|
resp.raise_for_status()
|
|
voices_data = await resp.json()
|
|
items = voices_data.get('items', []) if isinstance(voices_data, dict) else voices_data
|
|
result = {}
|
|
for v in items:
|
|
if isinstance(v, dict):
|
|
vid = v.get('voice_id', v.get('id', ''))
|
|
if vid:
|
|
result[vid] = v.get('name', vid)
|
|
return result
|
|
except Exception as e:
|
|
log.error(f'Error fetching Mistral voices: {e}')
|
|
|
|
return {}
|
|
|
|
|
|
@router.get('/voices')
|
|
async def get_voices(request: Request, user=Depends(get_verified_user)):
|
|
return {'voices': [{'id': k, 'name': v} for k, v in (await get_available_voices(request)).items()]}
|