diff --git a/backend/open_webui/utils/ask_user.py b/backend/open_webui/utils/ask_user.py new file mode 100644 index 0000000000..6c4168d2d7 --- /dev/null +++ b/backend/open_webui/utils/ask_user.py @@ -0,0 +1,144 @@ +from collections.abc import Callable + +from open_webui.utils.json_codec import JSONCodec + + +ASK_USER_NAME = 'ask_user' + + +def get_ask_user_tool_call(tool_calls: list[dict]) -> tuple[dict | None, str | None]: + ask_user_calls = [ + tool_call for tool_call in tool_calls if tool_call.get('function', {}).get('name') == ASK_USER_NAME + ] + if not ask_user_calls: + return None, None + if len(tool_calls) != 1: + return ask_user_calls[0], 'Error: ask_user must be called by itself after research.' + if len(ask_user_calls) != 1: + return ask_user_calls[0], 'Error: only one ask_user call is allowed per turn.' + return ask_user_calls[0], None + + +def normalize_ask_user_request(arguments: dict) -> dict: + questions = arguments.get('questions') + if not isinstance(questions, list) or not 1 <= len(questions) <= 3: + raise ValueError('ask_user requires 1-3 questions.') + + normalized_questions = [] + seen_ids = set() + allow_other = bool(arguments.get('allow_other', True)) + for index, question in enumerate(questions): + if not isinstance(question, dict): + raise ValueError('Each question must be an object.') + + question_id = str(question.get('id') or '').strip()[:64] + if not question_id: + raise ValueError('Each question requires a non-empty id.') + if question_id in seen_ids: + raise ValueError(f'Duplicate question id: {question_id}') + seen_ids.add(question_id) + + options = question.get('options') + if not isinstance(options, list) or not 2 <= len(options) <= 3: + raise ValueError('Each question requires 2-3 options.') + + normalized_options = [] + for option in options: + if not isinstance(option, dict): + raise ValueError('Each option must be an object.') + label = str(option.get('label') or '').strip()[:80] + description = str(option.get('description') or '').strip()[:240] + if not label or not description: + raise ValueError('Each option requires a label and description.') + normalized_options.append({'label': label, 'description': description}) + + question_text = str(question.get('question') or '').strip()[:500] + if not question_text: + raise ValueError('Each question requires question text.') + + normalized_questions.append( + { + 'id': question_id, + 'header': str(question.get('header') or '').strip()[:48] or f'Question {index + 1}', + 'question': question_text, + 'options': normalized_options, + 'allow_other': bool(question.get('allow_other', allow_other)), + } + ) + + timeout_ms = arguments.get('timeout_ms', 120_000) + if isinstance(timeout_ms, bool) or not isinstance(timeout_ms, int) or not 60_000 <= timeout_ms <= 240_000: + timeout_ms = 120_000 + + return { + 'questions': normalized_questions, + 'allow_other': allow_other, + 'timeout_ms': timeout_ms, + } + + +def stage_ask_user_tool_call( + tool_calls: list[dict], + output: list[dict], + make_output_id: Callable[[str], str], +) -> dict | None: + tool_call, error = get_ask_user_tool_call(tool_calls) + if not tool_call: + return None + + call_id = tool_call.get('id') or make_output_id('fc') + raw_arguments = tool_call.get('function', {}).get('arguments', '{}') + arguments = raw_arguments + + if not error: + try: + parsed_arguments = JSONCodec.loads(raw_arguments or '{}') + if not isinstance(parsed_arguments, dict): + raise ValueError('ask_user arguments must be an object.') + arguments = JSONCodec.dumps(normalize_ask_user_request(parsed_arguments)) + except (JSONCodec.JSONDecodeError, TypeError, ValueError) as exc: + error = f'Error: {exc}' + + item = { + 'type': 'function_call', + 'id': call_id or make_output_id('fc'), + 'call_id': call_id, + 'name': ASK_USER_NAME, + 'arguments': arguments, + 'status': 'completed' if error else 'pending', + } + + existing_item = next( + ( + existing + for existing in output + if existing.get('type') == 'function_call' + and ( + existing.get('call_id') == call_id + or existing.get('id') == tool_call.get('id') + or ( + not existing.get('call_id') + and existing.get('name') == ASK_USER_NAME + and existing.get('status') not in {'rejected', 'failed'} + ) + ) + ), + None, + ) + if existing_item: + existing_item.update(item) + else: + output.append(item) + + if error: + output.append( + { + 'type': 'function_call_output', + 'id': make_output_id('fco'), + 'call_id': call_id, + 'output': [{'type': 'input_text', 'text': error}], + 'status': 'completed', + } + ) + + return {'call_id': call_id, 'error': error, 'item': item} diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 50709b446b..4f533b8eb7 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -79,6 +79,7 @@ from open_webui.socket.main import ( from open_webui.utils.access_control import has_connection_access, has_permission from open_webui.utils.access_control.files import get_owner_accessible_folder_files from open_webui.utils.access_control.folders import has_folder_access +from open_webui.utils.ask_user import stage_ask_user_tool_call from open_webui.utils.chat import generate_chat_completion from open_webui.utils.chat_id import is_saved_chat_id from open_webui.utils.code_interpreter import execute_code_jupyter @@ -3114,6 +3115,12 @@ async def drain_approved_tool_calls(request, form_data, user, model, metadata) - event_emitter, event_caller = await get_event_emitter_and_caller(metadata) changed = False for item in approved_calls: + if item.get('name') == 'ask_user': + item['status'] = 'pending' + item.pop('approved', None) + changed = True + continue + tool_call = { 'id': item.get('call_id', ''), 'type': 'function', @@ -5094,11 +5101,12 @@ async def streaming_chat_response_handler(response, ctx): } responses_api_tool_calls = [] for item in output: - if item.get('type') == 'function_call' and item.get('call_id') not in handled_call_ids: + call_id = item.get('call_id') or item.get('id') or output_id('fc') + if item.get('type') == 'function_call' and call_id not in handled_call_ids: arguments = item.get('arguments', '{}') responses_api_tool_calls.append( { - 'id': item.get('call_id', ''), + 'id': call_id, 'index': len(responses_api_tool_calls), 'function': { 'name': item.get('name', ''), @@ -5148,6 +5156,22 @@ async def streaming_chat_response_handler(response, ctx): tool_call_iterations += 1 response_tool_calls = tool_calls.pop(0) + ask_user_stage = stage_ask_user_tool_call(response_tool_calls, output, output_id) + if ask_user_stage: + if ask_user_stage['error']: + await event_emitter({'type': 'chat:completion', 'data': {'output': full_output()}}) + continue + + if is_saved_chat_id(metadata.get('chat_id')) and metadata.get('message_id'): + await pause_for_tool_approval( + metadata['chat_id'], + metadata['message_id'], + output, + form_data, + metadata, + ) + await event_emitter({'type': 'chat:completion', 'data': {'output': full_output()}}) + return # Append function_call items for each tool call # (Responses API already has them from streaming, so skip duplicates) diff --git a/backend/open_webui/utils/tool_approval.py b/backend/open_webui/utils/tool_approval.py index 83bfd4dde9..de43c77550 100644 --- a/backend/open_webui/utils/tool_approval.py +++ b/backend/open_webui/utils/tool_approval.py @@ -51,6 +51,7 @@ async def resolve_tool_call_output( if not function_call: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='Tool call not found.') function_call.setdefault('call_id', form_data.call_id) + tool_name = function_call.get('name') if any( item.get('type') == 'function_call_output' and item.get('call_id') == form_data.call_id for item in output @@ -58,6 +59,8 @@ async def resolve_tool_call_output( raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail='Tool call has already been resolved.') if form_data.action == 'approve': + if tool_name == 'ask_user': + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='ask_user requires an answer or deny.') function_call['status'] = 'queued' function_call['approved'] = True elif form_data.action == 'reject': @@ -67,13 +70,21 @@ async def resolve_tool_call_output( 'type': 'function_call_output', 'id': f'fco_{form_data.call_id}', 'call_id': form_data.call_id, - 'output': [{'type': 'input_text', 'text': 'Tool call was denied by the user.'}], + 'output': [{'type': 'input_text', 'text': 'Error: tool call rejected by user.'}], 'status': 'rejected', } ) else: + if tool_name != 'ask_user': + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Tool call does not accept answers.') + if form_data.answers is None and not form_data.timed_out: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Answers are required for ask_user.') function_call['status'] = 'completed' - answer_payload = {'answers': form_data.answers, 'timed_out': form_data.timed_out} + answer_payload = ( + {'status': 'cancelled', 'answers': {}, 'timed_out': True} + if form_data.timed_out + else {'status': 'answered', 'answers': form_data.answers or {}} + ) output.append( { 'type': 'function_call_output', diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index bb682d4cdb..dd8eccc96f 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -542,9 +542,9 @@ async def get_builtin_tools( # Helper to check if a builtin tool category is enabled via meta.builtinTools # Defaults to True if not specified (backward compatible) - def is_builtin_tool_enabled(category: str) -> bool: + def is_builtin_tool_enabled(category: str, default: bool = True) -> bool: builtin_tools = model.get('info', {}).get('meta', {}).get('builtinTools', {}) - return builtin_tools.get(category, True) + return builtin_tools.get(category, default) # Helper to check user-level feature permission (admins always pass) user = extra_params.get('__user__', {}) @@ -584,7 +584,7 @@ async def get_builtin_tools( if is_builtin_tool_enabled('time'): builtin_functions.extend([get_current_timestamp, calculate_timestamp]) - if is_builtin_tool_enabled('user_input'): + if is_builtin_tool_enabled('user_input', True): builtin_functions.append(ask_user) metadata = extra_params.get('__metadata__') or {} diff --git a/src/lib/components/chat/Chat.svelte b/src/lib/components/chat/Chat.svelte index 1876ec92b2..2c08b4a1b4 100644 --- a/src/lib/components/chat/Chat.svelte +++ b/src/lib/components/chat/Chat.svelte @@ -77,6 +77,7 @@ getAllTags, getChatById, getTagsById, + resolveChatMessageToolCall, updateChatById, updateChatFolderIdById } from '$lib/apis/chats'; @@ -423,6 +424,131 @@ }; }; + const parseToolArguments = (args) => { + if (!args) { + return {}; + } + let value = args; + while (typeof value === 'string') { + try { + value = JSON.parse(value); + } catch { + break; + } + } + return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : {}; + }; + + const getPendingAskUserFromMessage = (message) => { + if (message?.role !== 'assistant' || !Array.isArray(message.output)) { + return null; + } + + const call = message.output.find( + (item) => + item?.type === 'function_call' && + item?.name === 'ask_user' && + item?.status === 'pending' && + (item?.call_id || item?.id) + ); + + if (call) { + return { message, call, args: parseToolArguments(call.arguments) }; + } + + return null; + }; + + const findPendingAskUser = (chatHistory) => { + if (!chatHistory?.messages) { + return null; + } + const messages = chatHistory.currentId + ? createMessagesList(chatHistory, chatHistory.currentId) + : Object.values(chatHistory.messages); + for (const message of [...messages].reverse()) { + const pending = getPendingAskUserFromMessage(message); + if (pending) return pending; + } + return null; + }; + + const messageHasPendingAskUser = (message) => { + return !!getPendingAskUserFromMessage(message); + }; + + const answerPendingAskUser = async (messageId, callId, answers, timedOut = false) => { + if (!$chatId || !messageId || !callId) { + return; + } + + await resolveChatMessageToolCall(localStorage.token, $chatId, messageId, callId, 'answer', { + answers, + timed_out: timedOut + }).catch(async (error) => { + toast.error(`${error}`); + await loadChat(); + }); + }; + + const rejectPendingAskUser = async (messageId, callId) => { + if (!$chatId || !messageId || !callId) { + return; + } + + await resolveChatMessageToolCall( + localStorage.token, + $chatId, + messageId, + callId, + 'reject' + ).catch(async (error) => { + toast.error(`${error}`); + await loadChat(); + }); + }; + + $: pendingAskUser = findPendingAskUser(history); + $: savedAskUserPrompt = pendingAskUser + ? { + show: true, + questions: Array.isArray(pendingAskUser.args?.questions) + ? pendingAskUser.args.questions + : [], + allowOther: pendingAskUser.args?.allow_other !== false, + timeoutMs: null, + onConfirm: (value) => { + void answerPendingAskUser( + pendingAskUser.message.id, + pendingAskUser.call.call_id || pendingAskUser.call.id, + value?.answers ?? {}, + false + ); + }, + onCancel: () => { + void rejectPendingAskUser( + pendingAskUser.message.id, + pendingAskUser.call.call_id || pendingAskUser.call.id + ); + } + } + : null; + + $: socketAskUserPrompt = { + show: showAskUserDialog, + questions: askUserQuestions, + allowOther: askUserAllowOther, + timeoutMs: askUserTimeoutMs, + onConfirm: (value) => { + showAskUserDialog = false; + eventCallback(value); + }, + onCancel: () => { + showAskUserDialog = false; + eventCallback({ status: 'cancelled', answers: {} }); + } + }; + const mergeChatVariableSchemas = (modelIds = [], availableModels = []) => { const byKey: Record = {}; const conflicts: any[] = []; @@ -2175,7 +2301,11 @@ } else { taskIds = null; // No active tasks and message incomplete → generation was interrupted - if (currentMessage?.role === 'assistant' && !currentMessage.done) { + if ( + currentMessage?.role === 'assistant' && + !currentMessage.done && + !messageHasPendingAskUser(currentMessage) + ) { currentMessage.done = true; } } @@ -2586,6 +2716,7 @@ } history.messages[message.id] = message; + history = history; if (done) { message.done = true; @@ -4192,20 +4323,7 @@ {onUpdate} messageQueue={$chatRequestQueues[$chatId] ?? []} {chatTasks} - askUser={{ - show: showAskUserDialog, - questions: askUserQuestions, - allowOther: askUserAllowOther, - timeoutMs: askUserTimeoutMs, - onConfirm: (value) => { - showAskUserDialog = false; - eventCallback(value); - }, - onCancel: () => { - showAskUserDialog = false; - eventCallback({ status: 'cancelled', answers: {} }); - } - }} + askUser={savedAskUserPrompt ?? socketAskUserPrompt} onQueueSendNow={sendQueuedMessageNow} onQueueEdit={editQueuedMessage} onQueueDelete={deleteQueuedMessage} @@ -4296,20 +4414,7 @@ {onUpdate} messageQueue={$chatRequestQueues[$chatId] ?? []} {chatTasks} - askUser={{ - show: showAskUserDialog, - questions: askUserQuestions, - allowOther: askUserAllowOther, - timeoutMs: askUserTimeoutMs, - onConfirm: (value) => { - showAskUserDialog = false; - eventCallback(value); - }, - onCancel: () => { - showAskUserDialog = false; - eventCallback({ status: 'cancelled', answers: {} }); - } - }} + askUser={savedAskUserPrompt ?? socketAskUserPrompt} onQueueSendNow={sendQueuedMessageNow} onQueueEdit={editQueuedMessage} onQueueDelete={deleteQueuedMessage} @@ -4354,6 +4459,7 @@ {onUpload} {onUpdate} messageQueue={$chatRequestQueues[$chatId] ?? []} + askUser={savedAskUserPrompt ?? socketAskUserPrompt} onQueueSendNow={sendQueuedMessageNow} onQueueEdit={editQueuedMessage} onQueueDelete={deleteQueuedMessage} diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 83298a6bf5..fd9ca1d54e 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -153,11 +153,20 @@ export let history; export let taskIds = null; + export let askUser: AskUserPrompt = { + show: false, + questions: [], + allowOther: true, + timeoutMs: null, + onConfirm: (_value: any) => {}, + onCancel: () => {} + }; $: isActive = - (taskIds && taskIds.length > 0) || - (history.currentId && history.messages[history.currentId]?.done != true) || - generating; + !askUser?.show && + ((taskIds && taskIds.length > 0) || + (history.currentId && history.messages[history.currentId]?.done != true) || + generating); $: canCompact = !!history?.currentId; export let prompt = ''; @@ -182,15 +191,6 @@ export let onQueueEdit: (id: string) => void = () => {}; export let onQueueDelete: (id: string) => void = () => {}; export let onUpdate: (data?: { file?: any }) => void = () => {}; - export let askUser: AskUserPrompt = { - show: false, - questions: [], - allowOther: true, - timeoutMs: null, - onConfirm: (_value: any) => {}, - onCancel: () => {} - }; - export let chatTasks = []; let inputContent = null; diff --git a/src/lib/components/chat/Messages/Markdown/ConsecutiveDetailsGroup.svelte b/src/lib/components/chat/Messages/Markdown/ConsecutiveDetailsGroup.svelte index 7a67160460..443ca05932 100644 --- a/src/lib/components/chat/Messages/Markdown/ConsecutiveDetailsGroup.svelte +++ b/src/lib/components/chat/Messages/Markdown/ConsecutiveDetailsGroup.svelte @@ -168,7 +168,7 @@ {/if} - {#if resolvable && pendingToolTokens.length === 1} + {#if resolvable && pendingToolTokens.length === 1 && pendingToolTokens[0]?.attributes?.name !== 'ask_user'} {@const pendingCallId = pendingToolTokens[0]?.attributes?.id ?? ''} - - + {#if token?.attributes?.name !== 'ask_user'} + + + + + {/if} {/each} diff --git a/src/lib/components/chat/Messages/structuredOutput.ts b/src/lib/components/chat/Messages/structuredOutput.ts index d7ad7a8474..d0299c4f35 100644 --- a/src/lib/components/chat/Messages/structuredOutput.ts +++ b/src/lib/components/chat/Messages/structuredOutput.ts @@ -145,7 +145,8 @@ function buildToolCallToken(item: OutputItem, toolOutputByCallId: Record void = () => {}; export let onQueueEdit: (id: string) => void = () => {}; export let onQueueDelete: (id: string) => void = () => {}; - + export let askUser = { + show: false, + questions: [], + allowOther: true, + timeoutMs: null, + onConfirm: (_value: any) => {}, + onCancel: () => {} + }; export let dragged = false; @@ -251,6 +258,7 @@ {onQueueSendNow} {onQueueEdit} {onQueueDelete} + {askUser} {onWebSearchToggle} on:chatVariables on:submit={(e) => { diff --git a/src/lib/components/common/ToolCallDisplay.svelte b/src/lib/components/common/ToolCallDisplay.svelte index 6aeb98d1e6..c3105bca44 100644 --- a/src/lib/components/common/ToolCallDisplay.svelte +++ b/src/lib/components/common/ToolCallDisplay.svelte @@ -93,12 +93,16 @@ $: result = resultContent || decode(attributes?.result ?? ''); $: files = parseJSONString(decode(attributes?.files ?? '')); $: embeds = parseJSONString(decode(attributes?.embeds ?? '')); + $: isAskUser = attributes?.name === 'ask_user'; + $: needsInput = isAskUser && attributes?.status === 'pending'; + $: needsApproval = !isAskUser && attributes?.status === 'pending' && resolvable; $: args = - open || (Array.isArray(embeds) && embeds.length > 0) ? decode(attributes?.arguments ?? '') : ''; + open || needsApproval || needsInput || (Array.isArray(embeds) && embeds.length > 0) + ? decode(attributes?.arguments ?? '') + : ''; $: isDone = attributes?.done === 'true'; - $: needsApproval = attributes?.status === 'pending' && resolvable; $: isRejected = attributes?.status === 'rejected'; - $: isExecuting = !needsApproval && attributes?.done && attributes?.done !== 'true'; + $: isExecuting = !needsApproval && !needsInput && attributes?.done && attributes?.done !== 'true'; $: parsedArgs = parseArguments(args); $: parsedResult = parseJSONString(result); @@ -178,6 +182,8 @@ - {#if needsApproval} + {#if needsApproval && !isAskUser}