diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index ff7692c38c..12584752b7 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -5,6 +5,7 @@ import copy import inspect import json import logging +import mimetypes import os import random import re @@ -246,6 +247,62 @@ def output_id(prefix: str) -> str: return f'{prefix}_{uuid4().hex[:24]}' +def build_terminal_file_tool_result( + tool_function_name: str, + tool_function_params: dict, + tool_result: Any, + tool: dict | None, + metadata: dict | None, +) -> dict | None: + if isinstance(tool_result, (list, tuple)) and tool_result and isinstance(tool_result[0], dict): + tool_result = tool_result[0] + + if ( + tool_function_name != 'display_file' + or tool_function_params.get('inline') is not True + or not isinstance(tool_result, dict) + or tool_result.get('exists') is False + ): + return None + + tool_id = (tool or {}).get('tool_id', '') + terminal_id = metadata.get('terminal_id') if metadata else None + if isinstance(tool_id, str) and tool_id.startswith('terminal:'): + terminal_id = tool_id.split(':', 1)[1] + + server_url = ((tool or {}).get('server') or {}).get('url') + terminal_selector = terminal_id or server_url + path = tool_result.get('path') or tool_function_params.get('path') + if not terminal_selector or not path: + return None + mime_type, _ = mimetypes.guess_type(path) + mime_type = mime_type or 'application/octet-stream' + + return { + **tool_result, + 'type': 'file', + 'source': 'open_terminal', + 'displayed': True, + 'terminal_selector': terminal_selector, + **({'terminal_id': terminal_id} if terminal_id else {}), + **({'terminal_url': server_url} if server_url and not terminal_id else {}), + 'session_id': metadata.get('chat_id') if metadata else None, + 'path': path, + 'full_path': tool_result.get('full_path') or path, + 'name': tool_result.get('name') or os.path.basename(path), + 'mime_type': tool_result.get('mime_type') or tool_result.get('content_type') or mime_type, + 'content_type': tool_result.get('content_type') or tool_result.get('mime_type') or mime_type, + } + + +def tool_result_content(tool_result: Any) -> str: + if not tool_result: + return '' + if isinstance(tool_result, (dict, list)): + return JSONCodec.dumps(tool_result, ensure_ascii=False) + return str(tool_result) + + def merge_streamed_reasoning_details(target: list, details) -> None: items = details if isinstance(details, list) else [details] for item in items: @@ -1171,6 +1228,8 @@ async def terminal_event_handler( return if tool_function_name == 'display_file': + if tool_function_params.get('inline') is True: + return path = tool_function_params.get('path', '') if not path: return @@ -3120,6 +3179,10 @@ async def execute_tool_call_for_output(request, form_data, user, metadata, event except Exception as e: result = {'error': str(e)} + terminal_file_result = build_terminal_file_tool_result(name, params, result, tool, metadata) + if terminal_file_result: + result = terminal_file_result + result, files, embeds = await process_tool_result( request, name, @@ -3134,7 +3197,7 @@ async def execute_tool_call_for_output(request, form_data, user, metadata, event return { 'tool_call_id': tool_call.get('id', ''), - 'content': str(result) if result else '', + 'content': tool_result_content(result), **({'files': files} if files else {}), **({'embeds': embeds} if embeds else {}), } @@ -5562,6 +5625,16 @@ async def streaming_chat_response_handler(response, ctx): ) continue + terminal_file_result = build_terminal_file_tool_result( + tool_function_name, + tool_function_params, + tool_result, + tool, + metadata, + ) + if terminal_file_result: + tool_result = terminal_file_result + tool_result, tool_result_files, tool_result_embeds = await process_tool_result( request, tool_function_name, @@ -5607,7 +5680,7 @@ async def streaming_chat_response_handler(response, ctx): results.append( { 'tool_call_id': tool_call_id, - 'content': str(tool_result) if tool_result else '', + 'content': tool_result_content(tool_result), **({'files': tool_result_files} if tool_result_files else {}), **({'embeds': tool_result_embeds} if tool_result_embeds else {}), } diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 720f6226dc..f36633cd17 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -952,6 +952,26 @@ def clean_openai_tool_schema(spec: dict) -> dict: return cleaned_spec +def add_terminal_display_file_inline_param(spec: dict) -> dict: + spec = copy.deepcopy(spec) + if spec.get('name') != 'display_file': + return spec + + spec['description'] = ( + f"{spec.get('description', '')} " + "Set inline=true when the file should be shown inline in the chat message instead of opening the file viewer. " + "After calling display_file with inline=true, do not emit Markdown image or link syntax for that file." + ).strip() + parameters = spec.setdefault('parameters', {'type': 'object', 'properties': {}, 'required': []}) + parameters.setdefault('type', 'object') + properties = parameters.setdefault('properties', {}) + properties['inline'] = { + 'type': 'boolean', + 'description': 'Show the file inline in the chat message instead of opening the file viewer.', + } + return spec + + @cache def get_builtin_function_introspection(func: Callable): try: @@ -1408,7 +1428,7 @@ async def get_terminal_tools( tools_dict = {} for spec in specs: function_name = spec['name'] - tool_spec = clean_openai_tool_schema(spec) + tool_spec = clean_openai_tool_schema(add_terminal_display_file_inline_param(spec)) if function_name == 'run_command' and terminal_cwd: tool_spec['description'] = ( @@ -1555,7 +1575,10 @@ async def get_tool_servers_data(servers: list[dict[str, Any]]) -> list[dict[str, response = { 'openapi': response, 'info': response.get('info', {}), - 'specs': convert_openapi_to_tool_payload(response), + 'specs': [ + add_terminal_display_file_inline_param(spec) + for spec in convert_openapi_to_tool_payload(response) + ], } openapi_data = response.get('openapi', {}) diff --git a/src/lib/components/chat/Messages/StructuredOutputRenderer.svelte b/src/lib/components/chat/Messages/StructuredOutputRenderer.svelte index 3468181a64..faeff5f179 100644 --- a/src/lib/components/chat/Messages/StructuredOutputRenderer.svelte +++ b/src/lib/components/chat/Messages/StructuredOutputRenderer.svelte @@ -1,6 +1,7 @@ + +
+
+ + + + +
+ + {#if expanded} + {#if unavailable} +
+ {t('Terminal unavailable')} +
+ {:else if error} +
{error}
+ {:else} +
+ + {#if !loading && fileImageUrl === null && fileVideoUrl === null && fileAudioUrl === null && filePdfData === null && fileSqliteData === null && fileDocxData === null && fileContent === null && fileOfficeHtml === null && fileOfficeSlides === null} +
+ {t('No preview available')} +
+ {/if} +
+ {/if} + {/if} +
diff --git a/src/lib/components/chat/Messages/structuredOutput.ts b/src/lib/components/chat/Messages/structuredOutput.ts index bb5eee4e66..177c02ffbf 100644 --- a/src/lib/components/chat/Messages/structuredOutput.ts +++ b/src/lib/components/chat/Messages/structuredOutput.ts @@ -57,6 +57,11 @@ export type OutputDisplayItem = type: 'detail_group'; id: string; tokens: OutputDetailToken[]; + } + | { + type: 'file'; + id: string; + item: Record; }; type ResponseStreamEvent = { @@ -142,6 +147,48 @@ function getToolResultText(item?: OutputItem): string { .join(''); } +function parseJSONStringValue(value: unknown): unknown { + if (typeof value !== 'string') { + return value; + } + + let parsed: unknown = value.trim(); + while (typeof parsed === 'string') { + try { + parsed = JSON.parse(parsed); + } catch { + break; + } + } + return parsed; +} + +function getInlineFileFromToolOutput(callItem?: OutputItem, resultItem?: OutputItem) { + if (!callItem || !resultItem || callItem.name !== 'display_file') { + return null; + } + + const args = parseJSONStringValue(callItem.arguments) as Record; + if (!args || typeof args !== 'object' || args.inline !== true) { + return null; + } + + const result = parseJSONStringValue(getToolResultText(resultItem)) as Record; + if ( + !result || + typeof result !== 'object' || + result.type !== 'file' || + result.source !== 'open_terminal' || + result.exists === false || + !result.path || + !result.terminal_selector + ) { + return null; + } + + return result; +} + function buildToolCallToken(item: OutputItem, toolOutputByCallId: Record) { const callId = item.call_id ?? item.id ?? ''; const resultItem = toolOutputByCallId[callId]; @@ -297,10 +344,13 @@ export function buildOutputDisplayItems(output: OutputItem[] = []): OutputDispla const displayItems: OutputDisplayItem[] = []; const currentDetailTokens: OutputDetailToken[] = []; const toolOutputByCallId: Record = {}; + const toolCallByCallId: Record = {}; for (const item of output) { if (item?.type === 'function_call_output' && item.call_id) { toolOutputByCallId[item.call_id] = item; + } else if (item?.type === 'function_call' && (item.call_id || item.id)) { + toolCallByCallId[item.call_id ?? item.id ?? ''] = item; } } @@ -323,6 +373,18 @@ export function buildOutputDisplayItems(output: OutputItem[] = []): OutputDispla output.forEach((item, index) => { if (item?.type === 'function_call_output') { + const inlineFile = getInlineFileFromToolOutput( + toolCallByCallId[item.call_id ?? ''], + item + ); + if (inlineFile) { + flushDetails(); + displayItems.push({ + type: 'file', + id: item.id ?? `file-${index}`, + item: inlineFile + }); + } return; } diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 3f39c62430..cb0591ed50 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -475,15 +475,16 @@ ); console.log('executeToolServer', res); + const result = Array.isArray(res) ? res[0] : res; - if (data?.name === 'display_file' && data?.params?.path) { - if (res?.exists !== false) { + if (data?.name === 'display_file' && data?.params?.path && data?.params?.inline !== true) { + if (result?.exists !== false) { displayFileHandler(data.params.path, { showControls, showFileNavPath }); } } if (['write_file'].includes(data?.name) && data?.params?.path) { - showFileNavDir.set(res?.path ?? data.params.path); + showFileNavDir.set(result?.path ?? data.params.path); } if (cb) {