refac

backend/open_webui/utils/middleware.py

@@ -5,6 +5,7 @@ import copy
 import inspect
 import json
 import logging
+import mimetypes
 import os
 import random
 import re
@@ -246,6 +247,62 @@ def output_id(prefix: str) -> str:
     return f'{prefix}_{uuid4().hex[:24]}'
 
 
+def build_terminal_file_tool_result(
+    tool_function_name: str,
+    tool_function_params: dict,
+    tool_result: Any,
+    tool: dict | None,
+    metadata: dict | None,
+) -> dict | None:
+    if isinstance(tool_result, (list, tuple)) and tool_result and isinstance(tool_result[0], dict):
+        tool_result = tool_result[0]
+
+    if (
+        tool_function_name != 'display_file'
+        or tool_function_params.get('inline') is not True
+        or not isinstance(tool_result, dict)
+        or tool_result.get('exists') is False
+    ):
+        return None
+
+    tool_id = (tool or {}).get('tool_id', '')
+    terminal_id = metadata.get('terminal_id') if metadata else None
+    if isinstance(tool_id, str) and tool_id.startswith('terminal:'):
+        terminal_id = tool_id.split(':', 1)[1]
+
+    server_url = ((tool or {}).get('server') or {}).get('url')
+    terminal_selector = terminal_id or server_url
+    path = tool_result.get('path') or tool_function_params.get('path')
+    if not terminal_selector or not path:
+        return None
+    mime_type, _ = mimetypes.guess_type(path)
+    mime_type = mime_type or 'application/octet-stream'
+
+    return {
+        **tool_result,
+        'type': 'file',
+        'source': 'open_terminal',
+        'displayed': True,
+        'terminal_selector': terminal_selector,
+        **({'terminal_id': terminal_id} if terminal_id else {}),
+        **({'terminal_url': server_url} if server_url and not terminal_id else {}),
+        'session_id': metadata.get('chat_id') if metadata else None,
+        'path': path,
+        'full_path': tool_result.get('full_path') or path,
+        'name': tool_result.get('name') or os.path.basename(path),
+        'mime_type': tool_result.get('mime_type') or tool_result.get('content_type') or mime_type,
+        'content_type': tool_result.get('content_type') or tool_result.get('mime_type') or mime_type,
+    }
+
+
+def tool_result_content(tool_result: Any) -> str:
+    if not tool_result:
+        return ''
+    if isinstance(tool_result, (dict, list)):
+        return JSONCodec.dumps(tool_result, ensure_ascii=False)
+    return str(tool_result)
+
+
 def merge_streamed_reasoning_details(target: list, details) -> None:
     items = details if isinstance(details, list) else [details]
     for item in items:
@@ -1171,6 +1228,8 @@ async def terminal_event_handler(
         return
 
     if tool_function_name == 'display_file':
+        if tool_function_params.get('inline') is True:
+            return
         path = tool_function_params.get('path', '')
         if not path:
             return
@@ -3120,6 +3179,10 @@ async def execute_tool_call_for_output(request, form_data, user, metadata, event
     except Exception as e:
         result = {'error': str(e)}
 
+    terminal_file_result = build_terminal_file_tool_result(name, params, result, tool, metadata)
+    if terminal_file_result:
+        result = terminal_file_result
+
     result, files, embeds = await process_tool_result(
         request,
         name,
@@ -3134,7 +3197,7 @@ async def execute_tool_call_for_output(request, form_data, user, metadata, event
 
     return {
         'tool_call_id': tool_call.get('id', ''),
-        'content': str(result) if result else '',
+        'content': tool_result_content(result),
         **({'files': files} if files else {}),
         **({'embeds': embeds} if embeds else {}),
     }
@@ -5562,6 +5625,16 @@ async def streaming_chat_response_handler(response, ctx):
                             )
                             continue
 
+                        terminal_file_result = build_terminal_file_tool_result(
+                            tool_function_name,
+                            tool_function_params,
+                            tool_result,
+                            tool,
+                            metadata,
+                        )
+                        if terminal_file_result:
+                            tool_result = terminal_file_result
+
                         tool_result, tool_result_files, tool_result_embeds = await process_tool_result(
                             request,
                             tool_function_name,
@@ -5607,7 +5680,7 @@ async def streaming_chat_response_handler(response, ctx):
                         results.append(
                             {
                                 'tool_call_id': tool_call_id,
-                                'content': str(tool_result) if tool_result else '',
+                                'content': tool_result_content(tool_result),
                                 **({'files': tool_result_files} if tool_result_files else {}),
                                 **({'embeds': tool_result_embeds} if tool_result_embeds else {}),
                             }

backend/open_webui/utils/tools.py

@@ -952,6 +952,26 @@ def clean_openai_tool_schema(spec: dict) -> dict:
     return cleaned_spec
 
 
+def add_terminal_display_file_inline_param(spec: dict) -> dict:
+    spec = copy.deepcopy(spec)
+    if spec.get('name') != 'display_file':
+        return spec
+
+    spec['description'] = (
+        f"{spec.get('description', '')} "
+        "Set inline=true when the file should be shown inline in the chat message instead of opening the file viewer. "
+        "After calling display_file with inline=true, do not emit Markdown image or link syntax for that file."
+    ).strip()
+    parameters = spec.setdefault('parameters', {'type': 'object', 'properties': {}, 'required': []})
+    parameters.setdefault('type', 'object')
+    properties = parameters.setdefault('properties', {})
+    properties['inline'] = {
+        'type': 'boolean',
+        'description': 'Show the file inline in the chat message instead of opening the file viewer.',
+    }
+    return spec
+
+
 @cache
 def get_builtin_function_introspection(func: Callable):
     try:
@@ -1408,7 +1428,7 @@ async def get_terminal_tools(
     tools_dict = {}
     for spec in specs:
         function_name = spec['name']
-        tool_spec = clean_openai_tool_schema(spec)
+        tool_spec = clean_openai_tool_schema(add_terminal_display_file_inline_param(spec))
 
         if function_name == 'run_command' and terminal_cwd:
             tool_spec['description'] = (
@@ -1555,7 +1575,10 @@ async def get_tool_servers_data(servers: list[dict[str, Any]]) -> list[dict[str,
         response = {
             'openapi': response,
             'info': response.get('info', {}),
-            'specs': convert_openapi_to_tool_payload(response),
+            'specs': [
+                add_terminal_display_file_inline_param(spec)
+                for spec in convert_openapi_to_tool_payload(response)
+            ],
         }
 
         openapi_data = response.get('openapi', {})

src/lib/components/chat/Messages/StructuredOutputRenderer.svelte

@@ -1,6 +1,7 @@
 <script lang="ts">
 	import Collapsible from '$lib/components/common/Collapsible.svelte';
 	import ToolCallDisplay from '$lib/components/common/ToolCallDisplay.svelte';
+	import TerminalOutputFile from './TerminalOutputFile.svelte';
 	import { resolveChatMessageToolCall } from '$lib/apis/chats';
 	import { settings } from '$lib/stores';
 	import { toast } from 'svelte-sonner';
@@ -161,6 +162,8 @@
 				{/each}
 			</div>
 		</ConsecutiveDetailsGroup>
+	{:else if displayItem.type === 'file'}
+		<TerminalOutputFile item={displayItem.item} {chatId} />
 	{:else}
 		{@const detailToken = displayItem.token}
 		{#if detailToken.attributes?.type === 'tool_calls'}

src/lib/components/chat/Messages/TerminalOutputFile.svelte

@@ -0,0 +1,265 @@
+<script lang="ts">
+	import { getContext, onDestroy } from 'svelte';
+	import type { Writable } from 'svelte/store';
+	import type { i18n as i18nType } from 'i18next';
+	import { toast } from 'svelte-sonner';
+	import { settings, selectedTerminalId, showControls, showFileNavPath, terminalServers } from '$lib/stores';
+	import { downloadFileBlob, readFile } from '$lib/apis/terminal';
+	import FilePreview from '$lib/components/chat/FileNav/FilePreview.svelte';
+	import Icon from '$lib/components/chat/FileNav/Icon.svelte';
+	import { fileIconName } from '$lib/components/chat/FileNav/fileIcon';
+
+	export let item: any;
+	export let chatId = '';
+
+	const i18n = getContext<Writable<i18nType>>('i18n');
+
+	let expanded = true;
+	let loading = false;
+	let loadedKey = '';
+	let error = '';
+	let objectUrls: string[] = [];
+
+	let fileImageUrl: string | null = null;
+	let fileVideoUrl: string | null = null;
+	let fileAudioUrl: string | null = null;
+	let filePdfData: ArrayBuffer | null = null;
+	let fileSqliteData: ArrayBuffer | null = null;
+	let fileDocxData: ArrayBuffer | null = null;
+	let fileContent: string | null = null;
+	let fileOfficeHtml: string | null = null;
+	let fileOfficeSlides: string[] | null = null;
+	let currentSlide = 0;
+	let excelSheetNames: string[] = [];
+	let selectedExcelSheet = '';
+	let excelWorkbook: any = null;
+
+	const IMAGE_EXTS = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'ico', 'avif']);
+	const VIDEO_EXTS = new Set(['mp4', 'webm', 'mov', 'ogv', 'avi', 'mkv']);
+	const AUDIO_EXTS = new Set(['mp3', 'wav', 'ogg', 'oga', 'flac', 'm4a', 'aac', 'wma', 'opus']);
+	const SQLITE_EXTS = new Set(['db', 'sqlite', 'sqlite3', 'db3']);
+	const OFFICE_EXTS = new Set(['docx', 'xlsx', 'xls', 'pptx']);
+
+	$: path = String(item?.full_path || item?.path || '');
+	$: name = String(item?.name || path.split('/').filter(Boolean).at(-1) || 'file');
+	$: selector = item?.terminal_selector;
+	$: terminal = resolveTerminal();
+	$: unavailable = !terminal;
+	$: previewKey = terminal ? `${terminal.url}|${terminal.key}|${path}|${chatId}` : '';
+	$: previewClass = isImage(path)
+		? 'h-[22rem]'
+		: isPdf(path) || isOffice(path) || getExt(path) === 'html' || getExt(path) === 'htm' || getExt(path) === 'svg'
+			? 'h-96'
+			: 'h-72';
+	$: if (expanded && terminal && path && previewKey !== loadedKey && !loading) {
+		void loadPreview(previewKey);
+	}
+
+	const getExt = (value: string) => value.split('.').pop()?.toLowerCase() ?? '';
+	const isImage = (value: string) => IMAGE_EXTS.has(getExt(value));
+	const isVideo = (value: string) => VIDEO_EXTS.has(getExt(value));
+	const isAudio = (value: string) => AUDIO_EXTS.has(getExt(value));
+	const isSqlite = (value: string) => SQLITE_EXTS.has(getExt(value));
+	const isPdf = (value: string) => getExt(value) === 'pdf';
+	const isOffice = (value: string) => OFFICE_EXTS.has(getExt(value));
+
+	const t = (key: string, vars?: Record<string, unknown>) => $i18n?.t?.(key, vars) ?? key;
+
+	function resolveTerminal(): { url: string; key: string } | null {
+		if (!selector || $selectedTerminalId !== selector) return null;
+
+		const systemTerminal = ($terminalServers ?? []).find((server: any) => server.id === selector);
+		if (systemTerminal?.url) {
+			return { url: systemTerminal.url, key: localStorage.token };
+		}
+
+		const directTerminal = (($settings as any)?.terminalServers ?? []).find(
+			(server: any) => server.url === selector && server.enabled
+		);
+		if (directTerminal?.url) {
+			return { url: directTerminal.url, key: directTerminal.key ?? '' };
+		}
+
+		return null;
+	}
+
+	function clearPreview() {
+		for (const url of objectUrls) URL.revokeObjectURL(url);
+		objectUrls = [];
+		fileImageUrl = null;
+		fileVideoUrl = null;
+		fileAudioUrl = null;
+		filePdfData = null;
+		fileSqliteData = null;
+		fileDocxData = null;
+		fileContent = null;
+		fileOfficeHtml = null;
+		fileOfficeSlides = null;
+		currentSlide = 0;
+		excelSheetNames = [];
+		selectedExcelSheet = '';
+		excelWorkbook = null;
+	}
+
+	async function blobForPreview() {
+		if (!terminal) return null;
+		return downloadFileBlob(terminal.url, terminal.key, path, chatId || undefined);
+	}
+
+	async function loadExcelSheet(sheet: string) {
+		if (!excelWorkbook) return;
+		selectedExcelSheet = sheet;
+		const { excelToTable } = await import('$lib/utils/excelToTable');
+		const result = await excelToTable(excelWorkbook.Sheets[selectedExcelSheet]);
+		const DOMPurify = (await import('dompurify')).default;
+		fileOfficeHtml = DOMPurify.sanitize(result.html);
+	}
+
+	async function loadPreview(key: string) {
+		loadedKey = key;
+		loading = true;
+		error = '';
+		clearPreview();
+
+		try {
+			if (isImage(path) || isVideo(path) || isAudio(path)) {
+				const result = await blobForPreview();
+				if (!result) throw new Error(t('Preview failed'));
+				const url = URL.createObjectURL(result.blob);
+				objectUrls = [...objectUrls, url];
+				if (isImage(path)) fileImageUrl = url;
+				else if (isVideo(path)) fileVideoUrl = url;
+				else fileAudioUrl = url;
+			} else if (isPdf(path) || isSqlite(path) || isOffice(path)) {
+				const result = await blobForPreview();
+				if (!result) throw new Error(t('Preview failed'));
+				const arrayBuffer = await result.blob.arrayBuffer();
+				const ext = getExt(path);
+
+				if (isPdf(path)) {
+					filePdfData = arrayBuffer;
+				} else if (isSqlite(path)) {
+					fileSqliteData = arrayBuffer;
+				} else if (ext === 'docx') {
+					fileDocxData = arrayBuffer;
+				} else if (ext === 'xlsx' || ext === 'xls') {
+					const XLSX = await import('xlsx');
+					excelWorkbook = XLSX.read(new Uint8Array(arrayBuffer), { type: 'array' });
+					excelSheetNames = excelWorkbook.SheetNames;
+					if (excelSheetNames.length > 0) await loadExcelSheet(excelSheetNames[0]);
+				} else if (ext === 'pptx') {
+					const { pptxToImages } = await import('$lib/utils/pptxToHtml');
+					const result = await pptxToImages(arrayBuffer);
+					fileOfficeSlides = result.images;
+				}
+			} else if (terminal) {
+				fileContent = await readFile(terminal.url, terminal.key, path, chatId || undefined);
+			}
+		} catch (e) {
+			error = e instanceof Error ? e.message : t('Preview failed');
+		} finally {
+			loading = false;
+		}
+	}
+
+	function openInFiles() {
+		if (unavailable || !path) return;
+		showControls.set(true);
+		showFileNavPath.set(path);
+	}
+
+	async function downloadFile() {
+		if (!terminal || !path) return;
+		const result = await downloadFileBlob(terminal.url, terminal.key, path, chatId || undefined);
+		if (!result) {
+			toast.error(t('Download failed'));
+			return;
+		}
+
+		const url = URL.createObjectURL(result.blob);
+		const a = document.createElement('a');
+		a.href = url;
+		a.download = result.filename;
+		a.click();
+		URL.revokeObjectURL(url);
+	}
+
+	onDestroy(clearPreview);
+</script>
+
+<div
+	class="my-2 w-full overflow-hidden rounded-xl border border-gray-200 bg-white dark:border-white/8 dark:bg-gray-950/20"
+>
+	<div class="flex h-8 items-center {expanded ? 'border-b border-gray-100 dark:border-white/8' : ''}">
+		<button
+			type="button"
+			class="flex h-full min-w-0 flex-1 items-center gap-2 px-2.5 text-left"
+			on:click={() => (expanded = !expanded)}
+			aria-expanded={expanded}
+		>
+			<div class="flex size-5 shrink-0 items-center justify-center text-gray-500 dark:text-gray-400">
+				<Icon name={fileIconName(name || path || '', 'file')} size={14} />
+			</div>
+			<div class="min-w-0 flex-1 truncate text-xs font-medium text-gray-800 dark:text-gray-100">
+				{name}
+			</div>
+		</button>
+
+		<button
+			type="button"
+			class="mr-1 flex size-6 shrink-0 items-center justify-center rounded text-gray-400 transition-colors hover:text-gray-700 disabled:opacity-40 disabled:hover:text-gray-400 dark:text-gray-500 dark:hover:text-gray-200 dark:disabled:hover:text-gray-500"
+			disabled={unavailable}
+			on:click|stopPropagation={downloadFile}
+			aria-label={t('Download')}
+		>
+			<Icon name="download" size={13} />
+		</button>
+		<button
+			type="button"
+			class="mr-1 flex size-6 shrink-0 items-center justify-center rounded text-gray-400 transition-colors hover:text-gray-700 disabled:opacity-40 disabled:hover:text-gray-400 dark:text-gray-500 dark:hover:text-gray-200 dark:disabled:hover:text-gray-500"
+			disabled={unavailable}
+			on:click|stopPropagation={openInFiles}
+			aria-label={t('Open')}
+		>
+			<Icon name="external-link" size={13} />
+		</button>
+	</div>
+
+	{#if expanded}
+		{#if unavailable}
+			<div class="px-4 py-3 text-xs text-gray-500 dark:text-gray-400">
+				{t('Terminal unavailable')}
+			</div>
+		{:else if error}
+			<div class="px-4 py-3 text-xs text-red-500 dark:text-red-400">{error}</div>
+		{:else}
+			<div class="{previewClass} max-h-[75vh] min-h-24 resize-y overflow-hidden bg-gray-50 dark:bg-gray-950">
+				<FilePreview
+					selectedFile={path}
+					fileLoading={loading}
+					{fileImageUrl}
+					{fileVideoUrl}
+					{fileAudioUrl}
+					{filePdfData}
+					{fileSqliteData}
+					{fileDocxData}
+					{fileContent}
+					baseUrl={terminal?.url ?? ''}
+					apiKey={terminal?.key ?? ''}
+					{fileOfficeHtml}
+					{fileOfficeSlides}
+					{currentSlide}
+					{excelSheetNames}
+					{selectedExcelSheet}
+					onSheetChange={loadExcelSheet}
+					readOnly={true}
+				/>
+				{#if !loading && fileImageUrl === null && fileVideoUrl === null && fileAudioUrl === null && filePdfData === null && fileSqliteData === null && fileDocxData === null && fileContent === null && fileOfficeHtml === null && fileOfficeSlides === null}
+					<div class="flex h-full items-center justify-center px-3 text-xs text-gray-500 dark:text-gray-400">
+						{t('No preview available')}
+					</div>
+				{/if}
+			</div>
+		{/if}
+	{/if}
+</div>

src/lib/components/chat/Messages/structuredOutput.ts

@@ -57,6 +57,11 @@ export type OutputDisplayItem =
 			type: 'detail_group';
 			id: string;
 			tokens: OutputDetailToken[];
+	  }
+	| {
+			type: 'file';
+			id: string;
+			item: Record<string, unknown>;
 	  };
 
 type ResponseStreamEvent = {
@@ -142,6 +147,48 @@ function getToolResultText(item?: OutputItem): string {
 		.join('');
 }
 
+function parseJSONStringValue(value: unknown): unknown {
+	if (typeof value !== 'string') {
+		return value;
+	}
+
+	let parsed: unknown = value.trim();
+	while (typeof parsed === 'string') {
+		try {
+			parsed = JSON.parse(parsed);
+		} catch {
+			break;
+		}
+	}
+	return parsed;
+}
+
+function getInlineFileFromToolOutput(callItem?: OutputItem, resultItem?: OutputItem) {
+	if (!callItem || !resultItem || callItem.name !== 'display_file') {
+		return null;
+	}
+
+	const args = parseJSONStringValue(callItem.arguments) as Record<string, unknown>;
+	if (!args || typeof args !== 'object' || args.inline !== true) {
+		return null;
+	}
+
+	const result = parseJSONStringValue(getToolResultText(resultItem)) as Record<string, unknown>;
+	if (
+		!result ||
+		typeof result !== 'object' ||
+		result.type !== 'file' ||
+		result.source !== 'open_terminal' ||
+		result.exists === false ||
+		!result.path ||
+		!result.terminal_selector
+	) {
+		return null;
+	}
+
+	return result;
+}
+
 function buildToolCallToken(item: OutputItem, toolOutputByCallId: Record<string, OutputItem>) {
 	const callId = item.call_id ?? item.id ?? '';
 	const resultItem = toolOutputByCallId[callId];
@@ -297,10 +344,13 @@ export function buildOutputDisplayItems(output: OutputItem[] = []): OutputDispla
 	const displayItems: OutputDisplayItem[] = [];
 	const currentDetailTokens: OutputDetailToken[] = [];
 	const toolOutputByCallId: Record<string, OutputItem> = {};
+	const toolCallByCallId: Record<string, OutputItem> = {};
 
 	for (const item of output) {
 		if (item?.type === 'function_call_output' && item.call_id) {
 			toolOutputByCallId[item.call_id] = item;
+		} else if (item?.type === 'function_call' && (item.call_id || item.id)) {
+			toolCallByCallId[item.call_id ?? item.id ?? ''] = item;
 		}
 	}
 
@@ -323,6 +373,18 @@ export function buildOutputDisplayItems(output: OutputItem[] = []): OutputDispla
 
 	output.forEach((item, index) => {
 		if (item?.type === 'function_call_output') {
+			const inlineFile = getInlineFileFromToolOutput(
+				toolCallByCallId[item.call_id ?? ''],
+				item
+			);
+			if (inlineFile) {
+				flushDetails();
+				displayItems.push({
+					type: 'file',
+					id: item.id ?? `file-${index}`,
+					item: inlineFile
+				});
+			}
 			return;
 		}
 

src/routes/+layout.svelte

@@ -475,15 +475,16 @@
 			);
 
 			console.log('executeToolServer', res);
+			const result = Array.isArray(res) ? res[0] : res;
 
-			if (data?.name === 'display_file' && data?.params?.path) {
-				if (res?.exists !== false) {
+			if (data?.name === 'display_file' && data?.params?.path && data?.params?.inline !== true) {
+				if (result?.exists !== false) {
 					displayFileHandler(data.params.path, { showControls, showFileNavPath });
 				}
 			}
 
 			if (['write_file'].includes(data?.name) && data?.params?.path) {
-				showFileNavDir.set(res?.path ?? data.params.path);
+				showFileNavDir.set(result?.path ?? data.params.path);
 			}
 
 			if (cb) {