From d99ac7d3f83b25161ca775229150c8f7c74cceee Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Sun, 28 Jun 2026 23:02:38 -0500 Subject: [PATCH] refac --- backend/open_webui/env.py | 3 +++ backend/open_webui/retrieval/loaders/main.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index cdae0886a5..f33e1f3278 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -753,6 +753,9 @@ BYPASS_RETRIEVAL_ACCESS_CONTROL = os.getenv('BYPASS_RETRIEVAL_ACCESS_CONTROL', ' # for non-admin users. When False (default), unknown collection names are # denied — closing the legacy unscoped namespace. ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS = os.getenv('ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS', 'False').lower() == 'true' +MINERU_MAX_MARKDOWN_BYTES = ( + int(os.getenv('MINERU_MAX_MARKDOWN_BYTES')) if os.getenv('MINERU_MAX_MARKDOWN_BYTES') else None +) # When enabled, skips pydub-based preprocessing (format conversion, compression, # and chunked splitting) before sending files to processing engines. Useful when diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index 7841ee272b..6dbab9d050 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -17,7 +17,12 @@ from langchain_community.document_loaders import ( YoutubeLoader, ) from langchain_core.documents import Document -from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL, GLOBAL_LOG_LEVEL, REQUESTS_VERIFY +from open_webui.env import ( + AIOHTTP_CLIENT_SESSION_SSL, + GLOBAL_LOG_LEVEL, + MINERU_MAX_MARKDOWN_BYTES, + REQUESTS_VERIFY, +) from open_webui.retrieval.loaders.datalab_marker import DatalabMarkerLoader from open_webui.retrieval.loaders.external_document import ExternalDocumentLoader from open_webui.retrieval.loaders.mineru import MinerULoader @@ -518,7 +523,6 @@ class Loader: mineru_timeout = int(mineru_timeout) except ValueError: mineru_timeout = 300 - loader = MinerULoader( file_path=file_path, api_mode=self.kwargs.get('MINERU_API_MODE', 'local'), @@ -526,6 +530,7 @@ class Loader: api_key=self.kwargs.get('MINERU_API_KEY', ''), params=self.kwargs.get('MINERU_PARAMS', {}), timeout=mineru_timeout, + max_markdown_bytes=MINERU_MAX_MARKDOWN_BYTES, ) elif ( self.engine == 'mistral_ocr'