github-starred/open-webui
2
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-07-19 04:10:26 -05:00
Code Issues 2.4k Packages Projects Releases 129 Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-06-01 11:08:58 -07:00
parent cc15a01778
commit 1bbb2b933d
1 changed files with 144 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/retrieval/loaders/main.py
+144 -36
View File
@@ -233,34 +233,7 @@ class Loader:
def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]:
loader = self._get_loader(filename, file_content_type, file_path)
try:
docs = loader.load()
except (UnicodeDecodeError, RuntimeError, TimeoutError) as e:
# Only retry with latin-1 for text-encoding loaders (TextLoader,
# CSVLoader). Binary format loaders (PDF, DOCX, PPTX, …) can
# raise RuntimeError for unrelated reasons; retrying those as
# plain text would produce garbage.
if not isinstance(loader, (TextLoader, CSVLoader)):
raise
# TextLoader/CSVLoader with autodetect_encoding=True can fail when:
# - chardet times out (5 s hardcoded in langchain)
# - chardet returns a wrong/low-confidence encoding
# - the file contains non-UTF-8 bytes (Latin-1, Windows-1252, …)
#
# Latin-1 is a safe last resort: every byte 0x00–0xFF is valid,
# and ftfy.fix_text() (applied below) repairs most mojibake that
# results from treating Windows-1252 content as Latin-1.
log.warning(
"Primary loader failed for %s (%s), retrying with latin-1 encoding: %s",
filename,
type(e).__name__,
e,
)
fallback_loader = TextLoader(file_path, encoding="latin-1")
docs = fallback_loader.load()
docs = loader.load()
return [Document(page_content=ftfy.fix_text(doc.page_content), metadata=doc.metadata) for doc in docs]
async def aload(self, filename: str, file_content_type: str, file_path: str) -> list[Document]:
@@ -283,6 +256,140 @@ class Loader:
and not file_content_type.find('html') >= 0
)
def _detect_text_encoding(self, file_path: str) -> str:
"""Detect the encoding of a text file with CJK-aware fallbacks.
Langchain's ``TextLoader`` uses chardet internally when
``autodetect_encoding=True``, but chardet frequently misidentifies
CJK encodings (e.g. GB18030 detected as GB2312 or even Cyrillic).
This method replaces that by:
1. Trying UTF-8 first (fast path for the vast majority of files).
2. Using chardet as a *hint* to prioritise the right CJK codec
family, but mapping subset names to their superset
(e.g. GB2312 → gb18030).
3. Validating that decoded text actually contains CJK characters,
guarding against codecs that "succeed" but produce garbage.
4. Falling back to latin-1 (always valid, ftfy fixes mojibake later).
"""
try:
with open(file_path, "rb") as f:
raw = f.read()
except OSError:
return "utf-8"
if not raw:
return "utf-8"
# Fast path: most files are UTF-8
try:
raw.decode("utf-8")
return "utf-8"
except UnicodeDecodeError:
pass
# Use chardet as a hint, not as ground truth
import chardet
detected = chardet.detect(raw)
detected_enc = (detected.get("encoding") or "").lower().replace("-", "").replace("_", "")
# Map chardet's detected encoding to the correct superset codec.
# chardet often reports GB2312 for content that is actually GB18030;
# GB18030 is a strict superset of both GB2312 and GBK.
_ENC_FAMILY = {
"gb2312": "gb18030",
"gb18030": "gb18030",
"gbk": "gb18030",
"big5": "big5",
"euckr": "euc-kr",
"eucjp": "euc-jp",
"iso2022jp": "euc-jp",
"shiftjis": "shift_jis",
}
# Build priority list: chardet-hinted codec first, then remaining CJK
base_order = ["gb18030", "big5", "euc-kr", "euc-jp"]
hinted = _ENC_FAMILY.get(detected_enc)
if hinted and hinted in base_order:
ordered = [hinted] + [e for e in base_order if e != hinted]
else:
ordered = base_order
for enc in ordered:
try:
text = raw.decode(enc)
if text.strip() and self._has_cjk_characters(text):
log.info(
"Detected encoding %s for %s (chardet guessed %s)",
enc,
file_path,
detected.get("encoding"),
)
return enc
except (UnicodeDecodeError, LookupError):
continue
# If chardet gave a non-CJK answer that isn't in our family map,
# try it directly — it might be a valid Western encoding.
chardet_encoding = detected.get("encoding")
if chardet_encoding:
try:
raw.decode(chardet_encoding)
log.info(
"Using chardet-detected encoding %s for %s",
chardet_encoding,
file_path,
)
return chardet_encoding
except (UnicodeDecodeError, LookupError):
pass
# latin-1 is the ultimate fallback: every byte 0x00–0xFF is valid.
# ftfy.fix_text() (applied downstream) repairs most mojibake that
# results from treating Windows-1252 content as Latin-1.
log.info("Falling back to latin-1 encoding for %s", file_path)
return "latin-1"
@staticmethod
def _has_cjk_characters(text: str, threshold: float = 0.05) -> bool:
"""Check if decoded text contains a meaningful proportion of CJK characters.
This guards against codecs that technically "succeed" but decode the
bytes into wrong Unicode codepoints (e.g. PUA chars, random symbols).
A genuine CJK document should have at least ``threshold`` fraction of
its non-whitespace characters in CJK Unicode blocks.
"""
if not text:
return False
cjk_count = 0
total = 0
for ch in text:
if ch.isspace():
continue
total += 1
cp = ord(ch)
if (
0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= cp <= 0x4DBF # CJK Extension A
or 0x20000 <= cp <= 0x2A6DF # CJK Extension B
or 0x2A700 <= cp <= 0x2B73F # CJK Extension C
or 0x2B740 <= cp <= 0x2B81F # CJK Extension D
or 0xF900 <= cp <= 0xFAFF # CJK Compatibility Ideographs
or 0x3000 <= cp <= 0x303F # CJK Symbols and Punctuation
or 0x3040 <= cp <= 0x309F # Hiragana
or 0x30A0 <= cp <= 0x30FF # Katakana
or 0xAC00 <= cp <= 0xD7AF # Hangul Syllables
or 0xFF00 <= cp <= 0xFFEF # Halfwidth and Fullwidth Forms
):
cjk_count += 1
if total == 0:
return False
return (cjk_count / total) >= threshold
def _get_loader(self, filename: str, file_content_type: str, file_path: str):
file_ext = filename.split('.')[-1].lower()
@@ -300,7 +407,7 @@ class Loader:
)
elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'):
if self._is_text_file(file_ext, file_content_type):
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
else:
loader = TikaLoader(
url=self.kwargs.get('TIKA_SERVER_URL'),
@@ -352,7 +459,7 @@ class Loader:
)
elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'):
if self._is_text_file(file_ext, file_content_type):
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
else:
# Build params for DoclingLoader
params = self.kwargs.get('DOCLING_PARAMS', {})
@@ -437,7 +544,7 @@ class Loader:
mode=self.kwargs.get('PDF_LOADER_MODE', 'page'),
)
elif file_ext == 'csv':
loader = CSVLoader(file_path, autodetect_encoding=True)
loader = CSVLoader(file_path, encoding=self._detect_text_encoding(file_path))
elif file_ext == 'rst':
try:
from langchain_community.document_loaders import UnstructuredRSTLoader
@@ -449,7 +556,7 @@ class Loader:
'Falling back to plain text loading for .rst file. '
'Install it with: pip install unstructured'
)
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
elif file_ext == 'xml':
try:
from langchain_community.document_loaders import UnstructuredXMLLoader
@@ -461,11 +568,11 @@ class Loader:
'Falling back to plain text loading for .xml file. '
'Install it with: pip install unstructured'
)
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
elif file_ext in ['htm', 'html']:
loader = BSHTMLLoader(file_path, open_encoding='unicode_escape')
elif file_ext == 'md':
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
elif file_content_type == 'application/epub+zip':
try:
from langchain_community.document_loaders import UnstructuredEPubLoader
@@ -534,8 +641,9 @@ class Loader:
'Install it with: pip install unstructured'
)
elif self._is_text_file(file_ext, file_content_type):
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
else:
loader = TextLoader(file_path, autodetect_encoding=True)
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
return loader