diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index ede890793b..68d6b79702 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -233,34 +233,7 @@ class Loader: def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: loader = self._get_loader(filename, file_content_type, file_path) - - try: - docs = loader.load() - except (UnicodeDecodeError, RuntimeError, TimeoutError) as e: - # Only retry with latin-1 for text-encoding loaders (TextLoader, - # CSVLoader). Binary format loaders (PDF, DOCX, PPTX, …) can - # raise RuntimeError for unrelated reasons; retrying those as - # plain text would produce garbage. - if not isinstance(loader, (TextLoader, CSVLoader)): - raise - - # TextLoader/CSVLoader with autodetect_encoding=True can fail when: - # - chardet times out (5 s hardcoded in langchain) - # - chardet returns a wrong/low-confidence encoding - # - the file contains non-UTF-8 bytes (Latin-1, Windows-1252, …) - # - # Latin-1 is a safe last resort: every byte 0x00–0xFF is valid, - # and ftfy.fix_text() (applied below) repairs most mojibake that - # results from treating Windows-1252 content as Latin-1. - log.warning( - "Primary loader failed for %s (%s), retrying with latin-1 encoding: %s", - filename, - type(e).__name__, - e, - ) - fallback_loader = TextLoader(file_path, encoding="latin-1") - docs = fallback_loader.load() - + docs = loader.load() return [Document(page_content=ftfy.fix_text(doc.page_content), metadata=doc.metadata) for doc in docs] async def aload(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: @@ -283,6 +256,140 @@ class Loader: and not file_content_type.find('html') >= 0 ) + def _detect_text_encoding(self, file_path: str) -> str: + """Detect the encoding of a text file with CJK-aware fallbacks. + + Langchain's ``TextLoader`` uses chardet internally when + ``autodetect_encoding=True``, but chardet frequently misidentifies + CJK encodings (e.g. GB18030 detected as GB2312 or even Cyrillic). + This method replaces that by: + + 1. Trying UTF-8 first (fast path for the vast majority of files). + 2. Using chardet as a *hint* to prioritise the right CJK codec + family, but mapping subset names to their superset + (e.g. GB2312 → gb18030). + 3. Validating that decoded text actually contains CJK characters, + guarding against codecs that "succeed" but produce garbage. + 4. Falling back to latin-1 (always valid, ftfy fixes mojibake later). + """ + try: + with open(file_path, "rb") as f: + raw = f.read() + except OSError: + return "utf-8" + + if not raw: + return "utf-8" + + # Fast path: most files are UTF-8 + try: + raw.decode("utf-8") + return "utf-8" + except UnicodeDecodeError: + pass + + # Use chardet as a hint, not as ground truth + import chardet + + detected = chardet.detect(raw) + detected_enc = (detected.get("encoding") or "").lower().replace("-", "").replace("_", "") + + # Map chardet's detected encoding to the correct superset codec. + # chardet often reports GB2312 for content that is actually GB18030; + # GB18030 is a strict superset of both GB2312 and GBK. + _ENC_FAMILY = { + "gb2312": "gb18030", + "gb18030": "gb18030", + "gbk": "gb18030", + "big5": "big5", + "euckr": "euc-kr", + "eucjp": "euc-jp", + "iso2022jp": "euc-jp", + "shiftjis": "shift_jis", + } + + # Build priority list: chardet-hinted codec first, then remaining CJK + base_order = ["gb18030", "big5", "euc-kr", "euc-jp"] + hinted = _ENC_FAMILY.get(detected_enc) + if hinted and hinted in base_order: + ordered = [hinted] + [e for e in base_order if e != hinted] + else: + ordered = base_order + + for enc in ordered: + try: + text = raw.decode(enc) + if text.strip() and self._has_cjk_characters(text): + log.info( + "Detected encoding %s for %s (chardet guessed %s)", + enc, + file_path, + detected.get("encoding"), + ) + return enc + except (UnicodeDecodeError, LookupError): + continue + + # If chardet gave a non-CJK answer that isn't in our family map, + # try it directly — it might be a valid Western encoding. + chardet_encoding = detected.get("encoding") + if chardet_encoding: + try: + raw.decode(chardet_encoding) + log.info( + "Using chardet-detected encoding %s for %s", + chardet_encoding, + file_path, + ) + return chardet_encoding + except (UnicodeDecodeError, LookupError): + pass + + # latin-1 is the ultimate fallback: every byte 0x00–0xFF is valid. + # ftfy.fix_text() (applied downstream) repairs most mojibake that + # results from treating Windows-1252 content as Latin-1. + log.info("Falling back to latin-1 encoding for %s", file_path) + return "latin-1" + + @staticmethod + def _has_cjk_characters(text: str, threshold: float = 0.05) -> bool: + """Check if decoded text contains a meaningful proportion of CJK characters. + + This guards against codecs that technically "succeed" but decode the + bytes into wrong Unicode codepoints (e.g. PUA chars, random symbols). + A genuine CJK document should have at least ``threshold`` fraction of + its non-whitespace characters in CJK Unicode blocks. + """ + if not text: + return False + + cjk_count = 0 + total = 0 + for ch in text: + if ch.isspace(): + continue + total += 1 + cp = ord(ch) + if ( + 0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs + or 0x3400 <= cp <= 0x4DBF # CJK Extension A + or 0x20000 <= cp <= 0x2A6DF # CJK Extension B + or 0x2A700 <= cp <= 0x2B73F # CJK Extension C + or 0x2B740 <= cp <= 0x2B81F # CJK Extension D + or 0xF900 <= cp <= 0xFAFF # CJK Compatibility Ideographs + or 0x3000 <= cp <= 0x303F # CJK Symbols and Punctuation + or 0x3040 <= cp <= 0x309F # Hiragana + or 0x30A0 <= cp <= 0x30FF # Katakana + or 0xAC00 <= cp <= 0xD7AF # Hangul Syllables + or 0xFF00 <= cp <= 0xFFEF # Halfwidth and Fullwidth Forms + ): + cjk_count += 1 + + if total == 0: + return False + + return (cjk_count / total) >= threshold + def _get_loader(self, filename: str, file_content_type: str, file_path: str): file_ext = filename.split('.')[-1].lower() @@ -300,7 +407,7 @@ class Loader: ) elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'): if self._is_text_file(file_ext, file_content_type): - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) else: loader = TikaLoader( url=self.kwargs.get('TIKA_SERVER_URL'), @@ -352,7 +459,7 @@ class Loader: ) elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'): if self._is_text_file(file_ext, file_content_type): - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) else: # Build params for DoclingLoader params = self.kwargs.get('DOCLING_PARAMS', {}) @@ -437,7 +544,7 @@ class Loader: mode=self.kwargs.get('PDF_LOADER_MODE', 'page'), ) elif file_ext == 'csv': - loader = CSVLoader(file_path, autodetect_encoding=True) + loader = CSVLoader(file_path, encoding=self._detect_text_encoding(file_path)) elif file_ext == 'rst': try: from langchain_community.document_loaders import UnstructuredRSTLoader @@ -449,7 +556,7 @@ class Loader: 'Falling back to plain text loading for .rst file. ' 'Install it with: pip install unstructured' ) - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) elif file_ext == 'xml': try: from langchain_community.document_loaders import UnstructuredXMLLoader @@ -461,11 +568,11 @@ class Loader: 'Falling back to plain text loading for .xml file. ' 'Install it with: pip install unstructured' ) - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) elif file_ext in ['htm', 'html']: loader = BSHTMLLoader(file_path, open_encoding='unicode_escape') elif file_ext == 'md': - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) elif file_content_type == 'application/epub+zip': try: from langchain_community.document_loaders import UnstructuredEPubLoader @@ -534,8 +641,9 @@ class Loader: 'Install it with: pip install unstructured' ) elif self._is_text_file(file_ext, file_content_type): - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) else: - loader = TextLoader(file_path, autodetect_encoding=True) + loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) return loader +