mirror of
https://github.com/open-webui/open-webui.git
synced 2026-11-17 15:53:59 -06:00
fix: uploaded files lose Chinese punctuation and quotes (#31655)
Text extracted from any uploaded file had full-width punctuation like :(),!? turned into ASCII :(),!? and curly quotes like “ ” turned into straight quotes, so both the file preview and the model saw altered text. Extracted text now keeps these characters as written, while garbled text from wrong encodings (like café becoming café) is still repaired. Files uploaded before this change keep the altered text until they are uploaded again. Fixes #17087
This commit is contained in:
@@ -342,7 +342,12 @@ class Loader:
|
||||
docs = loader.load()
|
||||
# ftfy's auto mode unescapes entities on every line before the first literal '<', rewriting the document.
|
||||
return [
|
||||
Document(page_content=ftfy.fix_text(doc.page_content, unescape_html=False), metadata=doc.metadata)
|
||||
Document(
|
||||
page_content=ftfy.fix_text(
|
||||
doc.page_content, unescape_html=False, fix_character_width=False, uncurl_quotes=False
|
||||
),
|
||||
metadata=doc.metadata,
|
||||
)
|
||||
for doc in docs
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user