github-starred/open-webui
2
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-07-16 14:39:31 -05:00
Code Issues 2.4k Packages Projects Releases 129 Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-05-14 02:56:44 +09:00
parent 2a7524056a
commit f607337582
2 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
backend/open_webui/env.py
View File

@@ -514,6 +514,11 @@ AIOHTTP_CLIENT_SESSION_SSL = os.getenv('AIOHTTP_CLIENT_SESSION_SSL', 'True').low
# When False (default), outbound HTTP requests do not follow 3xx redirects.
AIOHTTP_CLIENT_ALLOW_REDIRECTS = os.getenv('AIOHTTP_CLIENT_ALLOW_REDIRECTS', 'False').lower() == 'true'
# Optional User-Agent override for outbound web-loader fetches. When set,
# SafeWebBaseLoader sends this value instead of the default python-requests UA
# which is aggressively blocked by Cloudflare, Wikipedia, and similar services.
USER_AGENT = os.getenv('USER_AGENT', '')
_model_list_timeout_raw = os.getenv(
'AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST',
os.getenv('AIOHTTP_CLIENT_TIMEOUT_OPENAI_MODEL_LIST', '10'),

12
backend/open_webui/retrieval/web/utils.py
View File

@@ -42,7 +42,7 @@ from open_webui.config import (
WEB_LOADER_TIMEOUT,
)
from open_webui.constants import ERROR_MESSAGES
from open_webui.env import AIOHTTP_CLIENT_ALLOW_REDIRECTS, AIOHTTP_CLIENT_SESSION_SSL
from open_webui.env import AIOHTTP_CLIENT_ALLOW_REDIRECTS, AIOHTTP_CLIENT_SESSION_SSL, USER_AGENT
from open_webui.retrieval.loaders.external_web import ExternalWebLoader
from open_webui.retrieval.loaders.tavily import TavilyLoader
from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
@@ -491,6 +491,15 @@ class SafeWebBaseLoader(WebBaseLoader):
"""
super().__init__(*args, **kwargs)
self.trust_env = trust_env
# Propagate USER_AGENT env var so that both the sync _scrape() and
# async _fetch() paths present a real UA instead of python-requests/2.x
# which gets blocked by Cloudflare, Wikipedia, and similar bot-detection.
# _fetch() forwards self.session.headers to the aiohttp session, so
# setting it here covers both code-paths.
if USER_AGENT:
self.session.headers['User-Agent'] = USER_AGENT
# Prevent redirect-based SSRF on the synchronous _scrape() path.
# validate_url() is called once on the originally-submitted URL, but the
# parent WebBaseLoader's _scrape() invokes self.session.get(url, **self.requests_kwargs)
@@ -519,7 +528,6 @@ class SafeWebBaseLoader(WebBaseLoader):
async with session.get(
url,
**(self.requests_kwargs | kwargs),
allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS,
) as response:
if self.raise_for_status:
response.raise_for_status()