This commit is contained in:
Timothy Jaeryang Baek
2026-03-17 17:58:01 -05:00
parent fcf7208352
commit de3317e26b
220 changed files with 17200 additions and 22836 deletions

View File

@@ -28,11 +28,11 @@ def search_google_pse(
Returns:
list[SearchResult]: A list of SearchResult objects.
"""
url = "https://www.googleapis.com/customsearch/v1"
url = 'https://www.googleapis.com/customsearch/v1'
headers = {"Content-Type": "application/json"}
headers = {'Content-Type': 'application/json'}
if referer:
headers["Referer"] = referer
headers['Referer'] = referer
all_results = []
start_index = 1 # Google PSE start parameter is 1-based
@@ -40,21 +40,19 @@ def search_google_pse(
while count > 0:
num_results_this_page = min(count, 10) # Google PSE max results per page is 10
params = {
"cx": search_engine_id,
"q": query,
"key": api_key,
"num": num_results_this_page,
"start": start_index,
'cx': search_engine_id,
'q': query,
'key': api_key,
'num': num_results_this_page,
'start': start_index,
}
response = requests.request("GET", url, headers=headers, params=params)
response = requests.request('GET', url, headers=headers, params=params)
response.raise_for_status()
json_response = response.json()
results = json_response.get("items", [])
results = json_response.get('items', [])
if results: # check if results are returned. If not, no more pages to fetch.
all_results.extend(results)
count -= len(
results
) # Decrement count by the number of results fetched in this page.
count -= len(results) # Decrement count by the number of results fetched in this page.
start_index += 10 # Increment start index for the next page
else:
break # No more results from Google PSE, break the loop
@@ -64,9 +62,9 @@ def search_google_pse(
return [
SearchResult(
link=result["link"],
title=result.get("title"),
snippet=result.get("snippet"),
link=result['link'],
title=result.get('title'),
snippet=result.get('snippet'),
)
for result in all_results
]