zaclys
/
searxng
mirror of https://github.com/searxng/searxng.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2102 from MarcAbonce/wikipedia-api-cleanup 

Fix Wikipedia's paragraph extraction
This commit is contained in:
Adam Tauber 2020-07-27 14:46:41 +02:00 committed by GitHub
parent 6d18769ccf 77b9faa8df
commit 1185c06a87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
searx/engines/wikipedia.py
View File

@ -49,29 +49,6 @@ def request(query, params):
return params
# get first meaningful paragraph
# this should filter out disambiguation pages and notes above first paragraph
# "magic numbers" were obtained by fine tuning
def extract_first_paragraph(content, title, image):
first_paragraph = None
failed_attempts = 0
for paragraph in content.split('\n'):
starts_with_title = paragraph.lower().find(title.lower(), 0, len(title) + 35)
length = len(paragraph)
if length >= 200 or (starts_with_title >= 0 and (image or length >= 150)):
first_paragraph = paragraph
break
failed_attempts += 1
if failed_attempts > 3:
return None
return first_paragraph
# get response from search-request
def response(resp):
results = []
@ -97,10 +74,7 @@ def response(resp):
if image:
image = image.get('source')
extract = page.get('extract')
summary = extract_first_paragraph(extract, title, image)
summary = summary.replace('() ', '')
summary = page.get('extract', '').split('\n')[0].replace('()', '')
# link to wikipedia article
wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \