ZaclysIndustries
/
searxngRebrandZaclys
forked from zaclys/searxng
2
0
Fork 0
Code Issues Pull Requests Activity

Merge pull request #1650 from dalf/wikidata 

[fix] wikidata engine: faster processing, remove one HTTP redirection.
This commit is contained in:
Alexandre Flament 2019-07-29 08:28:50 +02:00 committed by GitHub
parent cbc5e13275 6e0285b2db
commit 7f56c78876
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 164 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

254
searx/engines/wikidata.py
View File

@ -20,6 +20,7 @@ from searx.utils import match_language
from json import loads
from lxml.html import fromstring
from lxml import etree
logger = logger.getChild('wikidata')
result_count = 1
@ -27,23 +28,23 @@ result_count = 1
# urls
wikidata_host = 'https://www.wikidata.org'
url_search = wikidata_host \
+ '/w/index.php?{query}'
+ '/w/index.php?{query}&ns0=1'
wikidata_api = wikidata_host + '/w/api.php'
url_detail = wikidata_api\
+ '?action=parse&format=json&{query}'\
+ '&redirects=1&prop=text%7Cdisplaytitle%7Clanglinks%7Crevid'\
+ '&disableeditsection=1&disabletidy=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'
+ '&redirects=1&prop=text%7Cdisplaytitle%7Cparsewarnings'\
+ '&disableeditsection=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'
url_map = 'https://www.openstreetmap.org/'\
+ '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
# xpaths
div_ids_xpath = '//div[@id]'
wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'
title_xpath = '//*[contains(@class,"wikibase-title-label")]'
description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
property_xpath = '//div[@id="{propertyid}"]'
label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'
url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'
wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\
@ -56,6 +57,31 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator
calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
# xpath_cache
xpath_cache = {}
def get_xpath(xpath_str):
result = xpath_cache.get(xpath_str, None)
if not result:
result = etree.XPath(xpath_str)
xpath_cache[xpath_str] = result
return result
def eval_xpath(element, xpath_str):
xpath = get_xpath(xpath_str)
return xpath(element)
def get_id_cache(result):
id_cache = {}
for e in eval_xpath(result, div_ids_xpath):
id = e.get('id')
if id.startswith('P'):
id_cache[id] = e
return id_cache
def request(query, params):
params['url'] = url_search.format(
@ -65,8 +91,9 @@ def request(query, params):
def response(resp):
results = []
html = fromstring(resp.text)
search_results = html.xpath(wikidata_ids_xpath)
htmlparser = etree.HTMLParser()
html = fromstring(resp.content.decode("utf-8"), parser=htmlparser)
search_results = eval_xpath(html, wikidata_ids_xpath)
if resp.search_params['language'].split('-')[0] == 'all':
language = 'en'
@ -78,13 +105,13 @@ def response(resp):
wikidata_id = search_result.split('/')[-1]
url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
htmlresponse = get(url)
jsonresponse = loads(htmlresponse.text)
results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])
jsonresponse = loads(htmlresponse.content.decode("utf-8"))
results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'], htmlparser)
return results
def getDetail(jsonresponse, wikidata_id, language, locale):
def getDetail(jsonresponse, wikidata_id, language, locale, htmlparser):
results = []
urls = []
attributes = []
@ -95,21 +122,23 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
if not title or not result:
return results
title = fromstring(title)
for elem in title.xpath(language_fallback_xpath):
title = fromstring(title, parser=htmlparser)
for elem in eval_xpath(title, language_fallback_xpath):
elem.getparent().remove(elem)
title = extract_text(title.xpath(title_xpath))
title = extract_text(eval_xpath(title, title_xpath))
result = fromstring(result)
for elem in result.xpath(language_fallback_xpath):
result = fromstring(result, parser=htmlparser)
for elem in eval_xpath(result, language_fallback_xpath):
elem.getparent().remove(elem)
description = extract_text(result.xpath(description_xpath))
description = extract_text(eval_xpath(result, description_xpath))
id_cache = get_id_cache(result)
# URLS
# official website
add_url(urls, result, 'P856', results=results)
add_url(urls, result, id_cache, 'P856', results=results)
# wikipedia
wikipedia_link_count = 0
@ -130,30 +159,30 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
# if wikipedia_link_count == 0:
# more wikis
add_url(urls, result, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')
add_url(urls, result, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')
add_url(urls, result, default_label='Wikimedia Commons', link_type='commonswiki')
add_url(urls, result, id_cache, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')
add_url(urls, result, id_cache, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')
add_url(urls, result, id_cache, default_label='Wikimedia Commons', link_type='commonswiki')
add_url(urls, result, 'P625', 'OpenStreetMap', link_type='geo')
add_url(urls, result, id_cache, 'P625', 'OpenStreetMap', link_type='geo')
# musicbrainz
add_url(urls, result, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')
add_url(urls, result, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')
add_url(urls, result, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')
add_url(urls, result, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')
add_url(urls, result, id_cache, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')
add_url(urls, result, id_cache, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')
add_url(urls, result, id_cache, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')
add_url(urls, result, id_cache, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')
# IMDb
add_url(urls, result, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')
add_url(urls, result, id_cache, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')
# source code repository
add_url(urls, result, 'P1324')
add_url(urls, result, id_cache, 'P1324')
# blog
add_url(urls, result, 'P1581')
add_url(urls, result, id_cache, 'P1581')
# social media links
add_url(urls, result, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')
add_url(urls, result, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')
add_url(urls, result, 'P2002', 'Twitter', 'https://twitter.com/')
add_url(urls, result, 'P2013', 'Facebook', 'https://facebook.com/')
add_url(urls, result, 'P2003', 'Instagram', 'https://instagram.com/')
add_url(urls, result, id_cache, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')
add_url(urls, result, id_cache, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')
add_url(urls, result, id_cache, 'P2002', 'Twitter', 'https://twitter.com/')
add_url(urls, result, id_cache, 'P2013', 'Facebook', 'https://facebook.com/')
add_url(urls, result, id_cache, 'P2003', 'Instagram', 'https://instagram.com/')
urls.append({'title': 'Wikidata',
'url': 'https://www.wikidata.org/wiki/'
@ -163,132 +192,132 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
# DATES
# inception date
add_attribute(attributes, result, 'P571', date=True)
add_attribute(attributes, id_cache, 'P571', date=True)
# dissolution date
add_attribute(attributes, result, 'P576', date=True)
add_attribute(attributes, id_cache, 'P576', date=True)
# start date
add_attribute(attributes, result, 'P580', date=True)
add_attribute(attributes, id_cache, 'P580', date=True)
# end date
add_attribute(attributes, result, 'P582', date=True)
add_attribute(attributes, id_cache, 'P582', date=True)
# date of birth
add_attribute(attributes, result, 'P569', date=True)
add_attribute(attributes, id_cache, 'P569', date=True)
# date of death
add_attribute(attributes, result, 'P570', date=True)
add_attribute(attributes, id_cache, 'P570', date=True)
# date of spacecraft launch
add_attribute(attributes, result, 'P619', date=True)
add_attribute(attributes, id_cache, 'P619', date=True)
# date of spacecraft landing
add_attribute(attributes, result, 'P620', date=True)
add_attribute(attributes, id_cache, 'P620', date=True)
# nationality
add_attribute(attributes, result, 'P27')
add_attribute(attributes, id_cache, 'P27')
# country of origin
add_attribute(attributes, result, 'P495')
add_attribute(attributes, id_cache, 'P495')
# country
add_attribute(attributes, result, 'P17')
add_attribute(attributes, id_cache, 'P17')
# headquarters
add_attribute(attributes, result, 'Q180')
add_attribute(attributes, id_cache, 'Q180')
# PLACES
# capital
add_attribute(attributes, result, 'P36', trim=True)
add_attribute(attributes, id_cache, 'P36', trim=True)
# head of state
add_attribute(attributes, result, 'P35', trim=True)
add_attribute(attributes, id_cache, 'P35', trim=True)
# head of government
add_attribute(attributes, result, 'P6', trim=True)
add_attribute(attributes, id_cache, 'P6', trim=True)
# type of government
add_attribute(attributes, result, 'P122')
add_attribute(attributes, id_cache, 'P122')
# official language
add_attribute(attributes, result, 'P37')
add_attribute(attributes, id_cache, 'P37')
# population
add_attribute(attributes, result, 'P1082', trim=True)
add_attribute(attributes, id_cache, 'P1082', trim=True)
# area
add_attribute(attributes, result, 'P2046')
add_attribute(attributes, id_cache, 'P2046')
# currency
add_attribute(attributes, result, 'P38', trim=True)
add_attribute(attributes, id_cache, 'P38', trim=True)
# heigth (building)
add_attribute(attributes, result, 'P2048')
add_attribute(attributes, id_cache, 'P2048')
# MEDIA
# platform (videogames)
add_attribute(attributes, result, 'P400')
add_attribute(attributes, id_cache, 'P400')
# author
add_attribute(attributes, result, 'P50')
add_attribute(attributes, id_cache, 'P50')
# creator
add_attribute(attributes, result, 'P170')
add_attribute(attributes, id_cache, 'P170')
# director
add_attribute(attributes, result, 'P57')
add_attribute(attributes, id_cache, 'P57')
# performer
add_attribute(attributes, result, 'P175')
add_attribute(attributes, id_cache, 'P175')
# developer
add_attribute(attributes, result, 'P178')
add_attribute(attributes, id_cache, 'P178')
# producer
add_attribute(attributes, result, 'P162')
add_attribute(attributes, id_cache, 'P162')
# manufacturer
add_attribute(attributes, result, 'P176')
add_attribute(attributes, id_cache, 'P176')
# screenwriter
add_attribute(attributes, result, 'P58')
add_attribute(attributes, id_cache, 'P58')
# production company
add_attribute(attributes, result, 'P272')
add_attribute(attributes, id_cache, 'P272')
# record label
add_attribute(attributes, result, 'P264')
add_attribute(attributes, id_cache, 'P264')
# publisher
add_attribute(attributes, result, 'P123')
add_attribute(attributes, id_cache, 'P123')
# original network
add_attribute(attributes, result, 'P449')
add_attribute(attributes, id_cache, 'P449')
# distributor
add_attribute(attributes, result, 'P750')
add_attribute(attributes, id_cache, 'P750')
# composer
add_attribute(attributes, result, 'P86')
add_attribute(attributes, id_cache, 'P86')
# publication date
add_attribute(attributes, result, 'P577', date=True)
add_attribute(attributes, id_cache, 'P577', date=True)
# genre
add_attribute(attributes, result, 'P136')
add_attribute(attributes, id_cache, 'P136')
# original language
add_attribute(attributes, result, 'P364')
add_attribute(attributes, id_cache, 'P364')
# isbn
add_attribute(attributes, result, 'Q33057')
add_attribute(attributes, id_cache, 'Q33057')
# software license
add_attribute(attributes, result, 'P275')
add_attribute(attributes, id_cache, 'P275')
# programming language
add_attribute(attributes, result, 'P277')
add_attribute(attributes, id_cache, 'P277')
# version
add_attribute(attributes, result, 'P348', trim=True)
add_attribute(attributes, id_cache, 'P348', trim=True)
# narrative location
add_attribute(attributes, result, 'P840')
add_attribute(attributes, id_cache, 'P840')
# LANGUAGES
# number of speakers
add_attribute(attributes, result, 'P1098')
add_attribute(attributes, id_cache, 'P1098')
# writing system
add_attribute(attributes, result, 'P282')
add_attribute(attributes, id_cache, 'P282')
# regulatory body
add_attribute(attributes, result, 'P1018')
add_attribute(attributes, id_cache, 'P1018')
# language code
add_attribute(attributes, result, 'P218')
add_attribute(attributes, id_cache, 'P218')
# OTHER
# ceo
add_attribute(attributes, result, 'P169', trim=True)
add_attribute(attributes, id_cache, 'P169', trim=True)
# founder
add_attribute(attributes, result, 'P112')
add_attribute(attributes, id_cache, 'P112')
# legal form (company/organization)
add_attribute(attributes, result, 'P1454')
add_attribute(attributes, id_cache, 'P1454')
# operator
add_attribute(attributes, result, 'P137')
add_attribute(attributes, id_cache, 'P137')
# crew members (tripulation)
add_attribute(attributes, result, 'P1029')
add_attribute(attributes, id_cache, 'P1029')
# taxon
add_attribute(attributes, result, 'P225')
add_attribute(attributes, id_cache, 'P225')
# chemical formula
add_attribute(attributes, result, 'P274')
add_attribute(attributes, id_cache, 'P274')
# winner (sports/contests)
add_attribute(attributes, result, 'P1346')
add_attribute(attributes, id_cache, 'P1346')
# number of deaths
add_attribute(attributes, result, 'P1120')
add_attribute(attributes, id_cache, 'P1120')
# currency code
add_attribute(attributes, result, 'P498')
add_attribute(attributes, id_cache, 'P498')
image = add_image(result)
image = add_image(id_cache)
if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:
results.append({
@ -310,43 +339,42 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
# only returns first match
def add_image(result):
def add_image(id_cache):
# P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon
property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']
for property_id in property_ids:
image = result.xpath(property_xpath.replace('{propertyid}', property_id))
if image:
image_name = image[0].xpath(media_xpath)
image = id_cache.get(property_id, None)
if image is not None:
image_name = eval_xpath(image, media_xpath)
image_src = url_image.replace('{filename}', extract_text(image_name[0]))
return image_src
# setting trim will only returned high ranked rows OR the first row
def add_attribute(attributes, result, property_id, default_label=None, date=False, trim=False):
attribute = result.xpath(property_xpath.replace('{propertyid}', property_id))
if attribute:
def add_attribute(attributes, id_cache, property_id, default_label=None, date=False, trim=False):
attribute = id_cache.get(property_id, None)
if attribute is not None:
if default_label:
label = default_label
else:
label = extract_text(attribute[0].xpath(label_xpath))
label = extract_text(eval_xpath(attribute, label_xpath))
label = label[0].upper() + label[1:]
if date:
trim = True
# remove calendar name
calendar_name = attribute[0].xpath(calendar_name_xpath)
calendar_name = eval_xpath(attribute, calendar_name_xpath)
for calendar in calendar_name:
calendar.getparent().remove(calendar)
concat_values = ""
values = []
first_value = None
for row in attribute[0].xpath(property_row_xpath):
if not first_value or not trim or row.xpath(preferred_rank_xpath):
value = row.xpath(value_xpath)
for row in eval_xpath(attribute, property_row_xpath):
if not first_value or not trim or eval_xpath(row, preferred_rank_xpath):
value = eval_xpath(row, value_xpath)
if not value:
continue
value = extract_text(value)
@ -369,18 +397,18 @@ def add_attribute(attributes, result, property_id, default_label=None, date=Fals
# requires property_id unless it's a wiki link (defined in link_type)
def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, results=None, link_type=None):
def add_url(urls, result, id_cache, property_id=None, default_label=None, url_prefix=None, results=None,
link_type=None):
links = []
# wiki links don't have property in wikidata page
if link_type and 'wiki' in link_type:
links.append(get_wikilink(result, link_type))
else:
dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id))
if dom_element:
dom_element = dom_element[0]
dom_element = id_cache.get(property_id, None)
if dom_element is not None:
if not default_label:
label = extract_text(dom_element.xpath(label_xpath))
label = extract_text(eval_xpath(dom_element, label_xpath))
label = label[0].upper() + label[1:]
if link_type == 'geo':
@ -390,7 +418,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
links.append(get_imdblink(dom_element, url_prefix))
else:
url_results = dom_element.xpath(url_xpath)
url_results = eval_xpath(dom_element, url_xpath)
for link in url_results:
if link is not None:
if url_prefix:
@ -410,7 +438,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
def get_imdblink(result, url_prefix):
imdb_id = result.xpath(value_xpath)
imdb_id = eval_xpath(result, value_xpath)
if imdb_id:
imdb_id = extract_text(imdb_id)
id_prefix = imdb_id[:2]
@ -430,7 +458,7 @@ def get_imdblink(result, url_prefix):
def get_geolink(result):
coordinates = result.xpath(value_xpath)
coordinates = eval_xpath(result, value_xpath)
if not coordinates:
return None
coordinates = extract_text(coordinates[0])
@ -477,7 +505,7 @@ def get_geolink(result):
def get_wikilink(result, wikiid):
url = result.xpath(wikilink_xpath.replace('{wikiid}', wikiid))
url = eval_xpath(result, wikilink_xpath.replace('{wikiid}', wikiid))
if not url:
return None
url = url[0]

38
tests/unit/engines/test_wikidata.py
View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
from lxml.html import fromstring
from lxml import etree
from collections import defaultdict
import mock
from searx.engines import wikidata
@ -30,12 +31,12 @@ class TestWikidataEngine(SearxTestCase):
wikidata.supported_languages = ['en', 'es']
wikidata.language_aliases = {}
response = mock.Mock(text='<html></html>', search_params={"language": "en"})
response = mock.Mock(content='<html></html>'.encode("utf-8"), search_params={"language": "en"})
self.assertEqual(wikidata.response(response), [])
def test_getDetail(self):
response = {}
results = wikidata.getDetail(response, "Q123", "en", "en-US")
results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser())
self.assertEqual(results, [])
title_html = '<div><div class="wikibase-title-label">Test</div></div>'
@ -52,7 +53,7 @@ class TestWikidataEngine(SearxTestCase):
"""
response = {"parse": {"displaytitle": title_html, "text": html}}
results = wikidata.getDetail(response, "Q123", "en", "en-US")
results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser())
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')
@ -92,7 +93,7 @@ class TestWikidataEngine(SearxTestCase):
"""
response = {"parse": {"displaytitle": title_html, "text": html}}
results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")
results = wikidata.getDetail(response, "Q123", "yua", "yua_MX", etree.HTMLParser())
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'Official website')
self.assertEqual(results[0]['url'], 'https://officialsite.com')
@ -139,8 +140,8 @@ class TestWikidataEngine(SearxTestCase):
</div>
"""
html_etree = fromstring(html)
image_src = wikidata.add_image(html_etree)
id_cache = wikidata.get_id_cache(html_etree)
image_src = wikidata.add_image(id_cache)
self.assertEqual(image_src,
"https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")
@ -197,8 +198,9 @@ class TestWikidataEngine(SearxTestCase):
</div>
"""
html_etree = fromstring(html)
id_cache = wikidata.get_id_cache(html_etree)
image_src = wikidata.add_image(html_etree)
image_src = wikidata.add_image(id_cache)
self.assertEqual(image_src,
"https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")
@ -232,11 +234,12 @@ class TestWikidataEngine(SearxTestCase):
"""
attributes = []
html_etree = fromstring(html)
id_cache = wikidata.get_id_cache(html_etree)
wikidata.add_attribute(attributes, html_etree, "Fail")
wikidata.add_attribute(attributes, id_cache, "Fail")
self.assertEqual(attributes, [])
wikidata.add_attribute(attributes, html_etree, "P27")
wikidata.add_attribute(attributes, id_cache, "P27")
self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["label"], "Country of citizenship")
self.assertEqual(attributes[0]["value"], "United Kingdom")
@ -271,7 +274,8 @@ class TestWikidataEngine(SearxTestCase):
"""
attributes = []
html_etree = fromstring(html)
wikidata.add_attribute(attributes, html_etree, "P569", date=True)
id_cache = wikidata.get_id_cache(html_etree)
wikidata.add_attribute(attributes, id_cache, "P569", date=True)
self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["label"], "Date of birth")
self.assertEqual(attributes[0]["value"], "27 January 1832")
@ -319,14 +323,16 @@ class TestWikidataEngine(SearxTestCase):
"""
attributes = []
html_etree = fromstring(html)
wikidata.add_attribute(attributes, html_etree, "P6")
id_cache = wikidata.get_id_cache(html_etree)
wikidata.add_attribute(attributes, id_cache, "P6")
self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["label"], "Head of government")
self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")
attributes = []
html_etree = fromstring(html)
wikidata.add_attribute(attributes, html_etree, "P6", trim=True)
id_cache = wikidata.get_id_cache(html_etree)
wikidata.add_attribute(attributes, id_cache, "P6", trim=True)
self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["value"], "Actual Prime Minister")
@ -357,12 +363,13 @@ class TestWikidataEngine(SearxTestCase):
"""
urls = []
html_etree = fromstring(html)
wikidata.add_url(urls, html_etree, 'P856')
id_cache = wikidata.get_id_cache(html_etree)
wikidata.add_url(urls, html_etree, id_cache, 'P856')
self.assertEquals(len(urls), 1)
self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)
urls = []
results = []
wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)
wikidata.add_url(urls, html_etree, id_cache, 'P856', 'custom label', results=results)
self.assertEquals(len(urls), 1)
self.assertEquals(len(results), 1)
self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)
@ -405,7 +412,8 @@ class TestWikidataEngine(SearxTestCase):
"""
urls = []
html_etree = fromstring(html)
wikidata.add_url(urls, html_etree, 'P856')
id_cache = wikidata.get_id_cache(html_etree)
wikidata.add_url(urls, html_etree, id_cache, 'P856')
self.assertEquals(len(urls), 2)
self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)