mirror of https://github.com/searxng/searxng.git
Merge pull request #612 from stepshal/trailing
Remove trailing whitespace
This commit is contained in:
commit
fcc3264f80
|
@ -22,4 +22,3 @@ def response(resp):
|
|||
resp: requests response object
|
||||
'''
|
||||
return [{'url': '', 'title': '', 'content': ''}]
|
||||
|
||||
|
|
|
@ -5,17 +5,17 @@ import unicodedata
|
|||
import string
|
||||
from urllib import urlencode
|
||||
from requests import get
|
||||
|
||||
|
||||
languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}
|
||||
|
||||
|
||||
url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)
|
||||
url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='
|
||||
url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='
|
||||
url_wikidata_search_template='http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
|
||||
|
||||
wmflabs_queries = [
|
||||
|
||||
wmflabs_queries = [
|
||||
'CLAIM[31:8142]', # all devise
|
||||
]
|
||||
|
||||
|
||||
db = {
|
||||
'iso4217' : {
|
||||
},
|
||||
|
@ -26,7 +26,7 @@ db = {
|
|||
|
||||
def remove_accents(data):
|
||||
return unicodedata.normalize('NFKD', data).lower()
|
||||
|
||||
|
||||
|
||||
def normalize_name(name):
|
||||
return re.sub(' +',' ', remove_accents(name.lower()).replace('-', ' '))
|
||||
|
@ -66,22 +66,22 @@ def get_property_value(data, name):
|
|||
prop = data.get('claims', {}).get(name, {})
|
||||
if len(prop) == 0:
|
||||
return None
|
||||
|
||||
value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')
|
||||
|
||||
value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')
|
||||
if value == '':
|
||||
return None
|
||||
|
||||
return value
|
||||
|
||||
|
||||
|
||||
def parse_currency(data):
|
||||
iso4217 = get_property_value(data, 'P498')
|
||||
|
||||
|
||||
if iso4217 is not None:
|
||||
unit = get_property_value(data, 'P558')
|
||||
if unit is not None:
|
||||
add_currency_name(unit, iso4217)
|
||||
|
||||
|
||||
labels = data.get('labels', {})
|
||||
for language in languages:
|
||||
name = labels.get(language, {}).get('value', None)
|
||||
|
@ -95,22 +95,22 @@ def parse_currency(data):
|
|||
alias = aliases[language][i].get('value', None)
|
||||
add_currency_name(alias, iso4217)
|
||||
|
||||
|
||||
|
||||
def fetch_data(wikidata_ids):
|
||||
url = url_template.format(query=urlencode({'ids' : '|'.join(wikidata_ids)}))
|
||||
htmlresponse = get(url)
|
||||
jsonresponse = json.loads(htmlresponse.content)
|
||||
entities = jsonresponse.get('entities', {})
|
||||
|
||||
|
||||
for pname in entities:
|
||||
pvalue = entities.get(pname)
|
||||
parse_currency(pvalue)
|
||||
|
||||
|
||||
|
||||
|
||||
def add_q(i):
|
||||
return "Q" + str(i)
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_data_batch(wikidata_ids):
|
||||
while len(wikidata_ids) > 0:
|
||||
if len(wikidata_ids) > 50:
|
||||
|
@ -119,8 +119,8 @@ def fetch_data_batch(wikidata_ids):
|
|||
else:
|
||||
fetch_data(wikidata_ids)
|
||||
wikidata_ids = []
|
||||
|
||||
|
||||
|
||||
|
||||
def wdq_query(query):
|
||||
url = url_wmflabs_template + query
|
||||
htmlresponse = get(url)
|
||||
|
@ -131,23 +131,23 @@ def wdq_query(query):
|
|||
print "error for query '" + query + "' :" + error
|
||||
|
||||
fetch_data_batch(qlist)
|
||||
|
||||
|
||||
|
||||
|
||||
def wd_query(query, offset=0):
|
||||
qlist = []
|
||||
|
||||
|
||||
url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))
|
||||
htmlresponse = get(url)
|
||||
jsonresponse = json.loads(htmlresponse.content)
|
||||
for r in jsonresponse.get('query', {}).get('search', {}):
|
||||
qlist.append(r.get('title', ''))
|
||||
fetch_data_batch(qlist)
|
||||
|
||||
|
||||
## fetch ##
|
||||
for q in wmflabs_queries:
|
||||
wdq_query(q)
|
||||
|
||||
# static
|
||||
# static
|
||||
add_currency_name(u"euro", 'EUR')
|
||||
add_currency_name(u"euros", 'EUR')
|
||||
add_currency_name(u"dollar", 'USD')
|
||||
|
|
Loading…
Reference in New Issue