diff --git a/searx/engines/wolframalpha_api.py b/searx/engines/wolframalpha_api.py index 303c6c165..ad80fb0f4 100644 --- a/searx/engines/wolframalpha_api.py +++ b/searx/engines/wolframalpha_api.py @@ -1,40 +1,56 @@ -# Wolfram Alpha (Maths) +# Wolfram Alpha (Science) # -# @website http://www.wolframalpha.com -# @provide-api yes (http://api.wolframalpha.com/v2/) +# @website https://www.wolframalpha.com +# @provide-api yes (https://api.wolframalpha.com/v2/) # # @using-api yes # @results XML # @stable yes -# @parse result +# @parse url, infobox from urllib import urlencode from lxml import etree -from re import search # search-url -base_url = 'http://api.wolframalpha.com/v2/query' -search_url = base_url + '?appid={api_key}&{query}&format=plaintext' -site_url = 'http://www.wolframalpha.com/input/?{query}' +search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}' +site_url = 'https://www.wolframalpha.com/input/?{query}' api_key = '' # defined in settings.yml # xpath variables failure_xpath = '/queryresult[attribute::success="false"]' answer_xpath = '//pod[attribute::primary="true"]/subpod/plaintext' input_xpath = '//pod[starts-with(attribute::title, "Input")]/subpod/plaintext' +pods_xpath = '//pod' +subpods_xpath = './subpod' +pod_title_xpath = './@title' +plaintext_xpath = './plaintext' +image_xpath = './img' +img_src_xpath = './@src' +img_alt_xpath = './@alt' + +# pods to display as image in infobox +# this pods do return a plaintext, but they look better and are more useful as images +image_pods = {'Visual representation', + 'Manipulatives illustration'} # do search-request def request(query, params): params['url'] = search_url.format(query=urlencode({'input': query}), api_key=api_key) + params['headers']['Referer'] = site_url.format(query=urlencode({'i': query})) return params # replace private user area characters to make text legible def replace_pua_chars(text): - pua_chars = {u'\uf74c': 'd', + pua_chars = {u'\uf522': u'\u2192', + u'\uf7b1': u'\u2115', + u'\uf7b4': u'\u211a', + u'\uf7b5': u'\u211d', + u'\uf7bd': u'\u2124', + u'\uf74c': 'd', u'\uf74d': u'\u212f', u'\uf74e': 'i', u'\uf7d9': '='} @@ -55,23 +71,45 @@ def response(resp): if search_results.xpath(failure_xpath): return [] - # parse answers - answers = search_results.xpath(answer_xpath) - if answers: - for answer in answers: - answer = replace_pua_chars(answer.text) + infobox_title = search_results.xpath(input_xpath) + if infobox_title: + infobox_title = replace_pua_chars(infobox_title[0].text) - results.append({'answer': answer}) + pods = search_results.xpath(pods_xpath) + result_chunks = [] + for pod in pods: + pod_title = replace_pua_chars(pod.xpath(pod_title_xpath)[0]) - # if there's no input section in search_results, check if answer has the input embedded (before their "=" sign) - try: - query_input = search_results.xpath(input_xpath)[0].text - except IndexError: - query_input = search(u'([^\uf7d9]+)', answers[0].text).group(1) + subpods = pod.xpath(subpods_xpath) + if not subpods: + continue + + for subpod in subpods: + content = subpod.xpath(plaintext_xpath)[0].text + image = subpod.xpath(image_xpath) + if content and pod_title not in image_pods: + content = replace_pua_chars(content) + result_chunks.append({'label': pod_title, 'value': content}) + + # if there's no input pod, infobox_title is content of first pod + if not infobox_title: + infobox_title = content + + elif image: + result_chunks.append({'label': pod_title, + 'image': {'src': image[0].xpath(img_src_xpath)[0], + 'alt': image[0].xpath(img_alt_xpath)[0]}}) + + if not result_chunks: + return [] + + results.append({'infobox': infobox_title, + 'attributes': result_chunks, + 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) # append link to site - result_url = site_url.format(query=urlencode({'i': query_input.encode('utf-8')})) - results.append({'url': result_url, - 'title': query_input + " - Wolfram|Alpha"}) + results.append({'url': resp.request.headers['Referer'], + 'title': 'Wolfram|Alpha', + 'content': infobox_title}) return results diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index 639dccf24..a5fbdeae7 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -1,23 +1,23 @@ -# WolframAlpha (Maths) +# Wolfram|Alpha (Science) # -# @website http://www.wolframalpha.com/ -# @provide-api yes (http://api.wolframalpha.com/v2/) +# @website https://www.wolframalpha.com/ +# @provide-api yes (https://api.wolframalpha.com/v2/) # # @using-api no -# @results HTML +# @results JSON # @stable no -# @parse answer +# @parse url, infobox from cgi import escape from json import loads from time import time from urllib import urlencode +from lxml.etree import XML from searx.poolrequests import get as http_get # search-url url = 'https://www.wolframalpha.com/' -search_url = url + 'input/?{query}' search_url = url + 'input/json.jsp'\ '?async=true'\ @@ -33,13 +33,25 @@ search_url = url + 'input/json.jsp'\ '&sponsorcategories=true'\ '&statemethod=deploybutton' -# xpath variables -scripts_xpath = '//script' -title_xpath = '//title' -failure_xpath = '//p[attribute::class="pfail"]' +referer_url = url + 'input/?{query}' + token = {'value': '', 'last_updated': None} +# xpath variables +success_xpath = '/pod[attribute::error="false"]' +plaintext_xpath = './plaintext' +title_xpath = './@title' +image_xpath = './img' +img_src_xpath = './img/@src' +img_alt_xpath = './img/@alt' + +# pods to display as image in infobox +# this pods do return a plaintext, but they look better and are more useful as images +image_pods = {'Visual representation', + 'Manipulatives illustration', + 'Symbol'} + # seems, wolframalpha resets its token in every hour def obtain_token(): @@ -62,13 +74,42 @@ def request(query, params): if time() - token['last_updated'] > 3600: obtain_token() params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value']) - params['headers']['Referer'] = 'https://www.wolframalpha.com/input/?i=' + query + params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query})) return params +# get additional pod +# NOTE: this makes an additional requests to server, so the response will take longer and might reach timeout +def get_async_pod(url): + pod = {'subpods': []} + + try: + resp = http_get(url, timeout=2.0) + + resp_pod = XML(resp.content) + if resp_pod.xpath(success_xpath): + + for subpod in resp_pod: + plaintext = subpod.xpath(plaintext_xpath)[0].text + if plaintext: + pod['subpods'].append({'title': subpod.xpath(title_xpath)[0], + 'plaintext': plaintext}) + elif subpod.xpath(image_xpath): + pod['subpods'].append({'title': subpod.xpath(title_xpath)[0], + 'plaintext': '', + 'img': {'src': subpod.xpath(img_src_xpath)[0], + 'alt': subpod.xpath(img_alt_xpath)[0]}}) + except: + pass + + return pod + + # get response from search-request def response(resp): + results = [] + resp_json = loads(resp.text) if not resp_json['queryresult']['success']: @@ -76,20 +117,45 @@ def response(resp): # TODO handle resp_json['queryresult']['assumptions'] result_chunks = [] + infobox_title = None for pod in resp_json['queryresult']['pods']: pod_title = pod.get('title', '') + if 'subpods' not in pod: - continue + # comment this section if your requests always reach timeout + if pod['async']: + result = get_async_pod(pod['async']) + if result: + pod = result + else: + continue + + # infobox title is input or text content on first pod + if pod_title.startswith('Input') or not infobox_title: + try: + infobox_title = pod['subpods'][0]['plaintext'] + except: + infobox_title = '' + pass + for subpod in pod['subpods']: - if 'img' in subpod: - result_chunks.append(u'

{0}
{2}

' - .format(escape(pod_title or subpod['img']['alt']), - escape(subpod['img']['src']), - escape(subpod['img']['alt']))) + if subpod['plaintext'] != '' and pod_title not in image_pods: + # append unless it's not an actual answer + if subpod['plaintext'] != '(requires interactivity)': + result_chunks.append({'label': pod_title, 'value': subpod['plaintext']}) + + elif 'img' in subpod: + result_chunks.append({'label': pod_title, 'image': subpod['img']}) if not result_chunks: return [] - return [{'url': resp.request.headers['Referer'].decode('utf-8'), - 'title': 'Wolframalpha', - 'content': ''.join(result_chunks)}] + results.append({'infobox': infobox_title, + 'attributes': result_chunks, + 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) + + results.append({'url': resp.request.headers['Referer'], + 'title': 'Wolfram|Alpha', + 'content': infobox_title}) + + return results diff --git a/searx/settings.yml b/searx/settings.yml index 7607ce94d..6e4316b08 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -310,10 +310,10 @@ engines: shortcut : wa # You can use the engine using the official stable API, but you need an API key # See : http://products.wolframalpha.com/api/ - # engine : wolframalpha_api - # api_key: 'apikey' # required! + # engine : wolframalpha_api + # api_key: '5952JX-X52L3VKWT8' # required! engine : wolframalpha_noapi - timeout: 6.0 + timeout: 10.0 categories : science #The blekko technology and team have joined IBM Watson! -> https://blekko.com/