From 64cccae99e625f3ebd879f94797decd0d824608d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 26 Nov 2020 17:22:54 +0100 Subject: [PATCH] [mod] various engines: use eval_xpath* functions and searx.exceptions.* Engine list: ahmia, duckduckgo_images, elasticsearch, google, google_images, google_videos, youtube_api --- searx/engines/ahmia.py | 14 ++++---- searx/engines/duckduckgo_images.py | 8 ++--- searx/engines/elasticsearch.py | 3 +- searx/engines/google.py | 52 +++++++++++++----------------- searx/engines/google_images.py | 6 ++-- searx/engines/google_videos.py | 10 +++--- searx/engines/youtube_api.py | 3 +- 7 files changed, 44 insertions(+), 52 deletions(-) diff --git a/searx/engines/ahmia.py b/searx/engines/ahmia.py index d9fcc6ca7..7a2ae0075 100644 --- a/searx/engines/ahmia.py +++ b/searx/engines/ahmia.py @@ -12,7 +12,7 @@ from urllib.parse import urlencode, urlparse, parse_qs from lxml.html import fromstring -from searx.engines.xpath import extract_url, extract_text +from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath # engine config categories = ['onions'] @@ -50,17 +50,17 @@ def response(resp): # trim results so there's not way too many at once first_result_index = page_size * (resp.search_params.get('pageno', 1) - 1) - all_results = dom.xpath(results_xpath) + all_results = eval_xpath_list(dom, results_xpath) trimmed_results = all_results[first_result_index:first_result_index + page_size] # get results for result in trimmed_results: # remove ahmia url and extract the actual url for the result - raw_url = extract_url(result.xpath(url_xpath), search_url) + raw_url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url) cleaned_url = parse_qs(urlparse(raw_url).query).get('redirect_url', [''])[0] - title = extract_text(result.xpath(title_xpath)) - content = extract_text(result.xpath(content_xpath)) + title = extract_text(eval_xpath(result, title_xpath)) + content = extract_text(eval_xpath(result, content_xpath)) results.append({'url': cleaned_url, 'title': title, @@ -68,11 +68,11 @@ def response(resp): 'is_onion': True}) # get spelling corrections - for correction in dom.xpath(correction_xpath): + for correction in eval_xpath_list(dom, correction_xpath): results.append({'correction': extract_text(correction)}) # get number of results - number_of_results = dom.xpath(number_of_results_xpath) + number_of_results = eval_xpath(dom, number_of_results_xpath) if number_of_results: try: results.append({'number_of_results': int(extract_text(number_of_results))}) diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py index 438a8d54c..009f81cca 100644 --- a/searx/engines/duckduckgo_images.py +++ b/searx/engines/duckduckgo_images.py @@ -15,6 +15,7 @@ from json import loads from urllib.parse import urlencode +from searx.exceptions import SearxEngineAPIException from searx.engines.duckduckgo import get_region_code from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import from searx.poolrequests import get @@ -37,7 +38,7 @@ def get_vqd(query, headers): res = get(query_url, headers=headers) content = res.text if content.find('vqd=\'') == -1: - raise Exception('Request failed') + raise SearxEngineAPIException('Request failed') vqd = content[content.find('vqd=\'') + 5:] vqd = vqd[:vqd.find('\'')] return vqd @@ -71,10 +72,7 @@ def response(resp): results = [] content = resp.text - try: - res_json = loads(content) - except: - raise Exception('Cannot parse results') + res_json = loads(content) # parse results for result in res_json['results']: diff --git a/searx/engines/elasticsearch.py b/searx/engines/elasticsearch.py index 99e93d876..081736c1c 100644 --- a/searx/engines/elasticsearch.py +++ b/searx/engines/elasticsearch.py @@ -1,5 +1,6 @@ from json import loads, dumps from requests.auth import HTTPBasicAuth +from searx.exceptions import SearxEngineAPIException base_url = 'http://localhost:9200' @@ -107,7 +108,7 @@ def response(resp): resp_json = loads(resp.text) if 'error' in resp_json: - raise Exception(resp_json['error']) + raise SearxEngineAPIException(resp_json['error']) for result in resp_json['hits']['hits']: r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()} diff --git a/searx/engines/google.py b/searx/engines/google.py index 83b18a9a0..17ab21f6a 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -20,9 +20,10 @@ Definitions`_. from urllib.parse import urlencode, urlparse from lxml import html -from flask_babel import gettext from searx import logger -from searx.utils import match_language, extract_text, eval_xpath +from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex +from searx.exceptions import SearxEngineCaptchaException + logger = logger.getChild('google engine') @@ -131,14 +132,6 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a' spelling_suggestion_xpath = '//div[@class="med"]/p/a' -def extract_text_from_dom(result, xpath): - """returns extract_text on the first result selected by the xpath or None""" - r = eval_xpath(result, xpath) - if len(r) > 0: - return extract_text(r[0]) - return None - - def get_lang_country(params, lang_list, custom_aliases): """Returns a tuple with *langauage* on its first and *country* on its second position.""" @@ -210,10 +203,10 @@ def response(resp): # detect google sorry resp_url = urlparse(resp.url) if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect': - raise RuntimeWarning('sorry.google.com') + raise SearxEngineCaptchaException() if resp_url.path.startswith('/sorry'): - raise RuntimeWarning(gettext('CAPTCHA required')) + raise SearxEngineCaptchaException() # which subdomain ? # subdomain = resp.search_params.get('google_subdomain') @@ -229,18 +222,17 @@ def response(resp): logger.debug("did not found 'answer'") # results --> number_of_results - try: - _txt = eval_xpath(dom, '//div[@id="result-stats"]//text()')[0] - _digit = ''.join([n for n in _txt if n.isdigit()]) - number_of_results = int(_digit) - results.append({'number_of_results': number_of_results}) - - except Exception as e: # pylint: disable=broad-except - logger.debug("did not 'number_of_results'") - logger.error(e, exc_info=True) + try: + _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0) + _digit = ''.join([n for n in _txt if n.isdigit()]) + number_of_results = int(_digit) + results.append({'number_of_results': number_of_results}) + except Exception as e: # pylint: disable=broad-except + logger.debug("did not 'number_of_results'") + logger.error(e, exc_info=True) # parse results - for result in eval_xpath(dom, results_xpath): + for result in eval_xpath_list(dom, results_xpath): # google *sections* if extract_text(eval_xpath(result, g_section_with_header)): @@ -248,14 +240,14 @@ def response(resp): continue try: - title_tag = eval_xpath(result, title_xpath) - if not title_tag: + title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None) + if title_tag is None: # this not one of the common google results *section* logger.debug('ingoring
section: missing title') continue - title = extract_text(title_tag[0]) - url = eval_xpath(result, href_xpath)[0] - content = extract_text_from_dom(result, content_xpath) + title = extract_text(title_tag) + url = eval_xpath_getindex(result, href_xpath, 0) + content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True) results.append({ 'url': url, 'title': title, @@ -270,11 +262,11 @@ def response(resp): continue # parse suggestion - for suggestion in eval_xpath(dom, suggestion_xpath): + for suggestion in eval_xpath_list(dom, suggestion_xpath): # append suggestion results.append({'suggestion': extract_text(suggestion)}) - for correction in eval_xpath(dom, spelling_suggestion_xpath): + for correction in eval_xpath_list(dom, spelling_suggestion_xpath): results.append({'correction': extract_text(correction)}) # return results @@ -286,7 +278,7 @@ def _fetch_supported_languages(resp): ret_val = {} dom = html.fromstring(resp.text) - radio_buttons = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]') + radio_buttons = eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]') for x in radio_buttons: name = x.get("data-name") diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index a3daf6070..9ef1be753 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -26,8 +26,8 @@ Definitions`_. from urllib.parse import urlencode, urlparse, unquote from lxml import html -from flask_babel import gettext from searx import logger +from searx.exceptions import SearxEngineCaptchaException from searx.utils import extract_text, eval_xpath from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import @@ -128,10 +128,10 @@ def response(resp): # detect google sorry resp_url = urlparse(resp.url) if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect': - raise RuntimeWarning('sorry.google.com') + raise SearxEngineCaptchaException() if resp_url.path.startswith('/sorry'): - raise RuntimeWarning(gettext('CAPTCHA required')) + raise SearxEngineCaptchaException() # which subdomain ? # subdomain = resp.search_params.get('google_subdomain') diff --git a/searx/engines/google_videos.py b/searx/engines/google_videos.py index 1e6c8b3ee..eedefbf45 100644 --- a/searx/engines/google_videos.py +++ b/searx/engines/google_videos.py @@ -13,7 +13,7 @@ from datetime import date, timedelta from urllib.parse import urlencode from lxml import html -from searx.utils import extract_text +from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex import re # engine dependent config @@ -66,11 +66,11 @@ def response(resp): dom = html.fromstring(resp.text) # parse results - for result in dom.xpath('//div[@class="g"]'): + for result in eval_xpath_list(dom, '//div[@class="g"]'): - title = extract_text(result.xpath('.//h3')) - url = result.xpath('.//div[@class="r"]/a/@href')[0] - content = extract_text(result.xpath('.//span[@class="st"]')) + title = extract_text(eval_xpath(result, './/h3')) + url = eval_xpath_getindex(result, './/div[@class="r"]/a/@href', 0) + content = extract_text(eval_xpath(result, './/span[@class="st"]')) # get thumbnails script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text) diff --git a/searx/engines/youtube_api.py b/searx/engines/youtube_api.py index 4a205ed6c..8c12ac4d2 100644 --- a/searx/engines/youtube_api.py +++ b/searx/engines/youtube_api.py @@ -11,6 +11,7 @@ from json import loads from dateutil import parser from urllib.parse import urlencode +from searx.exceptions import SearxEngineAPIException # engine dependent config categories = ['videos', 'music'] @@ -48,7 +49,7 @@ def response(resp): search_results = loads(resp.text) if 'error' in search_results and 'message' in search_results['error']: - raise Exception(search_results['error']['message']) + raise SearxEngineAPIException(search_results['error']['message']) # return empty array if there are no results if 'items' not in search_results: