From de179ecc5b8a966f0a46d266c1a04bb2865b27c4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 28 May 2020 15:17:59 +0200 Subject: [PATCH 1/6] [fix] remove debug print from commit e5305f8 Signed-off-by: Markus Heiser --- searx/engines/gigablast.py | 1 - 1 file changed, 1 deletion(-) diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 2bb29a9fe..607bbfcb3 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -62,7 +62,6 @@ def init(engine_settings=None): # do search-request def request(query, params): - print("EXTRAPARAM:", extra_param) offset = (params['pageno'] - 1) * number_of_results if params['language'] == 'all': From 151038255810ed972187ee5d7921e3f044557665 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 28 May 2020 15:18:47 +0200 Subject: [PATCH 2/6] utils/fetch_languages.py: print more meaningfull messages Signed-off-by: Markus Heiser --- utils/fetch_languages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py index ddebb4111..77ec0bf20 100644 --- a/utils/fetch_languages.py +++ b/utils/fetch_languages.py @@ -28,10 +28,10 @@ def fetch_supported_languages(): names.sort() for engine_name in names: - print("fetching languages of engine %s" % engine_name) - if hasattr(engines[engine_name], 'fetch_supported_languages'): engines_languages[engine_name] = engines[engine_name].fetch_supported_languages() + print("fetched %s languages from engine %s" % ( + len(engines_languages[engine_name]), engine_name)) if type(engines_languages[engine_name]) == list: engines_languages[engine_name] = sorted(engines_languages[engine_name]) From 57c7b90edd30ad0d24231a7e338c770c09de2dd4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 28 May 2020 15:21:01 +0200 Subject: [PATCH 3/6] [fix] gigablast does no longer support *supported_languages_url* Since there are zero results, we can remove it: $ make engines.languages fetch languages .. ... fetched 0 languages from engine gigablast Signed-off-by: Markus Heiser --- searx/engines/gigablast.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 607bbfcb3..a71dd1161 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -41,8 +41,6 @@ url_xpath = './/url' title_xpath = './/title' content_xpath = './/sum' -supported_languages_url = 'https://gigablast.com/search?&rxikd=1' - extra_param = '' # gigablast requires a random extra parameter # which can be extracted from the source code of the search page @@ -108,21 +106,3 @@ def response(resp): # return results return results - - -# get supported languages from their site -def _fetch_supported_languages(resp): - supported_languages = [] - dom = fromstring(resp.text) - links = eval_xpath(dom, '//span[@id="menu2"]/a') - for link in links: - href = eval_xpath(link, './@href')[0].split('lang%3A') - if len(href) == 2: - code = href[1].split('_') - if len(code) == 2: - code = code[0] + '-' + code[1].upper() - else: - code = code[0] - supported_languages.append(code) - - return supported_languages From a18760b322ba0006672b3d8f09362b407b39b4c1 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 28 May 2020 19:24:03 +0200 Subject: [PATCH 4/6] [fix] revise of the gigablast engine (WIP) The gigablast API has changed and seems to have some quirks, this is the first revise. More work (hacks) are needed. Signed-off-by: Markus Heiser --- searx/engines/gigablast.py | 127 +++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 68 deletions(-) diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index a71dd1161..37221c556 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -9,14 +9,11 @@ @stable yes @parse url, title, content """ +# pylint: disable=missing-function-docstring, invalid-name -import random -from json import loads from time import time -from lxml.html import fromstring -from searx.poolrequests import get +from json import loads from searx.url_utils import urlencode -from searx.utils import eval_xpath # engine dependent config categories = ['general'] @@ -26,83 +23,77 @@ language_support = True safesearch = True # search-url + base_url = 'https://gigablast.com/' -search_string = 'search?{query}'\ - '&n={number_of_results}'\ - '&c=main'\ - '&s={offset}'\ - '&format=json'\ - '&langcountry={lang}'\ - '&ff={safesearch}'\ - '&rand={rxikd}' -# specific xpath variables -results_xpath = '//response//result' -url_xpath = './/url' -title_xpath = './/title' -content_xpath = './/sum' - -extra_param = '' # gigablast requires a random extra parameter -# which can be extracted from the source code of the search page - - -def parse_extra_param(text): - global extra_param - param_lines = [x for x in text.splitlines() if x.startswith('var url=') or x.startswith('url=url+')] - extra_param = '' - for l in param_lines: - extra_param += l.split("'")[1] - extra_param = extra_param.split('&')[-1] - - -def init(engine_settings=None): - parse_extra_param(get('http://gigablast.com/search?c=main&qlangcountry=en-us&q=south&s=10').text) - # do search-request -def request(query, params): - offset = (params['pageno'] - 1) * number_of_results +def request(query, params): # pylint: disable=unused-argument - if params['language'] == 'all': - language = 'xx' - else: - language = params['language'].replace('-', '_').lower() - if language.split('-')[0] != 'zh': - language = language.split('-')[0] + # see API http://www.gigablast.com/api.html#/search + # Take into account, that the API has some quirks .. + + query_args = dict( + c = 'main' + , n = number_of_results + , format = 'json' + , q = query + # The gigablast HTTP client sends a random number and a nsga argument + # (the values don't seem to matter) + , rand = int(time() * 1000) + , nsga = int(time() * 1000) + ) + + page_no = (params['pageno'] - 1) + if page_no: + # API quirk; adds +2 to the number_of_results + offset = (params['pageno'] - 1) * number_of_results + query_args['s'] = offset + + if params['language'] and params['language'] != 'all': + query_args['qlangcountry'] = params['language'] + query_args['qlang'] = params['language'].split('-')[0] if params['safesearch'] >= 1: - safesearch = 1 - else: - safesearch = 0 + query_args['ff'] = 1 - # rxieu is some kind of hash from the search query, but accepts random atm - search_path = search_string.format(query=urlencode({'q': query}), - offset=offset, - number_of_results=number_of_results, - lang=language, - rxikd=int(time() * 1000), - safesearch=safesearch) - - params['url'] = base_url + search_path + '&' + extra_param + search_url = 'search?' + urlencode(query_args) + params['url'] = base_url + search_url return params - # get response from search-request def response(resp): results = [] - # parse results - try: - response_json = loads(resp.text) - except: - parse_extra_param(resp.text) - raise Exception('extra param expired, please reload') - + response_json = loads(resp.text) for result in response_json['results']: - # append result - results.append({'url': result['url'], - 'title': result['title'], - 'content': result['sum']}) + # see "Example JSON Output (&format=json)" + # at http://www.gigablast.com/api.html#/search + + # sort out meaningless result + + title = result.get('title') + if len(title) < 2: + continue + + url = result.get('url') + if len(url) < 9: + continue + + content = result.get('sum') + if len(content) < 5: + continue + + # extend fields + + subtitle = result.get('title') + if len(subtitle) > 3: + title += " - " + subtitle + + results.append(dict( + url = url + , title = title + , content = content + )) - # return results return results From d44b4eda645446293cae664b8df823181a432c2e Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 29 May 2020 08:52:11 +0200 Subject: [PATCH 5/6] [fix] gigablaste engine: switch from pep8 to pylint test Signed-off-by: Markus Heiser --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index abd0eddb8..494ff401b 100644 --- a/Makefile +++ b/Makefile @@ -195,6 +195,7 @@ test.pylint: pyenvinstall $(call cmd,pylint,\ searx/preferences.py \ searx/testing.py \ + searx/engines/gigablast.py \ ) endif @@ -204,7 +205,7 @@ endif test.pep8: pyenvinstall @echo "TEST pep8" - $(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests + $(Q)$(PY_ENV_ACT); pep8 --exclude='searx/static, searx/engines/gigablast.py' --max-line-length=120 --ignore "E402,W503" searx tests test.unit: pyenvinstall @echo "TEST tests/unit" From ee5d2b319b878e34308cf0761375ca1481edf473 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 29 May 2020 12:48:58 +0200 Subject: [PATCH 6/6] [fix] gigablast requires a random extra parameter Signed-off-by: Markus Heiser --- searx/engines/gigablast.py | 63 +++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 37221c556..b139c2a9f 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later """ Gigablast (Web) @@ -11,20 +12,52 @@ """ # pylint: disable=missing-function-docstring, invalid-name -from time import time +import re from json import loads +# from searx import logger from searx.url_utils import urlencode +from searx.poolrequests import get # engine dependent config categories = ['general'] -paging = True -number_of_results = 10 +# gigablast's pagination is totally damaged, don't use it +paging = False language_support = True safesearch = True # search-url +base_url = 'https://gigablast.com' + +# ugly hack: gigablast requires a random extra parameter which can be extracted +# from the source code of the gigablast HTTP client +extra_param = '' +extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10' + +def parse_extra_param(text): + + # example: + # + # var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n'; + # uxrl=uxrl+'sab=730863287'; + # + # extra_param --> "rand=1590740241635&nsab=730863287" + + global extra_param # pylint: disable=global-statement + re_var= None + for line in text.splitlines(): + if re_var is None and extra_param_path in line: + var = line.split("=")[0].split()[1] # e.g. var --> 'uxrl' + re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)") + extra_param = line.split("'")[1][len(extra_param_path):] + continue + if re_var is not None and re_var.search(line): + extra_param += re_var.search(line).group(1) + break + # logger.debug('gigablast extra_param="%s"', extra_param) + +def init(engine_settings=None): # pylint: disable=unused-argument + parse_extra_param(get(base_url + extra_param_path).text) -base_url = 'https://gigablast.com/' # do search-request def request(query, params): # pylint: disable=unused-argument @@ -34,21 +67,12 @@ def request(query, params): # pylint: disable=unused-argument query_args = dict( c = 'main' - , n = number_of_results , format = 'json' , q = query - # The gigablast HTTP client sends a random number and a nsga argument - # (the values don't seem to matter) - , rand = int(time() * 1000) - , nsga = int(time() * 1000) + , dr = 1 + , showgoodimages = 0 ) - page_no = (params['pageno'] - 1) - if page_no: - # API quirk; adds +2 to the number_of_results - offset = (params['pageno'] - 1) * number_of_results - query_args['s'] = offset - if params['language'] and params['language'] != 'all': query_args['qlangcountry'] = params['language'] query_args['qlang'] = params['language'].split('-')[0] @@ -56,8 +80,8 @@ def request(query, params): # pylint: disable=unused-argument if params['safesearch'] >= 1: query_args['ff'] = 1 - search_url = 'search?' + urlencode(query_args) - params['url'] = base_url + search_url + search_url = '/search?' + urlencode(query_args) + params['url'] = base_url + search_url + extra_param return params @@ -66,6 +90,9 @@ def response(resp): results = [] response_json = loads(resp.text) + + # logger.debug('gigablast returns %s results', len(response_json['results'])) + for result in response_json['results']: # see "Example JSON Output (&format=json)" # at http://www.gigablast.com/api.html#/search @@ -87,7 +114,7 @@ def response(resp): # extend fields subtitle = result.get('title') - if len(subtitle) > 3: + if len(subtitle) > 3 and subtitle != title: title += " - " + subtitle results.append(dict(