From 9c68980ea1b7092a81a10252955b65e3529956fa Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 16 Apr 2022 15:30:09 +0200 Subject: [PATCH] [fix] startpage engine: language/region & time support & fix CAPTCHA One reason for the often seen CAPTCHA of the startpage requests are the incomplete requests SearXNG sends to startpage.com. To avoid CAPTCHA we need to send a well formed HTTP POST request with a cookie, we need to form a request that is identical to the request build by startpage.com itself: - in the cookie the **region** is selected - in the POST arguments the **language** is selected Based on the *engine_properties* boilerplate, SearXNG's startpage engine now implements a `_fetch_engine_properties()` function to fetch regions & languages from startpage.com. This patch is a complete new implementation of the request() function, reversed engineered from the startpage.com page. The new implementation adds - time-range support - save-search support to the startpage engine which has been missed in the past. Closes: https://github.com/searxng/searxng/issues/1081 Signed-off-by: Markus Heiser --- searx/autocomplete.py | 8 +- searx/data/engines_languages.json | 383 +++++++++++------------------- searx/engines/startpage.py | 254 ++++++++++++++------ 3 files changed, 317 insertions(+), 328 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 014f81685..f8a482deb 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -11,9 +11,9 @@ from lxml import etree from httpx import HTTPError from searx import settings -from searx.data import ENGINES_LANGUAGES from searx.network import get as http_get from searx.exceptions import SearxEngineResponseException +from searx.engines import engines # a _fetch_supported_properites() for XPath engines isn't available right now # _brave = ENGINES_LANGUAGES['brave'].keys() @@ -103,9 +103,11 @@ def seznam(query, _lang): def startpage(query, lang): # startpage autocompleter - lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english') + engine = engines['startpage'] + _, engine_language, _ = engine.get_engine_locale(lang) + url = 'https://startpage.com/suggestions?{query}' - resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui}))) + resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': engine_language}))) data = resp.json() return [e['text'] for e in data.get('suggestions', []) if 'text' in e] diff --git a/searx/data/engines_languages.json b/searx/data/engines_languages.json index fe648499a..5ea3044c3 100644 --- a/searx/data/engines_languages.json +++ b/searx/data/engines_languages.json @@ -1561,255 +1561,140 @@ "zh-HK" ], "startpage": { - "af": { - "alias": "afrikaans" - }, - "am": { - "alias": "amharic" - }, - "ar": { - "alias": "arabic" - }, - "az": { - "alias": "azerbaijani" - }, - "be": { - "alias": "belarusian" - }, - "bg": { - "alias": "bulgarian" - }, - "bn": { - "alias": "bengali" - }, - "bs": { - "alias": "bosnian" - }, - "ca": { - "alias": "catalan" - }, - "cs": { - "alias": "czech" - }, - "cy": { - "alias": "welsh" - }, - "da": { - "alias": "dansk" - }, - "de": { - "alias": "deutsch" - }, - "el": { - "alias": "greek" - }, - "en": { - "alias": "english" - }, - "en-GB": { - "alias": "english_uk" - }, - "eo": { - "alias": "esperanto" - }, - "es": { - "alias": "espanol" - }, - "et": { - "alias": "estonian" - }, - "eu": { - "alias": "basque" - }, - "fa": { - "alias": "persian" - }, - "fi": { - "alias": "suomi" - }, - "fo": { - "alias": "faroese" - }, - "fr": { - "alias": "francais" - }, - "fy": { - "alias": "frisian" - }, - "ga": { - "alias": "irish" - }, - "gd": { - "alias": "gaelic" - }, - "gl": { - "alias": "galician" - }, - "gu": { - "alias": "gujarati" - }, - "he": { - "alias": "hebrew" - }, - "hi": { - "alias": "hindi" - }, - "hr": { - "alias": "croatian" - }, - "hu": { - "alias": "hungarian" - }, - "ia": { - "alias": "interlingua" - }, - "id": { - "alias": "indonesian" - }, - "is": { - "alias": "icelandic" - }, - "it": { - "alias": "italiano" - }, - "ja": { - "alias": "nihongo" - }, - "jv": { - "alias": "javanese" - }, - "ka": { - "alias": "georgian" - }, - "kn": { - "alias": "kannada" - }, - "ko": { - "alias": "hangul" - }, - "la": { - "alias": "latin" - }, - "lt": { - "alias": "lithuanian" - }, - "lv": { - "alias": "latvian" - }, - "mai": { - "alias": "bihari" - }, - "mk": { - "alias": "macedonian" - }, - "ml": { - "alias": "malayalam" - }, - "mr": { - "alias": "marathi" - }, - "ms": { - "alias": "malay" - }, - "mt": { - "alias": "maltese" - }, - "nb": { - "alias": "norsk" - }, - "ne": { - "alias": "nepali" - }, - "nl": { - "alias": "nederlands" - }, - "oc": { - "alias": "occitan" - }, - "pa": { - "alias": "punjabi" - }, - "pl": { - "alias": "polski" - }, - "pt": { - "alias": "portugues" - }, - "ro": { - "alias": "romanian" - }, - "ru": { - "alias": "russian" - }, - "si": { - "alias": "sinhalese" - }, - "sk": { - "alias": "slovak" - }, - "sl": { - "alias": "slovenian" - }, - "sq": { - "alias": "albanian" - }, - "sr": { - "alias": "serbian" - }, - "su": { - "alias": "sudanese" - }, - "sv": { - "alias": "svenska" - }, - "sw": { - "alias": "swahili" - }, - "ta": { - "alias": "tamil" - }, - "te": { - "alias": "telugu" - }, - "th": { - "alias": "thai" - }, - "ti": { - "alias": "tigrinya" - }, - "tl": { - "alias": "tagalog" - }, - "tr": { - "alias": "turkce" - }, - "uk": { - "alias": "ukrainian" - }, - "ur": { - "alias": "urdu" - }, - "uz": { - "alias": "uzbek" - }, - "vi": { - "alias": "vietnamese" - }, - "xh": { - "alias": "xhosa" - }, - "zh": { - "alias": "jiantizhongwen" - }, - "zh-HK": { - "alias": "fantizhengwen" - }, - "zh-TW": { - "alias": "fantizhengwen" - }, - "zu": { - "alias": "zulu" - } + "languages": { + "af": "afrikaans", + "am": "amharic", + "ar": "arabic", + "az": "azerbaijani", + "be": "belarusian", + "bg": "bulgarian", + "bn": "bengali", + "bs": "bosnian", + "ca": "catalan", + "cs": "czech", + "cy": "welsh", + "da": "dansk", + "de": "deutsch", + "el": "greek", + "en": "english_uk", + "eo": "esperanto", + "es": "espanol", + "et": "estonian", + "eu": "basque", + "fa": "persian", + "fi": "suomi", + "fo": "faroese", + "fr": "francais", + "fy": "frisian", + "ga": "irish", + "gd": "gaelic", + "gl": "galician", + "gu": "gujarati", + "he": "hebrew", + "hi": "hindi", + "hr": "croatian", + "hu": "hungarian", + "ia": "interlingua", + "id": "indonesian", + "is": "icelandic", + "it": "italiano", + "ja": "nihongo", + "jv": "javanese", + "ka": "georgian", + "kn": "kannada", + "ko": "hangul", + "la": "latin", + "lt": "lithuanian", + "lv": "latvian", + "mai": "bihari", + "mk": "macedonian", + "ml": "malayalam", + "mr": "marathi", + "ms": "malay", + "mt": "maltese", + "nb": "norsk", + "ne": "nepali", + "nl": "nederlands", + "oc": "occitan", + "pa": "punjabi", + "pl": "polski", + "pt": "portugues", + "ro": "romanian", + "ru": "russian", + "si": "sinhalese", + "sk": "slovak", + "sl": "slovenian", + "sq": "albanian", + "sr": "serbian", + "su": "sudanese", + "sv": "svenska", + "sw": "swahili", + "ta": "tamil", + "te": "telugu", + "th": "thai", + "ti": "tigrinya", + "tl": "tagalog", + "tr": "turkce", + "uk": "ukrainian", + "ur": "urdu", + "uz": "uzbek", + "vi": "vietnamese", + "xh": "xhosa", + "zh": "jiantizhongwen", + "zh_Hant": "fantizhengwen", + "zu": "zulu" + }, + "regions": { + "ar-EG": "ar_EG", + "bg-BG": "bg_BG", + "ca-ES": "ca_ES", + "cs-CZ": "cs_CZ", + "da-DK": "da_DK", + "de-AT": "de_AT", + "de-CH": "de_CH", + "de-DE": "de_DE", + "el-GR": "el_GR", + "en-AU": "en_AU", + "en-CA": "en_CA", + "en-GB": "en-GB_GB", + "en-IE": "en_IE", + "en-MY": "en_MY", + "en-NZ": "en_NZ", + "en-US": "en_US", + "en-ZA": "en_ZA", + "es-AR": "es_AR", + "es-CL": "es_CL", + "es-ES": "es_ES", + "es-US": "es_US", + "es-UY": "es_UY", + "fi-FI": "fi_FI", + "fil-PH": "fil_PH", + "fr-BE": "fr_BE", + "fr-CA": "fr_CA", + "fr-CH": "fr_CH", + "fr-FR": "fr_FR", + "hi-IN": "hi_IN", + "it-CH": "it_CH", + "it-IT": "it_IT", + "ja-JP": "ja_JP", + "ko-KR": "ko_KR", + "ms-MY": "ms_MY", + "nb-NO": "no_NO", + "nl-BE": "nl_BE", + "nl-NL": "nl_NL", + "pl-PL": "pl_PL", + "pt-BR": "pt-BR_BR", + "pt-PT": "pt_PT", + "ro-RO": "ro_RO", + "ru-BY": "ru_BY", + "ru-RU": "ru_RU", + "sv-SE": "sv_SE", + "tr-TR": "tr_TR", + "zh-CN": "zh-CN_CN", + "zh-HK": "zh-TW_HK", + "zh-TW": "zh-TW_TW" + }, + "type": "engine_properties" }, "wikidata": { "ab": { diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 7b87808b9..fe4d54682 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -7,17 +7,17 @@ import re from time import time -from urllib.parse import urlencode from unicodedata import normalize, combining from datetime import datetime, timedelta +from collections import OrderedDict from dateutil import parser from lxml import html -from babel import Locale -from babel.localedata import locale_identifiers + +import babel from searx.network import get -from searx.utils import extract_text, eval_xpath, match_language +from searx.utils import extract_text, eval_xpath from searx.exceptions import ( SearxEngineResponseException, SearxEngineCaptchaException, @@ -36,16 +36,21 @@ about = { # engine dependent config categories = ['general', 'web'] -# there is a mechanism to block "bot" search -# (probably the parameter qid), require -# storing of qid's between mulitble search-calls paging = True -supported_languages_url = 'https://www.startpage.com/do/settings' +number_of_results = 5 + +safesearch = True +filter_mapping = {0: '0', 1: '1', 2: '1'} + +time_range_support = True +time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'} + +supported_properties_url = 'https://www.startpage.com/do/settings' # search-url -base_url = 'https://startpage.com/' -search_url = base_url + 'sp/search?' +base_url = 'https://www.startpage.com/' +search_url = base_url + 'sp/search' # specific xpath variables # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"] @@ -104,42 +109,99 @@ def get_sc_code(headers): return sc_code -# do search-request +def get_engine_locale(language): + + if language == 'all': + language = 'en-US' + locale = babel.Locale.parse(language, sep='-') + + engine_language = supported_properties['languages'].get(locale.language) + if not engine_language: + logger.debug("startpage does NOT support language: %s", locale.language) + + engine_region = None + if locale.territory: + engine_region = supported_properties['regions'].get(locale.language + '-' + locale.territory) + if not engine_region: + logger.debug("no region in selected (only lang: '%s'), using region 'all'", language) + engine_region = 'all' + + logger.debug( + "UI language: %s --> engine language: %s // engine region: %s", + language, engine_language, engine_region + ) + return locale, engine_language, engine_region + + def request(query, params): - # pylint: disable=line-too-long - # The format string from Startpage's FFox add-on [1]:: - # - # https://www.startpage.com/do/dsearch?query={searchTerms}&cat=web&pl=ext-ff&language=__MSG_extensionUrlLanguage__&extVersion=1.3.0 - # - # [1] https://addons.mozilla.org/en-US/firefox/addon/startpage-private-search/ + locale, engine_language, engine_region = get_engine_locale(params['language']) + # prepare HTTP headers + ac_lang = locale.language + if locale.territory: + ac_lang = "%s-%s,%s;q=0.5" % (locale.language, locale.territory, locale.language) + logger.debug("headers.Accept-Language --> %s", ac_lang) + params['headers']['Accept-Language'] = ac_lang + params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' + + # build arguments args = { 'query': query, - 'page': params['pageno'], 'cat': 'web', - # 'pl': 'ext-ff', - # 'extVersion': '1.3.0', - # 'abp': "-1", - 'sc': get_sc_code(params['headers']), + 't': 'device', + 'sc': get_sc_code(params['headers']), # hint: this func needs HTTP headers + 'with_date' : time_range_dict.get(params['time_range'], '') } - # set language if specified - if params['language'] != 'all': - lang_code = match_language(params['language'], supported_languages, fallback=None) - if lang_code: - language_name = supported_languages[lang_code]['alias'] - args['language'] = language_name - args['lui'] = language_name + if engine_language: + args['language'] = engine_language + args['lui'] = engine_language + + if params['pageno'] == 1: + args['abp'] = ['-1', '-1'] + + else: + args['page'] = params['pageno'] + args['abp'] = '-1' + + # build cookie + lang_homepage = 'english' + cookie = OrderedDict() + cookie['date_time'] = 'world' + cookie['disable_family_filter'] = filter_mapping[params['safesearch']] + cookie['disable_open_in_new_window'] = '0' + cookie['enable_post_method'] = '1' # hint: POST + cookie['enable_proxy_safety_suggest'] = '1' + cookie['enable_stay_control'] = '1' + cookie['instant_answers'] = '1' + cookie['lang_homepage'] = 's/device/%s/' % lang_homepage + cookie['num_of_results'] = '10' + cookie['suggestions'] = '1' + cookie['wt_unit'] = 'celsius' + + if engine_language: + cookie['language'] = engine_language + cookie['language_ui'] = engine_language + + if engine_region: + cookie['search_results_region'] = engine_region + + params['cookies']['preferences'] = 'N1N'.join([ "%sEEE%s" % x for x in cookie.items() ]) + logger.debug('cookie preferences: %s', params['cookies']['preferences']) + params['method'] = 'POST' + + logger.debug("data: %s", args) + params['data'] = args + + params['url'] = search_url - params['url'] = search_url + urlencode(args) return params # get response from search-request def response(resp): results = [] - dom = html.fromstring(resp.text) # parse results @@ -201,62 +263,102 @@ def response(resp): return results -# get supported languages from their site -def _fetch_supported_languages(resp): - # startpage's language selector is a mess each option has a displayed name - # and a value, either of which may represent the language name in the native - # script, the language name in English, an English transliteration of the - # native name, the English name of the writing script used by the language, - # or occasionally something else entirely. +def _fetch_engine_properties(resp, engine_properties): - # this cases are so special they need to be hardcoded, a couple of them are mispellings - language_names = { - 'english_uk': 'en-GB', - 'fantizhengwen': ['zh-TW', 'zh-HK'], - 'hangul': 'ko', - 'malayam': 'ml', - 'norsk': 'nb', - 'sinhalese': 'si', - 'sudanese': 'su', + # startpage's language & region selectors are a mess. + # + # regions: + # in the list of regions there are tags we need to map to common + # region tags: + # - pt-BR_BR --> pt_BR + # - zh-CN_CN --> zh_Hans_CN + # - zh-TW_TW --> zh_Hant_TW + # - zh-TW_HK --> zh_Hant_HK + # - en-GB_GB --> en_GB + # and there is at least one tag with a three letter language tag (ISO 639-2) + # - fil_PH --> fil_PH + # + # languages: + # + # The displayed name in startpage's settings page depend on the location + # of the IP when the 'Accept-Language' HTTP header is unset (in tha + # language update script we use "en-US,en;q=0.5" to get uniform names + # independent from the IP). + # + # Each option has a displayed name and a value, either of which + # may represent the language name in the native script, the language name + # in English, an English transliteration of the native name, the English + # name of the writing script used by the language, or occasionally + # something else entirely. + + dom = html.fromstring(resp.text) + + # regions + + sp_region_names = [] + for option in dom.xpath('//form[@name="settings"]//select[@name="search_results_region"]/option'): + sp_region_names.append(option.get('value')) + + for sp_region_tag in sp_region_names: + if sp_region_tag == 'all': + continue + if '-' in sp_region_tag: + l, r = sp_region_tag.split('-') + r = r.split('_')[-1] + locale = babel.Locale.parse(l +'_'+ r, sep='_') + else: + locale = babel.Locale.parse(sp_region_tag, sep='_') + + region_tag = locale.language + '-' + locale.territory + # print("internal: %s --> engine: %s" % (region_tag, sp_region_tag)) + engine_properties['regions'][region_tag] = sp_region_tag + + # languages + + catalog_engine2code = { + name.lower(): lang_code + for lang_code, name in babel.Locale('en').languages.items() } - # get the English name of every language known by babel - language_names.update( - { - # fmt: off - name.lower(): lang_code - # pylint: disable=protected-access - for lang_code, name in Locale('en')._data['languages'].items() - # fmt: on - } - ) - # get the native name of every language known by babel - for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()): - native_name = Locale(lang_code).get_language_name().lower() + + for lang_code in filter( + lambda lang_code: lang_code.find('_') == -1, + babel.localedata.locale_identifiers() + ): + native_name = babel.Locale(lang_code).get_language_name().lower() # add native name exactly as it is - language_names[native_name] = lang_code + catalog_engine2code[native_name] = lang_code # add "normalized" language name (i.e. français becomes francais and español becomes espanol) unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name))) if len(unaccented_name) == len(unaccented_name.encode()): # add only if result is ascii (otherwise "normalization" didn't work) - language_names[unaccented_name] = lang_code + catalog_engine2code[unaccented_name] = lang_code + + # values that can't be determined by babel's languages names + + catalog_engine2code.update({ + 'english_uk': 'en', + # traditional chinese used in .. + 'fantizhengwen': 'zh_Hant', + # Korean alphabet + 'hangul': 'ko', + # Malayalam is one of 22 scheduled languages of India. + 'malayam': 'ml', + 'norsk': 'nb', + 'sinhalese': 'si', + }) - dom = html.fromstring(resp.text) - sp_lang_names = [] for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'): - sp_lang_names.append((option.get('value'), extract_text(option).lower())) + engine_lang = option.get('value') + name = extract_text(option).lower() - supported_languages = {} - for sp_option_value, sp_option_text in sp_lang_names: - lang_code = language_names.get(sp_option_value) or language_names.get(sp_option_text) - if isinstance(lang_code, str): - supported_languages[lang_code] = {'alias': sp_option_value} - elif isinstance(lang_code, list): - for _lc in lang_code: - supported_languages[_lc] = {'alias': sp_option_value} - else: - print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text)) + lang_code = catalog_engine2code.get(engine_lang) + if lang_code is None: + lang_code = catalog_engine2code[name] - return supported_languages + # print("internal: %s --> engine: %s" % (lang_code, engine_lang)) + engine_properties['languages'][lang_code] = engine_lang + + return engine_properties