zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

[fix] startpage engine: language/region & time support & fix CAPTCHA

Browse source 
One reason for the often seen CAPTCHA of the startpage requests are the
incomplete requests SearXNG sends to startpage.com.  To avoid CAPTCHA we need to
send a well formed HTTP POST request with a cookie, we need to form a request
that is identical to the request build by startpage.com itself:

- in the cookie the **region** is selected
- in the POST arguments the **language** is selected

Based on the *engine_properties* boilerplate, SearXNG's startpage engine now
implements a `_fetch_engine_properties()` function to fetch regions & languages
from startpage.com.

This patch is a complete new implementation of the request() function, reversed
engineered from the startpage.com page.  The new implementation adds

- time-range support
- save-search support

to the startpage engine which has been missed in the past.

Closes: https://github.com/searxng/searxng/issues/1081
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2022-04-16 15:30:09 +02:00
parent 4a28a5e6f6
commit 9c68980ea1
3 changed files with 317 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
searx/autocomplete.py
View file

@ -11,9 +11,9 @@ from lxml import etree
from httpx import HTTPError
from searx import settings
from searx.data import ENGINES_LANGUAGES
from searx.network import get as http_get
from searx.exceptions import SearxEngineResponseException
from searx.engines import engines
# a _fetch_supported_properites() for XPath engines isn't available right now
# _brave = ENGINES_LANGUAGES['brave'].keys()
@ -103,9 +103,11 @@ def seznam(query, _lang):
def startpage(query, lang):
# startpage autocompleter
lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
engine = engines['startpage']
_, engine_language, _ = engine.get_engine_locale(lang)
url = 'https://startpage.com/suggestions?{query}'
resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': engine_language})))
data = resp.json()
return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

383
searx/data/engines_languages.json
View file

@ -1561,255 +1561,140 @@
"zh-HK"
],
"startpage": {
"af": {
"alias": "afrikaans"
},
"am": {
"alias": "amharic"
},
"ar": {
"alias": "arabic"
},
"az": {
"alias": "azerbaijani"
},
"be": {
"alias": "belarusian"
},
"bg": {
"alias": "bulgarian"
},
"bn": {
"alias": "bengali"
},
"bs": {
"alias": "bosnian"
},
"ca": {
"alias": "catalan"
},
"cs": {
"alias": "czech"
},
"cy": {
"alias": "welsh"
},
"da": {
"alias": "dansk"
},
"de": {
"alias": "deutsch"
},
"el": {
"alias": "greek"
},
"en": {
"alias": "english"
},
"en-GB": {
"alias": "english_uk"
},
"eo": {
"alias": "esperanto"
},
"es": {
"alias": "espanol"
},
"et": {
"alias": "estonian"
},
"eu": {
"alias": "basque"
},
"fa": {
"alias": "persian"
},
"fi": {
"alias": "suomi"
},
"fo": {
"alias": "faroese"
},
"fr": {
"alias": "francais"
},
"fy": {
"alias": "frisian"
},
"ga": {
"alias": "irish"
},
"gd": {
"alias": "gaelic"
},
"gl": {
"alias": "galician"
},
"gu": {
"alias": "gujarati"
},
"he": {
"alias": "hebrew"
},
"hi": {
"alias": "hindi"
},
"hr": {
"alias": "croatian"
},
"hu": {
"alias": "hungarian"
},
"ia": {
"alias": "interlingua"
},
"id": {
"alias": "indonesian"
},
"is": {
"alias": "icelandic"
},
"it": {
"alias": "italiano"
},
"ja": {
"alias": "nihongo"
},
"jv": {
"alias": "javanese"
},
"ka": {
"alias": "georgian"
},
"kn": {
"alias": "kannada"
},
"ko": {
"alias": "hangul"
},
"la": {
"alias": "latin"
},
"lt": {
"alias": "lithuanian"
},
"lv": {
"alias": "latvian"
},
"mai": {
"alias": "bihari"
},
"mk": {
"alias": "macedonian"
},
"ml": {
"alias": "malayalam"
},
"mr": {
"alias": "marathi"
},
"ms": {
"alias": "malay"
},
"mt": {
"alias": "maltese"
},
"nb": {
"alias": "norsk"
},
"ne": {
"alias": "nepali"
},
"nl": {
"alias": "nederlands"
},
"oc": {
"alias": "occitan"
},
"pa": {
"alias": "punjabi"
},
"pl": {
"alias": "polski"
},
"pt": {
"alias": "portugues"
},
"ro": {
"alias": "romanian"
},
"ru": {
"alias": "russian"
},
"si": {
"alias": "sinhalese"
},
"sk": {
"alias": "slovak"
},
"sl": {
"alias": "slovenian"
},
"sq": {
"alias": "albanian"
},
"sr": {
"alias": "serbian"
},
"su": {
"alias": "sudanese"
},
"sv": {
"alias": "svenska"
},
"sw": {
"alias": "swahili"
},
"ta": {
"alias": "tamil"
},
"te": {
"alias": "telugu"
},
"th": {
"alias": "thai"
},
"ti": {
"alias": "tigrinya"
},
"tl": {
"alias": "tagalog"
},
"tr": {
"alias": "turkce"
},
"uk": {
"alias": "ukrainian"
},
"ur": {
"alias": "urdu"
},
"uz": {
"alias": "uzbek"
},
"vi": {
"alias": "vietnamese"
},
"xh": {
"alias": "xhosa"
},
"zh": {
"alias": "jiantizhongwen"
},
"zh-HK": {
"alias": "fantizhengwen"
},
"zh-TW": {
"alias": "fantizhengwen"
},
"zu": {
"alias": "zulu"
}
"languages": {
"af": "afrikaans",
"am": "amharic",
"ar": "arabic",
"az": "azerbaijani",
"be": "belarusian",
"bg": "bulgarian",
"bn": "bengali",
"bs": "bosnian",
"ca": "catalan",
"cs": "czech",
"cy": "welsh",
"da": "dansk",
"de": "deutsch",
"el": "greek",
"en": "english_uk",
"eo": "esperanto",
"es": "espanol",
"et": "estonian",
"eu": "basque",
"fa": "persian",
"fi": "suomi",
"fo": "faroese",
"fr": "francais",
"fy": "frisian",
"ga": "irish",
"gd": "gaelic",
"gl": "galician",
"gu": "gujarati",
"he": "hebrew",
"hi": "hindi",
"hr": "croatian",
"hu": "hungarian",
"ia": "interlingua",
"id": "indonesian",
"is": "icelandic",
"it": "italiano",
"ja": "nihongo",
"jv": "javanese",
"ka": "georgian",
"kn": "kannada",
"ko": "hangul",
"la": "latin",
"lt": "lithuanian",
"lv": "latvian",
"mai": "bihari",
"mk": "macedonian",
"ml": "malayalam",
"mr": "marathi",
"ms": "malay",
"mt": "maltese",
"nb": "norsk",
"ne": "nepali",
"nl": "nederlands",
"oc": "occitan",
"pa": "punjabi",
"pl": "polski",
"pt": "portugues",
"ro": "romanian",
"ru": "russian",
"si": "sinhalese",
"sk": "slovak",
"sl": "slovenian",
"sq": "albanian",
"sr": "serbian",
"su": "sudanese",
"sv": "svenska",
"sw": "swahili",
"ta": "tamil",
"te": "telugu",
"th": "thai",
"ti": "tigrinya",
"tl": "tagalog",
"tr": "turkce",
"uk": "ukrainian",
"ur": "urdu",
"uz": "uzbek",
"vi": "vietnamese",
"xh": "xhosa",
"zh": "jiantizhongwen",
"zh_Hant": "fantizhengwen",
"zu": "zulu"
},
"regions": {
"ar-EG": "ar_EG",
"bg-BG": "bg_BG",
"ca-ES": "ca_ES",
"cs-CZ": "cs_CZ",
"da-DK": "da_DK",
"de-AT": "de_AT",
"de-CH": "de_CH",
"de-DE": "de_DE",
"el-GR": "el_GR",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en-GB_GB",
"en-IE": "en_IE",
"en-MY": "en_MY",
"en-NZ": "en_NZ",
"en-US": "en_US",
"en-ZA": "en_ZA",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-ES": "es_ES",
"es-US": "es_US",
"es-UY": "es_UY",
"fi-FI": "fi_FI",
"fil-PH": "fil_PH",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"fr-FR": "fr_FR",
"hi-IN": "hi_IN",
"it-CH": "it_CH",
"it-IT": "it_IT",
"ja-JP": "ja_JP",
"ko-KR": "ko_KR",
"ms-MY": "ms_MY",
"nb-NO": "no_NO",
"nl-BE": "nl_BE",
"nl-NL": "nl_NL",
"pl-PL": "pl_PL",
"pt-BR": "pt-BR_BR",
"pt-PT": "pt_PT",
"ro-RO": "ro_RO",
"ru-BY": "ru_BY",
"ru-RU": "ru_RU",
"sv-SE": "sv_SE",
"tr-TR": "tr_TR",
"zh-CN": "zh-CN_CN",
"zh-HK": "zh-TW_HK",
"zh-TW": "zh-TW_TW"
},
"type": "engine_properties"
},
"wikidata": {
"ab": {

250
searx/engines/startpage.py
View file

@ -7,17 +7,17 @@
import re
from time import time
from urllib.parse import urlencode
from unicodedata import normalize, combining
from datetime import datetime, timedelta
from collections import OrderedDict
from dateutil import parser
from lxml import html
from babel import Locale
from babel.localedata import locale_identifiers
import babel
from searx.network import get
from searx.utils import extract_text, eval_xpath, match_language
from searx.utils import extract_text, eval_xpath
from searx.exceptions import (
SearxEngineResponseException,
SearxEngineCaptchaException,
@ -36,16 +36,21 @@ about = {
# engine dependent config
categories = ['general', 'web']
# there is a mechanism to block "bot" search
# (probably the parameter qid), require
# storing of qid's between mulitble search-calls
paging = True
supported_languages_url = 'https://www.startpage.com/do/settings'
number_of_results = 5
safesearch = True
filter_mapping = {0: '0', 1: '1', 2: '1'}
time_range_support = True
time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
supported_properties_url = 'https://www.startpage.com/do/settings'
# search-url
base_url = 'https://startpage.com/'
search_url = base_url + 'sp/search?'
base_url = 'https://www.startpage.com/'
search_url = base_url + 'sp/search'
# specific xpath variables
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
@ -104,42 +109,99 @@ def get_sc_code(headers):
return sc_code
# do search-request
def get_engine_locale(language):
if language == 'all':
language = 'en-US'
locale = babel.Locale.parse(language, sep='-')
engine_language = supported_properties['languages'].get(locale.language)
if not engine_language:
logger.debug("startpage does NOT support language: %s", locale.language)
engine_region = None
if locale.territory:
engine_region = supported_properties['regions'].get(locale.language + '-' + locale.territory)
if not engine_region:
logger.debug("no region in selected (only lang: '%s'), using region 'all'", language)
engine_region = 'all'
logger.debug(
"UI language: %s --> engine language: %s // engine region: %s",
language, engine_language, engine_region
)
return locale, engine_language, engine_region
def request(query, params):
# pylint: disable=line-too-long
# The format string from Startpage's FFox add-on [1]::
#
# https://www.startpage.com/do/dsearch?query={searchTerms}&cat=web&pl=ext-ff&language=__MSG_extensionUrlLanguage__&extVersion=1.3.0
#
# [1] https://addons.mozilla.org/en-US/firefox/addon/startpage-private-search/
locale, engine_language, engine_region = get_engine_locale(params['language'])
# prepare HTTP headers
ac_lang = locale.language
if locale.territory:
ac_lang = "%s-%s,%s;q=0.5" % (locale.language, locale.territory, locale.language)
logger.debug("headers.Accept-Language --> %s", ac_lang)
params['headers']['Accept-Language'] = ac_lang
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
# build arguments
args = {
'query': query,
'page': params['pageno'],
'cat': 'web',
# 'pl': 'ext-ff',
# 'extVersion': '1.3.0',
# 'abp': "-1",
'sc': get_sc_code(params['headers']),
't': 'device',
'sc': get_sc_code(params['headers']), # hint: this func needs HTTP headers
'with_date' : time_range_dict.get(params['time_range'], '')
}
# set language if specified
if params['language'] != 'all':
lang_code = match_language(params['language'], supported_languages, fallback=None)
if lang_code:
language_name = supported_languages[lang_code]['alias']
args['language'] = language_name
args['lui'] = language_name
if engine_language:
args['language'] = engine_language
args['lui'] = engine_language
if params['pageno'] == 1:
args['abp'] = ['-1', '-1']
else:
args['page'] = params['pageno']
args['abp'] = '-1'
# build cookie
lang_homepage = 'english'
cookie = OrderedDict()
cookie['date_time'] = 'world'
cookie['disable_family_filter'] = filter_mapping[params['safesearch']]
cookie['disable_open_in_new_window'] = '0'
cookie['enable_post_method'] = '1' # hint: POST
cookie['enable_proxy_safety_suggest'] = '1'
cookie['enable_stay_control'] = '1'
cookie['instant_answers'] = '1'
cookie['lang_homepage'] = 's/device/%s/' % lang_homepage
cookie['num_of_results'] = '10'
cookie['suggestions'] = '1'
cookie['wt_unit'] = 'celsius'
if engine_language:
cookie['language'] = engine_language
cookie['language_ui'] = engine_language
if engine_region:
cookie['search_results_region'] = engine_region
params['cookies']['preferences'] = 'N1N'.join([ "%sEEE%s" % x for x in cookie.items() ])
logger.debug('cookie preferences: %s', params['cookies']['preferences'])
params['method'] = 'POST'
logger.debug("data: %s", args)
params['data'] = args
params['url'] = search_url
params['url'] = search_url + urlencode(args)
return params
# get response from search-request
def response(resp):
results = []
dom = html.fromstring(resp.text)
# parse results
@ -201,62 +263,102 @@ def response(resp):
return results
# get supported languages from their site
def _fetch_supported_languages(resp):
# startpage's language selector is a mess each option has a displayed name
# and a value, either of which may represent the language name in the native
# script, the language name in English, an English transliteration of the
# native name, the English name of the writing script used by the language,
# or occasionally something else entirely.
def _fetch_engine_properties(resp, engine_properties):
# this cases are so special they need to be hardcoded, a couple of them are mispellings
language_names = {
'english_uk': 'en-GB',
'fantizhengwen': ['zh-TW', 'zh-HK'],
'hangul': 'ko',
'malayam': 'ml',
'norsk': 'nb',
'sinhalese': 'si',
'sudanese': 'su',
}
# startpage's language & region selectors are a mess.
#
# regions:
# in the list of regions there are tags we need to map to common
# region tags:
# - pt-BR_BR --> pt_BR
# - zh-CN_CN --> zh_Hans_CN
# - zh-TW_TW --> zh_Hant_TW
# - zh-TW_HK --> zh_Hant_HK
# - en-GB_GB --> en_GB
# and there is at least one tag with a three letter language tag (ISO 639-2)
# - fil_PH --> fil_PH
#
# languages:
#
# The displayed name in startpage's settings page depend on the location
# of the IP when the 'Accept-Language' HTTP header is unset (in tha
# language update script we use "en-US,en;q=0.5" to get uniform names
# independent from the IP).
#
# Each option has a displayed name and a value, either of which
# may represent the language name in the native script, the language name
# in English, an English transliteration of the native name, the English
# name of the writing script used by the language, or occasionally
# something else entirely.
# get the English name of every language known by babel
language_names.update(
{
# fmt: off
dom = html.fromstring(resp.text)
# regions
sp_region_names = []
for option in dom.xpath('//form[@name="settings"]//select[@name="search_results_region"]/option'):
sp_region_names.append(option.get('value'))
for sp_region_tag in sp_region_names:
if sp_region_tag == 'all':
continue
if '-' in sp_region_tag:
l, r = sp_region_tag.split('-')
r = r.split('_')[-1]
locale = babel.Locale.parse(l +'_'+ r, sep='_')
else:
locale = babel.Locale.parse(sp_region_tag, sep='_')
region_tag = locale.language + '-' + locale.territory
# print("internal: %s --> engine: %s" % (region_tag, sp_region_tag))
engine_properties['regions'][region_tag] = sp_region_tag
# languages
catalog_engine2code = {
name.lower(): lang_code
# pylint: disable=protected-access
for lang_code, name in Locale('en')._data['languages'].items()
# fmt: on
for lang_code, name in babel.Locale('en').languages.items()
}
)
# get the native name of every language known by babel
for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()):
native_name = Locale(lang_code).get_language_name().lower()
for lang_code in filter(
lambda lang_code: lang_code.find('_') == -1,
babel.localedata.locale_identifiers()
):
native_name = babel.Locale(lang_code).get_language_name().lower()
# add native name exactly as it is
language_names[native_name] = lang_code
catalog_engine2code[native_name] = lang_code
# add "normalized" language name (i.e. français becomes francais and español becomes espanol)
unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name)))
if len(unaccented_name) == len(unaccented_name.encode()):
# add only if result is ascii (otherwise "normalization" didn't work)
language_names[unaccented_name] = lang_code
catalog_engine2code[unaccented_name] = lang_code
# values that can't be determined by babel's languages names
catalog_engine2code.update({
'english_uk': 'en',
# traditional chinese used in ..
'fantizhengwen': 'zh_Hant',
# Korean alphabet
'hangul': 'ko',
# Malayalam is one of 22 scheduled languages of India.
'malayam': 'ml',
'norsk': 'nb',
'sinhalese': 'si',
})
dom = html.fromstring(resp.text)
sp_lang_names = []
for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'):
sp_lang_names.append((option.get('value'), extract_text(option).lower()))
engine_lang = option.get('value')
name = extract_text(option).lower()
supported_languages = {}
for sp_option_value, sp_option_text in sp_lang_names:
lang_code = language_names.get(sp_option_value) or language_names.get(sp_option_text)
if isinstance(lang_code, str):
supported_languages[lang_code] = {'alias': sp_option_value}
elif isinstance(lang_code, list):
for _lc in lang_code:
supported_languages[_lc] = {'alias': sp_option_value}
else:
print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text))
lang_code = catalog_engine2code.get(engine_lang)
if lang_code is None:
lang_code = catalog_engine2code[name]
return supported_languages
# print("internal: %s --> engine: %s" % (lang_code, engine_lang))
engine_properties['languages'][lang_code] = engine_lang
return engine_properties