From 75bb8c45d0cf5b0607604da63d0b4731e10714d9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 10 Aug 2022 09:45:48 +0200 Subject: [PATCH 1/4] [mod] decouple qwant's categories from SearXNG's categories By using new property `qwant_categ:` the category of qwant is no longer bound to the category of SearXNG. Signed-off-by: Markus Heiser --- searx/engines/qwant.py | 28 ++++++++++------------------ searx/settings.yml | 4 ++++ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py index a1799491a..e8b1d7739 100644 --- a/searx/engines/qwant.py +++ b/searx/engines/qwant.py @@ -9,16 +9,16 @@ https://www.qwant.com/ queries. This implementation is used by different qwant engines in the settings.yml:: - name: qwant - categories: general + qwant_categ: web ... - name: qwant news - categories: news + qwant_categ: news ... - name: qwant images - categories: images + qwant_categ: images ... - name: qwant videos - categories: videos + qwant_categ: videos ... """ @@ -50,13 +50,7 @@ about = { categories = [] paging = True supported_languages_url = about['website'] - -category_to_keyword = { - 'general': 'web', - 'news': 'news', - 'images': 'images', - 'videos': 'videos', -} +qwant_categ = None # web|news|inages|videos # search-url url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}' @@ -64,10 +58,9 @@ url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={o def request(query, params): """Qwant search request""" - keyword = category_to_keyword[categories[0]] count = 10 # web: count must be equal to 10 - if keyword == 'images': + if qwant_categ == 'images': count = 50 offset = (params['pageno'] - 1) * count # count + offset must be lower than 250 @@ -78,7 +71,7 @@ def request(query, params): offset = min(offset, 40) params['url'] = url.format( - keyword=keyword, + keyword=qwant_categ, query=urlencode({'q': query}), offset=offset, count=count, @@ -103,7 +96,6 @@ def response(resp): """Get response from Qwant's search request""" # pylint: disable=too-many-locals, too-many-branches, too-many-statements - keyword = category_to_keyword[categories[0]] results = [] # load JSON result @@ -125,7 +117,7 @@ def response(resp): # raise for other errors raise_for_httperror(resp) - if keyword == 'web': + if qwant_categ == 'web': # The WEB query contains a list named 'mainline'. This list can contain # different result types (e.g. mainline[0]['type'] returns type of the # result items in mainline[0]['items'] @@ -136,7 +128,7 @@ def response(resp): # result['items']. mainline = data.get('result', {}).get('items', []) mainline = [ - {'type': keyword, 'items': mainline}, + {'type': qwant_categ, 'items': mainline}, ] # return empty array if there are no results @@ -146,7 +138,7 @@ def response(resp): for row in mainline: mainline_type = row.get('type', 'web') - if mainline_type != keyword: + if mainline_type != qwant_categ: continue if mainline_type == 'ads': diff --git a/searx/settings.yml b/searx/settings.yml index 6cae30c48..9530a4420 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1198,6 +1198,7 @@ engines: results: HTML - name: qwant + qwant_categ: web engine: qwant shortcut: qw categories: [general, web] @@ -1206,6 +1207,7 @@ engines: rosebud: *test_rosebud - name: qwant news + qwant_categ: news engine: qwant shortcut: qwn categories: news @@ -1213,6 +1215,7 @@ engines: network: qwant - name: qwant images + qwant_categ: images engine: qwant shortcut: qwi categories: [images, web] @@ -1220,6 +1223,7 @@ engines: network: qwant - name: qwant videos + qwant_categ: videos engine: qwant shortcut: qwv categories: [videos, web] From 9ae409a05a0980ae70590303a83d983011831a80 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 12 Aug 2022 17:46:20 +0200 Subject: [PATCH 2/4] [mod] add locale.get_engine_locale to get predictable results The match_language function sometimes returns incorrect results which is why a new function get_engine_locale is required. A bugfix of the match_language is not easily possible, because there is almost no documentation for it and already the call parameters are undefined. E.g. the function processes values like the ones from yahoo:: "yahoo": [ "ar", ... "zh_chs", "zh_cht" ] The get_engine_locale has been documented in detail, there is a clear description of the assumptions as well as the requirements and approximation rules (read doc-string for more details):: Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to corresponding *engine locales*: : { # SearXNG string : engine-string 'ca-ES' : 'ca_ES', 'fr-BE' : 'fr_BE', 'fr-CA' : 'fr_CA', 'fr-CH' : 'fr_CH', 'fr' : 'fr_FR', ... 'pl-PL' : 'pl_PL', 'pt-PT' : 'pt_PT' } .. hint:: The *SearXNG locale* string has to be known by babel! In the following you will find a comparison: >>> import babel.languages >>> from searx.utils import match_language >>> from searx.locales import get_engine_locale Assume we have an engine that supports the follwoing locales: >>> lang_list = { ... "zh-CN": "zh_CN", ... "zh-HK": "zh_HK", ... "nl-BE": "nl_BE", ... "fr-CA": "fr_CA", ... } Assumption: A. When a user selects a language the results should be optimized according to the selected language. B. When user selects a language and a territory the results should be optimized with first priority on territory and second on language. ---- Example: (Assumption A.) A user selects region 'zh-TW' which should end in zh_HK hint: CN is 'Hans' and HK ('Hant') fits better to TW ('Hant') >>> get_engine_locale('zh-TW', lang_list) 'zh_HK' >>> lang_list[match_language('zh-TW', lang_list)] 'zh_CN' ---- Example: (Assumption A.) A user selects only the language 'zh' which should end in CN >>> get_engine_locale('zh', lang_list) 'zh_CN' >>> lang_list[match_language('zh', lang_list)] 'zh_CN' ---- Example: (Assumption B.) A user selects region 'fr-BE' which should end in nl-BE hint: priority should be on the territory the user selected. If the user prefers 'fr' he will select 'fr' without a region tag. >>> get_engine_locale('fr-BE', lang_list, default='unknown') 'nl_BE' >>> match_language('fr-BE', lang_list, fallback='unknown') 'fr-CA' ---- Example: (Assumption A.) A user selects only the language 'fr' which should end in fr_CA >>> get_engine_locale('fr', lang_list) 'fr_CA' >>> lang_list[match_language('fr', lang_list)] 'fr_CA' ---- The difference in priority on the territory is best shown with a engine that supports the following locales: >>> lang_list = { ... "fr-FR": "fr_FR", ... "fr-CA": "fr_CA", ... "en-GB": "en_GB", ... "nl-BE": "nl_BE", ... } ---- Example: (Assumption A.) A user selects only a language >>> get_engine_locale('en', lang_list) 'en_GB' >>> match_language('en', lang_list) 'en-GB' hint: the engine supports fr_FR and fr_CA since no territory is given, fr_FR takes priority .. >>> get_engine_locale('fr', lang_list) 'fr_FR' >>> lang_list[match_language('fr', lang_list)] 'fr_FR' ---- Example: (Assumption B.) A user selects region 'fr-BE' which should end in nl-BE >>> get_engine_locale('fr-BE', lang_list) 'nl_BE' >>> lang_list[match_language('fr-BE', lang_list)] 'fr_FR' ---- If the user selects a language and there are two locales like the following: >>> lang_list = { ... "fr-BE": "fr_BE", ... "fr-CH": "fr_CH", ... } >>> >>> get_engine_locale('fr', lang_list) 'fr_BE' >>> lang_list[match_language('fr', lang_list)] 'fr_BE' Looks like both functions return the same value, but match_language depends on the order of the dictionary (which is not predictable): >>> lang_list = { ... "fr-CH": "fr_CH", ... "fr-BE": "fr_BE", ... } >>> get_engine_locale('fr', lang_list) 'fr_BE' >>> lang_list[match_language('fr', lang_list)] 'fr_CH' >>> The get_engine_locale selects the locale by looking at the "population percent" and this percentage has an higher amount in BE (68.%) compared to CH (21%) Signed-off-by: Markus Heiser --- searx/locales.py | 137 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/searx/locales.py b/searx/locales.py index 3deff6863..227c95051 100644 --- a/searx/locales.py +++ b/searx/locales.py @@ -10,6 +10,8 @@ import pathlib from babel import Locale from babel.support import Translations +import babel.languages +import babel.core import flask_babel import flask from flask.ctx import has_request_context @@ -150,3 +152,138 @@ def locales_initialize(directory=None): LOCALE_NAMES[tag] = get_locale_descr(locale, dirname) if locale.text_direction == 'rtl': RTL_LOCALES.add(tag) + + +def get_engine_locale(searxng_locale, engine_locales, default=None): + """Return engine's language (aka locale) string that best fits to argument + ``searxng_locale``. + + Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to + corresponding *engine locales*: + + : { + # SearXNG string : engine-string + 'ca-ES' : 'ca_ES', + 'fr-BE' : 'fr_BE', + 'fr-CA' : 'fr_CA', + 'fr-CH' : 'fr_CH', + 'fr' : 'fr_FR', + ... + 'pl-PL' : 'pl_PL', + 'pt-PT' : 'pt_PT' + } + + .. hint:: + + The *SearXNG locale* string has to be known by babel! + + If there is no direct 1:1 mapping, this functions tries to narrow down + engine's language (locale). If no value can be determined by these + approximation attempts the ``default`` value is returned. + + Assumptions: + + A. When user select a language the results should be optimized according to + the selected language. + + B. When user select a language and a territory the results should be + optimized with first priority on terrirtory and second on language. + + First approximation rule (*by territory*): + + When the user selects a locale with terrirtory (and a language), the + territory has priority over the language. If any of the offical languages + in the terrirtory is supported by the engine (``engine_locales``) it will + be used. + + Second approximation rule (*by language*): + + If "First approximation rule" brings no result or the user selects only a + language without a terrirtory. Check in which territories the language + has an offical status and if one of these territories is supported by the + engine. + + """ + # pylint: disable=too-many-branches + + engine_locale = engine_locales.get(searxng_locale) + + if engine_locale is not None: + # There was a 1:1 mapping (e.g. "fr-BE --> fr_BE" or "fr --> fr_FR"), no + # need to narrow language nor territory. + return engine_locale + + locale = babel.Locale.parse(searxng_locale, sep='-') + + # SearXNG's selected locale is not supported by the engine .. + + if locale.territory: + # Try to narrow by *offical* languages in the territory (??-XX). + + for official_language in babel.languages.get_official_languages(locale.territory, de_facto=True): + searxng_locale = official_language + '-' + locale.territory + engine_locale = engine_locales.get(searxng_locale) + if engine_locale is not None: + return engine_locale + + # Engine does not support one of the offical languages in the territory or + # there is only a language selected without a territory. + + # Now lets have a look if the searxng_lang (the language selected by the + # user) is a offical language in other territories. If so, check if + # engine does support the searxng_lang in this other territory. + + if locale.language: + + searxng_lang = locale.language + if locale.script: + searxng_lang += '_' + locale.script + + terr_lang_dict = {} + for territory, langs in babel.core.get_global("territory_languages").items(): + if not langs.get(searxng_lang, {}).get('official_status'): + continue + terr_lang_dict[territory] = langs.get(searxng_lang) + + # first: check fr-FR, de-DE .. is supported by the engine + + territory = locale.language.upper() + if terr_lang_dict.get(territory): + searxng_locale = locale.language + '-' + territory + engine_locale = engine_locales.get(searxng_locale) + if engine_locale is not None: + return engine_locale + + # second: sort by population_percent and take first match + + # drawback of "population percent": if there is a terrirtory with a + # small number of people (e.g 100) but the majority speaks the + # language, then the percentage migth be 100% (--> 100 people) but in + # a different terrirtory with more people (e.g. 10.000) where only 10% + # speak the language the total amount of speaker is higher (--> 200 + # people). + # + # By example: The population of Saint-Martin is 33.000, of which 100% + # speak French, but this is less than the 30% of the approximately 2.5 + # million Belgian citizens + # + # - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official' + # - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official' + + terr_lang_list = [] + for k, v in terr_lang_dict.items(): + terr_lang_list.append((k, v)) + + for territory, _lang in sorted(terr_lang_list, key=lambda item: item[1]['population_percent'], reverse=True): + searxng_locale = locale.language + '-' + territory + engine_locale = engine_locales.get(searxng_locale) + if engine_locale is not None: + return engine_locale + + # No luck: narrow by "language from territory" and "territory from language" + # does not fit to a locale supported by the engine. + + if engine_locale is None: + engine_locale = default + + return default From 6579d6d5585a0ce58aaf223f7b1767ed69a0e428 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 12 Aug 2022 18:37:09 +0200 Subject: [PATCH 3/4] [fix] qwant - API error::locale must be one .. The request function should not request a language (aka locale) that is not supported by qwant. Select a locale like zh-TW ends in qwant's API error: ERROR searx.engines.qwant news: exception : \ API error::locale must be one of the following values: \ en_gb, en_ie, en_us, en_ca, en_my, en_au, en_nz, de_de, de_ch, de_at, fr_fr, \ fr_be, fr_ch, fr_ca, fr_ad, fc_ca, co_fr, es_es, es_ar, es_cl, es_co, es_mx, \ es_pe, es_ad, ca_es, ca_ad, ca_fr, eu_es, eu_fr, it_it, it_ch, pt_pt, pt_ad, \ nl_be, nl_nl The existing searx.utils.match_language function is unsuitable for this purpose, it is replaced by function searx.locales.get_engine_locale that is based on the methods from the babel package. The quant's _fetch_supported_languages function has been revised to filter out languages 8aka locales) not supported by qwant. Signed-off-by: Markus Heiser --- searx/data/engines_languages.json | 313 ++++++++++++++---------------- searx/engines/qwant.py | 77 +++++--- 2 files changed, 203 insertions(+), 187 deletions(-) diff --git a/searx/data/engines_languages.json b/searx/data/engines_languages.json index 1d4d9b2e3..3ee6fad79 100644 --- a/searx/data/engines_languages.json +++ b/searx/data/engines_languages.json @@ -1396,170 +1396,155 @@ "sv", "zh" ], - "qwant": [ - "bg-BG", - "ca-ES", - "cs-CZ", - "da-DK", - "de-AT", - "de-CH", - "de-DE", - "el-GR", - "en-AU", - "en-CA", - "en-GB", - "en-IE", - "en-MY", - "en-NZ", - "en-US", - "es-AR", - "es-CL", - "es-ES", - "es-MX", - "et-EE", - "fi-FI", - "fr-BE", - "fr-CA", - "fr-CH", - "fr-FR", - "hu-HU", - "it-CH", - "it-IT", - "ko-KR", - "nb-NO", - "nl-BE", - "nl-NL", - "pl-PL", - "pt-PT", - "ro-RO", - "sv-SE", - "th-TH", - "zh-CN", - "zh-HK" - ], - "qwant images": [ - "bg-BG", - "ca-ES", - "cs-CZ", - "da-DK", - "de-AT", - "de-CH", - "de-DE", - "el-GR", - "en-AU", - "en-CA", - "en-GB", - "en-IE", - "en-MY", - "en-NZ", - "en-US", - "es-AR", - "es-CL", - "es-ES", - "es-MX", - "et-EE", - "fi-FI", - "fr-BE", - "fr-CA", - "fr-CH", - "fr-FR", - "hu-HU", - "it-CH", - "it-IT", - "ko-KR", - "nb-NO", - "nl-BE", - "nl-NL", - "pl-PL", - "pt-PT", - "ro-RO", - "sv-SE", - "th-TH", - "zh-CN", - "zh-HK" - ], - "qwant news": [ - "bg-BG", - "ca-ES", - "cs-CZ", - "da-DK", - "de-AT", - "de-CH", - "de-DE", - "el-GR", - "en-AU", - "en-CA", - "en-GB", - "en-IE", - "en-MY", - "en-NZ", - "en-US", - "es-AR", - "es-CL", - "es-ES", - "es-MX", - "et-EE", - "fi-FI", - "fr-BE", - "fr-CA", - "fr-CH", - "fr-FR", - "hu-HU", - "it-CH", - "it-IT", - "ko-KR", - "nb-NO", - "nl-BE", - "nl-NL", - "pl-PL", - "pt-PT", - "ro-RO", - "sv-SE", - "th-TH", - "zh-CN", - "zh-HK" - ], - "qwant videos": [ - "bg-BG", - "ca-ES", - "cs-CZ", - "da-DK", - "de-AT", - "de-CH", - "de-DE", - "el-GR", - "en-AU", - "en-CA", - "en-GB", - "en-IE", - "en-MY", - "en-NZ", - "en-US", - "es-AR", - "es-CL", - "es-ES", - "es-MX", - "et-EE", - "fi-FI", - "fr-BE", - "fr-CA", - "fr-CH", - "fr-FR", - "hu-HU", - "it-CH", - "it-IT", - "ko-KR", - "nb-NO", - "nl-BE", - "nl-NL", - "pl-PL", - "pt-PT", - "ro-RO", - "sv-SE", - "th-TH", - "zh-CN", - "zh-HK" - ], + "qwant": { + "bg-BG": "bg_BG", + "ca-ES": "ca_ES", + "cs-CZ": "cs_CZ", + "da-DK": "da_DK", + "de-AT": "de_AT", + "de-CH": "de_CH", + "de-DE": "de_DE", + "el-GR": "el_GR", + "en-AU": "en_AU", + "en-CA": "en_CA", + "en-GB": "en_GB", + "en-IE": "en_IE", + "en-MY": "en_MY", + "en-NZ": "en_NZ", + "en-US": "en_US", + "es-AR": "es_AR", + "es-CL": "es_CL", + "es-ES": "es_ES", + "es-MX": "es_MX", + "et-EE": "et_EE", + "fi-FI": "fi_FI", + "fr-BE": "fr_BE", + "fr-CA": "fr_CA", + "fr-CH": "fr_CH", + "fr-FR": "fr_FR", + "hu-HU": "hu_HU", + "it-CH": "it_CH", + "it-IT": "it_IT", + "ko-KR": "ko_KR", + "nb-NO": "nb_NO", + "nl-BE": "nl_BE", + "nl-NL": "nl_NL", + "pl-PL": "pl_PL", + "pt-PT": "pt_PT", + "ro-RO": "ro_RO", + "sv-SE": "sv_SE", + "th-TH": "th_TH", + "zh-CN": "zh_CN", + "zh-HK": "zh_HK" + }, + "qwant images": { + "bg-BG": "bg_BG", + "ca-ES": "ca_ES", + "cs-CZ": "cs_CZ", + "da-DK": "da_DK", + "de-AT": "de_AT", + "de-CH": "de_CH", + "de-DE": "de_DE", + "el-GR": "el_GR", + "en-AU": "en_AU", + "en-CA": "en_CA", + "en-GB": "en_GB", + "en-IE": "en_IE", + "en-MY": "en_MY", + "en-NZ": "en_NZ", + "en-US": "en_US", + "es-AR": "es_AR", + "es-CL": "es_CL", + "es-ES": "es_ES", + "es-MX": "es_MX", + "et-EE": "et_EE", + "fi-FI": "fi_FI", + "fr-BE": "fr_BE", + "fr-CA": "fr_CA", + "fr-CH": "fr_CH", + "fr-FR": "fr_FR", + "hu-HU": "hu_HU", + "it-CH": "it_CH", + "it-IT": "it_IT", + "ko-KR": "ko_KR", + "nb-NO": "nb_NO", + "nl-BE": "nl_BE", + "nl-NL": "nl_NL", + "pl-PL": "pl_PL", + "pt-PT": "pt_PT", + "ro-RO": "ro_RO", + "sv-SE": "sv_SE", + "th-TH": "th_TH", + "zh-CN": "zh_CN", + "zh-HK": "zh_HK" + }, + "qwant news": { + "ca-ES": "ca_ES", + "de-AT": "de_AT", + "de-CH": "de_CH", + "de-DE": "de_DE", + "en-AU": "en_AU", + "en-CA": "en_CA", + "en-GB": "en_GB", + "en-IE": "en_IE", + "en-MY": "en_MY", + "en-NZ": "en_NZ", + "en-US": "en_US", + "es-AR": "es_AR", + "es-CL": "es_CL", + "es-ES": "es_ES", + "es-MX": "es_MX", + "fr-BE": "fr_BE", + "fr-CA": "fr_CA", + "fr-CH": "fr_CH", + "fr-FR": "fr_FR", + "it-CH": "it_CH", + "it-IT": "it_IT", + "nl-BE": "nl_BE", + "nl-NL": "nl_NL", + "pt-PT": "pt_PT" + }, + "qwant videos": { + "bg-BG": "bg_BG", + "ca-ES": "ca_ES", + "cs-CZ": "cs_CZ", + "da-DK": "da_DK", + "de-AT": "de_AT", + "de-CH": "de_CH", + "de-DE": "de_DE", + "el-GR": "el_GR", + "en-AU": "en_AU", + "en-CA": "en_CA", + "en-GB": "en_GB", + "en-IE": "en_IE", + "en-MY": "en_MY", + "en-NZ": "en_NZ", + "en-US": "en_US", + "es-AR": "es_AR", + "es-CL": "es_CL", + "es-ES": "es_ES", + "es-MX": "es_MX", + "et-EE": "et_EE", + "fi-FI": "fi_FI", + "fr-BE": "fr_BE", + "fr-CA": "fr_CA", + "fr-CH": "fr_CH", + "fr-FR": "fr_FR", + "hu-HU": "hu_HU", + "it-CH": "it_CH", + "it-IT": "it_IT", + "ko-KR": "ko_KR", + "nb-NO": "nb_NO", + "nl-BE": "nl_BE", + "nl-NL": "nl_NL", + "pl-PL": "pl_PL", + "pt-PT": "pt_PT", + "ro-RO": "ro_RO", + "sv-SE": "sv_SE", + "th-TH": "th_TH", + "zh-CN": "zh_CN", + "zh-HK": "zh_HK" + }, "startpage": { "af": { "alias": "afrikaans" diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py index e8b1d7739..47956fc97 100644 --- a/searx/engines/qwant.py +++ b/searx/engines/qwant.py @@ -30,11 +30,11 @@ from datetime import ( from json import loads from urllib.parse import urlencode from flask_babel import gettext +import babel -from searx.utils import match_language from searx.exceptions import SearxEngineAPIException from searx.network import raise_for_httperror - +from searx.locales import get_engine_locale # about about = { @@ -52,12 +52,26 @@ paging = True supported_languages_url = about['website'] qwant_categ = None # web|news|inages|videos +# fmt: off +qwant_news_locales = [ + 'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au', + 'en_ca', 'en_gb', 'en_ie', 'en_my', 'en_nz', 'en_us', 'es_ad', 'es_ar', + 'es_cl', 'es_co', 'es_es', 'es_mx', 'es_pe', 'eu_es', 'eu_fr', 'fc_ca', + 'fr_ad', 'fr_be', 'fr_ca', 'fr_ch', 'fr_fr', 'it_ch', 'it_it', 'nl_be', + 'nl_nl', 'pt_ad', 'pt_pt', +] +# fmt: on + # search-url url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}' def request(query, params): """Qwant search request""" + + if not query: + return None + count = 10 # web: count must be equal to 10 if qwant_categ == 'images': @@ -77,16 +91,9 @@ def request(query, params): count=count, ) - # add language tag - if params['language'] == 'all': - params['url'] += '&locale=en_US' - else: - language = match_language( - params['language'], - supported_languages, - language_aliases, - ) - params['url'] += '&locale=' + language.replace('-', '_') + # add quant's locale + q_locale = get_engine_locale(params['language'], supported_languages, default='en_US') + params['url'] += '&locale=' + q_locale params['raise_for_httperror'] = False return params @@ -230,19 +237,43 @@ def response(resp): return results -# get supported languages from their site def _fetch_supported_languages(resp): - # list of regions is embedded in page as a js object - response_text = resp.text - response_text = response_text[response_text.find('INITIAL_PROPS') :] - response_text = response_text[response_text.find('{') : response_text.find('')] - regions_json = loads(response_text) + text = resp.text + text = text[text.find('INITIAL_PROPS') :] + text = text[text.find('{') : text.find('')] - supported_languages = [] - for country, langs in regions_json['locales'].items(): - for lang in langs['langs']: - lang_code = "{lang}-{country}".format(lang=lang, country=country) - supported_languages.append(lang_code) + q_initial_props = loads(text) + q_locales = q_initial_props.get('locales') + q_valid_locales = [] + + for country, v in q_locales.items(): + for lang in v['langs']: + _locale = "{lang}_{country}".format(lang=lang, country=country) + + if qwant_categ == 'news' and _locale.lower() not in qwant_news_locales: + # qwant-news does not support all locales from qwant-web: + continue + + q_valid_locales.append(_locale) + + supported_languages = {} + + for q_locale in q_valid_locales: + try: + locale = babel.Locale.parse(q_locale, sep='_') + except babel.core.UnknownLocaleError: + print("ERROR: can't determine babel locale of quant's locale %s" % q_locale) + continue + + # note: supported_languages (dict) + # + # dict's key is a string build up from a babel.Locale object / the + # notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and + # language) notation and dict's values are the locale strings used by + # the engine. + + searxng_locale = locale.language + '-' + locale.territory # --> params['language'] + supported_languages[searxng_locale] = q_locale return supported_languages From 27385e789853b41e81493301bf5538616e484c90 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 12 Aug 2022 18:37:46 +0200 Subject: [PATCH 4/4] [mod] qwant - add safesearch option Closes: https://github.com/searxng/searxng/issues/1640 Signed-off-by: Markus Heiser --- searx/engines/qwant.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py index 47956fc97..6de2176d0 100644 --- a/searx/engines/qwant.py +++ b/searx/engines/qwant.py @@ -52,6 +52,9 @@ paging = True supported_languages_url = about['website'] qwant_categ = None # web|news|inages|videos +safesearch = True +safe_search_map = {0: '&safesearch=0', 1: '&safesearch=1', 2: '&safesearch=2'} + # fmt: off qwant_news_locales = [ 'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au', @@ -95,6 +98,9 @@ def request(query, params): q_locale = get_engine_locale(params['language'], supported_languages, default='en_US') params['url'] += '&locale=' + q_locale + # add safesearch option + params['url'] += safe_search_map.get(params['safesearch'], '') + params['raise_for_httperror'] = False return params