zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 18:24:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

[fix] qwant - API error::locale must be one ..

Browse Source 
The request function should not request a language (aka locale) that is not
supported by qwant. Select a locale like zh-TW ends in qwant's API error:

  ERROR searx.engines.qwant news: exception : \
  API error::locale must be one of the following values: \
    en_gb, en_ie, en_us, en_ca, en_my, en_au, en_nz, de_de, de_ch, de_at, fr_fr, \
    fr_be, fr_ch, fr_ca, fr_ad, fc_ca, co_fr, es_es, es_ar, es_cl, es_co, es_mx, \
    es_pe, es_ad, ca_es, ca_ad, ca_fr, eu_es, eu_fr, it_it, it_ch, pt_pt, pt_ad, \
    nl_be, nl_nl

The existing searx.utils.match_language function is unsuitable for this purpose,
it is replaced by function searx.locales.get_engine_locale that is based on the
methods from the babel package.

The quant's _fetch_supported_languages function has been revised to filter out
languages 8aka locales) not supported by qwant.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2022-08-12 18:37:09 +02:00
parent 9ae409a05a
commit 6579d6d558
2 changed files with 203 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

313
searx/data/engines_languages.json
View File

@ -1396,170 +1396,155 @@
"sv", "sv",
"zh" "zh"
], ],
"qwant": [ "qwant": {
"bg-BG", "bg-BG": "bg_BG",
"ca-ES", "ca-ES": "ca_ES",
"cs-CZ", "cs-CZ": "cs_CZ",
"da-DK", "da-DK": "da_DK",
"de-AT", "de-AT": "de_AT",
"de-CH", "de-CH": "de_CH",
"de-DE", "de-DE": "de_DE",
"el-GR", "el-GR": "el_GR",
"en-AU", "en-AU": "en_AU",
"en-CA", "en-CA": "en_CA",
"en-GB", "en-GB": "en_GB",
"en-IE", "en-IE": "en_IE",
"en-MY", "en-MY": "en_MY",
"en-NZ", "en-NZ": "en_NZ",
"en-US", "en-US": "en_US",
"es-AR", "es-AR": "es_AR",
"es-CL", "es-CL": "es_CL",
"es-ES", "es-ES": "es_ES",
"es-MX", "es-MX": "es_MX",
"et-EE", "et-EE": "et_EE",
"fi-FI", "fi-FI": "fi_FI",
"fr-BE", "fr-BE": "fr_BE",
"fr-CA", "fr-CA": "fr_CA",
"fr-CH", "fr-CH": "fr_CH",
"fr-FR", "fr-FR": "fr_FR",
"hu-HU", "hu-HU": "hu_HU",
"it-CH", "it-CH": "it_CH",
"it-IT", "it-IT": "it_IT",
"ko-KR", "ko-KR": "ko_KR",
"nb-NO", "nb-NO": "nb_NO",
"nl-BE", "nl-BE": "nl_BE",
"nl-NL", "nl-NL": "nl_NL",
"pl-PL", "pl-PL": "pl_PL",
"pt-PT", "pt-PT": "pt_PT",
"ro-RO", "ro-RO": "ro_RO",
"sv-SE", "sv-SE": "sv_SE",
"th-TH", "th-TH": "th_TH",
"zh-CN", "zh-CN": "zh_CN",
"zh-HK" "zh-HK": "zh_HK"
], },
"qwant images": [ "qwant images": {
"bg-BG", "bg-BG": "bg_BG",
"ca-ES", "ca-ES": "ca_ES",
"cs-CZ", "cs-CZ": "cs_CZ",
"da-DK", "da-DK": "da_DK",
"de-AT", "de-AT": "de_AT",
"de-CH", "de-CH": "de_CH",
"de-DE", "de-DE": "de_DE",
"el-GR", "el-GR": "el_GR",
"en-AU", "en-AU": "en_AU",
"en-CA", "en-CA": "en_CA",
"en-GB", "en-GB": "en_GB",
"en-IE", "en-IE": "en_IE",
"en-MY", "en-MY": "en_MY",
"en-NZ", "en-NZ": "en_NZ",
"en-US", "en-US": "en_US",
"es-AR", "es-AR": "es_AR",
"es-CL", "es-CL": "es_CL",
"es-ES", "es-ES": "es_ES",
"es-MX", "es-MX": "es_MX",
"et-EE", "et-EE": "et_EE",
"fi-FI", "fi-FI": "fi_FI",
"fr-BE", "fr-BE": "fr_BE",
"fr-CA", "fr-CA": "fr_CA",
"fr-CH", "fr-CH": "fr_CH",
"fr-FR", "fr-FR": "fr_FR",
"hu-HU", "hu-HU": "hu_HU",
"it-CH", "it-CH": "it_CH",
"it-IT", "it-IT": "it_IT",
"ko-KR", "ko-KR": "ko_KR",
"nb-NO", "nb-NO": "nb_NO",
"nl-BE", "nl-BE": "nl_BE",
"nl-NL", "nl-NL": "nl_NL",
"pl-PL", "pl-PL": "pl_PL",
"pt-PT", "pt-PT": "pt_PT",
"ro-RO", "ro-RO": "ro_RO",
"sv-SE", "sv-SE": "sv_SE",
"th-TH", "th-TH": "th_TH",
"zh-CN", "zh-CN": "zh_CN",
"zh-HK" "zh-HK": "zh_HK"
], },
"qwant news": [ "qwant news": {
"bg-BG", "ca-ES": "ca_ES",
"ca-ES", "de-AT": "de_AT",
"cs-CZ", "de-CH": "de_CH",
"da-DK", "de-DE": "de_DE",
"de-AT", "en-AU": "en_AU",
"de-CH", "en-CA": "en_CA",
"de-DE", "en-GB": "en_GB",
"el-GR", "en-IE": "en_IE",
"en-AU", "en-MY": "en_MY",
"en-CA", "en-NZ": "en_NZ",
"en-GB", "en-US": "en_US",
"en-IE", "es-AR": "es_AR",
"en-MY", "es-CL": "es_CL",
"en-NZ", "es-ES": "es_ES",
"en-US", "es-MX": "es_MX",
"es-AR", "fr-BE": "fr_BE",
"es-CL", "fr-CA": "fr_CA",
"es-ES", "fr-CH": "fr_CH",
"es-MX", "fr-FR": "fr_FR",
"et-EE", "it-CH": "it_CH",
"fi-FI", "it-IT": "it_IT",
"fr-BE", "nl-BE": "nl_BE",
"fr-CA", "nl-NL": "nl_NL",
"fr-CH", "pt-PT": "pt_PT"
"fr-FR", },
"hu-HU", "qwant videos": {
"it-CH", "bg-BG": "bg_BG",
"it-IT", "ca-ES": "ca_ES",
"ko-KR", "cs-CZ": "cs_CZ",
"nb-NO", "da-DK": "da_DK",
"nl-BE", "de-AT": "de_AT",
"nl-NL", "de-CH": "de_CH",
"pl-PL", "de-DE": "de_DE",
"pt-PT", "el-GR": "el_GR",
"ro-RO", "en-AU": "en_AU",
"sv-SE", "en-CA": "en_CA",
"th-TH", "en-GB": "en_GB",
"zh-CN", "en-IE": "en_IE",
"zh-HK" "en-MY": "en_MY",
], "en-NZ": "en_NZ",
"qwant videos": [ "en-US": "en_US",
"bg-BG", "es-AR": "es_AR",
"ca-ES", "es-CL": "es_CL",
"cs-CZ", "es-ES": "es_ES",
"da-DK", "es-MX": "es_MX",
"de-AT", "et-EE": "et_EE",
"de-CH", "fi-FI": "fi_FI",
"de-DE", "fr-BE": "fr_BE",
"el-GR", "fr-CA": "fr_CA",
"en-AU", "fr-CH": "fr_CH",
"en-CA", "fr-FR": "fr_FR",
"en-GB", "hu-HU": "hu_HU",
"en-IE", "it-CH": "it_CH",
"en-MY", "it-IT": "it_IT",
"en-NZ", "ko-KR": "ko_KR",
"en-US", "nb-NO": "nb_NO",
"es-AR", "nl-BE": "nl_BE",
"es-CL", "nl-NL": "nl_NL",
"es-ES", "pl-PL": "pl_PL",
"es-MX", "pt-PT": "pt_PT",
"et-EE", "ro-RO": "ro_RO",
"fi-FI", "sv-SE": "sv_SE",
"fr-BE", "th-TH": "th_TH",
"fr-CA", "zh-CN": "zh_CN",
"fr-CH", "zh-HK": "zh_HK"
"fr-FR", },
"hu-HU",
"it-CH",
"it-IT",
"ko-KR",
"nb-NO",
"nl-BE",
"nl-NL",
"pl-PL",
"pt-PT",
"ro-RO",
"sv-SE",
"th-TH",
"zh-CN",
"zh-HK"
],
"startpage": { "startpage": {
"af": { "af": {
"alias": "afrikaans" "alias": "afrikaans"

77
searx/engines/qwant.py
View File

@ -30,11 +30,11 @@ from datetime import (
from json import loads from json import loads
from urllib.parse import urlencode from urllib.parse import urlencode
from flask_babel import gettext from flask_babel import gettext
import babel
from searx.utils import match_language
from searx.exceptions import SearxEngineAPIException from searx.exceptions import SearxEngineAPIException
from searx.network import raise_for_httperror from searx.network import raise_for_httperror
from searx.locales import get_engine_locale
# about # about
about = { about = {
@ -52,12 +52,26 @@ paging = True
supported_languages_url = about['website'] supported_languages_url = about['website']
qwant_categ = None # web|news|inages|videos qwant_categ = None # web|news|inages|videos
# fmt: off
qwant_news_locales = [
'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au',
'en_ca', 'en_gb', 'en_ie', 'en_my', 'en_nz', 'en_us', 'es_ad', 'es_ar',
'es_cl', 'es_co', 'es_es', 'es_mx', 'es_pe', 'eu_es', 'eu_fr', 'fc_ca',
'fr_ad', 'fr_be', 'fr_ca', 'fr_ch', 'fr_fr', 'it_ch', 'it_it', 'nl_be',
'nl_nl', 'pt_ad', 'pt_pt',
]
# fmt: on
# search-url # search-url
url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}' url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
def request(query, params): def request(query, params):
"""Qwant search request""" """Qwant search request"""
if not query:
return None
count = 10 # web: count must be equal to 10 count = 10 # web: count must be equal to 10
if qwant_categ == 'images': if qwant_categ == 'images':
@ -77,16 +91,9 @@ def request(query, params):
count=count, count=count,
) )
# add language tag # add quant's locale
if params['language'] == 'all': q_locale = get_engine_locale(params['language'], supported_languages, default='en_US')
params['url'] += '&locale=en_US' params['url'] += '&locale=' + q_locale
else:
language = match_language(
params['language'],
supported_languages,
language_aliases,
)
params['url'] += '&locale=' + language.replace('-', '_')
params['raise_for_httperror'] = False params['raise_for_httperror'] = False
return params return params
@ -230,19 +237,43 @@ def response(resp):
return results return results
# get supported languages from their site
def _fetch_supported_languages(resp): def _fetch_supported_languages(resp):
# list of regions is embedded in page as a js object
response_text = resp.text
response_text = response_text[response_text.find('INITIAL_PROPS') :]
response_text = response_text[response_text.find('{') : response_text.find('</script>')]
regions_json = loads(response_text) text = resp.text
text = text[text.find('INITIAL_PROPS') :]
text = text[text.find('{') : text.find('</script>')]
supported_languages = [] q_initial_props = loads(text)
for country, langs in regions_json['locales'].items(): q_locales = q_initial_props.get('locales')
for lang in langs['langs']: q_valid_locales = []
lang_code = "{lang}-{country}".format(lang=lang, country=country)
supported_languages.append(lang_code) for country, v in q_locales.items():
for lang in v['langs']:
_locale = "{lang}_{country}".format(lang=lang, country=country)
if qwant_categ == 'news' and _locale.lower() not in qwant_news_locales:
# qwant-news does not support all locales from qwant-web:
continue
q_valid_locales.append(_locale)
supported_languages = {}
for q_locale in q_valid_locales:
try:
locale = babel.Locale.parse(q_locale, sep='_')
except babel.core.UnknownLocaleError:
print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
continue
# note: supported_languages (dict)
#
# dict's key is a string build up from a babel.Locale object / the
# notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
# language) notation and dict's values are the locale strings used by
# the engine.
searxng_locale = locale.language + '-' + locale.territory # --> params['language']
supported_languages[searxng_locale] = q_locale
return supported_languages return supported_languages