mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
Add searx.locales.SupportedLocales
This commit is contained in:
Alexandre Flament
parent
9c68980ea1
commit
7f78b69e86
7 changed files with 187 additions and 147 deletions
|
|
@ -13,13 +13,14 @@ usage::
|
|||
|
||||
import sys
|
||||
import copy
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
from os.path import realpath, dirname
|
||||
from babel.localedata import locale_identifiers
|
||||
from searx import logger, settings
|
||||
from searx.data import ENGINES_LANGUAGES
|
||||
from searx.utils import load_module, match_language
|
||||
from searx.locales import SupportedLocales
|
||||
|
||||
|
||||
logger = logger.getChild('engines')
|
||||
|
|
@ -35,7 +36,7 @@ ENGINE_DEFAULT_ARGS = {
|
|||
"timeout": settings["outgoing"]["request_timeout"],
|
||||
"shortcut": "-",
|
||||
"categories": ["general"],
|
||||
"language_support" : False,
|
||||
"language_support": False,
|
||||
"paging": False,
|
||||
"safesearch": False,
|
||||
"time_range_support": False,
|
||||
|
|
@ -56,11 +57,13 @@ class Engine: # pylint: disable=too-few-public-methods
|
|||
engine: str
|
||||
shortcut: str
|
||||
categories: List[str]
|
||||
supported_languages: List[str]
|
||||
supported_locales: SupportedLocales
|
||||
# language support, either by selecting a region or by selecting a language
|
||||
language_support: bool
|
||||
about: dict
|
||||
inactive: bool
|
||||
disabled: bool
|
||||
# language support, either by selecting a region or by selecting a language
|
||||
language_support: bool
|
||||
paging: bool
|
||||
safesearch: bool
|
||||
time_range_support: bool
|
||||
|
|
@ -141,25 +144,6 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
|
|||
|
||||
return engine
|
||||
|
||||
def engine_properties_template():
|
||||
"""A dictionary with languages and regions to map from SearXNG' languages &
|
||||
region tags to engine's language & region tags::
|
||||
|
||||
engine_properties = {
|
||||
'type' : 'engine_properties',
|
||||
'regions': {
|
||||
# 'ca-ES' : <engine's region name>
|
||||
},
|
||||
'languages': {
|
||||
# 'ca' : <engine's language name>
|
||||
},
|
||||
}
|
||||
"""
|
||||
return {
|
||||
'type' : 'engine_properties',
|
||||
'regions': {},
|
||||
'languages': {},
|
||||
}
|
||||
|
||||
def set_loggers(engine, engine_name):
|
||||
# set the logger for engine
|
||||
|
|
@ -197,10 +181,10 @@ def update_engine_attributes(engine: Engine, engine_data):
|
|||
def set_language_attributes(engine: Engine):
|
||||
# assign supported languages from json file
|
||||
|
||||
supported_properties = None
|
||||
data: Any = None
|
||||
|
||||
if engine.name in ENGINES_LANGUAGES:
|
||||
supported_properties = ENGINES_LANGUAGES[engine.name]
|
||||
data = ENGINES_LANGUAGES[engine.name]
|
||||
|
||||
elif engine.engine in ENGINES_LANGUAGES:
|
||||
# The key of the dictionary ENGINES_LANGUAGES is the *engine name*
|
||||
|
|
@ -208,48 +192,48 @@ def set_language_attributes(engine: Engine):
|
|||
# settings.yml to use the same origin engine (python module) these
|
||||
# additional engines can use the languages from the origin engine.
|
||||
# For this use the configured ``engine: ...`` from settings.yml
|
||||
supported_properties = ENGINES_LANGUAGES[engine.engine]
|
||||
data = ENGINES_LANGUAGES[engine.engine]
|
||||
|
||||
if not supported_properties:
|
||||
# supported_locales is always defined
|
||||
engine.supported_locales = SupportedLocales.loads(data)
|
||||
engine.language_support = not engine.supported_locales.empty()
|
||||
|
||||
if data is None:
|
||||
return
|
||||
|
||||
if isinstance(supported_properties, dict) and supported_properties.get('type') == 'engine_properties':
|
||||
engine.supported_properties = supported_properties
|
||||
engine.language_support = len(supported_properties['languages']) or len(supported_properties['regions'])
|
||||
if engine.language_support:
|
||||
# to do: implement engine.language equivalent by calling a method of SupportedLocales
|
||||
return
|
||||
|
||||
else:
|
||||
# depricated: does not work for engines that do support languages
|
||||
# based on a region.
|
||||
engine.supported_languages = supported_properties
|
||||
engine.language_support = len(engine.supported_languages) > 0
|
||||
# deprecated: does not work for engines that do support languages based on a region.
|
||||
engine.supported_languages = data
|
||||
engine.language_support = len(engine.supported_languages) > 0
|
||||
|
||||
if hasattr(engine, 'language'):
|
||||
# For an engine, when there is `language: ...` in the YAML settings, the
|
||||
# engine supports only one language, in this case
|
||||
# engine.supported_languages should contains this value defined in
|
||||
# settings.yml
|
||||
if engine.language not in engine.supported_languages:
|
||||
raise ValueError(
|
||||
"settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language)
|
||||
)
|
||||
if hasattr(engine, 'language'):
|
||||
# For an engine, when there is `language: ...` in the YAML settings, the engine supports only one language,
|
||||
# in this case engine.supported_languages should contains this value defined in settings.yml
|
||||
if engine.language not in engine.supported_languages:
|
||||
raise ValueError(
|
||||
"settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language)
|
||||
)
|
||||
|
||||
if isinstance(engine.supported_languages, dict):
|
||||
engine.supported_languages = {engine.language: engine.supported_languages[engine.language]}
|
||||
else:
|
||||
engine.supported_languages = [engine.language]
|
||||
if isinstance(engine.supported_languages, dict):
|
||||
engine.supported_languages = {engine.language: engine.supported_languages[engine.language]}
|
||||
else:
|
||||
engine.supported_languages = [engine.language]
|
||||
|
||||
if not hasattr(engine, 'language_aliases'):
|
||||
engine.language_aliases = {}
|
||||
# find custom aliases for non standard language codes
|
||||
for engine_lang in engine.supported_languages:
|
||||
iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
|
||||
if (
|
||||
iso_lang
|
||||
and iso_lang != engine_lang
|
||||
and not engine_lang.startswith(iso_lang)
|
||||
and iso_lang not in engine.supported_languages
|
||||
):
|
||||
engine.language_aliases[iso_lang] = engine_lang
|
||||
if not hasattr(engine, 'language_aliases'):
|
||||
engine.language_aliases = {}
|
||||
# find custom aliases for non standard language codes
|
||||
for engine_lang in engine.supported_languages:
|
||||
iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
|
||||
if (
|
||||
iso_lang
|
||||
and iso_lang != engine_lang
|
||||
and not engine_lang.startswith(iso_lang)
|
||||
and iso_lang not in engine.supported_languages
|
||||
):
|
||||
engine.language_aliases[iso_lang] = engine_lang
|
||||
|
||||
|
||||
def update_attributes_for_tor(engine: Engine) -> bool:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import babel
|
|||
|
||||
from searx.network import get
|
||||
from searx.utils import extract_text, eval_xpath
|
||||
from searx.locales import SupportedLocales
|
||||
from searx.exceptions import (
|
||||
SearxEngineResponseException,
|
||||
SearxEngineCaptchaException,
|
||||
|
|
@ -46,7 +47,7 @@ filter_mapping = {0: '0', 1: '1', 2: '1'}
|
|||
time_range_support = True
|
||||
time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
|
||||
|
||||
supported_properties_url = 'https://www.startpage.com/do/settings'
|
||||
supported_locales_url = 'https://www.startpage.com/do/settings'
|
||||
|
||||
# search-url
|
||||
base_url = 'https://www.startpage.com/'
|
||||
|
|
@ -109,33 +110,12 @@ def get_sc_code(headers):
|
|||
return sc_code
|
||||
|
||||
|
||||
def get_engine_locale(language):
|
||||
|
||||
if language == 'all':
|
||||
language = 'en-US'
|
||||
locale = babel.Locale.parse(language, sep='-')
|
||||
|
||||
engine_language = supported_properties['languages'].get(locale.language)
|
||||
if not engine_language:
|
||||
logger.debug("startpage does NOT support language: %s", locale.language)
|
||||
|
||||
engine_region = None
|
||||
if locale.territory:
|
||||
engine_region = supported_properties['regions'].get(locale.language + '-' + locale.territory)
|
||||
if not engine_region:
|
||||
logger.debug("no region in selected (only lang: '%s'), using region 'all'", language)
|
||||
engine_region = 'all'
|
||||
|
||||
logger.debug(
|
||||
"UI language: %s --> engine language: %s // engine region: %s",
|
||||
language, engine_language, engine_region
|
||||
)
|
||||
return locale, engine_language, engine_region
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
locale, engine_language, engine_region = get_engine_locale(params['language'])
|
||||
locale, engine_language, engine_region = params['locale'], params['engine_language'], params['engine_region']
|
||||
|
||||
if engine_region is None:
|
||||
engine_region = 'all'
|
||||
|
||||
# prepare HTTP headers
|
||||
ac_lang = locale.language
|
||||
|
|
@ -151,7 +131,7 @@ def request(query, params):
|
|||
'cat': 'web',
|
||||
't': 'device',
|
||||
'sc': get_sc_code(params['headers']), # hint: this func needs HTTP headers
|
||||
'with_date' : time_range_dict.get(params['time_range'], '')
|
||||
'with_date': time_range_dict.get(params['time_range'], ''),
|
||||
}
|
||||
|
||||
if engine_language:
|
||||
|
|
@ -187,7 +167,7 @@ def request(query, params):
|
|||
if engine_region:
|
||||
cookie['search_results_region'] = engine_region
|
||||
|
||||
params['cookies']['preferences'] = 'N1N'.join([ "%sEEE%s" % x for x in cookie.items() ])
|
||||
params['cookies']['preferences'] = 'N1N'.join(["%sEEE%s" % x for x in cookie.items()])
|
||||
logger.debug('cookie preferences: %s', params['cookies']['preferences'])
|
||||
params['method'] = 'POST'
|
||||
|
||||
|
|
@ -263,7 +243,7 @@ def response(resp):
|
|||
return results
|
||||
|
||||
|
||||
def _fetch_engine_properties(resp, engine_properties):
|
||||
def _fetch_supported_locales(resp):
|
||||
|
||||
# startpage's language & region selectors are a mess.
|
||||
#
|
||||
|
|
@ -291,6 +271,8 @@ def _fetch_engine_properties(resp, engine_properties):
|
|||
# name of the writing script used by the language, or occasionally
|
||||
# something else entirely.
|
||||
|
||||
supported_locales = SupportedLocales(all_language='en-US')
|
||||
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
# regions
|
||||
|
|
@ -305,27 +287,21 @@ def _fetch_engine_properties(resp, engine_properties):
|
|||
if '-' in sp_region_tag:
|
||||
l, r = sp_region_tag.split('-')
|
||||
r = r.split('_')[-1]
|
||||
locale = babel.Locale.parse(l +'_'+ r, sep='_')
|
||||
locale = babel.Locale.parse(l + '_' + r, sep='_')
|
||||
else:
|
||||
locale = babel.Locale.parse(sp_region_tag, sep='_')
|
||||
|
||||
region_tag = locale.language + '-' + locale.territory
|
||||
# print("internal: %s --> engine: %s" % (region_tag, sp_region_tag))
|
||||
engine_properties['regions'][region_tag] = sp_region_tag
|
||||
supported_locales.regions[region_tag] = sp_region_tag
|
||||
|
||||
# languages
|
||||
|
||||
catalog_engine2code = {
|
||||
name.lower(): lang_code
|
||||
for lang_code, name in babel.Locale('en').languages.items()
|
||||
}
|
||||
catalog_engine2code = {name.lower(): lang_code for lang_code, name in babel.Locale('en').languages.items()}
|
||||
|
||||
# get the native name of every language known by babel
|
||||
|
||||
for lang_code in filter(
|
||||
lambda lang_code: lang_code.find('_') == -1,
|
||||
babel.localedata.locale_identifiers()
|
||||
):
|
||||
for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, babel.localedata.locale_identifiers()):
|
||||
native_name = babel.Locale(lang_code).get_language_name().lower()
|
||||
# add native name exactly as it is
|
||||
catalog_engine2code[native_name] = lang_code
|
||||
|
|
@ -338,17 +314,19 @@ def _fetch_engine_properties(resp, engine_properties):
|
|||
|
||||
# values that can't be determined by babel's languages names
|
||||
|
||||
catalog_engine2code.update({
|
||||
'english_uk': 'en',
|
||||
# traditional chinese used in ..
|
||||
'fantizhengwen': 'zh_Hant',
|
||||
# Korean alphabet
|
||||
'hangul': 'ko',
|
||||
# Malayalam is one of 22 scheduled languages of India.
|
||||
'malayam': 'ml',
|
||||
'norsk': 'nb',
|
||||
'sinhalese': 'si',
|
||||
})
|
||||
catalog_engine2code.update(
|
||||
{
|
||||
'english_uk': 'en',
|
||||
# traditional chinese used in ..
|
||||
'fantizhengwen': 'zh_Hant',
|
||||
# Korean alphabet
|
||||
'hangul': 'ko',
|
||||
# Malayalam is one of 22 scheduled languages of India.
|
||||
'malayam': 'ml',
|
||||
'norsk': 'nb',
|
||||
'sinhalese': 'si',
|
||||
}
|
||||
)
|
||||
|
||||
for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'):
|
||||
engine_lang = option.get('value')
|
||||
|
|
@ -359,6 +337,6 @@ def _fetch_engine_properties(resp, engine_properties):
|
|||
lang_code = catalog_engine2code[name]
|
||||
|
||||
# print("internal: %s --> engine: %s" % (lang_code, engine_lang))
|
||||
engine_properties['languages'][lang_code] = engine_lang
|
||||
supported_locales.languages[lang_code] = engine_lang
|
||||
|
||||
return engine_properties
|
||||
return supported_locales
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue