Add engine locales (1/n)

searx/engines/__init__.py

@@ -13,14 +13,14 @@ usage::
 
 import sys
 import copy
-from typing import Dict, List, Optional
+import dataclasses
+from typing import Dict, List, Optional, Any
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
 from searx import logger, settings
-from searx.data import ENGINES_LANGUAGES
-from searx.network import get
-from searx.utils import load_module, match_language, gen_useragent
+from searx.data import ENGINES_LANGUAGES, ENGINES_LOCALES
+from searx.utils import load_module, match_language
 
 
 logger = logger.getChild('engines')
@@ -52,6 +52,27 @@ ENGINE_DEFAULT_ARGS = {
 OTHER_CATEGORY = 'other'
 
 
+@dataclasses.dataclass
+class EngineLocales:
+    """The class is intended to be instanciated for each engine."""
+
+    regions: Dict[str, str] = dataclasses.field(default_factory=dict)
+    """
+    .. code:: python
+       {
+           'fr-BE' : <engine's region name>,
+       }
+    """
+
+    languages: Dict[str, str] = dataclasses.field(default_factory=dict)
+    """
+    .. code:: python
+       {
+           'ca' : <engine's language name>,
+       }
+    """
+
+
 class Engine:  # pylint: disable=too-few-public-methods
     """This class is currently never initialized and only used for type hinting."""
 
@@ -59,15 +80,17 @@ class Engine:  # pylint: disable=too-few-public-methods
     engine: str
     shortcut: str
     categories: List[str]
-    supported_languages: List[str]
     about: dict
     inactive: bool
     disabled: bool
-    language_support: bool
     paging: bool
     safesearch: bool
     time_range_support: bool
     timeout: float
+    language_support: bool
+    engine_locales: EngineLocales
+    supported_languages: List[str]
+    language_aliases: Dict[str, str]
 
 
 # Defaults for the namespace of an engine module, see :py:func:`load_engine`
@@ -85,15 +108,15 @@ engine_shortcuts = {}
 """
 
 
-def load_engine(engine_data: dict) -> Optional[Engine]:
-    """Load engine from ``engine_data``.
+def load_engine(engine_setting: Dict[str, Any]) -> Optional[Engine]:
+    """Load engine from ``engine_setting``.
 
-    :param dict engine_data:  Attributes from YAML ``settings:engines/<engine>``
+    :param dict engine_setting:  Attributes from YAML ``settings:engines/<engine>``
     :return: initialized namespace of the ``<engine>``.
 
     1. create a namespace and load module of the ``<engine>``
     2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`
-    3. update namespace with values from ``engine_data``
+    3. update namespace with values from ``engine_setting``
 
     If engine *is active*, return namespace of the engine, otherwise return
     ``None``.
@@ -107,7 +130,7 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
 
     """
 
-    engine_name = engine_data['name']
+    engine_name = engine_setting['name']
     if '_' in engine_name:
         logger.error('Engine name contains underscore: "{}"'.format(engine_name))
         return None
@@ -115,10 +138,10 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
     if engine_name.lower() != engine_name:
         logger.warn('Engine name is not lowercase: "{}", converting to lowercase'.format(engine_name))
         engine_name = engine_name.lower()
-        engine_data['name'] = engine_name
+        engine_setting['name'] = engine_name
 
     # load_module
-    engine_module = engine_data['engine']
+    engine_module = engine_setting['engine']
     try:
         engine = load_module(engine_module + '.py', ENGINE_DIR)
     except (SyntaxError, KeyboardInterrupt, SystemExit, SystemError, ImportError, RuntimeError):
@@ -128,9 +151,10 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
         logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None
 
-    update_engine_attributes(engine, engine_data)
-    set_language_attributes(engine)
+    update_engine_attributes(engine, engine_setting)
     update_attributes_for_tor(engine)
+    if not set_engine_locales(engine):
+        set_language_attributes(engine)
 
     if not is_engine_active(engine):
         return None
@@ -165,15 +189,15 @@ def set_loggers(engine, engine_name):
             module.logger = logger.getChild(module_engine_name)
 
 
-def update_engine_attributes(engine: Engine, engine_data):
-    # set engine attributes from engine_data
-    for param_name, param_value in engine_data.items():
+def update_engine_attributes(engine: Engine, engine_setting: Dict[str, Any]):
+    # set engine attributes from engine_setting
+    for param_name, param_value in engine_setting.items():
         if param_name == 'categories':
             if isinstance(param_value, str):
                 param_value = list(map(str.strip, param_value.split(',')))
             engine.categories = param_value
         elif hasattr(engine, 'about') and param_name == 'about':
-            engine.about = {**engine.about, **engine_data['about']}
+            engine.about = {**engine.about, **engine_setting['about']}
         else:
             setattr(engine, param_name, param_value)
 
@@ -183,6 +207,28 @@ def update_engine_attributes(engine: Engine, engine_data):
             setattr(engine, arg_name, copy.deepcopy(arg_value))
 
 
+def set_engine_locales(engine: Engine):
+    engine_locales_key = None
+
+    if engine.name in ENGINES_LOCALES:
+        engine_locales_key = engine.name
+    elif engine.engine in ENGINES_LOCALES:
+        # The key of the dictionary engine_data_dict is the *engine name*
+        # configured in settings.xml.  When multiple engines are configured in
+        # settings.yml to use the same origin engine (python module) these
+        # additional engines can use the languages from the origin engine.
+        # For this use the configured ``engine: ...`` from settings.yml
+        engine_locales_key = engine.engine
+    else:
+        return False
+
+    print(engine.name, ENGINES_LOCALES[engine_locales_key])
+    engine.engine_locales = EngineLocales(**ENGINES_LOCALES[engine_locales_key])
+    # language_support
+    engine.language_support = len(engine.engine_locales.regions) > 0 or len(engine.engine_locales.languages) > 0
+    return True
+
+
 def set_language_attributes(engine: Engine):
     # assign supported languages from json file
     if engine.name in ENGINES_LANGUAGES:
@@ -225,17 +271,6 @@ def set_language_attributes(engine: Engine):
     # language_support
     engine.language_support = len(engine.supported_languages) > 0
 
-    # assign language fetching method if auxiliary method exists
-    if hasattr(engine, '_fetch_supported_languages'):
-        headers = {
-            'User-Agent': gen_useragent(),
-            'Accept-Language': "en-US,en;q=0.5",  # bing needs to set the English language
-        }
-        engine.fetch_supported_languages = (
-            # pylint: disable=protected-access
-            lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers))
-        )
-
 
 def update_attributes_for_tor(engine: Engine) -> bool:
     if using_tor_proxy(engine) and hasattr(engine, 'onion_url'):
@@ -294,8 +329,8 @@ def load_engines(engine_list):
     engine_shortcuts.clear()
     categories.clear()
     categories['general'] = []
-    for engine_data in engine_list:
-        engine = load_engine(engine_data)
+    for engine_setting in engine_list:
+        engine = load_engine(engine_setting)
         if engine:
             register_engine(engine)
     return engines