WIP

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/autocomplete.py

@@ -15,9 +15,6 @@ from searx.network import get as http_get
 from searx.exceptions import SearxEngineResponseException
 from searx.engines import engines
 
-# a _fetch_supported_properites() for XPath engines isn't available right now
-# _brave = ENGINES_LANGUAGES['brave'].keys()
-
 
 def get(*args, **kwargs):
     if 'timeout' not in kwargs:

searx/data/__init__.py

@@ -7,7 +7,7 @@
 """
 
 __all__ = [
-    'ENGINES_LANGUAGES',
+    'ENGINES_DATAS',
     'CURRENCIES',
     'USER_AGENTS',
     'EXTERNAL_URLS',
@@ -42,7 +42,7 @@ def ahmia_blacklist_loader():
         return f.read().split()
 
 
-ENGINES_LANGUAGES = _load('engines_languages.json')
+ENGINES_DATAS = _load('engines_datas.json')
 CURRENCIES = _load('currencies.json')
 USER_AGENTS = _load('useragents.json')
 EXTERNAL_URLS = _load('external_urls.json')

searx/engines/__init__.py

@@ -12,19 +12,22 @@ usage::
 """
 
 import sys
+import json
 import copy
+import pathlib
 import dataclasses
 from typing import Dict, List, Optional
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
 from searx import logger, settings
-from searx.data import ENGINES_LANGUAGES
 from searx.utils import load_module, match_language
-
+from searx.data import data_dir, ENGINES_DATAS
 
 logger = logger.getChild('engines')
-ENGINE_DIR = dirname(realpath(__file__))
+
+ENGINE_DIR = pathlib.Path(__file__).parent
+
 BABEL_LANGS = [
     lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]
     for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())
@@ -52,32 +55,58 @@ OTHER_CATEGORY = 'other'
 
 
 @dataclasses.dataclass
-class EngineProperties(dict):
-    """
-
-    The class is intended to be instanciated for each engine.
-    """
+class EngineData:
+    """The class is intended to be instanciated for each engine."""
 
     regions: Dict[str, str] = dataclasses.field(default_factory=dict)
     """
-    {
-        'fr-BE' : <engine's region name>
-    },
+    .. code:: python
+
+       {
+           'fr-BE' : <engine's region name>,
+       }
     """
 
     languages: Dict[str, str] = dataclasses.field(default_factory=dict)
     """
-    {
-        'ca' : <engine's language name>
-    },
+    .. code:: python
+
+       {
+           'ca' : <engine's language name>,
+       }
     """
 
-    def asdict(self):
-        return {
-            'type': 'engine_properties',
-            'regions': self.regions,
-            'languages': self.languages,
-        }
+    data_type: str = 'engine_data'
+    """Data type, default is 'engine_data' for vintage use 'supported_languages'"""
+
+
+class EngineDataEncoder(json.JSONEncoder):
+    """Encodes :class:`EngineData` to a serializable object, see
+    :class:`json.JSONEncoder`."""
+
+    def default(self, o):
+        """Return dictionary of a :class:`EngineData` object."""
+        if isinstance(o, EngineData):
+            return o.__dict__
+        return super().default(o)
+
+
+class EngineDataDict(dict):
+    """A python dictionary to map :class:`EngineData` by engine name."""
+
+    ENGINE_DATA_FILE = (data_dir / 'engines_datas.json').resolve()
+
+    def save_data(self):
+        with open(self.ENGINE_DATA_FILE, 'w', encoding='utf-8') as f:
+            json.dump(self, f, indent=2, sort_keys=True, cls=EngineDataEncoder)
+
+    @classmethod
+    def from_data(cls):
+        """Instantiate :class:`EngineDataDict` object from :py:obj:`ENGINES_DATAS`"""
+        obj = EngineDataDict()
+        for k, v in ENGINES_DATAS.items():
+            obj[k] = EngineData(**v)
+        return obj
 
 
 class Engine:  # pylint: disable=too-few-public-methods
@@ -96,7 +125,7 @@ class Engine:  # pylint: disable=too-few-public-methods
     safesearch: bool
     time_range_support: bool
     timeout: float
-    properties: EngineProperties
+    engine_data: EngineData
 
 
 # Defaults for the namespace of an engine module, see :py:func:`load_engine`
@@ -110,18 +139,19 @@ engine_shortcuts = {}
 
     engine_shortcuts[engine.shortcut] = engine.name
 
+:meta hide-value:
 """
 
 
-def load_engine(engine_data: dict) -> Optional[Engine]:
-    """Load engine from ``engine_data``.
+def load_engine(engine_setting: dict) -> Optional[Engine]:
+    """Load engine from ``engine_setting``.
 
-    :param dict engine_data:  Attributes from YAML ``settings:engines/<engine>``
+    :param dict engine_setting:  Attributes from YAML ``settings:engines/<engine>``
     :return: initialized namespace of the ``<engine>``.
 
     1. create a namespace and load module of the ``<engine>``
     2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`
-    3. update namespace with values from ``engine_data``
+    3. update namespace with values from ``engine_setting``
 
     If engine *is active*, return namespace of the engine, otherwise return
     ``None``.
@@ -135,7 +165,7 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
 
     """
 
-    engine_name = engine_data['name']
+    engine_name = engine_setting['name']
     if '_' in engine_name:
         logger.error('Engine name contains underscore: "{}"'.format(engine_name))
         return None
@@ -143,10 +173,10 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
     if engine_name.lower() != engine_name:
         logger.warn('Engine name is not lowercase: "{}", converting to lowercase'.format(engine_name))
         engine_name = engine_name.lower()
-        engine_data['name'] = engine_name
+        engine_setting['name'] = engine_name
 
     # load_module
-    engine_module = engine_data['engine']
+    engine_module = engine_setting['engine']
     try:
         engine = load_module(engine_module + '.py', ENGINE_DIR)
     except (SyntaxError, KeyboardInterrupt, SystemExit, SystemError, ImportError, RuntimeError):
@@ -156,7 +186,7 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
         logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None
 
-    update_engine_attributes(engine, engine_data)
+    update_engine_attributes(engine, engine_setting)
     set_language_attributes(engine)
     update_attributes_for_tor(engine)
 
@@ -193,15 +223,15 @@ def set_loggers(engine, engine_name):
             module.logger = logger.getChild(module_engine_name)
 
 
-def update_engine_attributes(engine: Engine, engine_data):
-    # set engine attributes from engine_data
-    for param_name, param_value in engine_data.items():
+def update_engine_attributes(engine: Engine, engine_setting):
+    # set engine attributes from engine_setting
+    for param_name, param_value in engine_setting.items():
         if param_name == 'categories':
             if isinstance(param_value, str):
                 param_value = list(map(str.strip, param_value.split(',')))
             engine.categories = param_value
         elif hasattr(engine, 'about') and param_name == 'about':
-            engine.about = {**engine.about, **engine_data['about']}
+            engine.about = {**engine.about, **engine_setting['about']}
         else:
             setattr(engine, param_name, param_value)
 
@@ -211,33 +241,36 @@ def update_engine_attributes(engine: Engine, engine_data):
             setattr(engine, arg_name, copy.deepcopy(arg_value))
 
 
-def set_language_attributes(engine: Engine):
+def set_language_attributes(engine: Engine):  # pylint: disable=too-many-branches
     # assign supported languages from json file
 
-    supported_properties = None
+    engine_data_dict = EngineDataDict.from_data()
+    engine_data = None
 
-    if engine.name in ENGINES_LANGUAGES:
-        supported_properties = ENGINES_LANGUAGES[engine.name]
+    if engine.name in engine_data_dict:
+        engine_data = engine_data_dict[engine.name]
 
-    elif engine.engine in ENGINES_LANGUAGES:
-        # The key of the dictionary ENGINES_LANGUAGES is the *engine name*
+    elif engine.engine in engine_data_dict:
+        # The key of the dictionary engine_data_dict is the *engine name*
         # configured in settings.xml.  When multiple engines are configured in
         # settings.yml to use the same origin engine (python module) these
         # additional engines can use the languages from the origin engine.
         # For this use the configured ``engine: ...`` from settings.yml
-        supported_properties = ENGINES_LANGUAGES[engine.engine]
+        engine_data = engine_data_dict[engine.engine]
 
-    if not supported_properties:
+    if not engine_data:
         return
 
-    if isinstance(supported_properties, dict) and supported_properties.get('type') == 'engine_properties':
-        engine.supported_properties = supported_properties
-        engine.language_support = len(supported_properties['languages']) or len(supported_properties['regions'])
+    if engine_data.data_type == 'engine_data':
+        engine.engine_data = engine_data
+        engine.language_support = len(engine_data.languages) or len(engine_data.regions)
+
+    elif engine_data.data_type == 'supported_languages':
+        # vintage
 
-    else:
         # depricated: does not work for engines that do support languages
         # based on a region.
-        engine.supported_languages = supported_properties
+        engine.supported_languages = engine_data.languages
         engine.language_support = len(engine.supported_languages) > 0
 
         if hasattr(engine, 'language'):
@@ -268,6 +301,9 @@ def set_language_attributes(engine: Engine):
             ):
                 engine.language_aliases[iso_lang] = engine_lang
 
+    else:
+        raise TypeError('unknown type of engine data: %s' % engine_data.data_type)
+
 
 def update_attributes_for_tor(engine: Engine) -> bool:
     if using_tor_proxy(engine) and hasattr(engine, 'onion_url'):

searx/engines/google.py

@@ -136,7 +136,7 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
     :param dict param: request parameters of the engine
 
     :param list lang_list: list of supported languages of the engine
-        :py:obj:`ENGINES_LANGUAGES[engine-name] <searx.data.ENGINES_LANGUAGES>`
+        :py:obj:`ENGINES_DATAS[engine-name].languages <searx.data.ENGINES_DATAS>`
 
     :param dict lang_list: custom aliases for non standard language codes
         (used when calling :py:func:`searx.utils.match_language`)

searx/engines/qwant.py

@@ -49,7 +49,7 @@ about = {
 # engine dependent config
 categories = []
 paging = True
-supported_properties_url = about['website']
+engine_data_url = about['website']
 qwant_categ = None  # web|news|inages|videos
 
 safesearch = True
@@ -95,7 +95,7 @@ def request(query, params):
     )
 
     # add quant's locale
-    q_locale = get_engine_locale(params['language'], supported_properties['regions'], default='en_US')
+    q_locale = get_engine_locale(params['language'], engine_data.regions, default='en_US')
     params['url'] += '&locale=' + q_locale
 
     # add safesearch option
@@ -243,7 +243,7 @@ def response(resp):
     return results
 
 
-def _fetch_engine_properties(resp, engine_properties):
+def _fetch_engine_data(resp, engine_data):
 
     text = resp.text
     text = text[text.find('INITIAL_PROPS') :]
@@ -270,7 +270,7 @@ def _fetch_engine_properties(resp, engine_properties):
             print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
             continue
 
-        # note: engine_properties.regions (dict)
+        # note: engine_data.regions (dict)
         #
         #   dict's key is a string build up from a babel.Locale object / the
         #   notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
@@ -278,6 +278,6 @@ def _fetch_engine_properties(resp, engine_properties):
         #   the engine.
 
         searxng_locale = locale.language + '-' + locale.territory  # --> params['language']
-        engine_properties.regions[searxng_locale] = q_locale
+        engine_data.regions[searxng_locale] = q_locale
 
-    return engine_properties
+    return engine_data

searx/engines/startpage.py

@@ -48,7 +48,7 @@ filter_mapping = {0: '0', 1: '1', 2: '1'}
 time_range_support = True
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 
-supported_properties_url = 'https://www.startpage.com/do/settings'
+engine_data_url = 'https://www.startpage.com/do/settings'
 
 # search-url
 base_url = 'https://www.startpage.com/'
@@ -249,7 +249,7 @@ def response(resp):
     return results
 
 
-def _fetch_engine_properties(resp, engine_properties):
+def _fetch_engine_data(resp, engine_data):
 
     # startpage's language & region selectors are a mess.
     #
@@ -341,7 +341,7 @@ def _fetch_engine_properties(resp, engine_properties):
 
         region_tag = locale.language + '-' + locale.territory
         # print("SearXNG locale tag: %s --> Engine tag: %s" % (region_tag, engine_region_tag))
-        engine_properties.regions[region_tag] = engine_region_tag
+        engine_data.regions[region_tag] = engine_region_tag
 
     # languages
 
@@ -385,6 +385,6 @@ def _fetch_engine_properties(resp, engine_properties):
             lang_code = catalog_engine2code[name]
 
         # print("SearXNG language tag: %s --> Engine tag: %s" % (lang_code, engine_lang))
-        engine_properties.languages[lang_code] = engine_lang
+        engine_data.languages[lang_code] = engine_lang
 
-    return engine_properties
+    return engine_data

searx/languages.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # list of language codes
-# this file is generated automatically by utils/fetch_languages.py
+# this file is generated automatically by:
+#
+#   ./manage pyenv.cmd searxng_extra/update/update_languages.py
 language_codes = (
     ('af-ZA', 'Afrikaans', 'Suid-Afrika', 'Afrikaans', '\U0001f1ff\U0001f1e6'),
     ('ar-EG', 'العربية', 'مصر', 'Arabic', '\U0001f1ea\U0001f1ec'),

searx/locales.py

@@ -19,13 +19,14 @@ from searx import logger
 
 logger = logger.getChild('locales')
 
+LANGUAGES_FILE = pathlib.Path(__file__).parent / 'languages.py'
 
 # safe before monkey patching flask_babel.get_translations
 _flask_babel_get_translations = flask_babel.get_translations
 
 LOCALE_NAMES = {}
-"""Mapping of locales and their description.  Locales e.g. 'fr' or 'pt-BR' (see
-:py:obj:`locales_initialize`)."""
+"""Mapping of locales and their description.  Locales e.g. ``fr`` or ``pt-BR``
+(see :py:obj:`locales_initialize`)."""
 
 RTL_LOCALES: Set[str] = set()
 """List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (see
@@ -161,6 +162,8 @@ def get_engine_locale(searxng_locale, engine_locales, default=None):
     Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to
     corresponding *engine locales*:
 
+    .. code:: python
+
       <engine>: {
           # SearXNG string : engine-string
           'ca-ES'          : 'ca_ES',