[mod] engines_languages.json: add new type engine_properties

This patch adds the boilerplate code, needed to fetch properties from engines.
In the past we only fetched *languages* but some engines need *regions* to
parameterize the engine request.

To fit into our *fetch language* procedures the boilerplate is implemented in
the `searxng_extra/update/update_languages.py` and the *engine_properties* are
stored along in the `searx/data/engines_languages.json`.

This implementation is downward compatible to the `_fetch_fetch_languages()`
infrastructure we have.  If there comes the day we have all
`_fetch_fetch_languages()` implementations moved to `_fetch_engine_properties()`
implementations, we can rename the files and scripts.

The new type `engine_properties` is a dictionary with keys `languages` and
`regions`.  The values are dictionaries to map from SearXNG's language & region
to option values the engine does use::

    engine_properties = {
        'type' : 'engine_properties',  # <-- !!!
        'regions': {
            # 'ca-ES' : <engine's region name>
        },
        'languages': {
            # 'ca' : <engine's language name>
        },
    }

Similar to the `supported_languages`, in the engine the properties are available
under the name `supported_properties`.

Initial we start with languages & regions, but in a wider sense the type is
named *engine properties*.  Engines can store in whatever options they need and
may be in the future there is a need to fetch additional or complete different
properties.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/autocomplete.py

@@ -15,7 +15,7 @@ from searx.data import ENGINES_LANGUAGES
 from searx.network import get as http_get
 from searx.exceptions import SearxEngineResponseException
 
-# a fetch_supported_languages() for XPath engines isn't available right now
+# a _fetch_supported_properites() for XPath engines isn't available right now
 # _brave = ENGINES_LANGUAGES['brave'].keys()
 
 

searx/engines/__init__.py

@@ -19,8 +19,7 @@ from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
 from searx import logger, settings
 from searx.data import ENGINES_LANGUAGES
-from searx.network import get
-from searx.utils import load_module, match_language, gen_useragent
+from searx.utils import load_module, match_language
 
 
 logger = logger.getChild('engines')
@@ -36,8 +35,7 @@ ENGINE_DEFAULT_ARGS = {
     "timeout": settings["outgoing"]["request_timeout"],
     "shortcut": "-",
     "categories": ["general"],
-    "supported_languages": [],
-    "language_aliases": {},
+    "language_support" : False,
     "paging": False,
     "safesearch": False,
     "time_range_support": False,
@@ -58,10 +56,10 @@ class Engine:  # pylint: disable=too-few-public-methods
     engine: str
     shortcut: str
     categories: List[str]
-    supported_languages: List[str]
     about: dict
     inactive: bool
     disabled: bool
+    # language support, either by selecting a region or by selecting a language
     language_support: bool
     paging: bool
     safesearch: bool
@@ -143,6 +141,25 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
 
     return engine
 
+def engine_properties_template():
+    """A dictionary with languages and regions to map from SearXNG' languages &
+    region tags to engine's language & region tags::
+
+      engine_properties = {
+          'type' : 'engine_properties',
+          'regions': {
+              # 'ca-ES' : <engine's region name>
+          },
+          'languages': {
+              # 'ca' : <engine's language name>
+          },
+      }
+    """
+    return {
+        'type' : 'engine_properties',
+        'regions': {},
+        'languages': {},
+    }
 
 def set_loggers(engine, engine_name):
     # set the logger for engine
@@ -179,8 +196,11 @@ def update_engine_attributes(engine: Engine, engine_data):
 
 def set_language_attributes(engine: Engine):
     # assign supported languages from json file
+
+    supported_properties = None
+
     if engine.name in ENGINES_LANGUAGES:
-        engine.supported_languages = ENGINES_LANGUAGES[engine.name]
+        supported_properties = ENGINES_LANGUAGES[engine.name]
 
     elif engine.engine in ENGINES_LANGUAGES:
         # The key of the dictionary ENGINES_LANGUAGES is the *engine name*
@@ -188,47 +208,48 @@ def set_language_attributes(engine: Engine):
         # settings.yml to use the same origin engine (python module) these
         # additional engines can use the languages from the origin engine.
         # For this use the configured ``engine: ...`` from settings.yml
-        engine.supported_languages = ENGINES_LANGUAGES[engine.engine]
+        supported_properties = ENGINES_LANGUAGES[engine.engine]
 
-    if hasattr(engine, 'language'):
-        # For an engine, when there is `language: ...` in the YAML settings, the
-        # engine supports only one language, in this case
-        # engine.supported_languages should contains this value defined in
-        # settings.yml
-        if engine.language not in engine.supported_languages:
-            raise ValueError(
-                "settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language)
-            )
+    if not supported_properties:
+        return
 
-        if isinstance(engine.supported_languages, dict):
-            engine.supported_languages = {engine.language: engine.supported_languages[engine.language]}
-        else:
-            engine.supported_languages = [engine.language]
+    if isinstance(supported_properties, dict) and supported_properties.get('type') == 'engine_properties':
+        engine.supported_properties = supported_properties
+        engine.language_support = len(supported_properties['languages']) or len(supported_properties['regions'])
 
-    # find custom aliases for non standard language codes
-    for engine_lang in engine.supported_languages:
-        iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
-        if (
-            iso_lang
-            and iso_lang != engine_lang
-            and not engine_lang.startswith(iso_lang)
-            and iso_lang not in engine.supported_languages
-        ):
-            engine.language_aliases[iso_lang] = engine_lang
+    else:
+        # depricated: does not work for engines that do support languages
+        # based on a region.
+        engine.supported_languages = supported_properties
+        engine.language_support = len(engine.supported_languages) > 0
 
-    # language_support
-    engine.language_support = len(engine.supported_languages) > 0
+        if hasattr(engine, 'language'):
+            # For an engine, when there is `language: ...` in the YAML settings, the
+            # engine supports only one language, in this case
+            # engine.supported_languages should contains this value defined in
+            # settings.yml
+            if engine.language not in engine.supported_languages:
+                raise ValueError(
+                    "settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language)
+                )
 
-    # assign language fetching method if auxiliary method exists
-    if hasattr(engine, '_fetch_supported_languages'):
-        headers = {
-            'User-Agent': gen_useragent(),
-            'Accept-Language': "en-US,en;q=0.5",  # bing needs to set the English language
-        }
-        engine.fetch_supported_languages = (
-            # pylint: disable=protected-access
-            lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers))
-        )
+            if isinstance(engine.supported_languages, dict):
+                engine.supported_languages = {engine.language: engine.supported_languages[engine.language]}
+            else:
+                engine.supported_languages = [engine.language]
+
+        if not hasattr(engine, 'language_aliases'):
+            engine.language_aliases = {}
+        # find custom aliases for non standard language codes
+        for engine_lang in engine.supported_languages:
+            iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
+            if (
+                    iso_lang
+                    and iso_lang != engine_lang
+                    and not engine_lang.startswith(iso_lang)
+                    and iso_lang not in engine.supported_languages
+            ):
+                engine.language_aliases[iso_lang] = engine_lang
 
 
 def update_attributes_for_tor(engine: Engine) -> bool:

searx/engines/google_scholar.py

@@ -48,7 +48,6 @@ about = {
 # engine dependent config
 categories = ['science']
 paging = True
-language_support = True
 use_locale_domain = True
 time_range_support = True
 safesearch = False

searx/engines/google_videos.py

@@ -56,7 +56,6 @@ about = {
 
 categories = ['videos', 'web']
 paging = False
-language_support = True
 use_locale_domain = True
 time_range_support = True
 safesearch = True

searx/engines/yahoo_news.py

@@ -32,7 +32,6 @@ about = {
     "results": 'HTML',
 }
 
-language_support = False
 time_range_support = False
 safesearch = False
 paging = True

searx/engines/youtube_noapi.py

@@ -21,7 +21,6 @@ about = {
 # engine dependent config
 categories = ['videos', 'music']
 paging = True
-language_support = False
 time_range_support = True
 
 # search-url