[fix] move locale code from webapp.py to locales.py and fix #1303

To improve modularization this patch:

- moves *locale* related implementation from the webapp.py application to the
  locale.py module.

- The initialization of the locales is now done in the application (webapp) and
  is no longer done while importing searx.locales.

In the searx.locales module a new dictionary named `LOCALE_BEST_MATCH` has been
added.  In this dictionary we can map languages without a translation to
languages we have a translation for.

To fix #1303 zh-HK has been mapped to zh-Hant-TW (we do not need additional
translations of traditional Chinese)

Closes: https://github.com/searxng/searxng/issues/1303
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/locales.py

@@ -9,21 +9,104 @@ import os
 import pathlib
 
 from babel import Locale
+from babel.support import Translations
+import flask_babel
+import flask
+from flask.ctx import has_request_context
+from searx import logger
 
-LOCALE_NAMES = {
-    "oc": "Occitan",
-    "nl-BE": "Vlaams (Dutch, Belgium)",
-    "szl": "Ślōnski (Silesian)",
-}
-"""Mapping of locales and their description.  Locales e.g. 'fr' or 'pt-BR'
-(delimiter is *underline* '-')"""
+logger = logger.getChild('locales')
+
+
+# safe before monkey patching flask_babel.get_translations
+_flask_babel_get_translations = flask_babel.get_translations
+
+LOCALE_NAMES = {}
+"""Mapping of locales and their description.  Locales e.g. 'fr' or 'pt-BR' (see
+:py:obj:`locales_initialize`)."""
 
 RTL_LOCALES: Set[str] = set()
-"""List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (delimiter is
-*underline* '-')"""
+"""List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (see
+:py:obj:`locales_initialize`)."""
+
+ADDITIONAL_TRANSLATIONS = {
+    "oc": "Occitan",
+    "szl": "Ślōnski (Silesian)",
+}
+"""Additional languages SearXNG has translations for but not supported by
+python-babel (see :py:obj:`locales_initialize`)."""
+
+LOCALE_BEST_MATCH = {
+    "oc": 'fr-FR',
+    "szl": "pl",
+    "nl-BE": "nl",
+    "zh-HK": "zh-Hant-TW",
+}
+"""Map a locale we do not have a translations for to a locale we have a
+translation for. By example: use Taiwan version of the translation for Hong
+Kong."""
 
 
-def _get_name(locale, language_code):
+def localeselector():
+    locale = 'en'
+    if has_request_context():
+        value = flask.request.preferences.get_value('locale')
+        if value:
+            locale = value
+
+    # first, set the language that is not supported by babel
+    if locale in ADDITIONAL_TRANSLATIONS:
+        flask.request.form['use-translation'] = locale
+
+    # second, map locale to a value python-babel supports
+    locale = LOCALE_BEST_MATCH.get(locale, locale)
+
+    if locale == '':
+        # if there is an error loading the preferences
+        # the locale is going to be ''
+        locale = 'en'
+
+    # babel uses underscore instead of hyphen.
+    locale = locale.replace('-', '_')
+    return locale
+
+
+def get_translations():
+    """Monkey patch of flask_babel.get_translations"""
+    if has_request_context() and flask.request.form.get('use-translation') == 'oc':
+        babel_ext = flask_babel.current_app.extensions['babel']
+        return Translations.load(next(babel_ext.translation_directories), 'oc')
+    if has_request_context() and flask.request.form.get('use-translation') == 'szl':
+        babel_ext = flask_babel.current_app.extensions['babel']
+        return Translations.load(next(babel_ext.translation_directories), 'szl')
+    return _flask_babel_get_translations()
+
+
+def get_locale_descr(locale, locale_name):
+    """Get locale name e.g. 'Français - fr' or 'Português (Brasil) - pt-BR'
+
+    :param locale: instance of :py:class:`Locale`
+    :param locale_name: name e.g. 'fr'  or 'pt_BR' (delimiter is *underscore*)
+    """
+
+    native_language, native_territory = _get_locale_descr(locale, locale_name)
+    english_language, english_territory = _get_locale_descr(locale, 'en')
+
+    if native_territory == english_territory:
+        english_territory = None
+
+    if not native_territory and not english_territory:
+        if native_language == english_language:
+            return native_language
+        return native_language + ' (' + english_language + ')'
+
+    result = native_language + ', ' + native_territory + ' (' + english_language
+    if english_territory:
+        return result + ', ' + english_territory + ')'
+    return result + ')'
+
+
+def _get_locale_descr(locale, language_code):
     language_name = locale.get_language_name(language_code).capitalize()
     if language_name and ('a' <= language_name[0] <= 'z'):
         language_name = language_name.capitalize()
@@ -31,39 +114,34 @@ def _get_name(locale, language_code):
     return language_name, terrirtory_name
 
 
-def _get_locale_name(locale, locale_name):
-    """Get locale name e.g. 'Français - fr' or 'Português (Brasil) - pt-BR'
+def locales_initialize(directory=None):
+    """Initialize locales environment of the SearXNG session.
 
-    :param locale: instance of :py:class:`Locale`
-    :param locale_name: name e.g. 'fr'  or 'pt_BR' (delimiter is *underscore*)
+    - monkey patch :py:obj:`flask_babel.get_translations` by :obj:py:`get_translations`
+    - init global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`
     """
-    native_language, native_territory = _get_name(locale, locale_name)
-    english_language, english_territory = _get_name(locale, 'en')
-    if native_territory == english_territory:
-        english_territory = None
-    if not native_territory and not english_territory:
-        if native_language == english_language:
-            return native_language
-        return native_language + ' (' + english_language + ')'
-    result = native_language + ', ' + native_territory + ' (' + english_language
-    if english_territory:
-        return result + ', ' + english_territory + ')'
-    return result + ')'
 
+    directory = directory or pathlib.Path(__file__).parent / 'translations'
+    logger.debug("locales_initialize: %s", directory)
+    flask_babel.get_translations = get_translations
+
+    for tag, descr in ADDITIONAL_TRANSLATIONS.items():
+        LOCALE_NAMES[tag] = descr
+
+    for tag in LOCALE_BEST_MATCH:
+        descr = LOCALE_NAMES.get(tag)
+        if not descr:
+            locale = Locale.parse(tag, sep='-')
+            LOCALE_NAMES[tag] = get_locale_descr(locale, tag.replace('-', '_'))
 
-def initialize_locales(directory):
-    """Initialize global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`."""
     for dirname in sorted(os.listdir(directory)):
         # Based on https://flask-babel.tkte.ch/_modules/flask_babel.html#Babel.list_translations
         if not os.path.isdir(os.path.join(directory, dirname, 'LC_MESSAGES')):
             continue
-        locale_name = dirname.replace('_', '-')
-        info = LOCALE_NAMES.get(locale_name)
-        if not info:
+        tag = dirname.replace('_', '-')
+        descr = LOCALE_NAMES.get(tag)
+        if not descr:
             locale = Locale.parse(dirname)
-            LOCALE_NAMES[locale_name] = _get_locale_name(locale, dirname)
+            LOCALE_NAMES[tag] = get_locale_descr(locale, dirname)
             if locale.text_direction == 'rtl':
-                RTL_LOCALES.add(locale_name)
-
-
-initialize_locales(pathlib.Path(__file__).parent / 'translations')
+                RTL_LOCALES.add(tag)

searx/webapp.py

@@ -40,11 +40,8 @@ from flask import (
     send_from_directory,
 )
 from flask.wrappers import Response
-from flask.ctx import has_request_context
 from flask.json import jsonify
 
-from babel.support import Translations
-import flask_babel
 from flask_babel import (
     Babel,
     gettext,
@@ -114,11 +111,16 @@ from searx.metrics import (
 )
 from searx.flaskfix import patch_application
 
-# renaming names from searx imports ...
+from searx.locales import (
+    LOCALE_NAMES,
+    RTL_LOCALES,
+    localeselector,
+    locales_initialize,
+)
 
+# renaming names from searx imports ...
 from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
 from searx.languages import language_codes as languages
-from searx.locales import LOCALE_NAMES, RTL_LOCALES
 from searx.search import SearchWithPlugins, initialize as search_initialize
 from searx.network import stream as http_stream, set_context_network_name
 from searx.search.checker import get_result as checker_get_result
@@ -148,7 +150,6 @@ STATS_SORT_PARAMETERS = {
     'time': (False, 'total', 0),
     'reliability': (False, 'reliability', 100),
 }
-_INFO_PAGES = infopage.InfoPageSet()
 
 # Flask app
 app = Flask(__name__, static_folder=settings['ui']['static_path'], template_folder=templates_path)
@@ -192,10 +193,6 @@ exception_classname_to_text = {
 }
 
 
-# monkey patch for flask_babel.get_translations
-_flask_babel_get_translations = flask_babel.get_translations
-
-
 class ExtendedRequest(flask.Request):
     """This class is never initialized and only used for type checking."""
 
@@ -211,40 +208,9 @@ class ExtendedRequest(flask.Request):
 request = typing.cast(ExtendedRequest, flask.request)
 
 
-def _get_translations():
-    if has_request_context() and request.form.get('use-translation') == 'oc':
-        babel_ext = flask_babel.current_app.extensions['babel']
-        return Translations.load(next(babel_ext.translation_directories), 'oc')
-    if has_request_context() and request.form.get('use-translation') == 'szl':
-        babel_ext = flask_babel.current_app.extensions['babel']
-        return Translations.load(next(babel_ext.translation_directories), 'szl')
-    return _flask_babel_get_translations()
-
-
-flask_babel.get_translations = _get_translations
-
-
 @babel.localeselector
 def get_locale():
-    locale = 'en'
-
-    if has_request_context():
-        value = request.preferences.get_value('locale')
-        if value:
-            locale = value
-
-    if locale == 'oc':
-        request.form['use-translation'] = 'oc'
-        locale = 'fr_FR'
-    if locale == 'szl':
-        request.form['use-translation'] = 'szl'
-        locale = 'pl'
-    if locale == '':
-        # if there is an error loading the preferences
-        # the locale is going to be ''
-        locale = 'en'
-    # babel uses underscore instead of hyphen.
-    locale = locale.replace('-', '_')
+    locale = localeselector()
     logger.debug("%s uses locale `%s`", urllib.parse.quote(request.url), locale)
     return locale
 
@@ -564,12 +530,14 @@ def pre_request():
     if not preferences.get_value("language"):
         language = _get_browser_language(request, settings['search']['languages'])
         preferences.parse_dict({"language": language})
+        logger.debug('set language %s (from browser)', preferences.get_value("language"))
 
     # locale is defined neither in settings nor in preferences
     # use browser headers
     if not preferences.get_value("locale"):
         locale = _get_browser_language(request, LOCALE_NAMES.keys())
         preferences.parse_dict({"locale": locale})
+        logger.debug('set locale %s (from browser)', preferences.get_value("locale"))
 
     # request.user_plugins
     request.user_plugins = []  # pylint: disable=assigning-non-slot
@@ -1415,6 +1383,8 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
 
 # initialize the engines except on the first run of the werkzeug server.
 if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
+    locales_initialize()
+    _INFO_PAGES = infopage.InfoPageSet()
     plugin_initialize(app)
     search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
 

tests/unit/test_preferences.py

@@ -1,3 +1,4 @@
+from searx.locales import locales_initialize
 from searx.preferences import (
     EnumStringSetting,
     MapSetting,
@@ -8,6 +9,8 @@ from searx.preferences import (
 )
 from tests import SearxTestCase
 
+locales_initialize()
+
 
 class PluginStub:
     def __init__(self, plugin_id, default_on):