[fix] get correct locale with country from browser

Some of our interface locales include uppercase country codes,
which are separated by `_` instead of the more common `-`.
Also, a browser's `Accept-Language` header could be in lowercase.

This commit attempts to normalize those cases so a browser's
language+country codes can better match with our locales.

This solution assumes that our UI locales have nothing more than
language and optionally country. If we ever add a script specific
locale like `zh-Hant-TW` this would have to change to accomodate
that, but the idea would be pretty much the same as this fix.

searx/webapp.py

@@ -142,6 +142,7 @@ babel = Babel(app)
 
 rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'fa_IR', 'glk', 'he',
                'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
+ui_locale_codes = [l.replace('_', '-') for l in settings['locales'].keys()]
 
 # used when translating category names
 _category_names = (gettext('files'),
@@ -175,6 +176,9 @@ def _get_browser_or_settings_language(request, lang_list):
     for lang in request.headers.get("Accept-Language", "en").split(","):
         if ';' in lang:
             lang = lang.split(';')[0]
+        if '-' in lang:
+            lang_parts = lang.split('-')
+            lang = "{}-{}".format(lang_parts[0], lang_parts[-1].upper())
         locale = match_language(lang, lang_list, fallback=None)
         if locale is not None:
             return locale
@@ -194,13 +198,10 @@ def get_locale():
         locale_source = 'preferences'
     else:
         # use local from the browser
-        locale = _get_browser_or_settings_language(request, settings['locales'].keys())
+        locale = _get_browser_or_settings_language(request, ui_locale_codes)
+        locale = locale.replace('-', '_')
         locale_source = 'browser'
 
-    #
-    if locale == 'zh_TW':
-        locale = 'zh_Hant_TW'
-
     # see _get_translations function
     # and https://github.com/searx/searx/pull/1863
     if locale == 'oc':

tests/unit/test_webapp.py

@@ -198,6 +198,20 @@ class ViewsTestCase(SearxTestCase):
             result.data
         )
 
+    def test_browser_locale(self):
+        result = self.app.get('/preferences', headers={'Accept-Language': 'zh-tw;q=0.8'})
+        self.assertEqual(result.status_code, 200)
+        self.assertIn(
+            b'<option value="zh_TW" selected="selected">',
+            result.data,
+            'Interface locale ignored browser preference.'
+        )
+        self.assertIn(
+            b'<option value="zh-TW" selected="selected">',
+            result.data,
+            'Search language ignored browser preference.'
+        )
+
     def test_stats(self):
         result = self.app.get('/stats')
         self.assertEqual(result.status_code, 200)