From c9cd376186d12d2d281e655d0b5539d1359fe148 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 10 Oct 2022 19:31:22 +0200 Subject: [PATCH] [mod] replace searx.languages by searx.sxng_locales With the language and region tags from the EngineTraitsMap the handling of SearXNG's tags of languages and regions has been normalized and is no longer a *mystery*. The "languages" became "locales" that are supported by babel and by this, the update_engine_traits.py can be simplified a lot. Other code places can be simplified as well, but these simplifications should (respectively can) only be done when none of the engines work with the deprecated EngineTraits.supported_languages interface anymore. This commit replaces searx.languages by searx.sxng_locales and fix the naming of some names from "language" to "locale" (e.g. language_codes --> sxng_locales). Signed-off-by: Markus Heiser --- docs/src/searx.locales.rst | 12 + searx/query.py | 6 +- searx/settings_defaults.py | 8 +- searx/{languages.py => sxng_locales.py} | 83 +++- searx/templates/simple/filters/languages.html | 8 +- searx/templates/simple/preferences.html | 8 +- searx/utils.py | 10 +- searx/webapp.py | 4 +- searxng_extra/update/update_engine_traits.py | 394 ++++++------------ searxng_extra/update/update_osm_keys_tags.py | 4 +- 10 files changed, 233 insertions(+), 304 deletions(-) rename searx/{languages.py => sxng_locales.py} (54%) diff --git a/docs/src/searx.locales.rst b/docs/src/searx.locales.rst index 579247aff..2f13bfca1 100644 --- a/docs/src/searx.locales.rst +++ b/docs/src/searx.locales.rst @@ -4,5 +4,17 @@ Locales ======= +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + .. automodule:: searx.locales :members: + + +SearXNG's locale codes +====================== + +.. automodule:: searx.sxng_locales + :members: diff --git a/searx/query.py b/searx/query.py index dbc52ec75..751308baa 100644 --- a/searx/query.py +++ b/searx/query.py @@ -4,7 +4,7 @@ from abc import abstractmethod, ABC import re from searx import settings -from searx.languages import language_codes +from searx.sxng_locales import sxng_locales from searx.engines import categories, engines, engine_shortcuts from searx.external_bang import get_bang_definition_and_autocomplete from searx.search import EngineRef @@ -84,7 +84,7 @@ class LanguageParser(QueryPartParser): found = False # check if any language-code is equal with # declared language-codes - for lc in language_codes: + for lc in sxng_locales: lang_id, lang_name, country, english_name, _flag = map(str.lower, lc) # if correct language-code is found @@ -125,7 +125,7 @@ class LanguageParser(QueryPartParser): self.raw_text_query.autocomplete_list.append(lang) return - for lc in language_codes: + for lc in sxng_locales: if lc[0] not in settings['search']['languages']: continue lang_id, lang_name, country, english_name, _flag = map(str.lower, lc) diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 6e98076ff..7f657aa54 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -12,13 +12,13 @@ import logging from base64 import b64decode from os.path import dirname, abspath -from searx.languages import language_codes as languages +from .sxng_locales import sxng_locales searx_dir = abspath(dirname(__file__)) logger = logging.getLogger('searx') OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss'] -LANGUAGE_CODES = ['all', 'auto'] + list(l[0] for l in languages) +SXNG_LOCALE_TAGS = ['all', 'auto'] + list(l[0] for l in sxng_locales) SIMPLE_STYLE = ('auto', 'light', 'dark') CATEGORIES_AS_TABS = { 'general': {}, @@ -156,8 +156,8 @@ SCHEMA = { 'safe_search': SettingsValue((0, 1, 2), 0), 'autocomplete': SettingsValue(str, ''), 'autocomplete_min': SettingsValue(int, 4), - 'default_lang': SettingsValue(tuple(LANGUAGE_CODES + ['']), ''), - 'languages': SettingSublistValue(LANGUAGE_CODES, LANGUAGE_CODES), + 'default_lang': SettingsValue(tuple(SXNG_LOCALE_TAGS + ['']), ''), + 'languages': SettingSublistValue(SXNG_LOCALE_TAGS, SXNG_LOCALE_TAGS), 'ban_time_on_fail': SettingsValue(numbers.Real, 5), 'max_ban_time_on_fail': SettingsValue(numbers.Real, 120), 'suspended_times': { diff --git a/searx/languages.py b/searx/sxng_locales.py similarity index 54% rename from searx/languages.py rename to searx/sxng_locales.py index 377e7495b..110c4f17a 100644 --- a/searx/languages.py +++ b/searx/sxng_locales.py @@ -1,73 +1,128 @@ # -*- coding: utf-8 -*- -# list of language codes -# this file is generated automatically by utils/fetch_languages.py -language_codes = ( - ('af-ZA', 'Afrikaans', 'Suid-Afrika', 'Afrikaans', '\U0001f1ff\U0001f1e6'), +'''List of SearXNG's locale codes. + +This file is generated automatically by:: + + ./manage pyenv.cmd searxng_extra/update/update_engine_traits.py +''' + +sxng_locales = ( + ('ar', 'العربية', '', 'Arabic', '\U0001f310'), ('ar-EG', 'العربية', 'مصر', 'Arabic', '\U0001f1ea\U0001f1ec'), - ('be-BY', 'Беларуская', 'Беларусь', 'Belarusian', '\U0001f1e7\U0001f1fe'), + ('ar-SA', 'العربية', 'المملكة العربية السعودية', 'Arabic', '\U0001f1f8\U0001f1e6'), + ('bg', 'Български', '', 'Bulgarian', '\U0001f310'), ('bg-BG', 'Български', 'България', 'Bulgarian', '\U0001f1e7\U0001f1ec'), + ('ca', 'Català', '', 'Catalan', '\U0001f310'), ('ca-ES', 'Català', 'Espanya', 'Catalan', '\U0001f1ea\U0001f1f8'), + ('cs', 'Čeština', '', 'Czech', '\U0001f310'), ('cs-CZ', 'Čeština', 'Česko', 'Czech', '\U0001f1e8\U0001f1ff'), + ('da', 'Dansk', '', 'Danish', '\U0001f310'), ('da-DK', 'Dansk', 'Danmark', 'Danish', '\U0001f1e9\U0001f1f0'), ('de', 'Deutsch', '', 'German', '\U0001f310'), ('de-AT', 'Deutsch', 'Österreich', 'German', '\U0001f1e6\U0001f1f9'), ('de-CH', 'Deutsch', 'Schweiz', 'German', '\U0001f1e8\U0001f1ed'), ('de-DE', 'Deutsch', 'Deutschland', 'German', '\U0001f1e9\U0001f1ea'), + ('el', 'Ελληνικά', '', 'Greek', '\U0001f310'), ('el-GR', 'Ελληνικά', 'Ελλάδα', 'Greek', '\U0001f1ec\U0001f1f7'), ('en', 'English', '', 'English', '\U0001f310'), ('en-AU', 'English', 'Australia', 'English', '\U0001f1e6\U0001f1fa'), ('en-CA', 'English', 'Canada', 'English', '\U0001f1e8\U0001f1e6'), ('en-GB', 'English', 'United Kingdom', 'English', '\U0001f1ec\U0001f1e7'), ('en-IE', 'English', 'Ireland', 'English', '\U0001f1ee\U0001f1ea'), - ('en-MY', 'English', 'Malaysia', 'English', '\U0001f1f2\U0001f1fe'), + ('en-IN', 'English', 'India', 'English', '\U0001f1ee\U0001f1f3'), ('en-NZ', 'English', 'New Zealand', 'English', '\U0001f1f3\U0001f1ff'), + ('en-PH', 'English', 'Philippines', 'English', '\U0001f1f5\U0001f1ed'), + ('en-PK', 'English', 'Pakistan', 'English', '\U0001f1f5\U0001f1f0'), + ('en-SG', 'English', 'Singapore', 'English', '\U0001f1f8\U0001f1ec'), ('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'), + ('en-ZA', 'English', 'South Africa', 'English', '\U0001f1ff\U0001f1e6'), ('es', 'Español', '', 'Spanish', '\U0001f310'), ('es-AR', 'Español', 'Argentina', 'Spanish', '\U0001f1e6\U0001f1f7'), ('es-CL', 'Español', 'Chile', 'Spanish', '\U0001f1e8\U0001f1f1'), + ('es-CO', 'Español', 'Colombia', 'Spanish', '\U0001f1e8\U0001f1f4'), ('es-ES', 'Español', 'España', 'Spanish', '\U0001f1ea\U0001f1f8'), ('es-MX', 'Español', 'México', 'Spanish', '\U0001f1f2\U0001f1fd'), + ('es-PE', 'Español', 'Perú', 'Spanish', '\U0001f1f5\U0001f1ea'), + ('et', 'Eesti', '', 'Estonian', '\U0001f310'), ('et-EE', 'Eesti', 'Eesti', 'Estonian', '\U0001f1ea\U0001f1ea'), - ('fa-IR', 'فارسی', 'ایران', 'Persian', '\U0001f1ee\U0001f1f7'), + ('fi', 'Suomi', '', 'Finnish', '\U0001f310'), ('fi-FI', 'Suomi', 'Suomi', 'Finnish', '\U0001f1eb\U0001f1ee'), - ('fil-PH', 'Filipino', 'Pilipinas', 'Filipino', '\U0001f1f5\U0001f1ed'), ('fr', 'Français', '', 'French', '\U0001f310'), ('fr-BE', 'Français', 'Belgique', 'French', '\U0001f1e7\U0001f1ea'), ('fr-CA', 'Français', 'Canada', 'French', '\U0001f1e8\U0001f1e6'), ('fr-CH', 'Français', 'Suisse', 'French', '\U0001f1e8\U0001f1ed'), ('fr-FR', 'Français', 'France', 'French', '\U0001f1eb\U0001f1f7'), - ('he-IL', 'עברית', 'ישראל', 'Hebrew', '\U0001f1ee\U0001f1f1'), - ('hi-IN', 'हिन्दी', 'भारत', 'Hindi', '\U0001f1ee\U0001f1f3'), + ('he', 'עברית', '', 'Hebrew', '\U0001f1ee\U0001f1f7'), + ('hr', 'Hrvatski', '', 'Croatian', '\U0001f310'), ('hr-HR', 'Hrvatski', 'Hrvatska', 'Croatian', '\U0001f1ed\U0001f1f7'), + ('hu', 'Magyar', '', 'Hungarian', '\U0001f310'), ('hu-HU', 'Magyar', 'Magyarország', 'Hungarian', '\U0001f1ed\U0001f1fa'), + ('id', 'Indonesia', '', 'Indonesian', '\U0001f310'), ('id-ID', 'Indonesia', 'Indonesia', 'Indonesian', '\U0001f1ee\U0001f1e9'), - ('is-IS', 'Íslenska', 'Ísland', 'Icelandic', '\U0001f1ee\U0001f1f8'), + ('it', 'Italiano', '', 'Italian', '\U0001f310'), + ('it-CH', 'Italiano', 'Svizzera', 'Italian', '\U0001f1e8\U0001f1ed'), ('it-IT', 'Italiano', 'Italia', 'Italian', '\U0001f1ee\U0001f1f9'), + ('ja', '日本語', '', 'Japanese', '\U0001f310'), ('ja-JP', '日本語', '日本', 'Japanese', '\U0001f1ef\U0001f1f5'), + ('ko', '한국어', '', 'Korean', '\U0001f310'), ('ko-KR', '한국어', '대한민국', 'Korean', '\U0001f1f0\U0001f1f7'), + ('lt', 'Lietuvių', '', 'Lithuanian', '\U0001f310'), ('lt-LT', 'Lietuvių', 'Lietuva', 'Lithuanian', '\U0001f1f1\U0001f1f9'), + ('lv', 'Latviešu', '', 'Latvian', '\U0001f310'), ('lv-LV', 'Latviešu', 'Latvija', 'Latvian', '\U0001f1f1\U0001f1fb'), + ('nb', 'Norsk Bokmål', '', 'Norwegian Bokmål', '\U0001f310'), + ('nb-NO', 'Norsk Bokmål', 'Norge', 'Norwegian Bokmål', '\U0001f1f3\U0001f1f4'), ('nl', 'Nederlands', '', 'Dutch', '\U0001f310'), ('nl-BE', 'Nederlands', 'België', 'Dutch', '\U0001f1e7\U0001f1ea'), ('nl-NL', 'Nederlands', 'Nederland', 'Dutch', '\U0001f1f3\U0001f1f1'), - ('no-NO', 'Norsk', '', 'Norwegian (Bokmål)', '\U0001f1f3\U0001f1f4'), + ('pl', 'Polski', '', 'Polish', '\U0001f310'), ('pl-PL', 'Polski', 'Polska', 'Polish', '\U0001f1f5\U0001f1f1'), ('pt', 'Português', '', 'Portuguese', '\U0001f310'), ('pt-BR', 'Português', 'Brasil', 'Portuguese', '\U0001f1e7\U0001f1f7'), ('pt-PT', 'Português', 'Portugal', 'Portuguese', '\U0001f1f5\U0001f1f9'), + ('ro', 'Română', '', 'Romanian', '\U0001f310'), ('ro-RO', 'Română', 'România', 'Romanian', '\U0001f1f7\U0001f1f4'), + ('ru', 'Русский', '', 'Russian', '\U0001f310'), ('ru-RU', 'Русский', 'Россия', 'Russian', '\U0001f1f7\U0001f1fa'), + ('sk', 'Slovenčina', '', 'Slovak', '\U0001f310'), ('sk-SK', 'Slovenčina', 'Slovensko', 'Slovak', '\U0001f1f8\U0001f1f0'), + ('sl', 'Slovenščina', '', 'Slovenian', '\U0001f310'), ('sl-SI', 'Slovenščina', 'Slovenija', 'Slovenian', '\U0001f1f8\U0001f1ee'), - ('sr-RS', 'Српски', 'Србија', 'Serbian', '\U0001f1f7\U0001f1f8'), + ('sv', 'Svenska', '', 'Swedish', '\U0001f310'), ('sv-SE', 'Svenska', 'Sverige', 'Swedish', '\U0001f1f8\U0001f1ea'), - ('sw-TZ', 'Kiswahili', 'Tanzania', 'Swahili', '\U0001f1f9\U0001f1ff'), + ('th', 'ไทย', '', 'Thai', '\U0001f310'), ('th-TH', 'ไทย', 'ไทย', 'Thai', '\U0001f1f9\U0001f1ed'), + ('tr', 'Türkçe', '', 'Turkish', '\U0001f310'), ('tr-TR', 'Türkçe', 'Türkiye', 'Turkish', '\U0001f1f9\U0001f1f7'), + ('uk', 'Українська', '', 'Ukrainian', '\U0001f310'), ('uk-UA', 'Українська', 'Україна', 'Ukrainian', '\U0001f1fa\U0001f1e6'), + ('vi', 'Tiếng Việt', '', 'Vietnamese', '\U0001f310'), ('vi-VN', 'Tiếng Việt', 'Việt Nam', 'Vietnamese', '\U0001f1fb\U0001f1f3'), ('zh', '中文', '', 'Chinese', '\U0001f310'), ('zh-CN', '中文', '中国', 'Chinese', '\U0001f1e8\U0001f1f3'), ('zh-HK', '中文', '中國香港', 'Chinese', '\U0001f1ed\U0001f1f0'), ('zh-TW', '中文', '台灣', 'Chinese', '\U0001f1f9\U0001f1fc'), ) +''' +A list of five-digit tuples: + +0. SearXNG's internal locale tag (a language or region tag) +1. Name of the language (:py:obj:`babel.core.Locale.get_language_name`) +2. For region tags the name of the region (:py:obj:`babel.core.Locale.get_territory_name`). + Empty string for language tags. +3. English language name (from :py:obj:`babel.core.Locale.english_name`) +4. Unicode flag (emoji) that fits to SearXNG's internal region tag. Languages + are represented by a globe (🌐) + +.. code:: python + + ('en', 'English', '', 'English', '🌐'), + ('en-CA', 'English', 'Canada', 'English', '🇨🇦'), + ('en-US', 'English', 'United States', 'English', '🇺🇸'), + .. + ('fr', 'Français', '', 'French', '🌐'), + ('fr-BE', 'Français', 'Belgique', 'French', '🇧🇪'), + ('fr-CA', 'Français', 'Canada', 'French', '🇨🇦'), + +:meta hide-value: +''' diff --git a/searx/templates/simple/filters/languages.html b/searx/templates/simple/filters/languages.html index 54e07e209..a42a304a5 100644 --- a/searx/templates/simple/filters/languages.html +++ b/searx/templates/simple/filters/languages.html @@ -1,12 +1,12 @@ diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html index 9626b04d4..a0cc8efc2 100644 --- a/searx/templates/simple/preferences.html +++ b/searx/templates/simple/preferences.html @@ -115,10 +115,10 @@ {{ _('Search language') }}

{{- '' -}} {{- '' -}}

diff --git a/searx/utils.py b/searx/utils.py index e6180906b..f7a71b649 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -24,7 +24,7 @@ from babel.core import get_global from searx import settings from searx.data import USER_AGENTS, data_dir from searx.version import VERSION_TAG -from searx.languages import language_codes +from searx.sxng_locales import sxng_locales from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException from searx import logger @@ -53,8 +53,8 @@ _LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {} _FASTTEXT_MODEL: Optional["fasttext.FastText._FastText"] = None """fasttext model to predict laguage of a search term""" -SEARCH_LANGUAGE_CODES = frozenset([searxng_locale[0].split('-')[0] for searxng_locale in language_codes]) -"""Languages supported by most searxng engines (:py:obj:`searx.languages.language_codes`).""" +SEARCH_LANGUAGE_CODES = frozenset([searxng_locale[0].split('-')[0] for searxng_locale in sxng_locales]) +"""Languages supported by most searxng engines (:py:obj:`searx.sxng_locales.sxng_locales`).""" class _NotSetClass: # pylint: disable=too-few-public-methods @@ -355,11 +355,11 @@ def is_valid_lang(lang) -> Optional[Tuple[bool, str, str]]: is_abbr = len(lang) == 2 lang = lang.lower() if is_abbr: - for l in language_codes: + for l in sxng_locales: if l[0][:2] == lang: return (True, l[0][:2], l[3].lower()) return None - for l in language_codes: + for l in sxng_locales: if l[1].lower() == lang or l[3].lower() == lang: return (True, l[0][:2], l[3].lower()) return None diff --git a/searx/webapp.py b/searx/webapp.py index e62badaf7..fdd119128 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -121,8 +121,8 @@ from searx.locales import ( # renaming names from searx imports ... from searx.autocomplete import search_autocomplete, backends as autocomplete_backends -from searx.languages import language_codes as languages from searx.redisdb import initialize as redis_initialize +from searx.sxng_locales import sxng_locales from searx.search import SearchWithPlugins, initialize as search_initialize from searx.network import stream as http_stream, set_context_network_name from searx.search.checker import get_result as checker_get_result @@ -438,7 +438,7 @@ def render(template_name: str, **kwargs): kwargs['OTHER_CATEGORY'] = OTHER_CATEGORY # i18n - kwargs['language_codes'] = [l for l in languages if l[0] in settings['search']['languages']] + kwargs['sxng_locales'] = [l for l in sxng_locales if l[0] in settings['search']['languages']] locale = request.preferences.get_value('locale') kwargs['locale_rfc5646'] = _get_locale_rfc5646(locale) diff --git a/searxng_extra/update/update_engine_traits.py b/searxng_extra/update/update_engine_traits.py index 9630d6d24..7449912dc 100755 --- a/searxng_extra/update/update_engine_traits.py +++ b/searxng_extra/update/update_engine_traits.py @@ -18,17 +18,69 @@ the :origin:`CI Update data ... <.github/workflows/data-update.yml>` from unicodedata import lookup from pathlib import Path from pprint import pformat -from babel import Locale, UnknownLocaleError -from babel.languages import get_global -from babel.core import parse_locale +import babel from searx import settings, searx_dir from searx import network -from searx.engines import load_engines, engines +from searx.engines import load_engines from searx.enginelib.traits import EngineTraitsMap # Output files. -languages_file = Path(searx_dir) / 'languages.py' +languages_file = Path(searx_dir) / 'sxng_locales.py' +languages_file_header = """\ +# -*- coding: utf-8 -*- +'''List of SearXNG's locale codes. + +This file is generated automatically by:: + + ./manage pyenv.cmd searxng_extra/update/update_engine_traits.py +''' + +sxng_locales = ( +""" +languages_file_footer = """, +) +''' +A list of five-digit tuples: + +0. SearXNG's internal locale tag (a language or region tag) +1. Name of the language (:py:obj:`babel.core.Locale.get_language_name`) +2. For region tags the name of the region (:py:obj:`babel.core.Locale.get_territory_name`). + Empty string for language tags. +3. English language name (from :py:obj:`babel.core.Locale.english_name`) +4. Unicode flag (emoji) that fits to SearXNG's internal region tag. Languages + are represented by a globe (\U0001F310) + +.. code:: python + + ('en', 'English', '', 'English', '\U0001f310'), + ('en-CA', 'English', 'Canada', 'English', '\U0001f1e8\U0001f1e6'), + ('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'), + .. + ('fr', 'Français', '', 'French', '\U0001f310'), + ('fr-BE', 'Français', 'Belgique', 'French', '\U0001f1e7\U0001f1ea'), + ('fr-CA', 'Français', 'Canada', 'French', '\U0001f1e8\U0001f1e6'), + +:meta hide-value: +''' +""" + + +lang2emoji = { + 'ha': '\U0001F1F3\U0001F1EA', # Hausa / Niger + 'bs': '\U0001F1E7\U0001F1E6', # Bosnian / Bosnia & Herzegovina + 'jp': '\U0001F1EF\U0001F1F5', # Japanese + 'ua': '\U0001F1FA\U0001F1E6', # Ukrainian + 'he': '\U0001F1EE\U0001F1F7', # Hebrew +} + + +def main(): + load_engines(settings['engines']) + # traits_map = EngineTraitsMap.from_data() + traits_map = fetch_traits_map() + sxng_tag_list = filter_locales(traits_map) + write_languages_file(sxng_tag_list) def fetch_traits_map(): @@ -45,234 +97,69 @@ def fetch_traits_map(): return traits_map -# Get babel Locale object from lang_code if possible. -def get_locale(lang_code): - try: - locale = Locale.parse(lang_code, sep='-') - return locale - except (UnknownLocaleError, ValueError): - return None +def filter_locales(traits_map: EngineTraitsMap): + """Filter language & region tags by a threshold.""" + + min_eng_per_region = 11 + min_eng_per_lang = 13 + + _ = {} + for eng in traits_map.values(): + for reg in eng.regions.keys(): + _[reg] = _.get(reg, 0) + 1 + + regions = set(k for k, v in _.items() if v >= min_eng_per_region) + lang_from_region = set(k.split('-')[0] for k in regions) + + _ = {} + for eng in traits_map.values(): + for lang in eng.languages.keys(): + # ignore script types like zh_Hant, zh_Hans or sr_Latin, pa_Arab (they + # already counted by existence of 'zh' or 'sr', 'pa') + if '_' in lang: + # print("ignore %s" % lang) + continue + _[lang] = _.get(lang, 0) + 1 + + languages = set(k for k, v in _.items() if v >= min_eng_per_lang) + + sxng_tag_list = set() + sxng_tag_list.update(regions) + sxng_tag_list.update(lang_from_region) + sxng_tag_list.update(languages) + + return sxng_tag_list -lang2emoji = { - 'ha': '\U0001F1F3\U0001F1EA', # Hausa / Niger - 'bs': '\U0001F1E7\U0001F1E6', # Bosnian / Bosnia & Herzegovina - 'jp': '\U0001F1EF\U0001F1F5', # Japanese - 'ua': '\U0001F1FA\U0001F1E6', # Ukrainian - 'he': '\U0001F1EE\U0001F1F7', # Hebrew -} +def write_languages_file(sxng_tag_list): + language_codes = [] -def get_unicode_flag(lang_code): - """Determine a unicode flag (emoji) that fits to the ``lang_code``""" + for sxng_tag in sorted(sxng_tag_list): + sxng_locale: babel.Locale = babel.Locale.parse(sxng_tag, sep='-') - emoji = lang2emoji.get(lang_code.lower()) - if emoji: - return emoji + flag = get_unicode_flag(sxng_locale) or '' - if len(lang_code) == 2: - return '\U0001F310' - - language = territory = script = variant = '' - try: - language, territory, script, variant = parse_locale(lang_code, '-') - except ValueError as exc: - print(exc) - - # https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 - if not territory: - # https://www.unicode.org/emoji/charts/emoji-list.html#country-flag - emoji = lang2emoji.get(language) - if not emoji: - print( - "%s --> language: %s / territory: %s / script: %s / variant: %s" - % (lang_code, language, territory, script, variant) - ) - return emoji - - emoji = lang2emoji.get(territory.lower()) - if emoji: - return emoji - - try: - c1 = lookup('REGIONAL INDICATOR SYMBOL LETTER ' + territory[0]) - c2 = lookup('REGIONAL INDICATOR SYMBOL LETTER ' + territory[1]) - # print("%s --> territory: %s --> %s%s" %(lang_code, territory, c1, c2 )) - except KeyError as exc: - print("%s --> territory: %s --> %s" % (lang_code, territory, exc)) - return None - - return c1 + c2 - - -def get_territory_name(lang_code): - country_name = None - locale = get_locale(lang_code) - try: - if locale is not None: - country_name = locale.get_territory_name() - except FileNotFoundError as exc: - print("ERROR: %s --> %s" % (locale, exc)) - return country_name - - -def join_language_lists(traits_map: EngineTraitsMap): - """Join all languages of the engines into one list. The returned language list - contains language codes (``zh``) and region codes (``zh-TW``). The codes can - be parsed by babel:: - - babel.Locale.parse(language_list[n]) - """ - # pylint: disable=too-many-branches - language_list = {} - - for eng_name, eng_traits in traits_map.items(): - eng = engines[eng_name] - eng_codes = set() - - if eng_traits.data_type == 'traits_v1': - # items of type 'engine_traits' do have regions & languages, the - # list of eng_codes should contain both. - eng_codes.update(eng_traits.regions.keys()) - eng_codes.update(eng_traits.languages.keys()) - - elif eng_traits.data_type == 'supported_languages': - # vintage / deprecated - _codes = set() - if isinstance(eng_traits.supported_languages, dict): - _codes.update(eng_traits.supported_languages.keys()) - elif isinstance(eng_traits.supported_languages, list): - _codes.update(eng_traits.supported_languages) - else: - raise TypeError('engine.supported_languages type %s is unknown' % type(eng_traits.supported_languages)) - - for lang_code in _codes: - # apply custom fixes if necessary - if lang_code in getattr(eng, 'language_aliases', {}).values(): - lang_code = next(lc for lc, alias in eng.language_aliases.items() if lang_code == alias) - eng_codes.add(lang_code) - - for lang_code in eng_codes: - - locale = get_locale(lang_code) - - # ensure that lang_code uses standard language and country codes - if locale and locale.territory: - lang_code = "{lang}-{country}".format(lang=locale.language, country=locale.territory) - short_code = lang_code.split('-')[0] - - # add language without country if not in list - if short_code not in language_list: - if locale: - # get language's data from babel's Locale object - language_name = locale.get_language_name().title() - english_name = locale.english_name.split(' (')[0] - elif short_code in traits_map['wikipedia'].supported_languages: - # get language's data from wikipedia if not known by babel - language_name = traits_map['wikipedia'].supported_languages[short_code]['name'] - english_name = traits_map['wikipedia'].supported_languages[short_code]['english_name'] - else: - language_name = None - english_name = None - - # add language to list - language_list[short_code] = { - 'name': language_name, - 'english_name': english_name, - 'counter': set(), - 'countries': {}, - } - - # add language with country if not in list - if lang_code != short_code and lang_code not in language_list[short_code]['countries']: - country_name = '' - if locale: - # get country name from babel's Locale object - try: - country_name = locale.get_territory_name() - except FileNotFoundError as exc: - print("ERROR: %s --> %s" % (locale, exc)) - locale = None - - language_list[short_code]['countries'][lang_code] = { - 'country_name': country_name, - 'counter': set(), - } - - # count engine for both language_country combination and language alone - language_list[short_code]['counter'].add(eng_name) - if lang_code != short_code: - language_list[short_code]['countries'][lang_code]['counter'].add(eng_name) - - return language_list - - -# Filter language list so it only includes the most supported languages and countries -def filter_language_list(joined_languages_map): - min_engines_per_lang = 12 - min_engines_per_country = 7 - # pylint: disable=consider-using-dict-items, consider-iterating-dictionary - main_engines = [ - engine_name - for engine_name in engines.keys() - if 'general' in engines[engine_name].categories - and hasattr(engines[engine_name], 'supported_languages') - and engines[engine_name].supported_languages - and not engines[engine_name].disabled - ] - - # filter list to include only languages supported by most engines or all default general engines - filtered_languages = { - code: lang - for code, lang in joined_languages_map.items() - if ( - len(lang['counter']) >= min_engines_per_lang - or all(main_engine in lang['counter'] for main_engine in main_engines) + item = ( + sxng_tag, + sxng_locale.get_language_name().title(), + sxng_locale.get_territory_name() or '', + sxng_locale.english_name.split(' (')[0], + UnicodeEscape(flag), ) - } - def _copy_lang_data(lang, country_name=None): - new_dict = {} - new_dict['name'] = joined_languages_map[lang]['name'] - new_dict['english_name'] = joined_languages_map[lang]['english_name'] - if country_name: - new_dict['country_name'] = country_name - return new_dict + language_codes.append(item) - # for each language get country codes supported by most engines or at least one country code - filtered_languages_with_countries = {} - for lang, lang_data in filtered_languages.items(): - countries = lang_data['countries'] - filtered_countries = {} + language_codes = tuple(language_codes) - # get language's country codes with enough supported engines - for lang_country, country_data in countries.items(): - if len(country_data['counter']) >= min_engines_per_country: - filtered_countries[lang_country] = _copy_lang_data(lang, country_data['country_name']) - - # add language without countries too if there's more than one country to choose from - if len(filtered_countries) > 1: - filtered_countries[lang] = _copy_lang_data(lang, None) - elif len(filtered_countries) == 1: - lang_country = next(iter(filtered_countries)) - - # if no country has enough engines try to get most likely country code from babel - if not filtered_countries: - lang_country = None - subtags = get_global('likely_subtags').get(lang) - if subtags: - country_code = subtags.split('_')[-1] - if len(country_code) == 2: - lang_country = "{lang}-{country}".format(lang=lang, country=country_code) - - if lang_country: - filtered_countries[lang_country] = _copy_lang_data(lang, None) - else: - filtered_countries[lang] = _copy_lang_data(lang, None) - - filtered_languages_with_countries.update(filtered_countries) - - return filtered_languages_with_countries + with open(languages_file, 'w', encoding='utf-8') as new_file: + file_content = "{header} {language_codes}{footer}".format( + header=languages_file_header, + language_codes=pformat(language_codes, width=120, indent=4)[1:-1], + footer=languages_file_footer, + ) + new_file.write(file_content) + new_file.close() class UnicodeEscape(str): @@ -282,54 +169,29 @@ class UnicodeEscape(str): return "'" + "".join([chr(c) for c in self.encode('unicode-escape')]) + "'" -# Write languages.py. -def write_languages_file(languages): - file_headers = ( - "# -*- coding: utf-8 -*-", - "# list of language codes", - "# this file is generated automatically by utils/fetch_languages.py", - "language_codes = (\n", - ) +def get_unicode_flag(locale: babel.Locale): + """Determine a unicode flag (emoji) that fits to the ``locale``""" - language_codes = [] + emoji = lang2emoji.get(locale.language) + if emoji: + return emoji - for code in sorted(languages): + if not locale.territory: + return '\U0001F310' - name = languages[code]['name'] - if name is None: - print("ERROR: languages['%s'] --> %s" % (code, languages[code])) - continue + emoji = lang2emoji.get(locale.territory.lower()) + if emoji: + return emoji - flag = get_unicode_flag(code) or '' - item = ( - code, - languages[code]['name'].split(' (')[0], - get_territory_name(code) or '', - languages[code].get('english_name') or '', - UnicodeEscape(flag), - ) + try: + c1 = lookup('REGIONAL INDICATOR SYMBOL LETTER ' + locale.territory[0]) + c2 = lookup('REGIONAL INDICATOR SYMBOL LETTER ' + locale.territory[1]) + # print("OK : %s --> %s%s" % (locale, c1, c2)) + except KeyError as exc: + print("ERROR: %s --> %s" % (locale, exc)) + return None - language_codes.append(item) - - language_codes = tuple(language_codes) - - with open(languages_file, 'w', encoding='utf-8') as new_file: - file_content = "{file_headers} {language_codes},\n)\n".format( - # fmt: off - file_headers = '\n'.join(file_headers), - language_codes = pformat(language_codes, indent=4)[1:-1] - # fmt: on - ) - new_file.write(file_content) - new_file.close() - - -def main(): - load_engines(settings['engines']) - traits_map = fetch_traits_map() - joined_languages_map = join_language_lists(traits_map) - filtered_languages = filter_language_list(joined_languages_map) - write_languages_file(filtered_languages) + return c1 + c2 if __name__ == "__main__": diff --git a/searxng_extra/update/update_osm_keys_tags.py b/searxng_extra/update/update_osm_keys_tags.py index 72197498d..72f3d61c5 100755 --- a/searxng_extra/update/update_osm_keys_tags.py +++ b/searxng_extra/update/update_osm_keys_tags.py @@ -50,7 +50,7 @@ from pathlib import Path from searx import searx_dir from searx.network import set_timeout_for_thread from searx.engines import wikidata, set_loggers -from searx.languages import language_codes +from searx.sxng_locales import sxng_locales from searx.engines.openstreetmap import get_key_rank, VALUE_TO_LINK set_loggers(wikidata, 'wikidata') @@ -76,7 +76,7 @@ GROUP BY ?key ?item ?itemLabel ORDER BY ?key ?item ?itemLabel """ -LANGUAGES = [l[0].lower() for l in language_codes] +LANGUAGES = [l[0].lower() for l in sxng_locales] PRESET_KEYS = { ('wikidata',): {'en': 'Wikidata'},