From c03b0ea650e0a052e422881c6278a5c9a7cca60c Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 30 Jan 2023 18:40:22 +0100 Subject: [PATCH 1/3] [mod] add a Preferences.client property to store client prefs Signed-off-by: Markus Heiser --- searx/preferences.py | 61 ++++++++++++++++++++++++++++++++++++++++++-- searx/webapp.py | 19 ++++++-------- 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/searx/preferences.py b/searx/preferences.py index 3da6d5d16..8552305a7 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -8,9 +8,10 @@ from base64 import urlsafe_b64encode, urlsafe_b64decode from zlib import compress, decompress from urllib.parse import parse_qs, urlencode -from typing import Iterable, Dict, List +from typing import Iterable, Dict, List, Optional import flask +import babel from searx import settings, autocomplete from searx.enginelib import Engine @@ -287,10 +288,65 @@ class PluginsSetting(BooleanChoices): return [item[len('plugin_') :] for item in items] +class ClientPref: + """Container to assemble client prefferences and settings.""" + + # hint: searx.webapp.get_client_settings should be moved into this class + + locale: babel.Locale + """Locale prefered by the client.""" + + def __init__(self, locale: Optional[babel.Locale] = None): + self.locale = locale + + @property + def locale_tag(self): + if self.locale is None: + return None + tag = self.locale.language + if self.locale.territory: + tag += '-' + self.locale.territory + return tag + + @classmethod + def from_http_request(cls, http_request: flask.Request): + """Build ClientPref object from HTTP request. + + - `Accept-Language used for locale setting + `__ + + """ + al_header = http_request.headers.get("Accept-Language") + if not al_header: + return cls(locale=None) + + pairs = [] + for l in al_header.split(','): + # fmt: off + lang, qvalue = [_.strip() for _ in (l.split(';') + ['q=1',])[:2]] + # fmt: on + try: + qvalue = float(qvalue.split('=')[-1]) + locale = babel.Locale.parse(lang, sep='-') + except (ValueError, babel.core.UnknownLocaleError): + continue + pairs.append((locale, qvalue)) + pairs.sort(reverse=True, key=lambda x: x[1]) + return cls(locale=pairs[0][0]) + + class Preferences: """Validates and saves preferences to cookies""" - def __init__(self, themes: List[str], categories: List[str], engines: Dict[str, Engine], plugins: Iterable[Plugin]): + def __init__( + self, + themes: List[str], + categories: List[str], + engines: Dict[str, Engine], + plugins: Iterable[Plugin], + client: Optional[ClientPref] = None, + ): + super().__init__() self.key_value_settings: Dict[str, Setting] = { @@ -414,6 +470,7 @@ class Preferences: self.engines = EnginesSetting('engines', engines=engines.values()) self.plugins = PluginsSetting('plugins', plugins=plugins) self.tokens = SetSetting('tokens') + self.client = client or ClientPref() self.unknown_params: Dict[str, str] = {} def get_as_url_params(self): diff --git a/searx/webapp.py b/searx/webapp.py index 388b28c38..52b1ccc34 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -96,6 +96,7 @@ from searx.plugins import Plugin, plugins, initialize as plugin_initialize from searx.plugins.oa_doi_rewrite import get_doi_resolver from searx.preferences import ( Preferences, + ClientPref, ValidationException, ) from searx.answerers import ( @@ -221,16 +222,9 @@ babel = Babel(app, locale_selector=get_locale) def _get_browser_language(req, lang_list): - for lang in req.headers.get("Accept-Language", "en").split(","): - if ';' in lang: - lang = lang.split(';')[0] - if '-' in lang: - lang_parts = lang.split('-') - lang = "{}-{}".format(lang_parts[0], lang_parts[-1].upper()) - locale = match_locale(lang, lang_list, fallback=None) - if locale is not None: - return locale - return 'en' + client = ClientPref.from_http_request(req) + locale = match_locale(client.locale_tag, lang_list, fallback='en') + return locale def _get_locale_rfc5646(locale): @@ -512,7 +506,10 @@ def pre_request(): request.timings = [] # pylint: disable=assigning-non-slot request.errors = [] # pylint: disable=assigning-non-slot - preferences = Preferences(themes, list(categories.keys()), engines, plugins) # pylint: disable=redefined-outer-name + client_pref = ClientPref.from_http_request(request) + # pylint: disable=redefined-outer-name + preferences = Preferences(themes, list(categories.keys()), engines, plugins, client_pref) + user_agent = request.headers.get('User-Agent', '').lower() if 'webkit' in user_agent and 'android' in user_agent: preferences.key_value_settings['method'].value = 'GET' From d5ecda99303f1555dc34d824f9dca662bbdda24e Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 31 Jan 2023 12:40:23 +0100 Subject: [PATCH 2/3] [mod] move language recognition to get_search_query_from_webapp To set the language from language recognition and hold the value selected by the client, the previous implementation creates a copy of the SearchQuery object and manipulates the SearchQuery object by calling function replace_auto_language(). This patch tries to implement a similar functionality in a more central place, in function get_search_query_from_webapp() when the SearchQuery object is build up. Additional this patch uses the language preferred by the client, if language recognition does not have a match / the existing implementation does not care about client preferences and uses 'all' in case of no match. Signed-off-by: Markus Heiser --- searx/search/__init__.py | 43 ++-------------------------------------- searx/webadapter.py | 32 ++++++++++++++++++++++++++++-- searx/webapp.py | 20 ++++++------------- 3 files changed, 38 insertions(+), 57 deletions(-) diff --git a/searx/search/__init__.py b/searx/search/__init__.py index e5465880c..77121c426 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -22,7 +22,6 @@ from searx.network import initialize as initialize_network, check_network_config from searx.metrics import initialize as initialize_metrics, counter_inc, histogram_observe_time from searx.search.processors import PROCESSORS, initialize as initialize_processors from searx.search.checker import initialize as initialize_checker -from searx.utils import detect_language logger = logger.getChild('search') @@ -40,57 +39,19 @@ def initialize(settings_engines=None, enable_checker=False, check_network=False, initialize_checker() -def replace_auto_language(search_query: SearchQuery): - """ - Do nothing except if `search_query.lang` is "auto". - In this case: - * the value "auto" is replaced by the detected language of the query. - The default value is "all" when no language is detected. - * `search_query.locale` is updated accordingly - - Use :py:obj:`searx.utils.detect_language` with `only_search_languages=True` to keep - only languages supported by the engines. - """ - if search_query.lang != 'auto': - return - - detected_lang = detect_language(search_query.query, threshold=0.3, only_search_languages=True) - if detected_lang is None: - # fallback to 'all' if no language has been detected - search_query.lang = 'all' - search_query.locale = None - return - search_query.lang = detected_lang - try: - search_query.locale = babel.Locale.parse(search_query.lang) - except babel.core.UnknownLocaleError: - search_query.locale = None - - class Search: """Search information container""" __slots__ = "search_query", "result_container", "start_time", "actual_timeout" def __init__(self, search_query: SearchQuery): - """Initialize the Search - - search_query is copied - """ + """Initialize the Search""" # init vars super().__init__() + self.search_query = search_query self.result_container = ResultContainer() self.start_time = None self.actual_timeout = None - self.search_query = copy(search_query) - self.update_search_query(self.search_query) - - def update_search_query(self, search_query: SearchQuery): - """Update search_query. - - call replace_auto_language to replace the "auto" language - """ - replace_auto_language(search_query) def search_external_bang(self): """ diff --git a/searx/webadapter.py b/searx/webadapter.py index dbcf25058..121319eeb 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -6,6 +6,7 @@ from searx.query import RawTextQuery from searx.engines import categories, engines from searx.search import SearchQuery, EngineRef from searx.preferences import Preferences, is_locked +from searx.utils import detect_language # remove duplicate queries. @@ -214,7 +215,27 @@ def parse_engine_data(form): def get_search_query_from_webapp( preferences: Preferences, form: Dict[str, str] -) -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]: +) -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef], str]: + """Assemble data from preferences and request.form (from the HTML form) needed + in a search query. + + The returned tuple consits of: + + 1. instance of :py:obj:`searx.search.SearchQuery` + 2. instance of :py:obj:`searx.query.RawTextQuery` + 3. list of :py:obj:`searx.search.EngineRef` instances + 4. string with the *selected locale* of the query + + About language/locale: if the client selects the alias ``auto`` the + ``SearchQuery`` object is build up by the :py:obj:`detected language + `. If language recognition does not have a + match the language preferred by the :py:obj:`Preferences.client` is used. + If client does not have a preference, the default ``all`` is used. + + The *selected locale* in the tuple always represents the selected + language/locale and might differ from the language recognition. + + """ # no text for the query ? if not form.get('q'): raise SearxParameterException('q', '') @@ -229,13 +250,19 @@ def get_search_query_from_webapp( # set query query = raw_text_query.getQuery() query_pageno = parse_pageno(form) - query_lang = parse_lang(preferences, form, raw_text_query) query_safesearch = parse_safesearch(preferences, form) query_time_range = parse_time_range(form) query_timeout = parse_timeout(form, raw_text_query) external_bang = raw_text_query.external_bang engine_data = parse_engine_data(form) + query_lang = parse_lang(preferences, form, raw_text_query) + selected_locale = query_lang + + if query_lang == 'auto': + query_lang = detect_language(query, threshold=0.8, only_search_languages=True) + query_lang = query_lang or preferences.client.locale_tag or 'all' + if not is_locked('categories') and raw_text_query.specific: # if engines are calculated from query, # set categories by using that information @@ -265,4 +292,5 @@ def get_search_query_from_webapp( raw_text_query, query_engineref_list_unknown, query_engineref_list_notoken, + selected_locale, ) diff --git a/searx/webapp.py b/searx/webapp.py index 52b1ccc34..79255652f 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -84,6 +84,7 @@ from searx.webutils import ( from searx.webadapter import ( get_search_query_from_webapp, get_selected_categories, + parse_lang, ) from searx.utils import ( html_to_text, @@ -440,11 +441,7 @@ def render(template_name: str, **kwargs): kwargs['rtl'] = True if 'current_language' not in kwargs: - _locale = request.preferences.get_value('language') - if _locale in ('auto', 'all'): - kwargs['current_language'] = _locale - else: - kwargs['current_language'] = match_locale(_locale, settings['search']['languages']) + kwargs['current_language'] = parse_lang(request.preferences, {}, RawTextQuery('', [])) # values from settings kwargs['search_formats'] = [x for x in settings['search']['formats'] if x != 'html'] @@ -678,7 +675,9 @@ def search(): raw_text_query = None result_container = None try: - search_query, raw_text_query, _, _ = get_search_query_from_webapp(request.preferences, request.form) + search_query, raw_text_query, _, _, selected_locale = get_search_query_from_webapp( + request.preferences, request.form + ) # search = Search(search_query) # without plugins search = SearchWithPlugins(search_query, request.user_plugins, request) # pylint: disable=redefined-outer-name @@ -809,13 +808,6 @@ def search(): ) ) - if search_query.lang in ('auto', 'all'): - current_language = search_query.lang - else: - current_language = match_locale( - search_query.lang, settings['search']['languages'], fallback=request.preferences.get_value("language") - ) - # search_query.lang contains the user choice (all, auto, en, ...) # when the user choice is "auto", search.search_query.lang contains the detected language # otherwise it is equals to search_query.lang @@ -838,7 +830,7 @@ def search(): result_container.unresponsive_engines ), current_locale = request.preferences.get_value("locale"), - current_language = current_language, + current_language = selected_locale, search_language = match_locale( search.search_query.lang, settings['search']['languages'], From 8adbc4fcecd3c3acb50e719bfe3794615649c793 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 15 Apr 2023 22:24:59 +0200 Subject: [PATCH 3/3] [mod] settings.yml: enable language detection by default_lang (auto) Signed-off-by: Markus Heiser --- searx/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/settings.yml b/searx/settings.yml index fa6141dd4..81428a747 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -31,7 +31,7 @@ search: autocomplete_min: 4 # Default search language - leave blank to detect from browser information or # use codes from 'languages.py' - default_lang: "" + default_lang: "auto" # Available languages # languages: # - all