From f0c8e50d5e20e4e0fd6d577c038fd84ddf841da3 Mon Sep 17 00:00:00 2001 From: Allen <64094914+allendema@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:50:52 +0000 Subject: [PATCH] [enh] Allowing using multiple autocomplete engines --- searx/autocomplete.py | 31 +++++++++++++++++++++++++----- searx/preferences.py | 6 +++++- searx/settings.yml | 10 ++++++---- searx/settings_defaults.py | 2 +- searx/utils.py | 11 +++++++++++ tests/unit/test_preferences.py | 8 ++++++++ tests/unit/test_settings_loader.py | 1 + 7 files changed, 58 insertions(+), 11 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index e277c6631..5512669fd 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -11,6 +11,7 @@ import lxml from httpx import HTTPError from searx import settings +from searx.utils import unique from searx.engines import ( engines, google, @@ -252,11 +253,31 @@ backends = { } -def search_autocomplete(backend_name, query, sxng_locale): - backend = backends.get(backend_name) - if backend is None: - return [] +def search_autocomplete(backend_names, query, sxng_locale): + + enabled_backends = list(unique(backend_names)) + + len_enabled_backends = len(enabled_backends) + try: - return backend(query, sxng_locale) + results = [] + + for backend_name in enabled_backends: + backend = backends.get(backend_name) + if backend is None: + # if somehow 'searx.preferences.ValidationException' was not raised + continue + + backend_results = backend(query, sxng_locale) + if (len_enabled_backends > 2) and (len(backend_results) > 3): + # if more than 2 autocompleters: only get the first 3 results from each + + results.extend(backend_results[:3]) + + else: + results.extend(backend_results) + + return list(unique(results)) + except (HTTPError, SearxEngineResponseException): return [] diff --git a/searx/preferences.py b/searx/preferences.py index b4a10899e..7fb4982c4 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -96,6 +96,10 @@ class MultipleChoiceSetting(Setting): """Setting of values which can only come from the given choices""" def __init__(self, default_value: List[str], choices: Iterable[str], locked=False): + # backwards compat for autocomplete setting (was string, now is a list of strings) + if isinstance(default_value, str): + default_value = [str(val) for val in default_value.split(",")] + super().__init__(default_value, locked) self.choices = choices self._validate_selections(self.value) @@ -401,7 +405,7 @@ class Preferences: locked=is_locked('locale'), choices=list(LOCALE_NAMES.keys()) + [''] ), - 'autocomplete': EnumStringSetting( + 'autocomplete': MultipleChoiceSetting( settings['search']['autocomplete'], locked=is_locked('autocomplete'), choices=list(autocomplete.backends.keys()) + [''] diff --git a/searx/settings.yml b/searx/settings.yml index 859b4613d..bc1454a83 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -29,10 +29,12 @@ brand: search: # Filter results. 0: None, 1: Moderate, 2: Strict safe_search: 0 - # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "yandex", "mwmbl", - # "seznam", "startpage", "stract", "swisscows", "qwant", "wikipedia" - leave blank to turn it off - # by default. - autocomplete: "" + # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", + # "yandex", "mwmbl", "seznam", "startpage", "stract", "swisscows", "qwant", "wikipedia" + # Uncomment the section below and edit/add/remove each engine to turn them on by default. + # autocomplete: + # - duckduckgo + # - stract # minimun characters to type before autocompleter starts autocomplete_min: 4 # Default search language - leave blank to detect from browser information or diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 93b04257c..b9ae4572e 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -154,7 +154,7 @@ SCHEMA = { }, 'search': { 'safe_search': SettingsValue((0, 1, 2), 0), - 'autocomplete': SettingsValue(str, ''), + 'autocomplete': SettingsValue((list, str, False), ['']), 'autocomplete_min': SettingsValue(int, 4), 'default_lang': SettingsValue(tuple(SXNG_LOCALE_TAGS + ['']), ''), 'languages': SettingSublistValue(SXNG_LOCALE_TAGS, SXNG_LOCALE_TAGS), diff --git a/searx/utils.py b/searx/utils.py index f50618ea2..df177a5a8 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -329,6 +329,17 @@ def dict_subset(dictionary: MutableMapping, properties: Set[str]) -> Dict: return {k: dictionary[k] for k in properties if k in dictionary} +def unique(iterable): + """Yield unique elements from 'iterable' while preserving order + https://github.com/mikf/gallery-dl/blob/e03b99ba0ecbf653b89e68d00245da78694071fb/gallery_dl/util.py#L64C1-L71C26""" + seen = set() + add = seen.add + for element in iterable: + if element not in seen: + add(element) + yield element + + def get_torrent_size(filesize: str, filesize_multiplier: str) -> Optional[int]: """ diff --git a/tests/unit/test_preferences.py b/tests/unit/test_preferences.py index 5855c12a6..d51c306ff 100644 --- a/tests/unit/test_preferences.py +++ b/tests/unit/test_preferences.py @@ -66,6 +66,14 @@ class TestSettings(SearxTestCase): # pylint: disable=missing-class-docstring # multiple choice settings + def test_multiple_setting_default_value_invalid_commma_seperated(self): + with self.assertRaises(ValidationException): + MultipleChoiceSetting("duckduckgo,doesnotexist", choices=['duckduckgo', 'stract', 'qwant']) + + def test_multiple_setting_default_value_valid_commma_seperated(self): + setting = MultipleChoiceSetting("duckduckgo,stract", choices=['duckduckgo', 'stract', 'qwant']) + self.assertEqual(setting.get_value(), ['duckduckgo', 'stract']) + def test_multiple_setting_invalid_default_value(self): with self.assertRaises(ValidationException): MultipleChoiceSetting(['3', '4'], choices=['0', '1', '2']) diff --git a/tests/unit/test_settings_loader.py b/tests/unit/test_settings_loader.py index 088767597..f9f41b3f4 100644 --- a/tests/unit/test_settings_loader.py +++ b/tests/unit/test_settings_loader.py @@ -121,3 +121,4 @@ class TestUserSettings(SearxTestCase): # pylint: disable=missing-class-docstrin self.assertEqual(settings['server']['secret_key'], "user_settings_secret") engine_names = [engine['name'] for engine in settings['engines']] self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote']) + self.assertIsInstance(settings['search']['autocomplete'], (list, str))