From 752c224608a193325e6cdc510a5f4b9d0553d83c Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 15 Sep 2021 16:34:52 +0200 Subject: [PATCH] [enh] add surrogates to lower fork maintenance, some objects in SearXNG can be replaced. The configuration is done in settings.yml --- searx/query.py | 6 ++- searx/search/__init__.py | 8 +++- searx/settings.yml | 4 ++ searx/settings_defaults.py | 4 ++ searx/surrogates.py | 35 +++++++++++++++ searx/webapp.py | 5 +-- searx_extra/standalone_searx.py | 2 +- tests/unit/test_query.py | 70 ++++++++++++++--------------- tests/unit/test_search.py | 14 +++--- tests/unit/test_standalone_searx.py | 2 +- tests/unit/test_webapp.py | 4 +- 11 files changed, 102 insertions(+), 52 deletions(-) create mode 100644 searx/surrogates.py diff --git a/searx/query.py b/searx/query.py index 2e6a2aa4c..0a8981c89 100644 --- a/searx/query.py +++ b/searx/query.py @@ -8,6 +8,7 @@ from searx.engines import categories, engines, engine_shortcuts from searx.external_bang import get_bang_definition_and_autocomplete from searx.search import EngineRef from searx.webutils import VALID_LANGUAGE_CODE +from searx.surrogates import get_actual_object class QueryPartParser(ABC): @@ -232,7 +233,7 @@ class BangParser(QueryPartParser): self._add_autocomplete(first_char + engine_shortcut) -class RawTextQuery: +class DefaultRawTextQuery: """parse raw text query (the value from the html input)""" PARSER_CLASSES = [ @@ -328,3 +329,6 @@ class RawTextQuery: + f"autocomplete_list={self.autocomplete_list!r}\n " \ + f"query_parts={self.query_parts!r}\n " \ + f"user_query_parts={self.user_query_parts!r} >" + + +RawTextQuery = get_actual_object('searx.query.RawTextQuery', DefaultRawTextQuery) diff --git a/searx/search/__init__.py b/searx/search/__init__.py index 97515622d..53e43a19c 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -19,6 +19,7 @@ from searx.network import initialize as initialize_network from searx.metrics import initialize as initialize_metrics, counter_inc, histogram_observe_time from searx.search.processors import PROCESSORS, initialize as initialize_processors from searx.search.checker import initialize as initialize_checker +from searx.surrogates import get_actual_object logger = logger.getChild('search') @@ -34,7 +35,7 @@ def initialize(settings_engines=None, enable_checker=False): initialize_checker() -class Search: +class BasicSearch: """Search information container""" __slots__ = "search_query", "result_container", "start_time", "actual_timeout" @@ -171,7 +172,7 @@ class Search: return self.result_container -class SearchWithPlugins(Search): +class SearchWithPlugins(BasicSearch): """Inherit from the Search class, add calls to the plugins.""" __slots__ = 'ordered_plugin_list', 'request' @@ -201,3 +202,6 @@ class SearchWithPlugins(Search): self.result_container.close() return self.result_container + + +Search = get_actual_object('searx.search.Search', SearchWithPlugins) diff --git a/searx/settings.yml b/searx/settings.yml index 55d8603a5..8f0b454ae 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -29,6 +29,10 @@ search: formats: - html +# surrogates: +# searx.search.Search: bidule.search.SearchWithPlugins +# searx.query.RawTextQuery: searx.query.DefaultRawTextQuery + server: # If you change port, bind_address or base_url don't forget to rebuild # instance's enviroment (make buildenv) diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 62625b912..2bdcac4e2 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -135,6 +135,10 @@ SCHEMA = { 'public_instances': SettingsValue(str, None), 'wiki_url': SettingsValue(str, None), }, + 'surrogates': SettingsValue(dict, { + 'searx.search.Search': 'searx.search.SearchWithPlugins', + 'searx.query.RawTextQuery': 'searx.query.DefaultRawTextQuery', + }), 'search': { 'safe_search': SettingsValue((0,1,2), 0), 'autocomplete': SettingsValue(str, ''), diff --git a/searx/surrogates.py b/searx/surrogates.py new file mode 100644 index 000000000..850cfdb89 --- /dev/null +++ b/searx/surrogates.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +""" +Allow to replace some objects using settings.yml to lower fork maintenance. +""" + +import sys +from importlib import import_module +from functools import wraps + +from searx import settings, logger + +logger = logger.getChild('surrogates') + + +def _get_obj_by_name(name): + module_name, obj_name = name.rsplit('.', 1) + if module_name not in sys.modules: + module = import_module(module_name) + else: + module = sys.modules[module_name] + return getattr(module, obj_name, None) + + +def get_actual_object(name, obj): + surrogate_name = settings['surrogates'].get(name) + actual_obj = _get_obj_by_name(surrogate_name) if surrogate_name else obj + logger.info('Replace "%s" with "%s"', name, surrogate_name) + if not callable(actual_obj): + raise ValueError(f"{surrogate_name} is not callable") + + @wraps(obj) + def wrapped(*args, **kwargs): + return actual_obj(*args, **kwargs) + return wrapped diff --git a/searx/webapp.py b/searx/webapp.py index cffde08a3..4842d4397 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -109,7 +109,7 @@ from searx.flaskfix import patch_application from searx.autocomplete import search_autocomplete, backends as autocomplete_backends from searx.languages import language_codes as languages from searx.locales import LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES -from searx.search import SearchWithPlugins, initialize as search_initialize +from searx.search import Search, initialize as search_initialize from searx.network import stream as http_stream, set_context_network_name from searx.search.checker import get_result as checker_get_result @@ -652,8 +652,7 @@ def search(): search_query, raw_text_query, _, _ = get_search_query_from_webapp( request.preferences, request.form ) - # search = Search(search_query) # without plugins - search = SearchWithPlugins(search_query, request.user_plugins, request) # pylint: disable=redefined-outer-name + search = Search(search_query, request.user_plugins, request) # pylint: disable=redefined-outer-name result_container = search.search() diff --git a/searx_extra/standalone_searx.py b/searx_extra/standalone_searx.py index b30762d3f..8fdf72066 100755 --- a/searx_extra/standalone_searx.py +++ b/searx_extra/standalone_searx.py @@ -140,7 +140,7 @@ def json_serial(obj: Any) -> Any: def to_dict(search_query: searx.search.SearchQuery) -> Dict[str, Any]: """Get result from parsed arguments.""" - result_container = searx.search.Search(search_query).search() + result_container = searx.search.BasicSearch(search_query).search() result_container_json = { "search": { "q": search_query.query, diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index edb0a18f7..70c9e47ab 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -1,6 +1,6 @@ from searx import settings from searx.engines import load_engines -from searx.query import RawTextQuery +from searx.query import DefaultRawTextQuery from tests import SearxTestCase @@ -20,7 +20,7 @@ class TestQuery(SearxTestCase): def test_simple_query(self): query_text = 'the query' - query = RawTextQuery(query_text, []) + query = DefaultRawTextQuery(query_text, []) self.assertEqual(query.getFullQuery(), query_text) self.assertEqual(len(query.query_parts), 0) @@ -30,7 +30,7 @@ class TestQuery(SearxTestCase): def test_multiple_spaces_query(self): query_text = '\tthe query' - query = RawTextQuery(query_text, []) + query = DefaultRawTextQuery(query_text, []) self.assertEqual(query.getFullQuery(), 'the query') self.assertEqual(len(query.query_parts), 0) @@ -40,18 +40,18 @@ class TestQuery(SearxTestCase): def test_str_method(self): query_text = '<7 the query' - query = RawTextQuery(query_text, []) + query = DefaultRawTextQuery(query_text, []) self.assertEqual(str(query), '<7 the query') def test_repr_method(self): query_text = '<8 the query' - query = RawTextQuery(query_text, []) + query = DefaultRawTextQuery(query_text, []) r = repr(query) - self.assertTrue(r.startswith(f"