zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

[enh] add surrogates

Browse source 
to lower fork maintenance, some objects in SearXNG can be replaced.
The configuration is done in settings.yml
This commit is contained in:
Alexandre Flament 2021-09-15 16:34:52 +02:00
parent 602cbc2c99
commit 752c224608
11 changed files with 102 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
searx/query.py
View file

@ -8,6 +8,7 @@ from searx.engines import categories, engines, engine_shortcuts
from searx.external_bang import get_bang_definition_and_autocomplete
from searx.search import EngineRef
from searx.webutils import VALID_LANGUAGE_CODE
from searx.surrogates import get_actual_object
class QueryPartParser(ABC):
@ -232,7 +233,7 @@ class BangParser(QueryPartParser):
self._add_autocomplete(first_char + engine_shortcut)
class RawTextQuery:
class DefaultRawTextQuery:
"""parse raw text query (the value from the html input)"""
PARSER_CLASSES = [
@ -328,3 +329,6 @@ class RawTextQuery:
+ f"autocomplete_list={self.autocomplete_list!r}\n " \
+ f"query_parts={self.query_parts!r}\n " \
+ f"user_query_parts={self.user_query_parts!r} >"
RawTextQuery = get_actual_object('searx.query.RawTextQuery', DefaultRawTextQuery)

8
searx/search/__init__.py
View file

@ -19,6 +19,7 @@ from searx.network import initialize as initialize_network
from searx.metrics import initialize as initialize_metrics, counter_inc, histogram_observe_time
from searx.search.processors import PROCESSORS, initialize as initialize_processors
from searx.search.checker import initialize as initialize_checker
from searx.surrogates import get_actual_object
logger = logger.getChild('search')
@ -34,7 +35,7 @@ def initialize(settings_engines=None, enable_checker=False):
initialize_checker()
class Search:
class BasicSearch:
"""Search information container"""
__slots__ = "search_query", "result_container", "start_time", "actual_timeout"
@ -171,7 +172,7 @@ class Search:
return self.result_container
class SearchWithPlugins(Search):
class SearchWithPlugins(BasicSearch):
"""Inherit from the Search class, add calls to the plugins."""
__slots__ = 'ordered_plugin_list', 'request'
@ -201,3 +202,6 @@ class SearchWithPlugins(Search):
self.result_container.close()
return self.result_container
Search = get_actual_object('searx.search.Search', SearchWithPlugins)

4
searx/settings.yml
View file

@ -29,6 +29,10 @@ search:
formats:
- html
# surrogates:
# searx.search.Search: bidule.search.SearchWithPlugins
# searx.query.RawTextQuery: searx.query.DefaultRawTextQuery
server:
# If you change port, bind_address or base_url don't forget to rebuild
# instance's enviroment (make buildenv)

4
searx/settings_defaults.py
View file

@ -135,6 +135,10 @@ SCHEMA = {
'public_instances': SettingsValue(str, None),
'wiki_url': SettingsValue(str, None),
},
'surrogates': SettingsValue(dict, {
'searx.search.Search': 'searx.search.SearchWithPlugins',
'searx.query.RawTextQuery': 'searx.query.DefaultRawTextQuery',
}),
'search': {
'safe_search': SettingsValue((0,1,2), 0),
'autocomplete': SettingsValue(str, ''),

35
searx/surrogates.py Normal file
View file

@ -0,0 +1,35 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""
Allow to replace some objects using settings.yml to lower fork maintenance.
"""
import sys
from importlib import import_module
from functools import wraps
from searx import settings, logger
logger = logger.getChild('surrogates')
def _get_obj_by_name(name):
module_name, obj_name = name.rsplit('.', 1)
if module_name not in sys.modules:
module = import_module(module_name)
else:
module = sys.modules[module_name]
return getattr(module, obj_name, None)
def get_actual_object(name, obj):
surrogate_name = settings['surrogates'].get(name)
actual_obj = _get_obj_by_name(surrogate_name) if surrogate_name else obj
logger.info('Replace "%s" with "%s"', name, surrogate_name)
if not callable(actual_obj):
raise ValueError(f"{surrogate_name} is not callable")
@wraps(obj)
def wrapped(*args, **kwargs):
return actual_obj(*args, **kwargs)
return wrapped

5
searx/webapp.py
View file

@ -109,7 +109,7 @@ from searx.flaskfix import patch_application
from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
from searx.languages import language_codes as languages
from searx.locales import LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES
from searx.search import SearchWithPlugins, initialize as search_initialize
from searx.search import Search, initialize as search_initialize
from searx.network import stream as http_stream, set_context_network_name
from searx.search.checker import get_result as checker_get_result
@ -652,8 +652,7 @@ def search():
search_query, raw_text_query, _, _ = get_search_query_from_webapp(
request.preferences, request.form
)
# search = Search(search_query) # without plugins
search = SearchWithPlugins(search_query, request.user_plugins, request) # pylint: disable=redefined-outer-name
search = Search(search_query, request.user_plugins, request) # pylint: disable=redefined-outer-name
result_container = search.search()

2
searx_extra/standalone_searx.py
View file

@ -140,7 +140,7 @@ def json_serial(obj: Any) -> Any:
def to_dict(search_query: searx.search.SearchQuery) -> Dict[str, Any]:
"""Get result from parsed arguments."""
result_container = searx.search.Search(search_query).search()
result_container = searx.search.BasicSearch(search_query).search()
result_container_json = {
"search": {
"q": search_query.query,

70
tests/unit/test_query.py
View file

@ -1,6 +1,6 @@
from searx import settings
from searx.engines import load_engines
from searx.query import RawTextQuery
from searx.query import DefaultRawTextQuery
from tests import SearxTestCase
@ -20,7 +20,7 @@ class TestQuery(SearxTestCase):
def test_simple_query(self):
query_text = 'the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 0)
@ -30,7 +30,7 @@ class TestQuery(SearxTestCase):
def test_multiple_spaces_query(self):
query_text = '\tthe query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), 'the query')
self.assertEqual(len(query.query_parts), 0)
@ -40,18 +40,18 @@ class TestQuery(SearxTestCase):
def test_str_method(self):
query_text = '<7 the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(str(query), '<7 the query')
def test_repr_method(self):
query_text = '<8 the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
r = repr(query)
self.assertTrue(r.startswith(f"<RawTextQuery query='{query_text}' "))
self.assertTrue(r.startswith(f"<DefaultRawTextQuery query='{query_text}' "))
def test_change_query(self):
query_text = '<8 the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
another_query = query.changeQuery('another text')
self.assertEqual(query, another_query)
self.assertEqual(query.getFullQuery(), '<8 another text')
@ -63,7 +63,7 @@ class TestLanguageParser(SearxTestCase):
language = 'es-ES'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query = DefaultRawTextQuery(full_query, [])
self.assertEqual(query.getFullQuery(), full_query)
self.assertEqual(len(query.query_parts), 1)
@ -75,7 +75,7 @@ class TestLanguageParser(SearxTestCase):
language = 'english'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query = DefaultRawTextQuery(full_query, [])
self.assertEqual(query.getFullQuery(), full_query)
self.assertEqual(len(query.query_parts), 1)
@ -86,7 +86,7 @@ class TestLanguageParser(SearxTestCase):
language = 'all'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query = DefaultRawTextQuery(full_query, [])
self.assertEqual(query.getFullQuery(), full_query)
self.assertEqual(len(query.query_parts), 1)
@ -97,7 +97,7 @@ class TestLanguageParser(SearxTestCase):
language = 'not_a_language'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query = DefaultRawTextQuery(full_query, [])
self.assertEqual(query.getFullQuery(), full_query)
self.assertEqual(len(query.query_parts), 0)
@ -106,7 +106,7 @@ class TestLanguageParser(SearxTestCase):
def test_empty_colon_in_query(self):
query_text = 'the : query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 0)
@ -115,23 +115,23 @@ class TestLanguageParser(SearxTestCase):
def test_autocomplete_empty(self):
query_text = 'the query :'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.autocomplete_list, [":en", ":en_us", ":english", ":united_kingdom"])
def test_autocomplete(self):
query = RawTextQuery(':englis', [])
query = DefaultRawTextQuery(':englis', [])
self.assertEqual(query.autocomplete_list, [":english"])
query = RawTextQuery(':deutschla', [])
query = DefaultRawTextQuery(':deutschla', [])
self.assertEqual(query.autocomplete_list, [":deutschland"])
query = RawTextQuery(':new_zea', [])
query = DefaultRawTextQuery(':new_zea', [])
self.assertEqual(query.autocomplete_list, [":new_zealand"])
query = RawTextQuery(':hu-H', [])
query = DefaultRawTextQuery(':hu-H', [])
self.assertEqual(query.autocomplete_list, [":hu-hu"])
query = RawTextQuery(':v', [])
query = DefaultRawTextQuery(':v', [])
self.assertEqual(query.autocomplete_list, [":vi", ":tiếng việt"])
@ -139,7 +139,7 @@ class TestTimeoutParser(SearxTestCase):
def test_timeout_below100(self):
query_text = '<3 the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 1)
@ -148,7 +148,7 @@ class TestTimeoutParser(SearxTestCase):
def test_timeout_above100(self):
query_text = '<350 the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 1)
@ -157,7 +157,7 @@ class TestTimeoutParser(SearxTestCase):
def test_timeout_above1000(self):
query_text = '<3500 the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 1)
@ -167,7 +167,7 @@ class TestTimeoutParser(SearxTestCase):
def test_timeout_invalid(self):
# invalid number: it is not bang but it is part of the query
query_text = '<xxx the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 0)
@ -178,7 +178,7 @@ class TestTimeoutParser(SearxTestCase):
def test_timeout_autocomplete(self):
# invalid number: it is not bang but it is part of the query
query_text = 'the query <'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 0)
@ -192,7 +192,7 @@ class TestExternalBangParser(SearxTestCase):
def test_external_bang(self):
query_text = '!!ddg the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 1)
@ -200,7 +200,7 @@ class TestExternalBangParser(SearxTestCase):
def test_external_bang_not_found(self):
query_text = '!!notfoundbang the query'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(query.external_bang, None)
@ -208,7 +208,7 @@ class TestExternalBangParser(SearxTestCase):
def test_external_bang_autocomplete(self):
query_text = 'the query !!dd'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), '!!dd the query')
self.assertEqual(len(query.query_parts), 1)
@ -220,7 +220,7 @@ class TestExternalBangParser(SearxTestCase):
def test_external_bang_autocomplete_empty(self):
query_text = 'the query !!'
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), 'the query !!')
self.assertEqual(len(query.query_parts), 0)
@ -243,7 +243,7 @@ class TestBang(SearxTestCase):
for bang in TestBang.SPECIFIC_BANGS + TestBang.NOT_SPECIFIC_BANGS:
with self.subTest(msg="Check bang", bang=bang):
query_text = TestBang.THE_QUERY + ' ' + bang
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), bang + ' ' + TestBang.THE_QUERY)
self.assertEqual(query.query_parts, [bang])
@ -253,34 +253,34 @@ class TestBang(SearxTestCase):
for bang in TestBang.SPECIFIC_BANGS:
with self.subTest(msg="Check bang is specific", bang=bang):
query_text = TestBang.THE_QUERY + ' ' + bang
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertTrue(query.specific)
def test_not_specific(self):
for bang in TestBang.NOT_SPECIFIC_BANGS:
with self.subTest(msg="Check bang is not specific", bang=bang):
query_text = TestBang.THE_QUERY + ' ' + bang
query = RawTextQuery(query_text, [])
query = DefaultRawTextQuery(query_text, [])
self.assertFalse(query.specific)
def test_bang_not_found(self):
load_engines(TEST_ENGINES)
query = RawTextQuery('the query !bang_not_found', [])
query = DefaultRawTextQuery('the query !bang_not_found', [])
self.assertEqual(query.getFullQuery(), 'the query !bang_not_found')
def test_bang_autocomplete(self):
load_engines(TEST_ENGINES)
query = RawTextQuery('the query !dum', [])
query = DefaultRawTextQuery('the query !dum', [])
self.assertEqual(query.autocomplete_list, ['!dummy_engine'])
query = RawTextQuery('!dum the query', [])
query = DefaultRawTextQuery('!dum the query', [])
self.assertEqual(query.autocomplete_list, [])
self.assertEqual(query.getQuery(), '!dum the query')
def test_bang_autocomplete_empty(self):
load_engines(settings['engines'])
query = RawTextQuery('the query !', [])
query = DefaultRawTextQuery('the query !', [])
self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm'])
query = RawTextQuery('the query ?', ['osm'])
query = DefaultRawTextQuery('the query ?', ['osm'])
self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia'])

14
tests/unit/test_search.py
View file

@ -45,7 +45,7 @@ class SearchTestCase(SearxTestCase):
settings['outgoing']['max_request_timeout'] = None
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, None)
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
search.search()
self.assertEqual(search.actual_timeout, 3.0)
@ -53,7 +53,7 @@ class SearchTestCase(SearxTestCase):
settings['outgoing']['max_request_timeout'] = None
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, 5.0)
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
search.search()
self.assertEqual(search.actual_timeout, 3.0)
@ -61,7 +61,7 @@ class SearchTestCase(SearxTestCase):
settings['outgoing']['max_request_timeout'] = None
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, 1.0)
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
search.search()
self.assertEqual(search.actual_timeout, 1.0)
@ -69,7 +69,7 @@ class SearchTestCase(SearxTestCase):
settings['outgoing']['max_request_timeout'] = 10.0
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, 5.0)
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
search.search()
self.assertEqual(search.actual_timeout, 5.0)
@ -77,7 +77,7 @@ class SearchTestCase(SearxTestCase):
settings['outgoing']['max_request_timeout'] = 10.0
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, 15.0)
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
search.search()
self.assertEqual(search.actual_timeout, 10.0)
@ -86,7 +86,7 @@ class SearchTestCase(SearxTestCase):
[EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, None,
external_bang="yt")
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
results = search.search()
# For checking if the user redirected with the youtube external bang
self.assertTrue(results.redirect_url is not None)
@ -95,7 +95,7 @@ class SearchTestCase(SearxTestCase):
[EngineRef(PUBLIC_ENGINE_NAME, 'general')],
'en-US', SAFESEARCH, PAGENO, None, None)
search = searx.search.Search(search_query)
search = searx.search.BasicSearch(search_query)
results = search.search()
# This should not redirect
self.assertTrue(results.redirect_url is None)

2
tests/unit/test_standalone_searx.py
View file

@ -59,7 +59,7 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict_with_mock(self):
"""test to dict."""
with patch.object(sas.searx.search, 'Search') as mock_s:
with patch.object(sas.searx.search, 'BasicSearch') as mock_s:
m_search = mock_s().search()
m_sq = Mock()
self.assertEqual(

4
tests/unit/test_webapp.py
View file

@ -5,7 +5,7 @@ from urllib.parse import ParseResult
from mock import Mock
import searx.search.processors
from searx.search import Search
from searx.search import BasicSearch
from tests import SearxTestCase
@ -68,7 +68,7 @@ class ViewsTestCase(SearxTestCase):
redirect_url=None,
engine_data={})
self.setattr4test(Search, 'search', search_mock)
self.setattr4test(BasicSearch, 'search', search_mock)
def get_current_theme_name_mock(override=None):
if override: