diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index b51181974..42c205d9d 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -58,6 +58,8 @@ argument type information name string name of search-engine engine string name of searx-engine (filename without ``.py``) +enable_http bool enable HTTP + (by default only HTTPS is enabled). shortcut string shortcut of search-engine timeout string specific timeout for search-engine display_error_messages boolean display error messages on the web UI diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 7270724b6..2238ea1b9 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -50,6 +50,7 @@ engine_default_args = {'paging': False, 'timeout': settings['outgoing']['request_timeout'], 'shortcut': '-', 'disabled': False, + 'enable_http': False, 'suspend_end_time': 0, 'continuous_errors': 0, 'time_range_support': False, @@ -305,35 +306,3 @@ def initialize_engines(engine_list): if init_fn: logger.debug('%s engine: Starting background initialization', engine_name) threading.Thread(target=engine_init, args=(engine_name, init_fn)).start() - - _set_https_support_for_engine(engine) - - -def _set_https_support_for_engine(engine): - # check HTTPS support if it is not disabled - if engine.engine_type != 'offline' and not hasattr(engine, 'https_support'): - params = engine.request('http_test', { - 'method': 'GET', - 'headers': {}, - 'data': {}, - 'url': '', - 'cookies': {}, - 'verify': True, - 'auth': None, - 'pageno': 1, - 'time_range': None, - 'language': '', - 'safesearch': False, - 'is_test': True, - 'category': 'files', - 'raise_for_status': True, - 'engine_data': {}, - }) - - if 'url' not in params: - return - - parsed_url = urlparse(params['url']) - https_support = parsed_url.scheme == 'https' - - setattr(engine, 'https_support', https_support) diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 8b8681437..ab327251b 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -91,9 +91,10 @@ class SessionSinglePool(requests.Session): self.adapters.clear() https_adapter = threadLocal.__dict__.setdefault('https_adapter', next(https_adapters)) - http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters)) self.mount('https://', https_adapter) - self.mount('http://', http_adapter) + if get_enable_http_protocol(): + http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters)) + self.mount('http://', http_adapter) def close(self): """Call super, but clear adapters since there are managed globaly""" @@ -106,6 +107,17 @@ def set_timeout_for_thread(timeout, start_time=None): threadLocal.start_time = start_time +def set_enable_http_protocol(enable_http): + threadLocal.enable_http = enable_http + + +def get_enable_http_protocol(): + try: + return threadLocal.enable_http + except AttributeError: + return False + + def reset_time_for_thread(): threadLocal.total_time = 0 diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index df0ab8c21..1fc6444ad 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -131,6 +131,8 @@ class OnlineProcessor(EngineProcessor): poolrequests.set_timeout_for_thread(timeout_limit, start_time=start_time) # reset the HTTP total time poolrequests.reset_time_for_thread() + # enable HTTP only if explicitly enabled + poolrequests.set_enable_http_protocol(self.engine.enable_http) # suppose everything will be alright requests_exception = False diff --git a/searx/settings.yml b/searx/settings.yml index e45afb59b..9cf2be8c8 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -656,6 +656,7 @@ engines: - name : library genesis engine : xpath + enable_http: True search_url : http://libgen.rs/search.php?req={query} url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href title_xpath : //a[contains(@href,"book/")]/text()[1] diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html index fc20b8ca5..6253b9858 100644 --- a/searx/templates/oscar/preferences.html +++ b/searx/templates/oscar/preferences.html @@ -230,7 +230,7 @@ {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }} - {% if not search_engine.https_support %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }} + {% if search_engine.enable_http %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }} {{ shortcuts[search_engine.name] }} {{ support_toggle(stats[search_engine.name].supports_selected_language) }} {{ support_toggle(search_engine.safesearch==True) }} diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html index f091a97cf..dff7ffba6 100644 --- a/searx/templates/simple/preferences.html +++ b/searx/templates/simple/preferences.html @@ -121,7 +121,7 @@ {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %} {{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }} - {% if not search_engine.https_support %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }} + {% if search_engine.enable_http %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }} {{ shortcuts[search_engine.name] }} {{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }} {{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}