Merge pull request #2641 from dalf/disable_http_by_default

[mod] by default allow only HTTPS, not HTTP

docs/dev/engine_overview.rst

@@ -58,6 +58,8 @@ argument                type        information
 name                    string      name of search-engine
 engine                  string      name of searx-engine
                                     (filename without ``.py``)
+enable_http             bool        enable HTTP
+                                    (by default only HTTPS is enabled).
 shortcut                string      shortcut of search-engine
 timeout                 string      specific timeout for search-engine
 display_error_messages  boolean     display error messages on the web UI

searx/engines/__init__.py

@@ -50,6 +50,7 @@ engine_default_args = {'paging': False,
                        'timeout': settings['outgoing']['request_timeout'],
                        'shortcut': '-',
                        'disabled': False,
+                       'enable_http': False,
                        'suspend_end_time': 0,
                        'continuous_errors': 0,
                        'time_range_support': False,
@@ -305,35 +306,3 @@ def initialize_engines(engine_list):
             if init_fn:
                 logger.debug('%s engine: Starting background initialization', engine_name)
                 threading.Thread(target=engine_init, args=(engine_name, init_fn)).start()
-
-        _set_https_support_for_engine(engine)
-
-
-def _set_https_support_for_engine(engine):
-    # check HTTPS support if it is not disabled
-    if engine.engine_type != 'offline' and not hasattr(engine, 'https_support'):
-        params = engine.request('http_test', {
-            'method': 'GET',
-            'headers': {},
-            'data': {},
-            'url': '',
-            'cookies': {},
-            'verify': True,
-            'auth': None,
-            'pageno': 1,
-            'time_range': None,
-            'language': '',
-            'safesearch': False,
-            'is_test': True,
-            'category': 'files',
-            'raise_for_status': True,
-            'engine_data': {},
-        })
-
-        if 'url' not in params:
-            return
-
-        parsed_url = urlparse(params['url'])
-        https_support = parsed_url.scheme == 'https'
-
-        setattr(engine, 'https_support', https_support)

searx/poolrequests.py

@@ -91,9 +91,10 @@ class SessionSinglePool(requests.Session):
         self.adapters.clear()
 
         https_adapter = threadLocal.__dict__.setdefault('https_adapter', next(https_adapters))
-        http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters))
         self.mount('https://', https_adapter)
-        self.mount('http://', http_adapter)
+        if get_enable_http_protocol():
+            http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters))
+            self.mount('http://', http_adapter)
 
     def close(self):
         """Call super, but clear adapters since there are managed globaly"""
@@ -106,6 +107,17 @@ def set_timeout_for_thread(timeout, start_time=None):
     threadLocal.start_time = start_time
 
 
+def set_enable_http_protocol(enable_http):
+    threadLocal.enable_http = enable_http
+
+
+def get_enable_http_protocol():
+    try:
+        return threadLocal.enable_http
+    except AttributeError:
+        return False
+
+
 def reset_time_for_thread():
     threadLocal.total_time = 0
 

searx/search/processors/online.py

@@ -131,6 +131,8 @@ class OnlineProcessor(EngineProcessor):
         poolrequests.set_timeout_for_thread(timeout_limit, start_time=start_time)
         # reset the HTTP total time
         poolrequests.reset_time_for_thread()
+        # enable HTTP only if explicitly enabled
+        poolrequests.set_enable_http_protocol(self.engine.enable_http)
 
         # suppose everything will be alright
         requests_exception = False

searx/settings.yml

@@ -656,6 +656,7 @@ engines:
 
   - name : library genesis
     engine : xpath
+    enable_http: True
     search_url : http://libgen.rs/search.php?req={query}
     url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href
     title_xpath : //a[contains(@href,"book/")]/text()[1]

searx/templates/oscar/preferences.html

@@ -230,7 +230,7 @@
                                     <td class="onoff-checkbox">
                                         {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }}
                                     </td>
-                                    <th scope="row">{% if not search_engine.https_support %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th>
+                                    <th scope="row">{% if search_engine.enable_http %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th>
                                     <td class="name">{{ shortcuts[search_engine.name] }}
                                         <td>{{ support_toggle(stats[search_engine.name].supports_selected_language) }}</td>
                                         <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

searx/templates/simple/preferences.html

@@ -121,7 +121,7 @@
       {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %}
       <tr>
         <td class="engine_checkbox">{{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }}</td>
-        <th class="name">{% if not search_engine.https_support %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th>
+        <th class="name">{% if search_engine.enable_http %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th>
         <td class="shortcut">{{ shortcuts[search_engine.name] }}</td>
         <td>{{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }}</td>
         <td>{{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}</td>