[enh] suspend engines after error

The duration is based on the number of continuous errors, but maximized in one minute

searx/engines/__init__.py

@@ -40,7 +40,9 @@ engine_default_args = {'paging': False,
                        'safesearch': False,
                        'timeout': settings['outgoing']['request_timeout'],
                        'shortcut': '-',
-                       'disabled': False}
+                       'disabled': False,
+                       'suspend_end_time': 0,
+                       'continuous_errors': 0}
 
 
 def load_module(filename):

searx/search.py

@@ -34,16 +34,23 @@ number_of_searches = 0
 
 
 def search_request_wrapper(fn, url, engine_name, **kwargs):
+    ret = None
+    engine = engines[engine_name]
     try:
-        return fn(url, **kwargs)
+        ret = fn(url, **kwargs)
+        with threading.RLock():
+            engine.continuous_errors = 0
+            engine.suspend_end_time = 0
     except:
         # increase errors stats
         with threading.RLock():
-            engines[engine_name].stats['errors'] += 1
+            engine.stats['errors'] += 1
+            engine.continuous_errors += 1
+            engine.suspend_end_time = time() + min(60, engine.continuous_errors)
 
         # print engine name and specific error message
         logger.exception('engine crash: {0}'.format(engine_name))
-        return
+    return ret
 
 
 def threaded_requests(requests):
@@ -241,6 +248,10 @@ class Search(object):
                                     for engine in categories[categ]
                                     if (engine.name, categ) not in self.blocked_engines)
 
+        # remove suspended engines
+        self.engines = [e for e in self.engines
+                        if engines[e['name']].suspend_end_time <= time()]
+
     # do search-request
     def search(self, request):
         global number_of_searches