[enh] gevent/grequests changed to the built-in threading lib

requirements.txt

@@ -1,6 +1,6 @@
 flask
 flask-babel
-grequests
+requests
 lxml
 pyyaml
 python-dateutil

searx/search.py

@@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-import grequests
+import requests as requests_lib
+import threading
 import re
 from itertools import izip_longest, chain
 from datetime import datetime
@@ -32,6 +33,18 @@ from searx.query import Query
 number_of_searches = 0
 
 
+def threaded_requests(requests):
+    for fn, url, request_args in requests:
+        th = threading.Thread(
+            target=fn, args=(url,), kwargs=request_args, name=url,
+        )
+        th.start()
+
+    for th in threading.enumerate():
+        if th.name.startswith('http'):
+            th.join()
+
+
 # get default reqest parameter
 def default_request_params():
     return {
@@ -471,9 +484,9 @@ class Search(object):
 
             # specific type of request (GET or POST)
             if request_params['method'] == 'GET':
-                req = grequests.get
+                req = requests_lib.get
             else:
-                req = grequests.post
+                req = requests_lib.post
                 request_args['data'] = request_params['data']
 
             # ignoring empty urls
@@ -481,10 +494,10 @@ class Search(object):
                 continue
 
             # append request to list
-            requests.append(req(request_params['url'], **request_args))
+            requests.append((req, request_params['url'], request_args))
 
         # send all search-request
-        grequests.map(requests)
+        threaded_requests(requests)
 
         # update engine-specific stats
         for engine_name, engine_results in results.items():

searx/webapp.py

@@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-from gevent import monkey
-monkey.patch_all()
-
-
 if __name__ == '__main__':
     from sys import path
     from os.path import realpath, dirname

setup.py

@@ -41,7 +41,7 @@ setup(
     install_requires=[
         'flask',
         'flask-babel',
-        'grequests',
+        'requests',
         'lxml',
         'pyyaml',
         'setuptools',

versions.cfg

@@ -36,18 +36,6 @@ zc.recipe.testrunner = 2.0.0
 # WebTest==2.0.11
 beautifulsoup4 = 4.3.2
 
-# Required by:
-# grequests==0.2.0
-gevent = 1.0
-
-# Required by:
-# gevent==1.0
-greenlet = 0.4.2
-
-# Required by:
-# searx==0.1
-grequests = 0.2.0
-
 # Required by:
 # robotframework-httplibrary==0.4.2
 jsonpatch = 1.3