[mod] search refactor

searx/engines/__init__.py

@@ -2,6 +2,7 @@
 from os.path import realpath, dirname, splitext, join
 from os import listdir
 from imp import load_source
+import grequests
 
 engine_dir = dirname(realpath(__file__))
 
@@ -12,4 +13,40 @@ for filename in listdir(engine_dir):
     if filename.startswith('_') or not filename.endswith('.py'):
         continue
     filepath = join(engine_dir, filename)
-    engines.append(load_source(modname, filepath))
+    engine = load_source(modname, filepath)
+    if not hasattr(engine, 'request') or not hasattr(engine, 'response'):
+        continue
+    engines.append(engine)
+
+def default_request_params():
+    return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
+
+def make_callback(results, callback):
+    def process_callback(response, **kwargs):
+        results.extend(callback(response))
+    return process_callback
+
+def search(query, request):
+    global engines
+    requests = []
+    results = []
+    user_agent = request.headers.get('User-Agent', '')
+    for engine in engines:
+        headers = default_request_params()
+        headers['User-Agent'] = user_agent
+        request_params = engine.request(query, headers)
+        callback = make_callback(results, engine.response)
+        if request_params['method'] == 'GET':
+            req = grequests.get(request_params['url']
+                                ,headers=headers
+                                ,hooks=dict(response=callback)
+                                )
+        else:
+            req = grequests.post(request_params['url']
+                                ,data=request_params['data']
+                                ,headers=headers
+                                ,hooks=dict(response=callback)
+                                )
+        requests.append(req)
+    grequests.map(requests)
+    return results

searx/webapp.py

@@ -25,8 +25,7 @@ if __name__ == "__main__":
 from flask import Flask, request, flash, render_template
 import ConfigParser
 from os import getenv
-from searx.engines import engines
-import grequests
+from searx.engines import search
 
 cfg = ConfigParser.SafeConfigParser()
 cfg.read('/etc/searx.conf')
@@ -38,14 +37,6 @@ cfg.read('searx.conf')
 app = Flask(__name__)
 app.secret_key = cfg.get('app', 'secret_key')
 
-def default_request_params():
-    return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
-
-def make_callback(results, callback):
-    def process_callback(response, **kwargs):
-        results.extend(callback(response))
-    return process_callback
-
 @app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method=='POST':
@@ -53,30 +44,8 @@ def index():
             flash('Wrong post data')
             return render_template('index.html')
         query = request.form['q']
-        requests = []
-        results = []
-        user_agent = request.headers.get('User-Agent', '')
-        for engine in engines:
-            headers = default_request_params()
-            headers['User-Agent'] = user_agent
-            request_params = engine.request(query, headers)
-            callback = make_callback(results, engine.response)
-            if request_params['method'] == 'GET':
-                req = grequests.get(request_params['url']
-                                   ,headers=headers
-                                   ,hooks=dict(response=callback)
-                                   )
-            else:
-                req = grequests.post(request_params['url']
-                                    ,data=request_params['data']
-                                    ,headers=headers
-                                    ,hooks=dict(response=callback)
-                                    )
-            requests.append(req)
-        grequests.map(requests)
+        results = search(query, request)
         return render_template('results.html', results=results, q=query)
-
-
     return render_template('index.html')
 
 if __name__ == "__main__":