fix: robot fw, entry points, some flake8, package searx egg

.gitignore

@@ -1,20 +1,24 @@
-env
-engines.cfg
-.installed.cfg
 .coverage
-coverage/
+.installed.cfg
+engines.cfg
+env
+robot_log.html
+robot_output.xml
+robot_report.html
 setup.cfg
 
 *.pyc
 */*.pyc
 
 bin/
+build/
+covearge/
+develop-eggs/
+dist/
+eggs/
 include/
 lib/
-build/
-develop-eggs/
-eggs/
 local/
-searx.egg-info/
 parts/
+searx.egg-info/
 var/

Makefile

@@ -21,11 +21,7 @@ $(python):
 tests: .installed.cfg
 	@bin/test
 
-enginescfg:
+robot: .installed.cfg
-	@test -f ./engines.cfg || echo "Copying engines.cfg ..."
-	@cp --no-clobber engines.cfg_sample engines.cfg
-
-robot: .installed.cfg enginescfg
 	@bin/robot
 
 flake8: .installed.cfg
@@ -37,18 +33,18 @@ coverage: .installed.cfg
 	@bin/coverage report --show-missing
 	@bin/coverage html --directory ./coverage
 
-production: bin/buildout production.cfg setup.py enginescfg
+production: bin/buildout production.cfg setup.py
 	bin/buildout -c production.cfg $(options)
 	@echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"
 	@echo "* Hint 1: on production, disable debug mode and change secret_key"
 	@echo "* Hint 2: searx will be executed at server startup by crontab"
 	@echo "* Hint 3: to run immediatley, execute 'bin/supervisord'"
 
-minimal: bin/buildout minimal.cfg setup.py enginescfg
+minimal: bin/buildout minimal.cfg setup.py
 	bin/buildout -c minimal.cfg $(options)
 
 clean:
 	@rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs \
 		searx.egg-info lib include .coverage coverage
 
-.PHONY: all tests enginescfg robot flake8 coverage production minimal clean
+.PHONY: all tests robot flake8 coverage production minimal clean

buildout.cfg

@@ -16,8 +16,6 @@ recipe = zc.recipe.egg:script
 eggs = ${buildout:eggs}
 interpreter = py
 dependent-scripts = true
-entry-points =
-    searx-run=searx.webapp:run
 
 
 [robot]

minimal.cfg

@@ -13,5 +13,3 @@ parts +=
 recipe = zc.recipe.egg:script
 eggs = ${buildout:eggs}
 interpreter = py
-entry-points =
-    searx-run=searx.webapp:run

production.cfg

@@ -15,8 +15,6 @@ parts +=
 recipe = zc.recipe.egg:script
 eggs = ${buildout:eggs}
 interpreter = py
-entry-points =
-    searx-run=searx.webapp:run
 
 
 [supervisor]

searx/__init__.py

@@ -1,5 +1,5 @@
 from os import environ
-from os.path import realpath, dirname, join
+from os.path import realpath, dirname, join, abspath
 try:
     from yaml import load
 except:
@@ -7,8 +7,7 @@ except:
     stderr.write('[E] install pyyaml\n')
     exit(2)
 
-
+searx_dir = abspath(dirname(__file__))
-searx_dir  = realpath(dirname(realpath(__file__))+'/../')
 engine_dir = dirname(realpath(__file__))
 
 if 'SEARX_SETTINGS_PATH' in environ:
@@ -19,4 +18,3 @@ else:
 
 with open(settings_path) as settings_yaml:
     settings = load(settings_yaml)
-

searx/engines/__init__.py

@@ -35,6 +35,7 @@ engines = {}
 
 categories = {'general': []}
 
+
 def load_module(filename):
     modname = splitext(filename)[0]
     if modname in sys.modules:
@@ -58,38 +59,50 @@ for engine_data in settings['engines']:
             if engine_data['categories'] == 'none':
                 engine.categories = []
             else:
-                engine.categories = map(str.strip, engine_data['categories'].split(','))
+                engine.categories = map(
+                    str.strip, engine_data['categories'].split(','))
             continue
         setattr(engine, param_name, engine_data[param_name])
     for engine_attr in dir(engine):
         if engine_attr.startswith('_'):
             continue
         if getattr(engine, engine_attr) == None:
-            print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr)
+            print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr)  # noqa
             sys.exit(1)
     engines[engine.name] = engine
-    engine.stats = {'result_count': 0, 'search_count': 0, 'page_load_time': 0, 'score_count': 0, 'errors': 0}
+    engine.stats = {
+        'result_count': 0,
+        'search_count': 0,
+        'page_load_time': 0,
+        'score_count': 0,
+        'errors': 0
+    }
     if hasattr(engine, 'categories'):
         for category_name in engine.categories:
             categories.setdefault(category_name, []).append(engine)
     else:
         categories['general'].append(engine)
 
+
 def default_request_params():
-    return {'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
+    return {
+        'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
+
 
 def make_callback(engine_name, results, suggestions, callback, params):
     # creating a callback wrapper for the search engine results
     def process_callback(response, **kwargs):
         cb_res = []
         response.search_params = params
-        engines[engine_name].stats['page_load_time'] += (datetime.now() - params['started']).total_seconds()
+        engines[engine_name].stats['page_load_time'] += \
+            (datetime.now() - params['started']).total_seconds()
         try:
             search_results = callback(response)
         except Exception, e:
             engines[engine_name].stats['errors'] += 1
             results[engine_name] = cb_res
-            print '[E] Error with engine "{0}":\n\t{1}'.format(engine_name, str(e))
+            print '[E] Error with engine "{0}":\n\t{1}'.format(
+                engine_name, str(e))
             return
         for result in search_results:
             result['engine'] = engine_name
@@ -101,8 +114,10 @@ def make_callback(engine_name, results, suggestions, callback, params):
         results[engine_name] = cb_res
     return process_callback
 
+
 def score_results(results):
-    flat_res = filter(None, chain.from_iterable(izip_longest(*results.values())))
+    flat_res = filter(
+        None, chain.from_iterable(izip_longest(*results.values())))
     flat_len = len(flat_res)
     engines_len = len(results)
     results = []
@@ -116,8 +131,8 @@ def score_results(results):
         score = int((flat_len - i) / engines_len) * weight + 1
         duplicated = False
         for new_res in results:
-            p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path
+            p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path  # noqa
-            p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path
+            p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path  # noqa
             if res['parsed_url'].netloc == new_res['parsed_url'].netloc and\
                p1 == p2 and\
                res['parsed_url'].query == new_res['parsed_url'].query and\
@@ -125,7 +140,7 @@ def score_results(results):
                 duplicated = new_res
                 break
         if duplicated:
-            if len(res.get('content', '')) > len(duplicated.get('content', '')):
+            if len(res.get('content', '')) > len(duplicated.get('content', '')):  # noqa
                 duplicated['content'] = res['content']
             duplicated['score'] += score
             duplicated['engines'].append(res['engine'])
@@ -139,6 +154,7 @@ def score_results(results):
             results.append(res)
     return sorted(results, key=itemgetter('score'), reverse=True)
 
+
 def search(query, request, selected_engines):
     global engines, categories, number_of_searches
     requests = []
@@ -160,12 +176,19 @@ def search(query, request, selected_engines):
         request_params['started'] = datetime.now()
         request_params = engine.request(query, request_params)
 
-        callback = make_callback(selected_engine['name'], results, suggestions, engine.response, request_params)
+        callback = make_callback(
+            selected_engine['name'],
+            results,
+            suggestions,
+            engine.response,
+            request_params
+        )
 
-        request_args = dict(headers = request_params['headers']
+        request_args = dict(
-                           ,hooks   = dict(response=callback)
+            headers=request_params['headers'],
-                           ,cookies = request_params['cookies']
+            hooks=dict(response=callback),
-                           ,timeout = settings['server']['request_timeout']
+            cookies=request_params['cookies'],
+            timeout=settings['server']['request_timeout']
         )
 
         if request_params['method'] == 'GET':
@@ -192,6 +215,7 @@ def search(query, request, selected_engines):
 
     return results, suggestions
 
+
 def get_engines_stats():
     # TODO refactor
     pageloads = []
@@ -200,14 +224,15 @@ def get_engines_stats():
     errors = []
     scores_per_result = []
 
-    max_pageload = max_results = max_score = max_errors = max_score_per_result = 0
+    max_pageload = max_results = max_score = max_errors = max_score_per_result = 0  # noqa
     for engine in engines.values():
         if engine.stats['search_count'] == 0:
             continue
-        results_num = engine.stats['result_count']/float(engine.stats['search_count'])
+        results_num = \
-        load_times  = engine.stats['page_load_time']/float(engine.stats['search_count'])
+            engine.stats['result_count'] / float(engine.stats['search_count'])
+        load_times = engine.stats['page_load_time'] / float(engine.stats['search_count'])  # noqa
         if results_num:
-            score = engine.stats['score_count'] / float(engine.stats['search_count'])
+            score = engine.stats['score_count'] / float(engine.stats['search_count'])  # noqa
             score_per_result = score / results_num
         else:
             score = score_per_result = 0.0
@@ -220,7 +245,10 @@ def get_engines_stats():
         results.append({'avg': results_num, 'name': engine.name})
         scores.append({'avg': score, 'name': engine.name})
         errors.append({'avg': engine.stats['errors'], 'name': engine.name})
-        scores_per_result.append({'avg': score_per_result, 'name': engine.name})
+        scores_per_result.append({
+            'avg': score_per_result,
+            'name': engine.name
+        })
 
     for engine in pageloads:
         engine['percentage'] = int(engine['avg'] / max_pageload * 100)
@@ -240,10 +268,16 @@ def get_engines_stats():
         else:
             engine['percentage'] = 0
 
-
+    return [
-    return [('Page loads (sec)', sorted(pageloads, key=itemgetter('avg')))
+        ('Page loads (sec)', sorted(pageloads, key=itemgetter('avg'))),
-           ,('Number of results', sorted(results, key=itemgetter('avg'), reverse=True))
+        (
-           ,('Scores', sorted(scores, key=itemgetter('avg'), reverse=True))
+            'Number of results',
-           ,('Scores per result', sorted(scores_per_result, key=itemgetter('avg'), reverse=True))
+            sorted(results, key=itemgetter('avg'), reverse=True)
-           ,('Errors', sorted(errors, key=itemgetter('avg'), reverse=True))
+        ),
+        ('Scores', sorted(scores, key=itemgetter('avg'), reverse=True)),
+        (
+            'Scores per result',
+            sorted(scores_per_result, key=itemgetter('avg'), reverse=True)
+        ),
+        ('Errors', sorted(errors, key=itemgetter('avg'), reverse=True)),
     ]

searx/engines/bing.py

@@ -8,7 +8,8 @@ locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx
 
 
 def request(query, params):
-    search_path = search_string.format(query=urlencode({'q': query, 'setmkt': locale}))
+    search_path = search_string.format(
+        query=urlencode({'q': query, 'setmkt': locale}))
     #if params['category'] == 'images':
     #    params['url'] = base_url + 'images/' + search_path
     params['url'] = base_url + search_path

searx/engines/currency_convert.py

@@ -7,6 +7,7 @@ weight = 100
 
 parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I)
 
+
 def request(query, params):
     m = parser_re.match(query)
     if not m:
@@ -38,19 +39,21 @@ def response(resp):
     except:
         return results
 
-    title = '{0} {1} in {2} is {3}'.format(resp.search_params['ammount']
+    title = '{0} {1} in {2} is {3}'.format(
-                                          ,resp.search_params['from']
+        resp.search_params['ammount'],
-                                          ,resp.search_params['to']
+        resp.search_params['from'],
-                                          ,resp.search_params['ammount']*conversion_rate
+        resp.search_params['to'],
+        resp.search_params['ammount'] * conversion_rate
     )
 
     content = '1 {0} is {1} {2}'.format(resp.search_params['from'], conversion_rate, resp.search_params['to'])
     now_date = datetime.now().strftime('%Y%m%d')
     url = 'http://finance.yahoo.com/currency/converter-results/{0}/{1}-{2}-to-{3}.html'
-    url = url.format(now_date
+    url = url.format(
-                    ,resp.search_params['ammount']
+        now_date,
-                    ,resp.search_params['from'].lower()
+        resp.search_params['ammount'],
-                    ,resp.search_params['to'].lower()
+        resp.search_params['from'].lower(),
+        resp.search_params['to'].lower()
     )
     results.append({'title': title, 'content': content, 'url': url})
 

searx/engines/dailymotion.py

@@ -8,9 +8,11 @@ locale = 'en_US'
 # see http://www.dailymotion.com/doc/api/obj-video.html
 search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'
 
+
 def request(query, params):
     global search_url
-    params['url'] = search_url.format(query=urlencode({'search': query, 'localization': locale }))
+    params['url'] = search_url.format(
+        query=urlencode({'search': query, 'localization': locale}))
     return params
 
 
@@ -32,6 +34,7 @@ def response(resp):
         results.append({'url': url, 'title': title, 'content': content})
     return results
 
+
 def text_content_from_html(html_string):
     desc_html = html.fragment_fromstring(html_string, create_parent=True)
     return desc_html.text_content()

searx/settings_robot.py

@@ -1,16 +0,0 @@
-
-port = 11111
-
-secret_key = "ultrasecretkey" # change this!
-
-debug = False
-
-request_timeout = 5.0 # seconds
-
-weights = {} # 'search_engine_name': float(weight) | default is 1.0
-
-blacklist = [] # search engine blacklist
-
-categories = {} # custom search engine categories
-
-base_url = None # "https://your.domain.tld/" or None (to use request parameters)

searx/settings_robot.yml

@@ -0,0 +1,107 @@
+server:
+    port : 11111
+    secret_key : "ultrasecretkey" # change this!
+    debug : False
+    request_timeout : 3.0 # seconds
+    base_url: False
+
+engines:
+  - name : wikipedia
+    engine : mediawiki
+    url    : https://en.wikipedia.org/
+    number_of_results : 1
+
+  - name : bing
+    engine : bing
+    locale : en-US
+
+  - name : currency
+    engine : currency_convert
+    categories : general
+
+  - name : deviantart
+    engine : deviantart
+    categories : images
+
+  - name : ddg definitions
+    engine : duckduckgo_definitions
+
+  - name : duckduckgo
+    engine : duckduckgo
+    locale : en-us
+
+  - name : filecrop
+    engine : filecrop
+    categories : files
+
+  - name : flickr
+    engine : flickr
+    categories : images
+
+  - name : github
+    engine : github
+    categories : it
+
+  - name : google
+    engine        : json_engine
+    search_url    : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
+    categories    : general
+    url_query     : /responseData/results/unescapedUrl
+    content_query : /responseData/results/content
+    title_query   : /responseData/results/titleNoFormatting
+
+  - name : google images
+    engine : google_images
+    categories : images
+
+  - name : piratebay
+    engine : piratebay
+    categories : videos, music, files
+
+  - name : soundcloud
+    engine : soundcloud
+    categories : music
+
+  - name : stackoverflow
+    engine : stackoverflow
+    categories : it
+
+  - name : startpage
+    engine : startpage
+
+  - name : twitter
+    engine : twitter
+    categories : social media
+
+  - name : urbandictionary
+    engine        : xpath
+    search_url    : http://www.urbandictionary.com/define.php?term={query}
+    url_xpath     : //div[@class="word"]//a/@href
+    title_xpath   : //div[@class="word"]//a
+    content_xpath : //div[@class="definition"]
+
+  - name : yahoo
+    engine           : xpath
+    search_url       : http://search.yahoo.com/search?p={query}
+    results_xpath    : //div[@class="res"]
+    url_xpath        : .//h3/a/@href
+    title_xpath      : .//h3/a
+    content_xpath    : .//div[@class="abstr"]
+    suggestion_xpath : //div[@id="satat"]//a
+
+  - name : youtube
+    engine : youtube
+    categories : videos
+
+  - name : dailymotion
+    engine : dailymotion
+    locale : en_US
+    categories : videos
+
+  - name : vimeo
+    engine : vimeo
+    categories : videos
+    results_xpath : //div[@id="browse_content"]/ol/li
+    url_xpath : ./a/@href
+    title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()
+    content_xpath : ./a/img/@src

searx/testing.py

@@ -7,10 +7,10 @@ from unittest2 import TestCase
 
 import os
 import subprocess
-import sys
 
 
 class SearxTestLayer:
+    """Base layer for non-robot tests."""
 
     __name__ = u'SearxTestLayer'
 
@@ -36,24 +36,37 @@ class SearxRobotLayer(Layer):
 
     def setUp(self):
         os.setpgrp()  # create new process group, become its leader
+
+        # get program paths
         webapp = os.path.join(
             os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
             'webapp.py'
         )
         exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py')
+
+        # set robot settings path
+        os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath(
+            os.path.dirname(__file__) + '/settings_robot.yml')
+
+        # run the server
         self.server = subprocess.Popen(
-            [exe, webapp, 'settings_robot'],
+            [exe, webapp],
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT
         )
 
     def tearDown(self):
-        # TERM all processes in my group
+        # send TERM signal to all processes in my group, to stop subprocesses
         os.killpg(os.getpgid(self.server.pid), 15)
 
+        # remove previously set environment variable
+        del os.environ['SEARX_SETTINGS_PATH']
+
 
 SEARXROBOTLAYER = SearxRobotLayer()
 
 
 class SearxTestCase(TestCase):
+    """Base test case for non-robot tests."""
+
     layer = SearxTestLayer

searx/utils.py

@@ -5,10 +5,12 @@ import codecs
 import cStringIO
 import re
 
+
 def gen_useragent():
     # TODO
     return "Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"
 
+
 def highlight_content(content, query):
 
     if not content:
@@ -34,6 +36,7 @@ def highlight_content(content, query):
 
     return content
 
+
 class HTMLTextExtractor(HTMLParser):
     def __init__(self):
         HTMLParser.__init__(self)
@@ -54,6 +57,7 @@ class HTMLTextExtractor(HTMLParser):
     def get_text(self):
         return u''.join(self.result)
 
+
 def html_to_text(html):
     s = HTMLTextExtractor()
     s.feed(html)

searx/webapp.py

@@ -17,13 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-import os
-import sys
-if __name__ == "__main__":
-    sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))
-
 from searx import settings
-
 from flask import Flask, request, render_template, url_for, Response, make_response, redirect
 from searx.engines import search, categories, engines, get_engines_stats
 import json
@@ -33,11 +27,17 @@ from flask import send_from_directory
 from searx.utils import highlight_content, html_to_text
 
 
+import os
+
+
+app = Flask(
+    __name__,
+    static_folder=os.path.join(os.path.dirname(__file__), 'static'),
+    template_folder=os.path.join(os.path.dirname(__file__), 'templates')
+)
 
-app = Flask(__name__)
 app.secret_key = settings['server']['secret_key']
 
-
 #TODO configurable via settings.yml
 favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',
             'twitter', 'stackoverflow', 'github']
@@ -81,6 +81,7 @@ def render(template_name, **kwargs):
             kwargs['selected_categories'] = ['general']
     return render_template(template_name, **kwargs)
 
+
 def parse_query(query):
     query_engines = []
     query_parts = query.split()
@@ -159,22 +160,23 @@ def index():
         response.headers.add('Content-Disposition', 'attachment;Filename=searx_-_{0}.csv'.format('_'.join(query.split())))
         return response
     elif request_data.get('format') == 'rss':
-        response_rss = render('opensearch_response_rss.xml'
+        response_rss = render(
-                              ,results=results
+            'opensearch_response_rss.xml',
-                              ,q=request_data['q']
+            results=results,
-                              ,number_of_results=len(results)
+            q=request_data['q'],
-                              ,base_url=get_base_url()
+            number_of_results=len(results),
+            base_url=get_base_url()
         )
         return Response(response_rss, mimetype='text/xml')
 
-
+    return render(
-    return render('results.html'
+        'results.html',
-                 ,results=results
+        results=results,
-                 ,q=request_data['q']
+        q=request_data['q'],
-                 ,selected_categories=selected_categories
+        selected_categories=selected_categories,
-                 ,number_of_results=len(results)+len(featured_results)
+        number_of_results=len(results) + len(featured_results),
-                 ,featured_results=featured_results
+        featured_results=featured_results,
-                 ,suggestions=suggestions
+        suggestions=suggestions
     )
 
 
@@ -203,7 +205,10 @@ def preferences():
         if selected_categories:
             resp = make_response(redirect('/'))
             # cookie max age: 4 weeks
-            resp.set_cookie('categories', ','.join(selected_categories), max_age=60*60*24*7*4)
+            resp.set_cookie(
+                'categories', ','.join(selected_categories),
+                max_age=60 * 60 * 24 * 7 * 4
+            )
             return resp
     return render('preferences.html')
 
@@ -238,6 +243,7 @@ def opensearch():
                 mimetype="application/xml")
     return resp
 
+
 @app.route('/favicon.ico')
 def favicon():
     return send_from_directory(os.path.join(app.root_path, 'static/img'),
@@ -248,9 +254,10 @@ def run():
     from gevent import monkey
     monkey.patch_all()
 
-    app.run(debug        = settings['server']['debug']
+    app.run(
-           ,use_debugger = settings['server']['debug']
+        debug=settings['server']['debug'],
-           ,port         = settings['server']['port']
+        use_debugger=settings['server']['debug'],
+        port=settings['server']['port']
     )
 
 

setup.py

@@ -49,4 +49,19 @@ setup(
             'zope.testrunner',
         ]
     },
+    entry_points={
+        'console_scripts': [
+            'searx-run = searx.webapp:run'
+        ]
+    },
+    package_data={
+        'searx': [
+            'settings.yml',
+            '../README.md',
+            'static/*/*',
+            'templates/*.html',
+            'templates/result_templates/*.html',
+        ],
+    },
+
 )