tests and robot tests framework, build overhaul

.gitignore

@@ -1,5 +1,14 @@
 env
 engines.cfg
+.installed.cfg
+setup.cfg
 
 *.pyc
 */*.pyc
+
+bin/
+include/
+lib/
+build/
+develop-eggs/
+parts/

Makefile

@@ -0,0 +1,50 @@
+# convenience makefile to boostrap & run buildout
+# use `make options=-v` to run buildout with extra options
+
+version = 2.7
+python = bin/python
+options =
+
+all: .installed.cfg
+
+.installed.cfg: bin/buildout buildout.cfg setup.py
+	bin/buildout $(options)
+
+bin/buildout: $(python) buildout.cfg bootstrap.py
+	$(python) bootstrap.py
+	@touch $@
+
+$(python):
+	virtualenv -p python$(version) --no-site-packages .
+	@touch $@
+
+tests: .installed.cfg
+	@bin/test
+
+enginescfg:
+	@test -f ./engines.cfg || echo "Copying engines.cfg ..."
+	@cp --no-clobber engines.cfg_sample engines.cfg
+
+robot: .installed.cfg enginescfg
+	@bin/robot
+
+flake8: .installed.cfg
+	@bin/flake8 setup.py
+	@bin/flake8 ./searx/
+
+coverage: .installed.cfg
+	@bin/coverage run --source=./searx/ --branch bin/test
+	@bin/coverage report --show-missing
+	@bin/coverage html --directory ./coverage
+
+minimal: bin/buildout production.cfg setup.py enginescfg
+	bin/buildout -c production.cfg $(options)
+	@echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"
+	@echo "* Hint 1: on production, disable debug mode and change secret_key"
+	@echo "* Hint 2: to run server execute 'bin/searx-run'"
+
+clean:
+	@rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs \
+		searx.egg-info lib include .coverage coverage
+
+.PHONY: all tests enginescfg robot flake8 coverage minimal clean

README.md

@@ -29,6 +29,48 @@ List of [running instances](https://github.com/asciimoo/searx/wiki/Searx-instanc
 
 For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation)
 
+### Alternative (Recommended) Installation
+
+* clone source: `git clone git@github.com:asciimoo/searx.git && cd searx`
+* build in current folder: `make minimal`
+* run `bin/searx-run` to start the application
+
+
+### Development
+
+Just run `make`. Versions of dependencies are pinned down inside `versions.cfg` to produce most stable build.
+
+#### Command make
+
+##### `make`
+
+Builds development environment with testing support.
+
+##### `make tests`
+
+Runs tests. You can write tests [here](https://github.com/asciimoo/searx/tree/master/searx/tests) and remember 'untested code is broken code'.
+
+##### `make robot`
+
+Runs robot (Selenium) tests, you must have `firefox` installed because this functional tests actually run the browser and perform operations on it. Also searx is executed with [settings_robot](https://github.com/asciimoo/searx/blob/master/searx/settings_robot.py).
+
+##### `make flake8`
+
+'pep8 is a tool to check your Python code against some of the style conventions in [PEP 8](http://www.python.org/dev/peps/pep-0008/).'
+
+##### `make coverage`
+
+Checks coverage of tests, after running this, execute this: `firefox ./coverage/index.html`
+
+##### `make minimal`
+
+Used to make co-called production environment - without tests (you should ran tests before deploying searx on the server).
+
+##### `make clean`
+
+Deletes several folders and files (see `Makefile` for more), so that next time you run any other `make` command it will rebuild everithing.
+
+
 ### TODO
 
 * Moar engines
@@ -36,7 +78,9 @@ For all the details, follow this [step by step installation](https://github.com/
 * Language support
 * Documentation
 * Pagination
+* Fix `flake8` errors, `make flake8` will be merged into `make tests` when it does not fail anymore
 * Tests
+* When we have more tests, we can integrate Travis-CI
 
 
 ### Bugs

base.cfg

@@ -0,0 +1,23 @@
+[buildout]
+extends = versions.cfg
+versions = versions
+unzip = true
+newest = false
+extends = versions.cfg
+versions = versions
+prefer-final = true
+develop = .
+
+extensions =
+    buildout_versions
+
+eggs =
+    searx
+
+parts =
+    omelette
+
+
+[omelette]
+recipe = collective.recipe.omelette
+eggs = ${buildout:eggs}

bootstrap.py

@@ -0,0 +1,277 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+"""
+
+import os, shutil, sys, tempfile, urllib, urllib2, subprocess
+from optparse import OptionParser
+
+if sys.platform == 'win32':
+    def quote(c):
+        if ' ' in c:
+            return '"%s"' % c  # work around spawn lamosity on windows
+        else:
+            return c
+else:
+    quote = str
+
+# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments.
+stdout, stderr = subprocess.Popen(
+    [sys.executable, '-Sc',
+     'try:\n'
+     '    import ConfigParser\n'
+     'except ImportError:\n'
+     '    print 1\n'
+     'else:\n'
+     '    print 0\n'],
+    stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+has_broken_dash_S = bool(int(stdout.strip()))
+
+# In order to be more robust in the face of system Pythons, we want to
+# run without site-packages loaded.  This is somewhat tricky, in
+# particular because Python 2.6's distutils imports site, so starting
+# with the -S flag is not sufficient.  However, we'll start with that:
+if not has_broken_dash_S and 'site' in sys.modules:
+    # We will restart with python -S.
+    args = sys.argv[:]
+    args[0:0] = [sys.executable, '-S']
+    args = map(quote, args)
+    os.execv(sys.executable, args)
+# Now we are running with -S.  We'll get the clean sys.path, import site
+# because distutils will do it later, and then reset the path and clean
+# out any namespace packages from site-packages that might have been
+# loaded by .pth files.
+clean_path = sys.path[:]
+import site  # imported because of its side effects
+sys.path[:] = clean_path
+for k, v in sys.modules.items():
+    if k in ('setuptools', 'pkg_resources') or (
+        hasattr(v, '__path__') and
+        len(v.__path__) == 1 and
+        not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))):
+        # This is a namespace package.  Remove it.
+        sys.modules.pop(k)
+
+is_jython = sys.platform.startswith('java')
+
+setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
+distribute_source = 'http://python-distribute.org/distribute_setup.py'
+distribute_source = 'https://bitbucket.org/pypa/setuptools/raw/f657df1f1ed46596d236376649c99a470662b4ba/distribute_setup.py'
+
+# parsing arguments
+def normalize_to_url(option, opt_str, value, parser):
+    if value:
+        if '://' not in value:  # It doesn't smell like a URL.
+            value = 'file://%s' % (
+                urllib.pathname2url(
+                    os.path.abspath(os.path.expanduser(value))),)
+        if opt_str == '--download-base' and not value.endswith('/'):
+            # Download base needs a trailing slash to make the world happy.
+            value += '/'
+    else:
+        value = None
+    name = opt_str[2:].replace('-', '_')
+    setattr(parser.values, name, value)
+
+usage = '''\
+[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
+
+Bootstraps a buildout-based project.
+
+Simply run this script in a directory containing a buildout.cfg, using the
+Python that you want bin/buildout to use.
+
+Note that by using --setup-source and --download-base to point to
+local resources, you can keep this script from going over the network.
+'''
+
+parser = OptionParser(usage=usage)
+parser.add_option("-v", "--version", dest="version",
+                          help="use a specific zc.buildout version")
+parser.add_option("-d", "--distribute",
+                   action="store_true", dest="use_distribute", default=False,
+                   help="Use Distribute rather than Setuptools.")
+parser.add_option("--setup-source", action="callback", dest="setup_source",
+                  callback=normalize_to_url, nargs=1, type="string",
+                  help=("Specify a URL or file location for the setup file. "
+                        "If you use Setuptools, this will default to " +
+                        setuptools_source + "; if you use Distribute, this "
+                        "will default to " + distribute_source + "."))
+parser.add_option("--download-base", action="callback", dest="download_base",
+                  callback=normalize_to_url, nargs=1, type="string",
+                  help=("Specify a URL or directory for downloading "
+                        "zc.buildout and either Setuptools or Distribute. "
+                        "Defaults to PyPI."))
+parser.add_option("--eggs",
+                  help=("Specify a directory for storing eggs.  Defaults to "
+                        "a temporary directory that is deleted when the "
+                        "bootstrap script completes."))
+parser.add_option("-t", "--accept-buildout-test-releases",
+                  dest='accept_buildout_test_releases',
+                  action="store_true", default=False,
+                  help=("Normally, if you do not specify a --version, the "
+                        "bootstrap script and buildout gets the newest "
+                        "*final* versions of zc.buildout and its recipes and "
+                        "extensions for you.  If you use this flag, "
+                        "bootstrap and buildout will get the newest releases "
+                        "even if they are alphas or betas."))
+parser.add_option("-c", None, action="store", dest="config_file",
+                   help=("Specify the path to the buildout configuration "
+                         "file to be used."))
+
+options, args = parser.parse_args()
+
+if options.eggs:
+    eggs_dir = os.path.abspath(os.path.expanduser(options.eggs))
+else:
+    eggs_dir = tempfile.mkdtemp()
+
+if options.setup_source is None:
+    if options.use_distribute:
+        options.setup_source = distribute_source
+    else:
+        options.setup_source = setuptools_source
+
+if options.accept_buildout_test_releases:
+    args.insert(0, 'buildout:accept-buildout-test-releases=true')
+
+try:
+    import pkg_resources
+    import setuptools  # A flag.  Sometimes pkg_resources is installed alone.
+    if not hasattr(pkg_resources, '_distribute'):
+        raise ImportError
+except ImportError:
+    ez_code = urllib2.urlopen(
+        options.setup_source).read().replace('\r\n', '\n')
+    ez = {}
+    exec ez_code in ez
+    setup_args = dict(to_dir=eggs_dir, download_delay=0)
+    if options.download_base:
+        setup_args['download_base'] = options.download_base
+    if options.use_distribute:
+        setup_args['no_fake'] = True
+        if sys.version_info[:2] == (2, 4):
+            setup_args['version'] = '0.6.32'
+    ez['use_setuptools'](**setup_args)
+    if 'pkg_resources' in sys.modules:
+        reload(sys.modules['pkg_resources'])
+    import pkg_resources
+    # This does not (always?) update the default working set.  We will
+    # do it.
+    for path in sys.path:
+        if path not in pkg_resources.working_set.entries:
+            pkg_resources.working_set.add_entry(path)
+
+cmd = [quote(sys.executable),
+       '-c',
+       quote('from setuptools.command.easy_install import main; main()'),
+       '-mqNxd',
+       quote(eggs_dir)]
+
+if not has_broken_dash_S:
+    cmd.insert(1, '-S')
+
+find_links = options.download_base
+if not find_links:
+    find_links = os.environ.get('bootstrap-testing-find-links')
+if not find_links and options.accept_buildout_test_releases:
+    find_links = 'http://downloads.buildout.org/'
+if find_links:
+    cmd.extend(['-f', quote(find_links)])
+
+if options.use_distribute:
+    setup_requirement = 'distribute'
+else:
+    setup_requirement = 'setuptools'
+ws = pkg_resources.working_set
+setup_requirement_path = ws.find(
+    pkg_resources.Requirement.parse(setup_requirement)).location
+env = dict(
+    os.environ,
+    PYTHONPATH=setup_requirement_path)
+
+requirement = 'zc.buildout'
+version = options.version
+if version is None and not options.accept_buildout_test_releases:
+    # Figure out the most recent final version of zc.buildout.
+    import setuptools.package_index
+    _final_parts = '*final-', '*final'
+
+    def _final_version(parsed_version):
+        for part in parsed_version:
+            if (part[:1] == '*') and (part not in _final_parts):
+                return False
+        return True
+    index = setuptools.package_index.PackageIndex(
+        search_path=[setup_requirement_path])
+    if find_links:
+        index.add_find_links((find_links,))
+    req = pkg_resources.Requirement.parse(requirement)
+    if index.obtain(req) is not None:
+        best = []
+        bestv = None
+        for dist in index[req.project_name]:
+            distv = dist.parsed_version
+            if distv >= pkg_resources.parse_version('2dev'):
+                continue
+            if _final_version(distv):
+                if bestv is None or distv > bestv:
+                    best = [dist]
+                    bestv = distv
+                elif distv == bestv:
+                    best.append(dist)
+        if best:
+            best.sort()
+            version = best[-1].version
+
+if version:
+    requirement += '=='+version
+else:
+    requirement += '<2dev'
+
+cmd.append(requirement)
+
+if is_jython:
+    import subprocess
+    exitcode = subprocess.Popen(cmd, env=env).wait()
+else:  # Windows prefers this, apparently; otherwise we would prefer subprocess
+    exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
+if exitcode != 0:
+    sys.stdout.flush()
+    sys.stderr.flush()
+    print ("An error occurred when trying to install zc.buildout. "
+           "Look above this message for any errors that "
+           "were output by easy_install.")
+    sys.exit(exitcode)
+
+ws.add_entry(eggs_dir)
+ws.require(requirement)
+import zc.buildout.buildout
+
+# If there isn't already a command in the args, add bootstrap
+if not [a for a in args if '=' not in a]:
+    args.append('bootstrap')
+
+
+# if -c was provided, we push it back into args for buildout's main function
+if options.config_file is not None:
+    args[0:0] = ['-c', options.config_file]
+
+zc.buildout.buildout.main(args)
+if not options.eggs:  # clean up temporary egg directory
+    shutil.rmtree(eggs_dir)

buildout.cfg

@@ -0,0 +1,32 @@
+[buildout]
+extends = base.cfg
+develop = .
+
+eggs =
+    searx [test]
+
+parts +=
+    pyscripts
+    robot
+    test
+
+
+[pyscripts]
+recipe = zc.recipe.egg:script
+eggs = ${buildout:eggs}
+interpreter = py
+dependent-scripts = true
+entry-points =
+    searx-run=searx.webapp:run
+
+
+[robot]
+recipe = zc.recipe.testrunner
+eggs = ${buildout:eggs}
+defaults = ['--color', '--auto-progress', '--layer', 'SearxRobotLayer']
+
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = ${buildout:eggs}
+defaults = ['--color', '--auto-progress', '--layer', 'SearxTestLayer', '--layer', '!SearxRobotLayer']

production.cfg

@@ -0,0 +1,17 @@
+[buildout]
+extends = base.cfg
+develop = .
+
+eggs =
+    searx
+
+parts +=
+    pyscripts
+
+
+[pyscripts]
+recipe = zc.recipe.egg:script
+eggs = ${buildout:eggs}
+interpreter = py
+entry-points =
+    searx-run=searx.webapp:run

searx/settings_robot.py

@@ -0,0 +1,16 @@
+
+port = 11111
+
+secret_key = "ultrasecretkey" # change this!
+
+debug = False
+
+request_timeout = 5.0 # seconds
+
+weights = {} # 'search_engine_name': float(weight) | default is 1.0
+
+blacklist = [] # search engine blacklist
+
+categories = {} # custom search engine categories
+
+base_url = None # "https://your.domain.tld/" or None (to use request parameters)

searx/testing.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""Shared testing code."""
+
+from plone.testing import Layer
+from unittest2 import TestCase
+
+
+import os
+import subprocess
+import sys
+
+
+class SearxTestLayer:
+
+    __name__ = u'SearxTestLayer'
+
+    def setUp(cls):
+        pass
+    setUp = classmethod(setUp)
+
+    def tearDown(cls):
+        pass
+    tearDown = classmethod(tearDown)
+
+    def testSetUp(cls):
+        pass
+    testSetUp = classmethod(testSetUp)
+
+    def testTearDown(cls):
+        pass
+    testTearDown = classmethod(testTearDown)
+
+
+class SearxRobotLayer(Layer):
+    """Searx Robot Test Layer"""
+
+    def setUp(self):
+        os.setpgrp()  # create new process group, become its leader
+        webapp = os.path.join(
+            os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
+            'webapp.py'
+        )
+        exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py')
+        self.server = subprocess.Popen(
+            [exe, webapp, 'settings_robot'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT
+        )
+
+    def tearDown(self):
+        # TERM all processes in my group
+        os.killpg(os.getpgid(self.server.pid), 15)
+
+
+SEARXROBOTLAYER = SearxRobotLayer()
+
+
+class SearxTestCase(TestCase):
+    layer = SearxTestLayer

searx/tests/robot/test_basic.robot

@@ -0,0 +1,11 @@
+*** Settings ***
+Library         Selenium2Library  timeout=10  implicit_wait=0.5
+Test Setup      Open Browser  http://localhost:11111/
+Test Teardown   Close All Browsers
+
+
+*** Test Cases ***
+Front page
+    Page Should Contain  about
+    Page Should Contain  preferences
+

searx/tests/test_robot.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+from plone.testing import layered
+from robotsuite import RobotTestSuite
+from searx.testing import SEARXROBOTLAYER
+
+import os
+import unittest2 as unittest
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    current_dir = os.path.abspath(os.path.dirname(__file__))
+    robot_dir = os.path.join(current_dir, 'robot')
+    tests = [
+        os.path.join('robot', f) for f in
+        os.listdir(robot_dir) if f.endswith('.robot') and
+        f.startswith('test_')
+    ]
+    for test in tests:
+        suite.addTests([
+            layered(RobotTestSuite(test), layer=SEARXROBOTLAYER),
+        ])
+    return suite

searx/tests/test_unit.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+from searx.testing import SearxTestCase
+
+
+class UnitTestCase(SearxTestCase):
+
+    def test_flask(self):
+        import flask
+        self.assertIn('Flask', dir(flask))

searx/webapp.py

@@ -18,13 +18,20 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 '''
 
 import os
+import sys
 if __name__ == "__main__":
-    from sys import path
-    path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))
+    sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))
+
+# first argument is for specifying settings module, used mostly by robot tests
+from sys import argv
+if len(argv) == 2:
+    from importlib import import_module
+    settings = import_module('searx.' + argv[1])
+else:
+    from searx import settings
 
 from flask import Flask, request, render_template, url_for, Response, make_response, redirect
 from searx.engines import search, categories, engines, get_engines_stats
-from searx import settings
 import json
 import cStringIO
 from searx.utils import UnicodeWriter
@@ -226,7 +233,7 @@ def favicon():
                                'favicon.png', mimetype='image/vnd.microsoft.icon')
 
 
-if __name__ == "__main__":
+def run():
     from gevent import monkey
     monkey.patch_all()
 
@@ -234,3 +241,7 @@ if __name__ == "__main__":
            ,use_debugger = settings.debug
            ,port         = settings.port
            )
+
+
+if __name__ == "__main__":
+    run()

setup.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""Installer for Searx package."""
+
+from setuptools import setup
+from setuptools import find_packages
+
+import os
+
+
+def read(*rnames):
+    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+
+long_description = read('README.md')
+
+setup(
+    name='searx',
+    version="0.1",
+    description="",
+    long_description=long_description,
+    classifiers=[
+        "Programming Language :: Python",
+    ],
+    keywords='meta search engine',
+    author='Adam Tauber',
+    author_email='asciimoo@gmail.com',
+    url='https://github.com/asciimoo/searx',
+    license='GNU Affero General Public License',
+    packages=find_packages('.'),
+    zip_safe=False,
+    install_requires=[
+        'flask',
+        'grequests',
+        'lxml',
+        'setuptools',
+    ],
+    extras_require={
+        'test': [
+            'coverage',
+            'flake8',
+            'plone.testing',
+            'robotframework',
+            'robotframework-debuglibrary',
+            'robotframework-httplibrary',
+            'robotframework-selenium2library',
+            'robotsuite',
+            'unittest2',
+            'zope.testrunner',
+        ]
+    },
+)

versions.cfg

@@ -0,0 +1,93 @@
+[versions]
+Flask = 0.10.1
+Jinja2 = 2.7.2
+MarkupSafe = 0.18
+WebOb = 1.3.1
+WebTest = 2.0.11
+Werkzeug = 0.9.4
+buildout-versions = 1.7
+collective.recipe.omelette = 0.16
+coverage = 3.7.1
+decorator = 3.4.0
+docutils = 0.11
+flake8 = 2.1.0
+itsdangerous = 0.23
+mccabe = 0.2.1
+pep8 = 1.4.6
+plone.testing = 4.0.8
+pyflakes = 0.7.3
+requests = 2.2.0
+robotframework-debuglibrary = 0.3
+robotframework-httplibrary = 0.4.2
+robotframework-selenium2library = 1.5.0
+robotsuite = 1.4.2
+selenium = 2.39.0
+unittest2 = 0.5.1
+waitress = 0.8.8
+zc.recipe.testrunner = 2.0.0
+
+# Required by:
+# WebTest==2.0.11
+beautifulsoup4 = 4.3.2
+
+# Required by:
+# grequests==0.2.0
+gevent = 1.0
+
+# Required by:
+# gevent==1.0
+greenlet = 0.4.2
+
+# Required by:
+# searx==0.1
+grequests = 0.2.0
+
+# Required by:
+# robotframework-httplibrary==0.4.2
+jsonpatch = 1.3
+
+# Required by:
+# robotframework-httplibrary==0.4.2
+jsonpointer = 1.1
+
+# Required by:
+# robotsuite==1.4.2
+# searx==0.1
+lxml = 3.2.5
+
+# Required by:
+# robotframework-httplibrary==0.4.2
+robotframework = 2.8.3
+
+# Required by:
+# plone.testing==4.0.8
+# robotsuite==1.4.2
+# searx==0.1
+# zope.exceptions==4.0.6
+# zope.interface==4.0.5
+# zope.testrunner==4.4.1
+setuptools = 2.1
+
+# Required by:
+# zope.testrunner==4.4.1
+six = 1.5.2
+
+# Required by:
+# collective.recipe.omelette==0.16
+zc.recipe.egg = 2.0.1
+
+# Required by:
+# zope.testrunner==4.4.1
+zope.exceptions = 4.0.6
+
+# Required by:
+# zope.testrunner==4.4.1
+zope.interface = 4.0.5
+
+# Required by:
+# plone.testing==4.0.8
+zope.testing = 4.1.2
+
+# Required by:
+# zc.recipe.testrunner==2.0.0
+zope.testrunner = 4.4.1