From 66f48c2bf5b84f6f7b52b1ba7217a1aadf5717b7 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 10 Jan 2016 18:49:50 +0100 Subject: [PATCH 01/13] [fix] google markup change - closes #489 --- searx/engines/google.py | 4 ++-- searx/tests/engines/test_google.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/searx/engines/google.py b/searx/engines/google.py index e82260356..e6dacc3a8 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -90,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\ search_path = '/search' search_url = ('https://{hostname}' + search_path + - '?{query}&start={offset}&gbv=1&gws_rd=cr') + '?{query}&start={offset}&gbv=1&gws_rd=ssl') # other URLs map_hostname_start = 'maps.google.' @@ -99,7 +99,7 @@ redirect_path = '/url' images_path = '/images' # specific xpath variables -results_xpath = '//li[@class="g"]' +results_xpath = '//div[@class="g"]' url_xpath = './/h3/a/@href' title_xpath = './/h3' content_xpath = './/span[@class="st"]' diff --git a/searx/tests/engines/test_google.py b/searx/tests/engines/test_google.py index 04f12b830..37a83cae3 100644 --- a/searx/tests/engines/test_google.py +++ b/searx/tests/engines/test_google.py @@ -40,7 +40,7 @@ class TestGoogleEngine(SearxTestCase): self.assertEqual(google.response(response), []) html = """ -
  • +

    This is the title @@ -94,35 +94,35 @@ class TestGoogleEngine(SearxTestCase):

    -
    • -
  • + +

    This

    -
    • -
  • + +

    This is

    -
    • -
  • + +

    This is the

    -
    • -
  • + +

    This is the

    -
    • +

    suggestion title From f9186344b3642fb3d55d2dc46c96c6b25b8ccf41 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Fri, 1 Jan 2016 18:36:26 +0100 Subject: [PATCH 02/13] [mod] dependency versions --- requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index e021c4a25..80c08a453 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -flask -flask-babel -requests -lxml -pyyaml -pygments -python-dateutil -ndg-httpsclient -pyopenssl -pyasn1 -pyasn1-modules -certifi +certifi==2015.11.20.1 +flask==0.10.1 +flask-babel==0.9 +lxml==3.5.0 +ndg-httpsclient==0.4.0 +pyasn1==0.1.9 +pyasn1-modules==0.0.8 +pygments==2.0.2 +pyopenssl==0.15.1 +python-dateutil==2.4.2 +pyyaml==3.11 +requests==2.9.1 From 53979a7bf7669c803c2a493fbf136519f6a293e6 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 11:14:49 +0100 Subject: [PATCH 03/13] [mod] remove buildout/makefile infrastructure --- .gitignore | 12 - .travis.yml | 15 +- Makefile | 66 ------ base.cfg | 17 -- bootstrap.py | 210 ------------------ buildout.cfg | 30 --- manage.sh | 70 ++++++ minimal.cfg | 15 -- production.cfg | 34 --- requirements-dev.txt | 9 + searx/tests/test_engines.py | 45 ---- {searx/tests => tests}/__init__.py | 0 .../tests/engines => tests/robot}/__init__.py | 0 {searx/tests => tests}/robot/test_basic.robot | 0 {searx/tests => tests}/test_robot.py | 0 {searx/tests/robot => tests/unit}/__init__.py | 0 tests/unit/engines/__init__.py | 0 .../tests => tests/unit}/engines/test_bing.py | 0 .../unit}/engines/test_bing_images.py | 0 .../unit}/engines/test_bing_news.py | 0 .../unit}/engines/test_blekko_images.py | 0 .../unit}/engines/test_btdigg.py | 0 .../unit}/engines/test_currency_convert.py | 0 .../unit}/engines/test_dailymotion.py | 0 .../unit}/engines/test_deezer.py | 0 .../unit}/engines/test_deviantart.py | 0 .../tests => tests/unit}/engines/test_digg.py | 0 .../unit}/engines/test_duckduckgo.py | 0 .../engines/test_duckduckgo_definitions.py | 0 .../unit}/engines/test_dummy.py | 0 .../unit}/engines/test_faroo.py | 0 .../unit}/engines/test_flickr.py | 0 .../unit}/engines/test_flickr_noapi.py | 0 .../unit}/engines/test_gigablast.py | 0 .../unit}/engines/test_github.py | 0 .../unit}/engines/test_google.py | 0 .../unit}/engines/test_google_images.py | 0 .../unit}/engines/test_google_news.py | 0 .../unit}/engines/test_kickass.py | 0 .../unit}/engines/test_mediawiki.py | 0 .../unit}/engines/test_mixcloud.py | 0 .../unit}/engines/test_openstreetmap.py | 0 .../unit}/engines/test_photon.py | 0 .../unit}/engines/test_piratebay.py | 0 .../unit}/engines/test_qwant.py | 0 .../unit}/engines/test_searchcode_code.py | 0 .../unit}/engines/test_searchcode_doc.py | 0 .../unit}/engines/test_soundcloud.py | 0 .../unit}/engines/test_spotify.py | 0 .../unit}/engines/test_stackoverflow.py | 0 .../unit}/engines/test_startpage.py | 0 .../unit}/engines/test_subtitleseeker.py | 0 .../unit}/engines/test_swisscows.py | 0 .../unit}/engines/test_twitter.py | 0 .../unit}/engines/test_vimeo.py | 0 .../unit}/engines/test_www1x.py | 0 .../unit}/engines/test_www500px.py | 0 .../tests => tests/unit}/engines/test_yacy.py | 0 .../unit}/engines/test_yahoo.py | 0 .../unit}/engines/test_yahoo_news.py | 0 .../unit}/engines/test_youtube_api.py | 0 .../unit}/engines/test_youtube_noapi.py | 0 {searx/tests => tests/unit}/test_plugins.py | 0 {searx/tests => tests/unit}/test_results.py | 0 {searx/tests => tests/unit}/test_search.py | 0 {searx/tests => tests/unit}/test_utils.py | 0 {searx/tests => tests/unit}/test_webapp.py | 0 versions.cfg | 122 ---------- 68 files changed, 87 insertions(+), 558 deletions(-) delete mode 100644 Makefile delete mode 100644 base.cfg delete mode 100644 bootstrap.py delete mode 100644 buildout.cfg create mode 100755 manage.sh delete mode 100644 minimal.cfg delete mode 100644 production.cfg create mode 100644 requirements-dev.txt delete mode 100644 searx/tests/test_engines.py rename {searx/tests => tests}/__init__.py (100%) rename {searx/tests/engines => tests/robot}/__init__.py (100%) rename {searx/tests => tests}/robot/test_basic.robot (100%) rename {searx/tests => tests}/test_robot.py (100%) rename {searx/tests/robot => tests/unit}/__init__.py (100%) create mode 100644 tests/unit/engines/__init__.py rename {searx/tests => tests/unit}/engines/test_bing.py (100%) rename {searx/tests => tests/unit}/engines/test_bing_images.py (100%) rename {searx/tests => tests/unit}/engines/test_bing_news.py (100%) rename {searx/tests => tests/unit}/engines/test_blekko_images.py (100%) rename {searx/tests => tests/unit}/engines/test_btdigg.py (100%) rename {searx/tests => tests/unit}/engines/test_currency_convert.py (100%) rename {searx/tests => tests/unit}/engines/test_dailymotion.py (100%) rename {searx/tests => tests/unit}/engines/test_deezer.py (100%) rename {searx/tests => tests/unit}/engines/test_deviantart.py (100%) rename {searx/tests => tests/unit}/engines/test_digg.py (100%) rename {searx/tests => tests/unit}/engines/test_duckduckgo.py (100%) rename {searx/tests => tests/unit}/engines/test_duckduckgo_definitions.py (100%) rename {searx/tests => tests/unit}/engines/test_dummy.py (100%) rename {searx/tests => tests/unit}/engines/test_faroo.py (100%) rename {searx/tests => tests/unit}/engines/test_flickr.py (100%) rename {searx/tests => tests/unit}/engines/test_flickr_noapi.py (100%) rename {searx/tests => tests/unit}/engines/test_gigablast.py (100%) rename {searx/tests => tests/unit}/engines/test_github.py (100%) rename {searx/tests => tests/unit}/engines/test_google.py (100%) rename {searx/tests => tests/unit}/engines/test_google_images.py (100%) rename {searx/tests => tests/unit}/engines/test_google_news.py (100%) rename {searx/tests => tests/unit}/engines/test_kickass.py (100%) rename {searx/tests => tests/unit}/engines/test_mediawiki.py (100%) rename {searx/tests => tests/unit}/engines/test_mixcloud.py (100%) rename {searx/tests => tests/unit}/engines/test_openstreetmap.py (100%) rename {searx/tests => tests/unit}/engines/test_photon.py (100%) rename {searx/tests => tests/unit}/engines/test_piratebay.py (100%) rename {searx/tests => tests/unit}/engines/test_qwant.py (100%) rename {searx/tests => tests/unit}/engines/test_searchcode_code.py (100%) rename {searx/tests => tests/unit}/engines/test_searchcode_doc.py (100%) rename {searx/tests => tests/unit}/engines/test_soundcloud.py (100%) rename {searx/tests => tests/unit}/engines/test_spotify.py (100%) rename {searx/tests => tests/unit}/engines/test_stackoverflow.py (100%) rename {searx/tests => tests/unit}/engines/test_startpage.py (100%) rename {searx/tests => tests/unit}/engines/test_subtitleseeker.py (100%) rename {searx/tests => tests/unit}/engines/test_swisscows.py (100%) rename {searx/tests => tests/unit}/engines/test_twitter.py (100%) rename {searx/tests => tests/unit}/engines/test_vimeo.py (100%) rename {searx/tests => tests/unit}/engines/test_www1x.py (100%) rename {searx/tests => tests/unit}/engines/test_www500px.py (100%) rename {searx/tests => tests/unit}/engines/test_yacy.py (100%) rename {searx/tests => tests/unit}/engines/test_yahoo.py (100%) rename {searx/tests => tests/unit}/engines/test_yahoo_news.py (100%) rename {searx/tests => tests/unit}/engines/test_youtube_api.py (100%) rename {searx/tests => tests/unit}/engines/test_youtube_noapi.py (100%) rename {searx/tests => tests/unit}/test_plugins.py (100%) rename {searx/tests => tests/unit}/test_results.py (100%) rename {searx/tests => tests/unit}/test_search.py (100%) rename {searx/tests => tests/unit}/test_utils.py (100%) rename {searx/tests => tests/unit}/test_webapp.py (100%) delete mode 100644 versions.cfg diff --git a/.gitignore b/.gitignore index 760b1a8f3..d29971645 100644 --- a/.gitignore +++ b/.gitignore @@ -11,18 +11,6 @@ setup.cfg */*.pyc *~ -bin/ -build/ -coverage/ -develop-eggs/ -dist/ -eggs/ -include/ -lib/ -local/ -parts/ -searx.egg-info/ -var/ node_modules/ .tx/ diff --git a/.travis.yml b/.travis.yml index be6683510..57d623060 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,16 +11,17 @@ before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - npm install -g less grunt-cli - - ( cd searx/static/themes/oscar;npm install ) + - ( cd searx/static/themes/oscar;npm install; cd - ) install: - - "make" + - ./manage.sh update_packages + - ./manage.sh update_dev_packages - pip install coveralls script: - - "make flake8" - - "make robot" - - "make styles" - - "make grunt" - - make coverage + - ./manage.sh pep8_check + - ./manage.sh styles + - ./manage.sh grunt + - ./manage.sh coverage + - ./manage.sh robot after_success: coveralls notifications: diff --git a/Makefile b/Makefile deleted file mode 100644 index 5573e5447..000000000 --- a/Makefile +++ /dev/null @@ -1,66 +0,0 @@ -# convenience makefile to boostrap & run buildout -# use `make options=-v` to run buildout with extra options - -version = 2.7 -python = bin/python -options = - -all: .installed.cfg - -.installed.cfg: bin/buildout buildout.cfg setup.py - bin/buildout $(options) - -bin/buildout: $(python) buildout.cfg bootstrap.py - $(python) bootstrap.py - @touch $@ - -$(python): - virtualenv -p python$(version) --no-site-packages . - @touch $@ - -robot: .installed.cfg - @bin/robot - -flake8: .installed.cfg - @bin/flake8 setup.py - @bin/flake8 ./searx/ - -tests: .installed.cfg flake8 - @bin/test - @grunt test --gruntfile searx/static/themes/oscar/gruntfile.js - -coverage: .installed.cfg - @bin/coverage run bin/test - @bin/coverage report - @bin/coverage html - -production: bin/buildout production.cfg setup.py - bin/buildout -c production.cfg $(options) - @echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`" - @echo "* Hint 1: on production, disable debug mode and change secret_key" - @echo "* Hint 2: searx will be executed at server startup by crontab" - @echo "* Hint 3: to run immediatley, execute 'bin/supervisord'" - -minimal: bin/buildout minimal.cfg setup.py - bin/buildout -c minimal.cfg $(options) - -styles: - @lessc -x searx/static/themes/default/less/style.less > searx/static/themes/default/css/style.css - @lessc -x searx/static/themes/default/less/style-rtl.less > searx/static/themes/default/css/style-rtl.css - @lessc -x searx/static/themes/courgette/less/style.less > searx/static/themes/courgette/css/style.css - @lessc -x searx/static/themes/courgette/less/style-rtl.less > searx/static/themes/courgette/css/style-rtl.css - @lessc -x searx/static/less/bootstrap/bootstrap.less > searx/static/css/bootstrap.min.css - @lessc -x searx/static/themes/oscar/less/oscar/oscar.less > searx/static/themes/oscar/css/oscar.min.css - @lessc -x searx/static/themes/pix-art/less/style.less > searx/static/themes/pix-art/css/style.css - -grunt: - @grunt --gruntfile searx/static/themes/oscar/gruntfile.js - -locales: - @pybabel compile -d searx/translations - -clean: - @rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs eggs \ - searx.egg-info lib include .coverage coverage - -.PHONY: all tests robot flake8 coverage production minimal styles locales clean diff --git a/base.cfg b/base.cfg deleted file mode 100644 index 6e46e9eb7..000000000 --- a/base.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[buildout] -extends = versions.cfg -unzip = true -newest = false -prefer-final = true -develop = . - -eggs = - searx - -parts = - omelette - - -[omelette] -recipe = collective.recipe.omelette -eggs = ${buildout:eggs} diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100644 index a4599211f..000000000 --- a/bootstrap.py +++ /dev/null @@ -1,210 +0,0 @@ -############################################################################## -# -# Copyright (c) 2006 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Bootstrap a buildout-based project - -Simply run this script in a directory containing a buildout.cfg. -The script accepts buildout command-line options, so you can -use the -c option to specify an alternate configuration file. -""" - -import os -import shutil -import sys -import tempfile - -from optparse import OptionParser - -__version__ = '2015-07-01' -# See zc.buildout's changelog if this version is up to date. - -tmpeggs = tempfile.mkdtemp(prefix='bootstrap-') - -usage = '''\ -[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] - -Bootstraps a buildout-based project. - -Simply run this script in a directory containing a buildout.cfg, using the -Python that you want bin/buildout to use. - -Note that by using --find-links to point to local resources, you can keep -this script from going over the network. -''' - -parser = OptionParser(usage=usage) -parser.add_option("--version", - action="store_true", default=False, - help=("Return bootstrap.py version.")) -parser.add_option("-t", "--accept-buildout-test-releases", - dest='accept_buildout_test_releases', - action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas.")) -parser.add_option("-c", "--config-file", - help=("Specify the path to the buildout configuration " - "file to be used.")) -parser.add_option("-f", "--find-links", - help=("Specify a URL to search for buildout releases")) -parser.add_option("--allow-site-packages", - action="store_true", default=False, - help=("Let bootstrap.py use existing site packages")) -parser.add_option("--buildout-version", - help="Use a specific zc.buildout version") -parser.add_option("--setuptools-version", - help="Use a specific setuptools version") -parser.add_option("--setuptools-to-dir", - help=("Allow for re-use of existing directory of " - "setuptools versions")) - -options, args = parser.parse_args() -if options.version: - print("bootstrap.py version %s" % __version__) - sys.exit(0) - - -###################################################################### -# load/install setuptools - -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -ez = {} -if os.path.exists('ez_setup.py'): - exec(open('ez_setup.py').read(), ez) -else: - exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez) - -if not options.allow_site_packages: - # ez_setup imports site, which adds site packages - # this will remove them from the path to ensure that incompatible versions - # of setuptools are not in the path - import site - # inside a virtualenv, there is no 'getsitepackages'. - # We can't remove these reliably - if hasattr(site, 'getsitepackages'): - for sitepackage_path in site.getsitepackages(): - # Strip all site-packages directories from sys.path that - # are not sys.prefix; this is because on Windows - # sys.prefix is a site-package directory. - if sitepackage_path != sys.prefix: - sys.path[:] = [x for x in sys.path - if sitepackage_path not in x] - -setup_args = dict(to_dir=tmpeggs, download_delay=0) - -if options.setuptools_version is not None: - setup_args['version'] = options.setuptools_version -if options.setuptools_to_dir is not None: - setup_args['to_dir'] = options.setuptools_to_dir - -ez['use_setuptools'](**setup_args) -import setuptools -import pkg_resources - -# This does not (always?) update the default working set. We will -# do it. -for path in sys.path: - if path not in pkg_resources.working_set.entries: - pkg_resources.working_set.add_entry(path) - -###################################################################### -# Install buildout - -ws = pkg_resources.working_set - -setuptools_path = ws.find( - pkg_resources.Requirement.parse('setuptools')).location - -# Fix sys.path here as easy_install.pth added before PYTHONPATH -cmd = [sys.executable, '-c', - 'import sys; sys.path[0:0] = [%r]; ' % setuptools_path + - 'from setuptools.command.easy_install import main; main()', - '-mZqNxd', tmpeggs] - -find_links = os.environ.get( - 'bootstrap-testing-find-links', - options.find_links or - ('http://downloads.buildout.org/' - if options.accept_buildout_test_releases else None) - ) -if find_links: - cmd.extend(['-f', find_links]) - -requirement = 'zc.buildout' -version = options.buildout_version -if version is None and not options.accept_buildout_test_releases: - # Figure out the most recent final version of zc.buildout. - import setuptools.package_index - _final_parts = '*final-', '*final' - - def _final_version(parsed_version): - try: - return not parsed_version.is_prerelease - except AttributeError: - # Older setuptools - for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): - return False - return True - - index = setuptools.package_index.PackageIndex( - search_path=[setuptools_path]) - if find_links: - index.add_find_links((find_links,)) - req = pkg_resources.Requirement.parse(requirement) - if index.obtain(req) is not None: - best = [] - bestv = None - for dist in index[req.project_name]: - distv = dist.parsed_version - if _final_version(distv): - if bestv is None or distv > bestv: - best = [dist] - bestv = distv - elif distv == bestv: - best.append(dist) - if best: - best.sort() - version = best[-1].version -if version: - requirement = '=='.join((requirement, version)) -cmd.append(requirement) - -import subprocess -if subprocess.call(cmd) != 0: - raise Exception( - "Failed to execute command:\n%s" % repr(cmd)[1:-1]) - -###################################################################### -# Import and run buildout - -ws.add_entry(tmpeggs) -ws.require(requirement) -import zc.buildout.buildout - -if not [a for a in args if '=' not in a]: - args.append('bootstrap') - -# if -c was provided, we push it back into args for buildout' main function -if options.config_file is not None: - args[0:0] = ['-c', options.config_file] - -zc.buildout.buildout.main(args) -shutil.rmtree(tmpeggs) diff --git a/buildout.cfg b/buildout.cfg deleted file mode 100644 index b9e6d24ed..000000000 --- a/buildout.cfg +++ /dev/null @@ -1,30 +0,0 @@ -[buildout] -extends = base.cfg -develop = . - -eggs = - searx [test] - -parts += - pyscripts - robot - test - - -[pyscripts] -recipe = zc.recipe.egg:script -eggs = ${buildout:eggs} -interpreter = py -dependent-scripts = true - - -[robot] -recipe = zc.recipe.testrunner -eggs = ${buildout:eggs} -defaults = ['--color', '--auto-progress', '--layer', 'SearxRobotLayer'] - - -[test] -recipe = zc.recipe.testrunner -eggs = ${buildout:eggs} -defaults = ['--color', '--auto-progress', '--layer', 'SearxTestLayer', '--layer', '!SearxRobotLayer'] diff --git a/manage.sh b/manage.sh new file mode 100755 index 000000000..39a8e27d3 --- /dev/null +++ b/manage.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +BASE_DIR=$(dirname `readlink -f $0`) +PYTHONPATH=$BASE_DIR +SEARX_DIR="$BASE_DIR/searx" +ACTION=$1 + +update_packages() { + pip install --upgrade -r "$BASE_DIR/requirements.txt" +} + +update_dev_packages() { + pip install --upgrade -r "$BASE_DIR/requirements-dev.txt" +} + +pep8_check() { + echo '[!] Running pep8 check' + pep8 "$SEARX_DIR" "$BASE_DIR/tests" +} + +unit_tests() { + echo '[!] Running unit tests' + python -m nose2 -s "$BASE_DIR/tests/unit" +} + +py_test_coverage() { + echo '[!] Running python test coverage' + PYTHONPATH=`pwd` python -m nose2 -C --coverage "$SEARX_DIR" -s "$BASE_DIR/tests/unit" + coverage report + coverage html +} + +robot_tests() { + echo '[!] Running robot tests' + python -c "import zope.testrunner; import sys; sys.exit(zope.testrunner.run("\ +"['--color', '--auto-progress', '--layer', 'SearxRobotLayer', '--test-path', '$BASE_DIR']))" +} + +tests() { + set -e + pep8_check + unit_tests + robot_tests + set +e +} + +build_style() { + lessc -x "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2" +} + +styles() { + echo '[!] Building styles' + build_style themes/default/less/style.less themes/default/css/style.css + build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css + build_style themes/courgette/less/style.less themes/courgette/css/style.css + build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css + build_style less/bootstrap/bootstrap.less css/bootstrap.min.css + build_style themes/oscar/less/oscar/oscar.less themes/oscar/css/oscar.min.css + build_style themes/pix-art/less/style.less themes/pix-art/css/style.css +} + +grunt() { + grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js" +} + +locales() { + pybabel compile -d "$SEARX_DIR/translations" +} + +$ACTION diff --git a/minimal.cfg b/minimal.cfg deleted file mode 100644 index 339a2939f..000000000 --- a/minimal.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[buildout] -extends = base.cfg -develop = . - -eggs = - searx - -parts += - pyscripts - - -[pyscripts] -recipe = zc.recipe.egg:script -eggs = ${buildout:eggs} -interpreter = py diff --git a/production.cfg b/production.cfg deleted file mode 100644 index ea40682d8..000000000 --- a/production.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[buildout] -extends = base.cfg -develop = . - -eggs = - searx - -parts += - pyscripts - supervisor - crontab_reboot - - -[pyscripts] -recipe = zc.recipe.egg:script -eggs = ${buildout:eggs} -interpreter = py - - -[supervisor] -recipe = collective.recipe.supervisor -http-socket = unix -user = searxer -password = ohpleasedochangeme -file = /tmp/supervisor.sock -chmod = 0700 -programs = - 50 searx ${buildout:bin-directory}/searx-run - - -[crontab_reboot] -recipe = z3c.recipe.usercrontab -times = @reboot -command = ${buildout:bin-directory}/supervisord diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..d8c2395dd --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +flake8==2.5.1 +mock==1.0.1 +nose2==0.4.7 +nose2[coverage-plugin] +plone.testing==4.0.15 +robotframework-selenium2library==1.7.4 +robotsuite==1.7.0 +unittest2==1.1.0 +zope.testrunner==4.4.10 diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py deleted file mode 100644 index 793b77460..000000000 --- a/searx/tests/test_engines.py +++ /dev/null @@ -1,45 +0,0 @@ -from searx.tests.engines.test_bing import * # noqa -from searx.tests.engines.test_bing_images import * # noqa -from searx.tests.engines.test_bing_news import * # noqa -from searx.tests.engines.test_blekko_images import * # noqa -from searx.tests.engines.test_btdigg import * # noqa -from searx.tests.engines.test_currency_convert import * # noqa -from searx.tests.engines.test_dailymotion import * # noqa -from searx.tests.engines.test_deezer import * # noqa -from searx.tests.engines.test_deviantart import * # noqa -from searx.tests.engines.test_digg import * # noqa -from searx.tests.engines.test_duckduckgo import * # noqa -from searx.tests.engines.test_duckduckgo_definitions import * # noqa -from searx.tests.engines.test_dummy import * # noqa -from searx.tests.engines.test_faroo import * # noqa -from searx.tests.engines.test_flickr import * # noqa -from searx.tests.engines.test_flickr_noapi import * # noqa -from searx.tests.engines.test_gigablast import * # noqa -from searx.tests.engines.test_github import * # noqa -from searx.tests.engines.test_google import * # noqa -from searx.tests.engines.test_google_images import * # noqa -from searx.tests.engines.test_google_news import * # noqa -from searx.tests.engines.test_kickass import * # noqa -from searx.tests.engines.test_mediawiki import * # noqa -from searx.tests.engines.test_mixcloud import * # noqa -from searx.tests.engines.test_openstreetmap import * # noqa -from searx.tests.engines.test_photon import * # noqa -from searx.tests.engines.test_piratebay import * # noqa -from searx.tests.engines.test_qwant import * # noqa -from searx.tests.engines.test_searchcode_code import * # noqa -from searx.tests.engines.test_searchcode_doc import * # noqa -from searx.tests.engines.test_soundcloud import * # noqa -from searx.tests.engines.test_spotify import * # noqa -from searx.tests.engines.test_stackoverflow import * # noqa -from searx.tests.engines.test_startpage import * # noqa -from searx.tests.engines.test_subtitleseeker import * # noqa -from searx.tests.engines.test_swisscows import * # noqa -from searx.tests.engines.test_twitter import * # noqa -from searx.tests.engines.test_vimeo import * # noqa -from searx.tests.engines.test_www1x import * # noqa -from searx.tests.engines.test_www500px import * # noqa -from searx.tests.engines.test_yacy import * # noqa -from searx.tests.engines.test_yahoo import * # noqa -from searx.tests.engines.test_youtube_api import * # noqa -from searx.tests.engines.test_youtube_noapi import * # noqa -from searx.tests.engines.test_yahoo_news import * # noqa diff --git a/searx/tests/__init__.py b/tests/__init__.py similarity index 100% rename from searx/tests/__init__.py rename to tests/__init__.py diff --git a/searx/tests/engines/__init__.py b/tests/robot/__init__.py similarity index 100% rename from searx/tests/engines/__init__.py rename to tests/robot/__init__.py diff --git a/searx/tests/robot/test_basic.robot b/tests/robot/test_basic.robot similarity index 100% rename from searx/tests/robot/test_basic.robot rename to tests/robot/test_basic.robot diff --git a/searx/tests/test_robot.py b/tests/test_robot.py similarity index 100% rename from searx/tests/test_robot.py rename to tests/test_robot.py diff --git a/searx/tests/robot/__init__.py b/tests/unit/__init__.py similarity index 100% rename from searx/tests/robot/__init__.py rename to tests/unit/__init__.py diff --git a/tests/unit/engines/__init__.py b/tests/unit/engines/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/searx/tests/engines/test_bing.py b/tests/unit/engines/test_bing.py similarity index 100% rename from searx/tests/engines/test_bing.py rename to tests/unit/engines/test_bing.py diff --git a/searx/tests/engines/test_bing_images.py b/tests/unit/engines/test_bing_images.py similarity index 100% rename from searx/tests/engines/test_bing_images.py rename to tests/unit/engines/test_bing_images.py diff --git a/searx/tests/engines/test_bing_news.py b/tests/unit/engines/test_bing_news.py similarity index 100% rename from searx/tests/engines/test_bing_news.py rename to tests/unit/engines/test_bing_news.py diff --git a/searx/tests/engines/test_blekko_images.py b/tests/unit/engines/test_blekko_images.py similarity index 100% rename from searx/tests/engines/test_blekko_images.py rename to tests/unit/engines/test_blekko_images.py diff --git a/searx/tests/engines/test_btdigg.py b/tests/unit/engines/test_btdigg.py similarity index 100% rename from searx/tests/engines/test_btdigg.py rename to tests/unit/engines/test_btdigg.py diff --git a/searx/tests/engines/test_currency_convert.py b/tests/unit/engines/test_currency_convert.py similarity index 100% rename from searx/tests/engines/test_currency_convert.py rename to tests/unit/engines/test_currency_convert.py diff --git a/searx/tests/engines/test_dailymotion.py b/tests/unit/engines/test_dailymotion.py similarity index 100% rename from searx/tests/engines/test_dailymotion.py rename to tests/unit/engines/test_dailymotion.py diff --git a/searx/tests/engines/test_deezer.py b/tests/unit/engines/test_deezer.py similarity index 100% rename from searx/tests/engines/test_deezer.py rename to tests/unit/engines/test_deezer.py diff --git a/searx/tests/engines/test_deviantart.py b/tests/unit/engines/test_deviantart.py similarity index 100% rename from searx/tests/engines/test_deviantart.py rename to tests/unit/engines/test_deviantart.py diff --git a/searx/tests/engines/test_digg.py b/tests/unit/engines/test_digg.py similarity index 100% rename from searx/tests/engines/test_digg.py rename to tests/unit/engines/test_digg.py diff --git a/searx/tests/engines/test_duckduckgo.py b/tests/unit/engines/test_duckduckgo.py similarity index 100% rename from searx/tests/engines/test_duckduckgo.py rename to tests/unit/engines/test_duckduckgo.py diff --git a/searx/tests/engines/test_duckduckgo_definitions.py b/tests/unit/engines/test_duckduckgo_definitions.py similarity index 100% rename from searx/tests/engines/test_duckduckgo_definitions.py rename to tests/unit/engines/test_duckduckgo_definitions.py diff --git a/searx/tests/engines/test_dummy.py b/tests/unit/engines/test_dummy.py similarity index 100% rename from searx/tests/engines/test_dummy.py rename to tests/unit/engines/test_dummy.py diff --git a/searx/tests/engines/test_faroo.py b/tests/unit/engines/test_faroo.py similarity index 100% rename from searx/tests/engines/test_faroo.py rename to tests/unit/engines/test_faroo.py diff --git a/searx/tests/engines/test_flickr.py b/tests/unit/engines/test_flickr.py similarity index 100% rename from searx/tests/engines/test_flickr.py rename to tests/unit/engines/test_flickr.py diff --git a/searx/tests/engines/test_flickr_noapi.py b/tests/unit/engines/test_flickr_noapi.py similarity index 100% rename from searx/tests/engines/test_flickr_noapi.py rename to tests/unit/engines/test_flickr_noapi.py diff --git a/searx/tests/engines/test_gigablast.py b/tests/unit/engines/test_gigablast.py similarity index 100% rename from searx/tests/engines/test_gigablast.py rename to tests/unit/engines/test_gigablast.py diff --git a/searx/tests/engines/test_github.py b/tests/unit/engines/test_github.py similarity index 100% rename from searx/tests/engines/test_github.py rename to tests/unit/engines/test_github.py diff --git a/searx/tests/engines/test_google.py b/tests/unit/engines/test_google.py similarity index 100% rename from searx/tests/engines/test_google.py rename to tests/unit/engines/test_google.py diff --git a/searx/tests/engines/test_google_images.py b/tests/unit/engines/test_google_images.py similarity index 100% rename from searx/tests/engines/test_google_images.py rename to tests/unit/engines/test_google_images.py diff --git a/searx/tests/engines/test_google_news.py b/tests/unit/engines/test_google_news.py similarity index 100% rename from searx/tests/engines/test_google_news.py rename to tests/unit/engines/test_google_news.py diff --git a/searx/tests/engines/test_kickass.py b/tests/unit/engines/test_kickass.py similarity index 100% rename from searx/tests/engines/test_kickass.py rename to tests/unit/engines/test_kickass.py diff --git a/searx/tests/engines/test_mediawiki.py b/tests/unit/engines/test_mediawiki.py similarity index 100% rename from searx/tests/engines/test_mediawiki.py rename to tests/unit/engines/test_mediawiki.py diff --git a/searx/tests/engines/test_mixcloud.py b/tests/unit/engines/test_mixcloud.py similarity index 100% rename from searx/tests/engines/test_mixcloud.py rename to tests/unit/engines/test_mixcloud.py diff --git a/searx/tests/engines/test_openstreetmap.py b/tests/unit/engines/test_openstreetmap.py similarity index 100% rename from searx/tests/engines/test_openstreetmap.py rename to tests/unit/engines/test_openstreetmap.py diff --git a/searx/tests/engines/test_photon.py b/tests/unit/engines/test_photon.py similarity index 100% rename from searx/tests/engines/test_photon.py rename to tests/unit/engines/test_photon.py diff --git a/searx/tests/engines/test_piratebay.py b/tests/unit/engines/test_piratebay.py similarity index 100% rename from searx/tests/engines/test_piratebay.py rename to tests/unit/engines/test_piratebay.py diff --git a/searx/tests/engines/test_qwant.py b/tests/unit/engines/test_qwant.py similarity index 100% rename from searx/tests/engines/test_qwant.py rename to tests/unit/engines/test_qwant.py diff --git a/searx/tests/engines/test_searchcode_code.py b/tests/unit/engines/test_searchcode_code.py similarity index 100% rename from searx/tests/engines/test_searchcode_code.py rename to tests/unit/engines/test_searchcode_code.py diff --git a/searx/tests/engines/test_searchcode_doc.py b/tests/unit/engines/test_searchcode_doc.py similarity index 100% rename from searx/tests/engines/test_searchcode_doc.py rename to tests/unit/engines/test_searchcode_doc.py diff --git a/searx/tests/engines/test_soundcloud.py b/tests/unit/engines/test_soundcloud.py similarity index 100% rename from searx/tests/engines/test_soundcloud.py rename to tests/unit/engines/test_soundcloud.py diff --git a/searx/tests/engines/test_spotify.py b/tests/unit/engines/test_spotify.py similarity index 100% rename from searx/tests/engines/test_spotify.py rename to tests/unit/engines/test_spotify.py diff --git a/searx/tests/engines/test_stackoverflow.py b/tests/unit/engines/test_stackoverflow.py similarity index 100% rename from searx/tests/engines/test_stackoverflow.py rename to tests/unit/engines/test_stackoverflow.py diff --git a/searx/tests/engines/test_startpage.py b/tests/unit/engines/test_startpage.py similarity index 100% rename from searx/tests/engines/test_startpage.py rename to tests/unit/engines/test_startpage.py diff --git a/searx/tests/engines/test_subtitleseeker.py b/tests/unit/engines/test_subtitleseeker.py similarity index 100% rename from searx/tests/engines/test_subtitleseeker.py rename to tests/unit/engines/test_subtitleseeker.py diff --git a/searx/tests/engines/test_swisscows.py b/tests/unit/engines/test_swisscows.py similarity index 100% rename from searx/tests/engines/test_swisscows.py rename to tests/unit/engines/test_swisscows.py diff --git a/searx/tests/engines/test_twitter.py b/tests/unit/engines/test_twitter.py similarity index 100% rename from searx/tests/engines/test_twitter.py rename to tests/unit/engines/test_twitter.py diff --git a/searx/tests/engines/test_vimeo.py b/tests/unit/engines/test_vimeo.py similarity index 100% rename from searx/tests/engines/test_vimeo.py rename to tests/unit/engines/test_vimeo.py diff --git a/searx/tests/engines/test_www1x.py b/tests/unit/engines/test_www1x.py similarity index 100% rename from searx/tests/engines/test_www1x.py rename to tests/unit/engines/test_www1x.py diff --git a/searx/tests/engines/test_www500px.py b/tests/unit/engines/test_www500px.py similarity index 100% rename from searx/tests/engines/test_www500px.py rename to tests/unit/engines/test_www500px.py diff --git a/searx/tests/engines/test_yacy.py b/tests/unit/engines/test_yacy.py similarity index 100% rename from searx/tests/engines/test_yacy.py rename to tests/unit/engines/test_yacy.py diff --git a/searx/tests/engines/test_yahoo.py b/tests/unit/engines/test_yahoo.py similarity index 100% rename from searx/tests/engines/test_yahoo.py rename to tests/unit/engines/test_yahoo.py diff --git a/searx/tests/engines/test_yahoo_news.py b/tests/unit/engines/test_yahoo_news.py similarity index 100% rename from searx/tests/engines/test_yahoo_news.py rename to tests/unit/engines/test_yahoo_news.py diff --git a/searx/tests/engines/test_youtube_api.py b/tests/unit/engines/test_youtube_api.py similarity index 100% rename from searx/tests/engines/test_youtube_api.py rename to tests/unit/engines/test_youtube_api.py diff --git a/searx/tests/engines/test_youtube_noapi.py b/tests/unit/engines/test_youtube_noapi.py similarity index 100% rename from searx/tests/engines/test_youtube_noapi.py rename to tests/unit/engines/test_youtube_noapi.py diff --git a/searx/tests/test_plugins.py b/tests/unit/test_plugins.py similarity index 100% rename from searx/tests/test_plugins.py rename to tests/unit/test_plugins.py diff --git a/searx/tests/test_results.py b/tests/unit/test_results.py similarity index 100% rename from searx/tests/test_results.py rename to tests/unit/test_results.py diff --git a/searx/tests/test_search.py b/tests/unit/test_search.py similarity index 100% rename from searx/tests/test_search.py rename to tests/unit/test_search.py diff --git a/searx/tests/test_utils.py b/tests/unit/test_utils.py similarity index 100% rename from searx/tests/test_utils.py rename to tests/unit/test_utils.py diff --git a/searx/tests/test_webapp.py b/tests/unit/test_webapp.py similarity index 100% rename from searx/tests/test_webapp.py rename to tests/unit/test_webapp.py diff --git a/versions.cfg b/versions.cfg deleted file mode 100644 index 2cb86aba1..000000000 --- a/versions.cfg +++ /dev/null @@ -1,122 +0,0 @@ -[versions] -Babel = 1.3 -Flask = 0.10.1 -Flask-Babel = 0.9 -Jinja2 = 2.7.3 -MarkupSafe = 0.23 -Pygments = 2.0.2 -WebOb = 1.4.1 -WebTest = 2.0.18 -Werkzeug = 0.10.4 -collective.recipe.omelette = 0.16 -coverage = 3.7.1 -decorator = 3.4.2 -docutils = 0.12 -flake8 = 2.4.1 -itsdangerous = 0.24 -mccabe = 0.3.1 -mock = 1.0.1 -pep8 = 1.5.7 -plone.testing = 4.0.13 -pyflakes = 0.8.1 -pytz = 2015.4 -pyyaml = 3.11 -requests = 2.7.0 -robotframework-debuglibrary = 0.3 -robotframework-httplibrary = 0.4.2 -robotframework-selenium2library = 1.7.1 -robotsuite = 1.6.1 -selenium = 2.46.0 -speaklater = 1.3 -unittest2 = 1.0.1 -waitress = 0.8.9 -zc.recipe.testrunner = 2.0.0 -pyopenssl = 0.15.1 -ndg-httpsclient = 0.4.0 -pyasn1 = 0.1.8 -pyasn1-modules = 0.0.6 -certifi = 2015.11.20.1 - -cffi = 1.1.2 -cryptography = 0.9.1 - -# Required by: -# robotsuite==1.6.1 -# searx==0.7.0 -lxml = 3.4.4 - -# Required by: -# searx==0.7.0 -python-dateutil = 2.4.2 - -# Required by: -# searx==0.7.0 -# zope.exceptions==4.0.7 -# zope.interface==4.1.2 -# zope.testrunner==4.4.9 -setuptools = 18.0.1 - -# Required by: -# WebTest==2.0.18 -beautifulsoup4 = 4.3.2 - -# Required by: -# cryptography==0.9.1 -enum34 = 1.0.4 - -# Required by: -# cryptography==0.9.1 -idna = 2.0 - -# Required by: -# cryptography==0.9.1 -ipaddress = 1.0.7 - -# Required by: -# robotframework-httplibrary==0.4.2 -jsonpatch = 1.11 - -# Required by: -# robotframework-httplibrary==0.4.2 -jsonpointer = 1.9 - -# Required by: -# traceback2==1.4.0 -linecache2 = 1.0.0 - -# Required by: -# cffi==1.1.2 -pycparser = 2.12 - -# Required by: -# robotframework-httplibrary==0.4.2 -robotframework = 2.8.7 - -# Required by: -# robotsuite==1.6.1 -# zope.testrunner==4.4.9 -six = 1.9.0 - -# Required by: -# unittest2==1.0.1 -traceback2 = 1.4.0 - -# Required by: -# collective.recipe.omelette==0.16 -zc.recipe.egg = 2.0.1 - -# Required by: -# zope.testrunner==4.4.9 -zope.exceptions = 4.0.7 - -# Required by: -# zope.testrunner==4.4.9 -zope.interface = 4.1.2 - -# Required by: -# plone.testing==4.0.13 -zope.testing = 4.2.0 - -# Required by: -# zc.recipe.testrunner==2.0.0 -zope.testrunner = 4.4.9 From 2ad9b4eeea3ca3eb98f483f8542b9c6d02e8c33a Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 11:29:54 +0100 Subject: [PATCH 04/13] [fix] dev requirements duplication --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d8c2395dd..ee100c16a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ flake8==2.5.1 mock==1.0.1 -nose2==0.4.7 nose2[coverage-plugin] plone.testing==4.0.15 robotframework-selenium2library==1.7.4 From 3498b7c53fa73d627ff4f1b858f4dfd5008008f2 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 11:30:10 +0100 Subject: [PATCH 05/13] [fix] manage.sh command names --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57d623060..5ca043a80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,8 @@ script: - ./manage.sh pep8_check - ./manage.sh styles - ./manage.sh grunt - - ./manage.sh coverage - - ./manage.sh robot + - ./manage.sh py_test_coverage + - ./manage.sh robot_tests after_success: coveralls notifications: From 39d2e0427f496e321a38e8a57b01a2c6043d2339 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 12:15:47 +0100 Subject: [PATCH 06/13] [fix] robot tests and pep8 check --- .gitignore | 1 + manage.sh | 5 ++--- searx/testing.py | 20 +++++++++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d29971645..105f0191b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ env robot_log.html robot_output.xml robot_report.html +test_basic/ setup.cfg *.pyc diff --git a/manage.sh b/manage.sh index 39a8e27d3..cf2662850 100755 --- a/manage.sh +++ b/manage.sh @@ -15,7 +15,7 @@ update_dev_packages() { pep8_check() { echo '[!] Running pep8 check' - pep8 "$SEARX_DIR" "$BASE_DIR/tests" + pep8 --max-line-length=120 "$SEARX_DIR" "$BASE_DIR/tests" } unit_tests() { @@ -32,8 +32,7 @@ py_test_coverage() { robot_tests() { echo '[!] Running robot tests' - python -c "import zope.testrunner; import sys; sys.exit(zope.testrunner.run("\ -"['--color', '--auto-progress', '--layer', 'SearxRobotLayer', '--test-path', '$BASE_DIR']))" + PYTHONPATH=`pwd` python "$SEARX_DIR/testing.py" robot } tests() { diff --git a/searx/testing.py b/searx/testing.py index e22ecf8fe..bc04d0fb0 100644 --- a/searx/testing.py +++ b/searx/testing.py @@ -3,6 +3,7 @@ from plone.testing import Layer from unittest2 import TestCase +from os.path import dirname, join, abspath import os @@ -42,11 +43,11 @@ class SearxRobotLayer(Layer): os.path.abspath(os.path.dirname(os.path.realpath(__file__))), 'webapp.py' ) - exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py') + exe = 'python' # set robot settings path - os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath( - os.path.dirname(__file__) + '/settings_robot.yml') + os.environ['SEARX_SETTINGS_PATH'] = abspath( + dirname(__file__) + '/settings_robot.yml') # run the server self.server = subprocess.Popen( @@ -68,3 +69,16 @@ class SearxTestCase(TestCase): """Base test case for non-robot tests.""" layer = SearxTestLayer + + +if __name__ == '__main__': + from tests.test_robot import test_suite + import sys + from zope.testrunner.runner import Runner + + base_dir = abspath(join(dirname(__file__), '../tests')) + if sys.argv[1] == 'robot': + Runner(['--color', + '--auto-progress', + '--path', base_dir], + found_suites=[test_suite()]).run() From 52323a4b4ed00f63a63181bdf9d0cf3850f1de78 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 12:21:40 +0100 Subject: [PATCH 07/13] [fix] grunt action name --- .travis.yml | 2 +- manage.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5ca043a80..4376e4c2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ install: script: - ./manage.sh pep8_check - ./manage.sh styles - - ./manage.sh grunt + - ./manage.sh grunt_build - ./manage.sh py_test_coverage - ./manage.sh robot_tests after_success: diff --git a/manage.sh b/manage.sh index cf2662850..2c92432aa 100755 --- a/manage.sh +++ b/manage.sh @@ -58,7 +58,7 @@ styles() { build_style themes/pix-art/less/style.less themes/pix-art/css/style.css } -grunt() { +grunt_build() { grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js" } From 5c073fdd8329fc3094f78d1730586e3e7ac16c81 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 12:36:36 +0100 Subject: [PATCH 08/13] [enh] setup.py requirements ++ new folders --- setup.py | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/setup.py b/setup.py index 6888ab2cf..fce32bb0c 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,8 @@ def read(*rnames): long_description = read('README.rst') +requirements = map(str.strip, open('requirements.txt').readlines()) +dev_requirements = map(str.strip, open('requirements-dev.txt').readlines()) setup( name='searx', @@ -38,35 +40,9 @@ setup( license='GNU Affero General Public License', packages=find_packages('.'), zip_safe=False, - install_requires=[ - 'flask', - 'flask-babel', - 'requests', - 'lxml', - 'pyyaml', - 'pygments', - 'setuptools', - 'python-dateutil', - 'pyopenssl', - 'ndg-httpsclient', - 'pyasn1', - 'pyasn1-modules', - 'certifi' - ], + install_requires=requirements, extras_require={ - 'test': [ - 'coverage', - 'flake8', - 'mock', - 'plone.testing', - 'robotframework', - 'robotframework-debuglibrary', - 'robotframework-httplibrary', - 'robotframework-selenium2library', - 'robotsuite', - 'unittest2', - 'zope.testrunner', - ] + 'test': dev_requirements }, entry_points={ 'console_scripts': [ @@ -86,6 +62,9 @@ setup( 'static/*/*/*/*/*.*', 'templates/*/*.*', 'templates/*/*/*.*', + 'tests/*', + 'tests/*/*', + 'tests/*/*/*', 'translations/*/*/*' ], }, From 1e4f0bedc5ed70359c6a75ae93bf51eba8e7145e Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 16:04:32 +0100 Subject: [PATCH 09/13] [doc] command descriptions --- manage.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/manage.sh b/manage.sh index 2c92432aa..68bcd807c 100755 --- a/manage.sh +++ b/manage.sh @@ -66,4 +66,28 @@ locales() { pybabel compile -d "$SEARX_DIR/translations" } -$ACTION +help() { + [ -z "$1" ] || echo "Error: $1\n" + echo "Searx manage.sh help + +Commands +======== + grunt_build - Build js files + help - This text + locales - Compile locales + pep8_check - Pep8 validation + py_test_coverage - Unit test coverage + robot_tests - Run selenium tests + styles - Build less files + tests - Run all python tests (pep8, unit, robot) + unit_tests - Run unit tests + update_dev_packages - Check & update development only dependency changes + update_packages - Check & update dependency changes +" +} + +if type $ACTION 1>/dev/null; then + $ACTION +else + help "action not found" +fi From 0c84996a32710c6de4e00a0e19ed13194d8831f5 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 2 Jan 2016 16:18:43 +0100 Subject: [PATCH 10/13] [doc] remove obsolete build docs --- README.rst | 117 +---------------------------------------------------- 1 file changed, 2 insertions(+), 115 deletions(-) diff --git a/README.rst b/README.rst index cf1263c0f..c0993157d 100644 --- a/README.rst +++ b/README.rst @@ -7,31 +7,16 @@ engine `__. List of `running instances `__. -See the `wiki `__ for more information. +See the `documentation `__ and the `wiki `__ for more information. |Flattr searx| -Features -~~~~~~~~ - -- Tracking free -- Supports multiple output formats - - - json ``curl https://searx.me/?format=json&q=[query]`` - - csv ``curl https://searx.me/?format=csv&q=[query]`` - - opensearch/rss ``curl https://searx.me/?format=rss&q=[query]`` -- Opensearch support (you can set as default search engine) -- Configurable search engines/categories -- Different search languages -- Duckduckgo like !bang functionality with engine shortcuts -- Parallel queries - relatively fast - Installation ~~~~~~~~~~~~ - clone source: ``git clone git@github.com:asciimoo/searx.git && cd searx`` -- install dependencies: ``pip install -r requirements.txt`` +- install dependencies: ``./manage.sh update_packages`` - edit your `settings.yml `__ (set your ``secret_key``!) @@ -40,104 +25,6 @@ Installation For all the details, follow this `step by step installation `__ -Alternative (Recommended) Installation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- clone source: - ``git clone git@github.com:asciimoo/searx.git && cd searx`` -- build in current folder: ``make minimal`` -- run ``bin/searx-run`` to start the application - -Development -~~~~~~~~~~~ - -Just run ``make``. Versions of dependencies are pinned down inside -``versions.cfg`` to produce most stable build. Also remember, NO make -command should be run as root, not even ``make production`` - -Deployment -~~~~~~~~~~ - -- clone source: - ``git clone git@github.com:asciimoo/searx.git && cd searx`` -- build in current folder: ``make production`` -- run ``bin/supervisord`` to start the application - -Upgrading -~~~~~~~~~ - -- inside previously cloned searx directory run: ``git stash`` to - temporarily save any changes you have made -- pull source: ``git pull origin master`` -- re-build in current folder: ``make production`` -- run ``bin/supervisorctl stop searx`` to stop searx, if it does not, - then run ``fuser -k 8888/tcp`` -- run ``bin/supervisorctl reload`` to re-read supervisor config and - start searx - -Command make -~~~~~~~~~~~~ - -``make`` -'''''''' - -Builds development environment with testing support. - -``make tests`` -'''''''''''''' - -Runs tests. You can write tests -`here `__ and -remember 'untested code is broken code'. - -``make robot`` -'''''''''''''' - -Runs robot (Selenium) tests, you must have ``firefox`` installed because -this functional tests actually run the browser and perform operations on -it. Also searx is executed with -`settings\_robot `__. - -``make flake8`` -''''''''''''''' - -'pep8 is a tool to check your Python code against some of the style -conventions in `PEP 8 `__.' - -``make coverage`` -''''''''''''''''' - -Checks coverage of tests, after running this, execute this: -``firefox ./coverage/index.html`` - -``make production`` -''''''''''''''''''' - -Used to make co-called production environment - without tests (you -should ran tests before deploying searx on the server). This installs -supervisord, so if searx crashes, it will try to pick itself up again. -And crontab entry is added to start supervisord at server boot. - -``make minimal`` -'''''''''''''''' - -Minimal build - without test frameworks, the quickest build option. - -``make clean`` -'''''''''''''' - -Deletes several folders and files (see ``Makefile`` for more), so that -next time you run any other ``make`` command it will rebuild everithing. - -TODO -~~~~ - -- Moar engines -- Better ui -- Browser integration -- Documentation -- Tests - Bugs ~~~~ From 28a9b5113ef5827146b0fe9ba3a484de77564dd9 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 7 Jan 2016 19:56:59 +0100 Subject: [PATCH 11/13] [mod] update core deps with dev deps --- .travis.yml | 1 - manage.sh | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4376e4c2c..d0bc35094 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ before_install: - npm install -g less grunt-cli - ( cd searx/static/themes/oscar;npm install; cd - ) install: - - ./manage.sh update_packages - ./manage.sh update_dev_packages - pip install coveralls script: diff --git a/manage.sh b/manage.sh index 68bcd807c..7890f98ec 100755 --- a/manage.sh +++ b/manage.sh @@ -10,6 +10,7 @@ update_packages() { } update_dev_packages() { + update_packages pip install --upgrade -r "$BASE_DIR/requirements-dev.txt" } From c873ddd7fbbc6275d6f4eb5ff873768bf9c8a48c Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 10 Jan 2016 19:49:07 +0100 Subject: [PATCH 12/13] [fix] translation dev deps --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index ee100c16a..d9ec779b3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,10 @@ +babel==2.2.0 flake8==2.5.1 mock==1.0.1 nose2[coverage-plugin] plone.testing==4.0.15 robotframework-selenium2library==1.7.4 robotsuite==1.7.0 +transifex-client==0.11 unittest2==1.1.0 zope.testrunner==4.4.10 From 6a158ca2d2c5fdec597f1025c58fad90a57312dc Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 10 Jan 2016 20:29:07 +0100 Subject: [PATCH 13/13] [fix] manage.sh dev package update help --- manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.sh b/manage.sh index 7890f98ec..0cff4b352 100755 --- a/manage.sh +++ b/manage.sh @@ -82,7 +82,7 @@ Commands styles - Build less files tests - Run all python tests (pep8, unit, robot) unit_tests - Run unit tests - update_dev_packages - Check & update development only dependency changes + update_dev_packages - Check & update development and production dependency changes update_packages - Check & update dependency changes " }