Merge branch 'master' into http1.1

.travis.yml

@@ -4,6 +4,8 @@ cache:
   - npm
   - directories:
     - $HOME/.cache/pip
+addons:
+  firefox: "latest"
 language: python
 python:
   - "2.7"
@@ -12,6 +14,9 @@ before_install:
   - "sh -e /etc/init.d/xvfb start"
   - npm install less grunt-cli
   - ( cd searx/static/themes/oscar;npm install; cd - )
+  - mkdir -p ~/drivers; export PATH=~/drivers:$PATH;
+  - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz";
+  - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver;
 install:
   - ./manage.sh update_dev_packages
   - pip install coveralls

AUTHORS.rst

@@ -58,3 +58,5 @@ generally made searx better:
 - marc @a01200356
 - Harry Wood @harry-wood
 - Thomas Renard @threnard
+- Pydo `<https://github.com/pydo>`_
+- Athemis `<https://github.com/Athemis>`_

manage.sh

@@ -53,8 +53,8 @@ build_style() {
 
 styles() {
     echo '[!] Building styles'
-	build_style themes/default/less/style.less themes/default/css/style.css
-	build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css
+	build_style themes/legacy/less/style.less themes/legacy/css/style.css
+	build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css
 	build_style themes/courgette/less/style.less themes/courgette/css/style.css
 	build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css
 	build_style less/bootstrap/bootstrap.less css/bootstrap.min.css

requirements-dev.txt

@@ -3,8 +3,8 @@ mock==2.0.0
 nose2[coverage-plugin]
 pep8==1.7.0
 plone.testing==5.0.0
-robotframework-selenium2library==1.7.4
+robotframework-selenium2library==1.8.0
 robotsuite==1.7.0
-transifex-client==0.11
+transifex-client==0.12.2
 unittest2==1.1.0
-zope.testrunner==4.4.10
+zope.testrunner==4.5.1

requirements.txt

@@ -1,4 +1,4 @@
-certifi==2016.2.28
+certifi==2016.9.26
 flask==0.11.1
 flask-babel==0.11.1
 lxml==3.6.0

searx/engines/__init__.py

@@ -57,11 +57,17 @@ def load_module(filename):
 
 
 def load_engine(engine_data):
-    engine_name = engine_data['engine']
+
+    if '_' in engine_data['name']:
+        logger.error('Engine name conains underscore: "{}"'.format(engine_data['name']))
+        sys.exit(1)
+
+    engine_module = engine_data['engine']
+
     try:
-        engine = load_module(engine_name + '.py')
+        engine = load_module(engine_module + '.py')
     except:
-        logger.exception('Cannot load engine "{}"'.format(engine_name))
+        logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None
 
     for param_name in engine_data:

searx/engines/digbt.py

@@ -40,7 +40,7 @@ def response(resp):
     results = list()
     for result in search_res:
         url = urljoin(URL, result.xpath('.//a[@title]/@href')[0])
-        title = result.xpath('.//a[@title]/text()')[0]
+        title = extract_text(result.xpath('.//a[@title]'))
         content = extract_text(result.xpath('.//div[@class="files"]'))
         files_data = extract_text(result.xpath('.//div[@class="tail"]')).split()
         filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER])

searx/engines/kickass.py

@@ -16,13 +16,14 @@ from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size, convert_str_to_int
 
 # engine dependent config
 categories = ['videos', 'music', 'files']
 paging = True
 
 # search-url
-url = 'https://kickass.to/'
+url = 'https://kickass.cd/'
 search_url = url + 'search/{search_term}/{pageno}/'
 
 # specific xpath variables
@@ -57,41 +58,16 @@ def response(resp):
         href = urljoin(url, link.attrib['href'])
         title = extract_text(link)
         content = escape(extract_text(result.xpath(content_xpath)))
-        seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
-        leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
-        filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
-        filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
-        files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
+        seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
+        files = extract_text(result.xpath('.//td[contains(@class, "center")][2]'))
 
-        # convert seed to int if possible
-        if seed.isdigit():
-            seed = int(seed)
-        else:
-            seed = 0
+        seed = convert_str_to_int(seed)
+        leech = convert_str_to_int(leech)
 
-        # convert leech to int if possible
-        if leech.isdigit():
-            leech = int(leech)
-        else:
-            leech = 0
-
-        # convert filesize to byte if possible
-        try:
-            filesize = float(filesize)
-
-            # convert filesize to byte
-            if filesize_multiplier == 'TB':
-                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'GB':
-                filesize = int(filesize * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'MB':
-                filesize = int(filesize * 1024 * 1024)
-            elif filesize_multiplier == 'KB':
-                filesize = int(filesize * 1024)
-        except:
-            filesize = None
-
-        # convert files to int if possible
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
         if files.isdigit():
             files = int(files)
         else:

searx/engines/pdbe.py

@@ -0,0 +1,109 @@
+"""
+ PDBe (Protein Data Bank in Europe)
+
+ @website       https://www.ebi.ac.uk/pdbe
+ @provide-api   yes (https://www.ebi.ac.uk/pdbe/api/doc/search.html),
+                unlimited
+ @using-api     yes
+ @results       python dictionary (from json)
+ @stable        yes
+ @parse         url, title, content, img_src
+"""
+
+from json import loads
+from flask_babel import gettext
+
+categories = ['science']
+
+hide_obsolete = False
+
+# status codes of unpublished entries
+pdb_unpublished_codes = ['HPUB', 'HOLD', 'PROC', 'WAIT', 'AUTH', 'AUCO', 'REPL', 'POLC', 'REFI', 'TRSF', 'WDRN']
+# url for api query
+pdbe_solr_url = 'https://www.ebi.ac.uk/pdbe/search/pdb/select?'
+# base url for results
+pdbe_entry_url = 'https://www.ebi.ac.uk/pdbe/entry/pdb/{pdb_id}'
+# link to preview image of structure
+pdbe_preview_url = 'https://www.ebi.ac.uk/pdbe/static/entry/{pdb_id}_deposited_chain_front_image-200x200.png'
+
+
+def request(query, params):
+
+    params['url'] = pdbe_solr_url
+    params['method'] = 'POST'
+    params['data'] = {
+        'q': query,
+        'wt': "json"  # request response in parsable format
+    }
+    return params
+
+
+def construct_body(result):
+    # set title
+    title = result['title']
+
+    # construct content body
+    content = """{title}<br />{authors} {journal} <strong>{volume}</strong>&nbsp;{page} ({year})"""
+
+    # replace placeholders with actual content
+    try:
+        if result['journal']:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal=result['journal'], volume=result['journal_volume'],
+                page=result['journal_page'], year=result['citation_year'])
+        else:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal='', volume='', page='', year=result['release_year'])
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        content = None
+        img_src = None
+
+    # construct url for preview image
+    try:
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        img_src = None
+
+    return [title, content, img_src]
+
+
+def response(resp):
+
+    results = []
+    json = loads(resp.text)['response']['docs']
+
+    # parse results
+    for result in json:
+        # catch obsolete entries and mark them accordingly
+        if result['status'] in pdb_unpublished_codes:
+            continue
+        if hide_obsolete:
+            continue
+        if result['status'] == 'OBS':
+            # expand title to add some sort of warning message
+            title = gettext('{title}&nbsp;(OBSOLETE)').format(title=result['title'])
+            superseded_url = pdbe_entry_url.format(pdb_id=result['superseded_by'])
+
+            # since we can't construct a proper body from the response, we'll make up our own
+            msg_superseded = gettext("This entry has been superseded by")
+            content = '<em>{msg_superseded} \<a href="{url}">{pdb_id}</a></em>'.format(
+                msg_superseded=msg_superseded,
+                url=superseded_url,
+                pdb_id=result['superseded_by'], )
+
+            # obsoleted entries don't have preview images
+            img_src = None
+        else:
+            title, content, img_src = construct_body(result)
+
+        results.append({
+            'url': pdbe_entry_url.format(pdb_id=result['pdb_id']),
+            'title': title,
+            'content': content,
+            'img_src': img_src
+        })
+
+    return results

searx/engines/seedpeer.py

@@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     http://seedpeer.eu
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import quote
+from lxml import html
+from operator import itemgetter
+from searx.engines.xpath import extract_text
+
+
+url = 'http://www.seedpeer.eu/'
+search_url = url + 'search/{search_term}/7/{page_no}.html'
+# specific xpath variables
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'] - 1)
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    torrent_links = dom.xpath(torrent_xpath)
+    if len(torrent_links) > 0:
+        seeds = dom.xpath(seeds_xpath)
+        peers = dom.xpath(peers_xpath)
+        titles = dom.xpath(title_xpath)
+        sizes = dom.xpath(size_xpath)
+        ages = dom.xpath(age_xpath)
+    else:  # under ~5 results uses a different xpath
+        torrent_links = dom.xpath(alternative_torrent_xpath)
+        seeds = dom.xpath(alternative_seeds_xpath)
+        peers = dom.xpath(alternative_peers_xpath)
+        titles = dom.xpath(alternative_title_xpath)
+        sizes = dom.xpath(alternative_size_xpath)
+        ages = dom.xpath(alternative_age_xpath)
+    # return empty array if nothing is found
+    if not torrent_links:
+        return []
+
+    # parse results
+    for index, result in enumerate(torrent_links):
+        link = result.attrib.get('href')
+        href = urljoin(url, link)
+        results.append({'url': href,
+                        'title': titles[index].text_content(),
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
+                        'seed': seeds[index],
+                        'leech': peers[index],
+
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/settings.yml

@@ -18,6 +18,12 @@ ui:
     default_theme : oscar # ui theme
     default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
 
+# searx supports result proxification using an external service: https://github.com/asciimoo/morty
+# uncomment below section if you have running morty proxy
+#result_proxy:
+#    url : http://127.0.0.1:3000/
+#    key : your_morty_proxy_key
+
 outgoing: # communication with search engines
     request_timeout : 2.0 # seconds
     useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
@@ -301,6 +307,12 @@ engines:
     timeout : 6.0
     disabled : True
 
+  - name: kickass
+    engine : kickass
+    shortcut : kc
+    timeout : 4.0
+    disabled : True
+
   - name : microsoft academic
     engine : json_engine
     paging : True
@@ -339,6 +351,13 @@ engines:
     disabled : True
     shortcut : or
 
+  - name : pdbe
+    engine : pdbe
+    shortcut : pdb
+# Hide obsolete PDB entries.
+# Default is not to hide obsolete structures
+#    hide_obsolete : False
+
   - name : photon
     engine : photon
     shortcut : ph
@@ -377,7 +396,7 @@ engines:
     timeout : 10.0
     disabled : True
 
-  - name : scanr_structures
+  - name : scanr structures
     shortcut: scs
     engine : scanr_structures
     disabled : True
@@ -495,6 +514,12 @@ engines:
     timeout: 6.0
     categories : science
 
+  - name : seedpeer
+    engine : seedpeer
+    shortcut: speu
+    categories: files, music, videos
+    disabled: True
+
   - name : dictzone
     engine : dictzone
     shortcut : dc

searx/settings_robot.yml

@@ -15,7 +15,7 @@ server:
 
 ui:
     themes_path : ""
-    default_theme : default
+    default_theme : legacy
     default_locale : ""
 
 outgoing:
@@ -23,12 +23,12 @@ outgoing:
     useragent_suffix : ""
 
 engines:
-  - name : general_dummy
+  - name : general dummy
     engine : dummy
     categories : general
     shortcut : gd
 
-  - name : dummy_dummy
+  - name : dummy dummy
     engine : dummy
     categories : dummy
     shortcut : dd

searx/templates/courgette/base.html

@@ -22,7 +22,7 @@
         {% endblock %}
         {% block meta %}{% endblock %}
         {% block head %}
-        <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+        <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
         {% endblock %}
         <script type="text/javascript">
             searx = {};

searx/templates/legacy/404.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block content %}
 <div class="center">
     <h1>{{ _('Page not found') }}</h1>

searx/templates/legacy/about.html

@@ -1,6 +1,6 @@
-{% extends 'default/base.html' %}
+{% extends 'legacy/base.html' %}
 {% block content %}
-{% include 'default/github_ribbon.html' %}
+{% include 'legacy/github_ribbon.html' %}
 <div class="row"{% if rtl %} dir="ltr"{% endif %}>
     <h1>About <a href="{{ url_for('index') }}">searx</a></h1>
 

searx/templates/legacy/base.html

@@ -17,7 +17,7 @@
         {% endblock %}
         {% block meta %}{% endblock %}
         {% block head %}
-        <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+        <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
         {% endblock %}
     </head>
     <body>

searx/templates/legacy/index.html

@@ -1,8 +1,8 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block content %}
 <div class="center">
     <div class="title"><h1>searx</h1></div>
-    {% include 'default/search.html' %}
+    {% include 'legacy/search.html' %}
     <p class="top_margin">
     	{% if rtl %}
     	<a href="{{ url_for('preferences') }}" class="hmarg">{{ _('preferences') }}</a>
@@ -13,6 +13,6 @@
         {% endif %}
     </p>
 </div>
-{% include 'default/github_ribbon.html' %}
+{% include 'legacy/github_ribbon.html' %}
 {% endblock %}
 

searx/templates/legacy/preferences.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <div class="row">
@@ -8,7 +8,7 @@
     <fieldset>
         <legend>{{ _('Default categories') }}</legend>
         {% set display_tooltip = false %}
-        {% include 'default/categories.html' %}
+        {% include 'legacy/categories.html' %}
     </fieldset>
     <fieldset>
         <legend>{{ _('Search language') }}</legend>

searx/templates/legacy/results.html

@@ -1,10 +1,10 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block title %}{{ q }} - {% endblock %}
 {% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
 {% block content %}
 <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>
 <div class="small search center">
-    {% include 'default/search.html' %}
+    {% include 'legacy/search.html' %}
 </div>
 <div id="results">
     <div id="sidebar">
@@ -55,16 +55,16 @@
     {% if infoboxes %}
     <div id="infoboxes">
       {% for infobox in infoboxes %}
-         {% include 'default/infobox.html' %}
+         {% include 'legacy/infobox.html' %}
       {% endfor %}
     </div>
     {% endif %}
 
     {% for result in results %}
         {% if result['template'] %}
-            {% include get_result_template('default', result['template']) %}
+            {% include get_result_template('legacy', result['template']) %}
         {% else %}
-            {% include 'default/result_templates/default.html' %}
+            {% include 'legacy/result_templates/default.html' %}
         {% endif %}
     {% endfor %}
 

searx/templates/legacy/search.html

@@ -4,5 +4,5 @@
         <input type="submit" value="search" id="search_submit" />
     </div>
     {% set display_tooltip = true %}
-    {% include 'default/categories.html' %}
+    {% include 'legacy/categories.html' %}
 </form>

searx/templates/legacy/stats.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <h2>{{ _('Engine stats') }}</h2>

searx/templates/oscar/macros.html

@@ -33,6 +33,9 @@
         <span class="label label-default">{{ engine }}</span>
     {% endfor %}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+    {% if proxify %}
+    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+    {% endif %}
 </div>
     <div class="text-muted"><small>{{ result.pretty_url }}</small></div>
 {%- endmacro %}
@@ -44,6 +47,9 @@
         <span class="label label-default">{{ engine }}</span>
     {% endfor %}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+    {% if proxify %}
+    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+    {% endif %}
     <div class="text-muted"><small>{{ result.pretty_url }}</small></div>
 {%- endmacro %}
 

searx/templates/pix-art/preferences.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <div class="row">

searx/templates/pix-art/stats.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <h2>{{ _('Engine stats') }}</h2>

searx/utils.py

@@ -252,12 +252,27 @@ def get_torrent_size(filesize, filesize_multiplier):
             filesize = int(filesize * 1024 * 1024)
         elif filesize_multiplier == 'KB':
             filesize = int(filesize * 1024)
+        elif filesize_multiplier == 'TiB':
+            filesize = int(filesize * 1000 * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'GiB':
+            filesize = int(filesize * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'MiB':
+            filesize = int(filesize * 1000 * 1000)
+        elif filesize_multiplier == 'KiB':
+            filesize = int(filesize * 1000)
     except:
         filesize = None
 
     return filesize
 
 
+def convert_str_to_int(number_str):
+    if number_str.isdigit():
+        return int(number_str)
+    else:
+        return 0
+
+
 def is_valid_lang(lang):
     is_abbr = (len(lang) == 2)
     if is_abbr:

searx/webapp.py

@@ -22,10 +22,11 @@ if __name__ == '__main__':
     from os.path import realpath, dirname
     path.append(realpath(dirname(realpath(__file__)) + '/../'))
 
-import json
 import cStringIO
-import os
 import hashlib
+import hmac
+import json
+import os
 import requests
 
 from searx import logger
@@ -245,6 +246,20 @@ def url_for_theme(endpoint, override_theme=None, **values):
     return url_for(endpoint, **values)
 
 
+def proxify(url):
+    if url.startswith('//'):
+        url = 'https:' + url
+
+    if not settings.get('result_proxy'):
+        return url
+
+    h = hmac.new(settings['result_proxy']['key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
+
+    return '{0}?{1}'.format(settings['result_proxy']['url'],
+                            urlencode(dict(mortyurl=url.encode('utf-8'),
+                                           mortyhash=h)))
+
+
 def image_proxify(url):
 
     if url.startswith('//'):
@@ -253,8 +268,7 @@ def image_proxify(url):
     if not request.preferences.get_value('image_proxy'):
         return url
 
-    hash_string = url + settings['server']['secret_key']
-    h = hashlib.sha256(hash_string.encode('utf-8')).hexdigest()
+    h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
 
     return '{0}?{1}'.format(url_for('image_proxy'),
                             urlencode(dict(url=url.encode('utf-8'), h=h)))
@@ -313,6 +327,8 @@ def render(template_name, override_theme=None, **kwargs):
 
     kwargs['image_proxify'] = image_proxify
 
+    kwargs['proxify'] = proxify if settings.get('result_proxy') else None
+
     kwargs['get_result_template'] = get_result_template
 
     kwargs['theme'] = get_current_theme_name(override=override_theme)
@@ -602,7 +618,7 @@ def image_proxy():
     if not url:
         return '', 400
 
-    h = hashlib.sha256(url + settings['server']['secret_key'].encode('utf-8')).hexdigest()
+    h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
 
     if h != request.args.get('h'):
         return '', 400
@@ -660,6 +676,7 @@ Allow: /
 Allow: /about
 Disallow: /stats
 Disallow: /preferences
+Disallow: /*?*q=*
 """, mimetype='text/plain')
 
 

tests/robot/test_basic.robot

@@ -4,6 +4,14 @@ Test Setup      Open Browser  http://localhost:11111/
 Test Teardown   Close All Browsers
 
 
+*** Keywords ***
+Submit Preferences
+    Set Selenium Speed  2 seconds
+    Submit Form  id=search_form
+    Location Should Be  http://localhost:11111/
+    Set Selenium Speed  0 seconds
+
+
 *** Test Cases ***
 Front page
     Page Should Contain  about
@@ -24,8 +32,8 @@ Preferences page
     Page Should Contain  Preferences
     Page Should Contain  Default categories
     Page Should Contain  Currently used search engines
-    Page Should Contain  dummy_dummy
-    Page Should Contain  general_dummy
+    Page Should Contain  dummy dummy
+    Page Should Contain  general dummy
 
 Switch category
     Go To  http://localhost:11111/preferences
@@ -33,8 +41,7 @@ Switch category
     Page Should Contain Checkbox  category_dummy
     Click Element  xpath=//*[.="general"]
     Click Element  xpath=//*[.="dummy"]
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Checkbox Should Not Be Selected  category_general
     Checkbox Should Be Selected  category_dummy
 
@@ -43,8 +50,7 @@ Change language
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
     Select From List  locale  hu
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Page Should Contain  rólunk
     Page Should Contain  beállítások
 
@@ -53,13 +59,11 @@ Change method
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
     Select From List  method  GET
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  method  GET
     Select From List  method  POST
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  method  POST
 
@@ -67,10 +71,9 @@ Change theme
     Page Should Contain  about
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
-    List Selection Should Be  theme  default
+    List Selection Should Be  theme  legacy
     Select From List  theme  oscar
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  theme  oscar
 
@@ -80,8 +83,7 @@ Change safesearch
     Go To  http://localhost:11111/preferences
     List Selection Should Be  safesearch  None
     Select From List  safesearch  Strict
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  safesearch  Strict
 
@@ -91,8 +93,7 @@ Change image proxy
     Go To  http://localhost:11111/preferences
     List Selection Should Be  image_proxy  Disabled
     Select From List  image_proxy  Enabled
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  image_proxy  Enabled
 
@@ -102,8 +103,7 @@ Change search language
     Go To  http://localhost:11111/preferences
     List Selection Should Be  language  Automatic
     Select From List  language  Turkish (Turkey) - tr_TR
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  language  Turkish (Turkey) - tr_TR
 
@@ -113,8 +113,7 @@ Change autocomplete
     Go To  http://localhost:11111/preferences
     List Selection Should Be  autocomplete  -
     Select From List  autocomplete  google
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  autocomplete  google
 
@@ -126,8 +125,7 @@ Change allowed/disabled engines
     Element Should Contain  xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy']  Block
     Element Should Contain  xpath=//label[@class="deny"][@for='engine_general_general_dummy']  Block
     Click Element  xpath=//label[@class="deny"][@for='engine_general_general_dummy']
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Page Should Contain  about
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
@@ -139,18 +137,16 @@ Block a plugin
     Page Should Contain  about
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
-    List Selection Should Be  theme  default
+    List Selection Should Be  theme  legacy
     Select From List  theme  oscar
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  theme  oscar
     Page Should Contain  Plugins
     Click Link  Plugins
     Checkbox Should Not Be Selected  id=plugin_HTTPS_rewrite
     Click Element  xpath=//label[@for='plugin_HTTPS_rewrite']
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     Page Should Contain  Plugins
     Click Link  Plugins

tests/unit/engines/seedpeer_fixture.html

@@ -0,0 +1,110 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml">
+        <head>
+        </head>
+        <body>
+            <div id="header">
+                <div id="whoIsYou">
+                        <a href="/lang.php"><small>SeedPeer in your own language?</small></a>&nbsp;&nbsp;&nbsp;<a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
+                        </div>
+
+                <script type="text/javascript">
+                whoIsYou();
+                </script>
+                    <div id="search">
+                        <form action="/search.php" method="get">
+                            <input id="topsearchbar" name="search" value="narcos season 2" />
+                            <input type="submit" class="searchbutton" value="Torrents" />
+                            <input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
+                        </form>
+                        <div id="suggestion"></div>
+                    </div>
+                    <div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
+                    <div id="subtext"><a href="/">Home</a> &gt; <a href="/search.html">Torrent search</a> &gt; Narcos season 2 | page 1</div>
+            </div>
+        <div id="nav">
+            <ul>
+        <!--
+                <li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
+
+        -->
+                <li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
+                    <ul>
+                        <li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
+                        <li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
+                        <li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
+                        <li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
+                        <li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
+                        <li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
+                        <li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
+                    </ul>
+                    </li>
+                        <li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
+                        <li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
+                        <li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
+                        <li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
+                        <li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
+                    <ul>
+                        <li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
+                        <li><a title="Contact Us" href="/contact.html">Contact</a></li>
+                        <li><a title="Frequently  Asked Questions" href="/faq.html">FAQ</a></li>
+                        <li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
+                        <li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
+                    </ul>
+                    </li>
+                    <!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
+                </ul>
+            <div class="clear"></div>
+        </div>
+        <div id="body"><div id="pageTop"></div>
+        <div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
+        <span style="float:right">
+        <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
+        <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
+        </span>
+        <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a>&nbsp;2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
+        <span style="float:right">
+        <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
+        <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
+        </span>
+        <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
+        <table width="100%">
+        <tr>
+        <td width="30%">
+        <h2>Torrents Download</h2>
+        <a href="/">Torrent search</a><br />
+        <a href="/browse.html">Browse categories</a><br />
+        <a href="/verified.html">Verified Torrents</a><br />
+        <a href="/order-date.html">Today's torrents</a><br />
+        <a href="/yesterday.html">Yesterday's torrents</a><br />
+        <a href="/stats.html">Statistics</a><br />
+        <br />
+        <a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
+        </td>
+        <td width="30%"><h2>Cool Stuff</h2>
+        <a href="/promotional.php">Promotional</a><br />
+        <a href="/contact.html">Advertising Information</a><br />
+        <strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
+        <a href="http://www.utorrent.com">&micro;Torrent Client</a><br />
+        <a href="/blog">Seedpeer Blog</a><br />
+        </td>
+        <td width="30%"><h2>Links</h2>
+         <a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
+         <a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
+         <a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
+         <a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
+         <a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
+         <a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
+         <a href="https://monova.org" target="_blank">Monova</a><br />
+         <a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
+         <a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
+        </td>
+        <td width="10%"><div id="bottomlogo"></div></td>
+        </tr>
+        </table>
+        <br />
+        <br />
+        </div>
+        </div>
+        </body>
+        </html>

tests/unit/engines/test_digbt.py

@@ -28,7 +28,9 @@ class TestDigBTEngine(SearxTestCase):
         <table class="table">
             <tr><td class="x-item">
             <div>
-                <a title="The Big Bang Theory" class="title" href="/The-Big-Bang-Theory-d2.html">The Big Bang Theory</a>
+                <a title="The Big Bang Theory" class="title" href="/The-Big-Bang-Theory-d2.html">
+                    The Big <span class="highlight">Bang</span> Theory
+                </a>
                 <span class="ctime"><span style="color:red;">4 hours ago</span></span>
             </div>
             <div class="files">

tests/unit/engines/test_kickass.py

@@ -14,7 +14,7 @@ class TestKickassEngine(SearxTestCase):
         params = kickass.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])
-        self.assertIn('kickass.to', params['url'])
+        self.assertIn('kickass.cd', params['url'])
         self.assertFalse(params['verify'])
 
     def test_response(self):
@@ -84,7 +84,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">449 <span>bytes</span></td>
+                <td class="nobr center">449 bytes</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">10</td>
@@ -97,7 +97,7 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], 'This should be the title')
-        self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+        self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
         self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
@@ -191,7 +191,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>KB</span></td>
+                <td class="nobr center">1 KiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">10</td>
@@ -235,7 +235,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>MB</span></td>
+                <td class="nobr center">1 MiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">9</td>
@@ -279,7 +279,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>GB</span></td>
+                <td class="nobr center">1 GiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">8</td>
@@ -323,7 +323,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>TB</span></td>
+                <td class="nobr center">1 TiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">7</td>
@@ -367,7 +367,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">z <span>bytes</span></td>
+                <td class="nobr center">z bytes</td>
                 <td class="center">r</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">a</td>
@@ -380,17 +380,17 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 5)
         self.assertEqual(results[0]['title'], 'This should be the title')
-        self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+        self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
         self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
         self.assertEqual(results[0]['files'], 4)
         self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
         self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
-        self.assertEqual(results[0]['filesize'], 1024)
-        self.assertEqual(results[1]['filesize'], 1048576)
-        self.assertEqual(results[2]['filesize'], 1073741824)
-        self.assertEqual(results[3]['filesize'], 1099511627776)
+        self.assertEqual(results[0]['filesize'], 1000)
+        self.assertEqual(results[1]['filesize'], 1000000)
+        self.assertEqual(results[2]['filesize'], 1000000000)
+        self.assertEqual(results[3]['filesize'], 1000000000000)
         self.assertEqual(results[4]['seed'], 0)
         self.assertEqual(results[4]['leech'], 0)
         self.assertEqual(results[4]['files'], None)

tests/unit/engines/test_pdbe.py

@@ -0,0 +1,109 @@
+import mock
+from collections import defaultdict
+from searx.engines import pdbe
+from searx.testing import SearxTestCase
+
+
+class TestPdbeEngine(SearxTestCase):
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        params = pdbe.request(query, dicto)
+        self.assertTrue('url' in params)
+        self.assertTrue('ebi.ac.uk' in params['url'])
+        self.assertTrue('data' in params)
+        self.assertTrue('q' in params['data'])
+        self.assertTrue(query in params['data']['q'])
+        self.assertTrue('wt' in params['data'])
+        self.assertTrue('json' in params['data']['wt'])
+        self.assertTrue('method' in params)
+        self.assertTrue(params['method'] == 'POST')
+
+    def test_response(self):
+        self.assertRaises(AttributeError, pdbe.response, None)
+        self.assertRaises(AttributeError, pdbe.response, [])
+        self.assertRaises(AttributeError, pdbe.response, '')
+        self.assertRaises(AttributeError, pdbe.response, '[]')
+
+        json = """
+{
+  "response": {
+    "docs": [
+      {
+        "citation_title": "X-ray crystal structure of ferric Aplysia limacina myoglobin in different liganded states.",
+        "citation_year": 1993,
+        "entry_author_list": [
+          "Conti E, Moser C, Rizzi M, Mattevi A, Lionetti C, Coda A, Ascenzi P, Brunori M, Bolognesi M"
+        ],
+        "journal": "J. Mol. Biol.",
+        "journal_page": "498-508",
+        "journal_volume": "233",
+        "pdb_id": "2fal",
+        "status": "REL",
+        "title": "X-RAY CRYSTAL STRUCTURE OF FERRIC APLYSIA LIMACINA MYOGLOBIN IN DIFFERENT LIGANDED STATES"
+      }
+    ],
+    "numFound": 1,
+    "start": 0
+  },
+  "responseHeader": {
+    "QTime": 0,
+    "params": {
+      "q": "2fal",
+      "wt": "json"
+    },
+    "status": 0
+  }
+}
+"""
+
+        response = mock.Mock(text=json)
+        results = pdbe.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'],
+                         'X-RAY CRYSTAL STRUCTURE OF FERRIC APLYSIA LIMACINA MYOGLOBIN IN DIFFERENT LIGANDED STATES')
+        self.assertEqual(results[0]['url'], pdbe.pdbe_entry_url.format(pdb_id='2fal'))
+        self.assertEqual(results[0]['img_src'], pdbe.pdbe_preview_url.format(pdb_id='2fal'))
+        self.assertTrue('Conti E' in results[0]['content'])
+        self.assertTrue('X-ray crystal structure of ferric Aplysia limacina myoglobin in different liganded states.' in
+                        results[0]['content'])
+        self.assertTrue('1993' in results[0]['content'])
+
+        # Testing proper handling of PDB entries marked as obsolete
+        json = """
+{
+  "response": {
+    "docs": [
+      {
+        "citation_title": "Obsolete entry test",
+        "citation_year": 2016,
+        "entry_author_list": ["Doe J"],
+        "journal": "J. Obs.",
+        "journal_page": "1-2",
+        "journal_volume": "1",
+        "pdb_id": "xxxx",
+        "status": "OBS",
+        "title": "OBSOLETE ENTRY TEST",
+        "superseded_by": "yyyy"
+      }
+    ],
+    "numFound": 1,
+    "start": 0
+  },
+  "responseHeader": {
+    "QTime": 0,
+    "params": {
+      "q": "xxxx",
+      "wt": "json"
+    },
+    "status": 0
+  }
+}
+"""
+        response = mock.Mock(text=json)
+        results = pdbe.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'], 'OBSOLETE ENTRY TEST (OBSOLETE)')
+        self.assertTrue(results[0]['content'].startswith('<em>This entry has been superseded by'))

tests/unit/engines/test_seedpeer.py

@@ -0,0 +1,51 @@
+import mock
+from collections import defaultdict
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+from datetime import datetime
+
+
+class TestSeedPeerEngine(SearxTestCase):
+
+    html = ''
+    with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
+        html += fixture.read()
+
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['pageno'] = 1
+        params = seedpeer.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertIn(query, params['url'])
+        self.assertIn('seedpeer.eu', params['url'])
+
+    def test_response_raises_attr_error_on_empty_response(self):
+        self.assertRaises(AttributeError, seedpeer.response, None)
+        self.assertRaises(AttributeError, seedpeer.response, [])
+        self.assertRaises(AttributeError, seedpeer.response, '')
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+    def test_response_returns_empty_list(self):
+        response = mock.Mock(text='<html></html>')
+        self.assertEqual(seedpeer.response(response), [])
+
+    def test_response_returns_all_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertTrue(isinstance(results, list))
+        self.assertEqual(len(results), 2)
+
+    def test_response_returns_correct_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertEqual(
+            results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
+        )
+        self.assertEqual(
+            results[0]['url'],
+            'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
+        )
+        self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
+        self.assertEqual(results[0]['seed'], '861')
+        self.assertEqual(results[0]['leech'], '332')

tests/unit/test_webapp.py

@@ -44,7 +44,7 @@ class ViewsTestCase(SearxTestCase):
         webapp.Search.search = search_mock
 
         def get_current_theme_name_mock(override=None):
-            return 'default'
+            return 'legacy'
 
         webapp.get_current_theme_name = get_current_theme_name_mock
 
@@ -58,7 +58,7 @@ class ViewsTestCase(SearxTestCase):
     def test_index_html(self):
         result = self.app.post('/', data={'q': 'test'})
         self.assertIn(
-            '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>',  # noqa
+            '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/legacy/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>',  # noqa
             result.data
         )
         self.assertIn(