Merge pull request #170 from pointhi/little_fixes

Little fixes, add searx-autocompletion, more informations about torrents

searx/autocomplete.py

@@ -20,6 +20,85 @@ from lxml import etree
 from requests import get
 from json import loads
 from urllib import urlencode
+from searx.languages import language_codes
+from searx.engines import (
+    categories, engines, engine_shortcuts
+)
+
+
+def searx_bang(full_query):
+    '''check if the searchQuery contain a bang, and create fitting autocompleter results'''
+    # check if there is a query which can be parsed
+    if len(full_query.getSearchQuery()) == 0:
+        return []
+
+    results = []
+
+    # check if current query stats with !bang
+    if full_query.getSearchQuery()[0] == '!':
+        if len(full_query.getSearchQuery()) == 1:
+            # show some example queries
+            # TODO, check if engine is not avaliable
+            results.append("!images")
+            results.append("!wikipedia")
+            results.append("!osm")
+        else:
+            engine_query = full_query.getSearchQuery()[1:]
+
+            # check if query starts with categorie name
+            for categorie in categories:
+                if categorie.startswith(engine_query):
+                    results.append('!{categorie}'.format(categorie=categorie))
+
+            # check if query starts with engine name
+            for engine in engines:
+                if engine.startswith(engine_query.replace('_', ' ')):
+                    results.append('!{engine}'.format(engine=engine.replace(' ', '_')))
+
+            # check if query starts with engine shortcut
+            for engine_shortcut in engine_shortcuts:
+                if engine_shortcut.startswith(engine_query):
+                    results.append('!{engine_shortcut}'.format(engine_shortcut=engine_shortcut))
+
+    # check if current query stats with :bang
+    elif full_query.getSearchQuery()[0] == ':':
+        if len(full_query.getSearchQuery()) == 1:
+            # show some example queries
+            results.append(":en")
+            results.append(":en_us")
+            results.append(":english")
+            results.append(":united_kingdom")
+        else:
+            engine_query = full_query.getSearchQuery()[1:]
+
+            for lc in language_codes:
+                lang_id, lang_name, country = map(str.lower, lc)
+
+                # check if query starts with language-id
+                if lang_id.startswith(engine_query):
+                    if len(engine_query) <= 2:
+                        results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0]))
+                    else:
+                        results.append(':{lang_id}'.format(lang_id=lang_id))
+
+                # check if query starts with language name
+                if lang_name.startswith(engine_query):
+                    results.append(':{lang_name}'.format(lang_name=lang_name))
+
+                # check if query starts with country
+                if country.startswith(engine_query.replace('_', ' ')):
+                    results.append(':{country}'.format(country=country.replace(' ', '_')))
+
+    # remove duplicates
+    result_set = set(results)
+
+    # remove results which are already contained in the query
+    for query_part in full_query.query_parts:
+        if query_part in result_set:
+            result_set.remove(query_part)
+
+    # convert result_set back to list
+    return list(result_set)
 
 
 def dbpedia(query):

searx/engines/kickass.py

@@ -24,6 +24,7 @@ search_url = url + 'search/{search_term}/{pageno}/'
 
 # specific xpath variables
 magnet_xpath = './/a[@title="Torrent magnet link"]'
+torrent_xpath = './/a[@title="Download torrent file"]'
 content_xpath = './/span[@class="font11px lightgrey block"]'
 
 
@@ -60,6 +61,9 @@ def response(resp):
                                        method="text"))
         seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
         leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
+        filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
+        filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
+        files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
 
         # convert seed to int if possible
         if seed.isdigit():
@@ -73,15 +77,42 @@ def response(resp):
         else:
             leech = 0
 
+        # convert filesize to byte if possible
+        try:
+            filesize = float(filesize)
+
+            # convert filesize to byte
+            if filesize_multiplier == 'TB':
+                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
+            elif filesize_multiplier == 'GB':
+                filesize = int(filesize * 1024 * 1024 * 1024)
+            elif filesize_multiplier == 'MB':
+                filesize = int(filesize * 1024 * 1024)
+            elif filesize_multiplier == 'kb':
+                filesize = int(filesize * 1024)
+        except:
+            filesize = None
+
+        # convert files to int if possible
+        if files.isdigit():
+            files = int(files)
+        else:
+            files = None
+
         magnetlink = result.xpath(magnet_xpath)[0].attrib['href']
 
+        torrentfile = result.xpath(torrent_xpath)[0].attrib['href']
+
         # append result
         results.append({'url': href,
                         'title': title,
                         'content': content,
                         'seed': seed,
                         'leech': leech,
+                        'filesize': filesize,
+                        'files': files,
                         'magnetlink': magnetlink,
+                        'torrentfile': torrentfile,
                         'template': 'torrent.html'})
 
     # return results sorted by seeder

searx/query.py

@@ -77,7 +77,7 @@ class Query(object):
                     if lang == lang_id\
                        or lang_id.startswith(lang)\
                        or lang == lang_name\
-                       or lang == country:
+                       or lang.replace('_', ' ') == country:
                         parse_next = True
                         self.languages.append(lang)
                         break

searx/templates/oscar/base.html

@@ -7,6 +7,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="generator" content="searx/{{ searx_version }}">
     <meta name="viewport" content="width=device-width, initial-scale=1 , maximum-scale=1.0, user-scalable=1" />
+    {% block meta %}{% endblock %}
     <title>{% block title %}{% endblock %}searx</title>
     
     <link rel="stylesheet" href="{{ url_for('static', filename='css/bootstrap.min.css') }}" type="text/css" />

searx/templates/oscar/result_templates/default.html

@@ -11,9 +11,7 @@
 
 {% if result.embedded %}
 <div id="result-media-{{ index }}" class="collapse">
-{% autoescape false %}
-   {{ result.embedded }}
-{% endautoescape %}
+   {{ result.embedded|safe }}
 </div>
 {% endif %}
 

searx/templates/oscar/result_templates/torrent.html

@@ -5,9 +5,20 @@
 {% if result.publishedDate %}<time class="text-muted" datetime="{{ result.pubdate }}" >{{ result.publishedDate }}</time>{% endif %}
 <small><a class="text-info" href="https://web.archive.org/web/{{ result.url }}">{{ icon('link') }} {{ _('cached') }}</a></small>
 
-<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span>, {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>
-<br/>
-<a href="{{ result.magnetlink }}" class="magnetlink">{{ icon('magnet') }} magnet link</a></p>
+<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>
+{% if result.filesize %}</br>{{ icon('floppy-disk') }} {{ _('Filesize') }} 
+    <span class="badge">
+        {% if result.filesize < 1024 %}{{ result.filesize }} Byte
+        {% elif result.filesize < 1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024) }} kb
+        {% elif result.filesize < 1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024) }} MB
+        {% elif result.filesize < 1024*1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024) }} GB
+        {% else %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024/1024) }} TB{% endif %}
+    </span>{% endif %}
+{% if result.files %}</br>{{ icon('file') }} {{ _('Number of Files') }} <span class="badge">{{ result.files }}</span>{% endif %}</p>
+<p class="result-content">
+    <a href="{{ result.magnetlink }}" class="magnetlink">{{ icon('magnet') }} {{ _('magnet link') }}</a>
+    {% if result.torrentfile %}</br><a href="{{ result.torrentfile }}" class="torrentfile">{{ icon('download-alt') }} {{ _('torrent file') }}</a>{% endif %}
+</p>
 
 {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}
 

searx/templates/oscar/result_templates/videos.html

@@ -11,9 +11,7 @@
 
 {% if result.embedded %}
 <div id="result-video-{{ index }}" class="collapse">
-{% autoescape false %}
-   {{ result.embedded }}
-{% endautoescape %}
+   {{ result.embedded|safe }}
 </div>
 {% endif %}
 

searx/templates/oscar/results.html

@@ -1,5 +1,6 @@
 {% extends "oscar/base.html" %}
 {% block title %}{{ q }} - {% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno+1 }}">{% endblock %}
 {% block content %}
     <div class="row">
         <div class="col-sm-8" id="main_results">

searx/webapp.py

@@ -46,7 +46,7 @@ from searx.languages import language_codes
 from searx.https_rewrite import https_url_rewrite
 from searx.search import Search
 from searx.query import Query
-from searx.autocomplete import backends as autocomplete_backends
+from searx.autocomplete import searx_bang, backends as autocomplete_backends
 from searx import logger
 
 
@@ -352,8 +352,13 @@ def autocompleter():
     if not completer:
         return '', 400
 
-    # run autocompletion
-    raw_results = completer(query.getSearchQuery())
+    # parse searx specific autocompleter results like !bang
+    raw_results = searx_bang(query)
+
+    # normal autocompletion results only appear if max 3. searx results returned
+    if len(raw_results) <= 3:
+        # run autocompletion
+        raw_results.extend(completer(query.getSearchQuery()))
 
     # parse results (write :language and !engine back to result string)
     results = []