[enh] Add onions category with Ahmia, Not Evil and Torch

Xpath engine and results template changed to account for the fact that
archive.org doesn't cache .onions, though some onion engines migth have
their own cache.

Disabled by default. Can be enabled by setting the SOCKS proxies to
wherever Tor is listening and setting using_tor_proxy as True.

Requires Tor and updating packages.

To avoid manually adding the timeout on each engine, you can set
extra_proxy_timeout to account for Tor's (or whatever proxy used) extra
time.

searx/engines/__init__.py

@@ -142,6 +142,17 @@ def load_engine(engine_data):
         engine.stats['page_load_time'] = 0
         engine.stats['page_load_count'] = 0
 
+    # tor related settings
+    if settings['outgoing'].get('using_tor_proxy'):
+        # use onion url if using tor.
+        if hasattr(engine, 'onion_url'):
+            engine.search_url = engine.onion_url + getattr(engine, 'search_path', '')
+    elif 'onions' in engine.categories:
+        # exclude onion engines if not using tor.
+        return None
+
+    engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)
+
     for category_name in engine.categories:
         categories.setdefault(category_name, []).append(engine)
 
@@ -252,8 +263,9 @@ def get_engines_stats(preferences):
 
 
 def load_engines(engine_list):
-    global engines
+    global engines, engine_shortcuts
     engines.clear()
+    engine_shortcuts.clear()
     for engine_data in engine_list:
         engine = load_engine(engine_data)
         if engine is not None:

searx/engines/ahmia.py

@@ -0,0 +1,82 @@
+"""
+ Ahmia (Onions)
+
+ @website      http://msydqstlz2kzerdg.onion
+ @provides-api no
+
+ @using-api    no
+ @results      HTML
+ @stable       no
+ @parse        url, title, content
+"""
+
+from urllib.parse import urlencode, urlparse, parse_qs
+from lxml.html import fromstring
+from searx.engines.xpath import extract_url, extract_text
+
+# engine config
+categories = ['onions']
+paging = True
+page_size = 10
+
+# search url
+search_url = 'http://msydqstlz2kzerdg.onion/search/?{query}'
+time_range_support = True
+time_range_dict = {'day': 1,
+                   'week': 7,
+                   'month': 30}
+
+# xpaths
+results_xpath = '//li[@class="result"]'
+url_xpath = './h4/a/@href'
+title_xpath = './h4/a[1]'
+content_xpath = './/p[1]'
+correction_xpath = '//*[@id="didYouMean"]//a'
+number_of_results_xpath = '//*[@id="totalResults"]'
+
+
+def request(query, params):
+    params['url'] = search_url.format(query=urlencode({'q': query}))
+
+    if params['time_range'] in time_range_dict:
+        params['url'] += '&' + urlencode({'d': time_range_dict[params['time_range']]})
+
+    return params
+
+
+def response(resp):
+    results = []
+    dom = fromstring(resp.text)
+
+    # trim results so there's not way too many at once
+    first_result_index = page_size * (resp.search_params.get('pageno', 1) - 1)
+    all_results = dom.xpath(results_xpath)
+    trimmed_results = all_results[first_result_index:first_result_index + page_size]
+
+    # get results
+    for result in trimmed_results:
+        # remove ahmia url and extract the actual url for the result
+        raw_url = extract_url(result.xpath(url_xpath), search_url)
+        cleaned_url = parse_qs(urlparse(raw_url).query).get('redirect_url', [''])[0]
+
+        title = extract_text(result.xpath(title_xpath))
+        content = extract_text(result.xpath(content_xpath))
+
+        results.append({'url': cleaned_url,
+                        'title': title,
+                        'content': content,
+                        'is_onion': True})
+
+    # get spelling corrections
+    for correction in dom.xpath(correction_xpath):
+        results.append({'correction': extract_text(correction)})
+
+    # get number of results
+    number_of_results = dom.xpath(number_of_results_xpath)
+    if number_of_results:
+        try:
+            results.append({'number_of_results': int(extract_text(number_of_results))})
+        except:
+            pass
+
+    return results

searx/engines/not_evil.py

@@ -0,0 +1,64 @@
+"""
+ not Evil (Onions)
+
+ @website     http://hss3uro2hsxfogfq.onion
+ @provide-api yes (http://hss3uro2hsxfogfq.onion/api.htm)
+
+ @using-api   no
+ @results     HTML
+ @stable      no
+ @parse       url, title, content
+"""
+
+from urllib.parse import urlencode
+from lxml import html
+from searx.engines.xpath import extract_text
+
+# engine dependent config
+categories = ['onions']
+paging = True
+page_size = 20
+
+# search-url
+base_url = 'http://hss3uro2hsxfogfq.onion/'
+search_url = 'index.php?{query}&hostLimit=20&start={pageno}&numRows={page_size}'
+
+# specific xpath variables
+results_xpath = '//*[@id="content"]/div/p'
+url_xpath = './span[1]'
+title_xpath = './a[1]'
+content_xpath = './text()'
+
+
+# do search-request
+def request(query, params):
+    offset = (params['pageno'] - 1) * page_size
+
+    params['url'] = base_url + search_url.format(pageno=offset,
+                                                 query=urlencode({'q': query}),
+                                                 page_size=page_size)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    # needed because otherwise requests guesses wrong encoding
+    resp.encoding = 'utf8'
+    dom = html.fromstring(resp.text)
+
+    # parse results
+    for result in dom.xpath(results_xpath):
+        url = extract_text(result.xpath(url_xpath)[0])
+        title = extract_text(result.xpath(title_xpath)[0])
+        content = extract_text(result.xpath(content_xpath))
+
+        # append result
+        results.append({'url': url,
+                        'title': title,
+                        'content': content,
+                        'is_onion': True})
+
+    return results

searx/engines/xpath.py

@@ -10,6 +10,8 @@ thumbnail_xpath = False
 paging = False
 suggestion_xpath = ''
 results_xpath = ''
+cached_xpath = ''
+cached_url = ''
 
 # parameters for engines with paging support
 #
@@ -36,6 +38,8 @@ def request(query, params):
 def response(resp):
     results = []
     dom = html.fromstring(resp.text)
+    is_onion = True if 'onions' in categories else False
+
     if results_xpath:
         for result in eval_xpath(dom, results_xpath):
             url = extract_url(eval_xpath(result, url_xpath), search_url)
@@ -49,15 +53,33 @@ def response(resp):
                 if len(thumbnail_xpath_result) > 0:
                     tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
 
+            # add alternative cached url if available
+            if cached_xpath:
+                tmp_result['cached_url'] = cached_url + extract_text(result.xpath(cached_xpath))
+
+            if is_onion:
+                tmp_result['is_onion'] = True
+
             results.append(tmp_result)
     else:
-        for url, title, content in zip(
-            (extract_url(x, search_url) for
-             x in eval_xpath(dom, url_xpath)),
-            map(extract_text, eval_xpath(dom, title_xpath)),
-            map(extract_text, eval_xpath(dom, content_xpath))
-        ):
-            results.append({'url': url, 'title': title, 'content': content})
+        if cached_xpath:
+            for url, title, content, cached in zip(
+                (extract_url(x, search_url) for
+                 x in dom.xpath(url_xpath)),
+                map(extract_text, dom.xpath(title_xpath)),
+                map(extract_text, dom.xpath(content_xpath)),
+                map(extract_text, dom.xpath(cached_xpath))
+            ):
+                results.append({'url': url, 'title': title, 'content': content,
+                                'cached_url': cached_url + cached, 'is_onion': is_onion})
+        else:
+            for url, title, content in zip(
+                (extract_url(x, search_url) for
+                 x in dom.xpath(url_xpath)),
+                map(extract_text, dom.xpath(title_xpath)),
+                map(extract_text, dom.xpath(content_xpath))
+            ):
+                results.append({'url': url, 'title': title, 'content': content, 'is_onion': is_onion})
 
     if not suggestion_xpath:
         return results