Merge pull request #1754 from MarcAbonce/seedpeer

Add Seedpeer again

searx/engines/seedpeer.py

@@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     https://seedpeer.me
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from lxml import html
+from json import loads
+from operator import itemgetter
+from searx.url_utils import quote, urljoin
+from searx.engines.xpath import extract_text
+
+
+url = 'https://seedpeer.me/'
+search_url = url + 'search/{search_term}?page={page_no}'
+torrent_file_url = url + 'torrent/{torrent_hash}'
+
+# specific xpath variables
+script_xpath = '//script[@type="text/javascript"][not(@src)]'
+torrent_xpath = '(//table)[2]/tbody/tr'
+link_xpath = '(./td)[1]/a/@href'
+age_xpath = '(./td)[2]'
+size_xpath = '(./td)[3]'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'])
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    result_rows = dom.xpath(torrent_xpath)
+
+    try:
+        script_element = dom.xpath(script_xpath)[0]
+        json_string = script_element.text[script_element.text.find('{'):]
+        torrents_json = loads(json_string)
+    except:
+        return []
+
+    # parse results
+    for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']):
+        title = torrent_json['name']
+        seed = int(torrent_json['seeds'])
+        leech = int(torrent_json['peers'])
+        size = int(torrent_json['size'])
+        torrent_hash = torrent_json['hash']
+
+        torrentfile = torrent_file_url.format(torrent_hash=torrent_hash)
+        magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash)
+
+        age = extract_text(torrent_row.xpath(age_xpath))
+        link = torrent_row.xpath(link_xpath)[0]
+
+        href = urljoin(url, link)
+
+        # append result
+        results.append({'url': href,
+                        'title': title,
+                        'content': age,
+                        'seed': seed,
+                        'leech': leech,
+                        'filesize': size,
+                        'torrentfile': torrentfile,
+                        'magnetlink': magnetlink,
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/settings.yml

@@ -748,6 +748,11 @@ engines:
     page_size : 10
     disabled : True
 
+  - name : seedpeer
+    shortcut : speu
+    engine : seedpeer
+    categories: files, music, videos
+
 #  - name : yacy
 #    engine : yacy
 #    shortcut : ya

searx/templates/courgette/result_templates/torrent.html

@@ -4,7 +4,7 @@
     {% endif %}
     <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>
     {% if result.content %}<span class="content">{{ result.content|safe }}</span><br />{% endif %}
-    {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}
+    {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}
     <span>
         {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} 
         {% if result.torrentfile %}<a href="{{ result.torrentfile }}" class="torrentfile" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('torrent file') }}</a>{% endif %}

searx/templates/legacy/result_templates/torrent.html

@@ -8,6 +8,6 @@
     <p>
         {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} 
         {% if result.torrentfile %}<a href="{{ result.torrentfile }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="torrentfile">{{ _('torrent file') }}</a>{% endif %} - 
-        {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} 
+        {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} 
     </p>
 </div>

searx/templates/oscar/result_templates/torrent.html

@@ -3,7 +3,7 @@
 {{ result_header(result, favicons) }}
 {{ result_sub_header(result) }}
 
-{% if result.seed %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}
+{% if result.seed is defined %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}
 {% if result.filesize %}<br />{{ icon('floppy-disk') }} {{ _('Filesize') }} 
     <span class="badge">
         {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/templates/simple/result_templates/torrent.html

@@ -6,7 +6,7 @@
 {% if result.magnetlink %}<p class="altlink"> &bull; {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</p>{% endif %}
 {% if result.torrentfile %}<p class="altlink"> &bull; {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</p>{% endif %}
 
-{% if result.seed %}<p class="stat"> &bull; {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}
+{% if result.seed is defined %}<p class="stat"> &bull; {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}
 
 {%- if result.filesize %}<p class="stat">{{ icon('floppy-disk') }} {{ _('Filesize') }}<span class="badge">
     {%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

tests/unit/engines/test_seedpeer.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from collections import defaultdict
+import mock
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+
+
+class TestBtdiggEngine(SearxTestCase):
+
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['pageno'] = 1
+        params = seedpeer.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertIn(query, params['url'])
+        self.assertIn('seedpeer', params['url'])
+
+    def test_response(self):
+        self.assertRaises(AttributeError, seedpeer.response, None)
+        self.assertRaises(AttributeError, seedpeer.response, [])
+        self.assertRaises(AttributeError, seedpeer.response, '')
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+        response = mock.Mock(text='<html></html>')
+        self.assertEqual(seedpeer.response(response), [])
+
+        html = u"""
+        <html>
+        <head>
+            <script></script>
+            <script type="text/javascript" src="not_here.js"></script>
+            <script type="text/javascript">
+                window.initialData=
+                {"data": {"list": [{"name": "Title", "seeds": "10", "peers": "20", "size": "1024", "hash": "abc123"}]}}
+            </script>
+        </head>
+        <body>
+            <table></table>
+            <table>
+                <thead><tr></tr></thead>
+                <tbody>
+                    <tr>
+                        <td><a href="link">Title</a></td>
+                        <td>1 year</td>
+                        <td>1 KB</td>
+                        <td>10</td>
+                        <td>20</td>
+                        <td></td>
+                    </tr>
+                </tbody>
+            </table>
+        </body>
+        </html>
+        """
+        response = mock.Mock(text=html)
+        results = seedpeer.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'], 'Title')
+        self.assertEqual(results[0]['url'], 'https://seedpeer.me/link')
+        self.assertEqual(results[0]['seed'], 10)
+        self.assertEqual(results[0]['leech'], 20)
+        self.assertEqual(results[0]['filesize'], 1024)
+        self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123')
+        self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123')