From 3a1c5876b16a51f64505c119283447b06f2a3d99 Mon Sep 17 00:00:00 2001 From: Noemi Vanyi Date: Sat, 13 Aug 2016 14:55:47 +0200 Subject: [PATCH 1/3] add digbt engine Unfortunately, it is quite slow so it is disabled. Furthermore, the display of number of files is wrong on digbt.org, so it is not displayed on searx. --- searx/engines/btdigg.py | 16 ++---------- searx/engines/digbt.py | 58 +++++++++++++++++++++++++++++++++++++++++ searx/settings.yml | 7 ++++- searx/utils.py | 18 +++++++++++++ 4 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 searx/engines/digbt.py diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index c2b22f003..ea6baf1c8 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -16,6 +16,7 @@ from urllib import quote from lxml import html from operator import itemgetter from searx.engines.xpath import extract_text +from searx.utils import get_torrent_size # engine dependent config categories = ['videos', 'music', 'files'] @@ -68,20 +69,7 @@ def response(resp): leech = 0 # convert filesize to byte if possible - try: - filesize = float(filesize) - - # convert filesize to byte - if filesize_multiplier == 'TB': - filesize = int(filesize * 1024 * 1024 * 1024 * 1024) - elif filesize_multiplier == 'GB': - filesize = int(filesize * 1024 * 1024 * 1024) - elif filesize_multiplier == 'MB': - filesize = int(filesize * 1024 * 1024) - elif filesize_multiplier == 'KB': - filesize = int(filesize * 1024) - except: - filesize = None + filesize = get_torrent_size(filesize, filesize_multiplier) # convert files to int if possible if files.isdigit(): diff --git a/searx/engines/digbt.py b/searx/engines/digbt.py new file mode 100644 index 000000000..c35327e8c --- /dev/null +++ b/searx/engines/digbt.py @@ -0,0 +1,58 @@ +""" + DigBT (Videos, Music, Files) + + @website https://digbt.org + @provide-api no + + @using-api no + @results HTML (using search portal) + @stable no (HTML can change) + @parse url, title, content, magnetlink +""" + +from urlparse import urljoin +from lxml import html +from searx.engines.xpath import extract_text +from searx.utils import get_torrent_size + +categories = ['videos', 'music', 'files'] +paging = True + +URL = 'https://digbt.org' +SEARCH_URL = URL + '/search/{query}-time-{pageno}' +FILESIZE = 3 +FILESIZE_MULTIPLIER = 4 + + +def request(query, params): + params['url'] = SEARCH_URL.format(query=query, pageno=params['pageno']) + + return params + + +def response(resp): + dom = html.fromstring(resp.content) + search_res = dom.xpath('.//td[@class="x-item"]') + + if not search_res: + return list() + + results = list() + for result in search_res: + url = urljoin(URL, result.xpath('.//a[@title]/@href')[0]) + title = result.xpath('.//a[@title]/text()')[0] + content = extract_text(result.xpath('.//div[@class="files"]')) + files_data = extract_text(result.xpath('.//div[@class="tail"]')).split() + filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER]) + magnetlink = result.xpath('.//div[@class="tail"]//a[@class="title"]/@href')[0] + + results.append({'url': url, + 'title': title, + 'content': content, + 'filesize': filesize, + 'magnetlink': magnetlink, + 'seed': 'N/A', + 'leech': 'N/A', + 'template': 'torrent.html'}) + + return results diff --git a/searx/settings.yml b/searx/settings.yml index f0d33e50e..885779af3 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -87,7 +87,7 @@ engines: - name : btdigg engine : btdigg shortcut : bt - + - name : crossref engine : json_engine paging : True @@ -118,6 +118,11 @@ engines: weight : 2 disabled : True + - name : digbt + engine : digbt + shortcut : dbt + timeout : 6.0 + - name : digg engine : digg shortcut : dg diff --git a/searx/utils.py b/searx/utils.py index aa8ce92a1..744142e36 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -237,3 +237,21 @@ def list_get(a_list, index, default=None): return a_list[index] else: return default + + +def get_torrent_size(filesize, filesize_multiplier): + try: + filesize = float(filesize) + + if filesize_multiplier == 'TB': + filesize = int(filesize * 1024 * 1024 * 1024 * 1024) + elif filesize_multiplier == 'GB': + filesize = int(filesize * 1024 * 1024 * 1024) + elif filesize_multiplier == 'MB': + filesize = int(filesize * 1024 * 1024) + elif filesize_multiplier == 'KB': + filesize = int(filesize * 1024) + except: + filesize = None + + return filesize From 6dd5f7a8c91eda450246e002010f579b9c6e59d8 Mon Sep 17 00:00:00 2001 From: Noemi Vanyi Date: Sat, 13 Aug 2016 16:16:54 +0200 Subject: [PATCH 2/3] add digbt unittest --- tests/unit/engines/test_digbt.py | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/unit/engines/test_digbt.py diff --git a/tests/unit/engines/test_digbt.py b/tests/unit/engines/test_digbt.py new file mode 100644 index 000000000..867188ed9 --- /dev/null +++ b/tests/unit/engines/test_digbt.py @@ -0,0 +1,59 @@ +from collections import defaultdict +import mock +from searx.engines import digbt +from searx.testing import SearxTestCase + + +class TestDigBTEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = digbt.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('digbt.org', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, digbt.response, None) + self.assertRaises(AttributeError, digbt.response, []) + self.assertRaises(AttributeError, digbt.response, '') + self.assertRaises(AttributeError, digbt.response, '[]') + + response = mock.Mock(content='') + self.assertEqual(digbt.response(response), []) + + html = """ + + +
+
+ The Big Bang Theory + 4 hours ago +
+
+
    +
  • The Big Bang Theory 2.9 GB
    • +
  • ....
    • +
+
+
+ Files: 1 Size: 2.9 GB Downloads: 1 Updated: 4 hours ago +     + + magnet-link + +     +
+
+ """ + response = mock.Mock(content=html) + results = digbt.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'The Big Bang Theory') + self.assertEqual(results[0]['url'], 'https://digbt.org/The-Big-Bang-Theory-d2.html') + self.assertEqual(results[0]['content'], 'The Big Bang Theory 2.9 GB ....') + self.assertEqual(results[0]['filesize'], 3113851289) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:a&dn=The+Big+Bang+Theory') From d320dd0efc44f178da5156fa152b8337f2e11281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Sun, 14 Aug 2016 01:05:21 +0200 Subject: [PATCH 3/3] diable digbt engine in settings.yml --- searx/settings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/searx/settings.yml b/searx/settings.yml index 885779af3..65995c07e 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -122,6 +122,7 @@ engines: engine : digbt shortcut : dbt timeout : 6.0 + disabled : True - name : digg engine : digg