From b538de568a6215082275fd3738555ea605a8cc6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Revol?= Date: Sat, 9 Jul 2016 17:44:27 +0200 Subject: [PATCH] Add INA search engine --- searx/engines/ina.py | 83 ++++++++++++++++++++++++++++++++++ searx/settings.yml | 6 +++ tests/unit/engines/test_ina.py | 64 ++++++++++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 searx/engines/ina.py create mode 100644 tests/unit/engines/test_ina.py diff --git a/searx/engines/ina.py b/searx/engines/ina.py new file mode 100644 index 000000000..86a39782b --- /dev/null +++ b/searx/engines/ina.py @@ -0,0 +1,83 @@ +# INA (Videos) +# +# @website https://www.ina.fr/ +# @provide-api no +# +# @using-api no +# @results HTML (using search portal) +# @stable no (HTML can change) +# @parse url, title, content, publishedDate, thumbnail +# +# @todo set content-parameter with correct data +# @todo embedded (needs some md5 from video page) + +from json import loads +from urllib import urlencode +from lxml import html +from HTMLParser import HTMLParser +from searx.engines.xpath import extract_text +from dateutil import parser + +# engine dependent config +categories = ['videos'] +paging = True +page_size = 48 + +# search-url +base_url = 'https://www.ina.fr' +search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}' + +# specific xpath variables +results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]' +url_xpath = './/a/@href' +title_xpath = './/h3[@class="h3--title media-heading"]' +thumbnail_xpath = './/img/@src' +publishedDate_xpath = './/span[@class="broadcast"]' +content_xpath = './/p[@class="media-body__summary"]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(ps=page_size, + start=params['pageno'] * page_size, + query=urlencode({'q': query})) + + return params + + +# get response from search-request +def response(resp): + results = [] + + # we get html in a JSON container... + response = loads(resp.text) + if "content" not in response: + return [] + dom = html.fromstring(response["content"]) + p = HTMLParser() + + # parse results + for result in dom.xpath(results_xpath): + videoid = result.xpath(url_xpath)[0] + url = base_url + videoid + title = p.unescape(extract_text(result.xpath(title_xpath))) + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) + if thumbnail[0] == '/': + thumbnail = base_url + thumbnail + d = extract_text(result.xpath(publishedDate_xpath)[0]) + d = d.split('/') + # force ISO date to avoid wrong parsing + d = "%s-%s-%s" % (d[2], d[1], d[0]) + publishedDate = parser.parse(d) + content = extract_text(result.xpath(content_xpath)) + + # append result + results.append({'url': url, + 'title': title, + 'content': content, + 'template': 'videos.html', + 'publishedDate': publishedDate, + 'thumbnail': thumbnail}) + + # return results + return results diff --git a/searx/settings.yml b/searx/settings.yml index 93087568c..558898886 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -254,6 +254,12 @@ engines: disabled : True shortcut : habr + - name : ina + engine : ina + shortcut : in + timeout : 6.0 + disabled : True + - name : mixcloud engine : mixcloud shortcut : mc diff --git a/tests/unit/engines/test_ina.py b/tests/unit/engines/test_ina.py new file mode 100644 index 000000000..2c87a0ff8 --- /dev/null +++ b/tests/unit/engines/test_ina.py @@ -0,0 +1,64 @@ +from collections import defaultdict +import mock +from searx.engines import ina +from searx.testing import SearxTestCase + + +class TestInaEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = ina.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('ina.fr' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, ina.response, None) + self.assertRaises(AttributeError, ina.response, []) + self.assertRaises(AttributeError, ina.response, '') + self.assertRaises(AttributeError, ina.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(ina.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(ina.response(response), []) + + json = """ + {"content":"\\t
\\n\\t\\n\ + \\n
\\n\ +
\\n\ + \\t\\t\\t\\t\\n\ + \\"Conf\\u00e9rence\\n\ + \\t\\t\\t\\t\\t<\/a>\\n\ + \\t\\t\\t\\t\\t
\\n\\t\\t\\t\\t\\t\\t

\\n\ + \\t\\t\\t\\t\\t\\t\\t\ + Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle <\/a>\\n\ + <\/h3>\\n\ +
\\n27\/11\/1967<\/span>\\n\ + 29321 vues<\/span>\\n\ + 01h 33m 07s<\/span>\\n\ + <\/div>\\n\ +

VERSION INTEGRALE DE LA CONFERENCE DE PRESSE DU GENERAL DE GAULLE . \ + - PA le Pr\\u00e9sident DE GAULLE : il ouvre les bras et s'assied. DP journalis...<\/p>\\n\ + <\/div>\\n<\/div>\\n" + } + """ + response = mock.Mock(text=json) + results = ina.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], u'Conf\xe9rence de presse du G\xe9n\xe9ral de Gaulle') + self.assertEqual(results[0]['url'], + 'https://www.ina.fr/video/CAF89035682/conference-de-presse-du-general-de-gaulle-video.html') + self.assertEqual(results[0]['content'], + u"VERSION INTEGRALE DE LA CONFERENCE DE PRESSE DU GENERAL DE GAULLE ." + u" - PA le Pr\u00e9sident DE GAULLE : il ouvre les bras et s'assied. DP journalis...")