From 3182ba7069b2aa2fba1a1b7879bd5f234a4d2868 Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 17:48:07 +0200 Subject: [PATCH] [fix] google news dom xpath fix --- searx/engines/google_news.py | 4 +- tests/unit/engines/test_google_news.py | 62 +++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index 7344b5289..8881d0dad 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -67,8 +67,8 @@ def response(resp): for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'): try: r = { - 'url': result.xpath('.//div[@class="_cnc"]//a/@href')[0], - 'title': ''.join(result.xpath('.//div[@class="_cnc"]//h3//text()')), + 'url': result.xpath('.//a[@class="l _PMs"]')[0].attrib.get("href"), + 'title': ''.join(result.xpath('.//a[@class="l _PMs"]//text()')), 'content': ''.join(result.xpath('.//div[@class="st"]//text()')), } except: diff --git a/tests/unit/engines/test_google_news.py b/tests/unit/engines/test_google_news.py index 6454dde47..20a75af48 100644 --- a/tests/unit/engines/test_google_news.py +++ b/tests/unit/engines/test_google_news.py @@ -37,14 +37,64 @@ class TestGoogleNewsEngine(SearxTestCase): self.assertEqual(google_news.response(response), []) html = u""" -
-
A(z) south témájának képe a következőből: CBC.ca

Meet Thuli Madonsela — South Africa's conscience

CBC.ca-9 órával ezelőtt
South African Public Protector
Who really governs South Africa?
Vélemény-News24-2016. dec. 8.
+

Search Results

+
+
+
+ +
+
+
+

+ Example title +

+
+ + Mac & i + + - + + Mar 21, 2016 +
+
Example description
+
+
+
+
+
+ + Story image for searx from Golem.de + +
+

+ Example title 2 +

+
+ + Golem.de + + - + + Oct 4, 2016 +
+
Example description 2
+
+
+
+
+
+ + """ # noqa response = mock.Mock(text=html) results = google_news.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], u'Meet Thuli Madonsela \u2014 South Africa\'s conscience') - self.assertEqual(results[0]['url'], 'http://this.is.the.url') - self.assertEqual(results[0]['content'], 'South African Public Protector') + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['title'], u'Example title') + self.assertEqual(results[0]['url'], 'https://example.com/') + self.assertEqual(results[0]['content'], 'Example description') + self.assertEqual(results[1]['title'], u'Example title 2') + self.assertEqual(results[1]['url'], 'https://example2.com/') + self.assertEqual(results[1]['content'], 'Example description 2') + self.assertEqual(results[1]['img_src'], 'https://example2.com/image.jpg')