From 44ce51f0c53139b109f7d00662780fa5def99df8 Mon Sep 17 00:00:00 2001 From: Michael Pfitzner Date: Fri, 14 Dec 2018 21:08:04 +0100 Subject: [PATCH 1/4] restore startpage search results --- searx/engines/startpage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 3e067597e..166c11f83 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -32,8 +32,9 @@ search_url = base_url + 'do/search' # specific xpath variables # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"] # not ads: div[@class="result"] are the direct childs of div[@id="results"] -results_xpath = '//div[@class="result"]' +results_xpath = '//li[contains(@class, "search-result") and contains(@class, "search-item")]' link_xpath = './/h3/a' +content_xpath = './p[@class="search-item__body"]' # do search-request @@ -79,8 +80,8 @@ def response(resp): title = extract_text(link) - if result.xpath('./p[@class="desc clk"]'): - content = extract_text(result.xpath('./p[@class="desc clk"]')) + if result.xpath(content_xpath): + content = extract_text(result.xpath(content_xpath)) else: content = '' From 5a20d9ecd198068fbacc38876afb4f0d9a8c0aa8 Mon Sep 17 00:00:00 2001 From: Michael Pfitzner Date: Fri, 14 Dec 2018 22:18:21 +0100 Subject: [PATCH 2/4] change the html in test_startpage.py to look more like the actual startpage site --- tests/unit/engines/test_startpage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/engines/test_startpage.py b/tests/unit/engines/test_startpage.py index bdb3356b7..4a7ca5b6f 100644 --- a/tests/unit/engines/test_startpage.py +++ b/tests/unit/engines/test_startpage.py @@ -31,14 +31,14 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(startpage.response(response), []) html = """ -
+
  • This should be the title

    -

    +

    This should be the content.

    @@ -56,7 +56,7 @@ class TestStartpageEngine(SearxTestCase): Mis en surbrillance

    -
  • + """ response = mock.Mock(text=html.encode('utf-8')) results = startpage.response(response) From 83a06d276de3cbaa0dabc9eca990e1298140898a Mon Sep 17 00:00:00 2001 From: Michael Pfitzner Date: Fri, 14 Dec 2018 22:29:57 +0100 Subject: [PATCH 3/4] change rest of test_startpage.py --- tests/unit/engines/test_startpage.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/engines/test_startpage.py b/tests/unit/engines/test_startpage.py index 4a7ca5b6f..4a8a443dd 100644 --- a/tests/unit/engines/test_startpage.py +++ b/tests/unit/engines/test_startpage.py @@ -67,14 +67,14 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(results[0]['content'], 'This should be the content.') html = """ -
    +
  • This should be the title

    -

    +

    This should be the content.

    @@ -92,20 +92,20 @@ class TestStartpageEngine(SearxTestCase): Mis en surbrillance

    -
  • -
    + +
  • -

    +

    This should be the content.

    www.speedtest.net/fr/

    -
  • - + """ response = mock.Mock(text=html.encode('utf-8')) results = startpage.response(response) From 73ec2ba74ec735bb5e989cab0336ecca24a93701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Bourrel?= Date: Wed, 19 Dec 2018 11:12:25 +0100 Subject: [PATCH 4/4] Update bing image engine with new image json format --- searx/engines/bing_images.py | 4 +--- tests/unit/engines/test_bing_images.py | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index 66e14c01f..876011f1d 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -88,9 +88,7 @@ def response(resp): url = json_data.get('purl') img_src = json_data.get('murl') - - thumb_json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('mad'))) - thumbnail = thumb_json_data.get('turl') + thumbnail = json_data.get('turl') # append result results.append({'template': 'images.html', diff --git a/tests/unit/engines/test_bing_images.py b/tests/unit/engines/test_bing_images.py index afc4cd6f0..cb1d550d9 100644 --- a/tests/unit/engines/test_bing_images.py +++ b/tests/unit/engines/test_bing_images.py @@ -52,7 +52,7 @@ class TestBingImagesEngine(SearxTestCase):
  • @@ -60,7 +60,7 @@ class TestBingImagesEngine(SearxTestCase):
    @@ -71,7 +71,7 @@ class TestBingImagesEngine(SearxTestCase):