[fix] replace the dead google images ajax api with a working one

searx/engines/google_images.py

@@ -2,41 +2,42 @@
  Google (Images)
 
  @website     https://www.google.com
- @provide-api yes (https://developers.google.com/web-search/docs/),
+ @provide-api yes (https://developers.google.com/custom-search/)
-              deprecated!
 
  @using-api   yes
- @results     JSON
+ @results     HTML chunk
- @stable      yes (but deprecated)
+ @stable      no
  @parse       url, title, img_src
 """
 
-from urllib import urlencode, unquote
+from urllib import urlencode
+from urlparse import parse_qs
 from json import loads
+from lxml import html
 
 # engine dependent config
 categories = ['images']
 paging = True
 safesearch = True
 
-# search-url
+search_url = 'https://www.google.com/search'\
-url = 'https://ajax.googleapis.com/'
+    '?{query}'\
-search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe={safesearch}&filter=off&{query}'
+    '&tbm=isch'\
+    '&ijn=1'\
+    '&start={offset}'
 
 
 # do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 8
+    offset = (params['pageno'] - 1) * 100
-
-    if params['safesearch'] == 0:
-        safesearch = 'off'
-    else:
-        safesearch = 'on'
 
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       offset=offset,
                                       safesearch=safesearch)
 
+    if safesearch and params['safesearch']:
+        params['url'] += '&' + urlencode({'safe': 'active'})
+
     return params
 
 
@@ -44,30 +45,28 @@ def request(query, params):
 def response(resp):
     results = []
 
-    search_res = loads(resp.text)
+    dom = html.fromstring(resp.text)
-
-    # return empty array if there are no results
-    if not search_res.get('responseData', {}).get('results'):
-        return []
 
     # parse results
-    for result in search_res['responseData']['results']:
+    for result in dom.xpath('//div[@data-ved]'):
-        href = result['originalContextUrl']
+        data_url = result.xpath('./a/@href')[0]
-        title = result['title']
+        data_query = {k: v[0] for k, v in parse_qs(data_url.split('?', 1)[1]).iteritems()}
-        if 'url' not in result:
+
-            continue
+        metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0])
-        thumbnail_src = result['tbUrl']
+
+        thumbnail_src = metadata['tu']
 
         # http to https
         thumbnail_src = thumbnail_src.replace("http://", "https://")
 
         # append result
-        results.append({'url': href,
+        results.append({'url': data_query['imgrefurl'],
-                        'title': title,
+                        'title': metadata['pt'],
-                        'content': result['content'],
+                        'content': metadata['s'],
-                        'thumbnail_src': thumbnail_src,
+                        'thumbnail_src': metadata['tu'],
-                        'img_src': unquote(result['url']),
+                        'img_src': data_query['imgurl'],
                         'template': 'images.html'})
 
     # return results
+    print len(results)
     return results

searx/tests/engines/test_google_images.py

@@ -10,15 +10,15 @@ class TestGoogleImagesEngine(SearxTestCase):
         query = 'test_query'
         dicto = defaultdict(dict)
         dicto['pageno'] = 1
+        dicto['safesearch'] = 1
         params = google_images.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])
-        self.assertIn('googleapis.com', params['url'])
+        self.assertIn('safe=active', params['url'])
-        self.assertIn('safe=on', params['url'])
 
         dicto['safesearch'] = 0
         params = google_images.request(query, dicto)
-        self.assertIn('safe=off', params['url'])
+        self.assertNotIn('safe', params['url'])
 
     def test_response(self):
         self.assertRaises(AttributeError, google_images.response, None)
@@ -26,88 +26,33 @@ class TestGoogleImagesEngine(SearxTestCase):
         self.assertRaises(AttributeError, google_images.response, '')
         self.assertRaises(AttributeError, google_images.response, '[]')
 
-        response = mock.Mock(text='{}')
+        response = mock.Mock(text='<div></div>')
         self.assertEqual(google_images.response(response), [])
 
-        response = mock.Mock(text='{"data": []}')
+        html = """
-        self.assertEqual(google_images.response(response), [])
+<div style="display:none">
-
+  <div eid="fWhnVq4Shqpp3pWo4AM" id="isr_scm_1" style="display:none"></div>
-        json = """
+  <div data-cei="fWhnVq4Shqpp3pWo4AM" class="rg_add_chunk"><!--m-->
-        {
+    <div class="rg_di rg_el ivg-i" data-ved="0ahUKEwjuxPWQts3JAhUGVRoKHd4KCjwQMwgDKAAwAA">
-        "responseData": {
+      <a href="/imgres?imgurl=http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png&amp;imgrefurl=http://www.clker.com/clipart-south-arrow.html&amp;h=598&amp;w=504&amp;tbnid=bQWQ9wz9loJmjM:&amp;docid=vlONkeBtERfDuM&amp;ei=fWhnVq4Shqpp3pWo4AM&amp;tbm=isch" jsaction="fire.ivg_o;mouseover:str.hmov;mouseout:str.hmou" class="rg_l"><img data-src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRsxy3gKnEX0lrwwpRxdPWyLJ8iZ--PXZ-ThbBA2_xXDG_bdQutMQ" data-sz="f" name="bQWQ9wz9loJmjM:" class="rg_i" alt="Image result for south" jsaction="load:str.tbn" onload="google.aft&&google.aft(this)">
-            "results": [
+        <div class="_aOd rg_ilm">
-            {
+          <div class="rg_ilmbg"><span class="rg_ilmn"> 504&nbsp;&#215;&nbsp;598 - clker.com </span>
-                "GsearchResultClass": "GimageSearch",
+          </div>
-                "width": "400",
+        </div>
-                "height": "400",
+      </a>
-                "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
+      <div class="rg_meta">
-                "tbWidth": "124",
+        {"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206}
-                "tbHeight": "124",
+      </div>
-                "unescapedUrl": "http://unescaped.url.jpg",
+    </div><!--n--><!--m-->
-                "url": "http://image.url.jpg",
+  </div>
-                "visibleUrl": "insolitebuzz.fr",
+</div>
-                "title": "This is the title",
+        """  # noqa
-                "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
+        response = mock.Mock(text=html)
-                "originalContextUrl": "http://this.is.the.url",
-                "content": "<b>test</b>",
-                "contentNoFormatting": "test",
-                "tbUrl": "http://thumbnail.url"
-            }
-            ]
-        },
-        "responseDetails": null,
-        "responseStatus": 200
-        }
-        """
-        response = mock.Mock(text=json)
         results = google_images.response(response)
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0]['title'], 'This is the title')
+        self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...')
-        self.assertEqual(results[0]['url'], 'http://this.is.the.url')
+        self.assertEqual(results[0]['url'], 'http://www.clker.com/clipart-south-arrow.html')
-        self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url')
+        self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/')
-        self.assertEqual(results[0]['img_src'], 'http://image.url.jpg')
+        self.assertEqual(results[0]['img_src'], 'http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png')
-        self.assertEqual(results[0]['content'], '<b>test</b>')
+        self.assertEqual(results[0]['content'], 'Download this image as:')
-
-        json = """
-        {
-        "responseData": {
-            "results": [
-            {
-                "GsearchResultClass": "GimageSearch",
-                "width": "400",
-                "height": "400",
-                "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
-                "tbWidth": "124",
-                "tbHeight": "124",
-                "unescapedUrl": "http://unescaped.url.jpg",
-                "visibleUrl": "insolitebuzz.fr",
-                "title": "This is the title",
-                "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
-                "originalContextUrl": "http://this.is.the.url",
-                "content": "<b>test</b>",
-                "contentNoFormatting": "test",
-                "tbUrl": "http://thumbnail.url"
-            }
-            ]
-        },
-        "responseDetails": null,
-        "responseStatus": 200
-        }
-        """
-        response = mock.Mock(text=json)
-        results = google_images.response(response)
-        self.assertEqual(type(results), list)
-        self.assertEqual(len(results), 0)
-
-        json = """
-        {
-        "responseData": {},
-        "responseDetails": null,
-        "responseStatus": 200
-        }
-        """
-        response = mock.Mock(text=json)
-        results = google_images.response(response)
-        self.assertEqual(type(results), list)
-        self.assertEqual(len(results), 0)