[enh] flickr_noapi: use complete JSON data block, add 'content', 'img_format', 'source', etc. (#1571)

Fetch complete JSON data block, use legend to extract images. 
Unquote urlencoded strings.
Add image description as 'content'. 
Add 'img_format' and 'source' data (needs PR #1567 to enable this data to be displayed). 
Show images which lack ownerid instead of discarding them.

searx/engines/flickr_noapi.py

@@ -16,8 +16,7 @@ from json import loads
 from time import time
 import re
 from searx.engines import logger
-from searx.url_utils import urlencode
+from searx.url_utils import urlencode, unquote
-
 
 logger = logger.getChild('flickr-noapi')
 
@@ -27,7 +26,7 @@ url = 'https://www.flickr.com/'
 search_url = url + 'search?{query}&page={page}'
 time_range_url = '&min_upload_date={start}&max_upload_date={end}'
 photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
-regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
+modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
 image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
 
 paging = True
@@ -57,40 +56,45 @@ def request(query, params):
 def response(resp):
     results = []
 
-    matches = regex.search(resp.text)
+    matches = modelexport_re.search(resp.text)
 
     if matches is None:
         return results
 
     match = matches.group(1)
-    search_results = loads(match)
+    model_export = loads(match)
 
-    if '_data' not in search_results:
+    if 'legend' not in model_export:
-        return []
+        return results
 
-    photos = search_results['_data']
+    legend = model_export['legend']
 
-    for photo in photos:
+    # handle empty page
+    if not legend or not legend[0]:
+        return results
 
-        # In paged configuration, the first pages' photos
+    for index in legend:
-        # are represented by a None object
+        photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])]
-        if photo is None:
+        author = unquote(photo.get('realname', ''))
-            continue
+        source = unquote(photo.get('username', '')) + ' @ Flickr'
+        title = unquote(photo.get('title', ''))
+        content = unquote(photo.get('description', ''))
 
         img_src = None
         # From the biggest to the lowest format
         for image_size in image_sizes:
             if image_size in photo['sizes']:
                 img_src = photo['sizes'][image_size]['url']
+                img_format = 'jpg ' \
+                    + str(photo['sizes'][image_size]['width']) \
+                    + 'x' \
+                    + str(photo['sizes'][image_size]['height'])
                 break
 
         if not img_src:
             logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
             continue
 
-        if 'ownerNsid' not in photo:
-            continue
-
         # For a bigger thumbnail, keep only the url_z, not the url_n
         if 'n' in photo['sizes']:
             thumbnail_src = photo['sizes']['n']['url']
@@ -99,19 +103,20 @@ def response(resp):
         else:
             thumbnail_src = img_src
 
+        if 'ownerNsid' not in photo:
+            # should not happen, disowned photo? Show it anyway
+            url = img_src
+        else:
             url = build_flickr_url(photo['ownerNsid'], photo['id'])
 
-        title = photo.get('title', '')
-
-        author = photo['username']
-
-        # append result
         results.append({'url': url,
                         'title': title,
                         'img_src': img_src,
                         'thumbnail_src': thumbnail_src,
-                        'content': '',
+                        'content': content,
                         'author': author,
+                        'source': source,
+                        'img_format': img_format,
                         'template': 'images.html'})
 
     return results

tests/unit/engines/test_flickr_noapi.py

@@ -27,20 +27,33 @@ class TestFlickrNoapiEngine(SearxTestCase):
         self.assertRaises(AttributeError, flickr_noapi.response, '')
         self.assertRaises(AttributeError, flickr_noapi.response, '[]')
 
-        response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":')
+        response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}')
         self.assertEqual(flickr_noapi.response(response), [])
 
-        response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":')
+        response = \
+            mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}')
         self.assertEqual(flickr_noapi.response(response), [])
 
         # everthing is ok test
         json = """
-        "search-photos-lite-models","photos":
+        modelExport: {
+          "legend": [
+            [
+              "search-photos-lite-models",
+              "0",
+              "photos",
+              "_data",
+              "0"
+            ]
+          ],
+          "main": {
+            "search-photos-lite-models": [
               {
+                "photos": {
                   "_data": [
                     {
                       "_flickrModelRegistry": "photo-lite-models",
-              "title": "This is the title",
+                      "title": "This%20is%20the%20title",
                       "username": "Owner",
                       "pathAlias": "klink692",
                       "realname": "Owner",
@@ -130,13 +143,16 @@ class TestFlickrNoapiEngine(SearxTestCase):
                         }
                       }
                     }
-          ],
+                  ]
-          "fetchedStart": true,
+                }
-          "fetchedEnd": false,
+              }
-          "totalItems": "4386039"
+            ]
-        },"totalItems":
+          }
+        }
         """
-        json = json.replace('\r\n', '').replace('\n', '').replace('\r', '')
+        # Flickr serves search results in a json block named 'modelExport' buried inside a script tag,
+        # this json is served as a single line terminating with a comma.
+        json = ''.join(json.split()) + ',\n'
         response = mock.Mock(text=json)
         results = flickr_noapi.response(response)
         self.assertEqual(type(results), list)
@@ -149,12 +165,24 @@ class TestFlickrNoapiEngine(SearxTestCase):
 
         # no n size, only the z size
         json = """
-        "search-photos-lite-models","photos":
+        modelExport: {
+          "legend": [
+            [
+              "search-photos-lite-models",
+              "0",
+              "photos",
+              "_data",
+              "0"
+            ]
+          ],
+          "main": {
+            "search-photos-lite-models": [
               {
+                "photos": {
                   "_data": [
                     {
                       "_flickrModelRegistry": "photo-lite-models",
-              "title": "This is the title",
+                      "title": "This%20is%20the%20title",
                       "username": "Owner",
                       "pathAlias": "klink692",
                       "realname": "Owner",
@@ -174,12 +202,14 @@ class TestFlickrNoapiEngine(SearxTestCase):
                         }
                       }
                     }
-          ],
+                  ]
-          "fetchedStart": true,
+                }
-          "fetchedEnd": false,
+              }
-          "totalItems": "4386039"
+            ]
-        },"totalItems":
+          }
+        }
         """
+        json = ''.join(json.split()) + ',\n'
         response = mock.Mock(text=json)
         results = flickr_noapi.response(response)
         self.assertEqual(type(results), list)
@@ -192,12 +222,24 @@ class TestFlickrNoapiEngine(SearxTestCase):
 
         # no z or n size
         json = """
-        "search-photos-lite-models","photos":
+        modelExport: {
+          "legend": [
+            [
+              "search-photos-lite-models",
+              "0",
+              "photos",
+              "_data",
+              "0"
+            ]
+          ],
+          "main": {
+            "search-photos-lite-models": [
               {
+                "photos": {
                   "_data": [
                     {
                       "_flickrModelRegistry": "photo-lite-models",
-              "title": "This is the title",
+                      "title": "This%20is%20the%20title",
                       "username": "Owner",
                       "pathAlias": "klink692",
                       "realname": "Owner",
@@ -217,12 +259,14 @@ class TestFlickrNoapiEngine(SearxTestCase):
                         }
                       }
                     }
-          ],
+                  ]
-          "fetchedStart": true,
+                }
-          "fetchedEnd": false,
+              }
-          "totalItems": "4386039"
+            ]
-        },"totalItems":
+          }
+        }
         """
+        json = ''.join(json.split()) + ',\n'
         response = mock.Mock(text=json)
         results = flickr_noapi.response(response)
         self.assertEqual(type(results), list)
@@ -235,8 +279,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
 
         # no image test
         json = """
-        "search-photos-lite-models","photos":
+        modelExport: {
+          "legend": [
+            [
+              "search-photos-lite-models",
+              "0",
+              "photos",
+              "_data",
+              "0"
+            ]
+          ],
+          "main": {
+            "search-photos-lite-models": [
               {
+                "photos": {
                   "_data": [
                     {
                       "_flickrModelRegistry": "photo-lite-models",
@@ -253,12 +309,14 @@ class TestFlickrNoapiEngine(SearxTestCase):
                       "sizes": {
                       }
                     }
-          ],
+                  ]
-          "fetchedStart": true,
+                }
-          "fetchedEnd": false,
+              }
-          "totalItems": "4386039"
+            ]
-        },"totalItems":
+          }
+        }
         """
+        json = ''.join(json.split()) + ',\n'
         response = mock.Mock(text=json)
         results = flickr_noapi.response(response)
         self.assertEqual(type(results), list)
@@ -266,51 +324,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
 
         # null test
         json = """
-        "search-photos-models","photos":
+        modelExport: {
+          "legend": [null],
+          "main": {
+            "search-photos-lite-models": [
               {
-          "_data": [null],
+                "photos": {
-          "fetchedStart": true,
+                  "_data": [null]
-          "fetchedEnd": false,
+                }
-          "totalItems": "4386039"
+              }
-        },"totalItems":
+            ]
-        """
+          }
-        response = mock.Mock(text=json)
+        }
-        results = flickr_noapi.response(response)
-        self.assertEqual(type(results), list)
-        self.assertEqual(len(results), 0)
-
-        # no ownerNsid test
-        json = """
-        "search-photos-lite-models","photos":
-        {
-          "_data": [
-            {
-              "_flickrModelRegistry": "photo-lite-models",
-              "title": "This is the title",
-              "username": "Owner",
-              "pathAlias": "klink692",
-              "realname": "Owner",
-              "license": 0,
-              "canComment": false,
-              "commentCount": 14,
-              "faveCount": 21,
-              "id": "14001294434",
-              "sizes": {
-                "o": {
-                  "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
-                  "width": 433,
-                  "height": 640,
-                  "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
-                  "key": "o"
-                }
-              }
-            }
-          ],
-          "fetchedStart": true,
-          "fetchedEnd": false,
-          "totalItems": "4386039"
-        },"totalItems":
         """
+        json = ''.join(json.split()) + ',\n'
         response = mock.Mock(text=json)
         results = flickr_noapi.response(response)
         self.assertEqual(type(results), list)
@@ -323,6 +350,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
             "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
         ]}
         """
+        json = ''.join(json.split()) + ',\n'
         response = mock.Mock(text=json)
         results = flickr_noapi.response(response)
         self.assertEqual(type(results), list)