Merge branch 'master' into devel_image_proxy

Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.8
 LABEL maintainer="searx <https://github.com/asciimoo/searx>"
 LABEL description="A privacy-respecting, hackable metasearch engine."
 
-ENV BASE_URL=False IMAGE_PROXY=False
+ENV BASE_URL=False IMAGE_PROXY=False HTTP_PROXY_URL= HTTPS_PROXY_URL=
 EXPOSE 8888
 WORKDIR /usr/local/searx
 CMD ["/sbin/tini","--","/usr/local/searx/run.sh"]
@@ -12,6 +12,9 @@ RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \
  && echo 'sed -i "s|base_url : False|base_url : $BASE_URL|g" searx/settings.yml' >> run.sh \
  && echo 'sed -i "s/image_proxy : False/image_proxy : $IMAGE_PROXY/g" searx/settings.yml' >> run.sh \
  && echo 'sed -i "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml' >> run.sh \
+ && echo 'if [ -n "$HTTP_PROXY_URL" ] || [ -n "$HTTPS_PROXY_URL" ]; then' >> run.sh \
+ && echo '  sed -i "s~^#    proxies :~    proxies:\\n      http: ${HTTP_PROXY_URL}\\n      https: ${HTTPS_PROXY_URL}\\n~" searx/settings.yml' >> run.sh \
+ && echo 'fi' >> run.sh \
  && echo 'python searx/webapp.py' >> run.sh \
  && chmod +x run.sh
 

searx/engines/unsplash.py

@@ -0,0 +1,52 @@
+"""
+ Unsplash
+
+ @website     https://unsplash.com
+ @provide-api yes (https://unsplash.com/developers)
+
+ @using-api   no
+ @results     JSON (using search portal's infiniscroll API)
+ @stable      no (JSON format could change any time)
+ @parse       url, title, img_src, thumbnail_src
+"""
+
+from searx.url_utils import urlencode, urlparse, urlunparse, parse_qsl
+from json import loads
+
+url = 'https://unsplash.com/'
+search_url = url + 'napi/search/photos?'
+categories = ['images']
+page_size = 20
+paging = True
+
+
+def clean_url(url):
+    parsed = urlparse(url)
+    query = [(k, v) for (k, v) in parse_qsl(parsed.query) if k not in ['ixid', 's']]
+
+    return urlunparse((parsed.scheme,
+                       parsed.netloc,
+                       parsed.path,
+                       parsed.params,
+                       urlencode(query),
+                       parsed.fragment))
+
+
+def request(query, params):
+    params['url'] = search_url + urlencode({'query': query, 'page': params['pageno'], 'per_page': page_size})
+    return params
+
+
+def response(resp):
+    results = []
+    json_data = loads(resp.text)
+
+    if 'results' in json_data:
+        for result in json_data['results']:
+            results.append({'template': 'images.html',
+                            'url': clean_url(result['links']['html']),
+                            'thumbnail_src': clean_url(result['urls']['thumb']),
+                            'img_src': clean_url(result['urls']['raw']),
+                            'title': result['description'],
+                            'content': ''})
+    return results

searx/settings.yml

@@ -616,6 +616,11 @@ engines:
 #    content_xpath : //*[@class="meaning"]
 #    shortcut : ud
 
+  - name : unsplash
+    engine : unsplash
+    disabled: True
+    shortcut : us
+
   - name : yahoo
     engine : yahoo
     shortcut : yh

tests/unit/engines/test_unsplash.py

@@ -0,0 +1,38 @@
+from collections import defaultdict
+import mock
+from searx.testing import SearxTestCase
+from searx.engines import unsplash
+
+
+class TestUnsplashEngine(SearxTestCase):
+    def test_request(self):
+        query = 'penguin'
+        _dict = defaultdict(dict)
+        _dict['pageno'] = 1
+        params = unsplash.request(query, _dict)
+
+        self.assertTrue('url' in params)
+        self.assertTrue(query in params['url'])
+
+    def test_response(self):
+        resp = mock.Mock(text='{}')
+        result = unsplash.response(resp)
+        self.assertEqual([], result)
+
+        resp.text = '{"results": []}'
+        result = unsplash.response(resp)
+        self.assertEqual([], result)
+
+        # Sourced from https://unsplash.com/napi/search/photos?query=penguin&xp=&per_page=20&page=2
+        with open('./tests/unit/engines/unsplash_fixture.json') as fixture:
+            resp.text = fixture.read()
+
+        result = unsplash.response(resp)
+        self.assertEqual(len(result), 2)
+        self.assertEqual(result[0]['title'], 'low angle photography of swimming penguin')
+        self.assertEqual(result[0]['url'], 'https://unsplash.com/photos/FY8d721UO_4')
+        self.assertEqual(result[0]['thumbnail_src'], 'https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80\
+&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max')
+        self.assertEqual(result[0]['img_src'], 'https://images.unsplash.com/photo-1523557148507-1b77641c7e7c\
+?ixlib=rb-0.3.5')
+        self.assertEqual(result[0]['content'], '')

tests/unit/engines/unsplash_fixture.json

@@ -0,0 +1,241 @@
+{
+    "total": 2,
+    "total_pages": 1,
+    "results": [
+        {
+            "id": "FY8d721UO_4",
+            "created_at": "2018-04-12T14:20:35-04:00",
+            "updated_at": "2018-08-28T20:58:33-04:00",
+            "width": 3891,
+            "height": 5829,
+            "color": "#152C33",
+            "description": "low angle photography of swimming penguin",
+            "urls": {
+                "raw": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&ixid=eyJhcHBfaWQiOjEyMDd9&s=095c5fc319c5a77c705f49ad63e0f195",
+                "full": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=85&fm=jpg&crop=entropy&cs=srgb&ixid=eyJhcHBfaWQiOjEyMDd9&s=74be977849c173d6929636d491a760c3",
+                "regular": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=ad65df26970bd010085f0ca25434de33",
+                "small": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=400&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=5d2edfd073c31eb8ee7b305222bdc5a2",
+                "thumb": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=a9b9e56e63efc6f4611a87ce7e9a48f8"
+            },
+            "links": {
+                "self": "https://api.unsplash.com/photos/FY8d721UO_4",
+                "html": "https://unsplash.com/photos/FY8d721UO_4",
+                "download": "https://unsplash.com/photos/FY8d721UO_4/download",
+                "download_location": "https://api.unsplash.com/photos/FY8d721UO_4/download"
+            },
+            "categories": [],
+            "sponsored": false,
+            "likes": 31,
+            "liked_by_user": false,
+            "current_user_collections": [],
+            "slug": null,
+            "user": {
+                "id": "N4gE4mrG8lE",
+                "updated_at": "2018-10-03T02:51:19-04:00",
+                "username": "gaspanik",
+                "name": "Masaaki Komori",
+                "first_name": "Masaaki",
+                "last_name": "Komori",
+                "twitter_username": "cipher",
+                "portfolio_url": "https://www.instagram.com/cipher/",
+                "bio": null,
+                "location": "Tokyo, JAPAN",
+                "links": {
+                    "self": "https://api.unsplash.com/users/gaspanik",
+                    "html": "https://unsplash.com/@gaspanik",
+                    "photos": "https://api.unsplash.com/users/gaspanik/photos",
+                    "likes": "https://api.unsplash.com/users/gaspanik/likes",
+                    "portfolio": "https://api.unsplash.com/users/gaspanik/portfolio",
+                    "following": "https://api.unsplash.com/users/gaspanik/following",
+                    "followers": "https://api.unsplash.com/users/gaspanik/followers"
+                },
+                "profile_image": {
+                    "small": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=32&w=32&s=9fe12f6d177bd6fdbd56d233a80c01a3",
+                    "medium": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=64&w=64&s=6ad7d156b62e438ae9dc794cba712fff",
+                    "large": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=128&w=128&s=13a08a2e72e7d11632410e92bd3a9406"
+                },
+                "instagram_username": "cipher",
+                "total_collections": 0,
+                "total_likes": 406,
+                "total_photos": 196
+            },
+            "tags": [
+                {
+                    "title": "animal"
+                },
+                {
+                    "title": "water"
+                },
+                {
+                    "title": "swim"
+                },
+                {
+                    "title": "aquarium"
+                },
+                {
+                    "title": "wallpaper"
+                },
+                {
+                    "title": "blue"
+                },
+                {
+                    "title": "sealife"
+                },
+                {
+                    "title": "wildlife"
+                },
+                {
+                    "title": "bird"
+                },
+                {
+                    "title": "deep sea"
+                },
+                {
+                    "title": "fish"
+                },
+                {
+                    "title": "water life"
+                }
+            ],
+            "photo_tags": [
+                {
+                    "title": "animal"
+                },
+                {
+                    "title": "water"
+                },
+                {
+                    "title": "swim"
+                },
+                {
+                    "title": "aquarium"
+                },
+                {
+                    "title": "wallpaper"
+                }
+            ]
+        },
+        {
+            "id": "ayKyc01xLWA",
+            "created_at": "2018-02-16T23:14:31-05:00",
+            "updated_at": "2018-08-28T20:48:27-04:00",
+            "width": 4928,
+            "height": 3264,
+            "color": "#161618",
+            "description": "black and white penguins on ice field",
+            "urls": {
+                "raw": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&ixid=eyJhcHBfaWQiOjEyMDd9&s=4e107a2bc49ab561ba6272eea2ec725d",
+                "full": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=85&fm=jpg&crop=entropy&cs=srgb&ixid=eyJhcHBfaWQiOjEyMDd9&s=f9b1e4d4572ab44efb2cf3d601d2b4d9",
+                "regular": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=4430cedb63841f1fe055d5005316cc96",
+                "small": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=400&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=ee73c7af22ce445d408e240821ce07af",
+                "thumb": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=934302390d383cad8c571905e3a80bac"
+            },
+            "links": {
+                "self": "https://api.unsplash.com/photos/ayKyc01xLWA",
+                "html": "https://unsplash.com/photos/ayKyc01xLWA",
+                "download": "https://unsplash.com/photos/ayKyc01xLWA/download",
+                "download_location": "https://api.unsplash.com/photos/ayKyc01xLWA/download"
+            },
+            "categories": [],
+            "sponsored": false,
+            "likes": 37,
+            "liked_by_user": false,
+            "current_user_collections": [],
+            "slug": null,
+            "user": {
+                "id": "tRb_KGw60Xk",
+                "updated_at": "2018-09-20T11:51:54-04:00",
+                "username": "ghost_cat",
+                "name": "Danielle Barnes",
+                "first_name": "Danielle",
+                "last_name": "Barnes",
+                "twitter_username": null,
+                "portfolio_url": null,
+                "bio": null,
+                "location": null,
+                "links": {
+                    "self": "https://api.unsplash.com/users/ghost_cat",
+                    "html": "https://unsplash.com/@ghost_cat",
+                    "photos": "https://api.unsplash.com/users/ghost_cat/photos",
+                    "likes": "https://api.unsplash.com/users/ghost_cat/likes",
+                    "portfolio": "https://api.unsplash.com/users/ghost_cat/portfolio",
+                    "following": "https://api.unsplash.com/users/ghost_cat/following",
+                    "followers": "https://api.unsplash.com/users/ghost_cat/followers"
+                },
+                "profile_image": {
+                    "small": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=32&w=32&s=751bf6a557763648d52ffd7e60e79436",
+                    "medium": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=64&w=64&s=e46cd1c8713035f045130e1b093b981e",
+                    "large": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=128&w=128&s=352eabcf107c3ce95fe51a18485f116b"
+                },
+                "instagram_username": null,
+                "total_collections": 0,
+                "total_likes": 0,
+                "total_photos": 21
+            },
+            "tags": [
+                {
+                    "title": "ice"
+                },
+                {
+                    "title": "bird"
+                },
+                {
+                    "title": "ice field"
+                },
+                {
+                    "title": "iceberg"
+                },
+                {
+                    "title": "snow"
+                },
+                {
+                    "title": "frozen"
+                },
+                {
+                    "title": "animal"
+                },
+                {
+                    "title": "wildlife"
+                },
+                {
+                    "title": "wild"
+                },
+                {
+                    "title": "antarctica"
+                },
+                {
+                    "title": "sunshine"
+                },
+                {
+                    "title": "daylight"
+                },
+                {
+                    "title": "wilderness"
+                },
+                {
+                    "title": "south pole"
+                },
+                {
+                    "title": "flock"
+                }
+            ],
+            "photo_tags": [
+                {
+                    "title": "ice"
+                },
+                {
+                    "title": "bird"
+                },
+                {
+                    "title": "ice field"
+                },
+                {
+                    "title": "iceberg"
+                },
+                {
+                    "title": "snow"
+                }
+            ]
+        }
+    ]
+}