From a2a8fbf6230892c2af8e2bb7b76ca3410c61b188 Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Fri, 20 Feb 2015 18:47:56 +0100 Subject: [PATCH 1/3] Add Spotify engine --- searx/engines/spotify.py | 60 ++++++++++++++ searx/settings.yml | 4 + searx/tests/engines/test_spotify.py | 124 ++++++++++++++++++++++++++++ searx/tests/test_engines.py | 1 + 4 files changed, 189 insertions(+) create mode 100644 searx/engines/spotify.py create mode 100644 searx/tests/engines/test_spotify.py diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py new file mode 100644 index 000000000..61f3721ec --- /dev/null +++ b/searx/engines/spotify.py @@ -0,0 +1,60 @@ +## Spotify (Music) +# +# @website https://spotify.com +# @provide-api yes (https://developer.spotify.com/web-api/search-item/) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, content, embedded + +from json import loads +from urllib import urlencode + +# engine dependent config +categories = ['music'] +paging = True + +# search-url +url = 'https://api.spotify.com/' +search_url = url + 'v1/search?{query}&type=track&offset={offset}' + +embedded_url = '' + + +# do search-request +def request(query, params): + offset = (params['pageno'] - 1) * 20 + + params['url'] = search_url.format(query=urlencode({'q': query}), + offset=offset) + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_res = loads(resp.text) + + # parse results + for result in search_res.get('tracks', {}).get('items', {}): + if result['type'] == 'track': + title = result['name'] + url = result['external_urls']['spotify'] + content = result['artists'][0]['name'] +\ + " • " +\ + result['album']['name'] +\ + " • " + result['name'] + embedded = embedded_url.format(audioid=result['id']) + + # append result + results.append({'url': url, + 'title': title, + 'embedded': embedded, + 'content': content}) + + # return results + return results diff --git a/searx/settings.yml b/searx/settings.yml index 8e2833ef0..0119a2084 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -195,6 +195,10 @@ engines: shortcut : scc disabled : True + - name : spotify + engine : spotify + shortcut : stf + - name : subtitleseeker engine : subtitleseeker shortcut : ss diff --git a/searx/tests/engines/test_spotify.py b/searx/tests/engines/test_spotify.py new file mode 100644 index 000000000..fd274abbd --- /dev/null +++ b/searx/tests/engines/test_spotify.py @@ -0,0 +1,124 @@ +from collections import defaultdict +import mock +from searx.engines import spotify +from searx.testing import SearxTestCase + + +class TestSpotifyEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = spotify.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('spotify.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, spotify.response, None) + self.assertRaises(AttributeError, spotify.response, []) + self.assertRaises(AttributeError, spotify.response, '') + self.assertRaises(AttributeError, spotify.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(spotify.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(spotify.response(response), []) + + json = """ + { + "tracks": { + "href": "https://api.spotify.com/v1/search?query=nosfell&offset=0&limit=20&type=track", + "items": [ + { + "album": { + "album_type": "album", + "external_urls": { + "spotify": "https://open.spotify.com/album/5c9ap1PBkSGLxT3J73toxA" + }, + "href": "https://api.spotify.com/v1/albums/5c9ap1PBkSGLxT3J73toxA", + "id": "5c9ap1PBkSGLxT3J73toxA", + "name": "Album Title", + "type": "album", + "uri": "spotify:album:5c9ap1PBkSGLxT3J73toxA" + }, + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/0bMc6b75FfZEpQHG1jifKu" + }, + "href": "https://api.spotify.com/v1/artists/0bMc6b75FfZEpQHG1jifKu", + "id": "0bMc6b75FfZEpQHG1jifKu", + "name": "Artist Name", + "type": "artist", + "uri": "spotify:artist:0bMc6b75FfZEpQHG1jifKu" + } + ], + "disc_number": 1, + "duration_ms": 202386, + "explicit": false, + "external_ids": { + "isrc": "FRV640600067" + }, + "external_urls": { + "spotify": "https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa" + }, + "href": "https://api.spotify.com/v1/tracks/2GzvFiedqW8hgqUpWcASZa", + "id": "1000", + "is_playable": true, + "name": "Title of track", + "popularity": 6, + "preview_url": "https://p.scdn.co/mp3-preview/7b8ecda580965a066b768c2647f877e43f7b1a0a", + "track_number": 3, + "type": "track", + "uri": "spotify:track:2GzvFiedqW8hgqUpWcASZa" + } + ], + "limit": 20, + "next": "https://api.spotify.com/v1/search?query=nosfell&offset=20&limit=20&type=track", + "offset": 0, + "previous": null, + "total": 107 + } + } + """ + response = mock.Mock(text=json) + results = spotify.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title of track') + self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa') + self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') + self.assertIn('1000', results[0]['embedded']) + + json = """ + { + "tracks": { + "href": "https://api.spotify.com/v1/search?query=nosfell&offset=0&limit=20&type=track", + "items": [ + { + "href": "https://api.spotify.com/v1/tracks/2GzvFiedqW8hgqUpWcASZa", + "id": "1000", + "is_playable": true, + "name": "Title of track", + "popularity": 6, + "preview_url": "https://p.scdn.co/mp3-preview/7b8ecda580965a066b768c2647f877e43f7b1a0a", + "track_number": 3, + "type": "album", + "uri": "spotify:track:2GzvFiedqW8hgqUpWcASZa" + } + ], + "limit": 20, + "next": "https://api.spotify.com/v1/search?query=nosfell&offset=20&limit=20&type=track", + "offset": 0, + "previous": null, + "total": 107 + } + } + """ + response = mock.Mock(text=json) + results = spotify.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index 9b1c12cb1..5770458f3 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -28,6 +28,7 @@ from searx.tests.engines.test_piratebay import * # noqa from searx.tests.engines.test_searchcode_code import * # noqa from searx.tests.engines.test_searchcode_doc import * # noqa from searx.tests.engines.test_soundcloud import * # noqa +from searx.tests.engines.test_spotify import * # noqa from searx.tests.engines.test_stackoverflow import * # noqa from searx.tests.engines.test_startpage import * # noqa from searx.tests.engines.test_subtitleseeker import * # noqa From aea0f866f62eccbeb0655f436144a41eaf7175fa Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Fri, 20 Feb 2015 18:50:30 +0100 Subject: [PATCH 2/3] Remove dead engines from config. I didn't remove the engines files because it always could be helpful if they come back. --- searx/settings.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/searx/settings.yml b/searx/settings.yml index 0119a2084..01afa3770 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -78,12 +78,6 @@ engines: # shortcut : fa # api_key : 'apikey' # required! -# down - website is under criminal investigation by the UK -# - name : filecrop -# engine : filecrop -# categories : files -# shortcut : fc - - name : 500px engine : www500px shortcut : px @@ -103,11 +97,6 @@ engines: # Or you can use the html non-stable engine, activated by default engine : flickr_noapi - - name : general-file - engine : generalfile - shortcut : gf - disabled : True - - name : gigablast engine : gigablast shortcut : gb From 56ad00bc3292c345dfaa5a671844e222b5f72e5b Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Fri, 20 Feb 2015 19:03:09 +0100 Subject: [PATCH 3/3] Correct a few UT --- searx/tests/engines/test_blekko_images.py | 11 ++++++++--- searx/tests/engines/test_google_images.py | 11 ++++++++--- searx/tests/engines/test_yahoo.py | 14 ++------------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/searx/tests/engines/test_blekko_images.py b/searx/tests/engines/test_blekko_images.py index 793fadbad..beb0853e3 100644 --- a/searx/tests/engines/test_blekko_images.py +++ b/searx/tests/engines/test_blekko_images.py @@ -12,9 +12,14 @@ class TestBlekkoImagesEngine(SearxTestCase): dicto['pageno'] = 0 dicto['safesearch'] = 1 params = blekko_images.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('blekko.com' in params['url']) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('blekko.com', params['url']) + self.assertIn('page', params['url']) + + dicto['pageno'] = 1 + params = blekko_images.request(query, dicto) + self.assertNotIn('page', params['url']) def test_response(self): self.assertRaises(AttributeError, blekko_images.response, None) diff --git a/searx/tests/engines/test_google_images.py b/searx/tests/engines/test_google_images.py index 6870ff52f..32d133334 100644 --- a/searx/tests/engines/test_google_images.py +++ b/searx/tests/engines/test_google_images.py @@ -11,9 +11,14 @@ class TestGoogleImagesEngine(SearxTestCase): dicto = defaultdict(dict) dicto['pageno'] = 1 params = google_images.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('googleapis.com' in params['url']) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('googleapis.com', params['url']) + self.assertIn('safe=on', params['url']) + + dicto['safesearch'] = 0 + params = google_images.request(query, dicto) + self.assertIn('safe=off', params['url']) def test_response(self): self.assertRaises(AttributeError, google_images.response, None) diff --git a/searx/tests/engines/test_yahoo.py b/searx/tests/engines/test_yahoo.py index cdd6fda29..11ef9db22 100644 --- a/searx/tests/engines/test_yahoo.py +++ b/searx/tests/engines/test_yahoo.py @@ -75,12 +75,6 @@ class TestYahooEngine(SearxTestCase):
  • -

    - This is the second title -

    This is the second content

    @@ -102,16 +96,12 @@ class TestYahooEngine(SearxTestCase): """ response = mock.Mock(text=html) results = yahoo.response(response) - print results self.assertEqual(type(results), list) - self.assertEqual(len(results), 3) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This is the title') self.assertEqual(results[0]['url'], 'https://this.is.the.url/') self.assertEqual(results[0]['content'], 'This is the content') - self.assertEqual(results[1]['title'], 'This is the second title') - self.assertEqual(results[1]['url'], 'https://this.is.the.second.url/') - self.assertEqual(results[1]['content'], 'This is the second content') - self.assertEqual(results[2]['suggestion'], 'This is the suggestion') + self.assertEqual(results[1]['suggestion'], 'This is the suggestion') html = """