forked from zaclys/searxng
Merge pull request #1068 from Apply55gx/genius
Continuation of PR #1004 (seems abandoned)
This commit is contained in:
Adam Tauber
GitHub
commit
00a7041daa
88
searx/engines/genius.py
Normal file
88
searx/engines/genius.py
Normal file
@ -0,0 +1,88 @@
|
||||
"""
|
||||
Genius
|
||||
|
||||
@website https://www.genius.com/
|
||||
@provide-api yes (https://docs.genius.com/)
|
||||
|
||||
@using-api yes
|
||||
@results JSON
|
||||
@stable yes
|
||||
@parse url, title, content, thumbnail, publishedDate
|
||||
"""
|
||||
|
||||
from json import loads
|
||||
from searx.url_utils import urlencode
|
||||
from datetime import datetime
|
||||
|
||||
# engine dependent config
|
||||
categories = ['music']
|
||||
paging = True
|
||||
language_support = False
|
||||
page_size = 5
|
||||
|
||||
url = 'https://genius.com/api/'
|
||||
search_url = url + 'search/{index}?{query}&page={pageno}&per_page={page_size}'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
params['url'] = search_url.format(query=urlencode({'q': query}),
|
||||
index='multi',
|
||||
page_size=page_size,
|
||||
pageno=params['pageno'])
|
||||
return params
|
||||
|
||||
|
||||
def parse_lyric(hit):
|
||||
try:
|
||||
content = hit['highlights'][0]['value']
|
||||
except:
|
||||
content = None
|
||||
timestamp = hit['result']['lyrics_updated_at']
|
||||
result = {'url': hit['result']['url'],
|
||||
'title': hit['result']['full_title'],
|
||||
'content': content,
|
||||
'thumbnail': hit['result']['song_art_image_thumbnail_url'],
|
||||
'template': 'videos.html'}
|
||||
if timestamp:
|
||||
result.update({'publishedDate': datetime.fromtimestamp(timestamp)})
|
||||
return result
|
||||
|
||||
|
||||
def parse_artist(hit):
|
||||
result = {'url': hit['result']['url'],
|
||||
'title': hit['result']['name'],
|
||||
'content': None,
|
||||
'thumbnail': hit['result']['image_url'],
|
||||
'template': 'videos.html'}
|
||||
return result
|
||||
|
||||
|
||||
def parse_album(hit):
|
||||
result = {'url': hit['result']['url'],
|
||||
'title': hit['result']['full_title'],
|
||||
'thumbnail': hit['result']['cover_art_url'],
|
||||
# 'thumbnail': hit['result']['cover_art_thumbnail_url'],
|
||||
'template': 'videos.html'}
|
||||
try:
|
||||
year = hit['result']['release_date_components']['year']
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
if year:
|
||||
result.update({'content': 'Released: {}'.format(year)})
|
||||
return result
|
||||
|
||||
parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
json = loads(resp.text)
|
||||
hits = [hit for section in json['response']['sections'] for hit in section['hits']]
|
||||
for hit in hits:
|
||||
try:
|
||||
func = parse[hit['type']]
|
||||
except KeyError:
|
||||
continue
|
||||
results.append(func(hit))
|
||||
return results
|
||||
@ -241,6 +241,10 @@ engines:
|
||||
shortcut : frk
|
||||
disabled : True
|
||||
|
||||
- name : genius
|
||||
engine : genius
|
||||
shortcut : gen
|
||||
|
||||
- name : gigablast
|
||||
engine : gigablast
|
||||
shortcut : gb
|
||||
|
||||
231
tests/unit/engines/test_genius.py
Normal file
231
tests/unit/engines/test_genius.py
Normal file
@ -0,0 +1,231 @@
|
||||
from collections import defaultdict
|
||||
import mock
|
||||
from datetime import datetime
|
||||
from searx.engines import genius
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
class TestGeniusEngine(SearxTestCase):
|
||||
|
||||
def test_request(self):
|
||||
query = 'test_query'
|
||||
dicto = defaultdict(dict)
|
||||
dicto['pageno'] = 1
|
||||
params = genius.request(query, dicto)
|
||||
self.assertTrue('url' in params)
|
||||
self.assertTrue(query in params['url'])
|
||||
self.assertTrue('genius.com' in params['url'])
|
||||
|
||||
def test_response(self):
|
||||
|
||||
json_empty = """
|
||||
{
|
||||
"meta": {
|
||||
"status": 200
|
||||
},
|
||||
"response": {
|
||||
"sections": [
|
||||
{
|
||||
"type": "top_hit",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "song",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "lyric",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "artist",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "album",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "tag",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "video",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "article",
|
||||
"hits": []
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"hits": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
resp = mock.Mock(text=json_empty)
|
||||
self.assertEqual(genius.response(resp), [])
|
||||
|
||||
json = """
|
||||
{
|
||||
"meta": {
|
||||
"status": 200
|
||||
},
|
||||
"response": {
|
||||
"sections": [
|
||||
{
|
||||
"type": "lyric",
|
||||
"hits": [
|
||||
{
|
||||
"highlights": [
|
||||
{
|
||||
"property": "lyrics",
|
||||
"value": "Sample lyrics",
|
||||
"snippet": true,
|
||||
"ranges": []
|
||||
}
|
||||
],
|
||||
"index": "lyric",
|
||||
"type": "song",
|
||||
"result": {
|
||||
"_type": "song",
|
||||
"annotation_count": 45,
|
||||
"api_path": "/songs/52916",
|
||||
"full_title": "J't'emmerde by MC Jean Gab'1",
|
||||
"header_image_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
|
||||
"header_image_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.1000x1000x1.jpg",
|
||||
"id": 52916,
|
||||
"instrumental": false,
|
||||
"lyrics_owner_id": 15586,
|
||||
"lyrics_state": "complete",
|
||||
"lyrics_updated_at": 1498744545,
|
||||
"path": "/Mc-jean-gab1-jtemmerde-lyrics",
|
||||
"pyongs_count": 4,
|
||||
"song_art_image_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
|
||||
"stats": {
|
||||
"hot": false,
|
||||
"unreviewed_annotations": 0,
|
||||
"pageviews": 62490
|
||||
},
|
||||
"title": "J't'emmerde",
|
||||
"title_with_featured": "J't'emmerde",
|
||||
"updated_by_human_at": 1498744546,
|
||||
"url": "https://genius.com/Mc-jean-gab1-jtemmerde-lyrics",
|
||||
"primary_artist": {
|
||||
"_type": "artist",
|
||||
"api_path": "/artists/12691",
|
||||
"header_image_url": "https://images.genius.com/c7847662a58f8c2b0f02a6e217d60907.960x657x1.jpg",
|
||||
"id": 12691,
|
||||
"image_url": "https://s3.amazonaws.com/rapgenius/Mc-jean-gab1.jpg",
|
||||
"index_character": "m",
|
||||
"is_meme_verified": false,
|
||||
"is_verified": false,
|
||||
"name": "MC Jean Gab'1",
|
||||
"slug": "Mc-jean-gab1",
|
||||
"url": "https://genius.com/artists/Mc-jean-gab1"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "artist",
|
||||
"hits": [
|
||||
{
|
||||
"highlights": [],
|
||||
"index": "artist",
|
||||
"type": "artist",
|
||||
"result": {
|
||||
"_type": "artist",
|
||||
"api_path": "/artists/191580",
|
||||
"header_image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
|
||||
"id": 191580,
|
||||
"image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
|
||||
"index_character": "a",
|
||||
"is_meme_verified": false,
|
||||
"is_verified": false,
|
||||
"name": "ASDF Guy",
|
||||
"slug": "Asdf-guy",
|
||||
"url": "https://genius.com/artists/Asdf-guy"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "album",
|
||||
"hits": [
|
||||
{
|
||||
"highlights": [],
|
||||
"index": "album",
|
||||
"type": "album",
|
||||
"result": {
|
||||
"_type": "album",
|
||||
"api_path": "/albums/132332",
|
||||
"cover_art_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
|
||||
"cover_art_url": "https://images.genius.com/xxx.600x600x1.jpg",
|
||||
"full_title": "ASD by A Skylit Drive",
|
||||
"id": 132332,
|
||||
"name": "ASD",
|
||||
"name_with_artist": "ASD (artist: A Skylit Drive)",
|
||||
"release_date_components": {
|
||||
"year": 2015,
|
||||
"month": null,
|
||||
"day": null
|
||||
},
|
||||
"url": "https://genius.com/albums/A-skylit-drive/Asd",
|
||||
"artist": {
|
||||
"_type": "artist",
|
||||
"api_path": "/artists/48712",
|
||||
"header_image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
|
||||
"id": 48712,
|
||||
"image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
|
||||
"index_character": "s",
|
||||
"is_meme_verified": false,
|
||||
"is_verified": false,
|
||||
"name": "A Skylit Drive",
|
||||
"slug": "A-skylit-drive",
|
||||
"url": "https://genius.com/artists/A-skylit-drive"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
resp = mock.Mock(text=json)
|
||||
results = genius.response(resp)
|
||||
|
||||
self.assertEqual(len(results), 3)
|
||||
self.assertEqual(type(results), list)
|
||||
|
||||
# check lyric parsing
|
||||
r = results[0]
|
||||
self.assertEqual(r['url'], 'https://genius.com/Mc-jean-gab1-jtemmerde-lyrics')
|
||||
self.assertEqual(r['title'], "J't'emmerde by MC Jean Gab'1")
|
||||
self.assertEqual(r['content'], "Sample lyrics")
|
||||
self.assertEqual(r['template'], 'videos.html')
|
||||
self.assertEqual(r['thumbnail'], 'https://images.genius.com/xxx.300x300x1.jpg')
|
||||
created = datetime.fromtimestamp(1498744545)
|
||||
self.assertEqual(r['publishedDate'], created)
|
||||
|
||||
# check artist parsing
|
||||
r = results[1]
|
||||
self.assertEqual(r['url'], 'https://genius.com/artists/Asdf-guy')
|
||||
self.assertEqual(r['title'], "ASDF Guy")
|
||||
self.assertEqual(r['content'], None)
|
||||
self.assertEqual(r['template'], 'videos.html')
|
||||
self.assertEqual(r['thumbnail'], 'https://assets.genius.com/images/default_avatar_300.png?1503090542')
|
||||
|
||||
# check album parsing
|
||||
r = results[2]
|
||||
self.assertEqual(r['url'], 'https://genius.com/albums/A-skylit-drive/Asd')
|
||||
self.assertEqual(r['title'], "ASD by A Skylit Drive")
|
||||
self.assertEqual(r['content'], "Released: 2015")
|
||||
self.assertEqual(r['template'], 'videos.html')
|
||||
self.assertEqual(r['thumbnail'], 'https://images.genius.com/xxx.600x600x1.jpg')
|
||||
Loading…
Reference in New Issue
Block a user