zaclys
/
searxng
mirror of https://github.com/searxng/searxng.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1444 from Venca24/devel_google_videos 

[fix] google videos engine
This commit is contained in:
Noémi Ványi 2019-01-05 18:08:05 +01:00 committed by GitHub
parent 899ba5d6de 2456b8f571
commit abcbcec0b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 41 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
searx/engines/google_videos.py
View File

@ -7,7 +7,7 @@
@using-api no @using-api no
@results HTML @results HTML
@stable no @stable no
@parse url, title, content @parse url, title, content, thumbnail
""" """
from datetime import date, timedelta from datetime import date, timedelta
@ -15,7 +15,7 @@ from json import loads
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
from searx.url_utils import urlencode from searx.url_utils import urlencode
import re
# engine dependent config # engine dependent config
categories = ['videos'] categories = ['videos']
@ -25,7 +25,7 @@ time_range_support = True
number_of_results = 10 number_of_results = 10
search_url = 'https://www.google.com/search'\ search_url = 'https://www.google.com/search'\
'?{query}'\ '?q={query}'\
'&tbm=vid'\ '&tbm=vid'\
'&{search_options}' '&{search_options}'
time_range_attr = "qdr:{range}" time_range_attr = "qdr:{range}"
@ -69,15 +69,27 @@ def response(resp):
# parse results # parse results
for result in dom.xpath('//div[@class="g"]'): for result in dom.xpath('//div[@class="g"]'):
title = extract_text(result.xpath('.//h3/a')) title = extract_text(result.xpath('.//h3'))
url = result.xpath('.//h3/a/@href')[0] url = result.xpath('.//div[@class="r"]/a/@href')[0]
content = extract_text(result.xpath('.//span[@class="st"]')) content = extract_text(result.xpath('.//span[@class="st"]'))
# get thumbnails
script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text)
id = result.xpath('.//div[@class="s"]//img/@id')[0]
thumbnails_data = re.findall('s=\'(.*?)(?:\\\\[a-z,1-9,\\\\]+\'|\')\;var ii=\[(?:|[\'vidthumb\d+\',]+)\'' + id,
script)
tmp = []
if len(thumbnails_data) != 0:
tmp = re.findall('(data:image/jpeg;base64,[a-z,A-Z,0-9,/,\+]+)', thumbnails_data[0])
thumbnail = ''
if len(tmp) != 0:
thumbnail = tmp[-1]
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'content': content, 'content': content,
'thumbnail': '', 'thumbnail': thumbnail,
'template': 'videos.html'}) 'template': 'videos.html'})
return results return results

23
tests/unit/engines/test_google_videos.py
View File

@ -30,16 +30,34 @@ class TestGoogleVideosEngine(SearxTestCase):
<div> <div>
<div> <div>
<div class="g"> <div class="g">
<div class="r">
<a href="url_1"><h3>Title 1</h3></a>
</div>
<div class="s">
<div> <div>
<h3><a href="url_1">Title 1</h3> <a>
<g-img>
<img id="vidthumb1">
</g-img>
</a>
</div>
</div> </div>
<div> <div>
<span class="st">Content 1</span> <span class="st">Content 1</span>
</div> </div>
</div> </div>
<div class="g"> <div class="g">
<div class="r">
<a href="url_2"><h3>Title 2</h3></a>
</div>
<div class="s">
<div> <div>
<h3><a href="url_2">Title 2</h3> <a>
<g-img>
<img id="vidthumb2">
</g-img>
</a>
</div>
</div> </div>
<div> <div>
<span class="st">Content 2</span> <span class="st">Content 2</span>
@ -47,6 +65,7 @@ class TestGoogleVideosEngine(SearxTestCase):
</div> </div>
</div> </div>
</div> </div>
<script>function _setImagesSrc(c,d,e){}</script>
""" """
response = mock.Mock(text=html) response = mock.Mock(text=html)
results = google_videos.response(response) results = google_videos.response(response)