From d13c5aa37b02e4df8d5aa7481d0fece3a341f8c2 Mon Sep 17 00:00:00 2001 From: asciimoo Date: Sun, 5 Jan 2014 00:46:42 +0100 Subject: [PATCH 1/9] [mod] unused imports and whitespaces purged --- searx/engines/filecrop.py | 10 ++++------ searx/engines/yacy.py | 10 +++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/searx/engines/filecrop.py b/searx/engines/filecrop.py index a1a933db9..52426b84a 100644 --- a/searx/engines/filecrop.py +++ b/searx/engines/filecrop.py @@ -1,6 +1,4 @@ -from json import loads from urllib import urlencode -from searx.utils import html_to_text from HTMLParser import HTMLParser url = 'http://www.filecrop.com/' @@ -10,7 +8,7 @@ class FilecropResultParser(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.__start_processing = False - + self.results = [] self.result = {} @@ -22,7 +20,7 @@ class FilecropResultParser(HTMLParser): if tag == 'tr': if ('bgcolor', '#edeff5') in attrs or ('bgcolor', '#ffffff') in attrs: self.__start_processing = True - + if not self.__start_processing: return @@ -50,7 +48,7 @@ class FilecropResultParser(HTMLParser): self.data_counter = 0 self.results.append(self.result) self.result = {} - + def handle_data(self, data): if not self.__start_processing: return @@ -59,7 +57,7 @@ class FilecropResultParser(HTMLParser): self.result['content'] += data + ' ' else: self.result['content'] = data + ' ' - + self.data_counter += 1 def request(query, params): diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py index e24edde56..c93ac522f 100644 --- a/searx/engines/yacy.py +++ b/searx/engines/yacy.py @@ -1,5 +1,5 @@ from json import loads -from urllib import urlencode, quote +from urllib import urlencode url = 'http://localhost:8090' search_url = '/yacysearch.json?{query}&maximumRecords=10' @@ -10,7 +10,7 @@ def request(query, params): def response(resp): raw_search_results = loads(resp.text) - + if not len(raw_search_results): return [] @@ -22,10 +22,10 @@ def response(resp): tmp_result = {} tmp_result['title'] = result['title'] tmp_result['url'] = result['link'] - tmp_result['content'] = '' - + tmp_result['content'] = '' + if len(result['description']): - tmp_result['content'] += result['description'] +"
" + tmp_result['content'] += result['description'] +"
" if len(result['pubDate']): tmp_result['content'] += result['pubDate'] + "
" From 53878fde2d46ac9c2d547de0212cdb13b5cb2b09 Mon Sep 17 00:00:00 2001 From: asciimoo Date: Sun, 5 Jan 2014 01:00:32 +0100 Subject: [PATCH 2/9] [fix] error percentage @ stats page --- searx/engines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 43ad1b52e..1ca11ff90 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -261,7 +261,7 @@ def get_engines_stats(): for engine in errors: if max_errors: - engine['percentage'] = int(engine['avg']/max_errors*100) + engine['percentage'] = int(float(engine['avg'])/max_errors*100) else: engine['percentage'] = 0 From 863d136dddc2b5947ac9ab615a91e085b2a307bb Mon Sep 17 00:00:00 2001 From: asciimoo Date: Sun, 5 Jan 2014 01:11:41 +0100 Subject: [PATCH 3/9] [fix] setting category cookie expiration to 4 weeks --- searx/webapp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/searx/webapp.py b/searx/webapp.py index b7e2a4674..2d48f2323 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -152,7 +152,8 @@ def preferences(): selected_categories.append(category) if selected_categories: resp = make_response(redirect('/')) - resp.set_cookie('categories', ','.join(selected_categories)) + # cookie max age: 4 weeks + resp.set_cookie('categories', ','.join(selected_categories), max_age=60*60*24*7*4) return resp return render('preferences.html') From 556b9dd2b0a17528a1c6f2f581e7a0173bc7d4cf Mon Sep 17 00:00:00 2001 From: asciimoo Date: Sun, 5 Jan 2014 12:24:38 +0100 Subject: [PATCH 4/9] [fix] id duplication --- searx/templates/about.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/templates/about.html b/searx/templates/about.html index b90757914..14a0080f9 100644 --- a/searx/templates/about.html +++ b/searx/templates/about.html @@ -37,7 +37,7 @@

It's ok if you don't trust us regarding the logs, take the code and run it yourself! decentralize!

How to add to firefox?

Install searx as a search engine on any version of Firefox! (javascript required)

-

Developer FAQ

+

Developer FAQ

New engines?

  • Edit your engines.cfg, see sample config
    • From e88cf0a0a877458e2babc8af43c5a77654036af6 Mon Sep 17 00:00:00 2001 From: Dalf Date: Sun, 5 Jan 2014 13:50:17 +0100 Subject: [PATCH 5/9] [mod] minor fixes (duckduck_definitions : if a ddg bang is in the query, avoid a useless redirect) --- searx/engines/duckduckgo_definitions.py | 2 +- searx/engines/flickr.py | 0 searx/engines/google_images.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 searx/engines/flickr.py mode change 100755 => 100644 searx/engines/google_images.py diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py index 2b180d0ff..2d97f8154 100644 --- a/searx/engines/duckduckgo_definitions.py +++ b/searx/engines/duckduckgo_definitions.py @@ -1,7 +1,7 @@ import json from urllib import urlencode -url = 'http://api.duckduckgo.com/?{query}&format=json&pretty=0' +url = 'http://api.duckduckgo.com/?{query}&format=json&pretty=0&no_redirect=1' def request(query, params): params['url'] = url.format(query=urlencode({'q': query})) diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py old mode 100755 new mode 100644 diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py old mode 100755 new mode 100644 From 49c85fce51c2b8431013d9269999eac5a6a9c038 Mon Sep 17 00:00:00 2001 From: Dalf Date: Sun, 5 Jan 2014 13:55:17 +0100 Subject: [PATCH 6/9] [fix] dailymotion engine : no more html tag in the description --- engines.cfg_sample | 1 + searx/engines/dailymotion.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/engines.cfg_sample b/engines.cfg_sample index d76d31214..e2668eeb1 100644 --- a/engines.cfg_sample +++ b/engines.cfg_sample @@ -82,5 +82,6 @@ categories = videos [dailymotion] engine = dailymotion +locale = en_US categories = videos diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py index 7046132f3..655485957 100644 --- a/searx/engines/dailymotion.py +++ b/searx/engines/dailymotion.py @@ -1,16 +1,17 @@ from urllib import urlencode +from lxml import html from json import loads from cgi import escape categories = ['videos'] -localization = 'en' +locale = 'en_US' # see http://www.dailymotion.com/doc/api/obj-video.html search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}' def request(query, params): global search_url - params['url'] = search_url.format(query=urlencode({'search': query, 'localization': localization })) + params['url'] = search_url.format(query=urlencode({'search': query, 'localization': locale })) return params @@ -27,6 +28,11 @@ def response(resp): else: content = '' if res['description']: - content += escape(res['description'][:500]) + description = text_content_from_html(res['description']) + content += description[:500] results.append({'url': url, 'title': title, 'content': content}) return results + +def text_content_from_html(html_string): + desc_html = html.fragment_fromstring(html_string, create_parent=True) + return desc_html.text_content() From bf56ec4fb1199ab46feeacb8a045d28b287baf47 Mon Sep 17 00:00:00 2001 From: Dalf Date: Sun, 5 Jan 2014 13:58:17 +0100 Subject: [PATCH 7/9] [mod] bing and duckduckgo engines : add / rename locale parameter --- engines.cfg_sample | 3 ++- searx/engines/bing.py | 4 ++-- searx/engines/duckduckgo.py | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/engines.cfg_sample b/engines.cfg_sample index e2668eeb1..27131e9dd 100644 --- a/engines.cfg_sample +++ b/engines.cfg_sample @@ -5,7 +5,7 @@ number_of_results = 1 [bing] engine = bing -language = en-us +locale = en-us [cc] engine=currency_convert @@ -20,6 +20,7 @@ engine = duckduckgo_definitions [duckduckgo] engine = duckduckgo +locale = en-us [flickr] engine = flickr diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 1002a29c6..6b0bf5a3f 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -4,11 +4,11 @@ from cgi import escape base_url = 'http://www.bing.com/' search_string = 'search?{query}' -language = 'en-us' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx +locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx def request(query, params): - search_path = search_string.format(query=urlencode({'q': query, 'setmkt': language})) + search_path = search_string.format(query=urlencode({'q': query, 'setmkt': locale})) #if params['category'] == 'images': # params['url'] = base_url + 'images/' + search_path params['url'] = base_url + search_path diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index d591854a5..4bf770972 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -3,10 +3,11 @@ from urllib import urlencode from searx.utils import html_to_text url = 'https://duckduckgo.com/' -search_url = url + 'd.js?{query}&l=us-en&p=1&s=0' +search_url = url + 'd.js?{query}&p=1&s=0' +locale = 'us-en' def request(query, params): - params['url'] = search_url.format(query=urlencode({'q': query})) + params['url'] = search_url.format(query=urlencode({'q': query, 'l': locale})) return params From a2928e8d8369f129303229d17dfabc896f59441d Mon Sep 17 00:00:00 2001 From: Dalf Date: Sun, 5 Jan 2014 14:00:10 +0100 Subject: [PATCH 8/9] [fix] startpage engine : characters with diacritic were preceded by whitespace, and cleaner way to parse the result. --- searx/engines/startpage.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 061c8158d..87c091e2d 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -19,14 +19,13 @@ def response(resp): global base_url results = [] dom = html.fromstring(resp.content) - for result in dom.xpath('//div[@class="result"]'): + # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"] + # not ads : div[@class="result"] are the direct childs of div[@id="results"] + for result in dom.xpath('//div[@id="results"]/div[@class="result"]'): link = result.xpath('.//h3/a')[0] url = link.attrib.get('href') parsed_url = urlparse(url) - # TODO better google link detection - if parsed_url.netloc.find('www.google.com') >= 0: - continue - title = ' '.join(link.xpath('.//text()')) - content = escape(' '.join(result.xpath('.//p[@class="desc"]//text()'))) + title = link.text_content() + content = result.xpath('./p[@class="desc"]')[0].text_content() results.append({'url': url, 'title': title, 'content': content}) return results From 3dc3fc77709d6f56cd42f748ae356e2915fa2286 Mon Sep 17 00:00:00 2001 From: Dalf Date: Sun, 5 Jan 2014 14:06:52 +0100 Subject: [PATCH 9/9] [mod][fix] xpath engine simplified, yahoo engine never returns truncated urls --- engines.cfg_sample | 14 ++++---- searx/engines/xpath.py | 76 +++++++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/engines.cfg_sample b/engines.cfg_sample index 27131e9dd..05593cea4 100644 --- a/engines.cfg_sample +++ b/engines.cfg_sample @@ -5,7 +5,7 @@ number_of_results = 1 [bing] engine = bing -locale = en-us +locale = en-US [cc] engine=currency_convert @@ -64,17 +64,17 @@ categories = social media [urbandictionary] engine = xpath search_url = http://www.urbandictionary.com/define.php?term={query} -url_xpath = //div[@id="entries"]//div[@class="word"]//a -title_xpath = //div[@id="entries"]//div[@class="word"]//span//text() -content_xpath = //div[@id="entries"]//div[@class="text"]//div[@class="definition"]//text() +url_xpath = //div[@id="entries"]//div[@class="word"]/a/@href +title_xpath = //div[@id="entries"]//div[@class="word"]/span +content_xpath = //div[@id="entries"]//div[@class="text"]/div[@class="definition"] [yahoo] engine = xpath search_url = http://search.yahoo.com/search?p={query} results_xpath = //div[@class="res"] -url_xpath = .//span[@class="url"]//text() -content_xpath = .//div[@class="abstr"]//text() -title_xpath = .//h3/a//text() +url_xpath = .//h3/a/@href +title_xpath = .//h3/a +content_xpath = .//div[@class="abstr"] suggestion_xpath = //div[@id="satat"]//a [youtube] diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index ad3a97ffa..e1ccad8de 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -1,5 +1,5 @@ from lxml import html -from urllib import urlencode +from urllib import urlencode, unquote from urlparse import urlparse, urljoin from cgi import escape from lxml.etree import _ElementStringResult @@ -11,32 +11,64 @@ title_xpath = None suggestion_xpath = '' results_xpath = '' -def extract_url(xpath_results): - url = '' - parsed_search_url = urlparse(search_url) +''' +if xpath_results is list, extract the text from each result and concat the list +if xpath_results is a xml element, extract all the text node from it ( text_content() method from lxml ) +if xpath_results is a string element, then it's already done +''' +def extract_text(xpath_results): if type(xpath_results) == list: + # it's list of result : concat everything using recursive call if not len(xpath_results): raise Exception('Empty url resultset') - if type(xpath_results[0]) == _ElementStringResult: - url = ''.join(xpath_results) - if url.startswith('//'): - url = parsed_search_url.scheme+url - elif url.startswith('/'): - url = urljoin(search_url, url) - #TODO - else: - url = xpath_results[0].attrib.get('href') + result = '' + for e in xpath_results: + result = result + extract_text(e) + return result + elif type(xpath_results) == _ElementStringResult: + # it's a string + return ''.join(xpath_results) else: - url = xpath_results.attrib.get('href') - if not url.startswith('http://') and not url.startswith('https://'): - url = 'http://'+url + # it's a element + return xpath_results.text_content() + + +def extract_url(xpath_results): + url = extract_text(xpath_results) + + if url.startswith('//'): + # add http or https to this kind of url //example.com/ + parsed_search_url = urlparse(search_url) + url = parsed_search_url.scheme+url + elif url.startswith('/'): + # fix relative url to the search engine + url = urljoin(search_url, url) + + # normalize url + url = normalize_url(url) + + return url + + +def normalize_url(url): parsed_url = urlparse(url) + + # add a / at this end of the url if there is no path if not parsed_url.netloc: raise Exception('Cannot parse url') if not parsed_url.path: url += '/' + + # FIXME : hack for yahoo + if parsed_url.hostname == 'search.yahoo.com' and parsed_url.path.startswith('/r'): + p = parsed_url.path + mark = p.find('/**') + if mark != -1: + return unquote(p[mark+3:]).decode('utf-8') + return url + def request(query, params): query = urlencode({'q': query})[2:] params['url'] = search_url.format(query=query) @@ -50,15 +82,19 @@ def response(resp): if results_xpath: for result in dom.xpath(results_xpath): url = extract_url(result.xpath(url_xpath)) - title = ' '.join(result.xpath(title_xpath)) - content = escape(' '.join(result.xpath(content_xpath))) + title = extract_text(result.xpath(title_xpath)[0 ]) + content = extract_text(result.xpath(content_xpath)[0]) results.append({'url': url, 'title': title, 'content': content}) else: - for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)): + for url, title, content in zip( + map(extract_url, dom.xpath(url_xpath)), \ + map(extract_text, dom.xpath(title_xpath)), \ + map(extract_text, dom.xpath(content_xpath)), \ + ): results.append({'url': url, 'title': title, 'content': content}) if not suggestion_xpath: return results for suggestion in dom.xpath(suggestion_xpath): - results.append({'suggestion': escape(''.join(suggestion.xpath('.//text()')))}) + results.append({'suggestion': extract_text(suggestion)}) return results