diff --git a/Dockerfile b/Dockerfile index 1815ddc57..b0b5a609d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,17 @@ FROM alpine:3.10 +ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"] +EXPOSE 8080 +VOLUME /etc/searx +VOLUME /var/log/uwsgi -ARG VERSION_GITCOMMIT=unknow -ARG SEARX_GIT_VERSION=unknow +ARG VERSION_GITCOMMIT=unknown +ARG SEARX_GIT_VERSION=unknown -ARG SEARX_GID=1000 -ARG SEARX_UID=1000 +ARG SEARX_GID=977 +ARG SEARX_UID=977 + +RUN addgroup -g ${SEARX_GID} searx && \ + adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx ARG TIMESTAMP_SETTINGS=0 ARG TIMESTAMP_UWSGI=0 @@ -16,19 +23,14 @@ ENV INSTANCE_NAME=searx \ BASE_URL= \ MORTY_KEY= \ MORTY_URL= -EXPOSE 8080 -VOLUME /etc/searx -VOLUME /var/log/uwsgi WORKDIR /usr/local/searx -RUN addgroup -g ${SEARX_GID} searx && \ - adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx COPY requirements.txt ./requirements.txt -RUN apk -U upgrade \ - && apk add -t build-dependencies \ +RUN apk upgrade --no-cache \ + && apk add --no-cache -t build-dependencies \ build-base \ py3-setuptools \ python3-dev \ @@ -38,7 +40,7 @@ RUN apk -U upgrade \ openssl-dev \ tar \ git \ - && apk add \ + && apk add --no-cache \ ca-certificates \ su-exec \ python3 \ @@ -50,8 +52,7 @@ RUN apk -U upgrade \ uwsgi-python3 \ && pip3 install --upgrade pip \ && pip3 install --no-cache -r requirements.txt \ - && apk del build-dependencies \ - && rm -f /var/cache/apk/* + && apk del build-dependencies COPY --chown=searx:searx . . @@ -62,7 +63,6 @@ RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \ echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \ fi -ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"] # Keep this argument at the end since it change each time ARG LABEL_DATE= diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 1e614867b..ed0b87dbd 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -18,7 +18,7 @@ from lxml import html from searx import logger, utils from searx.engines.xpath import extract_text from searx.url_utils import urlencode -from searx.utils import match_language, gen_useragent +from searx.utils import match_language, gen_useragent, eval_xpath logger = logger.getChild('bing engine') @@ -65,11 +65,11 @@ def response(resp): dom = html.fromstring(resp.text) # parse results - for result in dom.xpath('//div[@class="sa_cc"]'): - link = result.xpath('.//h3/a')[0] + for result in eval_xpath(dom, '//div[@class="sa_cc"]'): + link = eval_xpath(result, './/h3/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = extract_text(result.xpath('.//p')) + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, @@ -77,11 +77,11 @@ def response(resp): 'content': content}) # parse results again if nothing is found yet - for result in dom.xpath('//li[@class="b_algo"]'): - link = result.xpath('.//h2/a')[0] + for result in eval_xpath(dom, '//li[@class="b_algo"]'): + link = eval_xpath(result, './/h2/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = extract_text(result.xpath('.//p')) + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, @@ -89,7 +89,7 @@ def response(resp): 'content': content}) try: - result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()')) + result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()')) result_len_container = utils.to_string(result_len_container) if "-" in result_len_container: # Remove the part "from-to" for paginated request ... @@ -113,9 +113,9 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = html.fromstring(resp.text) - options = dom.xpath('//div[@id="limit-languages"]//input') + options = eval_xpath(dom, '//div[@id="limit-languages"]//input') for option in options: - code = option.xpath('./@id')[0].replace('_', '-') + code = eval_xpath(option, './@id')[0].replace('_', '-') if code == 'nb': code = 'no' supported_languages.append(code) diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py index 09db048cc..423af0971 100644 --- a/searx/engines/dictzone.py +++ b/searx/engines/dictzone.py @@ -11,7 +11,7 @@ import re from lxml import html -from searx.utils import is_valid_lang +from searx.utils import is_valid_lang, eval_xpath from searx.url_utils import urljoin categories = ['general'] @@ -47,14 +47,14 @@ def response(resp): dom = html.fromstring(resp.text) - for k, result in enumerate(dom.xpath(results_xpath)[1:]): + for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]): try: - from_result, to_results_raw = result.xpath('./td') + from_result, to_results_raw = eval_xpath(result, './td') except: continue to_results = [] - for to_result in to_results_raw.xpath('./p/a'): + for to_result in eval_xpath(to_results_raw, './p/a'): t = to_result.text_content() if t.strip(): to_results.append(to_result.text_content()) diff --git a/searx/engines/doku.py b/searx/engines/doku.py index a391be444..d20e66026 100644 --- a/searx/engines/doku.py +++ b/searx/engines/doku.py @@ -11,6 +11,7 @@ from lxml.html import fromstring from searx.engines.xpath import extract_text +from searx.utils import eval_xpath from searx.url_utils import urlencode # engine dependent config @@ -45,16 +46,16 @@ def response(resp): # parse results # Quickhits - for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'): + for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'): try: - res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1] + res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1] except: continue if not res_url: continue - title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title')) + title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title')) # append result results.append({'title': title, @@ -62,13 +63,13 @@ def response(resp): 'url': base_url + res_url}) # Search results - for r in doc.xpath('//dl[@class="search_results"]/*'): + for r in eval_xpath(doc, '//dl[@class="search_results"]/*'): try: if r.tag == "dt": - res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1] - title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title')) + res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1] + title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title')) elif r.tag == "dd": - content = extract_text(r.xpath('.')) + content = extract_text(eval_xpath(r, '.')) # append result results.append({'title': title, diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index e77ef0126..0d2c0af2d 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -18,7 +18,7 @@ from json import loads from searx.engines.xpath import extract_text from searx.poolrequests import get from searx.url_utils import urlencode -from searx.utils import match_language +from searx.utils import match_language, eval_xpath # engine dependent config categories = ['general'] @@ -106,19 +106,19 @@ def response(resp): doc = fromstring(resp.text) # parse results - for i, r in enumerate(doc.xpath(result_xpath)): + for i, r in enumerate(eval_xpath(doc, result_xpath)): if i >= 30: break try: - res_url = r.xpath(url_xpath)[-1] + res_url = eval_xpath(r, url_xpath)[-1] except: continue if not res_url: continue - title = extract_text(r.xpath(title_xpath)) - content = extract_text(r.xpath(content_xpath)) + title = extract_text(eval_xpath(r, title_xpath)) + content = extract_text(eval_xpath(r, content_xpath)) # append result results.append({'title': title, diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py index 957a13ea6..79d10c303 100644 --- a/searx/engines/duckduckgo_definitions.py +++ b/searx/engines/duckduckgo_definitions.py @@ -1,3 +1,14 @@ +""" +DuckDuckGo (definitions) + +- `Instant Answer API`_ +- `DuckDuckGo query`_ + +.. _Instant Answer API: https://duckduckgo.com/api +.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1 + +""" + import json from lxml import html from re import compile @@ -25,7 +36,8 @@ def result_to_text(url, text, htmlResult): def request(query, params): params['url'] = url.format(query=urlencode({'q': query})) language = match_language(params['language'], supported_languages, language_aliases) - params['headers']['Accept-Language'] = language.split('-')[0] + language = language.split('-')[0] + params['headers']['Accept-Language'] = language return params @@ -43,8 +55,9 @@ def response(resp): # add answer if there is one answer = search_res.get('Answer', '') - if answer != '': - results.append({'answer': html_to_text(answer)}) + if answer: + if search_res.get('AnswerType', '') not in ['calc']: + results.append({'answer': html_to_text(answer)}) # add infobox if 'Definition' in search_res: diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 444f18c1f..cf2f1a278 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -11,6 +11,7 @@ from lxml import html, etree import re from searx.engines.xpath import extract_text +from searx.utils import eval_xpath from searx.url_utils import quote, urljoin from searx import logger @@ -52,9 +53,9 @@ def response(resp): dom = html.fromstring(resp.text) try: - number_of_results_string = re.sub('[^0-9]', '', dom.xpath( - '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0] - ) + number_of_results_string =\ + re.sub('[^0-9]', '', + eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]) results.append({'number_of_results': int(number_of_results_string)}) @@ -62,12 +63,12 @@ def response(resp): logger.debug("Couldn't read number of results.") pass - for result in dom.xpath('//section[not(contains(@class, "essay"))]'): + for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'): try: - url = result.xpath('.//h2/a')[0].get('href') + url = eval_xpath(result, './/h2/a')[0].get('href') url = urljoin(base_url, url) - title = result.xpath('string(.//h2/a)').strip() - content = extract_text(result.xpath('.//p')) + title = eval_xpath(result, 'string(.//h2/a)').strip() + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, 'title': title, diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 6b0402233..a84f3f69d 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -15,6 +15,7 @@ from json import loads from time import time from lxml.html import fromstring from searx.url_utils import urlencode +from searx.utils import eval_xpath # engine dependent config categories = ['general'] @@ -99,9 +100,9 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = fromstring(resp.text) - links = dom.xpath('//span[@id="menu2"]/a') + links = eval_xpath(dom, '//span[@id="menu2"]/a') for link in links: - href = link.xpath('./@href')[0].split('lang%3A') + href = eval_xpath(link, './@href')[0].split('lang%3A') if len(href) == 2: code = href[1].split('_') if len(code) == 2: diff --git a/searx/engines/google.py b/searx/engines/google.py index 0a0d6ccb1..19bde710d 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -14,7 +14,7 @@ from lxml import html, etree from searx.engines.xpath import extract_text, extract_url from searx import logger from searx.url_utils import urlencode, urlparse, parse_qsl -from searx.utils import match_language +from searx.utils import match_language, eval_xpath logger = logger.getChild('google engine') @@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname): # returns extract_text on the first result selected by the xpath or None def extract_text_from_dom(result, xpath): - r = result.xpath(xpath) + r = eval_xpath(result, xpath) if len(r) > 0: return extract_text(r[0]) return None @@ -227,21 +227,21 @@ def response(resp): # convert the text to dom dom = html.fromstring(resp.text) - instant_answer = dom.xpath('//div[@id="_vBb"]//text()') + instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()') if instant_answer: results.append({'answer': u' '.join(instant_answer)}) try: - results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0] + results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0] .split()[1].replace(',', '')) results.append({'number_of_results': results_num}) except: pass # parse results - for result in dom.xpath(results_xpath): + for result in eval_xpath(dom, results_xpath): try: - title = extract_text(result.xpath(title_xpath)[0]) - url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname) + title = extract_text(eval_xpath(result, title_xpath)[0]) + url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname) parsed_url = urlparse(url, google_hostname) # map result @@ -250,7 +250,7 @@ def response(resp): continue # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start): # print "yooooo"*30 - # x = result.xpath(map_near) + # x = eval_xpath(result, map_near) # if len(x) > 0: # # map : near the location # results = results + parse_map_near(parsed_url, x, google_hostname) @@ -287,11 +287,11 @@ def response(resp): continue # parse suggestion - for suggestion in dom.xpath(suggestion_xpath): + for suggestion in eval_xpath(dom, suggestion_xpath): # append suggestion results.append({'suggestion': extract_text(suggestion)}) - for correction in dom.xpath(spelling_suggestion_xpath): + for correction in eval_xpath(dom, spelling_suggestion_xpath): results.append({'correction': extract_text(correction)}) # return results @@ -300,9 +300,9 @@ def response(resp): def parse_images(result, google_hostname): results = [] - for image in result.xpath(images_xpath): - url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname) - img_src = extract_text(image.xpath(image_img_src_xpath)[0]) + for image in eval_xpath(result, images_xpath): + url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname) + img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0]) # append result results.append({'url': url, @@ -389,10 +389,10 @@ def attributes_to_html(attributes): def _fetch_supported_languages(resp): supported_languages = {} dom = html.fromstring(resp.text) - options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]') + options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]') for option in options: - code = option.xpath('./@value')[0].split('_')[-1] - name = option.xpath('./@data-name')[0].title() + code = eval_xpath(option, './@value')[0].split('_')[-1] + name = eval_xpath(option, './@data-name')[0].title() supported_languages[code] = {"name": name} return supported_languages diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py new file mode 100644 index 000000000..f9b1f99c8 --- /dev/null +++ b/searx/engines/seedpeer.py @@ -0,0 +1,78 @@ +# Seedpeer (Videos, Music, Files) +# +# @website https://seedpeer.me +# @provide-api no (nothing found) +# +# @using-api no +# @results HTML (using search portal) +# @stable yes (HTML can change) +# @parse url, title, content, seed, leech, magnetlink + +from lxml import html +from json import loads +from operator import itemgetter +from searx.url_utils import quote, urljoin +from searx.engines.xpath import extract_text + + +url = 'https://seedpeer.me/' +search_url = url + 'search/{search_term}?page={page_no}' +torrent_file_url = url + 'torrent/{torrent_hash}' + +# specific xpath variables +script_xpath = '//script[@type="text/javascript"][not(@src)]' +torrent_xpath = '(//table)[2]/tbody/tr' +link_xpath = '(./td)[1]/a/@href' +age_xpath = '(./td)[2]' +size_xpath = '(./td)[3]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(search_term=quote(query), + page_no=params['pageno']) + return params + + +# get response from search-request +def response(resp): + results = [] + dom = html.fromstring(resp.text) + result_rows = dom.xpath(torrent_xpath) + + try: + script_element = dom.xpath(script_xpath)[0] + json_string = script_element.text[script_element.text.find('{'):] + torrents_json = loads(json_string) + except: + return [] + + # parse results + for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']): + title = torrent_json['name'] + seed = int(torrent_json['seeds']) + leech = int(torrent_json['peers']) + size = int(torrent_json['size']) + torrent_hash = torrent_json['hash'] + + torrentfile = torrent_file_url.format(torrent_hash=torrent_hash) + magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash) + + age = extract_text(torrent_row.xpath(age_xpath)) + link = torrent_row.xpath(link_xpath)[0] + + href = urljoin(url, link) + + # append result + results.append({'url': href, + 'title': title, + 'content': age, + 'seed': seed, + 'leech': leech, + 'filesize': size, + 'torrentfile': torrentfile, + 'magnetlink': magnetlink, + 'template': 'torrent.html'}) + + # return results sorted by seeder + return sorted(results, key=itemgetter('seed'), reverse=True) diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py index 870998545..284689bf6 100644 --- a/searx/engines/soundcloud.py +++ b/searx/engines/soundcloud.py @@ -51,7 +51,9 @@ def get_client_id(): if response.ok: tree = html.fromstring(response.content) - script_tags = tree.xpath("//script[contains(@src, '/assets/app')]") + # script_tags has been moved from /assets/app/ to /assets/ path. I + # found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js + script_tags = tree.xpath("//script[contains(@src, '/assets/')]") app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None] # extracts valid app_js urls from soundcloud.com content diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 0f0ec6e18..76567396f 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -16,6 +16,7 @@ from datetime import datetime, timedelta import re from searx.engines.xpath import extract_text from searx.languages import language_codes +from searx.utils import eval_xpath # engine dependent config categories = ['general'] @@ -70,8 +71,8 @@ def response(resp): dom = html.fromstring(resp.text) # parse results - for result in dom.xpath(results_xpath): - links = result.xpath(link_xpath) + for result in eval_xpath(dom, results_xpath): + links = eval_xpath(result, link_xpath) if not links: continue link = links[0] @@ -87,8 +88,8 @@ def response(resp): title = extract_text(link) - if result.xpath(content_xpath): - content = extract_text(result.xpath(content_xpath)) + if eval_xpath(result, content_xpath): + content = extract_text(eval_xpath(result, content_xpath)) else: content = '' diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index 5ea2b9958..e913b3915 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -16,7 +16,7 @@ from searx.poolrequests import get from searx.engines.xpath import extract_text from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url from searx.url_utils import urlencode -from searx.utils import match_language +from searx.utils import match_language, eval_xpath from json import loads from lxml.html import fromstring @@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]' media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a' -# xpath_cache -xpath_cache = {} - - -def get_xpath(xpath_str): - result = xpath_cache.get(xpath_str, None) - if not result: - result = etree.XPath(xpath_str) - xpath_cache[xpath_str] = result - return result - - -def eval_xpath(element, xpath_str): - xpath = get_xpath(xpath_str) - return xpath(element) - def get_id_cache(result): id_cache = {} diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 61494ce4e..b75896cc7 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -1,6 +1,6 @@ from lxml import html from lxml.etree import _ElementStringResult, _ElementUnicodeResult -from searx.utils import html_to_text +from searx.utils import html_to_text, eval_xpath from searx.url_utils import unquote, urlencode, urljoin, urlparse search_url = None @@ -104,15 +104,15 @@ def response(resp): results = [] dom = html.fromstring(resp.text) if results_xpath: - for result in dom.xpath(results_xpath): - url = extract_url(result.xpath(url_xpath), search_url) - title = extract_text(result.xpath(title_xpath)) - content = extract_text(result.xpath(content_xpath)) + for result in eval_xpath(dom, results_xpath): + url = extract_url(eval_xpath(result, url_xpath), search_url) + title = extract_text(eval_xpath(result, title_xpath)) + content = extract_text(eval_xpath(result, content_xpath)) tmp_result = {'url': url, 'title': title, 'content': content} # add thumbnail if available if thumbnail_xpath: - thumbnail_xpath_result = result.xpath(thumbnail_xpath) + thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath) if len(thumbnail_xpath_result) > 0: tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url) @@ -120,14 +120,14 @@ def response(resp): else: for url, title, content in zip( (extract_url(x, search_url) for - x in dom.xpath(url_xpath)), - map(extract_text, dom.xpath(title_xpath)), - map(extract_text, dom.xpath(content_xpath)) + x in eval_xpath(dom, url_xpath)), + map(extract_text, eval_xpath(dom, title_xpath)), + map(extract_text, eval_xpath(dom, content_xpath)) ): results.append({'url': url, 'title': title, 'content': content}) if not suggestion_xpath: return results - for suggestion in dom.xpath(suggestion_xpath): + for suggestion in eval_xpath(dom, suggestion_xpath): results.append({'suggestion': extract_text(suggestion)}) return results diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 73b78bcf7..36c1a11f8 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -14,7 +14,7 @@ from lxml import html from searx.engines.xpath import extract_text, extract_url from searx.url_utils import unquote, urlencode -from searx.utils import match_language +from searx.utils import match_language, eval_xpath # engine dependent config categories = ['general'] @@ -109,21 +109,21 @@ def response(resp): dom = html.fromstring(resp.text) try: - results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0] + results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0] .split()[0].replace(',', '')) results.append({'number_of_results': results_num}) except: pass # parse results - for result in dom.xpath(results_xpath): + for result in eval_xpath(dom, results_xpath): try: - url = parse_url(extract_url(result.xpath(url_xpath), search_url)) - title = extract_text(result.xpath(title_xpath)[0]) + url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url)) + title = extract_text(eval_xpath(result, title_xpath)[0]) except: continue - content = extract_text(result.xpath(content_xpath)[0]) + content = extract_text(eval_xpath(result, content_xpath)[0]) # append result results.append({'url': url, @@ -131,7 +131,7 @@ def response(resp): 'content': content}) # if no suggestion found, return results - suggestions = dom.xpath(suggestion_xpath) + suggestions = eval_xpath(dom, suggestion_xpath) if not suggestions: return results @@ -148,9 +148,9 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = html.fromstring(resp.text) - options = dom.xpath('//div[@id="yschlang"]/span/label/input') + options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input') for option in options: - code_parts = option.xpath('./@value')[0][5:].split('_') + code_parts = eval_xpath(option, './@value')[0][5:].split('_') if len(code_parts) == 2: code = code_parts[0] + '-' + code_parts[1].upper() else: diff --git a/searx/results.py b/searx/results.py index a127024c8..3b1e4bd62 100644 --- a/searx/results.py +++ b/searx/results.py @@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2): for url2 in infobox2.get('urls', []): unique_url = True - for url1 in infobox1.get('urls', []): - if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))): + parsed_url2 = urlparse(url2.get('url', '')) + for url1 in urls1: + if compare_urls(urlparse(url1.get('url', '')), parsed_url2): unique_url = False break if unique_url: @@ -188,8 +189,9 @@ class ResultContainer(object): add_infobox = True infobox_id = infobox.get('id', None) if infobox_id is not None: + parsed_url_infobox_id = urlparse(infobox_id) for existingIndex in self.infoboxes: - if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)): + if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id): merge_two_infoboxes(existingIndex, infobox) add_infobox = False diff --git a/searx/settings.yml b/searx/settings.yml index 0384052c1..c6f805331 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -748,6 +748,11 @@ engines: page_size : 10 disabled : True + - name : seedpeer + shortcut : speu + engine : seedpeer + categories: files, music, videos + # - name : yacy # engine : yacy # shortcut : ya diff --git a/searx/templates/courgette/result_templates/torrent.html b/searx/templates/courgette/result_templates/torrent.html index d659064d9..7f94a221e 100644 --- a/searx/templates/courgette/result_templates/torrent.html +++ b/searx/templates/courgette/result_templates/torrent.html @@ -4,7 +4,7 @@ {% endif %}
{% if result.magnetlink %}{{ _('magnet link') }}{% endif %} {% if result.torrentfile %}{{ _('torrent file') }}{% endif %} - - {% if result.seed %}{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}{% endif %} + {% if result.seed is defined %}{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}{% endif %}
diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index f5ea415e2..089367e36 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -3,7 +3,7 @@ {{ result_header(result, favicons) }} {{ result_sub_header(result) }} -{% if result.seed %}{{ icon('transfer') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }}{% endif %} +{% if result.seed is defined %}
{{ icon('transfer') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }}{% endif %}
{% if result.filesize %} • {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }} • {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }} • {{ icon('arrow-swap') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }} • {{ icon('arrow-swap') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }} {{ icon('floppy-disk') }} {{ _('Filesize') }}
{%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}
diff --git a/searx/utils.py b/searx/utils.py
index e61a134f7..5ea9dc89c 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -13,6 +13,7 @@ from numbers import Number
from os.path import splitext, join
from io import open
from random import choice
+from lxml.etree import XPath
import sys
import json
@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
+ "/data/useragents.json", 'r', encoding='utf-8').read())
+xpath_cache = dict()
lang_to_lc_cache = dict()
@@ -450,3 +452,16 @@ def get_engine_from_settings(name):
return engine
return {}
+
+
+def get_xpath(xpath_str):
+ result = xpath_cache.get(xpath_str, None)
+ if result is None:
+ result = XPath(xpath_str)
+ xpath_cache[xpath_str] = result
+ return result
+
+
+def eval_xpath(element, xpath_str):
+ xpath = get_xpath(xpath_str)
+ return xpath(element)
diff --git a/searx/webapp.py b/searx/webapp.py
index 183bf1975..7cf4106d3 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -157,20 +157,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None
@babel.localeselector
def get_locale():
- locale = request.accept_languages.best_match(settings['locales'].keys())
-
- if request.preferences.get_value('locale') != '':
- locale = request.preferences.get_value('locale')
+ if 'locale' in request.form\
+ and request.form['locale'] in settings['locales']:
+ return request.form['locale']
if 'locale' in request.args\
and request.args['locale'] in settings['locales']:
- locale = request.args['locale']
+ return request.args['locale']
- if 'locale' in request.form\
- and request.form['locale'] in settings['locales']:
- locale = request.form['locale']
+ if request.preferences.get_value('locale') != '':
+ return request.preferences.get_value('locale')
- return locale
+ return request.accept_languages.best_match(settings['locales'].keys())
# code-highlighter
diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py
new file mode 100644
index 000000000..2057c1cb1
--- /dev/null
+++ b/tests/unit/engines/test_seedpeer.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from collections import defaultdict
+import mock
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+
+
+class TestBtdiggEngine(SearxTestCase):
+
+ def test_request(self):
+ query = 'test_query'
+ dicto = defaultdict(dict)
+ dicto['pageno'] = 1
+ params = seedpeer.request(query, dicto)
+ self.assertIn('url', params)
+ self.assertIn(query, params['url'])
+ self.assertIn('seedpeer', params['url'])
+
+ def test_response(self):
+ self.assertRaises(AttributeError, seedpeer.response, None)
+ self.assertRaises(AttributeError, seedpeer.response, [])
+ self.assertRaises(AttributeError, seedpeer.response, '')
+ self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+ response = mock.Mock(text='')
+ self.assertEqual(seedpeer.response(response), [])
+
+ html = u"""
+
+
+
+
+
+
+
+
{{ icon('floppy-disk') }} {{ _('Filesize') }}
{% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}
diff --git a/searx/templates/simple/result_templates/torrent.html b/searx/templates/simple/result_templates/torrent.html
index 3c7fd15e8..71c775bc9 100644
--- a/searx/templates/simple/result_templates/torrent.html
+++ b/searx/templates/simple/result_templates/torrent.html
@@ -6,7 +6,7 @@
{% if result.magnetlink %}
+
+
+
+
+ """
+ response = mock.Mock(text=html)
+ results = seedpeer.response(response)
+ self.assertEqual(type(results), list)
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results[0]['title'], 'Title')
+ self.assertEqual(results[0]['url'], 'https://seedpeer.me/link')
+ self.assertEqual(results[0]['seed'], 10)
+ self.assertEqual(results[0]['leech'], 20)
+ self.assertEqual(results[0]['filesize'], 1024)
+ self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123')
+ self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123')
+
+
+
+
+ Title
+ 1 year
+ 1 KB
+ 10
+ 20
+
+