Merge branch 'master' into flask_perimeter

manage.sh

@@ -14,6 +14,36 @@ update_dev_packages() {
     pip install --upgrade -r "$BASE_DIR/requirements-dev.txt"
 }
 
+check_geckodriver() {
+    echo '[!] Checking geckodriver'
+    set -e
+    geckodriver -V 2>1 > /dev/null || NOTFOUND=1
+    set +e
+    if [ -z $NOTFOUND ]; then
+	return
+    fi
+    GECKODRIVER_VERSION="v0.11.1"
+    PLATFORM=`python -c "import platform; print platform.system().lower(), platform.architecture()[0]"`
+    case $PLATFORM in
+	"linux 32bit" | "linux2 32bit") ARCH="linux32";;
+	"linux 64bit" | "linux2 64bit") ARCH="linux64";;
+	"windows 32 bit") ARCH="win32";;
+	"windows 64 bit") ARCH="win64";;
+	"mac 64bit") ARCH="macos";;
+    esac
+    GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+    if [ -z "$VIRTUAL_ENV" ]; then
+	echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n  $GECKODRIVER_URL"
+	exit
+    else
+	echo "Installing $VIRTUAL_ENV from\n  $GECKODRIVER_URL"
+	FILE=`mktemp`
+	wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $VIRTUAL_ENV/bin/ -f $FILE geckodriver
+	rm $FILE
+	chmod 777 $VIRTUAL_ENV/bin/geckodriver
+    fi
+}
+
 pep8_check() {
     echo '[!] Running pep8 check'
     # ignored rules:
@@ -43,6 +73,7 @@ tests() {
     set -e
     pep8_check
     unit_tests
+    check_geckodriver
     robot_tests
     set +e
 }
@@ -88,6 +119,7 @@ Commands
     unit_tests           - Run unit tests
     update_dev_packages  - Check & update development and production dependency changes
     update_packages      - Check & update dependency changes
+    check_geckodriver    - Check & download geckodriver (required for robot_tests)
 "
 }
 

requirements.txt

@@ -1,12 +1,12 @@
 certifi==2016.9.26
-flask==0.11.1
+flask==0.12
 flask-babel==0.11.1
-lxml==3.6.0
+lxml==3.7.1
-ndg-httpsclient==0.4.1
+ndg-httpsclient==0.4.2
 pyasn1==0.1.9
 pyasn1-modules==0.0.8
 pygments==2.1.3
-pyopenssl==0.15.1
+pyopenssl==16.2.0
 python-dateutil==2.5.3
 pyyaml==3.11
-requests[socks]==2.10.0
+requests[socks]==2.12.4

searx/autocomplete.py

@@ -81,22 +81,22 @@ def searx_bang(full_query):
             engine_query = full_query.getSearchQuery()[1:]
 
             for lc in language_codes:
-                lang_id, lang_name, country, english_name = map(str.lower, lc)
+                lang_id, lang_name, country, english_name = map(unicode.lower, lc)
 
                 # check if query starts with language-id
                 if lang_id.startswith(engine_query):
                     if len(engine_query) <= 2:
-                        results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
+                        results.append(u':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
                     else:
-                        results.append(':{lang_id}'.format(lang_id=lang_id))
+                        results.append(u':{lang_id}'.format(lang_id=lang_id))
 
                 # check if query starts with language name
                 if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
-                    results.append(':{lang_name}'.format(lang_name=lang_name))
+                    results.append(u':{lang_name}'.format(lang_name=lang_name))
 
                 # check if query starts with country
                 if country.startswith(engine_query.replace('_', ' ')):
-                    results.append(':{country}'.format(country=country.replace(' ', '_')))
+                    results.append(u':{country}'.format(country=country.replace(' ', '_')))
 
     # remove duplicates
     result_set = set(results)

searx/engines/xpath.py

@@ -42,7 +42,9 @@ def extract_text(xpath_results):
         return ''.join(xpath_results)
     else:
         # it's a element
-        return html_to_text(xpath_results.text_content()).strip()
+        text = html.tostring(xpath_results, encoding='unicode', method='text', with_tail=False)
+        text = text.strip().replace('\n', ' ')
+        return ' '.join(text.split())
 
 
 def extract_url(xpath_results, search_url):

searx/static/plugins/js/infinite_scroll.js

@@ -4,13 +4,9 @@ $(document).ready(function() {
         if ($(document).height() - win.height() == win.scrollTop()) {
             var formData = $('#pagination form:last').serialize();
             if (formData) {
-                var pageno = $('#pagination input[name=pageno]:last').attr('value');
                 $('#pagination').html('<div class="loading-spinner"></div>');
                 $.post('./', formData, function (data) {
-                    var lastImageHref = $('.result-images:last a').attr('href');
                     var body = $(data);
-                    $('a[href^="#open-modal"]:last').attr('href', '#open-modal-1-' + pageno);
-                    body.find('.modal-image a:first').attr('href', lastImageHref);
                     $('#pagination').remove();
                     $('#main_results').append('<hr/>');
                     $('#main_results').append(body.find('.result'));

searx/static/themes/oscar/less/logicodev/modal-pic.less

@@ -1,77 +0,0 @@
-.modal-image {
-    position: fixed;
-    top: 0;
-    right: 0;
-    bottom: 0;
-    left: 0;
-    background: rgba(0,0,0,0.8);
-    z-index: 1000000001;
-    opacity:0 !important;
-    pointer-events: none;
-
-    button {
-        display: none;
-    }
-
-    &:target {
-        opacity: 1 !important;
-        pointer-events: auto;
-    }
-
-    & > div {
-        margin: 2% auto;
-        width: 97%;
-        background: @dim-gray;
-        border: @gray 0.1rem solid;
-    }
-
-    @media (min-width: 769px) {
-        & > div {
-            max-width: 60.0rem;
-        }
-    }
-
-    .image-paging-left {
-        margin-right: 1.0rem;
-        margin-top: 0.5rem;
-        width:15px;
-        height:15px;
-        background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA8AAAAPCAYAAAA71pVKAAAABmJLR0QA/wD/AP+gvaeTAAAACXBI
-WXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4AweDQoOuikqUQAAAB1pVFh0Q29tbWVudAAAAAAAQ3Jl
-YXRlZCB3aXRoIEdJTVBkLmUHAAAAiElEQVQoz6XTrQ0CQRCG4SesQHI5g6EAqIEewNLSVUACzfAT
-BApDDSgSBAaJORKyauf2czOZdybzl5SpxR5j3H/OUQHYoMMMNwE1fcUT5hFwUgPuenAxBDxHwRZb
-HKMgbPDCuiQ4ZfYDU6xwxTNafXDP1dOu3nP1heUJDnmCVAB/cMES7/+v+gIq0Bs3k6NL9AAAAABJ
-RU5ErkJggg==) 96% no-repeat;
-    }
-
-    .image-paging-right {
-        margin-left: 1.2rem;
-        margin-top: 0.5rem;
-        width:15px;
-        height:15px;
-        background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA8AAAAPCAYAAAA71pVKAAAABmJLR0QA/wD/AP+gvaeTAAAACXBI
-        WXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4AweDQon+JuyPQAAAB1pVFh0Q29tbWVudAAAAAAAQ3Jl
-        YXRlZCB3aXRoIEdJTVBkLmUHAAAAaklEQVQoz73TsQ2DUAxF0SMWAFEzwGcaWhgpEyAlbQYJMACj
-        sAINTaiIf8Tt3FzL9jPfDHijdoHiVK9o8EAlQMKM1z8EZUTQHoJnjmDJFUwYr17hTIcN/W2dwzOH
-        tx2+czhhCZ9oNH/6qh1F2RaYgWxrQwAAAABJRU5ErkJggg==);
-
-    }
-
-    .image-container::before {
-        display: block;
-        min-width: 1.0rem;
-        max-width: 60.0rem;
-        min-height: 10.0rem;
-        height: 30.0rem;
-        content: "";
-    }
-}
-
-.modal-close {
-    position:fixed;
-    top: 0;
-    left: 0;
-    height: 100% !important;
-    width: 100% !important;
-    z-index: -1;
-}

searx/static/themes/oscar/less/logicodev/oscar.less

@@ -19,5 +19,3 @@
 @import "cursor.less";
 
 @import "code.less";
-
-@import "modal-pic.less";

searx/static/themes/oscar/less/pointhi/oscar.less

@@ -17,7 +17,3 @@
 @import "code.less";
 
 @import "navbar.less";
-
-@import "../logicodev/variables.less";
-
-@import "../logicodev/modal-pic.less";

searx/templates/oscar/result_templates/images.html

@@ -1,54 +1,39 @@
-<a href="#open-modal-{{ index }}-{{ pageno }}">
+{% from 'oscar/macros.html' import draw_favicon %}
-    <img src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" class="img-thumbnail" id="img-result-thumb-{{ index }}" />
+
+<a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} data-toggle="modal" data-target="#modal-{{ index }}-{{pageno}}">
+    <img src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" class="img-thumbnail">
 </a>
 
-<style type="text/css" media="screen">
+<div class="modal fade" id="modal-{{ index }}-{{ pageno }}" tabindex="-1" role="dialog" aria-hidden="true">
-#open-modal-{{ index }}-{{ pageno }}:target .image-container::before {
+    <div class="modal-dialog">
-    background: url({{ image_proxify(result.img_src)|safe }}) no-repeat center/contain;
+        <div class="modal-wrapper">
-}
+            <div class="modal-header">
-</style>
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
-
+                <h4 class="modal-title">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result.title|striptags }}</h4>
-<div id="open-modal-{{ index }}-{{ pageno }}" class="modal-image">
-    <div class="container modal-dialog">
-        <div class="row">
-            <div class="col-md-12 col-sm-12 col-xs-12 modal-header">
-                <a {% if index != 1 %}href="#open-modal-{{ index-1 }}-{{ pageno }}"{% endif %}>
-                    <span class="pull-left image-paging-left"></span>
-                 </a>
-                <a href="#open-modal-{{ index+1 }}-{{ pageno }}">
-                    <span class="pull-right image-paging-right"></span>
-                </a>
-                <h4 class="modal-title image-title">{{ result.title|striptags }}</h4>
             </div>
-        </div>
+            <div class="modal-body">
-        <div class="row">
+                <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
-            <div class="col-md-12 col-sm-12 col-xs-12 modal-body">
-                <a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}><div class="image-container"></div></a>
-            </div>
-        </div>
-        {% if result.content %}
-        <div class="row">
-            <div class="col-md-12 hidden-sm hidden-xs modal-body">
-                <p class="result-content">{{ result.content|safe }}</p>
                 {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
+                {% if result.content %}
+                    <p class="result-content">
+                        {{ result.content }}
+                    </p>
+                {% endif %}
             </div>
-        </div>
+            <div class="modal-footer">
-        {% endif %}
+                <div class="clearfix"></div>
-        <div class="modal-footer">
+                <span class="label label-default pull-right">{{ result.engine }}</span>
-            <div class="row">
+                <p class="text-muted pull-left">{{ result.pretty_url }}</p>
-                <div class="col-md-10 col-xs-12">
+                <div class="clearfix"></div>
-                    <p class="text-muted pull-left">{{ result.pretty_url }}</p>
+				<div class="row">
+                    <div class="col-md-6">
+                        <a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('Get image') }}</a>
+                    </div>
+                    <div class="col-md-6">
+                        <a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('View source') }}</a>
+                    </div>
                 </div>
-                <div class="col-md-2 hidden-sm hidden-xs">
-                    <span class="label label-default pull-right">{{ result.engine }}</span>
-                </div>
-            </div>
-            <div class="row">
-                <a href="{{ result.url }}" class="btn btn-default">
-                    {{ _('View source') }}
-                </a>
             </div>
         </div>
     </div>
-    <a href="#img-result-thumb-{{ index }}-{{ pageno }}" class="modal-close"></a>
 </div>

utils/google_search.py

@@ -13,10 +13,9 @@ request_params = default_request_params()
 # Possible params
 # request_params['headers']['User-Agent'] = ''
 # request_params['category'] = ''
-# request_params['started'] = ''
-
 request_params['pageno'] = 1
 request_params['language'] = 'en_us'
+request_params['time_range'] = ''
 
 params = google.request(argv[1], request_params)
 
@@ -32,5 +31,5 @@ else:
     request_args['data'] = request_params['data']
 
 resp = req(request_params['url'], **request_args)
-
+resp.search_params = request_params
 print(dumps(google.response(resp)))

utils/standalone_searx.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+'''
+searx is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+searx is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with searx. If not, see < http://www.gnu.org/licenses/ >.
+
+(C) 2016- by Alexandre Flament, <alex@al-f.net>
+'''
+
+# set path
+from sys import path
+from os.path import realpath, dirname
+path.append(realpath(dirname(realpath(__file__)) + '/../'))
+
+# initialization
+from json import dumps
+from searx import settings
+import searx.query
+import searx.search
+import searx.engines
+import searx.preferences
+import argparse
+
+searx.engines.initialize_engines(settings['engines'])
+
+# command line parsing
+parser = argparse.ArgumentParser(description='Standalone searx.')
+parser.add_argument('query', type=str,
+                    help='Text query')
+parser.add_argument('--category', type=str, nargs='?',
+                    choices=searx.engines.categories.keys(),
+                    default='general',
+                    help='Search category')
+parser.add_argument('--lang', type=str, nargs='?',default='all',
+                    help='Search language')
+parser.add_argument('--pageno', type=int, nargs='?', default=1,
+                    help='Page number starting from 1')
+parser.add_argument('--safesearch', type=str, nargs='?', choices=['0', '1', '2'], default='0',
+                    help='Safe content filter from none to strict')
+parser.add_argument('--timerange', type=str, nargs='?', choices=['day', 'week', 'month', 'year'],
+                    help='Filter by time range')
+args = parser.parse_args()
+
+# search results for the query
+form = {
+    "q":args.query,
+    "categories":args.category.decode('utf-8'),
+    "pageno":str(args.pageno),
+    "language":args.lang,
+    "time_range":args.timerange
+}
+preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])
+preferences.key_value_settings['safesearch'].parse(args.safesearch)
+
+search_query = searx.search.get_search_query_from_webapp(preferences, form)
+search = searx.search.Search(search_query)
+result_container = search.search()
+
+# output
+from datetime import datetime
+
+def no_parsed_url(results):
+    for result in results:
+        del result['parsed_url']
+    return results
+
+def json_serial(obj):
+    """JSON serializer for objects not serializable by default json code"""
+    if isinstance(obj, datetime):
+        serial = obj.isoformat()
+        return serial
+    raise TypeError ("Type not serializable")
+
+result_container_json = {
+    "search": {
+        "q": search_query.query,
+        "pageno": search_query.pageno,
+        "lang": search_query.lang,
+        "safesearch": search_query.safesearch,
+        "timerange": search_query.time_range,
+        "engines": search_query.engines  
+    },
+    "results": no_parsed_url(result_container.get_ordered_results()),
+    "infoboxes": result_container.infoboxes,
+    "suggestions": list(result_container.suggestions),
+    "answers": list(result_container.answers),
+    "paging": result_container.paging,
+    "results_number": result_container.results_number()
+}
+
+print(dumps(result_container_json, sort_keys=True, indent=4, ensure_ascii=False, encoding="utf-8", default=json_serial))