[enh] search language support init

searx/engines/__init__.py

@@ -53,8 +53,14 @@ if not 'engines' in settings or not settings['engines']:
 for engine_data in settings['engines']:
     engine_name = engine_data['engine']
     engine = load_module(engine_name + '.py')
+
     if not hasattr(engine, 'paging'):
         engine.paging = False
+
+    if not hasattr(engine, 'language_support'):
+        #engine.language_support = False
+        engine.language_support = True
+
     for param_name in engine_data:
         if param_name == 'engine':
             continue
@@ -158,7 +164,7 @@ def score_results(results):
     return sorted(results, key=itemgetter('score'), reverse=True)
 
 
-def search(query, request, selected_engines, pageno=1):
+def search(query, request, selected_engines, pageno=1, lang='all'):
     global engines, categories, number_of_searches
     requests = []
     results = {}
@@ -176,11 +182,15 @@ def search(query, request, selected_engines, pageno=1):
         if pageno > 1 and not engine.paging:
             continue
 
+        if lang != 'all' and not engine.language_support:
+            continue
+
         request_params = default_request_params()
         request_params['headers']['User-Agent'] = user_agent
         request_params['category'] = selected_engine['category']
         request_params['started'] = datetime.now()
         request_params['pageno'] = pageno
+        request_params['language'] = lang
         request_params = engine.request(query, request_params)
 
         callback = make_callback(

searx/engines/bing.py

@@ -4,16 +4,22 @@ from cgi import escape
 
 base_url = 'http://www.bing.com/'
 search_string = 'search?{query}&first={offset}'
-locale = 'en-US'  # see http://msdn.microsoft.com/en-us/library/dd251064.aspx
-
 paging = True
+language_support = True
 
 
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
+    if params['language'] == 'all':
+        language = 'en-US'
+    else:
+        language = params['language'].replace('_', '-')
     search_path = search_string.format(
-        query=urlencode({'q': query, 'setmkt': locale}),
+        query=urlencode({'q': query, 'setmkt': language}),
         offset=offset)
+
+    params['cookies']['SRCHHPGUSR'] = \
+        'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]
     #if params['category'] == 'images':
     #    params['url'] = base_url + 'images/' + search_path
     params['url'] = base_url + search_path

searx/engines/google.py

@@ -5,16 +5,21 @@ from json import loads
 
 categories = ['general']
 
-paging = True
-
 url = 'https://ajax.googleapis.com/'
-search_url = url + 'ajax/services/search/web?v=1.0&start={offset}&rsz=large&safe=off&filter=off&{query}'  # noqa
+search_url = url + 'ajax/services/search/web?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}'  # noqa
+
+paging = True
+language_support = True
 
 
 def request(query, params):
     offset = (params['pageno'] - 1) * 8
+    language = 'en-US'
+    if params['language'] != 'all':
+        language = params['language'].replace('_', '-')
     params['url'] = search_url.format(offset=offset,
-                                      query=urlencode({'q': query}))
+                                      query=urlencode({'q': query}),
+                                      language=language)
     return params
 
 

searx/engines/wikipedia.py

@@ -0,0 +1,30 @@
+from json import loads
+from urllib import urlencode, quote
+
+url = 'https://{language}.wikipedia.org/'
+
+search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}'  # noqa
+
+number_of_results = 10
+
+language_support = True
+
+
+def request(query, params):
+    offset = (params['pageno'] - 1) * 10
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('_')[0]
+    params['language'] = language
+    params['url'] = search_url.format(query=urlencode({'srsearch': query}),
+                                      offset=offset,
+                                      language=language)
+    return params
+
+
+def response(resp):
+    search_results = loads(resp.text)
+    res = search_results.get('query', {}).get('search', [])
+    return [{'url': url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')),  # noqa
+        'title': result['title']} for result in res[:int(number_of_results)]]

searx/languages.py

@@ -0,0 +1,59 @@
+language_codes = (
+    ("ar_XA", "Arabic", "Arabia"),
+    ("bg_BG", "Bulgarian", "Bulgaria"),
+    ("cs_CZ", "Czech", "Czech Republic"),
+    ("de_DE", "German", "Germany"),
+    ("da_DK", "Danish", "Denmark"),
+    ("de_AT", "German", "Austria"),
+    ("de_CH", "German", "Switzerland"),
+    ("el_GR", "Greek", "Greece"),
+    ("en_AU", "English", "Australia"),
+    ("en_CA", "English", "Canada"),
+    ("en_GB", "English", "United Kingdom"),
+    ("en_ID", "English", "Indonesia"),
+    ("en_IE", "English", "Ireland"),
+    ("en_IN", "English", "India"),
+    ("en_MY", "English", "Malaysia"),
+    ("en_NZ", "English", "New Zealand"),
+    ("en_PH", "English", "Philippines"),
+    ("en_SG", "English", "Singapore"),
+    ("en_US", "English", "United States"),
+    ("en_XA", "English", "Arabia"),
+    ("en_ZA", "English", "South Africa"),
+    ("es_AR", "Spanish", "Argentina"),
+    ("es_CL", "Spanish", "Chile"),
+    ("es_ES", "Spanish", "Spain"),
+    ("es_MX", "Spanish", "Mexico"),
+    ("es_US", "Spanish", "United States"),
+    ("es_XL", "Spanish", "Latin America"),
+    ("et_EE", "Estonian", "Estonia"),
+    ("fi_FI", "Finnish", "Finland"),
+    ("fr_BE", "French", "Belgium"),
+    ("fr_CA", "French", "Canada"),
+    ("fr_CH", "French", "Switzerland"),
+    ("fr_FR", "French", "France"),
+    ("he_IL", "Hebrew", "Israel"),
+    ("hr_HR", "Croatian", "Croatia"),
+    ("hu_HU", "Hungarian", "Hungary"),
+    ("it_IT", "Italian", "Italy"),
+    ("ja_JP", "Japanese", "Japan"),
+    ("ko_KR", "Korean", "Korea"),
+    ("lt_LT", "Lithuanian", "Lithuania"),
+    ("lv_LV", "Latvian", "Latvia"),
+    ("nb_NO", "Norwegian", "Norway"),
+    ("nl_BE", "Dutch", "Belgium"),
+    ("nl_NL", "Dutch", "Netherlands"),
+    ("pl_PL", "Polish", "Poland"),
+    ("pt_BR", "Portuguese", "Brazil"),
+    ("pt_PT", "Portuguese", "Portugal"),
+    ("ro_RO", "Romanian", "Romania"),
+    ("ru_RU", "Russian", "Russia"),
+    ("sk_SK", "Slovak", "Slovak Republic"),
+    ("sl_SL", "Slovenian", "Slovenia"),
+    ("sv_SE", "Swedish", "Sweden"),
+    ("th_TH", "Thai", "Thailand"),
+    ("tr_TR", "Turkish", "Turkey"),
+    ("uk_UA", "Ukrainian", "Ukraine"),
+    ("zh_CN", "Chinese", "China"),
+    ("zh_HK", "Chinese", "Hong Kong SAR"),
+    ("zh_TW", "Chinese", "Taiwan"))

searx/settings.yml

@@ -7,8 +7,7 @@ server:
 
 engines:
   - name : wikipedia
-    engine : mediawiki
+    engine : wikipedia
-    url    : https://en.wikipedia.org/
     number_of_results : 1
     paging : False
 

searx/static/css/style.css

@@ -152,7 +152,7 @@ tr:hover td { background: #DDDDDD; }
 #results { margin: 10px; padding: 0; margin-bottom: 20px; }
 
 #sidebar { position: absolute; left: 54em; width: 12em; margin: 0 2px 5px 5px; padding: 0 2px 2px 2px; }
-#suggestions span { display: block; margin: 0 2px 10px 2px; padding: 0; }
+#suggestions span { display: block; margin: 0 2px 2px 2px; padding: 0; }
 #suggestions form { display: block; }
 #suggestions input { padding: 2px 6px; margin: 2px 4px;  font-size: 0.8em; display: inline-block; background: #3498DB; color: #FFFFFF; border-radius: 4px; border: 0; cursor: pointer; }
 
@@ -177,6 +177,7 @@ tr:hover td { background: #DDDDDD; }
 }
 
 #apis {
+    margin-top: 8px;
     clear: both;
 }
 

searx/templates/preferences.html

@@ -11,6 +11,17 @@
         {% include 'categories.html' %}
         </p>
     </fieldset>
+    <fieldset>
+        <legend>{{ _('Search language') }}</legend>
+        <p>
+        <select name='language'>
+            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
+            {% for lang_id,lang_name,country_name in language_codes %}
+            <option value={{ lang_id }} {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name}} ({{ country_name }})</option>
+            {% endfor %}
+        </select>
+        </p>
+    </fieldset>
     <fieldset>
         <legend>{{ _('Interface language') }}</legend>
         <p>

searx/webapp.py

@@ -29,6 +29,7 @@ from searx import settings, searx_dir
 from searx.engines import search, categories, engines, get_engines_stats
 from searx.utils import UnicodeWriter
 from searx.utils import highlight_content, html_to_text
+from searx.languages import language_codes
 
 from flask.ext.babel import Babel
 
@@ -117,6 +118,11 @@ def parse_query(query):
 @app.route('/', methods=['GET', 'POST'])
 def index():
     paging = False
+    lang = 'all'
+
+    if request.cookies.get('language')\
+       and request.cookies['language'] in (x[0] for x in language_codes):
+        lang = request.cookies['language']
 
     if request.method == 'POST':
         request_data = request.form
@@ -159,7 +165,11 @@ def index():
                                      'name': x.name}
                                     for x in categories[categ])
 
-    results, suggestions = search(query, request, selected_engines, pageno)
+    results, suggestions = search(query,
+                                  request,
+                                  selected_engines,
+                                  pageno,
+                                  lang)
 
     for result in results:
         if not paging and engines[result['engine']].paging:
@@ -232,6 +242,11 @@ def list_engines():
 
 @app.route('/preferences', methods=['GET', 'POST'])
 def preferences():
+    lang = None
+
+    if request.cookies.get('language')\
+       and request.cookies['language'] in (x[0] for x in language_codes):
+        lang = request.cookies['language']
 
     if request.method == 'POST':
         selected_categories = []
@@ -244,6 +259,10 @@ def preferences():
                 selected_categories.append(category)
             elif pd_name == 'locale' and pd in settings['locales']:
                 locale = pd
+            elif pd_name == 'language' and (pd == 'all' or
+                                            pd in (x[0] for
+                                                   x in language_codes)):
+                lang = pd
 
         resp = make_response(redirect('/'))
 
@@ -254,6 +273,13 @@ def preferences():
                 max_age=60 * 60 * 24 * 7 * 4
             )
 
+        if lang:
+            # cookie max age: 4 weeks
+            resp.set_cookie(
+                'language', lang,
+                max_age=60 * 60 * 24 * 7 * 4
+            )
+
         if selected_categories:
             # cookie max age: 4 weeks
             resp.set_cookie(
@@ -263,7 +289,9 @@ def preferences():
         return resp
     return render('preferences.html',
                   locales=settings['locales'],
-                  current_locale=get_locale())
+                  current_locale=get_locale(),
+                  current_language=lang or 'all',
+                  language_codes=language_codes)
 
 
 @app.route('/stats', methods=['GET'])