Merge branch 'master' into master

AUTHORS.rst

@@ -77,3 +77,9 @@ generally made searx better:
 - @pyrrh0n1c
 - @cclauss
 - QGW @moon2l
+- Pierre-Alain Toret @daftaupe
+- Matthew Olmsted @icegiant
+- Michael Tran @trankmichael
+- Joseph Nuthalapati @josephkiranbabu
+- @maiki
+- Richard Didier @zeph33

CHANGELOG.rst

@@ -1,3 +1,16 @@
+0.14.0 2018.02.19
+=================
+
+- New theme: oscar-logicodev dark
+- New engines
+
+  - AskSteem (general)
+- Autocompleter fix for py3
+- Engine fixes (pdbe, pubmed, gigablast, google, yacy, bing videos, microsoft academic)
+- "All" option is removed from languages
+- Minor UI changes
+- Translation updates
+
 0.13.1 2017.11.23
 =================
 

manage.sh

@@ -3,14 +3,6 @@
 BASE_DIR="$(dirname -- "`readlink -f -- "$0"`")"
 export PATH="$BASE_DIR/node_modules/.bin":$PATH
 
-# the script can be sourced to update the PATH
-# see https://stackoverflow.com/questions/2683279/how-to-detect-if-a-script-is-being-sourced
-if [ $_ != $0 ]; then
-    unset BASE_DIR
-    # sourced : exit now
-    return
-fi
-
 # subshell
 PYTHONPATH="$BASE_DIR"
 SEARX_DIR="$BASE_DIR/searx"
@@ -56,7 +48,7 @@ install_geckodriver() {
 
     if [ -z "$1" ]; then
         if [ -z "$VIRTUAL_ENV" ]; then
-            echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n  $GECKODRIVER_URL"
+            printf "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n  %s" "$GECKODRIVER_URL"
             exit
         else
             GECKODRIVER_DIR="$VIRTUAL_ENV/bin"
@@ -66,7 +58,7 @@ install_geckodriver() {
         mkdir -p -- "$GECKODRIVER_DIR"
     fi
 
-    echo "Installing $GECKODRIVER_DIR/geckodriver from\n  $GECKODRIVER_URL"
+    printf "Installing %s/geckodriver from\n  %s" "$GECKODRIVER_DIR" "$GECKODRIVER_URL"
 
     FILE="`mktemp`"
     wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "$GECKODRIVER_DIR" -f "$FILE" geckodriver

searx/engines/__init__.py

@@ -230,12 +230,12 @@ def load_engines(engine_list):
 
 def initialize_engines(engine_list):
     load_engines(engine_list)
-    for engine in engines.items():
+    for engine_name, engine in engines.items():
         if hasattr(engine, 'init'):
-            init_fn = getattr(engine, engine_attr)
+            init_fn = getattr(engine, 'init')
 
             def engine_init():
                 init_fn()
-                logger.debug('%s engine initialized', engine_data['name'])
+                logger.debug('%s engine initialized', engine_name)
-            logger.debug('Starting background initialization of %s engine', engine_data['name'])
+            logger.debug('Starting background initialization of %s engine', engine_name)
             threading.Thread(target=engine_init).start()

searx/engines/asksteem.py

@@ -0,0 +1,44 @@
+"""
+ Asksteem (general)
+
+ @website     https://asksteem.com/
+ @provide-api yes
+
+ @using-api   yes
+ @results     JSON (https://github.com/Hoxly/asksteem-docs/wiki)
+ @stable      yes
+ @parse       url, title, content
+"""
+
+from json import loads
+from searx.url_utils import urlencode
+
+# engine dependent config
+categories = ['general']
+paging = True
+language_support = False
+disabled = True
+
+# search-url
+search_url = 'https://api.asksteem.com/search?{params}'
+result_url = 'https://steemit.com/@{author}/{title}'
+
+
+# do search-request
+def request(query, params):
+    url = search_url.format(params=urlencode({'q': query, 'pg': params['pageno']}))
+    params['url'] = url
+    return params
+
+
+# get response from search-request
+def response(resp):
+    json = loads(resp.text)
+
+    results = []
+
+    for result in json.get('results', []):
+        results.append({'url': result_url.format(author=result['author'], title=result['permlink']),
+                        'title': result['title'],
+                        'content': result['summary']})
+    return results

searx/engines/microsoft_academic.py

@@ -0,0 +1,75 @@
+"""
+Microsoft Academic (Science)
+
+@website     https://academic.microsoft.com
+@provide-api yes
+@using-api   no
+@results     JSON
+@stable      no
+@parse       url, title, content
+"""
+
+from datetime import datetime
+from json import loads
+from uuid import uuid4
+
+from searx.url_utils import urlencode
+from searx.utils import html_to_text
+
+categories = ['images']
+paging = True
+result_url = 'https://academic.microsoft.com/api/search/GetEntityResults?{query}'
+
+
+def request(query, params):
+    correlation_id = uuid4()
+    msacademic = uuid4()
+    time_now = datetime.now()
+
+    params['url'] = result_url.format(query=urlencode({'correlationId': correlation_id}))
+    params['cookies']['msacademic'] = str(msacademic)
+    params['cookies']['ai_user'] = 'vhd0H|{now}'.format(now=str(time_now))
+    params['method'] = 'POST'
+    params['data'] = {
+        'Query': '@{query}@'.format(query=query),
+        'Limit': 10,
+        'Offset': params['pageno'] - 1,
+        'Filters': '',
+        'OrderBy': '',
+        'SortAscending': False,
+    }
+
+    return params
+
+
+def response(resp):
+    results = []
+    response_data = loads(resp.text)
+
+    for result in response_data['results']:
+        url = _get_url(result)
+        title = result['e']['dn']
+        content = _get_content(result)
+        results.append({
+            'url': url,
+            'title': html_to_text(title),
+            'content': html_to_text(content),
+        })
+
+    return results
+
+
+def _get_url(result):
+    if 's' in result['e']:
+        return result['e']['s'][0]['u']
+    return 'https://academic.microsoft.com/#/detail/{pid}'.format(pid=result['id'])
+
+
+def _get_content(result):
+    if 'd' in result['e']:
+        content = result['e']['d']
+        if len(content) > 300:
+            return content[:300] + '...'
+        return content
+
+    return ''

searx/settings.yml

@@ -66,6 +66,10 @@ engines:
     categories : science
     timeout : 4.0
 
+  - name : asksteem
+    engine : asksteem
+    shortcut : as
+
   - name : base
     engine : base
     shortcut : bs
@@ -398,15 +402,7 @@ engines:
     shortcut : lo
 
   - name : microsoft academic
-    engine : json_engine
+    engine : microsoft_academic
-    paging : True
-    search_url : https://academic.microsoft.com/api/search/GetEntityResults?query=%40{query}%40&filters=&offset={pageno}&limit=8&correlationId=undefined
-    results_query : results
-    url_query : u
-    title_query : dn
-    content_query : d
-    page_size : 8
-    first_page_num : 0
     categories : science
     shortcut : ma
 
@@ -719,6 +715,7 @@ locales:
     eo : Esperanto (Esperanto)
     es : Español (Spanish)
     fi : Suomi (Finnish)
+    fil : Wikang Filipino (Filipino)
     fr : Français (French)
     he : עברית (Hebrew)
     hr : Hrvatski (Croatian)
@@ -737,6 +734,7 @@ locales:
     tr : Türkçe (Turkish)
     uk : українська мова (Ukrainian)
     zh : 中文 (Chinese)
+    zh_TW : 國語 (Taiwanese Mandarin)
 
 doi_resolvers :
   oadoi.org : 'https://oadoi.org/'

searx/translations/es/LC_MESSAGES/messages.po

@@ -5,11 +5,12 @@
 # Translators:
 # Adam Tauber <asciimoo@gmail.com>, 2015
 # Alejandro León Aznar, 2014
-# Alejandro León Aznar, 2014-2017
+# Alejandro León Aznar, 2014-2018
 # Carmen Fernández B., 2016
 # juanda097 <juanda097@protonmail.ch>, 2016
 # juanda097 <juanda097@protonmail.ch>, 2017
 # Marc Abonce Seguin, 2016
+# Marc Abonce Seguin, 2018
 # Oscar <ocf@openmailbox.org>, 2015
 # rivera valdez <riveravaldezmail@gmail.com>, 2016
 # wefwefew ewfewfewf <nnnedmz0d@moakt.ws>, 2016
@@ -18,8 +19,8 @@ msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2017-11-01 21:31+0100\n"
-"PO-Revision-Date: 2017-12-18 06:34+0000\n"
+"PO-Revision-Date: 2018-02-16 05:29+0000\n"
-"Last-Translator: juanda097 <juanda097@protonmail.ch>\n"
+"Last-Translator: Marc Abonce Seguin\n"
 "Language-Team: Spanish (http://www.transifex.com/asciimoo/searx/language/es/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -738,7 +739,7 @@ msgstr "Atajo"
 #: searx/templates/oscar/preferences.html:164
 #: searx/templates/oscar/preferences.html:174
 msgid "Selected language"
-msgstr "Elegir idioma"
+msgstr "Idioma elegido"
 
 #: searx/templates/oscar/preferences.html:166
 #: searx/templates/oscar/preferences.html:172

searx/translations/nl/LC_MESSAGES/messages.po

@@ -4,15 +4,15 @@
 # 
 # Translators:
 # André Koot <meneer@tken.net>, 2014-2017
-# Nathan Follens, 2015-2017
+# Nathan Follens, 2015-2018
 # Rejo Zenger <rejo@zenger.nl>, 2016-2017
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2017-11-01 21:31+0100\n"
-"PO-Revision-Date: 2017-12-17 17:06+0000\n"
+"PO-Revision-Date: 2018-02-15 22:43+0000\n"
-"Last-Translator: André Koot <meneer@tken.net>\n"
+"Last-Translator: Nathan Follens\n"
 "Language-Team: Dutch (http://www.transifex.com/asciimoo/searx/language/nl/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -55,7 +55,7 @@ msgstr "afbeeldingen"
 
 #: searx/webapp.py:141
 msgid "videos"
-msgstr "video's"
+msgstr "video’s"
 
 #: searx/webapp.py:142
 msgid "it"
@@ -115,7 +115,7 @@ msgstr "Snelheid zoekmachine (sec)"
 
 #: searx/engines/__init__.py:198 searx/engines/flycheck___init__.py:205
 msgid "Page loads (sec)"
-msgstr "Pagina laadt (sec)"
+msgstr "Laden van pagina’s (sec)"
 
 #: searx/engines/__init__.py:202 searx/engines/flycheck___init__.py:209
 #: searx/templates/oscar/results.html:95
@@ -145,7 +145,7 @@ msgstr "Dit object is vervangen door"
 
 #: searx/engines/pubmed.py:74
 msgid "No abstract is available for this publication."
-msgstr "Voor deze publicatie is geen samenvatting beschikbaar."
+msgstr "Voor deze publicatie is geen abstract beschikbaar."
 
 #: searx/plugins/https_rewrite.py:32
 msgid "Rewrite HTTP links to HTTPS if possible"
@@ -157,7 +157,7 @@ msgstr "Oneindig scrollen"
 
 #: searx/plugins/infinite_scroll.py:4
 msgid "Automatically load next page when scrolling to bottom of current page"
-msgstr "Automatisch nieuwe pagina laden bij het bereiken van de onderkant van de huidige pagina"
+msgstr "Volgende pagina automatisch laden bij bereiken van de onderkant van de huidige pagina"
 
 #: searx/plugins/oa_doi_rewrite.py:9
 msgid "Open Access DOI rewrite"
@@ -195,11 +195,11 @@ msgstr "Zoekopdracht onmiddellijk uitvoeren wanneer een categorie geselecteerd w
 msgid ""
 "Displays your IP if the query is \"ip\" and your user agent if the query "
 "contains \"user agent\"."
-msgstr "Geeft je IP-adres weer als de zoekopdracht \"ip\" is en je gebruikersagent als de zoekopdracht \"user agent\" bevat."
+msgstr "Geeft je IP-adres weer als de zoekopdracht ‘ip’ is en je gebruikersagent als de zoekopdracht ‘user agent’ bevat."
 
 #: searx/plugins/tracker_url_remover.py:26
 msgid "Tracker URL remover"
-msgstr "Tracker-URL verwijderaar"
+msgstr "Tracker-URL-verwijderaar"
 
 #: searx/plugins/tracker_url_remover.py:27
 msgid "Remove trackers arguments from the returned URL"
@@ -213,7 +213,7 @@ msgstr "Sneltoetsen als in Vim"
 msgid ""
 "Navigate search results with Vim-like hotkeys (JavaScript required). Press "
 "\"h\" key on main or result page to get help."
-msgstr "Blader door zoekresultaten met sneltoetsen zoals die in Vim (JavaScript vereist). Druk op \"h\" op de hoofdpagina of de pagina met resultaten voor hulp."
+msgstr "Blader door zoekresultaten met sneltoetsen zoals die in Vim (JavaScript vereist). Druk op ‘h’ op de hoofdpagina of de pagina met resultaten voor hulp."
 
 #: searx/templates/courgette/404.html:4 searx/templates/legacy/404.html:4
 #: searx/templates/oscar/404.html:4 searx/templates/pix-art/404.html:4
@@ -365,7 +365,7 @@ msgstr "Geen"
 #: searx/templates/pix-art/preferences.html:39
 #: searx/templates/simple/preferences.html:131
 msgid "Themes"
-msgstr "Thema's"
+msgstr "Thema’s"
 
 #: searx/templates/courgette/preferences.html:83
 msgid "Color"
@@ -538,21 +538,21 @@ msgstr "Zoekmachinestatistieken"
 #: searx/templates/legacy/result_templates/images.html:4
 #: searx/templates/pix-art/result_templates/images.html:4
 msgid "original context"
-msgstr "originele inhoud"
+msgstr "oorspronkelijke context"
 
 #: searx/templates/courgette/result_templates/torrent.html:7
 #: searx/templates/legacy/result_templates/torrent.html:11
 #: searx/templates/oscar/result_templates/torrent.html:6
 #: searx/templates/simple/result_templates/torrent.html:9
 msgid "Seeder"
-msgstr "Aanbieder"
+msgstr "Seeders"
 
 #: searx/templates/courgette/result_templates/torrent.html:7
 #: searx/templates/legacy/result_templates/torrent.html:11
 #: searx/templates/oscar/result_templates/torrent.html:6
 #: searx/templates/simple/result_templates/torrent.html:9
 msgid "Leecher"
-msgstr "Ophaler"
+msgstr "Leechers"
 
 #: searx/templates/courgette/result_templates/torrent.html:9
 #: searx/templates/legacy/result_templates/torrent.html:9
@@ -597,7 +597,7 @@ msgstr "Uit"
 #: searx/templates/oscar/macros.html:34 searx/templates/oscar/macros.html:48
 #: searx/templates/simple/macros.html:43
 msgid "cached"
-msgstr "gecached"
+msgstr "gecachet"
 
 #: searx/templates/oscar/advanced.html:4
 msgid "Advanced settings"
@@ -673,7 +673,7 @@ msgstr "Welke taal wil je gebruiken voor het zoeken?"
 #: searx/templates/oscar/preferences.html:48
 #: searx/templates/simple/preferences.html:128
 msgid "Change the language of the layout"
-msgstr "Wijzig de taal van de layout"
+msgstr "Wijzig de taal van de lay-out"
 
 #: searx/templates/oscar/preferences.html:58
 #: searx/templates/simple/preferences.html:60
@@ -690,7 +690,7 @@ msgid ""
 "Change how forms are submited, <a "
 "href=\"http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\""
 " rel=\"external\">learn more about request methods</a>"
-msgstr "Bepaal hoe de formulieren worden ingestuurd, <a href=\"http://nl.wikipedia.org/wiki/Hypertext_Transfer_Protocol#HTTP-requests\" rel=\"external\">lees meer over request methodes</a>"
+msgstr "Bepaal hoe de formulieren worden ingestuurd, <a href=\"http://nl.wikipedia.org/wiki/Hypertext_Transfer_Protocol#HTTP-requests\" rel=\"external\">lees meer over opvraagmethodes</a>"
 
 #: searx/templates/oscar/preferences.html:87
 #: searx/templates/simple/preferences.html:71
@@ -700,7 +700,7 @@ msgstr "Filteren op inhoud"
 #: searx/templates/oscar/preferences.html:97
 #: searx/templates/simple/preferences.html:139
 msgid "Change searx layout"
-msgstr "Wijzig searx layout"
+msgstr "Opmaak van searx aanpassen"
 
 #: searx/templates/oscar/preferences.html:106
 #: searx/templates/oscar/preferences.html:111
@@ -714,13 +714,13 @@ msgstr "Stijl"
 
 #: searx/templates/oscar/preferences.html:122
 msgid "Open Access DOI resolver"
-msgstr "Open Access DOI oplossen"
+msgstr "Open Access DOI herschrijven"
 
 #: searx/templates/oscar/preferences.html:123
 msgid ""
 "Redirect to open-access versions of publications when available (plugin "
 "required)"
-msgstr "Doorverwijzen naar vrij toegankelijke versies van publicaties indien beschikbaar (plug-in vereist)"
+msgstr "Doorverwijzen naar vrij toegankelijke versies van publicaties, indien beschikbaar (plug-in vereist)"
 
 #: searx/templates/oscar/preferences.html:163
 #: searx/templates/oscar/preferences.html:175
@@ -753,7 +753,7 @@ msgstr "Max. duur"
 
 #: searx/templates/oscar/preferences.html:248
 msgid "This is the list of searx's instant answering modules."
-msgstr "Dit is het overzicht van searx's direct antwoordende modules."
+msgstr "Dit is het overzicht van de instantantwoordmodules van searx."
 
 #: searx/templates/oscar/preferences.html:252
 msgid "Name"
@@ -791,7 +791,7 @@ msgstr "Waarde"
 
 #: searx/templates/oscar/preferences.html:301
 msgid "Search URL of the currently saved preferences"
-msgstr "Zoek-URL van de momenteel opgeslagen voorkeuren"
+msgstr "Zoek-URL van de huidig opgeslagen voorkeuren"
 
 #: searx/templates/oscar/preferences.html:301
 msgid ""
@@ -815,7 +815,7 @@ msgstr "Zoekmachines konden geen resultaten ophalen"
 
 #: searx/templates/oscar/results.html:131
 msgid "Links"
-msgstr "Links"
+msgstr "Koppelingen"
 
 #: searx/templates/oscar/search.html:8
 #: searx/templates/oscar/search_full.html:11
@@ -855,7 +855,7 @@ msgstr "Vorig jaar"
 #: searx/templates/oscar/messages/first_time.html:6
 #: searx/templates/oscar/messages/no_data_available.html:3
 msgid "Heads up!"
-msgstr "Heads up!"
+msgstr "Opgelet!"
 
 #: searx/templates/oscar/messages/first_time.html:7
 msgid "It look like you are using searx first time."
@@ -881,7 +881,7 @@ msgstr "Zoekmachines konden geen resultaten ophalen."
 #: searx/templates/oscar/messages/no_results.html:10
 #: searx/templates/simple/messages/no_results.html:10
 msgid "Please, try again later or find another searx instance."
-msgstr "Probeer het later opnieuw, of gebruik een andere searx zoekserver."
+msgstr "Probeer het later opnieuw, of gebruik een andere instantie van searx."
 
 #: searx/templates/oscar/messages/no_results.html:14
 #: searx/templates/simple/messages/no_results.html:14
@@ -893,7 +893,7 @@ msgstr "Sorry!"
 msgid ""
 "we didn't find any results. Please use another query or search in more "
 "categories."
-msgstr "we kregen geen resultaat. Probeer een andere opvraag of zoek in meer categorieën."
+msgstr "We konden geen resultaten vinden. Probeer een andere zoekopdracht, of zoek in meer categorieën."
 
 #: searx/templates/oscar/messages/save_settings_successfull.html:7
 msgid "Well done!"
@@ -901,11 +901,11 @@ msgstr "Goed gedaan!"
 
 #: searx/templates/oscar/messages/save_settings_successfull.html:8
 msgid "Settings saved successfully."
-msgstr "Instellingen succesvol opgeslagen."
+msgstr "Instellingen opgeslagen."
 
 #: searx/templates/oscar/messages/unknow_error.html:7
 msgid "Oh snap!"
-msgstr "Oops!"
+msgstr "Oeps!"
 
 #: searx/templates/oscar/messages/unknow_error.html:8
 msgid "Something went wrong."

searx/translations/pt_BR/LC_MESSAGES/messages.po

@@ -8,13 +8,14 @@
 # Guimarães Mello <maeslor@cryptolab.net>, 2017
 # Neton Brício <fervelinux@gmail.com>, 2015
 # pizzaiolo, 2016
+# Shizuka 静香 <deusadaji@gmail.com>, 2018
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2017-11-01 21:31+0100\n"
-"PO-Revision-Date: 2017-11-13 19:51+0000\n"
+"PO-Revision-Date: 2018-02-04 10:39+0000\n"
-"Last-Translator: Adam Tauber <asciimoo@gmail.com>\n"
+"Last-Translator: Shizuka 静香 <deusadaji@gmail.com>\n"
 "Language-Team: Portuguese (Brazil) (http://www.transifex.com/asciimoo/searx/language/pt_BR/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -745,7 +746,7 @@ msgstr "Intervalo de tempo"
 #: searx/templates/oscar/preferences.html:171
 #: searx/templates/simple/preferences.html:92
 msgid "Avg. time"
-msgstr "Avg.tempo"
+msgstr "Tempo médio"
 
 #: searx/templates/oscar/preferences.html:168
 #: searx/templates/oscar/preferences.html:170
@@ -813,7 +814,7 @@ msgstr "Tente pesquisar por:"
 #: searx/templates/oscar/results.html:100
 #: searx/templates/simple/results.html:25
 msgid "Engines cannot retrieve results"
-msgstr "Os motores de busca não podem obter resultados"
+msgstr "Os motores de busca não conseguiram obter resultados"
 
 #: searx/templates/oscar/results.html:131
 msgid "Links"
@@ -832,7 +833,7 @@ msgstr "estatísticas"
 #: searx/templates/oscar/time-range.html:3
 #: searx/templates/simple/time-range.html:3
 msgid "Anytime"
-msgstr "A qualquer tempo"
+msgstr "A qualquer momento"
 
 #: searx/templates/oscar/time-range.html:6
 #: searx/templates/simple/time-range.html:6
@@ -861,7 +862,7 @@ msgstr "Atenção!"
 
 #: searx/templates/oscar/messages/first_time.html:7
 msgid "It look like you are using searx first time."
-msgstr "Parece que você está usando searx primeira vez."
+msgstr "Parece que você está usando o searx pela primeira vez."
 
 #: searx/templates/oscar/messages/no_cookies.html:3
 msgid "Information!"

searx/version.py

@@ -18,8 +18,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 # version of searx
 VERSION_MAJOR = 0
-VERSION_MINOR = 13
+VERSION_MINOR = 14
-VERSION_BUILD = 1
+VERSION_BUILD = 0
 
 VERSION_STRING = "{0}.{1}.{2}".format(VERSION_MAJOR,
                                       VERSION_MINOR,

searx/webapp.py

@@ -165,6 +165,9 @@ def get_locale():
        and request.form['locale'] in settings['locales']:
         locale = request.form['locale']
 
+    if locale == 'zh_TW':
+        locale = 'zh_Hant_TW'
+
     return locale