Merge pull request #634 from not-my-profile/powered-by

Introduce `categories_as_tabs` & group engines in tabs

docs/admin/engines/configured_engines.rst

@@ -16,11 +16,18 @@ Explanation of the :ref:`general engine configuration` shown in the table
 
    SearXNG supports {{engines | length}} search engines (of which {{enabled_engine_count}} are enabled by default).
 
-   {% for category, engines in engines.items() | groupby('1.categories.0') %}
+   {% for category, engines in categories_as_tabs.items() %}
 
    {{category}} search engines
    ---------------------------------------
 
+   {% for group, engines in engines | group_engines_in_tab %}
+
+   {% if loop.length > 1 %}
+   {{group}}
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+   {% endif %}
+
    .. flat-table::
       :header-rows: 2
       :stub-columns: 1
@@ -39,9 +46,9 @@ Explanation of the :ref:`general engine configuration` shown in the table
         - Safe search
         - Time range
 
-      {% for name, mod in engines | sort_engines %}
+      {% for mod in engines %}
 
-      * - `{{name}} <{{mod.about and mod.about.website}}>`_
+      * - `{{mod.name}} <{{mod.about and mod.about.website}}>`_
         - ``!{{mod.shortcut}}``
         - {%- if 'searx.engines.' + mod.__name__ in documented_modules %}
           :py:mod:`~searx.engines.{{mod.__name__}}`
@@ -65,3 +72,4 @@ Explanation of the :ref:`general engine configuration` shown in the table
 
      {% endfor %}
      {% endfor %}
+     {% endfor %}

docs/admin/engines/settings.rst

@@ -222,6 +222,26 @@ Communication with search engines.
 ``max_redirects`` :
   30 by default. Maximum redirect before it is an error.
 
+``categories_as_tabs:``
+-----------------------
+
+A list of the categories that are displayed as tabs in the user interface.
+Categories not listed here can still be searched with the :ref:`search-syntax`.
+
+.. code-block:: yaml
+
+  categories_as_tabs:
+    general:
+    images:
+    videos:
+    news:
+    map:
+    music:
+    it:
+    science:
+    files:
+    social media:
+
 .. _settings engine:
 
 Engine settings

docs/conf.py

@@ -39,7 +39,9 @@ exclude_patterns = ['build-templates/*.rst']
 
 import searx.engines
 import searx.plugins
+import searx.webutils
 searx.engines.load_engines(searx.settings['engines'])
+
 jinja_contexts = {
     'searx': {
         'engines': searx.engines.engines,
@@ -48,14 +50,12 @@ jinja_contexts = {
             'node': os.getenv('NODE_MINIMUM_VERSION')
         },
         'enabled_engine_count': sum(not x.disabled for x in searx.engines.engines.values()),
+        'categories': searx.engines.categories,
+        'categories_as_tabs': {c: searx.engines.categories[c] for c in searx.settings['categories_as_tabs']},
     },
 }
 jinja_filters = {
-    'sort_engines':
+    'group_engines_in_tab': searx.webutils.group_engines_in_tab,
-    lambda engines: sorted(
-        engines,
-        key=lambda engine: (engine[1].disabled, engine[1].about.get('language', ''), engine[0])
-    )
 }
 
 # Let the Jinja template in configured_engines.rst access documented_modules

searx/engines/__init__.py

@@ -13,6 +13,7 @@ usage::
 
 import sys
 import copy
+from typing import List
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
@@ -44,7 +45,29 @@ ENGINE_DEFAULT_ARGS = {
     "display_error_messages": True,
     "tokens": [],
 }
-"""Defaults for the namespace of an engine module, see :py:func:`load_engine`"""
+# set automatically when an engine does not have any tab category
+OTHER_CATEGORY = 'other'
+
+
+class Engine:  # pylint: disable=too-few-public-methods
+    """This class is currently never initialized and only used for type hinting."""
+
+    name: str
+    engine: str
+    shortcut: str
+    categories: List[str]
+    supported_languages: List[str]
+    about: dict
+    inactive: bool
+    disabled: bool
+    language_support: bool
+    paging: bool
+    safesearch: bool
+    time_range_support: bool
+    timeout: float
+
+
+# Defaults for the namespace of an engine module, see :py:func:`load_engine``
 
 categories = {'general': []}
 engines = {}
@@ -113,6 +136,9 @@ def load_engine(engine_data):
 
     set_loggers(engine, engine_name)
 
+    if not any(cat in settings['categories_as_tabs'] for cat in engine.categories):
+        engine.categories.append(OTHER_CATEGORY)
+
     return engine
 
 
@@ -138,6 +164,8 @@ def update_engine_attributes(engine, engine_data):
             if isinstance(param_value, str):
                 param_value = list(map(str.strip, param_value.split(',')))
             engine.categories = param_value
+        elif hasattr(engine, 'about') and param_name == 'about':
+            engine.about = {**engine.about, **engine_data['about']}
         else:
             setattr(engine, param_name, param_value)
 

searx/engines/apkmirror.py

@@ -24,7 +24,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['files']
+categories = ['files', 'apps']
 paging = True
 time_range_support = False
 

searx/engines/archlinux.py

@@ -20,7 +20,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['it']
+categories = ['it', 'software wikis']
 paging = True
 base_url = 'https://wiki.archlinux.org'
 

searx/engines/bing.py

@@ -20,7 +20,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
+categories = ['general', 'web']
 paging = True
 time_range_support = False
 safesearch = False

searx/engines/bing_images.py

@@ -27,7 +27,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['images']
+categories = ['images', 'web']
 paging = True
 safesearch = True
 time_range_support = True

searx/engines/bing_videos.py

@@ -26,7 +26,7 @@ about = {
     "results": 'HTML',
 }
 
-categories = ['videos']
+categories = ['videos', 'web']
 paging = True
 safesearch = True
 time_range_support = True

searx/engines/duckduckgo.py

@@ -27,7 +27,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
+categories = ['general', 'web']
 paging = True
 supported_languages_url = 'https://duckduckgo.com/util/u588.js'
 time_range_support = True

searx/engines/duckduckgo_images.py

@@ -27,7 +27,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['images']
+categories = ['images', 'web']
 paging = True
 safesearch = True
 

searx/engines/duden.py

@@ -19,7 +19,7 @@ about = {
     "language": 'de',
 }
 
-categories = ['general']
+categories = ['dictionaries']
 paging = True
 
 # search-url

searx/engines/etools.py

@@ -17,7 +17,7 @@ about = {
     "results": 'HTML',
 }
 
-categories = ['general']
+categories = ['general', 'web']
 paging = False
 safesearch = True
 

searx/engines/fdroid.py

@@ -18,7 +18,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['files']
+categories = ['files', 'apps']
 paging = True
 
 # search-url

searx/engines/genius.py

@@ -20,7 +20,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['music']
+categories = ['music', 'lyrics']
 paging = True
 page_size = 5
 

searx/engines/gentoo.py

@@ -18,7 +18,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['it']
+categories = ['it', 'software wikis']
 paging = True
 base_url = 'https://wiki.gentoo.org'
 

searx/engines/gigablast.py

@@ -22,7 +22,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
+categories = ['general', 'web']
 # gigablast's pagination is totally damaged, don't use it
 paging = False
 safesearch = True

searx/engines/github.py

@@ -17,7 +17,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['it']
+categories = ['it', 'repos']
 
 # search-url
 search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&{query}'  # noqa

searx/engines/google.py

@@ -41,7 +41,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
+categories = ['general', 'web']
 paging = True
 time_range_support = True
 safesearch = True

searx/engines/google_images.py

@@ -45,7 +45,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['images']
+categories = ['images', 'web']
 paging = False
 use_locale_domain = True
 time_range_support = True

searx/engines/google_videos.py

@@ -54,7 +54,7 @@ about = {
 
 # engine dependent config
 
-categories = ['videos']
+categories = ['videos', 'web']
 paging = False
 language_support = True
 use_locale_domain = True

searx/engines/imdb.py

@@ -27,9 +27,7 @@ about = {
     "results": 'HTML',
 }
 
-categories = [
+categories = []
-    'general',
-]
 paging = False
 
 # suggestion_url = "https://sg.media-imdb.com/suggestion/{letter}/{query}.json"

searx/engines/seznam.py

@@ -25,6 +25,7 @@ about = {
     "language": "cz",
 }
 
+categories = ['general', 'web']
 base_url = 'https://search.seznam.cz/'
 
 

searx/engines/sjp.py

@@ -21,7 +21,7 @@ about = {
     "language": 'pl',
 }
 
-categories = ['general']
+categories = ['dictionaries']
 paging = False
 
 URL = 'https://sjp.pwn.pl'

searx/engines/startpage.py

@@ -23,7 +23,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
+categories = ['general', 'web']
 # there is a mechanism to block "bot" search
 # (probably the parameter qid), require
 # storing of qid's between mulitble search-calls

searx/engines/translated.py

@@ -14,7 +14,7 @@ about = {
 }
 
 engine_type = 'online_dictionary'
-categories = ['general']
+categories = ['dictionaries']
 url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'
 web_url = 'https://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'
 weight = 100

searx/engines/yahoo.py

@@ -31,7 +31,7 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
+categories = ['general', 'web']
 paging = True
 time_range_support = True
 supported_languages_url = 'https://search.yahoo.com/preferences/languages'

searx/preferences.py

@@ -12,6 +12,7 @@ from urllib.parse import parse_qs, urlencode
 from searx import settings, autocomplete
 from searx.locales import LOCALE_NAMES
 from searx.webutils import VALID_LANGUAGE_CODE
+from searx.engines import OTHER_CATEGORY
 
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
@@ -271,6 +272,8 @@ class EnginesSetting(SwitchableSetting):
         transformed_choices = []
         for engine_name, engine in self.choices.items():  # pylint: disable=no-member,access-member-before-definition
             for category in engine.categories:
+                if not category in list(settings['categories_as_tabs'].keys()) + [OTHER_CATEGORY]:
+                    continue
                 transformed_choice = {}
                 transformed_choice['default_on'] = not engine.disabled
                 transformed_choice['id'] = '{}__{}'.format(engine_name, category)

searx/query.py

@@ -222,7 +222,7 @@ class BangParser(QueryPartParser):
         # check if query starts with categorie name
         for category in categories:
             if category.startswith(value):
-                self._add_autocomplete(first_char + category)
+                self._add_autocomplete(first_char + category.replace(' ', '_'))
 
         # check if query starts with engine name
         for engine in engines:

searx/settings.yml

@@ -82,12 +82,6 @@ ui:
     simple_style: auto
     # Open result links in a new tab by default
     # results_on_new_tab: false
-    # categories_order :
-    #   - general
-    #   - files
-    #   - map
-    #   - it
-    #   - science
 
 # Lock arbitrary settings on the preferences page.  To find the ID of the user
 # setting you want to lock, check the ID of the form on the page "preferences".
@@ -234,6 +228,18 @@ checker:
         result_container:
           - has_infobox
 
+categories_as_tabs:
+  general:
+  images:
+  videos:
+  news:
+  map:
+  music:
+  it:
+  science:
+  files:
+  social media:
+
 engines:
   - name: apk mirror
     engine: apkmirror
@@ -320,7 +326,7 @@ engines:
     url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href
     title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]
     content_xpath: //article[@class="repo-summary"]/p
-    categories: it
+    categories: [it, repos]
     timeout: 4.0
     disabled: true
     shortcut: bb
@@ -419,7 +425,7 @@ engines:
   - name: docker hub
     engine: docker_hub
     shortcut: dh
-    categories: it
+    categories: [it, packages]
 
   - name: erowid
     engine: xpath
@@ -430,7 +436,7 @@ engines:
     url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href
     title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text()
     content_xpath: //dl[@class="results-list"]/dd[@class="result-details"]
-    categories: general
+    categories: []
     shortcut: ew
     disabled: true
     about:
@@ -489,7 +495,8 @@ engines:
     content_xpath: //section[contains(@class, "word__defination")]
     first_page_num: 1
     shortcut: et
-    disabled: true
+    categories: [dictionaries]
+    disabled: false
     about:
       website: https://www.etymonline.com/
       wikidata_id: Q1188617
@@ -528,7 +535,7 @@ engines:
   - name: free software directory
     engine: mediawiki
     shortcut: fsd
-    categories: it
+    categories: [it, software wikis]
     base_url: https://directory.fsf.org/
     number_of_results: 5
     # what part of a page matches the query string: title, text, nearmatch
@@ -579,7 +586,7 @@ engines:
     title_query: name_with_namespace
     content_query: description
     page_size: 20
-    categories: it
+    categories: [it, repos]
     shortcut: gl
     timeout: 10.0
     disabled: true
@@ -605,7 +612,7 @@ engines:
     url_query: html_url
     title_query: name
     content_query: description
-    categories: it
+    categories: [it, repos]
     shortcut: cb
     disabled: true
     about:
@@ -671,7 +678,7 @@ engines:
     url_xpath: './/div[@class="RZEgze"]//div[@class="kCSSQe"]//a/@href'
     content_xpath: './/div[@class="RZEgze"]//a[@class="mnKHRc"]'
     thumbnail_xpath: './/div[@class="uzcko"]/div/span[1]//img/@data-src'
-    categories: files
+    categories: [files, apps]
     shortcut: gpa
     disabled: true
     about:
@@ -749,7 +756,7 @@ engines:
     url_xpath: './/div[@class="ans"]//a/@href'
     content_xpath: './/div[@class="from"]'
     page_size: 20
-    categories: it
+    categories: [it, packages]
     shortcut: ho
     about:
       website: https://hoogle.haskell.org/
@@ -844,7 +851,7 @@ engines:
     engine: xpath
     timeout: 4.0
     disabled: true
-    categories: music
+    categories: [music, lyrics]
     paging: true
     search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno}
     url_xpath: //td[@class="text-left visitedlyr"]/a/@href
@@ -899,7 +906,7 @@ engines:
     title_query: package/name
     content_query: package/description
     page_size: 25
-    categories: it
+    categories: [it, packages]
     disabled: true
     timeout: 5.0
     shortcut: npm
@@ -1008,7 +1015,7 @@ engines:
     url_query: url
     title_query: name
     content_query: description
-    categories: it
+    categories: [it, packages]
     disabled: true
     timeout: 5.0
     shortcut: pack
@@ -1065,7 +1072,7 @@ engines:
     content_xpath: ./p
     suggestion_xpath: /html/body/main/div/div/div/form/div/div[@class="callout-block"]/p/span/a[@class="link"]
     first_page_num: 1
-    categories: it
+    categories: [it, packages]
     about:
       website: https://pypi.org
       wikidata_id: Q2984686
@@ -1077,7 +1084,7 @@ engines:
   - name: qwant
     engine: qwant
     shortcut: qw
-    categories: general
+    categories: [general, web]
     disabled: false
     additional_tests:
       rosebud: *test_rosebud
@@ -1092,14 +1099,14 @@ engines:
   - name: qwant images
     engine: qwant
     shortcut: qwi
-    categories: images
+    categories: [images, web]
     disabled: false
     network: qwant
 
   - name: qwant videos
     engine: qwant
     shortcut: qwv
-    categories: videos
+    categories: [videos, web]
     disabled: false
     network: qwant
 
@@ -1159,19 +1166,19 @@ engines:
     engine: stackexchange
     shortcut: st
     api_site: 'stackoverflow'
-    categories: it
+    categories: [it, q&a]
 
   - name: askubuntu
     engine: stackexchange
     shortcut: ubuntu
     api_site: 'askubuntu'
-    categories: it
+    categories: [it, q&a]
 
   - name: superuser
     engine: stackexchange
     shortcut: su
     api_site: 'superuser'
-    categories: it
+    categories: [it, q&a]
 
   - name: searchcode code
     engine: searchcode_code
@@ -1354,7 +1361,7 @@ engines:
     url_query: URL
     title_query: Title
     content_query: Snippet
-    categories: general
+    categories: [general, web]
     shortcut: wib
     disabled: true
     about:
@@ -1413,11 +1420,11 @@ engines:
   - name: wiktionary
     engine: mediawiki
     shortcut: wt
-    categories: general
+    categories: [dictionaries]
     base_url: "https://{language}.wiktionary.org/"
     number_of_results: 5
     search_type: text
-    disabled: true
+    disabled: false
     about:
       website: https://www.wiktionary.org/
       wikidata_id: Q151
@@ -1467,7 +1474,7 @@ engines:
     engine: translated
     shortcut: tl
     timeout: 5.0
-    disabled: true
+    disabled: false
     # You can use without an API key, but you are limited to 1000 words/day
     # See: https://mymemory.translated.net/doc/usagelimits.php
     # api_key: ''
@@ -1501,6 +1508,7 @@ engines:
     shortcut: mjk
     engine: xpath
     paging: true
+    categories: [general, web]
     search_url: https://www.mojeek.com/search?q={query}&s={pageno}
     results_xpath: /html/body//div[@class="results"]/ul[@class="results-standard"]/li
     url_xpath: ./h2/a/@href
@@ -1520,6 +1528,7 @@ engines:
 
   - name: naver
     shortcut: nvr
+    categories: [general, web]
     engine: xpath
     paging: true
     search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno}
@@ -1549,7 +1558,7 @@ engines:
     content_xpath: ./span/p
     suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a
     first_page_num: 1
-    categories: it
+    categories: [it, packages]
     disabled: true
     about:
       website: https://rubygems.org/
@@ -1593,14 +1602,14 @@ engines:
     engine: wordnik
     shortcut: def
     base_url: https://www.wordnik.com/
-    categories: general
+    categories: [dictionaries]
     timeout: 5.0
-    disabled: true
+    disabled: false
 
   - name: woxikon.de synonyme
     engine: xpath
     shortcut: woxi
-    categories: general
+    categories: [dictionaries]
     timeout: 5.0
     disabled: true
     search_url: https://synonyme.woxikon.de/synonyme/{query}.php
@@ -1619,7 +1628,6 @@ engines:
     engine: sjp
     shortcut: sjp
     base_url: https://sjp.pwn.pl/
-    categories: general
     timeout: 5.0
     disabled: true
 
@@ -1652,7 +1660,7 @@ engines:
     title_xpath: //span[@class="snippet-title"]
     content_xpath: //p[1][@class="snippet-description"]
     suggestion_xpath: //div[@class="text-gray h6"]/a
-    categories: general
+    categories: [general, web]
     about:
       website: https://brave.com/search/
       wikidata_id: Q107355971

searx/settings_defaults.py

@@ -20,18 +20,18 @@ OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
 LANGUAGE_CODES = ['all'] + list(l[0] for l in languages)
 OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
 SIMPLE_STYLE = ('auto', 'light', 'dark')
-CATEGORY_ORDER = [
+CATEGORIES_AS_TABS = {
-    'general',
+    'general': {},
-    'images',
+    'images': {},
-    'videos',
+    'videos': {},
-    'news',
+    'news': {},
-    'map',
+    'map': {},
-    'music',
+    'music': {},
-    'it',
+    'it': {},
-    'science',
+    'science': {},
-    'files',
+    'files': {},
-    'social media',
+    'social media': {},
-]
+}
 STR_TO_BOOL = {
     '0': False,
     'false': False,
@@ -182,7 +182,6 @@ SCHEMA = {
         'results_on_new_tab': SettingsValue(bool, False),
         'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
-        'categories_order': SettingsValue(list, CATEGORY_ORDER),
     },
     'preferences': {
         'lock': SettingsValue(list, []),
@@ -213,6 +212,7 @@ SCHEMA = {
     'checker': {
         'off_when_debug': SettingsValue(bool, True),
     },
+    'categories_as_tabs': SettingsValue(dict, CATEGORIES_AS_TABS),
     'engines': SettingsValue(list, []),
     'doi_resolvers': {},
 }

searx/static/themes/simple/src/less/definitions.less

@@ -72,6 +72,7 @@
   /// Settings Colors
   --color-settings-tr-hover: #f7f7f7;
   --color-settings-engine-description-font: darken(#dcdcdc, 30%);
+  --color-settings-engine-group-background: #0001;
   /// Detail modal
   --color-result-detail-font: #fff;
   --color-result-detail-label-font: lightgray;
@@ -180,6 +181,7 @@
   /// Settings Colors
   --color-settings-tr-hover: #2d2d2d;
   --color-settings-engine-description-font: darken(#dcdcdc, 30%);
+  --color-settings-engine-group-background: #1a1919;
   /// Toolkit Colors
   --color-toolkit-badge-font: #fff;
   --color-toolkit-badge-background: #777;

searx/static/themes/simple/src/less/preferences.less

@@ -161,6 +161,12 @@
       }
     }
   }
+
+  .engine-group {
+    text-align: left;
+    font-weight: normal;
+    background: var(--color-settings-engine-group-background);
+  }
 }
 
 @media screen and (max-width: @tablet) {

searx/templates/oscar/categories.html

@@ -1,11 +1,11 @@
 <div id="categories">
 {%- if rtl -%}
-    {% for category in categories | reverse -%}
+    {% for category in categories_as_tabs | reverse -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}
 {%- else -%}
-    {% for category in categories -%}
+    {% for category in categories_as_tabs -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}

searx/templates/oscar/preferences.html

@@ -298,7 +298,7 @@
             <div class="tab-pane active_if_nojs" id="tab_engine">
                 <!-- Nav tabs -->
                 <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist">
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs + [OTHER_CATEGORY] %}
                     <li{% if loop.first %} class="active"{% endif %}><a href="#tab_engine_{{ categ|replace(' ', '_') }}" role="tab" data-toggle="tab">{{ _(categ) }}</a></li>
                     {% endfor %}
                 </ul>
@@ -317,10 +317,13 @@
                         </p>
                     </div>
 
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs + [OTHER_CATEGORY] %}
                     <noscript><label>{{ _(categ) }}</label>
                     </noscript>
                     <div class="tab-pane{% if loop.first %} active{% endif %} active_if_nojs" id="tab_engine_{{ categ|replace(' ', '_') }}">
+                        {% if categ == OTHER_CATEGORY %}
+                            <p>{{_('This tab does not show up for search results but you can search the engines listed here via bangs.')}}</p>
+                        {% endif %}
                         <div class="container-fluid">
                         <fieldset>
                           <div class="table-responsive">
@@ -348,7 +351,11 @@
                                     <th scope="col" class="text-right">{{ _("Allow") }}</th>
                                     {% endif %}
                                 </tr>
-                        {% for search_engine in engines_by_category[categ] %}
+                        {% for group, engines in engines_by_category[categ] | group_engines_in_tab %}
+                        {% if loop.length > 1 %}
+                        <tr><th colspan="9">{{_(group)}}</th></tr>
+                        {% endif %}
+                        {% for search_engine in engines %}
                             {% if not search_engine.private %}
                                 <tr>
                                     {% if not rtl %}
@@ -357,7 +364,11 @@
                                         </td>
                                         <th scope="row" data-engine-name="{{ search_engine.name }}"><span aria-labelledby="{{ 'tooltip_' + categ + '_' + search_engine.name }}">
                                             {%- if search_engine.enable_http %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif -%}
-                                            {{- search_engine.name -}}</span>
+                                            {{- search_engine.name -}}
+                                            {%- if search_engine.about and search_engine.about.language %}
+                                                ({{search_engine.about.language | upper}})
+                                            {%- endif %}
+                                            </span>
                                             {{- engine_about(search_engine, 'tooltip_' + categ + '_' + search_engine.name) -}}
                                         </th>
                                         <td class="name">{{ shortcuts[search_engine.name] }}</td>
@@ -382,6 +393,7 @@
                                     {% endif %}
                                 </tr>
                             {% endif %}
+                        {% endfor %}
                         {% endfor %}
                           </table>
                           </div>

searx/templates/simple/categories.html

@@ -14,7 +14,7 @@
 <div id="categories" class="search_categories">{{- '' -}}
     <div id="categories_container">
         {%- if display_tooltip %}<div class="help">{{ _('Click on the magnifier to perform search') }}</div>{% endif -%}
-        {%- for category in categories -%}
+        {%- for category in categories_as_tabs -%}
         <div class="category"><input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}"{% if category in selected_categories %} checked="checked"{% endif %}/>
             <label for="checkbox_{{ category|replace(' ', '_') }}" class="tooltips">
                 {{- icon_big(category_icons[category]) if category in category_icons  else icon_big('globe-outline') -}}

searx/templates/simple/preferences.html

@@ -274,8 +274,11 @@
   {{ tab_header('maintab', 'engines', _('Engines')) }}
     <p>{{ _('Currently used search engines') }}</p>
     {{ tabs_open() }}
-    {% for categ in all_categories %}
+    {% for categ in categories_as_tabs + [OTHER_CATEGORY] %}
     {{ tab_header('enginetab', 'category' + categ, _(categ)) }}
+    {% if categ == OTHER_CATEGORY %}
+      <p>{{_('This tab does not show up for search results but you can search the engines listed here via bangs.')}}</p>
+    {% endif %}
     <div class="scrollx">
     <table class="striped">
       <tr>
@@ -289,12 +292,22 @@
         <th>{{ _("Max time") }}</th>
         <th>{{ _("Reliability") }}</th>
       </tr>
-      {% for search_engine in engines_by_category[categ] %}
+      {% for group, engines in engines_by_category[categ] | group_engines_in_tab %}
+      {% if loop.length > 1 %}
+      <tr><th colspan="9" class="engine-group">{{_(group)}}</th></tr>
+      {% endif %}
+      {% for search_engine in engines %}
       {% if not search_engine.private %}
       {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %}
       <tr>
         <td class="engine_checkbox">{{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }}</td>
-        <th class="name" data-engine-name="{{ search_engine.name }}">{% if search_engine.enable_http %}{{ icon_big('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }} {{ engine_about(search_engine) }}</th>
+        <th class="name" data-engine-name="{{ search_engine.name }}">{% if search_engine.enable_http %}{{ icon_big('warning', 'No HTTPS') }}{% endif %}
+          {{ search_engine.name }}
+          {%- if search_engine.about and search_engine.about.language %}
+              ({{search_engine.about.language | upper}})
+          {%- endif %}
+          {{ engine_about(search_engine) }}
+        </th>
         <td class="shortcut">{{ shortcuts[search_engine.name] }}</td>
         <td>{{ checkbox(engine_id + '_supported_languages', supports[search_engine.name]['supports_selected_language'], true, true) }}</td>
         <td>{{ checkbox(engine_id + '_safesearch', supports[search_engine.name]['safesearch'], true, true) }}</td>
@@ -305,6 +318,7 @@
       </tr>
       {% endif %}
       {% endfor %}
+      {% endfor %}
     </table>
     </div>
     {{ tab_footer() }}

searx/webapp.py

@@ -59,6 +59,7 @@ from searx.settings_defaults import OUTPUT_FORMATS
 from searx.settings_loader import get_default_settings_path
 from searx.exceptions import SearxParameterException
 from searx.engines import (
+    OTHER_CATEGORY,
     categories,
     engines,
     engine_shortcuts,
@@ -73,6 +74,8 @@ from searx.webutils import (
     new_hmac,
     is_hmac_of,
     is_flask_run_cmdline,
+    DEFAULT_GROUP_NAME,
+    group_engines_in_tab,
 )
 from searx.webadapter import (
     get_search_query_from_webapp,
@@ -152,6 +155,7 @@ app = Flask(__name__, static_folder=settings['ui']['static_path'], template_fold
 app.jinja_env.trim_blocks = True
 app.jinja_env.lstrip_blocks = True
 app.jinja_env.add_extension('jinja2.ext.loopcontrols')  # pylint: disable=no-member
+app.jinja_env.filters['group_engines_in_tab'] = group_engines_in_tab  # pylint: disable=no-member
 app.secret_key = settings['server']['secret_key']
 
 babel = Babel(app)
@@ -169,6 +173,17 @@ _category_names = (
     gettext('map'),
     gettext('onions'),
     gettext('science'),
+    # non-tab categories
+    gettext('apps'),
+    gettext('dictionaries'),
+    gettext('lyrics'),
+    gettext('packages'),
+    gettext('q&a'),
+    gettext('repos'),
+    gettext('software wikis'),
+    gettext('web'),
+    gettext(DEFAULT_GROUP_NAME),
+    gettext(OTHER_CATEGORY),
 )
 
 _simple_style = (gettext('auto'), gettext('light'), gettext('dark'))
@@ -390,12 +405,6 @@ def get_translations():
     }
 
 
-def _get_ordered_categories():
-    ordered_categories = list(settings['ui']['categories_order'])
-    ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
-    return ordered_categories
-
-
 def _get_enable_categories(all_categories):
     disabled_engines = request.preferences.engines.get_disabled()
     enabled_categories = set(
@@ -430,8 +439,9 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['query_in_title'] = request.preferences.get_value('query_in_title')
     kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
     kwargs['theme'] = get_current_theme_name(override=override_theme)
-    kwargs['all_categories'] = _get_ordered_categories()
+    kwargs['categories_as_tabs'] = list(settings['categories_as_tabs'].keys())
-    kwargs['categories'] = _get_enable_categories(kwargs['all_categories'])
+    kwargs['categories'] = _get_enable_categories(categories.keys())
+    kwargs['OTHER_CATEGORY'] = OTHER_CATEGORY
 
     # i18n
     kwargs['language_codes'] = [l for l in languages if l[0] in settings['search']['languages']]

searx/webutils.py

@@ -5,11 +5,14 @@ import hashlib
 import hmac
 import re
 import inspect
+import itertools
+from typing import Iterable, List, Tuple
 
 from io import StringIO
 from codecs import getincrementalencoder
 
-from searx import logger
+from searx import logger, settings
+from searx.engines import Engine, OTHER_CATEGORY
 
 
 VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')
@@ -134,3 +137,28 @@ def is_flask_run_cmdline():
     if len(frames) < 2:
         return False
     return frames[-2].filename.endswith('flask/cli.py')
+
+
+DEFAULT_GROUP_NAME = 'others'
+
+
+def group_engines_in_tab(engines: Iterable[Engine]) -> List[Tuple[str, Iterable[Engine]]]:
+    """Groups an Iterable of engines by their first non tab category"""
+
+    def get_group(eng):
+        non_tab_categories = [
+            c for c in eng.categories if c not in list(settings['categories_as_tabs'].keys()) + [OTHER_CATEGORY]
+        ]
+        return non_tab_categories[0] if len(non_tab_categories) > 0 else DEFAULT_GROUP_NAME
+
+    groups = itertools.groupby(sorted(engines, key=get_group), get_group)
+
+    def group_sort_key(group):
+        return (group[0] == DEFAULT_GROUP_NAME, group[0].lower())
+
+    sorted_groups = sorted(((name, list(engines)) for name, engines in groups), key=group_sort_key)
+
+    def engine_sort_key(engine):
+        return (engine.about.get('language', ''), engine.name)
+
+    return [(groupname, sorted(engines, key=engine_sort_key)) for groupname, engines in sorted_groups]

tests/robot/settings_robot.yml

@@ -33,6 +33,10 @@ outgoing:
   request_timeout: 1.0  # seconds
   useragent_suffix: ""
 
+categories_as_tabs:
+  general:
+  dummy:
+
 engines:
   - name: general dummy
     engine: dummy