[enh] introduce categories_as_tabs

Previously all categories were displayed as search engine tabs.
This commit changes that so that only the categories listed under
categories_as_tabs in settings.yml are displayed.

This lets us introduce more categories without cluttering up the UI.
Categories not displayed as tabs  can still be searched with !bangs.

docs/admin/engines/configured_engines.rst

@@ -16,11 +16,18 @@ Explanation of the :ref:`general engine configuration` shown in the table
 
    SearXNG supports {{engines | length}} search engines (of which {{enabled_engine_count}} are enabled by default).
 
-   {% for category, engines in categories.items() %}
+   {% for category, engines in categories_as_tabs.items() %}
 
    {{category}} search engines
    ---------------------------------------
 
+   {% for group, engines in engines | group_engines_in_tab %}
+
+   {% if loop.length > 1 %}
+   {{group}}
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+   {% endif %}
+
    .. flat-table::
       :header-rows: 2
       :stub-columns: 1
@@ -39,7 +46,7 @@ Explanation of the :ref:`general engine configuration` shown in the table
         - Safe search
         - Time range
 
-      {% for mod in engines | sort_engines %}
+      {% for mod in engines %}
 
       * - `{{mod.name}} <{{mod.about and mod.about.website}}>`_
         - ``!{{mod.shortcut}}``
@@ -65,3 +72,4 @@ Explanation of the :ref:`general engine configuration` shown in the table
 
      {% endfor %}
      {% endfor %}
+     {% endfor %}

docs/admin/engines/settings.rst

@@ -219,6 +219,26 @@ Communication with search engines.
 ``max_redirects`` :
   30 by default. Maximum redirect before it is an error.
 
+``categories_as_tabs:``
+-----------------------
+
+A list of the categories that are displayed as tabs in the user interface.
+Categories not listed here can still be searched with the :ref:`search-syntax`.
+
+.. code-block:: yaml
+
+  categories_as_tabs:
+    - general
+    - images
+    - videos
+    - news
+    - map
+    - music
+    - it
+    - science
+    - files
+    - social media
+
 .. _settings engine:
 
 Engine settings

docs/conf.py

@@ -50,14 +50,11 @@ jinja_contexts = {
         },
         'enabled_engine_count': sum(not x.disabled for x in searx.engines.engines.values()),
         'categories': searx.engines.categories,
+        'categories_as_tabs': {c: searx.engines.categories[c] for c in searx.settings['categories_as_tabs']},
     },
 }
 jinja_filters = {
-    'sort_engines':
-    lambda engines: sorted(
-        engines,
-        key=lambda engine: (engine.about.get('language', ''), engine.name)
-    )
+    'group_engines_in_tab': searx.engines.group_engines_in_tab,
 }
 
 # Let the Jinja template in configured_engines.rst access documented_modules

searx/engines/__init__.py

@@ -13,6 +13,7 @@ usage::
 
 import sys
 import copy
+import itertools
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
@@ -260,3 +261,26 @@ def load_engines(engine_list):
         if engine:
             register_engine(engine)
     return engines
+
+
+DEFAULT_GROUP_NAME = 'others'
+
+
+def group_engines_in_tab(engines):  # pylint: disable=redefined-outer-name
+    def engine_sort_key(engine):
+        return (engine.about.get('language', ''), engine.name)
+
+    def group_sort_key(group):
+        return (group[0] == DEFAULT_GROUP_NAME, group[0].lower())
+
+    def get_group(eng):
+        non_tab_engines = [c for c in eng.categories if c not in settings['categories_as_tabs']]
+        return non_tab_engines[0] if len(non_tab_engines) > 0 else DEFAULT_GROUP_NAME
+
+    return [
+        (groupname, sorted(engines, key=engine_sort_key))
+        for groupname, engines in sorted(
+            ((name, list(engines)) for name, engines in itertools.groupby(sorted(engines, key=get_group), get_group)),
+            key=group_sort_key,
+        )
+    ]

searx/settings.yml

@@ -81,12 +81,6 @@ ui:
     simple_style: auto
     # Open result links in a new tab by default
     # results_on_new_tab: false
-    # categories_order :
-    #   - general
-    #   - files
-    #   - map
-    #   - it
-    #   - science
 
 # Lock arbitrary settings on the preferences page.  To find the ID of the user
 # setting you want to lock, check the ID of the form on the page "preferences".
@@ -233,6 +227,18 @@ checker:
         result_container:
           - has_infobox
 
+categories_as_tabs:
+  - general
+  - images
+  - videos
+  - news
+  - map
+  - music
+  - it
+  - science
+  - files
+  - social media
+
 engines:
   - name: apk mirror
     engine: apkmirror

searx/settings_defaults.py

@@ -20,7 +20,7 @@ OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
 LANGUAGE_CODES = ['all'] + list(l[0] for l in languages)
 OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
 SIMPLE_STYLE = ('auto', 'light', 'dark')
-CATEGORY_ORDER = [
+CATEGORIES_AS_TABS = [
     'general',
     'images',
     'videos',
@@ -181,7 +181,6 @@ SCHEMA = {
         'results_on_new_tab': SettingsValue(bool, False),
         'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
-        'categories_order': SettingsValue(list, CATEGORY_ORDER),
     },
     'preferences': {
         'lock': SettingsValue(list, []),
@@ -212,6 +211,7 @@ SCHEMA = {
     'checker': {
         'off_when_debug': SettingsValue(bool, True),
     },
+    'categories_as_tabs': SettingsValue(list, CATEGORIES_AS_TABS),
     'engines': SettingsValue(list, []),
     'doi_resolvers': {},
 }

searx/templates/oscar/categories.html

@@ -1,11 +1,11 @@
 <div id="categories">
 {%- if rtl -%}
-    {% for category in categories | reverse -%}
+    {% for category in categories_as_tabs | reverse -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}
 {%- else -%}
-    {% for category in categories -%}
+    {% for category in categories_as_tabs -%}
         <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
     {%- endfor %}

searx/templates/oscar/preferences.html

@@ -298,7 +298,7 @@
             <div class="tab-pane active_if_nojs" id="tab_engine">
                 <!-- Nav tabs -->
                 <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist">
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs %}
                     <li{% if loop.first %} class="active"{% endif %}><a href="#tab_engine_{{ categ|replace(' ', '_') }}" role="tab" data-toggle="tab">{{ _(categ) }}</a></li>
                     {% endfor %}
                 </ul>
@@ -317,7 +317,7 @@
                         </p>
                     </div>
 
-                    {% for categ in all_categories %}
+                    {% for categ in categories_as_tabs %}
                     <noscript><label>{{ _(categ) }}</label>
                     </noscript>
                     <div class="tab-pane{% if loop.first %} active{% endif %} active_if_nojs" id="tab_engine_{{ categ|replace(' ', '_') }}">

searx/templates/simple/categories.html

@@ -14,7 +14,7 @@
 <div id="categories" class="search_categories">{{- '' -}}
     <div id="categories_container">
         {%- if display_tooltip %}<div class="help">{{ _('Click on the magnifier to perform search') }}</div>{% endif -%}
-        {%- for category in categories -%}
+        {%- for category in categories_as_tabs -%}
         <div class="category"><input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}"{% if category in selected_categories %} checked="checked"{% endif %}/>
             <label for="checkbox_{{ category|replace(' ', '_') }}" class="tooltips">
                 {{- icon_big(category_icons[category]) if category in category_icons  else icon_big('globe-outline') -}}

searx/templates/simple/preferences.html

@@ -274,7 +274,7 @@
   {{ tab_header('maintab', 'engines', _('Engines')) }}
     <p>{{ _('Currently used search engines') }}</p>
     {{ tabs_open() }}
-    {% for categ in all_categories %}
+    {% for categ in categories_as_tabs %}
     {{ tab_header('enginetab', 'category' + categ, _(categ)) }}
     <div class="scrollx">
     <table class="striped">

searx/webapp.py

@@ -390,12 +390,6 @@ def get_translations():
     }
 
 
-def _get_ordered_categories():
-    ordered_categories = list(settings['ui']['categories_order'])
-    ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
-    return ordered_categories
-
-
 def _get_enable_categories(all_categories):
     disabled_engines = request.preferences.engines.get_disabled()
     enabled_categories = set(
@@ -430,8 +424,8 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['query_in_title'] = request.preferences.get_value('query_in_title')
     kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
     kwargs['theme'] = get_current_theme_name(override=override_theme)
-    kwargs['all_categories'] = _get_ordered_categories()
-    kwargs['categories'] = _get_enable_categories(kwargs['all_categories'])
+    kwargs['categories_as_tabs'] = settings['categories_as_tabs']
+    kwargs['categories'] = _get_enable_categories(categories.keys())
 
     # i18n
     kwargs['language_codes'] = [l for l in languages if l[0] in settings['search']['languages']]

tests/robot/settings_robot.yml

@@ -33,6 +33,10 @@ outgoing:
   request_timeout: 1.0  # seconds
   useragent_suffix: ""
 
+categories_as_tabs:
+  - general
+  - dummy
+
 engines:
   - name: general dummy
     engine: dummy