[doc] improve documentation of make targets and ./manage script

BTW force modularization of the ./mange script into sub modules:

- utils/lib_sxng_data.sh
- utils/lib_sxng_node.sh
- utils/lib_sxng_static.sh
- utils/lib_sxng_test.sh
- utils/lib_sxng_themes.sh

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Makefile

@@ -59,16 +59,16 @@ test.shell:
 		utils/brand.env \
 		$(MTOOLS) \
 		utils/lib.sh \
-		utils/lib_nvm.sh \
+		utils/lib_sxng*.sh \
-		utils/lib_static.sh \
 		utils/lib_go.sh \
+		utils/lib_nvm.sh \
 		utils/lib_redis.sh \
-		utils/filtron.sh \
-		utils/searx.sh \
 		utils/searxng.sh \
-		utils/morty.sh \
 		utils/lxc.sh \
-		utils/lxc-searxng.env
+		utils/lxc-searxng.env \
+		utils/searx.sh \
+		utils/filtron.sh \
+		utils/morty.sh
 	$(Q)$(MTOOLS) build_msg TEST "$@ OK"
 
 

docs/admin/buildhosts.rst

@@ -4,22 +4,30 @@
 Buildhosts
 ==========
 
-.. sidebar:: This article needs some work
-
-   If you have any contribution send us your :pull:`PR <../pulls>`, see
-   :ref:`how to contribute`.
-
 .. contents::
    :depth: 2
    :local:
    :backlinks: entry
 
-To get best results from build, it's recommend to install additional packages
+To get best results from build, it's recommend to install additional packages on
-on build hosts (see :ref:`searxng.sh`).::
+build hosts (see :ref:`searxng.sh`).
 
-  sudo -H ./utils/searxng.sh install buildhost
+.. _searxng.sh install buildhost:
 
-This will install packages needed by searx:
+Build and Development tools
+===========================
+
+To Install tools used by build and development tasks in once:
+
+.. tabs::
+
+  .. group-tab:: SearXNG's development tools
+
+     .. code:: sh
+
+        $ sudo -H ./utils/searxng.sh install buildhost
+
+This will install packages needed by SearXNG:
 
 .. kernel-include:: $DOCS_BUILD/includes/searxng.rst
    :start-after: START distro-packages
@@ -73,7 +81,7 @@ If your docs build (``make docs.html``) shows warnings like this::
             display), check the imgmath_latex setting
 
 you need to install additional packages on your build host, to get better HTML
-output.
+output (:ref:`install buildhost <searxng.sh install buildhost>`).
 
 .. tabs::
 
@@ -93,7 +101,7 @@ output.
 
       .. code-block:: sh
 
-         $ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
+         $ sudo dnf install graphviz graphviz-gd ImageMagick texlive-xetex-bin librsvg2-tools
 
 
 For PDF output you also need:
@@ -118,8 +126,7 @@ For PDF output you also need:
 
       	 $ sudo dnf install \
              texlive-collection-fontsrecommended texlive-collection-latex \
-		dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \
+             dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
-		ImageMagick
 
 .. _sh lint:
 
@@ -128,7 +135,8 @@ Lint shell scripts
 
 .. _ShellCheck: https://github.com/koalaman/shellcheck
 
-To lint shell scripts, we use ShellCheck_ - a shell script static analysis tool.
+To lint shell scripts we use ShellCheck_ - a shell script static analysis tool
+(:ref:`install buildhost <searxng.sh install buildhost>`).
 
 .. SNIP sh lint requirements
 

docs/admin/settings/settings_redis.rst

@@ -24,7 +24,10 @@ developer) account needs to be added to the *searxng-redis* group.
     rediss://[[username]:[password]]@localhost:6379/0
     unix://[[username]:[password]]@/path/to/socket.sock?db=0
 
-.. admonition:: Tip for developers
+.. _Redis Developer Notes:
+
+Redis Developer Notes
+=====================
 
 To set up a local redis instance, first set the socket path of the Redis DB
 in your YAML setting:
@@ -34,7 +37,10 @@ developer) account needs to be added to the *searxng-redis* group.
    redis:
      url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
 
-   Then use the following commands to install the redis instance ::
+Then use the following commands to install the redis instance (:ref:`manage
+redis.help`):
+
+.. code:: sh
 
    $ ./manage redis.build
    $ sudo -H ./manage redis.install

docs/build-templates/filtron.rst

@@ -1,53 +0,0 @@
-.. START create user
-
-.. tabs::
-
-  .. group-tab:: bash
-
-    .. code-block:: sh
-
-      $ sudo -H useradd --shell /bin/bash --system \\
-          --home-dir \"$SERVICE_HOME\" \\
-          --comment \"Privacy-respecting metasearch engine\" $SERVICE_USER
-
-      $ sudo -H mkdir \"$SERVICE_HOME\"
-      $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\"
-
-.. END create user
-
-.. START install go
-
-.. tabs::
-
-  .. group-tab:: os: linux / arch: amd64
-
-
-    .. code-block:: bash
-
-       $ cat > \"$GO_ENV\" <<EOF
-       export GOPATH=${SERVICE_HOME}/go-apps
-       export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
-       EOF
-       $ sudo -i -u \"${SERVICE_USER}\"
-       (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
-       (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
-       (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\
-                   \"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\"
-       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\"
-       (${SERVICE_USER}) $ which go
-       ${SERVICE_HOME}/local/go/bin/go
-
-.. END install go
-
-.. START install filtron
-
-.. tabs::
-
-  .. group-tab:: bash
-
-    .. code-block:: bash
-
-       $ sudo -i -u \"${SERVICE_USER}\"
-       (${SERVICE_USER}) $ go get -v -u github.com/searxng/filtron
-
-.. END install filtron

docs/build-templates/morty.rst

@@ -1,53 +0,0 @@
-.. START create user
-
-.. tabs::
-
-  .. group-tab:: bash
-
-    .. code-block:: sh
-
-      $ sudo -H useradd --shell /bin/bash --system \\
-          --home-dir \"$SERVICE_HOME\" \\
-          --comment \"Privacy-respecting metasearch engine\" $SERVICE_USER
-
-      $ sudo -H mkdir \"$SERVICE_HOME\"
-      $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\"
-
-.. END create user
-
-.. START install go
-
-.. tabs::
-
-  .. group-tab:: os: linux / arch: amd64
-
-
-    .. code-block:: bash
-
-       $ cat > \"$GO_ENV\" <<EOF
-       export GOPATH=${SERVICE_HOME}/go-apps
-       export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
-       EOF
-       $ sudo -i -u \"${SERVICE_USER}\"
-       (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
-       (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
-       (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\
-                   \"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\"
-       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\"
-       (${SERVICE_USER}) $ which go
-       ${SERVICE_HOME}/local/go/bin/go
-
-.. END install go
-
-.. START install morty
-
-.. tabs::
-
-  .. group-tab:: bash
-
-    .. code-block:: bash
-
-       $ sudo -i -u \"${SERVICE_USER}\"
-       (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty
-
-.. END install morty

docs/dev/contribution_guide.rst

@@ -148,7 +148,7 @@ live build
 Live build is like WYSIWYG.  If you want to edit the documentation, its
 recommended to use.  The Makefile target ``docs.live`` builds the docs, opens
 URL in your favorite browser and rebuilds every time a reST file has been
-changed.
+changed (:ref:`make docs.clean`).
 
 .. code:: sh
 
@@ -183,3 +183,8 @@ commit and push:
 .. code:: sh
 
    $ make docs.clean docs.gh-pages
+
+.. attention::
+
+   If you are working in your own brand, don't forgett to adjust your
+   :ref:`settings brand`.

docs/dev/makefile.rst

@@ -1,32 +1,49 @@
 .. _makefile:
 
-========
+=======================
-Makefile
+Makefile & ``./manage``
-========
+=======================
 
 .. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
 
+All relevant build and development tasks are implemented in the
+:origin:`./manage <manage>` script and for CI or IDE integration a small
+:origin:`Makefile` wrapper is available.  If you are not familiar with
+Makefiles, we recommend to read gnu-make_ introduction.
+
 .. sidebar:: build environment
 
    Before looking deeper at the targets, first read about :ref:`make
    install`.
 
-   To install system requirements follow :ref:`buildhosts`.
+   To install developer requirements follow :ref:`buildhosts`.
 
-All relevant build tasks are implemented in :origin:`manage` and for CI or
-IDE integration a small ``Makefile`` wrapper is available.  If you are not
-familiar with Makefiles, we recommend to read gnu-make_ introduction.
-
-The usage is simple, just type ``make {target-name}`` to *build* a target.
-Calling the ``help`` target gives a first overview (``make help``):
-
-.. program-output:: bash -c "cd ..; make --no-print-directory help"
 
 .. contents::
    :depth: 2
    :local:
    :backlinks: entry
 
+The usage is simple, just type ``make {target-name}`` to *build* a target.
+Calling the ``help`` target gives a first overview (``make help``):
+
+.. tabs::
+
+  .. group-tab:: ``make``
+
+     .. program-output:: bash -c "cd ..; make --no-print-directory help"
+
+
+  .. group-tab:: ``./manage``
+
+     The Makefile targets are implemented for comfort, if you can do without
+     tab-completion and need to have a more granular control, use
+     :origin:`manage` without the Makefile wrappers.
+
+     .. code:: sh
+
+        $ ./manage help
+
 .. _make install:
 
 Python environment (``make install``)
@@ -158,13 +175,36 @@ Node.js environment (``make node.env``)
    Manager) to install latest LTS of Node.js_ locally: there is no need to
    install nvm_ or npm_ on your system.
 
-Use ``make nvm.status`` to get the current status of you Node.js_ and nvm_ setup.
+To install NVM_ and Node.js_ in once you can use :ref:`make nvm.nodejs`.
 
-Here is the output you will typically get on a Ubuntu 20.04 system which serves
+.. _make nvm:
-only a `no longer active <https://nodejs.org/en/about/releases/>`_ Release
-`Node.js v10.19.0 <https://packages.ubuntu.com/focal/nodejs>`_.
 
-::
+NVM ``make nvm.install nvm.status``
+-----------------------------------
+
+Use ``make nvm.status`` to get the current status of your Node.js_ and nvm_
+setup.
+
+.. tabs::
+
+  .. group-tab:: nvm.install
+
+     .. code:: sh
+
+        $ LANG=C make nvm.install
+        INFO:  install (update) NVM at ./searxng/.nvm
+        INFO:  clone: https://github.com/nvm-sh/nvm.git
+          || Cloning into './searxng/.nvm'...
+        INFO:  checkout v0.39.4
+          || HEAD is now at 8fbf8ab v0.39.4
+
+  .. group-tab:: nvm.status (ubu2004)
+
+     Here is the output you will typically get on a Ubuntu 20.04 system which
+     serves only a `no longer active <https://nodejs.org/en/about/releases/>`_
+     Release `Node.js v10.19.0 <https://packages.ubuntu.com/focal/nodejs>`_.
+
+     .. code:: sh
 
         $ make nvm.status
         INFO:  Node.js is installed at /usr/bin/node
@@ -173,14 +213,11 @@ only a `no longer active <https://nodejs.org/en/about/releases/>`_ Release
         INFO:  npm is installed at /usr/bin/npm
         INFO:  npm is version 6.14.4
         WARN:  NVM is not installed
-  INFO:  to install NVM and Node.js (LTS) use: manage nvm install --lts
-
-To install you can also use :ref:`make nvm.nodejs`
 
 .. _make nvm.nodejs:
 
 ``make nvm.nodejs``
-===================
+-------------------
 
 Install latest Node.js_ LTS locally (uses nvm_)::
 
@@ -213,10 +250,29 @@ sources of the theme need to be rebuild.  You can do that by running::
   $ make themes.all
 
 Alternatively to ``themes.all`` you can run *live builds* of the theme you are
-modify::
+modify (:ref:`make themes`)::
 
   $ LIVE_THEME=simple make run
 
+.. _make format.python:
+
+``make format.python``
+======================
+
+Format Python sourcee code using `Black code style`_.  See ``$BLACK_OPTIONS``
+and ``$BLACK_TARGETS`` in :origin:`Makefile`.
+
+.. attention::
+
+   We stuck at Black 22.12.0, please read comment in PR `Bump black from 22.12.0
+   to 23.1.0`_
+
+.. _Bump black from 22.12.0 to 23.1.0:
+   https://github.com/searxng/searxng/pull/2159#pullrequestreview-1284094735
+
+.. _Black code style:
+   https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html
+
 .. _make clean:
 
 ``make clean``
@@ -237,18 +293,34 @@ calling ``make clean`` stop all processes using the :ref:`make install` or
 
 .. _make docs:
 
-``make docs docs.autobuild docs.clean``
+``make docs``
-=======================================
+=============
+
+Target ``docs`` builds the documentation:
+
+.. code:: bash
+
+   $ make docs
+   HTML ./docs --> file://
+   DOCS      build build/docs/includes
+   ...
+   The HTML pages are in dist/docs.
+
+.. _make docs.clean:
+
+``make docs.clean docs.live``
+----------------------------------
 
 We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /
 Documentation <contrib docs>` section.  If you want to edit the documentation
 read our :ref:`make docs.live` section.  If you are working in your own brand,
 adjust your :ref:`settings brand`.
 
+
 .. _make docs.gh-pages:
 
 ``make docs.gh-pages``
-======================
+----------------------
 
 To deploy on github.io first adjust your :ref:`settings brand`.  For any
 further read :ref:`deploy on github.io`.
@@ -261,17 +333,17 @@ further read :ref:`deploy on github.io`.
 Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``
 and ``test.robot``.  You can run tests selective, e.g.::
 
-  $ make test.pep8 test.unit test.sh
+  $ make test.pep8 test.unit test.shell
   TEST      test.pep8 OK
   ...
   TEST      test.unit OK
   ...
-  TEST      test.sh OK
+  TEST      test.shell OK
 
 .. _make test.shell:
 
 ``make test.shell``
-===================
+-------------------
 
 :ref:`sh lint` / if you have changed some bash scripting run this test before
 commit.
@@ -279,7 +351,7 @@ commit.
 .. _make test.pylint:
 
 ``make test.pylint``
-====================
+--------------------
 
 .. _Pylint: https://www.pylint.org/
 
@@ -289,8 +361,8 @@ found in project's root folder :origin:`.pylintrc`.
 
 .. _make search.checker:
 
-``search.checker.{engine name}``
+``make search.checker.{engine name}``
-================================
+=====================================
 
 To check all engines::
 
@@ -318,3 +390,63 @@ To filter out HTTP redirects (3xx_)::
     https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
     https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
     --
+
+.. _make themes:
+
+``make themes.*``
+=================
+
+.. sidebar:: further read
+
+   - :ref:`devquickstart`
+
+The :origin:`Makefile` targets ``make theme.*`` cover common tasks to build the
+theme(s).  The ``./manage themes.*`` command line can be used to convenient run
+common theme build tasks.
+
+.. program-output:: bash -c "cd ..; ./manage themes.help"
+
+To get live builds while modifying CSS & JS use (:ref:`make run`):
+
+.. code:: sh
+
+   $ LIVE_THEME=simple make run
+
+.. _make static.build:
+
+``make static.build.*``
+=======================
+
+.. sidebar:: further read
+
+   - :ref:`devquickstart`
+
+The :origin:`Makefile` targets ``static.build.*`` cover common tasks to build (a
+commit of) the static files.  The ``./manage static.build..*`` command line
+can be used to convenient run common build tasks of the satic files.
+
+.. program-output:: bash -c "cd ..; ./manage static.help"
+
+
+.. _manage redis.help:
+
+``./manage redis.help``
+=======================
+
+The ``./manage redis.*`` command line can be used to convenient run common Redis
+tasks (:ref:`Redis developer notes`).
+
+.. program-output:: bash -c "cd ..; ./manage redis.help"
+
+
+.. _manage go.help:
+
+``./manage go.help``
+====================
+
+The ``./manage go.*`` command line can be used to convenient run common `go
+(wiki)`_ tasks.
+
+.. _go (wiki): https://en.wikipedia.org/wiki/Go_(programming_language)
+
+.. program-output:: bash -c "cd ..; ./manage go.help"

docs/dev/quickstart.rst

@@ -7,8 +7,16 @@ Development Quickstart
 .. _npm: https://www.npmjs.com/
 .. _Node.js: https://nodejs.org/
 
-SearXNG loves developers, just clone and start hacking.  All the rest is done for
+
-you simply by using :ref:`make <makefile>`.
+.. sidebar:: further read
+
+   - :ref:`makefile`
+   - :ref:`buildhosts`
+
+SearXNG loves developers; Developers do not need to worry about tool chains, the
+usual developer tasks can be comfortably executed via :ref:`make <makefile>`.
+
+Don't hesitate, just clone SearXNG's sources and start hacking right now ..
 
 .. code:: bash
 
@@ -18,25 +26,23 @@ Here is how a minimal workflow looks like:
 
 1. *start* hacking
 2. *run* your code: :ref:`make run`
-3. *test* your code: :ref:`make test`
+3. *format & test* your code: :ref:`make format.python` and :ref:`make test`
 
 If you think at some point something fails, go back to *start*.  Otherwise,
 choose a meaningful commit message and we are happy to receive your pull
 request. To not end in *wild west* we have some directives, please pay attention
 to our ":ref:`how to contribute`" guideline.
 
-If you implement themes, you will need to setup a :ref:`make node.env` once:
+.. sidebar:: further read
 
-.. code:: bash
+   - :ref:`make nvm`
+   - :ref:`make themes`
 
-   make node.env
+If you implement themes, you will need to setup a :ref:`Node.js environment
+<make node.env>`: ``make node.env``
 
 Before you call *make run* (2.), you need to compile the modified styles and
-JavaScript:
+JavaScript: ``make themes.all``
-
-.. code:: bash
-
-   make themes.all
 
 Alternatively you can also compile selective the theme you have modified,
 e.g. the *simple* theme.
@@ -49,6 +55,10 @@ e.g. the *simple* theme.
 
    To get live builds while modifying CSS & JS use: ``LIVE_THEME=simple make run``
 
+.. sidebar:: further read
+
+   - :ref:`make static.build`
+
 If you finished your *tests* you can start to commit your changes.  To separate
 the modified source code from the build products first run:
 

manage

@@ -11,8 +11,23 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
 
-# shellcheck source=utils/lib_static.sh
+# shellcheck source=utils/lib_sxng_data.sh
-source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh"
+
+# shellcheck source=utils/lib_sxng_weblate.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh"
+
+# shellcheck source=utils/lib_sxng_static.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh"
+
+# shellcheck source=utils/lib_sxng_node.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh"
+
+# shellcheck source=utils/lib_sxng_themes.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
+
+# shellcheck source=utils/lib_sxng_test.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
 
 # shellcheck source=utils/lib_go.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
@@ -27,9 +42,8 @@ PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
 PYOBJECTS="searx"
 PY_SETUP_EXTRAS='[test]'
 GECKODRIVER_VERSION="v0.30.0"
-export NODE_MINIMUM_VERSION="16.13.0"
 # SPHINXOPTS=
-BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization")
+BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
 BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
 
 _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
@@ -73,13 +87,6 @@ buildenv:
   rebuild ./utils/brand.env
 webapp.:
   run       : run developer instance
-weblate.:
-  push.translations: push translation changes from SearXNG to Weblate's counterpart
-  to.translations: Update 'translations' branch with last additions from Weblate.
-data.:
-  all       : update searx/sxng_locales.py and searx/data/*
-  traits    : update searx/data/engine_traits.json & searx/sxng_locales.py
-  useragents: update searx/data/useragents.json with the most recent versions of Firefox
 docs.:
   html      : build HTML documentation
   live      : autobuild HTML documentation while editing
@@ -96,10 +103,6 @@ redis:
   build     : build redis binaries at $(redis._get_dist)
   install   : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
   help      : show more redis commands
-node.:
-  env       : download & install SearXNG's npm dependencies locally
-  env.dev   : download & install developer and CI tools
-  clean     : drop locally npm installations
 py.:
   build     : Build python packages at ./${PYDIST}
   clean     : delete virtualenv and intermediate py files
@@ -112,24 +115,16 @@ pypi.upload:
   Upload python packages to PyPi (to test use pypi.upload.test)
 format.:
   python    : format Python code source using black
-test.:
-  yamllint  : lint YAML files (YAMLLINT_FILES)
-  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
-  pyright   : static type check of python sources
-  black     : check black code format
-  unit      : run unit tests
-  coverage  : run unit tests with coverage
-  robot     : run robot test
-  rst       : test .rst files incl. README.rst
-  clean     : clean intermediate test stuff
-themes.:
-  all       : build all themes
-  simple    : build simple theme
 pygments.:
   less      : build LESS files for pygments
 EOF
     go.help
-    static_help
+    node.help
+    weblate.help
+    data.help
+    test.help
+    themes.help
+    static.help
     cat <<EOF
 environment ...
   SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
@@ -183,258 +178,6 @@ buildenv() {
     return "${PIPESTATUS[0]}"
 }
 
-TRANSLATIONS_WORKTREE="$CACHE/translations"
-
-weblate.translations.worktree() {
-
-    # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
-    # 'translations' from Weblate's counterpart (weblate) of the SearXNG
-    # (origin).
-    #
-    #     remote weblate https://translate.codeberg.org/git/searxng/searxng/
-
-    (   set -e
-        if ! git remote get-url weblate 2> /dev/null; then
-            git remote add weblate https://translate.codeberg.org/git/searxng/searxng/
-        fi
-        if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
-            pushd .
-            cd "${TRANSLATIONS_WORKTREE}"
-            git reset --hard HEAD
-            git pull origin translations
-            popd
-        else
-            mkdir -p "${TRANSLATIONS_WORKTREE}"
-            git worktree add "${TRANSLATIONS_WORKTREE}" translations
-        fi
-    )
-}
-
-weblate.to.translations() {
-
-    # Update 'translations' branch of SearXNG (origin) with last additions from
-    # Weblate.
-
-    # 1. Check if Weblate is locked, if not die with error message
-    # 2. On Weblate's counterpart (weblate), pull master and translations branch
-    #    from SearXNG (origin).
-    # 3. Commit changes made in a Weblate object on Weblate's counterpart
-    #    (weblate).
-    # 4. In translations worktree, merge changes of branch 'translations' from
-    #    remote 'weblate' and push it on branch 'translations' of 'origin'
-
-    (   set -e
-        pyenv.activate
-        if [ "$(wlc lock-status)" != "locked: True" ]; then
-            die 1 "weblate must be locked, currently: $(wlc lock-status)"
-        fi
-        # weblate: commit pending changes
-        wlc pull
-        wlc commit
-
-        # get the translations in a worktree
-        weblate.translations.worktree
-
-        pushd "${TRANSLATIONS_WORKTREE}"
-        git remote update weblate
-        git merge weblate/translations
-        git push
-        popd
-    )
-    dump_return $?
-}
-
-weblate.translations.commit() {
-
-    # Update 'translations' branch of SearXNG (origin) with last additions from
-    # Weblate.  Copy the changes to the master branch, compile translations and
-    # create a commit in the local branch (master)
-
-    local existing_commit_hash commit_body commit_message exitcode
-    (   set -e
-        pyenv.activate
-        # lock change on weblate
-        wlc lock
-
-        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
-        weblate.translations.worktree
-        existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1  --pretty=format:'%h')
-
-        # pull weblate commits
-        weblate.to.translations
-
-        # copy the changes to the master branch
-        cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
-
-        # compile translations
-        build_msg BABEL 'compile translation catalogs into binary MO files'
-        pybabel compile --statistics \
-                -d "searx/translations"
-        # git add/commit (no push)
-        commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
-        commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
-        git add searx/translations
-        git commit -m "${commit_message}"
-    )
-    exitcode=$?
-    (   # make sure to always unlock weblate
-        set -e
-        pyenv.cmd wlc unlock
-    )
-    dump_return $exitcode
-}
-
-weblate.push.translations() {
-
-    # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
-    # (weblate).
-
-    # In branch master of SearXNG (origin) check for meaningful changes in
-    # folder 'searx/translations', commit changes on branch 'translations' and
-    # at least, pull updated branches on Weblate's counterpart (weblate).
-
-    # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
-    #    'translations' from remote 'weblate'.
-    # 2. Stop if there is no meaningful change in the 'master' branch (origin),
-    #    compared to the 'translations' branch (weblate), otherwise ...
-    # 3. Update 'translations' branch of SearXNG (origin) with last additions
-    #    from Weblate.
-    # 5. Notify Weblate to pull updated 'master' & 'translations' branch.
-
-    local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
-          exitcode
-    messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
-    (   set -e
-        pyenv.activate
-        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
-        weblate.translations.worktree
-
-        # update messages.pot in the master branch
-        build_msg BABEL 'extract messages from source files and generate POT file'
-        pybabel extract -F babel.cfg \
-                -o "${messages_pot}" \
-                "searx/"
-
-        # stop if there is no meaningful change in the master branch
-        diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
-                            git diff -- "searx/translations/messages.pot")
-        if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
-            build_msg BABEL 'no changes detected, exiting'
-            return 42
-        fi
-        return 0
-    )
-    exitcode=$?
-    if [ "$exitcode" -eq 42 ]; then
-        return 0
-    fi
-    if [ "$exitcode" -gt 0 ]; then
-       return $exitcode
-    fi
-    (
-        set -e
-        pyenv.activate
-
-        # lock change on weblate
-        # weblate may add commit(s) since the call to "weblate.translations.worktree".
-        # this is not a problem because after this line, "weblate.to.translations"
-        # calls again "weblate.translations.worktree" which calls "git pull"
-        wlc lock
-
-        # save messages.pot in the translations branch for later
-        pushd "${TRANSLATIONS_WORKTREE}"
-        git stash push
-        popd
-
-        # merge weblate commits into the translations branch
-        weblate.to.translations
-
-        # restore messages.pot in the translations branch
-        pushd "${TRANSLATIONS_WORKTREE}"
-        git stash pop
-        popd
-
-        # update messages.po files in the master branch
-        build_msg BABEL 'update existing message catalogs from POT file'
-        pybabel update -N \
-            -i "${messages_pot}" \
-            -d "${TRANSLATIONS_WORKTREE}/searx/translations"
-
-        # git add/commit/push
-        last_commit_hash=$(git log -n1  --pretty=format:'%h')
-        last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
-
-        pushd "${TRANSLATIONS_WORKTREE}"
-        git add searx/translations
-        git commit \
-            -m "[translations] update messages.pot and messages.po files" \
-            -m "From ${last_commit_detail}"
-        git push
-        popd
-
-        # notify weblate to pull updated master & translations branch
-        wlc pull
-    )
-    exitcode=$?
-    (   # make sure to always unlock weblate
-        set -e
-        pyenv.activate
-        wlc unlock
-    )
-    dump_return $exitcode
-}
-
-data.all() {
-    (   set -e
-
-        pyenv.activate
-        data.traits
-        data.useragents
-
-        build_msg DATA "update searx/data/osm_keys_tags.json"
-        pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
-        build_msg DATA "update searx/data/ahmia_blacklist.txt"
-        python searxng_extra/update/update_ahmia_blacklist.py
-        build_msg DATA "update searx/data/wikidata_units.json"
-        python searxng_extra/update/update_wikidata_units.py
-        build_msg DATA "update searx/data/currencies.json"
-        python searxng_extra/update/update_currencies.py
-        build_msg DATA "update searx/data/external_bangs.json"
-        python searxng_extra/update/update_external_bangs.py
-        build_msg DATA "update searx/data/engine_descriptions.json"
-        python searxng_extra/update/update_engine_descriptions.py
-    )
-}
-
-
-data.traits() {
-    (   set -e
-        pyenv.activate
-        build_msg DATA "update searx/data/engine_traits.json"
-        python searxng_extra/update/update_engine_traits.py
-        build_msg ENGINES "update searx/sxng_locales.py"
-    )
-    dump_return $?
-}
-
-data.useragents() {
-    build_msg DATA "update searx/data/useragents.json"
-    pyenv.cmd python searxng_extra/update/update_firefox_version.py
-    dump_return $?
-}
-
-docs.prebuild() {
-    build_msg DOCS "build ${DOCS_BUILD}/includes"
-    (
-        set -e
-        [ "$VERBOSE" = "1" ] && set -x
-        mkdir -p "${DOCS_BUILD}/includes"
-        ./utils/searxng.sh searxng.doc.rst >  "${DOCS_BUILD}/includes/searxng.rst"
-        pyenv.cmd searxng_extra/docs_prebuild
-    )
-    dump_return $?
-}
-
 docker.push() {
     docker.build push
 }
@@ -554,44 +297,6 @@ gecko.driver() {
     dump_return $?
 }
 
-nodejs.ensure() {
-    if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
-        info_msg "install Node.js by NVM"
-        nvm.nodejs
-    fi
-}
-
-node.env() {
-    nodejs.ensure
-    (   set -e
-        build_msg INSTALL "./searx/static/themes/simple/package.json"
-        npm --prefix searx/static/themes/simple install
-    )
-    dump_return $?
-}
-
-node.env.dev() {
-    nodejs.ensure
-    build_msg INSTALL "./package.json: developer and CI tools"
-    npm install
-}
-
-node.clean() {
-    if ! required_commands npm 2>/dev/null; then
-        build_msg CLEAN "npm is not installed / ignore npm dependencies"
-        return 0
-    fi
-    build_msg CLEAN "themes -- locally installed npm dependencies"
-    (   set -e
-        npm --prefix searx/static/themes/simple run clean
-    )
-    build_msg CLEAN "locally installed developer and CI tools"
-    (   set -e
-        npm --prefix . run clean
-    )
-    dump_return $?
-}
-
 pygments.less() {
     build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
     if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
@@ -674,150 +379,6 @@ format.python() {
     dump_return $?
 }
 
-test.yamllint() {
-    build_msg TEST "[yamllint] \$YAMLLINT_FILES"
-    pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
-    dump_return $?
-}
-
-test.pylint() {
-    # shellcheck disable=SC2086
-    (   set -e
-        build_msg TEST "[pylint] \$PYLINT_FILES"
-        pyenv.activate
-        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
-            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
-            "${PYLINT_FILES[@]}"
-
-        build_msg TEST "[pylint] searx/engines"
-        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
-            --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
-            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
-            searx/engines
-
-        build_msg TEST "[pylint] searx tests"
-        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
-            --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
-	    --ignore=searx/engines \
-	    searx tests
-    )
-    dump_return $?
-}
-
-test.pyright() {
-    build_msg TEST "[pyright] static type check of python sources"
-    node.env.dev
-    # We run Pyright in the virtual environment because Pyright
-    # executes "python" to determine the Python version.
-    build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
-    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
-        | grep -v ".py$" \
-        | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
-        | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
-        | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
-    dump_return $?
-}
-
-test.black() {
-    build_msg TEST "[black] \$BLACK_TARGETS"
-    pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
-    dump_return $?
-}
-
-test.unit() {
-    build_msg TEST 'tests/unit'
-    pyenv.cmd python -m nose2 -s tests/unit
-    dump_return $?
-}
-
-test.coverage() {
-    build_msg TEST 'unit test coverage'
-    (   set -e
-        pyenv.activate
-        python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
-        coverage report
-        coverage html
-    )
-    dump_return $?
-}
-
-test.robot() {
-    build_msg TEST 'robot'
-    gecko.driver
-    PYTHONPATH=. pyenv.cmd python -m tests.robot
-    dump_return $?
-}
-
-test.rst() {
-    build_msg TEST "[reST markup] ${RST_FILES[*]}"
-    for rst in "${RST_FILES[@]}"; do
-        pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
-    done
-}
-
-test.pybabel() {
-    TEST_BABEL_FOLDER="build/test/pybabel"
-    build_msg TEST "[extract messages] pybabel"
-    mkdir -p "${TEST_BABEL_FOLDER}"
-    pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
-}
-
-test.clean() {
-    build_msg CLEAN  "test stuff"
-    rm -rf geckodriver.log .coverage coverage/
-    dump_return $?
-}
-
-themes.all() {
-    (   set -e
-        pygments.less
-        node.env
-        themes.simple
-    )
-    dump_return $?
-}
-
-themes.live() {
-    local LIVE_THEME="${LIVE_THEME:-${1}}"
-    case "${LIVE_THEME}" in
-        simple)
-            theme="searx/static/themes/${LIVE_THEME}"
-            ;;
-        '')
-            die_caller 42 "missing theme argument"
-            ;;
-        *)
-            die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'"
-            ;;
-    esac
-    build_msg GRUNT "theme: $1 (live build)"
-    nodejs.ensure
-    cd "${theme}"
-    {
-        npm install
-        npm run watch
-    } 2>&1 \
-        | prefix_stdout "${_Blue}THEME ${1} ${_creset}  " \
-        | grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |'
-}
-
-themes.simple() {
-    (   set -e
-        build_msg GRUNT "theme: simple"
-        npm --prefix searx/static/themes/simple run build
-    )
-    dump_return $?
-}
-
-themes.simple.test() {
-    build_msg TEST "theme: simple"
-    nodejs.ensure
-    npm --prefix searx/static/themes/simple install
-    npm --prefix searx/static/themes/simple run test
-    dump_return $?
-}
 
 PYLINT_FILES=()
 while IFS= read -r line; do

utils/lib_go.sh

@@ -20,9 +20,6 @@
 # shellcheck source=utils/lib.sh
 . /dev/null
 
-# shellcheck disable=SC2034
-declare main_cmd
-
 # configure golang environment
 # ----------------------------
 

utils/lib_sxng_data.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+data.help(){
+    cat <<EOF
+data.:
+  all       : update searx/sxng_locales.py and searx/data/*
+  traits    : update searx/data/engine_traits.json & searx/sxng_locales.py
+  useragents: update searx/data/useragents.json with the most recent versions of Firefox
+EOF
+}
+
+data.all() {
+    (   set -e
+
+        pyenv.activate
+        data.traits
+        data.useragents
+
+        build_msg DATA "update searx/data/osm_keys_tags.json"
+        pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
+        build_msg DATA "update searx/data/ahmia_blacklist.txt"
+        python searxng_extra/update/update_ahmia_blacklist.py
+        build_msg DATA "update searx/data/wikidata_units.json"
+        python searxng_extra/update/update_wikidata_units.py
+        build_msg DATA "update searx/data/currencies.json"
+        python searxng_extra/update/update_currencies.py
+        build_msg DATA "update searx/data/external_bangs.json"
+        python searxng_extra/update/update_external_bangs.py
+        build_msg DATA "update searx/data/engine_descriptions.json"
+        python searxng_extra/update/update_engine_descriptions.py
+    )
+}
+
+
+data.traits() {
+    (   set -e
+        pyenv.activate
+        build_msg DATA "update searx/data/engine_traits.json"
+        python searxng_extra/update/update_engine_traits.py
+        build_msg ENGINES "update searx/sxng_locales.py"
+    )
+    dump_return $?
+}
+
+data.useragents() {
+    build_msg DATA "update searx/data/useragents.json"
+    pyenv.cmd python searxng_extra/update/update_firefox_version.py
+    dump_return $?
+}
+
+docs.prebuild() {
+    build_msg DOCS "build ${DOCS_BUILD}/includes"
+    (
+        set -e
+        [ "$VERBOSE" = "1" ] && set -x
+        mkdir -p "${DOCS_BUILD}/includes"
+        ./utils/searxng.sh searxng.doc.rst >  "${DOCS_BUILD}/includes/searxng.rst"
+        pyenv.cmd searxng_extra/docs_prebuild
+    )
+    dump_return $?
+}

utils/lib_sxng_node.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+export NODE_MINIMUM_VERSION="16.13.0"
+
+node.help(){
+    cat <<EOF
+node.:
+  env       : download & install SearXNG's npm dependencies locally
+  env.dev   : download & install developer and CI tools
+  clean     : drop locally npm installations
+EOF
+}
+
+nodejs.ensure() {
+    if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
+        info_msg "install Node.js by NVM"
+        nvm.nodejs
+    fi
+}
+
+node.env() {
+    nodejs.ensure
+    (   set -e
+        build_msg INSTALL "./searx/static/themes/simple/package.json"
+        npm --prefix searx/static/themes/simple install
+    )
+    dump_return $?
+}
+
+node.env.dev() {
+    nodejs.ensure
+    build_msg INSTALL "./package.json: developer and CI tools"
+    npm install
+}
+
+node.clean() {
+    if ! required_commands npm 2>/dev/null; then
+        build_msg CLEAN "npm is not installed / ignore npm dependencies"
+        return 0
+    fi
+    build_msg CLEAN "themes -- locally installed npm dependencies"
+    (   set -e
+        npm --prefix searx/static/themes/simple run clean
+    )
+    build_msg CLEAN "locally installed developer and CI tools"
+    (   set -e
+        npm --prefix . run clean
+    )
+    dump_return $?
+}

utils/lib_sxng_static.sh

@@ -12,7 +12,7 @@ STATIC_BUILT_PATHS=(
     'searx/templates/simple/icons.html'
 )
 
-static_help(){
+static.help(){
     cat <<EOF
 static.build.:  ${STATIC_BUILD_COMMIT}
   commit    : build & commit /static folder

utils/lib_sxng_test.sh

@@ -0,0 +1,111 @@
+test.help(){
+    cat <<EOF
+test.:
+  yamllint  : lint YAML files (YAMLLINT_FILES)
+  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
+  pyright   : static type check of python sources
+  black     : check black code format
+  unit      : run unit tests
+  coverage  : run unit tests with coverage
+  robot     : run robot test
+  rst       : test .rst files incl. README.rst
+  clean     : clean intermediate test stuff
+EOF
+}
+
+test.yamllint() {
+    build_msg TEST "[yamllint] \$YAMLLINT_FILES"
+    pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
+    dump_return $?
+}
+
+test.pylint() {
+    # shellcheck disable=SC2086
+    (   set -e
+        build_msg TEST "[pylint] \$PYLINT_FILES"
+        pyenv.activate
+        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
+            "${PYLINT_FILES[@]}"
+
+        build_msg TEST "[pylint] searx/engines"
+        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
+            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
+            searx/engines
+
+        build_msg TEST "[pylint] searx tests"
+        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
+	    --ignore=searx/engines \
+	    searx tests
+    )
+    dump_return $?
+}
+
+test.pyright() {
+    build_msg TEST "[pyright] static type check of python sources"
+    node.env.dev
+    # We run Pyright in the virtual environment because Pyright
+    # executes "python" to determine the Python version.
+    build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
+    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
+        | grep -v ".py$" \
+        | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
+        | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
+        | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
+        | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
+        | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
+    dump_return $?
+}
+
+test.black() {
+    build_msg TEST "[black] \$BLACK_TARGETS"
+    pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
+    dump_return $?
+}
+
+test.unit() {
+    build_msg TEST 'tests/unit'
+    pyenv.cmd python -m nose2 -s tests/unit
+    dump_return $?
+}
+
+test.coverage() {
+    build_msg TEST 'unit test coverage'
+    (   set -e
+        pyenv.activate
+        python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
+        coverage report
+        coverage html
+    )
+    dump_return $?
+}
+
+test.robot() {
+    build_msg TEST 'robot'
+    gecko.driver
+    PYTHONPATH=. pyenv.cmd python -m tests.robot
+    dump_return $?
+}
+
+test.rst() {
+    build_msg TEST "[reST markup] ${RST_FILES[*]}"
+    for rst in "${RST_FILES[@]}"; do
+        pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
+    done
+}
+
+test.pybabel() {
+    TEST_BABEL_FOLDER="build/test/pybabel"
+    build_msg TEST "[extract messages] pybabel"
+    mkdir -p "${TEST_BABEL_FOLDER}"
+    pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
+}
+
+test.clean() {
+    build_msg CLEAN  "test stuff"
+    rm -rf geckodriver.log .coverage coverage/
+    dump_return $?
+}
+

utils/lib_sxng_themes.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+declare _Blue
+declare _creset
+
+themes.help(){
+    cat <<EOF
+themes.:
+  all       : build all themes
+  live      : to get live builds of CSS & JS use 'LIVE_THEME=simple make run'
+  simple.:
+    build   : build simple theme
+    test    : test simple theme
+EOF
+}
+
+themes.all() {
+    (   set -e
+        pygments.less
+        node.env
+        themes.simple
+    )
+    dump_return $?
+}
+
+themes.live() {
+    local LIVE_THEME="${LIVE_THEME:-${1}}"
+    case "${LIVE_THEME}" in
+        simple)
+            theme="searx/static/themes/${LIVE_THEME}"
+            ;;
+        '')
+            die_caller 42 "missing theme argument"
+            ;;
+        *)
+            die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'"
+            ;;
+    esac
+    build_msg GRUNT "theme: $1 (live build)"
+    nodejs.ensure
+    cd "${theme}"
+    {
+        npm install
+        npm run watch
+    } 2>&1 \
+        | prefix_stdout "${_Blue}THEME ${1} ${_creset}  " \
+        | grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |'
+}
+
+themes.simple() {
+    (   set -e
+        build_msg GRUNT "theme: simple"
+        npm --prefix searx/static/themes/simple run build
+    )
+    dump_return $?
+}
+
+themes.simple.test() {
+    build_msg TEST "theme: simple"
+    nodejs.ensure
+    npm --prefix searx/static/themes/simple install
+    npm --prefix searx/static/themes/simple run test
+    dump_return $?
+}

utils/lib_sxng_weblate.sh

@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+weblate.help(){
+    cat <<EOF
+weblate.:
+  push.translations: push translation changes from SearXNG to Weblate's counterpart
+  to.translations: Update 'translations' branch with last additions from Weblate.
+EOF
+}
+
+TRANSLATIONS_WORKTREE="$CACHE/translations"
+
+weblate.translations.worktree() {
+
+    # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
+    # 'translations' from Weblate's counterpart (weblate) of the SearXNG
+    # (origin).
+    #
+    #     remote weblate https://translate.codeberg.org/git/searxng/searxng/
+
+    (   set -e
+        if ! git remote get-url weblate 2> /dev/null; then
+            git remote add weblate https://translate.codeberg.org/git/searxng/searxng/
+        fi
+        if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
+            pushd .
+            cd "${TRANSLATIONS_WORKTREE}"
+            git reset --hard HEAD
+            git pull origin translations
+            popd
+        else
+            mkdir -p "${TRANSLATIONS_WORKTREE}"
+            git worktree add "${TRANSLATIONS_WORKTREE}" translations
+        fi
+    )
+}
+
+weblate.to.translations() {
+
+    # Update 'translations' branch of SearXNG (origin) with last additions from
+    # Weblate.
+
+    # 1. Check if Weblate is locked, if not die with error message
+    # 2. On Weblate's counterpart (weblate), pull master and translations branch
+    #    from SearXNG (origin).
+    # 3. Commit changes made in a Weblate object on Weblate's counterpart
+    #    (weblate).
+    # 4. In translations worktree, merge changes of branch 'translations' from
+    #    remote 'weblate' and push it on branch 'translations' of 'origin'
+
+    (   set -e
+        pyenv.activate
+        if [ "$(wlc lock-status)" != "locked: True" ]; then
+            die 1 "weblate must be locked, currently: $(wlc lock-status)"
+        fi
+        # weblate: commit pending changes
+        wlc pull
+        wlc commit
+
+        # get the translations in a worktree
+        weblate.translations.worktree
+
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git remote update weblate
+        git merge weblate/translations
+        git push
+        popd
+    )
+    dump_return $?
+}
+
+weblate.translations.commit() {
+
+    # Update 'translations' branch of SearXNG (origin) with last additions from
+    # Weblate.  Copy the changes to the master branch, compile translations and
+    # create a commit in the local branch (master)
+
+    local existing_commit_hash commit_body commit_message exitcode
+    (   set -e
+        pyenv.activate
+        # lock change on weblate
+        wlc lock
+
+        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
+        weblate.translations.worktree
+        existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1  --pretty=format:'%h')
+
+        # pull weblate commits
+        weblate.to.translations
+
+        # copy the changes to the master branch
+        cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
+
+        # compile translations
+        build_msg BABEL 'compile translation catalogs into binary MO files'
+        pybabel compile --statistics \
+                -d "searx/translations"
+        # git add/commit (no push)
+        commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
+        commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
+        git add searx/translations
+        git commit -m "${commit_message}"
+    )
+    exitcode=$?
+    (   # make sure to always unlock weblate
+        set -e
+        pyenv.cmd wlc unlock
+    )
+    dump_return $exitcode
+}
+
+weblate.push.translations() {
+
+    # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
+    # (weblate).
+
+    # In branch master of SearXNG (origin) check for meaningful changes in
+    # folder 'searx/translations', commit changes on branch 'translations' and
+    # at least, pull updated branches on Weblate's counterpart (weblate).
+
+    # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
+    #    'translations' from remote 'weblate'.
+    # 2. Stop if there is no meaningful change in the 'master' branch (origin),
+    #    compared to the 'translations' branch (weblate), otherwise ...
+    # 3. Update 'translations' branch of SearXNG (origin) with last additions
+    #    from Weblate.
+    # 5. Notify Weblate to pull updated 'master' & 'translations' branch.
+
+    local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
+          exitcode
+    messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
+    (   set -e
+        pyenv.activate
+        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
+        weblate.translations.worktree
+
+        # update messages.pot in the master branch
+        build_msg BABEL 'extract messages from source files and generate POT file'
+        pybabel extract -F babel.cfg \
+                -o "${messages_pot}" \
+                "searx/"
+
+        # stop if there is no meaningful change in the master branch
+        diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
+                            git diff -- "searx/translations/messages.pot")
+        if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
+            build_msg BABEL 'no changes detected, exiting'
+            return 42
+        fi
+        return 0
+    )
+    exitcode=$?
+    if [ "$exitcode" -eq 42 ]; then
+        return 0
+    fi
+    if [ "$exitcode" -gt 0 ]; then
+       return $exitcode
+    fi
+    (
+        set -e
+        pyenv.activate
+
+        # lock change on weblate
+        # weblate may add commit(s) since the call to "weblate.translations.worktree".
+        # this is not a problem because after this line, "weblate.to.translations"
+        # calls again "weblate.translations.worktree" which calls "git pull"
+        wlc lock
+
+        # save messages.pot in the translations branch for later
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git stash push
+        popd
+
+        # merge weblate commits into the translations branch
+        weblate.to.translations
+
+        # restore messages.pot in the translations branch
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git stash pop
+        popd
+
+        # update messages.po files in the master branch
+        build_msg BABEL 'update existing message catalogs from POT file'
+        pybabel update -N \
+            -i "${messages_pot}" \
+            -d "${TRANSLATIONS_WORKTREE}/searx/translations"
+
+        # git add/commit/push
+        last_commit_hash=$(git log -n1  --pretty=format:'%h')
+        last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
+
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git add searx/translations
+        git commit \
+            -m "[translations] update messages.pot and messages.po files" \
+            -m "From ${last_commit_detail}"
+        git push
+        popd
+
+        # notify weblate to pull updated master & translations branch
+        wlc pull
+    )
+    exitcode=$?
+    (   # make sure to always unlock weblate
+        set -e
+        pyenv.activate
+        wlc unlock
+    )
+    dump_return $exitcode
+}