From 782f73540e2d383ea122716507ccd9582918ab51 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 27 May 2022 18:43:14 +0200 Subject: [PATCH] [utils/searxng.sh] implement new script to install SearXNG Signed-off-by: Markus Heiser --- Makefile | 3 +- docs/dev/lxcdev.rst | 4 +- docs/utils/index.rst | 7 +- docs/utils/lxc.sh.rst | 46 +- searx/shared/redisdb.py | 6 +- utils/lib.sh | 47 +- utils/lib_install.sh | 7 +- utils/lib_redis.sh | 27 +- utils/{lxc-searx.env => lxc-searxng.env} | 51 +- utils/lxc.sh | 2 +- utils/morty.sh | 2 +- utils/searxng.sh | 894 ++++++++++++++++++ utils/searxng_check.py | 7 + .../etc/httpd/sites-available/searxng.conf | 41 + .../httpd/sites-available/searxng.conf:socket | 41 + .../nginx/default.apps-available/searxng.conf | 29 + .../searxng.conf:socket | 26 + utils/templates/etc/searxng/settings.yml | 78 +- .../etc/uwsgi/apps-archlinux/searxng.ini | 18 +- .../uwsgi/apps-archlinux/searxng.ini:socket | 17 - .../etc/uwsgi/apps-available/searxng.ini | 24 +- .../uwsgi/apps-available/searxng.ini:socket | 23 +- 22 files changed, 1182 insertions(+), 218 deletions(-) rename utils/{lxc-searx.env => lxc-searxng.env} (53%) create mode 100755 utils/searxng.sh create mode 100644 utils/templates/etc/httpd/sites-available/searxng.conf create mode 100644 utils/templates/etc/httpd/sites-available/searxng.conf:socket create mode 100644 utils/templates/etc/nginx/default.apps-available/searxng.conf create mode 100644 utils/templates/etc/nginx/default.apps-available/searxng.conf:socket diff --git a/Makefile b/Makefile index fe28e0f47..f21734793 100644 --- a/Makefile +++ b/Makefile @@ -66,9 +66,10 @@ test.shell: utils/lib_redis.sh \ utils/filtron.sh \ utils/searx.sh \ + utils/searxng.sh \ utils/morty.sh \ utils/lxc.sh \ - utils/lxc-searx.env \ + utils/lxc-searxng.env \ .config.sh $(Q)$(MTOOLS) build_msg TEST "$@ OK" diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst index 597f99925..85b262212 100644 --- a/docs/dev/lxcdev.rst +++ b/docs/dev/lxcdev.rst @@ -91,8 +91,8 @@ fork: $ git clone https://github.com/searxng/searxng.git searxng $ cd searxng -The :ref:`lxc-searx.env` consists of several images, see ``export -LXC_SUITE=(...`` near by :origin:`utils/lxc-searx.env#L19`. For this blog post +The :ref:`lxc-searxng.env` consists of several images, see ``export +LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`. For this blog post we exercise on a archlinux_ image. The container of this image is named ``searx-archlinux``. Lets build the container, but be sure that this container does not already exists, so first lets remove possible old one: diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 2077bbb13..62e02655b 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -35,10 +35,11 @@ Scripts to maintain services often dispose of common commands and environments. ``FORCE_TIMEOUT`` : environment Sets timeout for interactive prompts. If you want to run a script in batch job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a - reverse proxy for filtron on all containers of the :ref:`SearXNG suite - ` use :: + SearXNG server and nginx proxy on all containers of the :ref:`SearXNG suite + ` use:: - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx .. _toolboxing setup: diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index f09b033f4..291c4f147 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -23,7 +23,7 @@ With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of containers, what we call the: *lxc suite*. The *SearXNG suite* -(:origin:`lxc-searx.env `) is loaded by default, every time +(:origin:`lxc-searxng.env `) is loaded by default, every time you start the ``lxc.sh`` script (*you do not need to care about*). Before you can start with containers, you need to install and initiate LXD_ @@ -49,7 +49,7 @@ help>`. If you do not want to build all containers, **you can build just one**:: - $ sudo -H ./utils/lxc.sh build searx-ubu1804 + $ sudo -H ./utils/lxc.sh build searx-archlinux *Good to know ...* @@ -62,9 +62,9 @@ of:: In the containers, you can run what ever you want, e.g. to start a bash use:: - $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash - INFO: [searx-ubu1804] bash - root@searx-ubu1804:/share/searx# + $ sudo -H ./utils/lxc.sh cmd searx-archlinux bash + INFO: [searx-archlinux] bash + root@searx-archlinux:/share/searxng# If there comes the time you want to **get rid off all** the containers and **clean up local images** just type:: @@ -121,28 +121,26 @@ Install suite ============= To install the complete :ref:`SearXNG suite (includes searx, morty & filtron) -` into all LXC_ use:: +` into all LXC_ use:: $ sudo -H ./utils/lxc.sh install suite -The command above installs a SearXNG suite (see :ref:`installation scripts`). To -get the IP (URL) of the filtron service in the containers use ``show suite`` +The command above installs a SearXNG suite (see :ref:`installation scripts`). +To :ref:`install a nginx ` reverse proxy (or alternatively +use :ref:`apache `):: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx + +To get the IP (URL) of the SearXNG service in the containers use ``show suite`` command. To test instances from containers just open the URLs in your WEB-Browser:: - $ sudo ./utils/lxc.sh show suite | grep filtron - [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx - [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx - [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx - [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx - [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx - [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx + $ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL -To :ref:`install a nginx ` reverse proxy for filtron and -morty use (or alternatively use :ref:`apache `):: - - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install + [searxng-ubu2110] SEARXNG_URL : http://n.n.n.147/searxng + [searxng-ubu2004] SEARXNG_URL : http://n.n.n.246/searxng + [searxnggfedora35] SEARXNG_URL : http://n.n.n.140/searxng + [searxng-archlinux] SEARXNG_URL : http://n.n.n.165/searxng Running commands @@ -152,7 +150,7 @@ Running commands :ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the Makefile target ``test`` in the archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost + sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searxng.sh install buildhost sudo -H ./utils/lxc.sh cmd searx-archlinux make test @@ -164,7 +162,7 @@ The installation procedure to set up a :ref:`build host` takes its time. Installation in all containers will take more time (time for another cup of coffee).:: - sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost + sudo -H ./utils/lxc.sh cmd -- ./utils/searxng.sh install buildhost To build (live) documentation inside a archlinux_ container:: @@ -189,10 +187,10 @@ The ``--help`` output of the script is largely self-explanatory: .. program-output:: ../utils/lxc.sh --help -.. _lxc-searx.env: +.. _lxc-searxng.env: SearXNG suite ============= -.. literalinclude:: ../../utils/lxc-searx.env +.. literalinclude:: ../../utils/lxc-searxng.env :language: bash diff --git a/searx/shared/redisdb.py b/searx/shared/redisdb.py index da71d169c..bb7a0eeb4 100644 --- a/searx/shared/redisdb.py +++ b/searx/shared/redisdb.py @@ -19,10 +19,13 @@ A redis DB connect can be tested by:: """ +import os +import pwd import logging import redis from searx import get_setting + logger = logging.getLogger('searx.shared.redis') _client = None @@ -42,6 +45,7 @@ def init(): logger.info("connected redis DB --> %s", c.acl_whoami()) return True except redis.exceptions.ConnectionError as exc: - logger.error("can't connet redis DB ...") + _pw = pwd.getpwuid(os.getuid()) + logger.error("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid) logger.error(" %s", exc) return False diff --git a/utils/lib.sh b/utils/lib.sh index 257c15024..32db47524 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -195,7 +195,7 @@ wait_key(){ [[ -n $_t ]] && _t="-t $_t" printf "$msg" # shellcheck disable=SC2086 - read -r -s -n1 $_t + read -r -s -n1 $_t || true echo clean_stdin } @@ -1027,7 +1027,7 @@ nginx_include_apps_enabled() { local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;" local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;" - info_msg "checking existence: '${include_directive}' in file ${server_conf}" + info_msg "checking existence: '${include_directive}' in file ${server_conf}" if grep "${include_directive_re}" "${server_conf}"; then info_msg "OK, already exists." return @@ -1117,7 +1117,7 @@ apache_distro_setup() { APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" APACHE_MODULES="modules" - APACHE_PACKAGES="httpd" + APACHE_PACKAGES="httpd mod_ssl" ;; *) err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented" @@ -1249,8 +1249,6 @@ apache_dissable_site() { # ----- uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" -uWSGI_USER= -uWSGI_GROUP= # How distros manage uWSGI apps is very different. From uWSGI POV read: # - https://uwsgi-docs.readthedocs.io/en/latest/Management.html @@ -1276,13 +1274,14 @@ uWSGI_distro_setup() { ;; fedora-*|centos-7) # systemd --> /usr/lib/systemd/system/uwsgi.service - # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see - # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # There are some quirks abbout additional POSIX groups in uWSGI + # 2.0.x, read at least: https://github.com/unbit/uwsgi/issues/2099 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d" uWSGI_PACKAGES="uwsgi" - uWSGI_USER="uwsgi" - uWSGI_GROUP="uwsgi" ;; *) err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" @@ -1344,30 +1343,6 @@ uWSGI_restart() { esac } -uWSGI_prepare_app() { - - # usage: uWSGI_prepare_app - - [[ -z $1 ]] && die_caller 42 "missing argument " - - local APP="${1%.*}" - - case $DIST_ID-$DIST_VERS in - fedora-*|centos-7) - # in emperor mode, the uwsgi user is the owner of the sockets - info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" - ;; - *) - info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" - ;; - esac -} - - uWSGI_app_available() { # usage: uWSGI_app_available local CONF="$1" @@ -1378,7 +1353,7 @@ uWSGI_app_available() { uWSGI_install_app() { - # usage: uWSGI_install_app [