Merge pull request #686 from return42/lib_redis

Add redis DB and connector

Makefile

@@ -63,6 +63,7 @@ test.shell:
 		utils/lib_nvm.sh \
 		utils/lib_static.sh \
 		utils/lib_go.sh \
+		utils/lib_redis.sh \
 		utils/filtron.sh \
 		utils/searx.sh \
 		utils/morty.sh \

docs/admin/engines/settings.rst

@@ -142,6 +142,36 @@ Global Settings
 ``default_http_headers``:
   Set additional HTTP headers, see `#755 <https://github.com/searx/searx/issues/715>`__
 
+
+.. _settings redis:
+
+``redis:``
+----------
+
+.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
+
+``url``
+  URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`::
+
+    redis://[[username]:[password]]@localhost:6379/0
+    rediss://[[username]:[password]]@localhost:6379/0
+    unix://[[username]:[password]]@/path/to/socket.sock?db=0
+
+.. admonition:: Tip for developers
+
+   To set up a redis instance simply use::
+
+     $ ./manage redis.build
+     $ sudo -H ./manage redis.install
+
+   To get access rights to this instance, your developer account needs to be
+   added to the *searxng-redis* group::
+
+     $ sudo -H ./manage redis.addgrp "${USER}"
+     # don't forget to logout & login to get member of group
+
+.. _settings outgoing:
+
 ``outgoing:``
 -------------
 

docs/src/searx.shared.redisdb.rst

@@ -0,0 +1,8 @@
+.. _redis db:
+
+========
+Redis DB
+========
+
+.. automodule:: searx.shared.redisdb
+  :members:

manage

@@ -17,6 +17,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
 # shellcheck source=utils/lib_go.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
 
+# shellcheck source=utils/lib_redis.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"
+
 # config
 
 PYOBJECTS="searx"
@@ -74,9 +77,10 @@ docker.:
 gecko.driver:
   download & install geckodriver if not already installed (required for
   robot_tests)
-EOF
+redis:
-    nvm.help
+  build     : build redis binaries at $(redis._get_dist)
-    cat <<EOF
+  install   : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
+  help      : show more redis commands
 node.:
   env       : download & install npm dependencies locally
   clean     : drop locally npm installations

requirements.txt

@@ -13,3 +13,4 @@ uvloop==0.16.0
 httpx-socks[asyncio]==0.7.2
 langdetect==1.0.9
 setproctitle==1.2.2
+redis==4.1.0

searx/settings.yml

@@ -62,6 +62,10 @@ server:
     X-Robots-Tag: noindex, nofollow
     Referrer-Policy: no-referrer
 
+redis:
+  # https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
+  url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+
 ui:
   # Custom static path - leave it blank if you didn't change
   static_path: ""

searx/settings_defaults.py

@@ -170,6 +170,9 @@ SCHEMA = {
         'method': SettingsValue(('POST', 'GET'), 'POST'),
         'default_http_headers': SettingsValue(dict, {}),
     },
+    'redis': {
+        'url': SettingsValue(str, 'unix:///usr/local/searxng-redis/run/redis.sock?db=0'),
+    },
     'ui': {
         'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
         'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')),

searx/shared/redisdb.py

@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Implementation of the redis client (redis-py_).
+
+.. _redis-py: https://github.com/redis/redis-py
+
+This implementation uses the :ref:`settings redis` setup from ``settings.yml``.
+A redis DB connect can be tested by::
+
+  >>> from searx.shared import redisdb
+  >>> redisdb.init()
+  True
+  >>> db = redisdb.client()
+  >>> db.set("foo", "bar")
+  True
+  >>> db.get("foo")
+  b'bar'
+  >>>
+
+"""
+
+import logging
+import redis
+from searx import get_setting
+
+logger = logging.getLogger('searx.shared.redis')
+_client = None
+
+
+def client():
+    global _client  # pylint: disable=global-statement
+    if _client is None:
+        # not thread safe: in the worst case scenario, two or more clients are
+        # initialized only one is kept, the others are garbage collected.
+        _client = redis.Redis.from_url(get_setting('redis.url'))
+    return _client
+
+
+def init():
+    try:
+        c = client()
+        logger.info("connected redis DB --> %s", c.acl_whoami())
+        return True
+    except redis.exceptions.ConnectionError as exc:
+        logger.error("can't connet redis DB ...")
+        logger.error("  %s", exc)
+    return False

utils/lib_redis.sh

@@ -0,0 +1,348 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Tools to build and install redis [1] binaries & packages.
+#
+# [1] https://redis.io/download#installation
+#
+# 1. redis.devpkg (sudo)
+# 2. redis.build
+# 3. redis.install (sudo)
+#
+# systemd commands::
+#
+#    sudo -H systemctl status searxng-redis
+#    sudo -H journalctl -u searxng-redis
+#    sudo -H journalctl --vacuum-size=1M
+#
+# Test socket connection from client (local user)::
+#
+#    $ sudo -H ./manage redis.addgrp "${USER}"
+#    # logout & login to get member of group
+#    $ groups
+#    ... searxng-redis ...
+#    $ source /usr/local/searxng-redis/.redis_env
+#    $ which redis-cli
+#    /usr/local/searxng-redis/.local/bin/redis-cli
+#
+#    $ redis-cli -s /usr/local/searxng-redis/redis.sock
+#    redis /usr/local/searxng-redis/redis.sock> set foo bar
+#    OK
+#    redis /usr/local/searxng-redis/redis.sock> get foo
+#    "bar"
+#    [CTRL-D]
+
+
+# shellcheck disable=SC2091
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+REDIS_GIT_URL="https://github.com/redis/redis.git"
+REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}"
+
+REDIS_USER="searxng-redis"
+REDIS_HOME="/usr/local/${REDIS_USER}"
+REDIS_HOME_BIN="${REDIS_HOME}/.local/bin"
+REDIS_ENV="${REDIS_HOME}/.redis_env"
+
+REDIS_SERVICE_NAME="searxng-redis"
+REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service"
+
+# binaries to compile & install
+REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli)
+# link names of redis-server binary
+REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof)
+
+REDIS_CONF="${REDIS_HOME}/redis.conf"
+REDIS_CONF_TEMPLATE=$(cat <<EOF
+# Note that in order to read the configuration file, Redis must be
+# started with the file path as first argument:
+#
+# ./redis-server /path/to/redis.conf
+
+# bind 127.0.0.1 -::1
+protected-mode yes
+
+# Accept connections on the specified port, default is 6379 (IANA #815344).
+# If port 0 is specified Redis will not listen on a TCP socket.
+port 0
+
+# Specify the path for the Unix socket that will be used to listen for
+# incoming connections.
+
+unixsocket ${REDIS_HOME}/run/redis.sock
+unixsocketperm 770
+
+# The working directory.
+dir ${REDIS_HOME}/run
+
+# If you run Redis from upstart or systemd, Redis can interact with your
+# supervision tree.
+supervised auto
+
+pidfile ${REDIS_HOME}/run/redis.pid
+
+# log to the system logger
+syslog-enabled yes
+EOF
+)
+
+redis.help(){
+    cat <<EOF
+redis.:
+  devpkg    : install essential packages to compile redis
+  build     : build redis binaries at $(redis._get_dist)
+  install   : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
+  remove    : delete user (${REDIS_USER}) and remove service (${REDIS_SERVICE_NAME})
+  shell     : start bash interpreter from user ${REDIS_USER}
+  src       : clone redis source code to <path> and checkput ${REDIS_GIT_TAG}
+  useradd   : create user (${REDIS_USER}) at ${REDIS_HOME}
+  userdel   : delete user (${REDIS_USER})
+  addgrp    : add <user> to group (${REDIS_USER})
+  rmgrp     : remove <user> from group (${REDIS_USER})
+EOF
+}
+
+redis.devpkg() {
+
+    # Uses OS package manager to install the essential packages to build and
+    # compile sources
+
+    sudo_or_exit
+
+    case ${DIST_ID} in
+        ubuntu|debian)
+            pkg_install git build-essential
+            ;;
+        arch)
+            pkg_install git base-devel
+            ;;
+        fedora)
+            pkg_install git @development-tools
+            ;;
+        centos)
+            pkg_install git
+            yum groupinstall "Development Tools" -y
+            ;;
+        *)
+            err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS."
+            return 42
+            ;;
+    esac
+}
+
+redis.build() {
+
+    # usage: redis.build
+
+    rst_title "get redis sources" section
+    redis.src "${CACHE}/redis"
+
+    if ! required_commands gcc nm make gawk; then
+        sudo -H "$0" redis.devpkg
+    fi
+
+    rst_title "compile redis sources" section
+
+    pushd "${CACHE}/redis" &>/dev/null
+
+    if ask_yn "Do you run 'make distclean' first'?" Ny; then
+        $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout
+    fi
+
+    $(bash.cmd) -c "make" 2>&1 | prefix_stdout
+    if ask_yn "Do you run 'make test'?" Ny; then
+        $(bash.cmd) -c "make test" | prefix_stdout
+    fi
+
+    popd &>/dev/null
+
+    tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 |  prefix_stdout
+mkdir -p "$(redis._get_dist)"
+cd "${CACHE}/redis/src"
+cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)"
+EOF
+    info_msg "redis binaries available at $(redis._get_dist)"
+}
+
+
+redis.install() {
+    sudo_or_exit
+    (
+        set -e
+        redis.useradd
+        redis._install_bin
+        redis._install_conf
+        redis._install_service
+    )
+    dump_return $?
+}
+
+redis.remove() {
+    sudo_or_exit
+    (
+        set -e
+        redis._remove_service
+        redis.userdel
+    )
+    dump_return $?
+}
+
+redis.shell() {
+    interactive_shell "${REDIS_USER}"
+}
+
+redis.src() {
+
+    # usage: redis.src "${CACHE}/redis"
+
+    local dest="${1:-${CACHE}/redis}"
+
+    if [ -d "${dest}" ] ; then
+        info_msg "already cloned: $dest"
+        tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
+cd "${dest}"
+git fetch --all
+git reset --hard tags/${REDIS_GIT_TAG}
+EOF
+    else
+        tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
+mkdir -p "$(dirname "$dest")"
+cd "$(dirname "$dest")"
+git clone "${REDIS_GIT_URL}" "${dest}"
+EOF
+        tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
+cd "${dest}"
+git checkout tags/${REDIS_GIT_TAG} -b "build-branch"
+EOF
+    fi
+}
+
+redis.useradd(){
+
+    # usage: redis.useradd
+
+    rst_title "add user ${REDIS_USER}" section
+    echo
+    sudo_or_exit
+
+    # create user account
+    tee_stderr 0.5 <<EOF | sudo -H bash | prefix_stdout
+useradd --shell /bin/bash --system \
+ --home-dir "${REDIS_HOME}" \
+ --comment 'user that runs a redis instance' "${REDIS_USER}"
+mkdir -p "${REDIS_HOME}"
+chown -R "${REDIS_USER}:${REDIS_USER}" "${REDIS_HOME}"
+groups "${REDIS_USER}"
+EOF
+
+    # create App-ENV and add source it in the .profile
+    tee_stderr 0.5 <<EOF | sudo -H -u "${REDIS_USER}" bash | prefix_stdout
+mkdir -p "${REDIS_HOME_BIN}"
+echo "export PATH=${REDIS_HOME_BIN}:\\\$PATH" > "${REDIS_ENV}"
+grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile
+EOF
+}
+
+redis.userdel() {
+    sudo_or_exit
+    drop_service_account "${REDIS_USER}"
+    groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true
+}
+
+redis.addgrp() {
+
+    # usage: redis.addgrp <user>
+
+    [[ -z $1 ]] && die_caller 42 "missing argument <user>"
+    sudo -H gpasswd -a "$1" "${REDIS_USER}"
+}
+
+redis.rmgrp() {
+
+    # usage: redis.rmgrp <user>
+
+    [[ -z $1 ]] && die_caller 42 "missing argument <user>"
+    sudo -H gpasswd -d "$1" "${REDIS_USER}"
+
+}
+
+
+# private redis. functions
+# ------------------------
+
+redis._install_bin() {
+    local src
+    src="$(redis._get_dist)"
+    (
+        set -e
+        for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do
+            install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \
+                 "${src}/${redis_exe}" "${REDIS_HOME_BIN}"
+        done
+
+        pushd "${REDIS_HOME_BIN}" &> /dev/null
+        for redis_exe in "${REDIS_LINK_EXE[@]}"; do
+            info_msg "link redis-server --> ${redis_exe}"
+            sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}"
+        done
+        popd &> /dev/null
+
+    )
+}
+
+redis._install_conf() {
+        sudo -H -u "${REDIS_USER}" bash <<EOF
+mkdir -p "${REDIS_HOME}/run"
+echo '${REDIS_CONF_TEMPLATE}' > "${REDIS_CONF}"
+EOF
+}
+
+redis._install_service() {
+    systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
+}
+
+redis._remove_service() {
+    systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
+}
+
+redis._get_dist() {
+    if [ -z "${REDIS_DIST}" ]; then
+        echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)"
+    else
+        echo "${REDIS_DIST}"
+    fi
+}
+
+redis._arch() {
+    local ARCH
+    case "$(command uname -m)" in
+        "x86_64") ARCH=amd64 ;;
+        "aarch64") ARCH=arm64 ;;
+        "armv6" | "armv7l") ARCH=armv6l ;;
+        "armv8") ARCH=arm64 ;;
+        .*386.*) ARCH=386 ;;
+        ppc64*) ARCH=ppc64le ;;
+    *)  die 42 "ARCH is unknown: $(command uname -m)" ;;
+    esac
+    echo "${ARCH}"
+}
+
+# TODO: move this to the right place ..
+
+bash.cmd(){
+
+    # print cmd to get a bash in a non-root mode, even if we are in a sudo
+    # context.
+
+    local user="${USER}"
+    local bash_cmd="bash"
+
+    if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then
+        user="${SUDO_USER}"
+        bash_cmd="sudo -H -u ${SUDO_USER} bash"
+    fi
+
+    printf "%s" "${bash_cmd}"
+}

utils/templates/lib/systemd/system/searxng-redis.service

@@ -0,0 +1,42 @@
+[Unit]
+
+Description=SearXNG redis service
+After=syslog.target
+After=network.target
+Documentation=https://redis.io/documentation
+
+[Service]
+
+Type=simple
+User=${REDIS_USER}
+Group=${REDIS_USER}
+WorkingDirectory=${REDIS_HOME}
+Restart=always
+TimeoutStopSec=0
+
+Environment=USER=${REDIS_USER} HOME=${REDIS_HOME}
+ExecStart=${REDIS_HOME_BIN}/redis-server ${REDIS_CONF}
+ExecPaths=${REDIS_HOME_BIN}
+
+LimitNOFILE=65535
+NoNewPrivileges=true
+PrivateDevices=yes
+
+# ProtectSystem=full
+ProtectHome=yes
+ReadOnlyDirectories=/
+ReadWritePaths=-${REDIS_HOME}/run
+
+UMask=007
+PrivateTmp=yes
+
+MemoryDenyWriteExecute=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+RestrictRealtime=true
+RestrictNamespaces=true
+
+[Install]
+
+WantedBy=multi-user.target