mirror of https://github.com/searxng/searxng.git
Merge pull request #686 from return42/lib_redis
Add redis DB and connector
This commit is contained in:
Markus Heiser
GitHub
commit
977e9a4330
1
Makefile
1
Makefile
|
|
@ -63,6 +63,7 @@ test.shell:
|
|||
utils/lib_nvm.sh \
|
||||
utils/lib_static.sh \
|
||||
utils/lib_go.sh \
|
||||
utils/lib_redis.sh \
|
||||
utils/filtron.sh \
|
||||
utils/searx.sh \
|
||||
utils/morty.sh \
|
||||
|
|
|
|||
|
|
@ -142,6 +142,36 @@ Global Settings
|
|||
``default_http_headers``:
|
||||
Set additional HTTP headers, see `#755 <https://github.com/searx/searx/issues/715>`__
|
||||
|
||||
|
||||
.. _settings redis:
|
||||
|
||||
``redis:``
|
||||
----------
|
||||
|
||||
.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
|
||||
|
||||
``url``
|
||||
URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`::
|
||||
|
||||
redis://[[username]:[password]]@localhost:6379/0
|
||||
rediss://[[username]:[password]]@localhost:6379/0
|
||||
unix://[[username]:[password]]@/path/to/socket.sock?db=0
|
||||
|
||||
.. admonition:: Tip for developers
|
||||
|
||||
To set up a redis instance simply use::
|
||||
|
||||
$ ./manage redis.build
|
||||
$ sudo -H ./manage redis.install
|
||||
|
||||
To get access rights to this instance, your developer account needs to be
|
||||
added to the *searxng-redis* group::
|
||||
|
||||
$ sudo -H ./manage redis.addgrp "${USER}"
|
||||
# don't forget to logout & login to get member of group
|
||||
|
||||
.. _settings outgoing:
|
||||
|
||||
``outgoing:``
|
||||
-------------
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
.. _redis db:
|
||||
|
||||
========
|
||||
Redis DB
|
||||
========
|
||||
|
||||
.. automodule:: searx.shared.redisdb
|
||||
:members:
|
||||
10
manage
10
manage
|
|
@ -17,6 +17,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
|
|||
# shellcheck source=utils/lib_go.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
|
||||
|
||||
# shellcheck source=utils/lib_redis.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"
|
||||
|
||||
# config
|
||||
|
||||
PYOBJECTS="searx"
|
||||
|
|
@ -74,9 +77,10 @@ docker.:
|
|||
gecko.driver:
|
||||
download & install geckodriver if not already installed (required for
|
||||
robot_tests)
|
||||
EOF
|
||||
nvm.help
|
||||
cat <<EOF
|
||||
redis:
|
||||
build : build redis binaries at $(redis._get_dist)
|
||||
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
|
||||
help : show more redis commands
|
||||
node.:
|
||||
env : download & install npm dependencies locally
|
||||
clean : drop locally npm installations
|
||||
|
|
|
|||
|
|
@ -13,3 +13,4 @@ uvloop==0.16.0
|
|||
httpx-socks[asyncio]==0.7.2
|
||||
langdetect==1.0.9
|
||||
setproctitle==1.2.2
|
||||
redis==4.1.0
|
||||
|
|
|
|||
|
|
@ -62,6 +62,10 @@ server:
|
|||
X-Robots-Tag: noindex, nofollow
|
||||
Referrer-Policy: no-referrer
|
||||
|
||||
redis:
|
||||
# https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
|
||||
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
||||
|
||||
ui:
|
||||
# Custom static path - leave it blank if you didn't change
|
||||
static_path: ""
|
||||
|
|
|
|||
|
|
@ -170,6 +170,9 @@ SCHEMA = {
|
|||
'method': SettingsValue(('POST', 'GET'), 'POST'),
|
||||
'default_http_headers': SettingsValue(dict, {}),
|
||||
},
|
||||
'redis': {
|
||||
'url': SettingsValue(str, 'unix:///usr/local/searxng-redis/run/redis.sock?db=0'),
|
||||
},
|
||||
'ui': {
|
||||
'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
|
||||
'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""Implementation of the redis client (redis-py_).
|
||||
|
||||
.. _redis-py: https://github.com/redis/redis-py
|
||||
|
||||
This implementation uses the :ref:`settings redis` setup from ``settings.yml``.
|
||||
A redis DB connect can be tested by::
|
||||
|
||||
>>> from searx.shared import redisdb
|
||||
>>> redisdb.init()
|
||||
True
|
||||
>>> db = redisdb.client()
|
||||
>>> db.set("foo", "bar")
|
||||
True
|
||||
>>> db.get("foo")
|
||||
b'bar'
|
||||
>>>
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
import redis
|
||||
from searx import get_setting
|
||||
|
||||
logger = logging.getLogger('searx.shared.redis')
|
||||
_client = None
|
||||
|
||||
|
||||
def client():
|
||||
global _client # pylint: disable=global-statement
|
||||
if _client is None:
|
||||
# not thread safe: in the worst case scenario, two or more clients are
|
||||
# initialized only one is kept, the others are garbage collected.
|
||||
_client = redis.Redis.from_url(get_setting('redis.url'))
|
||||
return _client
|
||||
|
||||
|
||||
def init():
|
||||
try:
|
||||
c = client()
|
||||
logger.info("connected redis DB --> %s", c.acl_whoami())
|
||||
return True
|
||||
except redis.exceptions.ConnectionError as exc:
|
||||
logger.error("can't connet redis DB ...")
|
||||
logger.error(" %s", exc)
|
||||
return False
|
||||
|
|
@ -0,0 +1,348 @@
|
|||
#!/usr/bin/env bash
|
||||
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
#
|
||||
# Tools to build and install redis [1] binaries & packages.
|
||||
#
|
||||
# [1] https://redis.io/download#installation
|
||||
#
|
||||
# 1. redis.devpkg (sudo)
|
||||
# 2. redis.build
|
||||
# 3. redis.install (sudo)
|
||||
#
|
||||
# systemd commands::
|
||||
#
|
||||
# sudo -H systemctl status searxng-redis
|
||||
# sudo -H journalctl -u searxng-redis
|
||||
# sudo -H journalctl --vacuum-size=1M
|
||||
#
|
||||
# Test socket connection from client (local user)::
|
||||
#
|
||||
# $ sudo -H ./manage redis.addgrp "${USER}"
|
||||
# # logout & login to get member of group
|
||||
# $ groups
|
||||
# ... searxng-redis ...
|
||||
# $ source /usr/local/searxng-redis/.redis_env
|
||||
# $ which redis-cli
|
||||
# /usr/local/searxng-redis/.local/bin/redis-cli
|
||||
#
|
||||
# $ redis-cli -s /usr/local/searxng-redis/redis.sock
|
||||
# redis /usr/local/searxng-redis/redis.sock> set foo bar
|
||||
# OK
|
||||
# redis /usr/local/searxng-redis/redis.sock> get foo
|
||||
# "bar"
|
||||
# [CTRL-D]
|
||||
|
||||
|
||||
# shellcheck disable=SC2091
|
||||
# shellcheck source=utils/lib.sh
|
||||
. /dev/null
|
||||
|
||||
REDIS_GIT_URL="https://github.com/redis/redis.git"
|
||||
REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}"
|
||||
|
||||
REDIS_USER="searxng-redis"
|
||||
REDIS_HOME="/usr/local/${REDIS_USER}"
|
||||
REDIS_HOME_BIN="${REDIS_HOME}/.local/bin"
|
||||
REDIS_ENV="${REDIS_HOME}/.redis_env"
|
||||
|
||||
REDIS_SERVICE_NAME="searxng-redis"
|
||||
REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service"
|
||||
|
||||
# binaries to compile & install
|
||||
REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli)
|
||||
# link names of redis-server binary
|
||||
REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof)
|
||||
|
||||
REDIS_CONF="${REDIS_HOME}/redis.conf"
|
||||
REDIS_CONF_TEMPLATE=$(cat <<EOF
|
||||
# Note that in order to read the configuration file, Redis must be
|
||||
# started with the file path as first argument:
|
||||
#
|
||||
# ./redis-server /path/to/redis.conf
|
||||
|
||||
# bind 127.0.0.1 -::1
|
||||
protected-mode yes
|
||||
|
||||
# Accept connections on the specified port, default is 6379 (IANA #815344).
|
||||
# If port 0 is specified Redis will not listen on a TCP socket.
|
||||
port 0
|
||||
|
||||
# Specify the path for the Unix socket that will be used to listen for
|
||||
# incoming connections.
|
||||
|
||||
unixsocket ${REDIS_HOME}/run/redis.sock
|
||||
unixsocketperm 770
|
||||
|
||||
# The working directory.
|
||||
dir ${REDIS_HOME}/run
|
||||
|
||||
# If you run Redis from upstart or systemd, Redis can interact with your
|
||||
# supervision tree.
|
||||
supervised auto
|
||||
|
||||
pidfile ${REDIS_HOME}/run/redis.pid
|
||||
|
||||
# log to the system logger
|
||||
syslog-enabled yes
|
||||
EOF
|
||||
)
|
||||
|
||||
redis.help(){
|
||||
cat <<EOF
|
||||
redis.:
|
||||
devpkg : install essential packages to compile redis
|
||||
build : build redis binaries at $(redis._get_dist)
|
||||
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
|
||||
remove : delete user (${REDIS_USER}) and remove service (${REDIS_SERVICE_NAME})
|
||||
shell : start bash interpreter from user ${REDIS_USER}
|
||||
src : clone redis source code to <path> and checkput ${REDIS_GIT_TAG}
|
||||
useradd : create user (${REDIS_USER}) at ${REDIS_HOME}
|
||||
userdel : delete user (${REDIS_USER})
|
||||
addgrp : add <user> to group (${REDIS_USER})
|
||||
rmgrp : remove <user> from group (${REDIS_USER})
|
||||
EOF
|
||||
}
|
||||
|
||||
redis.devpkg() {
|
||||
|
||||
# Uses OS package manager to install the essential packages to build and
|
||||
# compile sources
|
||||
|
||||
sudo_or_exit
|
||||
|
||||
case ${DIST_ID} in
|
||||
ubuntu|debian)
|
||||
pkg_install git build-essential
|
||||
;;
|
||||
arch)
|
||||
pkg_install git base-devel
|
||||
;;
|
||||
fedora)
|
||||
pkg_install git @development-tools
|
||||
;;
|
||||
centos)
|
||||
pkg_install git
|
||||
yum groupinstall "Development Tools" -y
|
||||
;;
|
||||
*)
|
||||
err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS."
|
||||
return 42
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
redis.build() {
|
||||
|
||||
# usage: redis.build
|
||||
|
||||
rst_title "get redis sources" section
|
||||
redis.src "${CACHE}/redis"
|
||||
|
||||
if ! required_commands gcc nm make gawk; then
|
||||
sudo -H "$0" redis.devpkg
|
||||
fi
|
||||
|
||||
rst_title "compile redis sources" section
|
||||
|
||||
pushd "${CACHE}/redis" &>/dev/null
|
||||
|
||||
if ask_yn "Do you run 'make distclean' first'?" Ny; then
|
||||
$(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout
|
||||
fi
|
||||
|
||||
$(bash.cmd) -c "make" 2>&1 | prefix_stdout
|
||||
if ask_yn "Do you run 'make test'?" Ny; then
|
||||
$(bash.cmd) -c "make test" | prefix_stdout
|
||||
fi
|
||||
|
||||
popd &>/dev/null
|
||||
|
||||
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
|
||||
mkdir -p "$(redis._get_dist)"
|
||||
cd "${CACHE}/redis/src"
|
||||
cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)"
|
||||
EOF
|
||||
info_msg "redis binaries available at $(redis._get_dist)"
|
||||
}
|
||||
|
||||
|
||||
redis.install() {
|
||||
sudo_or_exit
|
||||
(
|
||||
set -e
|
||||
redis.useradd
|
||||
redis._install_bin
|
||||
redis._install_conf
|
||||
redis._install_service
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
redis.remove() {
|
||||
sudo_or_exit
|
||||
(
|
||||
set -e
|
||||
redis._remove_service
|
||||
redis.userdel
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
redis.shell() {
|
||||
interactive_shell "${REDIS_USER}"
|
||||
}
|
||||
|
||||
redis.src() {
|
||||
|
||||
# usage: redis.src "${CACHE}/redis"
|
||||
|
||||
local dest="${1:-${CACHE}/redis}"
|
||||
|
||||
if [ -d "${dest}" ] ; then
|
||||
info_msg "already cloned: $dest"
|
||||
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
|
||||
cd "${dest}"
|
||||
git fetch --all
|
||||
git reset --hard tags/${REDIS_GIT_TAG}
|
||||
EOF
|
||||
else
|
||||
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
cd "$(dirname "$dest")"
|
||||
git clone "${REDIS_GIT_URL}" "${dest}"
|
||||
EOF
|
||||
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
|
||||
cd "${dest}"
|
||||
git checkout tags/${REDIS_GIT_TAG} -b "build-branch"
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
redis.useradd(){
|
||||
|
||||
# usage: redis.useradd
|
||||
|
||||
rst_title "add user ${REDIS_USER}" section
|
||||
echo
|
||||
sudo_or_exit
|
||||
|
||||
# create user account
|
||||
tee_stderr 0.5 <<EOF | sudo -H bash | prefix_stdout
|
||||
useradd --shell /bin/bash --system \
|
||||
--home-dir "${REDIS_HOME}" \
|
||||
--comment 'user that runs a redis instance' "${REDIS_USER}"
|
||||
mkdir -p "${REDIS_HOME}"
|
||||
chown -R "${REDIS_USER}:${REDIS_USER}" "${REDIS_HOME}"
|
||||
groups "${REDIS_USER}"
|
||||
EOF
|
||||
|
||||
# create App-ENV and add source it in the .profile
|
||||
tee_stderr 0.5 <<EOF | sudo -H -u "${REDIS_USER}" bash | prefix_stdout
|
||||
mkdir -p "${REDIS_HOME_BIN}"
|
||||
echo "export PATH=${REDIS_HOME_BIN}:\\\$PATH" > "${REDIS_ENV}"
|
||||
grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile
|
||||
EOF
|
||||
}
|
||||
|
||||
redis.userdel() {
|
||||
sudo_or_exit
|
||||
drop_service_account "${REDIS_USER}"
|
||||
groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true
|
||||
}
|
||||
|
||||
redis.addgrp() {
|
||||
|
||||
# usage: redis.addgrp <user>
|
||||
|
||||
[[ -z $1 ]] && die_caller 42 "missing argument <user>"
|
||||
sudo -H gpasswd -a "$1" "${REDIS_USER}"
|
||||
}
|
||||
|
||||
redis.rmgrp() {
|
||||
|
||||
# usage: redis.rmgrp <user>
|
||||
|
||||
[[ -z $1 ]] && die_caller 42 "missing argument <user>"
|
||||
sudo -H gpasswd -d "$1" "${REDIS_USER}"
|
||||
|
||||
}
|
||||
|
||||
|
||||
# private redis. functions
|
||||
# ------------------------
|
||||
|
||||
redis._install_bin() {
|
||||
local src
|
||||
src="$(redis._get_dist)"
|
||||
(
|
||||
set -e
|
||||
for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do
|
||||
install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \
|
||||
"${src}/${redis_exe}" "${REDIS_HOME_BIN}"
|
||||
done
|
||||
|
||||
pushd "${REDIS_HOME_BIN}" &> /dev/null
|
||||
for redis_exe in "${REDIS_LINK_EXE[@]}"; do
|
||||
info_msg "link redis-server --> ${redis_exe}"
|
||||
sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}"
|
||||
done
|
||||
popd &> /dev/null
|
||||
|
||||
)
|
||||
}
|
||||
|
||||
redis._install_conf() {
|
||||
sudo -H -u "${REDIS_USER}" bash <<EOF
|
||||
mkdir -p "${REDIS_HOME}/run"
|
||||
echo '${REDIS_CONF_TEMPLATE}' > "${REDIS_CONF}"
|
||||
EOF
|
||||
}
|
||||
|
||||
redis._install_service() {
|
||||
systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
|
||||
}
|
||||
|
||||
redis._remove_service() {
|
||||
systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
|
||||
}
|
||||
|
||||
redis._get_dist() {
|
||||
if [ -z "${REDIS_DIST}" ]; then
|
||||
echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)"
|
||||
else
|
||||
echo "${REDIS_DIST}"
|
||||
fi
|
||||
}
|
||||
|
||||
redis._arch() {
|
||||
local ARCH
|
||||
case "$(command uname -m)" in
|
||||
"x86_64") ARCH=amd64 ;;
|
||||
"aarch64") ARCH=arm64 ;;
|
||||
"armv6" | "armv7l") ARCH=armv6l ;;
|
||||
"armv8") ARCH=arm64 ;;
|
||||
.*386.*) ARCH=386 ;;
|
||||
ppc64*) ARCH=ppc64le ;;
|
||||
*) die 42 "ARCH is unknown: $(command uname -m)" ;;
|
||||
esac
|
||||
echo "${ARCH}"
|
||||
}
|
||||
|
||||
# TODO: move this to the right place ..
|
||||
|
||||
bash.cmd(){
|
||||
|
||||
# print cmd to get a bash in a non-root mode, even if we are in a sudo
|
||||
# context.
|
||||
|
||||
local user="${USER}"
|
||||
local bash_cmd="bash"
|
||||
|
||||
if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then
|
||||
user="${SUDO_USER}"
|
||||
bash_cmd="sudo -H -u ${SUDO_USER} bash"
|
||||
fi
|
||||
|
||||
printf "%s" "${bash_cmd}"
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
[Unit]
|
||||
|
||||
Description=SearXNG redis service
|
||||
After=syslog.target
|
||||
After=network.target
|
||||
Documentation=https://redis.io/documentation
|
||||
|
||||
[Service]
|
||||
|
||||
Type=simple
|
||||
User=${REDIS_USER}
|
||||
Group=${REDIS_USER}
|
||||
WorkingDirectory=${REDIS_HOME}
|
||||
Restart=always
|
||||
TimeoutStopSec=0
|
||||
|
||||
Environment=USER=${REDIS_USER} HOME=${REDIS_HOME}
|
||||
ExecStart=${REDIS_HOME_BIN}/redis-server ${REDIS_CONF}
|
||||
ExecPaths=${REDIS_HOME_BIN}
|
||||
|
||||
LimitNOFILE=65535
|
||||
NoNewPrivileges=true
|
||||
PrivateDevices=yes
|
||||
|
||||
# ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
ReadOnlyDirectories=/
|
||||
ReadWritePaths=-${REDIS_HOME}/run
|
||||
|
||||
UMask=007
|
||||
PrivateTmp=yes
|
||||
|
||||
MemoryDenyWriteExecute=true
|
||||
ProtectKernelModules=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectControlGroups=true
|
||||
RestrictRealtime=true
|
||||
RestrictNamespaces=true
|
||||
|
||||
[Install]
|
||||
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Reference in New Issue