ZaclysIndustries/searxngRebrandZaclys
2
0
Fork 0
forked from zaclys/searxng
Code Issues Pull Requests Activity

Changes from the installation tests on (all) LXC containers.

Browse Source 
Tested and fixed HTTP & uWSGI installation on:

  ubu1604 ubu1804 ubu1910 ubu2004 fedora31 archlinux

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-04-08 18:38:36 +02:00
parent ee39a098ac
commit f693149cde
13 changed files with 263 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/utils/index.rst
View File

@ -19,20 +19,28 @@ developers.
.. _toolboxing common:
Common commands
===============
Common commands & environment
=============================
Scripts to maintain services often dispose of common commands and environments.
``shell``:
``shell`` : command
Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
troubleshooting.
``inspect service``:
``inspect service`` : command
Shows status and log of the service, most often you have a option to enable
more verbose debug logs. Very helpful for debugging, but be careful not to
enable debugging in a production environment!
``FORCE_TIMEOUT`` : environment
Sets timeout for interactive prompts. If you want to run a script in batch
job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
reverse proxy for filtron on all containers of the :ref:`searx suite
<lxc-searx.env>` use ::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
.. _toolboxing setup:
Tooling box setup

6
docs/utils/lxc.sh.rst
View File

@ -88,7 +88,11 @@ WEB-Browser::
[searx-fedora31] INFO: (eth0) filtron: http://n.n.n.18:4004/
[searx-archlinux] INFO: (eth0) filtron: http://n.n.n.12:4004/
To install a reverse proxy for filtron and morty use::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh apache install
Running commands
================

7
utils/filtron.sh
View File

@ -23,7 +23,6 @@ FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
FILTRON_ETC="/etc/filtron"
FILTRON_RULES="$FILTRON_ETC/rules.json"
FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
@ -447,7 +446,8 @@ This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SIT
install_apache
fi
echo
"${REPO_ROOT}/utils/searx.sh" install uwsgi
apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
info_msg "testing public url .."
@ -465,11 +465,12 @@ This removes apache site ${APACHE_FILTRON_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
apache_remove_site "$APACHE_FILTRON_SITE"
}
rst-doc() {

61
utils/lib.sh
View File

@ -389,7 +389,7 @@ install_template() {
local chmod="${pos_args[4]-644}"
info_msg "install (eval=$do_eval): ${dst}"
[[ -n $variant ]] && info_msg "variant: ${variant}"
[[ -n $variant ]] && info_msg "variant --> ${variant}"
if [[ ! -f "${template_origin}" ]] ; then
err_msg "${template_origin} does not exists"
@ -777,6 +777,7 @@ apache_dissable_site() {
ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
;;
esac
apache_reload
}
# uWSGI
@ -846,7 +847,7 @@ uWSGI_restart() {
if uWSGI_app_available "${CONF}"; then
systemctl restart "uwsgi@${CONF%.*}"
else
info_msg "in systemd template mode: ${CONF} not installed (nothing to restart)"
info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
fi
;;
fedora-*)
@ -854,7 +855,7 @@ uWSGI_restart() {
if uWSGI_app_enabled "${CONF}"; then
touch "${uWSGI_APPS_ENABLED}/${CONF}"
else
info_msg "in uWSGI emperor mode: ${CONF} not installed (nothing to restart)"
info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
fi
;;
*)
@ -864,6 +865,32 @@ uWSGI_restart() {
esac
}
uWSGI_prepare_app() {
# usage: uWSGI_prepare_app <myapp.ini>
local APP="${1%.*}"
if [[ -z $APP ]]; then
err_msg "uWSGI_prepare_app: missing arguments"
return 42
fi
case $DIST_ID-$DIST_VERS in
fedora-*)
# in emperor mode, the uwsgi user is the owner of the sockets
info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"
mkdir -p "/run/uwsgi/app/${APP}"
chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"
;;
*)
info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"
mkdir -p "/run/uwsgi/app/${APP}"
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"
;;
esac
}
uWSGI_app_available() {
# usage: uWSGI_app_available <myapp.ini>
local CONF="$1"
@ -888,6 +915,7 @@ uWSGI_install_app() {
*) pos_args+=("$i");;
esac
done
uWSGI_prepare_app "${pos_args[1]}"
mkdir -p "${uWSGI_APPS_AVAILABLE}"
install_template "${template_opts[@]}" \
"${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
@ -1281,3 +1309,30 @@ global_IPs(){
ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
}
primary_ip() {
case $DIST_ID in
arch)
echo "$(ip -o addr show \
| sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
| head -n 1)"
;;
*) echo "$(hostname -I | cut -d' ' -f1)" ;;
esac
}
# URL
# ---
url_replace_hostname(){
# usage: url_replace_hostname <url> <new hostname>
# to replace hostname by primary IP::
#
# url_replace_hostname http://searx-ubu1604/morty $(primary_ip)
# http://10.246.86.250/morty
echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
}

1
utils/lxc.sh
View File

@ -438,7 +438,6 @@ lxc_cmd() {
else
info_msg "lxc $* $i"
lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
echo
fi
done
}

16
utils/morty.sh
View File

@ -14,12 +14,16 @@ in_container && lxc_set_suite_env
# config
# ----------------------------------------------------------------------------
PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
PUBLIC_URL_MORTY="$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
PUBLIC_URL_MORTY="$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
if in_container; then
# container hostnames do not have a DNS entry, use primary IP
PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"
fi
# shellcheck disable=SC2034
MORTY_TIMEOUT=5
@ -425,7 +429,7 @@ This removes apache site ${APACHE_MORTY_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi

10
utils/searx.sh
View File

@ -748,6 +748,10 @@ excessively bot queries."
apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
rst_title "Install searx's uWSGI app (searx.ini)" section
echo
uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
@ -762,11 +766,15 @@ This removes apache site ${APACHE_SEARX_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
apache_remove_site "${APACHE_SEARX_SITE}"
rst_title "Remove searx's uWSGI app (searx.ini)" section
echo
uWSGI_remove_app "$SEARX_UWSGI_APP"
}
rst-doc() {

2
utils/templates/etc/httpd/sites-available/morty.conf
View File

@ -2,7 +2,7 @@
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog

2
utils/templates/etc/httpd/sites-available/searx.conf:filtron
View File

@ -2,7 +2,7 @@
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog

4
utils/templates/etc/uwsgi/apps-archlinux/searx.ini
View File

@ -74,7 +74,7 @@ http = ${SEARX_INTERNAL_HTTP}
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx/socket
# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
# socket = /run/uwsgi/app/searx/socket

80
utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket Normal file
View File

@ -0,0 +1,80 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
logger = systemd
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
socket = /run/uwsgi/app/searx/socket

4
utils/templates/etc/uwsgi/apps-available/searx.ini
View File

@ -73,7 +73,7 @@ http = ${SEARX_INTERNAL_HTTP}
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx/socket
# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket
# mkdir -p /run/uwsgi/app/searx
# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
# socket = /run/uwsgi/app/searx/socket

79
utils/templates/etc/uwsgi/apps-available/searx.ini:socket Normal file
View File

@ -0,0 +1,79 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python3,http
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
socket = /run/uwsgi/app/searx/socket