From 414a6105e7793e9499c64dd649eda43250971c50 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Jun 2021 19:01:07 +0200 Subject: [PATCH] [mod] load .config.sh from installation (utils/lib_install.sh) **new** utils/lib_install.sh: Used to initialize installation procedures - Modified source_dot_config function that - loads .config.sh from an existing installation (at SEARX_SRC). - initialize **SEARX_SRC_INIT_FILES** - functions like: - install_log_searx_instance() - install_searx_get_state() **modified** utils/searx.sh - obsolete environment SEARX_INSTANCE_NAME has been replaced by string 'SearXNG'. **modified** utils/filtron.sh, utils/morty.sh, utils/searx.sh - source utils/lib_install.sh - normalize logging of environment variables using new function install_log_searx_instance() **modified** utils/lib.sh - fix marginal typos **Installation scripts** The utils/lib_install.sh is sourced by the installations scripts: - utils/searx.sh - utils/morty.sh - utils/filtron.sh If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function loads this configuration (instead of './.config.sh'). **SEARX_SRC_INIT_FILES** Array of file names to sync into a installation at $SEARX_SRC. The file names are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). Most often theses are files like: - .config.sh - searx/settings.yml - utils/brand.env - ... Signed-off-by: Markus Heiser --- Makefile | 1 + utils/filtron.sh | 52 +++++------- utils/lib.sh | 6 +- utils/lib_install.sh | 184 +++++++++++++++++++++++++++++++++++++++++++ utils/morty.sh | 33 ++++---- utils/searx.sh | 67 +++++----------- 6 files changed, 244 insertions(+), 99 deletions(-) create mode 100755 utils/lib_install.sh diff --git a/Makefile b/Makefile index 2b9dfacce..6c22d5e2e 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,7 @@ test.shell: utils/brand.env \ $(MTOOLS) \ utils/lib.sh \ + utils/lib_install.sh \ utils/lib_static.sh \ utils/filtron.sh \ utils/searx.sh \ diff --git a/utils/filtron.sh b/utils/filtron.sh index 5680811ea..7469cd801 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck disable=SC2119,SC2001 +# shellcheck disable=SC2001 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -94,28 +90,17 @@ apache (${PUBLIC_URL}) nginx (${PUBLIC_URL}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_FILTRON_SITE} - filtron rules: ${FILTRON_RULES_TEMPLATE} - -If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL : ${PUBLIC_URL} - PUBLIC_HOST : ${PUBLIC_HOST} - SERVICE_USER : ${SERVICE_USER} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} +---- sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + FILTRON_TARGET : ${FILTRON_TARGET} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} EOF - if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container URL (IPv6): http://[${ip#*|}]:4005/" - else - # IPv4: - echo " container URL (IPv4): http://${ip#*|}:4005/" - fi - done - fi + + install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } @@ -349,16 +334,15 @@ inspect_service() { cat < [ []] # # First form uses $CACHE/ as destination folder, second form clones - # into . If repository is allready cloned, pull from and + # into . If repository is already cloned, pull from and # update working tree (if needed, the caller has to stash local changes). # # git clone https://github.com/searxng/searxng searx-src origin/master searxlogin diff --git a/utils/lib_install.sh b/utils/lib_install.sh new file mode 100755 index 000000000..19d438c43 --- /dev/null +++ b/utils/lib_install.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 +# shellcheck source=utils/lib.sh +. /dev/null + +# Initialize installation procedures: +# +# - Modified source_dot_config function that +# - loads .config.sh from an existing installation (at SEARX_SRC). +# - initialize **SEARX_SRC_INIT_FILES** +# - functions like: +# - install_log_searx_instance() +# - install_searx_get_state() +# +# usage: +# source lib_install.sh +# +# **Installation scripts** +# +# The utils/lib_install.sh is sourced by the installations scripts: +# +# - utils/searx.sh +# - utils/morty.sh +# - utils/filtron.sh +# +# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function +# loads this configuration (instead of './.config.sh'). + +# **SEARX_SRC_INIT_FILES** +# +# Array of file names to sync into a installation at $SEARX_SRC. The file names +# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). +# Most often theses are files like: +# - .config.sh +# - searx/settings.yml +# - utils/brand.env +# - ... + + +SEARX_SRC_INIT_FILES=() + +eval orig_"$(declare -f source_dot_config)" + +source_dot_config() { + + # Modified source_dot_config function that + # - loads .config.sh from an existing installation (at SEARX_SRC). + # - initialize SEARX_SRC_INIT_FILES + + if [ -z "$eval_SEARX_SRC" ]; then + export eval_SEARX_SRC='true' + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + if [ ! -r "${SEARX_SRC}" ]; then + build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" + orig_source_dot_config + return 0 + fi + build_msg INSTANCE "using instance at: ${SEARX_SRC}" + + # set and log DOT_CONFIG + if [ -r "${SEARX_SRC}/.config.sh" ]; then + build_msg INSTANCE "switching to ${SEARX_SRC}/.config.sh" + DOT_CONFIG="${SEARX_SRC}/.config.sh" + else + build_msg INSTANCE "using local config: ${DOT_CONFIG}" + fi + init_SEARX_SRC_INIT_FILES + fi +} + +init_SEARX_SRC_INIT_FILES(){ + # init environment SEARX_SRC_INIT_FILES + + # Monitor modified files in the working-tree from the local repository, only + # if the local file differs to the corresponding file in the instance. Most + # often theses are files like: + # + # - .config.sh + # - searx/settings.yml + # - utils/brand.env + # - ... + + # keep list empty if there is no installation + SEARX_SRC_INIT_FILES=() + if [ ! -r "$SEARX_SRC" ]; then + return 0 + fi + + local fname + local msg="" + + # Monitor local modified files from the repository, only if the local file + # differs to the corresponding file in the instance + + while IFS= read -r fname; do + if [ -z "$fname" ]; then + continue + fi + if [ -r "${SEARX_SRC}/${fname}" ]; then + # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then + SEARX_SRC_INIT_FILES+=("${fname}") + build_msg INSTANCE "local clone (workingtree), modified file: ./$fname" + msg="to update use: sudo -H ./utils/searx.sh install init-src" + fi + fi + done <<< "$(git diff --name-only)" + [ -n "$msg" ] && build_msg INSTANCE "$msg" +} + +install_log_searx_instance() { + + echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" + echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" + echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" + + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "---- container setup" + echo -e " ${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi +} + +install_searx_get_state(){ + + # usage: install_searx_get_state + # + # Prompts a string indicating the status of the installation procedure + # + # missing-searx-clone: + # There is no clone at ${SEARX_SRC} + # missing-searx-pyenv: + # There is no pyenv in ${SEARX_PYENV} + # installer-modified: + # There are files modified locally in the installer (clone), + # see ${SEARX_SRC_INIT_FILES} description. + # python-installed: + # Scripts can be executed in instance's environment + # - user: ${SERVICE_USER} + # - pyenv: ${SEARX_PYENV} + + if ! [ -r "${SEARX_SRC}" ]; then + echo "missing-searx-clone" + return + fi + if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then + echo "missing-searx-pyenv" + return + fi + if ! [ -r "${SEARX_SETTINGS_PATH}" ]; then + echo "missing-settings" + return + fi + if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + echo "installer-modified" + return + fi + echo "python-installed" +} + +# Initialization of the installation procedure +# -------------------------------------------- + +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +source_dot_config + +# shellcheck source=utils/lxc-searx.env +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env diff --git a/utils/morty.sh b/utils/morty.sh index 0a44ef574..afda41d22 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -1,15 +1,10 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -86,12 +81,15 @@ apache : ${PUBLIC_URL_MORTY} nginx (${PUBLIC_URL_MORTY}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_MORTY_SITE} - -If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY} - MORTY_LISTEN: ${MORTY_LISTEN} - SERVICE_USER: ${SERVICE_USER} +---- +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} + MORTY_LISTEN: : ${MORTY_LISTEN} EOF + + install_log_searx_instance if in_container; then # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) for ip in $(global_IPs) ; do @@ -335,11 +333,14 @@ inspect_service() { cat <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" -sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH" EOF } @@ -577,14 +555,14 @@ remove_searx_uwsgi() { } activate_service() { - rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "Activate SearXNG (service)" section echo uWSGI_enable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" } deactivate_service() { - rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "De-Activate SearXNG (service)" section echo uWSGI_disable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" @@ -609,7 +587,7 @@ EOF } enable_debug() { - warn_msg "Do not enable debug in production enviroments!!" + warn_msg "Do not enable debug in production environments!!" info_msg "try to enable debug mode ..." tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -680,14 +658,11 @@ inspect_service() { rst_title "service status & log" cat <