From 89df9d91412c2ebb8817e72c0ca9c8cfc5852f66 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 16 Jan 2020 14:01:38 +0100 Subject: [PATCH] utils/searx.sh: add script to install isolated searx service (WIP) Signed-off-by: Markus Heiser --- utils/filtron.sh | 2 +- utils/lib.sh | 61 +++++----- utils/searx.sh | 112 ++++++++++++------ .../etc/uwsgi/apps-available/searx.ini | 62 ++++++++++ 4 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 utils/templates/etc/uwsgi/apps-available/searx.ini diff --git a/utils/filtron.sh b/utils/filtron.sh index 50cce6e65..bf6016b0f 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh -*- +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # shellcheck disable=SC2119 # shellcheck source=utils/lib.sh diff --git a/utils/lib.sh b/utils/lib.sh index f43a117cb..827862b7c 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh -*- +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # shellcheck disable=SC2059,SC1117,SC2162,SC2004 ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" @@ -265,9 +265,9 @@ choose_one() { _t="" err_msg "invalid choice" done + eval "$env_name"='${list[${REPLY}]}' echo clean_stdin - eval "$env_name"='${list[${REPLY}]}' } install_template() { @@ -375,34 +375,32 @@ uWSGI_restart() { uWSGI_install_app() { - # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ... + # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini - local do_eval="" - local CONF + local no_eval="" + local CONF="" if [[ "$1" == "--no-eval" ]]; then no_eval=$1; shift fi - for CONF in "$@"; do - install_template "$no_eval" "${CONF}" root root 644 - uWSGI_enable_app "$(basename "${CONF}")" - info_msg "enabled uWSGI app: $(basename "${CONF}")" - done + CONF=$1 + # shellcheck disable=SC2086 + install_template $no_eval "${CONF}" root root 644 + uWSGI_enable_app "$(basename "${CONF}")" uWSGI_restart + info_msg "installed uWSGI app: $(basename "${CONF}")" } uWSGI_remove_app() { - # usage: uWSGI_remove_app ... + # usage: uWSGI_remove_app - local CONF - for CONF in "$@"; do - uWSGI_disable_app "$(basename "${CONF}")" - rm -f "$CONF" - info_msg "removed uWSGI app: $(basename "${CONF}")" - done + local CONF=$1 + uWSGI_disable_app "$(basename "${CONF}")" uWSGI_restart + rm -f "$CONF" + info_msg "removed uWSGI app: $(basename "${CONF}")" } # shellcheck disable=SC2164 @@ -416,6 +414,7 @@ uWSGI_enable_app() { return 42 fi pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + rm -f "$(basename "${CONF}")" # shellcheck disable=SC2226 ln -s "../apps-available/$(basename "${CONF}")" info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" @@ -454,7 +453,6 @@ pkg_install() { fi # shellcheck disable=SC2068 apt-get install -y $@ - wait_key 30 } pkg_remove() { @@ -468,7 +466,6 @@ pkg_remove() { return 42 fi apt-get purge --autoremove --ignore-missing -y "$@" - wait_key 30 } pkg_is_installed() { @@ -491,7 +488,7 @@ git_clone() { # git_clone [ []] # # First form uses $CACHE/ as destination folder, second form clones - # into . If repository is allready cloned, merge from origin and + # into . If repository is allready cloned, pull from and # update working tree (if needed, the caller has to stash local changes). # # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin @@ -501,7 +498,8 @@ git_clone() { local dest="$2" local branch="$3" local user="$4" - local prefix="" + local bash_cmd="bash" + local remote="origin" if [[ ! "${dest:0:1}" = "/" ]]; then dest="$CACHE/$dest" @@ -509,20 +507,21 @@ git_clone() { [[ -z $branch ]] && branch=master [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" - [[ -z $user ]] && prefix="sudo -H -u $user" + [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i" if [[ -d "${dest}" ]] ; then info_msg "already cloned: $dest" - pushd "${dest}" > /dev/null - $prefix git checkout -b "$(basename "$branch")" --track "$branch" - $prefix git pull --all - popd > /dev/null - + tee_stderr 0.1 <&1 | prefix_stdout " |$user| " +cd "${dest}" +git checkout -m -B "$branch" --track "$remote/$branch" +git pull --all +EOF else info_msg "clone into: $dest" - $prefix mkdir -p "$(dirname "$dest")" - pushd "${dest}" > /dev/null - git clone "$url" "$(basename "$dest")" - popd > /dev/null + tee_stderr 0.1 <&1 | prefix_stdout " |$user| " +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" +EOF fi } diff --git a/utils/searx.sh b/utils/searx.sh index 0d47820b0..6955f82a2 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh -*- +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # shellcheck disable=SC2119 # shellcheck source=utils/lib.sh @@ -15,8 +15,11 @@ SERVICE_USER="${SERVICE_NAME}" SERVICE_GROUP="${SERVICE_USER}" SERVICE_HOME="/home/${SERVICE_USER}" +# shellcheck disable=SC2034 +SEARX_URL="127.0.0.1:8888" + SEARX_GIT_URL="https://github.com/asciimoo/searx.git" -SEARX_GIT_BRANCH="origin/master" +SEARX_GIT_BRANCH="master" # FIXME: Arch Linux & RHEL should be added @@ -25,8 +28,8 @@ libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \ git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \ libffi-dev libssl-dev" -SEARX_VENV="${SEARX_HOME}/searx-venv" -SEARX_SRC="${SEARX_HOME}/searx-src" +SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARX_SRC="${SERVICE_HOME}/searx-src" SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" @@ -51,25 +54,26 @@ usage(){ usage: $(basename "$0") shell - $(basename "$0") install [all|user] + $(basename "$0") install [all|user|pyenv|searx-src] $(basename "$0") update [searx] - $(basename "$0") remove [all] + $(basename "$0") remove [all|user|pyenv|searx-src] $(basename "$0") activate [service] $(basename "$0") deactivate [service] $(basename "$0") show [service] shell start interactive shell from user ${SERVICE_USER} -install / remove all - complete setup of searx service +install / remove + all: complete (de-) installation of searx service + user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME + searx-src: clone $SEARX_GIT_URL + pyenv: create/remove virtualenv (python) in $SEARX_PYENV update searx Update searx installation of user ${SERVICE_USER} activate activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) -install user - add service user '$SERVICE_USER' at $SERVICE_HOME show service show service status and log EOF @@ -102,6 +106,8 @@ main(){ case $2 in all) install_all ;; user) assert_user ;; + pyenv) create_pyenv ;; + searx-src) clone_searx ;; *) usage "$_usage"; exit 42;; esac ;; update) @@ -115,6 +121,8 @@ main(){ case $2 in all) remove_all;; user) remove_user ;; + pyenv) remove_pyenv ;; + searx-src) remove_searx ;; *) usage "$_usage"; exit 42;; esac ;; activate) @@ -143,7 +151,7 @@ install_all() { wait_key clone_searx wait_key - create_venv + create_pyenv wait_key configure_searx wait_key @@ -167,7 +175,7 @@ update_searx() { cd ${SEARX_SRC} cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup git stash push -m "BACKUP -- 'update server' at ($(date))" -git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH" +git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH" git pull "$SEARX_GIT_BRANCH" ${SEARX_SRC}/manage.sh update_packages EOF @@ -185,16 +193,16 @@ EOF "start interactiv shell" case $action in "keep new configuration") - info_msg "continue using new settings file" - ;; + info_msg "continue using new settings file" + ;; "revert to the old configuration (backup file)") tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} EOF - ;; - "start interactiv shell") - interactive_shell - ;; + ;; + "start interactiv shell") + interactive_shell + ;; esac chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" @@ -208,7 +216,10 @@ EOF remove_all() { rst_title "De-Install $SERVICE_NAME (service)" - remove_service + if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then + return + fi + remove_searx_uwsgi wait_key remove_user } @@ -240,35 +251,71 @@ remove_user() { clone_searx(){ rst_title "Clone searx sources" section echo + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + return 42 + fi + export SERVICE_HOME + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ - "$SEARX_GIT_BRANCH" "$SERVICE_USER" + "$SEARX_GIT_BRANCH" "$SERVICE_USER" pushd "${SEARX_SRC}" > /dev/null tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd "${SEARX_SRC}" git config user.email "$ADMIN_EMAIL" git config user.name "$ADMIN_NAME" -git checkout "$SEARX_GIT_BRANCH" +git config --list EOF popd > /dev/null } -create_venv(){ - rst_title "Create virtualenv (python)" section +remove_searx() { + rst_title "Drop searx sources" section + if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + rm -rf "$SEARX_SRC" + else + rst_para "Leave searx sources unchanged." + fi +} - rst_para "Create venv in ${SEARX_VENV} and install needed python packages." +create_pyenv(){ + rst_title "Create virtualenv (python)" section echo + if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + err_msg "to create pyenv for searx, searx has to be cloned first" + return 42 + fi + info_msg "create pyenv in ${SEARX_PYENV}" + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_PYENV}" +python3 -m venv "${SEARX_PYENV}" +grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" -rm -rf "${SEARX_VENV}" -python3 -m venv "${SEARX_VENV}" -. ${SEARX_VENV}/bin/activate ${SEARX_SRC}/manage.sh update_packages EOF - tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" -grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \ - || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile -EOF +} +remove_pyenv(){ + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARX_PYENV}" } configure_searx(){ @@ -283,7 +330,7 @@ EOF } test_local_searx(){ - rstHeading "Testing searx instance localy" section + rst_title "Testing searx instance localy" section echo tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -293,7 +340,6 @@ sleep 1 curl --location --verbose --head --insecure http://127.0.0.1:8888/ sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" EOF - waitKEY } install_searx_uwsgi() { diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini new file mode 100644 index 000000000..138a57384 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -0,0 +1,62 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3 + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SERVICE_HOME} + + +# plugin http +# ----------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http + +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html +http = ${SEARX_URL}