From 924bf6551794e5962688c1e2962dc0c94869e2f1 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 29 Jan 2020 20:00:50 +0100 Subject: [PATCH] utils/searx.sh & filtron.sh: misc changes from first tests (WIP) Signed-off-by: Markus Heiser --- .gitignore | 1 + utils/dot_config | 8 + utils/filtron.sh | 157 +++++++++++++----- utils/lib.sh | 103 ++++++++++-- utils/searx.sh | 116 ++++++++----- .../sites-available/searx.conf:filtron | 3 + 6 files changed, 286 insertions(+), 102 deletions(-) create mode 100644 utils/dot_config create mode 100644 utils/templates/etc/apache2/sites-available/searx.conf:filtron diff --git a/.gitignore b/.gitignore index 3c998afae..42b76ca9e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ dist/ local/ gh-pages/ searx.egg-info/ +.config diff --git a/utils/dot_config b/utils/dot_config new file mode 100644 index 000000000..e4c283744 --- /dev/null +++ b/utils/dot_config @@ -0,0 +1,8 @@ +# -*- coding: utf-8; mode: sh -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# Set environment used by ./utils scripts like filtron.sh or searx.sh +# + +# Public URL of the searx instance +PUBLIC_URL="${PUBLIC_URL:-https://$(uname -n)/searx}" diff --git a/utils/filtron.sh b/utils/filtron.sh index 7defe1ec5..6ea001d42 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -5,6 +5,7 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config # ---------------------------------------------------------------------------- # config @@ -30,7 +31,9 @@ GO_ENV="${SERVICE_HOME}/.go_env" GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" GO_TAR=$(basename "$GO_PKG_URL") -APACHE_SITE="searx.conf" +# Apache Settings + +APACHE_FILTRON_SITE="searx.conf" # shellcheck disable=SC2034 CONFIG_FILES=( @@ -53,22 +56,32 @@ usage: $(basename "$0") remove [all] $(basename "$0") activate [service] $(basename "$0") deactivate [service] - $(basename "$0") show [service] + $(basename "$0") inspect [service] + $(basename "$0") apache [install|remove] + shell start interactive shell from user ${SERVICE_USER} -install / remove all - complete setup of filtron service +install / remove + all: complete setup of filtron service + user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME update filtron Update filtron installation of user ${SERVICE_USER} -activate +activate service activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) -install user - add service user '$SERVICE_USER' at $SERVICE_HOME -show service +inspect service show service status and log +apache + install: apache site with a reverse proxy (ProxyPass) + remove: apache site ${APACHE_FILTRON_SITE} + +If needed change the environment variable PUBLIC_URL of your WEB service in the +${DOT_CONFIG#"$REPO_ROOT/"} file: + + PUBLIC_URL : ${PUBLIC_URL} + EOF [ ! -z ${1+x} ] && echo -e "$1" } @@ -76,6 +89,10 @@ EOF main() { rst_title "$SERVICE_NAME" part + required_commands \ + dpkg apt-get install git wget curl \ + || exit + local _usage="ERROR: unknown or missing $1 command $2" case $1 in @@ -86,11 +103,11 @@ main() { sudo_or_exit interactive_shell ;; - show) + inspect) case $2 in service) sudo_or_exit - show_service + inspect_service ;; *) usage "$_usage"; exit 42;; esac ;; @@ -126,6 +143,14 @@ main() { service) deactivate_service ;; *) usage "$_usage"; exit 42;; esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + *) usage "ERROR: unknown or missing command $1"; exit 42;; esac } @@ -140,14 +165,29 @@ install_all() { wait_key install_service wait_key - if apache_is_installed; then - install_apache_site - wait_key + echo + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + } remove_all() { rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + remove_service wait_key remove_user @@ -155,18 +195,6 @@ remove_all() { wait_key } -filtron_is_available() { - curl --insecure "http://${FILTRON_LISTEN}" &>/dev/null -} - -api_is_available() { - curl --insecure "http://${FILTRON_API}" &>/dev/null -} - -target_is_available() { - curl --insecure "http://${FILTRON_TARGET}" &>/dev/null -} - install_service() { rst_title "Install System-D Unit ${SERVICE_NAME}.service" section echo @@ -191,7 +219,7 @@ systemctl enable $SERVICE_NAME.service systemctl restart $SERVICE_NAME.service EOF tee_stderr <&1 -systemctl status $SERVICE_NAME.service +systemctl status --no-pager $SERVICE_NAME.service EOF } @@ -265,9 +293,8 @@ mkdir -p \$HOME/local rm -rf \$HOME/local/go tar -C \$HOME/local -xzf ${CACHE}/${GO_TAR} EOF - echo sudo -i -u "$SERVICE_USER" </dev/null && echo "Go Installation not found in PATH!?!" +! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!" which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" EOF } @@ -293,9 +320,20 @@ go get -v -u github.com/asciimoo/filtron EOF } -show_service() { +inspect_service() { + rst_title "service status & log" - echo + + cat </dev/null; then + err_msg "missing command $1" + exit_val=42 + fi + shift + done + return $exit_val +} + rst_title() { # usage: rst_title [part|chapter|section] @@ -81,7 +112,7 @@ info_msg() { echo -e "INFO: $*"; } clean_stdin() { if [[ $(uname -s) != 'Darwin' ]]; then - while read -n1 -t 0.1; do : ; done + while read -r -n1 -t 0.1; do : ; done fi } @@ -93,7 +124,7 @@ wait_key(){ [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT [[ ! -z $_t ]] && _t="-t $_t" # shellcheck disable=SC2086 - read -s -n1 $_t -p "** press any [KEY] to continue **" + read -r -s -n1 $_t -p "** press any [KEY] to continue **" echo clean_stdin } @@ -124,7 +155,7 @@ ask_yn() { clean_stdin printf "$1 ${choice} " # shellcheck disable=SC2086 - read -n1 $_t + read -r -n1 $_t if [[ -z $REPLY ]]; then printf "$default\n"; break elif [[ $REPLY =~ ^[Yy]$ ]]; then @@ -156,7 +187,7 @@ tee_stderr () { local _t="0"; if [[ ! -z $1 ]] ; then _t="$1"; fi - (while read line; do + (while read -r line; do # shellcheck disable=SC2086 sleep $_t echo -e "$line" >&2 @@ -171,6 +202,7 @@ prefix_stdout () { if [[ ! -z $1 ]] ; then prefix="$1"; fi + # shellcheck disable=SC2162 (while IFS= read line; do echo -e "${prefix}$line" done) @@ -214,7 +246,7 @@ cache_download() { else wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? fi - if $exit_value; then + if [[ $exit_value = 0 ]]; then err_msg "failed to download: $1" fi fi @@ -238,7 +270,7 @@ choose_one() { list=("$@") echo -e "Menu::" - for ((i=1; i<= $(($max -1)); i++)); do + for ((i=1; i<= $((max -1)); i++)); do if [[ "$i" == "$default" ]]; then echo -e " $i.) ${list[$i]} [default]" else @@ -249,15 +281,15 @@ choose_one() { clean_stdin printf "$1 [$default] " - if (( 10 > $max )); then + if (( 10 > max )); then # shellcheck disable=SC2086 - read -n1 $_t + read -r -n1 $_t else # shellcheck disable=SC2086,SC2229 - read $_t + read -r $_t fi # selection fits - [[ $REPLY =~ ^-?[0-9]+$ ]] && (( $REPLY > 0 )) && (( $REPLY < $max )) && break + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break # take default [[ -z $REPLY ]] && REPLY=$default && break @@ -386,6 +418,28 @@ install_template() { done } + +service_is_available() { + + # usage: service_is_available + + local URL="$1" + if [[ -z $URL ]]; then + err_msg "service_is_available: missing arguments" + return 42 + fi + + http_code=$(curl -H 'Cache-Control: no-cache' \ + --silent -o /dev/null --head --write-out '%{http_code}' --insecure \ + "${URL}") + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "got $http_code from ${URL}" + fi + return $exit_val +} + + # Apache # ------ @@ -430,18 +484,32 @@ apache_install_site() { root root 644 apache_enable_site "${pos_args[1]}" - apache_reload info_msg "installed apache site: ${pos_args[1]}" } +apache_remove_site() { + + # usage: apache_remove_site + + info_msg "remove apache site: $1" + apache_dissable_site "$1" + rm -f "${APACHE_SITES_AVAILABE}/$1" +} + apache_enable_site() { - info_msg "enable apache site $1 .." + + # usage: apache_enable_site + + info_msg "enable apache site: $1" sudo -H a2ensite -q "$1" apache_reload } apache_dissable_site() { - info_msg "disable apache site $1 .." + + # usage: apache_disable_site + + info_msg "disable apache site: $1" sudo -H a2dissite -q "$1" apache_reload } @@ -456,7 +524,7 @@ uWSGI_restart() { # usage: uWSGI_restart() info_msg "restart uWSGI service" - sudo -H systemctl restart uwsgi + systemctl restart uwsgi } uWSGI_app_available() { @@ -498,10 +566,10 @@ uWSGI_remove_app() { # usage: uWSGI_remove_app local CONF="$1" + info_msg "remove uWSGI app: ${CONF}" uWSGI_disable_app "${CONF}" uWSGI_restart rm -f "${uWSGI_SETUP}/apps-available/${CONF}" - info_msg "removed uWSGI app: ${CONF}" } uWSGI_app_enabled() { @@ -542,6 +610,9 @@ uWSGI_disable_app() { return 42 fi rm -f "${uWSGI_SETUP}/apps-enabled/${CONF}" + # FIXME: restart uwsgi service won't stop wsgi forked processes of user searx. + # I had to kill them manually here ... + pkill -f "${uWSGI_SETUP}/apps-enabled/${CONF}" -9 info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)" } diff --git a/utils/searx.sh b/utils/searx.sh index 3092d2fa9..a352c0439 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck disable=SC2119 +# shellcheck disable=SC2001 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -SEARX_PUBLIC_URL="${SEARX_PUBLIC_URL:-https://$(uname -n)/searx}" -SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "$SEARX_PUBLIC_URL" \ -| sed -e 's,^.*://[^/]*\(/.*\),\1,g') }" -SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$SEARX_PUBLIC_URL" \ +SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/ +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ | sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" SERVICE_USER="searx" @@ -42,10 +43,10 @@ SEARX_APT_PACKAGES="\ # Apache Settings APACHE_APT_PACKAGES="\ - apache2 libapache2-mod-uwsgi \ + libapache2-mod-uwsgi \ " -SEARX_APACHE_SITE="searx.conf" +APACHE_SEARX_SITE="searx.conf" # shellcheck disable=SC2034 CONFIG_FILES=( @@ -74,6 +75,7 @@ usage: $(basename "$0") deactivate [service] $(basename "$0") inspect [service] $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] shell start interactive shell from user ${SERVICE_USER} @@ -82,23 +84,25 @@ install / remove user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME searx-src: clone $SEARX_GIT_URL pyenv: create/remove virtualenv (python) in $SEARX_PYENV - apache: install apache site for searx-uwsgi app update searx Update searx installation of user ${SERVICE_USER} -activate +activate service activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) inspect service run some small tests and inspect service's status and log option - set one of te available options + set one of the available options +apache + install: apache site with the searx uwsgi app + remove: apache site ${APACHE_FILTRON_SITE} -Use environment SEARX_PUBLIC_URL to set public URL of your WEB-Server: +If needed change the environment variable PUBLIC_URL of your WEB service in the +${DOT_CONFIG#"$REPO_ROOT/"} file: - SEARX_PUBLIC_URL : ${SEARX_PUBLIC_URL} - SEARX_URL_PATH : ${SEARX_URL_PATH} - SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + PUBLIC_URL : ${PUBLIC_URL} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} EOF [ ! -z ${1+x} ] && echo -e "$1" @@ -107,6 +111,10 @@ EOF main() { rst_title "$SEARX_INSTANCE_NAME" part + required_commands \ + dpkg systemctl apt-get install git wget curl \ + || exit + local _usage="ERROR: unknown or missing $1 command $2" case $1 in @@ -132,7 +140,6 @@ main() { user) assert_user ;; pyenv) create_pyenv ;; searx-src) clone_searx ;; - apache) install_apache_site ;; *) usage "$_usage"; exit 42;; esac ;; update) @@ -154,13 +161,13 @@ main() { sudo_or_exit case $2 in service) - activate_service; uWSGI_restart ;; + activate_service ;; *) usage "$_usage"; exit 42;; esac ;; deactivate) sudo_or_exit case $2 in - service) deactivate_service; uWSGI_restart ;; + service) deactivate_service ;; *) usage "$_usage"; exit 42;; esac ;; option) @@ -170,6 +177,14 @@ main() { debug-off) echo; disable_debug ;; *) usage "$_usage"; exit 42;; esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + *) usage "ERROR: unknown or missing command $1"; exit 42;; esac } @@ -191,9 +206,7 @@ install_all() { test_local_searx wait_key install_searx_uwsgi - if service_is_available "http://$SEARX_INTERNAL_URL" &>/dev/null; then - info_msg "URL http://$SEARX_INTERNAL_URL is available." - else + if ! service_is_available "http://$SEARX_INTERNAL_URL"; then err_msg "URL http://$SEARX_INTERNAL_URL not available, check searx & uwsgi setup!" fi if ask_yn "Do you want to inspect the installation?" Yn; then @@ -418,12 +431,14 @@ activate_service() { rst_title "Activate $SEARX_INSTANCE_NAME (service)" section echo uWSGI_enable_app "$SEARX_UWSGI_APP" + uWSGI_restart } deactivate_service() { rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section echo uWSGI_disable_app "$SEARX_UWSGI_APP" + uWSGI_restart } interactive_shell() { @@ -438,12 +453,6 @@ git --no-pager diff EOF } -service_is_available() { - curl -H 'Cache-Control: no-cache' -o /dev/null \ - --silent --head --write-out '%{http_code}' --insecure \ - "${1?missing URL argument}" -} - enable_debug() { info_msg "try to enable debug mode ..." tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" @@ -464,7 +473,16 @@ EOF inspect_service() { rst_title "service status & log" - echo + cat </dev/null; then - info_msg "uWSGI app (service) at http://$SEARX_INTERNAL_URL is available" - else + if ! service_is_available "http://$SEARX_INTERNAL_URL"; then err_msg "uWSGI app (service) at http://$SEARX_INTERNAL_URL is not available!" fi - if service_is_available "${SEARX_PUBLIC_URL}" &>/dev/null; then - info_msg "Public service at ${SEARX_PUBLIC_URL} is available" - else - err_msg "Public service at ${SEARX_PUBLIC_URL} is not available!" + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" fi local _debug_on @@ -530,7 +544,7 @@ inspect_service() { } install_apache_site() { - rst_title "Install Apache site $SEARX_APACHE_SITE" + rst_title "Install Apache site $APACHE_SEARX_SITE" rst_para "\ This installs the searx uwsgi app as apache site. If your server ist public to @@ -547,15 +561,29 @@ excessively bot queries." a2enmod uwsgi echo - apache_install_site --variant=uwsgi "${SEARX_APACHE_SITE}" + apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" - if service_is_available "${SEARX_PUBLIC_URL}" &>/dev/null; then - info_msg "Public service at ${SEARX_PUBLIC_URL} is available" - else - err_msg "Public service at ${SEARX_PUBLIC_URL} is not available!" + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" fi } +remove_apache_site() { + + rst_title "Remove Apache site ${APACHE_SEARX_SITE}" + + rst_para "\ +This removes apache site ${APACHE_SEARX_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "${APACHE_SEARX_SITE}" +} + # ---------------------------------------------------------------------------- main "$@" # ---------------------------------------------------------------------------- diff --git a/utils/templates/etc/apache2/sites-available/searx.conf:filtron b/utils/templates/etc/apache2/sites-available/searx.conf:filtron new file mode 100644 index 000000000..d2f5431a7 --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/searx.conf:filtron @@ -0,0 +1,3 @@ +# -*- coding: utf-8; mode: apache -*- + +ProxyPass "/searx" "http://127.0.0.1:4004/"