diff --git a/utils/filtron.sh b/utils/filtron.sh deleted file mode 100755 index 2536214e4..000000000 --- a/utils/filtron.sh +++ /dev/null @@ -1,629 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck disable=SC2001 - -# shellcheck source=utils/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_go.sh -source "${REPO_ROOT}/utils/lib_go.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" - -# ---------------------------------------------------------------------------- -# config -# ---------------------------------------------------------------------------- - -PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" - -FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ -| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" -[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ - -FILTRON_ETC="/etc/filtron" -FILTRON_RULES="$FILTRON_ETC/rules.json" -FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/filtron/rules.json}" - -FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" -FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" - -# The filtron target is the SearXNG installation, listenning on server.port at -# server.bind_address. The default of FILTRON_TARGET is taken from the YAML -# configuration, do not change this value without reinstalling the entire -# SearXNG suite including filtron & morty. -FILTRON_TARGET="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" - -SERVICE_NAME="filtron" -SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" -SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" - -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" - -GO_ENV="${SERVICE_HOME}/.go_env" -GO_VERSION="go1.17.2" - -APACHE_FILTRON_SITE="searxng.conf" -NGINX_FILTRON_SITE="searxng.conf" - -# shellcheck disable=SC2034 -CONFIG_FILES=( - "${FILTRON_RULES}" - "${SERVICE_SYSTEMD_UNIT}" -) - -# ---------------------------------------------------------------------------- -usage() { -# ---------------------------------------------------------------------------- - - # shellcheck disable=SC1117 - cat < "$(go_version)" ]]; then - warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - else - info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)" - fi - - if [ -f "${APACHE_SITES_AVAILABLE}/searx.conf" ]; then - warn_msg "old searx.conf apache site exists" - fi - - if [ -f "${NGINX_APPS_AVAILABLE}/searx.conf" ]; then - warn_msg "old searx.conf nginx site exists" - fi - -} - -go_version(){ - go.version "${SERVICE_USER}" -} - -remove_all() { - rst_title "De-Install $SERVICE_NAME (service)" - - rst_para "\ -It goes without saying that this script can only be used to remove -installations that were installed with this script." - - if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then - return 42 - fi - drop_service_account "${SERVICE_USER}" - rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout - if service_is_available "${PUBLIC_URL}"; then - MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10 - fi -} - -assert_user() { - rst_title "user $SERVICE_USER" section - echo - tee_stderr 1 <> ~/.profile -EOF -} - -filtron_is_installed() { - [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] -} - -install_filtron() { - rst_title "Install filtron in user's ~/go-apps" section - echo - go.install github.com/searxng/filtron@latest "${SERVICE_USER}" -} - -update_filtron() { - rst_title "Update filtron" section - echo - go.install github.com/searxng/filtron@latest "${SERVICE_USER}" -} - -install_rules() { - rst_title "Install filtron rules" - echo - if [[ ! -f "${FILTRON_RULES}" ]]; then - info_msg "install rules ${FILTRON_RULES_TEMPLATE}" - info_msg " --> ${FILTRON_RULES}" - mkdir -p "$(dirname "${FILTRON_RULES}")" - cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - return - fi - - if cmp --silent "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"; then - info_msg "${FILTRON_RULES} is up to date with" - info_msg "${FILTRON_RULES_TEMPLATE}" - return - fi - - rst_para "Diff between origin's rules file (+) and current (-):" - echo "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}" - $DIFF_CMD "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}" - - local action - choose_one action "What should happen to the rules file? " \ - "keep configuration unchanged" \ - "use origin rules" \ - "start interactive shell" - case $action in - "keep configuration unchanged") - info_msg "leave rules file unchanged" - ;; - "use origin rules") - backup_file "${FILTRON_RULES}" - info_msg "install origin rules" - cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - ;; - "start interactive shell") - backup_file "${FILTRON_RULES}" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -i - rst_para 'Diff between new rules file (-) and current (+):' - echo - $DIFF_CMD "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - wait_key - ;; - esac -} - -inspect_service() { - - rst_title "service status & log" - - cat < ${PUBLIC_URL}" - info_msg "internal URL --> http://${FILTRON_LISTEN}" - fi - - - local _debug_on - if ask_yn "Enable filtron debug mode?"; then - enable_debug - _debug_on=1 - fi - echo - systemctl --no-pager -l status "${SERVICE_NAME}" - echo - - info_msg "public URL --> ${PUBLIC_URL}" - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - while true; do - trap break 2 - journalctl -f -u "${SERVICE_NAME}" - done - - if [[ $_debug_on == 1 ]]; then - disable_debug - fi - return 0 -} - - -enable_debug() { - info_msg "try to enable debug mode ..." - python < - - # to replace hostname by primary IP:: - # - # url_replace_hostname http://searx-ubu1604/morty $(primary_ip) - # http://10.246.86.250/morty - - # shellcheck disable=SC2001 - echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|" -} diff --git a/utils/lib_go.sh b/utils/lib_go.sh deleted file mode 100755 index 314204e1a..000000000 --- a/utils/lib_go.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- -# SPDX-License-Identifier: AGPL-3.0-or-later -# -# Tools to install and maintain golang [1] binaries & packages. -# -# [1] https://golang.org/doc/devel/release#policy -# -# A simple *helloworld* test with user 'my_user' : -# -# sudo -H adduser my_user -# ./manage go.golang go1.17.3 my_user -# ./manage go.install github.com/go-training/helloworld@latest my_user -# ./manage go.bash my_user -# $ helloword -# Hello World!! -# -# Don't forget to remove 'my_user': sudo -H deluser --remove-home my_user - -# shellcheck source=utils/lib.sh -. /dev/null - -# shellcheck disable=SC2034 -declare main_cmd - -# configure golang environment -# ---------------------------- - -[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3" - -GO_DL_URL="https://golang.org/dl" - -# implement go functions -# ----------------------- - -go.help(){ - cat < [filename|sha256|size] - # - # kind: [archive|source|installer] - # os: [darwin|freebsd|linux|windows] - # arch: [amd64|arm64|386|armv6l|ppc64le|s390x] - - python3 - "$@" < [] - - local version fname sha size user userpr - local buf=() - - version="${1:-${GO_VERSION}}" - user="${2:-${USERNAME}}" - userpr=" ${_Yellow}|${user}|${_creset} " - - rst_title "Install Go in ${user}'s HOME" section - - mapfile -t buf < <( - go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size - ) - - if [ ${#buf[@]} -eq 0 ]; then - die 42 "can't find info of golang version: ${version}" - fi - fname="${buf[0]}" - sha="${buf[1]}" - size="$(numfmt --to=iec "${buf[2]}")" - - info_msg "Download go binary ${fname} (${size}B)" - cache_download "${GO_DL_URL}/${fname}" "${fname}" - - pushd "${CACHE}" &> /dev/null - echo "${sha} ${fname}" > "${fname}.sha256" - if ! sha256sum -c "${fname}.sha256" >/dev/null; then - die 42 "downloaded file ${fname} checksum does not match" - else - info_msg "${fname} checksum OK" - fi - popd &> /dev/null - - info_msg "install golang" - tee_stderr 0.1 < \$HOME/.go_env -echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env -EOF - info_msg "test golang installation" - sudo -i -u "${user}" < [] - - local package user userpr - - package="${1}" - user="${2:-${USERNAME}}" - userpr=" ${_Yellow}|${user}|${_creset} " - - if [ -z "${package}" ]; then - die 42 "${FUNCNAME[0]}() - missing argument: " - fi - tee_stderr 0.1 <] - - local user - user="${1:-${USERNAME}}" - sudo -i -u "${user}" bash --init-file "~${user}/.go_env" -} - -go.version(){ - local user - user="${1:-${USERNAME}}" - sudo -i -u "${user}" < "$(go_version)" ]]; then - warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - else - info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)" - fi -} - -go_version(){ - go.version "${SERVICE_USER}" -} - -remove_all() { - rst_title "De-Install $SERVICE_NAME (service)" - - rst_para "\ -It goes without saying that this script can only be used to remove -installations that were installed with this script." - - if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then - drop_service_account "${SERVICE_USER}" - fi -} - -assert_user() { - rst_title "user $SERVICE_USER" section - echo - tee_stderr 1 <> ~/.profile -EOF -} - -morty_is_installed() { - [[ -f $SERVICE_HOME/go-apps/bin/morty ]] -} - -install_morty() { - rst_title "Install morty in user's ~/go-apps" section - echo - go.install github.com/asciimoo/morty@latest "${SERVICE_USER}" -} - -update_morty() { - rst_title "Update morty" section - echo - go.install github.com/asciimoo/morty@latest "${SERVICE_USER}" -} - -set_service_env_debug() { - - # usage: set_service_env_debug [false|true] - - # shellcheck disable=SC2034 - local SERVICE_ENV_DEBUG="${1:-false}" - if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - fi -} - -inspect_service() { - - rst_title "service status & log" - - cat < ${PUBLIC_URL_MORTY}" - info_msg "morty URL --> http://${MORTY_LISTEN}" - fi - - local _debug_on - if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then - enable_debug - _debug_on=1 - else - systemctl --no-pager -l status "${SERVICE_NAME}" - fi - echo - - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - while true; do - trap break 2 - journalctl -f -u "${SERVICE_NAME}" - done - - if [[ $_debug_on == 1 ]]; then - FORCE_SELECTION=Y disable_debug - fi - return 0 -} - -enable_debug() { - warn_msg "Do not enable debug in production environments!!" - info_msg "Enabling debug option needs to reinstall systemd service!" - set_service_env_debug true -} - -disable_debug() { - info_msg "Disabling debug option needs to reinstall systemd service!" - set_service_env_debug false -} - - -set_new_key() { - rst_title "Set morty key" - echo - - MORTY_KEY="$(head -c 32 /dev/urandom | base64)" - info_msg "morty key: '${MORTY_KEY}'" - - warn_msg "this will need to reinstall services .." - MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key - - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" - "${REPO_ROOT}/utils/searx.sh" option image-proxy-on -} - - -install_apache_site() { - - rst_title "Install Apache site $APACHE_MORTY_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - apache_install_site "${APACHE_MORTY_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - fi -} - -remove_apache_site() { - - rst_title "Remove Apache site $APACHE_MORTY_SITE" - - rst_para "\ -This removes apache site ${APACHE_MORTY_SITE}." - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - fi - - apache_remove_site "$APACHE_MORTY_SITE" -} - -install_nginx_site() { - - rst_title "Install nginx site $NGINX_MORTY_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})" - - ! nginx_is_installed && err_msg "nginx is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_nginx - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - # shellcheck disable=SC2034 - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - # shellcheck disable=SC2034 - SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH) - nginx_install_app "${NGINX_MORTY_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - fi -} - -remove_nginx_site() { - - rst_title "Remove nginx site $NGINX_MORTY_SITE" - - rst_para "\ -This removes nginx site ${NGINX_MORTY_SITE}." - - ! nginx_is_installed && err_msg "nginx is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - fi - - nginx_remove_site "$NGINX_MORTY_SITE" - -} - -rst-doc() { - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\"" - - echo -e "\n.. START install systemd unit" - cat < - - - SecRuleEngine Off - - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://${MORTY_LISTEN} - RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} - - diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:filtron b/utils/templates/etc/httpd/sites-available/searxng.conf:filtron deleted file mode 100644 index 379d47e24..000000000 --- a/utils/templates/etc/httpd/sites-available/searxng.conf:filtron +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so -#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog -# CustomLog /dev/null combined env=dontlog - -# SecRuleRemoveById 981054 -# SecRuleRemoveById 981059 -# SecRuleRemoveById 981060 -# SecRuleRemoveById 950907 - - - - - SecRuleEngine Off - - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://${FILTRON_LISTEN} - RequestHeader set X-Script-Name ${FILTRON_URL_PATH} - - diff --git a/utils/templates/etc/nginx/default.apps-available/morty.conf b/utils/templates/etc/nginx/default.apps-available/morty.conf deleted file mode 100644 index 51f083985..000000000 --- a/utils/templates/etc/nginx/default.apps-available/morty.conf +++ /dev/null @@ -1,11 +0,0 @@ -# https://example.org/morty - -location /morty { - proxy_pass http://127.0.0.1:3000/; - - proxy_set_header Host \$host; - proxy_set_header Connection \$http_connection; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Scheme \$scheme; -} diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron b/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron deleted file mode 100644 index e25461c47..000000000 --- a/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron +++ /dev/null @@ -1,16 +0,0 @@ -# https://example.org/searx - -location ${SEARXNG_URL_PATH} { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host \$host; - proxy_set_header Connection \$http_connection; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Scheme \$scheme; - proxy_set_header X-Script-Name ${SEARXNG_URL_PATH}; -} - -location ${SEARXNG_URL_PATH}/static/ { - alias ${SEARX_SRC}/searx/static/; -} diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service deleted file mode 100644 index 3b0c6edcc..000000000 --- a/utils/templates/lib/systemd/system/filtron.service +++ /dev/null @@ -1,29 +0,0 @@ -[Unit] - -Description=${SERVICE_NAME} -After=syslog.target -After=network.target - -[Service] - -Type=simple -User=${SERVICE_USER} -Group=${SERVICE_GROUP} -WorkingDirectory=${SERVICE_HOME} -ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' - -Restart=always -Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} - -# Some distributions may not support these hardening directives. If you cannot -# start the service due to an unknown option, comment out the ones not supported -# by your version of systemd. - -ProtectSystem=full -PrivateDevices=yes -PrivateTmp=yes -NoNewPrivileges=true - -[Install] - -WantedBy=multi-user.target diff --git a/utils/templates/lib/systemd/system/morty.service b/utils/templates/lib/systemd/system/morty.service deleted file mode 100644 index 25b676b51..000000000 --- a/utils/templates/lib/systemd/system/morty.service +++ /dev/null @@ -1,29 +0,0 @@ -[Unit] - -Description=${SERVICE_NAME} -After=syslog.target -After=network.target - -[Service] - -Type=simple -User=${SERVICE_USER} -Group=${SERVICE_GROUP} -WorkingDirectory=${SERVICE_HOME} -ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '${MORTY_KEY}' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT} - -Restart=always -Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG} - -# Some distributions may not support these hardening directives. If you cannot -# start the service due to an unknown option, comment out the ones not supported -# by your version of systemd. - -ProtectSystem=full -PrivateDevices=yes -PrivateTmp=yes -NoNewPrivileges=true - -[Install] - -WantedBy=multi-user.target