mirror of
https://github.com/searxng/searxng
synced 2024-01-01 18:24:07 +00:00
utils/filtron.sh: various fix from first installation test (WIP)
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
parent
f20193155a
commit
4990b07b4b
@ -11,6 +11,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
||||
|
||||
FILTRON_ETC="/etc/filtron"
|
||||
|
||||
FILTRON_RULES="$FILTRON_ETC/rules.json"
|
||||
FILTRON_API="127.0.0.1:4005"
|
||||
FILTRON_LISTEN="127.0.0.1:4004"
|
||||
FILTRON_TARGET="127.0.0.1:8888"
|
||||
|
||||
SERVICE_NAME="filtron"
|
||||
SERVICE_USER="${SERVICE_NAME}"
|
||||
SERVICE_HOME="/home/${SERVICE_USER}"
|
||||
@ -23,6 +28,11 @@ GO_ENV="${SERVICE_HOME}/.go_env"
|
||||
GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
|
||||
GO_TAR=$(basename "$GO_PKG_URL")
|
||||
|
||||
CONFIG_FILES=(
|
||||
"${FILTRON_RULES}"
|
||||
"${SERVICE_SYSTEMD_UNIT}"
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
usage(){
|
||||
# ----------------------------------------------------------------------------
|
||||
@ -37,10 +47,16 @@ usage:
|
||||
$(basename "$0") remove [all]
|
||||
$(basename "$0") activate [server]
|
||||
$(basename "$0") deactivate [server]
|
||||
$(basename "$0") show [server]
|
||||
|
||||
shell - start interactive shell with user ${SERVICE_USER}
|
||||
install user - add service user '$SERVICE_USER' at $SERVICE_HOME
|
||||
|
||||
shell
|
||||
start interactive shell from user ${SERVICE_USER}
|
||||
show server
|
||||
show server status and log
|
||||
install / remove
|
||||
all - complete setup of filtron server
|
||||
install user
|
||||
add service user '$SERVICE_USER' at $SERVICE_HOME
|
||||
EOF
|
||||
[ ! -z ${1+x} ] && echo -e "$1"
|
||||
}
|
||||
@ -58,6 +74,14 @@ main(){
|
||||
sudo_or_exit
|
||||
interactive_shell
|
||||
;;
|
||||
show)
|
||||
case $2 in
|
||||
server)
|
||||
sudo_or_exit
|
||||
show_server
|
||||
;;
|
||||
*) usage "$_usage"; exit 42;;
|
||||
esac ;;
|
||||
install)
|
||||
sudo_or_exit
|
||||
case $2 in
|
||||
@ -91,21 +115,27 @@ main(){
|
||||
install_all() {
|
||||
rst_title "Install $SERVICE_NAME (service)"
|
||||
assert_user
|
||||
wait_key
|
||||
install_go
|
||||
wait_key
|
||||
install_filtron
|
||||
wait_key
|
||||
install_server
|
||||
wait_key
|
||||
}
|
||||
|
||||
remove_all() {
|
||||
rst_title "De-Install $SERVICE_NAME (service)"
|
||||
remove_server
|
||||
wait_key
|
||||
remove_user
|
||||
rm -rf "$FILTRON_ETC"
|
||||
rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
|
||||
wait_key
|
||||
}
|
||||
|
||||
install_server() {
|
||||
rst_title "Install System-D Unit ${SERVICE_NAME}.service ..." section
|
||||
rst_title "Install System-D Unit ${SERVICE_NAME}.service" section
|
||||
echo
|
||||
install_template ${SERVICE_SYSTEMD_UNIT} root root 644
|
||||
wait_key
|
||||
activate_server
|
||||
@ -116,12 +146,12 @@ remove_server() {
|
||||
return
|
||||
fi
|
||||
deactivate_server
|
||||
rm "${SERVICE_SYSTEMD_UNIT}"
|
||||
rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout
|
||||
}
|
||||
|
||||
|
||||
activate_server () {
|
||||
rst_title "Activate $SERVICE_NAME (service)" section
|
||||
echo
|
||||
tee_stderr <<EOF | bash 2>&1 | prefix_stdout
|
||||
systemctl enable $SERVICE_NAME.service
|
||||
systemctl restart $SERVICE_NAME.service
|
||||
@ -129,7 +159,6 @@ EOF
|
||||
tee_stderr <<EOF | bash 2>&1 | prefix_stdout
|
||||
systemctl status $SERVICE_NAME.service
|
||||
EOF
|
||||
wait_key
|
||||
}
|
||||
|
||||
deactivate_server () {
|
||||
@ -139,7 +168,6 @@ deactivate_server () {
|
||||
systemctl stop $SERVICE_NAME.service
|
||||
systemctl disable $SERVICE_NAME.service
|
||||
EOF
|
||||
wait_key
|
||||
}
|
||||
|
||||
assert_user() {
|
||||
@ -168,18 +196,18 @@ EOF
|
||||
remove_user() {
|
||||
rst_title "Drop $SERVICE_USER HOME" section
|
||||
if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then
|
||||
userdel -r -f "$SERVICE_USER"
|
||||
userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout
|
||||
else
|
||||
rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged."
|
||||
fi
|
||||
}
|
||||
|
||||
interactive_shell(){
|
||||
echo "// exit with STRG-D"
|
||||
echo "// exit with CTRL-D"
|
||||
sudo -H -u ${SERVICE_USER} -i
|
||||
}
|
||||
|
||||
_service_prefix="$SERVICE_USER@$(hostname) -->| "
|
||||
_service_prefix=" |$SERVICE_USER| "
|
||||
|
||||
install_go(){
|
||||
rst_title "Install Go in user's HOME" section
|
||||
@ -199,14 +227,29 @@ EOF
|
||||
! which go >/dev/null && echo "Go Installation not found in PATH!?!"
|
||||
which go >/dev/null && go version && echo "congratulations -- Go installation OK :)"
|
||||
EOF
|
||||
wait_key
|
||||
}
|
||||
|
||||
install_filtron() {
|
||||
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" | prefix_stdout "$_service_prefix"
|
||||
go get -v -u github.com/asciimoo/filtron 2>&1
|
||||
rst_title "Install filtron in user's ~/go-apps" section
|
||||
echo
|
||||
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix"
|
||||
go get -v -u github.com/asciimoo/filtron
|
||||
EOF
|
||||
install_template "$FILTRON_ETC/rules.json" root root 644
|
||||
install_template --no-eval "$FILTRON_RULES" root root 644
|
||||
}
|
||||
|
||||
show_server () {
|
||||
rst_title "server status & log"
|
||||
echo
|
||||
systemctl status filtron.service
|
||||
echo
|
||||
read -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log"
|
||||
echo
|
||||
while true; do
|
||||
trap break 2
|
||||
journalctl -f -u filtron
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
72
utils/lib.sh
72
utils/lib.sh
@ -22,6 +22,13 @@ if [[ -z "$SYSTEMD_UNITS" ]]; then
|
||||
SYSTEMD_UNITS="/lib/systemd/system"
|
||||
fi
|
||||
|
||||
if [[ -z ${DIFF_CMD} ]]; then
|
||||
DIFF_CMD="diff -u"
|
||||
if command -v colordiff >/dev/null; then
|
||||
DIFF_CMD="colordiff -u"
|
||||
fi
|
||||
fi
|
||||
|
||||
sudo_or_exit() {
|
||||
# usage: sudo_or_exit
|
||||
|
||||
@ -55,10 +62,10 @@ rst_para() {
|
||||
# usage: RST_INDENT=1 rst_para "lorem ipsum ..."
|
||||
local prefix=''
|
||||
if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then
|
||||
prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"
|
||||
echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
|
||||
prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"
|
||||
echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
|
||||
else
|
||||
echo -en "\n$*\n" | $FMT
|
||||
echo -en "\n$*\n" | $FMT
|
||||
fi
|
||||
}
|
||||
|
||||
@ -66,15 +73,23 @@ err_msg() { echo -e "ERROR: $*" >&2; }
|
||||
warn_msg() { echo -e "WARN: $*" >&2; }
|
||||
info_msg() { echo -e "INFO: $*"; }
|
||||
|
||||
clean_stdin() {
|
||||
if [[ $(uname -s) != 'Darwin' ]]; then
|
||||
while $(read -n1 -t 0.1); do : ; done
|
||||
fi
|
||||
}
|
||||
|
||||
wait_key(){
|
||||
# usage: waitKEY [<timeout in sec>]
|
||||
|
||||
clean_stdin
|
||||
local _t=$1
|
||||
[[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
|
||||
[[ ! -z $_t ]] && _t="-t $_t"
|
||||
# shellcheck disable=SC2086
|
||||
read -n1 $_t -p "** press any [KEY] to continue **"
|
||||
read -s -n1 $_t -p "** press any [KEY] to continue **"
|
||||
echo
|
||||
clean_stdin
|
||||
}
|
||||
|
||||
ask_yn() {
|
||||
@ -100,6 +115,7 @@ ask_yn() {
|
||||
esac
|
||||
echo
|
||||
while true; do
|
||||
clean_stdin
|
||||
printf "$1 ${choice} "
|
||||
# shellcheck disable=SC2086
|
||||
read -n1 $_t
|
||||
@ -117,6 +133,7 @@ ask_yn() {
|
||||
_t=""
|
||||
err_msg "invalid choice"
|
||||
done
|
||||
clean_stdin
|
||||
return $exit_val
|
||||
}
|
||||
|
||||
@ -144,7 +161,7 @@ tee_stderr () {
|
||||
prefix_stdout () {
|
||||
# usage: <cmd> | prefix_stdout [prefix]
|
||||
|
||||
local prefix="-->| "
|
||||
local prefix=" | "
|
||||
|
||||
if [[ ! -z $1 ]] ; then prefix="$1"; fi
|
||||
|
||||
@ -223,6 +240,7 @@ choose_one() {
|
||||
fi
|
||||
done
|
||||
while true; do
|
||||
clean_stdin
|
||||
printf "$1 [$default] "
|
||||
|
||||
if (( 10 > $max )); then
|
||||
@ -242,6 +260,7 @@ choose_one() {
|
||||
err_msg "invalid choice"
|
||||
done
|
||||
echo
|
||||
clean_stdin
|
||||
eval "$env_name"='${list[${REPLY}]}'
|
||||
}
|
||||
|
||||
@ -288,31 +307,48 @@ install_template() {
|
||||
|
||||
mkdir -p "$(dirname "${dst}")"
|
||||
|
||||
if [[ -f "${dst}" ]] ; then
|
||||
info_msg "file ${dst} allready exists on this host"
|
||||
if [[ ! -f "${dst}" ]]; then
|
||||
info_msg "install: ${template_file}"
|
||||
sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
|
||||
"${template_file}" "${dst}" | prefix_stdout
|
||||
return $?
|
||||
fi
|
||||
|
||||
if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
|
||||
info_msg "file ${dst} allready installed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info_msg "file ${dst} allready exists on this host"
|
||||
|
||||
while true; do
|
||||
choose_one _reply "choose next step with file $dst" \
|
||||
"replace file" \
|
||||
"leave file unchanged"
|
||||
"leave file unchanged" \
|
||||
"interactiv shell" \
|
||||
"diff files"
|
||||
|
||||
case $_reply in
|
||||
"replace file")
|
||||
info_msg "install: ${template_file}"
|
||||
sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
|
||||
"${template_file}" "${dst}" | prefix_stdout
|
||||
break
|
||||
;;
|
||||
"leave file unchanged")
|
||||
break
|
||||
;;
|
||||
"interactiv shell")
|
||||
echo "// exit with STRG-D"
|
||||
echo "// edit ${dst} to your needs"
|
||||
echo "// exit with CTRL-D"
|
||||
sudo -H -u "${owner}" -i
|
||||
;;
|
||||
$DIFF_CMD "${dst}" "${template_file}"
|
||||
if ask_yn "did you edit ${template_file} to your needs?"; then
|
||||
break
|
||||
fi
|
||||
;;
|
||||
"diff files")
|
||||
$DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
|
||||
esac
|
||||
|
||||
else
|
||||
info_msg "install: ${template_file}"
|
||||
sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
|
||||
"${template_file}" "${dst}" | prefix_stdout
|
||||
fi
|
||||
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
@ -1,56 +1,98 @@
|
||||
[
|
||||
[{
|
||||
"name":"search request",
|
||||
"filters":[
|
||||
"Param:q",
|
||||
"Path=^(/|/search)$"
|
||||
],
|
||||
"interval":60,
|
||||
"limit":15,
|
||||
"subrules":[
|
||||
{
|
||||
"name": "api limit",
|
||||
"interval": 60,
|
||||
"limit": 1000,
|
||||
"filters": ["Path=^/api"],
|
||||
"aggregations": ["Path"],
|
||||
"actions": [
|
||||
{"name": "block"}
|
||||
],
|
||||
"subrules": [
|
||||
{
|
||||
"name": "drop put",
|
||||
"interval": 60,
|
||||
"limit": 100,
|
||||
"filters": ["Method=PUT"],
|
||||
"aggregations": ["Header:X-Forwarded-For"],
|
||||
"actions": [
|
||||
{"name": "shell",
|
||||
"params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}}
|
||||
]
|
||||
}
|
||||
]
|
||||
"name":"roboagent limit",
|
||||
"interval":60,
|
||||
"limit":15,
|
||||
"filters":[
|
||||
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
|
||||
],
|
||||
"actions":[
|
||||
{"name": "log"},
|
||||
{
|
||||
"name":"block",
|
||||
"params":{
|
||||
"message":"Rate limit exceeded"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "log'n'block rss",
|
||||
"interval": 300,
|
||||
"limit": 2500,
|
||||
"filters": ["Path=^/$", "GET:format=rss"],
|
||||
"actions": [
|
||||
{"name": "log"},
|
||||
{"name": "block"}
|
||||
]
|
||||
"name":"botlimit",
|
||||
"limit":0,
|
||||
"stop":true,
|
||||
"filters":[
|
||||
"Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
|
||||
],
|
||||
"actions":[
|
||||
{"name": "log"},
|
||||
{
|
||||
"name":"block",
|
||||
"params":{
|
||||
"message":"Rate limit exceeded"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "log rule",
|
||||
"filters": ["Path=/"],
|
||||
"actions": [ {"name": "log"} ],
|
||||
"subrules": [
|
||||
{
|
||||
"name": "block missing accept-language",
|
||||
"filters": ["!Header:Accept-Language"],
|
||||
"actions": [
|
||||
{"name": "block"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "block curl",
|
||||
"filters": ["Header:User-Agent=[Cc]url"],
|
||||
"actions": [
|
||||
{"name": "block"}
|
||||
]
|
||||
}
|
||||
]
|
||||
"name":"IP limit",
|
||||
"interval":60,
|
||||
"limit":15,
|
||||
"stop":true,
|
||||
"aggregations":[
|
||||
"Header:X-Forwarded-For"
|
||||
],
|
||||
"actions":[
|
||||
{"name": "log"},
|
||||
{
|
||||
"name":"block",
|
||||
"params":{
|
||||
"message":"Rate limit exceeded"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name":"rss/json limit",
|
||||
"interval":60,
|
||||
"limit":15,
|
||||
"stop":true,
|
||||
"filters":[
|
||||
"Param:format=(csv|json|rss)"
|
||||
],
|
||||
"actions":[
|
||||
{"name": "log"},
|
||||
{
|
||||
"name":"block",
|
||||
"params":{
|
||||
"message":"Rate limit exceeded"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name":"useragent limit",
|
||||
"interval":60,
|
||||
"limit":15,
|
||||
"aggregations":[
|
||||
"Header:User-Agent"
|
||||
],
|
||||
"actions":[
|
||||
{"name": "log"},
|
||||
{
|
||||
"name":"block",
|
||||
"params":{
|
||||
"message":"Rate limit exceeded"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
]
|
||||
}]
|
||||
|
||||
@ -10,7 +10,7 @@ Type=simple
|
||||
User=${SERVICE_USER}
|
||||
Group=${SERVICE_GROUP}
|
||||
WorkingDirectory=${SERVICE_HOME}
|
||||
ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES}
|
||||
ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'
|
||||
|
||||
Restart=always
|
||||
Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user