diff --git a/.config.sh b/.config.sh index 3a17b8b29..0e41a73b0 100644 --- a/.config.sh +++ b/.config.sh @@ -2,48 +2,40 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck shell=bash disable=SC2034 # -# This environment is used by ./utils scripts like filtron.sh or searx.sh. The -# default values are *most flexible* and *best maintained*, you normally not -# need to change the defaults (except PUBLIC_URL). +# This file should be edited only ones just before the installation of any +# service is done. After the installation of the searx service a copy of this +# file is placed into the $SEARX_SRC of the instance, e.g.:: # -# Before you change any value here you have to uninstall any previous -# installation. Further is it recommended to backup your changes simply by -# adding them to you local brand (git branch):: +# /usr/local/searx/searx-src/.config.sh # -# git add .config +# .. hint:: +# +# Before you change a value here, You have to fully uninstall any previous +# installation of searx, morty and filtron services! -# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx" -# The default is taken from ./utils/brand.env. +# utils/searx.sh +# -------------- -PUBLIC_URL="${SEARX_URL}" +# The setup of the SearXNG instance is done in the settings.yml +# (SEARX_SETTINGS_PATH). Read the remarks in [1] carefully and don't forget to +# rebuild instance's environment (make buildenv) if needed. The settings.yml +# file of an already installed instance is shown by:: +# +# $ ./utils/searx.sh --help +# ---- SearXNG instance setup (already installed) +# SEARX_SETTINGS_PATH : /etc/searx/settings.yml +# SEARX_SRC : /usr/local/searx/searx-src +# +# [1] https://searxng.github.io/searxng/admin/engines/settings.html -if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then - # hint: Linux containers do not have DNS entries, lets use IPs - PUBLIC_URL="http://$(primary_ip)/searx" -fi - -# searx.sh -# --------- - -# SEARX_INTERNAL_HTTP="127.0.0.1:8888" -# SEARX_SETTINGS_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml" - -# Only change, if you maintain a searx brand in your searx fork (GIT_URL) which -# is not hold by branch 'master'. The branch has to be a local branch, in the -# repository from which you install (which is most often the case). If you want -# to install branch 'foo', don't forget to run 'git branch foo origin/foo' once. -# GIT_BRANCH="${GIT_BRANCH:-master}" - -# filtron.sh -# ---------- +# utils/filtron.sh +# ---------------- # FILTRON_API="127.0.0.1:4005" # FILTRON_LISTEN="127.0.0.1:4004" -# FILTRON_TARGET="127.0.0.1:8888" -# FILTRON_RULES_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml" -# morty.sh -# -------- +# utils/morty.sh +# -------------- # morty listen address # MORTY_LISTEN="127.0.0.1:3000" diff --git a/Makefile b/Makefile index 2b9dfacce..6c22d5e2e 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,7 @@ test.shell: utils/brand.env \ $(MTOOLS) \ utils/lib.sh \ + utils/lib_install.sh \ utils/lib_static.sh \ utils/filtron.sh \ utils/searx.sh \ diff --git a/docs/admin/api.rst b/docs/admin/api.rst index 7804a8664..8c782ea9c 100644 --- a/docs/admin/api.rst +++ b/docs/admin/api.rst @@ -87,7 +87,7 @@ HTML of the site. URL of the searx instance and values are customizable. .. code:: html -
+ diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 7335b9a2a..cab4ee493 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -37,16 +37,11 @@ see how you can simplify your *user defined* ``settings.yml``. Global Settings =============== -``general:`` +.. _settings global brand: + +``brand:`` ------------ -.. code:: yaml - - general: - debug: false # Debug mode, only for development - instance_name: "searxng" # displayed name - contact_url: false # mailto:contact@example.com - .. code:: yaml brand: @@ -57,22 +52,47 @@ Global Settings public_instances: https://searx.space wiki_url: https://github.com/searxng/searxng/wiki -``debug`` : +.. sidebar:: buildenv + + Changing a value tagged by :ref:`buildenv `, needs to + rebuild instance's environment :ref:`utils/brand.env `. + +``git_url`` & ``git_branch`` : :ref:`buildenv GIT_URL & GIT_BRANCH ` + Changes this, to point to your searx fork (branch). + +``issue_url`` : + If you host your own issue tracker change this URL. + +``docs_url`` : + If you host your own documentation change this URL. + +``public_instances`` : + If you host your own https://searx.space change this URL. + +``wiki_url`` : + Link to your wiki (or ``false``) + +.. _settings global general: + +``general:`` +------------ + +.. code:: yaml + + general: + debug: false # Debug mode, only for development + instance_name: "searxng" # displayed name + contact_url: false # mailto:contact@example.com + +``debug`` : ``$SEARX_DEBUG`` Allow a more detailed log if you run searx directly. Display *detailed* error messages in the browser too, so this must be deactivated in production. ``contact_url``: Contact ``mailto:`` address or WEB form. -``git_url`` and ``git_branch``: - Changes this, to point to your searx fork (branch). - -``docs_url`` - If you host your own documentation, change this URL. - -``wiki_url``: - Link to your wiki (or ``false``) +.. _settings global server: ``server:`` ----------- @@ -80,10 +100,10 @@ Global Settings .. code:: yaml server: + base_url: false # set custom base_url (or false) port: 8888 bind_address: "127.0.0.1" # address to listen on secret_key: "ultrasecretkey" # change this! - base_url: false # set custom base_url (or false) image_proxy: false # proxying image results through searx default_locale: "" # default interface locale default_theme: oscar # ui theme @@ -94,17 +114,24 @@ Global Settings X-Robots-Tag : noindex, nofollow Referrer-Policy : no-referrer -``port`` & ``bind_address``: +.. sidebar:: buildenv + + Changing a value tagged by :ref:`buildenv `, needs to + rebuild instance's environment :ref:`utils/brand.env `. + +``base_url`` : :ref:`buildenv SEARX_URL ` + The base URL where searx is deployed. Used to create correct inbound links. + If you change the value, don't forget to rebuild instance's environment + (:ref:`utils/brand.env `) + +``port`` & ``bind_address``: :ref:`buildenv SEARX_PORT & SEARX_BIND_ADDRESS ` Port number and *bind address* of the searx web application if you run it directly using ``python searx/webapp.py``. Doesn't apply to searx running on Apache or Nginx. -``secret_key`` : +``secret_key`` : ``$SEARX_SECRET`` Used for cryptography purpose. -``base_url`` : - The base URL where searx is deployed. Used to create correct inbound links. - ``image_proxy`` : Allow your instance of searx of being able to proxy images. Uses memory space. diff --git a/docs/admin/installation-searx.rst b/docs/admin/installation-searx.rst index 5c9985e8b..c5e1a05e7 100644 --- a/docs/admin/installation-searx.rst +++ b/docs/admin/installation-searx.rst @@ -94,7 +94,7 @@ For a *minimal setup*, configure like shown below – replace ``searx@$(uname .. group-tab:: Use default settings - .. literalinclude:: ../../utils/templates/etc/searx/use_default_settings.yml + .. literalinclude:: ../../utils/templates/etc/searx/settings.yml :language: yaml .. group-tab:: searx/settings.yml diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searx.rst index 33ed9601c..e4d5f06b3 100644 --- a/docs/build-templates/searx.rst +++ b/docs/build-templates/searx.rst @@ -134,7 +134,7 @@ ${fedora_build} .. code-block:: sh $ sudo -H mkdir -p \"$(dirname ${SEARX_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searx/use_default_settings.yml\" \\ + $ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searx/settings.yml\" \\ \"${SEARX_SETTINGS_PATH}\" .. group-tab:: searx/settings.yml @@ -152,7 +152,6 @@ ${fedora_build} .. code-block:: sh $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \"$SEARX_SETTINGS_PATH\" - $ sudo -H sed -i -e \"s/{instance_name}/searx@\$(uname -n)/g\" \"$SEARX_SETTINGS_PATH\" .. END searx config diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index b7472dad7..5bf3c13b2 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -81,6 +81,40 @@ the check fails if you edit the requirements listed in If you think, something goes wrong with your ./local environment or you change the :origin:`setup.py` file, you have to call :ref:`make clean`. +.. _make buildenv: + +``make buildenv`` +================= + +Rebuild instance's environment with the modified settings from the +:ref:`settings global brand` and :ref:`settings global server` section of your +:ref:`settings.yml `. + +We have all SearXNG setups are centralized in the :ref:`settings.yml` file. +This setup is available as long we are in a *installed instance*. E.g. the +*installed instance* on the server or the *installed developer instance* at +``./local`` (the later one is created by a :ref:`make install ` or :ref:`make run `). + +Tasks running outside of an *installed instance*, especially those tasks and +scripts running at (pre-) installation time do not have access to the SearXNG +setup (from a *installed instance*). Those tasks need a *build environment*. + +The ``make buildenv`` target will update the *build environment* in: + +- :origin:`utils/brand.env` + +Tasks running outside of an *installed instance*, need the following settings +from the YAML configuration: + +- ``GIT_URL`` from :ref:`brand.git_url ` +- ``GIT_BRANCH`` from :ref:`brand.git_branch ` + +- ``SEARX_URL`` from :ref:`server.base_url ` (aka + ``PUBLIC_URL``) +- ``SEARX_BIND_ADDRESS`` from :ref:`server.bind_address ` +- ``SEARX_PORT`` from :ref:`server.port ` + .. _make run: ``make run`` diff --git a/manage b/manage index f03343839..b974c3028 100755 --- a/manage +++ b/manage @@ -105,11 +105,32 @@ fi export DOCS_BUILD buildenv() { + + # settings file from repository's working tree are used by default + SEARX_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml" + + if [ -r '/etc/searx/settings.yml' ]; then + if ask_yn "should settings read from: /etc/searx/settings.yml"; then + SEARX_SETTINGS_PATH='/etc/searx/settings.yml' + fi + fi + export SEARX_SETTINGS_PATH SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \ | prefix_stdout "${_Blue}BUILDENV${_creset} " return "${PIPESTATUS[0]}" } +buildenv.unset_env(){ + # Some defaults in the settings.yml are taken from the environment, + # e.g. SEARX_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). In + # some tasks (e.g. test.robot) we do not want these envorionment applied. + unset GIT_URL + unset GIT_BRANCH + unset SEARX_URL + unset SEARX_PORT + unset SEARX_BIND_ADDRESS +} + babel.compile() { build_msg BABEL compile pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations" @@ -471,6 +492,7 @@ test.coverage() { test.robot() { build_msg TEST 'robot' + buildenv.unset_env gecko.driver PYTHONPATH=. pyenv.cmd python searx/testing.py robot dump_return $? diff --git a/searx/settings.yml b/searx/settings.yml index 70ac5ee49..ac9a14064 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -4,13 +4,16 @@ general: contact_url: false # mailto:contact@example.com brand: - git_url: https://github.com/searxng/searxng - git_branch: master - issue_url: https://github.com/searxng/searxng/issues new_issue_url: https://github.com/searxng/searxng/issues/new docs_url: https://searxng.github.io/searxng public_instances: https://searx.space wiki_url: https://github.com/searxng/searxng/wiki + issue_url: https://github.com/searxng/searxng/issues + + # If you change a value below don't forget to rebuild instance's enviroment + # (make buildenv) + git_url: https://github.com/return42/searxng + git_branch: fix-searx.sh search: # Filter results. 0: None, 1: Moderate, 2: Strict @@ -30,13 +33,16 @@ search: formats: [html, csv, json, rss] server: - port: 8888 - # address to listen on - bind_address: "127.0.0.1" + # If you change port, bind_address or base_url don't forget to rebuild + # instance's enviroment (make buildenv) + port: 7777 + bind_address: "127.0.0.12" + base_url: false # Possible values: false or "https://example.org/location". + + # If your instance owns a /etc/searx/settings.yml file, then set the following + # values there. + secret_key: "ultrasecretkey" # change this! - # Set custom base_url. Possible values: - # false or "https://your.custom.host/location/" - base_url: false # Proxying image results through searx image_proxy: false # 1.0 and 1.1 are supported diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 0fa1f973d..72d7dc588 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -147,7 +147,7 @@ SCHEMA = { 'formats': SettingsValue(list, OUTPUT_FORMATS), }, 'server': { - 'port': SettingsValue(int, 8888), + 'port': SettingsValue((int,str), 8888, 'SEARX_PORT'), 'bind_address': SettingsValue(str, '127.0.0.1', 'SEARX_BIND_ADDRESS'), 'secret_key': SettingsValue(str, environ_name='SEARX_SECRET'), 'base_url': SettingsValue((False, str), False), diff --git a/utils/brand.env b/utils/brand.env index e0c670a01..c09ea9d2f 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,8 +1,5 @@ +export GIT_URL='https://github.com/return42/searxng' +export GIT_BRANCH='fix-searx.sh' export SEARX_URL='' -export GIT_URL='https://github.com/searxng/searxng' -export GIT_BRANCH='master' -export ISSUE_URL='https://github.com/searxng/searxng/issues' -export DOCS_URL='https://searxng.github.io/searxng' -export PUBLIC_INSTANCES='https://searx.space' -export CONTACT_URL='' -export WIKI_URL='https://github.com/searxng/searxng/wiki' +export SEARX_PORT='7777' +export SEARX_BIND_ADDRESS='127.0.0.12' diff --git a/utils/build_env.py b/utils/build_env.py index c52111e81..dc55bd0a8 100644 --- a/utils/build_env.py +++ b/utils/build_env.py @@ -9,15 +9,12 @@ from os.path import realpath, dirname, join, sep, abspath repo_root = realpath(dirname(realpath(__file__)) + sep + '..') sys.path.insert(0, repo_root) -os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + '/settings.yml') # Under the assumption that a brand is always a fork assure that the settings # file from reposetorie's working tree is used to generate the build_env, not # from /etc/searx/settings.yml. os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + sep + 'settings.yml') -from searx import get_setting - def _env(*arg, **kwargs): val = get_setting(*arg, **kwargs) if val is True: @@ -26,20 +23,42 @@ def _env(*arg, **kwargs): val = '' return val +# If you add or remove variables here, do not forgett to update: +# - ./docs/admin/engines/settings.rst +# - ./docs/dev/makefile.rst (section make buildenv) +# - ./manage function buildenv.unset_env() + name_val = [ - ('SEARX_URL' , _env('server.base_url','')), - ('GIT_URL' , _env('brand.git_url', '')), - ('GIT_BRANCH' , _env('brand.git_branch', '')), - ('ISSUE_URL' , _env('brand.issue_url', '')), - ('DOCS_URL' , _env('brand.docs_url', '')), - ('PUBLIC_INSTANCES' , _env('brand.public_instances', '')), - ('CONTACT_URL' , _env('general.contact_url', '')), - ('WIKI_URL' , _env('brand.wiki_url', '')), + + ('GIT_URL' , 'brand.git_url'), + ('GIT_BRANCH' , 'brand.git_branch'), + + ('SEARX_URL' , 'server.base_url'), + ('SEARX_PORT' , 'server.port'), + ('SEARX_BIND_ADDRESS' , 'server.bind_address'), + ] brand_env = 'utils' + sep + 'brand.env' -print('build %s' % brand_env) +# Some defaults in the settings.yml are taken from the environment, +# e.g. SEARX_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). When the +# 'brand.env' file is created these enviroment variables should be unset first:: + +_unset = object() +for name, option in name_val: + if not os.environ.get(name, _unset) is _unset: + del os.environ[name] + +# After the variables are unset in the environ, we can import settings +# (get_setting) from searx module. + +from searx import get_setting + +print('build %s (settings from: %s)' % (brand_env, os.environ['SEARX_SETTINGS_PATH'])) +sys.path.insert(0, repo_root) +from searx import settings + with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f: - for name, val in name_val: - print("export %s='%s'" % (name, val), file=f) + for name, option in name_val: + print("export %s='%s'" % (name, _env(option)), file=f) diff --git a/utils/filtron.sh b/utils/filtron.sh index 5680811ea..3bd6f016d 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -1,21 +1,16 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck disable=SC2119,SC2001 +# shellcheck disable=SC2001 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ @@ -28,7 +23,12 @@ FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/e FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" -FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +# The filtron target is the SearXNG installation, listenning on server.port at +# server.bind_address. The default of FILTRON_TARGET is taken from the YAML +# configuration, do not change this value without reinstalling the entire +# SearXNG suite including filtron & morty. +FILTRON_TARGET="${SEARX_BIND_ADDRESS}:${SEARX_PORT}" SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" @@ -94,28 +94,17 @@ apache (${PUBLIC_URL}) nginx (${PUBLIC_URL}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_FILTRON_SITE} - filtron rules: ${FILTRON_RULES_TEMPLATE} - -If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL : ${PUBLIC_URL} - PUBLIC_HOST : ${PUBLIC_HOST} - SERVICE_USER : ${SERVICE_USER} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} +---- sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + FILTRON_TARGET : ${FILTRON_TARGET} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} EOF - if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container URL (IPv6): http://[${ip#*|}]:4005/" - else - # IPv4: - echo " container URL (IPv4): http://${ip#*|}:4005/" - fi - done - fi + + install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } @@ -349,16 +338,15 @@ inspect_service() { cat < [ []] # # First form uses $CACHE/ as destination folder, second form clones - # into . If repository is allready cloned, pull from and + # into . If repository is already cloned, pull from and # update working tree (if needed, the caller has to stash local changes). # # git clone https://github.com/searxng/searxng searx-src origin/master searxlogin diff --git a/utils/lib_install.sh b/utils/lib_install.sh new file mode 100755 index 000000000..4ad11d63d --- /dev/null +++ b/utils/lib_install.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 +# shellcheck source=utils/lib.sh +. /dev/null + +# Initialize installation procedures: +# +# - Modified source_dot_config function that +# - loads .config.sh from an existing installation (at SEARX_SRC). +# - initialize **SEARX_SRC_INIT_FILES** +# - functions like: +# - install_log_searx_instance() +# - install_searx_get_state() +# +# usage: +# source lib_install.sh +# +# **Installation scripts** +# +# The utils/lib_install.sh is sourced by the installations scripts: +# +# - utils/searx.sh +# - utils/morty.sh +# - utils/filtron.sh +# +# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function +# loads this configuration (instead of './.config.sh'). + +# **SEARX_SRC_INIT_FILES** +# +# Array of file names to sync into a installation at $SEARX_SRC. The file names +# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). +# Most often theses are files like: +# - .config.sh +# - searx/settings.yml +# - utils/brand.env +# - ... + + +SEARX_SRC_INIT_FILES=() + +eval orig_"$(declare -f source_dot_config)" + +source_dot_config() { + + # Modified source_dot_config function that + # - loads .config.sh from an existing installation (at SEARX_SRC). + # - initialize SEARX_SRC_INIT_FILES + + if [ -z "$eval_SEARX_SRC" ]; then + export eval_SEARX_SRC='true' + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) + SEARX_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SETTINGS_PATH) + if [ ! -r "${SEARX_SRC}" ]; then + info_msg "not yet cloned: ${SEARX_SRC}" + orig_source_dot_config + return 0 + fi + info_msg "using instance at: ${SEARX_SRC}" + + # set and log DOT_CONFIG + if [ -r "${SEARX_SRC}/.config.sh" ]; then + info_msg "switching to ${SEARX_SRC}/.config.sh" + DOT_CONFIG="${SEARX_SRC}/.config.sh" + else + info_msg "using local config: ${DOT_CONFIG}" + fi + init_SEARX_SRC_INIT_FILES + fi +} + +init_SEARX_SRC_INIT_FILES(){ + # init environment SEARX_SRC_INIT_FILES + + # Monitor modified files in the working-tree from the local repository, only + # if the local file differs to the corresponding file in the instance. Most + # often theses are files like: + # + # - .config.sh + # - searx/settings.yml + # - utils/brand.env + # - ... + + # keep list empty if there is no installation + SEARX_SRC_INIT_FILES=() + if [ ! -r "$SEARX_SRC" ]; then + return 0 + fi + + local fname + local msg="" + + # Monitor local modified files from the repository, only if the local file + # differs to the corresponding file in the instance + + while IFS= read -r fname; do + if [ -z "$fname" ]; then + continue + fi + if [ -r "${SEARX_SRC}/${fname}" ]; then + # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then + SEARX_SRC_INIT_FILES+=("${fname}") + info_msg "local clone (workingtree), modified file: ./$fname" + msg="to update use: sudo -H ./utils/searx.sh install init-src" + fi + fi + done <<< "$(git diff --name-only)" + [ -n "$msg" ] && info_msg "$msg" +} + +install_log_searx_instance() { + + echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" + echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SSEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}" + echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" + echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" + + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "---- container setup" + echo -e " ${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi +} + +install_searx_get_state(){ + + # usage: install_searx_get_state + # + # Prompts a string indicating the status of the installation procedure + # + # missing-searx-clone: + # There is no clone at ${SEARX_SRC} + # missing-searx-pyenv: + # There is no pyenv in ${SEARX_PYENV} + # installer-modified: + # There are files modified locally in the installer (clone), + # see ${SEARX_SRC_INIT_FILES} description. + # python-installed: + # Scripts can be executed in instance's environment + # - user: ${SERVICE_USER} + # - pyenv: ${SEARX_PYENV} + + if ! [ -r "${SEARX_SRC}" ]; then + echo "missing-searx-clone" + return + fi + if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then + echo "missing-searx-pyenv" + return + fi + if ! [ -r "${SEARX_SETTINGS_PATH}" ]; then + echo "missing-settings" + return + fi + if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + echo "installer-modified" + return + fi + echo "python-installed" +} + +# Initialization of the installation procedure +# -------------------------------------------- + +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +# SEARX_URL aka PUBLIC_URL: the public URL of the instance (e.g. +# "https://example.org/searx"). The value is taken from environment $SEARX_URL +# in ./utils/brand.env. This variable is a empty string if server.base_url in +# the settings.yml is set to 'false'. + +SEARX_URL="${SEARX_URL:-http://$(uname -n)}" +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARX_URL="http://$(primary_ip)" +fi +PUBLIC_URL="${SEARX_URL}" + +source_dot_config + +# shellcheck source=utils/lxc-searx.env +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env diff --git a/utils/lxc.sh b/utils/lxc.sh index f94e9841a..9a79606ca 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -568,7 +568,7 @@ check_connectivity() { info_msg "Most often the connectivity is blocked by a docker installation:" info_msg "Whenever docker is started (reboot) it sets the iptables policy " info_msg "for the FORWARD chain to DROP, see:" - info_msg " ${DOCS_URL}/utils/lxc.sh.html#internet-connectivity-docker" + info_msg " https://searxng.github.io/searxng/utils/lxc.sh.html#internet-connectivity-docker" iptables-save | grep ":FORWARD" fi return $ret_val diff --git a/utils/morty.sh b/utils/morty.sh index 0a44ef574..8be0e6ba6 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -1,15 +1,10 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -17,7 +12,6 @@ in_container && lxc_set_suite_env MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" - PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" # shellcheck disable=SC2034 @@ -86,12 +80,15 @@ apache : ${PUBLIC_URL_MORTY} nginx (${PUBLIC_URL_MORTY}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_MORTY_SITE} - -If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY} - MORTY_LISTEN: ${MORTY_LISTEN} - SERVICE_USER: ${SERVICE_USER} +---- +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} + MORTY_LISTEN: : ${MORTY_LISTEN} EOF + + install_log_searx_instance if in_container; then # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) for ip in $(global_IPs) ; do @@ -112,8 +109,9 @@ EOF info_searx() { # shellcheck disable=SC1117 cat <] $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy ] $(basename "$0") apache [install|remove] @@ -163,48 +155,34 @@ shell install / remove :all: complete (de-) installation of searx service :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :dot-config: copy ./config.sh to ${SEARX_SRC} :searx-src: clone $GIT_URL + :init-src: copy files (SEARX_SRC_INIT_FILES) to ${SEARX_SRC} :pyenv: create/remove virtualenv (python) in $SEARX_PYENV :uwsgi: install searx uWSGI application - :settings: reinstall settings from ${SEARX_SETTINGS_TEMPLATE} + :settings: reinstall settings from ${SEARX_SETTINGS_PATH} :packages: install needed packages from OS package manager :buildhost: install packages from OS package manager needed by buildhosts update searx - Update searx installation ($SERVICE_HOME) + Update SearXNG installation ($SERVICE_HOME) activate service activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) -inspect service - run some small tests and inspect service's status and log +inspect + :service: run some small tests and inspect service's status and log + :settings: inspect YAML setting from SearXNG instance (${SEARX_SRC}) option set one of the available options apache - :install: apache site with the searx uwsgi app + :install: apache site with the SearXNG uwsgi app :remove: apache site ${APACHE_FILTRON_SITE} - -searx settings: ${SEARX_SETTINGS_PATH} - -If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL : ${PUBLIC_URL} - SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} +---- sourced ${DOT_CONFIG} SERVICE_USER : ${SERVICE_USER} - SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP} + SERVICE_HOME : ${SERVICE_HOME} EOF - if in_container; then - # searx is listening on 127.0.0.1 and not available from outside container - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \ - "${_BBlack}inside${_creset} the container." - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container (IPv6): [${ip#*|}]" - else - # IPv4: - echo " container (IPv4): ${ip#*|}" - fi - done - fi + + install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } @@ -228,21 +206,45 @@ main() { sudo_or_exit inspect_service ;; + settings) + prompt_installation_setting "$3" + dump_return $? + ;; *) usage "$_usage"; exit 42;; esac ;; install) - rst_title "$SEARX_INSTANCE_NAME" part + rst_title "SearXNG (install)" part sudo_or_exit case $2 in all) install_all ;; - user) assert_user ;; - pyenv) create_pyenv ;; - searx-src) clone_searx ;; - settings) install_settings ;; + user) + verify_continue_install + assert_user + ;; + pyenv) + verify_continue_install + create_pyenv + ;; + searx-src) + verify_continue_install + clone_searx + install_DOT_CONFIG + init_SEARX_SRC + ;; + init-src) + init_SEARX_SRC + ;; + dot-config) + install_DOT_CONFIG + ;; + settings) + install_settings + ;; uwsgi) + verify_continue_install install_searx_uwsgi if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" fi ;; packages) @@ -261,6 +263,7 @@ main() { *) usage "$_usage"; exit 42;; esac ;; remove) + rst_title "SearXNG (remove)" part sudo_or_exit case $2 in all) remove_all;; @@ -307,13 +310,18 @@ main() { _service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} " install_all() { - rst_title "Install $SEARX_INSTANCE_NAME (service)" + rst_title "Install SearXNG (service)" + verify_continue_install pkg_install "$SEARX_PACKAGES" wait_key assert_user wait_key clone_searx wait_key + install_DOT_CONFIG + wait_key + init_SEARX_SRC + wait_key create_pyenv wait_key install_settings @@ -322,7 +330,7 @@ install_all() { wait_key install_searx_uwsgi if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" fi if ask_yn "Do you want to inspect the installation?" Ny; then inspect_service @@ -330,7 +338,7 @@ install_all() { } update_searx() { - rst_title "Update searx instance" + rst_title "Update SearXNG instance" echo tee_stderr 0.3 <&1 | prefix_stdout "$_service_prefix" @@ -348,13 +356,13 @@ EOF } remove_all() { - rst_title "De-Install $SEARX_INSTANCE_NAME (service)" + rst_title "De-Install SearXNG (service)" rst_para "\ It goes without saying that this script can only be used to remove installations that were installed with this script." - if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then + if ! ask_yn "Do you really want to deinstall SearXNG?"; then return fi remove_searx_uwsgi @@ -388,14 +396,14 @@ clone_is_available() { # shellcheck disable=SC2164 clone_searx() { - rst_title "Clone searx sources" section + rst_title "Clone SearXNG sources" section echo if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" fi SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" if [[ ! "${SERVICE_HOME}" ]]; then - err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + err_msg "to clone SearXNG sources, user $SERVICE_USER hast to be created first" return 42 fi if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then @@ -421,54 +429,169 @@ EOF popd > /dev/null } -install_settings() { - rst_title "${SEARX_SETTINGS_PATH}" section - if ! clone_is_available; then - err_msg "you have to install searx first" - exit 42 - fi - mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" +prompt_installation_status(){ - if [[ ! -f "${SEARX_SETTINGS_PATH}" ]]; then - info_msg "install settings ${SEARX_SETTINGS_TEMPLATE}" - info_msg " --> ${SEARX_SETTINGS_PATH}" - cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - configure_searx - return - fi + local state branch remote remote_url instance_setting + state="$(install_searx_get_state)" + branch="$(git name-rev --name-only HEAD)" + remote="$(git config branch."${branch}".remote)" + remote_url="$(git config remote."${remote}".url)" - rst_para "Diff between origin's setting file (+) and current (-):" - echo "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_TEMPLATE}" - $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_TEMPLATE}" - - local action - choose_one action "What should happen to the settings file? " \ - "keep configuration unchanged" \ - "use origin settings" \ - "start interactive shell" - case $action in - "keep configuration unchanged") - info_msg "leave settings file unchanged" + case $state in + missing-searx-clone) + info_msg "${_BBlue}(status: $(install_searx_get_state))${_creset}" + return 0 ;; - "use origin settings") - backup_file "${SEARX_SETTINGS_PATH}" - info_msg "install origin settings" - cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - ;; - "start interactive shell") - backup_file "${SEARX_SETTINGS_PATH}" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -i - rst_para 'Diff between new setting file (-) and current (+):' - echo - $DIFF_CMD "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - wait_key + *) + warn_msg "SearXNG instance already installed at: $SEARX_SRC" + warn_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} " + instance_setting="$(prompt_installation_setting brand.git_url)" + if ! [ "$instance_setting" = "$remote_url" ]; then + warn_msg "instance's brand.git_url: '${instance_setting}'" \ + "differs from local clone's remote URL: ${remote_url}" + fi + instance_setting="$(prompt_installation_setting brand.git_branch)" + if ! [ "$instance_setting" = "$branch" ]; then + warn_msg "instance brand.git_branch: ${instance_setting}" \ + "differs from local clone's branch: ${branch}" + fi + return 42 ;; esac } +verify_continue_install(){ + if ! prompt_installation_status; then + MSG="[${_BCyan}KEY${_creset}] to continue installation / [${_BCyan}CTRL-C${_creset}] to exit" \ + wait_key + fi +} + +prompt_installation_setting(){ + + # usage: prompt_installation_setting brand.git_url + # + # Prompts the value of the (YAML) setting in the SearXNG instance. + + local _state + _state="$(install_searx_get_state)" + case $_state in + python-installed|installer-modified) + sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" < ${SEARX_SRC}/${fname}" + cp "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + break + ;; + "diff files") + $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" + ;; + "interactive shell") + backup_file "${SEARX_SRC}/${fname}" + echo -e "// edit ${_Red}${dst}${_creset} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${SERVICE_USER}" -i + $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" + echo + echo -e "// ${_BBlack}did you edit file ...${_creset}" + echo -en "// ${_Red}${dst}${_creset}" + if ask_yn "//${_BBlack}... to your needs?${_creset}"; then + break + fi + ;; + esac + done + done +} + +install_DOT_CONFIG(){ + rst_title "Update instance: ${SEARX_SRC}/.config.sh" section + + if cmp --silent "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh"; then + info_msg "${SEARX_SRC}/.config.sh is up to date" + return 0 + fi + + diff "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" + if ! ask_yn "Do you want to copy file .config.sh into instance?" Yn; then + return 42 + fi + backup_file "${SEARX_SRC}/.config.sh" + cp "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" +} + +install_settings() { + rst_title "${SEARX_SETTINGS_PATH}" section + + if ! clone_is_available; then + err_msg "you have to install SearXNG first" + exit 42 + fi + + mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" + install_template \ + "${SEARX_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + configure_searx +} + remove_settings() { - rst_title "remove searx settings" section + rst_title "remove SearXNG settings" section echo info_msg "delete ${SEARX_SETTINGS_PATH}" rm -f "${SEARX_SETTINGS_PATH}" @@ -476,10 +599,10 @@ remove_settings() { remove_searx() { rst_title "Drop searx sources" section - if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + if ask_yn "Do you really want to drop SearXNG sources ($SEARX_SRC)?"; then rm -rf "$SEARX_SRC" else - rst_para "Leave searx sources unchanged." + rst_para "Leave SearXNG sources unchanged." fi } @@ -491,7 +614,7 @@ create_pyenv() { rst_title "Create virtualenv (python)" section echo if [[ ! -f "${SEARX_SRC}/manage" ]]; then - err_msg "to create pyenv for searx, searx has to be cloned first" + err_msg "to create pyenv for SearXNG, SearXNG has to be cloned first" return 42 fi info_msg "create pyenv in ${SEARX_PYENV}" @@ -531,18 +654,17 @@ EOF } configure_searx() { - rst_title "Configure searx" section - rst_para "Setup searx config located at $SEARX_SETTINGS_PATH" + rst_title "Configure SearXNG" section + rst_para "Setup SearXNG config located at $SEARX_SETTINGS_PATH" echo tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" -sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH" EOF } test_local_searx() { - rst_title "Testing searx instance localy" section + rst_title "Testing SearXNG instance localy" section echo if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then @@ -564,27 +686,27 @@ EOF } install_searx_uwsgi() { - rst_title "Install searx's uWSGI app (searx.ini)" section + rst_title "Install SearXNG's uWSGI app (searx.ini)" section echo install_uwsgi uWSGI_install_app "$SEARX_UWSGI_APP" } remove_searx_uwsgi() { - rst_title "Remove searx's uWSGI app (searx.ini)" section + rst_title "Remove SearXNG's uWSGI app (searx.ini)" section echo uWSGI_remove_app "$SEARX_UWSGI_APP" } activate_service() { - rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "Activate SearXNG (service)" section echo uWSGI_enable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" } deactivate_service() { - rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "De-Activate SearXNG (service)" section echo uWSGI_disable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" @@ -609,7 +731,7 @@ EOF } enable_debug() { - warn_msg "Do not enable debug in production enviroments!!" + warn_msg "Do not enable debug in production environments!!" info_msg "try to enable debug mode ..." tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -680,14 +802,11 @@ inspect_service() { rst_title "service status & log" cat <