Merge remote-tracking branch 'upstream/master' into pulling

utils/brand.env

@@ -1,5 +0,0 @@
-export SEARXNG_URL=''
-export SEARXNG_PORT='8888'
-export SEARXNG_BIND_ADDRESS='127.0.0.1'
-export GIT_URL='https://github.com/Kvan7/searxng'
-export GIT_BRANCH='master'

utils/brand.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+build.env.export() {
+    GIT_BRANCH="$(git branch | grep '\*' | cut -d' ' -f2-)"
+    GIT_REMOTE="$(git config "branch.${GIT_BRANCH}.remote")"
+    GIT_URL="$(git config --get "remote.${GIT_REMOTE}.url")"
+    if [[ "${GIT_URL}" == git@* ]]; then
+        GIT_URL="${GIT_URL/://}"
+        GIT_URL="${GIT_URL/git@/https://}"
+    fi
+    if [[ "${GIT_URL}" == *.git ]]; then
+        GIT_URL="${GIT_URL%.git}"
+    fi
+
+    SEARXNG_URL="$(python "${REPO_ROOT}/utils/get_setting.py" server.base_url)"
+    SEARXNG_PORT="$(python "${REPO_ROOT}/utils/get_setting.py" server.port)"
+    SEARXNG_BIND_ADDRESS="$(python "${REPO_ROOT}/utils/get_setting.py" server.bind_address)"
+    export GIT_URL
+    export GIT_BRANCH
+    export SEARXNG_URL
+    export SEARXNG_PORT
+    export SEARXNG_BIND_ADDRESS
+
+}
+
+pushd "${REPO_ROOT}" &> /dev/null
+build.env.export
+popd &> /dev/null

utils/build_env.py

@@ -1,61 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""build environment used by shell scripts
-"""
-
-# set path
-import sys
-import os
-from os.path import realpath, dirname, join, sep, abspath
-
-repo_root = realpath(dirname(realpath(__file__)) + sep + '..')
-sys.path.insert(0, repo_root)
-
-# Assure that the settings file from repository's working tree is used to
-# generate the build_env, not from /etc/searxng/settings.yml.
-os.environ['SEARXNG_SETTINGS_PATH'] = join(repo_root, 'etc', 'settings.yml')
-
-def _env(*arg, **kwargs):
-    val = get_setting(*arg, **kwargs)
-    if val is True:
-        val = '1'
-    elif val is False:
-        val = ''
-    return val
-
-# If you add or remove variables here, do not forget to update:
-# - ./docs/admin/engines/settings.rst
-# - ./docs/dev/makefile.rst (section make buildenv)
-
-name_val = [
-
-    ('SEARXNG_URL'              , 'server.base_url'),
-    ('SEARXNG_PORT'             , 'server.port'),
-    ('SEARXNG_BIND_ADDRESS'     , 'server.bind_address'),
-
-]
-
-brand_env = 'utils' + sep + 'brand.env'
-
-# Some defaults in the settings.yml are taken from the environment,
-# e.g. SEARXNG_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`).  When the
-# 'brand.env' file is created these environment variables should be unset first::
-
-_unset = object()
-for name, option in name_val:
-    if not os.environ.get(name, _unset) is _unset:
-        del os.environ[name]
-
-# After the variables are unset in the environ, we can import from the searx
-# package (what will read the values from the settings.yml).
-
-from searx.version import GIT_URL, GIT_BRANCH
-from searx import get_setting
-
-print('build %s (settings from: %s)' % (brand_env, os.environ['SEARXNG_SETTINGS_PATH']))
-sys.path.insert(0, repo_root)
-
-with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f:
-    for name, option in name_val:
-        print("export %s='%s'" % (name, _env(option)), file=f)
-    print(f"export GIT_URL='{GIT_URL}'", file=f)
-    print(f"export GIT_BRANCH='{GIT_BRANCH}'", file=f)

utils/filtron.sh

@@ -4,8 +4,6 @@
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
-source "${REPO_ROOT}/utils/brand.env"
 
 # ----------------------------------------------------------------------------
 # config

utils/get_setting.py

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""build environment used by shell scripts
+"""
+
+# set path
+import sys
+import importlib.util
+import re
+
+from pathlib import Path
+
+repo_root = Path(__file__).resolve().parent.parent
+
+
+def main(setting_name):
+
+    settings_path = repo_root / "searx" / "settings.yml"
+    with open(settings_path) as f:
+        settings = parse_yaml(f.read())
+    print(get_setting_value(settings, setting_name))
+
+
+def get_setting_value(settings, name):
+    value = settings
+    for a in name.split("."):
+        value = value[a]
+    if value is True:
+        value = "1"
+    elif value is False:
+        value = ""
+    return value
+
+
+def parse_yaml(yaml_str):
+    """A simple YAML parser that converts a YAML string to a Python dictionary.
+    This parser can handle nested dictionaries, but does not handle list or JSON
+    like structures.
+
+    Good enough parser to get the values of server.base_url, server.port and
+    server.bind_address
+
+    """
+
+    def get_type_and_value_without_comment(line):
+        """Extract value without comment and quote
+
+        Returns a tuple:
+
+        1. str or None: str when the value is written inside quote, None otherwise
+        2. the value without quote if any
+        """
+        match = re.search(r"\"(.*)\"(\s+#)?|\'(.*)\'(\s+#)?|([^#]*)(\s+#)?", line)
+        if match:
+            g = match.groups()
+            if g[0] is not None:
+                return str, g[0]
+            elif g[2] is not None:
+                return str, g[2]
+            elif g[4] is not None:
+                return None, g[4].strip()
+        return None, line.strip()
+
+    # fmt: off
+    true_values = ("y", "Y", "yes", "Yes", "YES", "true", "True", "TRUE", "on", "On", "ON",)
+    false_values = ("n", "N", "no", "No", "NO", "false", "False", "FALSE", "off", "Off", "OFF",)
+    # fmt: on
+
+    def process_line(line):
+        """Extract key and value from a line, considering its indentation."""
+        if ": " in line:
+            key, value = line.split(": ", 1)
+            key = key.strip()
+            value_type, value = get_type_and_value_without_comment(value)
+            if value in true_values and value_type is None:
+                value = True
+            elif value in false_values and value_type is None:
+                value = False
+            elif value.replace(".", "").isdigit() and value_type is None:
+                for t in (int, float):
+                    try:
+                        value = t(value)
+                        break
+                    except ValueError:
+                        continue
+            return key, value
+        return None, None
+
+    def get_indentation_level(line):
+        """Determine the indentation level of a line."""
+        return len(line) - len(line.lstrip())
+
+    yaml_dict = {}
+    lines = yaml_str.split("\n")
+    stack = [yaml_dict]
+
+    for line in lines:
+        if not line.strip():
+            continue  # Skip empty lines
+
+        indentation_level = get_indentation_level(line)
+        # Assuming 2 spaces per indentation level
+        # see .yamllint.yml
+        current_level = indentation_level // 2
+
+        # Adjust the stack based on the current indentation level
+        while len(stack) > current_level + 1:
+            stack.pop()
+
+        if line.endswith(":"):
+            key = line[0:-1].strip()
+            new_dict = {}
+            stack[-1][key] = new_dict
+            stack.append(new_dict)
+        else:
+            key, value = process_line(line)
+            if key is not None:
+                stack[-1][key] = value
+
+    return yaml_dict
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

utils/lib.sh

@@ -194,6 +194,7 @@ wait_key(){
     [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
     [[ -n $_t ]] && _t="-t $_t"
     printf "$msg"
+    # shellcheck disable=SC2229
     # shellcheck disable=SC2086
     read -r -s -n1 $_t || true
     echo
@@ -227,7 +228,7 @@ ask_yn() {
     while true; do
         clean_stdin
         printf "$1 ${choice} "
-        # shellcheck disable=SC2086
+        # shellcheck disable=SC2086,SC2229
         read -r -n1 $_t
         if [[ -z $REPLY ]]; then
             printf "$default\n"; break
@@ -261,7 +262,7 @@ tee_stderr () {
     if [[ -n $1 ]] ; then _t="$1"; fi
 
     (while read -r line; do
-         # shellcheck disable=SC2086
+         # shellcheck disable=SC2086,SC2229
          sleep $_t
          echo -e "$line" >&2
          echo "$line"
@@ -367,7 +368,7 @@ choose_one() {
         printf "$1 [${_BGreen}$default${_creset}] "
 
         if (( 10 > max )); then
-            # shellcheck disable=SC2086
+            # shellcheck disable=SC2086,SC2229
             read -r -n1 $_t
         else
             # shellcheck disable=SC2086,SC2229
@@ -708,6 +709,7 @@ pyenv.uninstall() {
 	pyenv.cmd python setup.py develop --uninstall 2>&1 \
             | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
     else
+        # shellcheck disable=SC2086
 	pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \
             | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
     fi

utils/lib_sxng_static.sh

@@ -43,7 +43,7 @@ is.static.build.commit() {
 
     # check all files of the commit belongs to $STATIC_BUILT_PATHS
     commit_files=$(git diff-tree --no-commit-id --name-only -r "${commit_sha}")
-    for i in ${STATIC_BUILT_PATHS[*]}; do
+    for i in "${STATIC_BUILT_PATHS[@]}"; do
         # remove files of ${STATIC_BUILT_PATHS}
         commit_files=$(echo "${commit_files}" | grep -v "^${i}")
     done

utils/lxc.sh

@@ -4,8 +4,6 @@
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
-source "${REPO_ROOT}/utils/brand.env"
 
 # load environment of the LXC suite
 LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}"

utils/morty.sh

@@ -3,8 +3,6 @@
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
-source "${REPO_ROOT}/utils/brand.env"
 
 # ----------------------------------------------------------------------------
 # config

utils/searx.sh

@@ -4,8 +4,6 @@
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
-source "${REPO_ROOT}/utils/brand.env"
 
 # ----------------------------------------------------------------------------
 # config

utils/searxng.sh

@@ -9,8 +9,8 @@ SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}"
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 # shellcheck source=utils/lib_redis.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh"
-# shellcheck source=utils/brand.env
-source "${REPO_ROOT}/utils/brand.env"
+# shellcheck source=utils/brand.sh
+source "${REPO_ROOT}/utils/brand.sh"
 
 SERVICE_NAME="searxng"
 SERVICE_USER="searxng"
@@ -159,7 +159,7 @@ searxng.instance.env() {
         echo "  SEARXNG_INTERNAL_HTTP: ${SEARXNG_INTERNAL_HTTP}"
     fi
     cat <<EOF
-environment ${SEARXNG_SRC}/utils/brand.env:
+environment:
   GIT_URL              : ${GIT_URL}
   GIT_BRANCH           : ${GIT_BRANCH}
   SEARXNG_URL          : ${SEARXNG_URL}
@@ -177,6 +177,7 @@ main() {
 
     case $1 in
         --getenv)  var="$2"; echo "${!var}"; exit 0;;
+        --cmd)  shift; "$@";;
         -h|--help) usage; exit 0;;
         install)
             sudo_or_exit
@@ -526,7 +527,6 @@ searxng.install.settings() {
 
     if ! [[ -f "${SEARXNG_SRC}/.git/config" ]]; then
         die "Before install settings, first install SearXNG."
-        exit 42
     fi
 
     mkdir -p "$(dirname "${SEARXNG_SETTINGS_PATH}")"
@@ -607,8 +607,8 @@ searxng.install.uwsgi.http() {
 
 searxng.install.uwsgi.socket() {
     rst_para "Install ${SEARXNG_UWSGI_APP} using socket at: ${SEARXNG_UWSGI_SOCKET}"
-    mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})"
-    chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})"
+    mkdir -p "$(dirname "${SEARXNG_UWSGI_SOCKET}")"
+    chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname "${SEARXNG_UWSGI_SOCKET}")"
 
     case $DIST_ID-$DIST_VERS in
         fedora-*)

utils/searxng_check.py

@@ -7,6 +7,9 @@ import os
 import sys
 import logging
 import warnings
+from pathlib import Path
+
+repo_root = Path(__file__).resolve().parent.parent
 
 LOG_FORMAT_DEBUG = '%(levelname)-7s %(name)-30.30s: %(message)s'
 logging.basicConfig(level=logging.getLevelName('DEBUG'), format=LOG_FORMAT_DEBUG)
@@ -26,6 +29,12 @@ if os.path.isfile(OLD_SETTING):
         ))
     warnings.warn(msg, DeprecationWarning)
 
+OLD_BRAND_ENV = repo_root / 'utils' / 'brand.env'
+
+if os.path.isfile(OLD_BRAND_ENV):
+    msg = ('%s is no longer needed, remove the file' % (OLD_BRAND_ENV))
+    warnings.warn(msg, DeprecationWarning)
+
 from searx import redisdb, get_setting
 
 if not redisdb.initialize():

utils/templates/etc/uwsgi/apps-archlinux/searxng.ini

@@ -47,6 +47,9 @@ plugin = python
 # default behaviour is for performance reasons.
 enable-threads = true
 
+# Number of workers (usually CPU count)
+workers = ${UWSGI_WORKERS:-%k}
+threads = ${UWSGI_THREADS:-4}
 
 # plugin: python
 # --------------

utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket

@@ -47,6 +47,9 @@ plugin = python
 # default behaviour is for performance reasons.
 enable-threads = true
 
+# Number of workers (usually CPU count)
+workers = ${UWSGI_WORKERS:-%k}
+threads = ${UWSGI_THREADS:-4}
 
 # plugin: python
 # --------------

utils/templates/etc/uwsgi/apps-available/searxng.ini

@@ -50,6 +50,9 @@ plugin = python3,http
 # default behaviour is for performance reasons.
 enable-threads = true
 
+# Number of workers (usually CPU count)
+workers = ${UWSGI_WORKERS:-%k}
+threads = ${UWSGI_THREADS:-4}
 
 # plugin: python
 # --------------

utils/templates/etc/uwsgi/apps-available/searxng.ini:socket

@@ -50,6 +50,9 @@ plugin = python3,http
 # default behaviour is for performance reasons.
 enable-threads = true
 
+# Number of workers (usually CPU count)
+workers = ${UWSGI_WORKERS:-%k}
+threads = ${UWSGI_THREADS:-4}
 
 # plugin: python
 # --------------