mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
bce9669d74
Allow propogating specific files to format for black Update build messages for black update hook Update hook further, add related doc
377 lines
11 KiB
Bash
Executable file
377 lines
11 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
# shellcheck disable=SC2034
|
|
main_cmd="$(basename "$0")"
|
|
|
|
# shellcheck source=utils/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
|
|
|
|
# shellcheck source=utils/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
|
|
|
|
# shellcheck source=utils/lib_sxng_data.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh"
|
|
|
|
# shellcheck source=utils/lib_sxng_weblate.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh"
|
|
|
|
# shellcheck source=utils/lib_sxng_static.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh"
|
|
|
|
# shellcheck source=utils/lib_sxng_node.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh"
|
|
|
|
# shellcheck source=utils/lib_sxng_themes.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
|
|
|
|
# shellcheck source=utils/lib_sxng_test.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
|
|
|
|
# shellcheck source=utils/lib_go.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
|
|
|
|
# shellcheck source=utils/lib_redis.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"
|
|
|
|
PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
|
|
|
|
# config
|
|
|
|
PYOBJECTS="searx"
|
|
PY_SETUP_EXTRAS='[test]'
|
|
GECKODRIVER_VERSION="v0.35.0"
|
|
# SPHINXOPTS=
|
|
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
|
|
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
|
|
|
|
_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
|
|
# set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
|
|
if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then
|
|
export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0"
|
|
fi
|
|
|
|
YAMLLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
|
YAMLLINT_FILES+=("$line")
|
|
fi
|
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
|
|
|
|
RST_FILES=(
|
|
'README.rst'
|
|
)
|
|
|
|
help() {
|
|
nvm.help
|
|
cat <<EOF
|
|
webapp.:
|
|
run : run developer instance
|
|
docs.:
|
|
html : build HTML documentation
|
|
live : autobuild HTML documentation while editing
|
|
gh-pages : deploy on gh-pages branch
|
|
prebuild : build reST include files (./${DOCS_BUILD}/includes)
|
|
clean : clean documentation build
|
|
docker.:
|
|
build : build docker image
|
|
push : build and push docker image
|
|
gecko.driver:
|
|
download & install geckodriver if not already installed (required for
|
|
robot_tests)
|
|
redis:
|
|
build : build redis binaries at $(redis._get_dist)
|
|
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
|
|
help : show more redis commands
|
|
py.:
|
|
build : Build python packages at ./${PYDIST}
|
|
clean : delete virtualenv and intermediate py files
|
|
pyenv.:
|
|
install : developer install of SearXNG into virtualenv
|
|
uninstall : uninstall developer installation
|
|
cmd ... : run command ... in virtualenv
|
|
OK : test if virtualenv is OK
|
|
format.:
|
|
python : format Python code source using black
|
|
git.:
|
|
install : Installs developer git utilities, such as hooks
|
|
uninstall : Removes developer git utilities previously installed
|
|
pygments.:
|
|
less : build LESS files for pygments
|
|
EOF
|
|
go.help
|
|
node.help
|
|
weblate.help
|
|
data.help
|
|
test.help
|
|
themes.help
|
|
static.help
|
|
cat <<EOF
|
|
environment ...
|
|
SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
|
|
EOF
|
|
}
|
|
|
|
|
|
if [ "$VERBOSE" = "1" ]; then
|
|
SPHINX_VERBOSE="-v"
|
|
PYLINT_VERBOSE="-v"
|
|
fi
|
|
|
|
# needed by sphinx-docs
|
|
export DOCS_BUILD
|
|
|
|
webapp.run() {
|
|
local parent_proc="$$"
|
|
(
|
|
if [ "${LIVE_THEME}" ]; then
|
|
( themes.live "${LIVE_THEME}" )
|
|
kill $parent_proc
|
|
fi
|
|
)&
|
|
(
|
|
sleep 3
|
|
xdg-open http://127.0.0.1:8888/
|
|
)&
|
|
SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
|
|
}
|
|
|
|
docker.push() {
|
|
docker.build push
|
|
}
|
|
|
|
docker.buildx() {
|
|
docker.build buildx
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
docker.build() {
|
|
pyenv.install
|
|
|
|
local SEARXNG_GIT_VERSION
|
|
local VERSION_GITCOMMIT
|
|
local GITHUB_USER
|
|
local SEARXNG_IMAGE_NAME
|
|
local BUILD
|
|
|
|
build_msg DOCKER build
|
|
# run installation in a subprocess and activate pyenv
|
|
|
|
# See https://www.shellcheck.net/wiki/SC1001 and others ..
|
|
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
|
|
( set -e
|
|
pyenv.activate
|
|
|
|
# Check if it is a git repository
|
|
if [ ! -d .git ]; then
|
|
die 1 "This is not Git repository"
|
|
fi
|
|
if [ ! -x "$(which git)" ]; then
|
|
die 1 "git is not installed"
|
|
fi
|
|
|
|
if ! git remote get-url origin 2> /dev/null; then
|
|
die 1 "there is no remote origin"
|
|
fi
|
|
|
|
# This is a git repository
|
|
git update-index -q --refresh
|
|
python -m searx.version freeze
|
|
eval "$(python -m searx.version)"
|
|
|
|
# Get the last git commit id
|
|
VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2)
|
|
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
|
|
|
|
# define the docker image name
|
|
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
|
|
SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
|
|
|
|
BUILD="build"
|
|
if [ "$1" = "buildx" ]; then
|
|
# buildx includes the push option
|
|
CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache"
|
|
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
|
|
shift
|
|
fi
|
|
build_msg DOCKER "Build command: ${BUILD}"
|
|
|
|
# build Docker image
|
|
build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
|
|
# shellcheck disable=SC2086
|
|
docker $BUILD \
|
|
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
|
|
--build-arg GIT_URL="${GIT_URL}" \
|
|
--build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \
|
|
--build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
|
|
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
|
|
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
|
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
|
|
--build-arg LABEL_VCS_URL="${GIT_URL}" \
|
|
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
|
|
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
|
|
-t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
|
|
|
|
if [ "$1" = "push" ]; then
|
|
docker push "${SEARXNG_IMAGE_NAME}:latest"
|
|
docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}"
|
|
fi
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
gecko.driver() {
|
|
pyenv.install
|
|
|
|
build_msg INSTALL "gecko.driver"
|
|
# run installation in a subprocess and activate pyenv
|
|
( set -e
|
|
pyenv.activate
|
|
|
|
INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
|
|
set +e
|
|
if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
|
|
build_msg INSTALL "geckodriver already installed"
|
|
return
|
|
fi
|
|
PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
|
|
case "$PLATFORM" in
|
|
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
|
|
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
|
|
"windows 32 bit") ARCH="win32";;
|
|
"windows 64 bit") ARCH="win64";;
|
|
"mac 64bit") ARCH="macos";;
|
|
esac
|
|
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
|
|
|
|
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
|
|
|
|
FILE="$(mktemp)"
|
|
wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
|
|
rm -- "$FILE"
|
|
chmod 755 -- "${PY_ENV_BIN}/geckodriver"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
pygments.less() {
|
|
build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
|
|
if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
|
|
build_msg PYGMENTS "building LESS files for pygments failed"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
py.build() {
|
|
build_msg BUILD "python package ${PYDIST}"
|
|
pyenv.cmd python setup.py \
|
|
sdist -d "${PYDIST}" \
|
|
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
|
|
}
|
|
|
|
py.clean() {
|
|
build_msg CLEAN pyenv
|
|
( set -e
|
|
pyenv.drop
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
|
|
find . -name '*.pyc' -exec rm -f {} +
|
|
find . -name '*.pyo' -exec rm -f {} +
|
|
find . -name __pycache__ -exec rm -rf {} +
|
|
)
|
|
}
|
|
|
|
pyenv.check() {
|
|
cat <<EOF
|
|
import yaml
|
|
print('import yaml --> OK')
|
|
EOF
|
|
}
|
|
|
|
pyenv.install() {
|
|
|
|
if ! pyenv.OK; then
|
|
py.clean > /dev/null
|
|
fi
|
|
if pyenv.install.OK > /dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
( set -e
|
|
pyenv
|
|
build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
|
|
"${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
|
|
)
|
|
local exit_val=$?
|
|
if [ ! $exit_val -eq 0 ]; then
|
|
die 42 "error while pip install (${PY_ENV_BIN})"
|
|
fi
|
|
}
|
|
|
|
pyenv.uninstall() {
|
|
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
|
|
pyenv.cmd python setup.py develop --uninstall 2>&1 \
|
|
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
|
|
|
}
|
|
|
|
format.python() {
|
|
if [ -z "$1" ]; then
|
|
build_msg TEST "[format.python] black \$BLACK_OPTIONS \$BLACK_TARGETS"
|
|
pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
|
else
|
|
build_msg TEST "[format.python] black \$BLACK_OPTIONS $@"
|
|
pyenv.cmd black "${BLACK_OPTIONS[@]}" "$@"
|
|
fi
|
|
}
|
|
|
|
git.install() {
|
|
if [ -d ".git/hooks" ]; then
|
|
cp .githooks/pre-commit .git/hooks/pre-commit
|
|
else
|
|
build_msg INSTALL "Could not find '.git/hooks', hooks were not installed"
|
|
fi
|
|
}
|
|
|
|
git.uninstall() {
|
|
if [ -f ".git/hooks/pre-commit" ]; then
|
|
rm .git/hooks/pre-commit
|
|
fi
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
main() {
|
|
|
|
local _type
|
|
local cmd="$1"; shift
|
|
|
|
if [ "$cmd" == "" ]; then
|
|
help
|
|
err_msg "missing command"
|
|
return 42
|
|
fi
|
|
|
|
case "$cmd" in
|
|
--getenv) var="$1"; echo "${!var}";;
|
|
--help) help;;
|
|
--*)
|
|
help
|
|
err_msg "unknown option $cmd"
|
|
return 42
|
|
;;
|
|
*)
|
|
_type="$(type -t "$cmd")"
|
|
if [ "$_type" != 'function' ]; then
|
|
err_msg "unknown command: $cmd / use --help"
|
|
return 42
|
|
else
|
|
"$cmd" "$@"
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|