mirror of
https://github.com/searxng/searxng
synced 2024-01-01 18:24:07 +00:00
14ba56fd1a
The dependencies and build process are described in the gruntfiles.js and packages.json files. It makes sense to move the clean up to these files too.
515 lines
15 KiB
Bash
Executable File
515 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# shellcheck disable=SC2031
|
|
|
|
# shellcheck source=utils/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
|
|
# shellcheck source=utils/brand.env
|
|
source "${REPO_ROOT}/utils/brand.env"
|
|
source_dot_config
|
|
|
|
# config
|
|
|
|
PYOBJECTS="searx"
|
|
PY_SETUP_EXTRAS='[test]'
|
|
GECKODRIVER_VERSION="v0.28.0"
|
|
# SPHINXOPTS=
|
|
|
|
pylint.FILES() {
|
|
|
|
# List files tagged by comment:
|
|
#
|
|
# # lint: pylint
|
|
#
|
|
# These py files are linted by test.pylint(), all other files are linted by
|
|
# test.pep8()
|
|
|
|
grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searx_extra tests
|
|
}
|
|
|
|
YAMLLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
YAMLLINT_FILES+=("$line")
|
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searx/*.yml')"
|
|
|
|
PYLINT_SEARX_DISABLE_OPTION="\
|
|
I,C,R,\
|
|
W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
|
|
E1136"
|
|
PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
|
|
PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
|
|
|
|
help() {
|
|
cat <<EOF
|
|
buildenv
|
|
rebuild ./utils/brand.env
|
|
babel.compile
|
|
pybabel compile ./searx/translations
|
|
data.*
|
|
all : update searx/languages.py and ./data/*
|
|
languages : update searx/data/engines_languages.json & searx/languages.py
|
|
useragents: update searx/data/useragents.json with the most recent versions of Firefox.
|
|
docs.*
|
|
html : build HTML documentation
|
|
live : autobuild HTML documentation while editing
|
|
gh-pages : deploy on gh-pages branch
|
|
prebuild : build reST include files (./${DOCS_BUILD}/includes)
|
|
clean : clean documentation build
|
|
docker
|
|
build : build docker image
|
|
push : build and push docker image
|
|
gecko.driver
|
|
download & install geckodriver if not already installed (required for
|
|
robot_tests)
|
|
node.*
|
|
env : download & install npm dependencies locally
|
|
clean : drop npm installations
|
|
py.*
|
|
build : Build python packages at ./${PYDIST}
|
|
clean : delete virtualenv and intermediate py files
|
|
pyenv.* :
|
|
install : developer install of searx into virtualenv
|
|
uninstall : uninstall developer installation
|
|
cmd ... : run command ... in virtualenv
|
|
OK : test if virtualenv is OK
|
|
pypi.upload:
|
|
Upload python packages to PyPi (to test use pypi.upload.test)
|
|
test.* :
|
|
pylint : lint PYLINT_FILES, searx/engines, searx & tests
|
|
pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
|
|
unit : run unit tests
|
|
coverage : run unit tests with coverage
|
|
robot : run robot test
|
|
clean : clean intermediate test stuff
|
|
themes.* :
|
|
all : build all themes
|
|
oscar : build oscar theme
|
|
simple : build simple theme
|
|
EOF
|
|
}
|
|
|
|
|
|
if [ "$VERBOSE" = "1" ]; then
|
|
SPHINX_VERBOSE="-v"
|
|
PYLINT_VERBOSE="-v"
|
|
fi
|
|
|
|
# needed by sphinx-docs
|
|
export DOCS_BUILD
|
|
|
|
buildenv() {
|
|
SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
|
|
| prefix_stdout "${_Blue}BUILDENV${_creset} "
|
|
return "${PIPESTATUS[0]}"
|
|
}
|
|
|
|
babel.compile() {
|
|
build_msg BABEL compile
|
|
pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
|
|
dump_return $?
|
|
}
|
|
|
|
|
|
data.all() {
|
|
data.languages
|
|
data.useragents
|
|
data.osm_keys_tags
|
|
build_msg DATA "update searx/data/ahmia_blacklist.txt"
|
|
pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
|
|
build_msg DATA "update searx/data/wikidata_units.json"
|
|
pyenv.cmd python searx_extra/update/update_wikidata_units.py
|
|
build_msg DATA "update searx/data/currencies.json"
|
|
pyenv.cmd python searx_extra/update/update_currencies.py
|
|
}
|
|
|
|
|
|
data.languages() {
|
|
( set -e
|
|
build_msg ENGINES "fetch languages .."
|
|
pyenv.cmd python searx_extra/update/update_languages.py
|
|
build_msg ENGINES "update update searx/languages.py"
|
|
build_msg DATA "update searx/data/engines_languages.json"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
data.useragents() {
|
|
build_msg DATA "update searx/data/useragents.json"
|
|
pyenv.cmd python searx_extra/update/update_firefox_version.py
|
|
dump_return $?
|
|
}
|
|
|
|
data.osm_keys_tags() {
|
|
build_msg DATA "update searx/data/osm_keys_tags.json"
|
|
pyenv.cmd python searx_extra/update/update_osm_keys_tags.py
|
|
dump_return $?
|
|
}
|
|
|
|
docs.prebuild() {
|
|
build_msg DOCS "build ${DOCS_BUILD}/includes"
|
|
(
|
|
set -e
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
mkdir -p "${DOCS_BUILD}/includes"
|
|
./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
|
|
./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
|
|
./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
docker.push() {
|
|
docker.build push
|
|
}
|
|
|
|
docker.buildx() {
|
|
docker.build buildx
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
docker.build() {
|
|
pyenv.install
|
|
|
|
build_msg DOCKER build
|
|
# run installation in a subprocess and activate pyenv
|
|
|
|
# See https://www.shellcheck.net/wiki/SC1001 and others ..
|
|
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
|
|
( set -e
|
|
# shellcheck source=/dev/null
|
|
source "${PY_ENV_BIN}/activate"
|
|
|
|
# Check if it is a git repository
|
|
if [ ! -d .git ]; then
|
|
die 1 "This is not Git repository"
|
|
fi
|
|
if [ ! -x "$(which git)" ]; then
|
|
die 1 "git is not installed"
|
|
fi
|
|
|
|
if ! git remote get-url origin 2> /dev/null; then
|
|
die 1 "there is no remote origin"
|
|
fi
|
|
|
|
# This is a git repository
|
|
|
|
# "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
|
|
# awk to remove the "v" and the "g"
|
|
SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
|
|
|
|
# add the suffix "-dirty" if the repository has uncommited change
|
|
# /!\ HACK for searx/searx: ignore utils/brand.env
|
|
git update-index -q --refresh
|
|
if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
|
|
SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
|
|
fi
|
|
|
|
# Get the last git commit id, will be added to the Searx version (see Dockerfile)
|
|
VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
|
|
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
|
|
|
|
# Check consistency between the git tag and the searx/version.py file
|
|
# /! HACK : parse Python file with bash /!
|
|
# otherwise it is not possible build the docker image without all Python
|
|
# dependencies ( version.py loads __init__.py )
|
|
# SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
|
|
SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
|
|
if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
|
|
err_msg "git tag: $SEARX_GIT_VERSION"
|
|
err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
|
|
die 1 "Inconsistency between the last git tag and the searx/version.py file"
|
|
fi
|
|
|
|
# define the docker image name
|
|
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
|
|
SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
|
|
|
|
BUILD="build"
|
|
if [ "$1" = "buildx" ]; then
|
|
# buildx includes the push option
|
|
CACHE_TAG="${SEARX_IMAGE_NAME}:latest-build-cache"
|
|
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
|
|
shift
|
|
fi
|
|
build_msg DOCKER "Build command: ${BUILD}"
|
|
|
|
# build Docker image
|
|
build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
|
|
# shellcheck disable=SC2086
|
|
docker $BUILD \
|
|
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
|
|
--build-arg GIT_URL="${GIT_URL}" \
|
|
--build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
|
|
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
|
|
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
|
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
|
|
--build-arg LABEL_VCS_URL="${GIT_URL}" \
|
|
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
|
|
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
|
|
-t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
|
|
|
|
if [ "$1" = "push" ]; then
|
|
docker push "${SEARX_IMAGE_NAME}:latest"
|
|
docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
|
|
fi
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
gecko.driver() {
|
|
pyenv.install
|
|
|
|
build_msg INSTALL "gecko.driver"
|
|
# run installation in a subprocess and activate pyenv
|
|
( set -e
|
|
# shellcheck source=/dev/null
|
|
source "${PY_ENV_BIN}/activate"
|
|
|
|
# TODO : check the current geckodriver version
|
|
geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
|
|
set +e
|
|
if [ -z "$NOTFOUND" ]; then
|
|
build_msg INSTALL "geckodriver already installed"
|
|
return
|
|
fi
|
|
PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
|
|
case "$PLATFORM" in
|
|
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
|
|
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
|
|
"windows 32 bit") ARCH="win32";;
|
|
"windows 64 bit") ARCH="win64";;
|
|
"mac 64bit") ARCH="macos";;
|
|
esac
|
|
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
|
|
|
|
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
|
|
|
|
FILE="$(mktemp)"
|
|
wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
|
|
rm -- "$FILE"
|
|
chmod 755 -- "${PY_ENV_BIN}/geckodriver"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
node.env() {
|
|
# shellcheck disable=SC2230
|
|
which npm &> /dev/null || die 1 'node.env - npm is not found!'
|
|
|
|
( set -e
|
|
|
|
build_msg INSTALL "theme: oscar"
|
|
npm --prefix searx/static/themes/oscar install
|
|
|
|
build_msg INSTALL "theme: simple"
|
|
npm --prefix searx/static/themes/simple install
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
node.clean() {
|
|
build_msg CLEAN "locally installed npm dependencies"
|
|
( set -e
|
|
npm --prefix searx/static/themes/oscar run clean
|
|
npm --prefix searx/static/themes/simple run clean
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
py.build() {
|
|
build_msg BUILD "[pylint] python package ${PYDIST}"
|
|
pyenv.cmd python setup.py \
|
|
sdist -d "${PYDIST}" \
|
|
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
|
|
}
|
|
|
|
py.clean() {
|
|
build_msg CLEAN pyenv
|
|
( set -e
|
|
pyenv.drop
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
|
|
find . -name '*.pyc' -exec rm -f {} +
|
|
find . -name '*.pyo' -exec rm -f {} +
|
|
find . -name __pycache__ -exec rm -rf {} +
|
|
)
|
|
}
|
|
|
|
pyenv.check() {
|
|
cat <<EOF
|
|
import yaml
|
|
print('import yaml --> OK')
|
|
EOF
|
|
}
|
|
|
|
pyenv.install() {
|
|
|
|
if ! pyenv.OK; then
|
|
py.clean > /dev/null
|
|
fi
|
|
if pyenv.install.OK > /dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
( set -e
|
|
pyenv
|
|
build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
|
|
"${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
|
|
buildenv
|
|
)
|
|
local exit_val=$?
|
|
if [ ! $exit_val -eq 0 ]; then
|
|
die 42 "error while pip install (${PY_ENV_BIN})"
|
|
fi
|
|
}
|
|
|
|
pyenv.uninstall() {
|
|
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
|
|
pyenv.cmd python setup.py develop --uninstall 2>&1 \
|
|
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
|
|
|
}
|
|
|
|
pypi.upload() {
|
|
py.clean
|
|
py.build
|
|
# https://github.com/pypa/twine
|
|
pyenv.cmd twine upload "${PYDIST}"/*
|
|
}
|
|
|
|
pypi.upload.test() {
|
|
py.clean
|
|
py.build
|
|
pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
|
|
}
|
|
|
|
test.yamllint() {
|
|
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
|
pyenv.cmd yamllint --format parsable "${YAMLLINT_FILES[@]}"
|
|
}
|
|
|
|
test.pylint() {
|
|
# shellcheck disable=SC2086
|
|
( set -e
|
|
build_msg TEST "[pylint] \$PYLINT_FILES"
|
|
pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
"${PYLINT_FILES[@]}"
|
|
|
|
build_msg TEST "[pylint] searx/engines"
|
|
pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--disable="${PYLINT_SEARX_DISABLE_OPTION}" \
|
|
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
|
searx/engines
|
|
|
|
build_msg TEST "[pylint] searx tests"
|
|
pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--disable="${PYLINT_SEARX_DISABLE_OPTION}" \
|
|
--ignore=searx/engines \
|
|
searx tests
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
test.pep8() {
|
|
build_msg TEST 'pycodestyle (formerly pep8)'
|
|
local _exclude=""
|
|
printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
|
|
pyenv.cmd pycodestyle \
|
|
--exclude="searx/static, searx/languages.py, $_exclude " \
|
|
--max-line-length=120 \
|
|
--ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
|
|
searx tests
|
|
dump_return $?
|
|
}
|
|
|
|
test.unit() {
|
|
build_msg TEST 'tests/unit'
|
|
pyenv.cmd python -m nose2 -s tests/unit
|
|
dump_return $?
|
|
}
|
|
|
|
test.coverage() {
|
|
build_msg TEST 'unit test coverage'
|
|
( set -e
|
|
pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
|
pyenv.cmd coverage report
|
|
pyenv.cmd coverage html
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
test.robot() {
|
|
build_msg TEST 'robot'
|
|
gecko.driver
|
|
PYTHONPATH=. pyenv.cmd python searx/testing.py robot
|
|
dump_return $?
|
|
}
|
|
|
|
test.clean() {
|
|
build_msg CLEAN "test stuff"
|
|
rm -rf geckodriver.log .coverage coverage/
|
|
dump_return $?
|
|
}
|
|
|
|
themes.all() {
|
|
( set -e
|
|
node.env
|
|
themes.oscar
|
|
themes.simple
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
themes.oscar() {
|
|
build_msg GRUNT "theme: oscar"
|
|
npm --prefix searx/static/themes/oscar run build
|
|
dump_return $?
|
|
}
|
|
|
|
themes.simple() {
|
|
build_msg GRUNT "theme: simple"
|
|
npm --prefix searx/static/themes/simple run build
|
|
dump_return $?
|
|
}
|
|
|
|
PYLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
PYLINT_FILES+=("$line")
|
|
done <<< "$(pylint.FILES)"
|
|
|
|
# shellcheck disable=SC2119
|
|
main() {
|
|
|
|
local _type
|
|
local cmd="$1"; shift
|
|
|
|
if [ "$cmd" == "" ]; then
|
|
help
|
|
err_msg "missing command"
|
|
return 42
|
|
fi
|
|
|
|
case "$cmd" in
|
|
--getenv) var="$1"; echo "${!var}";;
|
|
--help) help;;
|
|
--*)
|
|
help
|
|
err_msg "unknown option $cmd"
|
|
return 42
|
|
;;
|
|
*)
|
|
_type="$(type -t "$cmd")"
|
|
if [ "$_type" != 'function' ]; then
|
|
err_msg "unknown command $1 / use --help"
|
|
return 42
|
|
else
|
|
"$cmd" "$@"
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|