mirror of https://github.com/searxng/searxng.git
856 lines
25 KiB
Bash
Executable File
856 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
# shellcheck disable=SC2034
|
|
main_cmd="$(basename "$0")"
|
|
|
|
# shellcheck source=utils/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
|
|
|
|
# shellcheck source=utils/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
|
|
|
|
# shellcheck source=utils/lib_static.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
|
|
|
|
# shellcheck source=utils/lib_go.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
|
|
|
|
# shellcheck source=utils/lib_redis.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"
|
|
|
|
PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
|
|
|
|
# config
|
|
|
|
PYOBJECTS="searx"
|
|
PY_SETUP_EXTRAS='[test]'
|
|
GECKODRIVER_VERSION="v0.30.0"
|
|
export NODE_MINIMUM_VERSION="16.13.0"
|
|
# SPHINXOPTS=
|
|
BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization")
|
|
BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
|
|
|
|
_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
|
|
# set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
|
|
if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then
|
|
export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0"
|
|
fi
|
|
|
|
pylint.FILES() {
|
|
|
|
# List files tagged by comment:
|
|
#
|
|
# # lint: pylint
|
|
#
|
|
# These py files are linted by test.pylint()
|
|
|
|
grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
|
|
find . -name searxng.msg
|
|
}
|
|
|
|
YAMLLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
YAMLLINT_FILES+=("$line")
|
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
|
|
|
|
RST_FILES=(
|
|
'README.rst'
|
|
)
|
|
|
|
PYLINT_SEARXNG_DISABLE_OPTION="\
|
|
I,C,R,\
|
|
W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
|
|
E1136"
|
|
PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,logger,categories"
|
|
PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
|
|
|
|
help() {
|
|
nvm.help
|
|
cat <<EOF
|
|
buildenv:
|
|
rebuild ./utils/brand.env
|
|
webapp.:
|
|
run : run developer instance
|
|
weblate.:
|
|
push.translations: push translation changes from SearXNG to Weblate's counterpart
|
|
to.translations: Update 'translations' branch with last additions from Weblate.
|
|
data.:
|
|
all : update searx/languages.py and ./data/*
|
|
languages : update searx/data/engines_languages.json & searx/languages.py
|
|
useragents: update searx/data/useragents.json with the most recent versions of Firefox.
|
|
docs.:
|
|
html : build HTML documentation
|
|
live : autobuild HTML documentation while editing
|
|
gh-pages : deploy on gh-pages branch
|
|
prebuild : build reST include files (./${DOCS_BUILD}/includes)
|
|
clean : clean documentation build
|
|
docker.:
|
|
build : build docker image
|
|
push : build and push docker image
|
|
gecko.driver:
|
|
download & install geckodriver if not already installed (required for
|
|
robot_tests)
|
|
redis:
|
|
build : build redis binaries at $(redis._get_dist)
|
|
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
|
|
help : show more redis commands
|
|
node.:
|
|
env : download & install npm dependencies locally
|
|
clean : drop locally npm installations
|
|
py.:
|
|
build : Build python packages at ./${PYDIST}
|
|
clean : delete virtualenv and intermediate py files
|
|
pyenv.:
|
|
install : developer install of SearXNG into virtualenv
|
|
uninstall : uninstall developer installation
|
|
cmd ... : run command ... in virtualenv
|
|
OK : test if virtualenv is OK
|
|
pypi.upload:
|
|
Upload python packages to PyPi (to test use pypi.upload.test)
|
|
format.:
|
|
python : format Python code source using black
|
|
test.:
|
|
yamllint : lint YAML files (YAMLLINT_FILES)
|
|
pylint : lint PYLINT_FILES, searx/engines, searx & tests
|
|
pyright : static type check of python sources
|
|
black : check black code format
|
|
unit : run unit tests
|
|
coverage : run unit tests with coverage
|
|
robot : run robot test
|
|
rst : test .rst files incl. README.rst
|
|
clean : clean intermediate test stuff
|
|
themes.:
|
|
all : build all themes
|
|
simple : build simple theme
|
|
pygments.:
|
|
less : build LESS files for pygments
|
|
EOF
|
|
go.help
|
|
static_help
|
|
cat <<EOF
|
|
environment ...
|
|
SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
|
|
EOF
|
|
}
|
|
|
|
|
|
if [ "$VERBOSE" = "1" ]; then
|
|
SPHINX_VERBOSE="-v"
|
|
PYLINT_VERBOSE="-v"
|
|
fi
|
|
|
|
# needed by sphinx-docs
|
|
export DOCS_BUILD
|
|
|
|
webapp.run() {
|
|
local parent_proc="$$"
|
|
(
|
|
if [ "${LIVE_THEME}" ]; then
|
|
( themes.live "${LIVE_THEME}" )
|
|
kill $parent_proc
|
|
fi
|
|
)&
|
|
(
|
|
sleep 3
|
|
xdg-open http://127.0.0.1:8888/
|
|
)&
|
|
SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
|
|
}
|
|
|
|
buildenv() {
|
|
|
|
# settings file from repository's working tree are used by default
|
|
SEARXNG_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml"
|
|
|
|
if [ -f /etc/searx/settings.yml ]; then
|
|
err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/"
|
|
fi
|
|
|
|
if [ -r '/etc/searxng/settings.yml' ]; then
|
|
if ask_yn "should settings read from: /etc/searxng/settings.yml"; then
|
|
SEARXNG_SETTINGS_PATH='/etc/searxng/settings.yml'
|
|
fi
|
|
fi
|
|
export SEARXNG_SETTINGS_PATH
|
|
(
|
|
set -e
|
|
SEARXNG_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
|
|
| prefix_stdout "${_Blue}BUILDENV${_creset} "
|
|
)
|
|
return "${PIPESTATUS[0]}"
|
|
}
|
|
|
|
TRANSLATIONS_WORKTREE="$CACHE/translations"
|
|
|
|
weblate.translations.worktree() {
|
|
|
|
# Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
|
# 'translations' from Weblate's counterpart (weblate) of the SearXNG
|
|
# (origin).
|
|
#
|
|
# remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
|
|
|
|
( set -e
|
|
if ! git remote get-url weblate 2> /dev/null; then
|
|
git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
|
|
fi
|
|
if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
|
|
pushd .
|
|
cd "${TRANSLATIONS_WORKTREE}"
|
|
git reset --hard HEAD
|
|
git pull origin translations
|
|
popd
|
|
else
|
|
mkdir -p "${TRANSLATIONS_WORKTREE}"
|
|
git worktree add "${TRANSLATIONS_WORKTREE}" translations
|
|
fi
|
|
)
|
|
}
|
|
|
|
weblate.to.translations() {
|
|
|
|
# Update 'translations' branch of SearXNG (origin) with last additions from
|
|
# Weblate.
|
|
|
|
# 1. Check if Weblate is locked, if not die with error message
|
|
# 2. On Weblate's counterpart (weblate), pull master and translations branch
|
|
# from SearXNG (origin).
|
|
# 3. Commit changes made in a Weblate object on Weblate's counterpart
|
|
# (weblate).
|
|
# 4. In translations worktree, merge changes of branch 'translations' from
|
|
# remote 'weblate' and push it on branch 'translations' of 'origin'
|
|
|
|
( set -e
|
|
pyenv.activate
|
|
if [ "$(wlc lock-status)" != "locked: True" ]; then
|
|
die 1 "weblate must be locked, currently: $(wlc lock-status)"
|
|
fi
|
|
# weblate: commit pending changes
|
|
wlc pull
|
|
wlc commit
|
|
|
|
# get the translations in a worktree
|
|
weblate.translations.worktree
|
|
|
|
pushd "${TRANSLATIONS_WORKTREE}"
|
|
git remote update weblate
|
|
git merge weblate/translations
|
|
git push
|
|
popd
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
weblate.translations.commit() {
|
|
|
|
# Update 'translations' branch of SearXNG (origin) with last additions from
|
|
# Weblate. Copy the changes to the master branch, compile translations and
|
|
# create a commit in the local branch (master)
|
|
|
|
local existing_commit_hash commit_body commit_message exitcode
|
|
( set -e
|
|
pyenv.activate
|
|
# lock change on weblate
|
|
wlc lock
|
|
|
|
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
|
weblate.translations.worktree
|
|
existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
|
|
|
|
# pull weblate commits
|
|
weblate.to.translations
|
|
|
|
# copy the changes to the master branch
|
|
cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
|
|
|
|
# compile translations
|
|
build_msg BABEL 'compile translation catalogs into binary MO files'
|
|
pybabel compile --statistics \
|
|
-d "searx/translations"
|
|
# git add/commit (no push)
|
|
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
|
|
commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
|
|
git add searx/translations
|
|
git commit -m "${commit_message}"
|
|
)
|
|
exitcode=$?
|
|
( # make sure to always unlock weblate
|
|
set -e
|
|
pyenv.cmd wlc unlock
|
|
)
|
|
dump_return $exitcode
|
|
}
|
|
|
|
weblate.push.translations() {
|
|
|
|
# Push *translation changes* from SearXNG (origin) to Weblate's counterpart
|
|
# (weblate).
|
|
|
|
# In branch master of SearXNG (origin) check for meaningful changes in
|
|
# folder 'searx/translations', commit changes on branch 'translations' and
|
|
# at least, pull updated branches on Weblate's counterpart (weblate).
|
|
|
|
# 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
|
# 'translations' from remote 'weblate'.
|
|
# 2. Stop if there is no meaningful change in the 'master' branch (origin),
|
|
# compared to the 'translations' branch (weblate), otherwise ...
|
|
# 3. Update 'translations' branch of SearXNG (origin) with last additions
|
|
# from Weblate.
|
|
# 5. Notify Weblate to pull updated 'master' & 'translations' branch.
|
|
|
|
local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
|
|
exitcode
|
|
messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
|
|
( set -e
|
|
pyenv.activate
|
|
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
|
weblate.translations.worktree
|
|
|
|
# update messages.pot in the master branch
|
|
build_msg BABEL 'extract messages from source files and generate POT file'
|
|
pybabel extract -F babel.cfg \
|
|
-o "${messages_pot}" \
|
|
"searx/"
|
|
|
|
# stop if there is no meaningful change in the master branch
|
|
diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
|
|
git diff -- "searx/translations/messages.pot")
|
|
if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
|
|
build_msg BABEL 'no changes detected, exiting'
|
|
return 42
|
|
fi
|
|
return 0
|
|
)
|
|
exitcode=$?
|
|
if [ "$exitcode" -eq 42 ]; then
|
|
return 0
|
|
fi
|
|
if [ "$exitcode" -gt 0 ]; then
|
|
return $exitcode
|
|
fi
|
|
(
|
|
set -e
|
|
pyenv.activate
|
|
|
|
# lock change on weblate
|
|
# weblate may add commit(s) since the call to "weblate.translations.worktree".
|
|
# this is not a problem because after this line, "weblate.to.translations"
|
|
# calls again "weblate.translations.worktree" which calls "git pull"
|
|
wlc lock
|
|
|
|
# save messages.pot in the translations branch for later
|
|
pushd "${TRANSLATIONS_WORKTREE}"
|
|
git stash push
|
|
popd
|
|
|
|
# merge weblate commits into the translations branch
|
|
weblate.to.translations
|
|
|
|
# restore messages.pot in the translations branch
|
|
pushd "${TRANSLATIONS_WORKTREE}"
|
|
git stash pop
|
|
popd
|
|
|
|
# update messages.po files in the master branch
|
|
build_msg BABEL 'update existing message catalogs from POT file'
|
|
pybabel update -N \
|
|
-i "${messages_pot}" \
|
|
-d "${TRANSLATIONS_WORKTREE}/searx/translations"
|
|
|
|
# git add/commit/push
|
|
last_commit_hash=$(git log -n1 --pretty=format:'%h')
|
|
last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
|
|
|
|
pushd "${TRANSLATIONS_WORKTREE}"
|
|
git add searx/translations
|
|
git commit \
|
|
-m "[translations] update messages.pot and messages.po files" \
|
|
-m "From ${last_commit_detail}"
|
|
git push
|
|
popd
|
|
|
|
# notify weblate to pull updated master & translations branch
|
|
wlc pull
|
|
)
|
|
exitcode=$?
|
|
( # make sure to always unlock weblate
|
|
set -e
|
|
pyenv.activate
|
|
wlc unlock
|
|
)
|
|
dump_return $exitcode
|
|
}
|
|
|
|
data.all() {
|
|
( set -e
|
|
pyenv.activate
|
|
data.languages
|
|
data.useragents
|
|
data.osm_keys_tags
|
|
build_msg DATA "update searx/data/ahmia_blacklist.txt"
|
|
python searxng_extra/update/update_ahmia_blacklist.py
|
|
build_msg DATA "update searx/data/wikidata_units.json"
|
|
python searxng_extra/update/update_wikidata_units.py
|
|
build_msg DATA "update searx/data/currencies.json"
|
|
python searxng_extra/update/update_currencies.py
|
|
)
|
|
}
|
|
|
|
|
|
data.languages() {
|
|
( set -e
|
|
pyenv.activate
|
|
build_msg ENGINES "fetch languages .."
|
|
python searxng_extra/update/update_languages.py
|
|
build_msg ENGINES "update update searx/languages.py"
|
|
build_msg DATA "update searx/data/engines_languages.json"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
data.useragents() {
|
|
build_msg DATA "update searx/data/useragents.json"
|
|
pyenv.cmd python searxng_extra/update/update_firefox_version.py
|
|
dump_return $?
|
|
}
|
|
|
|
data.osm_keys_tags() {
|
|
build_msg DATA "update searx/data/osm_keys_tags.json"
|
|
pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
|
|
dump_return $?
|
|
}
|
|
|
|
docs.prebuild() {
|
|
build_msg DOCS "build ${DOCS_BUILD}/includes"
|
|
(
|
|
set -e
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
mkdir -p "${DOCS_BUILD}/includes"
|
|
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
|
|
pyenv.cmd searxng_extra/docs_prebuild
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
docker.push() {
|
|
docker.build push
|
|
}
|
|
|
|
docker.buildx() {
|
|
docker.build buildx
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
docker.build() {
|
|
pyenv.install
|
|
|
|
local SEARXNG_GIT_VERSION
|
|
local VERSION_GITCOMMIT
|
|
local GITHUB_USER
|
|
local SEARXNG_IMAGE_NAME
|
|
local BUILD
|
|
|
|
build_msg DOCKER build
|
|
# run installation in a subprocess and activate pyenv
|
|
|
|
# See https://www.shellcheck.net/wiki/SC1001 and others ..
|
|
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
|
|
( set -e
|
|
pyenv.activate
|
|
|
|
# Check if it is a git repository
|
|
if [ ! -d .git ]; then
|
|
die 1 "This is not Git repository"
|
|
fi
|
|
if [ ! -x "$(which git)" ]; then
|
|
die 1 "git is not installed"
|
|
fi
|
|
|
|
if ! git remote get-url origin 2> /dev/null; then
|
|
die 1 "there is no remote origin"
|
|
fi
|
|
|
|
# This is a git repository
|
|
git update-index -q --refresh
|
|
python -m searx.version freeze
|
|
eval "$(python -m searx.version)"
|
|
|
|
# Get the last git commit id
|
|
VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2)
|
|
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
|
|
|
|
# define the docker image name
|
|
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
|
|
SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
|
|
|
|
BUILD="build"
|
|
if [ "$1" = "buildx" ]; then
|
|
# buildx includes the push option
|
|
CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache"
|
|
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
|
|
shift
|
|
fi
|
|
build_msg DOCKER "Build command: ${BUILD}"
|
|
|
|
# build Docker image
|
|
build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
|
|
# shellcheck disable=SC2086
|
|
docker $BUILD \
|
|
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
|
|
--build-arg GIT_URL="${GIT_URL}" \
|
|
--build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \
|
|
--build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
|
|
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
|
|
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
|
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
|
|
--build-arg LABEL_VCS_URL="${GIT_URL}" \
|
|
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
|
|
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
|
|
-t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
|
|
|
|
if [ "$1" = "push" ]; then
|
|
docker push "${SEARXNG_IMAGE_NAME}:latest"
|
|
docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}"
|
|
fi
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
gecko.driver() {
|
|
pyenv.install
|
|
|
|
build_msg INSTALL "gecko.driver"
|
|
# run installation in a subprocess and activate pyenv
|
|
( set -e
|
|
pyenv.activate
|
|
|
|
INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
|
|
set +e
|
|
if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
|
|
build_msg INSTALL "geckodriver already installed"
|
|
return
|
|
fi
|
|
PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
|
|
case "$PLATFORM" in
|
|
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
|
|
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
|
|
"windows 32 bit") ARCH="win32";;
|
|
"windows 64 bit") ARCH="win64";;
|
|
"mac 64bit") ARCH="macos";;
|
|
esac
|
|
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
|
|
|
|
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
|
|
|
|
FILE="$(mktemp)"
|
|
wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
|
|
rm -- "$FILE"
|
|
chmod 755 -- "${PY_ENV_BIN}/geckodriver"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
nodejs.ensure() {
|
|
if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
|
|
info_msg "install Node.js by NVM"
|
|
nvm.nodejs
|
|
fi
|
|
}
|
|
|
|
node.env() {
|
|
nodejs.ensure
|
|
( set -e
|
|
|
|
build_msg INSTALL "searx/static/themes/simple/package.json"
|
|
npm --prefix searx/static/themes/simple install
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
node.env.devtools() {
|
|
nodejs.ensure
|
|
build_msg INSTALL "package.json: developer and CI tools"
|
|
npm install
|
|
}
|
|
|
|
node.clean() {
|
|
if ! required_commands npm 2>/dev/null; then
|
|
build_msg CLEAN "npm is not installed / ignore npm dependencies"
|
|
return 0
|
|
fi
|
|
build_msg CLEAN "themes -- locally installed npm dependencies"
|
|
( set -e
|
|
npm --prefix searx/static/themes/simple run clean
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
pygments.less() {
|
|
build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
|
|
if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
|
|
build_msg PYGMENTS "building LESS files for pygments failed"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
py.build() {
|
|
build_msg BUILD "python package ${PYDIST}"
|
|
pyenv.cmd python setup.py \
|
|
sdist -d "${PYDIST}" \
|
|
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
|
|
}
|
|
|
|
py.clean() {
|
|
build_msg CLEAN pyenv
|
|
( set -e
|
|
pyenv.drop
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
|
|
find . -name '*.pyc' -exec rm -f {} +
|
|
find . -name '*.pyo' -exec rm -f {} +
|
|
find . -name __pycache__ -exec rm -rf {} +
|
|
)
|
|
}
|
|
|
|
pyenv.check() {
|
|
cat <<EOF
|
|
import yaml
|
|
print('import yaml --> OK')
|
|
EOF
|
|
}
|
|
|
|
pyenv.install() {
|
|
|
|
if ! pyenv.OK; then
|
|
py.clean > /dev/null
|
|
fi
|
|
if pyenv.install.OK > /dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
( set -e
|
|
pyenv
|
|
build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
|
|
"${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
|
|
buildenv
|
|
)
|
|
local exit_val=$?
|
|
if [ ! $exit_val -eq 0 ]; then
|
|
die 42 "error while pip install (${PY_ENV_BIN})"
|
|
fi
|
|
}
|
|
|
|
pyenv.uninstall() {
|
|
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
|
|
pyenv.cmd python setup.py develop --uninstall 2>&1 \
|
|
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
|
|
|
}
|
|
|
|
pypi.upload() {
|
|
py.clean
|
|
py.build
|
|
# https://github.com/pypa/twine
|
|
pyenv.cmd twine upload "${PYDIST}"/*
|
|
}
|
|
|
|
pypi.upload.test() {
|
|
py.clean
|
|
py.build
|
|
pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
|
|
}
|
|
|
|
format.python() {
|
|
build_msg TEST "[format.python] black \$BLACK_TARGETS"
|
|
pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
|
dump_return $?
|
|
}
|
|
|
|
test.yamllint() {
|
|
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
|
pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
|
|
dump_return $?
|
|
}
|
|
|
|
test.pylint() {
|
|
# shellcheck disable=SC2086
|
|
( set -e
|
|
build_msg TEST "[pylint] \$PYLINT_FILES"
|
|
pyenv.activate
|
|
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
|
"${PYLINT_FILES[@]}"
|
|
|
|
build_msg TEST "[pylint] searx/engines"
|
|
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
|
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
|
searx/engines
|
|
|
|
build_msg TEST "[pylint] searx tests"
|
|
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
|
--ignore=searx/engines \
|
|
searx tests
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
test.pyright() {
|
|
build_msg TEST "[pyright] static type check of python sources"
|
|
node.env.devtools
|
|
# We run Pyright in the virtual environment because Pyright
|
|
# executes "python" to determine the Python version.
|
|
build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
|
|
pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
|
|
| grep -v ".py$" \
|
|
| grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
|
|
| grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
|
|
| grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
|
|
| grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
|
|
| grep -v '/engines/.*.py.* - warning: "categories" is not defined'
|
|
dump_return $?
|
|
}
|
|
|
|
test.black() {
|
|
build_msg TEST "[black] \$BLACK_TARGETS"
|
|
pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
|
dump_return $?
|
|
}
|
|
|
|
test.unit() {
|
|
build_msg TEST 'tests/unit'
|
|
pyenv.cmd python -m nose2 -s tests/unit
|
|
dump_return $?
|
|
}
|
|
|
|
test.coverage() {
|
|
build_msg TEST 'unit test coverage'
|
|
( set -e
|
|
pyenv.activate
|
|
python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
|
coverage report
|
|
coverage html
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
test.robot() {
|
|
build_msg TEST 'robot'
|
|
gecko.driver
|
|
PYTHONPATH=. pyenv.cmd python -m tests.robot
|
|
dump_return $?
|
|
}
|
|
|
|
test.rst() {
|
|
build_msg TEST "[reST markup] ${RST_FILES[*]}"
|
|
for rst in "${RST_FILES[@]}"; do
|
|
pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
|
|
done
|
|
}
|
|
|
|
test.pybabel() {
|
|
TEST_BABEL_FOLDER="build/test/pybabel"
|
|
build_msg TEST "[extract messages] pybabel"
|
|
mkdir -p "${TEST_BABEL_FOLDER}"
|
|
pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
|
|
}
|
|
|
|
test.clean() {
|
|
build_msg CLEAN "test stuff"
|
|
rm -rf geckodriver.log .coverage coverage/
|
|
dump_return $?
|
|
}
|
|
|
|
themes.all() {
|
|
( set -e
|
|
pygments.less
|
|
node.env
|
|
themes.simple
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
themes.live() {
|
|
local LIVE_THEME="${LIVE_THEME:-${1}}"
|
|
case "${LIVE_THEME}" in
|
|
simple)
|
|
theme="searx/static/themes/${LIVE_THEME}"
|
|
;;
|
|
'')
|
|
die_caller 42 "missing theme argument"
|
|
;;
|
|
*)
|
|
die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'"
|
|
;;
|
|
esac
|
|
build_msg GRUNT "theme: $1 (live build)"
|
|
nodejs.ensure
|
|
cd "${theme}"
|
|
{
|
|
npm install
|
|
npm run watch
|
|
} 2>&1 \
|
|
| prefix_stdout "${_Blue}THEME ${1} ${_creset} " \
|
|
| grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |'
|
|
}
|
|
|
|
themes.simple() {
|
|
( set -e
|
|
build_msg GRUNT "theme: simple"
|
|
npm --prefix searx/static/themes/simple run build
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
themes.simple.test() {
|
|
build_msg TEST "theme: simple"
|
|
nodejs.ensure
|
|
npm --prefix searx/static/themes/simple install
|
|
npm --prefix searx/static/themes/simple run test
|
|
dump_return $?
|
|
}
|
|
|
|
PYLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
PYLINT_FILES+=("$line")
|
|
done <<< "$(pylint.FILES)"
|
|
|
|
# shellcheck disable=SC2119
|
|
main() {
|
|
|
|
local _type
|
|
local cmd="$1"; shift
|
|
|
|
if [ "$cmd" == "" ]; then
|
|
help
|
|
err_msg "missing command"
|
|
return 42
|
|
fi
|
|
|
|
case "$cmd" in
|
|
--getenv) var="$1"; echo "${!var}";;
|
|
--help) help;;
|
|
--*)
|
|
help
|
|
err_msg "unknown option $cmd"
|
|
return 42
|
|
;;
|
|
*)
|
|
_type="$(type -t "$cmd")"
|
|
if [ "$_type" != 'function' ]; then
|
|
err_msg "unknown command: $cmd / use --help"
|
|
return 42
|
|
else
|
|
"$cmd" "$@"
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|