mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
Use rye
This commit is contained in:
Alexandre Flament
parent
33c099022b
commit
6672d532fd
16 changed files with 526 additions and 408 deletions
238
utils/lib.sh
238
utils/lib.sh
|
|
@ -527,15 +527,12 @@ service_is_available() {
|
|||
# python
|
||||
# ------
|
||||
|
||||
PY="${PY:=3}"
|
||||
PYTHON="${PYTHON:=python$PY}"
|
||||
PY_ENV="${PY_ENV:=local/py${PY}}"
|
||||
PY_ENV="${PY_ENV:=.venv}"
|
||||
PY_ENV_BIN="${PY_ENV}/bin"
|
||||
PY_ENV_REQ="${PY_ENV_REQ:=${REPO_ROOT}/requirements*.txt}"
|
||||
PY_ENV_SHA256_FILE="${PY_ENV}/requirements.sha256"
|
||||
PY_ENV_REQ="${REPO_ROOT}/pyproject.toml ${REPO_ROOT}/requirements*.lock"
|
||||
|
||||
# List of python packages (folders) or modules (files) installed by command:
|
||||
# pyenv.install
|
||||
PYOBJECTS="${PYOBJECTS:=.}"
|
||||
PYENV_INSTALL_DONE=0
|
||||
|
||||
# folder where the python distribution takes place
|
||||
PYDIST="${PYDIST:=dist}"
|
||||
|
|
@ -543,88 +540,10 @@ PYDIST="${PYDIST:=dist}"
|
|||
# folder where the intermediate build files take place
|
||||
PYBUILD="${PYBUILD:=build/py${PY}}"
|
||||
|
||||
# https://www.python.org/dev/peps/pep-0508/#extras
|
||||
#PY_SETUP_EXTRAS='[develop,test]'
|
||||
PY_SETUP_EXTRAS="${PY_SETUP_EXTRAS:=[develop,test]}"
|
||||
export RYE_VERSION="0.28.0"
|
||||
export RYE_HOME="${REPO_ROOT}/.local"
|
||||
export PATH="$RYE_HOME/shims:$PATH"
|
||||
|
||||
PIP_BOILERPLATE=( pip wheel setuptools )
|
||||
|
||||
# shellcheck disable=SC2120
|
||||
pyenv() {
|
||||
|
||||
# usage: pyenv [vtenv_opts ...]
|
||||
#
|
||||
# vtenv_opts: see 'pip install --help'
|
||||
#
|
||||
# Builds virtualenv with 'requirements*.txt' (PY_ENV_REQ) installed. The
|
||||
# virtualenv will be reused by validating sha256sum of the requirement
|
||||
# files.
|
||||
|
||||
required_commands \
|
||||
sha256sum "${PYTHON}" \
|
||||
|| exit
|
||||
|
||||
local pip_req=()
|
||||
|
||||
if ! pyenv.OK > /dev/null; then
|
||||
rm -f "${PY_ENV}/${PY_ENV_REQ}.sha256"
|
||||
pyenv.drop > /dev/null
|
||||
build_msg PYENV "[virtualenv] installing ${PY_ENV_REQ} into ${PY_ENV}"
|
||||
|
||||
"${PYTHON}" -m venv "$@" "${PY_ENV}"
|
||||
"${PY_ENV_BIN}/python" -m pip install -U "${PIP_BOILERPLATE[@]}"
|
||||
|
||||
for i in ${PY_ENV_REQ}; do
|
||||
pip_req=( "${pip_req[@]}" "-r" "$i" )
|
||||
done
|
||||
|
||||
(
|
||||
[ "$VERBOSE" = "1" ] && set -x
|
||||
# shellcheck disable=SC2086
|
||||
"${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \
|
||||
&& sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256"
|
||||
)
|
||||
fi
|
||||
pyenv.OK
|
||||
}
|
||||
|
||||
_pyenv_OK=''
|
||||
pyenv.OK() {
|
||||
|
||||
# probes if pyenv exists and runs the script from pyenv.check
|
||||
|
||||
[ "$_pyenv_OK" == "OK" ] && return 0
|
||||
|
||||
if [ ! -f "${PY_ENV_BIN}/python" ]; then
|
||||
build_msg PYENV "[virtualenv] missing ${PY_ENV_BIN}/python"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${PY_ENV}/requirements.sha256" ] \
|
||||
|| ! sha256sum -c "${PY_ENV}/requirements.sha256" > /dev/null 2>&1; then
|
||||
build_msg PYENV "[virtualenv] requirements.sha256 failed"
|
||||
sed 's/^/ [virtualenv] - /' <"${PY_ENV}/requirements.sha256"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$VERBOSE" = "1" ]; then
|
||||
pyenv.check \
|
||||
| "${PY_ENV_BIN}/python" 2>&1 \
|
||||
| prefix_stdout "${_Blue}PYENV ${_creset}[check] "
|
||||
else
|
||||
pyenv.check | "${PY_ENV_BIN}/python" 1>/dev/null
|
||||
fi
|
||||
|
||||
local err=${PIPESTATUS[1]}
|
||||
if [ "$err" -ne "0" ]; then
|
||||
build_msg PYENV "[check] python test failed"
|
||||
return "$err"
|
||||
fi
|
||||
|
||||
[ "$VERBOSE" = "1" ] && build_msg PYENV "OK"
|
||||
_pyenv_OK="OK"
|
||||
return 0
|
||||
}
|
||||
|
||||
pyenv.drop() {
|
||||
|
||||
|
|
@ -634,98 +553,6 @@ pyenv.drop() {
|
|||
|
||||
}
|
||||
|
||||
pyenv.check() {
|
||||
|
||||
# Prompts a python script with additional checks. Used by pyenv.OK to check
|
||||
# if virtualenv is ready to install python objects. This function should be
|
||||
# overwritten by the application script.
|
||||
|
||||
local imp=""
|
||||
|
||||
for i in "${PIP_BOILERPLATE[@]}"; do
|
||||
imp="$imp, $i"
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
import ${imp#,*}
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
pyenv.install() {
|
||||
|
||||
if ! pyenv.OK; then
|
||||
py.clean > /dev/null
|
||||
fi
|
||||
if ! pyenv.install.OK > /dev/null; then
|
||||
build_msg PYENV "[install] ${PYOBJECTS}"
|
||||
if ! pyenv.OK >/dev/null; then
|
||||
pyenv
|
||||
fi
|
||||
for i in ${PYOBJECTS}; do
|
||||
build_msg PYENV "[install] pip install -e '$i${PY_SETUP_EXTRAS}'"
|
||||
"${PY_ENV_BIN}/python" -m pip install -e "$i${PY_SETUP_EXTRAS}"
|
||||
done
|
||||
fi
|
||||
pyenv.install.OK
|
||||
}
|
||||
|
||||
_pyenv_install_OK=''
|
||||
pyenv.install.OK() {
|
||||
|
||||
[ "$_pyenv_install_OK" == "OK" ] && return 0
|
||||
|
||||
local imp=""
|
||||
local err=""
|
||||
|
||||
if [ "." = "${PYOBJECTS}" ]; then
|
||||
imp="import $(basename "$(pwd)")"
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
for i in ${PYOBJECTS}; do imp="$imp, $i"; done
|
||||
imp="import ${imp#,*} "
|
||||
fi
|
||||
(
|
||||
[ "$VERBOSE" = "1" ] && set -x
|
||||
"${PY_ENV_BIN}/python" -c "import sys; sys.path.pop(0); $imp;" 2>/dev/null
|
||||
)
|
||||
|
||||
err=$?
|
||||
if [ "$err" -ne "0" ]; then
|
||||
build_msg PYENV "[install] python installation test failed"
|
||||
return "$err"
|
||||
fi
|
||||
|
||||
build_msg PYENV "[install] OK"
|
||||
_pyenv_install_OK="OK"
|
||||
return 0
|
||||
}
|
||||
|
||||
pyenv.uninstall() {
|
||||
|
||||
build_msg PYENV "[uninstall] ${PYOBJECTS}"
|
||||
|
||||
if [ "." = "${PYOBJECTS}" ]; then
|
||||
pyenv.cmd python setup.py develop --uninstall 2>&1 \
|
||||
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \
|
||||
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
pyenv.cmd() {
|
||||
pyenv.install
|
||||
( set -e
|
||||
# shellcheck source=/dev/null
|
||||
source "${PY_ENV_BIN}/activate"
|
||||
[ "$VERBOSE" = "1" ] && set -x
|
||||
"$@"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
pyenv.activate() {
|
||||
pyenv.install
|
||||
|
|
@ -734,6 +561,53 @@ pyenv.activate() {
|
|||
}
|
||||
|
||||
|
||||
_pyenv_OK=''
|
||||
pyenv.OK() {
|
||||
# probes if pyenv exists and runs the script from pyenv.check
|
||||
[ "$_pyenv_OK" == "OK" ] && return 0
|
||||
if [ ! -f "${PY_ENV_SHA256_FILE}" ] || ! sha256sum -c "${PY_ENV_SHA256_FILE}" > /dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
_pyenv_OK="OK"
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
pyenv.install() {
|
||||
if [ ! -d "${RYE_HOME}" ]; then # FIXME : check the RYE_VERSION
|
||||
mkdir -p "${RYE_HOME}"
|
||||
RYE_URL="https://github.com/astral-sh/rye/releases/download/${RYE_VERSION}/rye-x86_64-linux.gz"
|
||||
PYTHON_PATH="$(which python)"
|
||||
RYE_TMP="$RYE_HOME/rye"
|
||||
curl -L "$RYE_URL" | gzip -d > "$RYE_TMP"
|
||||
chmod +x "$RYE_TMP"
|
||||
# use --toolchain "${PYTHON_PATH}" only if the version meet the rye requirement
|
||||
$RYE_TMP self install --no-modify-path --yes
|
||||
rm "$RYE_TMP"
|
||||
# don't use uv because of Sphinx<=7.1.2; python_version == '3.8'
|
||||
# echo "use-uv = true" >> "${RYE_HOME}/config.toml"
|
||||
fi
|
||||
if ! pyenv.OK > /dev/null; then
|
||||
"$RYE_HOME/shims/rye" sync
|
||||
PYENV_INSTALL_DONE=1
|
||||
sha256sum ${PY_ENV_REQ} > "${PY_ENV_SHA256_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
pyenv.uninstall() {
|
||||
# TODO
|
||||
echo "Not implemented"
|
||||
}
|
||||
|
||||
|
||||
rye() {
|
||||
pyenv.install
|
||||
"$RYE_HOME/shims/rye" $@
|
||||
}
|
||||
|
||||
|
||||
|
||||
# Sphinx doc
|
||||
# ----------
|
||||
|
||||
|
|
@ -746,7 +620,7 @@ docs.html() {
|
|||
pyenv.install
|
||||
docs.prebuild
|
||||
# shellcheck disable=SC2086
|
||||
PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-build \
|
||||
PATH="${PY_ENV_BIN}:${PATH}" rye run sphinx-build \
|
||||
${SPHINX_VERBOSE} ${SPHINXOPTS} \
|
||||
-b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"
|
||||
dump_return $?
|
||||
|
|
@ -757,7 +631,7 @@ docs.live() {
|
|||
pyenv.install
|
||||
docs.prebuild
|
||||
# shellcheck disable=SC2086
|
||||
PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-autobuild \
|
||||
PATH="${PY_ENV_BIN}:${PATH}" rye run sphinx-autobuild \
|
||||
${SPHINX_VERBOSE} ${SPHINXOPTS} --open-browser --host 0.0.0.0 \
|
||||
-b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"
|
||||
dump_return $?
|
||||
|
|
|
|||
|
|
@ -14,32 +14,30 @@ EOF
|
|||
data.all() {
|
||||
( set -e
|
||||
|
||||
pyenv.activate
|
||||
data.traits
|
||||
data.useragents
|
||||
data.locales
|
||||
data.locales
|
||||
|
||||
build_msg DATA "update searx/data/osm_keys_tags.json"
|
||||
pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
|
||||
rye run python searxng_extra/update/update_osm_keys_tags.py
|
||||
build_msg DATA "update searx/data/ahmia_blacklist.txt"
|
||||
python searxng_extra/update/update_ahmia_blacklist.py
|
||||
rye run python searxng_extra/update/update_ahmia_blacklist.py
|
||||
build_msg DATA "update searx/data/wikidata_units.json"
|
||||
python searxng_extra/update/update_wikidata_units.py
|
||||
rye run python searxng_extra/update/update_wikidata_units.py
|
||||
build_msg DATA "update searx/data/currencies.json"
|
||||
python searxng_extra/update/update_currencies.py
|
||||
rye run python searxng_extra/update/update_currencies.py
|
||||
build_msg DATA "update searx/data/external_bangs.json"
|
||||
python searxng_extra/update/update_external_bangs.py
|
||||
rye run python searxng_extra/update/update_external_bangs.py
|
||||
build_msg DATA "update searx/data/engine_descriptions.json"
|
||||
python searxng_extra/update/update_engine_descriptions.py
|
||||
rye run python searxng_extra/update/update_engine_descriptions.py
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
data.traits() {
|
||||
( set -e
|
||||
pyenv.activate
|
||||
build_msg DATA "update searx/data/engine_traits.json"
|
||||
python searxng_extra/update/update_engine_traits.py
|
||||
rye run python searxng_extra/update/update_engine_traits.py
|
||||
build_msg ENGINES "update searx/sxng_locales.py"
|
||||
)
|
||||
dump_return $?
|
||||
|
|
@ -47,15 +45,14 @@ data.traits() {
|
|||
|
||||
data.useragents() {
|
||||
build_msg DATA "update searx/data/useragents.json"
|
||||
pyenv.cmd python searxng_extra/update/update_firefox_version.py
|
||||
rye run python searxng_extra/update/update_firefox_version.py
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
data.locales() {
|
||||
( set -e
|
||||
pyenv.activate
|
||||
build_msg DATA "update searx/data/locales.json"
|
||||
python searxng_extra/update/update_locales.py
|
||||
rye run python searxng_extra/update/update_locales.py
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
|
@ -67,7 +64,7 @@ docs.prebuild() {
|
|||
[ "$VERBOSE" = "1" ] && set -x
|
||||
mkdir -p "${DOCS_BUILD}/includes"
|
||||
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
|
||||
pyenv.cmd searxng_extra/docs_prebuild
|
||||
rye run searxng_extra/docs_prebuild
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ EOF
|
|||
|
||||
test.yamllint() {
|
||||
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
||||
pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
|
||||
rye run yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
|
|
@ -23,19 +23,18 @@ test.pylint() {
|
|||
# shellcheck disable=SC2086
|
||||
( set -e
|
||||
build_msg TEST "[pylint] \$PYLINT_FILES"
|
||||
pyenv.activate
|
||||
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||
rye run python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
||||
"${PYLINT_FILES[@]}"
|
||||
|
||||
build_msg TEST "[pylint] searx/engines"
|
||||
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||
rye run python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
||||
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
||||
searx/engines
|
||||
|
||||
build_msg TEST "[pylint] searx tests"
|
||||
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||
rye run python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
||||
--ignore=searx/engines \
|
||||
searx tests
|
||||
|
|
@ -49,35 +48,35 @@ test.pyright() {
|
|||
# We run Pyright in the virtual environment because Pyright
|
||||
# executes "python" to determine the Python version.
|
||||
build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
|
||||
pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
|
||||
| grep -v ".py$" \
|
||||
| grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
|
||||
| grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
|
||||
| grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
|
||||
| grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
|
||||
| grep -v '/engines/.*.py.* - warning: "categories" is not defined'
|
||||
# FIXME
|
||||
# rye run npx --no-install pyright -p pyrightconfig-ci.json \
|
||||
# | grep -v ".py$" \
|
||||
# | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
|
||||
# | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
|
||||
# | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
|
||||
# | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
|
||||
# | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
test.black() {
|
||||
build_msg TEST "[black] \$BLACK_TARGETS"
|
||||
pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
||||
rye run black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
test.unit() {
|
||||
build_msg TEST 'tests/unit'
|
||||
pyenv.cmd python -m nose2 -s tests/unit
|
||||
rye run python -m nose2 -s tests/unit
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
test.coverage() {
|
||||
build_msg TEST 'unit test coverage'
|
||||
( set -e
|
||||
pyenv.activate
|
||||
python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
||||
coverage report
|
||||
coverage html
|
||||
rye run python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
||||
rye run coverage report
|
||||
rye run coverage html
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
|
@ -85,14 +84,14 @@ test.coverage() {
|
|||
test.robot() {
|
||||
build_msg TEST 'robot'
|
||||
gecko.driver
|
||||
PYTHONPATH=. pyenv.cmd python -m tests.robot
|
||||
PYTHONPATH=. rye run python -m tests.robot
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
test.rst() {
|
||||
build_msg TEST "[reST markup] ${RST_FILES[*]}"
|
||||
for rst in "${RST_FILES[@]}"; do
|
||||
pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
|
||||
rye run rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
|
||||
done
|
||||
}
|
||||
|
||||
|
|
@ -100,7 +99,7 @@ test.pybabel() {
|
|||
TEST_BABEL_FOLDER="build/test/pybabel"
|
||||
build_msg TEST "[extract messages] pybabel"
|
||||
mkdir -p "${TEST_BABEL_FOLDER}"
|
||||
pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
|
||||
rye run pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
|
||||
}
|
||||
|
||||
test.clean() {
|
||||
|
|
@ -108,4 +107,3 @@ test.clean() {
|
|||
rm -rf geckodriver.log .coverage coverage/
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ weblate.translations.commit() {
|
|||
exitcode=$?
|
||||
( # make sure to always unlock weblate
|
||||
set -e
|
||||
pyenv.cmd wlc unlock
|
||||
rye run wlc unlock
|
||||
)
|
||||
dump_return $exitcode
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue