From 036933599bc48a94cc722ad7d207b9de81ff85d2 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 22 Feb 2021 19:38:45 +0100 Subject: [PATCH] [enh] utils/lib.sh - commands pyenv, pyenv.drop pyenv.(un)install Implement a boilerplate to manage performance optimized virtualenv builds. Shell scripts can use (e.g.) 'pyenv.cmd' to execute command in the virtualenv without having to worry about whether and how the environment is provided. :: pyenv.cmd which python ..../local/py3/bin/python pyenv.cmd which pip ..../local/py3/bin/pip If pyenv.cmd released multiple times the installation will only rebuild if the function 'pyenv.OK' fails. Function 'pyenv.OK' make some test to validate that the virtualenv exists and works as expected. The check also fails if requirements listed requirements-dev.txt and requirements.txt has been edited. Among these tests 'pyenv.OK' calls 'pyenv.check' which implements a python script that validate the python installation. Here is an example how a 'pyenv.check' implementation could look like:: pyenv.check() { cat < OK') EOF } Signed-off-by: Markus Heiser --- utils/lib.sh | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 211 insertions(+), 4 deletions(-) diff --git a/utils/lib.sh b/utils/lib.sh index 8ae6bdd44..4475b5149 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -86,7 +86,7 @@ set_terminal_colors() { _Red='\e[0;31m' _Green='\e[0;32m' _Yellow='\e[0;33m' - _Blue='\e[0;34m' + _Blue='\e[0;94m' _Violet='\e[0;35m' _Cyan='\e[0;36m' @@ -95,12 +95,12 @@ set_terminal_colors() { _BRed='\e[1;31m' _BGreen='\e[1;32m' _BYellow='\e[1;33m' - _BBlue='\e[1;34m' + _BBlue='\e[1;94m' _BPurple='\e[1;35m' _BCyan='\e[1;36m' } -if [ ! -p /dev/stdout ]; then +if [ ! -p /dev/stdout ] && [ ! "$TERM" = 'dumb' ] && [ ! "$TERM" = 'unknown' ]; then set_terminal_colors fi @@ -152,6 +152,12 @@ err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } +build_msg() { + local tag="$1 " + shift + echo -e "${_Blue}${tag:0:10}${_creset}$*" +} + clean_stdin() { if [[ $(uname -s) != 'Darwin' ]]; then while read -r -n1 -t 0.1; do : ; done @@ -496,6 +502,203 @@ service_is_available() { return "$exit_val" } +# python +# ------ + +PY="${PY:=3}" +PYTHON="${PYTHON:=python$PY}" +PY_ENV="${PY_ENV:=local/py${PY}}" +PY_ENV_BIN="${PY_ENV}/bin" +PY_ENV_REQ="${PY_ENV_REQ:=${REPO_ROOT}/requirements*.txt}" + +# List of python packages (folders) or modules (files) installed by command: +# pyenv.install +PYOBJECTS="${PYOBJECTS:=.}" + +# folder where the python distribution takes place +PYDIST="${PYDIST:=dist}" + +# folder where the intermediate build files take place +PYBUILD="${PYBUILD:=build/py${PY}}" + +# https://www.python.org/dev/peps/pep-0508/#extras +#PY_SETUP_EXTRAS='[develop,test]' +PY_SETUP_EXTRAS="${PY_SETUP_EXTRAS:=[develop,test]}" + +PIP_BOILERPLATE=( pip wheel setuptools ) + +# shellcheck disable=SC2120 +pyenv() { + + # usage: pyenv [vtenv_opts ...] + # + # vtenv_opts: see 'pip install --help' + # + # Builds virtualenv with 'requirements*.txt' (PY_ENV_REQ) installed. The + # virtualenv will be reused by validating sha256sum of the requirement + # files. + + required_commands \ + sha256sum "${PYTHON}" \ + || exit + + local pip_req=() + + if ! pyenv.OK > /dev/null; then + rm -f "${PY_ENV}/${PY_ENV_REQ}.sha256" + pyenv.drop > /dev/null + build_msg PYENV "[virtualenv] installing ${PY_ENV_REQ} into ${PY_ENV}" + + "${PYTHON}" -m venv "$@" "${PY_ENV}" + "${PY_ENV_BIN}/python" -m pip install -U "${PIP_BOILERPLATE[@]}" + + for i in ${PY_ENV_REQ}; do + pip_req=( "${pip_req[@]}" "-r" "$i" ) + done + + ( + [ "$VERBOSE" = "1" ] && set -x + # shellcheck disable=SC2086 + "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \ + && sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256" + ) + fi + pyenv.OK +} + +_pyenv_OK='' +pyenv.OK() { + + # probes if pyenv exists and runs the script from pyenv.check + + [ "$_pyenv_OK" == "OK" ] && return 0 + + if [ ! -f "${PY_ENV_BIN}/python" ]; then + build_msg PYENV "[virtualenv] missing ${PY_ENV_BIN}/python" + return 1 + fi + + if [ ! -f "${PY_ENV}/requirements.sha256" ] \ + || ! sha256sum --check --status <"${PY_ENV}/requirements.sha256" 2>/dev/null; then + build_msg PYENV "[virtualenv] requirements.sha256 failed" + sed 's/^/ [virtualenv] - /' <"${PY_ENV}/requirements.sha256" + return 1 + fi + + pyenv.check \ + | "${PY_ENV_BIN}/python" 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[check] " + + local err=${PIPESTATUS[1]} + if [ "$err" -ne "0" ]; then + build_msg PYENV "[check] python test failed" + return "$err" + fi + + build_msg PYENV "OK" + _pyenv_OK="OK" + return 0 +} + +pyenv.drop() { + + build_msg PYENV "[virtualenv] drop ${PY_ENV}" + rm -rf "${PY_ENV}" + _pyenv_OK='' + +} + +pyenv.check() { + + # Prompts a python script with additional checks. Used by pyenv.OK to check + # if virtualenv is ready to install python objects. This function should be + # overwritten by the application script. + + local imp="" + + for i in "${PIP_BOILERPLATE[@]}"; do + imp="$imp, $i" + done + + cat < /dev/null + fi + if ! pyenv.install.OK > /dev/null; then + build_msg PYENV "[install] ${PYOBJECTS}" + if ! pyenv.OK >/dev/null; then + pyenv + fi + for i in ${PYOBJECTS}; do + build_msg PYENV "[install] pip install -e '$i${PY_SETUP_EXTRAS}'" + "${PY_ENV_BIN}/python" -m pip install -e "$i${PY_SETUP_EXTRAS}" + done + fi + pyenv.install.OK +} + +_pyenv_install_OK='' +pyenv.install.OK() { + + [ "$_pyenv_install_OK" == "OK" ] && return 0 + + local imp="" + local err="" + + if [ "." = "${PYOBJECTS}" ]; then + imp="import $(basename "$(pwd)")" + else + # shellcheck disable=SC2086 + for i in ${PYOBJECTS}; do imp="$imp, $i"; done + imp="import ${imp#,*} " + fi + ( + [ "$VERBOSE" = "1" ] && set -x + "${PY_ENV_BIN}/python" -c "import sys; sys.path.pop(0); $imp;" 2>/dev/null + ) + + err=$? + if [ "$err" -ne "0" ]; then + build_msg PYENV "[install] python installation test failed" + return "$err" + fi + + build_msg PYENV "[install] OK" + _pyenv_install_OK="OK" + return 0 +} + +pyenv.uninstall() { + + build_msg PYENV "[uninstall] ${PYOBJECTS}" + + if [ "." = "${PYOBJECTS}" ]; then + pyenv.cmd python setup.py develop --uninstall 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + else + pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + fi +} + + +pyenv.cmd() { + pyenv.install + ( set -e + # shellcheck source=/dev/null + source "${PY_ENV_BIN}/activate" + [ "$VERBOSE" = "1" ] && set -x + "$@" + ) +} + # golang # ------ @@ -1250,7 +1453,7 @@ pkg_install() { centos) # shellcheck disable=SC2068 yum install -y $@ - ;; + ;; esac } @@ -1382,6 +1585,10 @@ LXC_ENV_FOLDER= if in_container; then # shellcheck disable=SC2034 LXC_ENV_FOLDER="lxc-env/$(hostname)/" + PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}" + PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}" + PYDIST="${LXC_ENV_FOLDER}${PYDIST}" + PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}" fi lxc_init_container_env() {