From 3877dbc76455313b1ef02844215b5e5798907567 Mon Sep 17 00:00:00 2001 From: Grant Lanham Date: Sat, 20 Jan 2024 16:52:11 -0500 Subject: [PATCH] create basic integration tests with .env file support for specific engines --- .env.test | 3 + Makefile | 2 +- manage | 130 ++++++++++++++++-------------- requirements-dev.txt | 1 + tests/integration/__init__.py | 0 tests/integration/test_engines.py | 91 +++++++++++++++++++++ utils/lib_sxng_test.sh | 40 +++++---- 7 files changed, 189 insertions(+), 78 deletions(-) create mode 100644 .env.test create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_engines.py diff --git a/.env.test b/.env.test new file mode 100644 index 000000000..f0495f953 --- /dev/null +++ b/.env.test @@ -0,0 +1,3 @@ +# Add comma separated list of engines to test. Match the file name. If no values provided, all engines will be tested +# TEST_INTEGRATION_ENGINES=google,bing,yahoo +TEST_INTEGRATION_ENGINES= diff --git a/Makefile b/Makefile index fa0753fff..987a3ac68 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ MANAGE += py.build py.clean MANAGE += pyenv pyenv.install pyenv.uninstall MANAGE += pypi.upload pypi.upload.test MANAGE += format.python -MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean +MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.int test.clean MANAGE += themes.all themes.simple themes.simple.test pygments.less MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs diff --git a/manage b/manage index 7e60cea3f..807b32d2a 100755 --- a/manage +++ b/manage @@ -45,6 +45,8 @@ GECKODRIVER_VERSION="v0.33.0" # SPHINXOPTS= BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization") BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests") +# add one or more engines, comma seperated, here to only test a subset of engines +INTEGRATION_ENGINES="google" _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock" # set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists. @@ -66,8 +68,8 @@ pylint.FILES() { YAMLLINT_FILES=() while IFS= read -r line; do - YAMLLINT_FILES+=("$line") -done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')" + YAMLLINT_FILES+=("$line") +done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')" RST_FILES=( 'README.rst' @@ -129,7 +131,6 @@ environment ... EOF } - if [ "$VERBOSE" = "1" ]; then SPHINX_VERBOSE="-v" PYLINT_VERBOSE="-v" @@ -142,14 +143,14 @@ webapp.run() { local parent_proc="$$" ( if [ "${LIVE_THEME}" ]; then - ( themes.live "${LIVE_THEME}" ) + (themes.live "${LIVE_THEME}") kill $parent_proc fi - )& + ) & ( sleep 3 xdg-open http://127.0.0.1:8888/ - )& + ) & SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp } @@ -176,19 +177,20 @@ docker.build() { # See https://www.shellcheck.net/wiki/SC1001 and others .. # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 - ( set -e + ( + set -e pyenv.activate # Check if it is a git repository if [ ! -d .git ]; then - die 1 "This is not Git repository" + die 1 "This is not Git repository" fi if [ ! -x "$(which git)" ]; then - die 1 "git is not installed" + die 1 "git is not installed" fi - if ! git remote get-url origin 2> /dev/null; then - die 1 "there is no remote origin" + if ! git remote get-url origin 2>/dev/null; then + die 1 "there is no remote origin" fi # This is a git repository @@ -217,22 +219,22 @@ docker.build() { build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" # shellcheck disable=SC2086 docker $BUILD \ - --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ - --build-arg GIT_URL="${GIT_URL}" \ - --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ - --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ - --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ - --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ - --build-arg LABEL_VCS_URL="${GIT_URL}" \ - --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ - --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ - -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . + --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ + --build-arg GIT_URL="${GIT_URL}" \ + --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ + --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ + --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ + --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ + --build-arg LABEL_VCS_URL="${GIT_URL}" \ + --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ + --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ + -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . if [ "$1" = "push" ]; then - docker push "${SEARXNG_IMAGE_NAME}:latest" - docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" - fi + docker push "${SEARXNG_IMAGE_NAME}:latest" + docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" + fi ) dump_return $? } @@ -243,10 +245,11 @@ gecko.driver() { build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv - ( set -e + ( + set -e pyenv.activate - INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" + INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" set +e if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then build_msg INSTALL "geckodriver already installed" @@ -254,13 +257,13 @@ gecko.driver() { fi PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in - "linux 32bit" | "linux2 32bit") ARCH="linux32";; - "linux 64bit" | "linux2 64bit") ARCH="linux64";; - "windows 32 bit") ARCH="win32";; - "windows 64 bit") ARCH="win64";; - "mac 64bit") ARCH="macos";; + "linux 32bit" | "linux2 32bit") ARCH="linux32" ;; + "linux 64bit" | "linux2 64bit") ARCH="linux64" ;; + "windows 32 bit") ARCH="win32" ;; + "windows 64 bit") ARCH="win64" ;; + "mac 64bit") ARCH="macos" ;; esac - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz" build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" @@ -284,13 +287,14 @@ pygments.less() { py.build() { build_msg BUILD "python package ${PYDIST}" pyenv.cmd python setup.py \ - sdist -d "${PYDIST}" \ - bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" + sdist -d "${PYDIST}" \ + bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv - ( set -e + ( + set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info @@ -301,7 +305,7 @@ py.clean() { } pyenv.check() { - cat < OK') EOF @@ -310,13 +314,14 @@ EOF pyenv.install() { if ! pyenv.OK; then - py.clean > /dev/null + py.clean >/dev/null fi - if pyenv.install.OK > /dev/null; then + if pyenv.install.OK >/dev/null; then return 0 fi - ( set -e + ( + set -e pyenv build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}" @@ -329,8 +334,8 @@ pyenv.install() { pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" - pyenv.cmd python setup.py develop --uninstall 2>&1 \ - | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + pyenv.cmd python setup.py develop --uninstall 2>&1 | + prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } @@ -353,17 +358,17 @@ format.python() { dump_return $? } - PYLINT_FILES=() while IFS= read -r line; do - PYLINT_FILES+=("$line") -done <<< "$(pylint.FILES)" + PYLINT_FILES+=("$line") +done <<<"$(pylint.FILES)" # shellcheck disable=SC2119 main() { local _type - local cmd="$1"; shift + local cmd="$1" + shift if [ "$cmd" == "" ]; then help @@ -372,22 +377,25 @@ main() { fi case "$cmd" in - --getenv) var="$1"; echo "${!var}";; - --help) help;; - --*) - help - err_msg "unknown option $cmd" + --getenv) + var="$1" + echo "${!var}" + ;; + --help) help ;; + --*) + help + err_msg "unknown option $cmd" + return 42 + ;; + *) + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + err_msg "unknown command: $cmd / use --help" return 42 - ;; - *) - _type="$(type -t "$cmd")" - if [ "$_type" != 'function' ]; then - err_msg "unknown command: $cmd / use --help" - return 42 - else - "$cmd" "$@" - fi - ;; + else + "$cmd" "$@" + fi + ;; esac } diff --git a/requirements-dev.txt b/requirements-dev.txt index f22ba25dd..181dd3843 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -21,3 +21,4 @@ aiounittest==1.4.2 yamllint==1.33.0 wlc==1.13 coloredlogs==15.0.1 +python-dotenv==1.0.0 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_engines.py b/tests/integration/test_engines.py new file mode 100644 index 000000000..cdd006411 --- /dev/null +++ b/tests/integration/test_engines.py @@ -0,0 +1,91 @@ +from os import getenv +from searx import settings, engines, settings +from searx.search import SearchQuery, Search, EngineRef, initialize +from tests import SearxTestCase +from typing import Tuple, Optional +import sys +import logging +from flask import Flask +from dotenv import load_dotenv, find_dotenv + +logger = logging.getLogger() +logger.level = logging.INFO +stream_handler = logging.StreamHandler(sys.stdout) +logger.addHandler(stream_handler) + +SAFESEARCH = 0 +PAGENO = 1 + + +def test_single_engine(app: Flask, engine_name: str) -> Tuple[str, Optional[Exception], int]: + logger.debug('---------------------------') + logger.info(f'Testing Engine: {engine_name}') + try: + with app.test_request_context(): + # test your app context code + search_query = SearchQuery( + 'test', [EngineRef(engine_name, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None + ) + search = Search(search_query) + info = search.search() + return (engine_name, None, info.results_length()) + except Exception as e: + return (engine_name, e, 0) + finally: + logger.debug('---------------------------') + + +def get_specific_engines() -> list[str]: + load_dotenv(find_dotenv("../../.env.test", raise_error_if_not_found=True)) + integration_engines = getenv("TEST_INTEGRATION_ENGINES") + if integration_engines is None or integration_engines == '': + return [] + return integration_engines.split(',') + + +class TestEnginesSingleSearch(SearxTestCase): + @classmethod + def setUpClass(cls): + cls.app = Flask(__name__) + specific_engines = get_specific_engines() + + if len(specific_engines) > 0: + cls.engines = [eng for eng in settings['engines'] if eng['name'] in specific_engines] + else: + cls.engines = settings['engines'] + + cls.engine_names = [eng['name'] for eng in cls.engines] + + initialize(cls.engines) + + @classmethod + def tearDownClass(cls): + settings['outgoing']['using_tor_proxy'] = False + settings['outgoing']['extra_proxy_timeout'] = 0 + + def test_all_engines(self): + results = [test_single_engine(self.app, engine_name) for engine_name in self.engine_names] + engines_passed = [] + engines_exception = [] + engines_no_results = [] + for r in results: + if r[1] is not None: + engines_exception.append(r) + elif r[2] <= 0: + engines_no_results.append(r) + else: + engines_passed.append(r) + + def log_results(lst, name: str, level: int): + logger.log(level, f'{name}: {len(lst)}') + for e in lst: + logger.log(level, f'{name}: {e[0]}') + if e[1] is not None: + logger.log(level, f'{name}: {e[1]}') + + log_results(engines_passed, 'engines_passed', logging.INFO) + log_results(engines_exception, 'engines_exception', logging.ERROR) + log_results(engines_no_results, 'engines_no_results', logging.WARN) + + self.assertEqual(len(engines_exception), 0) + self.assertEqual(len(engines_no_results), 0) diff --git a/utils/lib_sxng_test.sh b/utils/lib_sxng_test.sh index 41a20d86f..5d4514456 100644 --- a/utils/lib_sxng_test.sh +++ b/utils/lib_sxng_test.sh @@ -1,4 +1,4 @@ -test.help(){ +test.help() { cat < /dev/null || die 42 "fix issue in $rst" + pyenv.cmd rst2html.py --halt error "$rst" >/dev/null || die 42 "fix issue in $rst" done } @@ -103,9 +106,14 @@ test.pybabel() { pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx } -test.clean() { - build_msg CLEAN "test stuff" - rm -rf geckodriver.log .coverage coverage/ +test.int() { + build_msg TEST 'tests/integration' + pyenv.cmd python -m nose2 -s tests/integration dump_return $? } +test.clean() { + build_msg CLEAN "test stuff" + rm -rf geckodriver.log .coverage coverage/ + dump_return $? +}