create basic integration tests with .env file support for specific engines

.env.test

@@ -0,0 +1,3 @@
+# Add comma separated list of engines to test. Match the file name. If no values provided, all engines will be tested
+# TEST_INTEGRATION_ENGINES=google,bing,yahoo
+TEST_INTEGRATION_ENGINES=

Makefile

@@ -84,7 +84,7 @@ MANAGE += py.build py.clean
 MANAGE += pyenv pyenv.install pyenv.uninstall
 MANAGE += pypi.upload pypi.upload.test
 MANAGE += format.python
-MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean
+MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.int test.clean
 MANAGE += themes.all themes.simple themes.simple.test pygments.less
 MANAGE += static.build.commit static.build.drop static.build.restore
 MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs

manage

@@ -45,6 +45,8 @@ GECKODRIVER_VERSION="v0.33.0"
 # SPHINXOPTS=
 BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
 BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
+# add one or more engines, comma seperated, here to only test a subset of engines
+INTEGRATION_ENGINES="google"
 
 _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
 # set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
@@ -66,8 +68,8 @@ pylint.FILES() {
 
 YAMLLINT_FILES=()
 while IFS= read -r line; do
-   YAMLLINT_FILES+=("$line")
-done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
+    YAMLLINT_FILES+=("$line")
+done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
 
 RST_FILES=(
     'README.rst'
@@ -129,7 +131,6 @@ environment ...
 EOF
 }
 
-
 if [ "$VERBOSE" = "1" ]; then
     SPHINX_VERBOSE="-v"
     PYLINT_VERBOSE="-v"
@@ -142,14 +143,14 @@ webapp.run() {
     local parent_proc="$$"
     (
         if [ "${LIVE_THEME}" ]; then
-            ( themes.live "${LIVE_THEME}" )
+            (themes.live "${LIVE_THEME}")
             kill $parent_proc
         fi
-    )&
+    ) &
     (
         sleep 3
         xdg-open http://127.0.0.1:8888/
-    )&
+    ) &
     SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
 }
 
@@ -176,19 +177,20 @@ docker.build() {
 
     # See https://www.shellcheck.net/wiki/SC1001 and others ..
     # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
-    (   set -e
+    (
+        set -e
         pyenv.activate
 
         # Check if it is a git repository
         if [ ! -d .git ]; then
-	    die 1 "This is not Git repository"
+            die 1 "This is not Git repository"
         fi
         if [ ! -x "$(which git)" ]; then
-	    die 1 "git is not installed"
+            die 1 "git is not installed"
         fi
 
-        if ! git remote get-url origin 2> /dev/null; then
-	    die 1 "there is no remote origin"
+        if ! git remote get-url origin 2>/dev/null; then
+            die 1 "there is no remote origin"
         fi
 
         # This is a git repository
@@ -217,22 +219,22 @@ docker.build() {
         build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
         # shellcheck disable=SC2086
         docker $BUILD \
-         --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
-         --build-arg GIT_URL="${GIT_URL}" \
-         --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \
-         --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
-         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
-         --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
-         --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
-         --build-arg LABEL_VCS_URL="${GIT_URL}" \
-         --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
-         --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
-         -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
+            --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
+            --build-arg GIT_URL="${GIT_URL}" \
+            --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \
+            --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
+            --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
+            --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+            --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
+            --build-arg LABEL_VCS_URL="${GIT_URL}" \
+            --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
+            --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
+            -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
 
         if [ "$1" = "push" ]; then
-	        docker push "${SEARXNG_IMAGE_NAME}:latest"
-	        docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}"
-	    fi
+            docker push "${SEARXNG_IMAGE_NAME}:latest"
+            docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}"
+        fi
     )
     dump_return $?
 }
@@ -243,10 +245,11 @@ gecko.driver() {
 
     build_msg INSTALL "gecko.driver"
     # run installation in a subprocess and activate pyenv
-    (   set -e
+    (
+        set -e
         pyenv.activate
 
-        INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
+        INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
         set +e
         if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
             build_msg INSTALL "geckodriver already installed"
@@ -254,13 +257,13 @@ gecko.driver() {
         fi
         PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
         case "$PLATFORM" in
-            "linux 32bit" | "linux2 32bit") ARCH="linux32";;
-            "linux 64bit" | "linux2 64bit") ARCH="linux64";;
-            "windows 32 bit") ARCH="win32";;
-            "windows 64 bit") ARCH="win64";;
-            "mac 64bit") ARCH="macos";;
+        "linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
+        "linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
+        "windows 32 bit") ARCH="win32" ;;
+        "windows 64 bit") ARCH="win64" ;;
+        "mac 64bit") ARCH="macos" ;;
         esac
-        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
 
         build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
 
@@ -284,13 +287,14 @@ pygments.less() {
 py.build() {
     build_msg BUILD "python package ${PYDIST}"
     pyenv.cmd python setup.py \
-              sdist -d "${PYDIST}" \
-              bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
+        sdist -d "${PYDIST}" \
+        bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
 }
 
 py.clean() {
     build_msg CLEAN pyenv
-    (   set -e
+    (
+        set -e
         pyenv.drop
         [ "$VERBOSE" = "1" ] && set -x
         rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
@@ -301,7 +305,7 @@ py.clean() {
 }
 
 pyenv.check() {
-    cat  <<EOF
+    cat <<EOF
 import yaml
 print('import yaml --> OK')
 EOF
@@ -310,13 +314,14 @@ EOF
 pyenv.install() {
 
     if ! pyenv.OK; then
-        py.clean > /dev/null
+        py.clean >/dev/null
     fi
-    if pyenv.install.OK > /dev/null; then
+    if pyenv.install.OK >/dev/null; then
         return 0
     fi
 
-    (   set -e
+    (
+        set -e
         pyenv
         build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
         "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
@@ -329,8 +334,8 @@ pyenv.install() {
 
 pyenv.uninstall() {
     build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
-    pyenv.cmd python setup.py develop --uninstall 2>&1 \
-        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+    pyenv.cmd python setup.py develop --uninstall 2>&1 |
+        prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
 
 }
 
@@ -353,17 +358,17 @@ format.python() {
     dump_return $?
 }
 
-
 PYLINT_FILES=()
 while IFS= read -r line; do
-   PYLINT_FILES+=("$line")
-done <<< "$(pylint.FILES)"
+    PYLINT_FILES+=("$line")
+done <<<"$(pylint.FILES)"
 
 # shellcheck disable=SC2119
 main() {
 
     local _type
-    local cmd="$1"; shift
+    local cmd="$1"
+    shift
 
     if [ "$cmd" == "" ]; then
         help
@@ -372,22 +377,25 @@ main() {
     fi
 
     case "$cmd" in
-        --getenv) var="$1"; echo "${!var}";;
-        --help) help;;
-        --*)
-            help
-            err_msg "unknown option $cmd"
+    --getenv)
+        var="$1"
+        echo "${!var}"
+        ;;
+    --help) help ;;
+    --*)
+        help
+        err_msg "unknown option $cmd"
+        return 42
+        ;;
+    *)
+        _type="$(type -t "$cmd")"
+        if [ "$_type" != 'function' ]; then
+            err_msg "unknown command: $cmd / use --help"
             return 42
-            ;;
-        *)
-            _type="$(type -t "$cmd")"
-            if [ "$_type" != 'function' ]; then
-                err_msg "unknown command: $cmd / use --help"
-                return 42
-            else
-                "$cmd" "$@"
-            fi
-            ;;
+        else
+            "$cmd" "$@"
+        fi
+        ;;
     esac
 }
 

requirements-dev.txt

@@ -21,3 +21,4 @@ aiounittest==1.4.2
 yamllint==1.33.0
 wlc==1.13
 coloredlogs==15.0.1
+python-dotenv==1.0.0

tests/integration/test_engines.py

@@ -0,0 +1,91 @@
+from os import getenv
+from searx import settings, engines, settings
+from searx.search import SearchQuery, Search, EngineRef, initialize
+from tests import SearxTestCase
+from typing import Tuple, Optional
+import sys
+import logging
+from flask import Flask
+from dotenv import load_dotenv, find_dotenv
+
+logger = logging.getLogger()
+logger.level = logging.INFO
+stream_handler = logging.StreamHandler(sys.stdout)
+logger.addHandler(stream_handler)
+
+SAFESEARCH = 0
+PAGENO = 1
+
+
+def test_single_engine(app: Flask, engine_name: str) -> Tuple[str, Optional[Exception], int]:
+    logger.debug('---------------------------')
+    logger.info(f'Testing Engine: {engine_name}')
+    try:
+        with app.test_request_context():
+            # test your app context code
+            search_query = SearchQuery(
+                'test', [EngineRef(engine_name, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None
+            )
+            search = Search(search_query)
+            info = search.search()
+            return (engine_name, None, info.results_length())
+    except Exception as e:
+        return (engine_name, e, 0)
+    finally:
+        logger.debug('---------------------------')
+
+
+def get_specific_engines() -> list[str]:
+    load_dotenv(find_dotenv("../../.env.test", raise_error_if_not_found=True))
+    integration_engines = getenv("TEST_INTEGRATION_ENGINES")
+    if integration_engines is None or integration_engines == '':
+        return []
+    return integration_engines.split(',')
+
+
+class TestEnginesSingleSearch(SearxTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.app = Flask(__name__)
+        specific_engines = get_specific_engines()
+
+        if len(specific_engines) > 0:
+            cls.engines = [eng for eng in settings['engines'] if eng['name'] in specific_engines]
+        else:
+            cls.engines = settings['engines']
+
+        cls.engine_names = [eng['name'] for eng in cls.engines]
+
+        initialize(cls.engines)
+
+    @classmethod
+    def tearDownClass(cls):
+        settings['outgoing']['using_tor_proxy'] = False
+        settings['outgoing']['extra_proxy_timeout'] = 0
+
+    def test_all_engines(self):
+        results = [test_single_engine(self.app, engine_name) for engine_name in self.engine_names]
+        engines_passed = []
+        engines_exception = []
+        engines_no_results = []
+        for r in results:
+            if r[1] is not None:
+                engines_exception.append(r)
+            elif r[2] <= 0:
+                engines_no_results.append(r)
+            else:
+                engines_passed.append(r)
+
+        def log_results(lst, name: str, level: int):
+            logger.log(level, f'{name}: {len(lst)}')
+            for e in lst:
+                logger.log(level, f'{name}: {e[0]}')
+                if e[1] is not None:
+                    logger.log(level, f'{name}: {e[1]}')
+
+        log_results(engines_passed, 'engines_passed', logging.INFO)
+        log_results(engines_exception, 'engines_exception', logging.ERROR)
+        log_results(engines_no_results, 'engines_no_results', logging.WARN)
+
+        self.assertEqual(len(engines_exception), 0)
+        self.assertEqual(len(engines_no_results), 0)

utils/lib_sxng_test.sh

@@ -1,4 +1,4 @@
-test.help(){
+test.help() {
     cat <<EOF
 test.:
   yamllint  : lint YAML files (YAMLLINT_FILES)
@@ -9,6 +9,7 @@ test.:
   coverage  : run unit tests with coverage
   robot     : run robot test
   rst       : test .rst files incl. README.rst
+  int       : run integraiton tests on engines
   clean     : clean intermediate test stuff
 EOF
 }
@@ -21,7 +22,8 @@ test.yamllint() {
 
 test.pylint() {
     # shellcheck disable=SC2086
-    (   set -e
+    (
+        set -e
         build_msg TEST "[pylint] \$PYLINT_FILES"
         pyenv.activate
         python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
@@ -37,8 +39,8 @@ test.pylint() {
         build_msg TEST "[pylint] searx tests"
         python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
             --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
-	    --ignore=searx/engines \
-	    searx tests
+            --ignore=searx/engines \
+            searx tests
     )
     dump_return $?
 }
@@ -49,13 +51,13 @@ test.pyright() {
     # We run Pyright in the virtual environment because Pyright
     # executes "python" to determine the Python version.
     build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
-    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
-        | grep -v ".py$" \
-        | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
-        | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
-        | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
+    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json |
+        grep -v ".py$" |
+        grep -v '/engines/.*.py.* - warning: "logger" is not defined' |
+        grep -v '/plugins/.*.py.* - error: "logger" is not defined' |
+        grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' |
+        grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' |
+        grep -v '/engines/.*.py.* - warning: "categories" is not defined'
     dump_return $?
 }
 
@@ -73,7 +75,8 @@ test.unit() {
 
 test.coverage() {
     build_msg TEST 'unit test coverage'
-    (   set -e
+    (
+        set -e
         pyenv.activate
         python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
         coverage report
@@ -92,7 +95,7 @@ test.robot() {
 test.rst() {
     build_msg TEST "[reST markup] ${RST_FILES[*]}"
     for rst in "${RST_FILES[@]}"; do
-        pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
+        pyenv.cmd rst2html.py --halt error "$rst" >/dev/null || die 42 "fix issue in $rst"
     done
 }
 
@@ -103,9 +106,14 @@ test.pybabel() {
     pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
 }
 
-test.clean() {
-    build_msg CLEAN  "test stuff"
-    rm -rf geckodriver.log .coverage coverage/
+test.int() {
+    build_msg TEST 'tests/integration'
+    pyenv.cmd python -m nose2 -s tests/integration
     dump_return $?
 }
 
+test.clean() {
+    build_msg CLEAN "test stuff"
+    rm -rf geckodriver.log .coverage coverage/
+    dump_return $?
+}