create basic integration tests with .env file support for specific engines

.env.test

@@ -0,0 +1,3 @@
+# Add comma separated list of engines to test. Match the file name. If no values provided, all engines will be tested
+# TEST_INTEGRATION_ENGINES=google,bing,yahoo
+TEST_INTEGRATION_ENGINES=

Makefile

@@ -84,7 +84,7 @@ MANAGE += py.build py.clean
 MANAGE += pyenv pyenv.install pyenv.uninstall
 MANAGE += pypi.upload pypi.upload.test
 MANAGE += format.python
-MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean
+MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.int test.clean
 MANAGE += themes.all themes.simple themes.simple.test pygments.less
 MANAGE += static.build.commit static.build.drop static.build.restore
 MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs

manage

@@ -45,6 +45,8 @@ GECKODRIVER_VERSION="v0.33.0"
 # SPHINXOPTS=
 BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
 BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
+# add one or more engines, comma seperated, here to only test a subset of engines
+INTEGRATION_ENGINES="google"
 
 _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
 # set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
@@ -129,7 +131,6 @@ environment ...
 EOF
 }
 
-
 if [ "$VERBOSE" = "1" ]; then
     SPHINX_VERBOSE="-v"
     PYLINT_VERBOSE="-v"
@@ -176,7 +177,8 @@ docker.build() {
 
     # See https://www.shellcheck.net/wiki/SC1001 and others ..
     # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
-    (   set -e
+    (
+        set -e
         pyenv.activate
 
         # Check if it is a git repository
@@ -243,7 +245,8 @@ gecko.driver() {
 
     build_msg INSTALL "gecko.driver"
     # run installation in a subprocess and activate pyenv
-    (   set -e
+    (
+        set -e
         pyenv.activate
 
         INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
@@ -260,7 +263,7 @@ gecko.driver() {
         "windows 64 bit") ARCH="win64" ;;
         "mac 64bit") ARCH="macos" ;;
         esac
-        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
 
         build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
 
@@ -290,7 +293,8 @@ py.build() {
 
 py.clean() {
     build_msg CLEAN pyenv
-    (   set -e
+    (
+        set -e
         pyenv.drop
         [ "$VERBOSE" = "1" ] && set -x
         rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
@@ -316,7 +320,8 @@ pyenv.install() {
         return 0
     fi
 
-    (   set -e
+    (
+        set -e
         pyenv
         build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
         "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
@@ -329,8 +334,8 @@ pyenv.install() {
 
 pyenv.uninstall() {
     build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
-    pyenv.cmd python setup.py develop --uninstall 2>&1 \
-        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+    pyenv.cmd python setup.py develop --uninstall 2>&1 |
+        prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
 
 }
 
@@ -353,7 +358,6 @@ format.python() {
     dump_return $?
 }
 
-
 PYLINT_FILES=()
 while IFS= read -r line; do
     PYLINT_FILES+=("$line")
@@ -363,7 +367,8 @@ done <<< "$(pylint.FILES)"
 main() {
 
     local _type
-    local cmd="$1"; shift
+    local cmd="$1"
+    shift
 
     if [ "$cmd" == "" ]; then
         help
@@ -372,7 +377,10 @@ main() {
     fi
 
     case "$cmd" in
-        --getenv) var="$1"; echo "${!var}";;
+    --getenv)
+        var="$1"
+        echo "${!var}"
+        ;;
     --help) help ;;
     --*)
         help

requirements-dev.txt

@@ -21,3 +21,4 @@ aiounittest==1.4.2
 yamllint==1.33.0
 wlc==1.13
 coloredlogs==15.0.1
+python-dotenv==1.0.0

tests/integration/test_engines.py

@@ -0,0 +1,91 @@
+from os import getenv
+from searx import settings, engines, settings
+from searx.search import SearchQuery, Search, EngineRef, initialize
+from tests import SearxTestCase
+from typing import Tuple, Optional
+import sys
+import logging
+from flask import Flask
+from dotenv import load_dotenv, find_dotenv
+
+logger = logging.getLogger()
+logger.level = logging.INFO
+stream_handler = logging.StreamHandler(sys.stdout)
+logger.addHandler(stream_handler)
+
+SAFESEARCH = 0
+PAGENO = 1
+
+
+def test_single_engine(app: Flask, engine_name: str) -> Tuple[str, Optional[Exception], int]:
+    logger.debug('---------------------------')
+    logger.info(f'Testing Engine: {engine_name}')
+    try:
+        with app.test_request_context():
+            # test your app context code
+            search_query = SearchQuery(
+                'test', [EngineRef(engine_name, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None
+            )
+            search = Search(search_query)
+            info = search.search()
+            return (engine_name, None, info.results_length())
+    except Exception as e:
+        return (engine_name, e, 0)
+    finally:
+        logger.debug('---------------------------')
+
+
+def get_specific_engines() -> list[str]:
+    load_dotenv(find_dotenv("../../.env.test", raise_error_if_not_found=True))
+    integration_engines = getenv("TEST_INTEGRATION_ENGINES")
+    if integration_engines is None or integration_engines == '':
+        return []
+    return integration_engines.split(',')
+
+
+class TestEnginesSingleSearch(SearxTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.app = Flask(__name__)
+        specific_engines = get_specific_engines()
+
+        if len(specific_engines) > 0:
+            cls.engines = [eng for eng in settings['engines'] if eng['name'] in specific_engines]
+        else:
+            cls.engines = settings['engines']
+
+        cls.engine_names = [eng['name'] for eng in cls.engines]
+
+        initialize(cls.engines)
+
+    @classmethod
+    def tearDownClass(cls):
+        settings['outgoing']['using_tor_proxy'] = False
+        settings['outgoing']['extra_proxy_timeout'] = 0
+
+    def test_all_engines(self):
+        results = [test_single_engine(self.app, engine_name) for engine_name in self.engine_names]
+        engines_passed = []
+        engines_exception = []
+        engines_no_results = []
+        for r in results:
+            if r[1] is not None:
+                engines_exception.append(r)
+            elif r[2] <= 0:
+                engines_no_results.append(r)
+            else:
+                engines_passed.append(r)
+
+        def log_results(lst, name: str, level: int):
+            logger.log(level, f'{name}: {len(lst)}')
+            for e in lst:
+                logger.log(level, f'{name}: {e[0]}')
+                if e[1] is not None:
+                    logger.log(level, f'{name}: {e[1]}')
+
+        log_results(engines_passed, 'engines_passed', logging.INFO)
+        log_results(engines_exception, 'engines_exception', logging.ERROR)
+        log_results(engines_no_results, 'engines_no_results', logging.WARN)
+
+        self.assertEqual(len(engines_exception), 0)
+        self.assertEqual(len(engines_no_results), 0)

utils/lib_sxng_test.sh

@@ -9,6 +9,7 @@ test.:
   coverage  : run unit tests with coverage
   robot     : run robot test
   rst       : test .rst files incl. README.rst
+  int       : run integraiton tests on engines
   clean     : clean intermediate test stuff
 EOF
 }
@@ -21,7 +22,8 @@ test.yamllint() {
 
 test.pylint() {
     # shellcheck disable=SC2086
-    (   set -e
+    (
+        set -e
         build_msg TEST "[pylint] \$PYLINT_FILES"
         pyenv.activate
         python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
@@ -49,13 +51,13 @@ test.pyright() {
     # We run Pyright in the virtual environment because Pyright
     # executes "python" to determine the Python version.
     build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
-    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
-        | grep -v ".py$" \
-        | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
-        | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
-        | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
+    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json |
+        grep -v ".py$" |
+        grep -v '/engines/.*.py.* - warning: "logger" is not defined' |
+        grep -v '/plugins/.*.py.* - error: "logger" is not defined' |
+        grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' |
+        grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' |
+        grep -v '/engines/.*.py.* - warning: "categories" is not defined'
     dump_return $?
 }
 
@@ -73,7 +75,8 @@ test.unit() {
 
 test.coverage() {
     build_msg TEST 'unit test coverage'
-    (   set -e
+    (
+        set -e
         pyenv.activate
         python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
         coverage report
@@ -103,9 +106,14 @@ test.pybabel() {
     pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
 }
 
+test.int() {
+    build_msg TEST 'tests/integration'
+    pyenv.cmd python -m nose2 -s tests/integration
+    dump_return $?
+}
+
 test.clean() {
     build_msg CLEAN "test stuff"
     rm -rf geckodriver.log .coverage coverage/
     dump_return $?
 }
-