forked from zaclys/searxng
6e5f22e558
Implementations of the *traits* of the engines.
Engine's traits are fetched from the origin engine and stored in a JSON file in
the *data folder*. Most often traits are languages and region codes and their
mapping from SearXNG's representation to the representation in the origin search
engine.
To load traits from the persistence::
searx.enginelib.traits.EngineTraitsMap.from_data()
For new traits new properties can be added to the class::
searx.enginelib.traits.EngineTraits
.. hint::
Implementation is downward compatible to the deprecated *supported_languages
method* from the vintage implementation.
The vintage code is tagged as *deprecated* an can be removed when all engines
has been ported to the *traits method*.
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
192 lines
7.5 KiB
Python
192 lines
7.5 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# lint: pylint
|
|
|
|
"""Abstract base classes for engine request processores.
|
|
|
|
"""
|
|
|
|
import threading
|
|
from abc import abstractmethod, ABC
|
|
from timeit import default_timer
|
|
from typing import Dict, Union
|
|
|
|
from searx import settings, logger
|
|
from searx.engines import engines
|
|
from searx.network import get_time_for_thread, get_network
|
|
from searx.metrics import histogram_observe, counter_inc, count_exception, count_error
|
|
from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineResponseException
|
|
from searx.utils import get_engine_from_settings
|
|
|
|
logger = logger.getChild('searx.search.processor')
|
|
SUSPENDED_STATUS: Dict[Union[int, str], 'SuspendedStatus'] = {}
|
|
|
|
|
|
class SuspendedStatus:
|
|
"""Class to handle suspend state."""
|
|
|
|
__slots__ = 'suspend_end_time', 'suspend_reason', 'continuous_errors', 'lock'
|
|
|
|
def __init__(self):
|
|
self.lock = threading.Lock()
|
|
self.continuous_errors = 0
|
|
self.suspend_end_time = 0
|
|
self.suspend_reason = None
|
|
|
|
@property
|
|
def is_suspended(self):
|
|
return self.suspend_end_time >= default_timer()
|
|
|
|
def suspend(self, suspended_time, suspend_reason):
|
|
with self.lock:
|
|
# update continuous_errors / suspend_end_time
|
|
self.continuous_errors += 1
|
|
if suspended_time is None:
|
|
suspended_time = min(
|
|
settings['search']['max_ban_time_on_fail'],
|
|
self.continuous_errors * settings['search']['ban_time_on_fail'],
|
|
)
|
|
self.suspend_end_time = default_timer() + suspended_time
|
|
self.suspend_reason = suspend_reason
|
|
logger.debug('Suspend for %i seconds', suspended_time)
|
|
|
|
def resume(self):
|
|
with self.lock:
|
|
# reset the suspend variables
|
|
self.continuous_errors = 0
|
|
self.suspend_end_time = 0
|
|
self.suspend_reason = None
|
|
|
|
|
|
class EngineProcessor(ABC):
|
|
"""Base classes used for all types of reqest processores."""
|
|
|
|
__slots__ = 'engine', 'engine_name', 'lock', 'suspended_status', 'logger'
|
|
|
|
def __init__(self, engine, engine_name: str):
|
|
self.engine = engine
|
|
self.engine_name = engine_name
|
|
self.logger = engines[engine_name].logger
|
|
key = get_network(self.engine_name)
|
|
key = id(key) if key else self.engine_name
|
|
self.suspended_status = SUSPENDED_STATUS.setdefault(key, SuspendedStatus())
|
|
|
|
def initialize(self):
|
|
try:
|
|
self.engine.init(get_engine_from_settings(self.engine_name))
|
|
except SearxEngineResponseException as exc:
|
|
self.logger.warn('Fail to initialize // %s', exc)
|
|
except Exception: # pylint: disable=broad-except
|
|
self.logger.exception('Fail to initialize')
|
|
else:
|
|
self.logger.debug('Initialized')
|
|
|
|
@property
|
|
def has_initialize_function(self):
|
|
return hasattr(self.engine, 'init')
|
|
|
|
def handle_exception(self, result_container, exception_or_message, suspend=False):
|
|
# update result_container
|
|
if isinstance(exception_or_message, BaseException):
|
|
exception_class = exception_or_message.__class__
|
|
module_name = getattr(exception_class, '__module__', 'builtins')
|
|
module_name = '' if module_name == 'builtins' else module_name + '.'
|
|
error_message = module_name + exception_class.__qualname__
|
|
else:
|
|
error_message = exception_or_message
|
|
result_container.add_unresponsive_engine(self.engine_name, error_message)
|
|
# metrics
|
|
counter_inc('engine', self.engine_name, 'search', 'count', 'error')
|
|
if isinstance(exception_or_message, BaseException):
|
|
count_exception(self.engine_name, exception_or_message)
|
|
else:
|
|
count_error(self.engine_name, exception_or_message)
|
|
# suspend the engine ?
|
|
if suspend:
|
|
suspended_time = None
|
|
if isinstance(exception_or_message, SearxEngineAccessDeniedException):
|
|
suspended_time = exception_or_message.suspended_time
|
|
self.suspended_status.suspend(suspended_time, error_message) # pylint: disable=no-member
|
|
|
|
def _extend_container_basic(self, result_container, start_time, search_results):
|
|
# update result_container
|
|
result_container.extend(self.engine_name, search_results)
|
|
engine_time = default_timer() - start_time
|
|
page_load_time = get_time_for_thread()
|
|
result_container.add_timing(self.engine_name, engine_time, page_load_time)
|
|
# metrics
|
|
counter_inc('engine', self.engine_name, 'search', 'count', 'successful')
|
|
histogram_observe(engine_time, 'engine', self.engine_name, 'time', 'total')
|
|
if page_load_time is not None:
|
|
histogram_observe(page_load_time, 'engine', self.engine_name, 'time', 'http')
|
|
|
|
def extend_container(self, result_container, start_time, search_results):
|
|
if getattr(threading.current_thread(), '_timeout', False):
|
|
# the main thread is not waiting anymore
|
|
self.handle_exception(result_container, 'timeout', None)
|
|
else:
|
|
# check if the engine accepted the request
|
|
if search_results is not None:
|
|
self._extend_container_basic(result_container, start_time, search_results)
|
|
self.suspended_status.resume()
|
|
|
|
def extend_container_if_suspended(self, result_container):
|
|
if self.suspended_status.is_suspended:
|
|
result_container.add_unresponsive_engine(
|
|
self.engine_name, self.suspended_status.suspend_reason, suspended=True
|
|
)
|
|
return True
|
|
return False
|
|
|
|
def get_params(self, search_query, engine_category):
|
|
"""Returns a set of (see :ref:`request params <engine request arguments>`) or
|
|
``None`` if request is not supported.
|
|
|
|
Not supported conditions (``None`` is returned):
|
|
|
|
- A page-number > 1 when engine does not support paging.
|
|
- A time range when the engine does not support time range.
|
|
"""
|
|
# if paging is not supported, skip
|
|
if search_query.pageno > 1 and not self.engine.paging:
|
|
return None
|
|
|
|
# if time_range is not supported, skip
|
|
if search_query.time_range and not self.engine.time_range_support:
|
|
return None
|
|
|
|
params = {}
|
|
params['category'] = engine_category
|
|
params['pageno'] = search_query.pageno
|
|
params['safesearch'] = search_query.safesearch
|
|
params['time_range'] = search_query.time_range
|
|
params['engine_data'] = search_query.engine_data.get(self.engine_name, {})
|
|
params['searxng_locale'] = search_query.lang
|
|
|
|
# deprecated / vintage --> use params['searxng_locale']
|
|
#
|
|
# Conditions related to engine's traits are implemented in engine.traits
|
|
# module. Don't do 'locale' decissions here in the abstract layer of the
|
|
# search processor, just pass the value from user's choice unchanged to
|
|
# the engine request.
|
|
|
|
if hasattr(self.engine, 'language') and self.engine.language:
|
|
params['language'] = self.engine.language
|
|
else:
|
|
params['language'] = search_query.lang
|
|
|
|
return params
|
|
|
|
@abstractmethod
|
|
def search(self, query, params, result_container, start_time, timeout_limit):
|
|
pass
|
|
|
|
def get_tests(self):
|
|
tests = getattr(self.engine, 'tests', None)
|
|
if tests is None:
|
|
tests = getattr(self.engine, 'additional_tests', {})
|
|
tests.update(self.get_default_tests())
|
|
return tests
|
|
|
|
def get_default_tests(self):
|
|
return {}
|