mirror of https://github.com/searxng/searxng.git
[mod] searx.metrics & searx.search: use the engine loggers
metrics & processors use the engine logger
This commit is contained in:
parent
76e0f6807c
commit
b513917ef9
|
@ -5,7 +5,8 @@ from urllib.parse import urlparse
|
||||||
from httpx import HTTPError, HTTPStatusError
|
from httpx import HTTPError, HTTPStatusError
|
||||||
from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException,
|
from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException,
|
||||||
SearxEngineAccessDeniedException)
|
SearxEngineAccessDeniedException)
|
||||||
from searx import logger, searx_parent_dir
|
from searx import searx_parent_dir
|
||||||
|
from searx.engines import engines
|
||||||
|
|
||||||
|
|
||||||
errors_per_engines = {}
|
errors_per_engines = {}
|
||||||
|
@ -47,7 +48,7 @@ class ErrorContext:
|
||||||
def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
|
def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
|
||||||
errors_for_engine = errors_per_engines.setdefault(engine_name, {})
|
errors_for_engine = errors_per_engines.setdefault(engine_name, {})
|
||||||
errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
|
errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
|
||||||
logger.debug('%s: %s', engine_name, str(error_context))
|
engines[engine_name].logger.warning('%s', str(error_context))
|
||||||
|
|
||||||
|
|
||||||
def get_trace(traces):
|
def get_trace(traces):
|
||||||
|
|
|
@ -147,7 +147,7 @@ class Search:
|
||||||
if th.is_alive():
|
if th.is_alive():
|
||||||
th._timeout = True
|
th._timeout = True
|
||||||
self.result_container.add_unresponsive_engine(th._engine_name, 'timeout')
|
self.result_container.add_unresponsive_engine(th._engine_name, 'timeout')
|
||||||
logger.warning('engine timeout: {0}'.format(th._engine_name))
|
PROCESSORS[th._engine_name].logger.error('engine timeout')
|
||||||
|
|
||||||
def search_standard(self):
|
def search_standard(self):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -65,6 +65,6 @@ def initialize(engine_list):
|
||||||
processor = get_processor(engine, engine_name)
|
processor = get_processor(engine, engine_name)
|
||||||
initialize_processor(processor)
|
initialize_processor(processor)
|
||||||
if processor is None:
|
if processor is None:
|
||||||
logger.error('Error get processor for engine %s', engine_name)
|
engine.logger.error('Error get processor for engine %s', engine_name)
|
||||||
else:
|
else:
|
||||||
PROCESSORS[engine_name] = processor
|
PROCESSORS[engine_name] = processor
|
||||||
|
|
|
@ -9,8 +9,8 @@ import threading
|
||||||
from abc import abstractmethod, ABC
|
from abc import abstractmethod, ABC
|
||||||
from timeit import default_timer
|
from timeit import default_timer
|
||||||
|
|
||||||
from searx import logger
|
from searx import settings, logger
|
||||||
from searx.engines import settings
|
from searx.engines import engines
|
||||||
from searx.network import get_time_for_thread, get_network
|
from searx.network import get_time_for_thread, get_network
|
||||||
from searx.metrics import histogram_observe, counter_inc, count_exception, count_error
|
from searx.metrics import histogram_observe, counter_inc, count_exception, count_error
|
||||||
from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineResponseException
|
from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineResponseException
|
||||||
|
@ -43,7 +43,7 @@ class SuspendedStatus:
|
||||||
self.continuous_errors * settings['search']['ban_time_on_fail'])
|
self.continuous_errors * settings['search']['ban_time_on_fail'])
|
||||||
self.suspend_end_time = default_timer() + suspended_time
|
self.suspend_end_time = default_timer() + suspended_time
|
||||||
self.suspend_reason = suspend_reason
|
self.suspend_reason = suspend_reason
|
||||||
logger.debug('Suspend engine for %i seconds', suspended_time)
|
logger.debug('Suspend for %i seconds', suspended_time)
|
||||||
|
|
||||||
def resume(self):
|
def resume(self):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
|
@ -56,11 +56,12 @@ class SuspendedStatus:
|
||||||
class EngineProcessor(ABC):
|
class EngineProcessor(ABC):
|
||||||
"""Base classes used for all types of reqest processores."""
|
"""Base classes used for all types of reqest processores."""
|
||||||
|
|
||||||
__slots__ = 'engine', 'engine_name', 'lock', 'suspended_status'
|
__slots__ = 'engine', 'engine_name', 'lock', 'suspended_status', 'logger'
|
||||||
|
|
||||||
def __init__(self, engine, engine_name):
|
def __init__(self, engine, engine_name):
|
||||||
self.engine = engine
|
self.engine = engine
|
||||||
self.engine_name = engine_name
|
self.engine_name = engine_name
|
||||||
|
self.logger = engines[engine_name].logger
|
||||||
key = get_network(self.engine_name)
|
key = get_network(self.engine_name)
|
||||||
key = id(key) if key else self.engine_name
|
key = id(key) if key else self.engine_name
|
||||||
self.suspended_status = SUSPENDED_STATUS.setdefault(key, SuspendedStatus())
|
self.suspended_status = SUSPENDED_STATUS.setdefault(key, SuspendedStatus())
|
||||||
|
@ -69,11 +70,11 @@ class EngineProcessor(ABC):
|
||||||
try:
|
try:
|
||||||
self.engine.init(get_engine_from_settings(self.engine_name))
|
self.engine.init(get_engine_from_settings(self.engine_name))
|
||||||
except SearxEngineResponseException as exc:
|
except SearxEngineResponseException as exc:
|
||||||
logger.warn('%s engine: Fail to initialize // %s', self.engine_name, exc)
|
self.logger.warn('Fail to initialize // %s', exc)
|
||||||
except Exception: # pylint: disable=broad-except
|
except Exception: # pylint: disable=broad-except
|
||||||
logger.exception('%s engine: Fail to initialize', self.engine_name)
|
self.logger.exception('Fail to initialize')
|
||||||
else:
|
else:
|
||||||
logger.debug('%s engine: Initialized', self.engine_name)
|
self.logger.debug('Initialized')
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def has_initialize_function(self):
|
def has_initialize_function(self):
|
||||||
|
|
|
@ -5,10 +5,8 @@
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from searx import logger
|
|
||||||
from .abstract import EngineProcessor
|
from .abstract import EngineProcessor
|
||||||
|
|
||||||
logger = logger.getChild('searx.search.processor.offline')
|
|
||||||
|
|
||||||
class OfflineProcessor(EngineProcessor):
|
class OfflineProcessor(EngineProcessor):
|
||||||
"""Processor class used by ``offline`` engines"""
|
"""Processor class used by ``offline`` engines"""
|
||||||
|
@ -24,7 +22,7 @@ class OfflineProcessor(EngineProcessor):
|
||||||
self.extend_container(result_container, start_time, search_results)
|
self.extend_container(result_container, start_time, search_results)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
# do not record the error
|
# do not record the error
|
||||||
logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e))
|
self.logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e))
|
||||||
except Exception as e: # pylint: disable=broad-except
|
except Exception as e: # pylint: disable=broad-except
|
||||||
self.handle_exception(result_container, e)
|
self.handle_exception(result_container, e)
|
||||||
logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))
|
self.logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))
|
||||||
|
|
|
@ -10,7 +10,6 @@ import asyncio
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
import searx.network
|
import searx.network
|
||||||
from searx import logger
|
|
||||||
from searx.utils import gen_useragent
|
from searx.utils import gen_useragent
|
||||||
from searx.exceptions import (
|
from searx.exceptions import (
|
||||||
SearxEngineAccessDeniedException,
|
SearxEngineAccessDeniedException,
|
||||||
|
@ -20,7 +19,6 @@ from searx.exceptions import (
|
||||||
from searx.metrics.error_recorder import count_error
|
from searx.metrics.error_recorder import count_error
|
||||||
from .abstract import EngineProcessor
|
from .abstract import EngineProcessor
|
||||||
|
|
||||||
logger = logger.getChild('searx.search.processor.online')
|
|
||||||
|
|
||||||
def default_request_params():
|
def default_request_params():
|
||||||
"""Default request parameters for ``online`` engines."""
|
"""Default request parameters for ``online`` engines."""
|
||||||
|
@ -146,31 +144,37 @@ class OnlineProcessor(EngineProcessor):
|
||||||
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
|
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
|
||||||
# requests timeout (connect or read)
|
# requests timeout (connect or read)
|
||||||
self.handle_exception(result_container, e, suspend=True)
|
self.handle_exception(result_container, e, suspend=True)
|
||||||
logger.error("engine {0} : HTTP requests timeout"
|
self.logger.error(
|
||||||
"(search duration : {1} s, timeout: {2} s) : {3}"
|
"HTTP requests timeout (search duration : {0} s, timeout: {1} s) : {2}"
|
||||||
.format(self.engine_name, default_timer() - start_time,
|
.format(
|
||||||
|
default_timer() - start_time,
|
||||||
timeout_limit,
|
timeout_limit,
|
||||||
e.__class__.__name__))
|
e.__class__.__name__
|
||||||
|
)
|
||||||
|
)
|
||||||
except (httpx.HTTPError, httpx.StreamError) as e:
|
except (httpx.HTTPError, httpx.StreamError) as e:
|
||||||
# other requests exception
|
# other requests exception
|
||||||
self.handle_exception(result_container, e, suspend=True)
|
self.handle_exception(result_container, e, suspend=True)
|
||||||
logger.exception("engine {0} : requests exception"
|
self.logger.exception(
|
||||||
"(search duration : {1} s, timeout: {2} s) : {3}"
|
"requests exception (search duration : {0} s, timeout: {1} s) : {2}"
|
||||||
.format(self.engine_name, default_timer() - start_time,
|
.format(
|
||||||
|
default_timer() - start_time,
|
||||||
timeout_limit,
|
timeout_limit,
|
||||||
e))
|
e
|
||||||
|
)
|
||||||
|
)
|
||||||
except SearxEngineCaptchaException as e:
|
except SearxEngineCaptchaException as e:
|
||||||
self.handle_exception(result_container, e, suspend=True)
|
self.handle_exception(result_container, e, suspend=True)
|
||||||
logger.exception('engine {0} : CAPTCHA'.format(self.engine_name))
|
self.logger.exception('CAPTCHA')
|
||||||
except SearxEngineTooManyRequestsException as e:
|
except SearxEngineTooManyRequestsException as e:
|
||||||
self.handle_exception(result_container, e, suspend=True)
|
self.handle_exception(result_container, e, suspend=True)
|
||||||
logger.exception('engine {0} : Too many requests'.format(self.engine_name))
|
self.logger.exception('Too many requests')
|
||||||
except SearxEngineAccessDeniedException as e:
|
except SearxEngineAccessDeniedException as e:
|
||||||
self.handle_exception(result_container, e, suspend=True)
|
self.handle_exception(result_container, e, suspend=True)
|
||||||
logger.exception('engine {0} : Searx is blocked'.format(self.engine_name))
|
self.logger.exception('Searx is blocked')
|
||||||
except Exception as e: # pylint: disable=broad-except
|
except Exception as e: # pylint: disable=broad-except
|
||||||
self.handle_exception(result_container, e)
|
self.handle_exception(result_container, e)
|
||||||
logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))
|
self.logger.exception('exception : {0}'.format(e))
|
||||||
|
|
||||||
def get_default_tests(self):
|
def get_default_tests(self):
|
||||||
tests = {}
|
tests = {}
|
||||||
|
|
Loading…
Reference in New Issue