Merge pull request #688 from dalf/settings_enable_stats

[enh] settings.yml: implement general.enable_metrics

docs/admin/engines/settings.rst

@@ -81,6 +81,9 @@ Global Settings
 ``contact_url``:
   Contact ``mailto:`` address or WEB form.
 
+``enable_metrics``:
+  Enabled by default. Record various anonymous metrics availabled at ``/stats``,
+  ``/stats/errors`` and ``/preferences``.
 
 .. _settings global server:
 

searx/metrics/__init__.py

@@ -9,7 +9,7 @@ from timeit import default_timer
 from operator import itemgetter
 
 from searx.engines import engines
-from .models import HistogramStorage, CounterStorage
+from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
 from .error_recorder import count_error, count_exception, errors_per_engines
 
 __all__ = [
@@ -69,14 +69,18 @@ def counter(*args):
     return counter_storage.get(*args)
 
 
-def initialize(engine_names=None):
+def initialize(engine_names=None, enabled=True):
     """
     Initialize metrics
     """
     global counter_storage, histogram_storage  # pylint: disable=global-statement
 
-    counter_storage = CounterStorage()
-    histogram_storage = HistogramStorage()
+    if enabled:
+        counter_storage = CounterStorage()
+        histogram_storage = HistogramStorage()
+    else:
+        counter_storage = VoidCounterStorage()
+        histogram_storage = HistogramStorage(histogram_class=VoidHistogram)
 
     # max_timeout = max of all the engine.timeout
     max_timeout = 2

searx/metrics/error_recorder.py

@@ -9,7 +9,7 @@ from searx.exceptions import (
     SearxEngineAPIException,
     SearxEngineAccessDeniedException,
 )
-from searx import searx_parent_dir
+from searx import searx_parent_dir, settings
 from searx.engines import engines
 
 
@@ -165,6 +165,8 @@ def get_error_context(framerecords, exception_classname, log_message, log_parame
 
 
 def count_exception(engine_name: str, exc: Exception, secondary: bool = False) -> None:
+    if not settings['general']['enable_metrics']:
+        return
     framerecords = inspect.trace()
     try:
         exception_classname = get_exception_classname(exc)
@@ -178,6 +180,8 @@ def count_exception(engine_name: str, exc: Exception, secondary: bool = False) -
 def count_error(
     engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None, secondary: bool = False
 ) -> None:
+    if not settings['general']['enable_metrics']:
+        return
     framerecords = list(reversed(inspect.stack()[1:]))
     try:
         error_context = get_error_context(framerecords, None, log_message, log_parameters or (), secondary)

searx/metrics/models.py

@@ -102,16 +102,17 @@ class Histogram:
 
 class HistogramStorage:
 
-    __slots__ = 'measures'
+    __slots__ = 'measures', 'histogram_class'
 
-    def __init__(self):
+    def __init__(self, histogram_class=Histogram):
         self.clear()
+        self.histogram_class = histogram_class
 
     def clear(self):
         self.measures = {}
 
     def configure(self, width, size, *args):
-        measure = Histogram(width, size)
+        measure = self.histogram_class(width, size)
         self.measures[args] = measure
         return measure
 
@@ -154,3 +155,13 @@ class CounterStorage:
         logger.debug("Counters:")
         for k in ks:
             logger.debug("- %-60s %s", '|'.join(k), self.counters[k])
+
+
+class VoidHistogram(Histogram):
+    def observe(self, value):
+        pass
+
+
+class VoidCounterStorage(CounterStorage):
+    def add(self, value, *args):
+        pass

searx/search/__init__.py

@@ -24,13 +24,13 @@ from searx.search.checker import initialize as initialize_checker
 logger = logger.getChild('search')
 
 
-def initialize(settings_engines=None, enable_checker=False, check_network=False):
+def initialize(settings_engines=None, enable_checker=False, check_network=False, enable_metrics=True):
     settings_engines = settings_engines or settings['engines']
     load_engines(settings_engines)
     initialize_network(settings_engines, settings['outgoing'])
     if check_network:
         check_network_configuration()
-    initialize_metrics([engine['name'] for engine in settings_engines])
+    initialize_metrics([engine['name'] for engine in settings_engines], enable_metrics)
     initialize_processors(settings_engines)
     if enable_checker:
         initialize_checker()

searx/settings.yml

@@ -2,6 +2,7 @@ general:
   debug: false              # Debug mode, only for development
   instance_name: "SearXNG"  # displayed name
   contact_url: false        # mailto:contact@example.com
+  enable_metrics: true      # record stats
 
 brand:
   new_issue_url: https://github.com/searxng/searxng/issues/new

searx/settings_defaults.py

@@ -142,6 +142,7 @@ SCHEMA = {
         'debug': SettingsValue(bool, False, 'SEARXNG_DEBUG'),
         'instance_name': SettingsValue(str, 'SearXNG'),
         'contact_url': SettingsValue((None, False, str), None),
+        'enable_metrics': SettingsValue(bool, True),
     },
     'brand': {
         'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'),

searx/webapp.py

@@ -1333,7 +1333,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
 # initialize the engines except on the first run of the werkzeug server.
 if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
     plugin_initialize(app)
-    search_initialize(enable_checker=True, check_network=True)
+    search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
 
 
 def run():