[enh] add searx.shared

shared dictionary between the workers (UWSGI or werkzeug)
scheduler: run a task once every x seconds (UWSGI or werkzeug)

dockerfiles/uwsgi.ini

@@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static
 static-expires = /* 864000
 static-gzip-all = True
 offload-threads = %k
+
+# Cache
+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1

searx/shared/__init__.py

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import logging
+
+logger = logging.getLogger('searx.shared')
+
+try:
+    import uwsgi
+except:
+    # no uwsgi
+    from .shared_simple import SimpleSharedDict as SharedDict, schedule
+    logger.info('Use shared_simple implementation')
+else:
+    try:
+        uwsgi.cache_update('dummy', b'dummy')
+        if uwsgi.cache_get('dummy') != b'dummy':
+            raise Exception()
+    except:
+        # uwsgi.ini configuration problem: disable all scheduling
+        logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
+                     'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1')
+        from .shared_simple import SimpleSharedDict as SharedDict
+
+        def schedule(delay, func, *args):
+            pass
+    else:
+        # uwsgi
+        from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
+        logger.info('Use shared_uwsgi implementation')
+
+storage = SharedDict()

searx/shared/shared_abstract.py

@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+class SharedDict:
+
+    def get_int(self, key):
+        pass
+
+    def set_int(self, key, value):
+        pass
+
+    def get_str(self, key):
+        pass
+
+    def set_str(self, key, value):
+        pass

searx/shared/shared_simple.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import threading
+
+from . import shared_abstract
+
+
+class SimpleSharedDict(shared_abstract.SharedDict):
+
+    __slots__ = 'd',
+
+    def __init__(self):
+        self.d = {}
+
+    def get_int(self, key):
+        return self.d.get(key, None)
+
+    def set_int(self, key, value):
+        self.d[key] = value
+
+    def get_str(self, key):
+        return self.d.get(key, None)
+
+    def set_str(self, key, value):
+        self.d[key] = value
+
+
+def schedule(delay, func, *args):
+    def call_later():
+        t = threading.Timer(delay, wrapper)
+        t.daemon = True
+        t.start()
+
+    def wrapper():
+        call_later()
+        func(*args)
+
+    call_later()

searx/shared/shared_uwsgi.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import time
+import uwsgi  # pylint: disable=E0401
+from . import shared_abstract
+
+
+_last_signal = 10
+
+
+class UwsgiCacheSharedDict(shared_abstract.SharedDict):
+
+    def get_int(self, key):
+        value = uwsgi.cache_get(key)
+        if value is None:
+            return value
+        else:
+            return int.from_bytes(value, 'big')
+
+    def set_int(self, key, value):
+        b = value.to_bytes(4, 'big')
+        uwsgi.cache_update(key, b)
+
+    def get_str(self, key):
+        value = uwsgi.cache_get(key)
+        if value is None:
+            return value
+        else:
+            return value.decode('utf-8')
+
+    def set_str(self, key, value):
+        b = value.encode('utf-8')
+        uwsgi.cache_update(key, b)
+
+
+def schedule(delay, func, *args):
+    """
+    Can be implemented using a spooler.
+    https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html
+
+    To make the uwsgi configuration simple, use the alternative implementation.
+    """
+    global _last_signal
+
+    def sighandler(signum):
+        now = int(time.time())
+        uwsgi.lock()
+        try:
+            updating = uwsgi.cache_get('updating')
+            if updating is not None:
+                updating = int.from_bytes(updating, 'big')
+                if now - updating < delay:
+                    return
+            uwsgi.cache_update('updating', now.to_bytes(4, 'big'))
+        finally:
+            uwsgi.unlock()
+        func(*args)
+
+    signal_num = _last_signal
+    _last_signal += 1
+    uwsgi.register_signal(signal_num, 'worker', sighandler)
+    uwsgi.add_timer(signal_num, delay)

utils/templates/etc/uwsgi/apps-archlinux/searx.ini

@@ -82,4 +82,7 @@ http = ${SEARX_INTERNAL_HTTP}
 #   mkdir -p /run/uwsgi/app/searx
 #   chown -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx
 #
-# socket = /run/uwsgi/app/searx/socket
+# socket = /run/uwsgi/app/searx/socket
+
+# Cache
+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1

utils/templates/etc/uwsgi/apps-available/searx.ini

@@ -82,3 +82,6 @@ http = ${SEARX_INTERNAL_HTTP}
 #   chown -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx
 #
 # socket = /run/uwsgi/app/searx/socket
+
+# Cache
+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1