[mod] shared.storage: implement a backend to store in the redis DB

- implement shared_redis.RedisCacheSharedDict to store key/value pairs in the
  redis DB.  The default key_prefix is 'SearXNG_SharedDict'.

- simplify searx.shared and import RedisCacheSharedDict, set
  key_prefix to 'SearXNG_storage'

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2022-09-24 19:56:14 +02:00
parent bfd6f61849
commit 4809bf705c
3 changed files with 66 additions and 24 deletions

View file

@ -1,39 +1,48 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""Module implements a :py:obj:`storage`."""
__all__ = ['schedule', 'storage']
import logging import logging
import importlib import importlib
from . import redisdb
from . import shared_redis
from . import shared_simple
logger = logging.getLogger('searx.shared') logger = logging.getLogger('searx.shared')
__all__ = ['SharedDict', 'schedule'] SharedDict = None
schedule = None
if redisdb.init():
SharedDict = shared_redis.RedisCacheSharedDict
logger.info('use redis DB for SharedDict')
try: try:
from . import shared_uwsgi
uwsgi = importlib.import_module('uwsgi') uwsgi = importlib.import_module('uwsgi')
except: uwsgi.cache_update('dummy', b'dummy')
# no uwsgi if uwsgi.cache_get('dummy') != b'dummy':
from .shared_simple import SimpleSharedDict as SharedDict, schedule raise Exception()
logger.info('Use shared_simple implementation') schedule = shared_uwsgi.schedule
else: logger.info('use shared_uwsgi for schedule')
try:
uwsgi.cache_update('dummy', b'dummy')
if uwsgi.cache_get('dummy') != b'dummy':
raise Exception()
except:
# uwsgi.ini configuration problem: disable all scheduling
logger.error(
'uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
'cache2 = name=searxngcache,items=2000,blocks=2000,blocksize=4096,bitmap=1'
)
from .shared_simple import SimpleSharedDict as SharedDict
def schedule(delay, func, *args): if SharedDict is None:
return False SharedDict = shared_simple.SimpleSharedDict
logger.info('use shared_uwsgi for SharedDict')
else: except Exception: # pylint: disable=broad-except
# uwsgi logger.debug('skip uwsgi setup ..', exc_info=1)
from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
logger.info('Use shared_uwsgi implementation') if SharedDict is None:
logger.info('use shared_simple for SharedDict')
SharedDict = shared_simple.SimpleSharedDict
storage = SharedDict() if schedule is None:
logger.info('use shared_simple for schedule')
schedule = shared_simple.schedule
storage = SharedDict(key_prefix='SearXNG_storage')

View file

@ -5,6 +5,10 @@ from typing import Optional
class SharedDict(ABC): class SharedDict(ABC):
def __init__(self, *args, **kwarg):
pass
@abstractmethod @abstractmethod
def get_int(self, key: str) -> Optional[int]: def get_int(self, key: str) -> Optional[int]:
pass pass

View file

@ -0,0 +1,29 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pyright: strict
"""Module implements a :py:class:`shared_abstract.SharedDict` to store data in a
redis DB (:py:obj:`redisdb`)."""
from typing import Optional
from . import shared_abstract
from . import redisdb
class RedisCacheSharedDict(shared_abstract.SharedDict):
"""Store key/value in the redis DB, the default prefix of the key in the DB is
``SearXNG_SharedDict`` (see :py.obj:`searx.redislib.purge_by_prefix`)."""
def __init__(self, key_prefix='SearXNG_SharedDict'):
self.key_prefix = key_prefix
def get_int(self, key: str) -> Optional[int]:
return int(redisdb.client().get(self.key_prefix + key))
def set_int(self, key: str, value: int):
redisdb.client().set(self.key_prefix + key, str(value).encode())
def get_str(self, key: str) -> Optional[str]:
value = redisdb.client().get(self.key_prefix + key)
return None if value is None else value.decode()
def set_str(self, key: str, value: str):
redisdb.client().set(self.key_prefix + key, value.encode())