mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
[mod] shared.storage: implement a backend to store in the redis DB
- implement shared_redis.RedisCacheSharedDict to store key/value pairs in the redis DB. The default key_prefix is 'SearXNG_SharedDict'. - simplify searx.shared and import RedisCacheSharedDict, set key_prefix to 'SearXNG_storage' Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
parent
bfd6f61849
commit
4809bf705c
3 changed files with 66 additions and 24 deletions
|
@ -1,39 +1,48 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""Module implements a :py:obj:`storage`."""
|
||||
|
||||
__all__ = ['schedule', 'storage']
|
||||
|
||||
import logging
|
||||
import importlib
|
||||
|
||||
from . import redisdb
|
||||
from . import shared_redis
|
||||
from . import shared_simple
|
||||
|
||||
logger = logging.getLogger('searx.shared')
|
||||
|
||||
__all__ = ['SharedDict', 'schedule']
|
||||
SharedDict = None
|
||||
schedule = None
|
||||
|
||||
if redisdb.init():
|
||||
SharedDict = shared_redis.RedisCacheSharedDict
|
||||
logger.info('use redis DB for SharedDict')
|
||||
|
||||
try:
|
||||
from . import shared_uwsgi
|
||||
uwsgi = importlib.import_module('uwsgi')
|
||||
except:
|
||||
# no uwsgi
|
||||
from .shared_simple import SimpleSharedDict as SharedDict, schedule
|
||||
uwsgi.cache_update('dummy', b'dummy')
|
||||
if uwsgi.cache_get('dummy') != b'dummy':
|
||||
raise Exception()
|
||||
|
||||
logger.info('Use shared_simple implementation')
|
||||
else:
|
||||
try:
|
||||
uwsgi.cache_update('dummy', b'dummy')
|
||||
if uwsgi.cache_get('dummy') != b'dummy':
|
||||
raise Exception()
|
||||
except:
|
||||
# uwsgi.ini configuration problem: disable all scheduling
|
||||
logger.error(
|
||||
'uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
|
||||
'cache2 = name=searxngcache,items=2000,blocks=2000,blocksize=4096,bitmap=1'
|
||||
)
|
||||
from .shared_simple import SimpleSharedDict as SharedDict
|
||||
schedule = shared_uwsgi.schedule
|
||||
logger.info('use shared_uwsgi for schedule')
|
||||
|
||||
def schedule(delay, func, *args):
|
||||
return False
|
||||
if SharedDict is None:
|
||||
SharedDict = shared_simple.SimpleSharedDict
|
||||
logger.info('use shared_uwsgi for SharedDict')
|
||||
|
||||
else:
|
||||
# uwsgi
|
||||
from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.debug('skip uwsgi setup ..', exc_info=1)
|
||||
|
||||
logger.info('Use shared_uwsgi implementation')
|
||||
if SharedDict is None:
|
||||
logger.info('use shared_simple for SharedDict')
|
||||
SharedDict = shared_simple.SimpleSharedDict
|
||||
|
||||
storage = SharedDict()
|
||||
if schedule is None:
|
||||
logger.info('use shared_simple for schedule')
|
||||
schedule = shared_simple.schedule
|
||||
|
||||
storage = SharedDict(key_prefix='SearXNG_storage')
|
||||
|
|
|
@ -5,6 +5,10 @@ from typing import Optional
|
|||
|
||||
|
||||
class SharedDict(ABC):
|
||||
|
||||
def __init__(self, *args, **kwarg):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_int(self, key: str) -> Optional[int]:
|
||||
pass
|
||||
|
|
29
searx/shared/shared_redis.py
Normal file
29
searx/shared/shared_redis.py
Normal file
|
@ -0,0 +1,29 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
# pyright: strict
|
||||
"""Module implements a :py:class:`shared_abstract.SharedDict` to store data in a
|
||||
redis DB (:py:obj:`redisdb`)."""
|
||||
|
||||
from typing import Optional
|
||||
from . import shared_abstract
|
||||
from . import redisdb
|
||||
|
||||
class RedisCacheSharedDict(shared_abstract.SharedDict):
|
||||
"""Store key/value in the redis DB, the default prefix of the key in the DB is
|
||||
``SearXNG_SharedDict`` (see :py.obj:`searx.redislib.purge_by_prefix`)."""
|
||||
|
||||
def __init__(self, key_prefix='SearXNG_SharedDict'):
|
||||
self.key_prefix = key_prefix
|
||||
|
||||
def get_int(self, key: str) -> Optional[int]:
|
||||
return int(redisdb.client().get(self.key_prefix + key))
|
||||
|
||||
def set_int(self, key: str, value: int):
|
||||
redisdb.client().set(self.key_prefix + key, str(value).encode())
|
||||
|
||||
def get_str(self, key: str) -> Optional[str]:
|
||||
value = redisdb.client().get(self.key_prefix + key)
|
||||
return None if value is None else value.decode()
|
||||
|
||||
def set_str(self, key: str, value: str):
|
||||
redisdb.client().set(self.key_prefix + key, value.encode())
|
Loading…
Add table
Reference in a new issue