zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

allow many rate limiters per engine

Browse source
This commit is contained in:
Marc Abonce Seguin 2022-03-27 20:21:37 -06:00
parent 93982f6d8e
commit 6eea3e7c1c
3 changed files with 18 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/admin/engines/settings.rst
View file

@ -423,8 +423,8 @@ engine is shown. Most of the options have a default value or even are optional.
max_keepalive_connections: 10 max_keepalive_connections: 10
keepalive_expiry: 5.0 keepalive_expiry: 5.0
rate_limit: rate_limit:
max_requests: 200 - max_requests: 200
interval: 60 interval: 60
proxies: proxies:
http: http:
- http://proxy1:8080 - http://proxy1:8080

10
searx/engines/__init__.py
View file

@ -44,7 +44,7 @@ ENGINE_DEFAULT_ARGS = {
"enable_http": False, "enable_http": False,
"using_tor_proxy": False, "using_tor_proxy": False,
"display_error_messages": True, "display_error_messages": True,
"rate_limit": { "max_requests": float('inf'), "interval": 1 }, "rate_limit": [{ "max_requests": float('inf'), "interval": 1 }],
"tokens": [], "tokens": [],
"about": {}, "about": {},
} }
@ -170,10 +170,10 @@ def update_engine_attributes(engine: Engine, engine_data):
elif hasattr(engine, 'about') and param_name == 'about': elif hasattr(engine, 'about') and param_name == 'about':
engine.about = {**engine.about, **engine_data['about']} engine.about = {**engine.about, **engine_data['about']}
elif hasattr(engine, 'rate_limit') and param_name == 'rate_limit': elif hasattr(engine, 'rate_limit') and param_name == 'rate_limit':
engine.rate_limit = { engine.rate_limit = [{
'max_requests': int(param_value.get('max_requests')), 'max_requests': int(rate_limit.get('max_requests')),
'interval': int(param_value.get('interval', 1)) 'interval': int(rate_limit.get('interval', 1))
} } for rate_limit in param_value]
else: else:
setattr(engine, param_name, param_value) setattr(engine, param_name, param_value)

27
searx/plugins/engine_rate_limiter.py
View file

@ -11,9 +11,6 @@ Enable ``settings.yml``:
- ``interval: ...`` number of seconds before rate limiting resets (optional, by default 1 second) - ``interval: ...`` number of seconds before rate limiting resets (optional, by default 1 second)
""" """
import hmac
from searx import settings
from searx.engines import engines from searx.engines import engines
from searx.shared import redisdb from searx.shared import redisdb
@ -43,26 +40,24 @@ def check_rate_limiter(engine_name, limit, interval):
""" """
script_sha = redis_client.script_load(lua_script) script_sha = redis_client.script_load(lua_script)
secret_key_bytes = bytes(settings['server']['secret_key'], encoding='utf-8') key = f'rate_limiter_{engine_name}_{limit}r/{interval}s'.encode()
m = hmac.new(secret_key_bytes, digestmod='sha256')
m.update(bytes(engine_name, encoding='utf-8'))
key = m.digest()
requestsCounter = redis_client.evalsha(script_sha, 1, key, limit, interval) requestsCounter = redis_client.evalsha(script_sha, 1, key, limit, interval)
return int(requestsCounter) return int(requestsCounter)
def below_rate_limit(engine_name): def below_rate_limit(engine_name):
engine = engines[engine_name] engine = engines[engine_name]
max_requests = engine.rate_limit['max_requests'] is_below_rate_limit = True
interval = engine.rate_limit['interval'] for rate_limit in engine.rate_limit:
max_requests = rate_limit['max_requests']
interval = rate_limit['interval']
if max_requests == float('inf'):
continue
if max_requests < check_rate_limiter(engine_name, max_requests, interval):
is_below_rate_limit = False
logger.debug(f"{engine_name} exceeded rate limit of {max_requests} requests per {interval} seconds") # pylint: disable=undefined-variable
if max_requests == float('inf'): return is_below_rate_limit
return True
if max_requests >= check_rate_limiter(engine_name, max_requests, interval):
return True
logger.debug(f"{engine_name} exceeded rate limit of {max_requests} requests per {interval} seconds") # pylint: disable=undefined-variable
return False
def pre_search(_, search): def pre_search(_, search):