mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
allow many rate limiters per engine
This commit is contained in:
parent
93982f6d8e
commit
6eea3e7c1c
3 changed files with 18 additions and 23 deletions
|
@ -423,8 +423,8 @@ engine is shown. Most of the options have a default value or even are optional.
|
|||
max_keepalive_connections: 10
|
||||
keepalive_expiry: 5.0
|
||||
rate_limit:
|
||||
max_requests: 200
|
||||
interval: 60
|
||||
- max_requests: 200
|
||||
interval: 60
|
||||
proxies:
|
||||
http:
|
||||
- http://proxy1:8080
|
||||
|
|
|
@ -44,7 +44,7 @@ ENGINE_DEFAULT_ARGS = {
|
|||
"enable_http": False,
|
||||
"using_tor_proxy": False,
|
||||
"display_error_messages": True,
|
||||
"rate_limit": { "max_requests": float('inf'), "interval": 1 },
|
||||
"rate_limit": [{ "max_requests": float('inf'), "interval": 1 }],
|
||||
"tokens": [],
|
||||
"about": {},
|
||||
}
|
||||
|
@ -170,10 +170,10 @@ def update_engine_attributes(engine: Engine, engine_data):
|
|||
elif hasattr(engine, 'about') and param_name == 'about':
|
||||
engine.about = {**engine.about, **engine_data['about']}
|
||||
elif hasattr(engine, 'rate_limit') and param_name == 'rate_limit':
|
||||
engine.rate_limit = {
|
||||
'max_requests': int(param_value.get('max_requests')),
|
||||
'interval': int(param_value.get('interval', 1))
|
||||
}
|
||||
engine.rate_limit = [{
|
||||
'max_requests': int(rate_limit.get('max_requests')),
|
||||
'interval': int(rate_limit.get('interval', 1))
|
||||
} for rate_limit in param_value]
|
||||
else:
|
||||
setattr(engine, param_name, param_value)
|
||||
|
||||
|
|
|
@ -11,9 +11,6 @@ Enable ``settings.yml``:
|
|||
- ``interval: ...`` number of seconds before rate limiting resets (optional, by default 1 second)
|
||||
"""
|
||||
|
||||
import hmac
|
||||
|
||||
from searx import settings
|
||||
from searx.engines import engines
|
||||
from searx.shared import redisdb
|
||||
|
||||
|
@ -43,26 +40,24 @@ def check_rate_limiter(engine_name, limit, interval):
|
|||
"""
|
||||
script_sha = redis_client.script_load(lua_script)
|
||||
|
||||
secret_key_bytes = bytes(settings['server']['secret_key'], encoding='utf-8')
|
||||
m = hmac.new(secret_key_bytes, digestmod='sha256')
|
||||
m.update(bytes(engine_name, encoding='utf-8'))
|
||||
key = m.digest()
|
||||
|
||||
key = f'rate_limiter_{engine_name}_{limit}r/{interval}s'.encode()
|
||||
requestsCounter = redis_client.evalsha(script_sha, 1, key, limit, interval)
|
||||
return int(requestsCounter)
|
||||
|
||||
|
||||
def below_rate_limit(engine_name):
|
||||
engine = engines[engine_name]
|
||||
max_requests = engine.rate_limit['max_requests']
|
||||
interval = engine.rate_limit['interval']
|
||||
is_below_rate_limit = True
|
||||
for rate_limit in engine.rate_limit:
|
||||
max_requests = rate_limit['max_requests']
|
||||
interval = rate_limit['interval']
|
||||
if max_requests == float('inf'):
|
||||
continue
|
||||
if max_requests < check_rate_limiter(engine_name, max_requests, interval):
|
||||
is_below_rate_limit = False
|
||||
logger.debug(f"{engine_name} exceeded rate limit of {max_requests} requests per {interval} seconds") # pylint: disable=undefined-variable
|
||||
|
||||
if max_requests == float('inf'):
|
||||
return True
|
||||
if max_requests >= check_rate_limiter(engine_name, max_requests, interval):
|
||||
return True
|
||||
logger.debug(f"{engine_name} exceeded rate limit of {max_requests} requests per {interval} seconds") # pylint: disable=undefined-variable
|
||||
return False
|
||||
return is_below_rate_limit
|
||||
|
||||
|
||||
def pre_search(_, search):
|
||||
|
|
Loading…
Add table
Reference in a new issue