add rate limiting per engine

This commit is contained in:
Marc Abonce Seguin 2022-03-20 19:22:44 -06:00
parent dec04c0ed6
commit 93982f6d8e
3 changed files with 97 additions and 0 deletions

View file

@ -422,6 +422,9 @@ engine is shown. Most of the options have a default value or even are optional.
max_connections: 100
max_keepalive_connections: 10
keepalive_expiry: 5.0
rate_limit:
max_requests: 200
interval: 60
proxies:
http:
- http://proxy1:8080
@ -487,6 +490,15 @@ engine is shown. Most of the options have a default value or even are optional.
- ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)
- ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)
``rate_limit``: optional
Limit how many outgoing requests is SearXNG going to send to the engines.
Requires :ref:`Redis <settings redis>`.
- ``max_requests`` is the maximum number of requests that will be sent to this
engine per interval.
- ``interval`` (optional) is the number of seconds before this engine's rate
limiter is reset. Defaults to 1 second if unspecified.
.. note::
A few more options are possible, but they are pretty specific to some

View file

@ -44,6 +44,7 @@ ENGINE_DEFAULT_ARGS = {
"enable_http": False,
"using_tor_proxy": False,
"display_error_messages": True,
"rate_limit": { "max_requests": float('inf'), "interval": 1 },
"tokens": [],
"about": {},
}
@ -168,6 +169,11 @@ def update_engine_attributes(engine: Engine, engine_data):
engine.categories = param_value
elif hasattr(engine, 'about') and param_name == 'about':
engine.about = {**engine.about, **engine_data['about']}
elif hasattr(engine, 'rate_limit') and param_name == 'rate_limit':
engine.rate_limit = {
'max_requests': int(param_value.get('max_requests')),
'interval': int(param_value.get('interval', 1))
}
else:
setattr(engine, param_name, param_value)

View file

@ -0,0 +1,79 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pyright: basic
"""Rate limit outgoing requests per engine
Enable ``settings.yml``:
- ``redis.url: ...`` check the value, see :ref:`settings redis`
- ``rate_limit: ...`` max_requests and interval, as specified below
- ``max_requests: ...`` max number of requests for that engine per interval
- ``interval: ...`` number of seconds before rate limiting resets (optional, by default 1 second)
"""
import hmac
from searx import settings
from searx.engines import engines
from searx.shared import redisdb
name = "Engine rate limiter"
description = "Limit the number of outgoing requests per engine"
default_on = True
preference_section = 'service'
def check_rate_limiter(engine_name, limit, interval):
redis_client = redisdb.client()
lua_script = """
local engine = KEYS[1]
local limit = ARGV[1]
local interval = ARGV[2]
local count = redis.call('GET', engine)
if count and count > limit then
return count
else
local newCount = redis.call('INCR', engine)
if newCount == 1 then
redis.call('EXPIRE', engine, interval)
end
return newCount
end
"""
script_sha = redis_client.script_load(lua_script)
secret_key_bytes = bytes(settings['server']['secret_key'], encoding='utf-8')
m = hmac.new(secret_key_bytes, digestmod='sha256')
m.update(bytes(engine_name, encoding='utf-8'))
key = m.digest()
requestsCounter = redis_client.evalsha(script_sha, 1, key, limit, interval)
return int(requestsCounter)
def below_rate_limit(engine_name):
engine = engines[engine_name]
max_requests = engine.rate_limit['max_requests']
interval = engine.rate_limit['interval']
if max_requests == float('inf'):
return True
if max_requests >= check_rate_limiter(engine_name, max_requests, interval):
return True
logger.debug(f"{engine_name} exceeded rate limit of {max_requests} requests per {interval} seconds") # pylint: disable=undefined-variable
return False
def pre_search(_, search):
allowed_engines = list(filter(lambda e: below_rate_limit(e.name), search.search_query.engineref_list))
search.search_query.engineref_list = allowed_engines
return bool(allowed_engines)
def init(*args, **kwargs): # pylint: disable=unused-argument
logger.debug("init engine rate limiter DB") # pylint: disable=undefined-variable
if not redisdb.init():
logger.error("init engine rate limiter DB failed!!!") # pylint: disable=undefined-variable
return False
return True