[mod] plugins/limiter.py - use searx.redislib.incr_sliding_window

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/plugins/limiter.py

@@ -13,11 +13,11 @@ Enable the plugin in ``settings.yml``:
 - ``redis.url: ...`` check the value, see :ref:`settings redis`
 """
 
-import hmac
 import re
 from flask import request
 
 from searx.shared import redisdb
+from searx.redislib import incr_sliding_window
 
 name = "Request limiter"
 description = "Limit the number of request"
@@ -36,8 +36,9 @@ re_bot = re.compile(
 )
 
 
-def is_accepted_request(inc_get_counter) -> bool:
+def is_accepted_request() -> bool:
     # pylint: disable=too-many-return-statements
+    redis_client = redisdb.client()
     user_agent = request.headers.get('User-Agent', '')
     x_forwarded_for = request.headers.get('X-Forwarded-For', '')
 
@@ -47,71 +48,46 @@ def is_accepted_request(inc_get_counter) -> bool:
         return True
 
     if request.path == '/search':
-        c_burst = inc_get_counter(interval=20, keys=[b'IP limit, burst', x_forwarded_for])
+        c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20)
-        c_10min = inc_get_counter(interval=600, keys=[b'IP limit, 10 minutes', x_forwarded_for])
+        c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600)
         if c_burst > 15 or c_10min > 150:
+            logger.debug("to many request")  # pylint: disable=undefined-variable
             return False
 
         if re_bot.match(user_agent):
+            logger.debug("detected bot")  # pylint: disable=undefined-variable
             return False
 
         if len(request.headers.get('Accept-Language', '').strip()) == '':
+            logger.debug("missing Accept-Language")  # pylint: disable=undefined-variable
             return False
 
         if request.headers.get('Connection') == 'close':
+            logger.debug("got Connection=close")  # pylint: disable=undefined-variable
             return False
 
         accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')]
         if 'gzip' not in accept_encoding_list or 'deflate' not in accept_encoding_list:
+            logger.debug("suspicious Accept-Encoding")  # pylint: disable=undefined-variable
             return False
 
         if 'text/html' not in request.accept_mimetypes:
+            logger.debug("Accept-Encoding misses text/html")  # pylint: disable=undefined-variable
             return False
 
         if request.args.get('format', 'html') != 'html':
-            c = inc_get_counter(interval=3600, keys=[b'API limit', x_forwarded_for])
+            c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600)
             if c > 4:
+                logger.debug("API limit exceeded")  # pylint: disable=undefined-variable
                 return False
     return True
 
 
-def create_inc_get_counter(redis_client, secret_key_bytes):
+def pre_request():
-    lua_script = """
+    if not is_accepted_request():
-    local slidingWindow = KEYS[1]
-    local key = KEYS[2]
-    local now = tonumber(redis.call('TIME')[1])
-    local id = redis.call('INCR', 'counter')
-    if (id > 2^46)
-    then
-        redis.call('SET', 'count', 0)
-    end
-    redis.call('ZREMRANGEBYSCORE', key, 0, now - slidingWindow)
-    redis.call('ZADD', key, now, id)
-    local result = redis.call('ZCOUNT', key, 0, now+1)
-    redis.call('EXPIRE', key, slidingWindow)
-    return result
-    """
-    script_sha = redis_client.script_load(lua_script)
-
-    def inc_get_counter(interval, keys):
-        m = hmac.new(secret_key_bytes, digestmod='sha256')
-        for k in keys:
-            m.update(bytes(str(k), encoding='utf-8') or b'')
-            m.update(b"\0")
-        key = m.digest()
-        return redis_client.evalsha(script_sha, 2, interval, key)
-
-    return inc_get_counter
-
-
-def create_pre_request(get_aggregation_count):
-    def pre_request():
-        if not is_accepted_request(get_aggregation_count):
         return '', 429
     return None
 
-    return pre_request
-
 
 def init(app, settings):
     if not settings['server']['limiter']:
@@ -122,8 +98,5 @@ def init(app, settings):
         logger.error("init limiter DB failed!!!")  # pylint: disable=undefined-variable
         return False
 
-    redis_client = redisdb.client()
+    app.before_request(pre_request)
-    secret_key_bytes = bytes(settings['server']['secret_key'], encoding='utf-8')
-    inc_get_counter = create_inc_get_counter(redis_client, secret_key_bytes)
-    app.before_request(create_pre_request(inc_get_counter))
     return True