[mod] limiter: reduce request rates for requests without a ping

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/plugins/limiter.py

@@ -18,7 +18,7 @@ from flask import request
 
 from searx import redisdb
 from searx.plugins import logger
-from searx.redislib import incr_sliding_window
+from searx.redislib import incr_sliding_window, secret_hash
 
 name = "Request limiter"
 description = "Limit the number of request"
@@ -41,6 +41,18 @@ block_user_agent = re.compile(
     + r')'
 )
 
+PING_KEY = 'SearXNG_limiter.ping'
+TOKEN_KEY = 'SearXNG_limiter.token'
+
+
+def ping():
+    redis_client = redisdb.client()
+    user_agent = request.headers.get('User-Agent', 'unknown')
+    x_forwarded_for = request.headers.get('X-Forwarded-For', '')
+
+    ping_key = PING_KEY + user_agent + x_forwarded_for
+    redis_client.set(secret_hash(ping_key), 1, ex=600)
+
 
 def is_accepted_request() -> bool:
     # pylint: disable=too-many-return-statements
@@ -57,9 +69,20 @@ def is_accepted_request() -> bool:
 
     if request.path == '/search':
 
+        c_burst_max = 2
+        c_10min_max = 10
+
+        ping_key = PING_KEY + user_agent + x_forwarded_for
+        if redis_client.get(secret_hash(ping_key)):
+            logger.debug('got a ping')
+            c_burst_max = 15
+            c_10min_max = 150
+        else:
+            logger.debug('missing a ping')
+
         c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20)
         c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600)
-        if c_burst > 15 or c_10min > 150:
+        if c_burst > c_burst_max or c_10min > c_10min_max:
             logger.debug("BLOCK %s: to many request", x_forwarded_for)
             return False
 

searx/templates/simple/base.html

@@ -17,6 +17,9 @@
   {% else %}
   <link rel="stylesheet" href="{{ url_for('static', filename='css/searxng.min.css') }}" type="text/css" media="screen" />
   {% endif %}
+  {% if get_setting('server.limiter') %}
+  <link rel="stylesheet" href="/limiter.css" type="text/css" media="screen" />
+  {% endif %}
   {% block styles %}{% endblock %}
   <!--[if gte IE 9]>-->
   <script src="{{ url_for('static', filename='js/searxng.head.min.js') }}" client_settings="{{ client_settings }}"></script>

searx/webapp.py

@@ -93,7 +93,7 @@ from searx.utils import (
 )
 from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
 from searx.query import RawTextQuery
-from searx.plugins import Plugin, plugins, initialize as plugin_initialize
+from searx.plugins import limiter, Plugin, plugins, initialize as plugin_initialize
 from searx.plugins.oa_doi_rewrite import get_doi_resolver
 from searx.preferences import (
     Preferences,
@@ -642,6 +642,12 @@ def health():
     return Response('OK', mimetype='text/plain')
 
 
+@app.route('/limiter.css', methods=['GET', 'POST'])
+def limiter_css():
+    limiter.ping()
+    return Response('', mimetype='text/css')
+
+
 @app.route('/search', methods=['GET', 'POST'])
 def search():
     """Search query in q and return results.