From afd8fcce36dc03c4ca0ff5878938b2b50459a924 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 12 Feb 2023 12:14:15 +0100 Subject: [PATCH] [mod] plugin limiter: improve the log messages In debug mode more detailed logging is needed to evaluate if an access should have been blocked by the limiter. BTW: remove duplicate code checking bot signature ``re_bot.match(user_agent)`` Signed-off-by: Markus Heiser --- searx/plugins/limiter.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index b66a0805c..baf2c1726 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -17,13 +17,14 @@ import re from flask import request from searx import redisdb +from searx.plugins import logger from searx.redislib import incr_sliding_window name = "Request limiter" description = "Limit the number of request" default_on = False preference_section = 'service' - +logger = logger.getChild('limiter') re_bot = re.compile( r'(' @@ -42,44 +43,52 @@ def is_accepted_request() -> bool: user_agent = request.headers.get('User-Agent', '') x_forwarded_for = request.headers.get('X-Forwarded-For', '') - if request.path == '/image_proxy': - if re_bot.match(user_agent): - return False - return True + if re_bot.match(user_agent): + logger.debug("BLOCK %s: detected bot", x_forwarded_for) + return False if request.path == '/search': c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20) c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600) if c_burst > 15 or c_10min > 150: - logger.debug("to many request") # pylint: disable=undefined-variable - return False - - if re_bot.match(user_agent): - logger.debug("detected bot") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: to many request", x_forwarded_for) return False if len(request.headers.get('Accept-Language', '').strip()) == '': - logger.debug("missing Accept-Language") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: missing Accept-Language", x_forwarded_for) return False if request.headers.get('Connection') == 'close': - logger.debug("got Connection=close") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: got Connection=close", x_forwarded_for) return False accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] if 'gzip' not in accept_encoding_list and 'deflate' not in accept_encoding_list: - logger.debug("suspicious Accept-Encoding") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: suspicious Accept-Encoding", x_forwarded_for) return False if 'text/html' not in request.accept_mimetypes: - logger.debug("Accept-Encoding misses text/html") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: Accept-Encoding misses text/html", x_forwarded_for) return False if request.args.get('format', 'html') != 'html': c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600) if c > 4: - logger.debug("API limit exceeded") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: API limit exceeded", x_forwarded_for) return False + + logger.debug( + "OK %s: '%s'" % (x_forwarded_for, request.path) + + " || form: %s" % request.form + + " || Accept: %s" % request.headers.get('Accept', '') + + " || Accept-Language: %s" % request.headers.get('Accept-Language', '') + + " || Accept-Encoding: %s" % request.headers.get('Accept-Encoding', '') + + " || Content-Type: %s" % request.headers.get('Content-Type', '') + + " || Content-Length: %s" % request.headers.get('Content-Length', '') + + " || Connection: %s" % request.headers.get('Connection', '') + + " || User-Agent: %s" % user_agent + ) + return True