Merge pull request #2186 from return42/limiter-debug-log

[mod] plugin limiter: improve the log messages
This commit is contained in:
Markus Heiser 2023-04-01 10:11:17 +02:00 committed by GitHub
commit 7592d85982
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -17,13 +17,14 @@ import re
from flask import request from flask import request
from searx import redisdb from searx import redisdb
from searx.plugins import logger
from searx.redislib import incr_sliding_window from searx.redislib import incr_sliding_window
name = "Request limiter" name = "Request limiter"
description = "Limit the number of request" description = "Limit the number of request"
default_on = False default_on = False
preference_section = 'service' preference_section = 'service'
logger = logger.getChild('limiter')
re_bot = re.compile( re_bot = re.compile(
r'(' r'('
@ -42,44 +43,52 @@ def is_accepted_request() -> bool:
user_agent = request.headers.get('User-Agent', '') user_agent = request.headers.get('User-Agent', '')
x_forwarded_for = request.headers.get('X-Forwarded-For', '') x_forwarded_for = request.headers.get('X-Forwarded-For', '')
if request.path == '/image_proxy':
if re_bot.match(user_agent): if re_bot.match(user_agent):
logger.debug("BLOCK %s: detected bot", x_forwarded_for)
return False return False
return True
if request.path == '/search': if request.path == '/search':
c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20) c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20)
c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600) c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600)
if c_burst > 15 or c_10min > 150: if c_burst > 15 or c_10min > 150:
logger.debug("to many request") # pylint: disable=undefined-variable logger.debug("BLOCK %s: to many request", x_forwarded_for)
return False
if re_bot.match(user_agent):
logger.debug("detected bot") # pylint: disable=undefined-variable
return False return False
if len(request.headers.get('Accept-Language', '').strip()) == '': if len(request.headers.get('Accept-Language', '').strip()) == '':
logger.debug("missing Accept-Language") # pylint: disable=undefined-variable logger.debug("BLOCK %s: missing Accept-Language", x_forwarded_for)
return False return False
if request.headers.get('Connection') == 'close': if request.headers.get('Connection') == 'close':
logger.debug("got Connection=close") # pylint: disable=undefined-variable logger.debug("BLOCK %s: got Connection=close", x_forwarded_for)
return False return False
accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')]
if 'gzip' not in accept_encoding_list and 'deflate' not in accept_encoding_list: if 'gzip' not in accept_encoding_list and 'deflate' not in accept_encoding_list:
logger.debug("suspicious Accept-Encoding") # pylint: disable=undefined-variable logger.debug("BLOCK %s: suspicious Accept-Encoding", x_forwarded_for)
return False return False
if 'text/html' not in request.accept_mimetypes: if 'text/html' not in request.accept_mimetypes:
logger.debug("Accept-Encoding misses text/html") # pylint: disable=undefined-variable logger.debug("BLOCK %s: Accept-Encoding misses text/html", x_forwarded_for)
return False return False
if request.args.get('format', 'html') != 'html': if request.args.get('format', 'html') != 'html':
c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600) c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600)
if c > 4: if c > 4:
logger.debug("API limit exceeded") # pylint: disable=undefined-variable logger.debug("BLOCK %s: API limit exceeded", x_forwarded_for)
return False return False
logger.debug(
"OK %s: '%s'" % (x_forwarded_for, request.path)
+ " || form: %s" % request.form
+ " || Accept: %s" % request.headers.get('Accept', '')
+ " || Accept-Language: %s" % request.headers.get('Accept-Language', '')
+ " || Accept-Encoding: %s" % request.headers.get('Accept-Encoding', '')
+ " || Content-Type: %s" % request.headers.get('Content-Type', '')
+ " || Content-Length: %s" % request.headers.get('Content-Length', '')
+ " || Connection: %s" % request.headers.get('Connection', '')
+ " || User-Agent: %s" % user_agent
)
return True return True