From 52f1452c09ab2ec74aa5898d9ea749f33a71a814 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 27 May 2023 21:36:34 +0200 Subject: [PATCH] [mod] limiter: ip_limt - monitore suspicious IPs To intercept bots that get their IPs from a range of IPs, there is a ``SUSPICIOUS_IP_WINDOW``. In this window the suspicious IPs are stored for a longer time. IPs stored in this sliding window have a maximum of ``SUSPICIOUS_IP_MAX`` accesses before they are blocked. As soon as the IP makes a request that is not suspicious, the sliding window for this IP is droped. Signed-off-by: Markus Heiser --- searx/botdetection/ip_limit.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/searx/botdetection/ip_limit.py b/searx/botdetection/ip_limit.py index e72015190..9cffff7f0 100644 --- a/searx/botdetection/ip_limit.py +++ b/searx/botdetection/ip_limit.py @@ -25,6 +25,13 @@ the request rates are reduced: - :py:obj:`BURST_MAX` -> :py:obj:`BURST_MAX_SUSPICIOUS` - :py:obj:`LONG_MAX` -> :py:obj:`LONG_MAX_SUSPICIOUS` +To intercept bots that get their IPs from a range of IPs, there is a +:py:obj:`SUSPICIOUS_IP_WINDOW`. In this window the suspicious IPs are stored +for a longer time. IPs stored in this sliding window have a maximum of +:py:obj:`SUSPICIOUS_IP_MAX` accesses before they are blocked. As soon as the IP +makes a request that is not suspicious, the sliding window for this IP is +droped. + .. _X-Forwarded-For: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For @@ -37,7 +44,7 @@ from searx.tools import config from searx import redisdb from searx import logger -from searx.redislib import incr_sliding_window +from searx.redislib import incr_sliding_window, drop_counter from . import link_token @@ -67,6 +74,12 @@ API_WONDOW = 3600 API_MAX = 4 """Maximum requests from one IP in the :py:obj:`API_WONDOW`""" +SUSPICIOUS_IP_WINDOW = 3600 * 24 +"""Time (sec) before sliding window for one suspicious IP expires.""" + +SUSPICIOUS_IP_MAX = 3 +"""Maximum requests from one suspicious IP in the :py:obj:`SUSPICIOUS_IP_WINDOW`.""" + def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple[int, str]]: redis_client = redisdb.client() @@ -81,10 +94,18 @@ def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple return 429, "BLOCK %s: API limit exceeded" suspicious = False + suspicious_ip_counter = 'IP limit - SUSPICIOUS_IP_WINDOW:' + x_forwarded_for + if cfg['botdetection.ip_limit.link_token']: suspicious = link_token.is_suspicious(request) if suspicious: + + # this IP is suspicious: count requests from this IP + c = incr_sliding_window(redis_client, suspicious_ip_counter, SUSPICIOUS_IP_WINDOW) + if c > SUSPICIOUS_IP_MAX: + return 429, f"bot detected, too many request from {x_forwarded_for} in SUSPICIOUS_IP_WINDOW" + c = incr_sliding_window(redis_client, 'IP limit - BURST_WINDOW:' + x_forwarded_for, BURST_WINDOW) if c > BURST_MAX_SUSPICIOUS: return 429, f"bot detected, too many request from {x_forwarded_for} in BURST_MAX_SUSPICIOUS" @@ -94,6 +115,11 @@ def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple return 429, f"bot detected, too many request from {x_forwarded_for} in LONG_MAX_SUSPICIOUS" else: + + if cfg['botdetection.ip_limit.link_token']: + # this IP is no longer suspicious: release ip again / delete the counter of this IP + drop_counter(redis_client, suspicious_ip_counter) + c = incr_sliding_window(redis_client, 'IP limit - BURST_WINDOW:' + x_forwarded_for, BURST_WINDOW) if c > BURST_MAX: return 429, f"bot detected, too many request from {x_forwarded_for} in BURST_MAX"