zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

fix: 429 when using Cloudflare

Browse source
This commit is contained in:
capric98 2022-04-07 00:21:21 +08:00
parent 4135a74001
commit 0eeb22bf82
1 changed files with 38 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
searx/plugins/limiter.py
View file

@ -36,10 +36,29 @@ re_bot = re.compile(
) )
def cdn_reported_ip(headers: dict) -> str:
# some commonly used CDNs:
cdn_keys = [
"CF-Connecting-IP", # Cloudflare
"Fastly-Client-IP", # Fastly
"True-Client-IP", # Akamai
"ar-real-ip", # ArvanCloud
]
for key in cdn_keys:
if (v := headers.get(key)):
return v
return ""
def is_accepted_request(inc_get_counter) -> bool: def is_accepted_request(inc_get_counter) -> bool:
# pylint: disable=too-many-return-statements # pylint: disable=too-many-return-statements
user_agent = request.headers.get('User-Agent', '') user_agent = request.headers.get('User-Agent', '')
x_forwarded_for = request.headers.get('X-Forwarded-For', '') client_ip = request.headers.get('X-Forwarded-For', '')
# uncomment below if you want to use the CDN reported client ip:
# client_ip = cdn_reported_ip(request.headers)
if request.path == '/image_proxy': if request.path == '/image_proxy':
if re_bot.match(user_agent): if re_bot.match(user_agent):
@ -47,31 +66,37 @@ def is_accepted_request(inc_get_counter) -> bool:
return True return True
if request.path == '/search': if request.path == '/search':
c_burst = inc_get_counter(interval=20, keys=[b'IP limit, burst', x_forwarded_for]) c_burst = inc_get_counter(interval=20, keys=[b'IP limit, burst', client_ip])
c_10min = inc_get_counter(interval=600, keys=[b'IP limit, 10 minutes', x_forwarded_for]) c_10min = inc_get_counter(interval=600, keys=[b'IP limit, 10 minutes', client_ip])
if c_burst > 15 or c_10min > 150: if c_burst > 15 or c_10min > 150:
return False return False
if re_bot.match(user_agent): if re_bot.match(user_agent):
return False return False
if len(request.headers.get('Accept-Language', '').strip()) == '': if request.headers.get('Accept-Language', '').strip():
return False return False
if request.headers.get('Connection') == 'close': # If SearXNG is behind Cloudflare, all requests will get 429 because
return False # Cloudflare uses "Connection: close" by default.
# if request.headers.get('Connection') == 'close':
# return False
accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] # "Accept-Encoding: gzip" will result in 429 because 'deflate' is not in accept_encoding_list...
if 'gzip' not in accept_encoding_list or 'deflate' not in accept_encoding_list: # accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')]
return False # if 'gzip' not in accept_encoding_list or 'deflate' not in accept_encoding_list:
# return False
if 'text/html' not in request.accept_mimetypes: if 'text/html' not in request.accept_mimetypes:
return False return False
if request.args.get('format', 'html') != 'html': # IDK but maybe api limit should based on path not format?
c = inc_get_counter(interval=3600, keys=[b'API limit', x_forwarded_for]) # if request.args.get('format', 'html') != 'html':
if c > 4: # c = inc_get_counter(interval=3600, keys=[b'API limit', client_ip])
return False # if c > 4:
# return False
return True return True