zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

Replace Flask by Starlette (2/n)

Browse source
This commit is contained in:
Alexandre Flament 2021-08-13 19:02:01 +02:00
parent 78561ce7bb
commit 1a3c73cf6f
30 changed files with 994 additions and 2235 deletions
Download patch file Download diff file Expand all files Collapse all files

147
searx/network/client.py
View file

@ -3,7 +3,6 @@
# pylint: disable=missing-module-docstring, missing-function-docstring, global-statement
import asyncio
import logging
import threading
import httpcore
import httpx
@ -12,7 +11,7 @@ from python_socks import (
parse_proxy_url,
ProxyConnectionError,
ProxyTimeoutError,
ProxyError
ProxyError,
)
from searx import logger
@ -26,33 +25,38 @@ else:
uvloop.install()
logger = logger.getChild('searx.http.client')
logger = logger.getChild("searx.http.client")
LOOP = None
LOOP_LOCK = threading.Lock()
SSLCONTEXTS = {}
TRANSPORT_KWARGS = {
'backend': 'asyncio',
'trust_env': False,
"backend": "asyncio",
"trust_env": False,
}
# pylint: disable=protected-access
async def close_connections_for_url(
connection_pool: httpcore.AsyncConnectionPool,
url: httpcore._utils.URL ):
connection_pool: httpcore.AsyncConnectionPool, url: httpcore._utils.URL
):
origin = httpcore._utils.url_to_origin(url)
logger.debug('Drop connections for %r', origin)
logger.debug("Drop connections for %r", origin)
connections_to_close = connection_pool._connections_for_origin(origin)
for connection in connections_to_close:
await connection_pool._remove_from_pool(connection)
try:
await connection.aclose()
except httpcore.NetworkError as e:
logger.warning('Error closing an existing connection', exc_info=e)
logger.warning("Error closing an existing connection", exc_info=e)
# pylint: enable=protected-access
def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
def get_sslcontexts(
proxy_url=None, cert=None, verify=True, trust_env=True, http2=False
):
global SSLCONTEXTS
key = (proxy_url, cert, verify, trust_env, http2)
if key not in SSLCONTEXTS:
@ -98,7 +102,7 @@ class AsyncProxyTransportFixed(AsyncProxyTransport):
except httpcore.RemoteProtocolError as e:
# in case of httpcore.RemoteProtocolError: Server disconnected
await close_connections_for_url(self, url)
logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
logger.warning("httpcore.RemoteProtocolError: retry", exc_info=e)
# retry
except (httpcore.NetworkError, httpcore.ProtocolError) as e:
# httpcore.WriteError on HTTP/2 connection leaves a new opened stream
@ -124,35 +128,50 @@ class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
# raised by _keepalive_sweep()
# from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # pylint: disable=line-too-long
await close_connections_for_url(self._pool, url)
logger.warning('httpcore.CloseError: retry', exc_info=e)
logger.warning("httpcore.CloseError: retry", exc_info=e)
# retry
except httpcore.RemoteProtocolError as e:
# in case of httpcore.RemoteProtocolError: Server disconnected
await close_connections_for_url(self._pool, url)
logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
logger.warning("httpcore.RemoteProtocolError: retry", exc_info=e)
# retry
except (httpcore.ProtocolError, httpcore.NetworkError) as e:
await close_connections_for_url(self._pool, url)
raise e
def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit, retries):
def get_transport_for_socks_proxy(
verify, http2, local_address, proxy_url, limit, retries
):
global TRANSPORT_KWARGS
# support socks5h (requests compatibility):
# https://requests.readthedocs.io/en/master/user/advanced/#socks
# socks5:// hostname is resolved on client side
# socks5h:// hostname is resolved on proxy side
rdns = False
socks5h = 'socks5h://'
socks5h = "socks5h://"
if proxy_url.startswith(socks5h):
proxy_url = 'socks5://' + proxy_url[len(socks5h):]
proxy_url = "socks5://" + proxy_url[len(socks5h) :]
rdns = True
proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
verify = get_sslcontexts(proxy_url, None, True, False, http2) if verify is True else verify
(
proxy_type,
proxy_host,
proxy_port,
proxy_username,
proxy_password,
) = parse_proxy_url(proxy_url)
verify = (
get_sslcontexts(proxy_url, None, True, False, http2)
if verify is True
else verify
)
return AsyncProxyTransportFixed(
proxy_type=proxy_type, proxy_host=proxy_host, proxy_port=proxy_port,
username=proxy_username, password=proxy_password,
proxy_type=proxy_type,
proxy_host=proxy_host,
proxy_port=proxy_port,
username=proxy_username,
password=proxy_password,
rdns=rdns,
loop=get_loop(),
verify=verify,
@ -168,7 +187,9 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
def get_transport(verify, http2, local_address, proxy_url, limit, retries):
global TRANSPORT_KWARGS
verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify
verify = (
get_sslcontexts(None, None, True, False, http2) if verify is True else verify
)
return AsyncHTTPTransportFixed(
# pylint: disable=protected-access
verify=verify,
@ -184,30 +205,39 @@ def get_transport(verify, http2, local_address, proxy_url, limit, retries):
def iter_proxies(proxies):
# https://www.python-httpx.org/compatibility/#proxy-keys
if isinstance(proxies, str):
yield 'all://', proxies
yield "all://", proxies
elif isinstance(proxies, dict):
for pattern, proxy_url in proxies.items():
yield pattern, proxy_url
def new_client(
# pylint: disable=too-many-arguments
enable_http, verify, enable_http2,
max_connections, max_keepalive_connections, keepalive_expiry,
proxies, local_address, retries, max_redirects ):
# pylint: disable=too-many-arguments
enable_http,
verify,
enable_http2,
max_connections,
max_keepalive_connections,
keepalive_expiry,
proxies,
local_address,
retries,
max_redirects,
):
limit = httpx.Limits(
max_connections=max_connections,
max_keepalive_connections=max_keepalive_connections,
keepalive_expiry=keepalive_expiry
keepalive_expiry=keepalive_expiry,
)
# See https://www.python-httpx.org/advanced/#routing
mounts = {}
for pattern, proxy_url in iter_proxies(proxies):
if not enable_http and (pattern == 'http' or pattern.startswith('http://')):
if not enable_http and (pattern == "http" or pattern.startswith("http://")):
continue
if (proxy_url.startswith('socks4://')
or proxy_url.startswith('socks5://')
or proxy_url.startswith('socks5h://')
if (
proxy_url.startswith("socks4://")
or proxy_url.startswith("socks5://")
or proxy_url.startswith("socks5h://")
):
mounts[pattern] = get_transport_for_socks_proxy(
verify, enable_http2, local_address, proxy_url, limit, retries
@ -218,10 +248,39 @@ def new_client(
)
if not enable_http:
mounts['http://'] = AsyncHTTPTransportNoHttp()
mounts["http://"] = AsyncHTTPTransportNoHttp()
transport = get_transport(verify, enable_http2, local_address, None, limit, retries)
return httpx.AsyncClient(transport=transport, mounts=mounts, max_redirects=max_redirects)
return httpx.AsyncClient(
transport=transport, mounts=mounts, max_redirects=max_redirects
)
def create_loop():
# pylint: disable=consider-using-with
global LOOP_LOCK
LOOP_LOCK.acquire()
if LOOP:
return
def loop_thread():
global LOOP
try:
LOOP = asyncio.new_event_loop()
except: # pylint: disable=bare-except
logger.exception('Error on asyncio.new_event_loop()')
finally:
LOOP_LOCK.release()
if LOOP:
LOOP.run_forever()
thread = threading.Thread(
target=loop_thread,
name="asyncio_loop",
daemon=True,
)
thread.start()
LOOP_LOCK.acquire()
def get_loop():
@ -229,20 +288,10 @@ def get_loop():
if LOOP:
return LOOP
loop_ready = threading.Lock()
loop_ready.acquire()
def loop_thread():
global LOOP
LOOP = asyncio.new_event_loop()
loop_ready.release()
LOOP.run_forever()
thread = threading.Thread(
target=loop_thread,
name='asyncio_loop',
daemon=True,
)
thread.start()
loop_ready.acquire()
create_loop()
return LOOP
def set_loop(loop):
global LOOP
LOOP = loop

199
searx/network/network.py
View file

@ -3,7 +3,6 @@
# pylint: disable=global-statement
# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring
import atexit
import asyncio
import ipaddress
from itertools import cycle
@ -11,54 +10,61 @@ from itertools import cycle
import httpx
from .client import new_client, get_loop
from searx import logger
DEFAULT_NAME = '__DEFAULT__'
DEFAULT_NAME = "__DEFAULT__"
NETWORKS = {}
# requests compatibility when reading proxy settings from settings.yml
PROXY_PATTERN_MAPPING = {
'http': 'http://',
'https': 'https://',
'socks4': 'socks4://',
'socks5': 'socks5://',
'socks5h': 'socks5h://',
'http:': 'http://',
'https:': 'https://',
'socks4:': 'socks4://',
'socks5:': 'socks5://',
'socks5h:': 'socks5h://',
"http": "http://",
"https": "https://",
"socks4": "socks4://",
"socks5": "socks5://",
"socks5h": "socks5h://",
"http:": "http://",
"https:": "https://",
"socks4:": "socks4://",
"socks5:": "socks5://",
"socks5h:": "socks5h://",
}
ADDRESS_MAPPING = {
'ipv4': '0.0.0.0',
'ipv6': '::'
}
ADDRESS_MAPPING = {"ipv4": "0.0.0.0", "ipv6": "::"}
class Network:
__slots__ = (
'enable_http', 'verify', 'enable_http2',
'max_connections', 'max_keepalive_connections', 'keepalive_expiry',
'local_addresses', 'proxies', 'max_redirects', 'retries', 'retry_on_http_error',
'_local_addresses_cycle', '_proxies_cycle', '_clients'
"enable_http",
"verify",
"enable_http2",
"max_connections",
"max_keepalive_connections",
"keepalive_expiry",
"local_addresses",
"proxies",
"max_redirects",
"retries",
"retry_on_http_error",
"_local_addresses_cycle",
"_proxies_cycle",
"_clients",
)
def __init__(
# pylint: disable=too-many-arguments
self,
enable_http=True,
verify=True,
enable_http2=False,
max_connections=None,
max_keepalive_connections=None,
keepalive_expiry=None,
proxies=None,
local_addresses=None,
retries=0,
retry_on_http_error=None,
max_redirects=30 ):
# pylint: disable=too-many-arguments
self,
enable_http=True,
verify=True,
enable_http2=False,
max_connections=None,
max_keepalive_connections=None,
keepalive_expiry=None,
proxies=None,
local_addresses=None,
retries=0,
retry_on_http_error=None,
max_redirects=30,
):
self.enable_http = enable_http
self.verify = verify
@ -78,13 +84,13 @@ class Network:
def check_parameters(self):
for address in self.iter_ipaddresses():
if '/' in address:
if "/" in address:
ipaddress.ip_network(address, False)
else:
ipaddress.ip_address(address)
if self.proxies is not None and not isinstance(self.proxies, (str, dict)):
raise ValueError('proxies type has to be str, dict or None')
raise ValueError("proxies type has to be str, dict or None")
def iter_ipaddresses(self):
local_addresses = self.local_addresses
@ -99,7 +105,7 @@ class Network:
while True:
count = 0
for address in self.iter_ipaddresses():
if '/' in address:
if "/" in address:
for a in ipaddress.ip_network(address, False).hosts():
yield str(a)
count += 1
@ -115,7 +121,7 @@ class Network:
return
# https://www.python-httpx.org/compatibility/#proxy-keys
if isinstance(self.proxies, str):
yield 'all://', [self.proxies]
yield "all://", [self.proxies]
else:
for pattern, proxy_url in self.proxies.items():
pattern = PROXY_PATTERN_MAPPING.get(pattern, pattern)
@ -129,7 +135,10 @@ class Network:
proxy_settings[pattern] = cycle(proxy_urls)
while True:
# pylint: disable=stop-iteration-return
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
yield tuple(
(pattern, next(proxy_url_cycle))
for pattern, proxy_url_cycle in proxy_settings.items()
)
def get_client(self, verify=None, max_redirects=None):
verify = self.verify if verify is None else verify
@ -148,32 +157,43 @@ class Network:
dict(proxies),
local_address,
0,
max_redirects
max_redirects,
)
return self._clients[key]
async def aclose(self):
async def close_client(client):
async def close_client(client: httpx.AsyncClient):
try:
await client.aclose()
except httpx.HTTPError:
pass
await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
await asyncio.gather(
*[close_client(client) for client in self._clients.values()],
return_exceptions=False
)
@staticmethod
def get_kwargs_clients(kwargs):
kwargs_clients = {}
if 'verify' in kwargs:
kwargs_clients['verify'] = kwargs.pop('verify')
if 'max_redirects' in kwargs:
kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
if "verify" in kwargs:
kwargs_clients["verify"] = kwargs.pop("verify")
if "max_redirects" in kwargs:
kwargs_clients["max_redirects"] = kwargs.pop("max_redirects")
return kwargs_clients
def is_valid_respones(self, response):
# pylint: disable=too-many-boolean-expressions
if ((self.retry_on_http_error is True and 400 <= response.status_code <= 599)
or (isinstance(self.retry_on_http_error, list) and response.status_code in self.retry_on_http_error)
or (isinstance(self.retry_on_http_error, int) and response.status_code == self.retry_on_http_error)
if (
(self.retry_on_http_error is True and 400 <= response.status_code <= 599)
or (
isinstance(self.retry_on_http_error, list)
and response.status_code in self.retry_on_http_error
)
or (
isinstance(self.retry_on_http_error, int)
and response.status_code == self.retry_on_http_error
)
):
return False
return True
@ -209,39 +229,52 @@ class Network:
@classmethod
async def aclose_all(cls):
global NETWORKS
await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)
await asyncio.gather(
*[network.aclose() for network in NETWORKS.values()],
return_exceptions=False
)
@classmethod
def close_all(cls):
future = asyncio.run_coroutine_threadsafe(Network.aclose_all(), get_loop())
future.result()
def get_network(name=None):
global NETWORKS
return NETWORKS.get(name or DEFAULT_NAME)
if name:
return NETWORKS.get(name)
if DEFAULT_NAME not in NETWORKS:
NETWORKS[DEFAULT_NAME] = Network({})
return NETWORKS[DEFAULT_NAME]
def initialize(settings_engines=None, settings_outgoing=None):
# pylint: disable=import-outside-toplevel)
from searx.engines import engines
from searx import settings
# pylint: enable=import-outside-toplevel)
global NETWORKS
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
settings_engines = settings_engines or settings["engines"]
settings_outgoing = settings_outgoing or settings["outgoing"]
# default parameters for AsyncHTTPTransport
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
default_params = {
'enable_http': False,
'verify': True,
'enable_http2': settings_outgoing['enable_http2'],
'max_connections': settings_outgoing['pool_connections'],
'max_keepalive_connections': settings_outgoing['pool_maxsize'],
'keepalive_expiry': settings_outgoing['keepalive_expiry'],
'local_addresses': settings_outgoing['source_ips'],
'proxies': settings_outgoing['proxies'],
'max_redirects': settings_outgoing['max_redirects'],
'retries': settings_outgoing['retries'],
'retry_on_http_error': None,
"enable_http": False,
"verify": True,
"enable_http2": settings_outgoing["enable_http2"],
"max_connections": settings_outgoing["pool_connections"],
"max_keepalive_connections": settings_outgoing["pool_maxsize"],
"keepalive_expiry": settings_outgoing["keepalive_expiry"],
"local_addresses": settings_outgoing["source_ips"],
"proxies": settings_outgoing["proxies"],
"max_redirects": settings_outgoing["max_redirects"],
"retries": settings_outgoing["retries"],
"retry_on_http_error": None,
}
def new_network(params):
@ -254,22 +287,22 @@ def initialize(settings_engines=None, settings_outgoing=None):
def iter_networks():
nonlocal settings_engines
for engine_spec in settings_engines:
engine_name = engine_spec['name']
engine_name = engine_spec["name"]
engine = engines.get(engine_name)
if engine is None:
continue
network = getattr(engine, 'network', None)
network = getattr(engine, "network", None)
yield engine_name, engine, network
if NETWORKS:
done()
Network.close_all()
NETWORKS.clear()
NETWORKS[DEFAULT_NAME] = new_network({})
NETWORKS['ipv4'] = new_network({'local_addresses': '0.0.0.0'})
NETWORKS['ipv6'] = new_network({'local_addresses': '::'})
NETWORKS["ipv4"] = new_network({"local_addresses": "0.0.0.0"})
NETWORKS["ipv6"] = new_network({"local_addresses": "::"})
# define networks from outgoing.networks
for network_name, network in settings_outgoing['networks'].items():
for network_name, network in settings_outgoing["networks"].items():
NETWORKS[network_name] = new_network(network)
# define networks from engines.[i].network (except references)
@ -289,29 +322,3 @@ def initialize(settings_engines=None, settings_outgoing=None):
for engine_name, engine, network in iter_networks():
if isinstance(network, str):
NETWORKS[engine_name] = NETWORKS[network]
@atexit.register
def done():
"""Close all HTTP client
Avoid a warning at exit
see https://github.com/encode/httpx/blob/1a6e254f72d9fd5694a1c10a28927e193ab4f76b/httpx/_client.py#L1785
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register
"""
global NETWORKS
try:
loop = get_loop()
if loop:
future = asyncio.run_coroutine_threadsafe(Network.aclose_all(), loop)
# wait 3 seconds to close the HTTP clients
future.result(3)
except:
logger.exception('Exception while closing clients')
finally:
NETWORKS.clear()
NETWORKS[DEFAULT_NAME] = Network()