mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
Replace httpx by aiohttp
This commit is contained in:
Alexandre Flament
parent
065b4dab56
commit
4ea887471b
14 changed files with 533 additions and 352 deletions
|
|
@ -1,13 +1,59 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import asyncio
|
||||
from searx.network.utils import URLPattern
|
||||
from mock import patch
|
||||
|
||||
import httpx
|
||||
import aiohttp
|
||||
from multidict import CIMultiDict, CIMultiDictProxy
|
||||
import yarl
|
||||
|
||||
from searx.network.network import Network, NETWORKS, initialize
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
def create_fake_response(url, method='GET', content='', status_code=200):
|
||||
if isinstance(url, str):
|
||||
url = yarl.URL(url)
|
||||
if not isinstance(url, yarl.URL):
|
||||
raise ValueError('url must be of type yarl.URL. Currently of type ' + str(type(url)))
|
||||
loop = asyncio.get_event_loop()
|
||||
request_info = aiohttp.RequestInfo(
|
||||
url,
|
||||
method,
|
||||
CIMultiDictProxy(CIMultiDict()),
|
||||
url
|
||||
)
|
||||
response = aiohttp.ClientResponse(
|
||||
method,
|
||||
url,
|
||||
writer=None,
|
||||
continue100=False,
|
||||
timer=None,
|
||||
request_info=request_info,
|
||||
traces=[],
|
||||
loop=loop,
|
||||
session=None
|
||||
)
|
||||
|
||||
async def async_nothing():
|
||||
pass
|
||||
|
||||
def iter_content(_):
|
||||
yield content.encode()
|
||||
|
||||
response.status = status_code
|
||||
response._headers = {}
|
||||
response.read = async_nothing
|
||||
response.close = lambda: None
|
||||
response.release = async_nothing
|
||||
response.content = content.encode()
|
||||
response._body = response.content
|
||||
response.get_encoding = lambda: 'utf-8'
|
||||
response.iter_content = iter_content
|
||||
return response
|
||||
|
||||
|
||||
class TestNetwork(SearxTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -49,25 +95,26 @@ class TestNetwork(SearxTestCase):
|
|||
|
||||
def test_proxy_cycles(self):
|
||||
network = Network(proxies='http://localhost:1337')
|
||||
self.assertEqual(next(network._proxies_cycle), (('all://', 'http://localhost:1337'),))
|
||||
P = URLPattern
|
||||
self.assertEqual(next(network._proxies_cycle), ((P('all://'), 'http://localhost:1337'),))
|
||||
|
||||
network = Network(proxies={
|
||||
'https': 'http://localhost:1337',
|
||||
'http': 'http://localhost:1338'
|
||||
})
|
||||
self.assertEqual(next(network._proxies_cycle),
|
||||
(('https://', 'http://localhost:1337'), ('http://', 'http://localhost:1338')))
|
||||
((P('https://'), 'http://localhost:1337'), (P('http://'), 'http://localhost:1338')))
|
||||
self.assertEqual(next(network._proxies_cycle),
|
||||
(('https://', 'http://localhost:1337'), ('http://', 'http://localhost:1338')))
|
||||
((P('https://'), 'http://localhost:1337'), (P('http://'), 'http://localhost:1338')))
|
||||
|
||||
network = Network(proxies={
|
||||
'https': ['http://localhost:1337', 'http://localhost:1339'],
|
||||
'http': 'http://localhost:1338'
|
||||
})
|
||||
self.assertEqual(next(network._proxies_cycle),
|
||||
(('https://', 'http://localhost:1337'), ('http://', 'http://localhost:1338')))
|
||||
((P('https://'), 'http://localhost:1337'), (P('http://'), 'http://localhost:1338')))
|
||||
self.assertEqual(next(network._proxies_cycle),
|
||||
(('https://', 'http://localhost:1339'), ('http://', 'http://localhost:1338')))
|
||||
((P('https://'), 'http://localhost:1339'), (P('http://'), 'http://localhost:1338')))
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
Network(proxies=1)
|
||||
|
|
@ -75,45 +122,38 @@ class TestNetwork(SearxTestCase):
|
|||
def test_get_kwargs_clients(self):
|
||||
kwargs = {
|
||||
'verify': True,
|
||||
'max_redirects': 5,
|
||||
'timeout': 2,
|
||||
}
|
||||
kwargs_client = Network.get_kwargs_clients(kwargs)
|
||||
|
||||
self.assertEqual(len(kwargs_client), 2)
|
||||
self.assertEqual(len(kwargs), 1)
|
||||
|
||||
self.assertEqual(kwargs['timeout'], 2)
|
||||
self.assertEqual(len(kwargs_client), 1)
|
||||
self.assertEqual(len(kwargs), 0)
|
||||
|
||||
self.assertTrue(kwargs_client['verify'])
|
||||
self.assertEqual(kwargs_client['max_redirects'], 5)
|
||||
|
||||
async def test_get_client(self):
|
||||
network = Network(verify=True)
|
||||
client1 = network.get_client()
|
||||
client2 = network.get_client(verify=True)
|
||||
client3 = network.get_client(max_redirects=10)
|
||||
client4 = network.get_client(verify=True)
|
||||
client5 = network.get_client(verify=False)
|
||||
client6 = network.get_client(max_redirects=10)
|
||||
url = 'https://example.com'
|
||||
client1 = network.get_client(url)
|
||||
client2 = network.get_client(url, verify=True)
|
||||
client3 = network.get_client(url, verify=False)
|
||||
|
||||
self.assertEqual(client1, client2)
|
||||
self.assertEqual(client1, client4)
|
||||
self.assertNotEqual(client1, client3)
|
||||
self.assertNotEqual(client1, client5)
|
||||
self.assertEqual(client3, client6)
|
||||
|
||||
await network.aclose()
|
||||
|
||||
async def test_aclose(self):
|
||||
network = Network(verify=True)
|
||||
network.get_client()
|
||||
network.get_client('https://example.com')
|
||||
await network.aclose()
|
||||
|
||||
async def test_request(self):
|
||||
a_text = 'Lorem Ipsum'
|
||||
response = httpx.Response(status_code=200, text=a_text)
|
||||
with patch.object(httpx.AsyncClient, 'request', return_value=response):
|
||||
|
||||
async def get_response(*args, **kwargs):
|
||||
return create_fake_response(url='https://example.com/', status_code=200, content=a_text)
|
||||
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=get_response):
|
||||
network = Network(enable_http=True)
|
||||
response = await network.request('GET', 'https://example.com/')
|
||||
self.assertEqual(response.text, a_text)
|
||||
|
|
@ -128,37 +168,47 @@ class TestNetworkRequestRetries(SearxTestCase):
|
|||
def get_response_404_then_200(cls):
|
||||
first = True
|
||||
|
||||
async def get_response(*args, **kwargs):
|
||||
async def get_response(method, url, *args, **kwargs):
|
||||
nonlocal first
|
||||
if first:
|
||||
first = False
|
||||
return httpx.Response(status_code=403, text=TestNetworkRequestRetries.TEXT)
|
||||
return httpx.Response(status_code=200, text=TestNetworkRequestRetries.TEXT)
|
||||
return create_fake_response(
|
||||
method=method,
|
||||
url=url,
|
||||
status_code=403,
|
||||
content=TestNetworkRequestRetries.TEXT
|
||||
)
|
||||
return create_fake_response(
|
||||
method=method,
|
||||
url=url,
|
||||
status_code=200,
|
||||
content=TestNetworkRequestRetries.TEXT
|
||||
)
|
||||
return get_response
|
||||
|
||||
async def test_retries_ok(self):
|
||||
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
network = Network(enable_http=True, retries=1, retry_on_http_error=403)
|
||||
response = await network.request('GET', 'https://example.com/')
|
||||
self.assertEqual(response.text, TestNetworkRequestRetries.TEXT)
|
||||
await network.aclose()
|
||||
|
||||
async def test_retries_fail_int(self):
|
||||
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
network = Network(enable_http=True, retries=0, retry_on_http_error=403)
|
||||
response = await network.request('GET', 'https://example.com/')
|
||||
self.assertEqual(response.status_code, 403)
|
||||
await network.aclose()
|
||||
|
||||
async def test_retries_fail_list(self):
|
||||
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
network = Network(enable_http=True, retries=0, retry_on_http_error=[403, 429])
|
||||
response = await network.request('GET', 'https://example.com/')
|
||||
self.assertEqual(response.status_code, 403)
|
||||
await network.aclose()
|
||||
|
||||
async def test_retries_fail_bool(self):
|
||||
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
|
||||
network = Network(enable_http=True, retries=0, retry_on_http_error=True)
|
||||
response = await network.request('GET', 'https://example.com/')
|
||||
self.assertEqual(response.status_code, 403)
|
||||
|
|
@ -167,14 +217,19 @@ class TestNetworkRequestRetries(SearxTestCase):
|
|||
async def test_retries_exception_then_200(self):
|
||||
request_count = 0
|
||||
|
||||
async def get_response(*args, **kwargs):
|
||||
async def get_response(method, url, *args, **kwargs):
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
if request_count < 3:
|
||||
raise httpx.RequestError('fake exception', request=None)
|
||||
return httpx.Response(status_code=200, text=TestNetworkRequestRetries.TEXT)
|
||||
raise aiohttp.ClientError()
|
||||
return create_fake_response(
|
||||
url,
|
||||
method,
|
||||
status_code=200,
|
||||
content=TestNetworkRequestRetries.TEXT
|
||||
)
|
||||
|
||||
with patch.object(httpx.AsyncClient, 'request', new=get_response):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=get_response):
|
||||
network = Network(enable_http=True, retries=2)
|
||||
response = await network.request('GET', 'https://example.com/')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
|
@ -182,12 +237,12 @@ class TestNetworkRequestRetries(SearxTestCase):
|
|||
await network.aclose()
|
||||
|
||||
async def test_retries_exception(self):
|
||||
async def get_response(*args, **kwargs):
|
||||
raise httpx.RequestError('fake exception', request=None)
|
||||
def get_response(*args, **kwargs):
|
||||
raise aiohttp.ClientError()
|
||||
|
||||
with patch.object(httpx.AsyncClient, 'request', new=get_response):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=get_response):
|
||||
network = Network(enable_http=True, retries=0)
|
||||
with self.assertRaises(httpx.RequestError):
|
||||
with self.assertRaises(aiohttp.ClientError):
|
||||
await network.request('GET', 'https://example.com/')
|
||||
await network.aclose()
|
||||
|
||||
|
|
@ -200,40 +255,48 @@ class TestNetworkStreamRetries(SearxTestCase):
|
|||
def get_response_exception_then_200(cls):
|
||||
first = True
|
||||
|
||||
def stream(*args, **kwargs):
|
||||
async def stream(method, url, *args, **kwargs):
|
||||
nonlocal first
|
||||
if first:
|
||||
first = False
|
||||
raise httpx.RequestError('fake exception', request=None)
|
||||
return httpx.Response(status_code=200, text=TestNetworkStreamRetries.TEXT)
|
||||
raise aiohttp.ClientError()
|
||||
return create_fake_response(url, method, content=TestNetworkStreamRetries.TEXT, status_code=200)
|
||||
return stream
|
||||
|
||||
async def test_retries_ok(self):
|
||||
with patch.object(httpx.AsyncClient, 'stream', new=TestNetworkStreamRetries.get_response_exception_then_200()):
|
||||
with patch.object(
|
||||
aiohttp.ClientSession,
|
||||
'request',
|
||||
new=TestNetworkStreamRetries.get_response_exception_then_200()
|
||||
):
|
||||
network = Network(enable_http=True, retries=1, retry_on_http_error=403)
|
||||
response = network.stream('GET', 'https://example.com/')
|
||||
response = await network.request('GET', 'https://example.com/', read_response=False)
|
||||
self.assertEqual(response.text, TestNetworkStreamRetries.TEXT)
|
||||
await network.aclose()
|
||||
|
||||
async def test_retries_fail(self):
|
||||
with patch.object(httpx.AsyncClient, 'stream', new=TestNetworkStreamRetries.get_response_exception_then_200()):
|
||||
with patch.object(
|
||||
aiohttp.ClientSession,
|
||||
'request',
|
||||
new=TestNetworkStreamRetries.get_response_exception_then_200()
|
||||
):
|
||||
network = Network(enable_http=True, retries=0, retry_on_http_error=403)
|
||||
with self.assertRaises(httpx.RequestError):
|
||||
network.stream('GET', 'https://example.com/')
|
||||
with self.assertRaises(aiohttp.ClientError):
|
||||
await network.request('GET', 'https://example.com/', read_response=False)
|
||||
await network.aclose()
|
||||
|
||||
async def test_retries_exception(self):
|
||||
first = True
|
||||
|
||||
def stream(*args, **kwargs):
|
||||
async def request(method, url, *args, **kwargs):
|
||||
nonlocal first
|
||||
if first:
|
||||
first = False
|
||||
return httpx.Response(status_code=403, text=TestNetworkRequestRetries.TEXT)
|
||||
return httpx.Response(status_code=200, text=TestNetworkRequestRetries.TEXT)
|
||||
return create_fake_response(url, method, status_code=403, content=TestNetworkRequestRetries.TEXT)
|
||||
return create_fake_response(url, method, status_code=200, content=TestNetworkRequestRetries.TEXT)
|
||||
|
||||
with patch.object(httpx.AsyncClient, 'stream', new=stream):
|
||||
with patch.object(aiohttp.ClientSession, 'request', new=request):
|
||||
network = Network(enable_http=True, retries=0, retry_on_http_error=403)
|
||||
response = network.stream('GET', 'https://example.com/')
|
||||
response = await network.request('GET', 'https://example.com/', read_response=False)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
await network.aclose()
|
||||
|
|
|
|||
48
tests/unit/network/test_utils.py
Normal file
48
tests/unit/network/test_utils.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import random
|
||||
from yarl import URL
|
||||
|
||||
from searx.network.utils import URLPattern
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
class TestNetworkUtils(SearxTestCase):
|
||||
def test_pattern_priority(self):
|
||||
matchers = [
|
||||
URLPattern("all://"),
|
||||
URLPattern("http://"),
|
||||
URLPattern("http://example.com"),
|
||||
URLPattern("http://example.com:123"),
|
||||
]
|
||||
random.shuffle(matchers)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(
|
||||
sorted(matchers),
|
||||
[
|
||||
URLPattern("http://example.com:123"),
|
||||
URLPattern("http://example.com"),
|
||||
URLPattern("http://"),
|
||||
URLPattern("all://"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_url_matches(self):
|
||||
parameters = [
|
||||
("http://example.com", "http://example.com", True),
|
||||
("http://example.com", "https://example.com", False),
|
||||
("http://example.com", "http://other.com", False),
|
||||
("http://example.com:123", "http://example.com:123", True),
|
||||
("http://example.com:123", "http://example.com:456", False),
|
||||
("http://example.com:123", "http://example.com", False),
|
||||
("all://example.com", "http://example.com", True),
|
||||
("all://example.com", "https://example.com", True),
|
||||
("http://", "http://example.com", True),
|
||||
("http://", "https://example.com", False),
|
||||
("all://", "https://example.com:123", True),
|
||||
("", "https://example.com:123", True),
|
||||
]
|
||||
|
||||
for pattern, url, expected in parameters:
|
||||
pattern = URLPattern(pattern)
|
||||
self.assertEqual(pattern.matches(URL(url)), expected)
|
||||
Loading…
Add table
Add a link
Reference in a new issue