mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
122a9568de
Anecdotally, using SearX over unreliable proxies,
like tor, seems to be quite error prone.
SearX puts quite an effort to measure the
performance and reliability of engines, most
likely owning to those aspects being of
significant concern.
The patch here proposes to mitigate related
problems, by issuing concurrent redundant requests
through the specified proxies at once, returning
the first response that is not an error.
The functionality is enabled using the:
`proxy_request_redundancy` parameter within the
outgoing network settings or the engine settings.
Example:
```yaml
outgoing:
request_timeout: 8.0
proxies:
"all://":
- socks5h://tor:9050
- socks5h://tor1:9050
- socks5h://tor2:9050
- socks5h://tor3:9050
proxy_request_redundancy: 4
```
In this example, each network request will be
send 4 times, once through every proxy. The
first (non-error) response wins.
In my testing environment using several tor proxy
end-points, this approach almost entirely
removes engine errors related to timeouts
and denied requests. The latency of the
network system is also improved.
The implementation, uses a
`AsyncParallelTransport(httpx.AsyncBaseTransport)`
wrapper to wrap multiple sub-trasports,
and `asyncio.wait` to wait on the first completed
request.
The existing implementation of the network
proxy cycling has also been moved into the
`AsyncParallelTransport` class, which should
improve network client memoization and
performance.
TESTED:
- unit tests for the new functions and classes.
- tested on desktop PC with 10+ upstream proxies
and comparable request redundancy.
128 lines
5.5 KiB
Python
128 lines
5.5 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
"""Test module for the client and proxy handling code."""
|
|
|
|
from unittest.mock import patch, Mock
|
|
|
|
import httpx
|
|
|
|
from searx.network import client
|
|
from tests import SearxTestCase
|
|
|
|
|
|
class TestClient(SearxTestCase):
|
|
"""Tests for the client and proxy handling code."""
|
|
|
|
def test_get_single_transport(self):
|
|
t = client.get_single_transport(proxy_url="socks4://local:1080")
|
|
assert isinstance(t, client.AsyncProxyTransportFixed)
|
|
t = client.get_single_transport(proxy_url="socks5://local:1080")
|
|
assert isinstance(t, client.AsyncProxyTransportFixed)
|
|
t = client.get_single_transport(proxy_url="socks5h://local:1080")
|
|
assert isinstance(t, client.AsyncProxyTransportFixed)
|
|
t = client.get_single_transport(proxy_url="https://local:8080")
|
|
assert isinstance(t, httpx.AsyncHTTPTransport)
|
|
|
|
def test_get_parallel_transport(self):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
)
|
|
assert isinstance(t, client.AsyncParallelTransport)
|
|
|
|
@patch(
|
|
'searx.network.client.AsyncProxyTransportFixed.handle_async_request',
|
|
side_effect=[httpx.Response(200, html="<html/>"), httpx.Response(301, html="<html/>")],
|
|
)
|
|
async def test_parallel_transport_ok(self, handler_mock: Mock):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
)
|
|
request = httpx.Request(url="http://wiki.com", method="GET")
|
|
response = await t.handle_async_request(request)
|
|
assert response.status_code == 200
|
|
handler_mock.assert_called_once_with(request)
|
|
|
|
response = await t.handle_async_request(request)
|
|
assert response.status_code == 301
|
|
assert handler_mock.call_count == 2
|
|
|
|
@patch(
|
|
'searx.network.client.AsyncProxyTransportFixed.handle_async_request',
|
|
side_effect=[httpx.Response(403, html="<html/>"), httpx.Response(200, html="<html/>")],
|
|
)
|
|
async def test_parallel_transport_403(self, handler_mock: Mock):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
proxy_request_redundancy=2,
|
|
)
|
|
assert isinstance(t, client.AsyncParallelTransport)
|
|
request = httpx.Request(url="http://wiki.com", method="GET")
|
|
response = await t.handle_async_request(request)
|
|
handler_mock.assert_called_with(request)
|
|
assert response.status_code == 200
|
|
assert handler_mock.call_count == 2
|
|
|
|
@patch(
|
|
'searx.network.client.AsyncProxyTransportFixed.handle_async_request',
|
|
side_effect=[httpx.Response(404, html="<html/>"), httpx.Response(200, html="<html/>")],
|
|
)
|
|
async def test_parallel_transport_404(self, handler_mock: Mock):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
proxy_request_redundancy=2,
|
|
)
|
|
assert isinstance(t, client.AsyncParallelTransport)
|
|
request = httpx.Request(url="http://wiki.com", method="GET")
|
|
response = await t.handle_async_request(request)
|
|
handler_mock.assert_called_with(request)
|
|
assert response.status_code == 404
|
|
assert handler_mock.call_count == 2
|
|
|
|
@patch(
|
|
'searx.network.client.AsyncProxyTransportFixed.handle_async_request',
|
|
side_effect=[httpx.Response(403, html="<html/>"), httpx.Response(403, html="<html/>")],
|
|
)
|
|
async def test_parallel_transport_403_403(self, handler_mock: Mock):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
proxy_request_redundancy=2,
|
|
)
|
|
assert isinstance(t, client.AsyncParallelTransport)
|
|
request = httpx.Request(url="http://wiki.com", method="GET")
|
|
response = await t.handle_async_request(request)
|
|
handler_mock.assert_called_with(request)
|
|
assert response.status_code == 403
|
|
assert handler_mock.call_count == 2
|
|
|
|
@patch(
|
|
'searx.network.client.AsyncProxyTransportFixed.handle_async_request',
|
|
side_effect=[httpx.RequestError("OMG!"), httpx.Response(200, html="<html/>")],
|
|
)
|
|
async def test_parallel_transport_ex_ok(self, handler_mock: Mock):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
proxy_request_redundancy=2,
|
|
)
|
|
assert isinstance(t, client.AsyncParallelTransport)
|
|
request = httpx.Request(url="http://wiki.com", method="GET")
|
|
response = await t.handle_async_request(request)
|
|
handler_mock.assert_called_with(request)
|
|
assert response.status_code == 200
|
|
assert handler_mock.call_count == 2
|
|
|
|
@patch(
|
|
'searx.network.client.AsyncProxyTransportFixed.handle_async_request',
|
|
side_effect=[httpx.RequestError("OMG!"), httpx.RequestError("OMG!")],
|
|
)
|
|
async def test_parallel_transport_ex_ex(self, handler_mock: Mock):
|
|
t = client.get_transport(
|
|
proxy_urls=["socks5h://local:1080", "socks5h://local:1180"],
|
|
proxy_request_redundancy=2,
|
|
)
|
|
assert isinstance(t, client.AsyncParallelTransport)
|
|
request = httpx.Request(url="http://wiki.com", method="GET")
|
|
response = None
|
|
with self.assertRaises(httpx.RequestError):
|
|
response = await t.handle_async_request(request)
|
|
handler_mock.assert_called_with(request)
|
|
assert not response
|
|
assert handler_mock.call_count == 2
|