diff --git a/docs/admin/settings/settings_outgoing.rst b/docs/admin/settings/settings_outgoing.rst index 96b3466cd..c912c64fa 100644 --- a/docs/admin/settings/settings_outgoing.rst +++ b/docs/admin/settings/settings_outgoing.rst @@ -37,7 +37,7 @@ Communication with search engines. # Uncomment below if you want to make multiple request in parallel # through all the proxies at once: # - # proxy_request_redundancy: 10 + # proxy_request_redundancy: 4 # # Extra seconds to add in order to account for the time taken by the proxy # diff --git a/searx/network/client.py b/searx/network/client.py index f68248976..699a2cc4d 100644 --- a/searx/network/client.py +++ b/searx/network/client.py @@ -207,16 +207,16 @@ class AsyncParallelTransport(httpx.AsyncBaseTransport): network_logger: logging.Logger, ) -> None: """Init the parallel transport using a list of base `transports`.""" + self._logger = network_logger or logger self._transports = list(transports) if len(self._transports) == 0: msg = "Got an empty list of (proxy) transports." raise ValueError(msg) if proxy_request_redundancy < 1: - logger.warning("Invalid proxy_request_redundancy specified: %d", proxy_request_redundancy) + self._logger.warning("Invalid proxy_request_redundancy specified: %d", proxy_request_redundancy) proxy_request_redundancy = 1 self._proxy_request_redundancy = proxy_request_redundancy self._index = random.randrange(len(self._transports)) # noqa: S311 - self._logger = network_logger or logger async def handle_async_request( self, @@ -258,7 +258,7 @@ class AsyncParallelTransport(httpx.AsyncBaseTransport): if not result.is_error: response = result elif result.status_code == 404 and response is None: - response = result + error_response = response = result elif not error_response: self._logger.warning("Error response: %s for %s", result.status_code, request.url) error_response = result diff --git a/searx/network/network.py b/searx/network/network.py index 4941349b5..9613384c3 100644 --- a/searx/network/network.py +++ b/searx/network/network.py @@ -147,7 +147,8 @@ class Network: else: yield pattern, tuple(proxy_urls) else: - raise ValueError("`proxies` need to be either a string or a patthern to url dictionary.") + msg = "`proxies` need to be either a string or a patthern to url dictionary." + raise ValueError(msg) async def log_response(self, response: httpx.Response): request = response.request @@ -351,6 +352,7 @@ def initialize(settings_engines=None, settings_outgoing=None): } def new_network(params, logger_name=None): + nonlocal default_params result = {} result.update(default_params) result.update(params) @@ -358,7 +360,8 @@ def initialize(settings_engines=None, settings_outgoing=None): result['logger_name'] = logger_name return Network(**result) - def iter_engine_networks(): + def iter_networks(): + nonlocal settings_engines for engine_spec in settings_engines: engine_name = engine_spec['name'] engine = engines.get(engine_name) @@ -379,7 +382,7 @@ def initialize(settings_engines=None, settings_outgoing=None): NETWORKS[network_name] = new_network(network, logger_name=network_name) # define networks from engines.[i].network (except references) - for engine_name, engine, network in iter_engine_networks(): + for engine_name, engine, network in iter_networks(): if network is None: network = {} for attribute_name, attribute_value in default_params.items(): @@ -392,7 +395,7 @@ def initialize(settings_engines=None, settings_outgoing=None): NETWORKS[engine_name] = new_network(network, logger_name=engine_name) # define networks from engines.[i].network (references) - for engine_name, engine, network in iter_engine_networks(): + for engine_name, engine, network in iter_networks(): if isinstance(network, str): NETWORKS[engine_name] = NETWORKS[network] diff --git a/searx/settings.yml b/searx/settings.yml index 620cfb759..1743d2554 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -193,7 +193,7 @@ outgoing: # Uncomment below if you want to make multiple request in parallel # through all the proxies at once: # - # proxy_request_redundancy: 10 + # proxy_request_redundancy: 4 # # Extra seconds to add in order to account for the time taken by the proxy #