diff --git a/searx/network/__init__.py b/searx/network/__init__.py index c921bdecb..9e80a30a1 100644 --- a/searx/network/__init__.py +++ b/searx/network/__init__.py @@ -5,6 +5,7 @@ import asyncio import threading import concurrent.futures +from types import MethodType from timeit import default_timer import httpx @@ -161,19 +162,32 @@ def patch(url, data=None, **kwargs): def delete(url, **kwargs): return request('delete', url, **kwargs) + async def stream_chunk_to_queue(network, queue, method, url, **kwargs): try: async with network.stream(method, url, **kwargs) as response: queue.put(response) - async for chunk in response.aiter_bytes(65536): + # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding + # https://www.python-httpx.org/quickstart/#streaming-responses + async for chunk in response.aiter_raw(65536): if len(chunk) > 0: queue.put(chunk) + except httpx.ResponseClosed as e: + # the response was closed + pass except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e: queue.put(e) finally: queue.put(None) +def _close_response_method(self): + asyncio.run_coroutine_threadsafe( + self.aclose(), + get_loop() + ) + + def stream(method, url, **kwargs): """Replace httpx.stream. @@ -191,10 +205,19 @@ def stream(method, url, **kwargs): stream_chunk_to_queue(get_network(), queue, method, url, **kwargs), get_loop() ) + + # yield response + response = queue.get() + if isinstance(response, Exception): + raise response + response.close = MethodType(_close_response_method, response) + yield response + + # yield chunks chunk_or_exception = queue.get() while chunk_or_exception is not None: if isinstance(chunk_or_exception, Exception): raise chunk_or_exception yield chunk_or_exception chunk_or_exception = queue.get() - return future.result() + future.result() diff --git a/searx/network/network.py b/searx/network/network.py index e7dc5b56e..94e91593d 100644 --- a/searx/network/network.py +++ b/searx/network/network.py @@ -289,6 +289,14 @@ def initialize(settings_engines=None, settings_outgoing=None): if isinstance(network, str): NETWORKS[engine_name] = NETWORKS[network] + # the /image_proxy endpoint has a dedicated network. + # same parameters than the default network, but HTTP/2 is disabled. + # It decreases the CPU load average, and the total time is more or less the same + if 'image_proxy' not in NETWORKS: + image_proxy_params = default_params.copy() + image_proxy_params['enable_http2'] = False + NETWORKS['image_proxy'] = new_network(image_proxy_params) + @atexit.register def done(): diff --git a/searx/utils.py b/searx/utils.py index 5e2ab18c4..22824d829 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -262,11 +262,7 @@ def dict_subset(d, properties): >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D']) {'A': 'a'} """ - result = {} - for k in properties: - if k in d: - result[k] = d[k] - return result + return {k: d[k] for k in properties if k in d} def get_torrent_size(filesize, filesize_multiplier): diff --git a/searx/webapp.py b/searx/webapp.py index a1abea887..3b7e0f972 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -108,7 +108,7 @@ from searx.autocomplete import search_autocomplete, backends as autocomplete_bac from searx.languages import language_codes as languages from searx.locales import LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES from searx.search import SearchWithPlugins, initialize as search_initialize -from searx.network import stream as http_stream +from searx.network import stream as http_stream, set_context_network_name from searx.search.checker import get_result as checker_get_result from searx.settings_loader import get_default_settings_path @@ -1065,7 +1065,7 @@ def _is_selected_language_supported(engine, preferences): # pylint: disable=red @app.route('/image_proxy', methods=['GET']) def image_proxy(): - # pylint: disable=too-many-return-statements + # pylint: disable=too-many-return-statements, too-many-branches url = request.args.get('url') if not url: @@ -1076,17 +1076,21 @@ def image_proxy(): return '', 400 maximum_size = 5 * 1024 * 1024 - + forward_resp = False + resp = None try: - headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'}) - headers['User-Agent'] = gen_useragent() + request_headers = { + 'User-Agent': gen_useragent(), + 'Accept': 'image/webp,*/*', + 'Accept-Encoding': 'gzip, deflate', + 'Sec-GPC': '1', + 'DNT': '1', + } + set_context_network_name('image_proxy') stream = http_stream( method = 'GET', url = url, - headers = headers, - timeout = settings['outgoing']['request_timeout'], - allow_redirects = True, - max_redirects = 20 + headers = request_headers ) resp = next(stream) content_length = resp.headers.get('Content-Length') @@ -1095,32 +1099,37 @@ def image_proxy(): and int(content_length) > maximum_size ): return 'Max size', 400 - if resp.status_code == 304: - return '', resp.status_code - if resp.status_code != 200: - logger.debug( - 'image-proxy: wrong response code: {0}'.format( - resp.status_code)) + logger.debug('image-proxy: wrong response code: %i', resp.status_code) if resp.status_code >= 400: return '', resp.status_code return '', 400 - if not resp.headers.get('content-type', '').startswith('image/'): - logger.debug( - 'image-proxy: wrong content-type: {0}'.format( - resp.headers.get('content-type'))) + if not resp.headers.get('Content-Type', '').startswith('image/'): + logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', '')) return '', 400 + forward_resp = True + except httpx.HTTPError: + logger.exception('HTTP error') + return '', 400 + finally: + if resp and not forward_resp: + # the code is about to return an HTTP 400 error to the browser + # we make sure to close the response between searxng and the HTTP server + try: + resp.close() + except httpx.HTTPError: + logger.exception('HTTP error on closing') + + try: headers = dict_subset( resp.headers, - {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'} + {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'} ) - total_length = 0 - def forward_chunk(): - nonlocal total_length + total_length = 0 for chunk in stream: total_length += len(chunk) if total_length > maximum_size: