Replace httpx by aiohttp

This commit is contained in:
Alexandre Flament 2021-08-07 12:07:51 +02:00
parent 065b4dab56
commit 4ea887471b
14 changed files with 533 additions and 352 deletions

View file

@ -4,6 +4,7 @@ import typing
import types
import functools
import itertools
import asyncio
from time import time
from timeit import default_timer
from urllib.parse import urlparse
@ -11,7 +12,7 @@ from urllib.parse import urlparse
import re
from langdetect import detect_langs
from langdetect.lang_detect_exception import LangDetectException
import httpx
import aiohttp
from searx import network, logger
from searx.results import ResultContainer
@ -91,10 +92,10 @@ def _is_url_image(image_url):
if r.headers["content-type"].startswith('image/'):
return True
return False
except httpx.TimeoutException:
except asyncio.TimeoutError:
logger.error('Timeout for %s: %i', image_url, int(time() - a))
retry -= 1
except httpx.HTTPError:
except aiohttp.ClientError:
logger.exception('Exception for %s', image_url)
return False

View file

@ -7,7 +7,8 @@
from timeit import default_timer
import asyncio
import httpx
import aiohttp
import python_socks
import searx.network
from searx import logger
@ -143,7 +144,7 @@ class OnlineProcessor(EngineProcessor):
# send requests and parse the results
search_results = self._search_basic(query, params)
self.extend_container(result_container, start_time, search_results)
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
except (asyncio.TimeoutError, python_socks.ProxyTimeoutError) as e:
# requests timeout (connect or read)
self.handle_exception(result_container, e, suspend=True)
logger.error("engine {0} : HTTP requests timeout"
@ -151,7 +152,7 @@ class OnlineProcessor(EngineProcessor):
.format(self.engine_name, default_timer() - start_time,
timeout_limit,
e.__class__.__name__))
except (httpx.HTTPError, httpx.StreamError) as e:
except (aiohttp.ClientError, python_socks.ProxyError, python_socks.ProxyConnectionError) as e:
# other requests exception
self.handle_exception(result_container, e, suspend=True)
logger.exception("engine {0} : requests exception"