zaclys
/
searxng
mirror of https://github.com/searxng/searxng.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #549 from guyou/feature-pool-fine-tuning 

Use new settings to fine tune the connection pool
This commit is contained in:
Adam Tauber 2016-05-19 15:51:14 +02:00
parent 9247b1e777 d571a9d204
commit bf1eebb17f
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
searx/poolrequests.py
View File

@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
block=self._pool_block, **self._conn_params) block=self._pool_block, **self._conn_params)
connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code
maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor
if settings['outgoing'].get('source_ips'): if settings['outgoing'].get('source_ips'):
http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,
source_address=(source_ip, 0))
for source_ip in settings['outgoing']['source_ips']) for source_ip in settings['outgoing']['source_ips'])
https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,
source_address=(source_ip, 0))
for source_ip in settings['outgoing']['source_ips']) for source_ip in settings['outgoing']['source_ips'])
else: else:
http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))
https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))
class SessionSinglePool(requests.Session): class SessionSinglePool(requests.Session):

2
searx/settings.yml
View File

@ -21,6 +21,8 @@ ui:
outgoing: # communication with search engines outgoing: # communication with search engines
request_timeout : 2.0 # seconds request_timeout : 2.0 # seconds
useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
pool_connections : 100 # Number of different hosts
pool_maxsize : 10 # Number of simultaneous requests by host
# uncomment below section if you want to use a proxy # uncomment below section if you want to use a proxy
# see http://docs.python-requests.org/en/latest/user/advanced/#proxies # see http://docs.python-requests.org/en/latest/user/advanced/#proxies
# SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478 # SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478