forked from zaclys/searxng
Merge pull request #192 from dalf/connection-pool
[enh] improve response time. close #100
This commit is contained in:
commit
10891bdeab
@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||
|
||||
|
||||
from lxml import etree
|
||||
from requests import get
|
||||
from json import loads
|
||||
from urllib import urlencode
|
||||
from searx.languages import language_codes
|
||||
from searx.engines import (
|
||||
categories, engines, engine_shortcuts
|
||||
)
|
||||
from searx.poolrequests import get
|
||||
|
||||
|
||||
def searx_bang(full_query):
|
||||
|
@ -1,6 +1,6 @@
|
||||
import json
|
||||
from requests import get
|
||||
from urllib import urlencode
|
||||
from searx.poolrequests import get
|
||||
from searx.utils import format_date_by_locale
|
||||
|
||||
result_count = 1
|
||||
|
61
searx/poolrequests.py
Normal file
61
searx/poolrequests.py
Normal file
@ -0,0 +1,61 @@
|
||||
import requests
|
||||
|
||||
|
||||
the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
|
||||
the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
|
||||
|
||||
|
||||
class SessionSinglePool(requests.Session):
|
||||
|
||||
def __init__(self):
|
||||
global the_https_adapter, the_http_adapter
|
||||
super(SessionSinglePool, self).__init__()
|
||||
|
||||
# reuse the same adapters
|
||||
self.adapters.clear()
|
||||
self.mount('https://', the_https_adapter)
|
||||
self.mount('http://', the_http_adapter)
|
||||
|
||||
def close(self):
|
||||
"""Call super, but clear adapters since there are managed globaly"""
|
||||
self.adapters.clear()
|
||||
super(SessionSinglePool, self).close()
|
||||
|
||||
|
||||
def request(method, url, **kwargs):
|
||||
"""same as requests/requests/api.py request(...) except it use SessionSinglePool"""
|
||||
session = SessionSinglePool()
|
||||
response = session.request(method=method, url=url, **kwargs)
|
||||
session.close()
|
||||
return response
|
||||
|
||||
|
||||
def get(url, **kwargs):
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return request('get', url, **kwargs)
|
||||
|
||||
|
||||
def options(url, **kwargs):
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return request('options', url, **kwargs)
|
||||
|
||||
|
||||
def head(url, **kwargs):
|
||||
kwargs.setdefault('allow_redirects', False)
|
||||
return request('head', url, **kwargs)
|
||||
|
||||
|
||||
def post(url, data=None, json=None, **kwargs):
|
||||
return request('post', url, data=data, json=json, **kwargs)
|
||||
|
||||
|
||||
def put(url, data=None, **kwargs):
|
||||
return request('put', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def patch(url, data=None, **kwargs):
|
||||
return request('patch', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def delete(url, **kwargs):
|
||||
return request('delete', url, **kwargs)
|
@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
||||
'''
|
||||
|
||||
import requests as requests_lib
|
||||
import threading
|
||||
import re
|
||||
import searx.poolrequests as requests_lib
|
||||
from itertools import izip_longest, chain
|
||||
from operator import itemgetter
|
||||
from Queue import Queue
|
||||
@ -31,7 +31,6 @@ from searx.utils import gen_useragent
|
||||
from searx.query import Query
|
||||
from searx import logger
|
||||
|
||||
|
||||
logger = logger.getChild('search')
|
||||
|
||||
number_of_searches = 0
|
||||
|
@ -28,7 +28,6 @@ import os
|
||||
import hashlib
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from requests import get as http_get
|
||||
from itertools import chain
|
||||
from urllib import urlencode
|
||||
from flask import (
|
||||
@ -37,6 +36,7 @@ from flask import (
|
||||
)
|
||||
from flask.ext.babel import Babel, gettext, format_date
|
||||
from searx import settings, searx_dir
|
||||
from searx.poolrequests import get as http_get
|
||||
from searx.engines import (
|
||||
categories, engines, get_engines_stats, engine_shortcuts
|
||||
)
|
||||
|
Loading…
Reference in New Issue
Block a user