From d07cfd9089d05a6a81109518f03ba82660a4aef7 Mon Sep 17 00:00:00 2001 From: dalf Date: Wed, 21 Jan 2015 11:33:16 +0100 Subject: [enh] use one single http connection pool : improve response time. close #100 --- searx/poolrequests.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 searx/poolrequests.py (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py new file mode 100644 index 000000000..ef7a195f5 --- /dev/null +++ b/searx/poolrequests.py @@ -0,0 +1,61 @@ +import requests + + +the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100) +the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100) + + +class SessionSinglePool(requests.Session): + + def __init__(self): + global the_https_adapter, the_http_adapter + super(SessionSinglePool, self).__init__() + + # reuse the same adapters + self.adapters.clear() + self.mount('https://', the_https_adapter) + self.mount('http://', the_http_adapter) + + def close(self): + """Call super, but clear adapters since there are managed globaly""" + self.adapters.clear() + super(SessionSinglePool, self).close() + + +def request(method, url, **kwargs): + """same as requests/requests/api.py request(...) except it use SessionSinglePool""" + session = SessionSinglePool() + response = session.request(method=method, url=url, **kwargs) + session.close() + return response + + +def get(url, **kwargs): + kwargs.setdefault('allow_redirects', True) + return request('get', url, **kwargs) + + +def options(url, **kwargs): + kwargs.setdefault('allow_redirects', True) + return request('options', url, **kwargs) + + +def head(url, **kwargs): + kwargs.setdefault('allow_redirects', False) + return request('head', url, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + return request('post', url, data=data, json=json, **kwargs) + + +def put(url, data=None, **kwargs): + return request('put', url, data=data, **kwargs) + + +def patch(url, data=None, **kwargs): + return request('patch', url, data=data, **kwargs) + + +def delete(url, **kwargs): + return request('delete', url, **kwargs) -- cgit v1.2.3