diff options
| author | Kang-min Liu <gugod@gugod.org> | 2015-11-14 00:05:44 +0100 |
|---|---|---|
| committer | Kang-min Liu <gugod@gugod.org> | 2015-11-14 00:05:44 +0100 |
| commit | ac8759cd3ff99024864fd04d7c4bef5c3a00b971 (patch) | |
| tree | 30c3f8b61504532df926bbffedcc8df80a8e926e /searx/poolrequests.py | |
| parent | c7c6c35ccd7373d2107b70b92badb9b70d31905f (diff) | |
| parent | e98aef6fc4954681e58d774203d522f0ae478004 (diff) | |
Merge remote-tracking branch 'origin/master'
Diffstat (limited to 'searx/poolrequests.py')
| -rw-r--r-- | searx/poolrequests.py | 18 |
1 files changed, 10 insertions, 8 deletions
diff --git a/searx/poolrequests.py b/searx/poolrequests.py index e2a757665..4761f6ae8 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -1,5 +1,7 @@ import requests + from itertools import cycle +from threading import RLock from searx import settings @@ -39,11 +41,11 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) -if settings.get('source_ips'): +if settings['outgoing'].get('source_ips'): http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) - for source_ip in settings['source_ips']) + for source_ip in settings['outgoing']['source_ips']) https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) - for source_ip in settings['source_ips']) + for source_ip in settings['outgoing']['source_ips']) else: http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) @@ -55,9 +57,10 @@ class SessionSinglePool(requests.Session): super(SessionSinglePool, self).__init__() # reuse the same adapters - self.adapters.clear() - self.mount('https://', next(https_adapters)) - self.mount('http://', next(http_adapters)) + with RLock(): + self.adapters.clear() + self.mount('https://', next(https_adapters)) + self.mount('http://', next(http_adapters)) def close(self): """Call super, but clear adapters since there are managed globaly""" @@ -67,9 +70,8 @@ class SessionSinglePool(requests.Session): def request(method, url, **kwargs): """same as requests/requests/api.py request(...) except it use SessionSinglePool and force proxies""" - global settings session = SessionSinglePool() - kwargs['proxies'] = settings.get('outgoing_proxies', None) + kwargs['proxies'] = settings['outgoing'].get('proxies', None) response = session.request(method=method, url=url, **kwargs) session.close() return response |