summaryrefslogtreecommitdiff
path: root/searx/poolrequests.py
diff options
context:
space:
mode:
authorKang-min Liu <gugod@gugod.org>2015-11-14 00:05:44 +0100
committerKang-min Liu <gugod@gugod.org>2015-11-14 00:05:44 +0100
commitac8759cd3ff99024864fd04d7c4bef5c3a00b971 (patch)
tree30c3f8b61504532df926bbffedcc8df80a8e926e /searx/poolrequests.py
parentc7c6c35ccd7373d2107b70b92badb9b70d31905f (diff)
parente98aef6fc4954681e58d774203d522f0ae478004 (diff)
Merge remote-tracking branch 'origin/master'
Diffstat (limited to 'searx/poolrequests.py')
-rw-r--r--searx/poolrequests.py18
1 files changed, 10 insertions, 8 deletions
diff --git a/searx/poolrequests.py b/searx/poolrequests.py
index e2a757665..4761f6ae8 100644
--- a/searx/poolrequests.py
+++ b/searx/poolrequests.py
@@ -1,5 +1,7 @@
import requests
+
from itertools import cycle
+from threading import RLock
from searx import settings
@@ -39,11 +41,11 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
block=self._pool_block, **self._conn_params)
-if settings.get('source_ips'):
+if settings['outgoing'].get('source_ips'):
http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
- for source_ip in settings['source_ips'])
+ for source_ip in settings['outgoing']['source_ips'])
https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
- for source_ip in settings['source_ips'])
+ for source_ip in settings['outgoing']['source_ips'])
else:
http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
@@ -55,9 +57,10 @@ class SessionSinglePool(requests.Session):
super(SessionSinglePool, self).__init__()
# reuse the same adapters
- self.adapters.clear()
- self.mount('https://', next(https_adapters))
- self.mount('http://', next(http_adapters))
+ with RLock():
+ self.adapters.clear()
+ self.mount('https://', next(https_adapters))
+ self.mount('http://', next(http_adapters))
def close(self):
"""Call super, but clear adapters since there are managed globaly"""
@@ -67,9 +70,8 @@ class SessionSinglePool(requests.Session):
def request(method, url, **kwargs):
"""same as requests/requests/api.py request(...) except it use SessionSinglePool and force proxies"""
- global settings
session = SessionSinglePool()
- kwargs['proxies'] = settings.get('outgoing_proxies', None)
+ kwargs['proxies'] = settings['outgoing'].get('proxies', None)
response = session.request(method=method, url=url, **kwargs)
session.close()
return response