From 1fcf066a8188b28eb644ea304a131d40b1b341eb Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 2 Aug 2015 19:38:27 +0200 Subject: [mod] change settings file structure according to #314 --- searx/poolrequests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py index e2a757665..c44bdc7e2 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -39,11 +39,11 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) -if settings.get('source_ips'): +if settings['outgoing'].get('source_ips'): http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) - for source_ip in settings['source_ips']) + for source_ip in settings['outgoing']['source_ips']) https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) - for source_ip in settings['source_ips']) + for source_ip in settings['outgoing']['source_ips']) else: http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) @@ -69,7 +69,7 @@ def request(method, url, **kwargs): """same as requests/requests/api.py request(...) except it use SessionSinglePool and force proxies""" global settings session = SessionSinglePool() - kwargs['proxies'] = settings.get('outgoing_proxies', None) + kwargs['proxies'] = settings['outgoing'].get('proxies', None) response = session.request(method=method, url=url, **kwargs) session.close() return response -- cgit v1.2.3 From 357fc47811a1de5a273622881574dbde23c7cb59 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 31 Oct 2015 14:40:59 +0100 Subject: [fix] lock request pool generator --- searx/poolrequests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py index c44bdc7e2..4761f6ae8 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -1,5 +1,7 @@ import requests + from itertools import cycle +from threading import RLock from searx import settings @@ -55,9 +57,10 @@ class SessionSinglePool(requests.Session): super(SessionSinglePool, self).__init__() # reuse the same adapters - self.adapters.clear() - self.mount('https://', next(https_adapters)) - self.mount('http://', next(http_adapters)) + with RLock(): + self.adapters.clear() + self.mount('https://', next(https_adapters)) + self.mount('http://', next(http_adapters)) def close(self): """Call super, but clear adapters since there are managed globaly""" @@ -67,7 +70,6 @@ class SessionSinglePool(requests.Session): def request(method, url, **kwargs): """same as requests/requests/api.py request(...) except it use SessionSinglePool and force proxies""" - global settings session = SessionSinglePool() kwargs['proxies'] = settings['outgoing'].get('proxies', None) response = session.request(method=method, url=url, **kwargs) -- cgit v1.2.3