From f18a3a77ef894d6563543fe449321e52b16e93ad Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Thu, 28 Apr 2016 13:59:41 +0200 Subject: Use new settings to fine tune the connection pool In some circumstances, it is necessary to increase size of pool. --- searx/poolrequests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 13c6a906e..54e8dbcf6 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) +connect=settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code +maxsize=settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor if settings['outgoing'].get('source_ips'): - http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) - https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) else: - http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) - https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) + http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) + https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) class SessionSinglePool(requests.Session): -- cgit v1.2.3 From 3f395304f96a7c71390dd0a79d0ba2b3c404d504 Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Mon, 2 May 2016 19:25:09 +0200 Subject: Fix pep8 --- searx/poolrequests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 54e8dbcf6..f268df207 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -41,14 +41,14 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) -connect=settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code -maxsize=settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor +connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code +maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor if settings['outgoing'].get('source_ips'): http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, - source_address=(source_ip, 0)) + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) else: http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) -- cgit v1.2.3