diff options
| author | Adam Tauber <asciimoo@gmail.com> | 2016-05-19 15:51:14 +0200 |
|---|---|---|
| committer | Adam Tauber <asciimoo@gmail.com> | 2016-05-19 15:51:14 +0200 |
| commit | bf1eebb17f2a4e4454d59a59757e0265e33c9dcf (patch) | |
| tree | 0b8113c700076bd08f2ed104f8a35467502d47fc /searx | |
| parent | 9247b1e777f4b051136c1ebdd89d84725eb77c9a (diff) | |
| parent | d571a9d204dc4ff021bdc097dcaf87a5cc90dbe4 (diff) | |
Merge pull request #549 from guyou/feature-pool-fine-tuning
Use new settings to fine tune the connection pool
Diffstat (limited to 'searx')
| -rw-r--r-- | searx/poolrequests.py | 12 | ||||
| -rw-r--r-- | searx/settings.yml | 2 |
2 files changed, 10 insertions, 4 deletions
diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 13c6a906e..f268df207 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) +connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code +maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor if settings['outgoing'].get('source_ips'): - http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) - https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) else: - http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) - https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) + http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) + https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) class SessionSinglePool(requests.Session): diff --git a/searx/settings.yml b/searx/settings.yml index ff85684ac..96455fc23 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -21,6 +21,8 @@ ui: outgoing: # communication with search engines request_timeout : 2.0 # seconds useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator + pool_connections : 100 # Number of different hosts + pool_maxsize : 10 # Number of simultaneous requests by host # uncomment below section if you want to use a proxy # see http://docs.python-requests.org/en/latest/user/advanced/#proxies # SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478 |