From 3786920df975b11c0feb7d8564eb19b634d32977 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 16 Nov 2020 12:44:07 +0100 Subject: [enh] Add multiple outgoing proxies credits go to @bauruine see https://github.com/searx/searx/pull/1958 --- searx/poolrequests.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py index e03797ce2..1eedc84b8 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -111,6 +111,32 @@ def get_time_for_thread(): return threadLocal.total_time +def get_proxy_cycles(proxy_settings): + if not proxy_settings: + return None + # Backwards compatibility for single proxy in settings.yml + for protocol, proxy in proxy_settings.items(): + if isinstance(proxy, str): + proxy_settings[protocol] = [proxy] + + for protocol in proxy_settings: + proxy_settings[protocol] = cycle(proxy_settings[protocol]) + return proxy_settings + + +GLOBAL_PROXY_CYCLES = get_proxy_cycles(settings['outgoing'].get('proxies')) + + +def get_proxies(proxy_cycles): + if proxy_cycles: + return {protocol: next(proxy_cycle) for protocol, proxy_cycle in proxy_cycles.items()} + return None + + +def get_global_proxies(): + return get_proxies(GLOBAL_PROXY_CYCLES) + + def request(method, url, **kwargs): """same as requests/requests/api.py request(...)""" time_before_request = time() @@ -119,8 +145,8 @@ def request(method, url, **kwargs): session = SessionSinglePool() # proxies - if kwargs.get('proxies') is None: - kwargs['proxies'] = settings['outgoing'].get('proxies') + if not kwargs.get('proxies'): + kwargs['proxies'] = get_global_proxies() # timeout if 'timeout' in kwargs: -- cgit v1.2.3