From d14994dc73ba5c95382812581dac146d9eceaafa Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 5 Apr 2021 10:43:33 +0200 Subject: [httpx] replace searx.poolrequests by searx.network settings.yml: * outgoing.networks: * can contains network definition * propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections, keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries * retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time) * local_addresses can be "192.168.0.1/24" (it supports IPv6) * support_ipv4 & support_ipv6: both True by default see https://github.com/searx/searx/pull/1034 * each engine can define a "network" section: * either a full network description * either reference an existing network * all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master) --- searx/raise_for_httperror.py | 66 -------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 searx/raise_for_httperror.py (limited to 'searx/raise_for_httperror.py') diff --git a/searx/raise_for_httperror.py b/searx/raise_for_httperror.py deleted file mode 100644 index bd12df9a9..000000000 --- a/searx/raise_for_httperror.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -""" -Raise exception for an HTTP response is an error. -""" -from searx.exceptions import (SearxEngineCaptchaException, SearxEngineTooManyRequestsException, - SearxEngineAccessDeniedException) - - -def is_cloudflare_challenge(resp): - if resp.status_code in [429, 503]: - if ('__cf_chl_jschl_tk__=' in resp.text)\ - or ('/cdn-cgi/challenge-platform/' in resp.text - and 'orchestrate/jsch/v1' in resp.text - and 'window._cf_chl_enter(' in resp.text): - return True - if resp.status_code == 403 and '__cf_chl_captcha_tk__=' in resp.text: - return True - return False - - -def is_cloudflare_firewall(resp): - return resp.status_code == 403 and '1020' in resp.text - - -def raise_for_cloudflare_captcha(resp): - if resp.headers.get('Server', '').startswith('cloudflare'): - if is_cloudflare_challenge(resp): - # https://support.cloudflare.com/hc/en-us/articles/200170136-Understanding-Cloudflare-Challenge-Passage-Captcha- - # suspend for 2 weeks - raise SearxEngineCaptchaException(message='Cloudflare CAPTCHA', suspended_time=3600 * 24 * 15) - - if is_cloudflare_firewall(resp): - raise SearxEngineAccessDeniedException(message='Cloudflare Firewall', suspended_time=3600 * 24) - - -def raise_for_recaptcha(resp): - if resp.status_code == 503 \ - and '"https://www.google.com/recaptcha/' in resp.text: - raise SearxEngineCaptchaException(message='ReCAPTCHA', suspended_time=3600 * 24 * 7) - - -def raise_for_captcha(resp): - raise_for_cloudflare_captcha(resp) - raise_for_recaptcha(resp) - - -def raise_for_httperror(resp): - """Raise exception for an HTTP response is an error. - - Args: - resp (requests.Response): Response to check - - Raises: - requests.HTTPError: raise by resp.raise_for_status() - searx.exceptions.SearxEngineAccessDeniedException: raise when the HTTP status code is 402 or 403. - searx.exceptions.SearxEngineTooManyRequestsException: raise when the HTTP status code is 429. - searx.exceptions.SearxEngineCaptchaException: raise when if CATPCHA challenge is detected. - """ - if resp.status_code and resp.status_code >= 400: - raise_for_captcha(resp) - if resp.status_code in (402, 403): - raise SearxEngineAccessDeniedException(message='HTTP error ' + str(resp.status_code), - suspended_time=3600 * 24) - if resp.status_code == 429: - raise SearxEngineTooManyRequestsException() - resp.raise_for_status() -- cgit v1.2.3