From d703119d3a313a406482b121ee94c6afee3bc307 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 9 Dec 2020 21:23:20 +0100 Subject: [enh] add raise_for_httperror check HTTP response: * detect some comme CAPTCHA challenge (no solving). In this case the engine is suspended for long a time. * otherwise raise HTTPError as before the check is done in poolrequests.py (was before in search.py). update qwant, wikipedia, wikidata to use raise_for_httperror instead of raise_for_status --- searx/poolrequests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'searx/poolrequests.py') diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 1eedc84b8..25a6baed9 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -7,6 +7,7 @@ import requests from searx import settings from searx import logger +from searx.raise_for_httperror import raise_for_httperror logger = logger.getChild('poolrequests') @@ -156,6 +157,12 @@ def request(method, url, **kwargs): if timeout is not None: kwargs['timeout'] = timeout + # raise_for_error + check_for_httperror = True + if 'raise_for_httperror' in kwargs: + check_for_httperror = kwargs['raise_for_httperror'] + del kwargs['raise_for_httperror'] + # do request response = session.request(method=method, url=url, **kwargs) @@ -176,6 +183,10 @@ def request(method, url, **kwargs): if hasattr(threadLocal, 'total_time'): threadLocal.total_time += time_after_request - time_before_request + # raise an exception + if check_for_httperror: + raise_for_httperror(response) + return response -- cgit v1.2.3