From aae7830d14242ac1f98232f428654c5d2c9c5eb2 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 13 Apr 2021 15:21:53 +0200 Subject: [mod] refactoring: processors Report to the user suspended engines. searx.search.processor.abstract: * manages suspend time (per network). * reports suspended time to the ResultContainer (method extend_container_if_suspended) * adds the results to the ResultContainer (method extend_container) * handles exceptions (method handle_exception) --- searx/search/processors/online.py | 118 ++++++++++---------------------------- 1 file changed, 30 insertions(+), 88 deletions(-) (limited to 'searx/search/processors/online.py') diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index 66719ea9b..1483897d6 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -1,23 +1,21 @@ # SPDX-License-Identifier: AGPL-3.0-or-later from time import time -import threading import asyncio import httpx import searx.network -from searx.engines import settings from searx import logger from searx.utils import gen_useragent from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException, SearxEngineTooManyRequestsException,) -from searx.metrology.error_recorder import record_exception, record_error +from searx.metrology.error_recorder import record_error from searx.search.processors.abstract import EngineProcessor -logger = logger.getChild('search.processor.online') +logger = logger.getChild('searx.search.processor.online') def default_request_params(): @@ -41,11 +39,6 @@ class OnlineProcessor(EngineProcessor): if params is None: return None - # skip suspended engines - if self.engine.suspend_end_time >= time(): - logger.debug('Engine currently suspended: %s', self.engine_name) - return None - # add default params params.update(default_request_params()) @@ -130,89 +123,38 @@ class OnlineProcessor(EngineProcessor): # set the network searx.network.set_context_network_name(self.engine_name) - # suppose everything will be alright - http_exception = False - suspended_time = None - try: # send requests and parse the results search_results = self._search_basic(query, params) - - # check if the engine accepted the request - if search_results is not None: - # yes, so add results - result_container.extend(self.engine_name, search_results) - - # update engine time when there is no exception - engine_time = time() - start_time - page_load_time = searx.network.get_time_for_thread() - result_container.add_timing(self.engine_name, engine_time, page_load_time) - with threading.RLock(): - self.engine.stats['engine_time'] += engine_time - self.engine.stats['engine_time_count'] += 1 - # update stats with the total HTTP time - self.engine.stats['page_load_time'] += page_load_time - self.engine.stats['page_load_count'] += 1 - except Exception as e: - record_exception(self.engine_name, e) - - # Timing - engine_time = time() - start_time - page_load_time = searx.network.get_time_for_thread() - result_container.add_timing(self.engine_name, engine_time, page_load_time) - - # Record the errors - with threading.RLock(): - self.engine.stats['errors'] += 1 - - if (issubclass(e.__class__, (httpx.TimeoutException, asyncio.TimeoutError))): - result_container.add_unresponsive_engine(self.engine_name, 'HTTP timeout') - # requests timeout (connect or read) - logger.error("engine {0} : HTTP requests timeout" + self.extend_container(result_container, start_time, search_results) + except (httpx.TimeoutException, asyncio.TimeoutError) as e: + # requests timeout (connect or read) + self.handle_exception(result_container, 'HTTP timeout', e, suspend=True, display_exception=False) + logger.error("engine {0} : HTTP requests timeout" + "(search duration : {1} s, timeout: {2} s) : {3}" + .format(self.engine_name, time() - start_time, + timeout_limit, + e.__class__.__name__)) + except (httpx.HTTPError, httpx.StreamError) as e: + # other requests exception + self.handle_exception(result_container, 'HTTP error', e, suspend=True, display_exception=False) + logger.exception("engine {0} : requests exception" "(search duration : {1} s, timeout: {2} s) : {3}" - .format(self.engine_name, engine_time, timeout_limit, e.__class__.__name__)) - http_exception = True - elif (issubclass(e.__class__, (httpx.HTTPError, httpx.StreamError))): - result_container.add_unresponsive_engine(self.engine_name, 'HTTP error') - # other requests exception - logger.exception("engine {0} : requests exception" - "(search duration : {1} s, timeout: {2} s) : {3}" - .format(self.engine_name, engine_time, timeout_limit, e)) - http_exception = True - elif (issubclass(e.__class__, SearxEngineCaptchaException)): - result_container.add_unresponsive_engine(self.engine_name, 'CAPTCHA required') - logger.exception('engine {0} : CAPTCHA'.format(self.engine_name)) - suspended_time = e.suspended_time # pylint: disable=no-member - elif (issubclass(e.__class__, SearxEngineTooManyRequestsException)): - result_container.add_unresponsive_engine(self.engine_name, 'too many requests') - logger.exception('engine {0} : Too many requests'.format(self.engine_name)) - suspended_time = e.suspended_time # pylint: disable=no-member - elif (issubclass(e.__class__, SearxEngineAccessDeniedException)): - result_container.add_unresponsive_engine(self.engine_name, 'blocked') - logger.exception('engine {0} : Searx is blocked'.format(self.engine_name)) - suspended_time = e.suspended_time # pylint: disable=no-member - else: - result_container.add_unresponsive_engine(self.engine_name, 'unexpected crash') - # others errors - logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) - else: - if getattr(threading.current_thread(), '_timeout', False): - record_error(self.engine_name, 'Timeout') - - # suspend the engine if there is an HTTP error - # or suspended_time is defined - with threading.RLock(): - if http_exception or suspended_time: - # update continuous_errors / suspend_end_time - self.engine.continuous_errors += 1 - if suspended_time is None: - suspended_time = min(settings['search']['max_ban_time_on_fail'], - self.engine.continuous_errors * settings['search']['ban_time_on_fail']) - self.engine.suspend_end_time = time() + suspended_time - else: - # reset the suspend variables - self.engine.continuous_errors = 0 - self.engine.suspend_end_time = 0 + .format(self.engine_name, time() - start_time, + timeout_limit, + e)) + except SearxEngineCaptchaException as e: + self.handle_exception(result_container, 'CAPTCHA required', e, suspend=True, display_exception=False) + logger.exception('engine {0} : CAPTCHA'.format(self.engine_name)) + except SearxEngineTooManyRequestsException as e: + self.handle_exception(result_container, 'too many requests', e, suspend=True, display_exception=False) + logger.exception('engine {0} : Too many requests'.format(self.engine_name)) + except SearxEngineAccessDeniedException as e: + self.handle_exception(result_container, 'blocked', e, suspend=True, display_exception=False) + logger.exception('engine {0} : Searx is blocked'.format(self.engine_name)) + except Exception as e: + self.handle_exception(result_container, 'unexpected crash', e, display_exception=False) + logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) def get_default_tests(self): tests = {} -- cgit v1.2.3 From 7acd7ffc02d14d175ec2a99ba984e47d8cb65d7d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 14 Apr 2021 17:23:15 +0200 Subject: [enh] rewrite and enhance metrics --- searx/search/processors/online.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'searx/search/processors/online.py') diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index 1483897d6..bca74b746 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -10,7 +10,7 @@ from searx import logger from searx.utils import gen_useragent from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException, SearxEngineTooManyRequestsException,) -from searx.metrology.error_recorder import record_error +from searx.metrics.error_recorder import count_error from searx.search.processors.abstract import EngineProcessor @@ -90,9 +90,9 @@ class OnlineProcessor(EngineProcessor): status_code = str(response.status_code or '') reason = response.reason_phrase or '' hostname = response.url.host - record_error(self.engine_name, - '{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects), - (status_code, reason, hostname)) + count_error(self.engine_name, + '{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects), + (status_code, reason, hostname)) return response -- cgit v1.2.3 From c27fef1cdeeebcc17e21dbdc3dafad00de08a2ce Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 17 Apr 2021 18:15:50 +0200 Subject: [mod] metrics: add secondary parameter Some error won't stop the engine: * additional HTTP redirects for example * some invalid results secondary=True allows to flag these errors as not important. --- searx/search/processors/online.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'searx/search/processors/online.py') diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index bca74b746..c39937023 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -92,7 +92,8 @@ class OnlineProcessor(EngineProcessor): hostname = response.url.host count_error(self.engine_name, '{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects), - (status_code, reason, hostname)) + (status_code, reason, hostname), + secondary=True) return response -- cgit v1.2.3