From aae7830d14242ac1f98232f428654c5d2c9c5eb2 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 13 Apr 2021 15:21:53 +0200 Subject: [mod] refactoring: processors Report to the user suspended engines. searx.search.processor.abstract: * manages suspend time (per network). * reports suspended time to the ResultContainer (method extend_container_if_suspended) * adds the results to the ResultContainer (method extend_container) * handles exceptions (method handle_exception) --- searx/results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'searx/results.py') diff --git a/searx/results.py b/searx/results.py index b3b874118..c1a1819d4 100644 --- a/searx/results.py +++ b/searx/results.py @@ -369,9 +369,9 @@ class ResultContainer: return 0 return resultnum_sum / len(self._number_of_results) - def add_unresponsive_engine(self, engine_name, error_type, error_message=None): + def add_unresponsive_engine(self, engine_name, error_type, error_message=None, suspended=False): if engines[engine_name].display_error_messages: - self.unresponsive_engines.add((engine_name, error_type, error_message)) + self.unresponsive_engines.add((engine_name, error_type, error_message, suspended)) def add_timing(self, engine_name, engine_time, page_load_time): self.timings.append({ -- cgit v1.2.3 From 7acd7ffc02d14d175ec2a99ba984e47d8cb65d7d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 14 Apr 2021 17:23:15 +0200 Subject: [enh] rewrite and enhance metrics --- searx/results.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'searx/results.py') diff --git a/searx/results.py b/searx/results.py index c1a1819d4..41c150803 100644 --- a/searx/results.py +++ b/searx/results.py @@ -5,7 +5,7 @@ from threading import RLock from urllib.parse import urlparse, unquote from searx import logger from searx.engines import engines -from searx.metrology.error_recorder import record_error +from searx.metrics import histogram_observe, counter_add, count_error CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U) @@ -196,12 +196,10 @@ class ResultContainer: if len(error_msgs) > 0: for msg in error_msgs: - record_error(engine_name, 'some results are invalids: ' + msg) + count_error(engine_name, 'some results are invalids: ' + msg) if engine_name in engines: - with RLock(): - engines[engine_name].stats['search_count'] += 1 - engines[engine_name].stats['result_count'] += standard_result_count + histogram_observe(standard_result_count, 'engine', engine_name, 'result', 'count') if not self.paging and standard_result_count > 0 and engine_name in engines\ and engines[engine_name].paging: @@ -301,9 +299,8 @@ class ResultContainer: for result in self._merged_results: score = result_score(result) result['score'] = score - with RLock(): - for result_engine in result['engines']: - engines[result_engine].stats['score_count'] += score + for result_engine in result['engines']: + counter_add(score, 'engine', result_engine, 'score') results = sorted(self._merged_results, key=itemgetter('score'), reverse=True) -- cgit v1.2.3 From c27fef1cdeeebcc17e21dbdc3dafad00de08a2ce Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 17 Apr 2021 18:15:50 +0200 Subject: [mod] metrics: add secondary parameter Some error won't stop the engine: * additional HTTP redirects for example * some invalid results secondary=True allows to flag these errors as not important. --- searx/results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/results.py') diff --git a/searx/results.py b/searx/results.py index 41c150803..a1c1d8527 100644 --- a/searx/results.py +++ b/searx/results.py @@ -196,7 +196,7 @@ class ResultContainer: if len(error_msgs) > 0: for msg in error_msgs: - count_error(engine_name, 'some results are invalids: ' + msg) + count_error(engine_name, 'some results are invalids: ' + msg, secondary=True) if engine_name in engines: histogram_observe(standard_result_count, 'engine', engine_name, 'result', 'count') -- cgit v1.2.3