From 6e2872f43625aba71eba019e16f7fbd74743f590 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 5 Jan 2021 11:22:48 +0100 Subject: [enh] add searx.shared shared dictionary between the workers (UWSGI or werkzeug) scheduler: run a task once every x seconds (UWSGI or werkzeug) --- searx/shared/shared_uwsgi.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 searx/shared/shared_uwsgi.py (limited to 'searx/shared/shared_uwsgi.py') diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py new file mode 100644 index 000000000..136bf687e --- /dev/null +++ b/searx/shared/shared_uwsgi.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import time +import uwsgi # pylint: disable=E0401 +from . import shared_abstract + + +_last_signal = 10 + + +class UwsgiCacheSharedDict(shared_abstract.SharedDict): + + def get_int(self, key): + value = uwsgi.cache_get(key) + if value is None: + return value + else: + return int.from_bytes(value, 'big') + + def set_int(self, key, value): + b = value.to_bytes(4, 'big') + uwsgi.cache_update(key, b) + + def get_str(self, key): + value = uwsgi.cache_get(key) + if value is None: + return value + else: + return value.decode('utf-8') + + def set_str(self, key, value): + b = value.encode('utf-8') + uwsgi.cache_update(key, b) + + +def schedule(delay, func, *args): + """ + Can be implemented using a spooler. + https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html + + To make the uwsgi configuration simple, use the alternative implementation. + """ + global _last_signal + + def sighandler(signum): + now = int(time.time()) + uwsgi.lock() + try: + updating = uwsgi.cache_get('updating') + if updating is not None: + updating = int.from_bytes(updating, 'big') + if now - updating < delay: + return + uwsgi.cache_update('updating', now.to_bytes(4, 'big')) + finally: + uwsgi.unlock() + func(*args) + + signal_num = _last_signal + _last_signal += 1 + uwsgi.register_signal(signal_num, 'worker', sighandler) + uwsgi.add_timer(signal_num, delay) -- cgit v1.2.3 From a0c8b413a610e8cde49dbb321ba17b16200eb92f Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 11 Jan 2021 18:44:39 +0100 Subject: [mod] searx.shared: minor tweaks searx.shared.shared_abstract.SharedDict inherit from abc.ABC searx.shared.shared_uwsgi.schedule can schedule multiple functions without issue --- searx/shared/shared_uwsgi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'searx/shared/shared_uwsgi.py') diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py index 136bf687e..b42b5fa7b 100644 --- a/searx/shared/shared_uwsgi.py +++ b/searx/shared/shared_uwsgi.py @@ -44,14 +44,15 @@ def schedule(delay, func, *args): def sighandler(signum): now = int(time.time()) + key = 'scheduler_call_time_signal_' + str(signum) uwsgi.lock() try: - updating = uwsgi.cache_get('updating') + updating = uwsgi.cache_get(key) if updating is not None: updating = int.from_bytes(updating, 'big') if now - updating < delay: return - uwsgi.cache_update('updating', now.to_bytes(4, 'big')) + uwsgi.cache_update(key, now.to_bytes(4, 'big')) finally: uwsgi.unlock() func(*args) -- cgit v1.2.3 From 912c7e975c3943db798d748fa48d460467b66d30 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 13 Jan 2021 14:07:39 +0100 Subject: [fix] checker: don't run the checker when uwsgi is not properly configured Before this commit, even with the scheduler disabled, the checker was running at least once for each uwsgi worker. --- searx/shared/shared_uwsgi.py | 1 + 1 file changed, 1 insertion(+) (limited to 'searx/shared/shared_uwsgi.py') diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py index b42b5fa7b..a6dba9f59 100644 --- a/searx/shared/shared_uwsgi.py +++ b/searx/shared/shared_uwsgi.py @@ -61,3 +61,4 @@ def schedule(delay, func, *args): _last_signal += 1 uwsgi.register_signal(signal_num, 'worker', sighandler) uwsgi.add_timer(signal_num, delay) + return True -- cgit v1.2.3