From 6e2872f43625aba71eba019e16f7fbd74743f590 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 5 Jan 2021 11:22:48 +0100 Subject: [enh] add searx.shared shared dictionary between the workers (UWSGI or werkzeug) scheduler: run a task once every x seconds (UWSGI or werkzeug) --- searx/shared/__init__.py | 31 +++++++++++++++++++++ searx/shared/shared_abstract.py | 15 ++++++++++ searx/shared/shared_simple.py | 38 +++++++++++++++++++++++++ searx/shared/shared_uwsgi.py | 62 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 searx/shared/__init__.py create mode 100644 searx/shared/shared_abstract.py create mode 100644 searx/shared/shared_simple.py create mode 100644 searx/shared/shared_uwsgi.py (limited to 'searx/shared') diff --git a/searx/shared/__init__.py b/searx/shared/__init__.py new file mode 100644 index 000000000..83d3a2742 --- /dev/null +++ b/searx/shared/__init__.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import logging + +logger = logging.getLogger('searx.shared') + +try: + import uwsgi +except: + # no uwsgi + from .shared_simple import SimpleSharedDict as SharedDict, schedule + logger.info('Use shared_simple implementation') +else: + try: + uwsgi.cache_update('dummy', b'dummy') + if uwsgi.cache_get('dummy') != b'dummy': + raise Exception() + except: + # uwsgi.ini configuration problem: disable all scheduling + logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n' + 'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1') + from .shared_simple import SimpleSharedDict as SharedDict + + def schedule(delay, func, *args): + pass + else: + # uwsgi + from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule + logger.info('Use shared_uwsgi implementation') + +storage = SharedDict() diff --git a/searx/shared/shared_abstract.py b/searx/shared/shared_abstract.py new file mode 100644 index 000000000..3fede417e --- /dev/null +++ b/searx/shared/shared_abstract.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +class SharedDict: + + def get_int(self, key): + pass + + def set_int(self, key, value): + pass + + def get_str(self, key): + pass + + def set_str(self, key, value): + pass diff --git a/searx/shared/shared_simple.py b/searx/shared/shared_simple.py new file mode 100644 index 000000000..5b970aad9 --- /dev/null +++ b/searx/shared/shared_simple.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import threading + +from . import shared_abstract + + +class SimpleSharedDict(shared_abstract.SharedDict): + + __slots__ = 'd', + + def __init__(self): + self.d = {} + + def get_int(self, key): + return self.d.get(key, None) + + def set_int(self, key, value): + self.d[key] = value + + def get_str(self, key): + return self.d.get(key, None) + + def set_str(self, key, value): + self.d[key] = value + + +def schedule(delay, func, *args): + def call_later(): + t = threading.Timer(delay, wrapper) + t.daemon = True + t.start() + + def wrapper(): + call_later() + func(*args) + + call_later() diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py new file mode 100644 index 000000000..136bf687e --- /dev/null +++ b/searx/shared/shared_uwsgi.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import time +import uwsgi # pylint: disable=E0401 +from . import shared_abstract + + +_last_signal = 10 + + +class UwsgiCacheSharedDict(shared_abstract.SharedDict): + + def get_int(self, key): + value = uwsgi.cache_get(key) + if value is None: + return value + else: + return int.from_bytes(value, 'big') + + def set_int(self, key, value): + b = value.to_bytes(4, 'big') + uwsgi.cache_update(key, b) + + def get_str(self, key): + value = uwsgi.cache_get(key) + if value is None: + return value + else: + return value.decode('utf-8') + + def set_str(self, key, value): + b = value.encode('utf-8') + uwsgi.cache_update(key, b) + + +def schedule(delay, func, *args): + """ + Can be implemented using a spooler. + https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html + + To make the uwsgi configuration simple, use the alternative implementation. + """ + global _last_signal + + def sighandler(signum): + now = int(time.time()) + uwsgi.lock() + try: + updating = uwsgi.cache_get('updating') + if updating is not None: + updating = int.from_bytes(updating, 'big') + if now - updating < delay: + return + uwsgi.cache_update('updating', now.to_bytes(4, 'big')) + finally: + uwsgi.unlock() + func(*args) + + signal_num = _last_signal + _last_signal += 1 + uwsgi.register_signal(signal_num, 'worker', sighandler) + uwsgi.add_timer(signal_num, delay) -- cgit v1.2.3 From a0c8b413a610e8cde49dbb321ba17b16200eb92f Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 11 Jan 2021 18:44:39 +0100 Subject: [mod] searx.shared: minor tweaks searx.shared.shared_abstract.SharedDict inherit from abc.ABC searx.shared.shared_uwsgi.schedule can schedule multiple functions without issue --- searx/shared/shared_abstract.py | 8 +++++++- searx/shared/shared_uwsgi.py | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'searx/shared') diff --git a/searx/shared/shared_abstract.py b/searx/shared/shared_abstract.py index 3fede417e..b1c72aabe 100644 --- a/searx/shared/shared_abstract.py +++ b/searx/shared/shared_abstract.py @@ -1,15 +1,21 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +from abc import ABC, abstractmethod -class SharedDict: +class SharedDict(ABC): + + @abstractmethod def get_int(self, key): pass + @abstractmethod def set_int(self, key, value): pass + @abstractmethod def get_str(self, key): pass + @abstractmethod def set_str(self, key, value): pass diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py index 136bf687e..b42b5fa7b 100644 --- a/searx/shared/shared_uwsgi.py +++ b/searx/shared/shared_uwsgi.py @@ -44,14 +44,15 @@ def schedule(delay, func, *args): def sighandler(signum): now = int(time.time()) + key = 'scheduler_call_time_signal_' + str(signum) uwsgi.lock() try: - updating = uwsgi.cache_get('updating') + updating = uwsgi.cache_get(key) if updating is not None: updating = int.from_bytes(updating, 'big') if now - updating < delay: return - uwsgi.cache_update('updating', now.to_bytes(4, 'big')) + uwsgi.cache_update(key, now.to_bytes(4, 'big')) finally: uwsgi.unlock() func(*args) -- cgit v1.2.3 From 912c7e975c3943db798d748fa48d460467b66d30 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 13 Jan 2021 14:07:39 +0100 Subject: [fix] checker: don't run the checker when uwsgi is not properly configured Before this commit, even with the scheduler disabled, the checker was running at least once for each uwsgi worker. --- searx/shared/__init__.py | 2 +- searx/shared/shared_simple.py | 1 + searx/shared/shared_uwsgi.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) (limited to 'searx/shared') diff --git a/searx/shared/__init__.py b/searx/shared/__init__.py index 83d3a2742..cbe24d239 100644 --- a/searx/shared/__init__.py +++ b/searx/shared/__init__.py @@ -22,7 +22,7 @@ else: from .shared_simple import SimpleSharedDict as SharedDict def schedule(delay, func, *args): - pass + return False else: # uwsgi from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule diff --git a/searx/shared/shared_simple.py b/searx/shared/shared_simple.py index 5b970aad9..48d8cb822 100644 --- a/searx/shared/shared_simple.py +++ b/searx/shared/shared_simple.py @@ -36,3 +36,4 @@ def schedule(delay, func, *args): func(*args) call_later() + return True diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py index b42b5fa7b..a6dba9f59 100644 --- a/searx/shared/shared_uwsgi.py +++ b/searx/shared/shared_uwsgi.py @@ -61,3 +61,4 @@ def schedule(delay, func, *args): _last_signal += 1 uwsgi.register_signal(signal_num, 'worker', sighandler) uwsgi.add_timer(signal_num, delay) + return True -- cgit v1.2.3