The checker requires Redis

Remove the abstraction in searx.shared.SharedDict.
Implement a basic and dedicated scheduler for the checker using a Redis script.

1 files changed, 36 insertions, 0 deletions

diff --git a/searx/search/checker/scheduler.lua b/searx/search/checker/scheduler.lua
new file mode 100644
index 000000000..b3c6023fe
--- /dev/null
+++ b/searx/search/checker/scheduler.lua
@@ -0,0 +1,36 @@
+-- SPDX-License-Identifier: AGPL-3.0-or-later
+--
+-- This script is not a string in scheduler.py, so editors can provide syntax highlighting.
+
+-- The Redis KEY is defined here and not in Python on purpose:
+-- only this LUA script can read and update this key to avoid lock and concurrency issues.
+local redis_key = 'SearXNG_checker_next_call_ts'
+
+local now = redis.call('TIME')[1]
+local start_after_from = ARGV[1]
+local start_after_to = ARGV[2]
+local every_from = ARGV[3]
+local every_to = ARGV[4]
+
+local next_call_ts = redis.call('GET', redis_key)
+
+if (next_call_ts == false or next_call_ts == nil) then
+    -- the scheduler has never run on this Redis instance, so:
+    -- 1/ the scheduler does not run now
+    -- 2/ the next call is a random time between start_after_from and start_after_to
+    local delay = start_after_from + math.random(start_after_to - start_after_from)
+    redis.call('SET', redis_key, now + delay)
+    return { false, delay }
+end
+
+-- next_call_ts is defined
+-- --> if now is lower than next_call_ts then we don't run the embedded checker
+-- --> if now is higher then we update next_call_ts and ask to run the embedded checker now.
+local call_now = next_call_ts <= now
+if call_now then
+    -- the checker runs now, define the timestamp of the next call:
+    -- this is a random delay between every_from and every_to
+    local periodic_delay = every_from + math.random(every_to - every_from)
+    next_call_ts = redis.call('INCRBY', redis_key, periodic_delay)
+end
+return { call_now, next_call_ts - now }