From fe419e355bf1527c51e3aee98495d08b89510320 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 15 Jul 2022 18:38:32 +0200 Subject: The checker requires Redis Remove the abstraction in searx.shared.SharedDict. Implement a basic and dedicated scheduler for the checker using a Redis script. --- searx/search/checker/scheduler.lua | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 searx/search/checker/scheduler.lua (limited to 'searx/search/checker/scheduler.lua') diff --git a/searx/search/checker/scheduler.lua b/searx/search/checker/scheduler.lua new file mode 100644 index 000000000..b3c6023fe --- /dev/null +++ b/searx/search/checker/scheduler.lua @@ -0,0 +1,36 @@ +-- SPDX-License-Identifier: AGPL-3.0-or-later +-- +-- This script is not a string in scheduler.py, so editors can provide syntax highlighting. + +-- The Redis KEY is defined here and not in Python on purpose: +-- only this LUA script can read and update this key to avoid lock and concurrency issues. +local redis_key = 'SearXNG_checker_next_call_ts' + +local now = redis.call('TIME')[1] +local start_after_from = ARGV[1] +local start_after_to = ARGV[2] +local every_from = ARGV[3] +local every_to = ARGV[4] + +local next_call_ts = redis.call('GET', redis_key) + +if (next_call_ts == false or next_call_ts == nil) then + -- the scheduler has never run on this Redis instance, so: + -- 1/ the scheduler does not run now + -- 2/ the next call is a random time between start_after_from and start_after_to + local delay = start_after_from + math.random(start_after_to - start_after_from) + redis.call('SET', redis_key, now + delay) + return { false, delay } +end + +-- next_call_ts is defined +-- --> if now is lower than next_call_ts then we don't run the embedded checker +-- --> if now is higher then we update next_call_ts and ask to run the embedded checker now. +local call_now = next_call_ts <= now +if call_now then + -- the checker runs now, define the timestamp of the next call: + -- this is a random delay between every_from and every_to + local periodic_delay = every_from + math.random(every_to - every_from) + next_call_ts = redis.call('INCRBY', redis_key, periodic_delay) +end +return { call_now, next_call_ts - now } -- cgit v1.2.3 From e92755d358df5b34b0181f48f8ba02c7f2939e8f Mon Sep 17 00:00:00 2001 From: Alexandre FLAMENT Date: Fri, 14 Oct 2022 13:27:07 +0000 Subject: Initialize Redis in searx/webapp.py settings.yml: * The default URL was unix:///usr/local/searxng-redis/run/redis.sock?db=0 * The default URL is now "false" The default URL makes the log difficult to deal with: if the admin didn't install a Redis instance, the logs record a false error. It worked before because SearXNG initialized the Redis connection when the limiter started. In this commit, SearXNG initializes Redis in searx/webapp.py so various components can use Redis without taking care of the initialization step. --- searx/search/checker/scheduler.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'searx/search/checker/scheduler.lua') diff --git a/searx/search/checker/scheduler.lua b/searx/search/checker/scheduler.lua index b3c6023fe..c5ebb05bc 100644 --- a/searx/search/checker/scheduler.lua +++ b/searx/search/checker/scheduler.lua @@ -18,8 +18,8 @@ if (next_call_ts == false or next_call_ts == nil) then -- the scheduler has never run on this Redis instance, so: -- 1/ the scheduler does not run now -- 2/ the next call is a random time between start_after_from and start_after_to - local delay = start_after_from + math.random(start_after_to - start_after_from) - redis.call('SET', redis_key, now + delay) + local initial_delay = math.random(start_after_from, start_after_to) + redis.call('SET', redis_key, now + initial_delay) return { false, delay } end @@ -30,7 +30,7 @@ local call_now = next_call_ts <= now if call_now then -- the checker runs now, define the timestamp of the next call: -- this is a random delay between every_from and every_to - local periodic_delay = every_from + math.random(every_to - every_from) + local periodic_delay = math.random(every_from, every_to) next_call_ts = redis.call('INCRBY', redis_key, periodic_delay) end return { call_now, next_call_ts - now } -- cgit v1.2.3