summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGaspard d'Hautefeuille <github@dhautefeuille.eu>2025-07-09 07:55:37 +0200
committerGitHub <noreply@github.com>2025-07-09 07:55:37 +0200
commitf798ddd4922d793d5e6ccb7c4111810d549ff4f4 (patch)
tree223aa9d26deb176d983cd8e1bed51ff2cff71eff
parentbd593d0bad2189f57657bbcfa2c5e86f795c680e (diff)
[mod] migrate from Redis to Valkey (#4795)
This patch migrates from `redis==5.2.1` [1] to `valkey==6.1.0` [2]. The migration to valkey is necessary because the company behind Redis has decided to abandon the open source license. After experiencing a drop in user numbers, they now want to run it under a dual license again. But this move demonstrates once again how unreliable the company is and how it treats open source developers. To review first, read the docs:: $ make docs.live Follow the instructions to remove redis: - http://0.0.0.0:8000/admin/settings/settings_redis.html Config and install a local valkey DB: - http://0.0.0.0:8000/admin/settings/settings_valkey.html [1] https://pypi.org/project/redis/ [2] https://pypi.org/project/valkey/ Co-authored-by: HLFH <gaspard@dhautefeuille.eu> Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>
-rw-r--r--.devcontainer/Dockerfile2
-rw-r--r--Makefile1
-rw-r--r--docs/admin/arch_public.dot11
-rw-r--r--docs/admin/installation-uwsgi.rst12
-rw-r--r--docs/admin/searx.limiter.rst2
-rw-r--r--docs/admin/settings/index.rst3
-rw-r--r--docs/admin/settings/settings_redis.rst44
-rw-r--r--docs/admin/settings/settings_server.rst2
-rw-r--r--docs/admin/settings/settings_valkey.rst53
-rw-r--r--docs/admin/update-searxng.rst6
-rw-r--r--docs/conf.py2
-rw-r--r--docs/dev/engines/offline/nosql-engines.rst23
-rw-r--r--docs/dev/lxcdev.rst2
-rw-r--r--docs/dev/makefile.rst11
-rw-r--r--docs/src/searx.redisdb.rst8
-rw-r--r--docs/src/searx.redislib.rst8
-rw-r--r--docs/src/searx.valkeydb.rst8
-rw-r--r--docs/src/searx.valkeylib.rst8
-rwxr-xr-xmanage18
-rw-r--r--requirements.txt2
-rw-r--r--searx/botdetection/__init__.py8
-rw-r--r--searx/botdetection/ip_limit.py24
-rw-r--r--searx/botdetection/link_token.py32
-rw-r--r--searx/engines/valkey_server.py (renamed from searx/engines/redis_server.py)36
-rw-r--r--searx/limiter.py22
-rw-r--r--searx/redisdb.py69
-rw-r--r--searx/search/checker/background.py34
-rw-r--r--searx/search/checker/scheduler.lua12
-rw-r--r--searx/search/checker/scheduler.py20
-rw-r--r--searx/settings.yml13
-rw-r--r--searx/settings_defaults.py4
-rw-r--r--searx/valkeydb.py65
-rw-r--r--searx/valkeylib.py (renamed from searx/redislib.py)52
-rwxr-xr-xsearx/webapp.py4
-rwxr-xr-xutils/lib.sh2
-rwxr-xr-xutils/lib_redis.sh290
-rwxr-xr-xutils/lib_valkey.sh73
-rwxr-xr-xutils/searxng.sh133
-rw-r--r--utils/searxng_check.py11
-rw-r--r--utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources6
-rw-r--r--utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources6
-rw-r--r--utils/templates/etc/searxng/settings.yml6
-rw-r--r--utils/templates/lib/systemd/system/searxng-redis.service42
43 files changed, 467 insertions, 723 deletions
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index 9e5871918..e278a7a84 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
FROM mcr.microsoft.com/devcontainers/base:debian
RUN apt-get update && \
- apt-get -y install python3 python3-venv redis firefox-esr graphviz imagemagick librsvg2-bin fonts-dejavu shellcheck
+ apt-get -y install python3 python3-venv valkey firefox-esr graphviz imagemagick librsvg2-bin fonts-dejavu shellcheck
diff --git a/Makefile b/Makefile
index 3071609ed..bd8c4442c 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,7 @@ test.shell:
utils/lib_go.sh \
utils/lib_nvm.sh \
utils/lib_redis.sh \
+ utils/lib_valkey.sh \
utils/searxng.sh \
utils/lxc.sh \
utils/lxc-searxng.env
diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot
index 49b03d157..c131186d0 100644
--- a/docs/admin/arch_public.dot
+++ b/docs/admin/arch_public.dot
@@ -7,7 +7,8 @@ digraph G {
rp [label="reverse proxy"];
static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray];
uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searxng.sh.html"]
- redis [label="redis DB", shape=cylinder];
+ valkey [label="valkey DB", shape=cylinder];
+
searxng1 [label="SearXNG #1", fontcolor=blue3];
searxng2 [label="SearXNG #2", fontcolor=blue3];
searxng3 [label="SearXNG #3", fontcolor=blue3];
@@ -21,10 +22,10 @@ digraph G {
{ rank=same; static rp };
rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray];
rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"];
- uwsgi -> searxng1 -> redis;
- uwsgi -> searxng2 -> redis;
- uwsgi -> searxng3 -> redis;
- uwsgi -> searxng4 -> redis;
+ uwsgi -> searxng1 -> valkey;
+ uwsgi -> searxng2 -> valkey;
+ uwsgi -> searxng3 -> valkey;
+ uwsgi -> searxng4 -> valkey;
}
}
diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst
index 78da22f45..a2152409e 100644
--- a/docs/admin/installation-uwsgi.rst
+++ b/docs/admin/installation-uwsgi.rst
@@ -235,19 +235,19 @@ major release is from Dec. 2013, since the there had been only bugfix releases
**In Tyrant mode, there is no way to get additional groups, and the uWSGI
process misses additional permissions that may be needed.**
-For example on Fedora (RHEL): If you try to install a redis DB with socket
+For example on Fedora (RHEL): If you try to install a valkey DB with socket
communication and you want to connect to it from the SearXNG uWSGI, you will see a
*Permission denied* in the log of your instance::
- ERROR:searx.redisdb: [searxng (993)] can't connect redis DB ...
- ERROR:searx.redisdb: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied.
+ ERROR:searx.valkeydb: [searxng (993)] can't connect valkey DB ...
+ ERROR:searx.valkeydb: Error 13 connecting to unix socket: /usr/local/searxng-valkey/run/valkey.sock. Permission denied.
ERROR:searx.plugins.limiter: init limiter DB failed!!!
Even if your *searxng* user of the uWSGI process is added to additional groups
-to give access to the socket from the redis DB::
+to give access to the socket from the valkey DB::
$ groups searxng
- searxng : searxng searxng-redis
+ searxng : searxng searxng-valkey
To see the effective groups of the uwsgi process, you have to look at the status
of the process, by example::
@@ -257,7 +257,7 @@ of the process, by example::
searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini
Here you can see that the additional "Groups" of PID 186 are unset (missing gid
-of ``searxng-redis``)::
+of ``searxng-valkey``)::
$ cat /proc/186/task/186/status
...
diff --git a/docs/admin/searx.limiter.rst b/docs/admin/searx.limiter.rst
index c23635571..6fac23868 100644
--- a/docs/admin/searx.limiter.rst
+++ b/docs/admin/searx.limiter.rst
@@ -6,7 +6,7 @@ Limiter
.. sidebar:: info
- The limiter requires a :ref:`Redis <settings redis>` database.
+ The limiter requires a :ref:`Valkey <settings valkey>` database.
.. contents::
:depth: 2
diff --git a/docs/admin/settings/index.rst b/docs/admin/settings/index.rst
index eb0e07a53..b9d0408ea 100644
--- a/docs/admin/settings/index.rst
+++ b/docs/admin/settings/index.rst
@@ -20,8 +20,7 @@ Settings
settings_server
settings_ui
settings_redis
+ settings_valkey
settings_outgoing
settings_categories_as_tabs
settings_plugins
-
-
diff --git a/docs/admin/settings/settings_redis.rst b/docs/admin/settings/settings_redis.rst
index 9fb067553..e1221ac32 100644
--- a/docs/admin/settings/settings_redis.rst
+++ b/docs/admin/settings/settings_redis.rst
@@ -4,46 +4,34 @@
``redis:``
==========
-.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
+.. _Valkey: https://valkey.io
-A redis DB can be connected by an URL, in :py:obj:`searx.redisdb` you
-will find a description to test your redis connection in SearXNG. When using
-sockets, don't forget to check the access rights on the socket::
+.. attention::
- ls -la /usr/local/searxng-redis/run/redis.sock
- srwxrwx--- 1 searxng-redis searxng-redis ... /usr/local/searxng-redis/run/redis.sock
+ SearXNG is switching from the Redis DB to Valkey_. The configuration
+ description of Valkey_ in SearXNG can be found here: :ref:`settings
+ <settings valkey>`.
-In this example read/write access is given to the *searxng-redis* group. To get
-access rights to redis instance (the socket), your SearXNG (or even your
-developer) account needs to be added to the *searxng-redis* group.
-
-``url`` : ``$SEARXNG_REDIS_URL``
- URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`::
-
- redis://[[username]:[password]]@localhost:6379/0
- rediss://[[username]:[password]]@localhost:6379/0
- unix://[[username]:[password]]@/path/to/socket.sock?db=0
+If you have built and installed a local Redis DB for SearXNG, it is recommended
+to uninstall it now and replace it with the installation of a Valkey_ DB.
.. _Redis Developer Notes:
Redis Developer Notes
=====================
-To set up a local redis instance, first set the socket path of the Redis DB
-in your YAML setting:
+To uninstall SearXNG's local Redis DB you can use:
-.. code:: yaml
+.. code:: sh
- redis:
- url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+ # stop your SearXNG instance
+ $ ./utils/searxng.sh remove.redis
-Then use the following commands to install the redis instance (:ref:`manage
-redis.help`):
+Remove the Redis DB in your YAML setting:
-.. code:: sh
+.. code:: yaml
- $ ./manage redis.build
- $ sudo -H ./manage redis.install
- $ sudo -H ./manage redis.addgrp "${USER}"
- # don't forget to logout & login to get member of group
+ redis:
+ url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+To install Valkey_ read: :ref:`Valkey Developer Notes`
diff --git a/docs/admin/settings/settings_server.rst b/docs/admin/settings/settings_server.rst
index 84908d43f..59c0d7791 100644
--- a/docs/admin/settings/settings_server.rst
+++ b/docs/admin/settings/settings_server.rst
@@ -36,7 +36,7 @@
``limiter`` : ``$SEARXNG_LIMITER``
Rate limit the number of request on the instance, block some bots. The
- :ref:`limiter` requires a :ref:`settings redis` database.
+ :ref:`limiter` requires a :ref:`settings valkey` database.
.. _public_instance:
diff --git a/docs/admin/settings/settings_valkey.rst b/docs/admin/settings/settings_valkey.rst
new file mode 100644
index 000000000..396d5c8ad
--- /dev/null
+++ b/docs/admin/settings/settings_valkey.rst
@@ -0,0 +1,53 @@
+.. _settings valkey:
+
+===========
+``valkey:``
+===========
+
+.. _Valkey:
+ https://valkey.io
+.. _Valkey-Installation:
+ https://valkey.io/topics/installation/
+.. _There are several ways to specify a database number:
+ https://valkey-py.readthedocs.io/en/stable/connections.html#valkey.Valkey.from_url
+
+A Valkey_ DB can be connected by an URL, in section :ref:`valkey db` you will
+find a description to test your valkey connection in SearXNG.
+
+``url`` : ``$SEARXNG_VALKEY_URL``
+ URL to connect valkey database. `There are several ways to specify a database
+ number`_::
+
+ valkey://[[username]:[password]]@localhost:6379/0
+ valkeys://[[username]:[password]]@localhost:6379/0
+ unix://[[username]:[password]]@/path/to/socket.sock?db=0
+
+ When using sockets, don't forget to check the access rights on the socket::
+
+ ls -la /usr/local/searxng-valkey/run/valkey.sock
+ srwxrwx--- 1 searxng-valkey searxng-valkey ... /usr/local/searxng-valkey/run/valkey.sock
+
+ In this example read/write access is given to the *searxng-valkey* group. To
+ get access rights to valkey instance (the socket), your SearXNG (or even your
+ developer) account needs to be added to the *searxng-valkey* group.
+
+
+.. _Valkey Developer Notes:
+
+Valkey Developer Notes
+======================
+
+To set up a local Valkey_ DB, set the URL connector in your YAML setting:
+
+.. code:: yaml
+
+ valkey:
+ url: valkey://localhost:6379/0
+
+To install a local Valkey_ DB from package manager read `Valkey-Installation`_
+or use:
+
+.. code:: sh
+
+ $ ./utils/searxng.sh install valkey
+ # restart your SearXNG instance
diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst
index 16715f00d..d4a197603 100644
--- a/docs/admin/update-searxng.rst
+++ b/docs/admin/update-searxng.rst
@@ -56,7 +56,7 @@ SearXNG is growing rapidly, the services and opportunities are change every now
and then, to name just a few:
- Bot protection has been switched from filtron to SearXNG's :ref:`limiter
- <limiter>`, this requires a :ref:`Redis <settings redis>` database.
+ <limiter>`, this requires a :ref:`Valkey <settings valkey>` database.
- To save bandwidth :ref:`cache busting <static_use_hash>` has been implemented.
To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi
@@ -89,5 +89,5 @@ to see if there are some left overs. In this example there exists a *old*
--------------
ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/
...
- INFO searx.redisdb : connecting to Redis db=0 path='/usr/local/searxng-redis/run/redis.sock'
- INFO searx.redisdb : connected to Redis
+ INFO searx.valkeydb : connecting to Valkey db=0 path='/usr/local/searxng-valkey/run/valkey.sock'
+ INFO searx.valkeydb : connected to Valkey
diff --git a/docs/conf.py b/docs/conf.py
index 468c160cb..a7221e48b 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -149,7 +149,7 @@ intersphinx_mapping = {
"jinja": ("https://jinja.palletsprojects.com/en/stable/", None),
"linuxdoc" : ("https://return42.github.io/linuxdoc/", None),
"sphinx" : ("https://www.sphinx-doc.org/en/master/", None),
- "redis": ('https://redis.readthedocs.io/en/stable/', None),
+ "valkey": ('https://valkey-py.readthedocs.io/en/stable/', None),
}
issues_github_path = "searxng/searxng"
diff --git a/docs/dev/engines/offline/nosql-engines.rst b/docs/dev/engines/offline/nosql-engines.rst
index 76f5cfb61..551600c4e 100644
--- a/docs/dev/engines/offline/nosql-engines.rst
+++ b/docs/dev/engines/offline/nosql-engines.rst
@@ -7,7 +7,7 @@ NoSQL databases
.. sidebar:: further read
- `NoSQL databases <https://en.wikipedia.org/wiki/NoSQL>`_
- - `redis.io <https://redis.io/>`_
+ - `valkey.io <https://valkey.io/>`_
- `MongoDB <https://www.mongodb.com>`_
.. contents::
@@ -22,7 +22,7 @@ NoSQL databases
The following `NoSQL databases`_ are supported:
-- :ref:`engine redis_server`
+- :ref:`engine valkey_server`
- :ref:`engine mongodb`
All of the engines above are just commented out in the :origin:`settings.yml
@@ -45,7 +45,7 @@ section :ref:`private engines`.
Extra Dependencies
==================
-For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to
+For using :ref:`engine valkey_server` or :ref:`engine mongodb` you need to
install additional packages in Python's Virtual Environment of your SearXNG
instance. To switch into the environment (:ref:`searxng-src`) you can use
:ref:`searxng.sh`::
@@ -61,20 +61,20 @@ Configure the engines
their structure.
-.. _engine redis_server:
+.. _engine valkey_server:
-Redis Server
-------------
+Valkey Server
+-------------
-.. _redis: https://github.com/andymccurdy/redis-py#installation
+.. _valkey: https://github.com/andymccurdy/valkey-py#installation
.. sidebar:: info
- - ``pip install`` redis_
- - redis.io_
- - :origin:`redis_server.py <searx/engines/redis_server.py>`
+ - ``pip install`` valkey_
+ - valkey.io_
+ - :origin:`valkey_server.py <searx/engines/valkey_server.py>`
-.. automodule:: searx.engines.redis_server
+.. automodule:: searx.engines.valkey_server
:members:
@@ -94,4 +94,3 @@ MongoDB
.. automodule:: searx.engines.mongodb
:members:
-
diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst
index 9edd9f672..22c16ff0d 100644
--- a/docs/dev/lxcdev.rst
+++ b/docs/dev/lxcdev.rst
@@ -140,7 +140,7 @@ proxy :ref:`installation nginx` into the archlinux container run:
.. sidebar:: Fully functional SearXNG suite
From here on you have a fully functional SearXNG suite (including a
- :ref:`redis db`).
+ :ref:`valkey db`).
In such a SearXNG suite admins can maintain and access the debug log of the
services quite easy.
diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst
index a66ce2f22..4781b2ce6 100644
--- a/docs/dev/makefile.rst
+++ b/docs/dev/makefile.rst
@@ -362,17 +362,6 @@ can be used to convenient run common build tasks of the static files.
.. program-output:: bash -c "cd ..; ./manage static.help"
-.. _manage redis.help:
-
-``./manage redis.help``
-=======================
-
-The ``./manage redis.*`` command line can be used to convenient run common Redis
-tasks (:ref:`Redis developer notes`).
-
-.. program-output:: bash -c "cd ..; ./manage redis.help"
-
-
.. _manage go.help:
``./manage go.help``
diff --git a/docs/src/searx.redisdb.rst b/docs/src/searx.redisdb.rst
deleted file mode 100644
index 625378c91..000000000
--- a/docs/src/searx.redisdb.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-.. _redis db:
-
-========
-Redis DB
-========
-
-.. automodule:: searx.redisdb
- :members:
diff --git a/docs/src/searx.redislib.rst b/docs/src/searx.redislib.rst
deleted file mode 100644
index b4604574c..000000000
--- a/docs/src/searx.redislib.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-.. _searx.redis:
-
-=============
-Redis Library
-=============
-
-.. automodule:: searx.redislib
- :members:
diff --git a/docs/src/searx.valkeydb.rst b/docs/src/searx.valkeydb.rst
new file mode 100644
index 000000000..165b88111
--- /dev/null
+++ b/docs/src/searx.valkeydb.rst
@@ -0,0 +1,8 @@
+.. _valkey db:
+
+==========
+Valkey DB
+==========
+
+.. automodule:: searx.valkeydb
+ :members:
diff --git a/docs/src/searx.valkeylib.rst b/docs/src/searx.valkeylib.rst
new file mode 100644
index 000000000..a0f935d2a
--- /dev/null
+++ b/docs/src/searx.valkeylib.rst
@@ -0,0 +1,8 @@
+.. _searx.valkey:
+
+==============
+Valkey Library
+==============
+
+.. automodule:: searx.valkeylib
+ :members:
diff --git a/manage b/manage
index ee2a29281..22f3ae821 100755
--- a/manage
+++ b/manage
@@ -35,8 +35,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
# shellcheck source=utils/lib_go.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
-# shellcheck source=utils/lib_redis.sh
-source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"
+# shellcheck source=utils/lib_valkey.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
# shellcheck source=utils/lib_sxng_vite.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"
@@ -52,12 +52,6 @@ GECKODRIVER_VERSION="v0.35.0"
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
-_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
-# set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
-if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then
- export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0"
-fi
-
YAMLLINT_FILES=()
while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
@@ -83,10 +77,8 @@ docs.:
gecko.driver:
download & install geckodriver if not already installed (required for
robot_tests)
-redis:
- build : build redis binaries at $(redis._get_dist)
- install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
- help : show more redis commands
+valkey:
+ install : create user (${VALKEY_USER}) and install systemd service (${VALKEY_SERVICE_NAME})
py.:
build : Build python packages at ./${PYDIST}
clean : delete virtualenv and intermediate py files
@@ -109,7 +101,7 @@ EOF
vite.help
cat <<EOF
environment ...
- SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
+ SEARXNG_VALKEY_URL : ${SEARXNG_VALKEY_URL}
EOF
}
diff --git a/requirements.txt b/requirements.txt
index f6601748b..732958a0c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,7 +12,7 @@ httpx-socks[asyncio]==0.10.0
Brotli==1.1.0
uvloop==0.21.0
setproctitle==1.3.6
-redis==5.2.1
+valkey==6.1.0
markdown-it-py==3.0.0
fasttext-predict==0.9.2.4
tomli==2.2.1; python_version < '3.11'
diff --git a/searx/botdetection/__init__.py b/searx/botdetection/__init__.py
index 51f437cf0..4079d97a9 100644
--- a/searx/botdetection/__init__.py
+++ b/searx/botdetection/__init__.py
@@ -12,11 +12,11 @@ from ._helpers import too_many_requests
__all__ = ['dump_request', 'get_network', 'get_real_ip', 'too_many_requests']
-redis_client = None
+valkey_client = None
cfg = None
-def init(_cfg, _redis_client):
- global redis_client, cfg # pylint: disable=global-statement
- redis_client = _redis_client
+def init(_cfg, _valkey_client):
+ global valkey_client, cfg # pylint: disable=global-statement
+ valkey_client = _valkey_client
cfg = _cfg
diff --git a/searx/botdetection/ip_limit.py b/searx/botdetection/ip_limit.py
index 161a9826e..93af8b7c5 100644
--- a/searx/botdetection/ip_limit.py
+++ b/searx/botdetection/ip_limit.py
@@ -6,8 +6,8 @@ Method ``ip_limit``
The ``ip_limit`` method counts request from an IP in *sliding windows*. If
there are to many requests in a sliding window, the request is evaluated as a
-bot request. This method requires a redis DB and needs a HTTP X-Forwarded-For_
-header. To take privacy only the hash value of an IP is stored in the redis DB
+bot request. This method requires a valkey DB and needs a HTTP X-Forwarded-For_
+header. To take privacy only the hash value of an IP is stored in the valkey DB
and at least for a maximum of 10 minutes.
The :py:obj:`.link_token` method can be used to investigate whether a request is
@@ -46,8 +46,8 @@ import flask
import werkzeug
from searx.extended_types import SXNG_Request
-from searx import redisdb
-from searx.redislib import incr_sliding_window, drop_counter
+from searx import valkeydb
+from searx.valkeylib import incr_sliding_window, drop_counter
from . import link_token
from . import config
@@ -97,14 +97,14 @@ def filter_request(
) -> werkzeug.Response | None:
# pylint: disable=too-many-return-statements
- redis_client = redisdb.client()
+ valkey_client = valkeydb.client()
if network.is_link_local and not cfg['botdetection.ip_limit.filter_link_local']:
logger.debug("network %s is link-local -> not monitored by ip_limit method", network.compressed)
return None
if request.args.get('format', 'html') != 'html':
- c = incr_sliding_window(redis_client, 'ip_limit.API_WINDOW:' + network.compressed, API_WINDOW)
+ c = incr_sliding_window(valkey_client, 'ip_limit.API_WINDOW:' + network.compressed, API_WINDOW)
if c > API_MAX:
return too_many_requests(network, "too many request in API_WINDOW")
@@ -114,12 +114,12 @@ def filter_request(
if not suspicious:
# this IP is no longer suspicious: release ip again / delete the counter of this IP
- drop_counter(redis_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed)
+ drop_counter(valkey_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed)
return None
# this IP is suspicious: count requests from this IP
c = incr_sliding_window(
- redis_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed, SUSPICIOUS_IP_WINDOW
+ valkey_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed, SUSPICIOUS_IP_WINDOW
)
if c > SUSPICIOUS_IP_MAX:
logger.error("BLOCK: too many request from %s in SUSPICIOUS_IP_WINDOW (redirect to /)", network)
@@ -127,22 +127,22 @@ def filter_request(
response.headers["Cache-Control"] = "no-store, max-age=0"
return response
- c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
+ c = incr_sliding_window(valkey_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
if c > BURST_MAX_SUSPICIOUS:
return too_many_requests(network, "too many request in BURST_WINDOW (BURST_MAX_SUSPICIOUS)")
- c = incr_sliding_window(redis_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
+ c = incr_sliding_window(valkey_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
if c > LONG_MAX_SUSPICIOUS:
return too_many_requests(network, "too many request in LONG_WINDOW (LONG_MAX_SUSPICIOUS)")
return None
# vanilla limiter without extensions counts BURST_MAX and LONG_MAX
- c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
+ c = incr_sliding_window(valkey_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
if c > BURST_MAX:
return too_many_requests(network, "too many request in BURST_WINDOW (BURST_MAX)")
- c = incr_sliding_window(redis_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
+ c = incr_sliding_window(valkey_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
if c > LONG_MAX:
return too_many_requests(network, "too many request in LONG_WINDOW (LONG_MAX)")
diff --git a/searx/botdetection/link_token.py b/searx/botdetection/link_token.py
index c255790cb..600796380 100644
--- a/searx/botdetection/link_token.py
+++ b/searx/botdetection/link_token.py
@@ -10,7 +10,7 @@ a ping by request a static URL.
.. note::
- This method requires a redis DB and needs a HTTP X-Forwarded-For_ header.
+ This method requires a valkey DB and needs a HTTP X-Forwarded-For_ header.
To get in use of this method a flask URL route needs to be added:
@@ -45,8 +45,8 @@ import string
import random
from searx import logger
-from searx import redisdb
-from searx.redislib import secret_hash
+from searx import valkeydb
+from searx.valkeylib import secret_hash
from searx.extended_types import SXNG_Request
from ._helpers import (
@@ -76,17 +76,17 @@ def is_suspicious(network: IPv4Network | IPv6Network, request: SXNG_Request, ren
:py:obj:`PING_LIVE_TIME`.
"""
- redis_client = redisdb.client()
- if not redis_client:
+ valkey_client = valkeydb.client()
+ if not valkey_client:
return False
ping_key = get_ping_key(network, request)
- if not redis_client.get(ping_key):
+ if not valkey_client.get(ping_key):
logger.info("missing ping (IP: %s) / request: %s", network.compressed, ping_key)
return True
if renew:
- redis_client.set(ping_key, 1, ex=PING_LIVE_TIME)
+ valkey_client.set(ping_key, 1, ex=PING_LIVE_TIME)
logger.debug("found ping for (client) network %s -> %s", network.compressed, ping_key)
return False
@@ -98,9 +98,9 @@ def ping(request: SXNG_Request, token: str):
The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
"""
- from . import redis_client, cfg # pylint: disable=import-outside-toplevel, cyclic-import
+ from . import valkey_client, cfg # pylint: disable=import-outside-toplevel, cyclic-import
- if not redis_client:
+ if not valkey_client:
return
if not token_is_valid(token):
return
@@ -110,7 +110,7 @@ def ping(request: SXNG_Request, token: str):
ping_key = get_ping_key(network, request)
logger.debug("store ping_key for (client) network %s (IP %s) -> %s", network.compressed, real_ip, ping_key)
- redis_client.set(ping_key, 1, ex=PING_LIVE_TIME)
+ valkey_client.set(ping_key, 1, ex=PING_LIVE_TIME)
def get_ping_key(network: IPv4Network | IPv6Network, request: SXNG_Request) -> str:
@@ -134,21 +134,21 @@ def token_is_valid(token) -> bool:
def get_token() -> str:
"""Returns current token. If there is no currently active token a new token
- is generated randomly and stored in the redis DB.
+ is generated randomly and stored in the valkey DB.
- :py:obj:`TOKEN_LIVE_TIME`
- :py:obj:`TOKEN_KEY`
"""
- redis_client = redisdb.client()
- if not redis_client:
- # This function is also called when limiter is inactive / no redis DB
+ valkey_client = valkeydb.client()
+ if not valkey_client:
+ # This function is also called when limiter is inactive / no valkey DB
# (see render function in webapp.py)
return '12345678'
- token = redis_client.get(TOKEN_KEY)
+ token = valkey_client.get(TOKEN_KEY)
if token:
token = token.decode('UTF-8')
else:
token = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
- redis_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
+ valkey_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
return token
diff --git a/searx/engines/redis_server.py b/searx/engines/valkey_server.py
index eebb5809b..b2d3dd26f 100644
--- a/searx/engines/redis_server.py
+++ b/searx/engines/valkey_server.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Redis is an open source (BSD licensed), in-memory data structure (key value
-based) store. Before configuring the ``redis_server`` engine, you must install
-the dependency redis_.
+"""Valkey is an open source (BSD licensed), in-memory data structure (key value
+based) store. Before configuring the ``valkey_server`` engine, you must install
+the dependency valkey_.
Configuration
=============
@@ -17,11 +17,11 @@ Below is an example configuration:
.. code:: yaml
- # Required dependency: redis
+ # Required dependency: valkey
- - name: myredis
+ - name: myvalkey
shortcut : rds
- engine: redis_server
+ engine: valkey_server
exact_match_only: false
host: '127.0.0.1'
port: 6379
@@ -34,13 +34,13 @@ Implementations
"""
-import redis # pylint: disable=import-error
+import valkey # pylint: disable=import-error
from searx.result_types import EngineResults
engine_type = 'offline'
-# redis connection variables
+# valkey connection variables
host = '127.0.0.1'
port = 6379
password = ''
@@ -50,12 +50,12 @@ db = 0
paging = False
exact_match_only = True
-_redis_client = None
+_valkey_client = None
def init(_engine_settings):
- global _redis_client # pylint: disable=global-statement
- _redis_client = redis.StrictRedis(
+ global _valkey_client # pylint: disable=global-statement
+ _valkey_client = valkey.StrictValkey(
host=host,
port=port,
db=db,
@@ -72,28 +72,28 @@ def search(query, _params) -> EngineResults:
res.add(res.types.KeyValue(kvmap=kvmap))
return res
- kvmap: dict[str, str] = _redis_client.hgetall(query)
+ kvmap: dict[str, str] = _valkey_client.hgetall(query)
if kvmap:
res.add(res.types.KeyValue(kvmap=kvmap))
elif " " in query:
qset, rest = query.split(" ", 1)
- for row in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
+ for row in _valkey_client.hscan_iter(qset, match='*{}*'.format(rest)):
res.add(res.types.KeyValue(kvmap={row[0]: row[1]}))
return res
def search_keys(query) -> list[dict]:
ret = []
- for key in _redis_client.scan_iter(match='*{}*'.format(query)):
- key_type = _redis_client.type(key)
+ for key in _valkey_client.scan_iter(match='*{}*'.format(query)):
+ key_type = _valkey_client.type(key)
res = None
if key_type == 'hash':
- res = _redis_client.hgetall(key)
+ res = _valkey_client.hgetall(key)
elif key_type == 'list':
- res = dict(enumerate(_redis_client.lrange(key, 0, -1)))
+ res = dict(enumerate(_valkey_client.lrange(key, 0, -1)))
if res:
- res['redis_key'] = key
+ res['valkey_key'] = key
ret.append(res)
return ret
diff --git a/searx/limiter.py b/searx/limiter.py
index 92b38c68f..99bc338d1 100644
--- a/searx/limiter.py
+++ b/searx/limiter.py
@@ -17,7 +17,7 @@ from the :ref:`botdetection`:
the time.
- Detection & dynamically :ref:`botdetection rate limit` of bots based on the
- behavior of the requests. For dynamically changeable IP lists a Redis
+ behavior of the requests. For dynamically changeable IP lists a Valkey
database is needed.
The prerequisite for IP based methods is the correct determination of the IP of
@@ -50,13 +50,13 @@ To enable the limiter activate:
...
limiter: true # rate limit the number of request on the instance, block some bots
-and set the redis-url connection. Check the value, it depends on your redis DB
-(see :ref:`settings redis`), by example:
+and set the valkey-url connection. Check the value, it depends on your valkey DB
+(see :ref:`settings valkey`), by example:
.. code:: yaml
- redis:
- url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+ valkey:
+ url: valkey://localhost:6379/0
Configure Limiter
@@ -102,7 +102,7 @@ import werkzeug
from searx import (
logger,
- redisdb,
+ valkeydb,
)
from searx import botdetection
from searx.extended_types import SXNG_Request, sxng_request
@@ -217,7 +217,7 @@ def pre_request():
def is_installed():
- """Returns ``True`` if limiter is active and a redis DB is available."""
+ """Returns ``True`` if limiter is active and a valkey DB is available."""
return _INSTALLED
@@ -229,15 +229,15 @@ def initialize(app: flask.Flask, settings):
# (e.g. the self_info plugin uses the botdetection to get client IP)
cfg = get_cfg()
- redis_client = redisdb.client()
- botdetection.init(cfg, redis_client)
+ valkey_client = valkeydb.client()
+ botdetection.init(cfg, valkey_client)
if not (settings['server']['limiter'] or settings['server']['public_instance']):
return
- if not redis_client:
+ if not valkey_client:
logger.error(
- "The limiter requires Redis, please consult the documentation: "
+ "The limiter requires Valkey, please consult the documentation: "
"https://docs.searxng.org/admin/searx.limiter.html"
)
if settings['server']['public_instance']:
diff --git a/searx/redisdb.py b/searx/redisdb.py
deleted file mode 100644
index bed0c347b..000000000
--- a/searx/redisdb.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Implementation of the redis client (redis-py_).
-
-.. _redis-py: https://github.com/redis/redis-py
-
-This implementation uses the :ref:`settings redis` setup from ``settings.yml``.
-A redis DB connect can be tested by::
-
- >>> from searx import redisdb
- >>> redisdb.initialize()
- True
- >>> db = redisdb.client()
- >>> db.set("foo", "bar")
- True
- >>> db.get("foo")
- b'bar'
- >>>
-
-"""
-
-import os
-import pwd
-import logging
-import redis
-from searx import get_setting
-
-
-OLD_REDIS_URL_DEFAULT_URL = 'unix:///usr/local/searxng-redis/run/redis.sock?db=0'
-"""This was the default Redis URL in settings.yml."""
-
-_CLIENT = None
-logger = logging.getLogger(__name__)
-
-
-def client() -> redis.Redis:
- return _CLIENT
-
-
-def initialize():
- global _CLIENT # pylint: disable=global-statement
- redis_url = get_setting('redis.url')
- if not redis_url:
- return False
- try:
- # create a client, but no connection is done
- _CLIENT = redis.Redis.from_url(redis_url)
-
- # log the parameters as seen by the redis lib, without the password
- kwargs = _CLIENT.get_connection_kwargs().copy()
- kwargs.pop('password', None)
- kwargs = ' '.join([f'{k}={v!r}' for k, v in kwargs.items()])
- logger.info("connecting to Redis %s", kwargs)
-
- # check the connection
- _CLIENT.ping()
-
- # no error: the redis connection is working
- logger.info("connected to Redis")
- return True
- except redis.exceptions.RedisError as e:
- _CLIENT = None
- _pw = pwd.getpwuid(os.getuid())
- logger.exception("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid)
- if redis_url == OLD_REDIS_URL_DEFAULT_URL and isinstance(e, redis.exceptions.ConnectionError):
- logger.info(
- "You can safely ignore the above Redis error if you don't use Redis. "
- "You can remove this error by setting redis.url to false in your settings.yml."
- )
- return False
diff --git a/searx/search/checker/background.py b/searx/search/checker/background.py
index 7333e6ad0..1890c77d5 100644
--- a/searx/search/checker/background.py
+++ b/searx/search/checker/background.py
@@ -8,18 +8,18 @@ import os
import signal
from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
-import redis.exceptions
+import valkey.exceptions
from searx import logger, settings, sxng_debug
-from searx.redisdb import client as get_redis_client
+from searx.valkeydb import client as get_valkey_client
from searx.exceptions import SearxSettingsException
from searx.search.processors import PROCESSORS
from searx.search.checker import Checker
from searx.search.checker.scheduler import scheduler_function
-REDIS_RESULT_KEY = 'SearXNG_checker_result'
-REDIS_LOCK_KEY = 'SearXNG_checker_lock'
+VALKEY_RESULT_KEY = 'SearXNG_checker_result'
+VALKEY_LOCK_KEY = 'SearXNG_checker_lock'
CheckerResult = Union['CheckerOk', 'CheckerErr', 'CheckerOther']
@@ -77,23 +77,23 @@ def _get_interval(every: Any, error_msg: str) -> Tuple[int, int]:
def get_result() -> CheckerResult:
- client = get_redis_client()
+ client = get_valkey_client()
if client is None:
- # without Redis, the checker is disabled
+ # without Valkey, the checker is disabled
return {'status': 'disabled'}
- serialized_result: Optional[bytes] = client.get(REDIS_RESULT_KEY)
+ serialized_result: Optional[bytes] = client.get(VALKEY_RESULT_KEY)
if serialized_result is None:
- # the Redis key does not exist
+ # the Valkey key does not exist
return {'status': 'unknown'}
return json.loads(serialized_result)
def _set_result(result: CheckerResult):
- client = get_redis_client()
+ client = get_valkey_client()
if client is None:
- # without Redis, the function does nothing
+ # without Valkey, the function does nothing
return
- client.set(REDIS_RESULT_KEY, json.dumps(result))
+ client.set(VALKEY_RESULT_KEY, json.dumps(result))
def _timestamp():
@@ -102,9 +102,9 @@ def _timestamp():
def run():
try:
- # use a Redis lock to make sure there is no checker running at the same time
+ # use a Valkey lock to make sure there is no checker running at the same time
# (this should not happen, this is a safety measure)
- with get_redis_client().lock(REDIS_LOCK_KEY, blocking_timeout=60, timeout=3600):
+ with get_valkey_client().lock(VALKEY_LOCK_KEY, blocking_timeout=60, timeout=3600):
logger.info('Starting checker')
result: CheckerOk = {'status': 'ok', 'engines': {}, 'timestamp': _timestamp()}
for name, processor in PROCESSORS.items():
@@ -118,7 +118,7 @@ def run():
_set_result(result)
logger.info('Check done')
- except redis.exceptions.LockError:
+ except valkey.exceptions.LockError:
_set_result({'status': 'error', 'timestamp': _timestamp()})
logger.exception('Error while running the checker')
except Exception: # pylint: disable=broad-except
@@ -149,9 +149,9 @@ def initialize():
logger.info('Checker scheduler is disabled')
return
- # make sure there is a Redis connection
- if get_redis_client() is None:
- logger.error('The checker requires Redis')
+ # make sure there is a Valkey connection
+ if get_valkey_client() is None:
+ logger.error('The checker requires Valkey')
return
# start the background scheduler
diff --git a/searx/search/checker/scheduler.lua b/searx/search/checker/scheduler.lua
index 0de9b404d..ec318ddd6 100644
--- a/searx/search/checker/scheduler.lua
+++ b/searx/search/checker/scheduler.lua
@@ -2,9 +2,9 @@
--
-- This script is not a string in scheduler.py, so editors can provide syntax highlighting.
--- The Redis KEY is defined here and not in Python on purpose:
+-- The Valkey KEY is defined here and not in Python on purpose:
-- only this LUA script can read and update this key to avoid lock and concurrency issues.
-local redis_key = 'SearXNG_checker_next_call_ts'
+local valkey_key = 'SearXNG_checker_next_call_ts'
local now = redis.call('TIME')[1]
local start_after_from = ARGV[1]
@@ -12,14 +12,14 @@ local start_after_to = ARGV[2]
local every_from = ARGV[3]
local every_to = ARGV[4]
-local next_call_ts = redis.call('GET', redis_key)
+local next_call_ts = redis.call('GET', valkey_key)
if (next_call_ts == false or next_call_ts == nil) then
- -- the scheduler has never run on this Redis instance, so:
+ -- the scheduler has never run on this Valkey instance, so:
-- 1/ the scheduler does not run now
-- 2/ the next call is a random time between start_after_from and start_after_to
local initial_delay = math.random(start_after_from, start_after_to)
- redis.call('SET', redis_key, now + initial_delay)
+ redis.call('SET', valkey_key, now + initial_delay)
return { false, initial_delay }
end
@@ -31,6 +31,6 @@ if call_now then
-- the checker runs now, define the timestamp of the next call:
-- this is a random delay between every_from and every_to
local periodic_delay = math.random(every_from, every_to)
- next_call_ts = redis.call('INCRBY', redis_key, periodic_delay)
+ next_call_ts = redis.call('INCRBY', valkey_key, periodic_delay)
end
return { call_now, next_call_ts - now }
diff --git a/searx/search/checker/scheduler.py b/searx/search/checker/scheduler.py
index c0d3f799a..b093a9ab7 100644
--- a/searx/search/checker/scheduler.py
+++ b/searx/search/checker/scheduler.py
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring
-"""Lame scheduler which use Redis as a source of truth:
-* the Redis key SearXNG_checker_next_call_ts contains the next time the embedded checker should run.
-* to avoid lock, a unique Redis script reads and updates the Redis key SearXNG_checker_next_call_ts.
-* this Redis script returns a list of two elements:
+"""Lame scheduler which use Valkey as a source of truth:
+* the Valkey key SearXNG_checker_next_call_ts contains the next time the embedded checker should run.
+* to avoid lock, a unique Valkey script reads and updates the Valkey key SearXNG_checker_next_call_ts.
+* this Valkey script returns a list of two elements:
* the first one is a boolean. If True, the embedded checker must run now in this worker.
- * the second element is the delay in second to wait before the next call to the Redis script.
+ * the second element is the delay in second to wait before the next call to the Valkey script.
This scheduler is not generic on purpose: if more feature are required, a dedicate scheduler must be used
(= a better scheduler should not use the web workers)
@@ -16,8 +16,8 @@ import time
from pathlib import Path
from typing import Callable
-from searx.redisdb import client as get_redis_client
-from searx.redislib import lua_script_storage
+from searx.valkeydb import client as get_valkey_client
+from searx.valkeylib import lua_script_storage
logger = logging.getLogger('searx.search.checker')
@@ -29,7 +29,7 @@ def scheduler_function(start_after_from: int, start_after_to: int, every_from: i
"""Run the checker periodically. The function never returns.
Parameters:
- * start_after_from and start_after_to: when to call "callback" for the first on the Redis instance
+ * start_after_from and start_after_to: when to call "callback" for the first on the Valkey instance
* every_from and every_to: after the first call, how often to call "callback"
There is no issue:
@@ -38,11 +38,11 @@ def scheduler_function(start_after_from: int, start_after_to: int, every_from: i
"""
scheduler_now_script = SCHEDULER_LUA.open().read()
while True:
- # ask the Redis script what to do
+ # ask the Valkey script what to do
# the script says
# * if the checker must run now.
# * how to long to way before calling the script again (it can be call earlier, but not later).
- script = lua_script_storage(get_redis_client(), scheduler_now_script)
+ script = lua_script_storage(get_valkey_client(), scheduler_now_script)
call_now, wait_time = script(args=[start_after_from, start_after_to, every_from, every_to])
# does the worker run the checker now?
diff --git a/searx/settings.yml b/searx/settings.yml
index cb7504efe..fb85f0ff4 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -110,9 +110,10 @@ server:
X-Robots-Tag: noindex, nofollow
Referrer-Policy: no-referrer
-redis:
- # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.
- # https://docs.searxng.org/admin/settings/settings_redis.html#settings-redis
+valkey:
+ # URL to connect valkey database. Is overwritten by ${SEARXNG_VALKEY_URL}.
+ # https://docs.searxng.org/admin/settings/settings_valkey.html#settings-valkey
+ # url: valkey://localhost:6379/0
url: false
ui:
@@ -1809,10 +1810,10 @@ engines:
shortcut: rt
disabled: true
- # Required dependency: redis
- # - name: myredis
+ # Required dependency: valkey
+ # - name: myvalkey
# shortcut : rds
- # engine: redis_server
+ # engine: valkey_server
# exact_match_only: false
# host: '127.0.0.1'
# port: 6379
diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py
index 4cee7e345..7e785e4d2 100644
--- a/searx/settings_defaults.py
+++ b/searx/settings_defaults.py
@@ -185,9 +185,13 @@ SCHEMA = {
'method': SettingsValue(('POST', 'GET'), 'POST', 'SEARXNG_METHOD'),
'default_http_headers': SettingsValue(dict, {}),
},
+ # redis is deprecated ..
'redis': {
'url': SettingsValue((None, False, str), False, 'SEARXNG_REDIS_URL'),
},
+ 'valkey': {
+ 'url': SettingsValue((None, False, str), False, 'SEARXNG_VALKEY_URL'),
+ },
'ui': {
'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
'static_use_hash': SettingsValue(bool, False, 'SEARXNG_STATIC_USE_HASH'),
diff --git a/searx/valkeydb.py b/searx/valkeydb.py
new file mode 100644
index 000000000..2817c6d0a
--- /dev/null
+++ b/searx/valkeydb.py
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Implementation of the valkey client (valkey-py_).
+
+.. _valkey-py: https://github.com/valkey-io/valkey-py
+
+This implementation uses the :ref:`settings valkey` setup from ``settings.yml``.
+A valkey DB connect can be tested by::
+
+ >>> from searx import valkeydb
+ >>> valkeydb.initialize()
+ True
+ >>> db = valkeydb.client()
+ >>> db.set("foo", "bar")
+ True
+ >>> db.get("foo")
+ b'bar'
+ >>>
+
+"""
+
+import os
+import pwd
+import logging
+import warnings
+
+import valkey
+from searx import get_setting
+
+
+_CLIENT = None
+logger = logging.getLogger(__name__)
+
+
+def client() -> valkey.Valkey:
+ return _CLIENT
+
+
+def initialize():
+ global _CLIENT # pylint: disable=global-statement
+ if get_setting('redis.url'):
+ warnings.warn("setting redis.url is deprecated, use valkey.url", DeprecationWarning)
+ valkey_url = get_setting('valkey.url') or get_setting('redis.url')
+ if not valkey_url:
+ return False
+ try:
+ # create a client, but no connection is done
+ _CLIENT = valkey.Valkey.from_url(valkey_url)
+
+ # log the parameters as seen by the valkey lib, without the password
+ kwargs = _CLIENT.get_connection_kwargs().copy()
+ kwargs.pop('password', None)
+ kwargs = ' '.join([f'{k}={v!r}' for k, v in kwargs.items()])
+ logger.info("connecting to Valkey %s", kwargs)
+
+ # check the connection
+ _CLIENT.ping()
+
+ # no error: the valkey connection is working
+ logger.info("connected to Valkey")
+ return True
+ except valkey.exceptions.ValkeyError:
+ _CLIENT = None
+ _pw = pwd.getpwuid(os.getuid())
+ logger.exception("[%s (%s)] can't connect valkey DB ...", _pw.pw_name, _pw.pw_uid)
+ return False
diff --git a/searx/redislib.py b/searx/valkeylib.py
index 5fa8f2dae..733988eb1 100644
--- a/searx/redislib.py
+++ b/searx/valkeylib.py
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
-"""A collection of convenient functions and redis/lua scripts.
+"""A collection of convenient functions and valkey/lua scripts.
-This code was partial inspired by the `Bullet-Proofing Lua Scripts in RedisPy`_
+This code was partial inspired by the `Bullet-Proofing Lua Scripts in ValkeyPy`_
article.
-.. _Bullet-Proofing Lua Scripts in RedisPy:
+.. _Bullet-Proofing Lua Scripts in ValkeyPy:
https://redis.com/blog/bullet-proofing-lua-scripts-in-redispy/
"""
@@ -19,8 +19,8 @@ LUA_SCRIPT_STORAGE = {}
def lua_script_storage(client, script):
- """Returns a redis :py:obj:`Script
- <redis.commands.core.CoreCommands.register_script>` instance.
+ """Returns a valkey :py:obj:`Script
+ <valkey.commands.core.CoreCommands.register_script>` instance.
Due to performance reason the ``Script`` object is instantiated only once
for a client (``client.register_script(..)``) and is cached in
@@ -28,7 +28,7 @@ def lua_script_storage(client, script):
"""
- # redis connection can be closed, lets use the id() of the redis connector
+ # valkey connection can be closed, lets use the id() of the valkey connector
# as key in the script-storage:
client_id = id(client)
@@ -64,8 +64,8 @@ def purge_by_prefix(client, prefix: str = "SearXNG_"):
:param prefix: prefix of the key to delete (default: ``SearXNG_``)
:type name: str
- .. _EXPIRE: https://redis.io/commands/expire/
- .. _DEL: https://redis.io/commands/del/
+ .. _EXPIRE: https://valkey.io/commands/expire/
+ .. _DEL: https://valkey.io/commands/del/
"""
script = lua_script_storage(client, PURGE_BY_PREFIX)
@@ -76,7 +76,7 @@ def secret_hash(name: str):
"""Creates a hash of the ``name``.
Combines argument ``name`` with the ``secret_key`` from :ref:`settings
- server`. This function can be used to get a more anonymized name of a Redis
+ server`. This function can be used to get a more anonymized name of a Valkey
KEY.
:param name: the name to create a secret hash for
@@ -112,12 +112,12 @@ return c
def incr_counter(client, name: str, limit: int = 0, expire: int = 0):
"""Increment a counter and return the new value.
- If counter with redis key ``SearXNG_counter_<name>`` does not exists it is
+ If counter with valkey key ``SearXNG_counter_<name>`` does not exists it is
created with initial value 1 returned. The replacement ``<name>`` is a
*secret hash* of the value from argument ``name`` (see
:py:func:`secret_hash`).
- The implementation of the redis counter is the lua script from string
+ The implementation of the valkey counter is the lua script from string
:py:obj:`INCR_COUNTER`.
:param name: name of the counter
@@ -133,8 +133,8 @@ def incr_counter(client, name: str, limit: int = 0, expire: int = 0):
:return: value of the incremented counter
:type return: int
- .. _EXPIRE: https://redis.io/commands/expire/
- .. _INCR: https://redis.io/commands/incr/
+ .. _EXPIRE: https://valkey.io/commands/expire/
+ .. _INCR: https://valkey.io/commands/incr/
A simple demo of a counter with expire time and limit::
@@ -157,7 +157,7 @@ def incr_counter(client, name: str, limit: int = 0, expire: int = 0):
def drop_counter(client, name):
- """Drop counter with redis key ``SearXNG_counter_<name>``
+ """Drop counter with valkey key ``SearXNG_counter_<name>``
The replacement ``<name>`` is a *secret hash* of the value from argument
``name`` (see :py:func:`incr_counter` and :py:func:`incr_sliding_window`).
@@ -182,7 +182,7 @@ return result
def incr_sliding_window(client, name: str, duration: int):
"""Increment a sliding-window counter and return the new value.
- If counter with redis key ``SearXNG_counter_<name>`` does not exists it is
+ If counter with valkey key ``SearXNG_counter_<name>`` does not exists it is
created with initial value 1 returned. The replacement ``<name>`` is a
*secret hash* of the value from argument ``name`` (see
:py:func:`secret_hash`).
@@ -196,27 +196,27 @@ def incr_sliding_window(client, name: str, duration: int):
:return: value of the incremented counter
:type return: int
- The implementation of the redis counter is the lua script from string
- :py:obj:`INCR_SLIDING_WINDOW`. The lua script uses `sorted sets in Redis`_
- to implement a sliding window for the redis key ``SearXNG_counter_<name>``
+ The implementation of the valkey counter is the lua script from string
+ :py:obj:`INCR_SLIDING_WINDOW`. The lua script uses `sorted sets in Valkey`_
+ to implement a sliding window for the valkey key ``SearXNG_counter_<name>``
(ZADD_). The current TIME_ is used to score the items in the sorted set and
the time window is moved by removing items with a score lower current time
minus *duration* time (ZREMRANGEBYSCORE_).
The EXPIRE_ time (the duration of the sliding window) is refreshed on each
call (increment) and if there is no call in this duration, the sorted
- set expires from the redis DB.
+ set expires from the valkey DB.
The return value is the amount of items in the sorted set (ZCOUNT_), what
means the number of calls in the sliding window.
- .. _Sorted sets in Redis:
- https://redis.com/ebook/part-1-getting-started/chapter-1-getting-to-know-redis/1-2-what-redis-data-structures-look-like/1-2-5-sorted-sets-in-redis/
- .. _TIME: https://redis.io/commands/time/
- .. _ZADD: https://redis.io/commands/zadd/
- .. _EXPIRE: https://redis.io/commands/expire/
- .. _ZREMRANGEBYSCORE: https://redis.io/commands/zremrangebyscore/
- .. _ZCOUNT: https://redis.io/commands/zcount/
+ .. _Sorted sets in Valkey:
+ https://valkey.com/ebook/part-1-getting-started/chapter-1-getting-to-know-valkey/1-2-what-valkey-data-structures-look-like/1-2-5-sorted-sets-in-valkey/
+ .. _TIME: https://valkey.io/commands/time/
+ .. _ZADD: https://valkey.io/commands/zadd/
+ .. _EXPIRE: https://valkey.io/commands/expire/
+ .. _ZREMRANGEBYSCORE: https://valkey.io/commands/zremrangebyscore/
+ .. _ZCOUNT: https://valkey.io/commands/zcount/
A simple demo of the sliding window::
diff --git a/searx/webapp.py b/searx/webapp.py
index 868d95e3e..15f79f151 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -118,7 +118,7 @@ from searx.locales import (
from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
from searx import favicons
-from searx.redisdb import initialize as redis_initialize
+from searx.valkeydb import initialize as valkey_initialize
from searx.sxng_locales import sxng_locales
import searx.search
from searx.network import stream as http_stream, set_context_network_name
@@ -1397,7 +1397,7 @@ def init():
return
locales_initialize()
- redis_initialize()
+ valkey_initialize()
searx.plugins.initialize(app)
metrics: bool = get_setting("general.enable_metrics") # type: ignore
diff --git a/utils/lib.sh b/utils/lib.sh
index 1ef273535..7886520c6 100755
--- a/utils/lib.sh
+++ b/utils/lib.sh
@@ -7,6 +7,8 @@
DIST_ID=$(source /etc/os-release; echo "$ID");
# shellcheck disable=SC2034
DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID");
+# shellcheck disable=SC2034
+DIST_VERSION_CODENAME=$(source /etc/os-release; echo "$VERSION_CODENAME");
ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
ADMIN_NAME="${ADMIN_NAME:-$USER}"
diff --git a/utils/lib_redis.sh b/utils/lib_redis.sh
index ba1435a86..3b481dc75 100755
--- a/utils/lib_redis.sh
+++ b/utils/lib_redis.sh
@@ -1,190 +1,26 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
-#
-# Tools to build and install redis [1] binaries & packages.
-#
-# [1] https://redis.io/download#installation
-#
-# 1. redis.devpkg (sudo)
-# 2. redis.build
-# 3. redis.install (sudo)
-#
-# systemd commands::
-#
-# sudo -H systemctl status searxng-redis
-# sudo -H journalctl -u searxng-redis
-# sudo -H journalctl --vacuum-size=1M
-#
-# Test socket connection from client (local user)::
-#
-# $ sudo -H ./manage redis.addgrp "${USER}"
-# # logout & login to get member of group
-# $ groups
-# ... searxng-redis ...
-# $ source /usr/local/searxng-redis/.redis_env
-# $ which redis-cli
-# /usr/local/searxng-redis/.local/bin/redis-cli
-#
-# $ redis-cli -s /usr/local/searxng-redis/redis.sock
-# redis /usr/local/searxng-redis/redis.sock> set foo bar
-# OK
-# redis /usr/local/searxng-redis/redis.sock> get foo
-# "bar"
-# [CTRL-D]
-
# shellcheck disable=SC2091
# shellcheck source=utils/lib.sh
. /dev/null
-REDIS_GIT_URL="https://github.com/redis/redis.git"
-REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}"
-
REDIS_USER="searxng-redis"
REDIS_GROUP="searxng-redis"
-REDIS_HOME="/usr/local/${REDIS_USER}"
-REDIS_HOME_BIN="${REDIS_HOME}/.local/bin"
-REDIS_ENV="${REDIS_HOME}/.redis_env"
-
REDIS_SERVICE_NAME="searxng-redis"
REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service"
-# binaries to compile & install
-REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli)
-# link names of redis-server binary
-REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof)
-
-REDIS_CONF="${REDIS_HOME}/redis.conf"
-REDIS_CONF_TEMPLATE=$(cat <<EOF
-# Note that in order to read the configuration file, Redis must be
-# started with the file path as first argument:
-#
-# ./redis-server /path/to/redis.conf
-
-# bind 127.0.0.1 -::1
-protected-mode yes
-
-# Accept connections on the specified port, default is 6379 (IANA #815344).
-# If port 0 is specified Redis will not listen on a TCP socket.
-port 0
-
-# Specify the path for the Unix socket that will be used to listen for
-# incoming connections.
-
-unixsocket ${REDIS_HOME}/run/redis.sock
-unixsocketperm 770
-
-# The working directory.
-dir ${REDIS_HOME}/run
-
-# If you run Redis from upstart or systemd, Redis can interact with your
-# supervision tree.
-supervised auto
-
-pidfile ${REDIS_HOME}/run/redis.pid
-
-# log to the system logger
-syslog-enabled yes
-EOF
-)
-
redis.help(){
cat <<EOF
redis.:
- devpkg : install essential packages to compile redis
- build : build redis binaries at $(redis._get_dist)
- install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
remove : delete user (${REDIS_USER}) and remove service (${REDIS_SERVICE_NAME})
- shell : start bash interpreter from user ${REDIS_USER}
- src : clone redis source code to <path> and checkput ${REDIS_GIT_TAG}
- useradd : create user (${REDIS_USER}) at ${REDIS_HOME}
userdel : delete user (${REDIS_USER})
- addgrp : add <user> to group (${REDIS_USER})
rmgrp : remove <user> from group (${REDIS_USER})
EOF
}
-redis.devpkg() {
-
- # Uses OS package manager to install the essential packages to build and
- # compile sources
-
- sudo_or_exit
-
- case ${DIST_ID} in
- ubuntu|debian)
- pkg_install git build-essential gawk
- ;;
- arch)
- pkg_install git base-devel
- ;;
- fedora)
- pkg_install git @development-tools
- ;;
- centos)
- pkg_install git
- yum groupinstall "Development Tools" -y
- ;;
- *)
- err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS."
- return 42
- ;;
- esac
-}
-
-redis.build() {
-
- # usage: redis.build
-
- rst_title "get redis sources" section
- redis.src "${CACHE}/redis"
-
- if ! required_commands gcc nm make gawk ; then
- info_msg "install development tools to get missing command(s) .."
- if [[ -n ${SUDO_USER} ]]; then
- sudo -H "$0" redis.devpkg
- else
- redis.devpkg
- fi
- fi
-
- rst_title "compile redis sources" section
-
- pushd "${CACHE}/redis" &>/dev/null
-
- if ask_yn "Do you run 'make distclean' first'?" Yn; then
- $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout
- fi
-
- $(bash.cmd) -c "make" 2>&1 | prefix_stdout
- if ask_yn "Do you run 'make test'?" Ny; then
- $(bash.cmd) -c "make test" | prefix_stdout
- fi
-
- popd &>/dev/null
-
- tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
-mkdir -p "$(redis._get_dist)"
-cd "${CACHE}/redis/src"
-cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)"
-EOF
- info_msg "redis binaries available at $(redis._get_dist)"
-}
-
-
-redis.install() {
- sudo_or_exit
- (
- set -e
- redis.useradd
- redis._install_bin
- redis._install_conf
- redis._install_service
- )
- dump_return $?
-}
redis.remove() {
sudo_or_exit
@@ -200,57 +36,6 @@ redis.shell() {
interactive_shell "${REDIS_USER}"
}
-redis.src() {
-
- # usage: redis.src "${CACHE}/redis"
-
- local dest="${1:-${CACHE}/redis}"
-
- if [ -d "${dest}" ] ; then
- info_msg "already cloned: $dest"
- tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
-cd "${dest}"
-git fetch --all
-git reset --hard tags/${REDIS_GIT_TAG}
-EOF
- else
- tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
-mkdir -p "$(dirname "$dest")"
-cd "$(dirname "$dest")"
-git clone "${REDIS_GIT_URL}" "${dest}"
-EOF
- tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
-cd "${dest}"
-git checkout tags/${REDIS_GIT_TAG} -b "build-branch"
-EOF
- fi
-}
-
-redis.useradd(){
-
- # usage: redis.useradd
-
- rst_title "add user ${REDIS_USER}" section
- echo
- sudo_or_exit
-
- # create user account
- tee_stderr 0.5 <<EOF | sudo -H bash | prefix_stdout
-useradd --shell /bin/bash --system \
- --home-dir "${REDIS_HOME}" \
- --comment 'user that runs a redis instance' "${REDIS_USER}"
-mkdir -p "${REDIS_HOME}"
-chown -R "${REDIS_USER}:${REDIS_GROUP}" "${REDIS_HOME}"
-groups "${REDIS_USER}"
-EOF
-
- # create App-ENV and add source it in the .profile
- tee_stderr 0.5 <<EOF | sudo -H -u "${REDIS_USER}" bash | prefix_stdout
-mkdir -p "${REDIS_HOME_BIN}"
-echo "export PATH=${REDIS_HOME_BIN}:\\\$PATH" > "${REDIS_ENV}"
-grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile
-EOF
-}
redis.userdel() {
sudo_or_exit
@@ -275,81 +60,6 @@ redis.rmgrp() {
}
-
-# private redis. functions
-# ------------------------
-
-redis._install_bin() {
- local src
- src="$(redis._get_dist)"
- (
- set -e
- for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do
- install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \
- "${src}/${redis_exe}" "${REDIS_HOME_BIN}"
- done
-
- pushd "${REDIS_HOME_BIN}" &> /dev/null
- for redis_exe in "${REDIS_LINK_EXE[@]}"; do
- info_msg "link redis-server --> ${redis_exe}"
- sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}"
- done
- popd &> /dev/null
-
- )
-}
-
-redis._install_conf() {
- sudo -H -u "${REDIS_USER}" bash <<EOF
-mkdir -p "${REDIS_HOME}/run"
-echo '${REDIS_CONF_TEMPLATE}' > "${REDIS_CONF}"
-EOF
-}
-
-redis._install_service() {
- systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
-}
-
redis._remove_service() {
systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
}
-
-redis._get_dist() {
- if [ -z "${REDIS_DIST}" ]; then
- echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)"
- else
- echo "${REDIS_DIST}"
- fi
-}
-
-redis._arch() {
- local ARCH
- case "$(command uname -m)" in
- "x86_64") ARCH=amd64 ;;
- "aarch64") ARCH=arm64 ;;
- "armv6" | "armv7l") ARCH=armv6l ;;
- "armv8") ARCH=arm64 ;;
- .*386.*) ARCH=386 ;;
- ppc64*) ARCH=ppc64le ;;
- *) die 42 "ARCH is unknown: $(command uname -m)" ;;
- esac
- echo "${ARCH}"
-}
-
-# TODO: move this to the right place ..
-
-bash.cmd(){
-
- # print cmd to get a bash in a non-root mode, even if we are in a sudo
- # context.
-
- local user="${USER}"
- local bash_cmd="bash"
-
- if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then
- user="${SUDO_USER}"
- bash_cmd="sudo -H -u ${SUDO_USER} bash"
- fi
-
- printf "%s" "${bash_cmd}"
-}
diff --git a/utils/lib_valkey.sh b/utils/lib_valkey.sh
new file mode 100755
index 000000000..7e7beb948
--- /dev/null
+++ b/utils/lib_valkey.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+valkey.distro.setup() {
+ # shellcheck disable=SC2034
+
+ case $DIST_ID in
+ ubuntu|debian)
+ VALKEY_PACKAGES="valkey-server"
+ ;;
+ arch|fedora|centos)
+ VALKEY_PACKAGES="valkey"
+ ;;
+ *)
+ err_msg "$DIST_ID: valkey not yet implemented"
+ ;;
+ esac
+}
+
+valkey.backports() {
+
+ case $DIST_ID in
+ debian)
+ info_msg "APT:: install debian-stable-backports.source / ${DIST_ID}-${DIST_VERS} (${DIST_VERSION_CODENAME})"
+ install_template /etc/apt/sources.list.d/debian-stable-backports.sources
+ apt update
+ ;;
+ ubuntu)
+ info_msg "APT:: install ubuntu-stable-backports.source / ${DIST_ID}-${DIST_VERS} (${DIST_VERSION_CODENAME})"
+ install_template /etc/apt/sources.list.d/ubuntu-stable-backports.sources
+ apt update
+ ;;
+ *)
+ info_msg "APT:: valkey.backports no implementation / ${DIST_ID}-${DIST_VERS} (${DIST_VERSION_CODENAME})"
+ ;;
+ esac
+}
+
+valkey.install(){
+ info_msg "installing valkey ..."
+ valkey.distro.setup
+
+ case $DIST_ID in
+ debian|ubuntu)
+ apt-cache show "${VALKEY_PACKAGES}" &> /dev/null || valkey.backports
+ pkg_install "${VALKEY_PACKAGES}"
+
+ # do some fix ...
+ # chown -R valkey:valkey /var/log/valkey/ /var/lib/valkey/ /etc/valkey/
+
+ # https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#PrivateUsers=
+ sed -i 's/PrivateUsers=true/# PrivateUsers=true/' /lib/systemd/system/valkey-server.service
+ sed -i 's/PrivateUsers=true/# PrivateUsers=true/' /lib/systemd/system/valkey-server@.service
+
+ systemd_activate_service valkey-server
+ ;;
+ arch|fedora|centos)
+ pkg_install "${VALKEY_PACKAGES}"
+ systemd_activate_service valkey
+ ;;
+ *)
+ # install backports if package is not in the current APT repos
+ pkg_install "${VALKEY_PACKAGES}"
+ ;;
+ esac
+
+ # case $DIST_ID-$DIST_VERS in
+ # arch-*|fedora-*|centos-7)
+ # systemctl enable nginx
+ # systemctl start nginx
+ # ;;
+ # esac
+}
diff --git a/utils/searxng.sh b/utils/searxng.sh
index 4ad2ef93f..663ca6d3d 100755
--- a/utils/searxng.sh
+++ b/utils/searxng.sh
@@ -9,6 +9,8 @@ SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}"
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/lib_redis.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh"
+# shellcheck source=utils/lib_valkey.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib_valkey.sh"
# shellcheck source=utils/brand.sh
source "${REPO_ROOT}/utils/brand.sh"
@@ -119,8 +121,8 @@ usage() {
# shellcheck disable=SC1117
cat <<EOF
usage:
- $(basename "$0") install [all|user|pyenv|settings|uwsgi|redis|nginx|apache|searxng-src|packages|buildhost]
- $(basename "$0") remove [all|user|pyenv|settings|uwsgi|redis|nginx|apache]
+ $(basename "$0") install [all|user|pyenv|settings|uwsgi|valkey|nginx|apache|searxng-src|packages|buildhost]
+ $(basename "$0") remove [all|user|pyenv|settings|uwsgi|valkey|nginx|apache]
$(basename "$0") instance [cmd|update|check|localtest|inspect]
install|remove:
all : complete (de-) installation of the SearXNG service
@@ -128,10 +130,13 @@ install|remove:
pyenv : virtualenv (python) in ${SEARXNG_PYENV}
settings : settings from ${SEARXNG_SETTINGS_PATH}
uwsgi : SearXNG's uWSGI app ${SEARXNG_UWSGI_APP}
- redis : build & install or remove a local redis server ${REDIS_HOME}/run/redis.sock
nginx : HTTP site ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}
apache : HTTP site ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}
install:
+ valkey : install a local valkey server
+remove:
+ redis : remove a local redis server ${REDIS_HOME}/run/redis.sock
+install:
searxng-src : clone ${GIT_URL} into ${SEARXNG_SRC}
packages : installs packages from OS package manager required by SearXNG
buildhost : installs packages from OS package manager required by a SearXNG buildhost
@@ -194,7 +199,7 @@ main() {
buildhost) searxng.install.buildhost;;
nginx) searxng.nginx.install;;
apache) searxng.apache.install;;
- redis) searxng.install.redis;;
+ valkey) searxng.install.valkey;;
*) usage "$_usage"; exit 42;;
esac
;;
@@ -208,6 +213,7 @@ main() {
uwsgi) searxng.remove.uwsgi;;
apache) searxng.apache.remove;;
remove) searxng.nginx.remove;;
+ valkey) searxng.remove.valkey;;
redis) searxng.remove.redis;;
*) usage "$_usage"; exit 42;;
esac
@@ -259,7 +265,7 @@ main() {
searxng.install.all() {
rst_title "SearXNG installation" part
- local redis_url
+ local valkey_url
rst_title "SearXNG"
searxng.install.packages
@@ -277,8 +283,8 @@ searxng.install.all() {
searxng.install.uwsgi
wait_key
- rst_title "Redis DB"
- searxng.install.redis.db
+ rst_title "Valkey DB"
+ searxng.install.valkey.db
rst_title "HTTP Server"
searxng.install.http.site
@@ -289,77 +295,35 @@ searxng.install.all() {
fi
}
-searxng.install.redis.db() {
- local redis_url
+searxng.install.valkey.db() {
+ local valkey_url
- redis_url=$(searxng.instance.get_setting redis.url)
- rst_para "\
-In your instance, redis DB connector is configured at:
+ valkey_url=$(searxng.instance.get_setting valkey.url)
- ${redis_url}
+ if [ "${valkey_url}" = "False" ]; then
+ rst_para "valkey DB connector is not configured in your instance"
+ else
+ rst_para "\
+In your instance, valkey DB connector is configured at:
+
+ ${valkey_url}
"
- if searxng.instance.exec python -c "from searx import redisdb; redisdb.initialize() or exit(42)"; then
- info_msg "SearXNG instance is able to connect redis DB."
- return
+ if searxng.instance.exec python -c "from searx import valkeydb; valkeydb.initialize() or exit(42)"; then
+ info_msg "SearXNG instance is able to connect valkey DB."
+ return
+ fi
fi
- if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then
- err_msg "SearXNG instance can't connect redis DB / check redis & your settings"
+
+ if ! [[ ${valkey_url} = valkey://localhost:6379/* ]]; then
+ err_msg "SearXNG instance can't connect valkey DB / check valkey & your settings"
return
fi
- rst_para ".. but this redis DB is not installed yet."
+ rst_para ".. but this valkey DB is not installed yet."
- case $DIST_ID-$DIST_VERS in
- fedora-*)
- # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the
- # Emperor will run the vassal using the UID/GID of the vassal
- # configuration file [1] (user and group of the app .ini file).
- #
- # HINT: without option ``emperor-tyrant-initgroups=true`` in
- # ``/etc/uwsgi.ini`` the process won't get the additional groups,
- # but this option is not available in 2.0.x branch [2][3] / on
- # fedora35 there is v2.0.20 installed --> no way to get additional
- # groups on fedora's tyrant mode.
- #
- # ERROR:searx.redisdb: [searxng (993)] can't connect redis DB ...
- # ERROR:searx.redisdb: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied.
- # ERROR:searx.plugins.limiter: init limiter DB failed!!!
- #
- # $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini'
- # searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini
- # searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini
- #
- # Additional groups:
- #
- # $ groups searxng
- # searxng : searxng searxng-redis
- #
- # Here you can see that the additional "Groups" of PID 186 are unset
- # (missing gid of searxng-redis)
- #
- # $ cat /proc/186/task/186/status
- # ...
- # Uid: 993 993 993 993
- # Gid: 993 993 993 993
- # FDSize: 128
- # Groups:
- # ...
- #
- # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting
- # [2] https://github.com/unbit/uwsgi/issues/2099
- # [3] https://github.com/unbit/uwsgi/pull/752
-
- rst_para "\
-Fedora uses emperor-tyrant mode / in this mode we had a lot of trouble with
-sockets and permissions of the vasals. We recommend to setup a redis DB
-and using redis:// TCP protocol in the settings.yml configuration."
- ;;
- *)
- if ask_yn "Do you want to install the redis DB now?" Yn; then
- searxng.install.redis
- uWSGI_restart "$SEARXNG_UWSGI_APP"
- fi
- ;;
- esac
+ if ask_yn "Do you want to install the valkey DB now?" Yn; then
+ searxng.install.valkey
+ uWSGI_restart "$SEARXNG_UWSGI_APP"
+ fi
}
searxng.install.http.site() {
@@ -380,16 +344,16 @@ searxng.install.http.site() {
}
searxng.remove.all() {
- local redis_url
+ local valkey_url
rst_title "De-Install SearXNG (service)"
if ! ask_yn "Do you really want to deinstall SearXNG?"; then
return
fi
- redis_url=$(searxng.instance.get_setting redis.url)
- if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then
- searxng.remove.redis
+ valkey_url=$(searxng.instance.get_setting valkey.url)
+ if ! [[ ${valkey_url} = unix://${VALKEY_HOME}/run/valkey.sock* ]]; then
+ searxng.remove.valkey
fi
searxng.remove.uwsgi
@@ -642,19 +606,18 @@ searxng.remove.uwsgi() {
uWSGI_remove_app "${SEARXNG_UWSGI_APP}"
}
-searxng.install.redis() {
- rst_title "SearXNG (install redis)"
- redis.build
- redis.install
- redis.addgrp "${SERVICE_USER}"
-}
-
searxng.remove.redis() {
rst_title "SearXNG (remove redis)"
redis.rmgrp "${SERVICE_USER}"
redis.remove
}
+searxng.install.valkey() {
+ rst_title "SearXNG (install valkey)"
+ valkey.install
+}
+
+
searxng.instance.localtest() {
rst_title "Test SearXNG instance locally" section
rst_para "Activate debug mode, start a minimal SearXNG "\
@@ -690,11 +653,11 @@ To install uWSGI use::
die 42 "SearXNG's uWSGI app not available"
fi
- if ! searxng.instance.exec python -c "from searx import redisdb; redisdb.initialize() or exit(42)"; then
+ if ! searxng.instance.exec python -c "from searx import valkeydb; valkeydb.initialize() or exit(42)"; then
rst_para "\
-The configured redis DB is not available: If your server is public to the
+The configured valkey DB is not available: If your server is public to the
internet, you should setup a bot protection to block excessively bot queries.
-Bot protection requires a redis DB. About bot protection visit the official
+Bot protection requires a valkey DB. About bot protection visit the official
SearXNG documentation and query for the word 'limiter'.
"
fi
diff --git a/utils/searxng_check.py b/utils/searxng_check.py
index 0bff756ad..b597b3e79 100644
--- a/utils/searxng_check.py
+++ b/utils/searxng_check.py
@@ -34,8 +34,11 @@ if os.path.isfile(OLD_BRAND_ENV):
msg = ('%s is no longer needed, remove the file' % (OLD_BRAND_ENV))
warnings.warn(msg, DeprecationWarning)
-from searx import redisdb, get_setting
+from searx import valkeydb, get_setting
-if not redisdb.initialize():
- warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2)
- warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2)
+if get_setting('redis.url'):
+ warnings.warn("setting redis.url is deprecated, use valkey.url", RuntimeWarning, stacklevel=2)
+
+if not valkeydb.initialize():
+ warnings.warn("can't connect to valkey DB at: %s" % get_setting('valkey.url'), RuntimeWarning, stacklevel=2)
+ warnings.warn("--> no bot protection without valkey DB", RuntimeWarning, stacklevel=2)
diff --git a/utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources b/utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources
new file mode 100644
index 000000000..251b2ccb7
--- /dev/null
+++ b/utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources
@@ -0,0 +1,6 @@
+Types: deb deb-src
+URIs: http://deb.debian.org/debian
+Suites: stable-backports
+Components: main contrib non-free non-free-firmware
+Enabled: yes
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
diff --git a/utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources b/utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources
new file mode 100644
index 000000000..a429a1349
--- /dev/null
+++ b/utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources
@@ -0,0 +1,6 @@
+Types: deb deb-src
+URIs: http://us.archive.ubuntu.com/ubuntu/
+Suites: ${DIST_VERSION_CODENAME}-backports
+Components: main multiverse restricted universe
+Enabled: yes
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
diff --git a/utils/templates/etc/searxng/settings.yml b/utils/templates/etc/searxng/settings.yml
index c5ecbd777..b5da38ebd 100644
--- a/utils/templates/etc/searxng/settings.yml
+++ b/utils/templates/etc/searxng/settings.yml
@@ -21,9 +21,9 @@ server:
# by ${SEARXNG_BASE_URL}.
# base_url: http://example.com/location
-redis:
- # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.
- url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+valkey:
+ # URL to connect valkey database. Is overwritten by ${SEARXNG_VALKEY_URL}.
+ url: valkey://localhost:6379/0
ui:
static_use_hash: true
diff --git a/utils/templates/lib/systemd/system/searxng-redis.service b/utils/templates/lib/systemd/system/searxng-redis.service
deleted file mode 100644
index d1d163f04..000000000
--- a/utils/templates/lib/systemd/system/searxng-redis.service
+++ /dev/null
@@ -1,42 +0,0 @@
-[Unit]
-
-Description=SearXNG redis service
-After=syslog.target
-After=network.target
-Documentation=https://redis.io/documentation
-
-[Service]
-
-Type=simple
-User=${REDIS_USER}
-Group=${REDIS_USER}
-WorkingDirectory=${REDIS_HOME}
-Restart=always
-TimeoutStopSec=0
-
-Environment=USER=${REDIS_USER} HOME=${REDIS_HOME}
-ExecStart=${REDIS_HOME_BIN}/redis-server ${REDIS_CONF}
-ExecPaths=${REDIS_HOME_BIN}
-
-LimitNOFILE=65535
-NoNewPrivileges=true
-PrivateDevices=yes
-
-# ProtectSystem=full
-ProtectHome=yes
-ReadOnlyDirectories=/
-ReadWritePaths=-${REDIS_HOME}/run
-
-UMask=007
-PrivateTmp=yes
-
-MemoryDenyWriteExecute=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectControlGroups=true
-RestrictRealtime=true
-RestrictNamespaces=true
-
-[Install]
-
-WantedBy=multi-user.target