From f798ddd4922d793d5e6ccb7c4111810d549ff4f4 Mon Sep 17 00:00:00 2001 From: Gaspard d'Hautefeuille Date: Wed, 9 Jul 2025 07:55:37 +0200 Subject: [mod] migrate from Redis to Valkey (#4795) This patch migrates from `redis==5.2.1` [1] to `valkey==6.1.0` [2]. The migration to valkey is necessary because the company behind Redis has decided to abandon the open source license. After experiencing a drop in user numbers, they now want to run it under a dual license again. But this move demonstrates once again how unreliable the company is and how it treats open source developers. To review first, read the docs:: $ make docs.live Follow the instructions to remove redis: - http://0.0.0.0:8000/admin/settings/settings_redis.html Config and install a local valkey DB: - http://0.0.0.0:8000/admin/settings/settings_valkey.html [1] https://pypi.org/project/redis/ [2] https://pypi.org/project/valkey/ Co-authored-by: HLFH Co-authored-by: Markus Heiser --- utils/lib.sh | 2 + utils/lib_redis.sh | 290 --------------------- utils/lib_valkey.sh | 73 ++++++ utils/searxng.sh | 133 ++++------ utils/searxng_check.py | 11 +- .../sources.list.d/debian-stable-backports.sources | 6 + .../sources.list.d/ubuntu-stable-backports.sources | 6 + utils/templates/etc/searxng/settings.yml | 6 +- .../lib/systemd/system/searxng-redis.service | 42 --- 9 files changed, 145 insertions(+), 424 deletions(-) create mode 100755 utils/lib_valkey.sh create mode 100644 utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources create mode 100644 utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources delete mode 100644 utils/templates/lib/systemd/system/searxng-redis.service (limited to 'utils') diff --git a/utils/lib.sh b/utils/lib.sh index 1ef273535..7886520c6 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -7,6 +7,8 @@ DIST_ID=$(source /etc/os-release; echo "$ID"); # shellcheck disable=SC2034 DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID"); +# shellcheck disable=SC2034 +DIST_VERSION_CODENAME=$(source /etc/os-release; echo "$VERSION_CODENAME"); ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" ADMIN_NAME="${ADMIN_NAME:-$USER}" diff --git a/utils/lib_redis.sh b/utils/lib_redis.sh index ba1435a86..3b481dc75 100755 --- a/utils/lib_redis.sh +++ b/utils/lib_redis.sh @@ -1,190 +1,26 @@ #!/usr/bin/env bash # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -# -# Tools to build and install redis [1] binaries & packages. -# -# [1] https://redis.io/download#installation -# -# 1. redis.devpkg (sudo) -# 2. redis.build -# 3. redis.install (sudo) -# -# systemd commands:: -# -# sudo -H systemctl status searxng-redis -# sudo -H journalctl -u searxng-redis -# sudo -H journalctl --vacuum-size=1M -# -# Test socket connection from client (local user):: -# -# $ sudo -H ./manage redis.addgrp "${USER}" -# # logout & login to get member of group -# $ groups -# ... searxng-redis ... -# $ source /usr/local/searxng-redis/.redis_env -# $ which redis-cli -# /usr/local/searxng-redis/.local/bin/redis-cli -# -# $ redis-cli -s /usr/local/searxng-redis/redis.sock -# redis /usr/local/searxng-redis/redis.sock> set foo bar -# OK -# redis /usr/local/searxng-redis/redis.sock> get foo -# "bar" -# [CTRL-D] - # shellcheck disable=SC2091 # shellcheck source=utils/lib.sh . /dev/null -REDIS_GIT_URL="https://github.com/redis/redis.git" -REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}" - REDIS_USER="searxng-redis" REDIS_GROUP="searxng-redis" -REDIS_HOME="/usr/local/${REDIS_USER}" -REDIS_HOME_BIN="${REDIS_HOME}/.local/bin" -REDIS_ENV="${REDIS_HOME}/.redis_env" - REDIS_SERVICE_NAME="searxng-redis" REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service" -# binaries to compile & install -REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli) -# link names of redis-server binary -REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof) - -REDIS_CONF="${REDIS_HOME}/redis.conf" -REDIS_CONF_TEMPLATE=$(cat < and checkput ${REDIS_GIT_TAG} - useradd : create user (${REDIS_USER}) at ${REDIS_HOME} userdel : delete user (${REDIS_USER}) - addgrp : add to group (${REDIS_USER}) rmgrp : remove from group (${REDIS_USER}) EOF } -redis.devpkg() { - - # Uses OS package manager to install the essential packages to build and - # compile sources - - sudo_or_exit - - case ${DIST_ID} in - ubuntu|debian) - pkg_install git build-essential gawk - ;; - arch) - pkg_install git base-devel - ;; - fedora) - pkg_install git @development-tools - ;; - centos) - pkg_install git - yum groupinstall "Development Tools" -y - ;; - *) - err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS." - return 42 - ;; - esac -} - -redis.build() { - - # usage: redis.build - - rst_title "get redis sources" section - redis.src "${CACHE}/redis" - - if ! required_commands gcc nm make gawk ; then - info_msg "install development tools to get missing command(s) .." - if [[ -n ${SUDO_USER} ]]; then - sudo -H "$0" redis.devpkg - else - redis.devpkg - fi - fi - - rst_title "compile redis sources" section - - pushd "${CACHE}/redis" &>/dev/null - - if ask_yn "Do you run 'make distclean' first'?" Yn; then - $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout - fi - - $(bash.cmd) -c "make" 2>&1 | prefix_stdout - if ask_yn "Do you run 'make test'?" Ny; then - $(bash.cmd) -c "make test" | prefix_stdout - fi - - popd &>/dev/null - - tee_stderr 0.1 <&1 | prefix_stdout -mkdir -p "$(redis._get_dist)" -cd "${CACHE}/redis/src" -cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)" -EOF - info_msg "redis binaries available at $(redis._get_dist)" -} - - -redis.install() { - sudo_or_exit - ( - set -e - redis.useradd - redis._install_bin - redis._install_conf - redis._install_service - ) - dump_return $? -} redis.remove() { sudo_or_exit @@ -200,57 +36,6 @@ redis.shell() { interactive_shell "${REDIS_USER}" } -redis.src() { - - # usage: redis.src "${CACHE}/redis" - - local dest="${1:-${CACHE}/redis}" - - if [ -d "${dest}" ] ; then - info_msg "already cloned: $dest" - tee_stderr 0.1 <&1 | prefix_stdout -cd "${dest}" -git fetch --all -git reset --hard tags/${REDIS_GIT_TAG} -EOF - else - tee_stderr 0.1 <&1 | prefix_stdout -mkdir -p "$(dirname "$dest")" -cd "$(dirname "$dest")" -git clone "${REDIS_GIT_URL}" "${dest}" -EOF - tee_stderr 0.1 <&1 | prefix_stdout -cd "${dest}" -git checkout tags/${REDIS_GIT_TAG} -b "build-branch" -EOF - fi -} - -redis.useradd(){ - - # usage: redis.useradd - - rst_title "add user ${REDIS_USER}" section - echo - sudo_or_exit - - # create user account - tee_stderr 0.5 < "${REDIS_ENV}" -grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile -EOF -} redis.userdel() { sudo_or_exit @@ -275,81 +60,6 @@ redis.rmgrp() { } - -# private redis. functions -# ------------------------ - -redis._install_bin() { - local src - src="$(redis._get_dist)" - ( - set -e - for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do - install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \ - "${src}/${redis_exe}" "${REDIS_HOME_BIN}" - done - - pushd "${REDIS_HOME_BIN}" &> /dev/null - for redis_exe in "${REDIS_LINK_EXE[@]}"; do - info_msg "link redis-server --> ${redis_exe}" - sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}" - done - popd &> /dev/null - - ) -} - -redis._install_conf() { - sudo -H -u "${REDIS_USER}" bash < "${REDIS_CONF}" -EOF -} - -redis._install_service() { - systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}" -} - redis._remove_service() { systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}" } - -redis._get_dist() { - if [ -z "${REDIS_DIST}" ]; then - echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)" - else - echo "${REDIS_DIST}" - fi -} - -redis._arch() { - local ARCH - case "$(command uname -m)" in - "x86_64") ARCH=amd64 ;; - "aarch64") ARCH=arm64 ;; - "armv6" | "armv7l") ARCH=armv6l ;; - "armv8") ARCH=arm64 ;; - .*386.*) ARCH=386 ;; - ppc64*) ARCH=ppc64le ;; - *) die 42 "ARCH is unknown: $(command uname -m)" ;; - esac - echo "${ARCH}" -} - -# TODO: move this to the right place .. - -bash.cmd(){ - - # print cmd to get a bash in a non-root mode, even if we are in a sudo - # context. - - local user="${USER}" - local bash_cmd="bash" - - if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then - user="${SUDO_USER}" - bash_cmd="sudo -H -u ${SUDO_USER} bash" - fi - - printf "%s" "${bash_cmd}" -} diff --git a/utils/lib_valkey.sh b/utils/lib_valkey.sh new file mode 100755 index 000000000..7e7beb948 --- /dev/null +++ b/utils/lib_valkey.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +valkey.distro.setup() { + # shellcheck disable=SC2034 + + case $DIST_ID in + ubuntu|debian) + VALKEY_PACKAGES="valkey-server" + ;; + arch|fedora|centos) + VALKEY_PACKAGES="valkey" + ;; + *) + err_msg "$DIST_ID: valkey not yet implemented" + ;; + esac +} + +valkey.backports() { + + case $DIST_ID in + debian) + info_msg "APT:: install debian-stable-backports.source / ${DIST_ID}-${DIST_VERS} (${DIST_VERSION_CODENAME})" + install_template /etc/apt/sources.list.d/debian-stable-backports.sources + apt update + ;; + ubuntu) + info_msg "APT:: install ubuntu-stable-backports.source / ${DIST_ID}-${DIST_VERS} (${DIST_VERSION_CODENAME})" + install_template /etc/apt/sources.list.d/ubuntu-stable-backports.sources + apt update + ;; + *) + info_msg "APT:: valkey.backports no implementation / ${DIST_ID}-${DIST_VERS} (${DIST_VERSION_CODENAME})" + ;; + esac +} + +valkey.install(){ + info_msg "installing valkey ..." + valkey.distro.setup + + case $DIST_ID in + debian|ubuntu) + apt-cache show "${VALKEY_PACKAGES}" &> /dev/null || valkey.backports + pkg_install "${VALKEY_PACKAGES}" + + # do some fix ... + # chown -R valkey:valkey /var/log/valkey/ /var/lib/valkey/ /etc/valkey/ + + # https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#PrivateUsers= + sed -i 's/PrivateUsers=true/# PrivateUsers=true/' /lib/systemd/system/valkey-server.service + sed -i 's/PrivateUsers=true/# PrivateUsers=true/' /lib/systemd/system/valkey-server@.service + + systemd_activate_service valkey-server + ;; + arch|fedora|centos) + pkg_install "${VALKEY_PACKAGES}" + systemd_activate_service valkey + ;; + *) + # install backports if package is not in the current APT repos + pkg_install "${VALKEY_PACKAGES}" + ;; + esac + + # case $DIST_ID-$DIST_VERS in + # arch-*|fedora-*|centos-7) + # systemctl enable nginx + # systemctl start nginx + # ;; + # esac +} diff --git a/utils/searxng.sh b/utils/searxng.sh index 4ad2ef93f..663ca6d3d 100755 --- a/utils/searxng.sh +++ b/utils/searxng.sh @@ -9,6 +9,8 @@ SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}" source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" # shellcheck source=utils/lib_redis.sh source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh" +# shellcheck source=utils/lib_valkey.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib_valkey.sh" # shellcheck source=utils/brand.sh source "${REPO_ROOT}/utils/brand.sh" @@ -119,8 +121,8 @@ usage() { # shellcheck disable=SC1117 cat < no way to get additional - # groups on fedora's tyrant mode. - # - # ERROR:searx.redisdb: [searxng (993)] can't connect redis DB ... - # ERROR:searx.redisdb: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. - # ERROR:searx.plugins.limiter: init limiter DB failed!!! - # - # $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' - # searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini - # searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini - # - # Additional groups: - # - # $ groups searxng - # searxng : searxng searxng-redis - # - # Here you can see that the additional "Groups" of PID 186 are unset - # (missing gid of searxng-redis) - # - # $ cat /proc/186/task/186/status - # ... - # Uid: 993 993 993 993 - # Gid: 993 993 993 993 - # FDSize: 128 - # Groups: - # ... - # - # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting - # [2] https://github.com/unbit/uwsgi/issues/2099 - # [3] https://github.com/unbit/uwsgi/pull/752 - - rst_para "\ -Fedora uses emperor-tyrant mode / in this mode we had a lot of trouble with -sockets and permissions of the vasals. We recommend to setup a redis DB -and using redis:// TCP protocol in the settings.yml configuration." - ;; - *) - if ask_yn "Do you want to install the redis DB now?" Yn; then - searxng.install.redis - uWSGI_restart "$SEARXNG_UWSGI_APP" - fi - ;; - esac + if ask_yn "Do you want to install the valkey DB now?" Yn; then + searxng.install.valkey + uWSGI_restart "$SEARXNG_UWSGI_APP" + fi } searxng.install.http.site() { @@ -380,16 +344,16 @@ searxng.install.http.site() { } searxng.remove.all() { - local redis_url + local valkey_url rst_title "De-Install SearXNG (service)" if ! ask_yn "Do you really want to deinstall SearXNG?"; then return fi - redis_url=$(searxng.instance.get_setting redis.url) - if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then - searxng.remove.redis + valkey_url=$(searxng.instance.get_setting valkey.url) + if ! [[ ${valkey_url} = unix://${VALKEY_HOME}/run/valkey.sock* ]]; then + searxng.remove.valkey fi searxng.remove.uwsgi @@ -642,19 +606,18 @@ searxng.remove.uwsgi() { uWSGI_remove_app "${SEARXNG_UWSGI_APP}" } -searxng.install.redis() { - rst_title "SearXNG (install redis)" - redis.build - redis.install - redis.addgrp "${SERVICE_USER}" -} - searxng.remove.redis() { rst_title "SearXNG (remove redis)" redis.rmgrp "${SERVICE_USER}" redis.remove } +searxng.install.valkey() { + rst_title "SearXNG (install valkey)" + valkey.install +} + + searxng.instance.localtest() { rst_title "Test SearXNG instance locally" section rst_para "Activate debug mode, start a minimal SearXNG "\ @@ -690,11 +653,11 @@ To install uWSGI use:: die 42 "SearXNG's uWSGI app not available" fi - if ! searxng.instance.exec python -c "from searx import redisdb; redisdb.initialize() or exit(42)"; then + if ! searxng.instance.exec python -c "from searx import valkeydb; valkeydb.initialize() or exit(42)"; then rst_para "\ -The configured redis DB is not available: If your server is public to the +The configured valkey DB is not available: If your server is public to the internet, you should setup a bot protection to block excessively bot queries. -Bot protection requires a redis DB. About bot protection visit the official +Bot protection requires a valkey DB. About bot protection visit the official SearXNG documentation and query for the word 'limiter'. " fi diff --git a/utils/searxng_check.py b/utils/searxng_check.py index 0bff756ad..b597b3e79 100644 --- a/utils/searxng_check.py +++ b/utils/searxng_check.py @@ -34,8 +34,11 @@ if os.path.isfile(OLD_BRAND_ENV): msg = ('%s is no longer needed, remove the file' % (OLD_BRAND_ENV)) warnings.warn(msg, DeprecationWarning) -from searx import redisdb, get_setting +from searx import valkeydb, get_setting -if not redisdb.initialize(): - warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2) - warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2) +if get_setting('redis.url'): + warnings.warn("setting redis.url is deprecated, use valkey.url", RuntimeWarning, stacklevel=2) + +if not valkeydb.initialize(): + warnings.warn("can't connect to valkey DB at: %s" % get_setting('valkey.url'), RuntimeWarning, stacklevel=2) + warnings.warn("--> no bot protection without valkey DB", RuntimeWarning, stacklevel=2) diff --git a/utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources b/utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources new file mode 100644 index 000000000..251b2ccb7 --- /dev/null +++ b/utils/templates/etc/apt/sources.list.d/debian-stable-backports.sources @@ -0,0 +1,6 @@ +Types: deb deb-src +URIs: http://deb.debian.org/debian +Suites: stable-backports +Components: main contrib non-free non-free-firmware +Enabled: yes +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg diff --git a/utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources b/utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources new file mode 100644 index 000000000..a429a1349 --- /dev/null +++ b/utils/templates/etc/apt/sources.list.d/ubuntu-stable-backports.sources @@ -0,0 +1,6 @@ +Types: deb deb-src +URIs: http://us.archive.ubuntu.com/ubuntu/ +Suites: ${DIST_VERSION_CODENAME}-backports +Components: main multiverse restricted universe +Enabled: yes +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg diff --git a/utils/templates/etc/searxng/settings.yml b/utils/templates/etc/searxng/settings.yml index c5ecbd777..b5da38ebd 100644 --- a/utils/templates/etc/searxng/settings.yml +++ b/utils/templates/etc/searxng/settings.yml @@ -21,9 +21,9 @@ server: # by ${SEARXNG_BASE_URL}. # base_url: http://example.com/location -redis: - # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}. - url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 +valkey: + # URL to connect valkey database. Is overwritten by ${SEARXNG_VALKEY_URL}. + url: valkey://localhost:6379/0 ui: static_use_hash: true diff --git a/utils/templates/lib/systemd/system/searxng-redis.service b/utils/templates/lib/systemd/system/searxng-redis.service deleted file mode 100644 index d1d163f04..000000000 --- a/utils/templates/lib/systemd/system/searxng-redis.service +++ /dev/null @@ -1,42 +0,0 @@ -[Unit] - -Description=SearXNG redis service -After=syslog.target -After=network.target -Documentation=https://redis.io/documentation - -[Service] - -Type=simple -User=${REDIS_USER} -Group=${REDIS_USER} -WorkingDirectory=${REDIS_HOME} -Restart=always -TimeoutStopSec=0 - -Environment=USER=${REDIS_USER} HOME=${REDIS_HOME} -ExecStart=${REDIS_HOME_BIN}/redis-server ${REDIS_CONF} -ExecPaths=${REDIS_HOME_BIN} - -LimitNOFILE=65535 -NoNewPrivileges=true -PrivateDevices=yes - -# ProtectSystem=full -ProtectHome=yes -ReadOnlyDirectories=/ -ReadWritePaths=-${REDIS_HOME}/run - -UMask=007 -PrivateTmp=yes - -MemoryDenyWriteExecute=true -ProtectKernelModules=true -ProtectKernelTunables=true -ProtectControlGroups=true -RestrictRealtime=true -RestrictNamespaces=true - -[Install] - -WantedBy=multi-user.target -- cgit v1.2.3