From ae0fcf3a4208401f5dc0296c29227a55f63163da Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 16 Jul 2025 15:47:36 +0200 Subject: [chore] overdue maintenance of shell scripts Removes obsolete scripts and fix various leftovers. Signed-off-by: Markus Heiser --- utils/lib.sh | 140 ------------ utils/lib_sxng_test.sh | 4 +- utils/lxc-searxng.env | 69 ------ utils/lxc.sh | 573 ------------------------------------------------- utils/makefile.include | 14 -- utils/makefile.lxc | 32 --- utils/searxng.sh | 12 -- 7 files changed, 2 insertions(+), 842 deletions(-) delete mode 100644 utils/lxc-searxng.env delete mode 100755 utils/lxc.sh delete mode 100644 utils/makefile.lxc (limited to 'utils') diff --git a/utils/lib.sh b/utils/lib.sh index 7886520c6..ff67c6f67 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -1626,146 +1626,6 @@ EOF fi } -# containers -# ---------- - -in_container() { - # Test if shell runs in a container. - # - # usage: in_container && echo "process running inside a LXC container" - # in_container || echo "process is not running inside a LXC container" - # - # sudo_or_exit - # hint: Reads init process environment, therefore root access is required! - # to be safe, take a look at the environment of process 1 (/sbin/init) - # grep -qa 'container=lxc' /proc/1/environ - - # see lxc_init_container_env - [[ -f /.lxcenv ]] -} - -LXC_ENV_FOLDER= -if in_container; then - # shellcheck disable=SC2034 - LXC_ENV_FOLDER="lxc-env/$(hostname)/" - PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}" - PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}" - PYDIST="${LXC_ENV_FOLDER}${PYDIST}" - PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}" - DOCS_DIST="${LXC_ENV_FOLDER}${DOCS_DIST}" - DOCS_BUILD="${LXC_ENV_FOLDER}${DOCS_BUILD}" -fi - -lxc_init_container_env() { - - # usage: lxc_init_container_env - - # Create a /.lxcenv file in the root folder. Call this once after the - # container is initial started and before installing any boilerplate stuff. - - info_msg "create /.lxcenv in container $1" - cat < - # - # lxc_image_copy "images:ubuntu/20.04" "ubu2004" - - if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then - info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}" - else - info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}" - lxc image copy "${LXC_SUITE[i]}" local: \ - --alias "${LXC_SUITE[i+1]}" | prefix_stdout - fi -} - -lxc_init_container() { - - # usage: lxc_init_container - - local image_name="$1" - local container_name="$2" - - if lxc info "${container_name}" &>/dev/null; then - info_msg "container '${container_name}' already exists" - else - info_msg "create container instance: ${container_name}" - lxc init "local:${image_name}" "${container_name}" - fi -} - -lxc_exists(){ - - # usage: lxc_exists || echo "container does not exists" - - lxc info "$1" &>/dev/null -} - -lxc_image_exists(){ - # usage: lxc_image_exists || echo "image does locally not exists" - - lxc image info "local:$1" &>/dev/null - -} - -lxc_delete_container() { - - # usage: lxc_delete_container - - if lxc info "$1" &>/dev/null; then - info_msg "stop & delete instance ${_BBlue}${1}${_creset}" - lxc stop "$1" &>/dev/null - lxc delete "$1" | prefix_stdout - else - warn_msg "instance '$1' does not exist / can't delete :o" - fi -} - -lxc_delete_local_image() { - - # usage: lxc_delete_local_image - - info_msg "delete image 'local:$i'" - lxc image delete "local:$i" -} - # IP # -- diff --git a/utils/lib_sxng_test.sh b/utils/lib_sxng_test.sh index b84852c1f..321584bf2 100755 --- a/utils/lib_sxng_test.sh +++ b/utils/lib_sxng_test.sh @@ -21,7 +21,7 @@ if [ "$VERBOSE" = "1" ]; then fi test.yamllint() { - build_msg TEST "[yamllint] \$YAMLLINT_FILES" + build_msg TEST "[yamllint] $YAMLLINT_FILES" pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}" dump_return $? } @@ -100,7 +100,7 @@ test.types.ci() { } test.black() { - build_msg TEST "[black] \$BLACK_TARGETS" + build_msg TEST "[black] $BLACK_TARGETS" pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}" dump_return $? } diff --git a/utils/lxc-searxng.env b/utils/lxc-searxng.env deleted file mode 100644 index 8cc0ce886..000000000 --- a/utils/lxc-searxng.env +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- -# SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck shell=bash - -# This file is a setup of a LXC suite. It is sourced from different context, do -# not manipulate the environment directly, implement functions and manipulate -# environment only in subshells. - -lxc_set_suite_env() { - - export LXC_SUITE_NAME="searxng" - - # name of https://images.linuxcontainers.org - export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" - export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}" - export LXC_SUITE=( - - # end of standard support see https://wiki.ubuntu.com/Releases - "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # LTS EOSS April 2025 - "$LINUXCONTAINERS_ORG_NAME:ubuntu/22.04" "ubu2204" # LTS EOSS April 2027 - - # EOL see https://fedoraproject.org/wiki/Releases - "$LINUXCONTAINERS_ORG_NAME:fedora/35" "fedora35" - - # rolling releases see https://www.archlinux.org/releng/releases/ - "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" - ) -} - -lxc_suite_install_info() { - ( - lxc_set_suite_env - cat <] - $_cmd copy [images] - $_cmd remove [containers||images] - $_cmd [start|stop] [containers|] - $_cmd show [images|suite|info|config []] - $_cmd cmd [--|] '...' - $_cmd install [suite|base []] - -build - :containers: build, launch all containers and 'install base' packages - :: build, launch container and 'install base' packages -copy: - :images: copy remote images of the suite into local storage -remove - :containers: delete all 'containers' or only - :images: delete local images of the suite -start/stop - :containers: start/stop all 'containers' from the suite - :: start/stop container from suite -show - :info: show info of all (or ) containers from LXC suite - :config: show config of all (or ) containers from the LXC suite - :suite: show services of all (or ) containers from the LXC suite - :images: show information of local images -cmd - use single quotes to evaluate in container's bash, e.g.: 'echo \$(hostname)' - -- run command '...' in all containers of the LXC suite - :: run command '...' in container -install - :base: prepare LXC; install basic packages - :suite: install LXC ${LXC_SUITE_NAME} suite into all (or ) containers - -EOF - usage_containers - [ -n "${1+x}" ] && err_msg "$1" -} - -usage_containers() { - lxc_suite_install_info - [ -n "${1+x}" ] && err_msg "$1" -} - -lxd_info() { - - cat < $2"; exit 42;; - esac - ;; - start|stop) - sudo_or_exit - case $2 in - ''|--|containers) lxc_cmd "$1" ;; - ${LXC_HOST_PREFIX}-*) - ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 - info_msg "lxc $1 $2" - lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " - ;; - *) usage "unknown or missing container $2"; exit 42;; - esac - ;; - show) - sudo_or_exit - case $2 in - suite) - case $3 in - ${LXC_HOST_PREFIX}-*) - lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ - | prefix_stdout "[${_BBlue}$3${_creset}] " - ;; - *) show_suite;; - esac - ;; - images) show_images ;; - config) - case $3 in - ${LXC_HOST_PREFIX}-*) - ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 - lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " - ;; - *) - rst_title "container configurations" - echo - lxc list "$LXC_HOST_PREFIX-" - echo - lxc_cmd config show - ;; - esac - ;; - info) - case $3 in - ${LXC_HOST_PREFIX}-*) - ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 - lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " - ;; - *) - rst_title "container info" - echo - lxc_cmd info - ;; - esac - ;; - *) usage "$_usage"; exit 42;; - esac - ;; - __show) - # wrapped show commands, called once in each container - case $2 in - suite) lxc_suite_info ;; - esac - ;; - cmd) - sudo_or_exit - shift - case $1 in - --) shift; lxc_exec "$@" ;; - ${LXC_HOST_PREFIX}-*) - ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42 - local name=$1 - shift - lxc_exec_cmd "${name}" "$@" - ;; - *) usage_containers "unknown container: $1" && exit 42 - esac - ;; - install) - sudo_or_exit - case $2 in - suite|base) - case $3 in - ${LXC_HOST_PREFIX}-*) - ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 - lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" - ;; - ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; - *) usage_containers "unknown container: $3" && exit 42 - esac - ;; - *) usage "$_usage"; exit 42 ;; - esac - ;; - __install) - # wrapped install commands, called once in each container - # shellcheck disable=SC2119 - case $2 in - suite) lxc_suite_install ;; - base) FORCE_TIMEOUT=0 lxc_install_base_packages ;; - esac - ;; - doc) - echo - echo ".. generic utils/lxc.sh documentation" - ;; - -*) usage "unknown option $1"; exit 42;; - *) usage "unknown or missing command $1"; exit 42;; - esac -} - - -build_all_containers() { - rst_title "Build all LXC containers of suite" - echo - usage_containers - lxc_copy_images_locally - lxc_init_all_containers - lxc_config_all_containers - lxc_boilerplate_all_containers - rst_title "install LXC base packages" section - echo - lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base - echo - lxc list "$LXC_HOST_PREFIX" -} - -build_container() { - rst_title "Build container $1" - - local remote_image - local container - local image - local boilerplate_script - - for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then - remote_image="${LXC_SUITE[i]}" - container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" - image="${LXC_SUITE[i+1]}" - boilerplate_script="${image}_boilerplate" - boilerplate_script="${!boilerplate_script}" - break - fi - done - echo - if [ -z "$container" ]; then - err_msg "container $1 unknown" - usage_containers - return 42 - fi - lxc_image_copy "${remote_image}" "${image}" - rst_title "init container" section - lxc_init_container "${image}" "${container}" - rst_title "configure container" section - lxc_config_container "${container}" - rst_title "run LXC boilerplate scripts" section - lxc_install_boilerplate "${container}" "$boilerplate_script" - echo - rst_title "install LXC base packages" section - lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \ - | prefix_stdout "[${_BBlue}${container}${_creset}] " - echo - lxc list "$container" -} - -remove_containers() { - rst_title "Remove all LXC containers of suite" - rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}" - echo - lxc list "$LXC_HOST_PREFIX-" - echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT - local default=Ny - [[ $FORCE_TIMEOUT = 0 ]] && default=Yn - if ask_yn "Do you really want to delete these containers" $default; then - for i in "${CONTAINERS[@]}"; do - lxc_delete_container "$i" - done - fi - echo - lxc list "$LXC_HOST_PREFIX-" -} - -# images -# ------ - -lxc_copy_images_locally() { - rst_title "copy images" section - for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}" - done - # lxc image list local: && wait_key -} - -lxc_delete_images_locally() { - rst_title "Delete LXC images" - rst_para "local existing images" - echo - lxc image list local: - echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" - if ask_yn "Do you really want to delete these images"; then - for i in "${LOCAL_IMAGES[@]}"; do - lxc_delete_local_image "$i" - done - fi - - for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do - if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then - lxc_delete_local_image "$i" - fi - done - - echo - lxc image list local: -} - -show_images(){ - rst_title "local images" - echo - lxc image list local: - echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" - wait_key - for i in "${LOCAL_IMAGES[@]}"; do - if lxc_image_exists "$i"; then - info_msg "lxc image info ${_BBlue}${i}${_creset}" - lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " - else - warn_msg "image ${_BBlue}$i${_creset} does not yet exists" - fi - done - -} - - -# container -# --------- - -show_suite(){ - rst_title "LXC suite ($LXC_HOST_PREFIX-*)" - echo - lxc list "$LXC_HOST_PREFIX-" - echo - for i in "${CONTAINERS[@]}"; do - if ! lxc_exists "$i"; then - warn_msg "container ${_BBlue}$i${_creset} does not yet exists" - else - lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ - | prefix_stdout "[${_BBlue}${i}${_creset}] " - echo - fi - done -} - -lxc_cmd() { - for i in "${CONTAINERS[@]}"; do - if ! lxc_exists "$i"; then - warn_msg "container ${_BBlue}$i${_creset} does not yet exists" - else - info_msg "lxc $* $i" - lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " - fi - done -} - -lxc_exec_cmd() { - local name="$1" - shift - exit_val= - info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" - lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" - exit_val=$? - if [[ $exit_val -ne 0 ]]; then - warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" - else - info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" - fi -} - -lxc_exec() { - for i in "${CONTAINERS[@]}"; do - if ! lxc_exists "$i"; then - warn_msg "container ${_BBlue}$i${_creset} does not yet exists" - else - lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] " - fi - done -} - -lxc_init_all_containers() { - rst_title "init all containers" section - - local image_name - local container_name - - for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" - done -} - -lxc_config_all_containers() { - rst_title "configure all containers" section - - for i in "${CONTAINERS[@]}"; do - lxc_config_container "${i}" - done -} - -lxc_config_container() { - info_msg "[${_BBlue}$1${_creset}] configure container ..." - - info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container" - # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps - echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\ - | lxc config set "$1" raw.idmap - - - info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" - # https://lxd.readthedocs.io/en/latest/instances/#type-disk - lxc config device add "$1" repo_share disk \ - source="${REPO_ROOT}" \ - path="${LXC_REPO_ROOT}" &>/dev/null - # lxc config show "$1" && wait_key -} - -lxc_boilerplate_all_containers() { - rst_title "run LXC boilerplate scripts" section - - local boilerplate_script - local image_name - - for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - - image_name="${LXC_SUITE[i+1]}" - boilerplate_script="${image_name}_boilerplate" - boilerplate_script="${!boilerplate_script}" - - lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script" - - if [[ -z "${boilerplate_script}" ]]; then - err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'" - fi - done -} - -lxc_install_boilerplate() { - - # usage: lxc_install_boilerplate - # - # usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}" - - local container_name="$1" - local boilerplate_script="$2" - - info_msg "[${_BBlue}${container_name}${_creset}] init .." - if lxc start -q "${container_name}" &>/dev/null; then - sleep 5 # guest needs some time to come up and get an IP - fi - if ! check_connectivity "${container_name}"; then - die 42 "Container ${container_name} has no internet connectivity!" - fi - lxc_init_container_env "${container_name}" - info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." - cat </dev/null; then - sleep 5 # guest needs some time to come up and get an IP - fi - if [[ -n "${boilerplate_script}" ]]; then - echo "${boilerplate_script}" \ - | lxc exec "${container_name}" -- bash \ - | prefix_stdout "[${_BBlue}${container_name}${_creset}] " - fi -} - -check_connectivity() { - local ret_val=0 - info_msg "check internet connectivity ..." - if ! lxc exec "${1}" -- ping -c 1 9.9.9.9 &>/dev/null; then - ret_val=1 - err_msg "no internet connectivity!" - info_msg "Most often the connectivity is blocked by a docker installation:" - info_msg "Whenever docker is started (reboot) it sets the iptables policy " - info_msg "for the FORWARD chain to DROP, see:" - info_msg " https://docs.searxng.org/utils/lxc.sh.html#internet-connectivity-docker" - iptables-save | grep ":FORWARD" - fi - return $ret_val -} - -# ---------------------------------------------------------------------------- -main "$@" -# ---------------------------------------------------------------------------- diff --git a/utils/makefile.include b/utils/makefile.include index 40f9d3302..c30b4e88f 100644 --- a/utils/makefile.include +++ b/utils/makefile.include @@ -1,22 +1,8 @@ # -*- coding: utf-8; mode: makefile-gmake -*- # SPDX-License-Identifier: AGPL-3.0-or-later -ifeq (,$(wildcard /.lxcenv.mk)) -PHONY += lxc-activate lxc-purge -lxc-activate: - @$(MAKE) -s -f "$$(dirname $(abspath $(lastword $(MAKEFILE_LIST))))/makefile.lxc" lxc-activate -lxc-purge: - $(Q)rm -rf ./lxc-env -else - include /.lxcenv.mk -endif - PHONY += make-help -ifeq (,$(wildcard /.lxcenv.mk)) make-help: -else -make-help: lxc-help -endif @echo 'options:' @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' @echo ' make V=2 [targets] 2 => give reason for rebuild of target' diff --git a/utils/makefile.lxc b/utils/makefile.lxc deleted file mode 100644 index 95a8d4317..000000000 --- a/utils/makefile.lxc +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8; mode: makefile-gmake -*- -# -# LXC environment -# =============== -# -# To activate/deactivate LXC makefile environment in a container, set/unset link -# from root '/.lxcenv.mk' to *this* file:: -# -# sudo make ./utils/makefile.lxc lxc-activate -# sudo make ./utils/makefile.lxc lxc-deactivate - -LXC_ENV_FOLDER=lxc-env/$(shell hostname)/ - -lxc-help:: - @echo ' LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)' - -# If not activated, serve target 'lxc-activate' .. -ifeq (,$(wildcard /.lxcenv.mk)) -PHONY += lxc-activate -lxc-activate: - ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk" -else -# .. and if activated, serve target 'lxc-deactivate'. -PHONY += lxc-deactivate -lxc-deactivate: - rm /.lxcenv.mk -$(LXC_ENV_FOLDER): - $(Q)mkdir -p $(LXC_ENV_FOLDER) - $(Q)echo placeholder > $(LXC_ENV_FOLDER).placeholder -endif - -.PHONY: $(PHONY) diff --git a/utils/searxng.sh b/utils/searxng.sh index 663ca6d3d..d4c3011c9 100755 --- a/utils/searxng.sh +++ b/utils/searxng.sh @@ -41,10 +41,6 @@ fi SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)/searxng}" SEARXNG_URL="${SEARXNG_URL%/}" # if exists, remove trailing slash -if in_container; then - # hint: Linux containers do not have DNS entries, lets use IPs - SEARXNG_URL="http://$(primary_ip)/searxng" -fi SEARXNG_URL_PATH="$(echo "${SEARXNG_URL}" | sed -e 's,^.*://[^/]*\(/.*\),\1,g')" [[ "${SEARXNG_URL_PATH}" == "${SEARXNG_URL}" ]] && SEARXNG_URL_PATH=/ @@ -114,9 +110,7 @@ esac _service_prefix=" ${_Yellow}|${SERVICE_USER}|${_creset} " -# ---------------------------------------------------------------------------- usage() { -# ---------------------------------------------------------------------------- # shellcheck disable=SC1117 cat <