From 414a6105e7793e9499c64dd649eda43250971c50 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Jun 2021 19:01:07 +0200 Subject: [mod] load .config.sh from installation (utils/lib_install.sh) **new** utils/lib_install.sh: Used to initialize installation procedures - Modified source_dot_config function that - loads .config.sh from an existing installation (at SEARX_SRC). - initialize **SEARX_SRC_INIT_FILES** - functions like: - install_log_searx_instance() - install_searx_get_state() **modified** utils/searx.sh - obsolete environment SEARX_INSTANCE_NAME has been replaced by string 'SearXNG'. **modified** utils/filtron.sh, utils/morty.sh, utils/searx.sh - source utils/lib_install.sh - normalize logging of environment variables using new function install_log_searx_instance() **modified** utils/lib.sh - fix marginal typos **Installation scripts** The utils/lib_install.sh is sourced by the installations scripts: - utils/searx.sh - utils/morty.sh - utils/filtron.sh If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function loads this configuration (instead of './.config.sh'). **SEARX_SRC_INIT_FILES** Array of file names to sync into a installation at $SEARX_SRC. The file names are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). Most often theses are files like: - .config.sh - searx/settings.yml - utils/brand.env - ... Signed-off-by: Markus Heiser --- utils/lib_install.sh | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100755 utils/lib_install.sh (limited to 'utils/lib_install.sh') diff --git a/utils/lib_install.sh b/utils/lib_install.sh new file mode 100755 index 000000000..19d438c43 --- /dev/null +++ b/utils/lib_install.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 +# shellcheck source=utils/lib.sh +. /dev/null + +# Initialize installation procedures: +# +# - Modified source_dot_config function that +# - loads .config.sh from an existing installation (at SEARX_SRC). +# - initialize **SEARX_SRC_INIT_FILES** +# - functions like: +# - install_log_searx_instance() +# - install_searx_get_state() +# +# usage: +# source lib_install.sh +# +# **Installation scripts** +# +# The utils/lib_install.sh is sourced by the installations scripts: +# +# - utils/searx.sh +# - utils/morty.sh +# - utils/filtron.sh +# +# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function +# loads this configuration (instead of './.config.sh'). + +# **SEARX_SRC_INIT_FILES** +# +# Array of file names to sync into a installation at $SEARX_SRC. The file names +# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). +# Most often theses are files like: +# - .config.sh +# - searx/settings.yml +# - utils/brand.env +# - ... + + +SEARX_SRC_INIT_FILES=() + +eval orig_"$(declare -f source_dot_config)" + +source_dot_config() { + + # Modified source_dot_config function that + # - loads .config.sh from an existing installation (at SEARX_SRC). + # - initialize SEARX_SRC_INIT_FILES + + if [ -z "$eval_SEARX_SRC" ]; then + export eval_SEARX_SRC='true' + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + if [ ! -r "${SEARX_SRC}" ]; then + build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" + orig_source_dot_config + return 0 + fi + build_msg INSTANCE "using instance at: ${SEARX_SRC}" + + # set and log DOT_CONFIG + if [ -r "${SEARX_SRC}/.config.sh" ]; then + build_msg INSTANCE "switching to ${SEARX_SRC}/.config.sh" + DOT_CONFIG="${SEARX_SRC}/.config.sh" + else + build_msg INSTANCE "using local config: ${DOT_CONFIG}" + fi + init_SEARX_SRC_INIT_FILES + fi +} + +init_SEARX_SRC_INIT_FILES(){ + # init environment SEARX_SRC_INIT_FILES + + # Monitor modified files in the working-tree from the local repository, only + # if the local file differs to the corresponding file in the instance. Most + # often theses are files like: + # + # - .config.sh + # - searx/settings.yml + # - utils/brand.env + # - ... + + # keep list empty if there is no installation + SEARX_SRC_INIT_FILES=() + if [ ! -r "$SEARX_SRC" ]; then + return 0 + fi + + local fname + local msg="" + + # Monitor local modified files from the repository, only if the local file + # differs to the corresponding file in the instance + + while IFS= read -r fname; do + if [ -z "$fname" ]; then + continue + fi + if [ -r "${SEARX_SRC}/${fname}" ]; then + # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then + SEARX_SRC_INIT_FILES+=("${fname}") + build_msg INSTANCE "local clone (workingtree), modified file: ./$fname" + msg="to update use: sudo -H ./utils/searx.sh install init-src" + fi + fi + done <<< "$(git diff --name-only)" + [ -n "$msg" ] && build_msg INSTANCE "$msg" +} + +install_log_searx_instance() { + + echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" + echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" + echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" + + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "---- container setup" + echo -e " ${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi +} + +install_searx_get_state(){ + + # usage: install_searx_get_state + # + # Prompts a string indicating the status of the installation procedure + # + # missing-searx-clone: + # There is no clone at ${SEARX_SRC} + # missing-searx-pyenv: + # There is no pyenv in ${SEARX_PYENV} + # installer-modified: + # There are files modified locally in the installer (clone), + # see ${SEARX_SRC_INIT_FILES} description. + # python-installed: + # Scripts can be executed in instance's environment + # - user: ${SERVICE_USER} + # - pyenv: ${SEARX_PYENV} + + if ! [ -r "${SEARX_SRC}" ]; then + echo "missing-searx-clone" + return + fi + if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then + echo "missing-searx-pyenv" + return + fi + if ! [ -r "${SEARX_SETTINGS_PATH}" ]; then + echo "missing-settings" + return + fi + if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + echo "installer-modified" + return + fi + echo "python-installed" +} + +# Initialization of the installation procedure +# -------------------------------------------- + +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +source_dot_config + +# shellcheck source=utils/lxc-searx.env +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env -- cgit v1.2.3 From f61c918dd4ea211f32874f1072ecfca0f39fcca4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Jun 2021 20:08:10 +0200 Subject: [mod] normalize .config.sh with settings.yml In commit 94851790 we have centralized all SearXNG setups in the settings.yml file: 94851790 [mod] move brand options from Makefile to settings.yml This step has not yet been completed for the installation procedures! Since all SearXNG setups are done in the settings.yml these environment variables needs to be removed from the ./conf.sh file. Scripts and other tasks running outside of an instance got the needed values from the ./utils/brand.env file. By example: ATM the environment variables of the ./config.sh file are in conflict with them from settings.yml: - PUBLIC_URL --> {server:base_url} - SEARX_INTERNAL_HTTP --> {server:bind_address}.{server:port} - GIT_BRANCH --> {brand:GIT_URL} These environment variable of a SearXNG instance and additional - SEARX_SETTINGS_TEMPLATE has been remove from the '.config.sh' file. With this patch, the main focus of ./conf.sh resists on environment variables needed for the installation of morty, filtron software. modified .config.sh: - removed no longer supported variables (see above) - add comment about: SearXNG setup in settings.yml modified utils/searx.sh: - SEARX_INTERNAL_HTTP no longer take from .config.sh - SEARX_SETTINGS_PATH /etc/searx/settings.yml - SEARX_SETTINGS_TEMPLATE obsolete modified utils/lib_install.sh: Initialize environment variables SEARX_PYENV, SEARX_SETTINGS_PATH and PUBLIC_URL. modified: utils/morty.sh Add missing hint about SEARX_SETTINGS_PATH and move PUBLIC_URL to utils/lib_install.sh modified: utils/morty.sh Move PUBLIC_URL to utils/lib_install.sh Renamed utils/templates/etc/searx/use_default_settings.yml -> settings.yml - removed option which can't be modified after installation - add some comments with examples Signed-off-by: Markus Heiser --- utils/lib_install.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'utils/lib_install.sh') diff --git a/utils/lib_install.sh b/utils/lib_install.sh index 19d438c43..a8cb23ab2 100755 --- a/utils/lib_install.sh +++ b/utils/lib_install.sh @@ -52,6 +52,8 @@ source_dot_config() { if [ -z "$eval_SEARX_SRC" ]; then export eval_SEARX_SRC='true' SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) + SEARX_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SETTINGS_PATH) if [ ! -r "${SEARX_SRC}" ]; then build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" orig_source_dot_config @@ -114,6 +116,7 @@ install_log_searx_instance() { echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SSEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}" echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" @@ -177,6 +180,18 @@ install_searx_get_state(){ # shellcheck source=utils/brand.env source "${REPO_ROOT}/utils/brand.env" +# SEARX_URL aka PUBLIC_URL: the public URL of the instance (e.g. +# "https://example.org/searx"). The value is taken from environment $SEARX_URL +# in ./utils/brand.env. This variable is a empty string if server.base_url in +# the settings.yml is set to 'false'. + +SEARX_URL="${SEARX_URL:-http://$(uname -n)}" +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARX_URL="http://$(primary_ip)" +fi +PUBLIC_URL="${SEARX_URL}" + source_dot_config # shellcheck source=utils/lxc-searx.env -- cgit v1.2.3 From 57b8f340a6689fafd0c8d92aebd2b263b090af18 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 12 Jul 2021 16:51:36 +0200 Subject: [mod] utils/searx.sh - add command 'inspect settings' Inspect YAML setting from SearXNG instance (${SEARX_SRC}):: utils/searx.sh inspect settings server.base_url utils/lib_install.sh should not log on stdout which is used for the output of function prompt_installation_setting(). Turned build_msg (stdout) into info_msg (stderr). Signed-off-by: Markus Heiser --- utils/lib_install.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'utils/lib_install.sh') diff --git a/utils/lib_install.sh b/utils/lib_install.sh index a8cb23ab2..4ad11d63d 100755 --- a/utils/lib_install.sh +++ b/utils/lib_install.sh @@ -55,18 +55,18 @@ source_dot_config() { SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) SEARX_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SETTINGS_PATH) if [ ! -r "${SEARX_SRC}" ]; then - build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" + info_msg "not yet cloned: ${SEARX_SRC}" orig_source_dot_config return 0 fi - build_msg INSTANCE "using instance at: ${SEARX_SRC}" + info_msg "using instance at: ${SEARX_SRC}" # set and log DOT_CONFIG if [ -r "${SEARX_SRC}/.config.sh" ]; then - build_msg INSTANCE "switching to ${SEARX_SRC}/.config.sh" + info_msg "switching to ${SEARX_SRC}/.config.sh" DOT_CONFIG="${SEARX_SRC}/.config.sh" else - build_msg INSTANCE "using local config: ${DOT_CONFIG}" + info_msg "using local config: ${DOT_CONFIG}" fi init_SEARX_SRC_INIT_FILES fi @@ -104,12 +104,12 @@ init_SEARX_SRC_INIT_FILES(){ # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then SEARX_SRC_INIT_FILES+=("${fname}") - build_msg INSTANCE "local clone (workingtree), modified file: ./$fname" + info_msg "local clone (workingtree), modified file: ./$fname" msg="to update use: sudo -H ./utils/searx.sh install init-src" fi fi done <<< "$(git diff --name-only)" - [ -n "$msg" ] && build_msg INSTANCE "$msg" + [ -n "$msg" ] && info_msg "$msg" } install_log_searx_instance() { -- cgit v1.2.3