diff options
| author | Markus Heiser <markus.heiser@darmarIT.de> | 2021-07-27 05:32:57 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-07-27 05:32:57 +0000 |
| commit | beff764d7402e6a834f6dbb0a28921fc2577defe (patch) | |
| tree | d1fd3beb026c1199d60ffc67b9c6e79ddb8f5f80 /utils | |
| parent | 80f5241387aea1381dd96ed2b91ca62de927ba23 (diff) | |
| parent | e946c3462c2b44fb154d0b6595f4f42605a326e5 (diff) | |
Merge pull request #216 from return42/fix-searx.sh
normalize environment of installation tasks (shell) with YAML settings
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/brand.env | 11 | ||||
| -rw-r--r-- | utils/build_env.py | 47 | ||||
| -rwxr-xr-x | utils/filtron.sh | 60 | ||||
| -rwxr-xr-x | utils/lib.sh | 6 | ||||
| -rwxr-xr-x | utils/lib_install.sh | 199 | ||||
| -rwxr-xr-x | utils/lxc.sh | 2 | ||||
| -rwxr-xr-x | utils/morty.sh | 41 | ||||
| -rwxr-xr-x | utils/searx.sh | 355 | ||||
| -rw-r--r-- | utils/templates/etc/searx/settings.yml (renamed from utils/templates/etc/searx/use_default_settings.yml) | 40 |
9 files changed, 551 insertions, 210 deletions
diff --git a/utils/brand.env b/utils/brand.env index e0c670a01..c09ea9d2f 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,8 +1,5 @@ +export GIT_URL='https://github.com/return42/searxng' +export GIT_BRANCH='fix-searx.sh' export SEARX_URL='' -export GIT_URL='https://github.com/searxng/searxng' -export GIT_BRANCH='master' -export ISSUE_URL='https://github.com/searxng/searxng/issues' -export DOCS_URL='https://searxng.github.io/searxng' -export PUBLIC_INSTANCES='https://searx.space' -export CONTACT_URL='' -export WIKI_URL='https://github.com/searxng/searxng/wiki' +export SEARX_PORT='7777' +export SEARX_BIND_ADDRESS='127.0.0.12' diff --git a/utils/build_env.py b/utils/build_env.py index c52111e81..dc55bd0a8 100644 --- a/utils/build_env.py +++ b/utils/build_env.py @@ -9,15 +9,12 @@ from os.path import realpath, dirname, join, sep, abspath repo_root = realpath(dirname(realpath(__file__)) + sep + '..') sys.path.insert(0, repo_root) -os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + '/settings.yml') # Under the assumption that a brand is always a fork assure that the settings # file from reposetorie's working tree is used to generate the build_env, not # from /etc/searx/settings.yml. os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + sep + 'settings.yml') -from searx import get_setting - def _env(*arg, **kwargs): val = get_setting(*arg, **kwargs) if val is True: @@ -26,20 +23,42 @@ def _env(*arg, **kwargs): val = '' return val +# If you add or remove variables here, do not forgett to update: +# - ./docs/admin/engines/settings.rst +# - ./docs/dev/makefile.rst (section make buildenv) +# - ./manage function buildenv.unset_env() + name_val = [ - ('SEARX_URL' , _env('server.base_url','')), - ('GIT_URL' , _env('brand.git_url', '')), - ('GIT_BRANCH' , _env('brand.git_branch', '')), - ('ISSUE_URL' , _env('brand.issue_url', '')), - ('DOCS_URL' , _env('brand.docs_url', '')), - ('PUBLIC_INSTANCES' , _env('brand.public_instances', '')), - ('CONTACT_URL' , _env('general.contact_url', '')), - ('WIKI_URL' , _env('brand.wiki_url', '')), + + ('GIT_URL' , 'brand.git_url'), + ('GIT_BRANCH' , 'brand.git_branch'), + + ('SEARX_URL' , 'server.base_url'), + ('SEARX_PORT' , 'server.port'), + ('SEARX_BIND_ADDRESS' , 'server.bind_address'), + ] brand_env = 'utils' + sep + 'brand.env' -print('build %s' % brand_env) +# Some defaults in the settings.yml are taken from the environment, +# e.g. SEARX_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). When the +# 'brand.env' file is created these enviroment variables should be unset first:: + +_unset = object() +for name, option in name_val: + if not os.environ.get(name, _unset) is _unset: + del os.environ[name] + +# After the variables are unset in the environ, we can import settings +# (get_setting) from searx module. + +from searx import get_setting + +print('build %s (settings from: %s)' % (brand_env, os.environ['SEARX_SETTINGS_PATH'])) +sys.path.insert(0, repo_root) +from searx import settings + with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f: - for name, val in name_val: - print("export %s='%s'" % (name, val), file=f) + for name, option in name_val: + print("export %s='%s'" % (name, _env(option)), file=f) diff --git a/utils/filtron.sh b/utils/filtron.sh index 5680811ea..3bd6f016d 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -1,21 +1,16 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck disable=SC2119,SC2001 +# shellcheck disable=SC2001 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ @@ -28,7 +23,12 @@ FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/e FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" -FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +# The filtron target is the SearXNG installation, listenning on server.port at +# server.bind_address. The default of FILTRON_TARGET is taken from the YAML +# configuration, do not change this value without reinstalling the entire +# SearXNG suite including filtron & morty. +FILTRON_TARGET="${SEARX_BIND_ADDRESS}:${SEARX_PORT}" SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" @@ -94,28 +94,17 @@ apache (${PUBLIC_URL}) nginx (${PUBLIC_URL}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_FILTRON_SITE} - filtron rules: ${FILTRON_RULES_TEMPLATE} - -If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL : ${PUBLIC_URL} - PUBLIC_HOST : ${PUBLIC_HOST} - SERVICE_USER : ${SERVICE_USER} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} +---- sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + FILTRON_TARGET : ${FILTRON_TARGET} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} EOF - if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container URL (IPv6): http://[${ip#*|}]:4005/" - else - # IPv4: - echo " container URL (IPv4): http://${ip#*|}:4005/" - fi - done - fi + + install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } @@ -349,16 +338,15 @@ inspect_service() { cat <<EOF -sourced ${DOT_CONFIG#"$REPO_ROOT/"} : - - PUBLIC_URL : ${PUBLIC_URL} - PUBLIC_HOST : ${PUBLIC_HOST} - FILTRON_URL_PATH : ${FILTRON_URL_PATH} +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + FILTRON_TARGET : ${FILTRON_TARGET} FILTRON_API : ${FILTRON_API} FILTRON_LISTEN : ${FILTRON_LISTEN} - FILTRON_TARGET : ${FILTRON_TARGET} - + FILTRON_URL_PATH : ${FILTRON_URL_PATH} EOF + install_log_searx_instance if service_account_is_available "$SERVICE_USER"; then info_msg "service account $SERVICE_USER available." diff --git a/utils/lib.sh b/utils/lib.sh index d2f5b757e..f29666226 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -462,11 +462,11 @@ install_template() { fi if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then - info_msg "file ${dst} allready installed" + info_msg "file ${dst} already installed" return 0 fi - info_msg "diffrent file ${dst} allready exists on this host" + info_msg "different file ${dst} already exists on this host" while true; do choose_one _reply "choose next step with file $dst" \ @@ -1642,7 +1642,7 @@ git_clone() { # git_clone <url> <path> [<branch> [<user>]] # # First form uses $CACHE/<name> as destination folder, second form clones - # into <path>. If repository is allready cloned, pull from <branch> and + # into <path>. If repository is already cloned, pull from <branch> and # update working tree (if needed, the caller has to stash local changes). # # git clone https://github.com/searxng/searxng searx-src origin/master searxlogin diff --git a/utils/lib_install.sh b/utils/lib_install.sh new file mode 100755 index 000000000..4ad11d63d --- /dev/null +++ b/utils/lib_install.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 +# shellcheck source=utils/lib.sh +. /dev/null + +# Initialize installation procedures: +# +# - Modified source_dot_config function that +# - loads .config.sh from an existing installation (at SEARX_SRC). +# - initialize **SEARX_SRC_INIT_FILES** +# - functions like: +# - install_log_searx_instance() +# - install_searx_get_state() +# +# usage: +# source lib_install.sh +# +# **Installation scripts** +# +# The utils/lib_install.sh is sourced by the installations scripts: +# +# - utils/searx.sh +# - utils/morty.sh +# - utils/filtron.sh +# +# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function +# loads this configuration (instead of './.config.sh'). + +# **SEARX_SRC_INIT_FILES** +# +# Array of file names to sync into a installation at $SEARX_SRC. The file names +# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). +# Most often theses are files like: +# - .config.sh +# - searx/settings.yml +# - utils/brand.env +# - ... + + +SEARX_SRC_INIT_FILES=() + +eval orig_"$(declare -f source_dot_config)" + +source_dot_config() { + + # Modified source_dot_config function that + # - loads .config.sh from an existing installation (at SEARX_SRC). + # - initialize SEARX_SRC_INIT_FILES + + if [ -z "$eval_SEARX_SRC" ]; then + export eval_SEARX_SRC='true' + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) + SEARX_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SETTINGS_PATH) + if [ ! -r "${SEARX_SRC}" ]; then + info_msg "not yet cloned: ${SEARX_SRC}" + orig_source_dot_config + return 0 + fi + info_msg "using instance at: ${SEARX_SRC}" + + # set and log DOT_CONFIG + if [ -r "${SEARX_SRC}/.config.sh" ]; then + info_msg "switching to ${SEARX_SRC}/.config.sh" + DOT_CONFIG="${SEARX_SRC}/.config.sh" + else + info_msg "using local config: ${DOT_CONFIG}" + fi + init_SEARX_SRC_INIT_FILES + fi +} + +init_SEARX_SRC_INIT_FILES(){ + # init environment SEARX_SRC_INIT_FILES + + # Monitor modified files in the working-tree from the local repository, only + # if the local file differs to the corresponding file in the instance. Most + # often theses are files like: + # + # - .config.sh + # - searx/settings.yml + # - utils/brand.env + # - ... + + # keep list empty if there is no installation + SEARX_SRC_INIT_FILES=() + if [ ! -r "$SEARX_SRC" ]; then + return 0 + fi + + local fname + local msg="" + + # Monitor local modified files from the repository, only if the local file + # differs to the corresponding file in the instance + + while IFS= read -r fname; do + if [ -z "$fname" ]; then + continue + fi + if [ -r "${SEARX_SRC}/${fname}" ]; then + # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then + SEARX_SRC_INIT_FILES+=("${fname}") + info_msg "local clone (workingtree), modified file: ./$fname" + msg="to update use: sudo -H ./utils/searx.sh install init-src" + fi + fi + done <<< "$(git diff --name-only)" + [ -n "$msg" ] && info_msg "$msg" +} + +install_log_searx_instance() { + + echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" + echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SSEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}" + echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" + echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" + + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "---- container setup" + echo -e " ${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi +} + +install_searx_get_state(){ + + # usage: install_searx_get_state + # + # Prompts a string indicating the status of the installation procedure + # + # missing-searx-clone: + # There is no clone at ${SEARX_SRC} + # missing-searx-pyenv: + # There is no pyenv in ${SEARX_PYENV} + # installer-modified: + # There are files modified locally in the installer (clone), + # see ${SEARX_SRC_INIT_FILES} description. + # python-installed: + # Scripts can be executed in instance's environment + # - user: ${SERVICE_USER} + # - pyenv: ${SEARX_PYENV} + + if ! [ -r "${SEARX_SRC}" ]; then + echo "missing-searx-clone" + return + fi + if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then + echo "missing-searx-pyenv" + return + fi + if ! [ -r "${SEARX_SETTINGS_PATH}" ]; then + echo "missing-settings" + return + fi + if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + echo "installer-modified" + return + fi + echo "python-installed" +} + +# Initialization of the installation procedure +# -------------------------------------------- + +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +# SEARX_URL aka PUBLIC_URL: the public URL of the instance (e.g. +# "https://example.org/searx"). The value is taken from environment $SEARX_URL +# in ./utils/brand.env. This variable is a empty string if server.base_url in +# the settings.yml is set to 'false'. + +SEARX_URL="${SEARX_URL:-http://$(uname -n)}" +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARX_URL="http://$(primary_ip)" +fi +PUBLIC_URL="${SEARX_URL}" + +source_dot_config + +# shellcheck source=utils/lxc-searx.env +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env diff --git a/utils/lxc.sh b/utils/lxc.sh index f94e9841a..9a79606ca 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -568,7 +568,7 @@ check_connectivity() { info_msg "Most often the connectivity is blocked by a docker installation:" info_msg "Whenever docker is started (reboot) it sets the iptables policy " info_msg "for the FORWARD chain to DROP, see:" - info_msg " ${DOCS_URL}/utils/lxc.sh.html#internet-connectivity-docker" + info_msg " https://searxng.github.io/searxng/utils/lxc.sh.html#internet-connectivity-docker" iptables-save | grep ":FORWARD" fi return $ret_val diff --git a/utils/morty.sh b/utils/morty.sh index 0a44ef574..8be0e6ba6 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -1,15 +1,10 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -17,7 +12,6 @@ in_container && lxc_set_suite_env MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" - PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" # shellcheck disable=SC2034 @@ -86,12 +80,15 @@ apache : ${PUBLIC_URL_MORTY} nginx (${PUBLIC_URL_MORTY}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_MORTY_SITE} - -If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY} - MORTY_LISTEN: ${MORTY_LISTEN} - SERVICE_USER: ${SERVICE_USER} +---- +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} + MORTY_LISTEN: : ${MORTY_LISTEN} EOF + + install_log_searx_instance if in_container; then # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) for ip in $(global_IPs) ; do @@ -112,8 +109,9 @@ EOF info_searx() { # shellcheck disable=SC1117 cat <<EOF -To activate result and image proxy in searx, edit settings.yml (read: -${DOCS_URL}/admin/morty.html):: +To activate result and image proxy in SearXNG read: + https://searxng.github.io/searxng/admin/morty.html +Check settings in file ${SEARX_SETTINGS_PATH} ... result_proxy: url : ${PUBLIC_URL_MORTY} server: @@ -237,7 +235,7 @@ install_all() { fi fi info_searx - if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then + if ask_yn "Add image and result proxy to SearXNG settings.yml?" Yn; then "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" "${REPO_ROOT}/utils/searx.sh" option image-proxy-on fi @@ -335,11 +333,14 @@ inspect_service() { cat <<EOF -sourced ${DOT_CONFIG#"$REPO_ROOT/"} : - - MORTY_LISTEN : ${MORTY_LISTEN} +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} + MORTY_LISTEN: : ${MORTY_LISTEN} EOF + install_log_searx_instance if service_account_is_available "$SERVICE_USER"; then info_msg "service account $SERVICE_USER available." @@ -402,7 +403,7 @@ EOF } enable_debug() { - warn_msg "Do not enable debug in production enviroments!!" + warn_msg "Do not enable debug in production environments!!" info_msg "Enabling debug option needs to reinstall systemd service!" set_service_env_debug true } diff --git a/utils/searx.sh b/utils/searx.sh index 40a61e4f4..f02611059 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -1,29 +1,22 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck disable=SC2001 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env + +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" - -SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}" +SEARX_INTERNAL_HTTP="${SEARX_BIND_ADDRESS}:${SEARX_PORT}" SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ | sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" [[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/ -SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ -| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" SERVICE_NAME="searx" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" @@ -35,8 +28,7 @@ SERVICE_GROUP="${SERVICE_USER}" GIT_BRANCH="${GIT_BRANCH:-master}" SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" SEARX_SRC="${SERVICE_HOME}/searx-src" -SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH:-/etc/searx/settings.yml}" -SEARX_SETTINGS_TEMPLATE="${SEARX_SETTINGS_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml}" +SEARX_SETTINGS_PATH="/etc/searx/settings.yml" SEARX_UWSGI_APP="searx.ini" # shellcheck disable=SC2034 SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" @@ -149,12 +141,12 @@ usage() { cat <<EOF usage:: $(basename "$0") shell - $(basename "$0") install [all|user|searx-src|pyenv|uwsgi|packages|settings|buildhost] + $(basename "$0") install [all|init-src|dot-config|user|searx-src|pyenv|uwsgi|packages|settings|buildhost] $(basename "$0") update [searx] $(basename "$0") remove [all|user|pyenv|searx-src] $(basename "$0") activate [service] $(basename "$0") deactivate [service] - $(basename "$0") inspect [service] + $(basename "$0") inspect [service|settings <key>] $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>] $(basename "$0") apache [install|remove] @@ -163,48 +155,34 @@ shell install / remove :all: complete (de-) installation of searx service :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :dot-config: copy ./config.sh to ${SEARX_SRC} :searx-src: clone $GIT_URL + :init-src: copy files (SEARX_SRC_INIT_FILES) to ${SEARX_SRC} :pyenv: create/remove virtualenv (python) in $SEARX_PYENV :uwsgi: install searx uWSGI application - :settings: reinstall settings from ${SEARX_SETTINGS_TEMPLATE} + :settings: reinstall settings from ${SEARX_SETTINGS_PATH} :packages: install needed packages from OS package manager :buildhost: install packages from OS package manager needed by buildhosts update searx - Update searx installation ($SERVICE_HOME) + Update SearXNG installation ($SERVICE_HOME) activate service activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) -inspect service - run some small tests and inspect service's status and log +inspect + :service: run some small tests and inspect service's status and log + :settings: inspect YAML setting <key> from SearXNG instance (${SEARX_SRC}) option set one of the available options apache - :install: apache site with the searx uwsgi app + :install: apache site with the SearXNG uwsgi app :remove: apache site ${APACHE_FILTRON_SITE} - -searx settings: ${SEARX_SETTINGS_PATH} - -If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL : ${PUBLIC_URL} - SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} +---- sourced ${DOT_CONFIG} SERVICE_USER : ${SERVICE_USER} - SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP} + SERVICE_HOME : ${SERVICE_HOME} EOF - if in_container; then - # searx is listening on 127.0.0.1 and not available from outside container - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \ - "${_BBlack}inside${_creset} the container." - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container (IPv6): [${ip#*|}]" - else - # IPv4: - echo " container (IPv4): ${ip#*|}" - fi - done - fi + + install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } @@ -228,21 +206,45 @@ main() { sudo_or_exit inspect_service ;; + settings) + prompt_installation_setting "$3" + dump_return $? + ;; *) usage "$_usage"; exit 42;; esac ;; install) - rst_title "$SEARX_INSTANCE_NAME" part + rst_title "SearXNG (install)" part sudo_or_exit case $2 in all) install_all ;; - user) assert_user ;; - pyenv) create_pyenv ;; - searx-src) clone_searx ;; - settings) install_settings ;; + user) + verify_continue_install + assert_user + ;; + pyenv) + verify_continue_install + create_pyenv + ;; + searx-src) + verify_continue_install + clone_searx + install_DOT_CONFIG + init_SEARX_SRC + ;; + init-src) + init_SEARX_SRC + ;; + dot-config) + install_DOT_CONFIG + ;; + settings) + install_settings + ;; uwsgi) + verify_continue_install install_searx_uwsgi if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" fi ;; packages) @@ -261,6 +263,7 @@ main() { *) usage "$_usage"; exit 42;; esac ;; remove) + rst_title "SearXNG (remove)" part sudo_or_exit case $2 in all) remove_all;; @@ -307,13 +310,18 @@ main() { _service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} " install_all() { - rst_title "Install $SEARX_INSTANCE_NAME (service)" + rst_title "Install SearXNG (service)" + verify_continue_install pkg_install "$SEARX_PACKAGES" wait_key assert_user wait_key clone_searx wait_key + install_DOT_CONFIG + wait_key + init_SEARX_SRC + wait_key create_pyenv wait_key install_settings @@ -322,7 +330,7 @@ install_all() { wait_key install_searx_uwsgi if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" fi if ask_yn "Do you want to inspect the installation?" Ny; then inspect_service @@ -330,7 +338,7 @@ install_all() { } update_searx() { - rst_title "Update searx instance" + rst_title "Update SearXNG instance" echo tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" @@ -348,13 +356,13 @@ EOF } remove_all() { - rst_title "De-Install $SEARX_INSTANCE_NAME (service)" + rst_title "De-Install SearXNG (service)" rst_para "\ It goes without saying that this script can only be used to remove installations that were installed with this script." - if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then + if ! ask_yn "Do you really want to deinstall SearXNG?"; then return fi remove_searx_uwsgi @@ -388,14 +396,14 @@ clone_is_available() { # shellcheck disable=SC2164 clone_searx() { - rst_title "Clone searx sources" section + rst_title "Clone SearXNG sources" section echo if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" fi SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" if [[ ! "${SERVICE_HOME}" ]]; then - err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + err_msg "to clone SearXNG sources, user $SERVICE_USER hast to be created first" return 42 fi if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then @@ -421,54 +429,169 @@ EOF popd > /dev/null } -install_settings() { - rst_title "${SEARX_SETTINGS_PATH}" section - if ! clone_is_available; then - err_msg "you have to install searx first" - exit 42 - fi - mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" - - if [[ ! -f "${SEARX_SETTINGS_PATH}" ]]; then - info_msg "install settings ${SEARX_SETTINGS_TEMPLATE}" - info_msg " --> ${SEARX_SETTINGS_PATH}" - cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - configure_searx - return - fi +prompt_installation_status(){ - rst_para "Diff between origin's setting file (+) and current (-):" - echo "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_TEMPLATE}" - $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_TEMPLATE}" + local state branch remote remote_url instance_setting + state="$(install_searx_get_state)" + branch="$(git name-rev --name-only HEAD)" + remote="$(git config branch."${branch}".remote)" + remote_url="$(git config remote."${remote}".url)" - local action - choose_one action "What should happen to the settings file? " \ - "keep configuration unchanged" \ - "use origin settings" \ - "start interactive shell" - case $action in - "keep configuration unchanged") - info_msg "leave settings file unchanged" + case $state in + missing-searx-clone) + info_msg "${_BBlue}(status: $(install_searx_get_state))${_creset}" + return 0 ;; - "use origin settings") - backup_file "${SEARX_SETTINGS_PATH}" - info_msg "install origin settings" - cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" + *) + warn_msg "SearXNG instance already installed at: $SEARX_SRC" + warn_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} " + instance_setting="$(prompt_installation_setting brand.git_url)" + if ! [ "$instance_setting" = "$remote_url" ]; then + warn_msg "instance's brand.git_url: '${instance_setting}'" \ + "differs from local clone's remote URL: ${remote_url}" + fi + instance_setting="$(prompt_installation_setting brand.git_branch)" + if ! [ "$instance_setting" = "$branch" ]; then + warn_msg "instance brand.git_branch: ${instance_setting}" \ + "differs from local clone's branch: ${branch}" + fi + return 42 ;; - "start interactive shell") - backup_file "${SEARX_SETTINGS_PATH}" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -i - rst_para 'Diff between new setting file (-) and current (+):' - echo - $DIFF_CMD "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - wait_key + esac +} + +verify_continue_install(){ + if ! prompt_installation_status; then + MSG="[${_BCyan}KEY${_creset}] to continue installation / [${_BCyan}CTRL-C${_creset}] to exit" \ + wait_key + fi +} + +prompt_installation_setting(){ + + # usage: prompt_installation_setting brand.git_url + # + # Prompts the value of the (YAML) setting in the SearXNG instance. + + local _state + _state="$(install_searx_get_state)" + case $_state in + python-installed|installer-modified) + sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" <<EOF +import sys +from searx import get_setting +name = "${1}" +unset = object() +value = get_setting(name, unset) +if value is unset: + sys.stderr.write("error: setting '%s' does not exists\n" % name) + sys.exit(42) +print(value) +sys.exit(0) +EOF + ;; + *) + return 42 ;; esac } +init_SEARX_SRC(){ + rst_title "Update instance: ${SEARX_SRC}/" section + + if ! clone_is_available; then + err_msg "you have to install SearXNG first" + return 1 + fi + + init_SEARX_SRC_INIT_FILES + + if [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + info_msg "no files registered in SEARX_SRC_INIT_FILES" + return 2 + fi + + echo + echo "Manipulating files like settings.yml can break existing installation!" + echo "Update instance with file(s) from: ${REPO_ROOT}" + echo + for i in "${SEARX_SRC_INIT_FILES[@]}"; do + echo "- $i" + done + if ! ask_yn "Do you really want to update these files in the instance?" Yn; then + return 42 + fi + for fname in "${SEARX_SRC_INIT_FILES[@]}"; do + while true; do + choose_one _reply "choose next step with file ${fname}" \ + "leave file unchanged" \ + "replace file" \ + "diff files" \ + "interactive shell" + + case $_reply in + "leave file unchanged") + break + ;; + "replace file") + info_msg "copy: ${REPO_ROOT}/${fname} --> ${SEARX_SRC}/${fname}" + cp "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + break + ;; + "diff files") + $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" + ;; + "interactive shell") + backup_file "${SEARX_SRC}/${fname}" + echo -e "// edit ${_Red}${dst}${_creset} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${SERVICE_USER}" -i + $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" + echo + echo -e "// ${_BBlack}did you edit file ...${_creset}" + echo -en "// ${_Red}${dst}${_creset}" + if ask_yn "//${_BBlack}... to your needs?${_creset}"; then + break + fi + ;; + esac + done + done +} + +install_DOT_CONFIG(){ + rst_title "Update instance: ${SEARX_SRC}/.config.sh" section + + if cmp --silent "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh"; then + info_msg "${SEARX_SRC}/.config.sh is up to date" + return 0 + fi + + diff "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" + if ! ask_yn "Do you want to copy file .config.sh into instance?" Yn; then + return 42 + fi + backup_file "${SEARX_SRC}/.config.sh" + cp "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" +} + +install_settings() { + rst_title "${SEARX_SETTINGS_PATH}" section + + if ! clone_is_available; then + err_msg "you have to install SearXNG first" + exit 42 + fi + + mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" + install_template \ + "${SEARX_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + configure_searx +} + remove_settings() { - rst_title "remove searx settings" section + rst_title "remove SearXNG settings" section echo info_msg "delete ${SEARX_SETTINGS_PATH}" rm -f "${SEARX_SETTINGS_PATH}" @@ -476,10 +599,10 @@ remove_settings() { remove_searx() { rst_title "Drop searx sources" section - if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + if ask_yn "Do you really want to drop SearXNG sources ($SEARX_SRC)?"; then rm -rf "$SEARX_SRC" else - rst_para "Leave searx sources unchanged." + rst_para "Leave SearXNG sources unchanged." fi } @@ -491,7 +614,7 @@ create_pyenv() { rst_title "Create virtualenv (python)" section echo if [[ ! -f "${SEARX_SRC}/manage" ]]; then - err_msg "to create pyenv for searx, searx has to be cloned first" + err_msg "to create pyenv for SearXNG, SearXNG has to be cloned first" return 42 fi info_msg "create pyenv in ${SEARX_PYENV}" @@ -531,18 +654,17 @@ EOF } configure_searx() { - rst_title "Configure searx" section - rst_para "Setup searx config located at $SEARX_SETTINGS_PATH" + rst_title "Configure SearXNG" section + rst_para "Setup SearXNG config located at $SEARX_SETTINGS_PATH" echo tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" -sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH" EOF } test_local_searx() { - rst_title "Testing searx instance localy" section + rst_title "Testing SearXNG instance localy" section echo if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then @@ -564,27 +686,27 @@ EOF } install_searx_uwsgi() { - rst_title "Install searx's uWSGI app (searx.ini)" section + rst_title "Install SearXNG's uWSGI app (searx.ini)" section echo install_uwsgi uWSGI_install_app "$SEARX_UWSGI_APP" } remove_searx_uwsgi() { - rst_title "Remove searx's uWSGI app (searx.ini)" section + rst_title "Remove SearXNG's uWSGI app (searx.ini)" section echo uWSGI_remove_app "$SEARX_UWSGI_APP" } activate_service() { - rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "Activate SearXNG (service)" section echo uWSGI_enable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" } deactivate_service() { - rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "De-Activate SearXNG (service)" section echo uWSGI_disable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" @@ -609,7 +731,7 @@ EOF } enable_debug() { - warn_msg "Do not enable debug in production enviroments!!" + warn_msg "Do not enable debug in production environments!!" info_msg "try to enable debug mode ..." tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -680,14 +802,11 @@ inspect_service() { rst_title "service status & log" cat <<EOF -sourced ${DOT_CONFIG#"$REPO_ROOT/"} : - - PUBLIC_URL : ${PUBLIC_URL} - SEARX_URL_PATH : ${SEARX_URL_PATH} - SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} - SEARX_INTERNAL_HTTP : ${SEARX_INTERNAL_HTTP} - +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} EOF + install_log_searx_instance if service_account_is_available "$SERVICE_USER"; then info_msg "Service account $SERVICE_USER exists." @@ -702,9 +821,9 @@ EOF fi if clone_is_available; then - info_msg "~$SERVICE_USER: Searx software is installed." + info_msg "~$SERVICE_USER: SearXNG software is installed." else - err_msg "~$SERVICE_USER: Missing searx software!" + err_msg "~$SERVICE_USER: Missing SearXNG software!" fi if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then @@ -737,7 +856,7 @@ EOF fi local _debug_on - if ask_yn "Enable searx debug mode?"; then + if ask_yn "Enable SearXNG debug mode?"; then enable_debug _debug_on=1 fi @@ -779,7 +898,7 @@ install_apache_site() { rst_title "Install Apache site $APACHE_SEARX_SITE" rst_para "\ -This installs the searx uwsgi app as apache site. If your server is public to +This installs the SearXNG uwsgi app as apache site. If your server is public to the internet, you should instead use a reverse proxy (filtron) to block excessively bot queries." @@ -793,7 +912,7 @@ excessively bot queries." apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" - rst_title "Install searx's uWSGI app (searx.ini)" section + rst_title "Install SearXNG's uWSGI app (searx.ini)" section echo uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP" @@ -817,7 +936,7 @@ This removes apache site ${APACHE_SEARX_SITE}." apache_remove_site "${APACHE_SEARX_SITE}" - rst_title "Remove searx's uWSGI app (searx.ini)" section + rst_title "Remove SearXNG's uWSGI app (searx.ini)" section echo uWSGI_remove_app "$SEARX_UWSGI_APP" } diff --git a/utils/templates/etc/searx/use_default_settings.yml b/utils/templates/etc/searx/settings.yml index 1e7530cdc..5320b0b2c 100644 --- a/utils/templates/etc/searx/use_default_settings.yml +++ b/utils/templates/etc/searx/settings.yml @@ -1,9 +1,14 @@ +# SearXNG settings, before editing this file read: +# +# https://searxng.github.io/searxng/admin/engines/settings.html + use_default_settings: true general: - debug: false # Debug mode, only for development - instance_name: "searxng" # displayed name - contact_url: false # mailto:contact@example.com + # Debug mode, only for development + debug: false + # change displayed name + # instance_name: "SearXNG" search: # Filter results. 0: None, 1: Moderate, 2: Strict @@ -11,24 +16,37 @@ search: # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", # "startpage", "swisscows", "qwant", "wikipedia" - leave blank to turn it off # by default. - autocomplete: "" + autocomplete: '' # Default search language - leave blank to detect from browser information or # use codes from 'languages.py' - default_lang: "" + default_lang: '' # remove format to deny access, use lower case. formats: [html, csv, json, rss] server: - port: 8888 - # address to listen on - bind_address: "127.0.0.1" secret_key: "ultrasecretkey" # change this! - # Set custom base_url. Possible values: - # false or "https://your.custom.host/location/" - # base_url: https://example.org/searx # Proxying image results through searx image_proxy: false # result_proxy: # url: http://127.0.0.1:3000/ # key: !!binary "your_morty_proxy_key" + +# plugins: +# - only_show_green_results + +# engines: +# +# - name: duckduckgo +# disabled: false +# +# - name: fdroid +# disabled: false +# +# - name: apk mirror +# disabled: false +# +# - name: mediathekviewweb +# engine: mediathekviewweb +# shortcut: mvw +# categories: general |