diff options
| author | Markus Heiser <markus.heiser@darmarIT.de> | 2022-07-30 13:45:12 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-07-30 13:45:12 +0200 |
| commit | 645c2a2ca1cf40ab9429366ac70c69d1ac8fd004 (patch) | |
| tree | 943270e4998815d360713e21615082293300c877 /utils | |
| parent | 0cf5310933ee4f6bff4a941f7296686bd671671f (diff) | |
| parent | e5323b8aa285d61bf011bbe7ca8f02818f19ed79 (diff) | |
Merge pull request #1332 from return42/searxng-install
Upgrade installation scripts and documentation
Diffstat (limited to 'utils')
24 files changed, 1301 insertions, 2638 deletions
diff --git a/utils/filtron.sh b/utils/filtron.sh index 2536214e4..ab207bbd8 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -4,56 +4,19 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_go.sh -source "${REPO_ROOT}/utils/lib_go.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" - -FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ -| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" -[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ - FILTRON_ETC="/etc/filtron" -FILTRON_RULES="$FILTRON_ETC/rules.json" -FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/filtron/rules.json}" - -FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" -FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" - -# The filtron target is the SearXNG installation, listenning on server.port at -# server.bind_address. The default of FILTRON_TARGET is taken from the YAML -# configuration, do not change this value without reinstalling the entire -# SearXNG suite including filtron & morty. -FILTRON_TARGET="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" - -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" -GO_ENV="${SERVICE_HOME}/.go_env" -GO_VERSION="go1.17.2" - -APACHE_FILTRON_SITE="searxng.conf" -NGINX_FILTRON_SITE="searxng.conf" - -# shellcheck disable=SC2034 -CONFIG_FILES=( - "${FILTRON_RULES}" - "${SERVICE_SYSTEMD_UNIT}" -) +APACHE_FILTRON_SITE="searx.conf" +NGINX_FILTRON_SITE="searx.conf" # ---------------------------------------------------------------------------- usage() { @@ -62,248 +25,45 @@ usage() { # shellcheck disable=SC1117 cat <<EOF usage:: - $(basename "$0") shell - $(basename "$0") install [all|user|rules] - $(basename "$0") reinstall all - $(basename "$0") update [filtron] - $(basename "$0") remove [all] - $(basename "$0") activate [service] - $(basename "$0") deactivate [service] - $(basename "$0") inspect [service] - $(basename "$0") option [debug-on|debug-off] - $(basename "$0") apache [install|remove] - $(basename "$0") nginx [install|remove] + $(basename "$0") remove all] + $(basename "$0") apache remove + $(basename "$0") nginx remove -shell - start interactive shell from user ${SERVICE_USER} -install / remove - :all: complete setup of filtron service - :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) - :rules: reinstall filtron rules $FILTRON_RULES -install - :check: check the filtron installation -reinstall: - :all: runs 'install/remove all' -update filtron - Update filtron installation ($SERVICE_HOME) -activate service - activate and start service daemon (systemd unit) -deactivate service - stop and deactivate service daemon (systemd unit) -inspect service - show service status and log -option - set one of the available options -apache (${PUBLIC_URL}) - :install: apache site with a reverse proxy (ProxyPass) - :remove: apache site ${APACHE_FILTRON_SITE} -nginx (${PUBLIC_URL}) - :install: nginx site with a reverse proxy (ProxyPass) - :remove: nginx site ${NGINX_FILTRON_SITE} -filtron rules: ${FILTRON_RULES_TEMPLATE} ----- sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} - FILTRON_URL_PATH : ${FILTRON_URL_PATH} +remove all : drop all components of the filtron service +apache remove : drop apache site ${APACHE_FILTRON_SITE} +nginx remove : drop nginx site ${NGINX_FILTRON_SITE} EOF - install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } main() { - required_commands \ - sudo install git wget curl \ - || exit - local _usage="unknown or missing $1 command $2" case $1 in - --getenv) var="$2"; echo "${!var}"; exit 0;; -h|--help) usage; exit 0;; - - shell) - sudo_or_exit - interactive_shell "${SERVICE_USER}" - ;; - inspect) - case $2 in - service) - sudo_or_exit - inspect_service - ;; - *) usage "$_usage"; exit 42;; - esac ;; - reinstall) - rst_title "re-install $SERVICE_NAME" part - sudo_or_exit - case $2 in - all) - remove_all - install_all - ;; - *) usage "$_usage"; exit 42;; - esac ;; - install) - rst_title "$SERVICE_NAME" part - sudo_or_exit - case $2 in - check) - rst_title "Check filtron installation" part - install_check - ;; - all) install_all ;; - user) assert_user ;; - rules) - install_rules - systemd_restart_service "${SERVICE_NAME}" - ;; - *) usage "$_usage"; exit 42;; - esac ;; - update) - sudo_or_exit - case $2 in - filtron) update_filtron ;; - *) usage "$_usage"; exit 42;; - esac ;; remove) sudo_or_exit case $2 in all) remove_all;; - user) drop_service_account "${SERVICE_USER}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - activate) - sudo_or_exit - case $2 in - service) systemd_activate_service "${SERVICE_NAME}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - deactivate) - sudo_or_exit - case $2 in - service) systemd_deactivate_service "${SERVICE_NAME}" ;; *) usage "$_usage"; exit 42;; esac ;; apache) sudo_or_exit case $2 in - install) install_apache_site ;; remove) remove_apache_site ;; *) usage "$_usage"; exit 42;; esac ;; nginx) sudo_or_exit case $2 in - install) install_nginx_site ;; remove) remove_nginx_site ;; *) usage "$_usage"; exit 42;; esac ;; - option) - sudo_or_exit - case $2 in - debug-on) echo; enable_debug ;; - debug-off) echo; disable_debug ;; - *) usage "$_usage"; exit 42;; - esac ;; - doc) rst-doc ;; *) usage "unknown or missing command $1"; exit 42;; esac } -install_all() { - rst_title "Install $SERVICE_NAME (service)" - assert_user - wait_key - go.golang "${GO_VERSION}" "${SERVICE_USER}" - wait_key - install_filtron - install_rules - wait_key - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - wait_key - echo - if ! service_is_available "http://${FILTRON_LISTEN}" ; then - err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}" - fi - if apache_is_installed; then - info_msg "Apache is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_apache_site - fi - elif nginx_is_installed; then - info_msg "nginx is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_nginx_site - fi - fi - if ask_yn "Do you want to inspect the installation?" Ny; then - inspect_service - fi - -} - -install_check() { - - if service_account_is_available "$SERVICE_USER"; then - info_msg "service account $SERVICE_USER available." - else - err_msg "service account $SERVICE_USER not available!" - fi - if go_is_available "$SERVICE_USER"; then - info_msg "~$SERVICE_USER: go is installed" - else - err_msg "~$SERVICE_USER: go is not installed" - fi - if filtron_is_installed; then - info_msg "~$SERVICE_USER: filtron app is installed" - else - err_msg "~$SERVICE_USER: filtron app is not installed!" - fi - - if ! service_is_available "http://${FILTRON_API}"; then - err_msg "API not available at: http://${FILTRON_API}" - fi - - if ! service_is_available "http://${FILTRON_LISTEN}" ; then - err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}" - fi - - if service_is_available "http://${FILTRON_TARGET}" ; then - info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" - fi - - if ! service_is_available "${PUBLIC_URL}"; then - warn_msg "Public service at ${PUBLIC_URL} is not available!" - if ! in_container; then - warn_msg "Check if public name is correct and routed or use the public IP from above." - fi - fi - - if [[ "${GO_VERSION}" > "$(go_version)" ]]; then - warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - else - info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)" - fi - - if [ -f "${APACHE_SITES_AVAILABLE}/searx.conf" ]; then - warn_msg "old searx.conf apache site exists" - fi - - if [ -f "${NGINX_APPS_AVAILABLE}/searx.conf" ]; then - warn_msg "old searx.conf nginx site exists" - fi - -} - -go_version(){ - go.version "${SERVICE_USER}" -} - remove_all() { rst_title "De-Install $SERVICE_NAME (service)" @@ -321,219 +81,6 @@ installations that were installed with this script." fi } -assert_user() { - rst_title "user $SERVICE_USER" section - echo - tee_stderr 1 <<EOF | bash | prefix_stdout -useradd --shell /bin/bash --system \ - --home-dir "$SERVICE_HOME" \ - --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER -mkdir "$SERVICE_HOME" -chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" -groups $SERVICE_USER -EOF - SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" - export SERVICE_HOME - echo "export SERVICE_HOME=$SERVICE_HOME" - - tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" -touch "$GO_ENV" -grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile -EOF -} - -filtron_is_installed() { - [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] -} - -install_filtron() { - rst_title "Install filtron in user's ~/go-apps" section - echo - go.install github.com/searxng/filtron@latest "${SERVICE_USER}" -} - -update_filtron() { - rst_title "Update filtron" section - echo - go.install github.com/searxng/filtron@latest "${SERVICE_USER}" -} - -install_rules() { - rst_title "Install filtron rules" - echo - if [[ ! -f "${FILTRON_RULES}" ]]; then - info_msg "install rules ${FILTRON_RULES_TEMPLATE}" - info_msg " --> ${FILTRON_RULES}" - mkdir -p "$(dirname "${FILTRON_RULES}")" - cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - return - fi - - if cmp --silent "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"; then - info_msg "${FILTRON_RULES} is up to date with" - info_msg "${FILTRON_RULES_TEMPLATE}" - return - fi - - rst_para "Diff between origin's rules file (+) and current (-):" - echo "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}" - $DIFF_CMD "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}" - - local action - choose_one action "What should happen to the rules file? " \ - "keep configuration unchanged" \ - "use origin rules" \ - "start interactive shell" - case $action in - "keep configuration unchanged") - info_msg "leave rules file unchanged" - ;; - "use origin rules") - backup_file "${FILTRON_RULES}" - info_msg "install origin rules" - cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - ;; - "start interactive shell") - backup_file "${FILTRON_RULES}" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -i - rst_para 'Diff between new rules file (-) and current (+):' - echo - $DIFF_CMD "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - wait_key - ;; - esac -} - -inspect_service() { - - rst_title "service status & log" - - cat <<EOF - -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} - FILTRON_URL_PATH : ${FILTRON_URL_PATH} -EOF - install_log_searx_instance - - install_check - - if in_container; then - lxc_suite_info - else - info_msg "public URL --> ${PUBLIC_URL}" - info_msg "internal URL --> http://${FILTRON_LISTEN}" - fi - - - local _debug_on - if ask_yn "Enable filtron debug mode?"; then - enable_debug - _debug_on=1 - fi - echo - systemctl --no-pager -l status "${SERVICE_NAME}" - echo - - info_msg "public URL --> ${PUBLIC_URL}" - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - while true; do - trap break 2 - journalctl -f -u "${SERVICE_NAME}" - done - - if [[ $_debug_on == 1 ]]; then - disable_debug - fi - return 0 -} - - -enable_debug() { - info_msg "try to enable debug mode ..." - python <<EOF -import sys, json - -debug = { - u'name': u'debug request' - , u'filters': [] - , u'interval': 0 - , u'limit': 0 - , u'actions': [{u'name': u'log'}] -} - -with open('$FILTRON_RULES') as rules: - j = json.load(rules) - -pos = None -for i in range(len(j)): - if j[i].get('name') == 'debug request': - pos = i - break -if pos is not None: - j[pos] = debug -else: - j.append(debug) -with open('$FILTRON_RULES', 'w') as rules: - json.dump(j, rules, indent=2, sort_keys=True) - -EOF - systemctl restart "${SERVICE_NAME}.service" -} - -disable_debug() { - info_msg "try to disable debug mode ..." - python <<EOF -import sys, json -with open('$FILTRON_RULES') as rules: - j = json.load(rules) - -pos = None -for i in range(len(j)): - if j[i].get('name') == 'debug request': - pos = i - break -if pos is not None: - del j[pos] - with open('$FILTRON_RULES', 'w') as rules: - json.dump(j, rules, indent=2, sort_keys=True) -EOF - systemctl restart "${SERVICE_NAME}.service" -} - -install_apache_site() { - - rst_title "Install Apache site $APACHE_FILTRON_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})" - - ! apache_is_installed && info_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL}"; then - err_msg "Public service at ${PUBLIC_URL} is not available!" - fi -} - remove_apache_site() { rst_title "Remove Apache site $APACHE_FILTRON_SITE" @@ -551,35 +98,6 @@ This removes apache site ${APACHE_FILTRON_SITE}." } -install_nginx_site() { - - rst_title "Install nginx site $NGINX_FILTRON_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})" - - ! nginx_is_installed && info_msg "nginx is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_nginx - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - # shellcheck disable=SC2034 - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - # shellcheck disable=SC2034 - SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH) - nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL}"; then - err_msg "Public service at ${PUBLIC_URL} is not available!" - fi -} - remove_nginx_site() { rst_title "Remove nginx site $NGINX_FILTRON_SITE" @@ -593,35 +111,8 @@ This removes nginx site ${NGINX_FILTRON_SITE}." return fi - nginx_remove_site "$FILTRON_FILTRON_SITE" - -} - - -rst-doc() { - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\"" - - echo -e "\n.. START install systemd unit" - cat <<EOF -.. tabs:: - - .. group-tab:: systemd - - .. code:: bash - -EOF - eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " - echo -e "\n.. END install systemd unit" + nginx_remove_app "$FILTRON_FILTRON_SITE" - # for DIST_NAME in ubuntu-20.04 arch fedora centos; do - # ( - # DIST_ID=${DIST_NAME%-*} - # DIST_VERS=${DIST_NAME#*-} - # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= - # # ... - # ) - # done } # ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh index 257c15024..32db47524 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -195,7 +195,7 @@ wait_key(){ [[ -n $_t ]] && _t="-t $_t" printf "$msg" # shellcheck disable=SC2086 - read -r -s -n1 $_t + read -r -s -n1 $_t || true echo clean_stdin } @@ -1027,7 +1027,7 @@ nginx_include_apps_enabled() { local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;" local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;" - info_msg "checking existence: '${include_directive}' in file ${server_conf}" + info_msg "checking existence: '${include_directive}' in file ${server_conf}" if grep "${include_directive_re}" "${server_conf}"; then info_msg "OK, already exists." return @@ -1117,7 +1117,7 @@ apache_distro_setup() { APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" APACHE_MODULES="modules" - APACHE_PACKAGES="httpd" + APACHE_PACKAGES="httpd mod_ssl" ;; *) err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented" @@ -1249,8 +1249,6 @@ apache_dissable_site() { # ----- uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" -uWSGI_USER= -uWSGI_GROUP= # How distros manage uWSGI apps is very different. From uWSGI POV read: # - https://uwsgi-docs.readthedocs.io/en/latest/Management.html @@ -1276,13 +1274,14 @@ uWSGI_distro_setup() { ;; fedora-*|centos-7) # systemd --> /usr/lib/systemd/system/uwsgi.service - # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see - # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # There are some quirks abbout additional POSIX groups in uWSGI + # 2.0.x, read at least: https://github.com/unbit/uwsgi/issues/2099 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d" uWSGI_PACKAGES="uwsgi" - uWSGI_USER="uwsgi" - uWSGI_GROUP="uwsgi" ;; *) err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" @@ -1344,30 +1343,6 @@ uWSGI_restart() { esac } -uWSGI_prepare_app() { - - # usage: uWSGI_prepare_app <myapp.ini> - - [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>" - - local APP="${1%.*}" - - case $DIST_ID-$DIST_VERS in - fedora-*|centos-7) - # in emperor mode, the uwsgi user is the owner of the sockets - info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" - ;; - *) - info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" - ;; - esac -} - - uWSGI_app_available() { # usage: uWSGI_app_available <myapp.ini> local CONF="$1" @@ -1378,7 +1353,7 @@ uWSGI_app_available() { uWSGI_install_app() { - # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> [{owner} [{group} [{chmod}]]] # # <template option>: see install_template @@ -1390,11 +1365,10 @@ uWSGI_install_app() { *) pos_args+=("$i");; esac done - uWSGI_prepare_app "${pos_args[1]}" mkdir -p "${uWSGI_APPS_AVAILABLE}" install_template "${template_opts[@]}" \ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ - root root 644 + "${pos_args[2]:-root}" "${pos_args[3]:-root}" "${pos_args[4]:-644}" uWSGI_enable_app "${pos_args[1]}" uWSGI_restart "${pos_args[1]}" info_msg "uWSGI app: ${pos_args[1]} is installed" @@ -1468,7 +1442,6 @@ uWSGI_enable_app() { mkdir -p "${uWSGI_APPS_ENABLED}" rm -f "${uWSGI_APPS_ENABLED}/${CONF}" ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" - chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}" info_msg "enabled uWSGI app: ${CONF}" ;; *) diff --git a/utils/lib_install.sh b/utils/lib_install.sh deleted file mode 100755 index 5d84c066f..000000000 --- a/utils/lib_install.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-or-later - -# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 -# shellcheck source=utils/lib.sh -. /dev/null - -# Initialize installation procedures: -# -# - Modified source_dot_config function that -# - loads .config.sh from an existing installation (at SEARX_SRC). -# - initialize **SEARX_SRC_INIT_FILES** -# - functions like: -# - install_log_searx_instance() -# - install_searx_get_state() -# -# usage: -# source lib_install.sh -# -# **Installation scripts** -# -# The utils/lib_install.sh is sourced by the installations scripts: -# -# - utils/searx.sh -# - utils/morty.sh -# - utils/filtron.sh -# -# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function -# loads this configuration (instead of './.config.sh'). - -# **SEARX_SRC_INIT_FILES** -# -# Array of file names to sync into a installation at $SEARX_SRC. The file names -# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). -# Most often theses are files like: -# - .config.sh -# - searx/settings.yml -# - utils/brand.env -# - ... - - -SEARX_SRC_INIT_FILES=() - -eval orig_"$(declare -f source_dot_config)" - -source_dot_config() { - - # Modified source_dot_config function that - # - loads .config.sh from an existing installation (at SEARX_SRC). - # - initialize SEARX_SRC_INIT_FILES - - if [ -z "$eval_SEARX_SRC" ]; then - export eval_SEARX_SRC='true' - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) - SEARXNG_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SETTINGS_PATH) - if [ ! -r "${SEARX_SRC}" ]; then - info_msg "not yet cloned: ${SEARX_SRC}" - orig_source_dot_config - return 0 - fi - info_msg "using instance at: ${SEARX_SRC}" - - # set and log DOT_CONFIG - if [ -r "${SEARX_SRC}/.config.sh" ]; then - info_msg "switching to ${SEARX_SRC}/.config.sh" - DOT_CONFIG="${SEARX_SRC}/.config.sh" - else - info_msg "using local config: ${DOT_CONFIG}" - fi - init_SEARX_SRC_INIT_FILES - fi -} - -init_SEARX_SRC_INIT_FILES(){ - # init environment SEARX_SRC_INIT_FILES - - # Monitor modified files in the working-tree from the local repository, only - # if the local file differs to the corresponding file in the instance. Most - # often theses are files like: - # - # - .config.sh - # - searx/settings.yml - # - utils/brand.env - # - ... - - # keep list empty if there is no installation - SEARX_SRC_INIT_FILES=() - if [ ! -r "$SEARX_SRC" ]; then - return 0 - fi - - local fname - local msg="" - local _prefix="" - if [[ -n ${SUDO_USER} ]]; then - _prefix="sudo -u ${SUDO_USER}" - fi - - # Monitor local modified files from the repository, only if the local file - # differs to the corresponding file in the instance - - while IFS= read -r fname; do - if [ -z "$fname" ]; then - continue - fi - if [ -r "${SEARX_SRC}/${fname}" ]; then - # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" - if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then - SEARX_SRC_INIT_FILES+=("${fname}") - info_msg "local clone (workingtree), modified file: ./$fname" - msg="to update use: sudo -H ./utils/searx.sh install init-src" - fi - fi - done <<< "$($_prefix git diff --name-only)" - [ -n "$msg" ] && info_msg "$msg" -} - -install_log_searx_instance() { - - echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" - echo -e " SEARXNG_SETTINGS_PATH : ${_BBlue}${SEARXNG_SETTINGS_PATH}${_creset}" - echo -e " SEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}" - echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" - echo -e " SEARXNG_URL : ${_BBlue}${SEARXNG_URL:-none}${_creset}" - - if in_container; then - # SearXNG is listening on 127.0.0.1 and not available from outside container - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - echo -e "---- container setup" - echo -e " ${_BBlack}HINT:${_creset} SearXNG only listen on loopback device" \ - "${_BBlack}inside${_creset} the container." - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container (IPv6): [${ip#*|}]" - else - # IPv4: - echo " container (IPv4): ${ip#*|}" - fi - done - fi -} - -install_searx_get_state(){ - - # usage: install_searx_get_state - # - # Prompts a string indicating the status of the installation procedure - # - # missing-searx-clone: - # There is no clone at ${SEARX_SRC} - # missing-searx-pyenv: - # There is no pyenv in ${SEARX_PYENV} - # installer-modified: - # There are files modified locally in the installer (clone), - # see ${SEARX_SRC_INIT_FILES} description. - # python-installed: - # Scripts can be executed in instance's environment - # - user: ${SERVICE_USER} - # - pyenv: ${SEARX_PYENV} - - if [ -f /etc/searx/settings.yml ]; then - err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/" - fi - - if ! [ -r "${SEARX_SRC}" ]; then - echo "missing-searx-clone" - return - fi - if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then - echo "missing-searx-pyenv" - return - fi - if ! [ -r "${SEARXNG_SETTINGS_PATH}" ]; then - echo "missing-settings" - return - fi - if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then - echo "installer-modified" - return - fi - echo "python-installed" -} - -# Initialization of the installation procedure -# -------------------------------------------- - -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" - -# SEARXNG_URL aka PUBLIC_URL: the public URL of the instance (e.g. -# "https://example.org/searx"). The value is taken from environment $SEARXNG_URL -# in ./utils/brand.env. This variable is a empty string if server.base_url in -# the settings.yml is set to 'false'. - -SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)}" -if in_container; then - # hint: Linux containers do not have DNS entries, lets use IPs - SEARXNG_URL="http://$(primary_ip)" -fi -PUBLIC_URL="${SEARXNG_URL}" - -source_dot_config - -# shellcheck source=utils/lxc-searx.env -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env diff --git a/utils/lib_redis.sh b/utils/lib_redis.sh index 5eaa1770f..ba1435a86 100755 --- a/utils/lib_redis.sh +++ b/utils/lib_redis.sh @@ -42,6 +42,8 @@ REDIS_GIT_URL="https://github.com/redis/redis.git" REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}" REDIS_USER="searxng-redis" +REDIS_GROUP="searxng-redis" + REDIS_HOME="/usr/local/${REDIS_USER}" REDIS_HOME_BIN="${REDIS_HOME}/.local/bin" REDIS_ENV="${REDIS_HOME}/.redis_env" @@ -113,7 +115,7 @@ redis.devpkg() { case ${DIST_ID} in ubuntu|debian) - pkg_install git build-essential + pkg_install git build-essential gawk ;; arch) pkg_install git base-devel @@ -139,15 +141,20 @@ redis.build() { rst_title "get redis sources" section redis.src "${CACHE}/redis" - if ! required_commands gcc nm make gawk; then - sudo -H "$0" redis.devpkg + if ! required_commands gcc nm make gawk ; then + info_msg "install development tools to get missing command(s) .." + if [[ -n ${SUDO_USER} ]]; then + sudo -H "$0" redis.devpkg + else + redis.devpkg + fi fi rst_title "compile redis sources" section pushd "${CACHE}/redis" &>/dev/null - if ask_yn "Do you run 'make distclean' first'?" Ny; then + if ask_yn "Do you run 'make distclean' first'?" Yn; then $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout fi @@ -158,7 +165,7 @@ redis.build() { popd &>/dev/null - tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout + tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout mkdir -p "$(redis._get_dist)" cd "${CACHE}/redis/src" cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)" @@ -233,7 +240,7 @@ useradd --shell /bin/bash --system \ --home-dir "${REDIS_HOME}" \ --comment 'user that runs a redis instance' "${REDIS_USER}" mkdir -p "${REDIS_HOME}" -chown -R "${REDIS_USER}:${REDIS_USER}" "${REDIS_HOME}" +chown -R "${REDIS_USER}:${REDIS_GROUP}" "${REDIS_HOME}" groups "${REDIS_USER}" EOF @@ -248,7 +255,7 @@ EOF redis.userdel() { sudo_or_exit drop_service_account "${REDIS_USER}" - groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true + groupdel "${REDIS_GROUP}" 2>&1 | prefix_stdout || true } redis.addgrp() { @@ -256,7 +263,7 @@ redis.addgrp() { # usage: redis.addgrp <user> [[ -z $1 ]] && die_caller 42 "missing argument <user>" - sudo -H gpasswd -a "$1" "${REDIS_USER}" + sudo -H gpasswd -a "$1" "${REDIS_GROUP}" } redis.rmgrp() { @@ -264,7 +271,7 @@ redis.rmgrp() { # usage: redis.rmgrp <user> [[ -z $1 ]] && die_caller 42 "missing argument <user>" - sudo -H gpasswd -d "$1" "${REDIS_USER}" + sudo -H gpasswd -d "$1" "${REDIS_GROUP}" } @@ -278,7 +285,7 @@ redis._install_bin() { ( set -e for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do - install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \ + install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \ "${src}/${redis_exe}" "${REDIS_HOME_BIN}" done diff --git a/utils/lxc-searx.env b/utils/lxc-searxng.env index 13b15522c..86279d4b3 100644 --- a/utils/lxc-searx.env +++ b/utils/lxc-searxng.env @@ -4,24 +4,18 @@ # This file is a setup of a LXC suite. It is sourced from different context, do # not manipulate the environment directly, implement functions and manipulate -# environment only is subshells! +# environment only in subshells. -# ---------------------------------------------------------------------------- -# config -# ---------------------------------------------------------------------------- - -# shellcheck disable=SC2034 -LXC_SUITE_NAME="searx" lxc_set_suite_env() { + + export LXC_SUITE_NAME="searxng" + # name of https://images.linuxcontainers.org export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}" export LXC_SUITE=( - # to disable containers, comment out lines .. - # end of standard support see https://wiki.ubuntu.com/Releases - "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023 "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # April 2025 "$LINUXCONTAINERS_ORG_NAME:ubuntu/21.10" "ubu2110" # July 2027 @@ -30,49 +24,27 @@ lxc_set_suite_env() { # rolling releases see https://www.archlinux.org/releng/releases/ "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" - - # EOL 30 June 2024 - "$LINUXCONTAINERS_ORG_NAME:centos/7" "centos7" ) - - PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" - if in_container; then - # container hostnames do not have a DNS entry: use primary IP! - PUBLIC_URL="http://$(primary_ip)/searx" - - # make GUEST's services public to the HOST - FILTRON_API="0.0.0.0:4005" - FILTRON_LISTEN="0.0.0.0:4004" - MORTY_LISTEN="0.0.0.0:3000" - - # export LXC specific environment - export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN - fi } lxc_suite_install_info() { ( lxc_set_suite_env cat <<EOF -LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL} - suite includes searx, morty & filtron -suite images: -$(echo " ${LOCAL_IMAGES[*]}" | $FMT) -suite containers: -$(echo " ${CONTAINERS[*]}" | $FMT) +LXC suite: ${LXC_SUITE_NAME} + Suite includes installation of SearXNG + images: ${LOCAL_IMAGES[*]} + containers: ${CONTAINERS[*]} EOF ) - } +} lxc_suite_install() { ( lxc_set_suite_env FORCE_TIMEOUT=0 export FORCE_TIMEOUT - "${LXC_REPO_ROOT}/utils/searx.sh" install all - "${LXC_REPO_ROOT}/utils/morty.sh" install all - "${LXC_REPO_ROOT}/utils/filtron.sh" install all - + "${LXC_REPO_ROOT}/utils/searxng.sh" install all rst_title "suite installation finished ($(hostname))" part lxc_suite_info echo @@ -88,10 +60,9 @@ lxc_suite_info() { else # IPv4: # shellcheck disable=SC2034,SC2031 - info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL" - info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY" info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/" fi done + "${LXC_REPO_ROOT}/utils/searxng.sh" searxng.instance.env ) } diff --git a/utils/lxc.sh b/utils/lxc.sh index 9754b5d75..418aea938 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -4,12 +4,11 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -source_dot_config # shellcheck source=utils/brand.env source "${REPO_ROOT}/utils/brand.env" # load environment of the LXC suite -LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}" +LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}" source "$LXC_ENV" lxc_set_suite_env diff --git a/utils/morty.sh b/utils/morty.sh index c4e7bdf52..a85b04bb8 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -3,10 +3,6 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_go.sh -source "${REPO_ROOT}/utils/lib_go.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -16,24 +12,9 @@ MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" -# shellcheck disable=SC2034 -MORTY_TIMEOUT=5 - SERVICE_NAME="morty" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" -# shellcheck disable=SC2034 -SERVICE_ENV_DEBUG=false - -GO_ENV="${SERVICE_HOME}/.go_env" -GO_VERSION="go1.17.2" - -# shellcheck disable=SC2034 -CONFIG_FILES=() # Apache Settings @@ -47,267 +28,45 @@ usage() { # shellcheck disable=SC1117 cat <<EOF usage:: - $(basename "$0") shell - $(basename "$0") install [all|check|user] - $(basename "$0") reinstall all - $(basename "$0") update [morty] - $(basename "$0") remove [all] - $(basename "$0") activate [service] - $(basename "$0") deactivate [service] - $(basename "$0") inspect [service] - $(basename "$0") option [debug-on|debug-off|new-key] - $(basename "$0") apache [install|remove] - $(basename "$0") nginx [install|remove] - $(basename "$0") info [searx] + $(basename "$0") remove all + $(basename "$0") apache remove + $(basename "$0") nginx remove -shell - start interactive shell from user ${SERVICE_USER} -install / remove - :all: complete setup of morty service - :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) -install - :check: check the morty installation -reinstall: - :all: runs 'install/remove all' -update morty - Update morty installation ($SERVICE_HOME) -activate service - activate and start service daemon (systemd unit) -deactivate service - stop and deactivate service daemon (systemd unit) -inspect service - show service status and log -option - set one of the available options - :new-key: set new morty key -apache : ${PUBLIC_URL_MORTY} - :install: apache site with a reverse proxy (ProxyPass) - :remove: apache site ${APACHE_MORTY_SITE} -nginx (${PUBLIC_URL_MORTY}) - :install: nginx site with a reverse proxy (ProxyPass) - :remove: nginx site ${NGINX_MORTY_SITE} ----- -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} - MORTY_LISTEN: : ${MORTY_LISTEN} +remove all : drop all components of the morty service +apache remove : drop apache site ${APACHE_MORTY_SITE} +nginx remove : drop nginx site ${NGINX_MORTY_SITE} EOF - install_log_searx_instance - if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container URL (IPv6): http://[${ip#*|}]:3000/" - else - # IPv4: - echo " container URL (IPv4): http://${ip#*|}:3000/" - fi - done - fi - echo - info_searx - [[ -n ${1} ]] && err_msg "$1" } -info_searx() { - # shellcheck disable=SC1117 - cat <<EOF -To activate result and image proxy in SearXNG read: - https://docs.searxng.org/admin/morty.html -Check settings in file ${SEARXNG_SETTINGS_PATH} ... - result_proxy: - url : ${PUBLIC_URL_MORTY} - server: - image_proxy : True -EOF -} - main() { - required_commands \ - sudo install git wget curl \ - || exit - local _usage="ERROR: unknown or missing $1 command $2" case $1 in - --getenv) var="$2"; echo "${!var}"; exit 0;; -h|--help) usage; exit 0;; - - shell) - sudo_or_exit - interactive_shell "${SERVICE_USER}" - ;; - inspect) - case $2 in - service) - sudo_or_exit - inspect_service - ;; - *) usage "$_usage"; exit 42;; - esac ;; - reinstall) - rst_title "re-install $SERVICE_NAME" part - sudo_or_exit - case $2 in - all) - remove_all - install_all - ;; - *) usage "$_usage"; exit 42;; - esac ;; - install) - rst_title "$SERVICE_NAME" part - sudo_or_exit - case $2 in - all) install_all ;; - check) - rst_title "Check morty installation" part - install_check - ;; - user) assert_user ;; - *) usage "$_usage"; exit 42;; - esac ;; - update) - sudo_or_exit - case $2 in - morty) update_morty ;; - *) usage "$_usage"; exit 42;; - esac ;; remove) sudo_or_exit case $2 in all) remove_all;; - user) drop_service_account "${SERVICE_USER}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - activate) - sudo_or_exit - case $2 in - service) systemd_activate_service "${SERVICE_NAME}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - deactivate) - sudo_or_exit - case $2 in - service) systemd_deactivate_service "${SERVICE_NAME}" ;; *) usage "$_usage"; exit 42;; esac ;; apache) sudo_or_exit case $2 in - install) install_apache_site ;; remove) remove_apache_site ;; *) usage "$_usage"; exit 42;; esac ;; nginx) sudo_or_exit case $2 in - install) install_nginx_site ;; remove) remove_nginx_site ;; *) usage "$_usage"; exit 42;; esac ;; - info) - case $2 in - searx) info_searx ;; - *) usage "$_usage"; exit 42;; - esac ;; - option) - sudo_or_exit - case $2 in - new-key) set_new_key ;; - debug-on) enable_debug ;; - debug-off) disable_debug ;; - *) usage "$_usage"; exit 42;; - esac ;; - doc) rst-doc ;; *) usage "ERROR: unknown or missing command $1"; exit 42;; esac } -install_all() { - - MORTY_KEY="$(head -c 32 /dev/urandom | base64)" - - rst_title "Install $SERVICE_NAME (service)" - assert_user - wait_key - go.golang "${GO_VERSION}" "${SERVICE_USER}" - wait_key - install_morty - wait_key - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - wait_key - if ! service_is_available "http://${MORTY_LISTEN}" ; then - err_msg "Morty is not listening on: http://${MORTY_LISTEN}" - fi - if apache_is_installed; then - info_msg "Apache is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_apache_site - fi - elif nginx_is_installed; then - info_msg "nginx is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_nginx_site - fi - fi - info_searx - if ask_yn "Add image and result proxy to SearXNG settings.yml?" Yn; then - "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" - "${REPO_ROOT}/utils/searx.sh" option image-proxy-on - fi - - if ask_yn "Do you want to inspect the installation?" Ny; then - inspect_service - fi - -} - -install_check() { - - if service_account_is_available "$SERVICE_USER"; then - info_msg "service account $SERVICE_USER available." - else - err_msg "service account $SERVICE_USER not available!" - fi - if go_is_available "$SERVICE_USER"; then - info_msg "~$SERVICE_USER: go is installed" - else - err_msg "~$SERVICE_USER: go is not installed" - fi - if morty_is_installed; then - info_msg "~$SERVICE_USER: morty app is installed" - else - err_msg "~$SERVICE_USER: morty app is not installed!" - fi - - if ! service_is_available "http://${MORTY_LISTEN}" ; then - err_msg "Morty is not listening on: http://${MORTY_LISTEN}" - echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." - wait_key - fi - - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - if ! in_container; then - warn_msg "Check if public name is correct and routed or use the public IP from above." - fi - fi - - if [[ "${GO_VERSION}" > "$(go_version)" ]]; then - warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - else - info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)" - fi -} - -go_version(){ - go.version "${SERVICE_USER}" -} remove_all() { rst_title "De-Install $SERVICE_NAME (service)" @@ -321,152 +80,6 @@ installations that were installed with this script." fi } -assert_user() { - rst_title "user $SERVICE_USER" section - echo - tee_stderr 1 <<EOF | bash | prefix_stdout -useradd --shell /bin/bash --system \ - --home-dir "$SERVICE_HOME" \ - --comment 'Web content sanitizer proxy' $SERVICE_USER -mkdir "$SERVICE_HOME" -chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" -groups $SERVICE_USER -EOF - SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" - export SERVICE_HOME - echo "export SERVICE_HOME=$SERVICE_HOME" - - tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" -touch $GO_ENV -grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile -EOF -} - -morty_is_installed() { - [[ -f $SERVICE_HOME/go-apps/bin/morty ]] -} - -install_morty() { - rst_title "Install morty in user's ~/go-apps" section - echo - go.install github.com/asciimoo/morty@latest "${SERVICE_USER}" -} - -update_morty() { - rst_title "Update morty" section - echo - go.install github.com/asciimoo/morty@latest "${SERVICE_USER}" -} - -set_service_env_debug() { - - # usage: set_service_env_debug [false|true] - - # shellcheck disable=SC2034 - local SERVICE_ENV_DEBUG="${1:-false}" - if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - fi -} - -inspect_service() { - - rst_title "service status & log" - - cat <<EOF - -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} - MORTY_LISTEN: : ${MORTY_LISTEN} - -EOF - install_log_searx_instance - - install_check - - if in_container; then - lxc_suite_info - else - info_msg "public URL --> ${PUBLIC_URL_MORTY}" - info_msg "morty URL --> http://${MORTY_LISTEN}" - fi - - local _debug_on - if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then - enable_debug - _debug_on=1 - else - systemctl --no-pager -l status "${SERVICE_NAME}" - fi - echo - - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - while true; do - trap break 2 - journalctl -f -u "${SERVICE_NAME}" - done - - if [[ $_debug_on == 1 ]]; then - FORCE_SELECTION=Y disable_debug - fi - return 0 -} - -enable_debug() { - warn_msg "Do not enable debug in production environments!!" - info_msg "Enabling debug option needs to reinstall systemd service!" - set_service_env_debug true -} - -disable_debug() { - info_msg "Disabling debug option needs to reinstall systemd service!" - set_service_env_debug false -} - - -set_new_key() { - rst_title "Set morty key" - echo - - MORTY_KEY="$(head -c 32 /dev/urandom | base64)" - info_msg "morty key: '${MORTY_KEY}'" - - warn_msg "this will need to reinstall services .." - MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key - - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" - "${REPO_ROOT}/utils/searx.sh" option image-proxy-on -} - - -install_apache_site() { - - rst_title "Install Apache site $APACHE_MORTY_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - apache_install_site "${APACHE_MORTY_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - fi -} remove_apache_site() { @@ -484,35 +97,6 @@ This removes apache site ${APACHE_MORTY_SITE}." apache_remove_site "$APACHE_MORTY_SITE" } -install_nginx_site() { - - rst_title "Install nginx site $NGINX_MORTY_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})" - - ! nginx_is_installed && err_msg "nginx is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_nginx - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - # shellcheck disable=SC2034 - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - # shellcheck disable=SC2034 - SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH) - nginx_install_app "${NGINX_MORTY_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - fi -} - remove_nginx_site() { rst_title "Remove nginx site $NGINX_MORTY_SITE" @@ -526,37 +110,10 @@ This removes nginx site ${NGINX_MORTY_SITE}." return fi - nginx_remove_site "$NGINX_MORTY_SITE" - -} - -rst-doc() { - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\"" - - echo -e "\n.. START install systemd unit" - cat <<EOF -.. tabs:: - - .. group-tab:: systemd + nginx_remove_app "$NGINX_MORTY_SITE" - .. code:: bash - -EOF - eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " - echo -e "\n.. END install systemd unit" - - # for DIST_NAME in ubuntu-20.04 arch fedora centos; do - # ( - # DIST_ID=${DIST_NAME%-*} - # DIST_VERS=${DIST_NAME#*-} - # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= - # # ... - # ) - # done } - # ---------------------------------------------------------------------------- main "$@" # ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh index ce118ed77..2f5e98c8e 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -5,130 +5,14 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" - # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -SEARX_INTERNAL_HTTP="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" - -SEARXNG_URL_PATH="${SEARXNG_URL_PATH:-$(echo "${PUBLIC_URL}" \ -| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" -[[ "${SEARXNG_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARXNG_URL_PATH=/ - SERVICE_NAME="searx" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" - -GIT_BRANCH="${GIT_BRANCH:-master}" -SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" -SEARX_SRC="${SERVICE_HOME}/searx-src" -SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml" -SEARXNG_UWSGI_APP="searxng.ini" -# shellcheck disable=SC2034 -SEARX_UWSGI_SOCKET="/run/uwsgi/app/searxng/socket" - -# apt packages -SEARX_PACKAGES_debian="\ -python3-dev python3-babel python3-venv -uwsgi uwsgi-plugin-python3 -git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev -shellcheck" - -BUILD_PACKAGES_debian="\ -firefox graphviz imagemagick texlive-xetex librsvg2-bin -texlive-latex-recommended texlive-extra-utils fonts-dejavu -latexmk" - -# pacman packages -SEARX_PACKAGES_arch="\ -python python-pip python-lxml python-babel -uwsgi uwsgi-plugin-python -git base-devel libxml2 -shellcheck" - -BUILD_PACKAGES_arch="\ -firefox graphviz imagemagick texlive-bin extra/librsvg -texlive-core texlive-latexextra ttf-dejavu" - -# dnf packages -SEARX_PACKAGES_fedora="\ -python python-pip python-lxml python-babel python3-devel -uwsgi uwsgi-plugin-python3 -git @development-tools libxml2 openssl -ShellCheck" - -BUILD_PACKAGES_fedora="\ -firefox graphviz graphviz-gd ImageMagick librsvg2-tools -texlive-xetex-bin texlive-collection-fontsrecommended -texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts -dejavu-sans-mono-fonts" - -# yum packages -# -# hint: We do no longer support yum packages, it is to complex to maintain -# automate installation of packages like npm. In the firts step we ignore -# CentOS-7 as developer & build platform (the inital patch which brought -# CentOS-7 supports was not intended to be a developer platform). - -SEARX_PACKAGES_centos="\ -python36 python36-pip python36-lxml python-babel -uwsgi uwsgi-plugin-python3 -git @development-tools libxml2 -ShellCheck" - -BUILD_PACKAGES_centos="\ -firefox graphviz graphviz-gd ImageMagick librsvg2-tools -texlive-xetex-bin texlive-collection-fontsrecommended -texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts -dejavu-sans-mono-fonts" - -case $DIST_ID-$DIST_VERS in - ubuntu-16.04|ubuntu-18.04) - SEARX_PACKAGES="${SEARX_PACKAGES_debian}" - BUILD_PACKAGES="${BUILD_PACKAGES_debian}" - APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" - ;; - ubuntu-20.04) - # https://askubuntu.com/a/1224710 - SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3" - BUILD_PACKAGES="${BUILD_PACKAGES_debian}" - ;; - ubuntu-*|debian-*) - SEARX_PACKAGES="${SEARX_PACKAGES_debian}" - BUILD_PACKAGES="${BUILD_PACKAGES_debian}" - ;; - arch-*) - SEARX_PACKAGES="${SEARX_PACKAGES_arch}" - BUILD_PACKAGES="${BUILD_PACKAGES_arch}" - ;; - fedora-*) - SEARX_PACKAGES="${SEARX_PACKAGES_fedora}" - BUILD_PACKAGES="${BUILD_PACKAGES_fedora}" - ;; - centos-7) - SEARX_PACKAGES="${SEARX_PACKAGES_centos}" - BUILD_PACKAGES="${BUILD_PACKAGES_centos}" - ;; -esac - -# Apache Settings -APACHE_SEARX_SITE="searxng.conf" - -# shellcheck disable=SC2034 -CONFIG_FILES=( - "${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}" -) - -# shellcheck disable=SC2034 -CONFIG_BACKUP_ENCRYPTED=( - "${SEARXNG_SETTINGS_PATH}" -) +SEARXNG_SETTINGS_PATH="/etc/searx/settings.yml" +SEARXNG_UWSGI_APP="searx.ini" # ---------------------------------------------------------------------------- usage() { @@ -137,286 +21,30 @@ usage() { # shellcheck disable=SC1117 cat <<EOF usage:: - $(basename "$0") shell - $(basename "$0") install [all|check|init-src|dot-config|user|searx-src|pyenv|uwsgi|packages|settings|buildhost] - $(basename "$0") reinstall all - $(basename "$0") update [searx] - $(basename "$0") remove [all|user|pyenv|searx-src] - $(basename "$0") activate [service] - $(basename "$0") deactivate [service] - $(basename "$0") inspect [service|settings <key>] - $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>] - $(basename "$0") apache [install|remove] + $(basename "$0") remove all -shell - start interactive shell from user ${SERVICE_USER} -install / remove - :all: complete (de-) installation of SearXNG service - :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) - :dot-config: copy ./config.sh to ${SEARX_SRC} - :searx-src: clone $GIT_URL - :init-src: copy files (SEARX_SRC_INIT_FILES) to ${SEARX_SRC} - :pyenv: create/remove virtualenv (python) in $SEARX_PYENV - :uwsgi: install SearXNG uWSGI application - :settings: reinstall settings from ${SEARXNG_SETTINGS_PATH} - :packages: install needed packages from OS package manager - :buildhost: install packages from OS package manager needed by buildhosts -install - :check: check the SearXNG installation -reinstall: - :all: runs 'install/remove all' -update searx - Update SearXNG installation ($SERVICE_HOME) -activate service - activate and start service daemon (systemd unit) -deactivate service - stop and deactivate service daemon (systemd unit) -inspect - :service: run some small tests and inspect service's status and log - :settings: inspect YAML setting <key> from SearXNG instance (${SEARX_SRC}) -option - set one of the available options -apache - :install: apache site with the SearXNG uwsgi app - :remove: apache site ${APACHE_FILTRON_SITE} ----- sourced ${DOT_CONFIG} - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} +remove all: complete uninstall of SearXNG service EOF - install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } main() { - required_commands \ - sudo systemctl install git wget curl \ - || exit local _usage="unknown or missing $1 command $2" case $1 in - --getenv) var="$2"; echo "${!var}"; exit 0;; - -h|--help) usage; exit 0;; - shell) - sudo_or_exit - interactive_shell "${SERVICE_USER}" - ;; - inspect) - case $2 in - service) - sudo_or_exit - inspect_service - ;; - settings) - prompt_installation_setting "$3" - dump_return $? - ;; - *) usage "$_usage"; exit 42;; - esac ;; - reinstall) - rst_title "re-install $SERVICE_NAME" part - sudo_or_exit - case $2 in - all) - remove_all - install_all - ;; - *) usage "$_usage"; exit 42;; - esac ;; - install) - sudo_or_exit - case $2 in - all) - rst_title "SearXNG (install)" part - install_all - ;; - check) - rst_title "SearXNG (check installation)" part - verify_continue_install - install_check - ;; - user) - rst_title "SearXNG (install user)" - verify_continue_install - assert_user - ;; - pyenv) - rst_title "SearXNG (install pyenv)" - verify_continue_install - create_pyenv - ;; - searx-src) - rst_title "SearXNG (install searx-src)" - verify_continue_install - assert_user - clone_searx - install_DOT_CONFIG - init_SEARX_SRC - ;; - init-src) - init_SEARX_SRC - ;; - dot-config) - install_DOT_CONFIG - ;; - settings) - install_settings - ;; - uwsgi) - rst_title "SearXNG (install uwsgi)" - verify_continue_install - install_searx_uwsgi - if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" - fi - ;; - packages) - rst_title "SearXNG (install packages)" - pkg_install "$SEARX_PACKAGES" - ;; - buildhost) - rst_title "SearXNG (install buildhost)" - pkg_install "$SEARX_PACKAGES" - pkg_install "$BUILD_PACKAGES" - ;; - *) usage "$_usage"; exit 42;; - esac ;; - update) - sudo_or_exit - case $2 in - searx) update_searx;; - *) usage "$_usage"; exit 42;; - esac ;; remove) rst_title "SearXNG (remove)" part sudo_or_exit case $2 in all) remove_all;; - user) drop_service_account "${SERVICE_USER}";; - pyenv) remove_pyenv ;; - searx-src) remove_searx ;; - *) usage "$_usage"; exit 42;; - esac ;; - activate) - sudo_or_exit - case $2 in - service) - activate_service ;; *) usage "$_usage"; exit 42;; esac ;; - deactivate) - sudo_or_exit - case $2 in - service) deactivate_service ;; - *) usage "$_usage"; exit 42;; - esac ;; - option) - sudo_or_exit - case $2 in - debug-on) echo; enable_debug ;; - debug-off) echo; disable_debug ;; - result-proxy) set_result_proxy "$3" "$4" ;; - image-proxy-on) enable_image_proxy ;; - image-proxy-off) disable_image_proxy ;; - *) usage "$_usage"; exit 42;; - esac ;; - apache) - sudo_or_exit - case $2 in - install) install_apache_site ;; - remove) remove_apache_site ;; - *) usage "$_usage"; exit 42;; - esac ;; - doc) rst-doc;; *) usage "unknown or missing command $1"; exit 42;; esac } -_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} " - -install_all() { - rst_title "Install SearXNG (service)" - verify_continue_install - pkg_install "$SEARX_PACKAGES" - wait_key - assert_user - wait_key - clone_searx - wait_key - install_DOT_CONFIG - wait_key - init_SEARX_SRC - wait_key - create_pyenv - wait_key - install_settings - wait_key - test_local_searx - wait_key - install_searx_uwsgi - if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" - fi - if ask_yn "Do you want to inspect the installation?" Ny; then - inspect_service - fi -} - -install_check() { - if service_account_is_available "$SERVICE_USER"; then - info_msg "Service account $SERVICE_USER exists." - else - err_msg "Service account $SERVICE_USER does not exists!" - fi - - if pyenv_is_available; then - info_msg "~$SERVICE_USER: python environment is available." - else - err_msg "~$SERVICE_USER: python environment is not available!" - fi - - if clone_is_available; then - info_msg "~$SERVICE_USER: SearXNG software is installed." - else - err_msg "~$SERVICE_USER: Missing SearXNG software!" - fi - - if uWSGI_app_enabled "$SEARXNG_UWSGI_APP"; then - info_msg "uWSGI app $SEARXNG_UWSGI_APP is enabled." - else - err_msg "uWSGI app $SEARXNG_UWSGI_APP not enabled!" - fi - - uWSGI_app_available "$SEARXNG_UWSGI_APP" \ - || err_msg "uWSGI app $SEARXNG_UWSGI_APP not available!" - - sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" "utils/searxng_check.py" - - if uWSGI_app_available 'searx.ini'; then - warn_msg "old searx.ini uWSGI app exists" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - fi -} - -update_searx() { - rst_title "Update SearXNG instance" - - rst_para "fetch from $GIT_URL and reset to origin/$GIT_BRANCH" - tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -git fetch origin "$GIT_BRANCH" -git reset --hard "origin/$GIT_BRANCH" -pip install -U pip -pip install -U setuptools -pip install -U wheel -pip install -U pyyaml -pip install -U -e . -EOF - install_settings - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - remove_all() { rst_title "De-Install SearXNG (service)" @@ -436,250 +64,6 @@ installations that were installed with this script." fi } -assert_user() { - rst_title "user $SERVICE_USER" section - echo - if getent passwd "$SERVICE_USER" > /dev/null; then - echo "user exists" - return 0 - fi - - tee_stderr 1 <<EOF | bash | prefix_stdout -useradd --shell /bin/bash --system \ - --home-dir "$SERVICE_HOME" \ - --comment 'Privacy-respecting metasearch engine' $SERVICE_USER -mkdir "$SERVICE_HOME" -chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" -groups $SERVICE_USER -EOF - #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" - #export SERVICE_HOME - #echo "export SERVICE_HOME=$SERVICE_HOME" -} - -clone_is_available() { - [[ -f "$SEARX_SRC/.git/config" ]] -} - -# shellcheck disable=SC2164 -clone_searx() { - rst_title "Clone SearXNG sources" section - echo - if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then - die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" - fi - SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" - if [[ ! "${SERVICE_HOME}" ]]; then - err_msg "to clone SearXNG sources, user $SERVICE_USER hast to be created first" - return 42 - fi - if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then - warn_msg "missing local branch ${GIT_BRANCH}" - info_msg "create local branch ${GIT_BRANCH} from start point: origin/${GIT_BRANCH}" - git branch "${GIT_BRANCH}" "origin/${GIT_BRANCH}" - fi - if [[ ! $(git rev-parse --abbrev-ref HEAD) == "${GIT_BRANCH}" ]]; then - warn_msg "take into account, installing branch $GIT_BRANCH while current branch is $(git rev-parse --abbrev-ref HEAD)" - fi - export SERVICE_HOME - git_clone "$REPO_ROOT" "$SEARX_SRC" \ - "$GIT_BRANCH" "$SERVICE_USER" - - pushd "${SEARX_SRC}" > /dev/null - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -cd "${SEARX_SRC}" -git remote set-url origin ${GIT_URL} -git config user.email "$ADMIN_EMAIL" -git config user.name "$ADMIN_NAME" -git config --list -EOF - popd > /dev/null -} - -prompt_installation_status(){ - - # shellcheck disable=SC2034 - local GIT_URL GIT_BRANCH VERSION_STRING VERSION_TAG - local ret_val state branch remote remote_url - state="$(install_searx_get_state)" - - case $state in - missing-searx-clone|missing-searx-pyenv) - info_msg "${_BBlue}(status: $(install_searx_get_state))${_creset}" - return 0 - ;; - *) - info_msg "SearXNG instance already installed at: $SEARX_SRC" - info_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} " - branch="$(git name-rev --name-only HEAD)" - remote="$(git config branch."${branch}".remote)" - remote_url="$(git config remote."${remote}".url)" - eval "$(get_installed_version_variables)" - - ret_val=0 - if ! [ "$GIT_URL" = "$remote_url" ]; then - warn_msg "instance's git URL: '${GIT_URL}'" \ - "differs from local clone's remote URL: ${remote_url}" - ret_val=42 - fi - if ! [ "$GIT_BRANCH" = "$branch" ]; then - warn_msg "instance git branch: ${GIT_BRANCH}" \ - "differs from local clone's branch: ${branch}" - ret_val=42 - fi - return $ret_val - ;; - esac -} - -verify_continue_install(){ - if ! prompt_installation_status; then - MSG="[${_BCyan}KEY${_creset}] to continue installation / [${_BCyan}CTRL-C${_creset}] to exit" \ - wait_key - fi -} - -prompt_installation_setting(){ - - # usage: prompt_installation_setting brand.docs_url - # - # Prompts the value of the (YAML) setting in the SearXNG instance. - - local _state - _state="$(install_searx_get_state)" - case $_state in - python-installed|installer-modified) - sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" <<EOF -import sys -from searx import get_setting -name = "${1}" -unset = object() -value = get_setting(name, unset) -if value is unset: - sys.stderr.write("error: setting '%s' does not exists\n" % name) - sys.exit(42) -print(value) -sys.exit(0) -EOF - ;; - *) - return 42 - ;; - esac -} - -get_installed_version_variables() { - - # usage: eval "$(get_installed_version_variables)" - # - # Set variables VERSION_STRING, VERSION_TAG, GIT_URL, GIT_BRANCH - - local _state - _state="$(install_searx_get_state)" - case $_state in - python-installed|installer-modified) - sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" -m searx.version;; - *) - return 42 - ;; - esac -} - -init_SEARX_SRC(){ - rst_title "Update instance: ${SEARX_SRC}/" section - - if ! clone_is_available; then - err_msg "you have to install SearXNG first" - return 1 - fi - - init_SEARX_SRC_INIT_FILES - - if [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then - info_msg "no files registered in SEARX_SRC_INIT_FILES" - return 2 - fi - - echo - echo "Update instance with file(s) from: ${REPO_ROOT}" - echo - for i in "${SEARX_SRC_INIT_FILES[@]}"; do - echo "- $i" - done - echo - echo "Be careful when modifying an existing installation." - if ! ask_yn "Do you really want to update these files in the instance?" Yn; then - return 42 - fi - for fname in "${SEARX_SRC_INIT_FILES[@]}"; do - while true; do - choose_one _reply "choose next step with file ${fname}" \ - "replace file" \ - "leave file unchanged" \ - "diff files" \ - "interactive shell" - - case $_reply in - "leave file unchanged") - break - ;; - "replace file") - info_msg "copy: ${REPO_ROOT}/${fname} --> ${SEARX_SRC}/${fname}" - cp "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" - break - ;; - "diff files") - $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" - ;; - "interactive shell") - backup_file "${SEARX_SRC}/${fname}" - echo -e "// edit ${_Red}${dst}${_creset} to your needs" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -u "${SERVICE_USER}" -i - $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" - echo - echo -e "// ${_BBlack}did you edit file ...${_creset}" - echo -en "// ${_Red}${dst}${_creset}" - if ask_yn "//${_BBlack}... to your needs?${_creset}"; then - break - fi - ;; - esac - done - done -} - -install_DOT_CONFIG(){ - rst_title "Update instance: ${SEARX_SRC}/.config.sh" section - - if cmp --silent "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh"; then - info_msg "${SEARX_SRC}/.config.sh is up to date" - return 0 - fi - - diff "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" - if ! ask_yn "Do you want to copy file .config.sh into instance?" Yn; then - return 42 - fi - backup_file "${SEARX_SRC}/.config.sh" - cp "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" -} - -install_settings() { - rst_title "${SEARXNG_SETTINGS_PATH}" section - - if ! clone_is_available; then - err_msg "you have to install SearXNG first" - exit 42 - fi - - mkdir -p "$(dirname "${SEARXNG_SETTINGS_PATH}")" - install_template --no-eval \ - "${SEARXNG_SETTINGS_PATH}" \ - "${SERVICE_USER}" "${SERVICE_GROUP}" - configure_searx -} - remove_settings() { rst_title "remove SearXNG settings" section echo @@ -687,419 +71,12 @@ remove_settings() { rm -f "${SEARXNG_SETTINGS_PATH}" } -remove_searx() { - rst_title "Drop SearXNG sources" section - if ask_yn "Do you really want to drop SearXNG sources ($SEARX_SRC)?"; then - rm -rf "$SEARX_SRC" - else - rst_para "Leave SearXNG sources unchanged." - fi -} - -pyenv_is_available() { - [[ -f "${SEARX_PYENV}/bin/activate" ]] -} - -create_pyenv() { - rst_title "Create virtualenv (python)" section - echo - if [[ ! -f "${SEARX_SRC}/manage" ]]; then - err_msg "to create pyenv for SearXNG, SearXNG has to be cloned first" - return 42 - fi - info_msg "create pyenv in ${SEARX_PYENV}" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -rm -rf "${SEARX_PYENV}" -python3 -m venv "${SEARX_PYENV}" -grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ - || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile -EOF - info_msg "inspect python's virtual environment" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -command -v python && python --version -EOF - wait_key - info_msg "install needed python packages" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -pip install -U pip -pip install -U setuptools -pip install -U wheel -pip install -U pyyaml -cd ${SEARX_SRC} -pip install -e . -EOF -} - -remove_pyenv() { - rst_title "Remove virtualenv (python)" section - if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then - return - fi - info_msg "remove pyenv activation from ~/.profile" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## -mv ~/.profile.## ~/.profile -EOF - rm -rf "${SEARX_PYENV}" -} - -configure_searx() { - rst_title "Configure SearXNG" section - rst_para "Setup SearXNG config located at $SEARXNG_SETTINGS_PATH" - echo - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARXNG_SETTINGS_PATH" -EOF -} - -test_local_searx() { - rst_title "Testing SearXNG instance localy" section - echo - - if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then - err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you" - err_msg "should stop that service before starting local tests!" - if ! ask_yn "Continue with local tests?"; then - return - fi - fi - sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -export SEARXNG_SETTINGS_PATH="${SEARXNG_SETTINGS_PATH}" -cd ${SEARX_SRC} -timeout 10 python searx/webapp.py & -sleep 3 -curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP -EOF - sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" -} - -install_searx_uwsgi() { - rst_title "Install SearXNG's uWSGI app (searxng.ini)" section - echo - install_uwsgi - uWSGI_install_app "$SEARXNG_UWSGI_APP" -} - remove_searx_uwsgi() { rst_title "Remove SearXNG's uWSGI app (searxng.ini)" section echo uWSGI_remove_app "$SEARXNG_UWSGI_APP" } -activate_service() { - rst_title "Activate SearXNG (service)" section - echo - uWSGI_enable_app "$SEARXNG_UWSGI_APP" - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -deactivate_service() { - rst_title "De-Activate SearXNG (service)" section - echo - uWSGI_disable_app "$SEARXNG_UWSGI_APP" - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -enable_image_proxy() { - info_msg "try to enable image_proxy ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/image_proxy: false/image_proxy: true/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -disable_image_proxy() { - info_msg "try to enable image_proxy ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/image_proxy: true/image_proxy: false/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -enable_debug() { - warn_msg "Do not enable debug in production environments!!" - info_msg "try to enable debug mode ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -disable_debug() { - info_msg "try to disable debug mode ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -set_result_proxy() { - - # usage: set_result_proxy <URL> [<key>] - - info_msg "try to set result proxy: '$1' ($2)" - cp "${SEARXNG_SETTINGS_PATH}" "${SEARXNG_SETTINGS_PATH}.bak" - _set_result_proxy "$1" "$2" > "${SEARXNG_SETTINGS_PATH}" -} - -_set_result_proxy() { - local line - local stage=0 - local url=" url: $1" - local key=" key: !!binary \"$2\"" - if [[ -z $2 ]]; then - key= - fi - - while IFS= read -r line - do - if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then - if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then - if [[ $stage = 0 ]]; then - stage=1 - echo "result_proxy:" - continue - elif [[ $stage = 2 ]]; then - continue - fi - fi - fi - if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then - if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then - [[ $stage = 1 ]] && echo "$url" - continue - elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then - [[ $stage = 1 ]] && [[ -n $key ]] && echo "$key" - continue - elif [[ $line =~ ^[[:space:]]*$ ]]; then - stage=2 - fi - fi - echo "$line" - done < "${SEARXNG_SETTINGS_PATH}.bak" -} - -function has_substring() { - [[ "$1" != "${2/$1/}" ]] -} -inspect_service() { - rst_title "service status & log" - cat <<EOF - -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} -EOF - install_log_searx_instance - - install_check - if in_container; then - lxc_suite_info - else - info_msg "public URL --> ${PUBLIC_URL}" - info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}" - fi - - if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!" - MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\ - wait_key - fi - - if ! service_is_available "${PUBLIC_URL}"; then - warn_msg "Public service at ${PUBLIC_URL} is not available!" - if ! in_container; then - warn_msg "Check if public name is correct and routed or use the public IP from above." - fi - fi - - local _debug_on - if ask_yn "Enable SearXNG debug mode?"; then - enable_debug - _debug_on=1 - fi - echo - - case $DIST_ID-$DIST_VERS in - ubuntu-*|debian-*) - systemctl --no-pager -l status "${SERVICE_NAME}" - ;; - arch-*) - systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" - ;; - fedora-*|centos-7) - systemctl --no-pager -l status uwsgi - ;; - esac - - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - - while true; do - trap break 2 - case $DIST_ID-$DIST_VERS in - ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;; - arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; - fedora-*|centos-7) journalctl -f -u uwsgi ;; - esac - done - - if [[ $_debug_on == 1 ]]; then - disable_debug - fi - return 0 -} - -install_apache_site() { - rst_title "Install Apache site $APACHE_SEARX_SITE" - - rst_para "\ -This installs the SearXNG uwsgi app as apache site. If your server is public to -the internet, you should instead use a reverse proxy (filtron) to block -excessively bot queries." - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" - - rst_title "Install SearXNG's uWSGI app (searxng.ini)" section - echo - uWSGI_install_app --variant=socket "$SEARXNG_UWSGI_APP" - - if ! service_is_available "${PUBLIC_URL}"; then - err_msg "Public service at ${PUBLIC_URL} is not available!" - fi -} - -remove_apache_site() { - - rst_title "Remove Apache site ${APACHE_SEARX_SITE}" - - rst_para "\ -This removes apache site ${APACHE_SEARX_SITE}." - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - fi - - apache_remove_site "${APACHE_SEARX_SITE}" - - rst_title "Remove SearXNG's uWSGI app (searxng.ini)" section - echo - uWSGI_remove_app "$SEARXNG_UWSGI_APP" -} - -rst-doc() { - local debian="${SEARX_PACKAGES_debian}" - local arch="${SEARX_PACKAGES_arch}" - local fedora="${SEARX_PACKAGES_fedora}" - local centos="${SEARX_PACKAGES_centos}" - local debian_build="${BUILD_PACKAGES_debian}" - local arch_build="${BUILD_PACKAGES_arch}" - local fedora_build="${BUILD_PACKAGES_fedora}" - local centos_build="${SEARX_PACKAGES_centos}" - debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - centos="$(echo "${centos}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - centos_build="$(echo "${centos_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\"" - - # I use ubuntu-20.04 here to demonstrate that versions are also suported, - # normaly debian-* and ubuntu-* are most the same. - - for DIST_NAME in ubuntu-20.04 arch fedora; do - ( - DIST_ID=${DIST_NAME%-*} - DIST_VERS=${DIST_NAME#*-} - [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= - uWSGI_distro_setup - - echo -e "\n.. START searxng uwsgi-description $DIST_NAME" - - case $DIST_ID-$DIST_VERS in - ubuntu-*|debian-*) cat <<EOF - -.. code:: bash - - # init.d --> /usr/share/doc/uwsgi/README.Debian.gz - # For uWSGI debian uses the LSB init process, this might be changed - # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 - - create ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} - enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ - start: sudo -H service uwsgi start ${SEARXNG_UWSGI_APP%.*} - restart: sudo -H service uwsgi restart ${SEARXNG_UWSGI_APP%.*} - stop: sudo -H service uwsgi stop ${SEARXNG_UWSGI_APP%.*} - disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - -EOF - ;; - arch-*) cat <<EOF - -.. code:: bash - - # systemd --> /usr/lib/systemd/system/uwsgi@.service - # For uWSGI archlinux uses systemd template units, see - # - http://0pointer.de/blog/projects/instances.html - # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd - - create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - enable: sudo -H systemctl enable uwsgi@${SEARXNG_UWSGI_APP%.*} - start: sudo -H systemctl start uwsgi@${SEARXNG_UWSGI_APP%.*} - restart: sudo -H systemctl restart uwsgi@${SEARXNG_UWSGI_APP%.*} - stop: sudo -H systemctl stop uwsgi@${SEARXNG_UWSGI_APP%.*} - disable: sudo -H systemctl disable uwsgi@${SEARXNG_UWSGI_APP%.*} - -EOF - ;; - fedora-*|centos-7) cat <<EOF - -.. code:: bash - - # systemd --> /usr/lib/systemd/system/uwsgi.service - # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see - # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html - - create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - -EOF - ;; - esac - echo -e ".. END searxng uwsgi-description $DIST_NAME" - - echo -e "\n.. START searxng uwsgi-appini $DIST_NAME" - echo ".. code:: bash" - echo - eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}")\"" | prefix_stdout " " - echo -e "\n.. END searxng uwsgi-appini $DIST_NAME" - - ) - done - -} # ---------------------------------------------------------------------------- main "$@" diff --git a/utils/searxng.sh b/utils/searxng.sh new file mode 100755 index 000000000..d9ecdcca3 --- /dev/null +++ b/utils/searxng.sh @@ -0,0 +1,1017 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# Script options from the environment: +SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}" + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/lib_redis.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +SERVICE_NAME="searxng" +SERVICE_USER="searxng" +SERVICE_HOME="/usr/local/searxng" +SERVICE_GROUP="searxng" + +SEARXNG_SRC="${SERVICE_HOME}/searxng-src" +# shellcheck disable=SC2034 +SEARXNG_STATIC="${SEARXNG_SRC}/searx/static" + +SEARXNG_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml" +SEARXNG_UWSGI_APP="searxng.ini" + +SEARXNG_INTERNAL_HTTP="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" +if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + SEARXNG_UWSGI_SOCKET="${SERVICE_HOME}/run/socket" +else + SEARXNG_UWSGI_SOCKET= +fi + +# SEARXNG_URL: the public URL of the instance (https://example.org/searxng). The +# value is taken from environment ${SEARXNG_URL} in ./utils/brand.env. This +# variable is an empty string if server.base_url in the settings.yml is set to +# 'false'. + +SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)/searxng}" +SEARXNG_URL="${SEARXNG_URL%/}" # if exists, remove trailing slash +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARXNG_URL="http://$(primary_ip)/searxng" +fi +SEARXNG_URL_PATH="$(echo "${SEARXNG_URL}" | sed -e 's,^.*://[^/]*\(/.*\),\1,g')" +[[ "${SEARXNG_URL_PATH}" == "${SEARXNG_URL}" ]] && SEARXNG_URL_PATH=/ + +# Apache settings + +APACHE_SEARXNG_SITE="searxng.conf" + +# nginx settings + +NGINX_SEARXNG_SITE="searxng.conf" + +# apt packages + +SEARXNG_PACKAGES_debian="\ +python3-dev python3-babel python3-venv +uwsgi uwsgi-plugin-python3 +git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev" + +SEARXNG_BUILD_PACKAGES_debian="\ +firefox graphviz imagemagick texlive-xetex librsvg2-bin +texlive-latex-recommended texlive-extra-utils fonts-dejavu +latexmk shellcheck" + +# pacman packages + +SEARXNG_PACKAGES_arch="\ +python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python +git base-devel libxml2" + +SEARXNG_BUILD_PACKAGES_arch="\ +firefox graphviz imagemagick texlive-bin extra/librsvg +texlive-core texlive-latexextra ttf-dejavu shellcheck" + +# dnf packages + +SEARXNG_PACKAGES_fedora="\ +python python-pip python-lxml python-babel python3-devel +uwsgi uwsgi-plugin-python3 +git @development-tools libxml2 openssl" + +SEARXNG_BUILD_PACKAGES_fedora="\ +firefox graphviz graphviz-gd ImageMagick librsvg2-tools +texlive-xetex-bin texlive-collection-fontsrecommended +texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts +dejavu-sans-mono-fonts ShellCheck" + +case $DIST_ID-$DIST_VERS in + ubuntu-18.04) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" + ;; + ubuntu-20.04) + # https://wiki.ubuntu.com/FocalFossa/ReleaseNotes#Python3_by_default + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian} python-is-python3" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + ;; + ubuntu-*|debian-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + ;; + arch-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_arch}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_arch}" + ;; + fedora-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_fedora}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_fedora}" + ;; +esac + +_service_prefix=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage: + $(basename "$0") install [all|user|pyenv|settings|uwsgi|redis|nginx|apache|searxng-src|packages|buildhost] + $(basename "$0") remove [all|user|pyenv|settings|uwsgi|redis|nginx|apache] + $(basename "$0") instance [cmd|update|check|localtest|inspect] +install|remove: + all : complete (de-) installation of the SearXNG service + user : service user '${SERVICE_USER}' (${SERVICE_HOME}) + pyenv : virtualenv (python) in ${SEARXNG_PYENV} + settings : settings from ${SEARXNG_SETTINGS_PATH} + uwsgi : SearXNG's uWSGI app ${SEARXNG_UWSGI_APP} + redis : build & install or remove a local redis server ${REDIS_HOME}/run/redis.sock + nginx : HTTP site ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE} + apache : HTTP site ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE} +install: + searxng-src : clone ${GIT_URL} into ${SEARXNG_SRC} + packages : installs packages from OS package manager required by SearXNG + buildhost : installs packages from OS package manager required by a SearXNG buildhost +instance: + update : update SearXNG instance (git fetch + reset & update settings.yml) + check : run checks from utils/searxng_check.py in the active installation + inspect : run some small tests and inspect SearXNG's server status and log + get_setting : get settings value from running SearXNG instance + cmd : run command in SearXNG instance's environment (e.g. bash) +EOF + searxng.instance.env + [[ -n ${1} ]] && err_msg "$1" +} + +searxng.instance.env() { + echo "uWSGI:" + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + echo " SEARXNG_UWSGI_SOCKET : ${SEARXNG_UWSGI_SOCKET}" + else + echo " SEARXNG_INTERNAL_HTTP: ${SEARXNG_INTERNAL_HTTP}" + fi + cat <<EOF +environment ${SEARXNG_SRC}/utils/brand.env: + GIT_URL : ${GIT_URL} + GIT_BRANCH : ${GIT_BRANCH} + SEARXNG_URL : ${SEARXNG_URL} + SEARXNG_PORT : ${SEARXNG_PORT} + SEARXNG_BIND_ADDRESS : ${SEARXNG_BIND_ADDRESS} +EOF +} + +main() { + required_commands \ + sudo systemctl install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + install) + sudo_or_exit + case $2 in + all) searxng.install.all;; + user) searxng.install.user;; + pyenv) searxng.install.pyenv;; + searxng-src) searxng.install.clone;; + settings) searxng.install.settings;; + uwsgi) searxng.install.uwsgi;; + packages) searxng.install.packages;; + buildhost) searxng.install.buildhost;; + nginx) searxng.nginx.install;; + apache) searxng.apache.install;; + redis) searxng.install.redis;; + *) usage "$_usage"; exit 42;; + esac + ;; + remove) + sudo_or_exit + case $2 in + all) searxng.remove.all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) searxng.remove.pyenv;; + settings) searxng.remove.settings;; + uwsgi) searxng.remove.uwsgi;; + apache) searxng.apache.remove;; + remove) searxng.nginx.remove;; + redis) searxng.remove.redis;; + *) usage "$_usage"; exit 42;; + esac + ;; + instance) + case $2 in + update) + sudo_or_exit + searxng.instance.update + ;; + check) + sudo_or_exit + searxng.instance.self.call searxng.check + ;; + inspect) + sudo_or_exit + searxng.instance.inspect + ;; + cmd) + sudo_or_exit + shift; shift; searxng.instance.exec "$@" + ;; + get_setting) + shift; shift; searxng.instance.get_setting "$@" + ;; + call) + # call a function in instance's environment + shift; shift; searxng.instance.self.call "$@" + ;; + _call) + shift; shift; "$@" + ;; + *) usage "$_usage"; exit 42;; + esac + ;; + *) + local cmd="$1" + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + usage "unknown or missing command $1" + exit 42 + else + "$cmd" "$@" + fi + ;; + esac +} + +searxng.install.all() { + rst_title "SearXNG installation" part + + local redis_url + + rst_title "SearXNG" + searxng.install.packages + wait_key 10 + searxng.install.user + wait_key 10 + searxng.install.clone + wait_key + searxng.install.pyenv + wait_key + searxng.install.settings + wait_key + searxng.instance.localtest + wait_key + searxng.install.uwsgi + wait_key + + rst_title "Redis DB" + searxng.install.redis.db + + rst_title "HTTP Server" + searxng.install.http.site + + rst_title "Finalize installation" + if ask_yn "Do you want to run some checks?" Yn; then + searxng.instance.self.call searxng.check + fi +} + +searxng.install.redis.db() { + local redis_url + + redis_url=$(searxng.instance.get_setting redis.url) + rst_para "\ +In your instance, redis DB connector is configured at: + + ${redis_url} +" + if searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + info_msg "SearXNG instance is able to connect redis DB." + return + fi + if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then + err_msg "SearXNG instance can't connect redis DB / check redis & your settings" + return + fi + rst_para ".. but this redis DB is not installed yet." + + case $DIST_ID-$DIST_VERS in + fedora-*) + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # + # HINT: without option ``emperor-tyrant-initgroups=true`` in + # ``/etc/uwsgi.ini`` the process won't get the additional groups, + # but this option is not available in 2.0.x branch [2][3] / on + # fedora35 there is v2.0.20 installed --> no way to get additional + # groups on fedora's tyrant mode. + # + # ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... + # ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + # ERROR:searx.plugins.limiter: init limiter DB failed!!! + # + # $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' + # searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini + # searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini + # + # Additional groups: + # + # $ groups searxng + # searxng : searxng searxng-redis + # + # Here you can see that the additional "Groups" of PID 186 are unset + # (missing gid of searxng-redis) + # + # $ cat /proc/186/task/186/status + # ... + # Uid: 993 993 993 993 + # Gid: 993 993 993 993 + # FDSize: 128 + # Groups: + # ... + # + # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting + # [2] https://github.com/unbit/uwsgi/issues/2099 + # [3] https://github.com/unbit/uwsgi/pull/752 + + rst_para "\ +Fedora uses emperor-tyrant mode / in this mode we had a lot of trouble with +sockets and permissions of the vasals. We recommend to setup a redis DB +and using redis:// TCP protocol in the settings.yml configuration." + ;; + *) + if ask_yn "Do you want to install the redis DB now?" Yn; then + searxng.install.redis + uWSGI_restart "$SEARXNG_UWSGI_APP" + fi + ;; + esac +} + +searxng.install.http.site() { + + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy" Yn; then + searxng.apache.install + fi + elif nginx_is_installed; then + info_msg "Nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy" Yn; then + searxng.nginx.install + fi + else + info_msg "Don't forget to install HTTP site." + fi +} + +searxng.remove.all() { + local redis_url + + rst_title "De-Install SearXNG (service)" + if ! ask_yn "Do you really want to deinstall SearXNG?"; then + return + fi + + redis_url=$(searxng.instance.get_setting redis.url) + if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then + searxng.remove.redis + fi + + searxng.remove.uwsgi + drop_service_account "${SERVICE_USER}" + searxng.remove.settings + wait_key + + if service_is_available "${SEARXNG_URL}"; then + MSG="** Don't forgett to remove your public site! (${SEARXNG_URL}) **" wait_key 10 + fi +} + +searxng.install.user() { + rst_title "SearXNG -- install user" section + echo + if getent passwd "${SERVICE_USER}" > /dev/null; then + echo "user already exists" + return 0 + fi + + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "${SERVICE_HOME}" \ + --comment 'Privacy-respecting metasearch engine' ${SERVICE_USER} +mkdir "${SERVICE_HOME}" +chown -R "${SERVICE_GROUP}:${SERVICE_GROUP}" "${SERVICE_HOME}" +groups ${SERVICE_USER} +EOF +} + +searxng.install.packages() { + TITLE="SearXNG -- install packages" pkg_install "${SEARXNG_PACKAGES}" +} + +searxng.install.buildhost() { + TITLE="SearXNG -- install buildhost packages" pkg_install \ + "${SEARXNG_PACKAGES} ${SEARXNG_BUILD_PACKAGES}" +} + +searxng.install.clone() { + rst_title "Clone SearXNG sources" section + if ! service_account_is_available "${SERVICE_USER}"; then + die 42 "To clone SearXNG, first install user ${SERVICE_USER}." + fi + echo + if ! sudo -i -u "${SERVICE_USER}" ls -d "$REPO_ROOT" > /dev/null; then + die 42 "user '${SERVICE_USER}' missed read permission: $REPO_ROOT" + fi + # SERVICE_HOME="$(sudo -i -u "${SERVICE_USER}" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone SearXNG sources, user ${SERVICE_USER} hast to be created first" + return 42 + fi + if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then + warn_msg "missing local branch ${GIT_BRANCH}" + info_msg "create local branch ${GIT_BRANCH} from start point: origin/${GIT_BRANCH}" + git branch "${GIT_BRANCH}" "origin/${GIT_BRANCH}" + fi + if [[ ! $(git rev-parse --abbrev-ref HEAD) == "${GIT_BRANCH}" ]]; then + warn_msg "take into account, installing branch $GIT_BRANCH while current branch is $(git rev-parse --abbrev-ref HEAD)" + fi + # export SERVICE_HOME + + # clone repo and add a safe.directory entry to git's system config / see + # https://github.com/searxng/searxng/issues/1251 + git_clone "$REPO_ROOT" "${SEARXNG_SRC}" \ + "$GIT_BRANCH" "${SERVICE_USER}" + git config --system --add safe.directory "${SEARXNG_SRC}" + + pushd "${SEARXNG_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARXNG_SRC}" +git remote set-url origin ${GIT_URL} +git config user.email "${ADMIN_EMAIL}" +git config user.name "${ADMIN_NAME}" +git config --list +EOF + popd > /dev/null +} + +searxng.install.pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARXNG_SRC}/manage" ]]; then + die 42 "To create pyenv for SearXNG, first install searxng-src." + fi + info_msg "create pyenv in ${SEARXNG_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARXNG_PYENV}" +python3 -m venv "${SEARXNG_PYENV}" +grep -qFs -- 'source ${SEARXNG_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARXNG_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U pyyaml +cd ${SEARXNG_SRC} +pip install -e . +EOF +} + +searxng.remove.pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARXNG_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARXNG_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARXNG_PYENV}" +} + +searxng.install.settings() { + rst_title "install ${SEARXNG_SETTINGS_PATH}" section + + if ! [[ -f "${SEARXNG_SRC}/.git/config" ]]; then + die "Before install settings, first install SearXNG." + exit 42 + fi + + mkdir -p "$(dirname "${SEARXNG_SETTINGS_PATH}")" + + DEFAULT_SELECT=1 \ + install_template --no-eval \ + "${SEARXNG_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "root" +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "${SEARXNG_SETTINGS_PATH}" +EOF +} + +searxng.remove.settings() { + rst_title "remove ${SEARXNG_SETTINGS_PATH}" section + if ask_yn "Do you want to delete the SearXNG settings?" Yn; then + rm -f "${SEARXNG_SETTINGS_PATH}" + fi +} + +searxng.check() { + rst_title "SearXNG checks" section + + for NAME in "searx" "filtron" "morty"; do + if service_account_is_available "${NAME}"; then + err_msg "There exists an old '${NAME}' account from a previous installation." + else + info_msg "[OK] (old) account '${NAME}' does not exists" + fi + done + + "${SEARXNG_PYENV}/bin/python" "${SEARXNG_SRC}/utils/searxng_check.py" +} + +searxng.instance.update() { + rst_title "Update SearXNG instance" + rst_para "fetch from $GIT_URL and reset to origin/$GIT_BRANCH" + tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +git fetch origin "$GIT_BRANCH" +git reset --hard "origin/$GIT_BRANCH" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U pyyaml +pip install -U -e . +EOF + rst_para "update instance's settings.yml from ${SEARXNG_SETTINGS_PATH}" + DEFAULT_SELECT=2 \ + install_template --no-eval \ + "${SEARXNG_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + + sudo -H -i <<EOF +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "${SEARXNG_SETTINGS_PATH}" +EOF + uWSGI_restart "${SEARXNG_UWSGI_APP}" +} + +searxng.install.uwsgi() { + rst_title "SearXNG (install uwsgi)" + install_uwsgi + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + searxng.install.uwsgi.socket + else + searxng.install.uwsgi.http + fi +} + +searxng.install.uwsgi.http() { + rst_para "Install ${SEARXNG_UWSGI_APP} at: http://${SEARXNG_INTERNAL_HTTP}" + uWSGI_install_app "${SEARXNG_UWSGI_APP}" + if ! searxng.uwsgi.available; then + err_msg "URL http://${SEARXNG_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" + fi +} + +searxng.install.uwsgi.socket() { + rst_para "Install ${SEARXNG_UWSGI_APP} using socket at: ${SEARXNG_UWSGI_SOCKET}" + mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" + + case $DIST_ID-$DIST_VERS in + fedora-*) + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting + uWSGI_install_app --variant=socket "${SEARXNG_UWSGI_APP}" "${SERVICE_USER}" "${SERVICE_GROUP}" + ;; + *) + uWSGI_install_app --variant=socket "${SEARXNG_UWSGI_APP}" + ;; + esac + sleep 5 + if ! searxng.uwsgi.available; then + err_msg "uWSGI socket not available at: ${SEARXNG_UWSGI_SOCKET}" + fi +} + +searxng.uwsgi.available() { + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + [[ -S "${SEARXNG_UWSGI_SOCKET}" ]] + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "uWSGI socket is located at: ${SEARXNG_UWSGI_SOCKET}" + fi + else + service_is_available "http://${SEARXNG_INTERNAL_HTTP}" + exit_val=$? + fi + return "$exit_val" +} + +searxng.remove.uwsgi() { + rst_title "Remove SearXNG's uWSGI app (${SEARXNG_UWSGI_APP})" section + echo + uWSGI_remove_app "${SEARXNG_UWSGI_APP}" +} + +searxng.install.redis() { + rst_title "SearXNG (install redis)" + redis.build + redis.install + redis.addgrp "${SERVICE_USER}" +} + +searxng.remove.redis() { + rst_title "SearXNG (remove redis)" + redis.rmgrp "${SERVICE_USER}" + redis.remove +} + +searxng.instance.localtest() { + rst_title "Test SearXNG instance localy" section + rst_para "Activate debug mode, start a minimal SearXNG "\ + "service and debug a HTTP request/response cycle." + + if service_is_available "http://${SEARXNG_INTERNAL_HTTP}" &>/dev/null; then + err_msg "URL/port http://${SEARXNG_INTERNAL_HTTP} is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + echo + searxng.instance.debug.on + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +export SEARXNG_SETTINGS_PATH="${SEARXNG_SETTINGS_PATH}" +cd ${SEARXNG_SRC} +timeout 10 python searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure ${SEARXNG_INTERNAL_HTTP} +EOF + echo + searxng.instance.debug.off +} + +searxng.install.http.pre() { + if ! searxng.uwsgi.available; then + rst_para "\ +To install uWSGI use:: + + $(basename "$0") install uwsgi +" + die 42 "SearXNG's uWSGI app not available" + fi + + if ! searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + rst_para "\ +The configured redis DB is not available: If your server is public to the +internet, you should setup a bot protection to block excessively bot queries. +Bot protection requires a redis DB. About bot protection visit the official +SearXNG documentation and query for the word 'limiter'. +" + fi +} + +searxng.apache.install() { + rst_title "Install Apache site ${APACHE_SEARXNG_SITE}" + rst_para "\ +This installs SearXNG's uWSGI app as apache site. The apache site is located at: +${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}." + searxng.install.http.pre + + if ! apache_is_installed; then + err_msg "Apache packages are not installed" + if ! ask_yn "Do you really want to continue and install apache packages?" Yn; then + return + else + FORCE_SELECTION=Y install_apache + fi + else + info_msg "Apache packages are installed [OK]" + fi + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + apache_install_site --variant=socket "${APACHE_SEARXNG_SITE}" + else + apache_install_site "${APACHE_SEARXNG_SITE}" + fi + + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + fi +} + +searxng.apache.remove() { + rst_title "Remove Apache site ${APACHE_SEARXNG_SITE}" + rst_para "\ +This removes apache site ${APACHE_SEARXNG_SITE}:: + + ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}" + + ! apache_is_installed && err_msg "Apache is not installed." + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + apache_remove_site "${APACHE_SEARXNG_SITE}" +} + +searxng.nginx.install() { + + rst_title "Install nginx site ${NGINX_SEARXNG_SITE}" + rst_para "\ +This installs SearXNG's uWSGI app as Nginx site. The Nginx site is located at: +${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE} and requires a uWSGI." + searxng.install.http.pre + + if ! nginx_is_installed ; then + err_msg "Nginx packages are not installed" + if ! ask_yn "Do you really want to continue and install Nginx packages?" Yn; then + return + else + FORCE_SELECTION=Y install_nginx + fi + else + info_msg "Nginx packages are installed [OK]" + fi + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + nginx_install_app --variant=socket "${NGINX_SEARXNG_SITE}" + else + nginx_install_app "${NGINX_SEARXNG_SITE}" + fi + + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + fi +} + +searxng.nginx.remove() { + rst_title "Remove Nginx site ${NGINX_SEARXNG_SITE}" + rst_para "\ +This removes Nginx site ${NGINX_SEARXNG_SITE}:: + + ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}" + + ! nginx_is_installed && err_msg "Nginx is not installed." + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + nginx_remove_app "${NGINX_SEARXNG_SITE}" +} + +searxng.instance.exec() { + if ! service_account_is_available "${SERVICE_USER}"; then + die 42 "can't execute: instance does not exists (missed account ${SERVICE_USER})" + fi + sudo -H -i -u "${SERVICE_USER}" \ + SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET}" \ + "$@" +} + +searxng.instance.self.call() { + # wrapper to call a function in instance's environment + info_msg "wrapper: utils/searxng.sh instance _call $*" + searxng.instance.exec "${SEARXNG_SRC}/utils/searxng.sh" instance _call "$@" +} + +searxng.instance.get_setting() { + searxng.instance.exec python <<EOF +from searx import get_setting +print(get_setting('$1')) +EOF +} + +searxng.instance.debug.on() { + warn_msg "Do not enable debug in a production environment!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARXNG_UWSGI_APP" +} + +searxng.instance.debug.off() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARXNG_UWSGI_APP" +} + +searxng.instance.inspect() { + rst_title "Inspect SearXNG instance" + echo + + searxng.instance.self.call _searxng.instance.inspect + + local _debug_on + if ask_yn "Enable SearXNG debug mode?"; then + searxng.instance.debug.on + _debug_on=1 + fi + echo + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # For uWSGI debian uses the LSB init process; for each configuration + # file new uWSGI daemon instance is started with additional option. + service uwsgi status "${SERVICE_NAME}" + ;; + arch-*) + systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" + ;; + fedora-*) + systemctl --no-pager -l status uwsgi + ;; + esac + + echo -e "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + + while true; do + trap break 2 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) tail -f "/var/log/uwsgi/app/${SERVICE_NAME%.*}.log" ;; + arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; + fedora-*) journalctl -f -u uwsgi ;; + esac + done + + if [[ $_debug_on == 1 ]]; then + searxng.instance.debug.off + fi + return 0 +} + +_searxng.instance.inspect() { + searxng.instance.env + + if in_container; then + # shellcheck source=utils/lxc-searxng.env + source "${REPO_ROOT}/utils/lxc-searxng.env" + lxc_suite_info + fi + + MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue${_creset}" + + if ! searxng.uwsgi.available; then + err_msg "SearXNG's uWSGI app not available" + wait_key + fi + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + wait_key + fi +} + +searxng.doc.rst() { + local debian="${SEARXNG_PACKAGES_debian}" + local arch="${SEARXNG_PACKAGES_arch}" + local fedora="${SEARXNG_PACKAGES_fedora}" + local debian_build="${SEARXNG_BUILD_PACKAGES_debian}" + local arch_build="${SEARXNG_BUILD_PACKAGES_arch}" + local fedora_build="${SEARXNG_BUILD_PACKAGES_fedora}" + debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + uwsgi_variant=':socket' + else + uwsgi_variant=':socket' + fi + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searxng.rst")\"" + + # I use ubuntu-20.04 here to demonstrate that versions are also suported, + # normaly debian-* and ubuntu-* are most the same. + + for DIST_NAME in ubuntu-20.04 arch fedora; do + ( + DIST_ID=${DIST_NAME%-*} + DIST_VERS=${DIST_NAME#*-} + [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + uWSGI_distro_setup + + echo -e "\n.. START searxng uwsgi-description $DIST_NAME" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) cat <<EOF + +.. code:: bash + + # init.d --> /usr/share/doc/uwsgi/README.Debian.gz + # For uWSGI debian uses the LSB init process, this might be changed + # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + + create ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} + enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ + start: sudo -H service uwsgi start ${SEARXNG_UWSGI_APP%.*} + restart: sudo -H service uwsgi restart ${SEARXNG_UWSGI_APP%.*} + stop: sudo -H service uwsgi stop ${SEARXNG_UWSGI_APP%.*} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + +EOF + ;; + arch-*) cat <<EOF + +.. code:: bash + + # systemd --> /usr/lib/systemd/system/uwsgi@.service + # For uWSGI archlinux uses systemd template units, see + # - http://0pointer.de/blog/projects/instances.html + # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + + create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + enable: sudo -H systemctl enable uwsgi@${SEARXNG_UWSGI_APP%.*} + start: sudo -H systemctl start uwsgi@${SEARXNG_UWSGI_APP%.*} + restart: sudo -H systemctl restart uwsgi@${SEARXNG_UWSGI_APP%.*} + stop: sudo -H systemctl stop uwsgi@${SEARXNG_UWSGI_APP%.*} + disable: sudo -H systemctl disable uwsgi@${SEARXNG_UWSGI_APP%.*} + +EOF + ;; + fedora-*|centos-7) cat <<EOF + +.. code:: bash + + # systemd --> /usr/lib/systemd/system/uwsgi.service + # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see + # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + + create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + +EOF + ;; + esac + echo -e ".. END searxng uwsgi-description $DIST_NAME" + + local _show_cursor="" # prevent from prefix_stdout's trailing show-cursor + + echo -e "\n.. START searxng uwsgi-appini $DIST_NAME" + echo ".. code:: bash" + echo + eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}${uwsgi_variant}")\"" | prefix_stdout " " + echo -e "\n.. END searxng uwsgi-appini $DIST_NAME" + + echo -e "\n.. START nginx socket" + echo ".. code:: nginx" + echo + eval "echo \"$(< "${TEMPLATES}/${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}:socket")\"" | prefix_stdout " " + echo -e "\n.. END nginx socket" + + echo -e "\n.. START nginx http" + echo ".. code:: nginx" + echo + eval "echo \"$(< "${TEMPLATES}/${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}")\"" | prefix_stdout " " + echo -e "\n.. END nginx http" + + echo -e "\n.. START apache socket" + echo ".. code:: apache" + echo + eval "echo \"$(< "${TEMPLATES}/${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}:socket")\"" | prefix_stdout " " + echo -e "\n.. END apache socket" + + echo -e "\n.. START apache http" + echo ".. code:: apache" + echo + eval "echo \"$(< "${TEMPLATES}/${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}")\"" | prefix_stdout " " + echo -e "\n.. END apache http" + ) + done + +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searxng_check.py b/utils/searxng_check.py index afd0c4056..39e774340 100644 --- a/utils/searxng_check.py +++ b/utils/searxng_check.py @@ -25,3 +25,10 @@ if os.path.isfile(OLD_SETTING): os.environ.get('SEARXNG_SETTINGS_PATH', '/etc/searxng/settings.yml') )) warnings.warn(msg, DeprecationWarning) + +from searx.shared import redisdb +from searx import get_setting + +if not redisdb.init(): + warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2) + warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2) diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json deleted file mode 100644 index fff70fa8f..000000000 --- a/utils/templates/etc/filtron/rules.json +++ /dev/null @@ -1,129 +0,0 @@ -[ - { - "name": "roboagent limit", - "filters": [ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)" - ], - "limit": 0, - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "botlimit", - "filters": [ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "limit": 0, - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "suspiciously frequent IP", - "filters": [], - "interval": 600, - "limit": 30, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions":[ - {"name":"log"} - ] - }, - { - "name": "search request", - "filters": [ - "Param:q", - "Path=^(/|/search)$" - ], - "interval": 61, - "limit": 999, - "subrules": [ - { - "name": "missing Accept-Language", - "filters": ["!Header:Accept-Language"], - "limit": 0, - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "suspiciously Connection=close header", - "filters": ["Header:Connection=close"], - "limit": 0, - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "IP limit", - "interval": 61, - "limit": 9, - "stop": true, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "rss/json limit", - "filters": [ - "Param:format=(csv|json|rss)" - ], - "interval": 121, - "limit": 2, - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "useragent limit", - "interval": 61, - "limit": 199, - "aggregations": [ - "Header:User-Agent" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - } - ] - } -] diff --git a/utils/templates/etc/httpd/sites-available/morty.conf b/utils/templates/etc/httpd/sites-available/morty.conf deleted file mode 100644 index daeb3635a..000000000 --- a/utils/templates/etc/httpd/sites-available/morty.conf +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so -#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog -# CustomLog /dev/null combined env=dontlog - -<Location ${PUBLIC_URL_PATH_MORTY} > - - <IfModule mod_security2.c> - SecRuleEngine Off - </IfModule> - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://${MORTY_LISTEN} - RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} - -</Location> diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf b/utils/templates/etc/httpd/sites-available/searxng.conf new file mode 100644 index 000000000..5278640c3 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searxng.conf @@ -0,0 +1,41 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so +# +# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARXNG_URL_PATH}> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + # add the trailing slash + RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/ + + ProxyPreserveHost On + ProxyPass http://${SEARXNG_INTERNAL_HTTP} + + # see flaskfix.py + RequestHeader set X-Scheme %{REQUEST_SCHEME}s + RequestHeader set X-Script-Name ${SEARXNG_URL_PATH} + + # see limiter.py + RequestHeader set X-Real-IP %{REMOTE_ADDR}s + RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s + +</Location> + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:filtron b/utils/templates/etc/httpd/sites-available/searxng.conf:filtron deleted file mode 100644 index 379d47e24..000000000 --- a/utils/templates/etc/httpd/sites-available/searxng.conf:filtron +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so -#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog -# CustomLog /dev/null combined env=dontlog - -# SecRuleRemoveById 981054 -# SecRuleRemoveById 981059 -# SecRuleRemoveById 981060 -# SecRuleRemoveById 950907 - -<Location ${FILTRON_URL_PATH} > - - <IfModule mod_security2.c> - SecRuleEngine Off - </IfModule> - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://${FILTRON_LISTEN} - RequestHeader set X-Script-Name ${FILTRON_URL_PATH} - -</Location> diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:socket b/utils/templates/etc/httpd/sites-available/searxng.conf:socket new file mode 100644 index 000000000..b55ea7560 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searxng.conf:socket @@ -0,0 +1,41 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so +# +# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARXNG_URL_PATH}> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + # add the trailing slash + RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/ + + ProxyPreserveHost On + ProxyPass unix:${SEARXNG_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searxng/ + + # see flaskfix.py + RequestHeader set X-Scheme %{REQUEST_SCHEME}s + RequestHeader set X-Script-Name ${SEARXNG_URL_PATH} + + # see limiter.py + RequestHeader set X-Real-IP %{REMOTE_ADDR}s + RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s + +</Location> + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi b/utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi deleted file mode 100644 index aabc125ae..000000000 --- a/utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so -# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog -# CustomLog /dev/null combined env=dontlog - -<Location ${SEARXNG_URL_PATH}> - - <IfModule mod_security2.c> - SecRuleEngine Off - </IfModule> - - Require all granted - - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/ - -</Location> diff --git a/utils/templates/etc/nginx/default.apps-available/morty.conf b/utils/templates/etc/nginx/default.apps-available/morty.conf deleted file mode 100644 index 51f083985..000000000 --- a/utils/templates/etc/nginx/default.apps-available/morty.conf +++ /dev/null @@ -1,11 +0,0 @@ -# https://example.org/morty - -location /morty { - proxy_pass http://127.0.0.1:3000/; - - proxy_set_header Host \$host; - proxy_set_header Connection \$http_connection; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Scheme \$scheme; -} diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron b/utils/templates/etc/nginx/default.apps-available/searxng.conf index e25461c47..7225a8f96 100644 --- a/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf @@ -1,16 +1,29 @@ -# https://example.org/searx - location ${SEARXNG_URL_PATH} { - proxy_pass http://127.0.0.1:4004/; + + proxy_pass http://${SEARXNG_INTERNAL_HTTP}; proxy_set_header Host \$host; proxy_set_header Connection \$http_connection; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + + # see flaskfix.py proxy_set_header X-Scheme \$scheme; proxy_set_header X-Script-Name ${SEARXNG_URL_PATH}; -} -location ${SEARXNG_URL_PATH}/static/ { - alias ${SEARX_SRC}/searx/static/; + # see limiter.py + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + + # proxy_buffering off; + # proxy_request_buffering off; + # proxy_buffer_size 8k; + } + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# location ${SEARXNG_URL_PATH}/static/ { +# alias ${SEARXNG_STATIC}/; +# } diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket new file mode 100644 index 000000000..7a74eab48 --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket @@ -0,0 +1,26 @@ +location ${SEARXNG_URL_PATH} { + + uwsgi_pass unix://${SEARXNG_UWSGI_SOCKET}; + + include uwsgi_params; + + uwsgi_param HTTP_HOST \$host; + uwsgi_param HTTP_CONNECTION \$http_connection; + + # see flaskfix.py + uwsgi_param HTTP_X_SCHEME \$scheme; + uwsgi_param HTTP_X_SCRIPT_NAME ${SEARXNG_URL_PATH}; + + # see limiter.py + uwsgi_param HTTP_X_REAL_IP \$remote_addr; + uwsgi_param HTTP_X_FORWARDED_FOR \$proxy_add_x_forwarded_for; +} + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# location ${SEARXNG_URL_PATH}/static/ { +# alias ${SEARXNG_STATIC}/; +# } diff --git a/utils/templates/etc/searxng/settings.yml b/utils/templates/etc/searxng/settings.yml index 860f4f5e9..aee21474d 100644 --- a/utils/templates/etc/searxng/settings.yml +++ b/utils/templates/etc/searxng/settings.yml @@ -1,46 +1,55 @@ -# SearXNG settings, before editing this file read: -# -# https://docs.searxng.org/admin/engines/settings.html +# SearXNG settings use_default_settings: true general: - # Debug mode, only for development debug: false - # change displayed name - # instance_name: "SearXNG" + instance_name: "SearXNG" search: - # Filter results. 0: None, 1: Moderate, 2: Strict - safe_search: 0 - # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", - # "startpage", "swisscows", "qwant", "wikipedia" - leave blank to turn it off - # by default. - autocomplete: '' - # Default search language - leave blank to detect from browser information or - # use codes from 'languages.py' - default_lang: '' - # remove format to deny access, use lower case. - formats: - - html + safe_search: 2 + autocomplete: 'duckduckgo' server: - secret_key: "ultrasecretkey" # change this! - # Proxying image results through SearXNG - image_proxy: false + secret_key: "ultrasecretkey" + limiter: true + image_proxy: true + +redis: + url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + +ui: + static_use_hash: true -# result_proxy: -# url: http://127.0.0.1:3000/ -# key: !!binary "your_morty_proxy_key" +# preferences: +# lock: +# - autocomplete +# - method + +enabled_plugins: + - 'Hash plugin' + - 'Search on category select' + - 'Self Informations' + - 'Tracker URL remover' + - 'Ahmia blacklist' + # - 'Hostname replace' # see hostname_replace configuration below + # - 'Infinite scroll' + # - 'Open Access DOI rewrite' + # - 'Vim-like hotkeys' # plugins: # - only_show_green_results -# engines: -# -# - name: duckduckgo -# disabled: false +# hostname_replace: # +# # twitter --> nitter +# '(www\.)?twitter\.com$': 'nitter.net' + +engines: + + - name: google + use_mobile_ui: true + # - name: fdroid # disabled: false # @@ -48,6 +57,13 @@ server: # disabled: false # # - name: mediathekviewweb -# engine: mediathekviewweb -# shortcut: mvw -# categories: general +# categories: TV +# disabled: false +# +# - name: invidious +# disabled: false +# base_url: +# - https://invidious.snopyta.org +# - https://invidious.tiekoetter.com +# - https://invidio.xamh.de +# - https://inv.riverside.rocks diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini index aaf55a807..04c32c662 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini @@ -16,7 +16,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -57,37 +57,27 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -http = ${SEARX_INTERNAL_HTTP} +http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: +# uWSGI serves the static files and in settings.yml we use:: # -# On some distributions you need to create the app folder for the sockets:: +# ui: +# static_use_hash: true # -# mkdir -p ${SEARX_UWSGI_SOCKET} -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} -# -# socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket index e8facda86..bbfaf63be 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket @@ -16,7 +16,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -57,37 +57,24 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. - -# using IP: -# -# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http -# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -# http = ${SEARX_INTERNAL_HTTP} +socket = ${SEARXNG_UWSGI_SOCKET} -# using unix-sockets: +# uWSGI serves the static files and in settings.yml we use:: # -# On some distributions you need to create the app folder for the sockets:: +# ui: +# static_use_hash: true # -# mkdir -p ${SEARX_UWSGI_SOCKET} -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} -# -socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini b/utils/templates/etc/uwsgi/apps-available/searxng.ini index 9dad84c16..5ea7d991a 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini @@ -6,7 +6,11 @@ # # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core -# Who will run the code +# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be +# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini). +# +# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting +# uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} @@ -16,7 +20,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -56,37 +60,27 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -http = ${SEARX_INTERNAL_HTTP} +http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: -# -# On some distributions you need to create the app folder for the sockets:: +# uWSGI serves the static files and in settings.yml we use:: # -# mkdir -p /run/uwsgi/app/searxng -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} +# ui: +# static_use_hash: true # -# socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket index c76d084e1..304ea3500 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket @@ -6,7 +6,11 @@ # # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core -# Who will run the code +# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be +# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini). +# +# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting +# uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} @@ -16,7 +20,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -56,37 +60,24 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# -# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http -# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html +socket = ${SEARXNG_UWSGI_SOCKET} -# http = ${SEARX_INTERNAL_HTTP} - -# using unix-sockets: -# -# On some distributions you need to create the app folder for the sockets:: +# uWSGI serves the static files and in settings.yml we use:: # -# mkdir -p ${SEARX_UWSGI_SOCKET} -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} +# ui: +# static_use_hash: true # -socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k |