diff options
Diffstat (limited to 'utils')
| -rwxr-xr-x | utils/filtron.sh | 445 | ||||
| -rwxr-xr-x | utils/lib.sh | 863 | ||||
| -rw-r--r-- | utils/makefile.python | 4 | ||||
| -rwxr-xr-x | utils/morty.sh | 406 | ||||
| -rwxr-xr-x | utils/searx.sh | 588 | ||||
| -rw-r--r-- | utils/templates/etc/apache2/sites-available/morty.conf | 28 | ||||
| -rw-r--r-- | utils/templates/etc/apache2/sites-available/searx.conf:filtron | 33 | ||||
| -rw-r--r-- | utils/templates/etc/apache2/sites-available/searx.conf:uwsgi | 27 | ||||
| -rw-r--r-- | utils/templates/etc/filtron/rules.json | 105 | ||||
| -rw-r--r-- | utils/templates/etc/uwsgi/apps-available/searx.ini | 62 | ||||
| -rw-r--r-- | utils/templates/lib/systemd/system/filtron.service | 29 | ||||
| -rw-r--r-- | utils/templates/lib/systemd/system/morty.service | 29 |
12 files changed, 2617 insertions, 2 deletions
diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 000000000..e97b9f014 --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,445 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2119,SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ + +FILTRON_ETC="/etc/filtron" + +FILTRON_RULES="$FILTRON_ETC/rules.json" + +FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" +FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" +FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# Apache Settings + +APACHE_FILTRON_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage:: + + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [filtron] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete setup of filtron service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) +update filtron + Update filtron installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options +apache : ${PUBLIC_URL} + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_FILTRON_SITE} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SERVICE_USER : ${SERVICE_USER} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_TARGET : ${FILTRON_TARGET} + +EOF + [ ! -z "${1+x}" ] && err_msg "$1" +} + +main() { + rst_title "$SERVICE_NAME" part + + required_commands \ + dpkg apt-get install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + filtron) update_filtron ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_filtron + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + echo + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + return 42 + fi + drop_service_account "${SERVICE_USER}" + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \ + --disabled-password --group --gecos 'Filtron' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + + +filtron_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] +} + +_svcpr=" |${SERVICE_USER}| " + +install_filtron() { + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_TARGET : ${FILTRON_TARGET} + +EOF + + apache_is_installed && info_msg "Apache is installed." + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if filtron_is_installed; then + info_msg "~$SERVICE_USER: filtron app is installed" + else + err_msg "~$SERVICE_USER: filtron app is not installed!" + fi + + if ! service_is_available "http://${FILTRON_API}"; then + err_msg "API not available at: http://${FILTRON_API}" + fi + + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + + if service_is_available "http://${FILTRON_TARGET}" ; then + info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" + fi + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + + +enable_debug() { + info_msg "try to enable debug mode ..." + python <<EOF +import sys, json + +debug = { + u'name': u'debug request' + , u'filters': [] + , u'interval': 0 + , u'limit': 0 + , u'actions': [{u'name': u'log'}] +} + +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + j[pos] = debug +else: + j.append(debug) +with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) + +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +disable_debug() { + info_msg "try to disable debug mode ..." + python <<EOF +import sys, json +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + del j[pos] + with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + a2enmod headers + a2enmod proxy + a2enmod proxy_http + + echo + apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This removes apache site ${APACHE_FILTRON_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "$APACHE_FILTRON_SITE" +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh new file mode 100755 index 000000000..59ad12229 --- /dev/null +++ b/utils/lib.sh @@ -0,0 +1,863 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2059,SC1117 + +ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" +ADMIN_NAME="${ADMIN_NAME:-$USER}" + +ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}" +ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}" + +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + while [ -h "${REPO_ROOT}" ] ; do + REPO_ROOT=$(readlink "${REPO_ROOT}") + done + REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P ) +fi + +if [[ -z ${TEMPLATES} ]]; then + TEMPLATES="${REPO_ROOT}/utils/templates" +fi + +if [[ -z "$CACHE" ]]; then + CACHE="${REPO_ROOT}/cache" +fi + +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + +DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}" + +source_dot_config() { + if [[ ! -e "${DOT_CONFIG}" ]]; then + err_msg "configuration does not extsts at: ${DOT_CONFIG}" + return 42 + fi + # shellcheck disable=SC1090 + source "${DOT_CONFIG}" +} + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +required_commands() { + + # usage: requires_commands [cmd1 ...] + + local exit_val=0 + while [ ! -z "$1" ]; do + + if ! command -v "$1" &>/dev/null; then + err_msg "missing command $1" + exit_val=42 + fi + shift + done + return $exit_val +} + +# colors +# ------ + +# shellcheck disable=SC2034 +set_terminal_colors() { + _colors=8 + _creset='\e[0m' # reset all attributes + + _Black='\e[0;30m' + _White='\e[1;37m' + _Red='\e[0;31m' + _Green='\e[0;32m' + _Yellow='\e[0;33m' + _Blue='\e[0;34m' + _Violet='\e[0;35m' + _Cyan='\e[0;36m' + + _BBlack='\e[1;30m' + _BWhite='\e[1;37m' + _BRed='\e[1;31m' + _BGreen='\e[1;32m' + _BYellow='\e[1;33m' + _BBlue='\e[1;34m' + _BPurple='\e[1;35m' + _BCyan='\e[1;36m' +} + +if [ ! -p /dev/stdout ]; then + set_terminal_colors +fi + +# reST +# ---- + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_title() { + # usage: rst_title <header-text> [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${_BGreen}${1//?/=}\n${_BCyan}${1}${_BGreen}\n${1//?/=}${_creset}\n";; + chapter) printf "\n${_BCyan}${1}\n${_BGreen}${1//?/=}${_creset}\n";; + section) printf "\n${_BCyan}${1}\n${_BGreen}${1//?/-}${_creset}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } +warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } +info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } + +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while read -r -n1 -t 0.1; do : ; done + fi +} + +wait_key(){ + # usage: waitKEY [<timeout in sec>] + + clean_stdin + local _t=$1 + local msg="${MSG}" + [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}" + + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + printf "$msg" + # shellcheck disable=SC2086 + read -r -s -n1 $_t + echo + clean_stdin +} + +ask_yn() { + # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + case "${FORCE_SELECTION:-${2}}" in + Y) return ${EXIT_YES} ;; + N) return ${EXIT_NO} ;; + Yn) + local exit_val=${EXIT_YES} + local choice="[${_BGreen}YES${_creset}/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[${_BGreen}NO${_creset}/yes]" + local default="No" + ;; + esac + echo + while true; do + clean_stdin + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -r -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + clean_stdin + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <<EOF | python -i + # print("hello") + # EOF + # ... + # >>> print("hello") + # hello + + local _t="0"; + if [[ ! -z $1 ]] ; then _t="$1"; fi + + (while read -r line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: <cmd> | prefix_stdout [prefix] + + local prefix="${_BYellow}-->|${_creset}" + + if [[ ! -z $1 ]] ; then prefix="${_BYellow}$1${_creset}"; fi + + # shellcheck disable=SC2162 + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line <line> <file> + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download <url> <local-filename> + + local exit_value=0 + + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if [[ ! $exit_value = 0 ]]; then + err_msg "failed to download: $1" + fi + fi +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one <name> "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "${_BGreen}Menu::${_creset}" + for ((i=1; i<= $((max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + clean_stdin + printf "$1 [${_BGreen}$default${_creset}] " + + if (( 10 > max )); then + # shellcheck disable=SC2086 + read -r -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read -r $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + eval "$env_name"='${list[${REPLY}]}' + echo + clean_stdin +} + +install_template() { + + # usage: + # + # install_template [--no-eval] [--variant=<name>] \ + # {file} [{owner} [{group} [{chmod}]]] + # + # E.g. the origin of variant 'raw' of /etc/updatedb.conf is:: + # + # ${TEMPLATES}/etc/updatedb.conf:raw + # + # To install variant 'raw' of /etc/updatedb.conf without evaluated + # replacements you can use:: + # + # install_template --variant=raw --no-eval \ + # /etc/updatedb.conf root root 644 + + local _reply="" + local do_eval=1 + local variant="" + local pos_args=("$0") + + for i in "$@"; do + case $i in + --no-eval) do_eval=0; shift ;; + --variant=*) variant=":${i#*=}"; shift ;; + *) pos_args+=("$i") ;; + esac + done + + local dst="${pos_args[1]}" + local template_origin="${TEMPLATES}${dst}${variant}" + local template_file="${TEMPLATES}${dst}" + + local owner="${pos_args[2]-$(id -un)}" + local group="${pos_args[3]-$(id -gn)}" + local chmod="${pos_args[4]-644}" + + info_msg "install (eval=$do_eval): ${dst}" + [[ ! -z $variant ]] && info_msg "variant: ${variant}" + + if [[ ! -f "${template_origin}" ]] ; then + err_msg "${template_origin} does not exists" + err_msg "... can't install $dst" + wait_key 30 + return 42 + fi + + if [[ "$do_eval" == "1" ]]; then + template_file="${CACHE}${dst}${variant}" + info_msg "BUILD template ${template_file}" + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")" + else + mkdir -p "$(dirname "${template_file}")" + fi + # shellcheck disable=SC2086 + eval "echo \"$(cat ${template_origin})\"" > "${template_file}" + if [[ ! -z ${SUDO_USER} ]]; then + chown "${SUDO_USER}:${SUDO_USER}" "${template_file}" + fi + else + template_file=$template_origin + fi + + mkdir -p "$(dirname "${dst}")" + + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "diffrent file ${dst} allready exists on this host" + + while true; do + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" \ + "interactiv shell" \ + "diff files" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + break + ;; + "leave file unchanged") + break + ;; + "interactiv shell") + echo "// edit ${dst} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${owner}" -i + $DIFF_CMD "${dst}" "${template_file}" + echo + echo "${_BBlack}did you edit file ...${_creset}" + printf " ${template_file}" + if ask_yn "... to your needs?"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout + esac + done +} + + +service_is_available() { + + # usage: service_is_available <URL> + + local URL="$1" + if [[ -z $URL ]]; then + err_msg "service_is_available: missing arguments" + return 42 + fi + + http_code=$(curl -H 'Cache-Control: no-cache' \ + --silent -o /dev/null --head --write-out '%{http_code}' --insecure \ + "${URL}") + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "got $http_code from ${URL}" + fi + case "$http_code" in + 404|410|423) exit_val=$http_code;; + esac + return "$exit_val" +} + +# golang +# ------ + +go_is_available() { + + # usage: go_is_available $SERVICE_USER && echo "go is installed!" + + sudo -i -u "${1}" which go &>/dev/null +} + +install_go() { + + # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + + local _svcpr=" |${3}| " + + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${1}" "${2}" + + tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr" +echo \$PATH +echo \$GOPATH +mkdir -p \$HOME/local +rm -rf \$HOME/local/go +tar -C \$HOME/local -xzf ${CACHE}/${2} +EOF + sudo -i -u "${3}" <<EOF | prefix_stdout +! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF +} + +# system accounts +# --------------- + +service_account_is_available() { + + # usage: service_account_is_available "$SERVICE_USER" && echo "OK" + + sudo -i -u "$1" echo \$HOME &>/dev/null +} + +drop_service_account() { + + # usage: drop_service_account "${SERVICE_USER}" + + rst_title "Drop ${1} HOME" section + if ask_yn "Do you really want to drop ${1} home folder?"; then + userdel -r -f "${1}" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "${1}") unchanged." + fi +} + +interactive_shell(){ + + # usage: interactive_shell "${SERVICE_USER}" + + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${1}" -i +} + + +# systemd +# ------- + +SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}" + +systemd_install_service() { + + # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + rst_title "Install System-D Unit ${1}" section + echo + install_template "${2}" root root 644 + wait_key + systemd_activate_service "${1}" +} + +systemd_remove_service() { + + # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then + return 42 + fi + systemd_deactivate_service "${1}" + rm "${2}" 2>&1 | prefix_stdout +} + +systemd_activate_service() { + + # usage: systemd_activate_service "${SERVICE_NAME}"w + + rst_title "Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl enable ${1}.service +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + +systemd_deactivate_service() { + + # usage: systemd_deactivate_service "${SERVICE_NAME}" + + rst_title "De-Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl stop ${1}.service +systemctl disable ${1}.service +EOF +} + + +# Apache +# ------ + +# FIXME: Arch Linux & RHEL should be added + +if [[ -z "${APACHE_SITES_AVAILABE}" ]]; then + APACHE_SITES_AVAILABE="/etc/apache2/sites-available" +fi + +apache_is_installed() { + (command -v apachectl \ + && command -v a2ensite \ + && command -v a2dissite ) &>/dev/null +} + +apache_reload() { + + info_msg "reload apache .." + echo + sudo -H apachectl configtest + sudo -H service apache2 force-reload +} + +apache_install_site() { + + # usage: apache_install_site [<template option> ...] <mysite.conf> + # + # <template option>: see install_template + + local template_opts=() + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + install_template "${template_opts[@]}" \ + "${APACHE_SITES_AVAILABE}/${pos_args[1]}" \ + root root 644 + + apache_enable_site "${pos_args[1]}" + info_msg "installed apache site: ${pos_args[1]}" +} + +apache_remove_site() { + + # usage: apache_remove_site <mysite.conf> + + info_msg "remove apache site: $1" + apache_dissable_site "$1" + rm -f "${APACHE_SITES_AVAILABE}/$1" +} + +apache_enable_site() { + + # usage: apache_enable_site <mysite.conf> + + info_msg "enable apache site: $1" + sudo -H a2ensite -q "$1" + apache_reload +} + +apache_dissable_site() { + + # usage: apache_disable_site <mysite.conf> + + info_msg "disable apache site: $1" + sudo -H a2dissite -q "$1" + apache_reload +} + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" + +uWSGI_restart() { + + # usage: uWSGI_restart() + + info_msg "restart uWSGI service" + systemctl restart uwsgi +} + +uWSGI_app_available() { + # usage: uWSGI_app_available <myapp.ini> + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_app_available: missing arguments" + return 42 + fi + [[ -f "${uWSGI_SETUP}/apps-available/${CONF}" ]] +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # + # <template option>: see install_template + + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + install_template "${template_opts[@]}" \ + "${uWSGI_SETUP}/apps-available/${pos_args[1]}" \ + root root 644 + + uWSGI_enable_app "${pos_args[1]}" + uWSGI_restart + info_msg "installed uWSGI app: ${pos_args[1]}" +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app <myapp.ini> + + local CONF="$1" + info_msg "remove uWSGI app: ${CONF}" + uWSGI_disable_app "${CONF}" + uWSGI_restart + rm -f "${uWSGI_SETUP}/apps-available/${CONF}" +} + +uWSGI_app_enabled() { + # usage: uWSGI_app_enabled <myapp.ini> + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_app_enabled: missing arguments" + return 42 + fi + [[ -f "${uWSGI_SETUP}/apps-enabled/${CONF}" ]] +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app <myapp.ini> + + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app: missing arguments" + return 42 + fi + pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + rm -f "$CONF" + # shellcheck disable=SC2226 + ln -s "../apps-available/${CONF}" + info_msg "enabled uWSGI app: ${CONF} (restart uWSGI required)" + popd >/dev/null +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app <myapp.ini> + + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app: missing arguments" + return 42 + fi + rm -f "${uWSGI_SETUP}/apps-enabled/${CONF}" + # FIXME: restart uwsgi service won't stop wsgi forked processes of user searx. + # I had to kill them manually here ... + pkill -f "${uWSGI_SETUP}/apps-enabled/${CONF}" -9 + info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)" +} + +# distro's package manager +# ------------------------ +# +# FIXME: Arch Linux & RHEL should be added +# + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + # shellcheck disable=SC2068 + apt-get install -y $@ +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + apt-get purge --autoremove --ignore-missing -y "$@" +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + dpkg -l "$1" &> /dev/null + return $? +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone <url> <name> [<branch> [<user>]] + # git_clone <url> <path> [<branch> [<user>]] + # + # First form uses $CACHE/<name> as destination folder, second form clones + # into <path>. If repository is allready cloned, pull from <branch> and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local bash_cmd="bash" + local remote="origin" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| " +cd "${dest}" +git checkout -m -B "$branch" --track "$remote/$branch" +git pull --all +EOF + else + info_msg "clone into: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| " +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" +EOF + fi +} diff --git a/utils/makefile.python b/utils/makefile.python index 4aa9d6b49..789cc4c90 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -192,7 +192,7 @@ quiet_cmd_pytest = TEST $@ # .. _installing: https://packaging.python.org/tutorials/installing-packages/ # quiet_cmd_pybuild = BUILD $@ - cmd_pybuild = $(PY_ENV_BIN)/$(PYTHON) setup.py \ + cmd_pybuild = $(PY_ENV_BIN)/python setup.py \ sdist -d $(PYDIST) \ bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) @@ -284,7 +284,7 @@ pyenv-python: pyenv-install # https://github.com/pypa/twine PHONY += upload-pypi -upload-pypi: pyclean pybuild +upload-pypi: pyclean pyenvinstall pybuild @$(PY_ENV_BIN)/twine upload $(PYDIST)/* .PHONY: $(PHONY) diff --git a/utils/morty.sh b/utils/morty.sh new file mode 100755 index 000000000..7a9db08a8 --- /dev/null +++ b/utils/morty.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL_PATH_MORTY="/morty" +PUBLIC_URL_MORTY="$(dirname "${PUBLIC_URL}")${PUBLIC_URL_PATH_MORTY}" + +MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" +# shellcheck disable=SC2034 +MORTY_TIMEOUT=5 + +SERVICE_NAME="morty" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_ENV_DEBUG=false + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# shellcheck disable=SC2034 +CONFIG_FILES=() + +# Apache Settings + +APACHE_MORTY_SITE="morty.conf" + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage:: + + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [morty] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + $(basename "$0") info [searx] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + all: complete setup of morty service + user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) +update morty + Update morty installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options +apache : ${PUBLIC_URL_MORTY} + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_MORTY_SITE} + +If needed, set the environment variable MORTY_LISTEN in the +${DOT_CONFIG#"$REPO_ROOT/"} file:: + + MORTY_LISTEN : ${MORTY_LISTEN} + SERVICE_USER : ${SERVICE_USER} +EOF + info_searx + [ ! -z "${1+x}" ] && err_msg "$1" +} + +info_searx() { + # shellcheck disable=SC1117 + cat <<EOF + +To activate morty in searx, add result_proxy to your settings.yml:: + + result_proxy: + url : ${PUBLIC_URL_MORTY}/ + +further read: ${DOCS_URL}/admin/morty.html + +EOF +} + +main() { + rst_title "$SERVICE_NAME" part + + required_commands \ + dpkg apt-get install git wget curl \ + || exit + + local _usage="ERROR: unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + morty) update_morty ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + info) + case $2 in + searx) info_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) enable_debug ;; + debug-off) disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + + *) usage "ERROR: unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_morty + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + info_searx + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + drop_service_account "${SERVICE_USER}" + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \ + --disabled-password --group --gecos 'Morty' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +morty_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/morty ]] +} + +_svcpr=" |${SERVICE_USER}| " + +install_morty() { + rst_title "Install morty in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +update_morty() { + rst_title "Update morty" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +set_service_env_debug() { + + # usage: set_service_env_debug [false|true] + + # shellcheck disable=SC2034 + local SERVICE_ENV_DEBUG="${1:-false}" + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + fi +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + MORTY_LISTEN : ${MORTY_LISTEN} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if morty_is_installed; then + info_msg "~$SERVICE_USER: morty app is installed" + else + err_msg "~$SERVICE_USER: morty app is not installed!" + fi + + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "morty URL --> http://${MORTY_LISTEN}" + info_msg "public URL --> ${PUBLIC_URL_MORTY}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + FORCE_SELECTION=Y disable_debug + fi + return 0 +} + + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "Enabling debug option needs to reinstall systemd service!" + set_service_env_debug true +} + +disable_debug() { + info_msg "Disabling debug option needs to reinstall systemd service!" + set_service_env_debug false +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + a2enmod headers + a2enmod proxy + a2enmod proxy_http + + echo + apache_install_site "${APACHE_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This removes apache site ${APACHE_MORTY_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "$APACHE_MORTY_SITE" +} +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..f02066ea0 --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,588 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +SEARX_INTERNAL_URL="${SEARX_INTERNAL_URL:-127.0.0.1:8888}" + +SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/ +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ +| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +SEARX_GIT_URL="${SEARX_GIT_URL:-https://github.com/asciimoo/searx.git}" +SEARX_GIT_BRANCH="${SEARX_GIT_BRANCH:-master}" +SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARX_SRC="${SERVICE_HOME}/searx-src" +SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" +SEARX_UWSGI_APP="searx.ini" +# shellcheck disable=SC2034 +SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" + +# FIXME: Arch Linux & RHEL should be added + +SEARX_APT_PACKAGES="\ + uwsgi uwsgi-plugin-python3 \ + git build-essential \ + libxslt-dev python3-dev python3-babel\ + zlib1g-dev libffi-dev libssl-dev \ +" + +# Apache Settings + +APACHE_APT_PACKAGES="\ + libapache2-mod-uwsgi \ +" + +APACHE_SEARX_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${uWSGI_SETUP}/apps-available/${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage:: + + $(basename "$0") shell + $(basename "$0") install [all|user|pyenv|searx-src|apache] + $(basename "$0") update [searx] + $(basename "$0") remove [all|user|pyenv|searx-src] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete (de-) installation of searx service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :searx-src: clone $SEARX_GIT_URL + :pyenv: create/remove virtualenv (python) in $SEARX_PYENV +update searx + Update searx installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + run some small tests and inspect service's status and log +option + set one of the available options +apache + :install: apache site with the searx uwsgi app + :remove: apache site ${APACHE_FILTRON_SITE} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SERVICE_USER : ${SERVICE_USER} + +EOF + [ ! -z "${1+x}" ] && err_msg "$1" +} + +main() { + rst_title "$SEARX_INSTANCE_NAME" part + + required_commands \ + dpkg systemctl apt-get install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + pyenv) create_pyenv ;; + searx-src) clone_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + searx) update_searx;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) remove_pyenv ;; + searx-src) remove_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) + activate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) deactivate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +_service_prefix=" |$SERVICE_USER| " + +install_all() { + rst_title "Install $SEARX_INSTANCE_NAME (service)" + pkg_install "$SEARX_APT_PACKAGES" + wait_key + assert_user + wait_key + clone_searx + wait_key + create_pyenv + wait_key + configure_searx + wait_key + test_local_searx + wait_key + install_searx_uwsgi + if ! service_is_available "http://$SEARX_INTERNAL_URL"; then + err_msg "URL http://$SEARX_INTERNAL_URL not available, check searx & uwsgi setup!" + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup +git stash push -m "BACKUP -- 'update server' at ($(date))" +git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH" +git pull "$SEARX_GIT_BRANCH" +${SEARX_SRC}/manage.sh update_packages +EOF + configure_searx + + rst_title "${SEARX_SETTINGS}" section + rst_para 'Diff between new setting file (<) and backup (>):' + echo + $DIFF_CMD "${SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup" + + local action + choose_one action "What should happen to the settings file? " \ + "keep new configuration" \ + "revert to the old configuration (backup file)" \ + "start interactiv shell" + case $action in + "keep new configuration") + info_msg "continue using new settings file" + ;; + "revert to the old configuration (backup file)") + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} +EOF + ;; + "start interactiv shell") + interactive_shell "${SERVICE_USER}" + ;; + esac + chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" + + # shellcheck disable=SC2016 + rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):' + echo + git_diff + wait_key + uWSGI_restart +} + +remove_all() { + rst_title "De-Install $SEARX_INSTANCE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then + return + fi + remove_searx_uwsgi + wait_key + drop_service_account "${SERVICE_USER}" + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home "$SERVICE_HOME" \ + --disabled-password --group --gecos 'searx' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + #export SERVICE_HOME + #echo "export SERVICE_HOME=$SERVICE_HOME" +} + +clone_is_available() { + [[ -f "$SEARX_SETTINGS" ]] +} + +# shellcheck disable=SC2164 +clone_searx() { + rst_title "Clone searx sources" section + echo + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + return 42 + fi + export SERVICE_HOME + + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ + "$SEARX_GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git config --list +EOF + popd > /dev/null +} + +remove_searx() { + rst_title "Drop searx sources" section + if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + rm -rf "$SEARX_SRC" + else + rst_para "Leave searx sources unchanged." + fi +} + +pyenv_is_available() { + [[ -f "${SEARX_PYENV}/bin/activate" ]] +} + +create_pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + err_msg "to create pyenv for searx, searx has to be cloned first" + return 42 + fi + info_msg "create pyenv in ${SEARX_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_PYENV}" +python3 -m venv "${SEARX_PYENV}" +grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +${SEARX_SRC}/manage.sh update_packages +EOF +} + +remove_pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARX_PYENV}" +} + +configure_searx() { + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS" + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS" +EOF +} + +test_local_searx() { + rst_title "Testing searx instance localy" section + echo + + if service_is_available "http://$SEARX_INTERNAL_URL" &>/dev/null; then + err_msg "URL/port http://$SEARX_INTERNAL_URL is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +timeout 10 python3 searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure $SEARX_INTERNAL_URL +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service() { + rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_enable_app "$SEARX_UWSGI_APP" + uWSGI_restart +} + +deactivate_service() { + rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_disable_app "$SEARX_UWSGI_APP" + uWSGI_restart +} + +git_diff() { + sudo -H -u "${SERVICE_USER}" -i <<EOF +cd ${SEARX_SRC} +git --no-pager diff +EOF +} + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +EOF + uWSGI_restart +} + +disable_debug() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF + uWSGI_restart +} + +inspect_service() { + rst_title "service status & log" + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SEARX_URL_PATH : ${SEARX_URL_PATH} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SEARX_INTERNAL_URL : ${SEARX_INTERNAL_URL} + +EOF + + apache_is_installed && info_msg "Apache is installed." + + if service_account_is_available "$SERVICE_USER"; then + info_msg "Service account $SERVICE_USER exists." + else + err_msg "Service account $SERVICE_USER does not exists!" + fi + + if pyenv_is_available; then + info_msg "~$SERVICE_USER: python environment is available." + else + err_msg "~$SERVICE_USER: python environment is not available!" + fi + + if clone_is_available; then + info_msg "~$SERVICE_USER: Searx software is installed." + else + err_msg "~$SERVICE_USER: Missing searx software!" + fi + + if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then + info_msg "uWSGI app $SEARX_UWSGI_APP is enabled." + else + err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!" + fi + + uWSGI_app_available "$SEARX_UWSGI_APP" \ + || err_msg "uWSGI app $SEARX_UWSGI_APP not available!" + + if ! service_is_available "http://${SEARX_INTERNAL_URL}"; then + err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_URL} is not available!" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi + + local _debug_on + if ask_yn "Enable searx debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${SEARX_INTERNAL_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + #journalctl -f -u "${SERVICE_NAME}" + tail -f /var/log/uwsgi/app/searx.log + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + +install_apache_site() { + rst_title "Install Apache site $APACHE_SEARX_SITE" + + rst_para "\ +This installs the searx uwsgi app as apache site. If your server ist public to +the internet you should instead use a reverse proxy (filtron) to block +excessively bot queries." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to install apache site for searx-uwsgi?"; then + return + fi + + pkg_install "$APACHE_APT_PACKAGES" + a2enmod uwsgi + + echo + apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site ${APACHE_SEARX_SITE}" + + rst_para "\ +This removes apache site ${APACHE_SEARX_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "${APACHE_SEARX_SITE}" +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/templates/etc/apache2/sites-available/morty.conf b/utils/templates/etc/apache2/sites-available/morty.conf new file mode 100644 index 000000000..6bcc77b78 --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/morty.conf @@ -0,0 +1,28 @@ +# -*- coding: utf-8; mode: apache -*- + +ProxyPreserveHost On + +<Location ${PUBLIC_URL_PATH_MORTY} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://${MORTY_LISTEN} + RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} + + # In Apache it seems, that setting HTTP_HOST header direct here does have no + # effect. I needed to set 'ProxyPreserveHost On' (see above). HTTP_HOST is + # needed by searx to render correct *Search URL* in the *Link* box and + # *saved preference*. + + # RequestHeader set Host ${PUBLIC_URL_PATH_MORTY} + +</Location> diff --git a/utils/templates/etc/apache2/sites-available/searx.conf:filtron b/utils/templates/etc/apache2/sites-available/searx.conf:filtron new file mode 100644 index 000000000..5ede66301 --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/searx.conf:filtron @@ -0,0 +1,33 @@ +# -*- coding: utf-8; mode: apache -*- + +ProxyPreserveHost On + +# SecRuleRemoveById 981054 +# SecRuleRemoveById 981059 +# SecRuleRemoveById 981060 +# SecRuleRemoveById 950907 + +<Location ${FILTRON_URL_PATH} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://${FILTRON_LISTEN} + RequestHeader set X-Script-Name ${FILTRON_URL_PATH} + + # In Apache it seems, that setting HTTP_HOST header direct here does have no + # effect. I needed to set 'ProxyPreserveHost On' (see above). HTTP_HOST is + # needed by searx to render correct *Search URL* in the *Link* box and + # *saved preference*. + + # RequestHeader set Host ${PUBLIC_HOST} + +</Location> diff --git a/utils/templates/etc/apache2/sites-available/searx.conf:uwsgi b/utils/templates/etc/apache2/sites-available/searx.conf:uwsgi new file mode 100644 index 000000000..21e01ac4e --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/searx.conf:uwsgi @@ -0,0 +1,27 @@ +# -*- coding: utf-8; mode: apache -*- + +<IfModule mod_uwsgi.c> + + # SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog + # CustomLog /dev/null combined env=dontlog + + <Location ${SEARX_URL_PATH}> + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Options FollowSymLinks Indexes + SetHandler uwsgi-handler + uWSGISocket ${SEARX_UWSGI_SOCKET} + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + </Location> + +</IfModule> diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json new file mode 100644 index 000000000..1c7005ae5 --- /dev/null +++ b/utils/templates/etc/filtron/rules.json @@ -0,0 +1,105 @@ +[ + { "name": "suspiciously frequent IP", + "filters": [], + "interval": 600, + "limit": 30, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"} + ] + }, + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": 61, + "limit": 999, + "subrules": [ + { + "name": "roboagent limit", + "interval": 61, + "limit": 1, + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "IP limit", + "interval": 61, + "limit": 9, + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "interval": 121, + "limit": 2, + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": 61, + "limit": 199, + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } +] diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini new file mode 100644 index 000000000..cff47f896 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -0,0 +1,62 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# plugin http +# ----------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http + +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html +http = ${SEARX_INTERNAL_URL} diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service new file mode 100644 index 000000000..3b0c6edcc --- /dev/null +++ b/utils/templates/lib/systemd/system/filtron.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target diff --git a/utils/templates/lib/systemd/system/morty.service b/utils/templates/lib/systemd/system/morty.service new file mode 100644 index 000000000..d463c5097 --- /dev/null +++ b/utils/templates/lib/systemd/system/morty.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT} + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target |