diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 8 | ||||
| -rw-r--r-- | docs/admin/buildhosts.rst | 50 | ||||
| -rw-r--r-- | docs/admin/filtron.rst | 41 | ||||
| -rw-r--r-- | docs/conf.py | 1 | ||||
| -rw-r--r-- | docs/dev/makefile.rst | 4 | ||||
| -rw-r--r-- | docs/dev/reST.rst | 5 | ||||
| -rw-r--r-- | requirements-dev.txt | 1 | ||||
| -rwxr-xr-x | utils/filtron.sh | 283 | ||||
| -rwxr-xr-x | utils/lib.sh | 354 | ||||
| -rw-r--r-- | utils/templates/etc/filtron/rules.json | 119 | ||||
| -rw-r--r-- | utils/templates/lib/systemd/system/filtron.service | 29 |
12 files changed, 878 insertions, 18 deletions
diff --git a/.gitignore b/.gitignore index 069dfd35b..3c998afae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # to sync with .dockerignore .coverage coverage/ +cache/ .installed.cfg engines.cfg env @@ -70,12 +70,16 @@ $(GH_PAGES):: # test # ---- -PHONY += test test.pylint test.pep8 test.unit test.robot +PHONY += test test.sh test.pylint test.pep8 test.unit test.robot # TODO: balance linting with pylint -test: test.pep8 test.unit test.robot +test: test.pep8 test.unit test.sh test.robot - make pylint +test.sh: + shellcheck -x utils/lib.sh + shellcheck -x utils/filtron.sh + test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index 5260da033..c1582cef2 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -35,8 +35,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF creation, its also needed for :ref:`math` when HTML output is build. To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered -as images (``sphinx.ext.imgmath`` extension). If your docs build (``make -docs``) shows warnings like this:: +as images (``sphinx.ext.imgmath`` extension). + +Here is the extract from the :origin:`docs/conf.py` file, setting math renderer +to ``imgmath``: + +.. literalinclude:: ../conf.py + :language: python + :start-after: # sphinx.ext.imgmath setup + :end-before: # sphinx.ext.imgmath setup END + +If your docs build (``make docs``) shows warnings like this:: WARNING: dot(1) not found, for better output quality install \ graphviz from http://www.graphviz.org @@ -47,8 +56,6 @@ docs``) shows warnings like this:: you need to install additional packages on your build host, to get better HTML output. -.. _system requirements: - .. tabs:: .. group-tab:: Ubuntu / debian @@ -94,10 +101,35 @@ For PDF output you also need: texlive-collection-fontsrecommended texlive-collection-latex \ dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts -.. _system requirements END: +.. _sh lint: -.. literalinclude:: ../conf.py - :language: python - :start-after: # sphinx.ext.imgmath setup - :end-before: # sphinx.ext.imgmath setup END +Lint shell scripts +================== + +.. _ShellCheck: https://github.com/koalaman/shellcheck + +To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool. + +.. SNIP sh lint requirements + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo apt install shellcheck + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S shellcheck + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo dnf install ShellCheck +.. SNAP sh lint requirements diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 07dcb9bc5..e8a2bfb15 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -2,12 +2,47 @@ How to protect an instance ========================== +.. _filtron: https://github.com/asciimoo/filtron + Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. -An application firewall, ``filtron`` solves exactly this problem. Information -on how to install it can be found at the `project page of filtron -<https://github.com/asciimoo/filtron>`__. +An application firewall, filtron_ solves exactly this problem. Filtron is just +a middleware between your web server (nginx, apache, ...) and searx. + + +filtron & go +============ + +.. _Go: https://golang.org/ +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + + +.. sidebar:: init system + + ATM the ``filtron.sh`` supports only systemd init process used by debian, + ubuntu and many other dists. If you have a working init.d file to start/stop + filtron service, please contribute. + +Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply +installed by ``go get`` package management (see `filtron README`_). If you use +filtron as middleware, a more isolated setup is recommended. + +#. Create a separated user account (``filtron``). +#. Download and install Go_ binary in users $HOME (``~filtron``). +#. Install filtron with the package management of Go_ (``go get -v -u + github.com/asciimoo/filtron``) +#. Setup a proper rule configuration :origin:`[ref] + <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``). +#. Setup a systemd service unit :origin:`[ref] + <utils/templates/lib/systemd/system/filtron.service>` + (``/lib/systemd/system/filtron.service``). + +To simplify such a installation and the maintenance of; use our script +``utils/filtron.sh``: + +.. program-output:: ../utils/filtron.sh --help + :ellipsis: 0,5 Sample configuration of filtron diff --git a/docs/conf.py b/docs/conf.py index af255e230..8c5f6b311 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,6 +61,7 @@ extensions = [ "pallets_sphinx_themes", "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja + "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive. 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling. "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index f5957001c..8e54aef48 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -11,6 +11,8 @@ Makefile Targets Before looking deeper at the targets, first read about :ref:`makefile setup` and :ref:`make pyenv`. + To install system requirements follow :ref:`buildhosts`. + With the aim to simplify development cycles, started with :pull:`1756` a ``Makefile`` based boilerplate was added. If you are not familiar with Makefiles, we recommend to read gnu-make_ introduction. @@ -170,7 +172,7 @@ e.g.: .. code:: sh - $ make test.pep8 test.unit + $ make test.pep8 test.unit test.sh . ./local/py3/bin/activate; ./manage.sh pep8_check [!] Running pep8 check . ./local/py3/bin/activate; ./manage.sh unit_tests diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 4dc1279f0..7c82ebe5a 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -1312,9 +1312,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from .. literalinclude:: ../admin/buildhosts.rst :language: reST - :start-after: .. _system requirements: - :end-before: .. _system requirements END: - + :start-after: .. SNIP sh lint requirements + :end-before: .. SNAP sh lint requirements .. _math: diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e8f617af..fe92bdce5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,3 +14,4 @@ selenium==3.141.0 linuxdoc @ git+http://github.com/return42/linuxdoc.git sphinx-jinja sphinx-tabs +sphinxcontrib-programoutput diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 000000000..5c8a738b0 --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,283 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2119 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +FILTRON_ETC="/etc/filtron" + +FILTRON_RULES="$FILTRON_ETC/rules.json" + +# shellcheck disable=SC2034 +FILTRON_API="127.0.0.1:4005" +# shellcheck disable=SC2034 +FILTRON_LISTEN="127.0.0.1:4004" +# shellcheck disable=SC2034 +FILTRON_TARGET="127.0.0.1:8888" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_NAME}" +SERVICE_HOME="/home/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + +# ---------------------------------------------------------------------------- +usage(){ +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage: + + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [filtron] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") show [service] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove all + complete setup of filtron service +update filtron + Update filtron installation of user ${SERVICE_USER} +activate + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +install user + add service user '$SERVICE_USER' at $SERVICE_HOME +show service + show service status and log +EOF + [ ! -z ${1+x} ] && echo -e "$1" +} + +main(){ + rst_title "$SERVICE_NAME" part + + local _usage="ERROR: unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell + ;; + show) + case $2 in + service) + sudo_or_exit + show_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + filtron) update_filtron ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) remove_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) activate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) deactivate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + *) usage "ERROR: unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go + wait_key + install_filtron + wait_key + install_service + wait_key +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + remove_service + wait_key + remove_user + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout + wait_key +} + +install_service() { + rst_title "Install System-D Unit ${SERVICE_NAME}.service" section + echo + install_template ${SERVICE_SYSTEMD_UNIT} root root 644 + wait_key + activate_service +} + +remove_service() { + if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then + return + fi + deactivate_service + rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout +} + +activate_service () { + rst_title "Activate $SERVICE_NAME (service)" section + echo + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl enable $SERVICE_NAME.service +systemctl restart $SERVICE_NAME.service +EOF + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl status $SERVICE_NAME.service +EOF +} + +deactivate_service () { + rst_title "De-Activate $SERVICE_NAME (service)" section + echo + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl stop $SERVICE_NAME.service +systemctl disable $SERVICE_NAME.service +EOF +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME --group --gecos 'Filtron' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u $SERVICE_USER +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +remove_user() { + rst_title "Drop $SERVICE_USER HOME" section + if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then + userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." + fi +} + +interactive_shell(){ + echo "// exit with CTRL-D" + sudo -H -u ${SERVICE_USER} -i +} + +_service_prefix=" |$SERVICE_USER| " + +install_go(){ + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${GO_PKG_URL}" "${GO_TAR}" + + tee_stderr 0.1 <<EOF | sudo -i -u "$SERVICE_USER" | prefix_stdout "$_service_prefix" +echo \$PATH +echo \$GOPATH +mkdir -p \$HOME/local +rm -rf \$HOME/local/go +tar -C \$HOME/local -xzf ${CACHE}/${GO_TAR} +EOF + echo + sudo -i -u "$SERVICE_USER" <<EOF | prefix_stdout +! which go >/dev/null && echo "Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF +} + +install_filtron() { + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix" +go get -v -u github.com/asciimoo/filtron +EOF + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix" +go get -v -u github.com/asciimoo/filtron +EOF +} + +show_service () { + rst_title "service status & log" + echo + systemctl status filtron.service + echo + read -r -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + echo + while true; do + trap break 2 + journalctl -f -u filtron + done + return 0 +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh new file mode 100755 index 000000000..fd6b92129 --- /dev/null +++ b/utils/lib.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2059,SC1117,SC2162,SC2004 + +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + while [ -h "${REPO_ROOT}" ] ; do + REPO_ROOT=$(readlink "${REPO_ROOT}") + done + REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P ) +fi + +if [[ -z ${TEMPLATES} ]]; then + TEMPLATES="${REPO_ROOT}/utils/templates" +fi + +if [[ -z "$CACHE" ]]; then + CACHE="${REPO_ROOT}/cache" +fi + +if [[ -z "$SYSTEMD_UNITS" ]]; then + SYSTEMD_UNITS="/lib/systemd/system" +fi + +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +rst_title() { + # usage: rst_title <header-text> [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${1//?/=}\n$1\n${1//?/=}\n";; + chapter) printf "\n${1}\n${1//?/=}\n";; + section) printf "\n${1}\n${1//?/-}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +err_msg() { echo -e "ERROR: $*" >&2; } +warn_msg() { echo -e "WARN: $*" >&2; } +info_msg() { echo -e "INFO: $*"; } + +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while read -n1 -t 0.1; do : ; done + fi +} + +wait_key(){ + # usage: waitKEY [<timeout in sec>] + + clean_stdin + local _t=$1 + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + # shellcheck disable=SC2086 + read -s -n1 $_t -p "** press any [KEY] to continue **" + echo + clean_stdin +} + +ask_yn() { + # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + case "${2}" in + Yn) + local exit_val=${EXIT_YES} + local choice="[YES/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[NO/yes]" + local default="No" + ;; + esac + echo + while true; do + clean_stdin + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + clean_stdin + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <<EOF | python -i + # print("hello") + # EOF + # ... + # >>> print("hello") + # hello + + local _t="0"; + if [[ ! -z $1 ]] ; then _t="$1"; fi + + (while read line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: <cmd> | prefix_stdout [prefix] + + local prefix=" | " + + if [[ ! -z $1 ]] ; then prefix="$1"; fi + + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line <line> <file> + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download <url> <local-filename> + + local exit_value=0 + + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if $exit_value; then + err_msg "failed to download: $1" + fi + fi +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one <name> "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "Menu::" + for ((i=1; i<= $(($max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " $i.) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + clean_stdin + printf "$1 [$default] " + + if (( 10 > $max )); then + # shellcheck disable=SC2086 + read -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( $REPLY > 0 )) && (( $REPLY < $max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + echo + clean_stdin + eval "$env_name"='${list[${REPLY}]}' +} + +install_template() { + + # usage: + # + # install_template [--no-eval] {file} [{owner} [{group} [{chmod}]]] + # + # install_template --no-eval /etc/updatedb.conf root root 644 + + local do_eval=1 + if [[ "$1" == "--no-eval" ]]; then + do_eval=0; shift + fi + local dst="${1}" + local owner=${2-$(id -un)} + local group=${3-$(id -gn)} + local chmod=${4-644} + local _reply="" + + info_msg "install: ${dst}" + + if [[ ! -f "${TEMPLATES}${dst}" ]] ; then + err_msg "${TEMPLATES}${dst} does not exists" + err_msg "... can't install $dst / exit installation with error 42" + wait_key 30 + return 42 + fi + + local template_file="${TEMPLATES}${dst}" + if [[ "$do_eval" == "1" ]]; then + info_msg "BUILD template ${template_file}" + if [[ -f "${TEMPLATES}${dst}" ]] ; then + template_file="${CACHE}${dst}" + mkdir -p "$(dirname "${template_file}")" + # shellcheck disable=SC2086 + eval "echo \"$(cat ${TEMPLATES}${dst})\"" > "${template_file}" + else + err_msg "failed ${template_file}" + return 42 + fi + fi + + mkdir -p "$(dirname "${dst}")" + + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "file ${dst} allready exists on this host" + + while true; do + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" \ + "interactiv shell" \ + "diff files" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + break + ;; + "leave file unchanged") + break + ;; + "interactiv shell") + echo "// edit ${dst} to your needs" + echo "// exit with CTRL-D" + sudo -H -u "${owner}" -i + $DIFF_CMD "${dst}" "${template_file}" + if ask_yn "did you edit ${template_file} to your needs?"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout + esac + done +} diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json new file mode 100644 index 000000000..634f5f2d6 --- /dev/null +++ b/utils/templates/etc/filtron/rules.json @@ -0,0 +1,119 @@ +[{ + "name":"suspiciously frequent queries", + "filters":[ + "Param:q", + "Path=^(/|/search)$" + ], + "interval":120, + "limit":9, + "actions":[ + {"name":"log"} + ] + }, + { + "name":"search request", + "filters":[ + "Param:q", + "Path=^(/|/search)$" + ], + "interval":120, + "limit":19, + "actions":[ + { + "name":"block", + "params":{ + "message":"common rate limit exceeded" + } + } + ], + "subrules":[ + { + "name":"roboagent limit", + "interval":60, + "limit":3, + "filters":[ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)" + ], + "actions":[ + {"name":"log"}, + { + "name":"block", + "params":{ + "message":"rate limit exceeded" + } + } + ] + }, + { + "name":"botlimit", + "interval":60, + "limit":0, + "stop":true, + "filters":[ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions":[ + {"name":"log"}, + { + "name":"block", + "params":{ + "message":"rate limit exceeded" + } + } + ] + }, + { + "name":"IP limit", + "interval":60, + "limit":13, + "stop":true, + "aggregations":[ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"}, + { + "name":"block", + "params":{ + "message":"rate limit exceeded" + } + } + ] + }, + { + "name":"rss/json limit", + "interval":60, + "limit":13, + "stop":true, + "filters":[ + "Param:format=(csv|json|rss)" + ], + "actions":[ + {"name":"log"}, + { + "name":"block", + "params":{ + "message":"rate limit exceeded" + } + } + ] + }, + { + "name":"useragent limit", + "interval":60, + "limit":13, + "aggregations":[ + "Header:User-Agent" + ], + "actions":[ + {"name":"log"}, + { + "name":"block", + "params":{ + "message":"rate limit exceeded" + } + } + ] + } + ] +}] diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service new file mode 100644 index 000000000..3b0c6edcc --- /dev/null +++ b/utils/templates/lib/systemd/system/filtron.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target |