Merge pull request #8 from return42/manage-script

Replace Makefile boilerplate by shell scripts

7 files changed, 330 insertions, 574 deletions

diff --git a/utils/lib.sh b/utils/lib.sh
index 8ae6bdd44..f2a879743 100755
--- a/utils/lib.sh
+++ b/utils/lib.sh
@@ -86,7 +86,7 @@ set_terminal_colors() {
     _Red='\e[0;31m'
     _Green='\e[0;32m'
     _Yellow='\e[0;33m'
-    _Blue='\e[0;34m'
+    _Blue='\e[0;94m'
     _Violet='\e[0;35m'
     _Cyan='\e[0;36m'
 
@@ -95,12 +95,12 @@ set_terminal_colors() {
     _BRed='\e[1;31m'
     _BGreen='\e[1;32m'
     _BYellow='\e[1;33m'
-    _BBlue='\e[1;34m'
+    _BBlue='\e[1;94m'
     _BPurple='\e[1;35m'
     _BCyan='\e[1;36m'
 }
 
-if [ ! -p /dev/stdout ]; then
+if [ ! -p /dev/stdout ] && [ ! "$TERM" = 'dumb' ] && [ ! "$TERM" = 'unknown' ]; then
     set_terminal_colors
 fi
 
@@ -152,6 +152,22 @@ err_msg()  { echo -e "${_BRed}ERROR:${_creset} $*" >&2; }
 warn_msg() { echo -e "${_BBlue}WARN:${_creset}  $*" >&2; }
 info_msg() { echo -e "${_BYellow}INFO:${_creset}  $*" >&2; }
 
+build_msg() {
+    local tag="$1        "
+    shift
+    echo -e "${_Blue}${tag:0:10}${_creset}$*"
+}
+
+dump_return() {
+
+    # Use this as last command in your function to prompt an ERROR message if
+    # the exit code is not zero.
+
+    local err=$1
+    [ "$err" -ne "0" ] && err_msg "${FUNCNAME[1]} exit with error ($err)"
+    return "$err"
+}
+
 clean_stdin() {
     if [[ $(uname -s) != 'Darwin' ]]; then
         while read -r -n1 -t 0.1; do : ; done
@@ -496,6 +512,295 @@ service_is_available() {
     return "$exit_val"
 }
 
+# python
+# ------
+
+PY="${PY:=3}"
+PYTHON="${PYTHON:=python$PY}"
+PY_ENV="${PY_ENV:=local/py${PY}}"
+PY_ENV_BIN="${PY_ENV}/bin"
+PY_ENV_REQ="${PY_ENV_REQ:=${REPO_ROOT}/requirements*.txt}"
+
+# List of python packages (folders) or modules (files) installed by command:
+# pyenv.install
+PYOBJECTS="${PYOBJECTS:=.}"
+
+# folder where the python distribution takes place
+PYDIST="${PYDIST:=dist}"
+
+# folder where the intermediate build files take place
+PYBUILD="${PYBUILD:=build/py${PY}}"
+
+# https://www.python.org/dev/peps/pep-0508/#extras
+#PY_SETUP_EXTRAS='[develop,test]'
+PY_SETUP_EXTRAS="${PY_SETUP_EXTRAS:=[develop,test]}"
+
+PIP_BOILERPLATE=( pip wheel setuptools )
+
+# shellcheck disable=SC2120
+pyenv() {
+
+    # usage:  pyenv [vtenv_opts ...]
+    #
+    #   vtenv_opts: see 'pip install --help'
+    #
+    # Builds virtualenv with 'requirements*.txt' (PY_ENV_REQ) installed.  The
+    # virtualenv will be reused by validating sha256sum of the requirement
+    # files.
+
+    required_commands \
+        sha256sum "${PYTHON}" \
+        || exit
+
+    local pip_req=()
+
+    if ! pyenv.OK > /dev/null; then
+        rm -f "${PY_ENV}/${PY_ENV_REQ}.sha256"
+        pyenv.drop > /dev/null
+        build_msg PYENV "[virtualenv] installing ${PY_ENV_REQ} into ${PY_ENV}"
+
+        "${PYTHON}" -m venv "$@" "${PY_ENV}"
+        "${PY_ENV_BIN}/python" -m pip install -U "${PIP_BOILERPLATE[@]}"
+
+        for i in ${PY_ENV_REQ}; do
+            pip_req=( "${pip_req[@]}" "-r" "$i" )
+        done
+
+        (
+            [ "$VERBOSE" = "1" ] && set -x
+            # shellcheck disable=SC2086
+            "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \
+                && sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256"
+        )
+    fi
+    pyenv.OK
+}
+
+_pyenv_OK=''
+pyenv.OK() {
+
+    # probes if pyenv exists and runs the script from pyenv.check
+
+    [ "$_pyenv_OK" == "OK" ] && return 0
+
+    if [ ! -f "${PY_ENV_BIN}/python" ]; then
+        build_msg PYENV "[virtualenv] missing ${PY_ENV_BIN}/python"
+        return 1
+    fi
+
+    if [ ! -f "${PY_ENV}/requirements.sha256" ] \
+        || ! sha256sum --check --status <"${PY_ENV}/requirements.sha256" 2>/dev/null; then
+        build_msg PYENV "[virtualenv] requirements.sha256 failed"
+        sed 's/^/          [virtualenv] - /' <"${PY_ENV}/requirements.sha256"
+        return 1
+    fi
+
+    pyenv.check \
+        | "${PY_ENV_BIN}/python" 2>&1 \
+        | prefix_stdout "${_Blue}PYENV     ${_creset}[check] "
+
+    local err=${PIPESTATUS[1]}
+    if [ "$err" -ne "0" ]; then
+        build_msg PYENV "[check] python test failed"
+        return "$err"
+    fi
+
+    build_msg PYENV "OK"
+    _pyenv_OK="OK"
+    return 0
+}
+
+pyenv.drop() {
+
+    build_msg PYENV "[virtualenv] drop ${PY_ENV}"
+    rm -rf "${PY_ENV}"
+    _pyenv_OK=''
+
+}
+
+pyenv.check() {
+
+    # Prompts a python script with additional checks. Used by pyenv.OK to check
+    # if virtualenv is ready to install python objects.  This function should be
+    # overwritten by the application script.
+
+    local imp=""
+
+    for i in "${PIP_BOILERPLATE[@]}"; do
+        imp="$imp, $i"
+    done
+
+    cat  <<EOF
+import ${imp#,*}
+
+EOF
+}
+
+pyenv.install() {
+
+    if ! pyenv.OK; then
+        py.clean > /dev/null
+    fi
+    if ! pyenv.install.OK > /dev/null; then
+        build_msg PYENV "[install] ${PYOBJECTS}"
+        if ! pyenv.OK >/dev/null; then
+            pyenv
+        fi
+        for i in ${PYOBJECTS}; do
+    	    build_msg PYENV "[install] pip install -e '$i${PY_SETUP_EXTRAS}'"
+    	    "${PY_ENV_BIN}/python" -m pip install -e "$i${PY_SETUP_EXTRAS}"
+        done
+    fi
+    pyenv.install.OK
+}
+
+_pyenv_install_OK=''
+pyenv.install.OK() {
+
+    [ "$_pyenv_install_OK" == "OK" ] && return 0
+
+    local imp=""
+    local err=""
+
+    if [ "." = "${PYOBJECTS}" ]; then
+        imp="import $(basename "$(pwd)")"
+    else
+        # shellcheck disable=SC2086
+        for i in ${PYOBJECTS}; do imp="$imp, $i"; done
+        imp="import ${imp#,*} "
+    fi
+    (
+        [ "$VERBOSE" = "1" ] && set -x
+        "${PY_ENV_BIN}/python" -c "import sys; sys.path.pop(0); $imp;" 2>/dev/null
+    )
+
+    err=$?
+    if [ "$err" -ne "0" ]; then
+        build_msg PYENV "[install] python installation test failed"
+        return "$err"
+    fi
+
+    build_msg PYENV "[install] OK"
+    _pyenv_install_OK="OK"
+    return 0
+}
+
+pyenv.uninstall() {
+
+    build_msg PYENV "[uninstall] ${PYOBJECTS}"
+
+    if [ "." = "${PYOBJECTS}" ]; then
+	pyenv.cmd python setup.py develop --uninstall 2>&1 \
+            | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+    else
+	pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \
+            | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+    fi
+}
+
+
+pyenv.cmd() {
+    pyenv.install
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+        [ "$VERBOSE" = "1" ] && set -x
+        "$@"
+    )
+}
+
+# Sphinx doc
+# ----------
+
+GH_PAGES="build/gh-pages"
+DOCS_DIST="${DOCS_DIST:=dist/docs}"
+DOCS_BUILD="${DOCS_BUILD:=build/docs}"
+
+docs.html() {
+    build_msg SPHINX "HTML ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")"
+    pyenv.install
+    docs.prebuild
+    # shellcheck disable=SC2086
+    PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-build \
+        ${SPHINX_VERBOSE} ${SPHINXOPTS} \
+	-b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"
+    dump_return $?
+}
+
+docs.live() {
+    build_msg SPHINX  "autobuild ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")"
+    pyenv.install
+    docs.prebuild
+    # shellcheck disable=SC2086
+    PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-autobuild \
+        ${SPHINX_VERBOSE} ${SPHINXOPTS} --open-browser --host 0.0.0.0 \
+	-b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"
+    dump_return $?
+}
+
+docs.clean() {
+    build_msg CLEAN "docs -- ${DOCS_BUILD} ${DOCS_DIST}"
+    # shellcheck disable=SC2115
+    rm -rf "${GH_PAGES}" "${DOCS_BUILD}" "${DOCS_DIST}"
+    dump_return $?
+}
+
+docs.prebuild() {
+    # Dummy function to run some actions before sphinx-doc build gets started.
+    # This finction needs to be overwritten by the application script.
+    true
+    dump_return $?
+}
+
+# shellcheck disable=SC2155
+docs.gh-pages() {
+
+    # The commit history in the gh-pages branch makes no sense, the history only
+    # inflates the repository unnecessarily.  Therefore a *new orphan* branch
+    # is created each time we deploy on the gh-pages branch.
+
+    docs.clean
+    docs.prebuild
+    docs.html
+
+    [ "$VERBOSE" = "1" ] && set -x
+    local head="$(git rev-parse HEAD)"
+    local branch="$(git name-rev --name-only HEAD)"
+    local remote="$(git config branch."${branch}".remote)"
+    local remote_url="$(git config remote."${remote}".url)"
+
+    build_msg GH-PAGES "prepare folder: ${GH_PAGES}"
+    build_msg GH-PAGES "remote of the gh-pages branch: ${remote} / ${remote_url}"
+    build_msg GH-PAGES "current branch: ${branch}"
+
+    # prepare the *orphan* gh-pages working tree
+    (
+        git worktree remove -f "${GH_PAGES}"
+        git branch -D gh-pages
+    ) &> /dev/null  || true
+    git worktree add --no-checkout "${GH_PAGES}" "${remote}/master"
+
+    pushd "${GH_PAGES}" &> /dev/null
+    git checkout --orphan gh-pages
+    git rm -rfq .
+    popd &> /dev/null
+
+    cp -r "${DOCS_DIST}"/* "${GH_PAGES}"/
+    touch "${GH_PAGES}/.nojekyll"
+    cat > "${GH_PAGES}/404.html" <<EOF
+<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>
+EOF
+
+    pushd "${GH_PAGES}" &> /dev/null
+    git add --all .
+    git commit -q -m "gh-pages build from: ${branch}@${head} (${remote_url})"
+    git push -f "${remote}" gh-pages
+    popd &> /dev/null
+
+    set +x
+    build_msg GH-PAGES "deployed"
+}
+
 # golang
 # ------
 
@@ -1250,7 +1555,7 @@ pkg_install() {
 	centos)
             # shellcheck disable=SC2068
             yum install -y $@
-            ;;    
+            ;;
     esac
 }
 
@@ -1382,6 +1687,12 @@ LXC_ENV_FOLDER=
 if in_container; then
     # shellcheck disable=SC2034
     LXC_ENV_FOLDER="lxc-env/$(hostname)/"
+    PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}"
+    PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}"
+    PYDIST="${LXC_ENV_FOLDER}${PYDIST}"
+    PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}"
+    DOCS_DIST="${LXC_ENV_FOLDER}${DOCS_DIST}"
+    DOCS_BUILD="${LXC_ENV_FOLDER}${DOCS_BUILD}"
 fi
 
 lxc_init_container_env() {
diff --git a/utils/lxc.sh b/utils/lxc.sh
index 79cb1c04f..f065bf3c7 100755
--- a/utils/lxc.sh
+++ b/utils/lxc.sh
@@ -142,11 +142,11 @@ main() {
     local _usage="unknown or missing $1 command $2"
 
     # don't check prerequisite when in recursion
-    if [[ ! $1 == __* ]]; then
+    if [[ ! $1 == __* ]] && [[ ! $1 == --help  ]]; then
         if ! in_container; then
             ! required_commands lxc && lxd_info && exit 42
         fi
-        [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 
+        [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42
     fi
 
     case $1 in
diff --git a/utils/makefile.include b/utils/makefile.include
index 879dcc23c..40f9d3302 100644
--- a/utils/makefile.include
+++ b/utils/makefile.include
@@ -1,4 +1,5 @@
 # -*- coding: utf-8; mode: makefile-gmake -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 ifeq (,$(wildcard /.lxcenv.mk))
 PHONY += lxc-activate lxc-purge
@@ -10,60 +11,26 @@ else
 	include /.lxcenv.mk
 endif
 
+PHONY += make-help
 ifeq (,$(wildcard /.lxcenv.mk))
 make-help:
 else
 make-help: lxc-help
 endif
+	@echo  'options:'
 	@echo  '  make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
 	@echo  '  make V=2   [targets] 2 => give reason for rebuild of target'
 
-quiet_cmd_common_clean = CLEAN     $@
-      cmd_common_clean = \
-	find . -name '*.orig' -exec rm -f {} +     ;\
-	find . -name '*.rej' -exec rm -f {} +      ;\
-	find . -name '*~' -exec rm -f {} +         ;\
-	find . -name '*.bak' -exec rm -f {} +      ;\
-
-FMT = cat
-ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0)
-FMT = fmt
-endif
-
-# MS-Windows
-#
-# For a minimal *make-environment*, I'am using the gnu-tools from:
-#
-# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc.
-#   https://github.com/gnu-mcu-eclipse/windows-build-tools/releases
-#
-# - git for Windows, which brings 'find', 'grep' etc.
-#   https://git-scm.com/download/win
-
-
-# normpath
-#
-# System-dependent normalization of the path name
-#
-#   usage: $(call normpath,/path/to/file)
-
-normpath = $1
-ifeq ($(OS),Windows_NT)
-  normpath = $(subst /,\,$1)
-endif
-
-
-# stolen from linux/Makefile
-#
-
 ifeq ("$(origin V)", "command line")
-  KBUILD_VERBOSE = $(V)
+  VERBOSE = $(V)
 endif
-ifndef KBUILD_VERBOSE
-  KBUILD_VERBOSE = 0
+ifndef VERBOSE
+  VERBOSE = 0
 endif
 
-ifeq ($(KBUILD_VERBOSE),1)
+export VERBOSE
+
+ifeq ($(VERBOSE),1)
   quiet =
   Q =
 else
@@ -75,14 +42,8 @@ endif
 #
 
 # Convenient variables
-comma   := ,
-quote   := "
-#" this comment is only for emacs highlighting
 squote  := '
 #' this comment is only for emacs highlighting
-empty   :=
-space   := $(empty) $(empty)
-space_escape := _-_SPACE_-_
 
 # Find any prerequisites that is newer than target or that does not exist.
 # PHONY targets skipped in both cases.
@@ -107,7 +68,7 @@ any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^)
 # (5) No dir/.target.cmd file (used to store command line)
 # (6) No dir/.target.cmd file and target not listed in $(targets)
 #     This is a good hint that there is a bug in the kbuild file
-ifeq ($(KBUILD_VERBOSE),2)
+ifeq ($(VERBOSE),2)
 why =                                                                        \
     $(if $(filter $@, $(PHONY)),- due to target is PHONY,                    \
         $(if $(wildcard $@),                                                 \
diff --git a/utils/makefile.python b/utils/makefile.python
deleted file mode 100644
index f4fd02197..000000000
--- a/utils/makefile.python
+++ /dev/null
@@ -1,269 +0,0 @@
-# -*- coding: utf-8; mode: makefile-gmake -*-
-
-# list of python packages (folders) or modules (files) of this build
-PYOBJECTS ?=
-
-SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python
-export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
-export PY_ENV PYDIST PYBUILD
-
-# folder where the python distribution takes place
-PYDIST   = ./$(LXC_ENV_FOLDER)dist
-# folder where the python intermediate build files take place
-PYBUILD  = ./$(LXC_ENV_FOLDER)build
-# python version to use
-PY       ?=3
-# $(PYTHON) points to the python interpreter from the OS!  The python from the
-# OS is needed e.g. to create a virtualenv.  For tasks inside the virtualenv the
-# interpeter from '$(PY_ENV_BIN)/python' is used.
-PYTHON   ?= python$(PY)
-PIP      ?= pip$(PY)
-PIP_INST ?= --user
-
-# https://www.python.org/dev/peps/pep-0508/#extras
-#PY_SETUP_EXTRAS ?= \[develop,test\]
-PY_SETUP_EXTRAS ?=
-
-PYDEBUG  ?= --pdb
-PYLINT_RC ?= .pylintrc
-
-TEST_FOLDER  ?= ./tests
-TEST         ?= .
-
-PY_ENV       = ./$(LXC_ENV_FOLDER)local/py$(PY)
-PY_ENV_BIN   = $(PY_ENV)/bin
-PY_ENV_ACT   = . $(PY_ENV_BIN)/activate
-
-ifeq ($(OS),Windows_NT)
-  PYTHON     = python
-  PY_ENV_BIN = $(PY_ENV)/Scripts
-  PY_ENV_ACT = $(PY_ENV_BIN)/activate
-endif
-
-VTENV_OPTS ?=
-
-python-help::
-	@echo  'makefile.python:'
-	@echo  '  pyenv | pyenv[un]install'
-	@echo  '     build $(PY_ENV) & [un]install python objects'
-	@echo  '  targts using pyenv $(PY_ENV):'
-	@echo  '    pylint    - run pylint *linting*'
-	@echo  '    pytest    - run *tox* test on python objects'
-	@echo  '    pydebug   - run tests within a PDB debug session'
-	@echo  '    pybuild   - build python packages ($(PYDIST) $(PYBUILD))'
-	@echo  '    pyclean   - clean intermediate python objects'
-	@echo  '  targets using system users environment:'
-	@echo  '    py[un]install - [un]install python objects in editable mode'
-	@echo  '    upload-pypi   - upload $(PYDIST)/* files to PyPi'
-	@echo  'options:'
-	@echo  '  make PY=3.7 [targets] => to eval targets with python 3.7 ($(PY))'
-	@echo  '  make PIP_INST=        => to set/unset pip install options ($(PIP_INST))'
-	@echo  '  make TEST=.           => choose test from $(TEST_FOLDER) (default "." runs all)'
-	@echo  '  make DEBUG=           => target "debug": do not invoke PDB on errors'
-	@echo  '  make PY_SETUP_EXTRAS  => also install extras_require from setup.py \[develop,test\]'
-	@echo  '  when using target "pydebug", set breakpoints within py-source by adding::'
-	@echo  '    DEBUG()'
-
-# ------------------------------------------------------------------------------
-# OS requirements
-# ------------------------------------------------------------------------------
-
-PHONY += msg-python-exe python-exe
-msg-python-exe:
-	@echo "\n  $(PYTHON) is required\n\n\
-  Make sure you have $(PYTHON) installed, grab it from\n\
-  https://www.python.org or install it from your package\n\
-  manager. On debian based OS these requirements are\n\
-  installed by::\n\n\
-    sudo -H add-apt-repository ppa:deadsnakes/ppa\n\
-    sudo -H apt update\n\
-    sudo -H apt-get install $(PYTHON) $(PYTHON)-venv\n"
-
-ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1)
-python-exe: msg-python-exe
-	$(error The '$(PYTHON)' command was not found)
-else
-python-exe:
-	@:
-endif
-
-msg-pip-exe:
-	@echo "\n  $(PIP) is required\n\n\
-  Make sure you have updated pip installed, grab it from\n\
-  https://pip.pypa.io or install it from your package\n\
-  manager. On debian based OS these requirements are\n\
-  installed by::\n\n\
-    sudo -H apt-get install python$(PY)-pip\n" | $(FMT)
-
-ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)
-pip-exe: msg-pip-exe
-	$(error The '$(PIP)' command was not found)
-else
-pip-exe:
-	@:
-endif
-
-# ------------------------------------------------------------------------------
-# commands
-# ------------------------------------------------------------------------------
-
-# $2 path to folder with setup.py, this uses pip from the OS
-quiet_cmd_pyinstall = INSTALL   $2
-      cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS)
-
-# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
-quiet_cmd_pyenvinstall = PYENV     install $2
-      cmd_pyenvinstall = \
-	if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \
-		rm -f $(PY_ENV)/requirements.sha256; \
-		$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) &&\
-		sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\
-	else \
-		echo "PYENV     $2 already installed"; \
-	fi
-
-# Uninstall the package.  Since pip does not uninstall the no longer needed
-# depencies (something like autoremove) the depencies remain.
-
-# $2 package name to uninstall, this uses pip from the OS.
-quiet_cmd_pyuninstall = UNINSTALL $2
-      cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2
-
-# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
-quiet_cmd_pyenvuninstall = PYENV     uninstall   $2
-      cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2
-
-# $2 path to folder where virtualenv take place
-quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
-      cmd_virtualenv  = \
-	if [ -d "./$(PY_ENV)" -a -x "./$(PY_ENV_BIN)/python" ]; then \
-		echo "PYENV     using virtualenv from $2"; \
-	else \
-		$(PYTHON) -m venv $(VTENV_OPTS) $2; \
-		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \
-		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \
-	fi
-
-# $2 path to lint
-quiet_cmd_pylint      = LINT      $@
-      cmd_pylint      = $(PY_ENV_BIN)/python -m pylint -j 0 --rcfile $(PYLINT_RC) $2
-
-quiet_cmd_pytest      = TEST      $@
-      cmd_pytest      = $(PY_ENV_BIN)/python -m tox -vv
-
-# setuptools, pip, easy_install its a mess full of cracks, a documentation hell
-# and broken by design ... all sucks, I really, really hate all this ... aaargh!
-#
-# About python packaging see `Python Packaging Authority`_.  Most of the names
-# here are mapped to ``setup(<name1>=..., <name2>=...)`` arguments in
-# ``setup.py``.  See `Packaging and distributing projects`_ about ``setup(...)``
-# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_.
-#
-# Further read:
-#
-# - pythonwheels_
-# - setuptools_
-# - packaging_
-# - sdist_
-# - installing_
-#
-# .. _`Python Packaging Authority`: https://www.pypa.io
-# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/
-# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/
-# .. _pythonwheels: https://pythonwheels.com/
-# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html
-# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects
-# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions
-# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels
-# .. _installing: https://packaging.python.org/tutorials/installing-packages/
-#
-quiet_cmd_pybuild     = BUILD     $@
-      cmd_pybuild     = $(PY_ENV_BIN)/python setup.py \
-			sdist -d $(PYDIST)  \
-			bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST)
-
-quiet_cmd_pyclean     = CLEAN     $@
-# remove 'build' folder since bdist_wheel does not care the --bdist-dir
-      cmd_pyclean     = \
-	rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info     ;\
-	find . -name '*.pyc' -exec rm -f {} +      ;\
-	find . -name '*.pyo' -exec rm -f {} +      ;\
-	find . -name __pycache__ -exec rm -rf {} +
-
-# ------------------------------------------------------------------------------
-# targets
-# ------------------------------------------------------------------------------
-
-# for installation use the pip from the OS!
-PHONY += pyinstall
-pyinstall: pip-exe
-	$(call cmd,pyinstall,.)
-
-PHONY += pyuninstall
-pyuninstall: pip-exe
-	$(call cmd,pyuninstall,$(PYOBJECTS))
-
-# for installation use the pip from PY_ENV (not the OS)!
-PHONY += pyenvinstall
-pyenvinstall: $(PY_ENV)
-	$(call cmd,pyenvinstall,.)
-
-PHONY += pyenvuninstall
-pyenvuninstall: $(PY_ENV)
-	$(call cmd,pyenvuninstall,$(PYOBJECTS))
-
-PHONY += pyclean
-pyclean:
-	$(call cmd,pyclean)
-
-# to build *local* environment, python from the OS is needed!
-pyenv: $(PY_ENV)
-$(PY_ENV): python-exe
-	$(call cmd,virtualenv,$(PY_ENV))
-
-PHONY += pylint-exe
-pylint-exe: $(PY_ENV)
-	@$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint
-
-PHONY += pylint
-pylint: pylint-exe
-	$(call cmd,pylint,$(PYOBJECTS))
-
-PHONY += pybuild
-pybuild: $(PY_ENV)
-	$(call cmd,pybuild)
-
-PHONY += pytest
-pytest: $(PY_ENV)
-	$(call cmd,pytest)
-
-PHONY += pydebug
-# set breakpoint with:
-#    DEBUG()
-# e.g. to run tests in debug mode in emacs use:
-#   'M-x pdb' ... 'make pydebug'
-pydebug: $(PY_ENV)
-	DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST)
-
-# runs python interpreter from ./local/py<N>/bin/python
-pyenv-python: pyenvinstall
-	$(PY_ENV_BIN)/python -i
-
-# With 'dependency_links=' setuptools supports dependencies on packages hosted
-# on other reposetories then PyPi, see "Packages Not On PyPI" [1].  The big
-# drawback is, due to security reasons (I don't know where the security gate on
-# PyPi is), this feature is not supported by pip [2]. Thats why an upload to
-# PyPi is required and since uploads via setuptools is not recommended, we have
-# to imstall / use twine ... its really a mess.
-#
-# [1] https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi
-# [2] https://github.com/pypa/pip/pull/1519
-
-# https://github.com/pypa/twine
-PHONY += upload-pypi upload-pypi-test
-upload-pypi: pyclean pyenvinstall pybuild
-	@$(PY_ENV_BIN)/twine upload $(PYDIST)/*
-
-upload-pypi-test: pyclean pyenvinstall pybuild
-	@$(PY_ENV_BIN)/twine upload -r testpypi $(PYDIST)/*
-.PHONY: $(PHONY)
diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx
deleted file mode 100644
index b674cf7f0..000000000
--- a/utils/makefile.sphinx
+++ /dev/null
@@ -1,199 +0,0 @@
-# -*- coding: utf-8; mode: makefile-gmake -*-
-
-export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST
-
-# You can set these variables from the command line.
-SPHINXOPTS  ?=
-SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
-SPHINX_CONF ?= conf.py
-
-DOCS_FOLDER = ./docs
-DOCS_BUILD  = ./$(LXC_ENV_FOLDER)build/docs
-DOCS_DIST   = ./$(LXC_ENV_FOLDER)dist/docs
-GH_PAGES    ?= build/gh-pages
-
-BOOKS_FOLDER = ./docs
-BOOKS_DIST   = ./$(LXC_ENV_FOLDER)dist/books
-
-ifeq ($(KBUILD_VERBOSE),1)
-  SPHINX_VERBOSE = "-v"
-else
-  SPHINX_VERBOSE =
-endif
-
-
-docs-help:
-	@echo  'makefile.sphinx:'
-	@echo  '  docs-clean	- clean intermediate doc objects'
-	@echo  '  $(GH_PAGES)	- create & upload github pages'
-	@echo  '  sphinx-pdf    - run sphinx latex & pdf targets'
-	@echo  ''
-	@echo  '  books/{name}.html : build only the HTML of document {name}'
-	@echo  '  valid values for books/{name}.html are:'
-	@echo  '    $(BOOKS_HTML)' | $(FMT)
-	@echo  '  books/{name}.pdf : build only the PDF of document {name}'
-	@echo  '  valid values for books/{name}.pdf are:'
-	@echo  '    $(BOOKS_PDF) ' | $(FMT)
-
-# ------------------------------------------------------------------------------
-# requirements
-# ------------------------------------------------------------------------------
-
-PHONY += msg-texlive texlive
-
-ifeq ($(shell which xelatex >/dev/null 2>&1; echo $$?), 1)
-texlive: msg-TeXLive
-	$(error The 'xelatex' command was not found)
-else
-texlive:
-	@:
-endif
-
-msg-texlive:
-	$(Q)echo "\n\
-The TeX/PDF output and the *math* extension require TexLive and latexmk:\n\n\
-  Make sure you have a updated TeXLive with XeTeX engine installed, grab it\n\
-  it from https://www.tug.org/texlive or install it from your package manager.\n\n\
-  Install latexmk from your package manager or visit https://ctan.org/pkg/latexmk\n\n\
-  Sphinx-doc produce (Xe)LaTeX files which might use additional TeX-packages\n\
-  and fonts. To process these LaTeX files, a TexLive installation with the\n\
-  additional packages is required. On debian based OS these requirements\n\
-  are installed by::\n\n\
-    sudo -H apt-get install\n\
-         latexmk\n\
-         texlive-base texlive-xetex texlive-latex-recommended\n\
-         texlive-extra-utils dvipng ttf-dejavu\n"
-
-# ------------------------------------------------------------------------------
-# commands
-# ------------------------------------------------------------------------------
-
-# $2 sphinx builder e.g. "html"
-# $3 path where configuration file (conf.py) is located
-# $4 sourcedir
-# $5 dest subfolder e.g. "man" for man pages at $(DOCS_DIST)/man
-
-quiet_cmd_sphinx = SPHINX    $@ --> file://$(abspath $(DOCS_DIST)/$5)
-      cmd_sphinx = SPHINX_CONF=$(abspath $4/$(SPHINX_CONF))\
-	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
-	-b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5
-
-quiet_cmd_sphinx_autobuild = SPHINX    $@ --> file://$(abspath $(DOCS_DIST)/$5)
-      cmd_sphinx_autobuild = PATH="$(PY_ENV_BIN):$(PATH)" $(PY_ENV_BIN)/sphinx-autobuild  $(SPHINX_VERBOSE) --open-browser --host 0.0.0.0 $(SPHINXOPTS)\
-	-b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5
-
-quiet_cmd_sphinx_clean = CLEAN     $@
-      cmd_sphinx_clean = rm -rf $(DOCS_BUILD) $(DOCS_DIST) $(GH_PAGES)/* $(GH_PAGES)/.buildinfo
-
-# ------------------------------------------------------------------------------
-# targets
-# ------------------------------------------------------------------------------
-
-# build PDF of whole documentation in: $(DOCS_DIST)/pdf
-
-PHONY += sphinx-pdf
-sphinx-pdf: sphinx-latex
-	$(Q)cd $(DOCS_BUILD)/latex/; make all-pdf
-	$(Q)mkdir -p $(DOCS_DIST)/pdf
-	$(Q)cp $(DOCS_BUILD)/latex/*.pdf $(DOCS_DIST)/pdf
-	@echo "SPHINX    *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)"
-
-PHONY += sphinx-latex
-sphinx-latex: pyenvinstall texlive
-	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
-	  -b latex \
-	  -c $(DOCS_FOLDER) \
-	  -d $(DOCS_BUILD)/.doctrees \
-	  $(DOCS_FOLDER) \
-	  $(DOCS_BUILD)/latex
-
-# Sphinx projects, we call them *books* (what is more common).  Books are
-# folders under $(BOOKS_FOLDER) containing a conf.py file. The HTML output goes
-# to folder $(BOOKS_DIST)/<name> while PDF is placed (BOOKS_DIST)/<name>/pdf
-
-BOOKS=$(patsubst $(BOOKS_FOLDER)/%/conf.py,books/%,$(wildcard $(BOOKS_FOLDER)/*/conf.py))
-
-# fine grained targets
-BOOKS_HTML  = $(patsubst %,%.html,$(BOOKS))
-BOOKS_CLEAN = $(patsubst %,%.clean,$(BOOKS))
-BOOKS_LATEX = $(patsubst %,%.latex,$(BOOKS))
-BOOKS_PDF   = $(patsubst %,%.pdf,$(BOOKS))
-BOOKS_LIVE  = $(patsubst %,%.live,$(BOOKS))
-
-$(BOOKS_DIST):
-	mkdir -p $(BOOKS_DIST)
-
-PHONY += $(BOOKS_HTML)
-$(BOOKS_HTML): pyenvinstall | $(BOOKS_DIST)
-	SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \
-	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
-	  -b html \
-	  -c $(DOCS_FOLDER) \
-	  -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
-	  $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \
-	  $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
-	@echo "SPHINX    $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
-
-PHONY += $(BOOKS_HTML)
-$(BOOKS_LIVE): pyenvinstall | $(BOOKS_DIST)
-	PATH="$(PY_ENV_BIN):$(PATH)" \
-	SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \
-	$(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\
-	  -b html \
-	  -c $(DOCS_FOLDER) \
-	  -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
-	  $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \
-	  $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
-
-$(BOOKS_PDF): %.pdf : %.latex
-	$(Q)cd $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@); make all-pdf
-	$(Q)mkdir -p $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf
-	$(Q)cp -v $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@)/*.pdf $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf
-	@echo "SPHINX    $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf"
-
-PHONY += $(BOOKS_LATEX)
-$(BOOKS_LATEX): pyenvinstall | $(BOOKS_DIST)
-	SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \
-	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
-	  -b latex \
-	  -c $(DOCS_FOLDER) \
-	  -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
-	  $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \
-	  $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
-	@echo "SPHINX    $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"
-
-$(BOOKS_CLEAN):
-	$(Q)rm -rf $(BOOKS_DIST)/$(patsubst books/%.clean,%,$@) \
-	       $(DOCS_BUILD)/books/$(patsubst books/%.clean,%,$@) \
-	       $(DOCS_BUILD)/latex/$(patsubst books/%.clean,%,$@)
-
-# github pages
-PHONY += prepare-gh-pages
-prepare-gh-pages:
-	cp -r $(DOCS_DIST)/* $(GH_PAGES)/
-	touch $(GH_PAGES)/.nojekyll
-	echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html
-
-PHONY += gh-pages
-gh-pages: docs-clean docs
-	- git worktree remove -f $(GH_PAGES) || exit 0
-	- git branch -D gh-pages || exit 0
-	git worktree add --no-checkout $(GH_PAGES) master
-	cd $(GH_PAGES); git checkout --orphan gh-pages && git rm -rfq .
-	$(MAKE) prepare-gh-pages
-	cd $(GH_PAGES);\
-		git add --all . ;\
-		git commit -q -m "make gh-pages: from $(shell git config --get remote.origin.url)@$(shell git rev-parse HEAD)" ;\
-		git push -f origin gh-pages
-
-PHONY += ci-gh-pages
-ci-gh-pages: docs-clean docs
-	rm -Rf $(GH_PAGES)
-	mkdir -p $(GH_PAGES)
-	$(MAKE) prepare-gh-pages
-
-PHONY += docs-clean
-docs-clean: $(BOOKS_CLEAN)
-	$(call cmd,sphinx_clean)
-
-.PHONY: $(PHONY)
diff --git a/utils/searx.sh b/utils/searx.sh
index d2dbe79b6..2d2358344 100755
--- a/utils/searx.sh
+++ b/utils/searx.sh
@@ -418,9 +418,9 @@ install_settings() {
         err_msg "you have to install searx first"
         exit 42
     fi
-    mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})"
+    mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")"
 
-    if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then
+    if [[ ! -f "${SEARX_SETTINGS_PATH}" ]]; then
         info_msg "install settings ${SEARX_SETTINGS_TEMPLATE}"
         info_msg "  --> ${SEARX_SETTINGS_PATH}"
         cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}"
@@ -481,7 +481,7 @@ pyenv_is_available() {
 create_pyenv() {
     rst_title "Create virtualenv (python)" section
     echo
-    if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then
+    if [[ ! -f "${SEARX_SRC}/manage" ]]; then
         err_msg "to create pyenv for searx, searx has to be cloned first"
         return 42
     fi
diff --git a/utils/site-python/sphinx_build_tools.py b/utils/site-python/sphinx_build_tools.py
deleted file mode 100644
index b9ebdeacc..000000000
--- a/utils/site-python/sphinx_build_tools.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# -*- coding: utf-8; mode: python -*-
-"""Implement some sphinx-build tools.
-
-"""
-
-import os
-import sys
-from sphinx.util.pycompat import execfile_
-
-# ------------------------------------------------------------------------------
-def load_sphinx_config(namespace):
-# ------------------------------------------------------------------------------
-
-    u"""Load an additional configuration file into *namespace*.
-
-    The name of the configuration file is taken from the environment
-    ``SPHINX_CONF``. The external configuration file extends (or overwrites) the
-    configuration values from the origin ``conf.py``.  With this you are able to
-    maintain *build themes*.  To your docs/conf.py add::
-
-        from sphinx_build_tools import load_sphinx_config
-        ...
-
-        # Since loadConfig overwrites settings from the global namespace, it has to be
-        # the last statement in the conf.py file
-
-        load_sphinx_config(globals())
-
-    """
-
-    config_file = os.environ.get("SPHINX_CONF", None)
-    if (config_file is not None
-        and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ):
-        config_file = os.path.abspath(config_file)
-
-        if os.path.isfile(config_file):
-            sys.stdout.write(
-                "load additional sphinx-config: %s\n"
-                % config_file)
-            config = namespace.copy()
-            config['__file__'] = config_file
-            execfile_(config_file, config)
-            del config['__file__']
-            namespace.update(config)
-        else:
-            sys.stderr.write(
-                "WARNING: additional sphinx-config not found: %s\n"
-                % config_file)