From f9b05a6c4443722ed866ed7195a6f273fcce779a Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 12 Apr 2021 17:24:00 +0200 Subject: [fix] utils/searx.sh - Double quote to prevent globbing Fix issues reported by `make test.sh` Signed-off-by: Markus Heiser --- utils/searx.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/utils/searx.sh b/utils/searx.sh index d2dbe79b6..578438698 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -418,9 +418,9 @@ install_settings() { err_msg "you have to install searx first" exit 42 fi - mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})" + mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" - if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then + if [[ ! -f "${SEARX_SETTINGS_PATH}" ]]; then info_msg "install settings ${SEARX_SETTINGS_TEMPLATE}" info_msg " --> ${SEARX_SETTINGS_PATH}" cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" -- cgit v1.2.3 From 036933599bc48a94cc722ad7d207b9de81ff85d2 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 22 Feb 2021 19:38:45 +0100 Subject: [enh] utils/lib.sh - commands pyenv, pyenv.drop pyenv.(un)install Implement a boilerplate to manage performance optimized virtualenv builds. Shell scripts can use (e.g.) 'pyenv.cmd' to execute command in the virtualenv without having to worry about whether and how the environment is provided. :: pyenv.cmd which python ..../local/py3/bin/python pyenv.cmd which pip ..../local/py3/bin/pip If pyenv.cmd released multiple times the installation will only rebuild if the function 'pyenv.OK' fails. Function 'pyenv.OK' make some test to validate that the virtualenv exists and works as expected. The check also fails if requirements listed requirements-dev.txt and requirements.txt has been edited. Among these tests 'pyenv.OK' calls 'pyenv.check' which implements a python script that validate the python installation. Here is an example how a 'pyenv.check' implementation could look like:: pyenv.check() { cat < OK') EOF } Signed-off-by: Markus Heiser --- utils/lib.sh | 215 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 211 insertions(+), 4 deletions(-) (limited to 'utils') diff --git a/utils/lib.sh b/utils/lib.sh index 8ae6bdd44..4475b5149 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -86,7 +86,7 @@ set_terminal_colors() { _Red='\e[0;31m' _Green='\e[0;32m' _Yellow='\e[0;33m' - _Blue='\e[0;34m' + _Blue='\e[0;94m' _Violet='\e[0;35m' _Cyan='\e[0;36m' @@ -95,12 +95,12 @@ set_terminal_colors() { _BRed='\e[1;31m' _BGreen='\e[1;32m' _BYellow='\e[1;33m' - _BBlue='\e[1;34m' + _BBlue='\e[1;94m' _BPurple='\e[1;35m' _BCyan='\e[1;36m' } -if [ ! -p /dev/stdout ]; then +if [ ! -p /dev/stdout ] && [ ! "$TERM" = 'dumb' ] && [ ! "$TERM" = 'unknown' ]; then set_terminal_colors fi @@ -152,6 +152,12 @@ err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } +build_msg() { + local tag="$1 " + shift + echo -e "${_Blue}${tag:0:10}${_creset}$*" +} + clean_stdin() { if [[ $(uname -s) != 'Darwin' ]]; then while read -r -n1 -t 0.1; do : ; done @@ -496,6 +502,203 @@ service_is_available() { return "$exit_val" } +# python +# ------ + +PY="${PY:=3}" +PYTHON="${PYTHON:=python$PY}" +PY_ENV="${PY_ENV:=local/py${PY}}" +PY_ENV_BIN="${PY_ENV}/bin" +PY_ENV_REQ="${PY_ENV_REQ:=${REPO_ROOT}/requirements*.txt}" + +# List of python packages (folders) or modules (files) installed by command: +# pyenv.install +PYOBJECTS="${PYOBJECTS:=.}" + +# folder where the python distribution takes place +PYDIST="${PYDIST:=dist}" + +# folder where the intermediate build files take place +PYBUILD="${PYBUILD:=build/py${PY}}" + +# https://www.python.org/dev/peps/pep-0508/#extras +#PY_SETUP_EXTRAS='[develop,test]' +PY_SETUP_EXTRAS="${PY_SETUP_EXTRAS:=[develop,test]}" + +PIP_BOILERPLATE=( pip wheel setuptools ) + +# shellcheck disable=SC2120 +pyenv() { + + # usage: pyenv [vtenv_opts ...] + # + # vtenv_opts: see 'pip install --help' + # + # Builds virtualenv with 'requirements*.txt' (PY_ENV_REQ) installed. The + # virtualenv will be reused by validating sha256sum of the requirement + # files. + + required_commands \ + sha256sum "${PYTHON}" \ + || exit + + local pip_req=() + + if ! pyenv.OK > /dev/null; then + rm -f "${PY_ENV}/${PY_ENV_REQ}.sha256" + pyenv.drop > /dev/null + build_msg PYENV "[virtualenv] installing ${PY_ENV_REQ} into ${PY_ENV}" + + "${PYTHON}" -m venv "$@" "${PY_ENV}" + "${PY_ENV_BIN}/python" -m pip install -U "${PIP_BOILERPLATE[@]}" + + for i in ${PY_ENV_REQ}; do + pip_req=( "${pip_req[@]}" "-r" "$i" ) + done + + ( + [ "$VERBOSE" = "1" ] && set -x + # shellcheck disable=SC2086 + "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \ + && sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256" + ) + fi + pyenv.OK +} + +_pyenv_OK='' +pyenv.OK() { + + # probes if pyenv exists and runs the script from pyenv.check + + [ "$_pyenv_OK" == "OK" ] && return 0 + + if [ ! -f "${PY_ENV_BIN}/python" ]; then + build_msg PYENV "[virtualenv] missing ${PY_ENV_BIN}/python" + return 1 + fi + + if [ ! -f "${PY_ENV}/requirements.sha256" ] \ + || ! sha256sum --check --status <"${PY_ENV}/requirements.sha256" 2>/dev/null; then + build_msg PYENV "[virtualenv] requirements.sha256 failed" + sed 's/^/ [virtualenv] - /' <"${PY_ENV}/requirements.sha256" + return 1 + fi + + pyenv.check \ + | "${PY_ENV_BIN}/python" 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[check] " + + local err=${PIPESTATUS[1]} + if [ "$err" -ne "0" ]; then + build_msg PYENV "[check] python test failed" + return "$err" + fi + + build_msg PYENV "OK" + _pyenv_OK="OK" + return 0 +} + +pyenv.drop() { + + build_msg PYENV "[virtualenv] drop ${PY_ENV}" + rm -rf "${PY_ENV}" + _pyenv_OK='' + +} + +pyenv.check() { + + # Prompts a python script with additional checks. Used by pyenv.OK to check + # if virtualenv is ready to install python objects. This function should be + # overwritten by the application script. + + local imp="" + + for i in "${PIP_BOILERPLATE[@]}"; do + imp="$imp, $i" + done + + cat < /dev/null + fi + if ! pyenv.install.OK > /dev/null; then + build_msg PYENV "[install] ${PYOBJECTS}" + if ! pyenv.OK >/dev/null; then + pyenv + fi + for i in ${PYOBJECTS}; do + build_msg PYENV "[install] pip install -e '$i${PY_SETUP_EXTRAS}'" + "${PY_ENV_BIN}/python" -m pip install -e "$i${PY_SETUP_EXTRAS}" + done + fi + pyenv.install.OK +} + +_pyenv_install_OK='' +pyenv.install.OK() { + + [ "$_pyenv_install_OK" == "OK" ] && return 0 + + local imp="" + local err="" + + if [ "." = "${PYOBJECTS}" ]; then + imp="import $(basename "$(pwd)")" + else + # shellcheck disable=SC2086 + for i in ${PYOBJECTS}; do imp="$imp, $i"; done + imp="import ${imp#,*} " + fi + ( + [ "$VERBOSE" = "1" ] && set -x + "${PY_ENV_BIN}/python" -c "import sys; sys.path.pop(0); $imp;" 2>/dev/null + ) + + err=$? + if [ "$err" -ne "0" ]; then + build_msg PYENV "[install] python installation test failed" + return "$err" + fi + + build_msg PYENV "[install] OK" + _pyenv_install_OK="OK" + return 0 +} + +pyenv.uninstall() { + + build_msg PYENV "[uninstall] ${PYOBJECTS}" + + if [ "." = "${PYOBJECTS}" ]; then + pyenv.cmd python setup.py develop --uninstall 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + else + pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + fi +} + + +pyenv.cmd() { + pyenv.install + ( set -e + # shellcheck source=/dev/null + source "${PY_ENV_BIN}/activate" + [ "$VERBOSE" = "1" ] && set -x + "$@" + ) +} + # golang # ------ @@ -1250,7 +1453,7 @@ pkg_install() { centos) # shellcheck disable=SC2068 yum install -y $@ - ;; + ;; esac } @@ -1382,6 +1585,10 @@ LXC_ENV_FOLDER= if in_container; then # shellcheck disable=SC2034 LXC_ENV_FOLDER="lxc-env/$(hostname)/" + PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}" + PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}" + PYDIST="${LXC_ENV_FOLDER}${PYDIST}" + PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}" fi lxc_init_container_env() { -- cgit v1.2.3 From d5eaa3fb314bb773ae6582528d4fae39224408e5 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 26 Feb 2021 17:23:12 +0100 Subject: [enh] utils/lib.sh - commands to build Sphinx-doc & deploy gh-pages Signed-off-by: Markus Heiser --- utils/lib.sh | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) (limited to 'utils') diff --git a/utils/lib.sh b/utils/lib.sh index 4475b5149..f2a879743 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -158,6 +158,16 @@ build_msg() { echo -e "${_Blue}${tag:0:10}${_creset}$*" } +dump_return() { + + # Use this as last command in your function to prompt an ERROR message if + # the exit code is not zero. + + local err=$1 + [ "$err" -ne "0" ] && err_msg "${FUNCNAME[1]} exit with error ($err)" + return "$err" +} + clean_stdin() { if [[ $(uname -s) != 'Darwin' ]]; then while read -r -n1 -t 0.1; do : ; done @@ -699,6 +709,98 @@ pyenv.cmd() { ) } +# Sphinx doc +# ---------- + +GH_PAGES="build/gh-pages" +DOCS_DIST="${DOCS_DIST:=dist/docs}" +DOCS_BUILD="${DOCS_BUILD:=build/docs}" + +docs.html() { + build_msg SPHINX "HTML ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")" + pyenv.install + docs.prebuild + # shellcheck disable=SC2086 + PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-build \ + ${SPHINX_VERBOSE} ${SPHINXOPTS} \ + -b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}" + dump_return $? +} + +docs.live() { + build_msg SPHINX "autobuild ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")" + pyenv.install + docs.prebuild + # shellcheck disable=SC2086 + PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-autobuild \ + ${SPHINX_VERBOSE} ${SPHINXOPTS} --open-browser --host 0.0.0.0 \ + -b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}" + dump_return $? +} + +docs.clean() { + build_msg CLEAN "docs -- ${DOCS_BUILD} ${DOCS_DIST}" + # shellcheck disable=SC2115 + rm -rf "${GH_PAGES}" "${DOCS_BUILD}" "${DOCS_DIST}" + dump_return $? +} + +docs.prebuild() { + # Dummy function to run some actions before sphinx-doc build gets started. + # This finction needs to be overwritten by the application script. + true + dump_return $? +} + +# shellcheck disable=SC2155 +docs.gh-pages() { + + # The commit history in the gh-pages branch makes no sense, the history only + # inflates the repository unnecessarily. Therefore a *new orphan* branch + # is created each time we deploy on the gh-pages branch. + + docs.clean + docs.prebuild + docs.html + + [ "$VERBOSE" = "1" ] && set -x + local head="$(git rev-parse HEAD)" + local branch="$(git name-rev --name-only HEAD)" + local remote="$(git config branch."${branch}".remote)" + local remote_url="$(git config remote."${remote}".url)" + + build_msg GH-PAGES "prepare folder: ${GH_PAGES}" + build_msg GH-PAGES "remote of the gh-pages branch: ${remote} / ${remote_url}" + build_msg GH-PAGES "current branch: ${branch}" + + # prepare the *orphan* gh-pages working tree + ( + git worktree remove -f "${GH_PAGES}" + git branch -D gh-pages + ) &> /dev/null || true + git worktree add --no-checkout "${GH_PAGES}" "${remote}/master" + + pushd "${GH_PAGES}" &> /dev/null + git checkout --orphan gh-pages + git rm -rfq . + popd &> /dev/null + + cp -r "${DOCS_DIST}"/* "${GH_PAGES}"/ + touch "${GH_PAGES}/.nojekyll" + cat > "${GH_PAGES}/404.html" < +EOF + + pushd "${GH_PAGES}" &> /dev/null + git add --all . + git commit -q -m "gh-pages build from: ${branch}@${head} (${remote_url})" + git push -f "${remote}" gh-pages + popd &> /dev/null + + set +x + build_msg GH-PAGES "deployed" +} + # golang # ------ @@ -1589,6 +1691,8 @@ if in_container; then PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}" PYDIST="${LXC_ENV_FOLDER}${PYDIST}" PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}" + DOCS_DIST="${LXC_ENV_FOLDER}${DOCS_DIST}" + DOCS_BUILD="${LXC_ENV_FOLDER}${DOCS_BUILD}" fi lxc_init_container_env() { -- cgit v1.2.3 From 438d35c38670a21fb646c3d3db0c07d0611c5cb4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 22 Feb 2021 20:10:18 +0100 Subject: [fix] do not spit out error messages for utils/lxc.sh --help Error:: $ utils/lxc.sh --help ERROR: missing command lxc ... This breaks also docs build:: $ make docs ... WARNING: Unexpected return code 42 from command '../utils/lxc.sh --help' ... Signed-off-by: Markus Heiser --- utils/lxc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/utils/lxc.sh b/utils/lxc.sh index 79cb1c04f..f065bf3c7 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -142,11 +142,11 @@ main() { local _usage="unknown or missing $1 command $2" # don't check prerequisite when in recursion - if [[ ! $1 == __* ]]; then + if [[ ! $1 == __* ]] && [[ ! $1 == --help ]]; then if ! in_container; then ! required_commands lxc && lxd_info && exit 42 fi - [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 + [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 fi case $1 in -- cgit v1.2.3 From d0e371f474627c878c179dc62610cb0437ae193b Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 17 Apr 2021 18:20:29 +0200 Subject: [mod] replace makefile boilerplate by 'manage' script Replaces the make targets with the bash scripts Signed-off-by: Markus Heiser --- utils/makefile.include | 59 ++----- utils/makefile.python | 269 -------------------------------- utils/makefile.sphinx | 199 ----------------------- utils/searx.sh | 2 +- utils/site-python/sphinx_build_tools.py | 48 ------ 5 files changed, 11 insertions(+), 566 deletions(-) delete mode 100644 utils/makefile.python delete mode 100644 utils/makefile.sphinx delete mode 100644 utils/site-python/sphinx_build_tools.py (limited to 'utils') diff --git a/utils/makefile.include b/utils/makefile.include index 879dcc23c..40f9d3302 100644 --- a/utils/makefile.include +++ b/utils/makefile.include @@ -1,4 +1,5 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +# SPDX-License-Identifier: AGPL-3.0-or-later ifeq (,$(wildcard /.lxcenv.mk)) PHONY += lxc-activate lxc-purge @@ -10,60 +11,26 @@ else include /.lxcenv.mk endif +PHONY += make-help ifeq (,$(wildcard /.lxcenv.mk)) make-help: else make-help: lxc-help endif + @echo 'options:' @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' @echo ' make V=2 [targets] 2 => give reason for rebuild of target' -quiet_cmd_common_clean = CLEAN $@ - cmd_common_clean = \ - find . -name '*.orig' -exec rm -f {} + ;\ - find . -name '*.rej' -exec rm -f {} + ;\ - find . -name '*~' -exec rm -f {} + ;\ - find . -name '*.bak' -exec rm -f {} + ;\ - -FMT = cat -ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0) -FMT = fmt -endif - -# MS-Windows -# -# For a minimal *make-environment*, I'am using the gnu-tools from: -# -# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc. -# https://github.com/gnu-mcu-eclipse/windows-build-tools/releases -# -# - git for Windows, which brings 'find', 'grep' etc. -# https://git-scm.com/download/win - - -# normpath -# -# System-dependent normalization of the path name -# -# usage: $(call normpath,/path/to/file) - -normpath = $1 -ifeq ($(OS),Windows_NT) - normpath = $(subst /,\,$1) -endif - - -# stolen from linux/Makefile -# - ifeq ("$(origin V)", "command line") - KBUILD_VERBOSE = $(V) + VERBOSE = $(V) endif -ifndef KBUILD_VERBOSE - KBUILD_VERBOSE = 0 +ifndef VERBOSE + VERBOSE = 0 endif -ifeq ($(KBUILD_VERBOSE),1) +export VERBOSE + +ifeq ($(VERBOSE),1) quiet = Q = else @@ -75,14 +42,8 @@ endif # # Convenient variables -comma := , -quote := " -#" this comment is only for emacs highlighting squote := ' #' this comment is only for emacs highlighting -empty := -space := $(empty) $(empty) -space_escape := _-_SPACE_-_ # Find any prerequisites that is newer than target or that does not exist. # PHONY targets skipped in both cases. @@ -107,7 +68,7 @@ any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^) # (5) No dir/.target.cmd file (used to store command line) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file -ifeq ($(KBUILD_VERBOSE),2) +ifeq ($(VERBOSE),2) why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ diff --git a/utils/makefile.python b/utils/makefile.python deleted file mode 100644 index f4fd02197..000000000 --- a/utils/makefile.python +++ /dev/null @@ -1,269 +0,0 @@ -# -*- coding: utf-8; mode: makefile-gmake -*- - -# list of python packages (folders) or modules (files) of this build -PYOBJECTS ?= - -SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python -export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH -export PY_ENV PYDIST PYBUILD - -# folder where the python distribution takes place -PYDIST = ./$(LXC_ENV_FOLDER)dist -# folder where the python intermediate build files take place -PYBUILD = ./$(LXC_ENV_FOLDER)build -# python version to use -PY ?=3 -# $(PYTHON) points to the python interpreter from the OS! The python from the -# OS is needed e.g. to create a virtualenv. For tasks inside the virtualenv the -# interpeter from '$(PY_ENV_BIN)/python' is used. -PYTHON ?= python$(PY) -PIP ?= pip$(PY) -PIP_INST ?= --user - -# https://www.python.org/dev/peps/pep-0508/#extras -#PY_SETUP_EXTRAS ?= \[develop,test\] -PY_SETUP_EXTRAS ?= - -PYDEBUG ?= --pdb -PYLINT_RC ?= .pylintrc - -TEST_FOLDER ?= ./tests -TEST ?= . - -PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY) -PY_ENV_BIN = $(PY_ENV)/bin -PY_ENV_ACT = . $(PY_ENV_BIN)/activate - -ifeq ($(OS),Windows_NT) - PYTHON = python - PY_ENV_BIN = $(PY_ENV)/Scripts - PY_ENV_ACT = $(PY_ENV_BIN)/activate -endif - -VTENV_OPTS ?= - -python-help:: - @echo 'makefile.python:' - @echo ' pyenv | pyenv[un]install' - @echo ' build $(PY_ENV) & [un]install python objects' - @echo ' targts using pyenv $(PY_ENV):' - @echo ' pylint - run pylint *linting*' - @echo ' pytest - run *tox* test on python objects' - @echo ' pydebug - run tests within a PDB debug session' - @echo ' pybuild - build python packages ($(PYDIST) $(PYBUILD))' - @echo ' pyclean - clean intermediate python objects' - @echo ' targets using system users environment:' - @echo ' py[un]install - [un]install python objects in editable mode' - @echo ' upload-pypi - upload $(PYDIST)/* files to PyPi' - @echo 'options:' - @echo ' make PY=3.7 [targets] => to eval targets with python 3.7 ($(PY))' - @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))' - @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)' - @echo ' make DEBUG= => target "debug": do not invoke PDB on errors' - @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]' - @echo ' when using target "pydebug", set breakpoints within py-source by adding::' - @echo ' DEBUG()' - -# ------------------------------------------------------------------------------ -# OS requirements -# ------------------------------------------------------------------------------ - -PHONY += msg-python-exe python-exe -msg-python-exe: - @echo "\n $(PYTHON) is required\n\n\ - Make sure you have $(PYTHON) installed, grab it from\n\ - https://www.python.org or install it from your package\n\ - manager. On debian based OS these requirements are\n\ - installed by::\n\n\ - sudo -H add-apt-repository ppa:deadsnakes/ppa\n\ - sudo -H apt update\n\ - sudo -H apt-get install $(PYTHON) $(PYTHON)-venv\n" - -ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1) -python-exe: msg-python-exe - $(error The '$(PYTHON)' command was not found) -else -python-exe: - @: -endif - -msg-pip-exe: - @echo "\n $(PIP) is required\n\n\ - Make sure you have updated pip installed, grab it from\n\ - https://pip.pypa.io or install it from your package\n\ - manager. On debian based OS these requirements are\n\ - installed by::\n\n\ - sudo -H apt-get install python$(PY)-pip\n" | $(FMT) - -ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1) -pip-exe: msg-pip-exe - $(error The '$(PIP)' command was not found) -else -pip-exe: - @: -endif - -# ------------------------------------------------------------------------------ -# commands -# ------------------------------------------------------------------------------ - -# $2 path to folder with setup.py, this uses pip from the OS -quiet_cmd_pyinstall = INSTALL $2 - cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS) - -# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) -quiet_cmd_pyenvinstall = PYENV install $2 - cmd_pyenvinstall = \ - if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \ - rm -f $(PY_ENV)/requirements.sha256; \ - $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) &&\ - sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\ - else \ - echo "PYENV $2 already installed"; \ - fi - -# Uninstall the package. Since pip does not uninstall the no longer needed -# depencies (something like autoremove) the depencies remain. - -# $2 package name to uninstall, this uses pip from the OS. -quiet_cmd_pyuninstall = UNINSTALL $2 - cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2 - -# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) -quiet_cmd_pyenvuninstall = PYENV uninstall $2 - cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2 - -# $2 path to folder where virtualenv take place -quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate - cmd_virtualenv = \ - if [ -d "./$(PY_ENV)" -a -x "./$(PY_ENV_BIN)/python" ]; then \ - echo "PYENV using virtualenv from $2"; \ - else \ - $(PYTHON) -m venv $(VTENV_OPTS) $2; \ - $(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \ - $(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \ - fi - -# $2 path to lint -quiet_cmd_pylint = LINT $@ - cmd_pylint = $(PY_ENV_BIN)/python -m pylint -j 0 --rcfile $(PYLINT_RC) $2 - -quiet_cmd_pytest = TEST $@ - cmd_pytest = $(PY_ENV_BIN)/python -m tox -vv - -# setuptools, pip, easy_install its a mess full of cracks, a documentation hell -# and broken by design ... all sucks, I really, really hate all this ... aaargh! -# -# About python packaging see `Python Packaging Authority`_. Most of the names -# here are mapped to ``setup(=..., =...)`` arguments in -# ``setup.py``. See `Packaging and distributing projects`_ about ``setup(...)`` -# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_. -# -# Further read: -# -# - pythonwheels_ -# - setuptools_ -# - packaging_ -# - sdist_ -# - installing_ -# -# .. _`Python Packaging Authority`: https://www.pypa.io -# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/ -# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/ -# .. _pythonwheels: https://pythonwheels.com/ -# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html -# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects -# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions -# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels -# .. _installing: https://packaging.python.org/tutorials/installing-packages/ -# -quiet_cmd_pybuild = BUILD $@ - cmd_pybuild = $(PY_ENV_BIN)/python setup.py \ - sdist -d $(PYDIST) \ - bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) - -quiet_cmd_pyclean = CLEAN $@ -# remove 'build' folder since bdist_wheel does not care the --bdist-dir - cmd_pyclean = \ - rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info ;\ - find . -name '*.pyc' -exec rm -f {} + ;\ - find . -name '*.pyo' -exec rm -f {} + ;\ - find . -name __pycache__ -exec rm -rf {} + - -# ------------------------------------------------------------------------------ -# targets -# ------------------------------------------------------------------------------ - -# for installation use the pip from the OS! -PHONY += pyinstall -pyinstall: pip-exe - $(call cmd,pyinstall,.) - -PHONY += pyuninstall -pyuninstall: pip-exe - $(call cmd,pyuninstall,$(PYOBJECTS)) - -# for installation use the pip from PY_ENV (not the OS)! -PHONY += pyenvinstall -pyenvinstall: $(PY_ENV) - $(call cmd,pyenvinstall,.) - -PHONY += pyenvuninstall -pyenvuninstall: $(PY_ENV) - $(call cmd,pyenvuninstall,$(PYOBJECTS)) - -PHONY += pyclean -pyclean: - $(call cmd,pyclean) - -# to build *local* environment, python from the OS is needed! -pyenv: $(PY_ENV) -$(PY_ENV): python-exe - $(call cmd,virtualenv,$(PY_ENV)) - -PHONY += pylint-exe -pylint-exe: $(PY_ENV) - @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint - -PHONY += pylint -pylint: pylint-exe - $(call cmd,pylint,$(PYOBJECTS)) - -PHONY += pybuild -pybuild: $(PY_ENV) - $(call cmd,pybuild) - -PHONY += pytest -pytest: $(PY_ENV) - $(call cmd,pytest) - -PHONY += pydebug -# set breakpoint with: -# DEBUG() -# e.g. to run tests in debug mode in emacs use: -# 'M-x pdb' ... 'make pydebug' -pydebug: $(PY_ENV) - DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST) - -# runs python interpreter from ./local/py/bin/python -pyenv-python: pyenvinstall - $(PY_ENV_BIN)/python -i - -# With 'dependency_links=' setuptools supports dependencies on packages hosted -# on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big -# drawback is, due to security reasons (I don't know where the security gate on -# PyPi is), this feature is not supported by pip [2]. Thats why an upload to -# PyPi is required and since uploads via setuptools is not recommended, we have -# to imstall / use twine ... its really a mess. -# -# [1] https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi -# [2] https://github.com/pypa/pip/pull/1519 - -# https://github.com/pypa/twine -PHONY += upload-pypi upload-pypi-test -upload-pypi: pyclean pyenvinstall pybuild - @$(PY_ENV_BIN)/twine upload $(PYDIST)/* - -upload-pypi-test: pyclean pyenvinstall pybuild - @$(PY_ENV_BIN)/twine upload -r testpypi $(PYDIST)/* -.PHONY: $(PHONY) diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx deleted file mode 100644 index b674cf7f0..000000000 --- a/utils/makefile.sphinx +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding: utf-8; mode: makefile-gmake -*- - -export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST - -# You can set these variables from the command line. -SPHINXOPTS ?= -SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build -SPHINX_CONF ?= conf.py - -DOCS_FOLDER = ./docs -DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs -DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs -GH_PAGES ?= build/gh-pages - -BOOKS_FOLDER = ./docs -BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books - -ifeq ($(KBUILD_VERBOSE),1) - SPHINX_VERBOSE = "-v" -else - SPHINX_VERBOSE = -endif - - -docs-help: - @echo 'makefile.sphinx:' - @echo ' docs-clean - clean intermediate doc objects' - @echo ' $(GH_PAGES) - create & upload github pages' - @echo ' sphinx-pdf - run sphinx latex & pdf targets' - @echo '' - @echo ' books/{name}.html : build only the HTML of document {name}' - @echo ' valid values for books/{name}.html are:' - @echo ' $(BOOKS_HTML)' | $(FMT) - @echo ' books/{name}.pdf : build only the PDF of document {name}' - @echo ' valid values for books/{name}.pdf are:' - @echo ' $(BOOKS_PDF) ' | $(FMT) - -# ------------------------------------------------------------------------------ -# requirements -# ------------------------------------------------------------------------------ - -PHONY += msg-texlive texlive - -ifeq ($(shell which xelatex >/dev/null 2>&1; echo $$?), 1) -texlive: msg-TeXLive - $(error The 'xelatex' command was not found) -else -texlive: - @: -endif - -msg-texlive: - $(Q)echo "\n\ -The TeX/PDF output and the *math* extension require TexLive and latexmk:\n\n\ - Make sure you have a updated TeXLive with XeTeX engine installed, grab it\n\ - it from https://www.tug.org/texlive or install it from your package manager.\n\n\ - Install latexmk from your package manager or visit https://ctan.org/pkg/latexmk\n\n\ - Sphinx-doc produce (Xe)LaTeX files which might use additional TeX-packages\n\ - and fonts. To process these LaTeX files, a TexLive installation with the\n\ - additional packages is required. On debian based OS these requirements\n\ - are installed by::\n\n\ - sudo -H apt-get install\n\ - latexmk\n\ - texlive-base texlive-xetex texlive-latex-recommended\n\ - texlive-extra-utils dvipng ttf-dejavu\n" - -# ------------------------------------------------------------------------------ -# commands -# ------------------------------------------------------------------------------ - -# $2 sphinx builder e.g. "html" -# $3 path where configuration file (conf.py) is located -# $4 sourcedir -# $5 dest subfolder e.g. "man" for man pages at $(DOCS_DIST)/man - -quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5) - cmd_sphinx = SPHINX_CONF=$(abspath $4/$(SPHINX_CONF))\ - $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ - -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5 - -quiet_cmd_sphinx_autobuild = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5) - cmd_sphinx_autobuild = PATH="$(PY_ENV_BIN):$(PATH)" $(PY_ENV_BIN)/sphinx-autobuild $(SPHINX_VERBOSE) --open-browser --host 0.0.0.0 $(SPHINXOPTS)\ - -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5 - -quiet_cmd_sphinx_clean = CLEAN $@ - cmd_sphinx_clean = rm -rf $(DOCS_BUILD) $(DOCS_DIST) $(GH_PAGES)/* $(GH_PAGES)/.buildinfo - -# ------------------------------------------------------------------------------ -# targets -# ------------------------------------------------------------------------------ - -# build PDF of whole documentation in: $(DOCS_DIST)/pdf - -PHONY += sphinx-pdf -sphinx-pdf: sphinx-latex - $(Q)cd $(DOCS_BUILD)/latex/; make all-pdf - $(Q)mkdir -p $(DOCS_DIST)/pdf - $(Q)cp $(DOCS_BUILD)/latex/*.pdf $(DOCS_DIST)/pdf - @echo "SPHINX *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)" - -PHONY += sphinx-latex -sphinx-latex: pyenvinstall texlive - $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ - -b latex \ - -c $(DOCS_FOLDER) \ - -d $(DOCS_BUILD)/.doctrees \ - $(DOCS_FOLDER) \ - $(DOCS_BUILD)/latex - -# Sphinx projects, we call them *books* (what is more common). Books are -# folders under $(BOOKS_FOLDER) containing a conf.py file. The HTML output goes -# to folder $(BOOKS_DIST)/ while PDF is placed (BOOKS_DIST)//pdf - -BOOKS=$(patsubst $(BOOKS_FOLDER)/%/conf.py,books/%,$(wildcard $(BOOKS_FOLDER)/*/conf.py)) - -# fine grained targets -BOOKS_HTML = $(patsubst %,%.html,$(BOOKS)) -BOOKS_CLEAN = $(patsubst %,%.clean,$(BOOKS)) -BOOKS_LATEX = $(patsubst %,%.latex,$(BOOKS)) -BOOKS_PDF = $(patsubst %,%.pdf,$(BOOKS)) -BOOKS_LIVE = $(patsubst %,%.live,$(BOOKS)) - -$(BOOKS_DIST): - mkdir -p $(BOOKS_DIST) - -PHONY += $(BOOKS_HTML) -$(BOOKS_HTML): pyenvinstall | $(BOOKS_DIST) - SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \ - $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ - -b html \ - -c $(DOCS_FOLDER) \ - -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \ - $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \ - $(BOOKS_DIST)/$(patsubst books/%.html,%,$@) - @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))" - -PHONY += $(BOOKS_HTML) -$(BOOKS_LIVE): pyenvinstall | $(BOOKS_DIST) - PATH="$(PY_ENV_BIN):$(PATH)" \ - SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \ - $(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\ - -b html \ - -c $(DOCS_FOLDER) \ - -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \ - $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \ - $(BOOKS_DIST)/$(patsubst books/%.live,%,$@) - -$(BOOKS_PDF): %.pdf : %.latex - $(Q)cd $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@); make all-pdf - $(Q)mkdir -p $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf - $(Q)cp -v $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@)/*.pdf $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf - @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf" - -PHONY += $(BOOKS_LATEX) -$(BOOKS_LATEX): pyenvinstall | $(BOOKS_DIST) - SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \ - $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ - -b latex \ - -c $(DOCS_FOLDER) \ - -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \ - $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \ - $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@) - @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))" - -$(BOOKS_CLEAN): - $(Q)rm -rf $(BOOKS_DIST)/$(patsubst books/%.clean,%,$@) \ - $(DOCS_BUILD)/books/$(patsubst books/%.clean,%,$@) \ - $(DOCS_BUILD)/latex/$(patsubst books/%.clean,%,$@) - -# github pages -PHONY += prepare-gh-pages -prepare-gh-pages: - cp -r $(DOCS_DIST)/* $(GH_PAGES)/ - touch $(GH_PAGES)/.nojekyll - echo "" > $(GH_PAGES)/404.html - -PHONY += gh-pages -gh-pages: docs-clean docs - - git worktree remove -f $(GH_PAGES) || exit 0 - - git branch -D gh-pages || exit 0 - git worktree add --no-checkout $(GH_PAGES) master - cd $(GH_PAGES); git checkout --orphan gh-pages && git rm -rfq . - $(MAKE) prepare-gh-pages - cd $(GH_PAGES);\ - git add --all . ;\ - git commit -q -m "make gh-pages: from $(shell git config --get remote.origin.url)@$(shell git rev-parse HEAD)" ;\ - git push -f origin gh-pages - -PHONY += ci-gh-pages -ci-gh-pages: docs-clean docs - rm -Rf $(GH_PAGES) - mkdir -p $(GH_PAGES) - $(MAKE) prepare-gh-pages - -PHONY += docs-clean -docs-clean: $(BOOKS_CLEAN) - $(call cmd,sphinx_clean) - -.PHONY: $(PHONY) diff --git a/utils/searx.sh b/utils/searx.sh index 578438698..2d2358344 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -481,7 +481,7 @@ pyenv_is_available() { create_pyenv() { rst_title "Create virtualenv (python)" section echo - if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + if [[ ! -f "${SEARX_SRC}/manage" ]]; then err_msg "to create pyenv for searx, searx has to be cloned first" return 42 fi diff --git a/utils/site-python/sphinx_build_tools.py b/utils/site-python/sphinx_build_tools.py deleted file mode 100644 index b9ebdeacc..000000000 --- a/utils/site-python/sphinx_build_tools.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8; mode: python -*- -"""Implement some sphinx-build tools. - -""" - -import os -import sys -from sphinx.util.pycompat import execfile_ - -# ------------------------------------------------------------------------------ -def load_sphinx_config(namespace): -# ------------------------------------------------------------------------------ - - u"""Load an additional configuration file into *namespace*. - - The name of the configuration file is taken from the environment - ``SPHINX_CONF``. The external configuration file extends (or overwrites) the - configuration values from the origin ``conf.py``. With this you are able to - maintain *build themes*. To your docs/conf.py add:: - - from sphinx_build_tools import load_sphinx_config - ... - - # Since loadConfig overwrites settings from the global namespace, it has to be - # the last statement in the conf.py file - - load_sphinx_config(globals()) - - """ - - config_file = os.environ.get("SPHINX_CONF", None) - if (config_file is not None - and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ): - config_file = os.path.abspath(config_file) - - if os.path.isfile(config_file): - sys.stdout.write( - "load additional sphinx-config: %s\n" - % config_file) - config = namespace.copy() - config['__file__'] = config_file - execfile_(config_file, config) - del config['__file__'] - namespace.update(config) - else: - sys.stderr.write( - "WARNING: additional sphinx-config not found: %s\n" - % config_file) -- cgit v1.2.3