From 7751b2955948664076409abaff4d4e4161a31c6f Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 8 Feb 2020 19:12:28 +0100 Subject: LXC: add LXC tooling box (initial, WIP) Signed-off-by: Markus Heiser --- utils/lxc.sh | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 utils/lxc.sh (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh new file mode 100755 index 000000000..d79262534 --- /dev/null +++ b/utils/lxc.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +# name of https://images.linuxcontainers.org +LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" +HOST_PREFIX="${HOST_PREFIX:-searx}" + +TEST_IMAGES=( + "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" + "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" + "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" + #"$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" + #"ubuntu-minimal:18.04" "ubu1804" + #"ubuntu-minimal:19.10" "ubu1910" +) + +REMOTE_IMAGES=() +LOCAL_IMAGES=() + +for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${TEST_IMAGES[i]}") + LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${TEST_IMAGES[i+1]}") +done + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + cat </dev/null; then + info_msg "image ${TEST_IMAGES[i]} already copied --> ${TEST_IMAGES[i+1]}" + else + info_msg "copy image locally ${TEST_IMAGES[i]} --> ${TEST_IMAGES[i+1]}" + lxc image copy "${TEST_IMAGES[i]}" local: \ + --alias "${TEST_IMAGES[i+1]}" prefix_stdout + fi + done + #lxc image list local: +} + +lxc_delete_images_localy() { + echo + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "delete image 'local:$i'" + lxc image delete "local:$i" + done + #lxc image list local: +} + +lxc_init_containers() { + echo + for i in "${LOCAL_IMAGES[@]}"; do + if lxc info "$HOST_PREFIX-$i" &>/dev/null; then + info_msg "conatiner '$HOST_PREFIX-$i' already exists" + else + info_msg "create conatiner instance: $HOST_PREFIX-$i" + lxc init "local:$i" "$HOST_PREFIX-$i" + fi + done + #lxc list "$HOST_PREFIX" +} + +lxc_delete_containers() { + echo + for i in "${LOCAL_IMAGES[@]}"; do + if lxc info "$HOST_PREFIX-$i" &>/dev/null; then + info_msg "stop & delete instance '$HOST_PREFIX-$i'" + lxc stop "$HOST_PREFIX-$i" &>/dev/null + lxc delete "$HOST_PREFIX-$i" | prefix_stdout + else + warn_msg "instance '$HOST_PREFIX-$i' does not exist / can't delete :o" + fi + done + #lxc list "$HOST_PREFIX" +} + + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- -- cgit v1.2.3 From 0b2b3255526110770bd419f11dfc4857c699ce77 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 13 Feb 2020 18:25:03 +0100 Subject: LXC: tooling box - add HOST shares and commandline to containers (WIP) Signed-off-by: Markus Heiser --- utils/lxc.sh | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 136 insertions(+), 16 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index d79262534..1762abd7c 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -9,6 +9,9 @@ source_dot_config # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- +# +# read also: +# - https://lxd.readthedocs.io/en/latest/ # name of https://images.linuxcontainers.org LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" @@ -17,10 +20,11 @@ HOST_PREFIX="${HOST_PREFIX:-searx}" TEST_IMAGES=( "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" - "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" + + # TODO: installation of searx & filtron not yet implemented .. + # + #"$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" #"$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" - #"ubuntu-minimal:18.04" "ubu1804" - #"ubuntu-minimal:19.10" "ubu1910" ) REMOTE_IMAGES=() @@ -31,6 +35,10 @@ for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${TEST_IMAGES[i+1]}") done +HOST_USER="${SUDO_USER:-$USER}" +HOST_USER_ID=$(id -u "${HOST_USER}") +HOST_GROUP_ID=$(id -g "${HOST_USER}") + # ---------------------------------------------------------------------------- usage() { # ---------------------------------------------------------------------------- @@ -39,13 +47,25 @@ usage() { usage:: - $(basename "$0") build [hosts] - $(basename "$0") delete [hosts] + $(basename "$0") build [containers] + $(basename "$0") delete [containers|subordinate] + $(basename "$0") [start|stop] [containers] + $(basename "$0") inspect [info|config] + $(basename "$0") cmd ... build / delete - build and/or delete all LXC hosts + :containers: build and delete all LXC containers +add / delete + :subordinate: lxd permission to map ${HOST_USER}'s user/group id through +start/stop + :containers: start/stop of all containers +inspect + :info: show info of all containers + :config: show config of all containers +cmd ... + run commandline ... in all containers -all LXC hosts: +all LXC containers: ${LOCAL_IMAGES[@]} EOF @@ -53,7 +73,6 @@ EOF } main() { - rst_title "LXC tooling box" part required_commands lxc || exit @@ -66,15 +85,43 @@ main() { build) sudo_or_exit case $2 in - hosts) build_instances ;; + containers) build_instances ;; *) usage "$_usage"; exit 42;; esac ;; delete) sudo_or_exit case $2 in - hosts) delete_instances ;; + containers) delete_instances ;; + subordinate) echo; del_subordinate_ids ;; + *) usage "$_usage"; exit 42;; + esac ;; + add) + sudo_or_exit + case $2 in + subordinate) echo; add_subordinate_ids ;; + *) usage "$_usage"; exit 42;; + esac ;; + start|stop) + sudo_or_exit + case $2 in + containers) lxc_cmd "$1" ;; *) usage "$_usage"; exit 42;; esac ;; + inspect) + sudo_or_exit + case $2 in + config) lxc_cmd config show;; + info) lxc_cmd info;; + *) usage "$_usage"; exit 42;; + esac ;; + cmd) + sudo_or_exit + shift + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "lxc exec ${_BBlue}${HOST_PREFIX}-${i}${_creset} -- ${_BGreen}${*}${_creset}" + lxc exec "${HOST_PREFIX}-${i}" -- "$@" + done + ;; *) usage "unknown or missing command $1"; exit 42;; esac @@ -83,19 +130,25 @@ main() { build_instances() { rst_title "Build LXC instances" lxc_copy_images_localy + #lxc image list local: && wait_key lxc_init_containers - - err_msg "WIP / sorry, not implemented yet :o" + lxc_config_containers + lxc list "$HOST_PREFIX" } delete_instances() { rst_title "Delete LXC instances" - echo -en "\\nLXC hosts(s)::\\n\\n ${LOCAL_IMAGES[*]}\\n" | $FMT + echo -en "\\nLXC containers(s)::\\n\\n ${LOCAL_IMAGES[*]}\\n" | $FMT if ask_yn "Do you really want to delete all images"; then lxc_delete_containers fi + # lxc list "$HOST_PREFIX" + # lxc image list local: && wait_key } +# images +# ------ + lxc_copy_images_localy() { echo for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do @@ -107,7 +160,6 @@ lxc_copy_images_localy() { --alias "${TEST_IMAGES[i+1]}" prefix_stdout fi done - #lxc image list local: } lxc_delete_images_localy() { @@ -119,6 +171,17 @@ lxc_delete_images_localy() { #lxc image list local: } +# container +# --------- + +lxc_cmd() { + echo + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "lxc $* $HOST_PREFIX-$i" + lxc "$@" "$HOST_PREFIX-$i" + done +} + lxc_init_containers() { echo for i in "${LOCAL_IMAGES[@]}"; do @@ -129,7 +192,25 @@ lxc_init_containers() { lxc init "local:$i" "$HOST_PREFIX-$i" fi done - #lxc list "$HOST_PREFIX" +} + +lxc_config_containers() { + echo + for i in "${LOCAL_IMAGES[@]}"; do + + info_msg "map uid/gid from host to conatiner: $HOST_PREFIX-$i" + # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps + echo -e -n "uid $HOST_USER_ID 1000\\ngid $HOST_GROUP_ID 1000"\ + | lxc config set "$HOST_PREFIX-$i" raw.idmap - + + info_msg "share ${REPO_ROOT} (repo_share) from HOST into container: $HOST_PREFIX-$i" + # https://lxd.readthedocs.io/en/latest/instances/#type-disk + lxc config device add "$HOST_PREFIX-$i" repo_share disk \ + source="${REPO_ROOT}" \ + path="/share/$(basename "${REPO_ROOT}")" + + # lxc config show "$HOST_PREFIX-$i" && wait_key + done } lxc_delete_containers() { @@ -143,7 +224,46 @@ lxc_delete_containers() { warn_msg "instance '$HOST_PREFIX-$i' does not exist / can't delete :o" fi done - #lxc list "$HOST_PREFIX" +} + +# subordinates +# ------------ +# +# see man: subgid(5), subuid(5), https://lxd.readthedocs.io/en/latest/userns-idmap +# +# E.g. in the HOST you have uid=1001(user) and/or gid=1001(user) :: +# +# root:1001:1 +# +# in the CONTAINER:: +# +# config: +# raw.idmap: | +# uid 1001 1000 +# gid 1001 1000 + +add_subordinate_ids() { + if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then + info_msg "lxd already has permission to map ${HOST_USER_ID}'s user/group id through" + else + info_msg "add lxd permission to map ${HOST_USER_ID}'s user/group id through" + usermod --add-subuids "${HOST_USER_ID}-${HOST_USER_ID}" \ + --add-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root + fi +} + +del_subordinate_ids() { + local out + if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then + # TODO: root user is always in use by process 1, how can we remove subordinates? + info_msg "remove lxd permission to map ${HOST_USER_ID}'s user/group id through" + out=$(usermod --del-subuids "${HOST_USER_ID}-${HOST_USER_ID}" --del-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root 2>&1) + if [ ! -z $? ]; then + err_msg "$out" + fi + else + info_msg "lxd does not have permission to map ${HOST_USER_ID}'s user/group id through" + fi } -- cgit v1.2.3 From e8cf22504650f742da247bc923abebccec869676 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 16 Feb 2020 18:18:15 +0100 Subject: LXC: tooling box - add boilerplate to containers Signed-off-by: Markus Heiser --- utils/lxc.sh | 109 +++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 29 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 1762abd7c..d0ad49bd1 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -27,12 +27,18 @@ TEST_IMAGES=( #"$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" ) +ubu1804_boilerplate=" +export DEBIAN_FRONTEND=noninteractive +apt-get install -y git curl wget +" +ubu1904_boilerplate="$ubu1804_boilerplate" + REMOTE_IMAGES=() LOCAL_IMAGES=() for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${TEST_IMAGES[i]}") - LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${TEST_IMAGES[i+1]}") + LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${HOST_PREFIX}-${TEST_IMAGES[i+1]}") done HOST_USER="${SUDO_USER:-$USER}" @@ -72,9 +78,26 @@ EOF [ ! -z "${1+x}" ] && err_msg "$1" } +lxd_info() { + + cat </dev/null; then info_msg "image ${TEST_IMAGES[i]} already copied --> ${TEST_IMAGES[i+1]}" else info_msg "copy image locally ${TEST_IMAGES[i]} --> ${TEST_IMAGES[i+1]}" lxc image copy "${TEST_IMAGES[i]}" local: \ - --alias "${TEST_IMAGES[i+1]}" prefix_stdout + --alias "${TEST_IMAGES[i+1]}" | prefix_stdout fi done } @@ -175,53 +211,68 @@ lxc_delete_images_localy() { # --------- lxc_cmd() { - echo for i in "${LOCAL_IMAGES[@]}"; do - info_msg "lxc $* $HOST_PREFIX-$i" - lxc "$@" "$HOST_PREFIX-$i" + info_msg "lxc $* $i" + lxc "$@" "$i" done } lxc_init_containers() { - echo for i in "${LOCAL_IMAGES[@]}"; do - if lxc info "$HOST_PREFIX-$i" &>/dev/null; then - info_msg "conatiner '$HOST_PREFIX-$i' already exists" + if lxc info "$i" &>/dev/null; then + info_msg "conatiner '$i' already exists" else - info_msg "create conatiner instance: $HOST_PREFIX-$i" - lxc init "local:$i" "$HOST_PREFIX-$i" + info_msg "create conatiner instance: $i" + lxc init "local:$i" "$i" fi done } lxc_config_containers() { - echo for i in "${LOCAL_IMAGES[@]}"; do + info_msg "configure container: ${_BBlue}${i}${_creset}" - info_msg "map uid/gid from host to conatiner: $HOST_PREFIX-$i" + info_msg "map uid/gid from host to container" # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps echo -e -n "uid $HOST_USER_ID 1000\\ngid $HOST_GROUP_ID 1000"\ - | lxc config set "$HOST_PREFIX-$i" raw.idmap - + | lxc config set "$i" raw.idmap - - info_msg "share ${REPO_ROOT} (repo_share) from HOST into container: $HOST_PREFIX-$i" + info_msg "share ${REPO_ROOT} (repo_share) from HOST into container" # https://lxd.readthedocs.io/en/latest/instances/#type-disk - lxc config device add "$HOST_PREFIX-$i" repo_share disk \ + lxc config device add "$i" repo_share disk \ source="${REPO_ROOT}" \ - path="/share/$(basename "${REPO_ROOT}")" + path="/share/$(basename "${REPO_ROOT}")" &>/dev/null + # lxc config show "$i" && wait_key + done +} - # lxc config show "$HOST_PREFIX-$i" && wait_key +lxc_boilerplate_containers() { + local shortname + local boilerplate_script + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + shortname="${TEST_IMAGES[i+1]}" + info_msg "install boilerplate: ${_BBlue}${HOST_PREFIX}-${shortname}${_creset}" + lxc start -q "${HOST_PREFIX}-${shortname}" &>/dev/null + boilerplate_script="${shortname}_boilerplate" + boilerplate_script="${!boilerplate_script}" + if [[ ! -z "${boilerplate_script}" ]]; then + echo "$boilerplate_script" \ + | lxc exec "${HOST_PREFIX}-${shortname}" -- bash \ + | prefix_stdout " ${HOST_PREFIX}-${shortname} | " + else + warn_msg "no boilerplate for instance '$i'" + fi done } lxc_delete_containers() { - echo for i in "${LOCAL_IMAGES[@]}"; do - if lxc info "$HOST_PREFIX-$i" &>/dev/null; then - info_msg "stop & delete instance '$HOST_PREFIX-$i'" - lxc stop "$HOST_PREFIX-$i" &>/dev/null - lxc delete "$HOST_PREFIX-$i" | prefix_stdout + if lxc info "$i" &>/dev/null; then + info_msg "stop & delete instance '$i'" + lxc stop "$i" &>/dev/null + lxc delete "$i" | prefix_stdout else - warn_msg "instance '$HOST_PREFIX-$i' does not exist / can't delete :o" + warn_msg "instance '$i' does not exist / can't delete :o" fi done } -- cgit v1.2.3 From ad3273986024c80cfe067d1b77983901a41b6d01 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 16 Feb 2020 20:07:37 +0100 Subject: shellcheck: fix usse -n instead of ! -z (SC2236 SC2237) - https://www.shellcheck.net/wiki/SC2236 -- Use -n instead of ! -z. - https://www.shellcheck.net/wiki/SC2237 -- Use [ -n .. ] instead of ! [ -z .... Signed-off-by: Markus Heiser --- utils/lxc.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index d0ad49bd1..0c198babc 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -31,6 +31,7 @@ ubu1804_boilerplate=" export DEBIAN_FRONTEND=noninteractive apt-get install -y git curl wget " +# shellcheck disable=SC2034 ubu1904_boilerplate="$ubu1804_boilerplate" REMOTE_IMAGES=() @@ -75,7 +76,7 @@ all LXC containers: ${LOCAL_IMAGES[@]} EOF - [ ! -z "${1+x}" ] && err_msg "$1" + [ -n "${1+x}" ] && err_msg "$1" } lxd_info() { @@ -255,7 +256,7 @@ lxc_boilerplate_containers() { lxc start -q "${HOST_PREFIX}-${shortname}" &>/dev/null boilerplate_script="${shortname}_boilerplate" boilerplate_script="${!boilerplate_script}" - if [[ ! -z "${boilerplate_script}" ]]; then + if [[ -n "${boilerplate_script}" ]]; then echo "$boilerplate_script" \ | lxc exec "${HOST_PREFIX}-${shortname}" -- bash \ | prefix_stdout " ${HOST_PREFIX}-${shortname} | " @@ -305,11 +306,13 @@ add_subordinate_ids() { del_subordinate_ids() { local out + local exit_value if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then # TODO: root user is always in use by process 1, how can we remove subordinates? info_msg "remove lxd permission to map ${HOST_USER_ID}'s user/group id through" out=$(usermod --del-subuids "${HOST_USER_ID}-${HOST_USER_ID}" --del-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root 2>&1) - if [ ! -z $? ]; then + exit_val=$? + if [ $exit_val -ne 0 ]; then err_msg "$out" fi else -- cgit v1.2.3 From a1f5f2ced805fad8941586fab66b16fcb11e948e Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 16 Feb 2020 22:26:03 +0100 Subject: LXC: minor fixes and renaming Signed-off-by: Markus Heiser --- utils/lxc.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 0c198babc..3d627e8b7 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -55,14 +55,14 @@ usage() { usage:: $(basename "$0") build [containers] - $(basename "$0") delete [containers|subordinate] + $(basename "$0") remove [containers|subordinate] $(basename "$0") [start|stop] [containers] $(basename "$0") inspect [info|config] $(basename "$0") cmd ... -build / delete - :containers: build and delete all LXC containers -add / delete +build / remove + :containers: build and remove all LXC containers +add / remove :subordinate: lxd permission to map ${HOST_USER}'s user/group id through start/stop :containers: start/stop of all containers @@ -86,7 +86,7 @@ lxd_info() { LXD is needed, to install run:: snap install lxd - lxc init --auto + lxd init --auto EOF } @@ -112,10 +112,10 @@ main() { containers) build_instances ;; *) usage "$_usage"; exit 42;; esac ;; - delete) + remove) sudo_or_exit case $2 in - containers) delete_instances ;; + containers) remove_instances ;; subordinate) echo; del_subordinate_ids ;; *) usage "$_usage"; exit 42;; esac ;; @@ -173,8 +173,8 @@ build_instances() { lxc list "$HOST_PREFIX" } -delete_instances() { - rst_title "Delete LXC instances" +remove_instances() { + rst_title "Remove LXC instances" echo -en "\\nLXC containers(s)::\\n\\n ${LOCAL_IMAGES[*]}\\n" | $FMT if ask_yn "Do you really want to delete all images"; then lxc_delete_containers @@ -219,12 +219,14 @@ lxc_cmd() { } lxc_init_containers() { - for i in "${LOCAL_IMAGES[@]}"; do - if lxc info "$i" &>/dev/null; then + local shortname + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + shortname="${TEST_IMAGES[i+1]}" + if lxc info "${HOST_PREFIX}-${shortname}" &>/dev/null; then info_msg "conatiner '$i' already exists" else info_msg "create conatiner instance: $i" - lxc init "local:$i" "$i" + lxc init "local:${shortname}" "${HOST_PREFIX}-${shortname}" fi done } -- cgit v1.2.3 From ccb96544f77814273bc0fbe2342633df69433669 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 18 Feb 2020 18:20:03 +0100 Subject: LXC: improved console messages & prompt Signed-off-by: Markus Heiser --- utils/lxc.sh | 77 ++++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 28 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 3d627e8b7..89c839cf1 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -17,6 +17,9 @@ source_dot_config LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" HOST_PREFIX="${HOST_PREFIX:-searx}" +# where all folders from HOST are mounted +LXC_SHARE_FOLDER="/share" + TEST_IMAGES=( "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" @@ -61,7 +64,7 @@ usage:: $(basename "$0") cmd ... build / remove - :containers: build and remove all LXC containers + :containers: build & launch (or remove) all LXC containers add / remove :subordinate: lxd permission to map ${HOST_USER}'s user/group id through start/stop @@ -129,7 +132,10 @@ main() { sudo_or_exit case $2 in containers) lxc_cmd "$1" ;; - *) usage "$_usage"; exit 42;; + *) + info_msg "lxc $1 $2" + lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " + ;; esac ;; inspect) sudo_or_exit @@ -142,12 +148,11 @@ main() { sudo_or_exit shift for i in "${LOCAL_IMAGES[@]}"; do - info_msg "call ${_BBlue}${i}${_creset} -- ${_BGreen}${*}${_creset}" - wait_key 3 + info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${*}${_creset}" lxc exec "${i}" -- "$@" exit_val=$? - if [ $exit_val -ne 0 ]; then - err_msg "$exit_val ${_BBlue}${i}${_creset} -- ${_BGreen}${*}${_creset}" + if [[ $exit_val -ne 0 ]]; then + err_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" fi done ;; @@ -162,7 +167,7 @@ build_instances() { rst_title "copy images" section echo lxc_copy_images_localy - lxc image list local: && wait_key + # lxc image list local: && wait_key echo rst_title "build containers" section echo @@ -175,6 +180,7 @@ build_instances() { remove_instances() { rst_title "Remove LXC instances" + lxc list "$HOST_PREFIX" echo -en "\\nLXC containers(s)::\\n\\n ${LOCAL_IMAGES[*]}\\n" | $FMT if ask_yn "Do you really want to delete all images"; then lxc_delete_containers @@ -214,64 +220,79 @@ lxc_delete_images_localy() { lxc_cmd() { for i in "${LOCAL_IMAGES[@]}"; do info_msg "lxc $* $i" - lxc "$@" "$i" + lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " done } lxc_init_containers() { - local shortname + + local image_name + local container_name + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do - shortname="${TEST_IMAGES[i+1]}" - if lxc info "${HOST_PREFIX}-${shortname}" &>/dev/null; then - info_msg "conatiner '$i' already exists" + + image_name="${TEST_IMAGES[i+1]}" + container_name="${HOST_PREFIX}-${image_name}" + + if lxc info "${container_name}" &>/dev/null; then + info_msg "container '${container_name}' already exists" else - info_msg "create conatiner instance: $i" - lxc init "local:${shortname}" "${HOST_PREFIX}-${shortname}" + info_msg "create conatiner instance: ${container_name}" + lxc init "local:${image_name}" "${container_name}" fi done } lxc_config_containers() { for i in "${LOCAL_IMAGES[@]}"; do - info_msg "configure container: ${_BBlue}${i}${_creset}" + info_msg "[${_BBlue}${i}${_creset}] configure container ..." - info_msg "map uid/gid from host to container" + info_msg "[${_BBlue}${i}${_creset}] map uid/gid from host to container" # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps echo -e -n "uid $HOST_USER_ID 1000\\ngid $HOST_GROUP_ID 1000"\ | lxc config set "$i" raw.idmap - - info_msg "share ${REPO_ROOT} (repo_share) from HOST into container" + info_msg "[${_BBlue}${i}${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" # https://lxd.readthedocs.io/en/latest/instances/#type-disk lxc config device add "$i" repo_share disk \ source="${REPO_ROOT}" \ - path="/share/$(basename "${REPO_ROOT}")" &>/dev/null + path="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" &>/dev/null # lxc config show "$i" && wait_key done } lxc_boilerplate_containers() { - local shortname + + local image_name + local container_name local boilerplate_script + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do - shortname="${TEST_IMAGES[i+1]}" - info_msg "install boilerplate: ${_BBlue}${HOST_PREFIX}-${shortname}${_creset}" - lxc start -q "${HOST_PREFIX}-${shortname}" &>/dev/null - boilerplate_script="${shortname}_boilerplate" + + image_name="${TEST_IMAGES[i+1]}" + container_name="${HOST_PREFIX}-${image_name}" + boilerplate_script="${image_name}_boilerplate" boilerplate_script="${!boilerplate_script}" + + info_msg "[${_BBlue}${container_name}${_creset}] install boilerplate" + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi if [[ -n "${boilerplate_script}" ]]; then - echo "$boilerplate_script" \ - | lxc exec "${HOST_PREFIX}-${shortname}" -- bash \ - | prefix_stdout " ${HOST_PREFIX}-${shortname} | " + echo "${boilerplate_script}" \ + | lxc exec "${container_name}" -- bash \ + | prefix_stdout "[${_BBlue}${container_name}${_creset}] " else - warn_msg "no boilerplate for instance '$i'" + err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'" fi + done } lxc_delete_containers() { for i in "${LOCAL_IMAGES[@]}"; do if lxc info "$i" &>/dev/null; then - info_msg "stop & delete instance '$i'" + info_msg "stop & delete instance ${_BBlue}${i}${_creset}" lxc stop "$i" &>/dev/null lxc delete "$i" | prefix_stdout else -- cgit v1.2.3 From e36e0f80aeec5d513b0bf8d4e4dc94c9fa4d98b4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 18 Feb 2020 18:40:34 +0100 Subject: LXC: added archlinux and fedora31 containers BTW: update all packages when installing conatiner's boilerplate Signed-off-by: Markus Heiser --- utils/lxc.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 89c839cf1..08205d374 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -23,20 +23,30 @@ LXC_SHARE_FOLDER="/share" TEST_IMAGES=( "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" - - # TODO: installation of searx & filtron not yet implemented .. - # - #"$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" - #"$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" + "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" + "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" ) ubu1804_boilerplate=" export DEBIAN_FRONTEND=noninteractive +apt-get update -y +apt-get upgrade -y apt-get install -y git curl wget " # shellcheck disable=SC2034 ubu1904_boilerplate="$ubu1804_boilerplate" +# shellcheck disable=SC2034 +archlinux_boilerplate=" +pacman -Syu --noconfirm +pacman -S --noconfirm git curl wget +" + +fedora31_boilerplate=" +dnf update -y +dnf install -y git curl wget +" + REMOTE_IMAGES=() LOCAL_IMAGES=() -- cgit v1.2.3 From 5fb6d4f508d9744a8d82160ca184729514bc18c8 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 23 Feb 2020 12:10:45 +0100 Subject: LXC: normalize package installation & user creation. utils/lib.sh: - get DIST_ID & DIST_VERSION from /etc/os-release - pkg_[install|remove|...] supports ubuntu, debian, archlinux & fedora utils/lxc.sh - Workaround for the "setrlimit(RLIMIT_CORE): Operation not permitted" error:: 'Set disable_coredump false' >> /etc/sudo.conf utils/[searx.sh|filtron.sh|morty.sh] - switched user creation from 'adduser' perl script to 'useradd' built-in command utils/searx.sh - install packages for ubuntu, debian, archlinux & fedora Signed-off-by: Markus Heiser --- utils/lxc.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 08205d374..8020b1346 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -39,12 +39,15 @@ ubu1904_boilerplate="$ubu1804_boilerplate" # shellcheck disable=SC2034 archlinux_boilerplate=" pacman -Syu --noconfirm -pacman -S --noconfirm git curl wget +pacman -S --noconfirm git curl wget sudo +echo 'Set disable_coredump false' >> /etc/sudo.conf " +# shellcheck disable=SC2034 fedora31_boilerplate=" dnf update -y -dnf install -y git curl wget +dnf install -y git curl wget hostname +echo 'Set disable_coredump false' >> /etc/sudo.conf " REMOTE_IMAGES=() @@ -162,7 +165,9 @@ main() { lxc exec "${i}" -- "$@" exit_val=$? if [[ $exit_val -ne 0 ]]; then - err_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + warn_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + else + info_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" fi done ;; -- cgit v1.2.3 From d5917cc029e2736b11412a570470c666af093ec9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 25 Feb 2020 20:20:17 +0100 Subject: utils/lib.sh: make uWSGI installation available for all distros support: ubuntu, debin, fedora, archlinux Signed-off-by: Markus Heiser --- utils/lxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 8020b1346..502f25366 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -344,7 +344,7 @@ add_subordinate_ids() { del_subordinate_ids() { local out - local exit_value + local exit_val if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then # TODO: root user is always in use by process 1, how can we remove subordinates? info_msg "remove lxd permission to map ${HOST_USER_ID}'s user/group id through" -- cgit v1.2.3 From af6acd3417bf53c151b9ba6068186c1e472a2776 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 26 Feb 2020 19:07:55 +0100 Subject: LXC: install searx-suite installs searx, filtron & morty on all containers Signed-off-by: Markus Heiser --- utils/lxc.sh | 55 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 502f25366..58528d591 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -19,6 +19,7 @@ HOST_PREFIX="${HOST_PREFIX:-searx}" # where all folders from HOST are mounted LXC_SHARE_FOLDER="/share" +LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" TEST_IMAGES=( "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" @@ -71,6 +72,7 @@ usage() { usage:: $(basename "$0") build [containers] + $(basename "$0") install [searx-suite] $(basename "$0") remove [containers|subordinate] $(basename "$0") [start|stop] [containers] $(basename "$0") inspect [info|config] @@ -87,6 +89,8 @@ inspect :config: show config of all containers cmd ... run commandline ... in all containers +install + :searx-suite: install searx suite, includes morty & filtron all LXC containers: ${LOCAL_IMAGES[@]} @@ -110,14 +114,24 @@ EOF main() { local exit_val - - if ! required_commands lxc; then - lxd_info - exit 42 - fi - local _usage="unknown or missing $1 command $2" + case $1 in + __install) + sudo_or_exit + case $2 in + searx-suite) install_searx_suite ;; + esac + exit + ;; + *) + if ! required_commands lxc; then + lxd_info + exit 42 + fi + ;; + esac + case $1 in --source-only) ;; -h|--help) usage; exit 0;; @@ -161,6 +175,7 @@ main() { sudo_or_exit shift for i in "${LOCAL_IMAGES[@]}"; do + exit_val= info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${*}${_creset}" lxc exec "${i}" -- "$@" exit_val=$? @@ -171,11 +186,37 @@ main() { fi done ;; + install) + sudo_or_exit + case $2 in + searx-suite) + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install $2${_creset}" + lxc exec "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" + done + ;; + *) usage "$_usage"; exit 42;; + esac ;; *) usage "unknown or missing command $1"; exit 42;; esac } +install_searx_suite() { + export FILTRON_API="0.0.0.0:4005" + export FILTRON_LISTEN="0.0.0.0:4004" + export MORTY_LISTEN="0.0.0.0:3000" + FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/searx.sh" install all + FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/morty.sh" install all + FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/filtron.sh" install all + rst_title "[$(hostname)] searx-suite installation finished" part + rst_para "IPs of the container ..." + echo + ip addr show | grep "inet\s*[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" + echo +} + + build_instances() { rst_title "Build LXC instances" @@ -271,7 +312,7 @@ lxc_config_containers() { # https://lxd.readthedocs.io/en/latest/instances/#type-disk lxc config device add "$i" repo_share disk \ source="${REPO_ROOT}" \ - path="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" &>/dev/null + path="${LXC_REPO_ROOT}" &>/dev/null # lxc config show "$i" && wait_key done } -- cgit v1.2.3 From 37c135f2cec6113aa2baaf1b52fe46fd64265975 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 27 Feb 2020 19:13:03 +0100 Subject: LXC: improved UX when working with a bunch of containers Signed-off-by: Markus Heiser --- utils/lxc.sh | 120 ++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 42 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 58528d591..ab9afcbb4 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -63,6 +63,12 @@ HOST_USER="${SUDO_USER:-$USER}" HOST_USER_ID=$(id -u "${HOST_USER}") HOST_GROUP_ID=$(id -g "${HOST_USER}") +searx_suite_set_env() { + export FILTRON_API="0.0.0.0:4005" + export FILTRON_LISTEN="0.0.0.0:4004" + export MORTY_LISTEN="0.0.0.0:3000" +} + # ---------------------------------------------------------------------------- usage() { # ---------------------------------------------------------------------------- @@ -74,8 +80,8 @@ usage:: $(basename "$0") build [containers] $(basename "$0") install [searx-suite] $(basename "$0") remove [containers|subordinate] - $(basename "$0") [start|stop] [containers] - $(basename "$0") inspect [info|config] + $(basename "$0") [start|stop] [containers|] + $(basename "$0") show [info|config|searx-suite] $(basename "$0") cmd ... build / remove @@ -83,10 +89,11 @@ build / remove add / remove :subordinate: lxd permission to map ${HOST_USER}'s user/group id through start/stop - :containers: start/stop of all containers -inspect - :info: show info of all containers - :config: show config of all containers + :containers: start/stop of all 'containers' or only +show + :info: show info of all containers + :config: show config of all containers + :searx-suite: show searx-suite services of all containers cmd ... run commandline ... in all containers install @@ -116,22 +123,10 @@ main() { local exit_val local _usage="unknown or missing $1 command $2" - case $1 in - __install) - sudo_or_exit - case $2 in - searx-suite) install_searx_suite ;; - esac - exit - ;; - *) - if ! required_commands lxc; then - lxd_info - exit 42 - fi - ;; - esac - + if [[ ! $1 == __* ]] && ! required_commands lxc; then + lxd_info + exit 42 + fi case $1 in --source-only) ;; -h|--help) usage; exit 0;; @@ -141,20 +136,23 @@ main() { case $2 in containers) build_instances ;; *) usage "$_usage"; exit 42;; - esac ;; + esac + ;; remove) sudo_or_exit case $2 in containers) remove_instances ;; subordinate) echo; del_subordinate_ids ;; *) usage "$_usage"; exit 42;; - esac ;; + esac + ;; add) sudo_or_exit case $2 in subordinate) echo; add_subordinate_ids ;; *) usage "$_usage"; exit 42;; - esac ;; + esac + ;; start|stop) sudo_or_exit case $2 in @@ -163,14 +161,27 @@ main() { info_msg "lxc $1 $2" lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " ;; - esac ;; - inspect) + esac + ;; + show) sudo_or_exit case $2 in config) lxc_cmd config show;; info) lxc_cmd info;; + searx-suite) + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install $2${_creset}" + lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show "$2" | prefix_stdout "[${i}] " + done + ;; *) usage "$_usage"; exit 42;; - esac ;; + esac + ;; + __show) + case $2 in + searx-suite) searx_suite_info ;; + esac + ;; cmd) sudo_or_exit shift @@ -192,30 +203,55 @@ main() { searx-suite) for i in "${LOCAL_IMAGES[@]}"; do info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install $2${_creset}" - lxc exec "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" + lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" | prefix_stdout "[${i}] " done ;; *) usage "$_usage"; exit 42;; - esac ;; + esac + ;; + __install) + case $2 in + searx-suite) searx_suite_install ;; + esac + ;; *) usage "unknown or missing command $1"; exit 42;; esac } -install_searx_suite() { - export FILTRON_API="0.0.0.0:4005" - export FILTRON_LISTEN="0.0.0.0:4004" - export MORTY_LISTEN="0.0.0.0:3000" - FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/searx.sh" install all - FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/morty.sh" install all - FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/filtron.sh" install all - rst_title "[$(hostname)] searx-suite installation finished" part - rst_para "IPs of the container ..." - echo - ip addr show | grep "inet\s*[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" - echo +searx_suite_install() { + ( + searx_suite_set_env + export FORCE_TIMEOUT=0 + "${LXC_REPO_ROOT}/utils/searx.sh" install all + "${LXC_REPO_ROOT}/utils/morty.sh" install all + "${LXC_REPO_ROOT}/utils/filtron.sh" install all + + rst_title "searx-suite installation finished ($(hostname))" part + searx_suite_info + echo + ) } +searx_suite_info() { + ( + searx_suite_set_env + rst_para "Services of the container $(hostname)" + for ip in $(hostname -I); do + echo + if [[ $ip =~ .*:.* ]]; then + : + # IPv6: not yet implemented / tested + # echo " searx (filtron) --> http://[$ip]:4004/" + # echo " morty --> http://[$ip]:3000/" + else + # IPv4: + echo " searx (filtron) --> http://$ip:4004/" + echo " morty --> http://$ip:3000/" + fi + done + ) +} build_instances() { rst_title "Build LXC instances" -- cgit v1.2.3 From 491cb95a1f910e17ac022ddfa73b68aae2fbc6e2 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 1 Mar 2020 18:28:10 +0100 Subject: utils/lxc.env: separate environment that is used in containers Signed-off-by: Markus Heiser --- utils/lxc.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index ab9afcbb4..f950cdb1b 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -5,6 +5,7 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" source_dot_config +source "${REPO_ROOT}/utils/lxc.env" # ---------------------------------------------------------------------------- # config @@ -63,12 +64,6 @@ HOST_USER="${SUDO_USER:-$USER}" HOST_USER_ID=$(id -u "${HOST_USER}") HOST_GROUP_ID=$(id -g "${HOST_USER}") -searx_suite_set_env() { - export FILTRON_API="0.0.0.0:4005" - export FILTRON_LISTEN="0.0.0.0:4004" - export MORTY_LISTEN="0.0.0.0:3000" -} - # ---------------------------------------------------------------------------- usage() { # ---------------------------------------------------------------------------- -- cgit v1.2.3 From c3e4753ce951f759844db13d98f9ad5b226f84b6 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 2 Mar 2020 19:00:19 +0100 Subject: docs: generic documentation from the installation scripts Signed-off-by: Markus Heiser --- utils/lxc.sh | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index f950cdb1b..674eab4a4 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -209,6 +209,11 @@ main() { searx-suite) searx_suite_install ;; esac ;; + doc) + echo + echo ".. generic utils/lxc.sh documentation" + ;; + *) usage "unknown or missing command $1"; exit 42;; esac -- cgit v1.2.3 From 7af991acda955c5d4edbf9405178fb099725d254 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 2 Mar 2020 19:01:08 +0100 Subject: LXC: add ubuntu 19.10 and 20.04 to the list of test images Signed-off-by: Markus Heiser --- utils/lxc.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 674eab4a4..7c7edccb3 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -25,6 +25,8 @@ LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" TEST_IMAGES=( "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" + "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" + "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" ) @@ -35,8 +37,10 @@ apt-get update -y apt-get upgrade -y apt-get install -y git curl wget " -# shellcheck disable=SC2034 ubu1904_boilerplate="$ubu1804_boilerplate" +ubu1910_boilerplate="$ubu1904_boilerplate" +# shellcheck disable=SC2034 +ubu2004_boilerplate="$ubu1910_boilerplate" # shellcheck disable=SC2034 archlinux_boilerplate=" -- cgit v1.2.3 From cbc08fdc26e96bf2cb02b76a30be095f5f60df9f Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 4 Mar 2020 19:56:40 +0100 Subject: docs: describe uwsgi setup of all suported distributions Signed-off-by: Markus Heiser --- utils/lxc.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 7c7edccb3..f9e6e6b7a 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -192,8 +192,9 @@ main() { if [[ $exit_val -ne 0 ]]; then warn_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" else - info_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + info_msg "[${_BBlue}${i}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" fi + echo done ;; install) -- cgit v1.2.3 From b1e90cff23eae2181d2b430e77471f488947d1a9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 7 Mar 2020 20:24:08 +0100 Subject: LXC: separate lxc-suite from lxc & improved command line. Signed-off-by: Markus Heiser --- utils/lxc.sh | 363 ++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 212 insertions(+), 151 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index f9e6e6b7a..b5ae59a7b 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -5,7 +5,11 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" source_dot_config -source "${REPO_ROOT}/utils/lxc.env" + +# load environment of the LXC suite +LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}" +source "$LXC_ENV" +lxc_set_suite_env # ---------------------------------------------------------------------------- # config @@ -14,33 +18,27 @@ source "${REPO_ROOT}/utils/lxc.env" # read also: # - https://lxd.readthedocs.io/en/latest/ -# name of https://images.linuxcontainers.org -LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" -HOST_PREFIX="${HOST_PREFIX:-searx}" +LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}" # where all folders from HOST are mounted LXC_SHARE_FOLDER="/share" LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" -TEST_IMAGES=( - "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" - "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" - "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" - "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" - "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" - "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" -) - -ubu1804_boilerplate=" +ubu1604_boilerplate=" export DEBIAN_FRONTEND=noninteractive apt-get update -y apt-get upgrade -y apt-get install -y git curl wget " +ubu1804_boilerplate="$ubu1604_boilerplate" ubu1904_boilerplate="$ubu1804_boilerplate" ubu1910_boilerplate="$ubu1904_boilerplate" + # shellcheck disable=SC2034 -ubu2004_boilerplate="$ubu1910_boilerplate" +ubu2004_boilerplate=" +$ubu1910_boilerplate +echo 'Set disable_coredump false' >> /etc/sudo.conf +" # shellcheck disable=SC2034 archlinux_boilerplate=" @@ -57,11 +55,13 @@ echo 'Set disable_coredump false' >> /etc/sudo.conf " REMOTE_IMAGES=() +CONTAINERS=() LOCAL_IMAGES=() -for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do - REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${TEST_IMAGES[i]}") - LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${HOST_PREFIX}-${TEST_IMAGES[i+1]}") +for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}") + CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}") + LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}") done HOST_USER="${SUDO_USER:-$USER}" @@ -71,36 +71,48 @@ HOST_GROUP_ID=$(id -g "${HOST_USER}") # ---------------------------------------------------------------------------- usage() { # ---------------------------------------------------------------------------- - + _cmd="$(basename "$0")" cat <] - $(basename "$0") show [info|config|searx-suite] - $(basename "$0") cmd ... - -build / remove - :containers: build & launch (or remove) all LXC containers + $_cmd build [containers] + $_cmd copy [images] + $_cmd remove [containers||images|subordinate] + $_cmd add [subordinate] + $_cmd [start|stop] [containers|] + $_cmd show [info|config|suite|images] + $_cmd cmd [--|] ... + $_cmd install [suite] + +build + :containers: build & launch all LXC containers of the suite +copy: + :images: copy remote images of the suite into local storage +remove + :containers: delete all 'containers' or only + :images: delete local images of the suite add / remove - :subordinate: lxd permission to map ${HOST_USER}'s user/group id through + :subordinate: LXD permission to map ${HOST_USER}'s user/group id through start/stop - :containers: start/stop of all 'containers' or only + :containers: start/stop all 'containers' from the suite + :: start/stop conatiner from suite show - :info: show info of all containers - :config: show config of all containers - :searx-suite: show searx-suite services of all containers -cmd ... - run commandline ... in all containers + :info: show info of all the containers from LXC suite + :config: show config of all the containers from the LXC suite + :suite: show services of all the containers from the LXC suite + :images: show information of local images +cmd + -- run command ... in all containers of the LXC suite + :: run command ... in container install - :searx-suite: install searx suite, includes morty & filtron + :suite: install LXC suite, includes morty & filtron -all LXC containers: - ${LOCAL_IMAGES[@]} +Images of the LXC suite: +$(echo " ${LOCAL_IMAGES[*]}" | $FMT) +Containers of the LXC suite: +$(echo " ${CONTAINERS[*]}" | $FMT) EOF [ -n "${1+x}" ] && err_msg "$1" } @@ -122,10 +134,12 @@ main() { local exit_val local _usage="unknown or missing $1 command $2" - if [[ ! $1 == __* ]] && ! required_commands lxc; then - lxd_info - exit 42 + # don't check prerequisite when in recursion + if [[ ! $1 == __* ]]; then + ! required_commands lxc && lxd_info && exit 42 + [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 fi + case $1 in --source-only) ;; -h|--help) usage; exit 0;; @@ -133,16 +147,28 @@ main() { build) sudo_or_exit case $2 in - containers) build_instances ;; + ''|containers) build_instances ;; + *) usage "$_usage"; exit 42;; + esac + ;; + copy) + case $2 in + ''|images) lxc_copy_images_localy;; *) usage "$_usage"; exit 42;; esac ;; remove) sudo_or_exit case $2 in - containers) remove_instances ;; + ''|containers) remove_instances ;; + images) lxc_delete_images_localy ;; subordinate) echo; del_subordinate_ids ;; - *) usage "$_usage"; exit 42;; + ${LXC_HOST_PREFIX}-*) + if ask_yn "Do you really want to delete conatiner $2"; then + lxc_delete_container "$2" + fi + ;; + *) usage "unknown (or mising) container $2"; exit 42;; esac ;; add) @@ -155,116 +181,86 @@ main() { start|stop) sudo_or_exit case $2 in - containers) lxc_cmd "$1" ;; - *) + ''|containers) lxc_cmd "$1" ;; + ${LXC_HOST_PREFIX}-*) info_msg "lxc $1 $2" lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " ;; + *) usage "ukknown or missing container $2"; exit 42;; esac ;; show) sudo_or_exit case $2 in - config) lxc_cmd config show;; - info) lxc_cmd info;; - searx-suite) - for i in "${LOCAL_IMAGES[@]}"; do - info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install $2${_creset}" - lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show "$2" | prefix_stdout "[${i}] " - done + suite) show_suite ;; + images) show_images ;; + config) + rst_title "container configurations" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + lxc_cmd config show + ;; + info) + rst_title "container info" + echo + lxc_cmd info ;; *) usage "$_usage"; exit 42;; esac ;; __show) case $2 in - searx-suite) searx_suite_info ;; + suite) lxc_suite_info ;; esac ;; cmd) sudo_or_exit shift - for i in "${LOCAL_IMAGES[@]}"; do - exit_val= - info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${*}${_creset}" - lxc exec "${i}" -- "$@" - exit_val=$? - if [[ $exit_val -ne 0 ]]; then - warn_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" - else - info_msg "[${_BBlue}${i}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" - fi - echo - done + case $1 in + --) + shift + for name in "${CONTAINERS[@]}"; do + lxc_exec_cmd "${name}" "$@" + done + ;; + ${LXC_HOST_PREFIX}-*) + local name=$1 + shift + lxc_exec_cmd "${name}" "$@" + ;; + + *) usage "unknown : $1"; exit 42 + ;; + esac ;; install) sudo_or_exit case $2 in - searx-suite) - for i in "${LOCAL_IMAGES[@]}"; do - info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install $2${_creset}" - lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" | prefix_stdout "[${i}] " - done - ;; - *) usage "$_usage"; exit 42;; + suite) install_suite ;; + *) usage "$_usage"; exit 42 ;; esac ;; __install) case $2 in - searx-suite) searx_suite_install ;; + suite) lxc_suite_install ;; esac ;; doc) echo echo ".. generic utils/lxc.sh documentation" ;; - - *) - usage "unknown or missing command $1"; exit 42;; + -*) usage "unknown option $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; esac } -searx_suite_install() { - ( - searx_suite_set_env - export FORCE_TIMEOUT=0 - "${LXC_REPO_ROOT}/utils/searx.sh" install all - "${LXC_REPO_ROOT}/utils/morty.sh" install all - "${LXC_REPO_ROOT}/utils/filtron.sh" install all - - rst_title "searx-suite installation finished ($(hostname))" part - searx_suite_info - echo - ) -} - -searx_suite_info() { - ( - searx_suite_set_env - rst_para "Services of the container $(hostname)" - for ip in $(hostname -I); do - echo - if [[ $ip =~ .*:.* ]]; then - : - # IPv6: not yet implemented / tested - # echo " searx (filtron) --> http://[$ip]:4004/" - # echo " morty --> http://[$ip]:3000/" - else - # IPv4: - echo " searx (filtron) --> http://$ip:4004/" - echo " morty --> http://$ip:3000/" - fi - done - ) -} build_instances() { rst_title "Build LXC instances" - - rst_title "copy images" section echo + add_subordinate_ids lxc_copy_images_localy - # lxc image list local: && wait_key echo rst_title "build containers" section echo @@ -272,64 +268,141 @@ build_instances() { lxc_config_containers lxc_boilerplate_containers echo - lxc list "$HOST_PREFIX" + lxc list "$LXC_HOST_PREFIX" } remove_instances() { rst_title "Remove LXC instances" - lxc list "$HOST_PREFIX" - echo -en "\\nLXC containers(s)::\\n\\n ${LOCAL_IMAGES[*]}\\n" | $FMT - if ask_yn "Do you really want to delete all images"; then - lxc_delete_containers + rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}" + echo + lxc list "$LXC_HOST_PREFIX-" + echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT + if ask_yn "Do you really want to delete these conatiners"; then + for i in "${CONTAINERS[@]}"; do + lxc_delete_container "$i" + done fi echo - lxc list "$HOST_PREFIX" - # lxc image list local: && wait_key + lxc list "$LXC_HOST_PREFIX-" } # images # ------ lxc_copy_images_localy() { - for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do - if lxc image info "local:${TEST_IMAGES[i+1]}" &>/dev/null; then - info_msg "image ${TEST_IMAGES[i]} already copied --> ${TEST_IMAGES[i+1]}" + rst_title "copy images" section + echo + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then + info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}" else - info_msg "copy image locally ${TEST_IMAGES[i]} --> ${TEST_IMAGES[i+1]}" - lxc image copy "${TEST_IMAGES[i]}" local: \ - --alias "${TEST_IMAGES[i+1]}" | prefix_stdout + info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}" + lxc image copy "${LXC_SUITE[i]}" local: \ + --alias "${LXC_SUITE[i+1]}" | prefix_stdout fi done + # lxc image list local: && wait_key } lxc_delete_images_localy() { + rst_title "Delete LXC images" + rst_para "local existing images" echo + lxc image list local: + echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" + if ask_yn "Do you really want to delete these images"; then + for i in "${LOCAL_IMAGES[@]}"; do + lxc_delete_local_image "$i" + done + fi + echo + lxc image list local: +} + +show_images(){ + rst_title "local images" + echo + lxc image list local: + echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" + wait_key for i in "${LOCAL_IMAGES[@]}"; do - info_msg "delete image 'local:$i'" - lxc image delete "local:$i" + if lxc_image_exists "$i"; then + info_msg "lxc image info ${_BBlue}${i}${_creset}" + lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + else + warn_msg "image ${_BBlue}$i${_creset} does not yet exists" + fi done - #lxc image list local: + } + # container # --------- +show_suite(){ + rst_title "LXC suite ($LXC_HOST_PREFIX-*)" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ + | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done +} + +install_suite() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install suite${_creset}" + lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install suite \ + | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done +} + lxc_cmd() { - for i in "${LOCAL_IMAGES[@]}"; do - info_msg "lxc $* $i" - lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + info_msg "lxc $* $i" + lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + echo + fi done } +lxc_exec_cmd() { + local name="$1" + shift + exit_val= + info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" + lxc exec "${name}" -- "$@" + exit_val=$? + if [[ $exit_val -ne 0 ]]; then + warn_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + else + info_msg "[${_BBlue}${i}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" + fi + echo +} + lxc_init_containers() { local image_name local container_name - for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - image_name="${TEST_IMAGES[i+1]}" - container_name="${HOST_PREFIX}-${image_name}" + image_name="${LXC_SUITE[i+1]}" + container_name="${LXC_HOST_PREFIX}-${image_name}" if lxc info "${container_name}" &>/dev/null; then info_msg "container '${container_name}' already exists" @@ -341,7 +414,7 @@ lxc_init_containers() { } lxc_config_containers() { - for i in "${LOCAL_IMAGES[@]}"; do + for i in "${CONTAINERS[@]}"; do info_msg "[${_BBlue}${i}${_creset}] configure container ..." info_msg "[${_BBlue}${i}${_creset}] map uid/gid from host to container" @@ -364,10 +437,10 @@ lxc_boilerplate_containers() { local container_name local boilerplate_script - for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - image_name="${TEST_IMAGES[i+1]}" - container_name="${HOST_PREFIX}-${image_name}" + image_name="${LXC_SUITE[i+1]}" + container_name="${LXC_HOST_PREFIX}-${image_name}" boilerplate_script="${image_name}_boilerplate" boilerplate_script="${!boilerplate_script}" @@ -386,18 +459,6 @@ lxc_boilerplate_containers() { done } -lxc_delete_containers() { - for i in "${LOCAL_IMAGES[@]}"; do - if lxc info "$i" &>/dev/null; then - info_msg "stop & delete instance ${_BBlue}${i}${_creset}" - lxc stop "$i" &>/dev/null - lxc delete "$i" | prefix_stdout - else - warn_msg "instance '$i' does not exist / can't delete :o" - fi - done -} - # subordinates # ------------ # -- cgit v1.2.3 From e7f69b63f1169a60f6f15b827dfe4f5748a41d2c Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 8 Mar 2020 02:41:45 +0100 Subject: LXC: from searx.sh, morty.sh and filtron.sh tests Signed-off-by: Markus Heiser --- utils/lxc.sh | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index b5ae59a7b..bcee16743 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -108,15 +108,29 @@ cmd install :suite: install LXC suite, includes morty & filtron -Images of the LXC suite: -$(echo " ${LOCAL_IMAGES[*]}" | $FMT) +EOF + usage_images + echo + usage_containers + echo + [ -n "${1+x}" ] && err_msg "$1" +} -Containers of the LXC suite: +usage_containers() { + cat < $2"; exit 42;; + *) usage "uknown or missing container $2"; exit 42;; esac ;; add) @@ -183,10 +198,11 @@ main() { case $2 in ''|containers) lxc_cmd "$1" ;; ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 info_msg "lxc $1 $2" lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " ;; - *) usage "ukknown or missing container $2"; exit 42;; + *) usage "uknown or missing container $2"; exit 42;; esac ;; show) @@ -225,13 +241,12 @@ main() { done ;; ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42 local name=$1 shift lxc_exec_cmd "${name}" "$@" ;; - - *) usage "unknown : $1"; exit 42 - ;; + *) usage "uknown or missing container $2"; exit 42;; esac ;; install) @@ -351,6 +366,7 @@ show_suite(){ else lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ | prefix_stdout "[${_BBlue}${i}${_creset}] " + echo fi done } @@ -384,12 +400,12 @@ lxc_exec_cmd() { shift exit_val= info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" - lxc exec "${name}" -- "$@" + lxc exec --cwd "${LXC_REPO_ROOT}" "${name}" -- "$@" exit_val=$? if [[ $exit_val -ne 0 ]]; then - warn_msg "[${_BBlue}${i}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" else - info_msg "[${_BBlue}${i}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" + info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" fi echo } -- cgit v1.2.3 From d48c7bf6783ba9220c88b3c3ccb7b2ce3721e905 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 8 Mar 2020 18:30:25 +0100 Subject: LXC: fixed raw.idmap & removed obsolete uid/gid subordinate Signed-off-by: Markus Heiser --- utils/lxc.sh | 67 +++++++----------------------------------------------------- 1 file changed, 7 insertions(+), 60 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index bcee16743..f9dd453ee 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -78,11 +78,10 @@ usage:: $_cmd build [containers] $_cmd copy [images] - $_cmd remove [containers||images|subordinate] - $_cmd add [subordinate] + $_cmd remove [containers||images] $_cmd [start|stop] [containers|] $_cmd show [info|config|suite|images] - $_cmd cmd [--|] ... + $_cmd cmd [--|] '...' $_cmd install [suite] build @@ -92,8 +91,6 @@ copy: remove :containers: delete all 'containers' or only :images: delete local images of the suite -add / remove - :subordinate: LXD permission to map ${HOST_USER}'s user/group id through start/stop :containers: start/stop all 'containers' from the suite :: start/stop conatiner from suite @@ -103,8 +100,9 @@ show :suite: show services of all the containers from the LXC suite :images: show information of local images cmd - -- run command ... in all containers of the LXC suite - :: run command ... in container + use single qoutes to evaluate in container's bash, e.g. 'echo $(hostname)' + -- run command '...' in all containers of the LXC suite + :: run command '...' in container install :suite: install LXC suite, includes morty & filtron @@ -176,7 +174,6 @@ main() { case $2 in ''|containers) remove_instances ;; images) lxc_delete_images_localy ;; - subordinate) echo; del_subordinate_ids ;; ${LXC_HOST_PREFIX}-*) ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 if ask_yn "Do you really want to delete conatiner $2"; then @@ -186,13 +183,6 @@ main() { *) usage "uknown or missing container $2"; exit 42;; esac ;; - add) - sudo_or_exit - case $2 in - subordinate) echo; add_subordinate_ids ;; - *) usage "$_usage"; exit 42;; - esac - ;; start|stop) sudo_or_exit case $2 in @@ -274,7 +264,6 @@ main() { build_instances() { rst_title "Build LXC instances" echo - add_subordinate_ids lxc_copy_images_localy echo rst_title "build containers" section @@ -400,7 +389,7 @@ lxc_exec_cmd() { shift exit_val= info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" - lxc exec --cwd "${LXC_REPO_ROOT}" "${name}" -- "$@" + lxc exec --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" exit_val=$? if [[ $exit_val -ne 0 ]]; then warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" @@ -435,7 +424,7 @@ lxc_config_containers() { info_msg "[${_BBlue}${i}${_creset}] map uid/gid from host to container" # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps - echo -e -n "uid $HOST_USER_ID 1000\\ngid $HOST_GROUP_ID 1000"\ + echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\ | lxc config set "$i" raw.idmap - info_msg "[${_BBlue}${i}${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" @@ -475,48 +464,6 @@ lxc_boilerplate_containers() { done } -# subordinates -# ------------ -# -# see man: subgid(5), subuid(5), https://lxd.readthedocs.io/en/latest/userns-idmap -# -# E.g. in the HOST you have uid=1001(user) and/or gid=1001(user) :: -# -# root:1001:1 -# -# in the CONTAINER:: -# -# config: -# raw.idmap: | -# uid 1001 1000 -# gid 1001 1000 - -add_subordinate_ids() { - if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then - info_msg "lxd already has permission to map ${HOST_USER_ID}'s user/group id through" - else - info_msg "add lxd permission to map ${HOST_USER_ID}'s user/group id through" - usermod --add-subuids "${HOST_USER_ID}-${HOST_USER_ID}" \ - --add-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root - fi -} - -del_subordinate_ids() { - local out - local exit_val - if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then - # TODO: root user is always in use by process 1, how can we remove subordinates? - info_msg "remove lxd permission to map ${HOST_USER_ID}'s user/group id through" - out=$(usermod --del-subuids "${HOST_USER_ID}-${HOST_USER_ID}" --del-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root 2>&1) - exit_val=$? - if [ $exit_val -ne 0 ]; then - err_msg "$out" - fi - else - info_msg "lxd does not have permission to map ${HOST_USER_ID}'s user/group id through" - fi -} - # ---------------------------------------------------------------------------- main "$@" -- cgit v1.2.3 From 80f7b658047a3541697ef5ae1aae897817b3f43c Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 9 Mar 2020 01:37:26 +0100 Subject: searx.sh: add buildhost cmd, installs OS packages for builds Signed-off-by: Markus Heiser --- utils/lxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index f9dd453ee..3c4d2016e 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -236,7 +236,7 @@ main() { shift lxc_exec_cmd "${name}" "$@" ;; - *) usage "uknown or missing container $2"; exit 42;; + *) usage "uknown or missing container $1"; exit 42;; esac ;; install) -- cgit v1.2.3 From 86e79488aab3ff434c6682a9464ba2eee49158e9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 15 Mar 2020 17:01:36 +0100 Subject: LXC: utils/makefile.lxc (inital) add /.lxcenv.mk to contaiiners Get LXC environment when building make targets. Signed-off-by: Markus Heiser --- utils/lxc.sh | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 3c4d2016e..64805272e 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -104,7 +104,8 @@ cmd -- run command '...' in all containers of the LXC suite :: run command '...' in container install - :suite: install LXC suite, includes morty & filtron + :suite: install LXC suite; ${lxc_suite_install_info} + :buildhost: prepare LXC; buildhost EOF usage_images @@ -224,12 +225,7 @@ main() { sudo_or_exit shift case $1 in - --) - shift - for name in "${CONTAINERS[@]}"; do - lxc_exec_cmd "${name}" "$@" - done - ;; + --) shift; lxc_exec "$@" ;; ${LXC_HOST_PREFIX}-*) ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42 local name=$1 @@ -242,13 +238,15 @@ main() { install) sudo_or_exit case $2 in - suite) install_suite ;; + suite) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install suite;; + buildhost) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install buildhost;; *) usage "$_usage"; exit 42 ;; esac ;; __install) case $2 in suite) lxc_suite_install ;; + buildhost) lxc_suite_prepare_buildhost ;; esac ;; doc) @@ -263,7 +261,6 @@ main() { build_instances() { rst_title "Build LXC instances" - echo lxc_copy_images_localy echo rst_title "build containers" section @@ -360,18 +357,6 @@ show_suite(){ done } -install_suite() { - for i in "${CONTAINERS[@]}"; do - if ! lxc_exists "$i"; then - warn_msg "container ${_BBlue}$i${_creset} does not yet exists" - else - info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install suite${_creset}" - lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install suite \ - | prefix_stdout "[${_BBlue}${i}${_creset}] " - fi - done -} - lxc_cmd() { for i in "${CONTAINERS[@]}"; do if ! lxc_exists "$i"; then @@ -389,14 +374,23 @@ lxc_exec_cmd() { shift exit_val= info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" - lxc exec --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" + lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" exit_val=$? if [[ $exit_val -ne 0 ]]; then warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" else info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" fi - echo +} + +lxc_exec() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done } lxc_init_containers() { @@ -449,7 +443,17 @@ lxc_boilerplate_containers() { boilerplate_script="${image_name}_boilerplate" boilerplate_script="${!boilerplate_script}" - info_msg "[${_BBlue}${container_name}${_creset}] install boilerplate" + info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + cat </dev/null; then sleep 5 # guest needs some time to come up and get an IP fi -- cgit v1.2.3 From d2cfe9ce5bb409472f10590aa1ae069b2a971a49 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 18 Mar 2020 15:34:46 +0100 Subject: LXC: add /.lxcenv Signed-off-by: Markus Heiser --- utils/lxc.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 64805272e..56450c4db 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -82,7 +82,7 @@ usage:: $_cmd [start|stop] [containers|] $_cmd show [info|config|suite|images] $_cmd cmd [--|] '...' - $_cmd install [suite] + $_cmd install [suite|buildhost] build :containers: build & launch all LXC containers of the suite @@ -443,10 +443,12 @@ lxc_boilerplate_containers() { boilerplate_script="${image_name}_boilerplate" boilerplate_script="${!boilerplate_script}" - info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." + info_msg "[${_BBlue}${container_name}${_creset}] init .." if lxc start -q "${container_name}" &>/dev/null; then sleep 5 # guest needs some time to come up and get an IP fi + lxc_init_container "${container_name}" + info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." cat < Date: Wed, 18 Mar 2020 17:47:48 +0100 Subject: ./utils/lxc.sh: add command 'install base' to install basic packages like git and *build essentials*:: $ ./utils/lxc.sh install base Signed-off-by: Markus Heiser --- utils/lxc.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 56450c4db..6070735ea 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -82,10 +82,10 @@ usage:: $_cmd [start|stop] [containers|] $_cmd show [info|config|suite|images] $_cmd cmd [--|] '...' - $_cmd install [suite|buildhost] + $_cmd install [suite|base|buildhost] build - :containers: build & launch all LXC containers of the suite + :containers: build, launch and 'install basic' packages on 'containers' copy: :images: copy remote images of the suite into local storage remove @@ -105,7 +105,8 @@ cmd :: run command '...' in container install :suite: install LXC suite; ${lxc_suite_install_info} - :buildhost: prepare LXC; buildhost + :base: prepare LXC; install basic packages + :buildhost: prepare LXC; install buildhost packages EOF usage_images @@ -217,6 +218,7 @@ main() { esac ;; __show) + # wrapped show commands, called once in each container case $2 in suite) lxc_suite_info ;; esac @@ -238,14 +240,17 @@ main() { install) sudo_or_exit case $2 in - suite) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install suite;; - buildhost) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install buildhost;; + suite|base|buildhost) + lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install $2 + ;; *) usage "$_usage"; exit 42 ;; esac ;; __install) + # wrapped install commands, called once in each container case $2 in suite) lxc_suite_install ;; + base) FORCE_TIMEOUT=0 lxc_install_base_packages ;; buildhost) lxc_suite_prepare_buildhost ;; esac ;; @@ -269,6 +274,8 @@ build_instances() { lxc_config_containers lxc_boilerplate_containers echo + lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base + echo lxc list "$LXC_HOST_PREFIX" } -- cgit v1.2.3 From 3e4d022d0425e4f7647573c97f105aa5208e5537 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 30 Mar 2020 11:03:46 +0200 Subject: [fix] make test.sh: fix various shellcheck error messages Signed-off-by: Markus Heiser --- utils/lxc.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 6070735ea..b03ce2af8 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -104,7 +104,7 @@ cmd -- run command '...' in all containers of the LXC suite :: run command '...' in container install - :suite: install LXC suite; ${lxc_suite_install_info} + :suite: install LXC suite; ${LXC_SUITE_INSTALL_INFO} :base: prepare LXC; install basic packages :buildhost: prepare LXC; install buildhost packages @@ -241,13 +241,14 @@ main() { sudo_or_exit case $2 in suite|base|buildhost) - lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install $2 + lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; *) usage "$_usage"; exit 42 ;; esac ;; __install) # wrapped install commands, called once in each container + # shellcheck disable=SC2119 case $2 in suite) lxc_suite_install ;; base) FORCE_TIMEOUT=0 lxc_install_base_packages ;; -- cgit v1.2.3 From dd53c45a2cb46f882a856869de26215942749cba Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 30 Mar 2020 18:47:01 +0200 Subject: docs: add utils/lxc.sh docs, normalize filtron, morty & searx docs Signed-off-by: Markus Heiser --- utils/lxc.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index b03ce2af8..2526a257d 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -324,6 +324,13 @@ lxc_delete_images_localy() { lxc_delete_local_image "$i" done fi + + for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do + if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then + lxc_delete_local_image "$i" + fi + done + echo lxc image list local: } -- cgit v1.2.3 From f27f6c6c5d372b257dfe671ec26eb73f6e12ed28 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 31 Mar 2020 18:25:40 +0200 Subject: docs: use make and uitls/searx.sh in containers Signed-off-by: Markus Heiser --- utils/lxc.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 2526a257d..612ca566d 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -82,7 +82,7 @@ usage:: $_cmd [start|stop] [containers|] $_cmd show [info|config|suite|images] $_cmd cmd [--|] '...' - $_cmd install [suite|base|buildhost] + $_cmd install [suite|base] build :containers: build, launch and 'install basic' packages on 'containers' @@ -106,7 +106,6 @@ cmd install :suite: install LXC suite; ${LXC_SUITE_INSTALL_INFO} :base: prepare LXC; install basic packages - :buildhost: prepare LXC; install buildhost packages EOF usage_images @@ -150,7 +149,9 @@ main() { # don't check prerequisite when in recursion if [[ ! $1 == __* ]]; then - ! required_commands lxc && lxd_info && exit 42 + if ! in_container; then + ! required_commands lxc && lxd_info && exit 42 + fi [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 fi @@ -240,7 +241,7 @@ main() { install) sudo_or_exit case $2 in - suite|base|buildhost) + suite|base) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; *) usage "$_usage"; exit 42 ;; @@ -252,7 +253,6 @@ main() { case $2 in suite) lxc_suite_install ;; base) FORCE_TIMEOUT=0 lxc_install_base_packages ;; - buildhost) lxc_suite_prepare_buildhost ;; esac ;; doc) -- cgit v1.2.3 From af988dbf71ce4b14d2056676d7d9691335266d83 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 3 Apr 2020 17:08:42 +0200 Subject: utils/lxc.sh: support build of selected containers $ sudo -H ./utils/lxc.sh build $ sudo -H ./utils/lxc.sh show [images|suite|info|config []] Signed-off-by: Markus Heiser --- utils/lxc.sh | 230 ++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 149 insertions(+), 81 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 612ca566d..813b50843 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -76,16 +76,17 @@ usage() { usage:: - $_cmd build [containers] + $_cmd build [containers|] $_cmd copy [images] $_cmd remove [containers||images] $_cmd [start|stop] [containers|] - $_cmd show [info|config|suite|images] + $_cmd show [images|suite|info|config []] $_cmd cmd [--|] '...' $_cmd install [suite|base] build - :containers: build, launch and 'install basic' packages on 'containers' + :containers: build, launch all containers and 'install base' packages + :: build, launch container and 'install base' packages copy: :images: copy remote images of the suite into local storage remove @@ -93,10 +94,10 @@ remove :images: delete local images of the suite start/stop :containers: start/stop all 'containers' from the suite - :: start/stop conatiner from suite + :: start/stop container from suite show - :info: show info of all the containers from LXC suite - :config: show config of all the containers from the LXC suite + :info: show info of all (or ) containers from LXC suite + :config: show config of all (or ) containers from the LXC suite :suite: show services of all the containers from the LXC suite :images: show information of local images cmd @@ -162,7 +163,8 @@ main() { build) sudo_or_exit case $2 in - ''|containers) build_instances ;; + ${LXC_HOST_PREFIX}-*) build_container "$2" ;; + ''|containers) build_all_containers ;; *) usage "$_usage"; exit 42;; esac ;; @@ -175,11 +177,11 @@ main() { remove) sudo_or_exit case $2 in - ''|containers) remove_instances ;; + ''|containers) remove_containers ;; images) lxc_delete_images_localy ;; ${LXC_HOST_PREFIX}-*) ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 - if ask_yn "Do you really want to delete conatiner $2"; then + if ask_yn "Do you really want to delete container $2"; then lxc_delete_container "$2" fi ;; @@ -201,19 +203,41 @@ main() { show) sudo_or_exit case $2 in - suite) show_suite ;; + suite) + case $3 in + ${LXC_HOST_PREFIX}-*) + lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ + | prefix_stdout "[${_BBlue}$3${_creset}] " + ;; + *) show_suite;; + esac + ;; images) show_images ;; config) - rst_title "container configurations" - echo - lxc list "$LXC_HOST_PREFIX-" - echo - lxc_cmd config show + case $3 in + ${LXC_HOST_PREFIX}-*) + lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " + ;; + *) + rst_title "container configurations" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + lxc_cmd config show + ;; + esac ;; info) - rst_title "container info" - echo - lxc_cmd info + case $3 in + ${LXC_HOST_PREFIX}-*) + lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " + ;; + *) + rst_title "container info" + echo + lxc_cmd info + ;; + esac ;; *) usage "$_usage"; exit 42;; esac @@ -265,28 +289,69 @@ main() { } -build_instances() { - rst_title "Build LXC instances" +build_all_containers() { + rst_title "Build all LXC containers of suite" + usage_containers lxc_copy_images_localy echo rst_title "build containers" section echo - lxc_init_containers - lxc_config_containers - lxc_boilerplate_containers + lxc_init_all_containers + lxc_config_all_containers + lxc_boilerplate_all_containers + rst_title "install LXC base packages" section echo lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base echo lxc list "$LXC_HOST_PREFIX" } -remove_instances() { - rst_title "Remove LXC instances" +build_container() { + rst_title "Build container $1" + + local remote_image + local container + local image + local boilerplate_script + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then + remote_image="${LXC_SUITE[i]}" + container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" + image="${LXC_SUITE[i+1]}" + boilerplate_script="${image}_boilerplate" + boilerplate_script="${!boilerplate_script}" + break + fi + done + echo + if [ -z "$container" ]; then + err_msg "container $1 unknown" + usage_containers + return 42 + fi + lxc_image_copy "${remote_image}" "${image}" + rst_title "init container" section + lxc_init_container "${image}" "${container}" + rst_title "configure container" section + lxc_config_container "${container}" + rst_title "run LXC boilerplate scripts" section + lxc_install_boilerplate "${container}" "$boilerplate_script" + echo + rst_title "install LXC base packages" section + lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \ + | prefix_stdout "[${_BBlue}${container}${_creset}] " + echo + lxc list "$container" +} + +remove_containers() { + rst_title "Remove all LXC containers of suite" rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}" echo lxc list "$LXC_HOST_PREFIX-" echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT - if ask_yn "Do you really want to delete these conatiners"; then + if ask_yn "Do you really want to delete these containers"; then for i in "${CONTAINERS[@]}"; do lxc_delete_container "$i" done @@ -302,13 +367,7 @@ lxc_copy_images_localy() { rst_title "copy images" section echo for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then - info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}" - else - info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}" - lxc image copy "${LXC_SUITE[i]}" local: \ - --alias "${LXC_SUITE[i+1]}" | prefix_stdout - fi + lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}" done # lxc image list local: && wait_key } @@ -408,81 +467,90 @@ lxc_exec() { done } -lxc_init_containers() { +lxc_init_all_containers() { + rst_title "init all containers" section local image_name local container_name for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - - image_name="${LXC_SUITE[i+1]}" - container_name="${LXC_HOST_PREFIX}-${image_name}" - - if lxc info "${container_name}" &>/dev/null; then - info_msg "container '${container_name}' already exists" - else - info_msg "create conatiner instance: ${container_name}" - lxc init "local:${image_name}" "${container_name}" - fi + lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${image_name}" done } -lxc_config_containers() { +lxc_config_all_containers() { + rst_title "configure all containers" section + for i in "${CONTAINERS[@]}"; do - info_msg "[${_BBlue}${i}${_creset}] configure container ..." - - info_msg "[${_BBlue}${i}${_creset}] map uid/gid from host to container" - # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps - echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\ - | lxc config set "$i" raw.idmap - - - info_msg "[${_BBlue}${i}${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" - # https://lxd.readthedocs.io/en/latest/instances/#type-disk - lxc config device add "$i" repo_share disk \ - source="${REPO_ROOT}" \ - path="${LXC_REPO_ROOT}" &>/dev/null - # lxc config show "$i" && wait_key + lxc_config_container "${i}" done } -lxc_boilerplate_containers() { +lxc_config_container() { + info_msg "[${_BBlue}$1${_creset}] configure container ..." + + info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container" + # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps + echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\ + | lxc config set "$1" raw.idmap - + + info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" + # https://lxd.readthedocs.io/en/latest/instances/#type-disk + lxc config device add "$1" repo_share disk \ + source="${REPO_ROOT}" \ + path="${LXC_REPO_ROOT}" &>/dev/null + # lxc config show "$1" && wait_key +} + +lxc_boilerplate_all_containers() { + rst_title "run LXC boilerplate scripts" section - local image_name - local container_name local boilerplate_script + local image_name for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do image_name="${LXC_SUITE[i+1]}" - container_name="${LXC_HOST_PREFIX}-${image_name}" boilerplate_script="${image_name}_boilerplate" boilerplate_script="${!boilerplate_script}" - info_msg "[${_BBlue}${container_name}${_creset}] init .." - if lxc start -q "${container_name}" &>/dev/null; then - sleep 5 # guest needs some time to come up and get an IP + lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script" + + if [[ -z "${boilerplate_script}" ]]; then + err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'" fi - lxc_init_container "${container_name}" - info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." - cat < + # + # usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}" + + local container_name="$1" + local boilerplate_script="$2" + + info_msg "[${_BBlue}${container_name}${_creset}] init .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." + cat </dev/null; then - sleep 5 # guest needs some time to come up and get an IP - fi - if [[ -n "${boilerplate_script}" ]]; then - echo "${boilerplate_script}" \ - | lxc exec "${container_name}" -- bash \ - | prefix_stdout "[${_BBlue}${container_name}${_creset}] " - else - err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'" - fi - - done + info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + if [[ -n "${boilerplate_script}" ]]; then + echo "${boilerplate_script}" \ + | lxc exec "${container_name}" -- bash \ + | prefix_stdout "[${_BBlue}${container_name}${_creset}] " + fi } -- cgit v1.2.3 From 3c19f2f1cef31fd512975c70a25442d078992a42 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 4 Apr 2020 12:40:14 +0200 Subject: utils/lxc.sh install: support installation on selected containers Signed-off-by: Markus Heiser --- utils/lxc.sh | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 813b50843..92dfff2d1 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -82,7 +82,7 @@ usage:: $_cmd [start|stop] [containers|] $_cmd show [images|suite|info|config []] $_cmd cmd [--|] '...' - $_cmd install [suite|base] + $_cmd install [suite|base []] build :containers: build, launch all containers and 'install base' packages @@ -105,32 +105,26 @@ cmd -- run command '...' in all containers of the LXC suite :: run command '...' in container install - :suite: install LXC suite; ${LXC_SUITE_INSTALL_INFO} :base: prepare LXC; install basic packages + :suite: install LXC ${LXC_SUITE_NAME} suite into all (or ) containers EOF - usage_images - echo usage_containers - echo [ -n "${1+x}" ] && err_msg "$1" } usage_containers() { cat < $1"; exit 42;; - esac + *) usage_containers "unknown container: $1" && exit 42 + esac ;; install) sudo_or_exit case $2 in suite|base) - lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" + ;; + '') lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; + *) usage_containers "unknown container: $3" && exit 42 + esac ;; *) usage "$_usage"; exit 42 ;; esac -- cgit v1.2.3 From c2caf9569c5b6e1a9d143382c2045983f7c6dd61 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 5 Apr 2020 15:15:28 +0200 Subject: [fix] creation of /.lxcenv in containers In utils/lib.sh there are two functions with the same name, but different tasks. Rename one of them from lxc_init_container() into lxc_init_container_env(). Signed-off-by: Markus Heiser --- utils/lxc.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 92dfff2d1..9eb28f498 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -538,6 +538,7 @@ lxc_install_boilerplate() { if lxc start -q "${container_name}" &>/dev/null; then sleep 5 # guest needs some time to come up and get an IP fi + lxc_init_container_env "${container_name}" info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." cat < Date: Tue, 7 Apr 2020 18:31:51 +0200 Subject: apache: normalize installation (docs and script)s over all distros Signed-off-by: Markus Heiser --- utils/lxc.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 9eb28f498..6a26f80eb 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -98,7 +98,7 @@ start/stop show :info: show info of all (or ) containers from LXC suite :config: show config of all (or ) containers from the LXC suite - :suite: show services of all the containers from the LXC suite + :suite: show services of all (or ) containers from the LXC suite :images: show information of local images cmd use single qoutes to evaluate in container's bash, e.g. 'echo $(hostname)' @@ -294,11 +294,9 @@ main() { build_all_containers() { rst_title "Build all LXC containers of suite" + echo usage_containers lxc_copy_images_localy - echo - rst_title "build containers" section - echo lxc_init_all_containers lxc_config_all_containers lxc_boilerplate_all_containers @@ -368,7 +366,6 @@ remove_containers() { lxc_copy_images_localy() { rst_title "copy images" section - echo for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}" done @@ -477,7 +474,7 @@ lxc_init_all_containers() { local container_name for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do - lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${image_name}" + lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" done } -- cgit v1.2.3 From f693149cded4f783380f8f02154bd9288b72cdd5 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 8 Apr 2020 18:38:36 +0200 Subject: Changes from the installation tests on (all) LXC containers. Tested and fixed HTTP & uWSGI installation on: ubu1604 ubu1804 ubu1910 ubu2004 fedora31 archlinux Signed-off-by: Markus Heiser --- utils/lxc.sh | 1 - 1 file changed, 1 deletion(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 6a26f80eb..6a8dce94f 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -438,7 +438,6 @@ lxc_cmd() { else info_msg "lxc $* $i" lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " - echo fi done } -- cgit v1.2.3 From 58d5da8b57c5aeab92f551e8d175be67537c351c Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 11 Apr 2020 13:19:11 +0200 Subject: nginx: normalize installation (docs and script)s over all distros This is the revision of the documentation about the varous nginx installation variants. It also implements the nginx installation scripts for morty and filtron. Signed-off-by: Markus Heiser --- utils/lxc.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index 6a8dce94f..ce306fe85 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -73,9 +73,7 @@ usage() { # ---------------------------------------------------------------------------- _cmd="$(basename "$0")" cat <] $_cmd copy [images] $_cmd remove [containers||images] @@ -101,7 +99,7 @@ show :suite: show services of all (or ) containers from the LXC suite :images: show information of local images cmd - use single qoutes to evaluate in container's bash, e.g. 'echo $(hostname)' + use single qoutes to evaluate in container's bash, e.g.: 'echo \$(hostname)' -- run command '...' in all containers of the LXC suite :: run command '...' in container install @@ -151,14 +149,14 @@ main() { fi case $1 in - --source-only) ;; + --getenv) var="$2"; echo "${!var}"; exit 0;; -h|--help) usage; exit 0;; build) sudo_or_exit case $2 in ${LXC_HOST_PREFIX}-*) build_container "$2" ;; - ''|containers) build_all_containers ;; + ''|--|containers) build_all_containers ;; *) usage "$_usage"; exit 42;; esac ;; @@ -171,7 +169,7 @@ main() { remove) sudo_or_exit case $2 in - ''|containers) remove_containers ;; + ''|--|containers) remove_containers ;; images) lxc_delete_images_localy ;; ${LXC_HOST_PREFIX}-*) ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 @@ -185,7 +183,7 @@ main() { start|stop) sudo_or_exit case $2 in - ''|containers) lxc_cmd "$1" ;; + ''|--|containers) lxc_cmd "$1" ;; ${LXC_HOST_PREFIX}-*) ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 info_msg "lxc $1 $2" @@ -203,7 +201,7 @@ main() { lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ | prefix_stdout "[${_BBlue}$3${_creset}] " ;; - *) show_suite;; + *|--) show_suite;; esac ;; images) show_images ;; @@ -213,7 +211,7 @@ main() { ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " ;; - *) + *|--) rst_title "container configurations" echo lxc list "$LXC_HOST_PREFIX-" @@ -228,7 +226,7 @@ main() { ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " ;; - *) + *|--) rst_title "container info" echo lxc_cmd info @@ -267,7 +265,7 @@ main() { ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; - '') lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; + ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; *) usage_containers "unknown container: $3" && exit 42 esac ;; -- cgit v1.2.3 From 99ff16c465ed4d3b98041bf308dfeb0918b535ab Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 13 Apr 2020 11:34:28 +0200 Subject: tooling box: added nginx + polished bash scripts and environment - add installation method for nginx sites, morty and filtron - clean up PUBLIC_URL environment in and outside of containers - clean up comand lines - handle uWSGI quirks on fedora (emperor mode) - handle Python quirks on debian (there is no 'python' command anymore) - lib.sh: add die and die_caller functions - lxc_suite_install_info is now a function - lint: shellcheck Signed-off-by: Markus Heiser --- utils/lxc.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index ce306fe85..a324bdaf2 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -112,14 +112,7 @@ EOF } usage_containers() { - cat < Date: Mon, 4 May 2020 16:15:23 +0200 Subject: archlinux: add package 'inetutils' to boilerplate Even it is a core component, some hoster do not have pre installed the 'inetutils' package. We do need this package for the 'hostname' command. Signed-off-by: Markus Heiser --- utils/lxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'utils/lxc.sh') diff --git a/utils/lxc.sh b/utils/lxc.sh index a324bdaf2..a0688bc07 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -43,7 +43,7 @@ echo 'Set disable_coredump false' >> /etc/sudo.conf # shellcheck disable=SC2034 archlinux_boilerplate=" pacman -Syu --noconfirm -pacman -S --noconfirm git curl wget sudo +pacman -S --noconfirm inetutils git curl wget sudo echo 'Set disable_coredump false' >> /etc/sudo.conf " -- cgit v1.2.3