From 0606cde105859d9bdb751825bf89952297b335ed Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 23 Jun 2021 12:48:31 +0200 Subject: [mod] add make targets to manage the build files in the /searx/static directory The idea is to avoid conflict when there are differents branches with changes are made on the static files. A solution is to ask the administrators to build the files from the sources, but it requires to install the npm dependencies. So the solution in this commit keep the sources and the build files in the same git repository. In one branch, the modification of the source (*.less, *.js) are commited without the built files. The built files are commited in a uniq commit, with a commit message "Static build" In case of merge or rebase, this commit can be dropped. New make targets: * static.build.commit.drop: drop the last "Static build". The command checks that there are only build files in the commit. * static.build.commit : call "make static.build.commit.drop" call "make themes.all" commit the files * static.git.restore.staged: git restore --staged * static.git.restore: git restore Related to https://github.com/searxng/searxng/issues/137 --- utils/manage_static.sh | 151 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 utils/manage_static.sh (limited to 'utils') diff --git a/utils/manage_static.sh b/utils/manage_static.sh new file mode 100755 index 000000000..3321f310e --- /dev/null +++ b/utils/manage_static.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +BUILD_COMMIT_MESSAGE="Static build" + +BUILT_PATHS=( + searx/static/themes/oscar/css + searx/static/themes/oscar/js + searx/static/themes/oscar/src/generated/pygments-logicodev.less + searx/static/themes/oscar/src/generated/pygments-pointhi.less + searx/static/themes/simple/css + searx/static/themes/simple/js + searx/static/themes/simple/src/generated/pygments.less +) + +CURRENT_BRANCH="$(git branch --show-current)" +STAGED_FILES=$(git diff --name-only --cached) + +git_log_current_branch() { + git log "heads/${CURRENT_BRANCH}" --not --exclude="${CURRENT_BRANCH}" --branches --remotes --pretty=format:"%h" +} + +is.build.commit() { + COMMIT_SHA=$1 + # check commit message + COMMIT_MESSAGE=$(git show -s --format=%s ${COMMIT_SHA}) + if [ "${COMMIT_MESSAGE}" != "${BUILD_COMMIT_MESSAGE}" ]; then + echo "Commit message of ${COMMIT_SHA} is '${COMMIT_MESSAGE}'" + return 1 + fi + + # check all files of the commit belongs to $BUILT_PATHS + COMMIT_FILES=$(git diff-tree --no-commit-id --name-only -r "${COMMIT_SHA}") + for i in ${BUILT_PATHS[*]}; do + # remove files of ${BUILT_PATHS} + COMMIT_FILES=$(echo "${COMMIT_FILES}" | grep -v "^${i}") + done + if [ -n "${COMMIT_FILES}" ]; then + echo "Commit $1 contains files that were not build: ${COMMIT_FILES}" + return 2 + fi + return 0 +} + +static.build.commit.drop() { + LAST_COMMIT_ID=$(git_log_current_branch | head -1) + + if [ -z "${LAST_COMMIT_ID}" ]; then + echo "Empty branch" + return 1 + fi + + is.build.commit "${LAST_COMMIT_ID}" + if [ $? -ne 0 ]; then + return $? + fi + echo "Drop last commit ${LAST_COMMIT_ID}" + git reset --hard HEAD~1 +} + +static.build.commit() { + # check for not commited files + NOT_COMMITED_FILES="$(git diff --name-only)" + if [ -n "${NOT_COMMITED_FILES}" ]; then + echo "Some files are not commited:" + echo "${NOT_COMMITED_FILES}" + return 1 + fi + + # check for staged files + if [ -n "${STAGED_FILES}" ]; then + echo "Some files are staged:" + echo "${STAGED_FILES}" + return 1 + fi + + # drop existing commit + static.commit.drop + if [ $? -ne 0 ]; then + return $? + fi + + ( + set -e + # build the themes + make themes.all + + # add build files + for built_path in ${BUILT_PATHS[@]}; do + git add -v "${built_path}" + done + + # check for modified files that are not staged + if [ -n "$(git diff --name-only)" ]; then + echo "make themes.all has created files that are not in BUILT_PATHS" + return 2 + fi + + # + git commit -m "Static build" + ) +} + +static.git.restore.staged() { + for i in ${BUILT_PATHS[*]}; do + STAGED_FILES_FOR_I=$(echo "${STAGED_FILES}" | grep "^${i}") + if [ -n "${STAGED_FILES_FOR_I}" ]; then + git restore --staged ${STAGED_FILES_FOR_I} + fi + done +} + +static.git.restore() { + static.git.restore.staged + + NOT_COMMITED_FILES="$(git diff --name-only)" + for i in ${BUILT_PATHS[*]}; do + NOT_COMMITED_FILES_FOR_I=$(echo "${NOT_COMMITED_FILES}" | grep "^${i}") + if [ -n "${NOT_COMMITED_FILES_FOR_I}" ]; then + git restore ${NOT_COMMITED_FILES_FOR_I} + fi + done +} + +main() { + case $1 in + static.build.commit.drop) + # drop last commit if it was made by the "commit" command + static.build.commit.drop + ;; + static.build.commit) + # call the "static.build.commit.drop" command, + # then "make themes.all" + # then commit the built files ($BUILT_PATHS). + static.build.commit + ;; + static.git.restore.staged) + # after "git add ." + # remove the built files + # so only the source are commited + static.git.restore.staged + ;; + static.git.restore) + # "git restore" of the built files. + static.git.restore + ;; + esac +} + +main "$@" -- cgit v1.2.3 From 2cb1f350869e0c7998d3c99d7d6ba38250ab7675 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 24 Jun 2021 15:15:34 +0200 Subject: [mod] utils/manage_static.sh: add script to shellcheck - add script to shellcheck, - fix error messages from shellcheck and - moved global variables to local variables (lower case) No functional change! Signed-off-by: Markus Heiser --- utils/manage_static.sh | 66 ++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 29 deletions(-) (limited to 'utils') diff --git a/utils/manage_static.sh b/utils/manage_static.sh index 3321f310e..f6eb1cae3 100755 --- a/utils/manage_static.sh +++ b/utils/manage_static.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -BUILD_COMMIT_MESSAGE="Static build" +BUILD_COMMIT_MESSAGE="[build] /static" BUILT_PATHS=( searx/static/themes/oscar/css @@ -14,70 +13,76 @@ BUILT_PATHS=( searx/static/themes/simple/src/generated/pygments.less ) -CURRENT_BRANCH="$(git branch --show-current)" -STAGED_FILES=$(git diff --name-only --cached) - git_log_current_branch() { - git log "heads/${CURRENT_BRANCH}" --not --exclude="${CURRENT_BRANCH}" --branches --remotes --pretty=format:"%h" + local branch + branch="$(git branch --show-current)" + git log "${branch}" --pretty=format:'%h' \ + --not --exclude="${branch}" --branches --remotes } is.build.commit() { - COMMIT_SHA=$1 + local commit_sha="$1" + local commit_message + local commit_files + # check commit message - COMMIT_MESSAGE=$(git show -s --format=%s ${COMMIT_SHA}) - if [ "${COMMIT_MESSAGE}" != "${BUILD_COMMIT_MESSAGE}" ]; then - echo "Commit message of ${COMMIT_SHA} is '${COMMIT_MESSAGE}'" + commit_message=$(git show -s --format=%s "${commit_sha}") + if [ "${commit_message}" != "${BUILD_COMMIT_MESSAGE}" ]; then + echo "Commit message of ${commit_sha} is '${commit_message}'" return 1 fi # check all files of the commit belongs to $BUILT_PATHS - COMMIT_FILES=$(git diff-tree --no-commit-id --name-only -r "${COMMIT_SHA}") + commit_files=$(git diff-tree --no-commit-id --name-only -r "${commit_sha}") for i in ${BUILT_PATHS[*]}; do # remove files of ${BUILT_PATHS} - COMMIT_FILES=$(echo "${COMMIT_FILES}" | grep -v "^${i}") + commit_files=$(echo "${commit_files}" | grep -v "^${i}") done - if [ -n "${COMMIT_FILES}" ]; then - echo "Commit $1 contains files that were not build: ${COMMIT_FILES}" + + if [ -n "${commit_files}" ]; then + echo "Commit $1 contains files that were not build: ${commit_files}" return 2 fi return 0 } static.build.commit.drop() { - LAST_COMMIT_ID=$(git_log_current_branch | head -1) + local last_commit_id + last_commit_id=$(git_log_current_branch | head -1) - if [ -z "${LAST_COMMIT_ID}" ]; then + if [ -z "${last_commit_id}" ]; then echo "Empty branch" return 1 fi - is.build.commit "${LAST_COMMIT_ID}" - if [ $? -ne 0 ]; then + if ! is.build.commit "${last_commit_id}"; then return $? fi - echo "Drop last commit ${LAST_COMMIT_ID}" + echo "Drop last commit ${last_commit_id}" git reset --hard HEAD~1 } static.build.commit() { + local staged_files + # check for not commited files - NOT_COMMITED_FILES="$(git diff --name-only)" - if [ -n "${NOT_COMMITED_FILES}" ]; then + if [ -n "$(git diff --name-only)" ]; then echo "Some files are not commited:" echo "${NOT_COMMITED_FILES}" return 1 fi + staged_files=$(git diff --name-only --cached) + # check for staged files - if [ -n "${STAGED_FILES}" ]; then + if [ -n "${staged_files}" ]; then echo "Some files are staged:" - echo "${STAGED_FILES}" + echo "${staged_files}" return 1 fi # drop existing commit - static.commit.drop - if [ $? -ne 0 ]; then + if static.commit.drop; then return $? fi @@ -87,7 +92,7 @@ static.build.commit() { make themes.all # add build files - for built_path in ${BUILT_PATHS[@]}; do + for built_path in "${BUILT_PATHS[@]}"; do git add -v "${built_path}" done @@ -96,16 +101,18 @@ static.build.commit() { echo "make themes.all has created files that are not in BUILT_PATHS" return 2 fi - - # - git commit -m "Static build" + git commit -m "${BUILD_COMMIT_MESSAGE}" ) } static.git.restore.staged() { + local STAGED_FILES + STAGED_FILES=$(git diff --name-only --cached) + for i in ${BUILT_PATHS[*]}; do STAGED_FILES_FOR_I=$(echo "${STAGED_FILES}" | grep "^${i}") if [ -n "${STAGED_FILES_FOR_I}" ]; then + # shellcheck disable=SC2086 git restore --staged ${STAGED_FILES_FOR_I} fi done @@ -118,6 +125,7 @@ static.git.restore() { for i in ${BUILT_PATHS[*]}; do NOT_COMMITED_FILES_FOR_I=$(echo "${NOT_COMMITED_FILES}" | grep "^${i}") if [ -n "${NOT_COMMITED_FILES_FOR_I}" ]; then + # shellcheck disable=SC2086 git restore ${NOT_COMMITED_FILES_FOR_I} fi done -- cgit v1.2.3 From 80710c666f48c78f65588c967b69ce008d7b799a Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 24 Jun 2021 17:23:01 +0200 Subject: [mod] utils/manage_static.sh: simplify git restore actions Signed-off-by: Markus Heiser --- utils/manage_static.sh | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) (limited to 'utils') diff --git a/utils/manage_static.sh b/utils/manage_static.sh index f6eb1cae3..8f58b5dbf 100755 --- a/utils/manage_static.sh +++ b/utils/manage_static.sh @@ -105,31 +105,6 @@ static.build.commit() { ) } -static.git.restore.staged() { - local STAGED_FILES - STAGED_FILES=$(git diff --name-only --cached) - - for i in ${BUILT_PATHS[*]}; do - STAGED_FILES_FOR_I=$(echo "${STAGED_FILES}" | grep "^${i}") - if [ -n "${STAGED_FILES_FOR_I}" ]; then - # shellcheck disable=SC2086 - git restore --staged ${STAGED_FILES_FOR_I} - fi - done -} - -static.git.restore() { - static.git.restore.staged - - NOT_COMMITED_FILES="$(git diff --name-only)" - for i in ${BUILT_PATHS[*]}; do - NOT_COMMITED_FILES_FOR_I=$(echo "${NOT_COMMITED_FILES}" | grep "^${i}") - if [ -n "${NOT_COMMITED_FILES_FOR_I}" ]; then - # shellcheck disable=SC2086 - git restore ${NOT_COMMITED_FILES_FOR_I} - fi - done -} main() { case $1 in @@ -147,11 +122,11 @@ main() { # after "git add ." # remove the built files # so only the source are commited - static.git.restore.staged + git restore --staged "${BUILT_PATHS[@]}" ;; static.git.restore) # "git restore" of the built files. - static.git.restore + git restore --worktree --staged "${BUILT_PATHS[@]}" ;; esac } -- cgit v1.2.3 From 03d5d14d98b06ce4d4a35f715b17eaccc4da7f86 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 25 Jun 2021 18:34:40 +0200 Subject: [mod] utils/manage_static.sh: remove git_log_current_branch Marginal simplification of the procedure to get only the last local commit which is not in remotes. Signed-off-by: Markus Heiser --- utils/manage_static.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'utils') diff --git a/utils/manage_static.sh b/utils/manage_static.sh index 8f58b5dbf..dc0fe2fed 100755 --- a/utils/manage_static.sh +++ b/utils/manage_static.sh @@ -13,13 +13,6 @@ BUILT_PATHS=( searx/static/themes/simple/src/generated/pygments.less ) -git_log_current_branch() { - local branch - branch="$(git branch --show-current)" - git log "${branch}" --pretty=format:'%h' \ - --not --exclude="${branch}" --branches --remotes -} - is.build.commit() { local commit_sha="$1" local commit_message @@ -46,9 +39,15 @@ is.build.commit() { return 0 } + static.build.commit.drop() { local last_commit_id - last_commit_id=$(git_log_current_branch | head -1) + local branch + + # get only last (option -n1) local commit not in remotes + branch="$(git branch --show-current)" + last_commit_id="$(git log -n1 "${branch}" --pretty=format:'%h'\ + --not --exclude="${branch}" --branches --remotes)" if [ -z "${last_commit_id}" ]; then echo "Empty branch" -- cgit v1.2.3 From 25b6309cf248a1c5c092780e98107d281999bd85 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 26 Jun 2021 08:46:20 +0200 Subject: [mod] move functions from utils/manage_static.sh to ./manage script The functions: - static.build.commit - static.build.commit.drop - static.build.restore are imported into the ./manage script. To avoid name collisions some variables and fucntions has been renamed by adding the prefix *static_*. Signed-off-by: Markus Heiser --- utils/lib_static.sh | 124 +++++++++++++++++++++++++++++++++++++++++++++ utils/manage_static.sh | 133 ------------------------------------------------- 2 files changed, 124 insertions(+), 133 deletions(-) create mode 100755 utils/lib_static.sh delete mode 100755 utils/manage_static.sh (limited to 'utils') diff --git a/utils/lib_static.sh b/utils/lib_static.sh new file mode 100755 index 000000000..0d4ba9bad --- /dev/null +++ b/utils/lib_static.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + + +STATIC_BUILD_COMMIT="[build] /static" +STATIC_BUILT_PATHS=( + searx/static/themes/oscar/css + searx/static/themes/oscar/js + searx/static/themes/oscar/src/generated/pygments-logicodev.less + searx/static/themes/oscar/src/generated/pygments-pointhi.less + searx/static/themes/simple/css + searx/static/themes/simple/js + searx/static/themes/simple/src/generated/pygments.less +) + +static_help(){ + cat </dev/null + + ( set -e + # build the themes + themes.all + + # add build files + for built_path in "${STATIC_BUILT_PATHS[@]}"; do + git add -v "${built_path}" + done + + # check for modified files that are not staged + if [ -n "$(git diff --name-only)" ]; then + die 42 "themes.all has created files that are not in STATIC_BUILT_PATHS" + fi + git commit -m "${STATIC_BUILD_COMMIT}" + ) +} + +static.build.restore() { + build_msg STATIC "git-restore of the built files (/static)" + git restore --staged "${STATIC_BUILT_PATHS[@]}" + git restore --worktree "${STATIC_BUILT_PATHS[@]}" +} diff --git a/utils/manage_static.sh b/utils/manage_static.sh deleted file mode 100755 index dc0fe2fed..000000000 --- a/utils/manage_static.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-or-later - -BUILD_COMMIT_MESSAGE="[build] /static" - -BUILT_PATHS=( - searx/static/themes/oscar/css - searx/static/themes/oscar/js - searx/static/themes/oscar/src/generated/pygments-logicodev.less - searx/static/themes/oscar/src/generated/pygments-pointhi.less - searx/static/themes/simple/css - searx/static/themes/simple/js - searx/static/themes/simple/src/generated/pygments.less -) - -is.build.commit() { - local commit_sha="$1" - local commit_message - local commit_files - - # check commit message - commit_message=$(git show -s --format=%s "${commit_sha}") - if [ "${commit_message}" != "${BUILD_COMMIT_MESSAGE}" ]; then - echo "Commit message of ${commit_sha} is '${commit_message}'" - return 1 - fi - - # check all files of the commit belongs to $BUILT_PATHS - commit_files=$(git diff-tree --no-commit-id --name-only -r "${commit_sha}") - for i in ${BUILT_PATHS[*]}; do - # remove files of ${BUILT_PATHS} - commit_files=$(echo "${commit_files}" | grep -v "^${i}") - done - - if [ -n "${commit_files}" ]; then - echo "Commit $1 contains files that were not build: ${commit_files}" - return 2 - fi - return 0 -} - - -static.build.commit.drop() { - local last_commit_id - local branch - - # get only last (option -n1) local commit not in remotes - branch="$(git branch --show-current)" - last_commit_id="$(git log -n1 "${branch}" --pretty=format:'%h'\ - --not --exclude="${branch}" --branches --remotes)" - - if [ -z "${last_commit_id}" ]; then - echo "Empty branch" - return 1 - fi - - if ! is.build.commit "${last_commit_id}"; then - return $? - fi - echo "Drop last commit ${last_commit_id}" - git reset --hard HEAD~1 -} - -static.build.commit() { - local staged_files - - # check for not commited files - if [ -n "$(git diff --name-only)" ]; then - echo "Some files are not commited:" - echo "${NOT_COMMITED_FILES}" - return 1 - fi - - staged_files=$(git diff --name-only --cached) - - # check for staged files - if [ -n "${staged_files}" ]; then - echo "Some files are staged:" - echo "${staged_files}" - return 1 - fi - - # drop existing commit - if static.commit.drop; then - return $? - fi - - ( - set -e - # build the themes - make themes.all - - # add build files - for built_path in "${BUILT_PATHS[@]}"; do - git add -v "${built_path}" - done - - # check for modified files that are not staged - if [ -n "$(git diff --name-only)" ]; then - echo "make themes.all has created files that are not in BUILT_PATHS" - return 2 - fi - git commit -m "${BUILD_COMMIT_MESSAGE}" - ) -} - - -main() { - case $1 in - static.build.commit.drop) - # drop last commit if it was made by the "commit" command - static.build.commit.drop - ;; - static.build.commit) - # call the "static.build.commit.drop" command, - # then "make themes.all" - # then commit the built files ($BUILT_PATHS). - static.build.commit - ;; - static.git.restore.staged) - # after "git add ." - # remove the built files - # so only the source are commited - git restore --staged "${BUILT_PATHS[@]}" - ;; - static.git.restore) - # "git restore" of the built files. - git restore --worktree --staged "${BUILT_PATHS[@]}" - ;; - esac -} - -main "$@" -- cgit v1.2.3