[translations] web integration

* make babel.translations.to.master: pull weblate updates
* make babel.master.to.translations: push .pot and .po files to weblate

1 files changed, 121 insertions, 32 deletions

diff --git a/manage b/manage
index 42c617f5b..7a7c98143 100755
--- a/manage
+++ b/manage
@@ -45,9 +45,8 @@ help() {
 buildenv:
   rebuild ./utils/brand.env
 babel.:
-  extract   : extract messages from source files and generate POT file
-  update    : update existing message catalogs from POT file
-  compile   : compile translation catalogs into binary MO files
+  master.to.translations: update the translations branch from the messages of the master branch.
+  translations.to.master: copy change from the translations branch to the master branch.
 data.:
   all       : update searx/languages.py and ./data/*
   languages : update searx/data/engines_languages.json & searx/languages.py
@@ -122,46 +121,136 @@ buildenv() {
     return "${PIPESTATUS[0]}"
 }
 
-babel.sha256sum() {
-    grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
-}
+TRANSLATIONS_WORKTREE="$CACHE/translations"
 
-ci.babel.update() {
-    local sha_before
+babel.setup.translations.worktree() {
     (   set -e
-        sha_before="$(babel.sha256sum)"
-        babel.extract
-        if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
-            build_msg BABEL 'no changes detected, exiting'
-            return 1
+        if ! git remote get-url weblate 2> /dev/null; then
+            git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
+        fi
+        if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
+            pushd .
+            cd "${TRANSLATIONS_WORKTREE}"
+            git reset --hard HEAD
+            git pull origin translations
+            popd
+        else
+            mkdir -p "${TRANSLATIONS_WORKTREE}"
+            git worktree add "${TRANSLATIONS_WORKTREE}" translations
         fi
-        babel.update
-        build_msg BABEL 'update done, edit .po files if required and run babel.compile'
     )
-    dump_return $?
 }
 
-babel.extract() {
-    build_msg BABEL 'extract messages from source files and generate POT file'
-    pyenv.cmd pybabel extract -F babel.cfg \
-            -o "searx/translations/messages.pot" \
-            "searx/"
+babel.weblate.to.translations() {
+    (   set -e
+        if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then
+            build_msg BABEL "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)"
+            exit 1
+        fi
+        # weblate: commit pending changes
+        pyenv.cmd wlc pull
+        pyenv.cmd wlc commit
+        # get the translations in a worktree
+        babel.setup.translations.worktree
+        cd "${TRANSLATIONS_WORKTREE}"
+        git remote update weblate
+        git merge weblate/translations
+        git push
+    )
     dump_return $?
 }
 
-babel.update() {
-    build_msg BABEL 'update existing message catalogs from POT file'
-    pyenv.cmd pybabel update -N \
-              -i "searx/translations/messages.pot" \
-              -d "searx/translations"
-    dump_return $?
+babel.translations.to.master() {
+    local existing_commit_hash commit_body commit_message exitcode
+    (   set -e
+        pyenv.cmd wlc lock
+        babel.setup.translations.worktree
+        existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1  --pretty=format:'%h')
+        # pull weblate commits
+        babel.weblate.to.translations
+        # copy the changes to the master branch
+        cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
+        # compile translations
+        build_msg BABEL 'compile translation catalogs into binary MO files'
+        pyenv.cmd pybabel compile --statistics \
+                -d "searx/translations"
+        # git add/commit (no push)
+        commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
+        commit_message=$(echo -e "[translations] update\n${commit_body}")
+        git add searx/translations
+        git commit -m "${commit_message}"
+    )
+    exitcode=$?
+    (   # make sure to always unlock weblate
+        set -e
+        pyenv.cmd wlc unlock
+    )
+    dump_return $exitcode
 }
 
-babel.compile() {
-    build_msg BABEL 'compile translation catalogs into binary MO files'
-    pyenv.cmd pybabel compile --statistics \
-              -d "searx/translations"
-    dump_return $?
+babel.master.to.translations() {
+    local messages_pot diff_messages_pot last_commit_hash last_commit_detail last_commit_message exitcode
+    (   set -e
+        # lock change on weblate
+        pyenv.cmd wlc lock
+
+        # get translation branch
+        babel.setup.translations.worktree
+
+        # update messages.pot
+        build_msg BABEL 'extract messages from source files and generate POT file'
+        messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
+        pyenv.cmd pybabel extract -F babel.cfg \
+                -o "${messages_pot}" \
+                "searx/"
+
+        # stop if there is no meaningful change
+        diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}"; git diff -- "searx/translations/messages.pot")
+        if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
+            build_msg BABEL 'no changes detected, exiting'
+            return 0
+        fi
+
+        # save messages.pot for later
+        cd "${TRANSLATIONS_WORKTREE}"
+        git stash push
+        cd -
+
+        # merge weblate commits into the translations branch
+        babel.weblate.to.translations
+
+        # restore messages.pot
+        cd "${TRANSLATIONS_WORKTREE}"
+        git stash pop
+        cd -
+
+        set -x
+
+        # update messages.po files
+        build_msg BABEL 'update existing message catalogs from POT file'
+        pyenv.cmd pybabel update -N \
+            -i "${messages_pot}" \
+            -d "${TRANSLATIONS_WORKTREE}/searx/translations"
+
+        # git add/commit/push
+        last_commit_hash=$(git log -n1  --pretty=format:'%h')
+        last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
+        last_commit_message=$(echo -e "[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}")
+        cd "${TRANSLATIONS_WORKTREE}"
+        git add searx/translations
+        git commit -m "${last_commit_message}"
+        git push
+        cd -
+
+        # notify weblate
+        pyenv.cmd wlc pull
+    )
+    exitcode=$?
+    (   # make sure to always unlock weblate
+        set -e
+        pyenv.cmd wlc unlock
+    )
+    dump_return $exitcode
 }
 
 data.all() {