From 901ab87717f642f2ab5ce68ef0d01be38ca6726d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 5 Aug 2021 13:57:48 +0200 Subject: [PATCH] [translations] web integration * make babel.translations.to.master: pull weblate updates * make babel.master.to.translations: push .pot and .po files to weblate --- .github/workflows/integration.yml | 40 +++--- .github/workflows/translations-update.yml | 56 ++++++++ .weblate | 3 + Makefile | 2 +- README.rst | 2 + docs/dev/translation.rst | 81 ++++-------- docs/dev/translation.svg | 45 +++++++ manage | 153 +++++++++++++++++----- requirements-dev.txt | 1 + 9 files changed, 278 insertions(+), 105 deletions(-) create mode 100644 .github/workflows/translations-update.yml create mode 100644 .weblate create mode 100644 docs/dev/translation.svg diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 78fc66e3f..886df3b2f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -91,7 +91,7 @@ jobs: COMMIT_MESSAGE: build from commit ${{ github.sha }} babel: - name: Babel + name: Update translations branch runs-on: ubuntu-20.04 if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} needs: @@ -102,32 +102,32 @@ jobs: - name: Checkout uses: actions/checkout@v2 with: - persist-credentials: false + fetch-depth: '0' + token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v2 with: python-version: '3.9' architecture: 'x64' + - name: Cache Python dependencies + id: cache-python + uses: actions/cache@v2 + with: + path: ./local + key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }} + - name: weblate & git setup + env: + WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} + run: | + mkdir -p ~/.config + echo "${WEBLATE_CONFIG}" > ~/.config/weblate + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" - name: Update transations id: update - continue-on-error: true - run: make V=1 ci.babel.update - - name: Open pull request - if: steps.update.outcome == 'success' - uses: peter-evans/create-pull-request@v3 - with: - commit-message: Update translations (pot, po) - committer: searx-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: update_translations_pot - delete-branch: true - draft: false - title: 'Update translations (pot, po)' - body: | - Update messages.pot and messages.po files - labels: | - translation + run: | + git restore utils/brand.env + make V=1 babel.master.to.translations dockers: name: Docker diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml new file mode 100644 index 000000000..274a093df --- /dev/null +++ b/.github/workflows/translations-update.yml @@ -0,0 +1,56 @@ +name: "Update translations" +on: + schedule: + - cron: "05 07 * * 5" + workflow_dispatch: + +jobs: + babel: + name: "translations: update master branch" + runs-on: ubuntu-20.04 + if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: '0' + token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - name: Cache Python dependencies + id: cache-python + uses: actions/cache@v2 + with: + path: ./local + key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }} + - name: weblate & git setup + env: + WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} + run: | + mkdir -p ~/.config + echo "${WEBLATE_CONFIG}" > ~/.config/weblate + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + - name: Merge and push transation updates + run: | + make V=1 babel.translations.to.master + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} + commit-message: Update translations + committer: searxng-bot + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: translations_update + delete-branch: true + draft: false + title: 'Update translations' + body: | + Update translations + labels: | + translation diff --git a/.weblate b/.weblate new file mode 100644 index 000000000..80bdba884 --- /dev/null +++ b/.weblate @@ -0,0 +1,3 @@ +[weblate] +url = https://weblate.bubu1.eu/api/ +translation = searxng/searxng diff --git a/Makefile b/Makefile index d48f5c531..cee1e4b97 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ test.shell: # wrap ./manage script MANAGE += buildenv -MANAGE += ci.babel.update babel.extract babel.update babel.compile +MANAGE += babel.translations.to.master babel.master.to.translations MANAGE += data.all data.languages data.useragents data.osm_keys_tags MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean MANAGE += docker.build docker.push docker.buildx diff --git a/README.rst b/README.rst index 1ab7ee201..a4d2a30ef 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_ .. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits :target: https://github.com/searxng/searxng/commits/master +.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg + :target: https://weblate.bubu1.eu/projects/searxng/ If you are looking for running instances, ready to use, then visit searx.space_. diff --git a/docs/dev/translation.rst b/docs/dev/translation.rst index 523dcf78c..be0cf5bef 100644 --- a/docs/dev/translation.rst +++ b/docs/dev/translation.rst @@ -4,69 +4,46 @@ Translation =========== -.. _searx@transifex: https://www.transifex.com/asciimoo/searx/ +.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/ -Translation currently takes place on `searx@transifex`_ +Translation takes place on `weblate.bubu1.eu`_ ( `documentation `_ ). -Requirements -============ +New messages on the master branch are extracted and pushed to Weblate automatically. -* Transifex account +Every Friday, a GitHub workflow creates a pull request with the updated translations (messages.mo, messages.po, messages.mo files). -Init Transifex project -====================== +.. image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg + :target: https://weblate.bubu1.eu/projects/searxng/ -After installing ``transifex`` using pip, run the following command to -initialize the project. - -.. code:: sh - - ./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/ - - -After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it -into the configuration file. - -Create a configuration file for ``tx`` named ``$HOME/.tx/config``. - -.. code:: ini - - [main] - host = https://www.transifex.com - [searx.messagespo] - file_filter = searx/translations//LC_MESSAGES/messages.po - source_file = messages.pot - source_lang = en - type = PO - - -Then run ``tx set``: - -.. code:: shell - - ./manage pyenv.cmd tx set --auto-local \ - -r searx.messagespo 'searx/translations//LC_MESSAGES/messages.po' \ - --source-lang en --type PO --source-file messages.pot --execute - - -Update translations +Weblate integration =================== -To retrieve the latest translations, pull it from Transifex. +Weblate monitors the `translations branch `_, not the master branch. -.. code:: sh +This branch contains only the .pot and pot files, nothing else. - ./manage pyenv.cmd tx pull -a - [?] Enter your api token: .... +Documentation +------------- -Then check the new languages. If strings translated are not enough, delete those -folders, because those should not be compiled. Call the command below to compile -the ``.po`` files. +* `wlc `_ +* `pybabel `_ +* `weblate workflow `_ -.. code:: shell +Worfklow +-------- - ./manage pyenv.cmd pybabel compile -d searx/translations +.. image:: translation.svg -After the compilation is finished commit the ``.po`` and ``.mo`` files and -create a PR. +wlc +--- + +All weblate integration is done by GitHub workflows, but if you want to use wlc, copy this content into `~/.config/weblate `_ : + +.. code-block:: ini + + [keys] + https://weblate.bubu1.eu/api/ = APIKEY + + +Replace `APIKEY` by `your API key `_. diff --git a/docs/dev/translation.svg b/docs/dev/translation.svg new file mode 100644 index 000000000..060e58420 --- /dev/null +++ b/docs/dev/translation.svg @@ -0,0 +1,45 @@ +master branchmaster branchtranslations_update branchtranslations_update branchtranslations branchtranslations branchweblate clone oftranslations branchweblate clone oftranslations branchweblatepending changesweblatepending changesfor each commit on master.github/workflows/integration.ymlmake babel.master.to.translationswlc lockwlc pullwlc commitgit merge weblate/translationspybabel extractextract messages, store messages.pot on translations branchpybabel update (messages.po)git add searx/translationsgit commitgit pushwlc unlockevery Friday.github/workflows/translations-update.ymlmake babel.translations.to.masterwlc lockwlc pullwlc commitgit merge weblate/translationspybabel compilecp searx/translationsgit addgit commitwlc unlockcreate or update pull request"Update translations"developper's reviewmerge pull request diff --git a/manage b/manage index 42c617f5b..7a7c98143 100755 --- a/manage +++ b/manage @@ -45,9 +45,8 @@ help() { buildenv: rebuild ./utils/brand.env babel.: - extract : extract messages from source files and generate POT file - update : update existing message catalogs from POT file - compile : compile translation catalogs into binary MO files + master.to.translations: update the translations branch from the messages of the master branch. + translations.to.master: copy change from the translations branch to the master branch. data.: all : update searx/languages.py and ./data/* languages : update searx/data/engines_languages.json & searx/languages.py @@ -122,46 +121,136 @@ buildenv() { return "${PIPESTATUS[0]}" } -babel.sha256sum() { - grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' ' +TRANSLATIONS_WORKTREE="$CACHE/translations" + +babel.setup.translations.worktree() { + ( set -e + if ! git remote get-url weblate 2> /dev/null; then + git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/ + fi + if [ -d "${TRANSLATIONS_WORKTREE}" ]; then + pushd . + cd "${TRANSLATIONS_WORKTREE}" + git reset --hard HEAD + git pull origin translations + popd + else + mkdir -p "${TRANSLATIONS_WORKTREE}" + git worktree add "${TRANSLATIONS_WORKTREE}" translations + fi + ) } -ci.babel.update() { - local sha_before +babel.weblate.to.translations() { ( set -e - sha_before="$(babel.sha256sum)" - babel.extract - if [ "$(babel.sha256sum)" = "${sha_before}" ]; then - build_msg BABEL 'no changes detected, exiting' - return 1 + if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then + build_msg BABEL "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)" + exit 1 fi - babel.update - build_msg BABEL 'update done, edit .po files if required and run babel.compile' + # weblate: commit pending changes + pyenv.cmd wlc pull + pyenv.cmd wlc commit + # get the translations in a worktree + babel.setup.translations.worktree + cd "${TRANSLATIONS_WORKTREE}" + git remote update weblate + git merge weblate/translations + git push ) dump_return $? } -babel.extract() { - build_msg BABEL 'extract messages from source files and generate POT file' - pyenv.cmd pybabel extract -F babel.cfg \ - -o "searx/translations/messages.pot" \ - "searx/" - dump_return $? +babel.translations.to.master() { + local existing_commit_hash commit_body commit_message exitcode + ( set -e + pyenv.cmd wlc lock + babel.setup.translations.worktree + existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h') + # pull weblate commits + babel.weblate.to.translations + # copy the changes to the master branch + cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx" + # compile translations + build_msg BABEL 'compile translation catalogs into binary MO files' + pyenv.cmd pybabel compile --statistics \ + -d "searx/translations" + # git add/commit (no push) + commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD") + commit_message=$(echo -e "[translations] update\n${commit_body}") + git add searx/translations + git commit -m "${commit_message}" + ) + exitcode=$? + ( # make sure to always unlock weblate + set -e + pyenv.cmd wlc unlock + ) + dump_return $exitcode } -babel.update() { - build_msg BABEL 'update existing message catalogs from POT file' - pyenv.cmd pybabel update -N \ - -i "searx/translations/messages.pot" \ - -d "searx/translations" - dump_return $? -} +babel.master.to.translations() { + local messages_pot diff_messages_pot last_commit_hash last_commit_detail last_commit_message exitcode + ( set -e + # lock change on weblate + pyenv.cmd wlc lock -babel.compile() { - build_msg BABEL 'compile translation catalogs into binary MO files' - pyenv.cmd pybabel compile --statistics \ - -d "searx/translations" - dump_return $? + # get translation branch + babel.setup.translations.worktree + + # update messages.pot + build_msg BABEL 'extract messages from source files and generate POT file' + messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot" + pyenv.cmd pybabel extract -F babel.cfg \ + -o "${messages_pot}" \ + "searx/" + + # stop if there is no meaningful change + diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}"; git diff -- "searx/translations/messages.pot") + if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then + build_msg BABEL 'no changes detected, exiting' + return 0 + fi + + # save messages.pot for later + cd "${TRANSLATIONS_WORKTREE}" + git stash push + cd - + + # merge weblate commits into the translations branch + babel.weblate.to.translations + + # restore messages.pot + cd "${TRANSLATIONS_WORKTREE}" + git stash pop + cd - + + set -x + + # update messages.po files + build_msg BABEL 'update existing message catalogs from POT file' + pyenv.cmd pybabel update -N \ + -i "${messages_pot}" \ + -d "${TRANSLATIONS_WORKTREE}/searx/translations" + + # git add/commit/push + last_commit_hash=$(git log -n1 --pretty=format:'%h') + last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}") + last_commit_message=$(echo -e "[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}") + cd "${TRANSLATIONS_WORKTREE}" + git add searx/translations + git commit -m "${last_commit_message}" + git push + cd - + + # notify weblate + pyenv.cmd wlc pull + ) + exitcode=$? + ( # make sure to always unlock weblate + set -e + pyenv.cmd wlc unlock + ) + dump_return $exitcode } data.all() { diff --git a/requirements-dev.txt b/requirements-dev.txt index a6f29db55..e8dc0221c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14 linuxdoc==20210324 aiounittest==1.4.0 yamllint==1.26.2 +wlc==1.12