diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 78fc66e3f..84ce51a04 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -91,7 +91,7 @@ jobs: COMMIT_MESSAGE: build from commit ${{ github.sha }} babel: - name: Babel + name: Update translations branch runs-on: ubuntu-20.04 if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} needs: @@ -102,32 +102,32 @@ jobs: - name: Checkout uses: actions/checkout@v2 with: - persist-credentials: false + fetch-depth: '0' + token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v2 with: python-version: '3.9' architecture: 'x64' + - name: Cache Python dependencies + id: cache-python + uses: actions/cache@v2 + with: + path: ./local + key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }} + - name: weblate & git setup + env: + WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} + run: | + mkdir -p ~/.config + echo "${WEBLATE_CONFIG}" > ~/.config/weblate + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" - name: Update transations id: update - continue-on-error: true - run: make V=1 ci.babel.update - - name: Open pull request - if: steps.update.outcome == 'success' - uses: peter-evans/create-pull-request@v3 - with: - commit-message: Update translations (pot, po) - committer: searx-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: update_translations_pot - delete-branch: true - draft: false - title: 'Update translations (pot, po)' - body: | - Update messages.pot and messages.po files - labels: | - translation + run: | + git restore utils/brand.env + make V=1 weblate.push.translations dockers: name: Docker diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml new file mode 100644 index 000000000..c3120ce1e --- /dev/null +++ b/.github/workflows/translations-update.yml @@ -0,0 +1,56 @@ +name: "Update translations" +on: + schedule: + - cron: "05 07 * * 5" + workflow_dispatch: + +jobs: + babel: + name: "create PR for additons from weblate" + runs-on: ubuntu-20.04 + if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: '0' + token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - name: Cache Python dependencies + id: cache-python + uses: actions/cache@v2 + with: + path: ./local + key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }} + - name: weblate & git setup + env: + WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} + run: | + mkdir -p ~/.config + echo "${WEBLATE_CONFIG}" > ~/.config/weblate + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + - name: Merge and push transation updates + run: | + make V=1 weblate.translations.commit + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} + commit-message: Update translations + committer: searxng-bot + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: translations_update + delete-branch: true + draft: false + title: 'Update translations' + body: | + Update translations + labels: | + translation diff --git a/.weblate b/.weblate new file mode 100644 index 000000000..80bdba884 --- /dev/null +++ b/.weblate @@ -0,0 +1,3 @@ +[weblate] +url = https://weblate.bubu1.eu/api/ +translation = searxng/searxng diff --git a/Makefile b/Makefile index d48f5c531..88edef5cf 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ test.shell: # wrap ./manage script MANAGE += buildenv -MANAGE += ci.babel.update babel.extract babel.update babel.compile +MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.languages data.useragents data.osm_keys_tags MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean MANAGE += docker.build docker.push docker.buildx diff --git a/README.rst b/README.rst index 1ab7ee201..a4d2a30ef 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_ .. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits :target: https://github.com/searxng/searxng/commits/master +.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg + :target: https://weblate.bubu1.eu/projects/searxng/ If you are looking for running instances, ready to use, then visit searx.space_. diff --git a/docs/dev/translation.rst b/docs/dev/translation.rst index 523dcf78c..922c29c54 100644 --- a/docs/dev/translation.rst +++ b/docs/dev/translation.rst @@ -4,69 +4,77 @@ Translation =========== -.. _searx@transifex: https://www.transifex.com/asciimoo/searx/ +.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/ +.. _Weblate: https://docs.weblate.org +.. _translations branch: https://github.com/searxng/searxng/tree/translations +.. _orphan branch: https://git-scm.com/docs/git-checkout#Documentation/git-checkout.txt---orphanltnewbranchgt +.. _Weblate repository: https://weblate.bubu1.eu/projects/searxng/searxng/#repository +.. _wlc: https://docs.weblate.org/en/latest/wlc.html -Translation currently takes place on `searx@transifex`_ +.. |translated| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg + :target: https://weblate.bubu1.eu/projects/searxng/ -Requirements -============ +.. sidebar:: |translated| -* Transifex account + - Weblate_ + - SearXNG `translations branch`_ + - SearXNG `Weblate repository`_ + - Weblate Client: wlc_ + - Babel Command-Line: `pybabel `_ + - `weblate workflow `_ -Init Transifex project -====================== +Translation takes place on weblate.bubu1.eu_. -After installing ``transifex`` using pip, run the following command to -initialize the project. +Translations which has been added by translators on the weblate.bubu1.eu_ UI are +committed to Weblate's counterpart of the SearXNG *origin* repository which is +located at ``https://weblate.bubu1.eu/git/searxng/searxng``. -.. code:: sh +There is no need to clone this repository, :ref:`SearXNG Weblate workflow` take +care of the synchronization with the *origin*. To avoid merging commits from +the counterpart directly on the ``master`` branch of *SearXNG origin*, a *pull +request* (PR) is created by this workflow. - ./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/ +Weblate monitors the `translations branch`_, not the ``master`` branch. This +branch is an `orphan branch`_, decoupled from the master branch (we already know +orphan branches from the ``gh-pages``). The `translations branch`_ contains +only the + +- ``translation/messages.pot`` and the +- ``translation/*/messages.po`` files, nothing else. -After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it -into the configuration file. +.. _SearXNG Weblate workflow: -Create a configuration file for ``tx`` named ``$HOME/.tx/config``. +.. figure:: translation.svg -.. code:: ini + SearXNG's PR workflow to be in sync with Weblate - [main] - host = https://www.transifex.com - [searx.messagespo] - file_filter = searx/translations//LC_MESSAGES/messages.po - source_file = messages.pot - source_lang = en - type = PO +Sync from *origin* to *weblate*: using ``make weblate.push.translations`` + For each commit on the ``master`` branch of SearXNG *origin* the GitHub job + :origin:`babel / Update translations branch + <.github/workflows/integration.yml>` checks for updated translations. +Sync from *weblate* to *origin*: using ``make weblate.translations.commit`` + Every Friday, the GitHub workflow :origin:`babel / create PR for additons from + weblate <.github/workflows/translations-update.yml>` creates a PR with the + updated translation files: -Then run ``tx set``: + - ``translation/messages.pot``, + - ``translation/*/messages.po`` and + - ``translation/*/messages.mo`` -.. code:: shell +wlc +=== - ./manage pyenv.cmd tx set --auto-local \ - -r searx.messagespo 'searx/translations//LC_MESSAGES/messages.po' \ - --source-lang en --type PO --source-file messages.pot --execute +.. _wlc configuration: https://docs.weblate.org/en/latest/wlc.html#wlc-config +.. _API key: https://weblate.bubu1.eu/accounts/profile/#api +All weblate integration is done by GitHub workflows, but if you want to use wlc_, +copy this content into `wlc configuration`_ in your HOME ``~/.config/weblate`` -Update translations -=================== +.. code-block:: ini -To retrieve the latest translations, pull it from Transifex. + [keys] + https://weblate.bubu1.eu/api/ = APIKEY -.. code:: sh - - ./manage pyenv.cmd tx pull -a - [?] Enter your api token: .... - -Then check the new languages. If strings translated are not enough, delete those -folders, because those should not be compiled. Call the command below to compile -the ``.po`` files. - -.. code:: shell - - ./manage pyenv.cmd pybabel compile -d searx/translations - - -After the compilation is finished commit the ``.po`` and ``.mo`` files and -create a PR. +Replace ``APIKEY`` by your `API key`_. diff --git a/docs/dev/translation.svg b/docs/dev/translation.svg new file mode 100644 index 000000000..70400cca7 --- /dev/null +++ b/docs/dev/translation.svg @@ -0,0 +1,45 @@ +master branchmaster branchtranslations_update branchtranslations_update branchtranslations branchtranslations branchweblate clone ofSearXNG repositoryweblate clone ofSearXNG repositoryweblatepending changesweblatepending changesfor each commit on master.github/workflows/integration.ymlmake weblate.push.translationswlc lockwlc pullwlc commitgit merge weblate/translationspybabel extractextract messages, store messages.pot on translations branchpybabel update (messages.po)git add searx/translationsgit commitgit pushwlc unlockevery Friday.github/workflows/translations-update.ymlmake weblate.translations.commitwlc lockwlc pullwlc commitgit merge weblate/translationspybabel compilecp searx/translationsgit addgit commitwlc unlockcreate or update pull request"Update translations"developper's reviewmerge pull request diff --git a/manage b/manage index 42c617f5b..a855512f5 100755 --- a/manage +++ b/manage @@ -45,9 +45,8 @@ help() { buildenv: rebuild ./utils/brand.env babel.: - extract : extract messages from source files and generate POT file - update : update existing message catalogs from POT file - compile : compile translation catalogs into binary MO files + master.to.translations: update the translations branch from the messages of the master branch. + translations.to.master: copy change from the translations branch to the master branch. data.: all : update searx/languages.py and ./data/* languages : update searx/data/engines_languages.json & searx/languages.py @@ -122,46 +121,185 @@ buildenv() { return "${PIPESTATUS[0]}" } -babel.sha256sum() { - grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' ' +TRANSLATIONS_WORKTREE="$CACHE/translations" + +weblate.translations.worktree() { + + # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch + # 'translations' from Weblate's counterpart (weblate) of the SearXNG + # (origin). + # + # remote weblate https://weblate.bubu1.eu/git/searxng/searxng/ + + ( set -e + if ! git remote get-url weblate 2> /dev/null; then + git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/ + fi + if [ -d "${TRANSLATIONS_WORKTREE}" ]; then + pushd . + cd "${TRANSLATIONS_WORKTREE}" + git reset --hard HEAD + git pull origin translations + popd + else + mkdir -p "${TRANSLATIONS_WORKTREE}" + git worktree add "${TRANSLATIONS_WORKTREE}" translations + fi + ) } -ci.babel.update() { - local sha_before +weblate.to.translations() { + + # Update 'translations' branch of SearXNG (origin) with last additions from + # Weblate. + + # 1. Check if Weblate is locked, if not die with error message + # 2. On Weblate's counterpart (weblate), pull master and translations branch + # from SearXNG (origin). + # 3. Commit changes made in a Weblate object on Weblate's counterpart + # (weblate). + # 4. In translations worktree, merge changes of branch 'translations' from + # remote 'weblate' and push it on branch 'translations' of 'origin' + ( set -e - sha_before="$(babel.sha256sum)" - babel.extract - if [ "$(babel.sha256sum)" = "${sha_before}" ]; then - build_msg BABEL 'no changes detected, exiting' - return 1 + if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then + die 1 "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)" fi - babel.update - build_msg BABEL 'update done, edit .po files if required and run babel.compile' + # weblate: commit pending changes + pyenv.cmd wlc pull + pyenv.cmd wlc commit + + # get the translations in a worktree + weblate.translations.worktree + + pushd "${TRANSLATIONS_WORKTREE}" + git remote update weblate + git merge weblate/translations + git push + popd ) dump_return $? } -babel.extract() { - build_msg BABEL 'extract messages from source files and generate POT file' - pyenv.cmd pybabel extract -F babel.cfg \ - -o "searx/translations/messages.pot" \ - "searx/" - dump_return $? +weblate.translations.commit() { + + # Update 'translations' branch of SearXNG (origin) with last additions from + # Weblate. Copy the changes to the master branch, compile translations and + # create a commit in the local branch (master) + + local existing_commit_hash commit_body commit_message exitcode + ( set -e + # lock change on weblate + pyenv.cmd wlc lock + + # get translations branch in git worktree (TRANSLATIONS_WORKTREE) + weblate.translations.worktree + existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h') + + # pull weblate commits + weblate.to.translations + + # copy the changes to the master branch + cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx" + + # compile translations + build_msg BABEL 'compile translation catalogs into binary MO files' + pyenv.cmd pybabel compile --statistics \ + -d "searx/translations" + # git add/commit (no push) + commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD") + commit_message=$(echo -e "[translations] update\n${commit_body}") + git add searx/translations + git commit -m "${commit_message}" + ) + exitcode=$? + ( # make sure to always unlock weblate + set -e + pyenv.cmd wlc unlock + ) + dump_return $exitcode } -babel.update() { - build_msg BABEL 'update existing message catalogs from POT file' - pyenv.cmd pybabel update -N \ - -i "searx/translations/messages.pot" \ - -d "searx/translations" - dump_return $? -} +weblate.push.translations() { -babel.compile() { - build_msg BABEL 'compile translation catalogs into binary MO files' - pyenv.cmd pybabel compile --statistics \ - -d "searx/translations" - dump_return $? + # Push *translation changes* from SearXNG (origin) to Weblate's counterpart + # (weblate). + + # In branch master of SearXNG (origin) check for meaningful changes in + # folder 'searx/translations', commit changes on branch 'translations' and + # at least, pull updated branches on Weblate's counterpart (weblate). + + # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch + # 'translations' from remote 'weblate'. + # 2. Stop if there is no meaningful change in the 'master' branch (origin), + # compared to the 'translations' branch (weblate), otherwise ... + # 3. Update 'translations' branch of SearXNG (origin) with last additions + # from Weblate. + # 5. Notify Weblate to pull updated 'master' & 'translations' branch. + + local messages_pot diff_messages_pot last_commit_hash last_commit_detail \ + last_commit_message exitcode + ( set -e + # lock change on weblate + pyenv.cmd wlc lock + + # get translations branch in git worktree (TRANSLATIONS_WORKTREE) + weblate.translations.worktree + + # update messages.pot in the master branch + build_msg BABEL 'extract messages from source files and generate POT file' + messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot" + pyenv.cmd pybabel extract -F babel.cfg \ + -o "${messages_pot}" \ + "searx/" + + # stop if there is no meaningful change in the master branch + diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\ + git diff -- "searx/translations/messages.pot") + if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then + build_msg BABEL 'no changes detected, exiting' + return 0 + fi + + # save messages.pot in the translations branch for later + pushd "${TRANSLATIONS_WORKTREE}" + git stash push + popd + + # merge weblate commits into the translations branch + weblate.to.translations + + # restore messages.pot in the translations branch + pushd "${TRANSLATIONS_WORKTREE}" + git stash pop + popd + + # update messages.po files in the master branch + build_msg BABEL 'update existing message catalogs from POT file' + pyenv.cmd pybabel update -N \ + -i "${messages_pot}" \ + -d "${TRANSLATIONS_WORKTREE}/searx/translations" + + # git add/commit/push + last_commit_hash=$(git log -n1 --pretty=format:'%h') + last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}") + last_commit_message="[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}" + + pushd "${TRANSLATIONS_WORKTREE}" + git add searx/translations + git commit -m "${last_commit_message}" + git push + popd + + # notify weblate to pull updated master & translations branch + pyenv.cmd wlc pull + ) + exitcode=$? + ( # make sure to always unlock weblate + set -e + pyenv.cmd wlc unlock + ) + dump_return $exitcode } data.all() { diff --git a/requirements-dev.txt b/requirements-dev.txt index a6f29db55..e8dc0221c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14 linuxdoc==20210324 aiounittest==1.4.0 yamllint==1.26.2 +wlc==1.12