mirror of
https://github.com/searxng/searxng
synced 2024-01-01 19:24:07 +01:00
Merge pull request #250 from dalf/weblate_v2
[translations] weblate integration
This commit is contained in:
Alexandre Flament
GitHub
commit
28b350eda6
9 changed files with 352 additions and 99 deletions
40
.github/workflows/integration.yml
vendored
40
.github/workflows/integration.yml
vendored
|
|
@ -91,7 +91,7 @@ jobs:
|
|||
COMMIT_MESSAGE: build from commit ${{ github.sha }}
|
||||
|
||||
babel:
|
||||
name: Babel
|
||||
name: Update translations branch
|
||||
runs-on: ubuntu-20.04
|
||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||
needs:
|
||||
|
|
@ -102,32 +102,32 @@ jobs:
|
|||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: '0'
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9'
|
||||
architecture: 'x64'
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./local
|
||||
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
||||
- name: weblate & git setup
|
||||
env:
|
||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||
git config --global user.name "searxng-bot"
|
||||
- name: Update transations
|
||||
id: update
|
||||
continue-on-error: true
|
||||
run: make V=1 ci.babel.update
|
||||
- name: Open pull request
|
||||
if: steps.update.outcome == 'success'
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
with:
|
||||
commit-message: Update translations (pot, po)
|
||||
committer: searx-bot <noreply@github.com>
|
||||
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||
signoff: false
|
||||
branch: update_translations_pot
|
||||
delete-branch: true
|
||||
draft: false
|
||||
title: 'Update translations (pot, po)'
|
||||
body: |
|
||||
Update messages.pot and messages.po files
|
||||
labels: |
|
||||
translation
|
||||
run: |
|
||||
git restore utils/brand.env
|
||||
make V=1 weblate.push.translations
|
||||
|
||||
dockers:
|
||||
name: Docker
|
||||
|
|
|
|||
56
.github/workflows/translations-update.yml
vendored
Normal file
56
.github/workflows/translations-update.yml
vendored
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
name: "Update translations"
|
||||
on:
|
||||
schedule:
|
||||
- cron: "05 07 * * 5"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
babel:
|
||||
name: "create PR for additons from weblate"
|
||||
runs-on: ubuntu-20.04
|
||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: '0'
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9'
|
||||
architecture: 'x64'
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./local
|
||||
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
||||
- name: weblate & git setup
|
||||
env:
|
||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||
git config --global user.name "searxng-bot"
|
||||
- name: Merge and push transation updates
|
||||
run: |
|
||||
make V=1 weblate.translations.commit
|
||||
- name: Create Pull Request
|
||||
id: cpr
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
with:
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
commit-message: Update translations
|
||||
committer: searxng-bot <searxng-bot@users.noreply.github.com>
|
||||
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||
signoff: false
|
||||
branch: translations_update
|
||||
delete-branch: true
|
||||
draft: false
|
||||
title: 'Update translations'
|
||||
body: |
|
||||
Update translations
|
||||
labels: |
|
||||
translation
|
||||
3
.weblate
Normal file
3
.weblate
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[weblate]
|
||||
url = https://weblate.bubu1.eu/api/
|
||||
translation = searxng/searxng
|
||||
2
Makefile
2
Makefile
|
|
@ -77,7 +77,7 @@ test.shell:
|
|||
# wrap ./manage script
|
||||
|
||||
MANAGE += buildenv
|
||||
MANAGE += ci.babel.update babel.extract babel.update babel.compile
|
||||
MANAGE += weblate.translations.commit weblate.push.translations
|
||||
MANAGE += data.all data.languages data.useragents data.osm_keys_tags
|
||||
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
|
||||
MANAGE += docker.build docker.push docker.buildx
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_
|
|||
.. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits
|
||||
:target: https://github.com/searxng/searxng/commits/master
|
||||
|
||||
.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
|
||||
:target: https://weblate.bubu1.eu/projects/searxng/
|
||||
|
||||
If you are looking for running instances, ready to use, then visit searx.space_.
|
||||
|
||||
|
|
|
|||
|
|
@ -4,69 +4,77 @@
|
|||
Translation
|
||||
===========
|
||||
|
||||
.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
|
||||
.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/
|
||||
.. _Weblate: https://docs.weblate.org
|
||||
.. _translations branch: https://github.com/searxng/searxng/tree/translations
|
||||
.. _orphan branch: https://git-scm.com/docs/git-checkout#Documentation/git-checkout.txt---orphanltnewbranchgt
|
||||
.. _Weblate repository: https://weblate.bubu1.eu/projects/searxng/searxng/#repository
|
||||
.. _wlc: https://docs.weblate.org/en/latest/wlc.html
|
||||
|
||||
Translation currently takes place on `searx@transifex`_
|
||||
.. |translated| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
|
||||
:target: https://weblate.bubu1.eu/projects/searxng/
|
||||
|
||||
Requirements
|
||||
============
|
||||
.. sidebar:: |translated|
|
||||
|
||||
* Transifex account
|
||||
- Weblate_
|
||||
- SearXNG `translations branch`_
|
||||
- SearXNG `Weblate repository`_
|
||||
- Weblate Client: wlc_
|
||||
- Babel Command-Line: `pybabel <http://babel.pocoo.org/en/latest/cmdline.html>`_
|
||||
- `weblate workflow <https://docs.weblate.org/en/latest/workflows.html>`_
|
||||
|
||||
Init Transifex project
|
||||
======================
|
||||
Translation takes place on weblate.bubu1.eu_.
|
||||
|
||||
After installing ``transifex`` using pip, run the following command to
|
||||
initialize the project.
|
||||
Translations which has been added by translators on the weblate.bubu1.eu_ UI are
|
||||
committed to Weblate's counterpart of the SearXNG *origin* repository which is
|
||||
located at ``https://weblate.bubu1.eu/git/searxng/searxng``.
|
||||
|
||||
.. code:: sh
|
||||
There is no need to clone this repository, :ref:`SearXNG Weblate workflow` take
|
||||
care of the synchronization with the *origin*. To avoid merging commits from
|
||||
the counterpart directly on the ``master`` branch of *SearXNG origin*, a *pull
|
||||
request* (PR) is created by this workflow.
|
||||
|
||||
./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
|
||||
Weblate monitors the `translations branch`_, not the ``master`` branch. This
|
||||
branch is an `orphan branch`_, decoupled from the master branch (we already know
|
||||
orphan branches from the ``gh-pages``). The `translations branch`_ contains
|
||||
only the
|
||||
|
||||
- ``translation/messages.pot`` and the
|
||||
- ``translation/*/messages.po`` files, nothing else.
|
||||
|
||||
|
||||
After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
|
||||
into the configuration file.
|
||||
.. _SearXNG Weblate workflow:
|
||||
|
||||
Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
|
||||
.. figure:: translation.svg
|
||||
|
||||
.. code:: ini
|
||||
SearXNG's PR workflow to be in sync with Weblate
|
||||
|
||||
[main]
|
||||
host = https://www.transifex.com
|
||||
[searx.messagespo]
|
||||
file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
|
||||
source_file = messages.pot
|
||||
source_lang = en
|
||||
type = PO
|
||||
Sync from *origin* to *weblate*: using ``make weblate.push.translations``
|
||||
For each commit on the ``master`` branch of SearXNG *origin* the GitHub job
|
||||
:origin:`babel / Update translations branch
|
||||
<.github/workflows/integration.yml>` checks for updated translations.
|
||||
|
||||
Sync from *weblate* to *origin*: using ``make weblate.translations.commit``
|
||||
Every Friday, the GitHub workflow :origin:`babel / create PR for additons from
|
||||
weblate <.github/workflows/translations-update.yml>` creates a PR with the
|
||||
updated translation files:
|
||||
|
||||
Then run ``tx set``:
|
||||
- ``translation/messages.pot``,
|
||||
- ``translation/*/messages.po`` and
|
||||
- ``translation/*/messages.mo``
|
||||
|
||||
.. code:: shell
|
||||
wlc
|
||||
===
|
||||
|
||||
./manage pyenv.cmd tx set --auto-local \
|
||||
-r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
|
||||
--source-lang en --type PO --source-file messages.pot --execute
|
||||
.. _wlc configuration: https://docs.weblate.org/en/latest/wlc.html#wlc-config
|
||||
.. _API key: https://weblate.bubu1.eu/accounts/profile/#api
|
||||
|
||||
All weblate integration is done by GitHub workflows, but if you want to use wlc_,
|
||||
copy this content into `wlc configuration`_ in your HOME ``~/.config/weblate``
|
||||
|
||||
Update translations
|
||||
===================
|
||||
.. code-block:: ini
|
||||
|
||||
To retrieve the latest translations, pull it from Transifex.
|
||||
[keys]
|
||||
https://weblate.bubu1.eu/api/ = APIKEY
|
||||
|
||||
.. code:: sh
|
||||
|
||||
./manage pyenv.cmd tx pull -a
|
||||
[?] Enter your api token: ....
|
||||
|
||||
Then check the new languages. If strings translated are not enough, delete those
|
||||
folders, because those should not be compiled. Call the command below to compile
|
||||
the ``.po`` files.
|
||||
|
||||
.. code:: shell
|
||||
|
||||
./manage pyenv.cmd pybabel compile -d searx/translations
|
||||
|
||||
|
||||
After the compilation is finished commit the ``.po`` and ``.mo`` files and
|
||||
create a PR.
|
||||
Replace ``APIKEY`` by your `API key`_.
|
||||
|
|
|
|||
45
docs/dev/translation.svg
Normal file
45
docs/dev/translation.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 17 KiB |
202
manage
202
manage
|
|
@ -45,9 +45,8 @@ help() {
|
|||
buildenv:
|
||||
rebuild ./utils/brand.env
|
||||
babel.:
|
||||
extract : extract messages from source files and generate POT file
|
||||
update : update existing message catalogs from POT file
|
||||
compile : compile translation catalogs into binary MO files
|
||||
master.to.translations: update the translations branch from the messages of the master branch.
|
||||
translations.to.master: copy change from the translations branch to the master branch.
|
||||
data.:
|
||||
all : update searx/languages.py and ./data/*
|
||||
languages : update searx/data/engines_languages.json & searx/languages.py
|
||||
|
|
@ -122,46 +121,185 @@ buildenv() {
|
|||
return "${PIPESTATUS[0]}"
|
||||
}
|
||||
|
||||
babel.sha256sum() {
|
||||
grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
|
||||
TRANSLATIONS_WORKTREE="$CACHE/translations"
|
||||
|
||||
weblate.translations.worktree() {
|
||||
|
||||
# Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
||||
# 'translations' from Weblate's counterpart (weblate) of the SearXNG
|
||||
# (origin).
|
||||
#
|
||||
# remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
|
||||
|
||||
( set -e
|
||||
if ! git remote get-url weblate 2> /dev/null; then
|
||||
git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
|
||||
fi
|
||||
if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
|
||||
pushd .
|
||||
cd "${TRANSLATIONS_WORKTREE}"
|
||||
git reset --hard HEAD
|
||||
git pull origin translations
|
||||
popd
|
||||
else
|
||||
mkdir -p "${TRANSLATIONS_WORKTREE}"
|
||||
git worktree add "${TRANSLATIONS_WORKTREE}" translations
|
||||
fi
|
||||
)
|
||||
}
|
||||
|
||||
ci.babel.update() {
|
||||
local sha_before
|
||||
weblate.to.translations() {
|
||||
|
||||
# Update 'translations' branch of SearXNG (origin) with last additions from
|
||||
# Weblate.
|
||||
|
||||
# 1. Check if Weblate is locked, if not die with error message
|
||||
# 2. On Weblate's counterpart (weblate), pull master and translations branch
|
||||
# from SearXNG (origin).
|
||||
# 3. Commit changes made in a Weblate object on Weblate's counterpart
|
||||
# (weblate).
|
||||
# 4. In translations worktree, merge changes of branch 'translations' from
|
||||
# remote 'weblate' and push it on branch 'translations' of 'origin'
|
||||
|
||||
( set -e
|
||||
sha_before="$(babel.sha256sum)"
|
||||
babel.extract
|
||||
if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
|
||||
build_msg BABEL 'no changes detected, exiting'
|
||||
return 1
|
||||
if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then
|
||||
die 1 "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)"
|
||||
fi
|
||||
babel.update
|
||||
build_msg BABEL 'update done, edit .po files if required and run babel.compile'
|
||||
# weblate: commit pending changes
|
||||
pyenv.cmd wlc pull
|
||||
pyenv.cmd wlc commit
|
||||
|
||||
# get the translations in a worktree
|
||||
weblate.translations.worktree
|
||||
|
||||
pushd "${TRANSLATIONS_WORKTREE}"
|
||||
git remote update weblate
|
||||
git merge weblate/translations
|
||||
git push
|
||||
popd
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
babel.extract() {
|
||||
build_msg BABEL 'extract messages from source files and generate POT file'
|
||||
pyenv.cmd pybabel extract -F babel.cfg \
|
||||
-o "searx/translations/messages.pot" \
|
||||
"searx/"
|
||||
dump_return $?
|
||||
weblate.translations.commit() {
|
||||
|
||||
# Update 'translations' branch of SearXNG (origin) with last additions from
|
||||
# Weblate. Copy the changes to the master branch, compile translations and
|
||||
# create a commit in the local branch (master)
|
||||
|
||||
local existing_commit_hash commit_body commit_message exitcode
|
||||
( set -e
|
||||
# lock change on weblate
|
||||
pyenv.cmd wlc lock
|
||||
|
||||
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
||||
weblate.translations.worktree
|
||||
existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
|
||||
|
||||
# pull weblate commits
|
||||
weblate.to.translations
|
||||
|
||||
# copy the changes to the master branch
|
||||
cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
|
||||
|
||||
# compile translations
|
||||
build_msg BABEL 'compile translation catalogs into binary MO files'
|
||||
pyenv.cmd pybabel compile --statistics \
|
||||
-d "searx/translations"
|
||||
# git add/commit (no push)
|
||||
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
|
||||
commit_message=$(echo -e "[translations] update\n${commit_body}")
|
||||
git add searx/translations
|
||||
git commit -m "${commit_message}"
|
||||
)
|
||||
exitcode=$?
|
||||
( # make sure to always unlock weblate
|
||||
set -e
|
||||
pyenv.cmd wlc unlock
|
||||
)
|
||||
dump_return $exitcode
|
||||
}
|
||||
|
||||
babel.update() {
|
||||
build_msg BABEL 'update existing message catalogs from POT file'
|
||||
pyenv.cmd pybabel update -N \
|
||||
-i "searx/translations/messages.pot" \
|
||||
-d "searx/translations"
|
||||
dump_return $?
|
||||
}
|
||||
weblate.push.translations() {
|
||||
|
||||
babel.compile() {
|
||||
build_msg BABEL 'compile translation catalogs into binary MO files'
|
||||
pyenv.cmd pybabel compile --statistics \
|
||||
-d "searx/translations"
|
||||
dump_return $?
|
||||
# Push *translation changes* from SearXNG (origin) to Weblate's counterpart
|
||||
# (weblate).
|
||||
|
||||
# In branch master of SearXNG (origin) check for meaningful changes in
|
||||
# folder 'searx/translations', commit changes on branch 'translations' and
|
||||
# at least, pull updated branches on Weblate's counterpart (weblate).
|
||||
|
||||
# 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
||||
# 'translations' from remote 'weblate'.
|
||||
# 2. Stop if there is no meaningful change in the 'master' branch (origin),
|
||||
# compared to the 'translations' branch (weblate), otherwise ...
|
||||
# 3. Update 'translations' branch of SearXNG (origin) with last additions
|
||||
# from Weblate.
|
||||
# 5. Notify Weblate to pull updated 'master' & 'translations' branch.
|
||||
|
||||
local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
|
||||
last_commit_message exitcode
|
||||
( set -e
|
||||
# lock change on weblate
|
||||
pyenv.cmd wlc lock
|
||||
|
||||
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
||||
weblate.translations.worktree
|
||||
|
||||
# update messages.pot in the master branch
|
||||
build_msg BABEL 'extract messages from source files and generate POT file'
|
||||
messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
|
||||
pyenv.cmd pybabel extract -F babel.cfg \
|
||||
-o "${messages_pot}" \
|
||||
"searx/"
|
||||
|
||||
# stop if there is no meaningful change in the master branch
|
||||
diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
|
||||
git diff -- "searx/translations/messages.pot")
|
||||
if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
|
||||
build_msg BABEL 'no changes detected, exiting'
|
||||
return 0
|
||||
fi
|
||||
|
||||
# save messages.pot in the translations branch for later
|
||||
pushd "${TRANSLATIONS_WORKTREE}"
|
||||
git stash push
|
||||
popd
|
||||
|
||||
# merge weblate commits into the translations branch
|
||||
weblate.to.translations
|
||||
|
||||
# restore messages.pot in the translations branch
|
||||
pushd "${TRANSLATIONS_WORKTREE}"
|
||||
git stash pop
|
||||
popd
|
||||
|
||||
# update messages.po files in the master branch
|
||||
build_msg BABEL 'update existing message catalogs from POT file'
|
||||
pyenv.cmd pybabel update -N \
|
||||
-i "${messages_pot}" \
|
||||
-d "${TRANSLATIONS_WORKTREE}/searx/translations"
|
||||
|
||||
# git add/commit/push
|
||||
last_commit_hash=$(git log -n1 --pretty=format:'%h')
|
||||
last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
|
||||
last_commit_message="[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}"
|
||||
|
||||
pushd "${TRANSLATIONS_WORKTREE}"
|
||||
git add searx/translations
|
||||
git commit -m "${last_commit_message}"
|
||||
git push
|
||||
popd
|
||||
|
||||
# notify weblate to pull updated master & translations branch
|
||||
pyenv.cmd wlc pull
|
||||
)
|
||||
exitcode=$?
|
||||
( # make sure to always unlock weblate
|
||||
set -e
|
||||
pyenv.cmd wlc unlock
|
||||
)
|
||||
dump_return $exitcode
|
||||
}
|
||||
|
||||
data.all() {
|
||||
|
|
|
|||
|
|
@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14
|
|||
linuxdoc==20210324
|
||||
aiounittest==1.4.0
|
||||
yamllint==1.26.2
|
||||
wlc==1.12
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue