forked from zaclys/searxng
[translations] web integration
* make babel.translations.to.master: pull weblate updates * make babel.master.to.translations: push .pot and .po files to weblate
This commit is contained in:
Alexandre Flament
parent
80372d87d9
commit
901ab87717
|
|
@ -91,7 +91,7 @@ jobs:
|
|||
COMMIT_MESSAGE: build from commit ${{ github.sha }}
|
||||
|
||||
babel:
|
||||
name: Babel
|
||||
name: Update translations branch
|
||||
runs-on: ubuntu-20.04
|
||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||
needs:
|
||||
|
|
@ -102,32 +102,32 @@ jobs:
|
|||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: '0'
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9'
|
||||
architecture: 'x64'
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./local
|
||||
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
||||
- name: weblate & git setup
|
||||
env:
|
||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||
git config --global user.name "searxng-bot"
|
||||
- name: Update transations
|
||||
id: update
|
||||
continue-on-error: true
|
||||
run: make V=1 ci.babel.update
|
||||
- name: Open pull request
|
||||
if: steps.update.outcome == 'success'
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
with:
|
||||
commit-message: Update translations (pot, po)
|
||||
committer: searx-bot <noreply@github.com>
|
||||
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||
signoff: false
|
||||
branch: update_translations_pot
|
||||
delete-branch: true
|
||||
draft: false
|
||||
title: 'Update translations (pot, po)'
|
||||
body: |
|
||||
Update messages.pot and messages.po files
|
||||
labels: |
|
||||
translation
|
||||
run: |
|
||||
git restore utils/brand.env
|
||||
make V=1 babel.master.to.translations
|
||||
|
||||
dockers:
|
||||
name: Docker
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
name: "Update translations"
|
||||
on:
|
||||
schedule:
|
||||
- cron: "05 07 * * 5"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
babel:
|
||||
name: "translations: update master branch"
|
||||
runs-on: ubuntu-20.04
|
||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: '0'
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9'
|
||||
architecture: 'x64'
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./local
|
||||
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
||||
- name: weblate & git setup
|
||||
env:
|
||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||
git config --global user.name "searxng-bot"
|
||||
- name: Merge and push transation updates
|
||||
run: |
|
||||
make V=1 babel.translations.to.master
|
||||
- name: Create Pull Request
|
||||
id: cpr
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
with:
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
commit-message: Update translations
|
||||
committer: searxng-bot <searxng-bot@users.noreply.github.com>
|
||||
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||
signoff: false
|
||||
branch: translations_update
|
||||
delete-branch: true
|
||||
draft: false
|
||||
title: 'Update translations'
|
||||
body: |
|
||||
Update translations
|
||||
labels: |
|
||||
translation
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[weblate]
|
||||
url = https://weblate.bubu1.eu/api/
|
||||
translation = searxng/searxng
|
||||
2
Makefile
2
Makefile
|
|
@ -77,7 +77,7 @@ test.shell:
|
|||
# wrap ./manage script
|
||||
|
||||
MANAGE += buildenv
|
||||
MANAGE += ci.babel.update babel.extract babel.update babel.compile
|
||||
MANAGE += babel.translations.to.master babel.master.to.translations
|
||||
MANAGE += data.all data.languages data.useragents data.osm_keys_tags
|
||||
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
|
||||
MANAGE += docker.build docker.push docker.buildx
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_
|
|||
.. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits
|
||||
:target: https://github.com/searxng/searxng/commits/master
|
||||
|
||||
.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
|
||||
:target: https://weblate.bubu1.eu/projects/searxng/
|
||||
|
||||
If you are looking for running instances, ready to use, then visit searx.space_.
|
||||
|
||||
|
|
|
|||
|
|
@ -4,69 +4,46 @@
|
|||
Translation
|
||||
===========
|
||||
|
||||
.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
|
||||
.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/
|
||||
|
||||
Translation currently takes place on `searx@transifex`_
|
||||
Translation takes place on `weblate.bubu1.eu`_ ( `documentation <https://docs.weblate.org/en/latest/index.html>`_ ).
|
||||
|
||||
Requirements
|
||||
============
|
||||
New messages on the master branch are extracted and pushed to Weblate automatically.
|
||||
|
||||
* Transifex account
|
||||
Every Friday, a GitHub workflow creates a pull request with the updated translations (messages.mo, messages.po, messages.mo files).
|
||||
|
||||
Init Transifex project
|
||||
======================
|
||||
.. image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
|
||||
:target: https://weblate.bubu1.eu/projects/searxng/
|
||||
|
||||
After installing ``transifex`` using pip, run the following command to
|
||||
initialize the project.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
|
||||
|
||||
|
||||
After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
|
||||
into the configuration file.
|
||||
|
||||
Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
|
||||
|
||||
.. code:: ini
|
||||
|
||||
[main]
|
||||
host = https://www.transifex.com
|
||||
[searx.messagespo]
|
||||
file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
|
||||
source_file = messages.pot
|
||||
source_lang = en
|
||||
type = PO
|
||||
|
||||
|
||||
Then run ``tx set``:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
./manage pyenv.cmd tx set --auto-local \
|
||||
-r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
|
||||
--source-lang en --type PO --source-file messages.pot --execute
|
||||
|
||||
|
||||
Update translations
|
||||
Weblate integration
|
||||
===================
|
||||
|
||||
To retrieve the latest translations, pull it from Transifex.
|
||||
Weblate monitors the `translations branch <https://github.com/searxng/searxng/tree/translations>`_, not the master branch.
|
||||
|
||||
.. code:: sh
|
||||
This branch contains only the .pot and pot files, nothing else.
|
||||
|
||||
./manage pyenv.cmd tx pull -a
|
||||
[?] Enter your api token: ....
|
||||
Documentation
|
||||
-------------
|
||||
|
||||
Then check the new languages. If strings translated are not enough, delete those
|
||||
folders, because those should not be compiled. Call the command below to compile
|
||||
the ``.po`` files.
|
||||
* `wlc <https://docs.weblate.org/en/latest/wlc.html>`_
|
||||
* `pybabel <http://babel.pocoo.org/en/latest/cmdline.html>`_
|
||||
* `weblate workflow <https://docs.weblate.org/en/latest/workflows.html>`_
|
||||
|
||||
.. code:: shell
|
||||
Worfklow
|
||||
--------
|
||||
|
||||
./manage pyenv.cmd pybabel compile -d searx/translations
|
||||
.. image:: translation.svg
|
||||
|
||||
|
||||
After the compilation is finished commit the ``.po`` and ``.mo`` files and
|
||||
create a PR.
|
||||
wlc
|
||||
---
|
||||
|
||||
All weblate integration is done by GitHub workflows, but if you want to use wlc, copy this content into `~/.config/weblate <https://docs.weblate.org/en/latest/wlc.html#wlc-config>`_ :
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[keys]
|
||||
https://weblate.bubu1.eu/api/ = APIKEY
|
||||
|
||||
|
||||
Replace `APIKEY` by `your API key <https://weblate.bubu1.eu/accounts/profile/#api>`_.
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 17 KiB |
153
manage
153
manage
|
|
@ -45,9 +45,8 @@ help() {
|
|||
buildenv:
|
||||
rebuild ./utils/brand.env
|
||||
babel.:
|
||||
extract : extract messages from source files and generate POT file
|
||||
update : update existing message catalogs from POT file
|
||||
compile : compile translation catalogs into binary MO files
|
||||
master.to.translations: update the translations branch from the messages of the master branch.
|
||||
translations.to.master: copy change from the translations branch to the master branch.
|
||||
data.:
|
||||
all : update searx/languages.py and ./data/*
|
||||
languages : update searx/data/engines_languages.json & searx/languages.py
|
||||
|
|
@ -122,46 +121,136 @@ buildenv() {
|
|||
return "${PIPESTATUS[0]}"
|
||||
}
|
||||
|
||||
babel.sha256sum() {
|
||||
grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
|
||||
TRANSLATIONS_WORKTREE="$CACHE/translations"
|
||||
|
||||
babel.setup.translations.worktree() {
|
||||
( set -e
|
||||
if ! git remote get-url weblate 2> /dev/null; then
|
||||
git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
|
||||
fi
|
||||
if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
|
||||
pushd .
|
||||
cd "${TRANSLATIONS_WORKTREE}"
|
||||
git reset --hard HEAD
|
||||
git pull origin translations
|
||||
popd
|
||||
else
|
||||
mkdir -p "${TRANSLATIONS_WORKTREE}"
|
||||
git worktree add "${TRANSLATIONS_WORKTREE}" translations
|
||||
fi
|
||||
)
|
||||
}
|
||||
|
||||
ci.babel.update() {
|
||||
local sha_before
|
||||
babel.weblate.to.translations() {
|
||||
( set -e
|
||||
sha_before="$(babel.sha256sum)"
|
||||
babel.extract
|
||||
if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
|
||||
build_msg BABEL 'no changes detected, exiting'
|
||||
return 1
|
||||
if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then
|
||||
build_msg BABEL "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)"
|
||||
exit 1
|
||||
fi
|
||||
babel.update
|
||||
build_msg BABEL 'update done, edit .po files if required and run babel.compile'
|
||||
# weblate: commit pending changes
|
||||
pyenv.cmd wlc pull
|
||||
pyenv.cmd wlc commit
|
||||
# get the translations in a worktree
|
||||
babel.setup.translations.worktree
|
||||
cd "${TRANSLATIONS_WORKTREE}"
|
||||
git remote update weblate
|
||||
git merge weblate/translations
|
||||
git push
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
babel.extract() {
|
||||
build_msg BABEL 'extract messages from source files and generate POT file'
|
||||
pyenv.cmd pybabel extract -F babel.cfg \
|
||||
-o "searx/translations/messages.pot" \
|
||||
"searx/"
|
||||
dump_return $?
|
||||
babel.translations.to.master() {
|
||||
local existing_commit_hash commit_body commit_message exitcode
|
||||
( set -e
|
||||
pyenv.cmd wlc lock
|
||||
babel.setup.translations.worktree
|
||||
existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
|
||||
# pull weblate commits
|
||||
babel.weblate.to.translations
|
||||
# copy the changes to the master branch
|
||||
cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
|
||||
# compile translations
|
||||
build_msg BABEL 'compile translation catalogs into binary MO files'
|
||||
pyenv.cmd pybabel compile --statistics \
|
||||
-d "searx/translations"
|
||||
# git add/commit (no push)
|
||||
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
|
||||
commit_message=$(echo -e "[translations] update\n${commit_body}")
|
||||
git add searx/translations
|
||||
git commit -m "${commit_message}"
|
||||
)
|
||||
exitcode=$?
|
||||
( # make sure to always unlock weblate
|
||||
set -e
|
||||
pyenv.cmd wlc unlock
|
||||
)
|
||||
dump_return $exitcode
|
||||
}
|
||||
|
||||
babel.update() {
|
||||
build_msg BABEL 'update existing message catalogs from POT file'
|
||||
pyenv.cmd pybabel update -N \
|
||||
-i "searx/translations/messages.pot" \
|
||||
-d "searx/translations"
|
||||
dump_return $?
|
||||
}
|
||||
babel.master.to.translations() {
|
||||
local messages_pot diff_messages_pot last_commit_hash last_commit_detail last_commit_message exitcode
|
||||
( set -e
|
||||
# lock change on weblate
|
||||
pyenv.cmd wlc lock
|
||||
|
||||
babel.compile() {
|
||||
build_msg BABEL 'compile translation catalogs into binary MO files'
|
||||
pyenv.cmd pybabel compile --statistics \
|
||||
-d "searx/translations"
|
||||
dump_return $?
|
||||
# get translation branch
|
||||
babel.setup.translations.worktree
|
||||
|
||||
# update messages.pot
|
||||
build_msg BABEL 'extract messages from source files and generate POT file'
|
||||
messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
|
||||
pyenv.cmd pybabel extract -F babel.cfg \
|
||||
-o "${messages_pot}" \
|
||||
"searx/"
|
||||
|
||||
# stop if there is no meaningful change
|
||||
diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}"; git diff -- "searx/translations/messages.pot")
|
||||
if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
|
||||
build_msg BABEL 'no changes detected, exiting'
|
||||
return 0
|
||||
fi
|
||||
|
||||
# save messages.pot for later
|
||||
cd "${TRANSLATIONS_WORKTREE}"
|
||||
git stash push
|
||||
cd -
|
||||
|
||||
# merge weblate commits into the translations branch
|
||||
babel.weblate.to.translations
|
||||
|
||||
# restore messages.pot
|
||||
cd "${TRANSLATIONS_WORKTREE}"
|
||||
git stash pop
|
||||
cd -
|
||||
|
||||
set -x
|
||||
|
||||
# update messages.po files
|
||||
build_msg BABEL 'update existing message catalogs from POT file'
|
||||
pyenv.cmd pybabel update -N \
|
||||
-i "${messages_pot}" \
|
||||
-d "${TRANSLATIONS_WORKTREE}/searx/translations"
|
||||
|
||||
# git add/commit/push
|
||||
last_commit_hash=$(git log -n1 --pretty=format:'%h')
|
||||
last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
|
||||
last_commit_message=$(echo -e "[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}")
|
||||
cd "${TRANSLATIONS_WORKTREE}"
|
||||
git add searx/translations
|
||||
git commit -m "${last_commit_message}"
|
||||
git push
|
||||
cd -
|
||||
|
||||
# notify weblate
|
||||
pyenv.cmd wlc pull
|
||||
)
|
||||
exitcode=$?
|
||||
( # make sure to always unlock weblate
|
||||
set -e
|
||||
pyenv.cmd wlc unlock
|
||||
)
|
||||
dump_return $exitcode
|
||||
}
|
||||
|
||||
data.all() {
|
||||
|
|
|
|||
|
|
@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14
|
|||
linuxdoc==20210324
|
||||
aiounittest==1.4.0
|
||||
yamllint==1.26.2
|
||||
wlc==1.12
|
||||
|
|
|
|||
Loading…
Reference in New Issue