Merge pull request #238 from dalf/update_babel

Automatic update of searx/translations/messages.pot and messages.po files.

.github/workflows/integration.yml

@@ -90,6 +90,45 @@ jobs:
         SINGLE_COMMIT: True
         COMMIT_MESSAGE: build from commit ${{ github.sha }}
 
+  babel:
+    name: Babel
+    runs-on: ubuntu-20.04
+    if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
+    needs:
+      - python
+      - themes
+      - documentation
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+        architecture: 'x64'
+    - name: Update transations
+      id: update
+      continue-on-error: true
+      run: make V=1 ci.babel.update
+    - name: Open pull request
+      if: steps.update.outcome == 'success'
+      uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update translations (pot, po)
+        committer: searx-bot <noreply@github.com>
+        author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+        signoff: false
+        branch: update_translations_pot
+        delete-branch: true
+        draft: false
+        title: 'Update translations (pot, po)'
+        body: |
+          Update messages.pot and messages.po files
+        labels: |
+          translation
+
   dockers:
     name: Docker
     if: github.ref == 'refs/heads/master'

.tx/config

@@ -1,7 +0,0 @@
-[main]
-host = https://www.transifex.com
-[searx.messagespo]
-file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
-source_file = messages.pot
-source_lang = en
-type = PO

Makefile

@@ -77,7 +77,7 @@ test.shell:
 # wrap ./manage script
 
 MANAGE += buildenv
-MANAGE += babel.compile
+MANAGE += ci.babel.update babel.extract babel.update babel.compile
 MANAGE += data.all data.languages data.useragents data.osm_keys_tags
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += docker.build docker.push docker.buildx

babel.cfg

@@ -1,3 +1,4 @@
+[ignore: **/node_modules/**]
 [python: **.py]
 [jinja2: **/templates/**.html]
 extensions=jinja2.ext.autoescape,jinja2.ext.with_

manage

@@ -44,8 +44,10 @@ help() {
     cat <<EOF
 buildenv:
   rebuild ./utils/brand.env
-babel.compile:
+babel.:
-  pybabel compile ./searx/translations
+  extract   : extract messages from source files and generate POT file
+  update    : update existing message catalogs from POT file
+  compile   : compile translation catalogs into binary MO files
 data.:
   all       : update searx/languages.py and ./data/*
   languages : update searx/data/engines_languages.json & searx/languages.py
@@ -120,12 +122,47 @@ buildenv() {
     return "${PIPESTATUS[0]}"
 }
 
-babel.compile() {
+babel.sha256sum() {
-    build_msg BABEL compile
+    grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
-    pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
+}
+
+ci.babel.update() {
+    local sha_before
+    (   set -e
+        sha_before="$(babel.sha256sum)"
+        babel.extract
+        if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
+            build_msg BABEL 'no changes detected, exiting'
+            return 1
+        fi
+        babel.update
+        build_msg BABEL 'update done, edit .po files if required and run babel.compile'
+    )
     dump_return $?
 }
 
+babel.extract() {
+    build_msg BABEL 'extract messages from source files and generate POT file'
+    pyenv.cmd pybabel extract -F babel.cfg \
+            -o "searx/translations/messages.pot" \
+            "searx/"
+    dump_return $?
+}
+
+babel.update() {
+    build_msg BABEL 'update existing message catalogs from POT file'
+    pyenv.cmd pybabel update -N \
+              -i "searx/translations/messages.pot" \
+              -d "searx/translations"
+    dump_return $?
+}
+
+babel.compile() {
+    build_msg BABEL 'compile translation catalogs into binary MO files'
+    pyenv.cmd pybabel compile --statistics \
+              -d "searx/translations"
+    dump_return $?
+}
 
 data.all() {
     data.languages

searx_extra/update/update_translations.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-
-# script to easily update translation language files
-
-# add new language:
-# pybabel init -i messages.pot -d searx/translations -l en
-
-SEARX_DIR='searx'
-
-pybabel extract -F babel.cfg -o messages.pot "$SEARX_DIR"
-for f in `ls "$SEARX_DIR"'/translations/'`; do
-    pybabel update -N -i messages.pot -d "$SEARX_DIR"'/translations/' -l "$f"
-done
-
-echo '[!] update done, edit .po files if required and run pybabel compile -d searx/translations/'