diff --git a/Makefile b/Makefile index 23f06a66..58235f99 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ .PHONY: validate_translations pull_translations push_translations install_transifex_clients PACKAGE_NAME := xblocks_contrib -EXTRACT_DIR := $(PACKAGE_NAME)/conf/locale/en/LC_MESSAGES +EXTRACT_DIR := conf/locale/en/LC_MESSAGES +COMBINED_LOCALE_DIR := $(PACKAGE_NAME)/conf/locale/en/LC_MESSAGES JS_TARGET := $(PACKAGE_NAME)/public/js/translations help: @@ -37,21 +38,40 @@ requirements: piptools ## install development environment requirements pip-sync -q requirements/dev.txt requirements/private.* # XBlock directories -XBLOCKS=$(shell find $(shell pwd)/$(PACKAGE_NAME) -maxdepth 2 -type d -name 'conf' -exec dirname {} \;) +XBLOCKS=$(shell find $(shell pwd)/$(PACKAGE_NAME) -mindepth 2 -maxdepth 2 -type d -name 'conf' -exec dirname {} \;) ## Localization targets extract_translations: ## extract strings to be translated, outputting .po files for each XBlock @for xblock in $(XBLOCKS); do \ echo "Extracting translations for $$xblock..."; \ - cd $$xblock && i18n_tool extract --no-segment --merge-po-files; \ - if [ -f $(EXTRACT_DIR)/django.po ]; then \ - mv $(EXTRACT_DIR)/django.po $(EXTRACT_DIR)/text.po; \ + cd $$xblock && i18n_tool extract --no-segment; \ + if [ -f $$xblock/$(EXTRACT_DIR)/djangojs.po ]; then \ + cd $$xblock/$(EXTRACT_DIR) && msgcat django.po djangojs.po -o django.po && rm -f djangojs.po; \ + fi; \ + if [ -f $$xblock/$(EXTRACT_DIR)/django.po ]; then \ + mv $$xblock/$(EXTRACT_DIR)/django.po $$xblock/$(EXTRACT_DIR)/text.po; \ fi; \ done + @# Merge all per-xblock text.po files into a single combined file for the + @# openedx-translations pipeline (OEP-58), which expects one conf/locale/en per repo. + @mkdir -p $(COMBINED_LOCALE_DIR) + @PO_FILES=""; \ + for xblock in $(XBLOCKS); do \ + if [ -f $$xblock/$(EXTRACT_DIR)/text.po ]; then \ + PO_FILES="$$PO_FILES $$xblock/$(EXTRACT_DIR)/text.po"; \ + fi; \ + done; \ + if [ -n "$$PO_FILES" ]; then \ + msgcat --use-first $$PO_FILES -o $(COMBINED_LOCALE_DIR)/django.po; \ + echo "Combined translation source file written to $(COMBINED_LOCALE_DIR)/django.po"; \ + fi compile_translations: ## compile translation files, outputting .mo files for each supported language for each XBlock - django-admin compilemessages --locale en + @for xblock in $(XBLOCKS); do \ + echo "Compiling translations for $$xblock..."; \ + cd $$xblock && django-admin compilemessages --locale en; \ + done detect_changed_source_translations: @for xblock in $(XBLOCKS); do \ @@ -65,7 +85,7 @@ dummy_translations: ## generate dummy translation (.po) files for each XBlock cd $$xblock && i18n_tool dummy; \ done -build_dummy_translations: dummy_translations compile_translations ## generate and compile dummy translation files +build_dummy_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files validate_translations: build_dummy_translations detect_changed_source_translations ## validate translations diff --git a/README.rst b/README.rst index c9898f8d..c4e72f10 100644 --- a/README.rst +++ b/README.rst @@ -70,6 +70,10 @@ Localization (l10n) is adapting a program to local language and cultural habits. For information on how to enable translations, visit the `Open edX XBlock tutorial on Internationalization `_. The included Makefile contains targets for extracting, compiling and validating translatable strings. +Each XBlock in this repository has its own translation configuration under +``xblocks_contrib//conf/locale/`` and its own Transifex resource mapping under +``xblocks_contrib//.tx/config``. All Make targets iterate over every XBlock automatically. + The general steps to provide multilingual messages for a Python program (or an XBlock) are: 1. Mark translatable strings. @@ -77,12 +81,24 @@ The general steps to provide multilingual messages for a Python program (or an X 3. Create language specific translations for each message in the catalogs. 4. Use ``gettext`` to translate strings. +Prerequisites +------------- + +Install the development requirements, which include `edx-i18n-tools `_ +and GNU gettext tools (``msgcat``, ``msgfmt``):: + + $ make requirements + +On macOS, install gettext via Homebrew if not already present:: + + $ brew install gettext + 1. Mark translatable strings ---------------------------- Mark translatable strings in python:: - from django.utils.translation import ugettext as _ + from django.utils.translation import gettext as _ # Translators: This comment will appear in the `.po` file. message = _("This will be marked.") @@ -92,15 +108,14 @@ for more information. You can also use ``gettext`` to mark strings in javascript:: - // Translators: This comment will appear in the `.po` file. var message = gettext("Custom message."); See `edx-developer-guide `__ for more information. -2. Run i18n tools to create Raw message catalogs ------------------------------------------------- +2. Run i18n tools to create raw message catalogs +------------------------------------------------- After marking strings as translatable we have to create the raw message catalogs. These catalogs are created in ``.po`` files. For more information see @@ -109,15 +124,23 @@ These catalogs can be created by running:: $ make extract_translations -This command will create the necessary ``.po`` files under -``xblocks-contrib/xblocks_contrib//conf/locale/en/LC_MESSAGES/text.po``. -The ``text.po`` file is created from the ``django-partial.po`` file created by -``django-admin makemessages`` (`makemessages documentation `_), -this is why you will not see a ``django-partial.po`` file. +This iterates over every XBlock and: -You will need to have `edx-i18n-tools` that you can get by: +1. Runs ``i18n_tool extract --no-segment`` to extract Python, HTML and JS strings. +2. Merges ``djangojs.po`` into ``django.po`` (if JS strings exist) using ``msgcat``. +3. Renames the result to ``text.po``. - $ make requirements +The output for each XBlock is a single file at:: + + xblocks_contrib//conf/locale/en/LC_MESSAGES/text.po + +Additionally, all per-xblock ``text.po`` files are merged into a single combined file at:: + + xblocks_contrib/conf/locale/en/LC_MESSAGES/django.po + +This combined file is used by the +`openedx-translations `_ pipeline (OEP-58) +to sync translations with Transifex. The per-xblock files are used for local development and testing. 3. Create language specific translations ---------------------------------------- @@ -131,8 +154,8 @@ The format of each entry is as follows:: # translator-comments A. extracted-comments - #: reference… - #, flag… + #: reference... + #, flag... #| msgid previous-untranslated-string msgid 'untranslated message' msgstr 'mensaje traducido (translated message)' @@ -140,55 +163,130 @@ The format of each entry is as follows:: For more information see `GNU PO file documentation `_. -To use translations from transifex use the follow Make target to pull translations:: +3.2 Transifex integration (via openedx-translations) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - $ make pull_translations +This project follows `OEP-58 `_ +for translation management. Translations are managed centrally through the +`openedx-translations `_ repository: -See `config instructions `_ for information on how to set up your -transifex credentials. +1. A daily GitHub Actions workflow in openedx-translations clones this repo, runs + ``make extract_translations``, and commits the combined ``django.po`` source file. +2. The Transifex GitHub App syncs source strings to the ``openedx-translations`` Transifex project + under the ``open-edx`` organization. +3. The Open edX translation community translates strings on Transifex. +4. Reviewed translations sync back to the openedx-translations repository. -See `transifex documentation `_ for more details about integrating -django with transiflex. +Since these XBlocks were extracted from edx-platform, many strings already have existing translations +in Transifex. The openedx-translations pipeline preserves these so translators do not need to +re-translate them. -3.2 Compile translations -~~~~~~~~~~~~~~~~~~~~~~~~ +**Direct Transifex CLI (alternative):** -Once translations are in place, use the following Make target to compile the translation catalogs ``.po`` into -``.mo`` message files:: +Each XBlock also has a ``.tx/config`` for direct Transifex CLI usage. This is useful for +testing or when the openedx-translations pipeline is not available. - $ make compile_translations +1. Install the Transifex CLI:: + + $ make install_transifex_client + +2. Set your Transifex API token (request access from the Open edX Open Source Team):: -The previous command will compile ``.po`` files using -``django-admin compilemessages`` (`compilemessages documentation `_). -After compiling the ``.po`` file(s), ``django-statici18n`` is used to create language specific catalogs. See -``django-statici18n`` `documentation `_ for more information. + $ export TX_TOKEN= -To upload translations to transiflex use the follow Make target:: +3. Pull or push translations:: + $ make pull_translations $ make push_translations -See `config instructions `_ for information on how to set up your -transifex credentials. +See `Transifex CLI configuration `_ for more details. + +3.3 Compile translations +~~~~~~~~~~~~~~~~~~~~~~~~~ -See `transifex documentation `_ for more details about integrating -django with transiflex. +Once translations are in place, use the following Make target to compile the translation catalogs ``.po`` into +``.mo`` message files:: - **Note:** To check if the source translation files (``.po``) are up-to-date run:: + $ make compile_translations - $ make detect_changed_source_translations +This runs ``django-admin compilemessages`` inside each XBlock directory. +See `compilemessages documentation `_. 4. Use ``gettext`` to translate strings --------------------------------------- Django will automatically use ``gettext`` and the compiled translations to translate strings. +Validating and testing translations +----------------------------------- + +**Validate the full translation pipeline** (extract, generate dummy translations, compile, and +check for source drift):: + + $ make validate_translations + +This is the target used in CI (via ``tox -e translations``). It runs the following targets in order: + +**Generate dummy (fake) translations** in the Esperanto (``eo``) and fake-RTL (``rtl``) locales for +visual testing:: + + $ make dummy_translations + +You can trigger the display by setting your browser's language to Esperanto and navigating to a page. +Instead of plain English strings you should see accented English like:: + + The Future of Online Education --> The Futuré of Onliné Education + +**Compile translations** (required after pull or dummy generation):: + + $ make compile_translations + +**Check if source translation files are up-to-date** with the current source code:: + + $ make detect_changed_source_translations + +Make targets reference +---------------------- + +.. list-table:: + :widths: 35 65 + :header-rows: 1 + + * - Target + - Description + * - ``make extract_translations`` + - Extract translatable strings into ``text.po`` for each XBlock + * - ``make compile_translations`` + - Compile ``.po`` files into ``.mo`` files for each XBlock + * - ``make dummy_translations`` + - Generate dummy Esperanto/RTL ``.po`` files for testing + * - ``make build_dummy_translations`` + - Generate and compile dummy translations + * - ``make validate_translations`` + - Full validation: dummy build + source drift detection (CI target) + * - ``make detect_changed_source_translations`` + - Check if source ``.po`` files are up-to-date + * - ``make pull_translations`` + - Pull translations from Transifex + * - ``make push_translations`` + - Extract and push source translations to Transifex + * - ``make install_transifex_client`` + - Install the Transifex CLI + Troubleshooting ~~~~~~~~~~~~~~~ -If there are any errors compiling ``.po`` files run the following command to validate your ``.po`` files:: +If there are any errors compiling ``.po`` files, validate them:: - $ make validate + $ make validate_translations See `django's i18n troubleshooting documentation `_ for more information. + +**Common issues:** + +- ``i18n_tool: command not found`` -- Run ``make requirements`` to install ``edx-i18n-tools``. +- ``msgcat: command not found`` -- Install GNU gettext (``brew install gettext`` on macOS). +- Transifex push/pull errors -- Ensure ``TX_TOKEN`` is set and you have access to the ``open-edx`` + organization. Run ``make install_transifex_client`` if the ``tx`` CLI is missing. diff --git a/docs/internationalization.rst b/docs/internationalization.rst index eb64470e..ed16c7e0 100644 --- a/docs/internationalization.rst +++ b/docs/internationalization.rst @@ -10,38 +10,84 @@ Follow the `internationalization coding guidelines`_ in the Open edX Developer's .. _internationalization coding guidelines: https://docs.openedx.org/en/latest/developers/references/developer_guide/internationalization/i18n.html +Project structure +***************** +Each XBlock in this repository manages its own translations independently: + +- **Config**: ``xblocks_contrib//conf/locale/config.yaml`` -- defines supported languages and + ignored directories. +- **Source strings**: ``xblocks_contrib//conf/locale/en/LC_MESSAGES/text.po`` -- extracted + English strings (Python, HTML, and JavaScript combined into a single file). +- **Transifex mapping**: ``xblocks_contrib//.tx/config`` -- maps the local ``conf/locale/`` + paths to the Transifex resource for that XBlock. +- **Combined source file**: ``xblocks_contrib/conf/locale/en/LC_MESSAGES/django.po`` -- all per-xblock + ``text.po`` files merged into one, used by the openedx-translations pipeline (OEP-58). + +All ``make`` targets iterate over every XBlock that has a ``conf/`` directory automatically. + Updating Translations ********************* -This project uses `Transifex`_ to translate content. After new features are developed the translation source files -should be pushed to Transifex. Our translation community will translate the content, after which we can retrieve the -translations. +This project follows `OEP-58`_ for translation management. Translations are managed centrally through +the `openedx-translations`_ repository rather than directly via Transifex. + +.. _OEP-58: https://docs.openedx.org/en/latest/developers/concepts/oep58.html +.. _openedx-translations: https://github.com/openedx/openedx-translations + +**How the pipeline works:** + +1. A daily GitHub Actions workflow in openedx-translations clones this repo and runs + ``make extract_translations``. +2. The combined ``django.po`` file at ``xblocks_contrib/conf/locale/en/`` is committed to + openedx-translations. +3. The Transifex GitHub App syncs source strings to the ``openedx-translations`` Transifex project + under the ``open-edx`` organization. +4. The Open edX translation community translates strings on `Transifex`_. +5. Reviewed translations sync back to the openedx-translations repository. + +Since these XBlocks were extracted from edx-platform, many strings already have existing translations +in Transifex. The pipeline preserves these so translators do not need to re-translate them. .. _Transifex: https://www.transifex.com/ -Pushing source translation files to Transifex requires access to the edx-platform. Request access from the Open Source -Team if you will be pushing translation files. You should also `configure the Transifex client`_ if you have not done so -already. +**Direct Transifex CLI (alternative):** -.. _configure the Transifex client: https://docs.transifex.com/client/config/ +Each XBlock also has a ``.tx/config`` for direct Transifex CLI usage when the openedx-translations +pipeline is not available. This requires access to the ``open-edx`` Transifex organization. -The `make` targets listed below can be used to push or pull translations. +1. Install the Transifex CLI: ``make install_transifex_client`` +2. Set your API token: ``export TX_TOKEN=`` .. list-table:: - :widths: 25 75 + :widths: 35 65 :header-rows: 1 * - Target - Description - * - pull_translations - - Pull translations from Transifex - * - push_translations - - Push source translation files to Transifex + * - ``make extract_translations`` + - Extract translatable strings into ``text.po`` for each XBlock + * - ``make compile_translations`` + - Compile ``.po`` files into ``.mo`` files for each XBlock + * - ``make pull_translations`` + - Pull translations from Transifex for each XBlock + * - ``make push_translations`` + - Extract and push source translations to Transifex for each XBlock + * - ``make validate_translations`` + - Full validation pipeline (dummy build + source drift detection) + * - ``make detect_changed_source_translations`` + - Check if source ``.po`` files are up-to-date with the code + * - ``make dummy_translations`` + - Generate dummy Esperanto/RTL ``.po`` files for testing + * - ``make build_dummy_translations`` + - Generate and compile dummy translations + * - ``make install_transifex_client`` + - Install the Transifex CLI Fake Translations ***************** As you develop features it may be helpful to know which strings have been marked for translation, and which are not. -Use the `fake_translations` make target for this purpose. This target will extract all strings marked for translation, -generate fake translations in the Esperanto (eo) language directory, and compile the translations. +Use the ``dummy_translations`` make target for this purpose. This target will extract all strings marked for +translation and generate fake translations in the Esperanto (eo) and fake-RTL (rtl) language directories. +Run ``make compile_translations`` afterwards (or use ``make build_dummy_translations`` to do both at once). You can trigger the display of the translations by setting your browser's language to Esperanto (eo), and navigating to a page on the site. Instead of plain English strings, you should see specially-accented English strings that look diff --git a/xblocks_contrib/annotatable/.tx/config b/xblocks_contrib/annotatable/.tx/config index 5f1f3f6d..24902752 100644 --- a/xblocks_contrib/annotatable/.tx/config +++ b/xblocks_contrib/annotatable/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:annotatable] -file_filter = annotatable/translations//LC_MESSAGES/text.po -source_file = annotatable/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-annotatable] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/discussion/.tx/config b/xblocks_contrib/discussion/.tx/config index b2a95c6f..e72ae1c4 100644 --- a/xblocks_contrib/discussion/.tx/config +++ b/xblocks_contrib/discussion/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:discussion] -file_filter = discussion/translations//LC_MESSAGES/text.po -source_file = discussion/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-discussion] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/html/.tx/config b/xblocks_contrib/html/.tx/config index d157595a..a4dd8e84 100644 --- a/xblocks_contrib/html/.tx/config +++ b/xblocks_contrib/html/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:html] -file_filter = html/translations//LC_MESSAGES/text.po -source_file = html/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-html] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/lti/.tx/config b/xblocks_contrib/lti/.tx/config index 65e33581..bd0e4061 100644 --- a/xblocks_contrib/lti/.tx/config +++ b/xblocks_contrib/lti/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:lti] -file_filter = lti/translations//LC_MESSAGES/text.po -source_file = lti/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-lti] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/poll/.tx/config b/xblocks_contrib/poll/.tx/config index 4455ffd1..d36d8212 100644 --- a/xblocks_contrib/poll/.tx/config +++ b/xblocks_contrib/poll/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:poll] -file_filter = poll/translations//LC_MESSAGES/text.po -source_file = poll/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-poll] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/problem/.tx/config b/xblocks_contrib/problem/.tx/config index 7e3be765..60910adb 100644 --- a/xblocks_contrib/problem/.tx/config +++ b/xblocks_contrib/problem/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:problem] -file_filter = problem/translations//LC_MESSAGES/text.po -source_file = problem/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-problem] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/video/.tx/config b/xblocks_contrib/video/.tx/config index 14b9a06a..d42f5547 100644 --- a/xblocks_contrib/video/.tx/config +++ b/xblocks_contrib/video/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:video] -file_filter = video/translations//LC_MESSAGES/text.po -source_file = video/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-video] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO diff --git a/xblocks_contrib/word_cloud/.tx/config b/xblocks_contrib/word_cloud/.tx/config index 3d7b52dc..f52a8494 100644 --- a/xblocks_contrib/word_cloud/.tx/config +++ b/xblocks_contrib/word_cloud/.tx/config @@ -1,8 +1,8 @@ [main] host = https://www.transifex.com -[o:open-edx:p:p:xblocks:r:word_cloud] -file_filter = word_cloud/translations//LC_MESSAGES/text.po -source_file = word_cloud/translations/en/LC_MESSAGES/text.po +[o:open-edx:p:openedx-translations:r:xblocks-contrib-word_cloud] +file_filter = conf/locale//LC_MESSAGES/text.po +source_file = conf/locale/en/LC_MESSAGES/text.po source_lang = en type = PO