diff --git a/.gitignore b/.gitignore index d6f1202bf4ce..5ac88bad430e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ conf/locale/fake*/LC_MESSAGES/*.po conf/locale/fake*/LC_MESSAGES/*.mo # this was a mistake in i18n_tools, now fixed. conf/locale/messages.mo +conf/plugins-locale/ ### Testing artifacts .testids/ diff --git a/Makefile b/Makefile index 8664111ebefb..5eba534e59cb 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,16 @@ extract_translations: ## extract localizable strings from sources push_translations: ## push source strings to Transifex for translation i18n_tool transifex push -pull_translations: ## pull translations from Transifex +pull_xblock_translations: +ifeq ($(OPENEDX_ATLAS_PULL),) + @echo "atlas is not used due to empty OPENEDX_ATLAS_PULL environment variable" +else + rm -rf conf/xblocks/locale + mkdir conf/xblocks/locale + python manage.py lms xblocks_atlas_pull_module +endif + +pull_translations: pull_xblock_translations ## pull translations from Transifex git clean -fdX conf/locale i18n_tool transifex pull i18n_tool extract diff --git a/cms/envs/production.py b/cms/envs/production.py index d04dfcd8acc0..760bd3f9522f 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -262,6 +262,8 @@ def get_env_setting(setting): # ], PREPEND_LOCALE_PATHS = ENV_TOKENS.get('PREPEND_LOCALE_PATHS', []) +PLUGINS_TRANSLATIONS_ROOT = REPO_ROOT / 'conf/plugins-locale' + #Timezone overrides TIME_ZONE = ENV_TOKENS.get('CELERY_TIMEZONE', CELERY_TIMEZONE) diff --git a/common/djangoapps/xblock_django/api.py b/common/djangoapps/xblock_django/api.py index 163683ab0c38..19dd4a8b74ef 100644 --- a/common/djangoapps/xblock_django/api.py +++ b/common/djangoapps/xblock_django/api.py @@ -2,6 +2,7 @@ API methods related to xblock state. """ +import pkg_resources from openedx.core.lib.cache_utils import CacheInvalidationManager from common.djangoapps.xblock_django.models import XBlockConfiguration, XBlockStudioConfiguration @@ -27,6 +28,14 @@ def disabled_xblocks(): return XBlockConfiguration.objects.current_set().filter(enabled=False) +def get_xblocks_entry_points(): + return [entry_point for entry_point in pkg_resources.iter_entry_points(group='xblock.v1')] + +def get_xblocks(): + return [entry_point.resolve() for entry_point in get_xblocks_entry_points()] + + + def authorable_xblocks(allow_unsupported=False, name=None): """ This method returns the QuerySet of XBlocks that can be created in Studio (by default, only fully supported diff --git a/common/djangoapps/xblock_django/management/commands/pull_plugins_translations.py b/common/djangoapps/xblock_django/management/commands/pull_plugins_translations.py new file mode 100644 index 000000000000..c2d10022cfd5 --- /dev/null +++ b/common/djangoapps/xblock_django/management/commands/pull_plugins_translations.py @@ -0,0 +1,71 @@ +""" +This command downloads the translations for the XBlocks that are installed. +""" +import shutil +import subprocess +import os.path + +from django.conf import settings + +from django.core.management.base import BaseCommand + +from ...api import get_xblocks_entry_points + + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument( + '--verbose|-v', + action='store_true', + default=False, + dest='verbose', + help='Verbose output.' + ) + + parser.add_argument( + '--list|-l', + action='store_true', + default=False, + dest='list', + help='List plugins module names.' + ) + + def handle(self, *args, **options): + # Remove previous translations + for dir_name in os.listdir(settings.PLUGINS_TRANSLATIONS_ROOT): + dir_path = os.path.join(settings.PLUGINS_TRANSLATIONS_ROOT, dir_name) + + if os.path.isdir(dir_path): + shutil.rmtree(dir_path, ignore_errors=True) + + xblock_module_names = set() + for entry_point in get_xblocks_entry_points(): + xblock = entry_point.resolve() + module_import_path = xblock.__module__ + parent_module, _rest = module_import_path.split('.', maxsplit=1) + if parent_module == 'xmodule': + if options['verbose']: + self.stdout.write(f'INFO: Skipped edx-platform XBlock "{entry_point.name}" ' + f'in module={module_import_path}') + else: + xblock_module_names.add(parent_module) + + sorted_xblock_module_names = list(sorted(xblock_module_names)) + + if options['list']: + self.stdout.write('\n'.join(sorted_xblock_module_names)) + else: + xblock_atlas_args_list = [ + f'translations/edx-platform-plugins/{module_name}:{module_name}' + for module_name in sorted_xblock_module_names + ] + + subprocess.run( + [ + 'atlas', 'pull', + '--repository=Zeit-Labs/openedx-translations', + '--branch=plugins-fix' + ] + xblock_atlas_args_list, + cwd=settings.PLUGINS_TRANSLATIONS_ROOT, + ) diff --git a/xmodule/modulestore/django.py b/xmodule/modulestore/django.py index 9e78b21a4ce6..87d99e180b88 100644 --- a/xmodule/modulestore/django.py +++ b/xmodule/modulestore/django.py @@ -8,6 +8,7 @@ from importlib import import_module import gettext import logging +from os import path from pkg_resources import resource_filename import re # lint-amnesty, pylint: disable=wrong-import-order @@ -29,6 +30,7 @@ from xmodule.modulestore.draft_and_published import BranchSettingMixin # lint-amnesty, pylint: disable=wrong-import-position from xmodule.modulestore.mixed import MixedModuleStore # lint-amnesty, pylint: disable=wrong-import-position from xmodule.util.xmodule_django import get_current_request_hostname # lint-amnesty, pylint: disable=wrong-import-position +from xmodule.waffle import ENABLE_ATLAS_TRANSLATIONS # We also may not always have the current request user (crum) module available try: @@ -369,8 +371,7 @@ class XBlockI18nService: def __init__(self, block=None): """ Attempt to load an XBlock-specific GNU gettext translator using the XBlock's own domain - translation catalog, currently expected to be found at: - /conf/locale//LC_MESSAGES/.po|mo + translation catalog. If we can't locate the domain translation catalog then we fall-back onto django.utils.translation, which will point to the system's own domain translation catalog This effectively achieves translations by coincidence for an XBlock which does not provide @@ -378,21 +379,66 @@ def __init__(self, block=None): """ self.translator = django.utils.translation if block: - xblock_class = getattr(block, 'unmixed_class', block.__class__) - xblock_resource = xblock_class.__module__ + if ENABLE_ATLAS_TRANSLATIONS.is_enabled(): + xblock_domain = 'django' + else: + xblock_domain = 'text' + + selected_language = get_language() + + xblock_locale_path = self.get_python_locale_directory(block) + if xblock_locale_path: + try: + self.translator = gettext.translation( + xblock_domain, + xblock_locale_path, + [to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)] + ) + except OSError: + # Fall back to the default Django translator if the XBlock translator is not found. + pass + + @staticmethod + def get_python_locale_directory(block): + """ + Return the XBlock locale directory with support for OEP-58 updated translation infrastructure. + + This function works in two modes: + 1. With ENABLE_ATLAS_TRANSLATIONS enabled it loads OEP-58 external translations. + 2. With ENABLE_ATLAS_TRANSLATIONS disabled it uses the old in-xblock translations which are + typically found at /conf/locale//LC_MESSAGES/.po|mo + + The second mode is going to be eventually deprecated when OEP-58 is fully in use. + """ + xblock_class = getattr(block, 'unmixed_class', block.__class__) + xblock_resource = xblock_class.__module__ + + if ENABLE_ATLAS_TRANSLATIONS.is_enabled(): + xblock_module_name = xblock_resource + xblock_locale_path = path.join(settings.PLUGINS_TRANSLATIONS_ROOT, xblock_module_name) + else: xblock_locale_dir = 'translations' xblock_locale_path = resource_filename(xblock_resource, xblock_locale_dir) - xblock_domain = 'text' - selected_language = get_language() - try: - self.translator = gettext.translation( - xblock_domain, - xblock_locale_path, - [to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)] - ) - except OSError: - # Fall back to the default Django translator if the XBlock translator is not found. - pass + + return xblock_locale_path + + @staticmethod + def get_javascript_locale_path(block): + """ + Return the XBlock compiled javascript i18n path with support for OEP-58 updated translation infrastructure. + + This function only works With ENABLE_ATLAS_TRANSLATIONS waffle switch enabled. + """ + xblock_class = getattr(block, 'unmixed_class', block.__class__) + xblock_resource = xblock_class.__module__ + selected_language = get_language() + + if ENABLE_ATLAS_TRANSLATIONS.is_enabled(): + xblock_module_name = xblock_resource + translations_dir = settings.XBLOCK_TRANSLATIONS_DIRECTORY + xblock_locale_path = path.join(translations_dir, xblock_module_name, selected_language, 'django.js') + if path.exists(xblock_locale_path): + return xblock_locale_path def __getattr__(self, name): name = 'gettext' if name == 'ugettext' else name diff --git a/xmodule/waffle.py b/xmodule/waffle.py new file mode 100644 index 000000000000..c224090a393b --- /dev/null +++ b/xmodule/waffle.py @@ -0,0 +1,20 @@ +""" +This module contains configuration settings via waffle flags for the xmodule modulestore. +""" + +from edx_toggles.toggles import WaffleSwitch + +# XModule Namespace +WAFFLE_NAMESPACE = 'xmodule' + +# .. toggle_name: xmodule.enable_atlas_translations +# .. toggle_implementation: WaffleSwitch +# .. toggle_default: False +# .. toggle_description: Waffle switch for loading XBlock translations from external directory in line with to OEP-58. +# .. toggle_use_cases: temporary +# .. toggle_creation_date: 2023-03-28 +# .. toggle_tickets: TODO: add here the docs.openedx.org document link. + +ENABLE_ATLAS_TRANSLATIONS = WaffleSwitch( + f'{WAFFLE_NAMESPACE}.enable_atlas_translations', __name__ +)