Merge pull request #462 from openedx/bmtcril/tx_to_openedx

refactor: Move translations to the openedx Transifex project

.github/workflows/test-tutor-aspects.yml

@@ -33,6 +33,20 @@ jobs:
         run: tutor config save
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@d5af243ce7bacb67384aa6c5b1fc5f169e30903e
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: true
       - name: Tutor build openedx
         run: tutor images build openedx aspects aspects-superset
       - name: Tutor start
@@ -71,6 +85,20 @@ jobs:
         run: tutor config save
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@d5af243ce7bacb67384aa6c5b1fc5f169e30903e
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: true
       - name: Tutor build openedx
         run: tutor images build openedx-dev aspects aspects-superset
       - name: Tutor start

.gitignore

@@ -9,3 +9,4 @@ __pycache__
 /.idea/
 venv/
 env/
+transifex_input.yaml

Makefile

@@ -29,6 +29,7 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)
 	pip install -r requirements/pip-tools.txt
 	$(UPGRADE) -o requirements/base.txt requirements/base.in
 	$(UPGRADE) -o requirements/dev.txt requirements/dev.in
+	$(UPGRADE) -o requirements/translations.txt requirements/translations.in
 
 requirements: ## Install packages from base requirement files
 	pip install -r requirements/pip.txt
@@ -42,6 +43,10 @@ dev-requirements: ## Install packages from developer requirement files
 	pip uninstall --yes $(PROJECT)
 	pip install -e .
 
+translation-requirements: ## Install packages from translation requirements
+	pip install -r requirements/pip.txt
+	pip install -r requirements/translations.txt
+
 build-pythonpackage: ## Build Python packages ready to upload to pypi
 	python setup.py sdist bdist_wheel
 
@@ -78,17 +83,15 @@ release-push:
 	git push origin $(TAG)
 
 ###### Additional commands
-push_translations:
-	TUTOR_ROOT=$(TUTOR_ROOT) tutor config save
-	python scripts/translate.py $(TUTOR_ROOT) push
+pull_translations: translation-requirements
+	rm -rf tutoraspects/templates/aspects/apps/superset/conf/locale/*;
+	atlas pull $(OPENEDX_ATLAS_ARGS) translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale/:tutoraspects/templates/aspects/apps/superset/conf/locale/
 
-compile_translations:
-	TUTOR_ROOT=$(TUTOR_ROOT) tutor config save
-	python scripts/translate.py $(TUTOR_ROOT) compile
+	@echo "Translations have been pulled via Atlas."
+	python scripts/translate.py . compile
 
-list_translations:
-	TUTOR_ROOT=$(TUTOR_ROOT) tutor config save
-	python scripts/translate.py $(TUTOR_ROOT) list
+extract_translations: translation-requirements
+	python scripts/translate.py . extract
 
 version: ## Print the current tutor version
 	@python -c 'import io, os; about = {}; exec(io.open(os.path.join("$(PACKAGE)", "__about__.py"), "rt", encoding="utf-8").read(), about); print(about["__version__"])'

requirements/base.in

@@ -1,4 +1,5 @@
 click
-tutor>=15
 bcrypt
+openedx-atlas
 transifex-python
+tutor>=15

requirements/base.txt

@@ -43,6 +43,8 @@ oauthlib==3.2.2
     # via
     #   kubernetes
     #   requests-oauthlib
+openedx-atlas==0.5.0
+    # via -r requirements/base.in
 parsimonious==0.10.0
     # via pyseeyou
 pyasn1==0.5.0

requirements/dev.txt

@@ -106,6 +106,8 @@ oauthlib==3.2.2
     #   -r requirements/base.txt
     #   kubernetes
     #   requests-oauthlib
+openedx-atlas==0.5.0
+    # via -r requirements/base.txt
 packaging==23.2
     # via
     #   black

requirements/translations.in

@@ -0,0 +1,2 @@
+# Requirements needed to run in openedx-translations to manage localization
+click

requirements/translations.txt

@@ -0,0 +1,8 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    make upgrade
+#
+click==8.1.7
+    # via -r requirements/translations.in

scripts/translate.py

@@ -2,24 +2,16 @@
 import sys
 
 import click
-from transifex.native import init
-from utils import (LANGUAGES, compile_translations, get_text_for_translations,
-                   push_translations)
-
-init(
-    os.getenv("TRANSIFEX_TOKEN"),
-    LANGUAGES,
-    os.getenv("TRANSIFEX_SECRET"),
-)
+from utils import compile_translations, extract_translations, get_text_for_translations
 
 
 @click.command()
 @click.argument("root")
 @click.argument("action")
 def command(root, action):
     """Interface for the translations."""
-    if action == "push":
-        push_translations(root)
+    if action == "extract":
+        extract_translations(root)
     elif action == "compile":
         compile_translations(root)
     elif action == "list":

scripts/utils.py

@@ -1,50 +1,46 @@
+import os
 import yaml
-from transifex.native import tx
-from transifex.native.parsing import SourceString
 
 ASSET_FOLDER_MAPPING = {
     "dashboard_title": "dashboards",
     "slice_name": "charts",
-    "database_name": "databases",
-    "table_name": "datasets",
 }
 
-LANGUAGES = [
-    "es",
-    "it",
-    "fr",
-    "zh",
-    "ja",
-    "de",
-    "pt",
-    "ru",
-    "ko",
-    "sk",
-    "sl",
-    "nl",
-]
-
 
 def get_text_for_translations(root_path):
-    assets_file = (
-        root_path + "/env/plugins/aspects/apps/superset/pythonpath/assets.yaml"
+    assets_path = (
+        os.path.join(root_path, "tutoraspects/templates/openedx-assets/assets/")
     )
 
+    print(f"Assets path: {assets_path}")
+
     strings = []
 
-    file = yaml.load(open(assets_file, "r"), Loader=yaml.FullLoader)
+    for root, dirs, files in os.walk(assets_path):
+        for file in files:
+            if not file.endswith(".yaml"):
+                continue
+            path = os.path.join(root, file)
+            print(f"Reading {path}")
+            with open(path, 'r') as asset_file:
+                asset_str = asset_file.read().replace("{{", "'")
+                asset_str = asset_str.replace("}}", "'")
 
-    for asset in file:
-        strings.extend(mark_text_for_translation(asset))
+            asset = yaml.safe_load(asset_str)
+            strings.extend(mark_text_for_translation(asset))
 
     return strings
 
 
 def mark_text_for_translation(asset):
-    """For every asset extract the text and mark it for translation"""
+    """
+    For every asset extract the text and mark it for translation
+    """
 
     def extract_text(asset, type):
-        """Extract text from an asset"""
+        """
+        Extract text from an asset
+        """
         strings = []
         if type == "dashboards":
             strings.append(asset["dashboard_title"])
@@ -73,64 +69,82 @@ def extract_text(asset, type):
             if asset["params"].get("y_axis_label"):
                 strings.append(asset["params"]["y_axis_label"])
 
-        elif type == "databases":
-            # WARNING: Databases are not translated
-            pass
-        elif type == "datasets":
-            # WARNING: Datasets are not translated
-            pass
         return strings
 
     for key, value in ASSET_FOLDER_MAPPING.items():
         if key in asset:
             strings = extract_text(asset, value)
             print(
-                f"Extracting text from {value} {asset.get('uuid')}",
-                strings,
+                f"Extracted {len(strings)} strings from {value} {asset.get('uuid')}"
             )
             return strings
 
+    # If we get here it's a type of asset that we don't translate, return nothing.
+    return []
 
-def compile_translations(root_path):
-    print("Fetching translations...")
-    tx.fetch_translations()
 
-    translation_file = (
-        "tutoraspects/templates/aspects/apps/superset/localization/locale.yaml"
+def compile_translations(root_path):
+    """
+    Combine translated files into the single file we use for translation.
+
+    This should be called after we pull translations using Atlas, see the
+    pull_translations make target.
+    """
+    translations_path = (
+        os.path.join(
+            root_path,
+            "tutoraspects/templates/aspects/apps/superset/conf/locale"
+        )
     )
-    file = open(translation_file, "w")
 
-    STRINGS = get_text_for_translations(root_path)
-
-    translations = {}
+    all_translations = {}
+    for root, dirs, files in os.walk(translations_path):
+        for file in files:
+            if not file.endswith(".yaml"):
+                continue
+
+            lang = root.split(os.sep)[-1]
+            path = os.path.join(root, file)
+            with open(path, 'r') as asset_file:
+                loc_str = asset_file.read()
+            all_translations[lang] = yaml.safe_load(loc_str)[lang]
+
+    out_path = (
+        os.path.join(
+            root_path,
+            "tutoraspects/templates/aspects/apps/superset/localization/locale.yaml"
+        )
+    )
 
-    print("Compiling translations...")
+    print(f"Writing all translations out to {out_path}")
+    with open(out_path, 'w') as outfile:
+        outfile.write("---\n")
+        yaml.safe_dump(all_translations, outfile)
+        outfile.write("\n{{ patch('superset-extra-asset-translations')}}\n")
 
-    for language in LANGUAGES:
-        print("Processing language", language)
-        translations[language] = {}
-        for string in STRINGS:
-            if not translations[language].get(string):
-                translation = tx.get_translation(string, language, None)
-                translations[language][string] = translation if translation else ""
 
-    file.write("---\n")
-    file.write(yaml.dump(translations))
 
-    file.write("\n{{ patch('superset-extra-asset-translations')}}\n")
+def extract_translations(root_path):
+    """
+    This gathers all translatable text from the Superset assets.
 
+    An English locale file is created, which openedx-translations will send to
+    Transifex for translation.
+    """
+    # The expectation is that this will end up at the site root, which should
+    # be cwd for make targets. This is a temporary file used only in the Github
+    # action in openedx-translations.
+    translation_file = "transifex_input.yaml"
 
-def push_translations(root_path):
-    print("Publishing translation strings...")
+    print("Gathering text for translations...")
+    STRINGS = set(get_text_for_translations(root_path))
+    translations = {'en': {}}
 
-    STRINGS = get_text_for_translations(root_path)
+    for string in STRINGS:
+        translations['en'][string] = string
 
-    source_strings = []
-    for text in STRINGS:
-        source_string = SourceString(
-            text,
-        )
-        source_strings.append(source_string)
+    print(f"Writing English strings to {translation_file}")
+    with open(translation_file, "w") as file:
+        file.write(yaml.dump(translations))
 
-    response_code, response_content = tx.push_source_strings(source_strings, purge=True)
-    print(response_code, response_content)
+    print("Done compiling translations.")

tutoraspects/templates/aspects/apps/superset/conf/README.md

@@ -0,0 +1,7 @@
+# Locale Files
+
+These files are managed via the OEP-58 process and should not be manually edited as they will be overwritten. The process is:
+
+1. A Github action on `openedx-translations` will pull this repo and run `make extract-translations` on this repo's Makefile. This pulls all translatable strings from the Superset assets. These are then uploaded to Transifex for translation.
+2. As translations are updated in Transifex they will be pushed to the correct language's files in `openedx-translations`.
+3. A Github action on this repo will run `make pull-translations` periodically. This will run the Atlas tool to synchronize the translation files in this directory with what is stored in `openedx-translations`. Then it will run a command to compile all of the separate translated files into one `locale.yaml` that is used to create the localized dashboard and chart strings during the Tutor init command. It will then automatically merge those changed files.