From e9ea0c00c20d1b4faed679717f399a8f75fab7fc Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Thu, 12 Oct 2023 09:55:24 -0400 Subject: [PATCH 1/4] feat: Add a new generic translation extraction type For non-Django, non-JS projects we should be able to extract translations just using make commands and a small set of assumptions about where the source English file can be found. This adds that functionality and an integration with tutor-contrib-aspects which needs it. --- .../extract-translation-source-files.yml | 92 +++++++++++++++++++ transifex.yml | 9 +- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extract-translation-source-files.yml b/.github/workflows/extract-translation-source-files.yml index e38cc9580c3..e3da3dcd4e1 100644 --- a/.github/workflows/extract-translation-source-files.yml +++ b/.github/workflows/extract-translation-source-files.yml @@ -254,6 +254,98 @@ jobs: # push changes to branch git push + # Non-Django, non-JS repositories with a generic extraction interface + generic-translations: + strategy: + # using max-parallel to avoid git push/pull issues when running in parallel + max-parallel: 1 + matrix: + repo: + - tutor-contrib-aspects + + runs-on: ubuntu-latest + continue-on-error: true + needs: [setup-branch] + + steps: + # Clones the openedx-translations repo + - name: clone openedx/openedx-translations + uses: actions/checkout@v3 + with: + ref: ${{ needs.setup-branch.outputs.branch }} + + # Clones the repository + - name: clone openedx/${{ matrix.repo }} + uses: actions/checkout@v3 + with: + repository: openedx/${{ matrix.repo }} + path: translations/${{ matrix.repo }} + + # Sets up Python + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + # Installs Python requirements from translations.txt + - name: install requirements + run: pip install -r requirements/translations.txt + + # Extracts the translation source files + - name: extract translation source files + run: | + cd translations/${{ matrix.repo }} + make extract_translations + + # Validate compilation of translation source files + - name: validate compilation of translation source files + run: | + cd translations/${{ matrix.repo }} + make compile_translations + + # git adds only the translation source files, found in conf/locale/en + - name: git add the translation source files + id: add-sources + run: | + # set identity + git config --global user.email "translations-bot@openedx.org" + git config --global user.name "edx-transifex-bot" + # Change directory to translations/${{ matrix.repo }} + cd translations/${{ matrix.repo }} + # finds the directory containing the english locale usually located + # in */*/conf/locale/en + # This is slightly different from the python-translations line above + # in that it ignores top level dotfiles and searches further down the + # directory tree, which may be needed in these use cases. + EN_DIR=$(find ./* -type d -regex ".*conf\/locale\/en$") + # If the directory is not found, exit with an error. This can happen + # if we add a repository that doesn't comply with OEP-58, or it + # doesn't have any translations yet + if [ -z "$EN_DIR" ]; then + echo "Missing English locale directory for ${{ matrix.repo }}!" + echo "exiting with error!" + exit 1 + fi + # remove translations/${{ matrix.repo }}/.git so we don't commit a + # submodule + rm -rf .git + + # use (-f) to force add the files even if they are ignored in the + # source repo + git add -A $EN_DIR -f + # Check the git statuses of the translation source files, this will + # store the number of files staged for the next step + echo "GIT_STATUS=$(git status $EN_DIR -s | wc -l)" >> $GITHUB_ENV + + # Attempts to commit the translation source files if there is a difference + - name: git commit the translation source files + if: "${{ env.GIT_STATUS > 0 }}" + run: | + # commit the changes + git commit -m "chore: add extracted translation source files from ${{ matrix.repo }}" + # push changes to branch + git push + merge-translations: runs-on: ubuntu-latest needs: [setup-branch, python-translations, js-translations] diff --git a/transifex.yml b/transifex.yml index 9cb8528d88c..a6017e4e5df 100644 --- a/transifex.yml +++ b/transifex.yml @@ -80,7 +80,7 @@ git: source_language: en source_file_dir: translations/FeedbackXBlock/feedback/conf/locale/en/ translation_files_expression: 'translations/FeedbackXBlock/feedback/conf/locale//' - + # frontend-app-account - filter_type: file file_format: KEYVALUEJSON @@ -250,6 +250,13 @@ git: source_file: translations/studio-frontend/src/i18n/transifex_input.json translation_files_expression: 'translations/studio-frontend/src/i18n/messages/.json' + # tutor-contrib-aspects + - filter_type: file + file_format: YAML_GENERIC + source_language: en + source_file: translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale/en/locale.yaml + translation_files_expression: 'translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale//locale.yaml' + # xblock-drag-and-drop-v2 - filter_type: dir file_format: PO From 8bcf0b53547232464aa4fc85309d2caae12c6ddf Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Fri, 13 Oct 2023 11:09:23 -0400 Subject: [PATCH 2/4] refactor: Move to a root level transifex_input.yaml This relies on the extract_translations make target in each "generic" repo to create a transifex_input.yaml in the root of the site, which greatly simplifies the process. --- .../extract-translation-source-files.yml | 45 ++++++------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/.github/workflows/extract-translation-source-files.yml b/.github/workflows/extract-translation-source-files.yml index e3da3dcd4e1..6082410c672 100644 --- a/.github/workflows/extract-translation-source-files.yml +++ b/.github/workflows/extract-translation-source-files.yml @@ -254,7 +254,9 @@ jobs: # push changes to branch git push - # Non-Django, non-JS repositories with a generic extraction interface + # Non-Django, non-JS repositories with a generic extraction interface. This + # assumes that a `extract_translations` make target exists and will create + # a transifex_input.yaml file in the root of the site. generic-translations: strategy: # using max-parallel to avoid git push/pull issues when running in parallel @@ -297,13 +299,8 @@ jobs: cd translations/${{ matrix.repo }} make extract_translations - # Validate compilation of translation source files - - name: validate compilation of translation source files - run: | - cd translations/${{ matrix.repo }} - make compile_translations - - # git adds only the translation source files, found in conf/locale/en + # git adds only the translation source file transifex_input.yaml from + # the repo root. - name: git add the translation source files id: add-sources run: | @@ -312,30 +309,16 @@ jobs: git config --global user.name "edx-transifex-bot" # Change directory to translations/${{ matrix.repo }} cd translations/${{ matrix.repo }} - # finds the directory containing the english locale usually located - # in */*/conf/locale/en - # This is slightly different from the python-translations line above - # in that it ignores top level dotfiles and searches further down the - # directory tree, which may be needed in these use cases. - EN_DIR=$(find ./* -type d -regex ".*conf\/locale\/en$") - # If the directory is not found, exit with an error. This can happen - # if we add a repository that doesn't comply with OEP-58, or it - # doesn't have any translations yet - if [ -z "$EN_DIR" ]; then - echo "Missing English locale directory for ${{ matrix.repo }}!" - echo "exiting with error!" - exit 1 - fi - # remove translations/${{ matrix.repo }}/.git so we don't commit a - # submodule - rm -rf .git - # use (-f) to force add the files even if they are ignored in the - # source repo - git add -A $EN_DIR -f - # Check the git statuses of the translation source files, this will - # store the number of files staged for the next step - echo "GIT_STATUS=$(git status $EN_DIR -s | wc -l)" >> $GITHUB_ENV + # remove translations/${{ matrix.repo }}/.git so we don't commit a submodule + rm -rf .git + # find transifex_input.yaml + TRANSIFEX_YAML_PATH=$(find . -name 'transifex_input.yaml') + # stage the transifex_input.yaml file generated by make, force it in + # case the file is in .gitignore (it should be) + git add $TRANSIFEX_YAML_PATH -f -v + # Check the git status of the translation source files + echo "GIT_STATUS=$(git status $TRANSIFEX_YAML_PATH -s | wc -l)" >> $GITHUB_ENV # Attempts to commit the translation source files if there is a difference - name: git commit the translation source files From 5703d4af3488f52ca6c625d70060718a5c04522e Mon Sep 17 00:00:00 2001 From: Brian Mesick <112640379+bmtcril@users.noreply.github.com> Date: Fri, 13 Oct 2023 11:39:18 -0400 Subject: [PATCH 3/4] fix: Update transifex.yml Co-authored-by: Brian Smith <112954497+brian-smith-tcril@users.noreply.github.com> --- transifex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transifex.yml b/transifex.yml index a6017e4e5df..64f22774013 100644 --- a/transifex.yml +++ b/transifex.yml @@ -254,7 +254,7 @@ git: - filter_type: file file_format: YAML_GENERIC source_language: en - source_file: translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale/en/locale.yaml + source_file: translations/tutor-contrib-aspects/transifex_input.yaml translation_files_expression: 'translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale//locale.yaml' # xblock-drag-and-drop-v2 From 94bb36a9ee15ae3d1143b9414e769eeabc077cac Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Mon, 16 Oct 2023 10:07:44 -0400 Subject: [PATCH 4/4] refactor: Include input file location in matrix --- .../extract-translation-source-files.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/extract-translation-source-files.yml b/.github/workflows/extract-translation-source-files.yml index 6082410c672..9af2c28abc3 100644 --- a/.github/workflows/extract-translation-source-files.yml +++ b/.github/workflows/extract-translation-source-files.yml @@ -262,8 +262,9 @@ jobs: # using max-parallel to avoid git push/pull issues when running in parallel max-parallel: 1 matrix: - repo: - - tutor-contrib-aspects + repository_config: + - repo: tutor-contrib-aspects + transifex_file_path: transifex_input.yaml runs-on: ubuntu-latest continue-on-error: true @@ -277,11 +278,11 @@ jobs: ref: ${{ needs.setup-branch.outputs.branch }} # Clones the repository - - name: clone openedx/${{ matrix.repo }} + - name: clone openedx/${{ matrix.repository_config.repo }} uses: actions/checkout@v3 with: - repository: openedx/${{ matrix.repo }} - path: translations/${{ matrix.repo }} + repository: openedx/${{ matrix.repository_config.repo }} + path: translations/${{ matrix.repository_config.repo }} # Sets up Python - name: setup python @@ -296,7 +297,7 @@ jobs: # Extracts the translation source files - name: extract translation source files run: | - cd translations/${{ matrix.repo }} + cd translations/${{ matrix.repository_config.repo }} make extract_translations # git adds only the translation source file transifex_input.yaml from @@ -307,31 +308,30 @@ jobs: # set identity git config --global user.email "translations-bot@openedx.org" git config --global user.name "edx-transifex-bot" - # Change directory to translations/${{ matrix.repo }} - cd translations/${{ matrix.repo }} + # Change directory to translations/${{ matrix.repository_config.repo }} + cd translations/${{ matrix.repository_config.repo }} - # remove translations/${{ matrix.repo }}/.git so we don't commit a submodule + # remove translations/${{ matrix.repository_config.repo }}/.git so we don't commit a submodule rm -rf .git - # find transifex_input.yaml - TRANSIFEX_YAML_PATH=$(find . -name 'transifex_input.yaml') + # stage the transifex_input.yaml file generated by make, force it in # case the file is in .gitignore (it should be) - git add $TRANSIFEX_YAML_PATH -f -v + git add ${{ matrix.repository_config.transifex_file_path }} -f -v # Check the git status of the translation source files - echo "GIT_STATUS=$(git status $TRANSIFEX_YAML_PATH -s | wc -l)" >> $GITHUB_ENV + echo "GIT_STATUS=$(git status ${{ matrix.repository_config.transifex_file_path }} -s | wc -l)" >> $GITHUB_ENV # Attempts to commit the translation source files if there is a difference - name: git commit the translation source files if: "${{ env.GIT_STATUS > 0 }}" run: | # commit the changes - git commit -m "chore: add extracted translation source files from ${{ matrix.repo }}" + git commit -m "chore: add extracted translation source files from ${{ matrix.repository_config.repo }}" # push changes to branch git push merge-translations: runs-on: ubuntu-latest - needs: [setup-branch, python-translations, js-translations] + needs: [setup-branch, python-translations, js-translations, generic-translations] steps: # Clones the openedx-translations repo on the automated/extract-translation-source-files-# branch