extract-translation-source-files.yml

# This workflow extracts translation source files from code and adds the extracted
# translation source files to the openedx-translations repository via an auto-merged pull
# request.

name: Extract Translation Source Files

on:
  workflow_dispatch:    # by request
    inputs:
      repo:
        description: 'Repository to extract translation source files from (leave blank to run all)'
        required: false
        type: string
      ref:
        description: 'If repository is specified, the ref to extract translation source files from (leave blank to use the default branch)'
        required: false
        default: ''
        type: string

  schedule:
    - cron: '0 0 * * *'  # every day at midnight

jobs:
  setup-matrix:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/github-script@v6
        id: generate_repo_matrix
        with:
          script: |
            // Need to use a workaround until https://github.com/actions/toolkit/issues/1576 is fixed
            // const selectedRepo = core.getInput('repo');
            const selectedRepo = ${{ toJSON(github.event.inputs.repo) }} ?? '';

            const allPythonRepos = [
              'AudioXBlock',
              'completion',
              'course-discovery',
              'credentials',
              'credentials-themes',
              'DoneXBlock',
              'ecommerce',
              'edx-ace',
              'edx-bulk-grades',
              'edx-enterprise',
              'edx-ora2',
              'edx-platform',
              'edx-proctoring',
              'FeedbackXBlock',
              'RecommenderXBlock',
              'xblock-drag-and-drop-v2',
              'xblock-free-text-response',
              'xblock-google-drive',
              'xblock-image-explorer',
              'xblock-image-modal',
              'xblock-lti-consumer',
              'xblock-qualtrics-survey',
              'xblock-sql-grader',
              'xblock-submit-and-compare',
            ];

            const allJavascriptRepos = [
              'frontend-app-admin-portal',
              'frontend-app-publisher',
              'frontend-app-account',
              'frontend-app-authn',
              'frontend-app-communications',
              'frontend-app-course-authoring',
              'frontend-app-discussions',
              'frontend-app-ecommerce',
              'frontend-app-enterprise-public-catalog',
              'frontend-app-gradebook',
              'frontend-app-learner-dashboard',
              'frontend-app-learner-record',
              'frontend-app-learning',
              'frontend-app-library-authoring',
              'frontend-app-ora-grading',
              'frontend-app-payment',
              'frontend-app-profile',
              'frontend-app-program-console',
              'frontend-app-support-tools',
              'frontend-component-footer',
              'frontend-component-header',
              'frontend-lib-content-components',
              'frontend-lib-special-exams',
              'frontend-platform',
              'paragon',
              'studio-frontend',
              'frontend-app-learner-portal-enterprise',
            ];

            const allGenericRepos = [
              {
                repo: 'tutor-contrib-aspects',
                transifex_file_path: 'transifex_input.yaml'
              },
            ]

            core.setOutput('hasPythonRepos', true);
            core.setOutput('hasJavascriptRepos', true);
            core.setOutput('hasGenericRepos', true);

            if (selectedRepo === '') {
              core.setOutput('pythonRepos', allPythonRepos);
              core.setOutput('javascriptRepos', allJavascriptRepos);
              core.setOutput('genericRepos', allGenericRepos);
              return;
            }

            if (allPythonRepos.includes(selectedRepo)) {
              core.setOutput('pythonRepos', [selectedRepo]);
            } else {
              core.setOutput('pythonRepos', []);
              core.setOutput('hasPythonRepos', false);
            }

            if (allJavascriptRepos.includes(selectedRepo)) {
              core.setOutput('javascriptRepos', [selectedRepo]);
            } else {
              core.setOutput('javascriptRepos', []);
              core.setOutput('hasJavascriptRepos', false);
            }

            const genericRepoConfig = allGenericRepos.find(repoConfig => repoConfig.repo === selectedRepo);
            if (genericRepoConfig !== undefined) {
              core.setOutput('genericRepos', [genericRepoConfig]);
            } else {
              core.setOutput('genericRepos', []);
              core.setOutput('hasGenericRepos', false);
            }

    outputs:
      has_python_repos: ${{ steps.generate_repo_matrix.outputs.hasPythonRepos }}
      python_repos: ${{ steps.generate_repo_matrix.outputs.pythonRepos }}
      has_js_repos: ${{ steps.generate_repo_matrix.outputs.hasJavascriptRepos }}
      js_repos: ${{ steps.generate_repo_matrix.outputs.javascriptRepos }}
      has_generic_repos: ${{ steps.generate_repo_matrix.outputs.hasGenericRepos }}
      generic_repos: ${{ steps.generate_repo_matrix.outputs.genericRepos }}

  setup-branch:
    runs-on: ubuntu-latest
    outputs:
      branch: ${{ steps.dynamic_branch.outputs.branch }}
    steps:
    # sets the job output that creates a unique branch name
    - name: dynamically set branch job output variable
      id: dynamic_branch
      run: echo "branch=automated/extract-translation-source-files-$(date +'%Y%m%dT%H%M%S')" >> $GITHUB_OUTPUT

    # clones the openedx-translations repo so a branch can be added in the next step
    - name: echo branch name
      id: date
      run: echo $BRANCH
    - name: clone openedx/openedx-translations
      uses: actions/checkout@v3

    # creates the branch where all the translations will be added to
    - name: make and push a new branch
      run: |
        git checkout -b ${{ steps.dynamic_branch.outputs.branch }}
        git push --set-upstream origin ${{ steps.dynamic_branch.outputs.branch }}

  python-translations:
    if: ${{ fromJson(needs.setup-matrix.outputs.has_python_repos) == true }}
    needs: [setup-branch, setup-matrix]
    strategy:
      # using max-parallel to avoid git push/pull issues when running in parallel
      max-parallel: 1
      matrix:
        repo: ${{ fromJson(needs.setup-matrix.outputs.python_repos) }}

    runs-on: ubuntu-latest
    continue-on-error: true

    steps:
      # Clones the openedx-translations repo
      - name: clone openedx/openedx-translations
        uses: actions/checkout@v3
        with:
          ref: ${{ needs.setup-branch.outputs.branch }}

      # Installs gettext, a dependency for extracting translation source files in django
      - name: install gettext
        run: sudo apt install -y gettext

      # Clones the  repository
      - name: clone ${{ github.repository_owner }}/${{ matrix.repo }}
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/${{ matrix.repo }}
          ref: ${{ github.event.inputs.ref }}
          path: translations/${{ matrix.repo }}

      - name: prepare the environment with edx-platform specific changes
        if: matrix.repo == 'edx-platform'
        run: |
          # This is needed for edx-platform base.txt packages
          sudo apt install -y libxml2-dev libxmlsec1-dev libxslt1-dev

          # There is a theme directory with it's own conf/locale directory, we need to remove that
          rm -rf translations/${{ matrix.repo }}/themes/conf

      # Setup Python
      - name: setup python
        uses: actions/setup-python@v4
        id: setup_python
        with:
          python-version: '3.8'
          cache: pip
          # The `edx-platf*rm` wildcard will be expanded to `edx-platform` if its requirement file exists,
          # otherwise it will be omitted
          cache-dependency-path: |
            requirements/translations.txt
            translations/edx-platf*rm/requirements/edx/base.txt

      # Installs Python requirements from translations.txt
      - name: install requirements
        run: |
          # Install requirements for all packages
          pip install -r requirements/translations.txt

          EDX_BASE_REQS=translations/edx-platform/requirements/edx/base.txt
          if test -f $EDX_BASE_REQS; then
            # Install edx-platform specific apps (e.g. wiki) which is required to extract translations from
            pip install -r $EDX_BASE_REQS
          fi

      # Extracts the translation source files
      - name: extract translation source files
        run: |
          cd translations/${{ matrix.repo }}
          make extract_translations
        env:
          IS_OPENEDX_TRANSLATIONS_WORKFLOW: yes

      # Validate compilation of translation source files
      - name: validate compilation of translation source files
        run: |
          cd translations/${{ matrix.repo }}
          django-admin compilemessages --locale en

      # Preparations so git adds only the translation source files, found in conf/locale/en
      - name: find the location of the translation source files
        id: add-sources
        run: |
          # set identity
          git config --global user.email "translations-bot@openedx.org"
          git config --global user.name "edx-transifex-bot"
          # Change directory to translations/${{ matrix.repo }}
          cd translations/${{ matrix.repo }}
          # finds the directory containing the english locale usually located in
          # */*/conf/locale/en
          # but also exclude any hidden directories that might exist for some reason
          EN_DIR=$(find . -type d -not -path '*/.*' -regex ".+conf\/locale\/en$")
          en_dir_count=$(echo "$EN_DIR" | wc -l)
          # If the repo complies with OEP-58, then we'll find only one directory that matches the above criteria
          # Otherwise, we'll consider it an error
          if [ $en_dir_count -gt 1 ]; then
            echo "Error: More than one English locale directory found for ${{ matrix.repo }}!"
            exit 1
          elif [ $en_dir_count -eq 0 ]; then
            echo "Error: Missing English locale directory for ${{ matrix.repo }}!"
            exit 1
          fi
          # Save EN_DIR value to be used in next steps
          echo "EN_DIR=$EN_DIR" >> $GITHUB_ENV
          # remove translations/${{ matrix.repo }}/.git so we don't commit a submodule
          rm -rf .git

      - name: add repo translation source files openedx-translations
        run: |
          # finds the django.po and djangojs.po files generated by make
          # extract_translations into EN_DIR and adds them
          cd translations/${{ matrix.repo }}
          DJANGO_PATH=$(find $EN_DIR -name 'django.po')
          DJANGOJS_PATH=$(find $EN_DIR -name 'djangojs.po')
          ############## Special support for XBlocks
          # if both files are not found, then it can be an XBlock (thus, text.po and textjs.po files)
          if [ -z "$DJANGO_PATH" ] && [ -z "$DJANGOJS_PATH" ]; then
            DJANGO_PATH=$(find $EN_DIR -name 'text.po')
            DJANGOJS_PATH=$(find $EN_DIR -name 'textjs.po')
            # Rename the files to django.po and djangojs.po, if exists
            if [ -n "$DJANGO_PATH" ]; then
              mv $DJANGO_PATH $(dirname $DJANGO_PATH)/django.po
              DJANGO_PATH=$(dirname $DJANGO_PATH)/django.po
            fi
            if [ -n "$DJANGOJS_PATH" ]; then
              mv $DJANGOJS_PATH $(dirname $DJANGOJS_PATH)/djangojs.po
              DJANGOJS_PATH=$(dirname $DJANGOJS_PATH)/djangojs.po
            fi
          fi
          ############## End of - Special support for XBlocks
          # use (-f) to force add the files even if they are ignored in the source repo
          git add $DJANGO_PATH $DJANGOJS_PATH -f -v
          # Check the git statuses of the translation source files
          echo "GIT_STATUS=$(git status $DJANGO_PATH $DJANGOJS_PATH -s | wc -l)" >> $GITHUB_ENV

      # Attempts to commit the translation source files if there is a difference
      - name: git commit the translation source files
        if: "${{ env.GIT_STATUS > 0 }}"
        run: |
          # commit the changes
          git commit -m "chore: add extracted translation source files from ${{ matrix.repo }}"
          # push changes to branch
          git push

  js-translations:
    if: ${{ !cancelled() && fromJson(needs.setup-matrix.outputs.has_js_repos) == true }}
    needs: [setup-branch, setup-matrix, python-translations]
    strategy:
      # using max-parallel to avoid git push/pull issues when running in parallel
      max-parallel: 1
      matrix:
        # repos missing extract_translations target in makefile:
        # * frontend-enterprise?
        # * frontend-build?
        # * frontend-app-learner-portal-programs
        # * frontend-component-cookie-policy-banner
        # * frontend-app-programs-dashboard
        # repos with errors running extract_translations
        # * frontend-template-application
        repo: ${{ fromJson(needs.setup-matrix.outputs.js_repos) }}
    runs-on: ubuntu-latest
    continue-on-error: true

    steps:
      # Clones the openedx-translations repo
      - name: clone openedx/openedx-translations
        uses: actions/checkout@v3
        with:
          ref: ${{ needs.setup-branch.outputs.branch }}

      # Clones the  repository
      - name: clone ${{ github.repository_owner }}/${{ matrix.repo }}
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/${{ matrix.repo }}
          ref: ${{ github.event.inputs.ref }}
          path: translations/${{ matrix.repo }}

      # Sets up node/npm
      - name: setup node
        uses: actions/setup-node@v3
        with:
          node-version: 16

      # Extracts the translation source files
      - name: extract translation source files
        run: |
          cd translations/${{ matrix.repo }}
          make extract_translations
      # git adds only the translation source files, found in src/i18n/transifex_input.json
      - name: git add the translation source files
        id: add-sources
        run: |
          # set identity
          git config --global user.email "translations-bot@openedx.org"
          git config --global user.name "edx-transifex-bot"
          # Change directory to translations/${{ matrix.repo }}
          cd translations/${{ matrix.repo }}
          # remove translations/${{ matrix.repo }}/.git so we don't commit a submodule
          rm -rf .git
          # find transifex_input.json
          TRANSIFEX_JSON_PATH=$(find . -name 'transifex_input.json')
          # stage the transifex_input.json file generated by make
          git add $TRANSIFEX_JSON_PATH -f -v
          # Check the git status of the translation source files
          echo "GIT_STATUS=$(git status $TRANSIFEX_JSON_PATH -s | wc -l)" >> $GITHUB_ENV
      # Attempts to commit the translation source files if there is a difference
      - name: git commit the translation source files
        if: "${{ env.GIT_STATUS > 0 }}"
        run: |
          # commit the changes
          git commit -m "chore: add extracted translation source files from ${{ matrix.repo }}"
          # push changes to branch
          git push

  # Non-Django, non-JS repositories with a generic extraction interface. This
  # assumes that a `extract_translations` make target exists and will create
  # a transifex_input.yaml file in the root of the site.
  generic-translations:
    if: ${{ !cancelled() && fromJson(needs.setup-matrix.outputs.has_generic_repos) == true }}
    needs: [setup-branch, setup-matrix, js-translations]
    strategy:
      # using max-parallel to avoid git push/pull issues when running in parallel
      max-parallel: 1
      matrix:
        repository_config: ${{ fromJson(needs.setup-matrix.outputs.generic_repos) }}

    runs-on: ubuntu-latest
    continue-on-error: true

    steps:
      # Clones the openedx-translations repo
      - name: clone openedx/openedx-translations
        uses: actions/checkout@v3
        with:
          ref: ${{ needs.setup-branch.outputs.branch }}

      # Clones the  repository
      - name: clone ${{ github.repository_owner }}/${{ matrix.repository_config.repo }}
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/${{ matrix.repository_config.repo }}
          ref: ${{ github.event.inputs.ref }}
          path: translations/${{ matrix.repository_config.repo }}

      # Sets up Python
      - name: setup python
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'

      # Installs Python requirements from translations.txt
      - name: install requirements
        run: pip install -r requirements/translations.txt

      # Extracts the translation source files
      - name: extract translation source files
        run: |
          cd translations/${{ matrix.repository_config.repo }}
          make extract_translations

      # git adds only the translation source file transifex_input.yaml from
      # the repo root.
      - name: git add the translation source files
        id: add-sources
        run: |
          # set identity
          git config --global user.email "translations-bot@openedx.org"
          git config --global user.name "edx-transifex-bot"
          # Change directory to translations/${{ matrix.repository_config.repo }}
          cd translations/${{ matrix.repository_config.repo }}

          # remove translations/${{ matrix.repository_config.repo }}/.git so we don't commit a submodule
          rm -rf .git

          # stage the transifex_input.yaml file generated by make, force it in
          # case the file is in .gitignore (it should be)
          git add ${{ matrix.repository_config.transifex_file_path }} -f -v
          # Check the git status of the translation source files
          echo "GIT_STATUS=$(git status ${{ matrix.repository_config.transifex_file_path }} -s | wc -l)" >> $GITHUB_ENV

      # Attempts to commit the translation source files if there is a difference
      - name: git commit the translation source files
        if: "${{ env.GIT_STATUS > 0 }}"
        run: |
          # commit the changes
          git commit -m "chore: add extracted translation source files from ${{ matrix.repository_config.repo }}"
          # push changes to branch
          git push

  merge-translations:
    runs-on: ubuntu-latest
    needs: [setup-branch, python-translations, js-translations, generic-translations]

    steps:
      # Clones the openedx-translations repo on the automated/extract-translation-source-files-# branch
      - name: clone openedx/openedx-translations
        uses: actions/checkout@v3
        with:
          ref: ${{ needs.setup-branch.outputs.branch }}
          fetch-depth: 0

      # Create a pull request
      - name: create pull request
        id: createPR
        env:
          # This token requires Write access to the openedx-translations repo
          GITHUB_TOKEN: ${{ secrets.EDX_TRANSIFEX_BOT_GITHUB_TOKEN }}
        run: |
          echo "COMMIT_COUNT=$(git rev-list HEAD ^origin/main | wc -l)" >> $GITHUB_ENV
          echo "PR_URL=$(gh pr create --title 'chore: add updated translation source files' --body 'This PR is auto-generated by [extract-translation-source-files](https://github.com/openedx/openedx-translations/blob/master/.github/workflows/extract-translation-source-files.yml).')" >> $GITHUB_ENV

      # Merges the pull request
      - name: merge pull request
        id: mergePR
        env:
          # This token requires Write access to the openedx-translations repo
          GITHUB_TOKEN: ${{ secrets.EDX_TRANSIFEX_BOT_GITHUB_TOKEN }}
        if: "${{ env.COMMIT_COUNT > 0 }}"
        run: gh pr merge ${{ env.PR_URL }} --merge --auto

      # Notify that branch did not merge because there were no new commits in the branch
      # and delete the branch now that it is unnecessary
      - name: notify of empty branch and delete branch
        if: (steps.mergePR.outcome == 'skipped')
        run: |
          echo "The branch was not merged because the branch had 0 commits."
          git push origin -d ${{ needs.setup-branch.outputs.branch }}