From 567c216fb9ab8b890e65e9377a0d3bf48e671244 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Sun, 20 Aug 2023 15:37:19 +0300 Subject: [PATCH] feat: sync from old transifex project This is a on-demand GitHub Actions workflow which will sync traslations and their status from the `open-edx/edx-platform` Transifex project (old) into `open-edx/openedx-translations` OEP-58 project (new). Refs: FC-0012 OEP-58 --- .github/workflows/sync-translations.yml | 141 +++++++++++++++ Makefile | 19 ++- requirements/sync.in | 2 + requirements/sync.txt | 8 + scripts/sync_translations.py | 217 ++++++++++++++++++++++++ 5 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sync-translations.yml create mode 100644 requirements/sync.in create mode 100644 requirements/sync.txt create mode 100644 scripts/sync_translations.py diff --git a/.github/workflows/sync-translations.yml b/.github/workflows/sync-translations.yml new file mode 100644 index 00000000000..039b1f15e0e --- /dev/null +++ b/.github/workflows/sync-translations.yml @@ -0,0 +1,141 @@ +name: Migrate translations from the old Transifex project + +on: +# schedule: # TODO: Maybe before merge +# # https://crontab.guru/#0_0_*_*_1 +# # At midnight on Mondays +# - cron: 0 0 * * 1" + workflow_dispatch: +# push: +# branches: [sync_translations] # TODO: Remove before merge + +jobs: + migrate-translations: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + batch: + - new_slug: edx-ora2 + old_slug: openassessment + old_project_slug: edx-platform + + - new_slug: edx-ora2-js + old_slug: openassessment-js + old_project_slug: edx-platform + + - new_slug: edx-proctoring + old_slug: edx-proctoring + old_project_slug: edx-platform + + - new_slug: studio-frontend + old_slug: studio-frontend + old_project_slug: edx-platform + + - new_slug: donexblock + old_slug: xblock-done + old_project_slug: xblocks + + - new_slug: xblock-drag-and-drop-v2 + old_slug: drag-and-drop-v2 + old_project_slug: xblocks + + - new_slug: xblock-free-text-response + old_slug: xblock-free-text-response + old_project_slug: xblocks + + - new_slug: course-discovery + old_slug: course_discovery + old_project_slug: edx-platform + + - new_slug: course-discovery + old_slug: course_discovery + old_project_slug: edx-platform + + - new_slug: course-discovery-js + old_slug: course_discovery-js + old_project_slug: edx-platform + + - new_slug: credentials-js + old_slug: credentials-js + old_project_slug: edx-platform + + - new_slug: credentials + old_slug: credentials + old_project_slug: edx-platform + + - new_slug: frontend-app-account + old_slug: frontend-app-account + old_project_slug: edx-platform + + - new_slug: frontend-app-authn + old_slug: frontend-app-authn + old_project_slug: edx-platform + + - new_slug: frontend-app-course-authoring + old_slug: frontend-app-course-authoring + old_project_slug: edx-platform + + - new_slug: frontend-app-discussions + old_slug: frontend-app-discussions + old_project_slug: edx-platform + + - new_slug: frontend-app-ecommerce + old_slug: frontend-app-ecommerce + old_project_slug: edx-platform + + - new_slug: frontend-app-gradebook + old_slug: frontend-app-gradebook + old_project_slug: edx-platform + + - new_slug: frontend-app-learner-dashboard + old_slug: frontend-app-learner-dashboard + old_project_slug: edx-platform + + - new_slug: frontend-app-learner-record + old_slug: frontend-app-learner-record + old_project_slug: edx-platform + + - new_slug: frontend-app-learning + old_slug: frontend-app-learning + old_project_slug: edx-platform + + - new_slug: frontend-app-profile + old_slug: frontend-app-profile + old_project_slug: edx-platform + + - new_slug: frontend-app-program-console + old_slug: frontend-app-program-manager + old_project_slug: edx-platform + + - new_slug: frontend-component-footer + old_slug: frontend-component-footer-edx + old_project_slug: edx-platform + + - new_slug: frontend-component-header + old_slug: frontend-component-header + old_project_slug: edx-platform + + - new_slug: paragon + old_slug: paragon + old_project_slug: edx-platform + + +# if: ${{ !github.event.inputs.resource || github.event.inputs.resource == matrix.new_resource_slug }} + steps: + - uses: actions/checkout@v3 + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install Python dependencies + run: make sync_requirements + + - name: Sync + env: + # `TX_LANGUAGES` list of languages is set in the `Makefile` + TX_NEW_SLUG: ${{ matrix.batch.new_slug }} + TX_OLD_SLUG: ${{ matrix.batch.old_slug }} + TX_OLD_PROJECT_SLUG: ${{ matrix.batch.old_project_slug }} + TX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }} + run: make sync_translations diff --git a/Makefile b/Makefile index f5c0b5c01cf..4fb849634c2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,10 @@ -.PHONY: piptools upgrade fix_transifex_resource_names transifex_resources_requirements validate_translation_files +.PHONY: piptools upgrade fix_transifex_resource_names transifex_resources_requirements validate_translation_files \ +sync_translations sync_translations_github_workflow + + +# Default languages for the sync_translations.py file +export TX_LANGUAGES := ar,de,fr_CA + piptools: pip install -q -r requirements/pip_tools.txt @@ -12,6 +18,7 @@ upgrade: piptools ## update the requirements/*.txt files with the latest packag pip-compile --rebuild --upgrade -o requirements/translations.txt requirements/translations.in pip-compile --rebuild --upgrade -o requirements/transifex.txt requirements/transifex.in pip-compile --rebuild --upgrade -o requirements/test.txt requirements/test.in + pip-compile --rebuild --upgrade -o requirements/sync.txt requirements/sync.in transifex_resources_requirements: ## Installs the requirements file @@ -37,3 +44,13 @@ validate_translation_files: ## Run basic validation to ensure files are compila @echo '-----------------------------------------' @echo 'Congratulations! Translation files are valid.' @echo '-----------------------------------------' + +sync_requirements: ## install sync.txt requirements + pip install -q -r requirements/sync.txt + +sync_translations: ## Syncs from the old projects to the new openedx-translations project + python scripts/sync_translations.py $(SYNC_ARGS) + +sync_translations_github_workflow: ## Run with parameters from .github/workflows/sync-translations.yml + make SYNC_ARGS="--simulate-github-workflow $(SYNC_ARGS)" sync_translations + diff --git a/requirements/sync.in b/requirements/sync.in new file mode 100644 index 00000000000..773d70cc460 --- /dev/null +++ b/requirements/sync.in @@ -0,0 +1,2 @@ +# requirements for sync_translations script +PyYAML diff --git a/requirements/sync.txt b/requirements/sync.txt new file mode 100644 index 00000000000..d2b30601a28 --- /dev/null +++ b/requirements/sync.txt @@ -0,0 +1,8 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/sync.txt requirements/sync.in +# +pyyaml==6.0.1 + # via -r requirements/sync.in diff --git a/scripts/sync_translations.py b/scripts/sync_translations.py new file mode 100644 index 00000000000..c43d9989407 --- /dev/null +++ b/scripts/sync_translations.py @@ -0,0 +1,217 @@ +import configparser +import os +import sys +from os.path import expanduser +import yaml + +from transifex.api import transifex_api +from transifex.api.jsonapi import exceptions + +NEW_PROJECT_SLUG = 'openedx-translations' +ORGANIZATION_SLUG = 'open-edx' + + +class Command: + + workflow_file_path = '.github/workflows/sync-translations.yml' + + def __init__(self, argv, tx_api, environ): + self.argv = argv + self.tx_api = tx_api + self.environ = environ + + def is_dry_run(self): + """ + Check if the script is running in dry-run mode. + """ + return '--dry-run' in self.argv + + def is_simulated_github_actions(self): + """ + Check if the script is running in simulated GitHub Actions mode. + """ + return '--simulate-github-workflow' in self.argv + + def get_resource_url(self, resource, project_slug): + return f'https://www.transifex.com/{ORGANIZATION_SLUG}/{project_slug}/{resource.slug}' + + def get_transifex_organization_projects(self): + """ + Get openedx-translations project from Transifex. + """ + tx_api_token = self.environ.get('TX_API_TOKEN') + if not tx_api_token: + config = configparser.ConfigParser() + config.read(expanduser('~/.transifexrc')) + tx_api_token = config['https://www.transifex.com']['password'] + + if not tx_api_token: + raise Exception( + 'Error: No auth token found. ' + 'Set transifex API token via TX_API_TOKEN environment variable or via the ~/.transifexrc file.' + ) + + self.tx_api.setup(auth=tx_api_token) + return self.tx_api.Organization.get(slug=ORGANIZATION_SLUG).fetch('projects') + + def get_resources_pair(self, new_slug, old_slug, old_project_slug): + """ + Load the old and new Transifex resources pair. + """ + projects = self.get_transifex_organization_projects() + new_project = projects.get(slug=NEW_PROJECT_SLUG) + + new_resource_id = f'o:{ORGANIZATION_SLUG}:p:{new_project.slug}:r:{new_slug}' + print(f'new resource id: {new_resource_id}') + try: + new_resource = self.tx_api.Resource.get(id=new_resource_id) + except exceptions.JsonApiException as error: + print(f'Error: New resource error: {new_resource_id}. Error: {error}') + sys.exit(1) + + old_resource_id = f'o:{ORGANIZATION_SLUG}:p:{old_project_slug}:r:{old_slug}' + print(f'old resource id: {old_resource_id}') + try: + old_resource = self.tx_api.Resource.get(id=old_resource_id) + except exceptions.JsonApiException as error: + print(f'Error: Old resource error: {new_resource_id}. Error: {error}') + sys.exit(1) + + return { + 'old_resource': old_resource, + 'new_resource': new_resource, + } + + def get_translations(self, language_code, resource): + """ + Get a list of translations for a given language and resource. + """ + language = self.tx_api.Language.get(code=language_code) + translations = self.tx_api.ResourceTranslation. \ + filter(resource=resource, language=language). \ + include('resource_string') + + return translations.all() + + def sync_translations(self, language_code, old_resource, new_resource): + """ + Sync specific language translations into the new Transifex resource. + """ + print(' syncing', language_code, '...') + old_translations = { + self.get_translation_id(translation): translation + for translation in self.get_translations(language_code=language_code, resource=old_resource) + } + + for new_translation in self.get_translations(language_code=language_code, resource=new_resource): + translation_id = self.get_translation_id(new_translation) + if old_translation := old_translations.get(translation_id): + updates = {} + for attr in ['reviewed', 'proofread', 'strings']: + if old_attr_value := getattr(old_translation, attr, None): + if old_attr_value != getattr(new_translation, attr, None): + updates[attr] = old_attr_value + + if updates: + print(translation_id, updates) + + if not self.is_dry_run(): + new_translation.save(**updates) + + def sync_tags(self, old_resource, new_resource): + """ + Sync tags from the old Transifex resource into the new Transifex resource. This process is language independent. + """ + old_resource_str = self.tx_api.ResourceString.filter(resource=old_resource) + new_resource_str = self.tx_api.ResourceString.filter(resource=new_resource) + + old_quick_lookup = { + item['attributes']['string_hash']: item['attributes']['tags'] for item in old_resource_str.to_dict()['data'] + } + + for new_info in new_resource_str.all(): + old_tags = old_quick_lookup.get(new_info.string_hash) + new_tags = new_info.tags + + if old_tags is None: # in case of new changes are not synced yet + continue + if len(new_tags) == 0 and len(old_tags) == 0: # nothing to compare + continue + + if len(new_tags) != len(old_tags) or set(new_tags) != set(old_tags): + print(f' - found tag difference for {new_info.string_hash}. overwriting: {new_tags} with {old_tags}') + + if not self.is_dry_run(): + new_info.save(tags=old_tags) + + def get_translation_id(self, translation): + """ + Build a unique identifier for a translation entry. + """ + return f'context:{translation.resource_string.context}:key:{translation.resource_string.key}' + + def get_languages(self): + """ + Get a list of languages to sync translations for. + """ + return self.environ['TX_LANGUAGES'].split(',') + + def sync_pair_into_new_resource(self, new_slug, old_slug, old_project_slug): + """ + Sync translations from both the edx-platform and XBlock projects into the new openedx-translations project. + """ + languages = self.get_languages() + resource_pair = self.get_resources_pair(new_slug, old_slug, old_project_slug) + + print(f'Syncing {resource_pair["new_resource"].name} from {resource_pair["old_resource"].name}...') + print(f'Syncing: {languages}') + print(f' - from: {self.get_resource_url(resource_pair["old_resource"], old_project_slug)}') + print(f' - to: {self.get_resource_url(resource_pair["new_resource"], NEW_PROJECT_SLUG)}') + + for lang_code in languages: + self.sync_translations(language_code=lang_code, **resource_pair) + + print('Syncing tags...') + self.sync_tags(**resource_pair) + + print('-' * 80, '\n') + + def run_from_workflow_yaml_file(self, workflow_configs): + """ + Run the script from a GitHub Actions migrate-from-transifex-old-project.yml workflow file. + """ + pairs_list = workflow_configs['jobs']['migrate-translations']['strategy']['matrix']['batch'] + + print('Verifying existence of resource pairs...') + for pair in pairs_list: + self.get_resources_pair( + new_slug=pair['new_slug'], + old_slug=pair['old_slug'], + old_project_slug=pair['old_project_slug'], + ) + print('\n', '-' * 80, '\n') + + for pair in pairs_list: + self.sync_pair_into_new_resource( + new_slug=pair['new_slug'], + old_slug=pair['old_slug'], + old_project_slug=pair['old_project_slug'], + ) + + def run(self): + if self.is_simulated_github_actions(): + with open(self.workflow_file_path) as workflow_file: + self.run_from_workflow_yaml_file( + workflow_configs=yaml.safe_load(workflow_file.read()), + ) + else: + self.sync_pair_into_new_resource( + new_slug=self.environ['TX_NEW_SLUG'], + old_slug=self.environ['TX_OLD_SLUG'], + old_project_slug=self.environ['TX_OLD_PROJECT_SLUG'], + ) + + +if __name__ == '__main__': + command = Command(sys.argv, environ=os.environ, tx_api=transifex_api) + command.run()