diff --git a/.github/workflows/fix-transifex-resource-names.yml b/.github/workflows/fix-transifex-resource-names.yml index 2042ae2119e..e6e72d1aa2b 100644 --- a/.github/workflows/fix-transifex-resource-names.yml +++ b/.github/workflows/fix-transifex-resource-names.yml @@ -9,6 +9,8 @@ on: paths: - 'transifex.yml' - '.github/workflows/extract-translation-source-files.yml' + schedule: # Also run monthly just in case there's a stall slug/name update + - cron: '0 0 1 * *' jobs: python-translations: @@ -28,7 +30,7 @@ jobs: # Run the script - name: Fix transifex automatic resource names env: - TRANSIFEX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }} + TRANSIFEX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }} run: | make transifex_resources_requirements make fix_transifex_resource_names diff --git a/.gitignore b/.gitignore index 6da04169ce6..22935c244ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ __pycache__ .pytest_cache +*,cover +.coverage +htmlcov # msgfmt sometimes leaves these behind in the root directory /*.mo diff --git a/Makefile b/Makefile index a9990de4783..f5c0b5c01cf 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ test_requirements: ## Installs test.txt requirements pip install -q -r requirements/test.txt test: ## Run scripts tests - pytest -v -s scripts/tests + pytest -v -s --cov=. --cov-report=term-missing --cov-report=html scripts/tests validate_translation_files: ## Run basic validation to ensure files are compilable find translations/ -name '*.po' \ diff --git a/requirements/pip.txt b/requirements/pip.txt index f36085a07d6..da0741c57b7 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.41.1 +wheel==0.41.2 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==23.2.1 # via -r requirements/pip.in -setuptools==68.1.2 +setuptools==68.2.0 # via -r requirements/pip.in diff --git a/requirements/pip_tools.txt b/requirements/pip_tools.txt index c38c47670b6..3a6a0e2404e 100644 --- a/requirements/pip_tools.txt +++ b/requirements/pip_tools.txt @@ -4,10 +4,12 @@ # # make upgrade # -build==0.10.0 +build==1.0.3 # via pip-tools click==8.1.7 # via pip-tools +importlib-metadata==6.8.0 + # via build packaging==23.1 # via build pip-tools==7.3.0 @@ -19,8 +21,10 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.41.1 +wheel==0.41.2 # via pip-tools +zipp==3.16.2 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/test.in b/requirements/test.in index 216f53e0079..35350a6e598 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -2,4 +2,5 @@ -r transifex.txt pytest +pytest-cov diff --git a/requirements/test.txt b/requirements/test.txt index 65bb8ad9d5a..10d0cb9ec89 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -asttokens==2.2.1 +asttokens==2.4.0 # via # -r requirements/transifex.txt # transifex-python @@ -20,6 +20,8 @@ click==8.1.7 # via # -r requirements/transifex.txt # transifex-python +coverage[toml]==7.3.1 + # via pytest-cov exceptiongroup==1.1.3 # via pytest future==0.18.3 @@ -30,7 +32,7 @@ gitdb==4.0.10 # via # -r requirements/transifex.txt # gitpython -gitpython==3.1.32 +gitpython==3.1.35 # via # -r requirements/transifex.txt # transifex-client @@ -46,19 +48,23 @@ parsimonious==0.10.0 # via # -r requirements/transifex.txt # pyseeyou -pluggy==1.2.0 +pluggy==1.3.0 # via pytest pyseeyou==1.0.2 # via # -r requirements/transifex.txt # transifex-python -pytest==7.4.0 +pytest==7.4.2 + # via + # -r requirements/test.in + # pytest-cov +pytest-cov==4.1.0 # via -r requirements/test.in python-slugify==4.0.1 # via # -r requirements/transifex.txt # transifex-client -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/transifex.txt # transifex-python @@ -85,7 +91,9 @@ text-unidecode==1.3 # -r requirements/transifex.txt # python-slugify tomli==2.0.1 - # via pytest + # via + # coverage + # pytest toolz==0.12.0 # via # -r requirements/transifex.txt diff --git a/requirements/transifex.txt b/requirements/transifex.txt index 3be9a78f5ee..fc27438235b 100644 --- a/requirements/transifex.txt +++ b/requirements/transifex.txt @@ -4,7 +4,7 @@ # # make upgrade # -asttokens==2.2.1 +asttokens==2.4.0 # via transifex-python certifi==2023.7.22 # via requests @@ -16,7 +16,7 @@ future==0.18.3 # via pyseeyou gitdb==4.0.10 # via gitpython -gitpython==3.1.32 +gitpython==3.1.35 # via transifex-client idna==3.4 # via requests @@ -26,7 +26,7 @@ pyseeyou==1.0.2 # via transifex-python python-slugify==4.0.1 # via transifex-client -pytz==2023.3 +pytz==2023.3.post1 # via transifex-python regex==2023.8.8 # via parsimonious diff --git a/requirements/translations.txt b/requirements/translations.txt index a614d1c6dbd..52803c8c9b6 100644 --- a/requirements/translations.txt +++ b/requirements/translations.txt @@ -6,7 +6,7 @@ # asgiref==3.7.2 # via django -django==3.2.20 +django==3.2.21 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # edx-i18n-tools @@ -16,7 +16,7 @@ path==16.7.1 # via edx-i18n-tools polib==1.2.0 # via edx-i18n-tools -pytz==2023.3 +pytz==2023.3.post1 # via django pyyaml==6.0.1 # via edx-i18n-tools diff --git a/scripts/fix_transifex_resource_names.py b/scripts/fix_transifex_resource_names.py index 5d260a2550c..fe2049cb408 100644 --- a/scripts/fix_transifex_resource_names.py +++ b/scripts/fix_transifex_resource_names.py @@ -1,10 +1,10 @@ """ -Set the openedx-translations to readable resource names: +Set the openedx-translations to readable resource names and slugs: Run via `$ make fix_transifex_resource_names`. -Transifex sets resource slug names to a long name which makes it unreadable by translators .e.g. +Transifex sets resource slug names and slugs to a long name which makes it unreadable by translators .e.g. - "translations..frontend-app-something..src-i18n-transifex-input--main" This script infer the resource name in two ways: @@ -14,7 +14,12 @@ 2) If no usable slug is available, infer the resource name from the categories e.g. - ["github#repository:openedx/openedx-translations#branch:main#path:translations/my-xblock/openassessment/conf/locale/en/LC_MESSAGES/djangojs.po"] - would result in "my-xblock" as resource name. + would result in "my-xblock-js" as resource name. + +Slugs are even worse, sometimes they're also the lengthy while other times they're just hashes e.g. + - "b8933764bdb3063ca09d6aa20341102f" + +This script updates slugs to be like names. """ import configparser @@ -23,9 +28,17 @@ from os import getenv from os.path import expanduser +from slugify import slugify from transifex.api import transifex_api +def is_dry_run(): + """ + Check if the script is running in debug mode. + """ + return '--dry-run' in sys.argv + + def get_transifex_project(): """ Get openedx-translations project from Transifex. @@ -48,14 +61,7 @@ def get_transifex_project(): return openedx_org.fetch('projects').get(slug='openedx-translations') -def get_repo_name_from_resource(resource): - if resource.slug.startswith('translations-'): - # example resource.slug = ( - # "translations-my-xblock-conf-locale-en-lc-messages-django-po--main" - # ) - new_name = re.sub(r'(^translations-|-src-i18n-.*--main$|-conf-locale-.*--main)', '', resource.slug) - return new_name - +def get_repo_slug_from_resource(resource): if resource.categories: github_repo_categories = [ category for category in resource.categories if 'github#repository' in category @@ -70,7 +76,23 @@ def get_repo_name_from_resource(resource): if '#path:translations/' in github_repo_info: path_name = github_repo_info.split('#path:translations/')[1] directory_name = path_name.split('/')[0] - return directory_name + if github_repo_info.endswith('js.po'): + return slugify(f'{directory_name}-js') + else: + return slugify(directory_name) + + if resource.slug.startswith('translations-'): + # example resource.slug: + # - "translations-my-xblock-conf-locale-en-lc-messages-django-po--main" + # - "translations-frontend-app-library-authoring-src-i18n-transifex-input-json--main" + # + results = re.search(r'translations-(.*)-(conf-locale|src-i18n)', resource.slug) + if results: + if new_name := results.group(1): + if 'djangojs-po' in resource.slug: + return f'{new_name}-js' + else: + return new_name def main(argv): @@ -79,31 +101,48 @@ def main(argv): print(__doc__) return - print('Updating openedx-translations project resource names:') + print('Updating openedx-translations project resource and slug names:') openedx_translations_proj = get_transifex_project() for resource in openedx_translations_proj.fetch('resources'): + print('------------') + print('Updating:') + print('Resource id:', resource.id) + print('Resource slug:', resource.slug) + print('Resource name:', resource.name) + print('Resource categories:', ', '.join(resource.categories)) + + new_name = get_repo_slug_from_resource(resource) + new_slug = get_repo_slug_from_resource(resource) + if resource.name.startswith('translations..'): - print('------------') - print('Updating:') - print('Resource id:', resource.id) - print('Resource slug:', resource.slug) - print('Resource name:', resource.name) - print('Resource categories:', ', '.join(resource.categories)) - - new_name = get_repo_name_from_resource(resource) - if new_name: - if '--dry-run' in argv: - print('Saving new name (dry run):', new_name, '\n') + if new_name and resource.name != new_name: + resource.name = new_name + if is_dry_run(): + print(f'\n### Saving new name "{new_name}" (dry-run) ###', '\n') else: - print('Saving new name:', new_name, '\n') - resource.name = new_name + print(f'\n### Saving new name "{new_name}" ###', '\n') resource.save('name') else: - print(f'Error: Unrecognized slug pattern "{resource.slug}" or categories "{resource.categories}" ' - f'to infer resource name from.') + print(f'Error: Unrecognized slug pattern or categories to infer resource resource name from.') + + if re.match('^[a-z0-9]{32}$', resource.slug) or resource.slug.startswith('translations-'): + if new_slug and resource.slug != new_slug: + resource.slug = new_slug + if is_dry_run(): + print(f'\n### Saving new slug "{new_slug}" (dry-run) ###', '\n') + else: + print(f'\n### Saving new slug "{new_slug}" ###', '\n') + try: + resource.save('slug') + except Exception as e: + # Slug is unique, so if it already exists, we get an error. + print(f'Error: {e}') + else: + print(f'Error: Unrecognized slug pattern or categories to infer resource slug from.') + else: - print(f'Skipping: "{resource.name}" because it seems to have a proper name (id={resource.id})') + print(f'Skipping: "{resource.name}" because it seems to have proper attributes') if __name__ == '__main__': diff --git a/scripts/tests/test_fix_transifex_resource_names.py b/scripts/tests/test_fix_transifex_resource_names.py index 1846c97d918..159ff999ee7 100644 --- a/scripts/tests/test_fix_transifex_resource_names.py +++ b/scripts/tests/test_fix_transifex_resource_names.py @@ -3,35 +3,49 @@ """ import unittest from unittest.mock import MagicMock -from ..fix_transifex_resource_names import get_repo_name_from_resource +from ..fix_transifex_resource_names import get_repo_slug_from_resource -def test_get_repo_name_from_resource_with_no_categories(): +def test_get_repo_slug_from_resource_with_no_categories(): resource = MagicMock() resource.slug = 'translations-my-xblock-conf-locale-en-lc-messages-django-po--main' resource.categories = [] - assert get_repo_name_from_resource(resource) == 'my-xblock' + assert get_repo_slug_from_resource(resource) == 'my-xblock' + + +def test_get_repo_slug_from_resource_slug_js_with_no_categories(): + resource = MagicMock() + resource.slug = 'translations-my-xblock-conf-locale-en-lc-messages-djangojs-po--main' + resource.categories = [] + assert get_repo_slug_from_resource(resource) == 'my-xblock-js' def test_get_repo_name_from_invalid_slug(): resource = MagicMock() resource.slug = 'some-gibberish-slug' resource.categories = [] - assert get_repo_name_from_resource(resource) == None + assert get_repo_slug_from_resource(resource) == None def test_get_repo_name_from_slug_and_categories(): """ - Slug takes precedence over categories. + Categories takes precedence over slug. """ resource = MagicMock() resource.slug = 'translations-my-xblock1-conf-locale-en-lc-messages-django-po--main' - resource.categories = ['github#repository:openedx/openedx-translations#branch:main#path:translations/my-xblock2/openassessment/conf/locale/en/LC_MESSAGES/djangojs.po'] # noqa - assert get_repo_name_from_resource(resource) == 'my-xblock1' + resource.categories = ['github#repository:openedx/openedx-translations#branch:main#path:translations/my-xblock2/openassessment/conf/locale/en/LC_MESSAGES/django.po'] # noqa + assert get_repo_slug_from_resource(resource) == 'my-xblock2' def test_get_repo_name_from_categories(): + resource = MagicMock() + resource.slug = 'some-gibberish-slug' + resource.categories = ['github#repository:openedx/openedx-translations#branch:main#path:translations/my-xblock2/openassessment/conf/locale/en/LC_MESSAGES/django.po'] # noqa + assert get_repo_slug_from_resource(resource) == 'my-xblock2' + + +def test_get_repo_name_from_categories_with_js(): resource = MagicMock() resource.slug = 'some-gibberish-slug' resource.categories = ['github#repository:openedx/openedx-translations#branch:main#path:translations/my-xblock2/openassessment/conf/locale/en/LC_MESSAGES/djangojs.po'] # noqa - assert get_repo_name_from_resource(resource) == 'my-xblock2' + assert get_repo_slug_from_resource(resource) == 'my-xblock2-js'