Merge pull request #1006 from Zeit-Labs/fix-slug

feat: fix random and lengthy slugs and support `-js` suffix

.github/workflows/fix-transifex-resource-names.yml

@@ -9,6 +9,8 @@ on:
     paths:
       - 'transifex.yml'
       - '.github/workflows/extract-translation-source-files.yml'
+  schedule:  # Also run monthly just in case there's a stall slug/name update
+    - cron: '0 0 1 * *'
 
 jobs:
   python-translations:
@@ -28,7 +30,7 @@ jobs:
       # Run the script
       - name: Fix transifex automatic resource names
         env:
-          TRANSIFEX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }} 
+          TRANSIFEX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }}
         run: |
           make transifex_resources_requirements
           make fix_transifex_resource_names

.gitignore

@@ -1,5 +1,8 @@
 __pycache__
 .pytest_cache
+*,cover
+.coverage
+htmlcov
 
 # msgfmt sometimes leaves these behind in the root directory
 /*.mo

Makefile

@@ -28,7 +28,7 @@ test_requirements:  ## Installs test.txt requirements
 	pip install -q -r requirements/test.txt
 
 test:  ## Run scripts tests
-	 pytest -v -s scripts/tests
+	 pytest -v -s --cov=. --cov-report=term-missing --cov-report=html scripts/tests
 
 validate_translation_files:  ## Run basic validation to ensure files are compilable
 	find translations/ -name '*.po' \

requirements/pip.txt

@@ -4,11 +4,11 @@
 #
 #    make upgrade
 #
-wheel==0.41.1
+wheel==0.41.2
     # via -r requirements/pip.in
 
 # The following packages are considered to be unsafe in a requirements file:
 pip==23.2.1
     # via -r requirements/pip.in
-setuptools==68.1.2
+setuptools==68.2.0
     # via -r requirements/pip.in

requirements/pip_tools.txt

@@ -4,10 +4,12 @@
 #
 #    make upgrade
 #
-build==0.10.0
+build==1.0.3
     # via pip-tools
 click==8.1.7
     # via pip-tools
+importlib-metadata==6.8.0
+    # via build
 packaging==23.1
     # via build
 pip-tools==7.3.0
@@ -19,8 +21,10 @@ tomli==2.0.1
     #   build
     #   pip-tools
     #   pyproject-hooks
-wheel==0.41.1
+wheel==0.41.2
     # via pip-tools
+zipp==3.16.2
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/test.in

@@ -2,4 +2,5 @@
 -r transifex.txt
 
 pytest
+pytest-cov
 

requirements/test.txt

@@ -4,7 +4,7 @@
 #
 #    make upgrade
 #
-asttokens==2.2.1
+asttokens==2.4.0
     # via
     #   -r requirements/transifex.txt
     #   transifex-python
@@ -20,6 +20,8 @@ click==8.1.7
     # via
     #   -r requirements/transifex.txt
     #   transifex-python
+coverage[toml]==7.3.1
+    # via pytest-cov
 exceptiongroup==1.1.3
     # via pytest
 future==0.18.3
@@ -30,7 +32,7 @@ gitdb==4.0.10
     # via
     #   -r requirements/transifex.txt
     #   gitpython
-gitpython==3.1.32
+gitpython==3.1.35
     # via
     #   -r requirements/transifex.txt
     #   transifex-client
@@ -46,19 +48,23 @@ parsimonious==0.10.0
     # via
     #   -r requirements/transifex.txt
     #   pyseeyou
-pluggy==1.2.0
+pluggy==1.3.0
     # via pytest
 pyseeyou==1.0.2
     # via
     #   -r requirements/transifex.txt
     #   transifex-python
-pytest==7.4.0
+pytest==7.4.2
+    # via
+    #   -r requirements/test.in
+    #   pytest-cov
+pytest-cov==4.1.0
     # via -r requirements/test.in
 python-slugify==4.0.1
     # via
     #   -r requirements/transifex.txt
     #   transifex-client
-pytz==2023.3
+pytz==2023.3.post1
     # via
     #   -r requirements/transifex.txt
     #   transifex-python
@@ -85,7 +91,9 @@ text-unidecode==1.3
     #   -r requirements/transifex.txt
     #   python-slugify
 tomli==2.0.1
-    # via pytest
+    # via
+    #   coverage
+    #   pytest
 toolz==0.12.0
     # via
     #   -r requirements/transifex.txt

requirements/transifex.txt

@@ -4,7 +4,7 @@
 #
 #    make upgrade
 #
-asttokens==2.2.1
+asttokens==2.4.0
     # via transifex-python
 certifi==2023.7.22
     # via requests
@@ -16,7 +16,7 @@ future==0.18.3
     # via pyseeyou
 gitdb==4.0.10
     # via gitpython
-gitpython==3.1.32
+gitpython==3.1.35
     # via transifex-client
 idna==3.4
     # via requests
@@ -26,7 +26,7 @@ pyseeyou==1.0.2
     # via transifex-python
 python-slugify==4.0.1
     # via transifex-client
-pytz==2023.3
+pytz==2023.3.post1
     # via transifex-python
 regex==2023.8.8
     # via parsimonious

requirements/translations.txt

@@ -6,7 +6,7 @@
 #
 asgiref==3.7.2
     # via django
-django==3.2.20
+django==3.2.21
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
     #   edx-i18n-tools
@@ -16,7 +16,7 @@ path==16.7.1
     # via edx-i18n-tools
 polib==1.2.0
     # via edx-i18n-tools
-pytz==2023.3
+pytz==2023.3.post1
     # via django
 pyyaml==6.0.1
     # via edx-i18n-tools

scripts/fix_transifex_resource_names.py

@@ -1,10 +1,10 @@
 """
-Set the openedx-translations to readable resource names:
+Set the openedx-translations to readable resource names and slugs:
 
 Run via `$ make fix_transifex_resource_names`.
 
 
-Transifex sets resource slug names to a long name which makes it unreadable by translators .e.g.
+Transifex sets resource slug names and slugs to a long name which makes it unreadable by translators .e.g.
  - "translations..frontend-app-something..src-i18n-transifex-input--main"
 
 This script infer the resource name in two ways:
@@ -14,7 +14,12 @@
 
 2) If no usable slug is available, infer the resource name from the categories e.g.
  - ["github#repository:openedx/openedx-translations#branch:main#path:translations/my-xblock/openassessment/conf/locale/en/LC_MESSAGES/djangojs.po"]
-   would result in "my-xblock" as resource name.
+   would result in "my-xblock-js" as resource name.
+
+Slugs are even worse, sometimes they're also the lengthy while other times they're just hashes e.g.
+ - "b8933764bdb3063ca09d6aa20341102f"
+
+This script updates slugs to be like names.
 """
 
 import configparser
@@ -23,9 +28,17 @@
 from os import getenv
 from os.path import expanduser
 
+from slugify import slugify
 from transifex.api import transifex_api
 
 
+def is_dry_run():
+    """
+    Check if the script is running in debug mode.
+    """
+    return '--dry-run' in sys.argv
+
+
 def get_transifex_project():
     """
     Get openedx-translations project from Transifex.
@@ -48,14 +61,7 @@ def get_transifex_project():
     return openedx_org.fetch('projects').get(slug='openedx-translations')
 
 
-def get_repo_name_from_resource(resource):
-    if resource.slug.startswith('translations-'):
-        # example resource.slug = (
-        #     "translations-my-xblock-conf-locale-en-lc-messages-django-po--main"
-        # )
-        new_name = re.sub(r'(^translations-|-src-i18n-.*--main$|-conf-locale-.*--main)', '', resource.slug)
-        return new_name
-
+def get_repo_slug_from_resource(resource):
     if resource.categories:
         github_repo_categories = [
             category for category in resource.categories if 'github#repository' in category
@@ -70,7 +76,23 @@ def get_repo_name_from_resource(resource):
             if '#path:translations/' in github_repo_info:
                 path_name = github_repo_info.split('#path:translations/')[1]
                 directory_name = path_name.split('/')[0]
-                return directory_name
+                if github_repo_info.endswith('js.po'):
+                    return slugify(f'{directory_name}-js')
+                else:
+                    return slugify(directory_name)
+
+    if resource.slug.startswith('translations-'):
+        # example resource.slug:
+        # - "translations-my-xblock-conf-locale-en-lc-messages-django-po--main"
+        # - "translations-frontend-app-library-authoring-src-i18n-transifex-input-json--main"
+        #
+        results = re.search(r'translations-(.*)-(conf-locale|src-i18n)', resource.slug)
+        if results:
+            if new_name := results.group(1):
+                if 'djangojs-po' in resource.slug:
+                    return f'{new_name}-js'
+                else:
+                    return new_name
 
 
 def main(argv):
@@ -79,31 +101,48 @@ def main(argv):
         print(__doc__)
         return
 
-    print('Updating openedx-translations project resource names:')
+    print('Updating openedx-translations project resource and slug names:')
 
     openedx_translations_proj = get_transifex_project()
     for resource in openedx_translations_proj.fetch('resources'):
+        print('------------')
+        print('Updating:')
+        print('Resource id:', resource.id)
+        print('Resource slug:', resource.slug)
+        print('Resource name:', resource.name)
+        print('Resource categories:', ', '.join(resource.categories))
+
+        new_name = get_repo_slug_from_resource(resource)
+        new_slug = get_repo_slug_from_resource(resource)
+
         if resource.name.startswith('translations..'):
-            print('------------')
-            print('Updating:')
-            print('Resource id:', resource.id)
-            print('Resource slug:', resource.slug)
-            print('Resource name:', resource.name)
-            print('Resource categories:', ', '.join(resource.categories))
-
-            new_name = get_repo_name_from_resource(resource)
-            if new_name:
-                if '--dry-run' in argv:
-                    print('Saving new name (dry run):', new_name, '\n')
+            if new_name and resource.name != new_name:
+                resource.name = new_name
+                if is_dry_run():
+                    print(f'\n### Saving new name "{new_name}" (dry-run) ###', '\n')
                 else:
-                    print('Saving new name:', new_name, '\n')
-                    resource.name = new_name
+                    print(f'\n### Saving new name "{new_name}" ###', '\n')
                     resource.save('name')
             else:
-                print(f'Error: Unrecognized slug pattern "{resource.slug}" or categories "{resource.categories}" '
-                      f'to infer resource name from.')
+                print(f'Error: Unrecognized slug pattern or categories to infer resource resource name from.')
+
+        if re.match('^[a-z0-9]{32}$', resource.slug) or resource.slug.startswith('translations-'):
+            if new_slug and resource.slug != new_slug:
+                resource.slug = new_slug
+                if is_dry_run():
+                    print(f'\n### Saving new slug "{new_slug}" (dry-run) ###', '\n')
+                else:
+                    print(f'\n### Saving new slug "{new_slug}" ###', '\n')
+                    try:
+                        resource.save('slug')
+                    except Exception as e:
+                        # Slug is unique, so if it already exists, we get an error.
+                        print(f'Error: {e}')
+            else:
+                print(f'Error: Unrecognized slug pattern or categories to infer resource slug from.')
+
         else:
-            print(f'Skipping: "{resource.name}" because it seems to have a proper name (id={resource.id})')
+            print(f'Skipping: "{resource.name}" because it seems to have proper attributes')
 
 
 if __name__ == '__main__':