cinnamon-spices-makepot: Improve runtime speed with concurrency (#666)

Co-authored-by: Gr3qi <[email protected]>

cinnamon-spices-makepot

@@ -6,7 +6,12 @@ import os
 import subprocess
 import sys
 
+from concurrent.futures import ProcessPoolExecutor
 from glob import glob
+from pathlib import Path
+from typing import Any, List, Literal, Optional, Tuple
+
+CapturedOutput = List[Tuple[Literal['stdout', 'stderr'], str]]
 
 
 def parse_args():
@@ -32,14 +37,20 @@ def parse_args():
     if args.uuid and args.uuid.endswith('/'):
         args.uuid = args.uuid.replace('/', '')
     if args.all and not args.uuid:
-        for file_path in os.listdir("."):
-            if os.path.isdir(file_path) and not file_path.startswith("."):
-                if args.install:
-                    install_po(file_path, True)
-                elif args.remove:
-                    remove_po(file_path, True)
-                else:
-                    make_pot(file_path, True)
+        folders = [file_path for file_path in os.listdir(
+            '.') if os.path.isdir(file_path) and not file_path.startswith('.')]
+
+        if args.install:
+            for folder in folders:
+                install_po(folder, True)
+        elif args.remove:
+            for folder in folders:
+                remove_po(folder, True)
+        else:
+            with ProcessPoolExecutor() as executor:
+                for output in executor.map(make_pot, folders):
+                    print_output(output)
+
     elif args.install and args.remove:
         print('Only -i/--install OR -r/--remove may be specified. Not both.')
         sys.exit(1)
@@ -48,11 +59,55 @@ def parse_args():
     elif args.remove and args.uuid:
         remove_po(args.uuid)
     elif args.uuid:
-        make_pot(args.uuid)
+        output = make_pot(args.uuid)
+        print_output(output)
     else:
         parser.print_help()
 
 
+def print_output(output: CapturedOutput):
+    """
+    Replay the output of a command, preserving stdout/stderr order in the
+    output list
+    """
+    # Flush is needed or we won't print in the right order
+    for stream, content in output:
+        file = sys.stdout if stream == 'stdout' else sys.stderr
+        print(content, end='', file=file, flush=True)
+
+    # Add a newline to the end of the output
+    print()
+
+
+def get_command_output(cmd: List[str], cwd: Optional[str] = None, check: bool = True, stderr: Optional[int] = None) -> CapturedOutput:
+    """
+    Gather command output while preserving stdout/stderr order and distinction
+
+    Based on https://stackoverflow.com/a/56918582/14100077
+
+    @param cmd: The command to run
+    @param cwd: The working directory to run the command in
+    @param check: Passed to subprocess.run when calling it
+
+    @return: A list of tuples, where the first element is the stream
+             ('stdout' or 'stderr') and the second element is the content
+    """
+    try:
+        capture_output = stderr is None
+        result = subprocess.run(cmd, cwd=cwd, check=check, stderr=stderr,
+                                text=True, capture_output=capture_output)
+        output: CapturedOutput = []
+        if result.stdout:
+            output.extend(('stdout', line)
+                          for line in result.stdout.splitlines(keepends=True))
+        if result.stderr:
+            output.extend(('stderr', line)
+                          for line in result.stderr.splitlines(keepends=True))
+        return output
+    except subprocess.CalledProcessError as e:
+        return [('stderr', str(e))]
+
+
 def install_po(uuid: str, _all: bool = False):
     """
     Install translation files locally from the po directory of the UUID
@@ -64,13 +119,13 @@ def install_po(uuid: str, _all: bool = False):
         print(f'Translations not found for: {uuid}')
         if not _all:
             sys.exit(1)
-    home = os.path.expanduser("~")
+    home = os.path.expanduser('~')
     locale_inst = f'{home}/.local/share/locale'
     if 'po' in contents:
         po_dir = f'{uuid_path}/po'
         for file in os.listdir(po_dir):
             if file.endswith('.po'):
-                lang = file.split(".")[0]
+                lang = file.split('.')[0]
                 locale_dir = os.path.join(locale_inst, lang, 'LC_MESSAGES')
                 os.makedirs(locale_dir, mode=0o755, exist_ok=True)
                 subprocess.run(['msgfmt', '-c', os.path.join(po_dir, file),
@@ -82,7 +137,7 @@ def remove_po(uuid: str, _all: bool = False):
     """
     Remove local translation files for the UUID
     """
-    home = os.path.expanduser("~")
+    home = os.path.expanduser('~')
     locale_inst = f'{home}/.local/share/locale'
     uuid_mo_list = glob(f'{locale_inst}/**/{uuid}.mo', recursive=True)
     if not uuid_mo_list:
@@ -93,71 +148,110 @@ def remove_po(uuid: str, _all: bool = False):
         os.remove(uuid_mo_file)
 
 
-def make_pot(uuid: str, _all: bool = False):
+def process_po(path_to_po: str, uuid: str) -> CapturedOutput:
+    """
+    Process existing .po files and return the output
+
+    @param path_to_po: The path to the .po file to process
+    @param uuid: The UUID of the applet
+
+    @return: A list of tuples, where the first element is the stream
+             ('stdout' or 'stderr') and the second element is the content
+    """
+    po_path = Path(path_to_po)
+    po_dir = str(po_path.parent)
+    po_file = po_path.name
+    po_lang = po_path.stem
+
+    commands = [
+        ['msguniq', '-o', po_file, po_file],
+        ['intltool-update', '-g', uuid, '-d', po_lang],
+        ['msgattrib', '--no-obsolete', '-w', '79', '-o', po_file, po_file]
+    ]
+
+    output: CapturedOutput = [('stdout', f'{po_lang} ')]
+    for cmd in commands:
+        output.extend(get_command_output(cmd, cwd=po_dir))
+
+    return output
+
+
+def make_pot(uuid: str) -> CapturedOutput:
     """
     Make the translation template file for the UUID
     """
-    _pwd = sys.argv[1] if not _all else f'{os.getcwd()}/{uuid}'
-    _return_pwd = '../../..'
-    po_dir = f'{_pwd}/files/{uuid}/po'
-    uuid_dir = f'{_pwd}/files/{uuid}'
+
+    output: CapturedOutput = []
+
+    folder = Path(f'{os.getcwd()}/{uuid}')
+    output_dir = folder.joinpath(f'files/{uuid}')
+    po_dir = output_dir.joinpath('po')
+
+    # Prepare the pot file
     pot_file = uuid + '.pot'
-    outfile = os.path.join(po_dir, pot_file)
-    if os.path.exists(outfile):
-        os.remove(outfile)
-    elif not os.path.exists(po_dir):
-        os.mkdir(po_dir)
-    subprocess.run(["cinnamon-xlet-makepot", "-o", outfile, _pwd],
-                   check=True)
-    os.chdir(uuid_dir)
-    pot_path = f'po/{pot_file}'
-    os.chmod(pot_path, 0o0644)
-    glade_list = glob('**/*.glade', recursive=True)
-    glade_list += glob('**/*.ui', recursive=True)
+    pot_file_path = po_dir.joinpath(pot_file)
+
+    if pot_file_path.exists():
+        pot_file_path.unlink()
+    elif not po_dir.exists():
+        po_dir.mkdir()
+
+    output += [('stdout', f'\nProcessing translation files for: {uuid}\n')]
+    output += get_command_output(['cinnamon-xlet-makepot',
+                                  '-o', pot_file_path, folder],
+                                 check=True, cwd=folder)
+
+    if os.path.exists(pot_file_path):
+        pot_file_path.chmod(0o0644)
+
+    # Extract translatable strings from glade
+    glade_list = glob('**/*.glade', recursive=True, root_dir=output_dir)
+    glade_list += glob('**/*.ui', recursive=True, root_dir=output_dir)
     for glade_file in glade_list:
-        subprocess.run(["xgettext", "-jL", "Glade", "-o", pot_path,
-                       glade_file], check=True)
-    shell_list = glob('**/*.sh', recursive=True)
+        output += get_command_output(['xgettext', '-jL', 'Glade',
+                                      '-o', pot_file_path, glade_file],
+                                     check=True, cwd=output_dir)
+
+    # Extract translatable strings from shell scripts
+    shell_list = glob('**/*.sh', recursive=True, root_dir=output_dir)
     for shell_file in shell_list:
-        subprocess.run(["xgettext", "-jL", "Shell", "-o", pot_path,
-                       shell_file], stderr=subprocess.DEVNULL, check=True)
-    if not _all:
-        os.chdir(f'{_return_pwd}')
-    metadata_file = 'metadata.json' if _all else f'{uuid}/files/{uuid}/metadata.json'
+        output += get_command_output(['xgettext', '-jL', 'Shell',
+                                      '-o', pot_file_path, shell_file],
+                                     check=True, cwd=output_dir,
+                                     stderr=subprocess.DEVNULL)
+
+    # Get the metadata file
+    metadata_file = f'{uuid}/files/{uuid}/metadata.json'
     try:
         with open(metadata_file, encoding='utf-8') as meta:
-            metadata = json.load(meta)
+            metadata: dict[str, Any] = json.load(meta)
         version = str(metadata['version'])
     except (FileNotFoundError, KeyError):
-        print(f"{uuid}: metadata.json or version not found")
-        version = "1.0"
+        output.append(
+            ('stdout', f'{uuid}: metadata.json or version not found\n'))
+        version = '1.0'
+
+    # Update the pot file with the metadata
     address = 'https://github.com/linuxmint/cinnamon-spices-extensions/issues'
-    subprocess.run(["xgettext", "-w", "79", "--package-name", uuid,
-                    "--foreign-user", "--msgid-bugs-address", address,
-                    "--package-version", version, "-o", outfile, outfile],
-                   check=True)
-    glob_path = 'po/*.po' if _all else f'{uuid}/**/*.po'
+    output.extend(get_command_output(['xgettext', '-w', '79', '--foreign-user',
+                                      '--package-name', uuid,
+                                      '--msgid-bugs-address', address,
+                                      '--package-version', version,
+                                      '-o', pot_file_path, pot_file_path],
+                                     check=True))
+
+    # Process the po files
+    glob_path = f'{output_dir}/**/*.po'
     po_list = glob(glob_path, recursive=True)
     for po_file in po_list:
         os.chmod(po_file, 0o0644)
 
-    for po_file in po_list:
-        *po_dir_list, po_ext = po_file.split('/')
-        po_dir_str = '/'.join(po_dir_list)
-        if not os.getcwd().endswith(po_dir_str):
-            os.chdir(po_dir_str)
-        po_lang = po_ext.split('.')[0]
-        subprocess.run(["msguniq", "-o", po_ext, po_ext], check=True)
-        subprocess.run(["intltool-update", "-g", uuid, "-d", po_lang],
-                       check=True)
-        subprocess.run(["msgattrib", "--no-obsolete", "-w", "79", "-o",
-                        po_ext, po_ext],
-                       check=True)
-    if po_list:
-        _return_pwd += '/..'
-    if _all:
-        os.chdir(_return_pwd)
-
-
-if __name__ == "__main__":
+    with ProcessPoolExecutor() as executor:
+        for lines in executor.map(process_po, po_list, [uuid] * len(po_list)):
+            output += lines
+
+    return output
+
+
+if __name__ == '__main__':
     parse_args()