Validate tests in parallel

src/sinol_make/commands/ingen/__init__.py

@@ -30,6 +30,8 @@ def configure_subparser(self, subparser):
                             help='path to ingen source file, for example prog/abcingen.cpp')
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
+        parser.add_argument('-c', '--cpus', type=int,
+                            help=f'number of cpus to use (default: {util.default_cpu_count()})')
         parsers.add_compilation_arguments(parser)
 
     def run(self, args: argparse.Namespace):
@@ -70,8 +72,5 @@ def run(self, args: argparse.Namespace):
             f.write("\n".join(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in"))))
 
         if not self.args.no_validate:
-            print(util.info('Validating input test contents.'))
             tests = sorted(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")))
-            for test in tests:
-                package_util.validate_test(test)
-            print(util.info('Input test contents are valid!'))
+            package_util.validate_tests(tests, self.args.cpus, 'input')

src/sinol_make/commands/outgen/__init__.py

@@ -129,7 +129,4 @@ def run(self, args: argparse.Namespace):
                 yaml.dump(md5_sums, f)
 
         if not self.args.no_validate:
-            print(util.info('Validating output test contents.'))
-            for test in sorted(outputs_to_generate):
-                package_util.validate_test(test)
-            print(util.info('Output test contents are valid!'))
+            package_util.validate_tests(outputs_to_generate, self.args.cpus, 'outputs')

src/sinol_make/helpers/package_util.py

@@ -3,8 +3,9 @@
 import yaml
 import glob
 import fnmatch
+import multiprocessing as mp
 from enum import Enum
-from typing import List, Union, Dict, Any
+from typing import List, Union, Dict, Any, Tuple
 
 from sinol_make import util
 from sinol_make.helpers import paths
@@ -345,11 +346,12 @@ def save_contest_type_to_cache(contest_type):
         contest_type_file.write(contest_type)
 
 
-def validate_test(test_path: str):
+def validate_test(test_path: str) -> Tuple[bool, str]:
     """
     Check if test doesn't contain leading/trailing whitespaces,
     has only one space between tokens and ends with newline.
     Exits with error if any of the conditions is not met.
+    :return: Tuple of two values: True if test is valid, error message otherwise.
     """
     basename = os.path.basename(test_path)
     num_empty = 0
@@ -358,21 +360,42 @@ def validate_test(test_path: str):
         for i, line in enumerate(lines):
             line = line.decode('utf-8')
             if len(line) > 0 and line[0] == ' ':
-                util.exit_with_error(f'Leading whitespace in {basename}:{i + 1}')
+                return False, util.error(f'Leading whitespace in {basename}:{i + 1}')
             if len(line) > 0 and (line[-2:] == '\r\n' or line[-2:] == '\n\r' or line[-1] == '\r'):
-                util.exit_with_error(f'Carriage return at the end of {basename}:{i + 1}')
+                return False, util.error(f'Carriage return at the end of {basename}:{i + 1}')
             if len(line) > 0 and line[-1] != '\n':
-                util.exit_with_error(f'No newline at the end of {basename}')
+                return False, util.error(f'No newline at the end of {basename}')
             if line == '\n' or line == '':
                 num_empty += 1
                 continue
             elif i == len(lines) - 1:
                 num_empty = 0
             if line[-2] == ' ':
-                util.exit_with_error(f'Trailing whitespace in {basename}:{i + 1}')
+                return False, util.error(f'Trailing whitespace in {basename}:{i + 1}')
             for j in range(len(line) - 1):
                 if line[j] == ' ' and line[j + 1] == ' ':
-                    util.exit_with_error(f'Tokens not separated by one space in {basename}:{i + 1}')
+                    return False, util.error(f'Tokens not separated by one space in {basename}:{i + 1}')
 
         if num_empty != 0:
-            util.exit_with_error(f'Exactly one empty line expected in {basename}')
+            return False, util.error(f'Exactly one empty line expected in {basename}')
+
+    return True, ''
+
+
+def validate_tests(tests: List[str], cpus: int, type: str = 'input'):
+    """
+    Validate all tests in parallel.
+    """
+    if not tests:
+        return
+    print(f'Validating {type} test contents.')
+    num_tests = len(tests)
+    finished = 0
+    with mp.Pool(cpus) as pool:
+        for valid, message in pool.imap(validate_test, tests):
+            if not valid:
+                util.exit_with_error(message)
+            finished += 1
+            print(f'Validated {finished}/{num_tests} tests', end='\r')
+    print()
+    print(util.info(f'All {type} tests are valid!'))