Pull upstream changes

.github/workflows/Arch.yaml

@@ -6,7 +6,7 @@ jobs:
     runs-on: self-hosted
     strategy:
         matrix:
-            python-version: ["3.7.17", "3.12.0"]
+            python-version: ["3.8", "3.12"]
     name: pytest-arch-python-${{ matrix.python-version }}
     container:
       image: archlinux:latest

.github/workflows/GithubRunner.yaml

@@ -5,7 +5,7 @@ jobs:
   pytest:
     strategy:
         matrix:
-            python-version: ["3.7.17", "3.12.0"]
+            python-version: ["3.8", "3.12"]
     name: pytest-github-runner-python-${{ matrix.python-version }}
     runs-on: ubuntu-latest
     steps:
@@ -18,6 +18,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install .[tests]
+          sudo apt update
           sudo apt install texlive-latex-recommended -y
           sudo apt install ghostscript -y
       - name: Run github runner pytest

.github/workflows/Ubuntu.yaml

@@ -6,7 +6,7 @@ jobs:
     runs-on: self-hosted
     strategy:
         matrix:
-            python-version: ["3.7.17", "3.12.0"]
+            python-version: ["3.8", "3.12"]
     name: pytest-ubuntu-python-${{ matrix.python-version }}
     container:
       image: ubuntu:latest

.github/workflows/macOS.yaml

@@ -7,7 +7,7 @@ jobs:
     runs-on: macos-latest
     strategy:
         matrix:
-            python-version: ["3.7", "3.12"]
+            python-version: ["3.10", "3.12"]
     name: pytest-macos-python-${{ matrix.python-version }}
     steps:
       - name: Checkout

README.md

@@ -65,7 +65,7 @@ activate-global-python-argcomplete
 
 ### Usage
 
-The availabe commands (see `st-make --help`) are:
+The available commands (see `st-make --help`) are:
 
 - `st-make run` -- Runs selected solutions (by default all solutions) on selected tests (by default all tests) with a given number
 of CPUs. Measures the solutions' time with oiejq, unless specified otherwise. After running the solutions, it
@@ -85,6 +85,9 @@ Run `st-make ingen --help` to see available flags.
 program to use, what tests to check and how many CPUs to use. Run `st-make inwer --help` to see available flags.
 - `st-make export` -- Creates archive ready to upload to Wyzwania, Talent-camp and other sio2 instances. Run `st-make export --help` to see all available flags.
 - `st-make doc` -- Compiles all LaTeX files in doc/ directory to PDF. Run `st-make doc --help` to see all available flags.
+- `st-make verify` -- Verifies the package. This command runs stress tests (if available), verifies the config,
+generates tests, generates problem statements, runs inwer and run all solutions. Ingen and inwer are compiled with
+address and UB sanitizers. Run `st-make verify --help` to see all available flags.
 - `st-make init [id]` -- Creates package from template [on github](https://github.com/Stowarzyszenie-Talent/st-make/tree/main/example_package) and sets task id to provided `[id]`. Requires an internet connection to run.
 
 You can also run multiple commands at once, for example:

example_package/README.md

@@ -449,7 +449,7 @@ an archive with the proper name, which sio2 uses as the task id.
 sinol_contest_type: talent
 ```
 
-sinol-make can behave differently depending on the value of `sinol_contest_type` key.
+st-make can behave differently depending on the value of `sinol_contest_type` key.
 Mainly, it affects how points are calculated.
 If the key is not specified, then (in st-make) `talent` is used. In sinol-make (OI version) is used 'default'.
 

example_package/config.yml

@@ -90,9 +90,17 @@ scores:
 
 # sinol_contest_type: talent
 
-# sinol-make can behave differently depending on the value of `sinol_contest_type` key.
+# st-make can behave differently depending on the value of `sinol_contest_type` key.
 # Mainly, it affects how points are calculated.
-# If the key is not specified, then (in st-make) `talent` is used. In sinol-make (OI version) is used 'default'.
+# If the key is not specified, then (in st-make) `talent` is used. In st-make (OI version) is used 'default'.
+
+
+### handwritten tests
+
+# You can specify which tests are static (handwritten). This allows st-make to differentiate between
+# old and handwritten tests. If this key is not present old tests won't be removed.
+# This key is optional and should be a list of tests.
+sinol_static_tests: ["abc0.in", "abc0a.in"]
 
 
 ### expected scores

setup.cfg

@@ -29,6 +29,7 @@ install_requires =
     dictdiffer
     importlib-resources
     psutil
+    packaging
 
 [options.packages.find]
 where = src

src/sinol_make/__init__.py

@@ -33,7 +33,7 @@ def configure_parsers():
 
 def check_oiejq():
     if util.is_linux() and not oiejq.check_oiejq():
-        print(util.warning('`oiejq` in `~/.local/bin/` not found, installing now...'))
+        print(util.warning('Up to date `oiejq` in `~/.local/bin/` not found, installing new version...'))
         try:
             if oiejq.install_oiejq():
                 print(util.info('`oiejq` was successfully installed.'))
@@ -50,35 +50,40 @@ def main_exn():
     parser = configure_parsers()
     arguments = []
     curr_args = []
+    commands = util.get_commands()
+    commands_dict = {command.get_name(): command for command in commands}
     for arg in sys.argv[1:]:
-        if arg in util.get_command_names() and not (len(curr_args) > 0 and curr_args[0] == 'init'):
+        if arg in commands_dict.keys() and not (len(curr_args) > 0 and curr_args[0] == 'init'):
             if curr_args:
                 arguments.append(curr_args)
             curr_args = [arg]
         else:
             curr_args.append(arg)
     if curr_args:
         arguments.append(curr_args)
-    commands = util.get_commands()
+    if not arguments:
+        parser.print_help()
+        exit(1)
     check_oiejq()
 
     for curr_args in arguments:
         args = parser.parse_args(curr_args)
-        command_found = False
-        for command in commands:
-            if command.get_name() == args.command:
-                command_found = True
-                if len(arguments) > 1:
-                    print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '='))
-                command.run(args)
-        if not command_found:
+        command = commands_dict.get(args.command, None)
+        if command:
+            if len(arguments) > 1:
+                print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '='))
+            command.run(args)
+        else:
             parser.print_help()
             exit(1)
 
 
 def main():
     new_version = None
     try:
+        if util.is_dev(__version__):
+            print(util.warning('You are using a development version of st-make. '
+                               'It may be unstable and contain bugs.'))
         new_version = util.check_for_updates(__version__)
         main_exn()
     except argparse.ArgumentError as err:
@@ -92,6 +97,12 @@ def main():
                              'https://github.com/Stowarzyszenie-Talent/st-make/#reporting-bugs-and-contributing-code')
     finally:
         if new_version is not None:
-            print(util.warning(
-                f'New version of sinol-make is available (your version: {__version__}, available version: '
-                f'{new_version}).\nYou can update it by running `pip3 install sinol-make --upgrade`.'))
+            if not util.is_dev(new_version):
+                print(util.warning(
+                    f'New version of st-make is available (your version: {__version__}, available version: '
+                    f'{new_version}).\nYou can update it by running `pip3 install st-make --upgrade`.'))
+            elif util.is_dev(new_version):
+                print(util.warning(
+                    f'New development version of st-make is available (your version: {__version__}, available '
+                    f'version: {new_version}).\nYou can update it by running `pip3 install st-make --pre --upgrade`.'
+                ))

src/sinol_make/commands/doc/__init__.py

@@ -18,7 +18,7 @@ def get_name(self):
         return "doc"
 
     def compile_file_latex_div(self, file_path):
-        print(util.info(f'Compiling {os.path.basename(file_path)} (latex to dvi)...'))
+        print(f'Compiling {os.path.basename(file_path)} (latex to dvi)...')
         os.chdir(os.path.dirname(file_path))
         subprocess.run(['latex', file_path])
         dvi_file = os.path.splitext(file_path)[0] + '.dvi'
@@ -35,7 +35,7 @@ def compile_file_latex_div(self, file_path):
         return True
 
     def compile_pdf_latex(self, file_path):
-        print(util.info(f'Compiling {os.path.basename(file_path)} (pdflatex)...'))
+        print(f'Compiling {os.path.basename(file_path)} (pdflatex)...')
         os.chdir(os.path.dirname(file_path))
         subprocess.run(['pdflatex', file_path])
         pdf_file = os.path.splitext(file_path)[0] + '.pdf'
@@ -62,7 +62,7 @@ def move_logs(self):
         for pattern in self.LOG_PATTERNS:
             for file in glob.glob(os.path.join(os.getcwd(), 'doc', pattern)):
                 os.rename(file, os.path.join(output_dir, os.path.basename(file)))
-        print(util.info(f'Compilation log files can be found in {os.path.relpath(output_dir, os.getcwd())}'))
+        print(f'Compilation log files can be found in {os.path.relpath(output_dir, os.getcwd())}')
 
     def configure_subparser(self, subparser: argparse.ArgumentParser):
         parser = subparser.add_parser(
@@ -76,6 +76,7 @@ def configure_subparser(self, subparser: argparse.ArgumentParser):
                                  ' pdflatex - uses pdflatex. Works with .png and .jpg images.\n'
                                  ' latex_dvi - uses latex and dvipdf. Works with .ps and .eps images.', default='auto')
         parser.add_argument('files', type=str, nargs='*', help='files to compile')
+        return parser
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)

src/sinol_make/commands/export/__init__.py

@@ -28,15 +28,13 @@ def configure_subparser(self, subparser: argparse.ArgumentParser):
             self.get_name(),
             help='Create archive for oioioi upload',
             description='Creates archive in the current directory ready to upload to sio2 or szkopul.')
-        parser.add_argument('-c', '--cpus', type=int,
-                            help=f'number of cpus to use to generate output files '
-                                 f'(default: {util.default_cpu_count()})',
-                            default=util.default_cpu_count())
+        parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files')
         parser.add_argument('--no-statement', dest='no_statement', action='store_true',
                             help='allow export without statement')
         parser.add_argument('--export-ocen', dest='export_ocen', action='store_true',
                             help='Create ocen archive')
         parsers.add_compilation_arguments(parser)
+        return parser
 
     def generate_input_tests(self):
         print('Generating tests...')
@@ -97,6 +95,7 @@ def create_ocen(self, target_dir: str):
         Creates ocen archive for sio2.
         :param target_dir: Path to exported package.
         """
+        print('Generating ocen archive...')
         attachments_dir = os.path.join(target_dir, 'attachments')
         if not os.path.exists(attachments_dir):
             os.makedirs(attachments_dir)
@@ -109,12 +108,27 @@ def create_ocen(self, target_dir: str):
             os.makedirs(in_dir)
             out_dir = os.path.join(ocen_dir, 'out')
             os.makedirs(out_dir)
+            num_tests = 0
             for ext in ['in', 'out']:
                 for test in glob.glob(os.path.join(tests_dir, ext, f'{self.task_id}0*.{ext}')) + \
                             glob.glob(os.path.join(tests_dir, ext, f'{self.task_id}*ocen.{ext}')):
                     shutil.copy(test, os.path.join(ocen_dir, ext, os.path.basename(test)))
-
-            shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir)
+                    num_tests += 1
+
+            dlazaw_dir = os.path.join(os.getcwd(), 'dlazaw')
+            if num_tests == 0:
+                print(util.warning('No ocen tests found.'))
+            elif os.path.exists(dlazaw_dir):
+                print(util.warning('Skipping ocen archive creation because dlazaw directory exists.'))
+            else:
+                shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir)
+
+            if os.path.exists(dlazaw_dir):
+                print('Archiving dlazaw directory and adding to attachments.')
+                os.makedirs(os.path.join(tmpdir, 'dlazaw'), exist_ok=True)
+                shutil.copytree(dlazaw_dir, os.path.join(tmpdir, 'dlazaw', 'dlazaw'))
+                shutil.make_archive(os.path.join(attachments_dir, 'dlazaw'), 'zip',
+                                    os.path.join(tmpdir, 'dlazaw'))
 
     def compile_statement(self):
         command = DocCommand()
@@ -168,7 +182,6 @@ def copy_package_required_files(self, target_dir: str):
 
         self.generate_output_files()
         if self.args.export_ocen:
-            print('Generating ocen archive...')
             self.create_ocen(target_dir)
 
     def clear_files(self, target_dir: str):

src/sinol_make/commands/gen/__init__.py

@@ -31,13 +31,12 @@ def configure_subparser(self, subparser):
                             help='path to ingen source file, for example prog/abcingen.cpp')
         parser.add_argument('-i', '--only-inputs', action='store_true', help='generate input files only')
         parser.add_argument('-o', '--only-outputs', action='store_true', help='generate output files only')
-        parser.add_argument('-c', '--cpus', type=int,
-                            help=f'number of cpus to use to generate output files '
-                                 f'(default: {util.default_cpu_count()})',
-                            default=util.default_cpu_count())
+        parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files')
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
+        return parser
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)

src/sinol_make/commands/ingen/__init__.py

@@ -30,19 +30,44 @@ def configure_subparser(self, subparser):
                             help='path to ingen source file, for example prog/abcingen.cpp')
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
+        parsers.add_cpus_argument(parser, 'number of cpus used for validating tests')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
+        return parser
+
+    def delete_dangling_files(self, dates):
+        to_delete = set()
+        for test in glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")):
+            basename = os.path.basename(test)
+            if basename in dates and dates[basename] == os.path.getmtime(test):
+                to_delete.add(os.path.basename(test))
+        if to_delete:
+            config = package_util.get_config()
+            if 'sinol_static_tests' not in config:
+                print(util.warning('Old input files won\'t be deleted, '
+                                   'because static tests are not defined. '
+                                   'You can define them in config.yml with `sinol_static_tests` key.'))
+            else:
+                static_files = config['sinol_static_tests']
+                if isinstance(static_files, str):
+                    static_files = [static_files]
+                static_files = set([os.path.basename(test) for test in static_files])
+                to_delete = to_delete - static_files
+                if to_delete:
+                    print('Cleaning up old input files.')
+                    for test in to_delete:
+                        os.remove(os.path.join(os.getcwd(), "in", test))
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)
 
         self.args = args
 
         self.task_id = package_util.get_task_id()
-        package_util.validate_test_names(self.task_id)
         util.change_stack_size_to_unlimited()
         self.ingen = get_ingen(self.task_id, args.ingen_path)
-        print(util.info(f'Using ingen file {os.path.basename(self.ingen)}'))
-        self.ingen_exe = compile_ingen(self.ingen, self.args, self.args.compile_mode)
+        print(f'Using ingen file {os.path.basename(self.ingen)}')
+        self.ingen_exe = compile_ingen(self.ingen, self.args, self.args.compile_mode, self.args.fsanitize)
 
         previous_tests = []
         try:
@@ -60,18 +85,11 @@ def run(self, args: argparse.Namespace):
         else:
             util.exit_with_error('Failed to generate input files.')
 
-        print(util.info('Cleaning up old input files.'))
-        for test in glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")):
-            basename = os.path.basename(test)
-            if basename in dates and dates[basename] == os.path.getmtime(test):
-                os.unlink(test)
+        self.delete_dangling_files(dates)
 
         with open(paths.get_cache_path("input_tests"), "w") as f:
             f.write("\n".join(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in"))))
 
         if not self.args.no_validate:
-            print(util.info('Validating input test contents.'))
             tests = sorted(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")))
-            for test in tests:
-                package_util.validate_test(test)
-            print(util.info('Input test contents are valid!'))
+            package_util.validate_tests(tests, self.args.cpus, 'input')