diff --git a/setup.cfg b/setup.cfg index 39193949..542bc314 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ install_requires = dictdiffer importlib-resources psutil + packaging [options.packages.find] where = src diff --git a/src/sinol_make/__init__.py b/src/sinol_make/__init__.py index f7aa5f70..c6c6d1e3 100644 --- a/src/sinol_make/__init__.py +++ b/src/sinol_make/__init__.py @@ -6,7 +6,7 @@ from sinol_make import util, oiejq -__version__ = "1.5.29" +__version__ = "1.6.0.dev3" def configure_parsers(): @@ -50,8 +50,10 @@ def main_exn(): parser = configure_parsers() arguments = [] curr_args = [] + commands = util.get_commands() + commands_dict = {command.get_name(): command for command in commands} for arg in sys.argv[1:]: - if arg in util.get_command_names() and not (len(curr_args) > 0 and curr_args[0] == 'init'): + if arg in commands_dict.keys() and not (len(curr_args) > 0 and curr_args[0] == 'init'): if curr_args: arguments.append(curr_args) curr_args = [arg] @@ -59,19 +61,19 @@ def main_exn(): curr_args.append(arg) if curr_args: arguments.append(curr_args) - commands = util.get_commands() + if not arguments: + parser.print_help() + exit(1) check_oiejq() for curr_args in arguments: args = parser.parse_args(curr_args) - command_found = False - for command in commands: - if command.get_name() == args.command: - command_found = True - if len(arguments) > 1: - print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '=')) - command.run(args) - if not command_found: + command = commands_dict.get(args.command, None) + if command: + if len(arguments) > 1: + print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '=')) + command.run(args) + else: parser.print_help() exit(1) @@ -79,6 +81,9 @@ def main_exn(): def main(): new_version = None try: + if util.is_dev(__version__): + print(util.warning('You are using a development version of sinol-make. ' + 'It may be unstable and contain bugs.')) new_version = util.check_for_updates(__version__) main_exn() except argparse.ArgumentError as err: @@ -92,6 +97,12 @@ def main(): 'https://github.com/sio2project/sinol-make/#reporting-bugs-and-contributing-code') finally: if new_version is not None: - print(util.warning( - f'New version of sinol-make is available (your version: {__version__}, available version: ' - f'{new_version}).\nYou can update it by running `pip3 install sinol-make --upgrade`.')) + if not util.is_dev(new_version): + print(util.warning( + f'New version of sinol-make is available (your version: {__version__}, available version: ' + f'{new_version}).\nYou can update it by running `pip3 install sinol-make --upgrade`.')) + elif util.is_dev(new_version): + print(util.warning( + f'New development version of sinol-make is available (your version: {__version__}, available ' + f'version: {new_version}).\nYou can update it by running `pip3 install sinol-make --pre --upgrade`.' + )) diff --git a/src/sinol_make/commands/doc/__init__.py b/src/sinol_make/commands/doc/__init__.py index 3d624bed..ace6f803 100644 --- a/src/sinol_make/commands/doc/__init__.py +++ b/src/sinol_make/commands/doc/__init__.py @@ -18,7 +18,7 @@ def get_name(self): return "doc" def compile_file_latex_div(self, file_path): - print(util.info(f'Compiling {os.path.basename(file_path)} (latex to dvi)...')) + print(f'Compiling {os.path.basename(file_path)} (latex to dvi)...') os.chdir(os.path.dirname(file_path)) subprocess.run(['latex', file_path]) dvi_file = os.path.splitext(file_path)[0] + '.dvi' @@ -35,7 +35,7 @@ def compile_file_latex_div(self, file_path): return True def compile_pdf_latex(self, file_path): - print(util.info(f'Compiling {os.path.basename(file_path)} (pdflatex)...')) + print(f'Compiling {os.path.basename(file_path)} (pdflatex)...') os.chdir(os.path.dirname(file_path)) subprocess.run(['pdflatex', file_path]) pdf_file = os.path.splitext(file_path)[0] + '.pdf' @@ -62,7 +62,7 @@ def move_logs(self): for pattern in self.LOG_PATTERNS: for file in glob.glob(os.path.join(os.getcwd(), 'doc', pattern)): os.rename(file, os.path.join(output_dir, os.path.basename(file))) - print(util.info(f'Compilation log files can be found in {os.path.relpath(output_dir, os.getcwd())}')) + print(f'Compilation log files can be found in {os.path.relpath(output_dir, os.getcwd())}') def configure_subparser(self, subparser: argparse.ArgumentParser): parser = subparser.add_parser( @@ -76,6 +76,7 @@ def configure_subparser(self, subparser: argparse.ArgumentParser): ' pdflatex - uses pdflatex. Works with .png and .jpg images.\n' ' latex_dvi - uses latex and dvipdf. Works with .ps and .eps images.', default='auto') parser.add_argument('files', type=str, nargs='*', help='files to compile') + return parser def run(self, args: argparse.Namespace): args = util.init_package_command(args) diff --git a/src/sinol_make/commands/export/__init__.py b/src/sinol_make/commands/export/__init__.py index 5d072040..eb727d99 100644 --- a/src/sinol_make/commands/export/__init__.py +++ b/src/sinol_make/commands/export/__init__.py @@ -28,15 +28,13 @@ def configure_subparser(self, subparser: argparse.ArgumentParser): self.get_name(), help='Create archive for oioioi upload', description='Creates archive in the current directory ready to upload to sio2 or szkopul.') - parser.add_argument('-c', '--cpus', type=int, - help=f'number of cpus to use to generate output files ' - f'(default: {util.default_cpu_count()})', - default=util.default_cpu_count()) + parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files') parser.add_argument('--no-statement', dest='no_statement', action='store_true', help='allow export without statement') parser.add_argument('--export-ocen', dest='export_ocen', action='store_true', help='Create ocen archive') parsers.add_compilation_arguments(parser) + return parser def generate_input_tests(self): print('Generating tests...') @@ -97,6 +95,7 @@ def create_ocen(self, target_dir: str): Creates ocen archive for sio2. :param target_dir: Path to exported package. """ + print('Generating ocen archive...') attachments_dir = os.path.join(target_dir, 'attachments') if not os.path.exists(attachments_dir): os.makedirs(attachments_dir) @@ -109,12 +108,27 @@ def create_ocen(self, target_dir: str): os.makedirs(in_dir) out_dir = os.path.join(ocen_dir, 'out') os.makedirs(out_dir) + num_tests = 0 for ext in ['in', 'out']: for test in glob.glob(os.path.join(tests_dir, ext, f'{self.task_id}0*.{ext}')) + \ glob.glob(os.path.join(tests_dir, ext, f'{self.task_id}*ocen.{ext}')): shutil.copy(test, os.path.join(ocen_dir, ext, os.path.basename(test))) - - shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir) + num_tests += 1 + + dlazaw_dir = os.path.join(os.getcwd(), 'dlazaw') + if num_tests == 0: + print(util.warning('No ocen tests found.')) + elif os.path.exists(dlazaw_dir): + print(util.warning('Skipping ocen arhive creation because dlazaw directory exists.')) + else: + shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir) + + if os.path.exists(dlazaw_dir): + print('Archiving dlazaw directory and adding to attachments.') + os.makedirs(os.path.join(tmpdir, 'dlazaw'), exist_ok=True) + shutil.copytree(dlazaw_dir, os.path.join(tmpdir, 'dlazaw', 'dlazaw')) + shutil.make_archive(os.path.join(attachments_dir, 'dlazaw'), 'zip', + os.path.join(tmpdir, 'dlazaw')) def compile_statement(self): command = DocCommand() @@ -168,7 +182,6 @@ def copy_package_required_files(self, target_dir: str): self.generate_output_files() if self.args.export_ocen: - print('Generating ocen archive...') self.create_ocen(target_dir) def clear_files(self, target_dir: str): diff --git a/src/sinol_make/commands/gen/__init__.py b/src/sinol_make/commands/gen/__init__.py index afb3b53f..539166b7 100644 --- a/src/sinol_make/commands/gen/__init__.py +++ b/src/sinol_make/commands/gen/__init__.py @@ -31,13 +31,14 @@ def configure_subparser(self, subparser): help='path to ingen source file, for example prog/abcingen.cpp') parser.add_argument('-i', '--only-inputs', action='store_true', help='generate input files only') parser.add_argument('-o', '--only-outputs', action='store_true', help='generate output files only') - parser.add_argument('-c', '--cpus', type=int, - help=f'number of cpus to use to generate output files ' - f'(default: {util.default_cpu_count()})', - default=util.default_cpu_count()) + parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files') parser.add_argument('-n', '--no-validate', default=False, action='store_true', help='do not validate test contents') + parser.add_argument('-f', '--fsanitize', default=False, action='store_true', + help='Use -fsanitize=address,undefined for ingen compilation. Warning: this may fail on some ' + 'systems. To fix this, run `sudo sysctl vm.mmap_rnd_bits = 28`.') parsers.add_compilation_arguments(parser) + return parser def run(self, args: argparse.Namespace): args = util.init_package_command(args) diff --git a/src/sinol_make/commands/ingen/__init__.py b/src/sinol_make/commands/ingen/__init__.py index af30fe2a..552ccdbe 100644 --- a/src/sinol_make/commands/ingen/__init__.py +++ b/src/sinol_make/commands/ingen/__init__.py @@ -30,9 +30,12 @@ def configure_subparser(self, subparser): help='path to ingen source file, for example prog/abcingen.cpp') parser.add_argument('-n', '--no-validate', default=False, action='store_true', help='do not validate test contents') - parser.add_argument('-c', '--cpus', type=int, - help=f'number of cpus to use (default: {util.default_cpu_count()})') + parsers.add_cpus_argument(parser, 'number of cpus used for validating tests') + parser.add_argument('-f', '--fsanitize', default=False, action='store_true', + help='Use -fsanitize=address,undefined for compilation. Warning: this may fail on some ' + 'systems. Tof fix this, run `sudo sysctl vm.mmap_rnd_bits = 28`.') parsers.add_compilation_arguments(parser) + return parser def run(self, args: argparse.Namespace): args = util.init_package_command(args) @@ -43,8 +46,8 @@ def run(self, args: argparse.Namespace): package_util.validate_test_names(self.task_id) util.change_stack_size_to_unlimited() self.ingen = get_ingen(self.task_id, args.ingen_path) - print(util.info(f'Using ingen file {os.path.basename(self.ingen)}')) - self.ingen_exe = compile_ingen(self.ingen, self.args, self.args.compile_mode) + print(f'Using ingen file {os.path.basename(self.ingen)}') + self.ingen_exe = compile_ingen(self.ingen, self.args, self.args.compile_mode, self.args.fsanitize) previous_tests = [] try: @@ -62,7 +65,7 @@ def run(self, args: argparse.Namespace): else: util.exit_with_error('Failed to generate input files.') - print(util.info('Cleaning up old input files.')) + print('Cleaning up old input files.') for test in glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")): basename = os.path.basename(test) if basename in dates and dates[basename] == os.path.getmtime(test): diff --git a/src/sinol_make/commands/ingen/ingen_util.py b/src/sinol_make/commands/ingen/ingen_util.py index 2b6608da..a8c03ca8 100644 --- a/src/sinol_make/commands/ingen/ingen_util.py +++ b/src/sinol_make/commands/ingen/ingen_util.py @@ -47,7 +47,7 @@ def get_ingen(task_id, ingen_path=None): return correct_ingen -def compile_ingen(ingen_path: str, args: argparse.Namespace, compilation_flags='default'): +def compile_ingen(ingen_path: str, args: argparse.Namespace, compilation_flags='default', use_fsanitize=False): """ Compiles ingen and returns path to compiled executable. If ingen_path is shell script, then it will be returned. @@ -57,7 +57,7 @@ def compile_ingen(ingen_path: str, args: argparse.Namespace, compilation_flags=' compilers = compiler.verify_compilers(args, [ingen_path]) ingen_exe, compile_log_path = compile.compile_file(ingen_path, package_util.get_executable(ingen_path), - compilers, compilation_flags, use_fsanitize=True, + compilers, compilation_flags, use_fsanitize=use_fsanitize, additional_flags='-D_INGEN') if ingen_exe is None: @@ -86,11 +86,21 @@ def run_ingen(ingen_exe, working_dir=None): print(util.bold(' Ingen output '.center(util.get_terminal_size()[1], '='))) process = subprocess.Popen([ingen_exe], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_dir, shell=is_shell) + whole_output = '' while process.poll() is None: - print(process.stdout.readline().decode('utf-8'), end='') - - print(process.stdout.read().decode('utf-8'), end='') + out = process.stdout.readline().decode('utf-8') + if out != '': + print(out, end='') + whole_output += out + out = process.stdout.read().decode('utf-8') + whole_output += out + print(out, end='') exit_code = process.returncode print(util.bold(' End of ingen output '.center(util.get_terminal_size()[1], '='))) + if util.has_sanitizer_error(whole_output, exit_code): + print(util.warning('Warning: if ingen failed due to sanitizer errors, you can either run ' + '`sudo sysctl vm.mmap_rnd_bits = 28` to fix this or disable sanitizers with the ' + '--no-fsanitize flag.')) + return exit_code == 0 diff --git a/src/sinol_make/commands/init/__init__.py b/src/sinol_make/commands/init/__init__.py index 8ae00b2f..d675d103 100644 --- a/src/sinol_make/commands/init/__init__.py +++ b/src/sinol_make/commands/init/__init__.py @@ -23,6 +23,7 @@ def configure_subparser(self, subparser: argparse.ArgumentParser): description='Create package from predefined template with given id.' ) parser.add_argument('task_id', type=str, help='Id of the task to create') + return parser def download_template(self): repo = 'https://github.com/sio2project/sinol-make.git' @@ -69,8 +70,7 @@ def run(self, args: argparse.Namespace): self.move_folder() self.update_config() - + self.used_tmpdir.cleanup() print(util.info(f'Successfully created task "{self.task_id}"')) - diff --git a/src/sinol_make/commands/inwer/__init__.py b/src/sinol_make/commands/inwer/__init__.py index 71245b97..39743f35 100644 --- a/src/sinol_make/commands/inwer/__init__.py +++ b/src/sinol_make/commands/inwer/__init__.py @@ -10,8 +10,7 @@ from sinol_make import util from sinol_make.structs.inwer_structs import TestResult, InwerExecution, VerificationResult, TableData -from sinol_make.helpers import package_util, compile, printer, paths -from sinol_make.helpers.parsers import add_compilation_arguments +from sinol_make.helpers import package_util, printer, paths, parsers from sinol_make.interfaces.BaseCommand import BaseCommand from sinol_make.commands.inwer import inwer_util @@ -37,9 +36,12 @@ def configure_subparser(self, subparser: argparse.ArgumentParser): help='path to inwer source file, for example prog/abcinwer.cpp') parser.add_argument('-t', '--tests', type=str, nargs='+', help='test to verify, for example in/abc{0,1}*') - parser.add_argument('-c', '--cpus', type=int, - help=f'number of cpus to use (default: {util.default_cpu_count()})') - add_compilation_arguments(parser) + parsers.add_cpus_argument(parser, 'number of cpus to use when verifying tests') + parser.add_argument('-f', '--fsanitize', default=False, action='store_true', + help='Use -fsanitize=address,undefined for compilation. Warning: this may fail on some ' + 'systems. Tof fix this, run `sudo sysctl vm.mmap_rnd_bits = 28`.') + parsers.add_compilation_arguments(parser) + return parser @staticmethod def verify_test(execution: InwerExecution) -> VerificationResult: @@ -94,11 +96,14 @@ def verify_and_print_table(self) -> Dict[str, TestResult]: thr.start() keyboard_interrupt = False + sanitizer_error = False try: with mp.Pool(self.cpus) as pool: for i, result in enumerate(pool.imap(self.verify_test, executions)): table_data.results[result.test_path].set_results(result.valid, result.output) table_data.i = i + if util.has_sanitizer_error(result.output, 0 if result.valid else 1): + sanitizer_error = True except KeyboardInterrupt: keyboard_interrupt = True @@ -108,6 +113,10 @@ def verify_and_print_table(self) -> Dict[str, TestResult]: print("\n".join(inwer_util.print_view(terminal_width, terminal_height, table_data)[0])) + if sanitizer_error: + print(util.warning('Warning: if inwer failed due to sanitizer errors, you can either run ' + '`sudo sysctl vm.mmap_rnd_bits = 28` to fix this or disable sanitizers with the ' + '--no-fsanitize flag.')) if keyboard_interrupt: util.exit_with_error('Keyboard interrupt.') @@ -203,7 +212,7 @@ def run(self, args: argparse.Namespace): print('Verifying tests: ' + util.bold(', '.join(self.tests))) util.change_stack_size_to_unlimited() - self.inwer_executable = inwer_util.compile_inwer(self.inwer, args, args.compile_mode) + self.inwer_executable = inwer_util.compile_inwer(self.inwer, args, args.compile_mode, args.fsanitize) results: Dict[str, TestResult] = self.verify_and_print_table() print('') @@ -218,4 +227,3 @@ def run(self, args: argparse.Namespace): print("Verifying tests order...") self.verify_tests_order() print(util.info('Verification successful.')) - exit(0) diff --git a/src/sinol_make/commands/inwer/inwer_util.py b/src/sinol_make/commands/inwer/inwer_util.py index 6d21f0c0..53618603 100644 --- a/src/sinol_make/commands/inwer/inwer_util.py +++ b/src/sinol_make/commands/inwer/inwer_util.py @@ -29,13 +29,13 @@ def get_inwer_path(task_id: str, path=None) -> Union[str, None]: return None -def compile_inwer(inwer_path: str, args: argparse.Namespace, compilation_flags='default'): +def compile_inwer(inwer_path: str, args: argparse.Namespace, compilation_flags='default', use_fsanitize=False): """ Compiles inwer and returns path to compiled executable and path to compile log. """ compilers = compiler.verify_compilers(args, [inwer_path]) inwer_exe, compile_log_path = compile.compile_file(inwer_path, package_util.get_executable(inwer_path), compilers, - compilation_flags, use_fsanitize=True, + compilation_flags, use_fsanitize=use_fsanitize, additional_flags='-D_INWER') if inwer_exe is None: diff --git a/src/sinol_make/commands/outgen/__init__.py b/src/sinol_make/commands/outgen/__init__.py index 126b7049..3a188484 100644 --- a/src/sinol_make/commands/outgen/__init__.py +++ b/src/sinol_make/commands/outgen/__init__.py @@ -26,14 +26,11 @@ def configure_subparser(self, subparser): help='Generate output files', description='Generate output files using the correct solution.' ) - - parser.add_argument('-c', '--cpus', type=int, - help=f'number of cpus to use to generate output files ' - f'(default: {util.default_cpu_count()})', - default=util.default_cpu_count()) + parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files') parser.add_argument('-n', '--no-validate', default=False, action='store_true', help='do not validate test contents') parsers.add_compilation_arguments(parser) + return parser def generate_outputs(self, outputs_to_generate): print(f'Generating output files for {len(outputs_to_generate)} tests on {self.args.cpus} cpus.') @@ -49,7 +46,7 @@ def generate_outputs(self, outputs_to_generate): for i, result in enumerate(pool.imap(generate_output, arguments)): results.append(result) if result: - print(util.info(f'Successfully generated output file {os.path.basename(arguments[i].output_test)}')) + print(f'Successfully generated output file {os.path.basename(arguments[i].output_test)}') else: print(util.error(f'Failed to generate output file {os.path.basename(arguments[i].output_test)}')) diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py index 79612732..3058ec0d 100644 --- a/src/sinol_make/commands/run/__init__.py +++ b/src/sinol_make/commands/run/__init__.py @@ -8,21 +8,23 @@ import psutil import glob import shutil +import os +import collections +import sys +import math +import dictdiffer +import multiprocessing as mp from io import StringIO from typing import Dict -from sinol_make import contest_types, oiejq +from sinol_make import contest_types, oiejq, util from sinol_make.structs.run_structs import ExecutionData, PrintData from sinol_make.structs.cache_structs import CacheTest, CacheFile -from sinol_make.helpers.parsers import add_compilation_arguments from sinol_make.interfaces.BaseCommand import BaseCommand from sinol_make.interfaces.Errors import CompilationError, CheckerOutputException, UnknownContestType -from sinol_make.helpers import compile, compiler, package_util, printer, paths, cache +from sinol_make.helpers import compile, compiler, package_util, printer, paths, cache, parsers from sinol_make.structs.status_structs import Status, ResultChange, PointsChange, ValidationResult, ExecutionResult, \ TotalPointsChange -import sinol_make.util as util -import yaml, os, collections, sys, re, math, dictdiffer -import multiprocessing as mp def color_memory(memory, limit): @@ -288,8 +290,7 @@ def configure_subparser(self, subparser): help='solutions to be run, for example prog/abc{b,s}*.{cpp,py}') parser.add_argument('-t', '--tests', type=str, nargs='+', help='tests to be run, for example in/abc{0,1}*') - parser.add_argument('-c', '--cpus', type=int, - help=f'number of cpus to use (default: {util.default_cpu_count()}') + parsers.add_cpus_argument(parser, 'number of cpus to use when running solutions') parser.add_argument('--tl', type=float, help='time limit for all tests (in s)') parser.add_argument('--ml', type=float, help='memory limit for all tests (in MB)') parser.add_argument('--hide-memory', dest='hide_memory', action='store_true', @@ -300,7 +301,11 @@ def configure_subparser(self, subparser): help='path to oiejq executable (default: `~/.local/bin/oiejq`)') parser.add_argument('-a', '--apply-suggestions', dest='apply_suggestions', action='store_true', help='apply suggestions from expected scores report') - add_compilation_arguments(parser) + parser.add_argument('--ignore-expected', dest='ignore_expected', action='store_true', + help='ignore expected scores from config.yml. When this flag is set, ' + 'the expected scores are not compared with the actual scores.') + parsers.add_compilation_arguments(parser) + return parser def parse_time(self, time_str): if len(time_str) < 3: return -1 @@ -1016,7 +1021,7 @@ def set_group_result(solution, group, result): self.config["sinol_expected_scores"] = self.convert_status_to_string(config_expected_scores) util.save_config(self.config) - print(util.info("Saved suggested expected scores description.")) + print("Saved suggested expected scores description.") else: util.exit_with_error("Use flag --apply-suggestions to apply suggestions.") @@ -1210,7 +1215,7 @@ def run(self, args): checker = package_util.get_files_matching_pattern(self.ID, f'{self.ID}chk.*') if len(checker) != 0: - print(util.info("Checker found: %s" % os.path.basename(checker[0]))) + print("Checker found: %s" % os.path.basename(checker[0])) self.checker = checker[0] self.compile_checker() else: @@ -1237,6 +1242,11 @@ def run(self, args): results, all_results = self.compile_and_run(solutions) self.check_errors(all_results) + if self.args.ignore_expected: + print(util.warning("Ignoring expected scores.")) + self.exit() + return + try: validation_results = self.validate_expected_scores(results) except Exception: diff --git a/src/sinol_make/commands/verify/__init__.py b/src/sinol_make/commands/verify/__init__.py new file mode 100644 index 00000000..f42dbe74 --- /dev/null +++ b/src/sinol_make/commands/verify/__init__.py @@ -0,0 +1,164 @@ +import os +import stat +import shutil +import argparse +import subprocess + +from sinol_make import util, contest_types +from sinol_make.helpers import parsers, package_util, paths +from sinol_make.interfaces.BaseCommand import BaseCommand +from sinol_make.commands.gen import Command as GenCommand +from sinol_make.commands.doc import Command as DocCommand +from sinol_make.commands.inwer import Command as InwerCommand, inwer_util +from sinol_make.commands.run import Command as RunCommand + + +class Command(BaseCommand): + """ + Class for `verify` command. + """ + + def get_name(self): + return "verify" + + def configure_subparser(self, subparser): + parser = subparser.add_parser( + self.get_name(), + help='Verify the package', + description='Verify the whole package. This command will first ' + 'run stress tests (if the file `prog/{task_id}stresstest.sh` exists), ' + 'verify the config, generate tests, generate problem ' + 'statements, run inwer and run all solutions. ' + 'Ingen and inwer are compiled with sanitizers (-fsanitize=address,undefined), ' + 'which may fail on some systems. To fix this, run `sudo sysctl vm.mmap_rnd_bits = 28` ' + 'or disable sanitizers with --no-fsanitize.' + ) + + parser.add_argument('-f', '--no-fsanitize', action='store_true', default=False, + help='do not use sanitizers for ingen and inwer programs') + parser.add_argument('-c', '--cpus', type=int, + help=f'number of cpus that sinol-make will use ' + f'(default: {util.default_cpu_count()})', + default=util.default_cpu_count()) + parser.add_argument('--ignore-expected', dest='ignore_expected', action='store_true', + help='ignore expected scores from config.yml. When this flag is set, ' + 'the expected scores are not compared with the actual scores. ' + 'This flag will be passed to the run command.') + parsers.add_compilation_arguments(parser) + + def remove_cache(self): + """ + Remove whole cache dir + """ + cache_dir = paths.get_cache_path() + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + + def check_extra_files(self): + """ + Checks if extra_compilation_files and extra_execution_files exist. + """ + extra_compilation_files = self.config.get('extra_compilation_files', []) + for file in extra_compilation_files: + if not os.path.exists(os.path.join(os.getcwd(), "prog", file)): + util.exit_with_error(f"Extra compilation file `{file}` does not exist. " + f"It should be in `prog` directory.") + if extra_compilation_files: + print(util.info("All extra compilation files exist.")) + + extra_execution_files = self.config.get('extra_execution_files', {}) + for lang, files in extra_execution_files.items(): + for file in files: + if not os.path.exists(os.path.join(os.getcwd(), "prog", file)): + util.exit_with_error(f"Extra execution file `{file}` for language `{lang}` does not exist. " + f"It should be in `prog` directory.") + if extra_execution_files: + print(util.info("All extra execution files exist.")) + + def verify_scores(self, scored_groups): + config_scores = self.config.get('scores', {}) + if not config_scores: + return + if '0' in scored_groups: + scored_groups.remove('0') + if 0 in scored_groups: + scored_groups.remove(0) + + for group in scored_groups: + if int(group) not in config_scores: + util.exit_with_error(f"Score for group '{group}' not found. " + f"You must either provide scores for all groups " + f"or not provide them at all (to have them assigned automatically).") + + for group in config_scores: + if int(group) not in scored_groups: + util.exit_with_error(f"Score for group '{group}' found in config, " + f"but no such test group exists in scored groups. " + f"You must either provide scores for all groups " + f"or not provide them at all (to have them assigned automatically).") + + print(util.info("All scores are provided for all groups.")) + + def prepare_args(self, command): + parser = argparse.ArgumentParser() + subparser = parser.add_subparsers(dest='command') + command_parser = command.configure_subparser(subparser) + command_args = command_parser.parse_args([]) + for key, value in vars(self.args).items(): + setattr(command_args, key, value) + setattr(command_args, 'fsanitize', not self.args.no_fsanitize) + return command_args + + def run_stresstests(self): + stresstests_path = os.path.join(os.getcwd(), 'prog', self.task_id + 'stresstest.sh') + if not os.path.exists(stresstests_path): + return + + print(util.bold(' Running stress tests '.center(util.get_terminal_size()[1], '='))) + print(f"See the comments in `prog/{self.task_id}stresstest.sh` for details.".center( + util.get_terminal_size()[1], ' ')) + st = os.stat(stresstests_path) + os.chmod(stresstests_path, st.st_mode | stat.S_IEXEC) + p = subprocess.Popen([stresstests_path], shell=True) + p.wait() + if p.returncode != 0: + util.exit_with_error("Stress tests failed.") + + def run(self, args: argparse.Namespace): + self.args = util.init_package_command(args) + self.config = package_util.get_config() + self.task_id = package_util.get_task_id() + self.contest = contest_types.get_contest_type() + + self.remove_cache() + self.check_extra_files() + self.contest.verify_pre_gen() + + # Run stresstests (if present) + self.run_stresstests() + + # Generate tests + print(util.bold(' Generating tests '.center(util.get_terminal_size()[1], '='))) + gen = GenCommand() + gen.run(self.prepare_args(gen)) + self.verify_scores(package_util.get_groups(package_util.get_all_inputs(self.task_id), self.task_id)) + + # Generate problem statements + print(util.bold(' Generating problem statements '.center(util.get_terminal_size()[1], '='))) + doc = DocCommand() + doc.run(self.prepare_args(doc)) + + # Run inwer + if inwer_util.get_inwer_path(self.task_id) is None: + print(util.warning("Package doesn't have inwer.")) + else: + print(util.bold(' Running inwer '.center(util.get_terminal_size()[1], '='))) + inwer = InwerCommand() + inwer.run(self.prepare_args(inwer)) + + # Run solutions + print(util.bold(' Running solutions '.center(util.get_terminal_size()[1], '='))) + run = RunCommand() + run.run(self.prepare_args(run)) + + print(util.info('Package verification successful.')) diff --git a/src/sinol_make/contest_types/default.py b/src/sinol_make/contest_types/default.py index 07015957..5e4a3be3 100644 --- a/src/sinol_make/contest_types/default.py +++ b/src/sinol_make/contest_types/default.py @@ -64,7 +64,7 @@ def get_possible_score(self, groups: List[int], scores: Dict[int, int]) -> int: :param scores: Dictionary: {"": } :return: Maximum possible score. """ - if groups[0] == 0 and len(groups) == 1: + if len(groups) == 0 or (groups[0] == 0 and len(groups) == 1): return 0 possible_score = 0 @@ -126,3 +126,9 @@ def max_score_per_test(self): Returns maximum score for single test """ return 100 + + def verify_pre_gen(self): + """ + Called by verify command before generating tests. + """ + pass diff --git a/src/sinol_make/contest_types/oi.py b/src/sinol_make/contest_types/oi.py index ea4f0bb4..fbe90930 100644 --- a/src/sinol_make/contest_types/oi.py +++ b/src/sinol_make/contest_types/oi.py @@ -1,5 +1,7 @@ import argparse +from sinol_make import util +from sinol_make.helpers import package_util from sinol_make.structs.status_structs import ExecutionResult from sinol_make.contest_types.default import DefaultContest @@ -28,3 +30,14 @@ def get_test_score(self, result: ExecutionResult, time_limit, memory_limit): return result.Points else: return 1 + int((result.Points - 1) * ((time_limit - result.Time) / (time_limit / 2.0))) + + def verify_pre_gen(self): + """ + Verify if scores sum up to 100. + """ + config = package_util.get_config() + if 'scores' not in config: + return + total_score = sum(config['scores'].values()) + if total_score != 100: + util.exit_with_error(f"Total score in config is {total_score}, but should be 100.") diff --git a/src/sinol_make/helpers/cache.py b/src/sinol_make/helpers/cache.py index 9f405c4f..cca819fd 100644 --- a/src/sinol_make/helpers/cache.py +++ b/src/sinol_make/helpers/cache.py @@ -35,7 +35,7 @@ def get_cache_file(solution_path: str) -> CacheFile: return CacheFile() -def check_compiled(file_path: str) -> Union[str, None]: +def check_compiled(file_path: str, compilation_flags: str, sanitizers: bool) -> Union[str, None]: """ Check if a file is compiled :param file_path: Path to the file @@ -44,7 +44,7 @@ def check_compiled(file_path: str) -> Union[str, None]: md5sum = util.get_file_md5(file_path) try: info = get_cache_file(file_path) - if info.md5sum == md5sum: + if info.md5sum == md5sum and info.compilation_flags == compilation_flags and info.sanitizers == sanitizers: exe_path = info.executable_path if os.path.exists(exe_path): return exe_path @@ -53,19 +53,18 @@ def check_compiled(file_path: str) -> Union[str, None]: return None -def save_compiled(file_path: str, exe_path: str, is_checker: bool = False): +def save_compiled(file_path: str, exe_path: str, compilation_flags: str, sanitizers: bool, is_checker: bool = False): """ Save the compiled executable path to cache in `.cache/md5sums/`, which contains the md5sum of the file and the path to the executable. :param file_path: Path to the file :param exe_path: Path to the compiled executable + :param compilation_flags: Compilation flags used + :param sanitizers: Whether -fsanitize=undefined,address was used :param is_checker: Whether the compiled file is a checker. If True, all cached tests are removed. """ - info = CacheFile() - info.executable_path = exe_path - info.md5sum = util.get_file_md5(file_path) + info = CacheFile(util.get_file_md5(file_path), exe_path, compilation_flags, sanitizers) info.save(file_path) - if is_checker: remove_results_cache() diff --git a/src/sinol_make/helpers/compile.py b/src/sinol_make/helpers/compile.py index 07500090..d3dc8958 100644 --- a/src/sinol_make/helpers/compile.py +++ b/src/sinol_make/helpers/compile.py @@ -37,10 +37,17 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp if use_fsanitize and util.is_macos_arm(): use_fsanitize = False + if compilation_flags == 'w': + compilation_flags = 'weak' + elif compilation_flags == 'o': + compilation_flags = 'oioioi' + elif compilation_flags == 'd': + compilation_flags = 'default' + if extra_compilation_files is None: extra_compilation_files = [] - compiled_exe = check_compiled(program) + compiled_exe = check_compiled(program, compilation_flags, use_fsanitize) if compiled_exe is not None: if compile_log is not None: compile_log.write(f'Using cached executable {compiled_exe}\n') @@ -53,12 +60,11 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp shutil.copy(file, os.path.join(os.path.dirname(output), os.path.basename(file))) gcc_compilation_flags = '' - if compilation_flags == 'weak' or compilation_flags == 'w': - compilation_flags = 'weak' + if compilation_flags == 'weak': gcc_compilation_flags = '' # Disable all warnings - elif compilation_flags == 'oioioi' or compilation_flags == 'o': + elif compilation_flags == 'oioioi': gcc_compilation_flags = ' -Wall -Wno-unused-result -Werror' # Same flags as oioioi - elif compilation_flags == 'default' or compilation_flags == 'd': + elif compilation_flags == 'default': gcc_compilation_flags = ' -Werror -Wall -Wextra -Wshadow -Wconversion -Wno-unused-result -Wfloat-equal' else: util.exit_with_error(f'Unknown compilation flags group: {compilation_flags}') @@ -108,7 +114,7 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp if process.returncode != 0: raise CompilationError('Compilation failed') else: - save_compiled(program, output, is_checker) + save_compiled(program, output, compilation_flags, use_fsanitize, is_checker) return True diff --git a/src/sinol_make/helpers/compiler.py b/src/sinol_make/helpers/compiler.py index 6b11bc36..8a1a68a4 100644 --- a/src/sinol_make/helpers/compiler.py +++ b/src/sinol_make/helpers/compiler.py @@ -1,12 +1,12 @@ from typing import List - +import subprocess import argparse +import sys import os -import sinol_make.util as util -import sys, subprocess - +from sinol_make import util from sinol_make.structs.compiler_structs import Compilers +from sinol_make.helpers.func_cache import cache_result def check_if_installed(compiler): @@ -22,6 +22,7 @@ def check_if_installed(compiler): return True +@cache_result() def get_c_compiler_path(): """ Get the C compiler @@ -41,6 +42,7 @@ def get_c_compiler_path(): return None +@cache_result() def get_cpp_compiler_path(): """ Get the C++ compiler @@ -62,6 +64,7 @@ def get_cpp_compiler_path(): return None +@cache_result() def get_python_interpreter_path(): """ Get the Python interpreter @@ -75,6 +78,7 @@ def get_python_interpreter_path(): return None +@cache_result() def get_java_compiler_path(): """ Get the Java compiler @@ -94,7 +98,8 @@ def get_default_compilers(): c_compiler_path=get_c_compiler_path(), cpp_compiler_path=get_cpp_compiler_path(), python_interpreter_path=get_python_interpreter_path(), - java_compiler_path=get_java_compiler_path() + # Java is not currently supported by sinol-make + # java_compiler_path=get_java_compiler_path() ) @@ -141,5 +146,5 @@ def verify_compilers(args: argparse.Namespace, solutions: List[str]) -> Compiler c_compiler_path=args.c_compiler_path, cpp_compiler_path=args.cpp_compiler_path, python_interpreter_path=args.python_interpreter_path, - java_compiler_path=args.java_compiler_path + java_compiler_path=None ) diff --git a/src/sinol_make/helpers/func_cache.py b/src/sinol_make/helpers/func_cache.py new file mode 100644 index 00000000..6b493489 --- /dev/null +++ b/src/sinol_make/helpers/func_cache.py @@ -0,0 +1,30 @@ +import os + +__cache = {} + + +def cache_result(cwd=False): + """ + Function to cache the result of a function. + """ + def decorator(func): + def wrapper(*args, **kwargs): + if cwd: + key = (func.__name__, os.getcwd()) + else: + key = func.__name__ + + if key in __cache: + return __cache[key] + result = func(*args, **kwargs) + __cache[key] = result + return result + return wrapper + return decorator + + +def clear_cache(): + """ + Function to clear the cache. + """ + __cache.clear() diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index 52dab16a..98d582b8 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -7,10 +7,12 @@ from enum import Enum from typing import List, Union, Dict, Any, Tuple +from sinol_make.helpers.func_cache import cache_result from sinol_make import util from sinol_make.helpers import paths +@cache_result(cwd=True) def get_task_id() -> str: config = get_config() if "sinol_task_id" in config: @@ -267,6 +269,14 @@ def get_memory_limit(test_path, config, lang, task_id, args=None): return _get_limit(LimitTypes.MEMORY_LIMIT, test_path, str_config, lang, task_id) +def get_in_tests_re(task_id: str) -> re.Pattern: + return re.compile(r'^%s(([0-9]+)([a-z]?[a-z0-9]*))\.in$' % re.escape(task_id)) + + +def get_out_tests_re(task_id: str) -> re.Pattern: + return re.compile(r'^%s(([0-9]+)([a-z]?[a-z0-9]*))\.out$' % re.escape(task_id)) + + def validate_test_names(task_id): """ Checks if all files in the package have valid names. @@ -279,12 +289,12 @@ def get_invalid_files(path, pattern): invalid_files.append(os.path.basename(file)) return invalid_files - in_test_re = re.compile(r'^(%s(([0-9]+)([a-z]?[a-z0-9]*))).in$' % (re.escape(task_id))) + in_test_re = get_in_tests_re(task_id) invalid_in_tests = get_invalid_files(os.path.join("in", "*.in"), in_test_re) if len(invalid_in_tests) > 0: util.exit_with_error(f'Input tests with invalid names: {", ".join(invalid_in_tests)}.') - out_test_re = re.compile(r'^(%s(([0-9]+)([a-z]?[a-z0-9]*))).out$' % (re.escape(task_id))) + out_test_re = get_out_tests_re(task_id) invalid_out_tests = get_invalid_files(os.path.join("out", "*.out"), out_test_re) if len(invalid_out_tests) > 0: util.exit_with_error(f'Output tests with invalid names: {", ".join(invalid_out_tests)}.') @@ -399,3 +409,12 @@ def validate_tests(tests: List[str], cpus: int, type: str = 'input'): print(f'Validated {finished}/{num_tests} tests', end='\r') print() print(util.info(f'All {type} tests are valid!')) + + +def get_all_inputs(task_id): + in_test_re = get_in_tests_re(task_id) + inputs = [] + for file in glob.glob(os.path.join(os.getcwd(), "in", "*.in")): + if in_test_re.match(os.path.basename(file)): + inputs.append(file) + return inputs diff --git a/src/sinol_make/helpers/parsers.py b/src/sinol_make/helpers/parsers.py index 83f1ae74..696cb339 100644 --- a/src/sinol_make/helpers/parsers.py +++ b/src/sinol_make/helpers/parsers.py @@ -1,7 +1,7 @@ import sys - import argparse +from sinol_make import util from sinol_make.helpers import compiler @@ -20,8 +20,9 @@ def add_compilation_arguments(parser: argparse.ArgumentParser): parser.add_argument('--python-interpreter-path', dest='python_interpreter_path', type=str, default=compiler.get_python_interpreter_path(), help='Python interpreter to use (default: python3)') - parser.add_argument('--java-compiler-path', dest='java_compiler_path', type=str, - default=compiler.get_java_compiler_path(), help='Java compiler to use (default: javac)') + # Java is not currently supported by sinol-make + # parser.add_argument('--java-compiler-path', dest='java_compiler_path', type=str, + # default=compiler.get_java_compiler_path(), help='Java compiler to use (default: javac)') parser.add_argument('--compile-mode', '-C', dest='compile_mode', choices=['default', 'oioioi', 'weak', 'd', 'o', 'w'], help='Warning flag groups used to compile C/C++ files. Available options:\n' ' default / d - uses default flags: \n' @@ -29,3 +30,10 @@ def add_compilation_arguments(parser: argparse.ArgumentParser): ' oioioi / o - uses the same flags as oioioi:\n' ' (-Wall -Wno-unused-result -Werror)' ' weak / w - disable all warning flags during C and C++ compilation', default='default') + + +def add_cpus_argument(parser: argparse.ArgumentParser, help: str): + parser.add_argument('-c', '--cpus', type=int, + help=f'{help} ' + f'(default: {util.default_cpu_count()})', + default=util.default_cpu_count()) diff --git a/src/sinol_make/structs/cache_structs.py b/src/sinol_make/structs/cache_structs.py index 5d1f3933..249bbdc8 100644 --- a/src/sinol_make/structs/cache_structs.py +++ b/src/sinol_make/structs/cache_structs.py @@ -43,20 +43,28 @@ class CacheFile: md5sum: str # Path to the executable executable_path: str + # Compilation flags used + compilation_flags: str + # Whether -fsanitize=undefined,address was used + sanitizers: bool # Test results tests: Dict[str, CacheTest] - def __init__(self, md5sum="", executable_path="", tests=None): + def __init__(self, md5sum="", executable_path="", compilation_flags="default", sanitizers=False, tests=None): if tests is None: tests = {} self.md5sum = md5sum self.executable_path = executable_path + self.compilation_flags = compilation_flags + self.sanitizers = sanitizers self.tests = tests def to_dict(self) -> Dict: return { "md5sum": self.md5sum, "executable_path": self.executable_path, + "compilation_flags": self.compilation_flags, + "sanitizers": self.sanitizers, "tests": {k: v.to_dict() for k, v in self.tests.items()} } @@ -65,6 +73,8 @@ def from_dict(dict) -> 'CacheFile': return CacheFile( md5sum=dict.get("md5sum", ""), executable_path=dict.get("executable_path", ""), + compilation_flags=dict.get("compilation_flags", "default"), + sanitizers=dict.get("sanitizers", False), tests={k: CacheTest( time_limit=v["time_limit"], memory_limit=v["memory_limit"], diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py index a8714b52..58548f30 100644 --- a/src/sinol_make/util.py +++ b/src/sinol_make/util.py @@ -1,28 +1,24 @@ import glob, importlib, os, sys, requests, yaml import math -import multiprocessing import platform import tarfile import hashlib import multiprocessing import resource from typing import Union +from packaging.version import parse as parse_version from sinol_make.contest_types import get_contest_type from sinol_make.helpers import paths, cache +from sinol_make.helpers.func_cache import cache_result from sinol_make.structs.status_structs import Status -__cache = {} - - +@cache_result() def get_commands(): """ Function to get an array of all available commands. """ - global __cache - if 'commands' in __cache: - return __cache['commands'] commands_path = glob.glob( os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -34,7 +30,6 @@ def get_commands(): temp = importlib.import_module('sinol_make.commands.' + os.path.basename(path), 'Command') commands.append(temp.Command()) - __cache['commands'] = commands return commands @@ -150,10 +145,18 @@ def import_importlib_resources(): return importlib -def check_for_updates(current_version) -> Union[str, None]: +def is_dev(version): + """ + Function to check if the version is a development version. + """ + return parse_version(version).is_devrelease + + +def check_for_updates(current_version, check=True) -> Union[str, None]: """ Function to check if there is a new version of sinol-make. :param current_version: current version of sinol-make + :param check: whether to check for new version :return: returns new version if there is one, None otherwise """ importlib = import_importlib_resources() @@ -164,8 +167,9 @@ def check_for_updates(current_version) -> Union[str, None]: # We check for new version asynchronously, so that it doesn't slow down the program. # If the main process exits, the check_version process will also exit. - process = multiprocessing.Process(target=check_version, daemon=True) - process.start() + if check: + process = multiprocessing.Process(target=check_version, daemon=True) + process.start() version_file = data_dir.joinpath("version") try: @@ -178,7 +182,9 @@ def check_for_updates(current_version) -> Union[str, None]: return None try: - if compare_versions(current_version, version) == -1: + if not is_dev(version) and parse_version(version) > parse_version(current_version): + return version + if is_dev(current_version) and is_dev(version) and parse_version(version) > parse_version(current_version): return version else: return None @@ -202,7 +208,9 @@ def check_version(): return data = request.json() - latest_version = data["info"]["version"] + versions = list(data["releases"].keys()) + versions.sort(key=parse_version) + latest_version = versions[-1] version_file = importlib.files("sinol_make").joinpath("data/version") try: @@ -217,26 +225,6 @@ def check_version(): pass -def compare_versions(version_a, version_b): - """ - Function to compare two versions. - Returns 1 if version_a > version_b, 0 if version_a == version_b, -1 if version_a < version_b. - """ - - def convert(version): - return tuple(map(int, version.split("."))) - - version_a = convert(version_a) - version_b = convert(version_b) - - if version_a > version_b: - return 1 - elif version_a == version_b: - return 0 - else: - return -1 - - def lines_diff(lines1, lines2): """ Function to compare two lists of lines. @@ -441,3 +429,7 @@ def exit_with_error(text, func=None): except TypeError: pass exit(1) + + +def has_sanitizer_error(output, exit_code): + return ('ELF_ET_DYN_BASE' in output or 'ASan' in output) and exit_code != 0 diff --git a/tests/commands/export/test_integration.py b/tests/commands/export/test_integration.py index a7a73647..1cc5196a 100644 --- a/tests/commands/export/test_integration.py +++ b/tests/commands/export/test_integration.py @@ -212,3 +212,50 @@ def test_no_ocen(create_package): with tarfile.open(f'{task_id}.tgz', "r") as tar: sinol_util.extract_tar(tar, tmpdir) assert not os.path.exists(os.path.join(tmpdir, task_id, "attachments", f"{task_id}ocen.zip")) + + +@pytest.mark.parametrize("create_package", [util.get_dlazaw_package()], indirect=True) +def test_no_ocen_and_dlazaw(create_package): + """ + Test if ocen archive is not created when there are no ocen tests. + Also test if dlazaw archive is created. + """ + parser = configure_parsers() + args = parser.parse_args(["export", "--no-statement"]) + command = Command() + command.run(args) + task_id = package_util.get_task_id() + + with tempfile.TemporaryDirectory() as tmpdir: + with tarfile.open(f'{task_id}.tgz', "r") as tar: + sinol_util.extract_tar(tar, tmpdir) + + assert not os.path.exists(os.path.join(tmpdir, task_id, "attachments", f"{task_id}ocen.zip")) + dlazaw_archive = os.path.join(tmpdir, task_id, "attachments", f"dlazaw.zip") + assert os.path.exists(dlazaw_archive) + + with zipfile.ZipFile(dlazaw_archive, "r") as zip: + zip.extractall(os.path.join(tmpdir)) + + assert os.path.exists(os.path.join(tmpdir, "dlazaw")) + assert os.path.exists(os.path.join(tmpdir, "dlazaw", "epic_file")) + + +@pytest.mark.parametrize("create_package", [util.get_ocen_package_path()], indirect=True) +def test_dlazaw_ocen(create_package): + """ + Test if ocen archive isn't created when dlazaw directory exists + """ + os.makedirs("dlazaw") + parser = configure_parsers() + args = parser.parse_args(["export", "--no-statement"]) + command = Command() + command.run(args) + task_id = package_util.get_task_id() + + with tempfile.TemporaryDirectory() as tmpdir: + with tarfile.open(f'{task_id}.tgz', "r") as tar: + sinol_util.extract_tar(tar, tmpdir) + + assert not os.path.exists(os.path.join(tmpdir, task_id, "attachments", f"{task_id}ocen.zip")) + assert os.path.join(tmpdir, task_id, "attachments", f"dlazaw.zip") diff --git a/tests/commands/gen/test_integration.py b/tests/commands/gen/test_integration.py index da8c5fa7..0fb0dc70 100644 --- a/tests/commands/gen/test_integration.py +++ b/tests/commands/gen/test_integration.py @@ -60,7 +60,7 @@ def test_correct_inputs(capsys, create_package): """ task_id = package_util.get_task_id() correct_solution = package_util.get_correct_solution(task_id) - cache.save_compiled(correct_solution, "exe") + cache.save_compiled(correct_solution, "exe", "default", False) simple_run() md5_sums = get_md5_sums(create_package) @@ -79,7 +79,7 @@ def test_changed_inputs(capsys, create_package): """ task_id = package_util.get_task_id() correct_solution = package_util.get_correct_solution(task_id) - cache.save_compiled(correct_solution, "exe") + cache.save_compiled(correct_solution, "exe", "default", False) simple_run() md5_sums = get_md5_sums(create_package) correct_md5 = md5_sums.copy() @@ -245,7 +245,7 @@ def test_fsanitize(create_package): pytest.skip("-fsanitize=address,undefined is not supported on Apple Silicon") for ingen in ["prog/geningen3.cpp", "prog/geningen4.cpp"]: with pytest.raises(SystemExit) as e: - simple_run([ingen]) + simple_run(["--fsanitize", ingen]) assert e.type == SystemExit assert e.value.code == 1 @@ -337,3 +337,34 @@ def test_outgen_cache_cleaning(create_package, capsys): simple_run(command="outgen") # Run should pass, because output file was regenerated and cache for this test was cleaned. RunCommand().run(args) + + +@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True) +def test_cache_remove_after_flags_change(create_package): + """ + Test if cache for a program is removed if compilation flags change or -fsanitize is disabled. + """ + def random_key_to_cache(): + cache_file = cache.get_cache_file("abcingen.cpp") + print(cache_file) + cache_dict = cache_file.to_dict() + cache_dict["random_key"] = "random_value" + with open(paths.get_cache_path("md5sums", "abcingen.cpp"), "w") as f: + yaml.dump(cache_dict, f) + + def check_assert(): + with open(paths.get_cache_path("md5sums", "abcingen.cpp"), "r") as f: + cache_dict = yaml.load(f, Loader=yaml.FullLoader) + assert "random_key" not in cache_dict + + # Generate cache + simple_run(command="gen") + random_key_to_cache() + simple_run(["--compile-mode", "oioioi"], command="gen") + check_assert() + + # Generate cache + simple_run(command="gen") + random_key_to_cache() + simple_run(["--fsanitize"], command="gen") + check_assert() diff --git a/tests/commands/inwer/test_integration.py b/tests/commands/inwer/test_integration.py index 9beb1297..3135003a 100644 --- a/tests/commands/inwer/test_integration.py +++ b/tests/commands/inwer/test_integration.py @@ -19,12 +19,7 @@ def test_default(capsys, create_package): parser = configure_parsers() args = parser.parse_args(["inwer"]) command = Command() - - with pytest.raises(SystemExit) as e: - command.run(args) - - assert e.type == SystemExit - assert e.value.code == 0 + command.run(args) out = capsys.readouterr().out assert "Verification successful." in out @@ -42,12 +37,7 @@ def test_specified_inwer(capsys, create_package): for inwer_path in ["prog/werinwer.cpp", "prog/werinwer7.cpp"]: args = parser.parse_args(["inwer", inwer_path]) command = Command() - - with pytest.raises(SystemExit) as e: - command.run(args) - - assert e.type == SystemExit - assert e.value.code == 0 + command.run(args) out = capsys.readouterr().out assert "Verification successful." in out @@ -98,12 +88,7 @@ def test_flag_tests(capsys, create_package): parser = configure_parsers() args = parser.parse_args(["inwer", "prog/werinwer.cpp", "--tests", "in/wer2a.in"]) command = Command() - - with pytest.raises(SystemExit) as e: - command.run(args) - - assert e.type == SystemExit - assert e.value.code == 0 + command.run(args) out = capsys.readouterr().out assert "Verification successful." in out @@ -137,10 +122,7 @@ def test_no_output(capsys, create_package): parser = configure_parsers() args = parser.parse_args(["inwer", "prog/werinwer4.cpp"]) command = Command() - - with pytest.raises(SystemExit) as e: - command.run(args) - assert e.value.code == 0 + command.run(args) out = capsys.readouterr().out assert "No output" in out @@ -154,7 +136,7 @@ def test_fsanitize(create_package): pytest.skip("-fsanitize=address,undefined is not supported on Apple Silicon") for inwer in ["prog/werinwer5.cpp", "prog/werinwer6.cpp"]: parser = configure_parsers() - args = parser.parse_args(["inwer", inwer]) + args = parser.parse_args(["inwer", "--fsanitize", inwer]) command = Command() with pytest.raises(SystemExit) as e: command.run(args) diff --git a/tests/commands/run/test_integration.py b/tests/commands/run/test_integration.py index da17eb97..bbca5927 100644 --- a/tests/commands/run/test_integration.py +++ b/tests/commands/run/test_integration.py @@ -785,3 +785,30 @@ def test_ghost_checker(create_package): with pytest.raises(SystemExit) as e: command.run(args) assert e.value.code == 1 + + +@pytest.mark.parametrize("create_package", [get_simple_package_path()], indirect=True) +def test_ignore_expected_flag(create_package, capsys): + """ + Test flag --ignore-expected. + """ + config = package_util.get_config() + del config["sinol_expected_scores"] + util.save_config(config) + + package_path = create_package + create_ins_outs(package_path) + parser = configure_parsers() + args = parser.parse_args(["run"]) + command = Command() + + with pytest.raises(SystemExit): + command.run(args) + out = capsys.readouterr().out + assert "Use flag --apply-suggestions to apply suggestions." in out + + args = parser.parse_args(["run", "--ignore-expected"]) + command = Command() + command.run(args) + out = capsys.readouterr().out + assert "Ignoring expected scores." in out diff --git a/tests/commands/verify/__init__.py b/tests/commands/verify/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/commands/verify/test_integration.py b/tests/commands/verify/test_integration.py new file mode 100644 index 00000000..68a54660 --- /dev/null +++ b/tests/commands/verify/test_integration.py @@ -0,0 +1,116 @@ +import os +import pytest + +from sinol_make import configure_parsers +from sinol_make import util as sm_util +from sinol_make.commands.verify import Command +from sinol_make.helpers import package_util +from tests import util +from tests.fixtures import create_package + + +def run(args=None): + if args is None: + args = [] + parser = configure_parsers() + command = Command() + args = parser.parse_args(['verify'] + args) + command.run(args) + + +@pytest.mark.parametrize("create_package", [util.get_simple_package_path(), util.get_inwer_package_path()], + indirect=True) +def test_simple_package(capsys, create_package): + """ + Test if simple package runs successfully. + """ + run() + + +@pytest.mark.parametrize("create_package", [util.get_stresstest_package_path()], indirect=True) +def test_stresstest_package(capsys, create_package): + """ + Test if stresstest.sh script runs. Then check if after failing the stresstest.sh script, the verify command + will fail as well. + """ + run() + out = capsys.readouterr().out + assert "Running stress tests" in out + + stresstest_path = os.path.join(create_package, "prog", "strstresstest.sh") + with open(stresstest_path, "r") as f: + code = f.read() + with open(stresstest_path, "w") as f: + f.write(code.replace("exit 0", "exit 1")) + + with pytest.raises(SystemExit) as e: + run() + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Running stress tests" in out + assert "Stress tests failed." in out + + +@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True) +def test_missing_extra_files(capsys, create_package): + """ + Test if missing extra files will cause the verify command to fail. + """ + config = package_util.get_config() + config["extra_compilation_files"] = ["missing_file"] + sm_util.save_config(config) + with pytest.raises(SystemExit) as e: + run() + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Extra compilation file `missing_file` does not exist." in out + + del config["extra_compilation_files"] + config["extra_execution_files"] = {"cpp": ["missing_file"]} + sm_util.save_config(config) + with pytest.raises(SystemExit) as e: + run() + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Extra execution file `missing_file` for language `cpp` does not exist." in out + + +@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True) +def test_invalid_scores(capsys, create_package): + """ + Test if invalid scores will cause the verify command to fail. + """ + config = package_util.get_config() + scores = config["scores"] + config["scores"] = {1: 100} + sm_util.save_config(config) + with pytest.raises(SystemExit) as e: + run() + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Score for group '2' not found." in out + + config["scores"] = scores + config["scores"][20] = 0 + sm_util.save_config(config) + with pytest.raises(SystemExit) as e: + run() + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Score for group '20' found in config" in out + + +@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True) +def test_scores_not_100(capsys, create_package): + """ + Test if scores not adding up to 100 will cause the verify command to fail. + """ + config = package_util.get_config() + config["sinol_contest_type"] = "oi" + config["scores"][1] -= 1 + sm_util.save_config(config) + with pytest.raises(SystemExit) as e: + run() + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Total score in config is 99, but should be 100." in out diff --git a/tests/helpers/test_cache.py b/tests/helpers/test_cache.py index be2423c7..52c37c77 100644 --- a/tests/helpers/test_cache.py +++ b/tests/helpers/test_cache.py @@ -13,25 +13,33 @@ def test_compilation_caching(): program = os.path.join(tmpdir, 'program.cpp') open(program, 'w').write('int main() { return 0; }') - assert cache.check_compiled(program) is None + assert cache.check_compiled(program, "default", False) is None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - exe_path = cache.check_compiled(program) + exe_path = cache.check_compiled(program, "default", False) assert exe_path is not None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - exe_path2 = cache.check_compiled(program) + exe_path2 = cache.check_compiled(program, "default", False) assert exe_path2 == exe_path open(program, 'w').write('int main() { return 1; }') - assert cache.check_compiled(program) is None + assert cache.check_compiled(program, "default", False) is None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - assert cache.check_compiled(program) is not None + assert cache.check_compiled(program, "default", False) is not None open(program, 'w').write('int main() { return 0; }') - assert cache.check_compiled(program) is None + assert cache.check_compiled(program, "default", False) is None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - assert cache.check_compiled(program) is not None + assert cache.check_compiled(program, "default", False) is not None + + assert cache.check_compiled(program, "default", True) is None + cache.save_compiled(program, exe_path, "default", True) + assert cache.check_compiled(program, "default", True) is not None + + assert cache.check_compiled(program, "oioioi", True) is None + cache.save_compiled(program, exe_path, "oioioi", True) + assert cache.check_compiled(program, "oioioi", True) is not None def test_cache(): @@ -72,7 +80,8 @@ def test_cache(): f.write("int main() { return 0; }") cache_file.save("abc.cpp") assert cache.get_cache_file("abc.cpp") == cache_file - cache.save_compiled("abc.cpp", "abc.e", is_checker=True) + cache.save_compiled("abc.cpp", "abc.e", "default", False, + is_checker=True) assert cache.get_cache_file("abc.cpp").tests == {} # Test that cache is cleared when extra compilation files change diff --git a/tests/helpers/test_compile.py b/tests/helpers/test_compile.py index 4447d93c..06703496 100644 --- a/tests/helpers/test_compile.py +++ b/tests/helpers/test_compile.py @@ -16,10 +16,11 @@ def test_compilation_caching(): with open(os.path.join(os.getcwd(), "test.e"), "w") as f: f.write("") - assert check_compiled(os.path.join(os.getcwd(), "test.txt")) is None + assert check_compiled(os.path.join(os.getcwd(), "test.txt"), "default", False) is None save_compiled(os.path.join(os.getcwd(), "test.txt"), - os.path.join(os.getcwd(), "test.e")) - assert check_compiled(os.path.join(os.getcwd(), "test.txt")) == os.path.join(os.getcwd(), "test.e") + os.path.join(os.getcwd(), "test.e"), "default", False) + assert check_compiled(os.path.join(os.getcwd(), "test.txt"), "default", False) == \ + os.path.join(os.getcwd(), "test.e") @pytest.mark.parametrize("create_package", [util.get_shell_ingen_pack_path()], indirect=True) diff --git a/tests/helpers/test_package_util.py b/tests/helpers/test_package_util.py index d821eb17..3d3538ce 100644 --- a/tests/helpers/test_package_util.py +++ b/tests/helpers/test_package_util.py @@ -3,7 +3,7 @@ from ..commands.run.util import create_ins from ..fixtures import * from tests import util -from sinol_make.helpers import package_util +from sinol_make.helpers import package_util, func_cache @pytest.mark.parametrize("create_package", [util.get_long_name_package_path()], indirect=True) @@ -12,6 +12,7 @@ def test_get_task_id(create_package): assert package_util.get_task_id() == "lpn" with open(os.path.join(package_path, "config.yml"), "w") as config_file: config_file.write("title: Long package name\n") + func_cache.clear_cache() with pytest.raises(SystemExit): package_util.get_task_id() diff --git a/tests/packages/dlazaw/config.yml b/tests/packages/dlazaw/config.yml new file mode 100644 index 00000000..8caed113 --- /dev/null +++ b/tests/packages/dlazaw/config.yml @@ -0,0 +1,5 @@ +title: Package with no ocen files and dlazaw dir +sinol_task_id: dla +sinol_contest_type: oi +time_limit: 1000 +memory_limit: 10240 diff --git a/tests/packages/dlazaw/dlazaw/epic_file b/tests/packages/dlazaw/dlazaw/epic_file new file mode 100644 index 00000000..f81fce04 --- /dev/null +++ b/tests/packages/dlazaw/dlazaw/epic_file @@ -0,0 +1 @@ +this is a file diff --git a/tests/packages/dlazaw/in/.gitkeep b/tests/packages/dlazaw/in/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/dlazaw/out/.gitkeep b/tests/packages/dlazaw/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/dlazaw/prog/dla.cpp b/tests/packages/dlazaw/prog/dla.cpp new file mode 100644 index 00000000..9d30bab0 --- /dev/null +++ b/tests/packages/dlazaw/prog/dla.cpp @@ -0,0 +1,9 @@ +#include + +using namespace std; + +int main() { + int a, b; + cin >> a >> b; + cout << a + b; +} diff --git a/tests/packages/dlazaw/prog/dlaingen.cpp b/tests/packages/dlazaw/prog/dlaingen.cpp new file mode 100644 index 00000000..7c84e777 --- /dev/null +++ b/tests/packages/dlazaw/prog/dlaingen.cpp @@ -0,0 +1,9 @@ +#include + +using namespace std; + +int main() { + ofstream f("ocen1a.in"); + f << "1 1\n"; + f.close(); +} diff --git a/tests/packages/stresstest/config.yml b/tests/packages/stresstest/config.yml new file mode 100644 index 00000000..2da7a23e --- /dev/null +++ b/tests/packages/stresstest/config.yml @@ -0,0 +1,11 @@ +title: Package with stresstest.sh (for `verify` command) +sinol_task_id: str +time_limit: 1000 +memory_limit: 10240 + +sinol_expected_scores: + str.cpp: + expected: + 1: {points: 50, status: OK} + 2: {points: 50, status: OK} + points: 100 diff --git a/tests/packages/stresstest/in/.gitkeep b/tests/packages/stresstest/in/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/stresstest/out/.gitkeep b/tests/packages/stresstest/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/stresstest/prog/str.cpp b/tests/packages/stresstest/prog/str.cpp new file mode 100644 index 00000000..1e57f863 --- /dev/null +++ b/tests/packages/stresstest/prog/str.cpp @@ -0,0 +1,9 @@ +#include + +using namespace std; + +int main() { + int a, b; + cin >> a >> b; + cout << a + b << "\n"; +} diff --git a/tests/packages/stresstest/prog/stringen.cpp b/tests/packages/stresstest/prog/stringen.cpp new file mode 100644 index 00000000..e947e92a --- /dev/null +++ b/tests/packages/stresstest/prog/stringen.cpp @@ -0,0 +1,12 @@ +#include + +using namespace std; + +int main() { + ofstream f("str1a.in"); + f << "1 3\n"; + f.close(); + f.open("str2a.in"); + f << "2 5\n"; + f.close(); +} diff --git a/tests/packages/stresstest/prog/strstresstest.sh b/tests/packages/stresstest/prog/strstresstest.sh new file mode 100644 index 00000000..de5d8729 --- /dev/null +++ b/tests/packages/stresstest/prog/strstresstest.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo Very hard stress test +exit 0 diff --git a/tests/packages/wer/config.yml b/tests/packages/wer/config.yml index 388f9a4d..c5b12807 100644 --- a/tests/packages/wer/config.yml +++ b/tests/packages/wer/config.yml @@ -6,3 +6,11 @@ scores: 1: 33 2: 33 3: 34 + +sinol_expected_scores: + wer.cpp: + expected: + 1: {points: 33, status: OK} + 2: {points: 33, status: OK} + 3: {points: 34, status: OK} + points: 100 diff --git a/tests/packages/wer/out/.gitkeep b/tests/packages/wer/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/wer/prog/wer.cpp b/tests/packages/wer/prog/wer.cpp index 8c8f29dd..ac799965 100644 --- a/tests/packages/wer/prog/wer.cpp +++ b/tests/packages/wer/prog/wer.cpp @@ -11,5 +11,5 @@ int main() { s += a; } s *= n; - cout << s; + cout << s << "\n"; } diff --git a/tests/test_multiple_arguments.py b/tests/test_multiple_arguments.py index 4a508c70..4607a318 100644 --- a/tests/test_multiple_arguments.py +++ b/tests/test_multiple_arguments.py @@ -21,7 +21,7 @@ def test_simple_package(create_package): @pytest.mark.parametrize("create_package", [util.get_inwer_package_path()], indirect=True) def test_inwer_package(create_package): - run("sinol-make ingen inwer run") + run("sinol-make gen inwer run") run("sinol-make ingen prog/weringen.cpp inwer prog/werinwer.cpp --tests wer1a.in run --tests wer2a.in") run("sinol-make ingen inwer run export --no-statement") diff --git a/tests/test_util.py b/tests/test_util.py index e95f69bf..3d259fab 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -44,28 +44,6 @@ def test_file_diff(): assert util.file_diff(a_file, b_file) is False -def test_compare_versions(): - """ - Tests for compare_versions function - """ - - assert util.compare_versions('1.0.0', '1.0.0') == 0 - assert util.compare_versions('1.0.0', '1.0.1') == -1 - assert util.compare_versions('1.0.1', '1.0.0') == 1 - assert util.compare_versions('1.0.0', '1.1.0') == -1 - assert util.compare_versions('1.1.0', '1.0.0') == 1 - assert util.compare_versions('1.0.0', '2.0.0') == -1 - assert util.compare_versions('2.0.0', '1.0.0') == 1 - with pytest.raises(ValueError): - util.compare_versions('1.0.0', '') - with pytest.raises(ValueError): - util.compare_versions('', '1.0.0') - with pytest.raises(ValueError): - util.compare_versions('1.0.0', 'abc') - with pytest.raises(ValueError): - util.compare_versions('abc', '1.0.0') - - @requests_mock.Mocker(kw="mocker") def test_check_version(**kwargs): """ @@ -74,6 +52,32 @@ def test_check_version(**kwargs): """ mocker = kwargs["mocker"] + def create_response(version): + return {"releases": {version: "something"}} + + mocker.get("https://pypi.python.org/pypi/sinol-make/json", json=create_response("1.0.0.dev2")) + util.check_version() + version = util.check_for_updates("1.0.0.dev1", False) + assert version == "1.0.0.dev2" + assert util.is_dev(version) + + mocker.get("https://pypi.python.org/pypi/sinol-make/json", json=create_response("1.0.0")) + util.check_version() + version = util.check_for_updates("1.0.0.dev1", False) + assert version == "1.0.0" + assert not util.is_dev(version) + + mocker.get("https://pypi.python.org/pypi/sinol-make/json", json=create_response("2.0.0.dev1")) + util.check_version() + version = util.check_for_updates("1.0.0", False) + assert version is None + + mocker.get("https://pypi.python.org/pypi/sinol-make/json", json=create_response("1.0.1")) + util.check_version() + version = util.check_for_updates("1.0.0", False) + assert version == "1.0.1" + assert not util.is_dev(version) + importlib = util.import_importlib_resources() data_dir = importlib.files('sinol_make').joinpath("data") @@ -85,7 +89,7 @@ def test_check_version(**kwargs): version_file.unlink() # Test correct request - mocker.get("https://pypi.python.org/pypi/sinol-make/json", json={"info": {"version": "1.0.0"}}) + mocker.get("https://pypi.python.org/pypi/sinol-make/json", json=create_response(("1.0.0"))) util.check_version() assert version_file.is_file() assert version_file.read_text() == "1.0.0" diff --git a/tests/util.py b/tests/util.py index c29d99c3..d4158735 100644 --- a/tests/util.py +++ b/tests/util.py @@ -164,6 +164,20 @@ def get_bad_tests_package_path(): return os.path.join(os.path.dirname(__file__), "packages", "bad_tests") +def get_dlazaw_package(): + """ + Get path to package with dlazaw dir and no ocen tests (/tests/packages/dlazaw) + """ + return os.path.join(os.path.dirname(__file__), "packages", "dlazaw") + + +def get_stresstest_package_path(): + """ + Get path to package with stresstest.sh (/tests/packages/stresstest) + """ + return os.path.join(os.path.dirname(__file__), "packages", "stresstest") + + def create_ins(package_path, task_id): """ Create .in files for package. @@ -202,6 +216,7 @@ def create_ins_outs(package_path): """ os.chdir(package_path) task_id = package_util.get_task_id() + print(task_id) create_ins(package_path, task_id) has_lib = package_util.any_files_matching_pattern(task_id, f"{task_id}lib.*") if not has_lib: