From b95c7e2ac69c4f15327d7d3cf3c737b5d5adad1e Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Fri, 19 Jul 2024 20:01:38 +0300 Subject: [PATCH] Interactive via IO support --- src/sinol_make/commands/gen/__init__.py | 6 + src/sinol_make/commands/outgen/__init__.py | 4 + src/sinol_make/commands/run/__init__.py | 402 ++++----------------- src/sinol_make/executors/__init__.py | 65 ++++ src/sinol_make/executors/detailed.py | 91 +++++ src/sinol_make/executors/oiejq.py | 85 +++++ src/sinol_make/executors/time.py | 112 ++++++ src/sinol_make/helpers/cache.py | 6 +- src/sinol_make/helpers/classinit.py | 90 +++++ src/sinol_make/helpers/compile.py | 6 +- src/sinol_make/interfaces/Errors.py | 2 +- src/sinol_make/structs/status_structs.py | 27 +- src/sinol_make/task_type/__init__.py | 172 +++++++++ src/sinol_make/task_type/interactive.py | 225 ++++++++++++ src/sinol_make/task_type/normal.py | 37 ++ tests/commands/run/test_unit.py | 6 + tests/helpers/test_cache.py | 2 +- 17 files changed, 986 insertions(+), 352 deletions(-) create mode 100644 src/sinol_make/executors/__init__.py create mode 100644 src/sinol_make/executors/detailed.py create mode 100644 src/sinol_make/executors/oiejq.py create mode 100644 src/sinol_make/executors/time.py create mode 100644 src/sinol_make/helpers/classinit.py create mode 100644 src/sinol_make/task_type/__init__.py create mode 100644 src/sinol_make/task_type/interactive.py create mode 100644 src/sinol_make/task_type/normal.py diff --git a/src/sinol_make/commands/gen/__init__.py b/src/sinol_make/commands/gen/__init__.py index 18719100..a1898799 100644 --- a/src/sinol_make/commands/gen/__init__.py +++ b/src/sinol_make/commands/gen/__init__.py @@ -5,6 +5,7 @@ from sinol_make.commands.outgen import Command as OutgenCommand from sinol_make.helpers import parsers from sinol_make.interfaces.BaseCommand import BaseCommand +from sinol_make.task_type import BaseTaskType class Command(BaseCommand): @@ -44,6 +45,7 @@ def run(self, args: argparse.Namespace): self.args = args self.ins = args.only_inputs self.outs = args.only_outputs + self.task_type = BaseTaskType.get_task_type() # If no arguments are specified, generate both input and output files. if not self.ins and not self.outs: self.ins = True @@ -53,6 +55,10 @@ def run(self, args: argparse.Namespace): command = IngenCommand() command.run(args) + if not self.task_type.run_outgen(): + print(util.warning("Outgen is not supported for this task type.")) + return + if self.outs: command = OutgenCommand() command.run(args) diff --git a/src/sinol_make/commands/outgen/__init__.py b/src/sinol_make/commands/outgen/__init__.py index 3a188484..bee68abf 100644 --- a/src/sinol_make/commands/outgen/__init__.py +++ b/src/sinol_make/commands/outgen/__init__.py @@ -10,6 +10,7 @@ from sinol_make.structs.gen_structs import OutputGenerationArguments from sinol_make.helpers import parsers, package_util, cache, paths from sinol_make.interfaces.BaseCommand import BaseCommand +from sinol_make.task_type import BaseTaskType class Command(BaseCommand): @@ -109,6 +110,9 @@ def run(self, args: argparse.Namespace): self.args = args self.task_id = package_util.get_task_id() + self.task_type = BaseTaskType.get_task_type() + if not self.task_type.run_outgen(): + util.exit_with_error('Output generation is not supported for this task type.') package_util.validate_test_names(self.task_id) util.change_stack_size_to_unlimited() cache.check_correct_solution(self.task_id) diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py index 161b9e5a..7320a685 100644 --- a/src/sinol_make/commands/run/__init__.py +++ b/src/sinol_make/commands/run/__init__.py @@ -1,11 +1,7 @@ # Modified version of https://sinol3.dasie.mimuw.edu.pl/oij/jury/package/-/blob/master/runner.py # Author of the original code: Bartosz Kostka # Version 0.6 (2021-08-29) -import subprocess -import signal import threading -import time -import psutil import glob import shutil import os @@ -21,10 +17,15 @@ from sinol_make.structs.run_structs import ExecutionData, PrintData from sinol_make.structs.cache_structs import CacheTest, CacheFile from sinol_make.interfaces.BaseCommand import BaseCommand -from sinol_make.interfaces.Errors import CompilationError, CheckerOutputException, UnknownContestType +from sinol_make.interfaces.Errors import CompilationError, UnknownContestType from sinol_make.helpers import compile, compiler, package_util, printer, paths, cache, parsers from sinol_make.structs.status_structs import Status, ResultChange, PointsChange, ValidationResult, ExecutionResult, \ TotalPointsChange +from sinol_make.task_type import BaseTaskType + +# Required for side effects +from sinol_make.task_type.normal import NormalTaskType # noqa +from sinol_make.task_type.interactive import InteractiveTaskType # noqa def color_memory(memory, limit): @@ -181,7 +182,7 @@ def print_table_end(): status_text = util.bold(util.color_green(group_status.ljust(6))) else: status_text = util.bold(util.color_red(group_status.ljust(6))) - print(f"{status_text}{str(points).rjust(3)}/{str(scores[group]).rjust(3)}", end=' | ') + print(f"{status_text}{str(int(points)).rjust(3)}/{str(scores[group]).rjust(3)}", end=' | ') program_groups_scores[program][group] = {"status": group_status, "points": points} print() for program in program_group: @@ -247,7 +248,7 @@ def print_group_seperator(): lang = package_util.get_file_lang(program) result = all_results[program][package_util.get_group(test, task_id)][test] if result.Points: - print(colorize_points(result.Points, contest.min_score_per_test(), + print(colorize_points(int(result.Points), contest.min_score_per_test(), contest.max_score_per_test()).ljust(13), end="") else: print(3*" ", end="") @@ -258,7 +259,6 @@ def print_group_seperator(): print_table_end() print() - sys.stdout = previous_stdout return output.getvalue().splitlines(), title, "Use arrows to move." @@ -268,11 +268,9 @@ class Command(BaseCommand): Class for running current task """ - def get_name(self): return 'run' - def configure_subparser(self, subparser): parser = subparser.add_parser( 'run', @@ -306,27 +304,17 @@ def configure_subparser(self, subparser): 'the expected scores are not compared with the actual scores.') parser.add_argument('--no-outputs', dest='allow_no_outputs', action='store_true', help='allow running the script without full outputs') + parser.add_argument('-o', '--comments', dest='comments', action='store_true', + help="show checker's comments") parsers.add_compilation_arguments(parser) return parser - def parse_time(self, time_str): - if len(time_str) < 3: return -1 - return int(time_str[:-2]) - - - def parse_memory(self, memory_str): - if len(memory_str) < 3: return -1 - return int(memory_str[:-2]) - - def extract_file_name(self, file_path): return os.path.split(file_path)[1] - def get_group(self, test_path): return package_util.get_group(test_path, self.ID) - def get_solution_from_exe(self, executable): file = os.path.splitext(executable)[0] for ext in self.SOURCE_EXTENSIONS: @@ -334,39 +322,35 @@ def get_solution_from_exe(self, executable): return file + ext util.exit_with_error("Source file not found for executable %s" % executable) - def get_executables(self, args_solutions): - return [package_util.get_executable(solution) for solution in package_util.get_solutions(self.ID, args_solutions)] - - def get_possible_score(self, groups): possible_score = 0 for group in groups: possible_score += self.scores[group] return possible_score - def get_output_file(self, test_path): return os.path.join("out", os.path.split(os.path.splitext(test_path)[0])[1]) + ".out" - def get_groups(self, tests): return sorted(list(set([self.get_group(test) for test in tests]))) - - def compile_solutions(self, solutions, is_checker=False): + def compile_solutions(self, solutions): os.makedirs(paths.get_compilation_log_path(), exist_ok=True) os.makedirs(paths.get_executables_path(), exist_ok=True) print("Compiling %d solutions..." % len(solutions)) - args = [(solution, True, is_checker) for solution in solutions] + args = [(solution, None, True, False, None) for solution in solutions] with mp.Pool(self.cpus) as pool: compilation_results = pool.starmap(self.compile, args) return compilation_results - - def compile(self, solution, use_extras = False, is_checker = False): + def compile(self, solution, dest=None, use_extras=False, clear_cache=False, name=None): compile_log_file = paths.get_compilation_log_path("%s.compile_log" % package_util.get_file_name(solution)) source_file = os.path.join(os.getcwd(), "prog", self.get_solution_from_exe(solution)) - output = paths.get_executables_path(package_util.get_executable(solution)) + if dest: + output = dest + else: + output = paths.get_executables_path(package_util.get_executable(solution)) + name = name or "file " + package_util.get_file_name(solution) extra_compilation_args = [] extra_compilation_files = [] @@ -384,255 +368,14 @@ def compile(self, solution, use_extras = False, is_checker = False): try: with open(compile_log_file, "w") as compile_log: compile.compile(source_file, output, self.compilers, compile_log, self.args.compile_mode, - extra_compilation_args, extra_compilation_files, is_checker=is_checker) - print(util.info("Compilation of file %s was successful." - % package_util.get_file_name(solution))) + extra_compilation_args, extra_compilation_files, clear_cache=clear_cache) + print(util.info(f"Compilation of {name} was successful.")) return True except CompilationError as e: - print(util.error("Compilation of file %s was unsuccessful." - % package_util.get_file_name(solution))) + print(util.error(f"Compilation of {name} was unsuccessful.")) compile.print_compile_log(compile_log_file) return False - def check_output_diff(self, output_file, answer_file): - """ - Checks whether the output file and the answer file are the same. - """ - return util.file_diff(output_file, answer_file) - - def check_output_checker(self, name, input_file, output_file, answer_file): - """ - Checks if the output file is correct with the checker. - Returns True if the output file is correct, False otherwise and number of points. - """ - command = [self.checker_executable, input_file, output_file, answer_file] - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - process.wait() - checker_output = process.communicate()[0].decode("utf-8").splitlines() - - if len(checker_output) == 0: - raise CheckerOutputException("Checker output is empty.") - - if checker_output[0].strip() == "OK": - points = 100 - if len(checker_output) >= 3: - try: - points = int(checker_output[2].strip()) - except ValueError: - pass - - return True, points - else: - return False, 0 - - - def check_output(self, name, input_file, output_file_path, output, answer_file_path): - """ - Checks if the output file is correct. - Returns a tuple (is correct, number of points). - """ - try: - has_checker = self.checker is not None - except AttributeError: - has_checker = False - - if not has_checker: - with open(answer_file_path, "r") as answer_file: - correct = util.lines_diff(output, answer_file.readlines()) - return correct, 100 if correct else 0 - else: - with open(output_file_path, "w") as output_file: - output_file.write("\n".join(output) + "\n") - return self.check_output_checker(name, input_file, output_file_path, answer_file_path) - - def execute_oiejq(self, name, timetool_path, executable, result_file_path, input_file_path, output_file_path, answer_file_path, - time_limit, memory_limit, hard_time_limit, execution_dir): - command = f'"{timetool_path}" "{executable}"' - env = os.environ.copy() - env["MEM_LIMIT"] = f'{memory_limit}K' - env["MEASURE_MEM"] = "1" - env["UNDER_OIEJQ"] = "1" - - timeout = False - with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file, \ - open(result_file_path, "w") as result_file: - process = subprocess.Popen(command, shell=True, stdin=input_file, stdout=output_file, - stderr=result_file, env=env, preexec_fn=os.setsid, cwd=execution_dir) - - def sigint_handler(signum, frame): - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except ProcessLookupError: - pass - sys.exit(1) - signal.signal(signal.SIGINT, sigint_handler) - - try: - process.wait(timeout=hard_time_limit) - except subprocess.TimeoutExpired: - timeout = True - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except ProcessLookupError: - pass - process.communicate() - - with open(result_file_path, "r") as result_file: - lines = result_file.read() - with open(output_file_path, "r") as output_file: - output = output_file.read() - result = ExecutionResult() - - if not timeout: - lines = lines.splitlines() - output = output.splitlines() - - for line in lines: - line = line.strip() - if ": " in line: - (key, value) = line.split(": ")[:2] - if key == "Time": - result.Time = self.parse_time(value) - elif key == "Memory": - result.Memory = self.parse_memory(value) - else: - setattr(result, key, value) - - if timeout: - result.Status = Status.TL - elif getattr(result, "Time") is not None and result.Time > time_limit: - result.Status = Status.TL - elif getattr(result, "Memory") is not None and result.Memory > memory_limit: - result.Status = Status.ML - elif getattr(result, "Status") is None: - result.Status = Status.RE - elif result.Status == "OK": # Here OK is a string, because it is set while parsing oiejq's output. - if result.Time > time_limit: - result.Status = Status.TL - elif result.Memory > memory_limit: - result.Status = Status.ML - else: - try: - correct, result.Points = self.check_output(name, input_file_path, output_file_path, output, answer_file_path) - if not correct: - result.Status = Status.WA - except CheckerOutputException as e: - result.Status = Status.CE - result.Error = e.message - else: - result.Status = result.Status[:2] - - return result - - - def execute_time(self, name, executable, result_file_path, input_file_path, output_file_path, answer_file_path, - time_limit, memory_limit, hard_time_limit, execution_dir): - if sys.platform == 'darwin': - time_name = 'gtime' - elif sys.platform == 'linux': - time_name = 'time' - elif sys.platform == 'win32' or sys.platform == 'cygwin': - raise Exception("Measuring time with GNU time on Windows is not supported.") - - command = [f'{time_name}', '-f', '%U\\n%M\\n%x', '-o', result_file_path, executable] - timeout = False - mem_limit_exceeded = False - with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file: - process = subprocess.Popen(command, stdin=input_file, stdout=output_file, stderr=subprocess.DEVNULL, - preexec_fn=os.setsid, cwd=execution_dir) - - def sigint_handler(signum, frame): - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except ProcessLookupError: - pass - sys.exit(1) - signal.signal(signal.SIGINT, sigint_handler) - - start_time = time.time() - while process.poll() is None: - try: - time_process = psutil.Process(process.pid) - executable_process = None - for child in time_process.children(): - if child.name() == executable: - executable_process = child - break - if executable_process is not None and executable_process.memory_info().rss > memory_limit * 1024: - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except ProcessLookupError: - pass - mem_limit_exceeded = True - break - except psutil.NoSuchProcess: - pass - - if time.time() - start_time > hard_time_limit: - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except ProcessLookupError: - pass - timeout = True - break - - with open(output_file_path, "r") as output_file: - output = output_file.read() - result = ExecutionResult() - program_exit_code = None - if not timeout: - output = output.splitlines() - with open(result_file_path, "r") as result_file: - lines = result_file.readlines() - if len(lines) == 3: - """ - If programs runs successfully, the output looks like this: - - first line is CPU time in seconds - - second line is memory in KB - - third line is exit code - This format is defined by -f flag in time command. - """ - result.Time = round(float(lines[0].strip()) * 1000) - result.Memory = int(lines[1].strip()) - program_exit_code = int(lines[2].strip()) - elif len(lines) > 0 and "Command terminated by signal " in lines[0]: - """ - If there was a runtime error, the first line is the error message with signal number. - For example: - Command terminated by signal 11 - """ - program_exit_code = int(lines[0].strip().split(" ")[-1]) - elif not mem_limit_exceeded: - result.Status = Status.RE - result.Error = "Unexpected output from time command: " + "".join(lines) - return result - - if program_exit_code is not None and program_exit_code != 0: - result.Status = Status.RE - elif timeout: - result.Status = Status.TL - elif mem_limit_exceeded: - result.Memory = memory_limit + 1 # Add one so that the memory is red in the table - result.Status = Status.ML - elif result.Time > time_limit: - result.Status = Status.TL - elif result.Memory > memory_limit: - result.Status = Status.ML - else: - try: - correct, result.Points = self.check_output(name, input_file_path, output_file_path, output, - answer_file_path) - if correct: - result.Status = Status.OK - else: - result.Status = Status.WA - except CheckerOutputException as e: - result.Status = Status.CE - result.Error = e.message - - return result - - def run_solution(self, data_for_execution: ExecutionData): """ Run an execution and return the result as ExecutionResult object. @@ -642,14 +385,10 @@ def run_solution(self, data_for_execution: ExecutionData): file_no_ext = paths.get_executions_path(name, package_util.extract_test_id(test, self.ID)) output_file = file_no_ext + ".out" result_file = file_no_ext + ".res" - hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0) + hard_time_limit = math.ceil(2 * time_limit / 1000.0) - if self.timetool_name == 'oiejq': - return self.execute_oiejq(name, timetool_path, executable, result_file, test, output_file, self.get_output_file(test), - time_limit, memory_limit, hard_time_limit_in_s, execution_dir) - elif self.timetool_name == 'time': - return self.execute_time(name, executable, result_file, test, output_file, self.get_output_file(test), - time_limit, memory_limit, hard_time_limit_in_s, execution_dir) + return self.task_type.run(time_limit, hard_time_limit, memory_limit, test, output_file, + self.get_output_file(test), result_file, executable, execution_dir) def run_solutions(self, compiled_commands, names, solutions, executables_dir): """ @@ -807,22 +546,15 @@ def validate_expected_scores(self, results): if group not in self.scores: util.exit_with_error(f'Group {group} doesn\'t have points specified in config file.') - if self.checker is None: - for solution in results.keys(): - new_expected_scores[solution] = { - "expected": results[solution], - "points": self.contest.get_global_score(results[solution], self.possible_score) - } - else: - for solution in results.keys(): - new_expected_scores[solution] = { - "expected": results[solution], - "points": self.contest.get_global_score(results[solution], self.possible_score) - } + for solution in results.keys(): + new_expected_scores[solution] = { + "expected": results[solution], + "points": self.contest.get_global_score(results[solution], self.possible_score) + } config_expected_scores = self.config.get("sinol_expected_scores", {}) used_solutions = results.keys() - if self.args.solutions == None and config_expected_scores: # If no solutions were specified, use all solutions from config + if self.args.solutions is None and config_expected_scores: # If no solutions were specified, use all solutions from config used_solutions = config_expected_scores.keys() used_solutions = list(used_solutions) @@ -952,7 +684,6 @@ def validate_expected_scores(self, results): unknown_change, ) - def print_expected_scores_diff(self, validation_results: ValidationResult): diff = validation_results config_expected_scores = self.config.get("sinol_expected_scores", {}) @@ -1027,13 +758,11 @@ def set_group_result(solution, group, result): else: util.exit_with_error("Use flag --apply-suggestions to apply suggestions.") - def set_constants(self): self.ID = package_util.get_task_id() self.SOURCE_EXTENSIONS = ['.c', '.cpp', '.py', '.java'] self.SOLUTIONS_RE = package_util.get_solutions_re(self.ID) - def validate_arguments(self, args): compilers = compiler.verify_compilers(args, package_util.get_solutions(self.ID, None)) @@ -1128,7 +857,7 @@ def validate_existence_of_outputs(self): print(util.warning('Missing output files for tests: ' + ', '.join( [self.extract_file_name(test) for test in missing_tests]))) - if self.args.allow_no_outputs != True: + if not self.args.allow_no_outputs: util.exit_with_error('There are tests without outputs. \n' 'Run outgen to fix this issue or add the --no-outputs flag to ignore the issue.') print(util.warning('Running only on tests with output files.')) @@ -1149,7 +878,7 @@ def check_are_any_tests_to_run(self): if len(example_tests) == len(self.tests): print(util.warning('Running only on example tests.')) - if not self.has_lib: + if not self.has_lib and self.task_type.run_outgen(): self.validate_existence_of_outputs() else: util.exit_with_error('There are no tests to run.') @@ -1161,40 +890,33 @@ def check_errors(self, results: Dict[str, Dict[str, Dict[str, ExecutionResult]]] :return: """ error_msg = "" + fail = False for solution in results: for group in results[solution]: for test in results[solution][group]: if results[solution][group][test].Error is not None: error_msg += f'Solution {solution} had an error on test {test}: {results[solution][group][test].Error}\n' + fail |= results[solution][group][test].Fail if error_msg != "": - util.exit_with_error(error_msg) - - def compile_checker(self): - checker_basename = os.path.basename(self.checker) - self.checker_executable = paths.get_executables_path(checker_basename + ".e") - - checker_compilation = self.compile_solutions([self.checker], is_checker=True) - if not checker_compilation[0]: - util.exit_with_error('Checker compilation failed.') + print(util.error(error_msg)) + if fail: + sys.exit(1) - def check_had_checker(self, has_checker): + def print_checker_comments(self, results: Dict[str, Dict[str, Dict[str, ExecutionResult]]]): """ - Checks if there was a checker and if it is now removed (or the other way around) and if so, removes tests cache. - In theory, removing cache after adding a checker is redundant, because during its compilation, the cache is - removed. + Prints checker's comments for all tests and solutions. """ - had_checker = os.path.exists(paths.get_cache_path("checker")) - if (had_checker and not has_checker) or (not had_checker and has_checker): - cache.remove_results_cache() - if has_checker: - with open(paths.get_cache_path("checker"), "w") as f: - f.write("") - else: - try: - os.remove(paths.get_cache_path("checker")) - except FileNotFoundError: - pass - + print(util.bold("Checker comments:")) + any_comments = False + for solution in results: + for group in results[solution]: + for test in results[solution][group]: + result = results[solution][group][test] + if result.Comment != "": + any_comments = True + print(util.bold(f"{solution} on {test}: ") + result.Comment) + if not any_comments: + print("No comments.") def run(self, args): args = util.init_package_command(args) @@ -1220,14 +942,16 @@ def run(self, args): cache.process_extra_execution_files(self.config.get("extra_execution_files", {}), self.ID) cache.remove_results_if_contest_type_changed(self.config.get("sinol_contest_type", "default")) - checker = package_util.get_files_matching_pattern(self.ID, f'{self.ID}chk.*') - if len(checker) != 0: - print("Checker found: %s" % os.path.basename(checker[0])) - self.checker = checker[0] - self.compile_checker() - else: - self.checker = None - self.check_had_checker(self.checker is not None) + task_type_cls = BaseTaskType.get_task_type() + self.task_type = task_type_cls(self.timetool_name, self.timetool_path) + additional_files = self.task_type.additional_files_to_compile() + os.makedirs(paths.get_compilation_log_path(), exist_ok=True) + os.makedirs(paths.get_executables_path(), exist_ok=True) + for file, dest, name, clear_cache, fail_on_error in additional_files: + print(f"Compiling {name}...") + success = self.compile(file, dest, False, clear_cache, name) + if not success and fail_on_error: + sys.exit(1) lib = package_util.get_files_matching_pattern(self.ID, f'{self.ID}lib.*') self.has_lib = len(lib) != 0 @@ -1249,6 +973,8 @@ def run(self, args): results, all_results = self.compile_and_run(solutions) self.check_errors(all_results) + if self.args.comments: + self.print_checker_comments(all_results) if self.args.ignore_expected: print(util.warning("Ignoring expected scores.")) self.exit() @@ -1256,11 +982,11 @@ def run(self, args): try: validation_results = self.validate_expected_scores(results) - except Exception: + except: self.config = util.try_fix_config(self.config) try: validation_results = self.validate_expected_scores(results) - except Exception: + except: util.exit_with_error("Validating expected scores failed. " "This probably means that `sinol_expected_scores` is broken. " "Delete it and run `sinol-make run --apply-suggestions` again.") diff --git a/src/sinol_make/executors/__init__.py b/src/sinol_make/executors/__init__.py new file mode 100644 index 00000000..c714c654 --- /dev/null +++ b/src/sinol_make/executors/__init__.py @@ -0,0 +1,65 @@ +import subprocess +from typing import List, Tuple, Union + +from sinol_make.structs.status_structs import ExecutionResult, Status + + +class BaseExecutor: + """ + Base class for executors. Executors are used to run commands and measure their time and memory usage. + """ + + def __init__(self): + pass + + def _wrap_command(self, command: List[str], result_file_path: str) -> List[str]: + """ + Wraps the command with the necessary tools to measure time and memory usage. + """ + raise NotImplementedError() + + def _execute(self, command: List[str], time_limit: int, hard_time_limit: int, memory_limit: int, + result_file_path: str, executable: str, execution_dir: str, stdin: int, stdout: int, + stderr: Union[None, int], fds_to_close: Union[None, List[int]], + *args, **kwargs) -> Tuple[bool, bool, int, List[str]]: + """ + This function should run subprocess.Popen with the given command and return a tuple of three values: + - bool: whether the process was terminated due to time limit + - bool: whether the process was terminated due to memory limit + - int: return code of the process + - List[str]: stderr of the process + """ + raise NotImplementedError() + + def _parse_result(self, tle, mle, return_code, result_file_path) -> ExecutionResult: + """ + Parses the result file and returns the result. + """ + raise NotImplementedError() + + def execute(self, command: List[str], time_limit, hard_time_limit, memory_limit, result_file_path, executable, + execution_dir, stdin=None, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + fds_to_close: Union[None, List[int]] = None, *args, **kwargs) -> ExecutionResult: + """ + Executes the command and returns the result, stdout and stderr. + """ + + command = self._wrap_command(command, result_file_path) + tle, mle, return_code, proc_stderr = self._execute(command, time_limit, hard_time_limit, memory_limit, + result_file_path, executable, execution_dir, stdin, stdout, + stderr, fds_to_close, *args, **kwargs) + result = self._parse_result(tle, mle, return_code, result_file_path) + if not result.Stderr: + result.Stderr = proc_stderr + if tle: + result.Status = Status.TL + result.Time = time_limit + 1 + elif mle: + result.Status = Status.ML + result.Memory = memory_limit + 1 # Add one so that the memory is red in the table + elif return_code != 0: + result.Status = Status.RE + result.Error = f"Solution returned with code {return_code}" + elif result.Status is None: + result.Status = Status.OK + return result diff --git a/src/sinol_make/executors/detailed.py b/src/sinol_make/executors/detailed.py new file mode 100644 index 00000000..e94d539d --- /dev/null +++ b/src/sinol_make/executors/detailed.py @@ -0,0 +1,91 @@ +import os +import signal +import subprocess +import time +import psutil +from typing import List, Tuple, Union + +from sinol_make.executors import BaseExecutor +from sinol_make.structs.status_structs import ExecutionResult, Status + + +class DetailedExecutor(BaseExecutor): + """ + Executor which doesn't use time or sio2jail for measuring time and memory usage. + """ + + def _wrap_command(self, command: List[str], result_file_path: str) -> List[str]: + return command + + def _execute(self, command: List[str], time_limit: int, hard_time_limit: int, memory_limit: int, + result_file_path: str, executable: str, execution_dir: str, stdin: int, stdout: int, + stderr: Union[None, int], fds_to_close: Union[None, List[int]], + *args, **kwargs) -> Tuple[bool, bool, int, List[str]]: + timeout = False + mem_used = 0 + if stderr is None: + stderr = subprocess.PIPE + process = subprocess.Popen(command, *args, stdin=stdin, stdout=stdout, stderr=stderr, + preexec_fn=os.setpgrp, cwd=execution_dir, **kwargs) + if fds_to_close is not None: + for fd in fds_to_close: + os.close(fd) + + start_time = time.time() + while process.poll() is None: + try: + time_process = psutil.Process(process.pid) + executable_process = None + for child in time_process.children(): + if child.name() == executable: + executable_process = child + break + mem_used = max(mem_used, executable_process.memory_info().rss) + if executable_process is not None: + mem_used = max(mem_used, executable_process.memory_info().rss) + if executable_process is not None and mem_used > memory_limit * 1024: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + break + except psutil.NoSuchProcess: + pass + + if time.time() - start_time > hard_time_limit: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + break + time_used = time.time() - start_time + mem_used = mem_used // 1024 + + with open(result_file_path, "w") as result_file: + result_file.write(f"{time_used}\n{mem_used}\n{process.returncode}\n") + + if stderr == subprocess.PIPE: + _, proc_stderr = process.communicate() + proc_stderr = proc_stderr.decode('utf-8').split('\n') + else: + proc_stderr = [] + return timeout, mem_used > memory_limit, 0, proc_stderr + + def _parse_result(self, tle, mle, return_code, result_file_path) -> ExecutionResult: + result = ExecutionResult() + program_exit_code = 0 + with open(result_file_path, "r") as result_file: + lines = result_file.readlines() + if len(lines) == 3: + result.Time = float(lines[0].strip()) + result.Memory = float(lines[1].strip()) + program_exit_code = int(lines[2].strip()) + else: + result.Status = Status.RE + result.Error = "Unexpected output from execution:\n" + "".join(lines) + result.Fail = True + if program_exit_code != 0: + result.Status = Status.RE + result.Error = f"Solution exited with code {program_exit_code}" + result.ExitSignal = os.WTERMSIG(program_exit_code) + return result diff --git a/src/sinol_make/executors/oiejq.py b/src/sinol_make/executors/oiejq.py new file mode 100644 index 00000000..c30371f2 --- /dev/null +++ b/src/sinol_make/executors/oiejq.py @@ -0,0 +1,85 @@ +import os +import signal +import subprocess +import sys +from typing import List, Tuple, Union + +from sinol_make.executors import BaseExecutor +from sinol_make.structs.status_structs import ExecutionResult, Status + + +class OiejqExecutor(BaseExecutor): + def __init__(self, oiejq_path): + super().__init__() + self.oiejq_path = oiejq_path + + def _wrap_command(self, command: List[str], result_file_path: str) -> List[str]: + return [f'"{self.oiejq_path}"'] + command + + def _execute(self, command: List[str], time_limit: int, hard_time_limit: int, memory_limit: int, + result_file_path: str, executable: str, execution_dir: str, stdin: int, stdout: int, + stderr: Union[None, int], fds_to_close: Union[None, List[int]], + *args, **kwargs) -> Tuple[bool, bool, int, List[str]]: + env = os.environ.copy() + env["MEM_LIMIT"] = f'{memory_limit}K' + env["MEASURE_MEM"] = "1" + env["UNDER_OIEJQ"] = "1" + + timeout = False + with open(result_file_path, "w") as result_file: + process = subprocess.Popen(' '.join(command), *args, shell=True, stdin=stdin, stdout=stdout, + stderr=result_file, env=env, preexec_fn=os.setpgrp, cwd=execution_dir, **kwargs) + if fds_to_close is not None: + for fd in fds_to_close: + os.close(fd) + + try: + process.wait(timeout=hard_time_limit) + except subprocess.TimeoutExpired: + timeout = True + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + + return timeout, False, 0, [] + + def _parse_time(self, time_str): + if len(time_str) < 3: return -1 + return int(time_str[:-2]) + + def _parse_memory(self, memory_str): + if len(memory_str) < 3: return -1 + return int(memory_str[:-2]) + + def _parse_result(self, tle, mle, return_code, result_file_path) -> ExecutionResult: + result = ExecutionResult() + if not tle: + with open(result_file_path, "r") as result_file: + lines = result_file.readlines() + + stderr = [] + i = 0 + while lines[i].strip() != "-------------------------": + stderr.append(lines[i]) + i += 1 + result.Stderr = stderr[:-1] # oiejq adds a blank line. + + for line in lines: + line = line.strip() + if ": " in line: + (key, value) = line.split(": ")[:2] + if key == "Time": + result.Time = self._parse_time(value) + elif key == "Memory": + result.Memory = self._parse_memory(value) + else: + setattr(result, key, value) + + if lines[-2].strip() == 'Details': + result.Error = lines[-1].strip() + if lines[-1].startswith("process exited due to signal"): + result.ExitSignal = int(lines[-1].strip()[len("process exited due to signal "):]) + result.Status = Status.from_str(result.Status) + return result diff --git a/src/sinol_make/executors/time.py b/src/sinol_make/executors/time.py new file mode 100644 index 00000000..58559139 --- /dev/null +++ b/src/sinol_make/executors/time.py @@ -0,0 +1,112 @@ +import os +import signal +import subprocess +import sys +import time +from typing import List, Tuple, Union + +import psutil +from sinol_make import util +from sinol_make.executors import BaseExecutor +from sinol_make.structs.status_structs import ExecutionResult, Status + + +class TimeExecutor(BaseExecutor): + def _wrap_command(self, command: List[str], result_file_path: str) -> List[str]: + if sys.platform == 'darwin': + time_name = 'gtime' + elif sys.platform == 'linux': + time_name = 'time' + else: + util.exit_with_error("Measuring time with GNU time on Windows is not supported.") + + return [f'{time_name}', '-f', '%U\\n%M\\n%x', '-o', result_file_path] + command + + def _execute(self, command: List[str], time_limit: int, hard_time_limit: int, memory_limit: int, + result_file_path: str, executable: str, execution_dir: str, stdin: int, stdout: int, + stderr: Union[None, int], fds_to_close: Union[None, List[int]], + *args, **kwargs) -> Tuple[bool, bool, int, List[str]]: + timeout = False + mem_limit_exceeded = False + if stderr is None: + stderr = subprocess.PIPE + process = subprocess.Popen(command, *args, stdin=stdin, stdout=stdout, stderr=stderr, + preexec_fn=os.setpgrp, cwd=execution_dir, **kwargs) + if fds_to_close is not None: + for fd in fds_to_close: + os.close(fd) + + start_time = time.time() + while process.poll() is None: + try: + time_process = psutil.Process(process.pid) + executable_process = None + for child in time_process.children(): + if child.name() == executable: + executable_process = child + break + if executable_process is not None and executable_process.memory_info().rss > memory_limit * 1024: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + mem_limit_exceeded = True + break + except psutil.NoSuchProcess: + pass + + if time.time() - start_time > hard_time_limit: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + timeout = True + break + + if stderr == subprocess.PIPE: + _, proc_stderr = process.communicate() + proc_stderr = proc_stderr.decode('utf-8').split('\n') + else: + proc_stderr = [] + return timeout, mem_limit_exceeded, 0, proc_stderr + + def _parse_result(self, tle, mle, return_code, result_file_path) -> ExecutionResult: + program_exit_code = None + result = ExecutionResult() + if not tle: + result.Time = 0 + result.Memory = 0 + with open(result_file_path, "r") as result_file: + lines = result_file.readlines() + if len(lines) == 4 and lines[0].startswith("Command exited with non-zero status"): + result.Status = Status.RE + exit_signal = int(lines[0].strip()[len("Command exited with non-zero status "):]) + program_exit_code = os.WTERMSIG(exit_signal) + elif len(lines) == 3: + """ + If programs runs successfully, the output looks like this: + - first line is CPU time in seconds + - second line is memory in KB + - third line is exit code + This format is defined by -f flag in time command. + """ + result.Time = round(float(lines[0].strip()) * 1000) + result.Memory = int(lines[1].strip()) + program_exit_code = int(lines[2].strip()) + elif len(lines) > 0 and "Command terminated by signal " in lines[0]: + """ + If there was a runtime error, the first line is the error message with signal number. + For example: + Command terminated by signal 11 + """ + program_exit_code = int(lines[0].strip().split(" ")[-1]) + elif not mle: + result.Status = Status.RE + result.Error = "Unexpected output from time command:\n" + "".join(lines) + result.Fail = True + + if program_exit_code is not None and program_exit_code != 0: + result.Status = Status.RE + result.Error = f"Solution exited with code {program_exit_code}" + result.ExitSignal = os.WTERMSIG(program_exit_code) + return result diff --git a/src/sinol_make/helpers/cache.py b/src/sinol_make/helpers/cache.py index cca819fd..0185ba5c 100644 --- a/src/sinol_make/helpers/cache.py +++ b/src/sinol_make/helpers/cache.py @@ -53,7 +53,7 @@ def check_compiled(file_path: str, compilation_flags: str, sanitizers: bool) -> return None -def save_compiled(file_path: str, exe_path: str, compilation_flags: str, sanitizers: bool, is_checker: bool = False): +def save_compiled(file_path: str, exe_path: str, compilation_flags: str, sanitizers: bool, clear_cache: bool = False): """ Save the compiled executable path to cache in `.cache/md5sums/`, which contains the md5sum of the file and the path to the executable. @@ -61,11 +61,11 @@ def save_compiled(file_path: str, exe_path: str, compilation_flags: str, sanitiz :param exe_path: Path to the compiled executable :param compilation_flags: Compilation flags used :param sanitizers: Whether -fsanitize=undefined,address was used - :param is_checker: Whether the compiled file is a checker. If True, all cached tests are removed. + :param clear_cache: Set to True if you want to delete all cached test results. """ info = CacheFile(util.get_file_md5(file_path), exe_path, compilation_flags, sanitizers) info.save(file_path) - if is_checker: + if clear_cache: remove_results_cache() diff --git a/src/sinol_make/helpers/classinit.py b/src/sinol_make/helpers/classinit.py new file mode 100644 index 00000000..53309163 --- /dev/null +++ b/src/sinol_make/helpers/classinit.py @@ -0,0 +1,90 @@ +# From oioioi/base/utils/__init__.py + +class ClassInitMeta(type): + """Meta class triggering __classinit__ on class intialization.""" + + def __init__(cls, class_name, bases, new_attrs): + super(ClassInitMeta, cls).__init__(class_name, bases, new_attrs) + cls.__classinit__() + + +class ClassInitBase(object, metaclass=ClassInitMeta): + """Abstract base class injecting ClassInitMeta meta class.""" + + @classmethod + def __classinit__(cls): + """ + Empty __classinit__ implementation. + + This must be a no-op as subclasses can't reliably call base class's + __classinit__ from their __classinit__s. + + Subclasses of __classinit__ should look like: + + .. python:: + + class MyClass(ClassInitBase): + + @classmethod + def __classinit__(cls): + # Need globals().get as MyClass may be still undefined. + super(globals().get('MyClass', cls), + cls).__classinit__() + ... + + class Derived(MyClass): + + @classmethod + def __classinit__(cls): + super(globals().get('Derived', cls), + cls).__classinit__() + ... + """ + pass + + +class RegisteredSubclassesBase(ClassInitBase): + """A base class for classes which should have a list of subclasses + available. + + The list of subclasses is available in their :attr:`subclasses` class + attributes. Classes which have *explicitly* set :attr:`abstract` class + attribute to ``True`` are not added to :attr:`subclasses`. + """ + + _subclasses_loaded = False + + @classmethod + def __classinit__(cls): + this_cls = globals().get('RegisteredSubclassesBase', cls) + super(this_cls, cls).__classinit__() + if this_cls is cls: + # This is RegisteredSubclassesBase class. + return + + assert 'subclasses' not in cls.__dict__, ( + '%s defines attribute subclasses, but has ' + 'RegisteredSubclassesMeta metaclass' % (cls,) + ) + cls.subclasses = [] + cls.abstract = cls.__dict__.get('abstract', False) + + def find_superclass(cls): + superclasses = [c for c in cls.__bases__ if issubclass(c, this_cls)] + if not superclasses: + return None + if len(superclasses) > 1: + raise AssertionError( + '%s derives from more than one ' + 'RegisteredSubclassesBase' % (cls.__name__,) + ) + superclass = superclasses[0] + return superclass + + # Add the class to all superclasses' 'subclasses' attribute, including + # self. + superclass = cls + while superclass is not this_cls: + if not cls.abstract: + superclass.subclasses.append(cls) + superclass = find_superclass(superclass) diff --git a/src/sinol_make/helpers/compile.py b/src/sinol_make/helpers/compile.py index d3dc8958..92fc7bdf 100644 --- a/src/sinol_make/helpers/compile.py +++ b/src/sinol_make/helpers/compile.py @@ -14,7 +14,7 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, compilation_flags='default', - extra_compilation_args=None, extra_compilation_files=None, is_checker=False, use_fsanitize=False): + extra_compilation_args=None, extra_compilation_files=None, clear_cache=False, use_fsanitize=False): """ Compile a program. :param program: Path to the program to compile @@ -24,7 +24,7 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp :param compilation_flags: Group of compilation flags to use :param extra_compilation_args: Extra compilation arguments :param extra_compilation_files: Extra compilation files - :param is_checker: Set to True if compiling a checker. This will remove all cached test results. + :param clear_cache: Set to True if you want to delete all cached test results. :param use_fsanitize: Whether to use fsanitize when compiling C/C++ programs. Sanitizes address and undefined behavior. """ if extra_compilation_args is None: @@ -114,7 +114,7 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp if process.returncode != 0: raise CompilationError('Compilation failed') else: - save_compiled(program, output, compilation_flags, use_fsanitize, is_checker) + save_compiled(program, output, compilation_flags, use_fsanitize, clear_cache) return True diff --git a/src/sinol_make/interfaces/Errors.py b/src/sinol_make/interfaces/Errors.py index 5766ca95..f52e95c8 100644 --- a/src/sinol_make/interfaces/Errors.py +++ b/src/sinol_make/interfaces/Errors.py @@ -10,7 +10,7 @@ def __str__(self): return self.message -class CheckerOutputException(SinolMakeException): +class CheckerException(SinolMakeException): def __init__(self, message): self.message = message diff --git a/src/sinol_make/structs/status_structs.py b/src/sinol_make/structs/status_structs.py index 2ff3ce11..2e2fe760 100644 --- a/src/sinol_make/structs/status_structs.py +++ b/src/sinol_make/structs/status_structs.py @@ -22,9 +22,9 @@ def __repr__(self): def from_str(status): if status == "CE": return Status.CE - elif status == "TL": + elif status == "TL" or status == "TLE": return Status.TL - elif status == "ML": + elif status == "ML" or status == "MLE": return Status.ML elif status == "RE": return Status.RE @@ -89,13 +89,26 @@ class ExecutionResult: Points: int # Error message Error: str - - def __init__(self, status=None, Time=None, Memory=None, Points=0, Error=None): + # Whether the program should fail due to the error + Fail: bool + # Exit signal of the process + ExitSignal: int + # Comment received from checker. + Comment: str + # Stderr of the program (used for checkers/interactors) + Stderr: List[str] + + def __init__(self, status=None, Time=None, Memory=None, Points=0, Error=None, Fail=False, ExitSignal=0, Comment="", + Stderr=None): self.Status = status self.Time = Time self.Memory = Memory self.Points = Points self.Error = Error + self.Fail = Fail + self.ExitSignal = ExitSignal + self.Comment = Comment + self.Stderr = Stderr if Stderr is not None else [] @staticmethod def from_dict(dict): @@ -104,7 +117,8 @@ def from_dict(dict): Time=dict.get("Time", None), Memory=dict.get("Memory", None), Points=dict.get("Points", 0), - Error=dict.get("Error", None) + Error=dict.get("Error", None), + Fail=dict.get("Fail", False), ) def to_dict(self): @@ -113,5 +127,6 @@ def to_dict(self): "Time": self.Time, "Memory": self.Memory, "Points": self.Points, - "Error": self.Error + "Error": self.Error, + "Fail": self.Fail, } diff --git a/src/sinol_make/task_type/__init__.py b/src/sinol_make/task_type/__init__.py new file mode 100644 index 00000000..705b785d --- /dev/null +++ b/src/sinol_make/task_type/__init__.py @@ -0,0 +1,172 @@ +import os +import subprocess +from fractions import Fraction +from typing import Tuple, List, Type + +from sinol_make import util +from sinol_make.executors.oiejq import OiejqExecutor +from sinol_make.executors.time import TimeExecutor +from sinol_make.helpers import package_util, paths, cache +from sinol_make.helpers.classinit import RegisteredSubclassesBase +from sinol_make.interfaces.Errors import CheckerException +from sinol_make.structs.status_structs import ExecutionResult + + +class BaseTaskType(RegisteredSubclassesBase): + abstract = True + + @classmethod + def identify(cls) -> Tuple[bool, int]: + """ + Returns a tuple of two values: + - bool: whether the task is of this type + - int: priority of this task type + """ + raise NotImplementedError() + + @classmethod + def get_task_type(cls) -> Type["BaseTaskType"]: + task_type = None + max_priority = -1 + for subclass in cls.subclasses: + is_task, priority = subclass.identify() + if is_task and priority > max_priority: + task_type = subclass + max_priority = priority + return task_type + + @staticmethod + def name() -> str: + """ + Returns the name of task type. + """ + raise NotImplementedError() + + def _check_task_type_changed(self): + """ + Checks if task type has changed and if so, deletes cache. + """ + name = self.name() + if os.path.exists(paths.get_cache_path("task_type")): + with open(paths.get_cache_path("task_type"), "r") as f: + if f.read().strip() != name: + cache.remove_results_cache() + with open(paths.get_cache_path("task_type"), "w") as f: + f.write(name) + + def __init__(self, timetool, oiejq_path): + super().__init__() + self.timetool = timetool + self.oiejq_path = oiejq_path + self.has_checker = False + self.checker_path = None + + if self.timetool == 'time': + self.executor = TimeExecutor() + elif self.timetool == 'oiejq': + self.executor = OiejqExecutor(oiejq_path) + else: + util.exit_with_error(f"Unknown timetool {self.timetool}") + self._check_task_type_changed() + + def _check_had_file(self, file, has_file): + """ + Checks if there was a file (e.g. checker) and if it is now removed (or the other way around) and if so, + removes tests cache. + """ + had_file = os.path.exists(paths.get_cache_path(file)) + if (had_file and not has_file) or (not had_file and has_file): + cache.remove_results_cache() + if has_file: + with open(paths.get_cache_path(file), "w") as f: + f.write("") + else: + try: + os.remove(paths.get_cache_path(file)) + except FileNotFoundError: + pass + + def additional_files_to_compile(self) -> List[Tuple[str, str, str, bool, bool]]: + """ + Returns a list of tuples of two values: + - str: source file path to compile + - str: path to the compiled file + - str: name of the compiled file + - bool: whether the program should fail on compilation error + - bool: whether all cache should be cleared on file change + """ + ret = [] + task_id = package_util.get_task_id() + checker = package_util.get_files_matching_pattern(task_id, f'{task_id}chk.*') + if len(checker) > 0: + self.has_checker = True + checker = checker[0] + checker_basename = os.path.basename(checker) + self.checker_path = paths.get_executables_path(checker_basename + ".e") + ret += [(checker, self.checker_path, "checker", True, True)] + self._check_had_file("checker", self.has_checker) + return ret + + @staticmethod + def run_outgen() -> bool: + """ + Whether outgen should be run. + """ + return True + + def _output_to_fraction(self, output_str): + if not output_str: + return Fraction(100, 1) + if isinstance(output_str, bytes): + output_str = output_str.decode('utf-8') + try: + return Fraction(output_str) + except ValueError: + raise CheckerException(f'Invalid checker output, expected float, percent or fraction, got "{output_str}"') + except ZeroDivisionError: + raise CheckerException(f'Zero division in checker output "{output_str}"') + except TypeError: + raise CheckerException(f'Invalid checker output "{output_str}"') + + def _parse_checker_output(self, output: List[str]) -> Tuple[bool, Fraction, str]: + while len(output) < 3: + output.append('') + + if output[0].strip() == "OK": + points = self._output_to_fraction(output[2]) + return True, points, output[1].strip() + else: + return False, Fraction(0, 1), output[1].strip() + + def _run_checker(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + proc = subprocess.Popen([self.checker_path, input_file_path, output_file_path, answer_file_path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc.wait() + output, stderr = proc.communicate() + if proc.returncode > 2: + return False, Fraction(0, 1), (f"Checker returned with code {proc.returncode}, " + f"stderr: '{stderr.decode('utf-8')}'") + return self._parse_checker_output(output.decode('utf-8').split('\n')) + + def _run_diff(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + same = util.file_diff(output_file_path, answer_file_path) + if same: + return True, Fraction(100, 1), "" + else: + return False, Fraction(0, 1), "" + + def check_output(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + """ + Runs the checker (or runs diff) and returns a tuple of three values: + - bool: whether the solution is correct + - Fraction: percentage of the score + - str: optional comment + """ + if self.has_checker: + return self._run_checker(input_file_path, output_file_path, answer_file_path) + else: + return self._run_diff(output_file_path, answer_file_path) + + def run(self, time_limit, hard_time_limit, memory_limit, input_file_path, output_file_path, answer_file_path, + result_file_path, executable, execution_dir) -> ExecutionResult: + raise NotImplementedError diff --git a/src/sinol_make/task_type/interactive.py b/src/sinol_make/task_type/interactive.py new file mode 100644 index 00000000..ff5058fb --- /dev/null +++ b/src/sinol_make/task_type/interactive.py @@ -0,0 +1,225 @@ +import os +import re +import signal +from threading import Thread +from typing import Tuple, List + +from sinol_make.executors.detailed import DetailedExecutor +from sinol_make.helpers import package_util, paths +from sinol_make.structs.status_structs import ExecutionResult, Status +from sinol_make.task_type import BaseTaskType +from sinol_make import util + + +class InteractiveTaskType(BaseTaskType): + INTERACTOR_MEMORY_LIMIT = 256 * 2 ** 10 + + class Pipes: + """ + Class for storing file descriptors for interactor and solution processes. + """ + r_interactor = None + w_interactor = None + r_solution = None + w_solution = None + + def __init__(self, r_interactor, w_interactor, r_solution, w_solution): + """ + Constructor for Pipes class. + :param r_interactor: file descriptor from which the interactor reads from the solution + :param w_interactor: file descriptor to which the interactor writes to the solution + :param r_solution: file descriptor from which the solution reads from the interactor + :param w_solution: file descriptor to which the solution writes to the interactor + """ + self.r_interactor = r_interactor + self.w_interactor = w_interactor + self.r_solution = r_solution + self.w_solution = w_solution + + class ExecutionWrapper(Thread): + def __init__(self, executor, *args, **kwargs): + super().__init__() + self.executor = executor + self.args = args + self.kwargs = kwargs + self.result = None + self.exception = None + + def run(self): + try: + self.result = self.executor.execute(*self.args, **self.kwargs) + except Exception as e: + self.exception = e + + @staticmethod + def get_interactor_re(task_id: str) -> re.Pattern: + """ + Returns regex pattern matching all solutions for given task. + :param task_id: Task id. + """ + return re.compile(r"^%ssoc\.(c|cpp|cc|py)$" % task_id) + + @classmethod + def identify(cls) -> Tuple[bool, int]: + task_id = package_util.get_task_id() + for file in os.listdir(os.path.join(os.getcwd(), "prog")): + if cls.get_interactor_re(task_id).match(file): + return True, 10 + return False, 0 + + @staticmethod + def name() -> str: + return "interactive" + + def __init__(self, timetool, oiejq_path): + super().__init__(timetool, oiejq_path) + self.has_checker = False + self.interactor = None + self.interactor_executor = DetailedExecutor() + + def additional_files_to_compile(self) -> List[Tuple[str, str, str, bool, bool]]: + ret = [] + task_id = package_util.get_task_id() + interactor = package_util.get_files_matching_pattern(task_id, f'{task_id}soc.*') + if len(interactor) > 0: + interactor = interactor[0] + interactor_basename = os.path.basename(interactor) + self.interactor = paths.get_executables_path(interactor_basename + ".e") + ret += [(interactor, self.interactor, "interactor", True, True)] + else: + util.exit_with_error(f"Interactor not found for task {task_id} (how did you manage to do this????)") + return ret + + @staticmethod + def run_outgen() -> bool: + # In interactive tasks via IO output files don't exist. + return False + + def _fill_result(self, result: ExecutionResult, iresult: ExecutionResult, interactor_output: List[str]): + sol_sig = result.ExitSignal + inter_sig = iresult.ExitSignal + + if interactor_output[0] != '': + ok, points, comment = self._parse_checker_output(interactor_output) + result.Points = float(points) + result.Comment = comment + if ok: + result.Status = Status.OK + else: + result.Status = Status.WA + elif iresult.Status != Status.OK and iresult.Status != Status.TL and inter_sig != signal.SIGPIPE: + result.Status = Status.RE + result.Error = (f"Interactor got {iresult.Status}. This would cause SE on sio. " + f"Interactor error: '{iresult.Error}'. " + f"Interactor stderr: {iresult.Stderr}. " + f"Interactor output: {interactor_output}") + result.Fail = True + elif result.Status is not None and result.Status != Status.OK and sol_sig != signal.SIGPIPE: + return + elif inter_sig == signal.SIGPIPE: + result.Status = Status.WA + result.Comment = "Solution exited prematurely" + elif iresult == Status.TL: + result.Status = Status.TL + result.Comment = "interactor time limit exceeded (user's solution or interactor can be the cause)" + else: + result.Status = Status.RE + result.Error = "Unexpected interactor error. Create an issue." + result.Fail = True + + def run(self, time_limit, hard_time_limit, memory_limit, input_file_path, output_file_path, answer_file_path, + result_file_path, executable, execution_dir) -> ExecutionResult: + config = package_util.get_config() + num_processes = config.get('num_processes', 1) + proc_pipes = [] + + file_no_ext = os.path.splitext(result_file_path)[0] + interactor_res_file = file_no_ext + "-soc.res" + processes_res_files = [file_no_ext + f"-{i}.res" for i in range(num_processes)] + + for _ in range(num_processes): + r1, w1 = os.pipe() + r2, w2 = os.pipe() + for fd in (r1, w1, r2, w2): + os.set_inheritable(fd, True) + proc_pipes.append(self.Pipes(r1, w2, r2, w1)) + + interactor_args = [str(num_processes)] + for pipes in proc_pipes: + interactor_args.extend([str(pipes.r_interactor), str(pipes.w_interactor)]) + + processes = [] + interactor_fds = [] + for pipes in proc_pipes: + interactor_fds.extend([pipes.r_interactor, pipes.w_interactor]) + + with open(input_file_path, "r") as inf, open(output_file_path, "w") as outf: + interactor = self.ExecutionWrapper( + self.interactor_executor, + [f'"{self.interactor}"'] + interactor_args, + time_limit * 2, + hard_time_limit * 2, + memory_limit, + interactor_res_file, + self.interactor, + execution_dir, + stdin=inf, + stdout=outf, + fds_to_close=interactor_fds, + pass_fds=interactor_fds, + ) + + for i in range(num_processes): + pipes = proc_pipes[i] + proc = self.ExecutionWrapper( + self.executor, + [f'"{executable}"', str(i)], + time_limit, + hard_time_limit, + memory_limit, + processes_res_files[i], + executable, + execution_dir, + stdin=pipes.r_solution, + stdout=pipes.w_solution, + fds_to_close=[pipes.r_solution, pipes.w_solution], + ) + processes.append(proc) + + for proc in processes: + proc.start() + interactor.start() + + for proc in processes: + proc.join() + interactor.join() + + result = ExecutionResult(Time=0, Memory=0) + if interactor.exception: + result.RE = True + result.Error = f"Interactor got an exception:\n" + str(interactor.exception) + for i in range(num_processes): + proc = processes[i] + if proc.exception: + result.RE = True + result.Error = f"Solution {i} got an exception:\n" + str(proc.exception) + + for proc in processes: + if proc.result.Status != Status.OK: + result = proc.result + break + result.Time = max(result.Time, proc.result.Time) + result.Memory = max(result.Memory, proc.result.Memory) + + iresult = interactor.result + + try: + with open(output_file_path, "r") as ires_file: + interactor_output = [line.rstrip() for line in ires_file.readlines()] + while len(interactor_output) < 3: + interactor_output.append("") + except FileNotFoundError: + interactor_output = [] + + self._fill_result(result, iresult, interactor_output) + return result diff --git a/src/sinol_make/task_type/normal.py b/src/sinol_make/task_type/normal.py new file mode 100644 index 00000000..0a6c1f12 --- /dev/null +++ b/src/sinol_make/task_type/normal.py @@ -0,0 +1,37 @@ +from typing import Tuple + +from sinol_make.interfaces.Errors import CheckerException +from sinol_make.structs.status_structs import ExecutionResult, Status +from sinol_make.task_type import BaseTaskType + + +class NormalTaskType(BaseTaskType): + @classmethod + def identify(cls) -> Tuple[bool, int]: + return True, 1 + + @staticmethod + def name() -> str: + return "normal" + + def run(self, time_limit, hard_time_limit, memory_limit, input_file_path, output_file_path, answer_file_path, + result_file_path, executable, execution_dir) -> ExecutionResult: + with open(input_file_path, "r") as inf, open(output_file_path, "w") as outf: + result = self.executor.execute([f'"{executable}"'], time_limit, hard_time_limit, memory_limit, + result_file_path, executable, execution_dir, stdin=inf, stdout=outf) + if result.Time > time_limit: + result.Status = Status.TL + elif result.Memory > memory_limit: + result.Status = Status.ML + elif result.Status == Status.OK: + try: + correct, points, comment = self.check_output(input_file_path, output_file_path, answer_file_path) + result.Points = float(points) + result.Comment = comment + if not correct: + result.Status = Status.WA + except CheckerException as e: + result.Status = Status.RE + result.Error = str(e) + result.Fail = True + return result diff --git a/tests/commands/run/test_unit.py b/tests/commands/run/test_unit.py index 262194a1..60318d97 100644 --- a/tests/commands/run/test_unit.py +++ b/tests/commands/run/test_unit.py @@ -3,6 +3,8 @@ from sinol_make import util, oiejq from sinol_make.structs.status_structs import Status, ResultChange, ValidationResult from sinol_make.helpers import package_util +from sinol_make.task_type.normal import NormalTaskType + from .util import * from ...util import * from ...fixtures import * @@ -27,6 +29,8 @@ def test_execution(create_package, time_tool): command = get_command(package_path) command.args.time_tool = time_tool command.timetool_name = time_tool + command.task_type = NormalTaskType(timetool=time_tool, oiejq_path=oiejq.get_oiejq_path(), has_checker=False, + checker_path=None) solution = "abc.cpp" executable = package_util.get_executable(solution) result = command.compile_solutions([solution]) @@ -59,6 +63,8 @@ def test_run_solutions(create_package, time_tool): command.time_limit = command.config["time_limit"] command.timetool_path = oiejq.get_oiejq_path() command.timetool_name = time_tool + command.task_type = NormalTaskType(timetool=time_tool, oiejq_path=oiejq.get_oiejq_path(), has_checker=False, + checker_path=None) def flatten_results(results): new_results = {} for solution in results.keys(): diff --git a/tests/helpers/test_cache.py b/tests/helpers/test_cache.py index 52c37c77..273ab1bd 100644 --- a/tests/helpers/test_cache.py +++ b/tests/helpers/test_cache.py @@ -81,7 +81,7 @@ def test_cache(): cache_file.save("abc.cpp") assert cache.get_cache_file("abc.cpp") == cache_file cache.save_compiled("abc.cpp", "abc.e", "default", False, - is_checker=True) + clear_cache=True) assert cache.get_cache_file("abc.cpp").tests == {} # Test that cache is cleared when extra compilation files change