From e66595fc3532d7a318e8fe67f6a7b0f51cc7a5f5 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Wed, 13 Sep 2023 12:57:56 +0200 Subject: [PATCH] Add functions for better searching for files --- src/sinol_make/commands/gen/gen_util.py | 13 +++--- src/sinol_make/commands/inwer/__init__.py | 2 +- src/sinol_make/commands/inwer/inwer_util.py | 2 +- src/sinol_make/commands/run/__init__.py | 8 ++-- src/sinol_make/helpers/package_util.py | 49 ++++++++++++++++++--- tests/commands/gen/test_unit.py | 6 --- tests/commands/inwer/test_integration.py | 16 ++++--- tests/commands/inwer/test_unit.py | 3 +- tests/commands/run/test_integration.py | 5 ++- tests/commands/run/test_unit.py | 8 ++-- tests/helpers/test_package_util.py | 8 ++-- tests/util.py | 18 ++++---- 12 files changed, 88 insertions(+), 50 deletions(-) diff --git a/src/sinol_make/commands/gen/gen_util.py b/src/sinol_make/commands/gen/gen_util.py index 42f9d94f..5123372f 100644 --- a/src/sinol_make/commands/gen/gen_util.py +++ b/src/sinol_make/commands/gen/gen_util.py @@ -16,15 +16,14 @@ def ingen_exists(task_id): :param task_id: task id, for example abc :return: True if exists, False otherwise """ - return len(glob.glob(os.path.join(os.getcwd(), 'prog', task_id + 'ingen.*'))) > 0 + return package_util.any_files_matching_pattern(task_id, f'{task_id}ingen.*') -def get_ingen(task_id=None, ingen_path=None): +def get_ingen(task_id, ingen_path=None): """ Find ingen source file in `prog/` directory. If `ingen_path` is specified, then it will be used (if exists). - :param task_id: task id, for example abc. If None, then - will return any ingen matching "*ingen.*" + :param task_id: task id, for example abc. :param ingen_path: path to ingen source file :return: path to ingen source file or None if not found """ @@ -35,9 +34,7 @@ def get_ingen(task_id=None, ingen_path=None): else: util.exit_with_error(f'Ingen source file {ingen_path} does not exist.') - if task_id is None: - task_id = '*' - ingen = glob.glob(os.path.join(os.getcwd(), 'prog', task_id + 'ingen.*')) + ingen = package_util.get_files_matching_pattern(task_id, f'{task_id}ingen.*') if len(ingen) == 0: util.exit_with_error(f'Ingen source file for task {task_id} does not exist.') @@ -78,7 +75,7 @@ def get_correct_solution(task_id): :param task_id: task id, for example abc :return: path to correct solution or None if not found """ - correct_solution = glob.glob(os.path.join(os.getcwd(), 'prog', task_id + '.*')) + correct_solution = package_util.get_files_matching_pattern(task_id, f'{task_id}.*') if len(correct_solution) == 0: util.exit_with_error(f'Correct solution for task {task_id} does not exist.') return correct_solution[0] diff --git a/src/sinol_make/commands/inwer/__init__.py b/src/sinol_make/commands/inwer/__init__.py index 41ace8f0..d30c252e 100644 --- a/src/sinol_make/commands/inwer/__init__.py +++ b/src/sinol_make/commands/inwer/__init__.py @@ -134,7 +134,7 @@ def run(self, args: argparse.Namespace): print(f'Verifying with inwer {util.bold(relative_path)}') self.cpus = args.cpus or mp.cpu_count() - self.tests = package_util.get_tests(args.tests) + self.tests = package_util.get_tests(self.task_id, args.tests) if len(self.tests) == 0: util.exit_with_error('No tests found.') diff --git a/src/sinol_make/commands/inwer/inwer_util.py b/src/sinol_make/commands/inwer/inwer_util.py index b9ec0653..58ed5a9a 100644 --- a/src/sinol_make/commands/inwer/inwer_util.py +++ b/src/sinol_make/commands/inwer/inwer_util.py @@ -18,7 +18,7 @@ def get_inwer_path(task_id: str, path = None) -> Union[str, None]: Returns path to inwer executable for given task or None if no inwer was found. """ if path is None: - inwers = glob.glob(os.path.join(os.getcwd(), 'prog', f'{task_id}inwer.*')) + inwers = package_util.get_files_matching_pattern(task_id, f'{task_id}inwer.*') if len(inwers) == 0: return None return inwers[0] diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py index af46abc4..447a5c9e 100644 --- a/src/sinol_make/commands/run/__init__.py +++ b/src/sinol_make/commands/run/__init__.py @@ -716,7 +716,7 @@ def get_whole_groups(self): Returns a list of groups for which all tests were run. """ group_sizes = {} - for test in package_util.get_tests(): + for test in package_util.get_tests(self.ID): group = package_util.get_group(test, self.ID) if group not in group_sizes: group_sizes[group] = 0 @@ -1005,7 +1005,7 @@ def exit(self): cnt=len(self.failed_compilations), letter='' if len(self.failed_compilations) == 1 else 's')) def set_scores(self): - self.tests = package_util.get_tests(self.args.tests) + self.tests = package_util.get_tests(self.ID, self.args.tests) self.groups = self.get_groups(self.tests) self.scores = collections.defaultdict(int) @@ -1129,7 +1129,7 @@ def run(self, args): print("Task: %s (tag: %s)" % (title, self.ID)) self.cpus = args.cpus or mp.cpu_count() - checker = glob.glob(os.path.join(os.getcwd(), "prog", f'{self.ID}chk.*')) + checker = package_util.get_files_matching_pattern(self.ID, f'{self.ID}chk.*') if len(checker) != 0: print(util.info("Checker found: %s" % os.path.basename(checker[0]))) self.checker = checker[0] @@ -1142,7 +1142,7 @@ def run(self, args): else: self.checker = None - lib = glob.glob(os.path.join(os.getcwd(), "prog", f'{self.ID}lib.*')) + lib = package_util.get_files_matching_pattern(self.ID, f'{self.ID}lib.*') self.has_lib = len(lib) != 0 self.set_scores() diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index d6a72289..1721d3de 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -2,6 +2,7 @@ import re import yaml import glob +import fnmatch from enum import Enum from typing import List, Union, Dict, Any @@ -34,28 +35,29 @@ def extract_test_id(test_path, task_id): return os.path.split(os.path.splitext(test_path)[0])[1][len(task_id):] -def get_group(test_path, task_id=None): +def get_group(test_path, task_id): if extract_test_id(test_path, task_id).endswith("ocen"): return 0 return int("".join(filter(str.isdigit, extract_test_id(test_path, task_id)))) -def get_test_key(test): - return get_group(test), test +def get_test_key(test, task_id): + return get_group(test, task_id), test -def get_tests(arg_tests: Union[List[str], None] = None) -> List[str]: +def get_tests(task_id: str, arg_tests: Union[List[str], None] = None) -> List[str]: """ Returns list of tests to run. + :param task_id: Task id. :param arg_tests: Tests specified in command line arguments. If None, all tests are returned. :return: List of tests to run. """ if arg_tests is None: all_tests = ["in/%s" % test for test in os.listdir("in/") if test[-3:] == ".in"] - return sorted(all_tests, key=get_test_key) + return sorted(all_tests, key=lambda test: get_test_key(test, task_id)) else: - return sorted(list(set(arg_tests)), key=get_test_key) + return sorted(list(set(arg_tests)), key=lambda test: get_test_key(test, task_id)) def get_file_name(file_path): @@ -109,7 +111,7 @@ def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: def _get_limit(limit_type: LimitTypes, test_path: str, config: Dict[str, Any], lang: str, task_id: str): test_id = extract_test_id(test_path, task_id) - test_group = str(get_group(test_path)) + test_group = str(get_group(test_path, task_id)) global_limit = _get_limit_from_dict(config, limit_type, test_id, test_group, test_path) override_limits_dict = config.get("override_limits", {}).get(lang, {}) overriden_limit = _get_limit_from_dict(override_limits_dict, limit_type, test_id, test_group, test_path) @@ -167,3 +169,36 @@ def get_invalid_files(path, pattern): invalid_out_tests = get_invalid_files(os.path.join("out", "*.out"), out_test_re) if len(invalid_out_tests) > 0: util.exit_with_error(f'Output tests with invalid names: {", ".join(invalid_out_tests)}.') + + +def get_all_code_files(task_id: str) -> List[str]: + """ + Returns all code files in package. + :param task_id: Task id. + :return: List of code files. + """ + result = glob.glob(os.path.join(os.getcwd(), "prog", f"{task_id}ingen.sh")) + for ext in ["c", "cpp", "py", "java"]: + result += glob.glob(os.path.join(os.getcwd(), f"prog/{task_id}*.{ext}")) + return result + + +def get_files_matching_pattern(task_id: str, pattern: str) -> List[str]: + """ + Returns all files in package matching given pattern. + :param task_id: Task id. + :param pattern: Pattern to match. + :return: List of files matching the pattern. + """ + all_files = get_all_code_files(task_id) + return [file for file in all_files if fnmatch.fnmatch(os.path.basename(file), pattern)] + + +def any_files_matching_pattern(task_id: str, pattern: str) -> bool: + """ + Returns True if any file in package matches given pattern. + :param task_id: Task id. + :param pattern: Pattern to match. + :return: True if any file in package matches given pattern. + """ + return len(get_files_matching_pattern(task_id, pattern)) > 0 diff --git a/tests/commands/gen/test_unit.py b/tests/commands/gen/test_unit.py index e82eea59..fe823cfd 100644 --- a/tests/commands/gen/test_unit.py +++ b/tests/commands/gen/test_unit.py @@ -20,9 +20,6 @@ def test_get_ingen(): shutil.copytree(simple_package_path, os.path.join(tmpdir, 'simple')) os.chdir(os.path.join(tmpdir, 'simple')) - ingen_path = gen_util.get_ingen() - assert os.path.basename(ingen_path) == "abcingen.cpp" - ingen_path = gen_util.get_ingen("abc") assert os.path.basename(ingen_path) == "abcingen.cpp" @@ -37,9 +34,6 @@ def test_get_ingen(): shutil.copytree(gen_package_path, os.path.join(tmpdir, 'gen')) os.chdir(os.path.join(tmpdir, 'gen')) - ingen_path = gen_util.get_ingen() - assert os.path.basename(ingen_path) == "geningen.sh" - ingen_path = gen_util.get_ingen("gen") assert os.path.basename(ingen_path) == "geningen.sh" diff --git a/tests/commands/inwer/test_integration.py b/tests/commands/inwer/test_integration.py index 767e2d4f..9fadfb6d 100644 --- a/tests/commands/inwer/test_integration.py +++ b/tests/commands/inwer/test_integration.py @@ -2,6 +2,7 @@ from sinol_make import configure_parsers from sinol_make.commands.inwer import Command +from sinol_make.helpers import package_util from tests import util from tests.fixtures import * @@ -12,7 +13,8 @@ def test_default(capsys, create_package): Test `inwer` command with no parameters. """ package_path = create_package - util.create_ins(package_path) + task_id = package_util.get_task_id() + util.create_ins(package_path, task_id) parser = configure_parsers() args = parser.parse_args(["inwer"]) command = Command() @@ -33,7 +35,8 @@ def test_specified_inwer(capsys, create_package): Test `inwer` command with specified inwer. """ package_path = create_package - util.create_ins(package_path) + task_id = package_util.get_task_id() + util.create_ins(package_path, task_id) parser = configure_parsers() args = parser.parse_args(["inwer", "prog/werinwer.cpp"]) command = Command() @@ -66,7 +69,8 @@ def test_asserting_inwer(capsys, create_package): Test `inwer` command with inwer that uses assert for verifying. """ package_path = create_package - util.create_ins(package_path) + task_id = package_util.get_task_id() + util.create_ins(package_path, task_id) parser = configure_parsers() args = parser.parse_args(["inwer", "prog/werinwer3.cpp"]) command = Command() @@ -87,7 +91,8 @@ def test_flag_tests(capsys, create_package): Test `inwer` command with --tests flag. """ package_path = create_package - util.create_ins(package_path) + task_id = package_util.get_task_id() + util.create_ins(package_path, task_id) parser = configure_parsers() args = parser.parse_args(["inwer", "prog/werinwer.cpp", "--tests", "in/wer2a.in"]) command = Command() @@ -125,7 +130,8 @@ def test_no_output(capsys, create_package): Test `inwer` command when inwer doesn't print anything. """ package_path = create_package - util.create_ins(package_path) + task_id = package_util.get_task_id() + util.create_ins(package_path, task_id) parser = configure_parsers() args = parser.parse_args(["inwer", "prog/werinwer4.cpp"]) command = Command() diff --git a/tests/commands/inwer/test_unit.py b/tests/commands/inwer/test_unit.py index 83c738a9..744fc404 100644 --- a/tests/commands/inwer/test_unit.py +++ b/tests/commands/inwer/test_unit.py @@ -36,7 +36,8 @@ def test_asserting_inwer(create_package): Test asserting inwer. """ package_path = create_package - util.create_ins(package_path) + task_id = package_util.get_task_id() + util.create_ins(package_path, task_id) inwer_path = os.path.join(os.getcwd(), 'prog', 'werinwer3.cpp') args = argparse.Namespace( c_compiler_path=compiler.get_c_compiler_path(), diff --git a/tests/commands/run/test_integration.py b/tests/commands/run/test_integration.py index d648f51a..ffe59bd0 100644 --- a/tests/commands/run/test_integration.py +++ b/tests/commands/run/test_integration.py @@ -59,7 +59,7 @@ def test_no_expected_scores(capsys, create_package, time_tool): out = capsys.readouterr().out assert "Solutions were added:" in out - solution = glob.glob(os.path.join(package_path, "prog", "???.*"))[0] + solution = package_util.get_files_matching_pattern(command.ID, f"{command.ID}.*")[0] assert os.path.basename(solution) in out @@ -219,7 +219,8 @@ def test_flag_solutions(capsys, create_package, time_tool): package_path = create_package create_ins_outs(package_path) - solutions = glob.glob(os.path.join(package_path, "prog", "????.*")) + task_id = package_util.get_task_id() + solutions = package_util.get_files_matching_pattern(task_id, f'{task_id}?.*') parser = configure_parsers() args = parser.parse_args(["run", "--solutions", solutions[0], "--time-tool", time_tool]) command = Command() diff --git a/tests/commands/run/test_unit.py b/tests/commands/run/test_unit.py index d0f80407..2af4d417 100644 --- a/tests/commands/run/test_unit.py +++ b/tests/commands/run/test_unit.py @@ -50,7 +50,7 @@ def test_execution(create_package, time_tool): assert result == [True] create_ins_outs(package_path) - test = package_util.get_tests(None)[0] + test = package_util.get_tests("abc", None)[0] with open(os.path.join(package_path, "config.yml"), "r") as config_file: config = yaml.load(config_file, Loader=yaml.FullLoader) @@ -78,7 +78,7 @@ def test_run_solutions(create_package, time_tool): command.args = argparse.Namespace(solutions_report=False, time_tool=time_tool, weak_compilation_flags=False, hide_memory=False) create_ins_outs(package_path) - command.tests = package_util.get_tests(None) + command.tests = package_util.get_tests("abc", None) command.groups = list(sorted(set([command.get_group(test) for test in command.tests]))) command.scores = command.config["scores"] command.possible_score = command.get_possible_score(command.groups) @@ -132,7 +132,7 @@ def test_validate_expected_scores_success(): os.chdir(get_simple_package_path()) command = get_command() command.scores = command.config["scores"] - command.tests = package_util.get_tests(None) + command.tests = package_util.get_tests("abc", None) # Test with correct expected scores. command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None) @@ -439,7 +439,7 @@ def test_get_valid_input_files(create_package): package_path = create_package command = get_command(package_path) create_ins_outs(package_path) - command.tests = package_util.get_tests(None) + command.tests = package_util.get_tests(command.ID, None) outputs = glob.glob(os.path.join(package_path, "out", "*.out")) os.unlink(outputs[0]) diff --git a/tests/helpers/test_package_util.py b/tests/helpers/test_package_util.py index d30eef8a..de5f294a 100644 --- a/tests/helpers/test_package_util.py +++ b/tests/helpers/test_package_util.py @@ -25,13 +25,15 @@ def test_extract_test_id(): def test_get_group(): - assert package_util.get_group("in/abc1a.in") == 1 + assert package_util.get_group("in/abc1a.in", "abc") == 1 + assert package_util.get_group("in/long_name2ocen.in", "long_name") == 0 def test_get_tests(create_package): - create_ins(create_package) os.chdir(create_package) - tests = package_util.get_tests(None) + task_id = package_util.get_task_id() + create_ins(create_package, task_id) + tests = package_util.get_tests("abc", None) assert tests == ["in/abc1a.in", "in/abc2a.in", "in/abc3a.in", "in/abc4a.in"] diff --git a/tests/util.py b/tests/util.py index 409d1f1a..d6d2454f 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,7 +2,7 @@ import glob import subprocess -from sinol_make.helpers import compile, paths +from sinol_make.helpers import compile, paths, package_util def get_simple_package_path(): @@ -87,11 +87,11 @@ def get_long_name_package_path(): return os.path.join(os.path.dirname(__file__), "packages", "long_package_name") -def create_ins(package_path): +def create_ins(package_path, task_id): """ Create .in files for package. """ - ingen = glob.glob(os.path.join(package_path, "prog", "*ingen.*"))[0] + ingen = package_util.get_files_matching_pattern(task_id, f'{task_id}ingen.*')[0] ingen_executable = paths.get_executables_path("ingen.e") os.makedirs(paths.get_executables_path(), exist_ok=True) assert compile.compile(ingen, ingen_executable) @@ -100,11 +100,11 @@ def create_ins(package_path): os.chdir(package_path) -def create_outs(package_path): +def create_outs(package_path, task_id): """ Create .out files for package. """ - solution = glob.glob(os.path.join(package_path, "prog", "???.*"))[0] + solution = package_util.get_files_matching_pattern(task_id, f'{task_id}.*')[0] solution_executable = paths.get_executables_path("solution.e") os.makedirs(paths.get_executables_path(), exist_ok=True) assert compile.compile(solution, solution_executable) @@ -120,7 +120,9 @@ def create_ins_outs(package_path): """ Create .in and .out files for package. """ - create_ins(package_path) - has_lib = len(glob.glob(os.path.join(package_path, "prog", "???lib.*"))) > 0 + os.chdir(package_path) + task_id = package_util.get_task_id() + create_ins(package_path, task_id) + has_lib = package_util.any_files_matching_pattern(task_id, f"{task_id}lib.*") if not has_lib: - create_outs(package_path) + create_outs(package_path, task_id)