diff --git a/.github/workflows/Arch.yaml b/.github/workflows/Arch.yaml index f3777c8c..6f354222 100644 --- a/.github/workflows/Arch.yaml +++ b/.github/workflows/Arch.yaml @@ -29,5 +29,7 @@ jobs: run: | pip3 install .[tests] - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" run: | python3 -m pytest -v diff --git a/.github/workflows/GithubRunner.yaml b/.github/workflows/GithubRunner.yaml index 9878e3f4..01f58643 100644 --- a/.github/workflows/GithubRunner.yaml +++ b/.github/workflows/GithubRunner.yaml @@ -19,5 +19,7 @@ jobs: run: | pip install .[tests] - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" run: | python -m pytest -v --github-runner diff --git a/.github/workflows/Ubuntu.yaml b/.github/workflows/Ubuntu.yaml index b63baad1..aaa11c09 100644 --- a/.github/workflows/Ubuntu.yaml +++ b/.github/workflows/Ubuntu.yaml @@ -35,5 +35,7 @@ jobs: pip3 install .[tests] apt install -y time - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" run: | python3 -m pytest -v diff --git a/.github/workflows/macOS.yaml b/.github/workflows/macOS.yaml index dbe1eda0..3fd54ba1 100644 --- a/.github/workflows/macOS.yaml +++ b/.github/workflows/macOS.yaml @@ -30,5 +30,7 @@ jobs: run: | python3 -m pip install .[tests] - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" run: | python3 -m pytest -v --time-tool time diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py index 958d3db3..b82b4d82 100644 --- a/src/sinol_make/commands/run/__init__.py +++ b/src/sinol_make/commands/run/__init__.py @@ -1057,7 +1057,6 @@ def exit(self): cnt=len(self.failed_compilations), letter='' if len(self.failed_compilations) == 1 else 's')) def set_scores(self): - self.tests = package_util.get_tests(self.ID, self.args.tests) self.groups = self.get_groups(self.tests) self.scores = collections.defaultdict(int) @@ -1069,6 +1068,11 @@ def set_scores(self): num_groups -= 1 self.scores[0] = 0 + # This only happens when running only on group 0. + if num_groups == 0: + self.possible_score = 0 + return + points_per_group = 100 // num_groups for group in self.groups: if group == 0: @@ -1138,7 +1142,7 @@ def check_are_any_tests_to_run(self): if not self.has_lib: self.validate_existence_of_outputs() else: - print(util.warning('There are no tests to run.')) + util.exit_with_error('There are no tests to run.') def check_errors(self, results: Dict[str, Dict[str, Dict[str, ExecutionResult]]]): """ @@ -1197,8 +1201,9 @@ def run(self, args): lib = package_util.get_files_matching_pattern(self.ID, f'{self.ID}lib.*') self.has_lib = len(lib) != 0 - self.set_scores() + self.tests = package_util.get_tests(self.ID, self.args.tests) self.check_are_any_tests_to_run() + self.set_scores() self.failed_compilations = [] solutions = self.get_solutions(self.args.solutions) diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index bf634a73..8ace8c8a 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -57,7 +57,11 @@ def get_tests(task_id: str, arg_tests: Union[List[str], None] = None) -> List[st if test[-3:] == ".in"] return sorted(all_tests, key=lambda test: get_test_key(test, task_id)) else: - return sorted(list(set(arg_tests)), key=lambda test: get_test_key(test, task_id)) + existing_tests = set() + for test in arg_tests: + if os.path.exists(test): + existing_tests.add(test) + return sorted(list(existing_tests), key=lambda test: get_test_key(test, task_id)) def get_file_name(file_path): diff --git a/tests/commands/run/test_integration.py b/tests/commands/run/test_integration.py index 0b846108..c9837a8c 100644 --- a/tests/commands/run/test_integration.py +++ b/tests/commands/run/test_integration.py @@ -238,6 +238,7 @@ def test_weak_compilation_flags(create_package): """ Test flag --weak-compilation-flags. """ + package_path = create_package parser = configure_parsers() args = parser.parse_args(["run", "--time-tool", "time"]) command = Command() @@ -503,3 +504,52 @@ def test_undocumented_test_limits_option(create_package, capsys): assert e.value.code == 1 out = capsys.readouterr().out assert "und1a.in: Specifying limit for a single test is not allowed in sinol-make." in out + + +@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_example_tests_package_path()], indirect=True) +def test_no_tests(create_package, time_tool, capsys): + """ + Test if `sinol-make` doesn't crash when there are no tests to run. + """ + parser = configure_parsers() + args = parser.parse_args(["run", "--time-tool", time_tool]) + command = Command() + with pytest.raises(SystemExit) as e: + command.run(args) + + assert e.value.code == 1 + out = capsys.readouterr().out + assert "There are no tests to run." in out + + +@pytest.mark.parametrize("create_package", [get_example_tests_package_path()], indirect=True) +def test_only_example_tests(create_package, time_tool, capsys): + """ + Test if `sinol-make` works only on example tests + """ + package_path = create_package + create_ins_outs(package_path) + parser = configure_parsers() + args = parser.parse_args(["run", "--time-tool", time_tool]) + command = Command() + command.run(args) + out = capsys.readouterr().out + assert "Expected scores are correct!" in out + + +@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_checker_package_path(), + get_library_package_path()], indirect=True) +def test_flag_tests_not_existing_tests(create_package, time_tool, capsys): + """ + Test flag --tests with not existing tests. + """ + package_path = create_package + create_ins_outs(package_path) + parser = configure_parsers() + args = parser.parse_args(["run", "--tests", "in/non_existing_file", "--time-tool", time_tool]) + command = Command() + with pytest.raises(SystemExit) as e: + command.run(args) + assert e.value.code == 1 + out = capsys.readouterr().out + assert "There are no tests to run." in out diff --git a/tests/commands/run/test_unit.py b/tests/commands/run/test_unit.py index 989c2883..eb1802a2 100644 --- a/tests/commands/run/test_unit.py +++ b/tests/commands/run/test_unit.py @@ -417,8 +417,8 @@ def test_set_scores(create_package): """ package_path = create_package command = get_command(package_path) - command.args = argparse.Namespace(tests=["in/abc0a.in", "in/abc1a.in", "in/abc2a.in", "in/abc3a.in", "in/abc4a.in", - "in/abc5a.in", "in/abc6a.in"]) + command.tests = ["in/abc0a.in", "in/abc1a.in", "in/abc2a.in", "in/abc3a.in", "in/abc4a.in", + "in/abc5a.in", "in/abc6a.in"] del command.config["scores"] command.set_scores() assert command.scores == { diff --git a/tests/packages/example_tests/config.yml b/tests/packages/example_tests/config.yml new file mode 100644 index 00000000..1ddcd9af --- /dev/null +++ b/tests/packages/example_tests/config.yml @@ -0,0 +1,8 @@ +title: Package with example tests +memory_limit: 10240 +time_limit: 1000 +sinol_expected_scores: + exa.cpp: + expected: {0: OK} + points: 0 +sinol_task_id: exa diff --git a/tests/packages/example_tests/in/.gitkeep b/tests/packages/example_tests/in/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/example_tests/out/.gitkeep b/tests/packages/example_tests/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/example_tests/prog/exa.cpp b/tests/packages/example_tests/prog/exa.cpp new file mode 100644 index 00000000..7d624fc6 --- /dev/null +++ b/tests/packages/example_tests/prog/exa.cpp @@ -0,0 +1,9 @@ +#include + +using namespace std; + +int main() { + int a, b; + cin >> a >> b; + cout << a + b << endl; +} diff --git a/tests/packages/example_tests/prog/exaingen.cpp b/tests/packages/example_tests/prog/exaingen.cpp new file mode 100644 index 00000000..c216a496 --- /dev/null +++ b/tests/packages/example_tests/prog/exaingen.cpp @@ -0,0 +1,17 @@ +#include + +using namespace std; + +int main() { + ofstream f("exa0.in"); + f << "1 1\n"; + f.close(); + + f.open("exa0a.in"); + f << "1 2\n"; + f.close(); + + f.open("exa0b.in"); + f << "1 3\n"; + f.close(); +} diff --git a/tests/packages/wcf/config.yml b/tests/packages/wcf/config.yml index b94ecced..05be7e6b 100644 --- a/tests/packages/wcf/config.yml +++ b/tests/packages/wcf/config.yml @@ -4,3 +4,8 @@ time_limit: 1000 scores: 1: 100 + +sinol_expected_scores: + wcf.cpp: + expected: {1: OK} + points: 100 diff --git a/tests/packages/wcf/in/wcf1a.in b/tests/packages/wcf/in/wcf1a.in new file mode 100644 index 00000000..2fb73a07 --- /dev/null +++ b/tests/packages/wcf/in/wcf1a.in @@ -0,0 +1 @@ +1 1 diff --git a/tests/packages/wcf/out/.gitkeep b/tests/packages/wcf/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/wcf/out/wcf1a.out b/tests/packages/wcf/out/wcf1a.out new file mode 100644 index 00000000..0cfbf088 --- /dev/null +++ b/tests/packages/wcf/out/wcf1a.out @@ -0,0 +1 @@ +2 diff --git a/tests/util.py b/tests/util.py index 74ecf9d1..e6468b27 100644 --- a/tests/util.py +++ b/tests/util.py @@ -109,6 +109,13 @@ def get_undocumented_options_package_path(): return os.path.join(os.path.dirname(__file__), "packages", "undocumented_options") +def get_example_tests_package_path(): + """ + Get path to package with only example tests (/tests/packages/example_tests) + """ + return os.path.join(os.path.dirname(__file__), "packages", "example_tests") + + def create_ins(package_path, task_id): """ Create .in files for package.