Multiple bugfixes for finding tests (#121)

* Fix error when running on group 0 and when there are no tests

* Fix error when non-
existing files are passed to --tests flag

* Add package with only example tests

* Add tests

* Fix tests

* Add pytest colors in workflows

.github/workflows/Arch.yaml

@@ -29,5 +29,7 @@ jobs:
         run: |
           pip3 install .[tests]
       - name: Run pytest
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
         run: |
           python3 -m pytest -v

.github/workflows/GithubRunner.yaml

@@ -19,5 +19,7 @@ jobs:
         run: |
           pip install .[tests]
       - name: Run pytest
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
         run: |
           python -m pytest -v --github-runner

.github/workflows/Ubuntu.yaml

@@ -35,5 +35,7 @@ jobs:
           pip3 install .[tests]
           apt install -y time
       - name: Run pytest
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
         run: |
           python3 -m pytest -v

.github/workflows/macOS.yaml

@@ -30,5 +30,7 @@ jobs:
         run: |
           python3 -m pip install .[tests]
       - name: Run pytest
+        env:
+          PYTEST_ADDOPTS: "--color=yes"
         run: |
           python3 -m pytest -v --time-tool time

src/sinol_make/commands/run/__init__.py

@@ -1057,7 +1057,6 @@ def exit(self):
                 cnt=len(self.failed_compilations), letter='' if len(self.failed_compilations) == 1 else 's'))
 
     def set_scores(self):
-        self.tests = package_util.get_tests(self.ID, self.args.tests)
         self.groups = self.get_groups(self.tests)
         self.scores = collections.defaultdict(int)
 
@@ -1069,6 +1068,11 @@ def set_scores(self):
                 num_groups -= 1
                 self.scores[0] = 0
 
+            # This only happens when running only on group 0.
+            if num_groups == 0:
+                self.possible_score = 0
+                return
+
             points_per_group = 100 // num_groups
             for group in self.groups:
                 if group == 0:
@@ -1138,7 +1142,7 @@ def check_are_any_tests_to_run(self):
             if not self.has_lib:
                 self.validate_existence_of_outputs()
         else:
-            print(util.warning('There are no tests to run.'))
+            util.exit_with_error('There are no tests to run.')
 
     def check_errors(self, results: Dict[str, Dict[str, Dict[str, ExecutionResult]]]):
         """
@@ -1197,8 +1201,9 @@ def run(self, args):
         lib = package_util.get_files_matching_pattern(self.ID, f'{self.ID}lib.*')
         self.has_lib = len(lib) != 0
 
-        self.set_scores()
+        self.tests = package_util.get_tests(self.ID, self.args.tests)
         self.check_are_any_tests_to_run()
+        self.set_scores()
         self.failed_compilations = []
         solutions = self.get_solutions(self.args.solutions)
 

src/sinol_make/helpers/package_util.py

@@ -57,7 +57,11 @@ def get_tests(task_id: str, arg_tests: Union[List[str], None] = None) -> List[st
                      if test[-3:] == ".in"]
         return sorted(all_tests, key=lambda test: get_test_key(test, task_id))
     else:
-        return sorted(list(set(arg_tests)), key=lambda test: get_test_key(test, task_id))
+        existing_tests = set()
+        for test in arg_tests:
+            if os.path.exists(test):
+                existing_tests.add(test)
+        return sorted(list(existing_tests), key=lambda test: get_test_key(test, task_id))
 
 
 def get_file_name(file_path):

tests/commands/run/test_integration.py

@@ -238,6 +238,7 @@ def test_weak_compilation_flags(create_package):
     """
     Test flag --weak-compilation-flags.
     """
+    package_path = create_package
     parser = configure_parsers()
     args = parser.parse_args(["run", "--time-tool", "time"])
     command = Command()
@@ -503,3 +504,52 @@ def test_undocumented_test_limits_option(create_package, capsys):
     assert e.value.code == 1
     out = capsys.readouterr().out
     assert "und1a.in: Specifying limit for a single test is not allowed in sinol-make." in out
+
+
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_example_tests_package_path()], indirect=True)
+def test_no_tests(create_package, time_tool, capsys):
+    """
+    Test if `sinol-make` doesn't crash when there are no tests to run.
+    """
+    parser = configure_parsers()
+    args = parser.parse_args(["run", "--time-tool", time_tool])
+    command = Command()
+    with pytest.raises(SystemExit) as e:
+        command.run(args)
+
+    assert e.value.code == 1
+    out = capsys.readouterr().out
+    assert "There are no tests to run." in out
+
+
+@pytest.mark.parametrize("create_package", [get_example_tests_package_path()], indirect=True)
+def test_only_example_tests(create_package, time_tool, capsys):
+    """
+    Test if `sinol-make` works only on example tests
+    """
+    package_path = create_package
+    create_ins_outs(package_path)
+    parser = configure_parsers()
+    args = parser.parse_args(["run", "--time-tool", time_tool])
+    command = Command()
+    command.run(args)
+    out = capsys.readouterr().out
+    assert "Expected scores are correct!" in out
+
+
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_checker_package_path(),
+                                            get_library_package_path()], indirect=True)
+def test_flag_tests_not_existing_tests(create_package, time_tool, capsys):
+    """
+    Test flag --tests with not existing tests.
+    """
+    package_path = create_package
+    create_ins_outs(package_path)
+    parser = configure_parsers()
+    args = parser.parse_args(["run", "--tests", "in/non_existing_file", "--time-tool", time_tool])
+    command = Command()
+    with pytest.raises(SystemExit) as e:
+        command.run(args)
+    assert e.value.code == 1
+    out = capsys.readouterr().out
+    assert "There are no tests to run." in out

tests/commands/run/test_unit.py

@@ -417,8 +417,8 @@ def test_set_scores(create_package):
     """
     package_path = create_package
     command = get_command(package_path)
-    command.args = argparse.Namespace(tests=["in/abc0a.in", "in/abc1a.in", "in/abc2a.in", "in/abc3a.in", "in/abc4a.in",
-                                             "in/abc5a.in", "in/abc6a.in"])
+    command.tests = ["in/abc0a.in", "in/abc1a.in", "in/abc2a.in", "in/abc3a.in", "in/abc4a.in",
+                     "in/abc5a.in", "in/abc6a.in"]
     del command.config["scores"]
     command.set_scores()
     assert command.scores == {

tests/packages/example_tests/config.yml

@@ -0,0 +1,8 @@
+title: Package with example tests
+memory_limit: 10240
+time_limit: 1000
+sinol_expected_scores:
+  exa.cpp:
+    expected: {0: OK}
+    points: 0
+sinol_task_id: exa

tests/packages/example_tests/prog/exa.cpp

@@ -0,0 +1,9 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b << endl;
+}

tests/packages/example_tests/prog/exaingen.cpp

@@ -0,0 +1,17 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    ofstream f("exa0.in");
+    f << "1 1\n";
+    f.close();
+
+    f.open("exa0a.in");
+    f << "1 2\n";
+    f.close();
+
+    f.open("exa0b.in");
+    f << "1 3\n";
+    f.close();
+}

tests/packages/wcf/config.yml

@@ -4,3 +4,8 @@ time_limit: 1000
 
 scores:
   1: 100
+
+sinol_expected_scores:
+  wcf.cpp:
+    expected: {1: OK}
+    points: 100

tests/packages/wcf/in/wcf1a.in

@@ -0,0 +1 @@
+1 1

tests/packages/wcf/out/wcf1a.out

@@ -0,0 +1 @@
+2

tests/util.py

@@ -109,6 +109,13 @@ def get_undocumented_options_package_path():
     return os.path.join(os.path.dirname(__file__), "packages", "undocumented_options")
 
 
+def get_example_tests_package_path():
+    """
+    Get path to package with only example tests (/tests/packages/example_tests)
+    """
+    return os.path.join(os.path.dirname(__file__), "packages", "example_tests")
+
+
 def create_ins(package_path, task_id):
     """
     Create .in files for package.