Add ICPC contest type (#128)

* Change format of `sinol_expected_scores`

* Fix tests

* Add icpc contest type

* Add test for version change

* Add icpc package

* Add icpc package

* Refactor

* Add unit tests for icpc contest type

* Refactor

* Add description of `sinol_contest_type: icpc`

* Update examples/config.yml

Co-authored-by: Tomasz Nowak <[email protected]>

* Lower version

* Remove printing of expected scores

* Fix tests

---------

Co-authored-by: Tomasz Nowak <[email protected]>

example_package/config.yml

@@ -73,9 +73,14 @@ sinol_task_id: abc
 # Possible values are:
 # - `default` - Points for a test can only be 100 or 0 (unless checker assigns points).
 #               Points for a group are calculated based of the lowest number of points for a test in this group.
+#               If scores are not defined in `scores` key, then all groups have the same number of points,
+#                summing up to 100.
 # - `oi` - Points for a test are unchanged if the execution time is less or equal to the time limit.
 #          Otherwise, number of points decreases linearly to one depending on the execution time.
 #          Points for a group are calculated same as in `default` mode.
+# - `icpc` - A test passes when the status is OK.
+#            A group passes when all tests in this group pass.
+#            A solution passes when all groups pass.
 sinol_contest_type: oi
 
 # sinol-make can check if the solutions run as expected when using `run` command.

src/sinol_make/__init__.py

@@ -61,6 +61,7 @@ def main_exn():
                 except Exception as err:
                     util.exit_with_error('`oiejq` could not be installed.\n' + err)
 
+            util.make_version_changes()
             command.run(args)
             exit(0)
 

src/sinol_make/commands/run/__init__.py

@@ -147,9 +147,11 @@ def print_table_end():
                         util.color_red(group_status)),
                           "%3s/%3s" % (points, scores[group]),
                           end=" | ")
-                program_scores[program] += points if group_status == Status.OK else 0
                 program_groups_scores[program][group] = {"status": group_status, "points": points}
             print()
+        for program in program_group:
+            program_scores[program] = contest.get_global_score(program_groups_scores[program], possible_score)
+
         print(8 * " ", end=" | ")
         for program in program_group:
             print(10 * " ", end=" | ")
@@ -258,9 +260,9 @@ def configure_subparser(self, subparser):
                             help=f'tool to measure time and memory usage (default: {default_timetool})')
         parser.add_argument('--oiejq-path', dest='oiejq_path', type=str,
                             help='path to oiejq executable (default: `~/.local/bin/oiejq`)')
-        add_compilation_arguments(parser)
         parser.add_argument('-a', '--apply-suggestions', dest='apply_suggestions', action='store_true',
                             help='apply suggestions from expected scores report')
+        add_compilation_arguments(parser)
 
     def parse_time(self, time_str):
         if len(time_str) < 3: return -1
@@ -638,7 +640,7 @@ def run_solutions(self, compiled_commands, names, solutions):
                 for test in self.tests:
                     all_results[name][self.get_group(test)][test] = ExecutionResult(Status.CE)
         print()
-        executions.sort(key = lambda x: (package_util.get_executable_key(x[1]), x[2]))
+        executions.sort(key = lambda x: (package_util.get_executable_key(x[1], self.ID), x[2]))
         program_groups_scores = collections.defaultdict(dict)
         print_data = PrintData(0)
 
@@ -695,19 +697,6 @@ def run_solutions(self, compiled_commands, names, solutions):
 
         return program_groups_scores, all_results
 
-
-    def calculate_points(self, results):
-        points = 0
-        for group, result in results.items():
-            if group != 0 and group not in self.scores:
-                util.exit_with_error(f'Group {group} doesn\'t have points specified in config file.')
-            if isinstance(result, str):
-                if result == Status.OK:
-                    points += self.scores[group]
-            elif isinstance(result, dict):
-                points += result["points"]
-        return points
-
     def compile_and_run(self, solutions):
         compilation_results = self.compile_solutions(solutions)
         for i in range(len(solutions)):
@@ -734,10 +723,6 @@ def _convert(obj):
                 return obj
         return _convert(dictionary)
 
-    def print_expected_scores(self, expected_scores):
-        yaml_dict = { "sinol_expected_scores": self.convert_status_to_string(expected_scores) }
-        print(yaml.dump(yaml_dict, default_flow_style=None))
-
     def get_whole_groups(self):
         """
         Returns a list of groups for which all tests were run.
@@ -770,33 +755,17 @@ def validate_expected_scores(self, results):
                 if group not in self.scores:
                     util.exit_with_error(f'Group {group} doesn\'t have points specified in config file.')
 
-        def convert_to_expected(results):
-            new_results = {}
-            for solution in results.keys():
-                new_results[solution] = {}
-                for group, result in results[solution].items():
-                    if result["status"] == Status.OK:
-                        if result["points"] == self.scores[group]:
-                            new_results[solution][group] = Status.OK
-                        else:
-                            new_results[solution][group] = result
-                    else:
-                        new_results[solution][group] = result["status"]
-            return new_results
-
-        results = convert_to_expected(results)
-
         if self.checker is None:
             for solution in results.keys():
                 new_expected_scores[solution] = {
                     "expected": results[solution],
-                    "points": self.calculate_points(results[solution])
+                    "points": self.contest.get_global_score(results[solution], self.possible_score)
                 }
         else:
             for solution in results.keys():
                 new_expected_scores[solution] = {
                     "expected": results[solution],
-                    "points": self.calculate_points(results[solution])
+                    "points": self.contest.get_global_score(results[solution], self.possible_score)
                 }
 
         config_expected_scores = self.config.get("sinol_expected_scores", {})
@@ -812,8 +781,7 @@ def convert_to_expected(results):
         used_groups = set()
         if self.args.tests == None and config_expected_scores: # If no groups were specified, use all groups from config
             for solution in config_expected_scores.keys():
-                for group in config_expected_scores[solution]["expected"]:
-                    used_groups.add(group)
+                used_groups.update(config_expected_scores[solution]["expected"].keys())
         else:
             used_groups = self.get_whole_groups()
 
@@ -848,17 +816,11 @@ def convert_to_expected(results):
                     if group in config_expected_scores[solution]["expected"]:
                         expected_scores[solution]["expected"][group] = config_expected_scores[solution]["expected"][group]
 
-                expected_scores[solution]["points"] = self.calculate_points(expected_scores[solution]["expected"])
+                expected_scores[solution]["points"] = self.contest.get_global_score(expected_scores[solution]["expected"],
+                                                                                    self.possible_score)
                 if len(expected_scores[solution]["expected"]) == 0:
                     del expected_scores[solution]
 
-        if self.args.tests is not None:
-            print("Showing expected scores only for groups with all tests run.")
-        print(util.bold("Expected scores from config:"))
-        self.print_expected_scores(expected_scores)
-        print(util.bold("\nExpected scores based on results:"))
-        self.print_expected_scores(new_expected_scores)
-
         expected_scores_diff = dictdiffer.diff(expected_scores, new_expected_scores)
         added_solutions = set()
         removed_solutions = set()
@@ -961,11 +923,17 @@ def print_points_change(solution, group, new_points, old_points):
             def delete_group(solution, group):
                 if group in config_expected_scores[solution]["expected"]:
                     del config_expected_scores[solution]["expected"][group]
-                    config_expected_scores[solution]["points"] = self.calculate_points(config_expected_scores[solution]["expected"])
+                    config_expected_scores[solution]["points"] = self.contest.get_global_score(
+                        config_expected_scores[solution]["expected"],
+                        self.possible_score
+                    )
 
             def set_group_result(solution, group, result):
                 config_expected_scores[solution]["expected"][group] = result
-                config_expected_scores[solution]["points"] = self.calculate_points(config_expected_scores[solution]["expected"])
+                config_expected_scores[solution]["points"] = self.contest.get_global_score(
+                    config_expected_scores[solution]["expected"],
+                    self.possible_score
+                )
 
 
             if self.args.apply_suggestions:
@@ -1055,33 +1023,7 @@ def set_scores(self):
         self.scores = collections.defaultdict(int)
 
         if 'scores' not in self.config.keys():
-            print(util.warning('Scores are not defined in config.yml. Points will be assigned equally to all groups.'))
-            num_groups = len(self.groups)
-            self.scores = {}
-            if self.groups[0] == 0:
-                num_groups -= 1
-                self.scores[0] = 0
-
-            # This only happens when running only on group 0.
-            if num_groups == 0:
-                self.possible_score = 0
-                return
-
-            points_per_group = 100 // num_groups
-            for group in self.groups:
-                if group == 0:
-                    continue
-                self.scores[group] = points_per_group
-
-            if points_per_group * num_groups != 100:
-                self.scores[self.groups[-1]] += 100 - points_per_group * num_groups
-
-            print("Points will be assigned as follows:")
-            total_score = 0
-            for group in self.scores:
-                print("%2d: %3d" % (group, self.scores[group]))
-                total_score += self.scores[group]
-            print()
+            self.scores = self.contest.assign_scores(self.groups)
         else:
             total_score = 0
             for group in self.config["scores"]:
@@ -1092,7 +1034,7 @@ def set_scores(self):
                 print(util.warning("WARN: Scores sum up to %d instead of 100." % total_score))
                 print()
 
-        self.possible_score = self.get_possible_score(self.groups)
+        self.possible_score = self.contest.get_possible_score(self.groups, self.scores)
 
     def get_valid_input_files(self):
         """

src/sinol_make/contest_types/__init__.py

@@ -2,6 +2,7 @@
 import yaml
 
 from sinol_make.contest_types.default import DefaultContest
+from sinol_make.contest_types.icpc import ICPCContest
 from sinol_make.contest_types.oi import OIContest
 from sinol_make.interfaces.Errors import UnknownContestType
 
@@ -15,5 +16,7 @@ def get_contest_type():
         return DefaultContest()
     elif contest_type == "oi":
         return OIContest()
+    elif contest_type == "icpc":
+        return ICPCContest()
     else:
         raise UnknownContestType(f'Unknown contest type "{contest_type}"')

src/sinol_make/contest_types/default.py

@@ -1,6 +1,7 @@
 from math import ceil
-from typing import List
+from typing import List, Dict
 
+from sinol_make import util
 from sinol_make.structs.status_structs import ExecutionResult
 
 
@@ -9,11 +10,86 @@ class DefaultContest:
     Default contest type.
     Points for tests are equal to points from execution result.
     Group score is equal to minimum score from tests.
+    Global score is sum of group scores.
+    Scores for groups are assigned equally.
+    Max possible score is sum of group scores.
     """
 
-    def get_test_score(self, result: ExecutionResult, time_limit, memory_limit):
+    def assign_scores(self, groups: List[int]) -> Dict[int, int]:
+        """
+        Returns dictionary with scores for each group.
+        Called if `scores` is not specified in config.
+        :param groups: List of groups
+        :return: Dictionary: {"<group>": <points>}
+        """
+        print(util.warning('Scores are not defined in config.yml. Points will be assigned equally to all groups.'))
+        num_groups = len(groups)
+        scores = {}
+        if groups[0] == 0:
+            num_groups -= 1
+            scores[0] = 0
+
+        # This only happens when running only on group 0.
+        if num_groups == 0:
+            return scores
+
+        points_per_group = 100 // num_groups
+        for group in groups:
+            if group == 0:
+                continue
+            scores[group] = points_per_group
+
+        if points_per_group * num_groups != 100:
+            scores[groups[-1]] += 100 - points_per_group * num_groups
+
+        print("Points will be assigned as follows:")
+        total_score = 0
+        for group in scores:
+            print("%2d: %3d" % (group, scores[group]))
+            total_score += scores[group]
+        print()
+        return scores
+
+    def get_possible_score(self, groups: List[int], scores: Dict[int, int]) -> int:
+        """
+        Get the maximum possible score.
+        :param groups: List of groups.
+        :param scores: Dictionary: {"<group>": <points>}
+        :return: Maximum possible score.
+        """
+        if groups[0] == 0 and len(groups) == 1:
+            return 0
+
+        possible_score = 0
+        for group in groups:
+            possible_score += scores[group]
+        return possible_score
+
+    def get_test_score(self, result: ExecutionResult, time_limit, memory_limit) -> int:
+        """
+        Returns points for test.
+        :param result: result of execution
+        :param time_limit: time limit for test
+        :param memory_limit: memory limit for test
+        :return: points for test
+        """
         return result.Points
 
-    def get_group_score(self, test_scores: List[int], group_max_score):
+    def get_group_score(self, test_scores: List[int], group_max_score) -> int:
+        """
+        Calculates group score based on tests scores.
+        :param test_scores: List of scores for tests
+        :param group_max_score: Maximum score for group
+        :return:
+        """
         min_score = min(test_scores)
         return int(ceil(group_max_score * (min_score / 100.0)))
+
+    def get_global_score(self, groups_scores: Dict[int, Dict], global_max_score) -> int:
+        """
+        Calculates global score based on groups scores.
+        :param groups_scores: Dictionary: {"<group>: {"status": Status, "points": <points for group>}
+        :param global_max_score: Maximum score for contest
+        :return: Global score
+        """
+        return sum(group["points"] for group in groups_scores.values())

src/sinol_make/contest_types/icpc.py

@@ -0,0 +1,31 @@
+from typing import List, Dict
+
+from sinol_make.structs.status_structs import ExecutionResult
+from sinol_make.contest_types import DefaultContest
+from sinol_make.structs.status_structs import Status
+
+
+class ICPCContest(DefaultContest):
+    """
+    Contest type for ACM ICPC type contest.
+    The possible score for one solution is 1 or 0.
+    The score is 0 if any of the tests fail.
+    """
+
+    def assign_scores(self, groups: List[int]) -> Dict[int, int]:
+        return {group: 1 for group in groups}
+
+    def get_possible_score(self, groups: List[int], scores: Dict[int, int]) -> int:
+        return 1
+
+    def get_test_score(self, result: ExecutionResult, time_limit, memory_limit):
+        if result.Status == Status.OK:
+            return 1
+        else:
+            return 0
+
+    def get_group_score(self, test_scores, group_max_score):
+        return min(test_scores)
+
+    def get_global_score(self, groups_scores: Dict[int, Dict], global_max_score):
+        return min(group["points"] for group in groups_scores.values())