From 25fe31e73164b0ca3883259f1424f605ad240f40 Mon Sep 17 00:00:00 2001
From: Mateusz Masiarz <matimasiarz@gmail.com>
Date: Sun, 24 Sep 2023 12:31:09 +0200
Subject: [PATCH] Add ICPC contest type (#128)

* Change format of `sinol_expected_scores`

* Fix tests

* Add icpc contest type

* Add test for version change

* Add icpc package

* Add icpc package

* Refactor

* Add unit tests for icpc contest type

* Refactor

* Add description of `sinol_contest_type: icpc`

* Update examples/config.yml

Co-authored-by: Tomasz Nowak <36604952+tonowak@users.noreply.github.com>

* Lower version

* Remove printing of expected scores

* Fix tests

---------

Co-authored-by: Tomasz Nowak <36604952+tonowak@users.noreply.github.com>
---
 example_package/config.yml                    |   5 +
 src/sinol_make/__init__.py                    |   1 +
 src/sinol_make/commands/run/__init__.py       |  98 +++----------
 src/sinol_make/contest_types/__init__.py      |   3 +
 src/sinol_make/contest_types/default.py       |  82 ++++++++++-
 src/sinol_make/contest_types/icpc.py          |  31 ++++
 src/sinol_make/helpers/package_util.py        |  29 ++--
 src/sinol_make/util.py                        |  38 +++++
 tests/commands/run/test_integration.py        |  78 ++---------
 tests/commands/run/test_unit.py               | 132 ++++++++----------
 tests/commands/run/util.py                    |   1 +
 tests/contest_types/test_default.py           |  19 +++
 tests/contest_types/test_icpc.py              |  20 +++
 tests/helpers/test_package_util.py            |   9 +-
 tests/packages/abc/config.yml                 |  70 ++++++----
 tests/packages/chk/config.yml                 |  13 +-
 tests/packages/example_tests/config.yml       |   6 +-
 tests/packages/hwr/config.yml                 |   8 +-
 tests/packages/icpc/config.yml                |  34 +++++
 tests/packages/icpc/in/.gitkeep               |   0
 tests/packages/icpc/out/.gitkeep              |   0
 tests/packages/icpc/prog/abc4.cpp             |  17 +++
 tests/packages/icpc/prog/acm.cpp              |   9 ++
 tests/packages/icpc/prog/acm1.cpp             |  12 ++
 tests/packages/icpc/prog/acm2.cpp             |  20 +++
 tests/packages/icpc/prog/acm3.cpp             |  34 +++++
 tests/packages/icpc/prog/acmingen.cpp         |  18 +++
 tests/packages/lib/config.yml                 |  12 +-
 tests/packages/lim/config.yml                 |  20 +--
 tests/packages/lsa/config.yml                 |  15 +-
 tests/packages/ovl/config.yml                 |   7 +-
 tests/packages/stc/config.yml                 |   5 +-
 .../packages/undocumented_options/config.yml  |   9 +-
 tests/packages/vso/config.yml                 |  26 ++--
 tests/packages/wcf/config.yml                 |   5 +-
 tests/util.py                                 |   7 +
 tests/version_changes/__init__.py             |   0
 .../test_expected_scores_format_change.py     |  31 ++++
 38 files changed, 617 insertions(+), 307 deletions(-)
 create mode 100644 src/sinol_make/contest_types/icpc.py
 create mode 100644 tests/contest_types/test_icpc.py
 create mode 100644 tests/packages/icpc/config.yml
 create mode 100644 tests/packages/icpc/in/.gitkeep
 create mode 100644 tests/packages/icpc/out/.gitkeep
 create mode 100644 tests/packages/icpc/prog/abc4.cpp
 create mode 100644 tests/packages/icpc/prog/acm.cpp
 create mode 100644 tests/packages/icpc/prog/acm1.cpp
 create mode 100644 tests/packages/icpc/prog/acm2.cpp
 create mode 100644 tests/packages/icpc/prog/acm3.cpp
 create mode 100644 tests/packages/icpc/prog/acmingen.cpp
 create mode 100644 tests/version_changes/__init__.py
 create mode 100644 tests/version_changes/test_expected_scores_format_change.py

diff --git a/example_package/config.yml b/example_package/config.yml
index b6228d8c..700f66ee 100644
--- a/example_package/config.yml
+++ b/example_package/config.yml
@@ -73,9 +73,14 @@ sinol_task_id: abc
 # Possible values are:
 # - `default` - Points for a test can only be 100 or 0 (unless checker assigns points).
 #               Points for a group are calculated based of the lowest number of points for a test in this group.
+#               If scores are not defined in `scores` key, then all groups have the same number of points,
+#                summing up to 100.
 # - `oi` - Points for a test are unchanged if the execution time is less or equal to the time limit.
 #          Otherwise, number of points decreases linearly to one depending on the execution time.
 #          Points for a group are calculated same as in `default` mode.
+# - `icpc` - A test passes when the status is OK.
+#            A group passes when all tests in this group pass.
+#            A solution passes when all groups pass.
 sinol_contest_type: oi
 
 # sinol-make can check if the solutions run as expected when using `run` command.
diff --git a/src/sinol_make/__init__.py b/src/sinol_make/__init__.py
index d54cc09a..15a5b656 100644
--- a/src/sinol_make/__init__.py
+++ b/src/sinol_make/__init__.py
@@ -61,6 +61,7 @@ def main_exn():
                 except Exception as err:
                     util.exit_with_error('`oiejq` could not be installed.\n' + err)
 
+            util.make_version_changes()
             command.run(args)
             exit(0)
 
diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py
index 5f305a5a..3bc4d649 100644
--- a/src/sinol_make/commands/run/__init__.py
+++ b/src/sinol_make/commands/run/__init__.py
@@ -147,9 +147,11 @@ def print_table_end():
                         util.color_red(group_status)),
                           "%3s/%3s" % (points, scores[group]),
                           end=" | ")
-                program_scores[program] += points if group_status == Status.OK else 0
                 program_groups_scores[program][group] = {"status": group_status, "points": points}
             print()
+        for program in program_group:
+            program_scores[program] = contest.get_global_score(program_groups_scores[program], possible_score)
+
         print(8 * " ", end=" | ")
         for program in program_group:
             print(10 * " ", end=" | ")
@@ -258,9 +260,9 @@ def configure_subparser(self, subparser):
                             help=f'tool to measure time and memory usage (default: {default_timetool})')
         parser.add_argument('--oiejq-path', dest='oiejq_path', type=str,
                             help='path to oiejq executable (default: `~/.local/bin/oiejq`)')
-        add_compilation_arguments(parser)
         parser.add_argument('-a', '--apply-suggestions', dest='apply_suggestions', action='store_true',
                             help='apply suggestions from expected scores report')
+        add_compilation_arguments(parser)
 
     def parse_time(self, time_str):
         if len(time_str) < 3: return -1
@@ -638,7 +640,7 @@ def run_solutions(self, compiled_commands, names, solutions):
                 for test in self.tests:
                     all_results[name][self.get_group(test)][test] = ExecutionResult(Status.CE)
         print()
-        executions.sort(key = lambda x: (package_util.get_executable_key(x[1]), x[2]))
+        executions.sort(key = lambda x: (package_util.get_executable_key(x[1], self.ID), x[2]))
         program_groups_scores = collections.defaultdict(dict)
         print_data = PrintData(0)
 
@@ -695,19 +697,6 @@ def run_solutions(self, compiled_commands, names, solutions):
 
         return program_groups_scores, all_results
 
-
-    def calculate_points(self, results):
-        points = 0
-        for group, result in results.items():
-            if group != 0 and group not in self.scores:
-                util.exit_with_error(f'Group {group} doesn\'t have points specified in config file.')
-            if isinstance(result, str):
-                if result == Status.OK:
-                    points += self.scores[group]
-            elif isinstance(result, dict):
-                points += result["points"]
-        return points
-
     def compile_and_run(self, solutions):
         compilation_results = self.compile_solutions(solutions)
         for i in range(len(solutions)):
@@ -734,10 +723,6 @@ def _convert(obj):
                 return obj
         return _convert(dictionary)
 
-    def print_expected_scores(self, expected_scores):
-        yaml_dict = { "sinol_expected_scores": self.convert_status_to_string(expected_scores) }
-        print(yaml.dump(yaml_dict, default_flow_style=None))
-
     def get_whole_groups(self):
         """
         Returns a list of groups for which all tests were run.
@@ -770,33 +755,17 @@ def validate_expected_scores(self, results):
                 if group not in self.scores:
                     util.exit_with_error(f'Group {group} doesn\'t have points specified in config file.')
 
-        def convert_to_expected(results):
-            new_results = {}
-            for solution in results.keys():
-                new_results[solution] = {}
-                for group, result in results[solution].items():
-                    if result["status"] == Status.OK:
-                        if result["points"] == self.scores[group]:
-                            new_results[solution][group] = Status.OK
-                        else:
-                            new_results[solution][group] = result
-                    else:
-                        new_results[solution][group] = result["status"]
-            return new_results
-
-        results = convert_to_expected(results)
-
         if self.checker is None:
             for solution in results.keys():
                 new_expected_scores[solution] = {
                     "expected": results[solution],
-                    "points": self.calculate_points(results[solution])
+                    "points": self.contest.get_global_score(results[solution], self.possible_score)
                 }
         else:
             for solution in results.keys():
                 new_expected_scores[solution] = {
                     "expected": results[solution],
-                    "points": self.calculate_points(results[solution])
+                    "points": self.contest.get_global_score(results[solution], self.possible_score)
                 }
 
         config_expected_scores = self.config.get("sinol_expected_scores", {})
@@ -812,8 +781,7 @@ def convert_to_expected(results):
         used_groups = set()
         if self.args.tests == None and config_expected_scores: # If no groups were specified, use all groups from config
             for solution in config_expected_scores.keys():
-                for group in config_expected_scores[solution]["expected"]:
-                    used_groups.add(group)
+                used_groups.update(config_expected_scores[solution]["expected"].keys())
         else:
             used_groups = self.get_whole_groups()
 
@@ -848,17 +816,11 @@ def convert_to_expected(results):
                     if group in config_expected_scores[solution]["expected"]:
                         expected_scores[solution]["expected"][group] = config_expected_scores[solution]["expected"][group]
 
-                expected_scores[solution]["points"] = self.calculate_points(expected_scores[solution]["expected"])
+                expected_scores[solution]["points"] = self.contest.get_global_score(expected_scores[solution]["expected"],
+                                                                                    self.possible_score)
                 if len(expected_scores[solution]["expected"]) == 0:
                     del expected_scores[solution]
 
-        if self.args.tests is not None:
-            print("Showing expected scores only for groups with all tests run.")
-        print(util.bold("Expected scores from config:"))
-        self.print_expected_scores(expected_scores)
-        print(util.bold("\nExpected scores based on results:"))
-        self.print_expected_scores(new_expected_scores)
-
         expected_scores_diff = dictdiffer.diff(expected_scores, new_expected_scores)
         added_solutions = set()
         removed_solutions = set()
@@ -961,11 +923,17 @@ def print_points_change(solution, group, new_points, old_points):
             def delete_group(solution, group):
                 if group in config_expected_scores[solution]["expected"]:
                     del config_expected_scores[solution]["expected"][group]
-                    config_expected_scores[solution]["points"] = self.calculate_points(config_expected_scores[solution]["expected"])
+                    config_expected_scores[solution]["points"] = self.contest.get_global_score(
+                        config_expected_scores[solution]["expected"],
+                        self.possible_score
+                    )
 
             def set_group_result(solution, group, result):
                 config_expected_scores[solution]["expected"][group] = result
-                config_expected_scores[solution]["points"] = self.calculate_points(config_expected_scores[solution]["expected"])
+                config_expected_scores[solution]["points"] = self.contest.get_global_score(
+                    config_expected_scores[solution]["expected"],
+                    self.possible_score
+                )
 
 
             if self.args.apply_suggestions:
@@ -1055,33 +1023,7 @@ def set_scores(self):
         self.scores = collections.defaultdict(int)
 
         if 'scores' not in self.config.keys():
-            print(util.warning('Scores are not defined in config.yml. Points will be assigned equally to all groups.'))
-            num_groups = len(self.groups)
-            self.scores = {}
-            if self.groups[0] == 0:
-                num_groups -= 1
-                self.scores[0] = 0
-
-            # This only happens when running only on group 0.
-            if num_groups == 0:
-                self.possible_score = 0
-                return
-
-            points_per_group = 100 // num_groups
-            for group in self.groups:
-                if group == 0:
-                    continue
-                self.scores[group] = points_per_group
-
-            if points_per_group * num_groups != 100:
-                self.scores[self.groups[-1]] += 100 - points_per_group * num_groups
-
-            print("Points will be assigned as follows:")
-            total_score = 0
-            for group in self.scores:
-                print("%2d: %3d" % (group, self.scores[group]))
-                total_score += self.scores[group]
-            print()
+            self.scores = self.contest.assign_scores(self.groups)
         else:
             total_score = 0
             for group in self.config["scores"]:
@@ -1092,7 +1034,7 @@ def set_scores(self):
                 print(util.warning("WARN: Scores sum up to %d instead of 100." % total_score))
                 print()
 
-        self.possible_score = self.get_possible_score(self.groups)
+        self.possible_score = self.contest.get_possible_score(self.groups, self.scores)
 
     def get_valid_input_files(self):
         """
diff --git a/src/sinol_make/contest_types/__init__.py b/src/sinol_make/contest_types/__init__.py
index ea83390b..309198df 100644
--- a/src/sinol_make/contest_types/__init__.py
+++ b/src/sinol_make/contest_types/__init__.py
@@ -2,6 +2,7 @@
 import yaml
 
 from sinol_make.contest_types.default import DefaultContest
+from sinol_make.contest_types.icpc import ICPCContest
 from sinol_make.contest_types.oi import OIContest
 from sinol_make.interfaces.Errors import UnknownContestType
 
@@ -15,5 +16,7 @@ def get_contest_type():
         return DefaultContest()
     elif contest_type == "oi":
         return OIContest()
+    elif contest_type == "icpc":
+        return ICPCContest()
     else:
         raise UnknownContestType(f'Unknown contest type "{contest_type}"')
diff --git a/src/sinol_make/contest_types/default.py b/src/sinol_make/contest_types/default.py
index 727570a0..b55df0ba 100644
--- a/src/sinol_make/contest_types/default.py
+++ b/src/sinol_make/contest_types/default.py
@@ -1,6 +1,7 @@
 from math import ceil
-from typing import List
+from typing import List, Dict
 
+from sinol_make import util
 from sinol_make.structs.status_structs import ExecutionResult
 
 
@@ -9,11 +10,86 @@ class DefaultContest:
     Default contest type.
     Points for tests are equal to points from execution result.
     Group score is equal to minimum score from tests.
+    Global score is sum of group scores.
+    Scores for groups are assigned equally.
+    Max possible score is sum of group scores.
     """
 
-    def get_test_score(self, result: ExecutionResult, time_limit, memory_limit):
+    def assign_scores(self, groups: List[int]) -> Dict[int, int]:
+        """
+        Returns dictionary with scores for each group.
+        Called if `scores` is not specified in config.
+        :param groups: List of groups
+        :return: Dictionary: {"<group>": <points>}
+        """
+        print(util.warning('Scores are not defined in config.yml. Points will be assigned equally to all groups.'))
+        num_groups = len(groups)
+        scores = {}
+        if groups[0] == 0:
+            num_groups -= 1
+            scores[0] = 0
+
+        # This only happens when running only on group 0.
+        if num_groups == 0:
+            return scores
+
+        points_per_group = 100 // num_groups
+        for group in groups:
+            if group == 0:
+                continue
+            scores[group] = points_per_group
+
+        if points_per_group * num_groups != 100:
+            scores[groups[-1]] += 100 - points_per_group * num_groups
+
+        print("Points will be assigned as follows:")
+        total_score = 0
+        for group in scores:
+            print("%2d: %3d" % (group, scores[group]))
+            total_score += scores[group]
+        print()
+        return scores
+
+    def get_possible_score(self, groups: List[int], scores: Dict[int, int]) -> int:
+        """
+        Get the maximum possible score.
+        :param groups: List of groups.
+        :param scores: Dictionary: {"<group>": <points>}
+        :return: Maximum possible score.
+        """
+        if groups[0] == 0 and len(groups) == 1:
+            return 0
+
+        possible_score = 0
+        for group in groups:
+            possible_score += scores[group]
+        return possible_score
+
+    def get_test_score(self, result: ExecutionResult, time_limit, memory_limit) -> int:
+        """
+        Returns points for test.
+        :param result: result of execution
+        :param time_limit: time limit for test
+        :param memory_limit: memory limit for test
+        :return: points for test
+        """
         return result.Points
 
-    def get_group_score(self, test_scores: List[int], group_max_score):
+    def get_group_score(self, test_scores: List[int], group_max_score) -> int:
+        """
+        Calculates group score based on tests scores.
+        :param test_scores: List of scores for tests
+        :param group_max_score: Maximum score for group
+        :return:
+        """
         min_score = min(test_scores)
         return int(ceil(group_max_score * (min_score / 100.0)))
+
+    def get_global_score(self, groups_scores: Dict[int, Dict], global_max_score) -> int:
+        """
+        Calculates global score based on groups scores.
+        :param groups_scores: Dictionary: {"<group>: {"status": Status, "points": <points for group>}
+        :param global_max_score: Maximum score for contest
+        :return: Global score
+        """
+        return sum(group["points"] for group in groups_scores.values())
diff --git a/src/sinol_make/contest_types/icpc.py b/src/sinol_make/contest_types/icpc.py
new file mode 100644
index 00000000..6031a75c
--- /dev/null
+++ b/src/sinol_make/contest_types/icpc.py
@@ -0,0 +1,31 @@
+from typing import List, Dict
+
+from sinol_make.structs.status_structs import ExecutionResult
+from sinol_make.contest_types import DefaultContest
+from sinol_make.structs.status_structs import Status
+
+
+class ICPCContest(DefaultContest):
+    """
+    Contest type for ACM ICPC type contest.
+    The possible score for one solution is 1 or 0.
+    The score is 0 if any of the tests fail.
+    """
+
+    def assign_scores(self, groups: List[int]) -> Dict[int, int]:
+        return {group: 1 for group in groups}
+
+    def get_possible_score(self, groups: List[int], scores: Dict[int, int]) -> int:
+        return 1
+
+    def get_test_score(self, result: ExecutionResult, time_limit, memory_limit):
+        if result.Status == Status.OK:
+            return 1
+        else:
+            return 0
+
+    def get_group_score(self, test_scores, group_max_score):
+        return min(test_scores)
+
+    def get_global_score(self, groups_scores: Dict[int, Dict], global_max_score):
+        return min(group["points"] for group in groups_scores.values())
diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py
index 3bc310a6..4afd68f9 100644
--- a/src/sinol_make/helpers/package_util.py
+++ b/src/sinol_make/helpers/package_util.py
@@ -41,6 +41,10 @@ def get_group(test_path, task_id):
     return int("".join(filter(str.isdigit, extract_test_id(test_path, task_id))))
 
 
+def get_groups(tests, task_id):
+    return sorted(list(set([get_group(test, task_id) for test in tests])))
+
+
 def get_test_key(test, task_id):
     return get_group(test, task_id), test
 
@@ -53,19 +57,26 @@ def get_solutions_re(task_id: str) -> re.Pattern:
     return re.compile(r"^%s[bs]?[0-9]*\.(cpp|cc|java|py|pas)$" % task_id)
 
 
-def get_executable_key(executable):
+def get_executable_key(executable, task_id):
     name = get_file_name(executable)
+    task_id_len = len(task_id)
     value = [0, 0]
-    if name[3] == 's':
+    if name[task_id_len] == 's':
         value[0] = 1
-        suffix = name.split(".")[0][4:]
-    elif name[3] == 'b':
+        suffix = name.split(".")[0][(task_id_len + 1):]
+    elif name[task_id_len] == 'b':
         value[0] = 2
-        suffix = name.split(".")[0][4:]
+        suffix = name.split(".")[0][(task_id_len + 1):]
     else:
-        suffix = name.split(".")[0][3:]
+        suffix = name.split(".")[0][task_id_len:]
     if suffix != "":
-        value[1] = int(suffix)
+        i = 0
+        digits = ""
+        while i < len(suffix) and suffix[i].isdigit():
+            digits += suffix[i]
+            i += 1
+        if digits != "":
+            value[1] = int(digits)
     return tuple(value)
 
 
@@ -123,7 +134,7 @@ def get_solutions(task_id: str, args_solutions: Union[List[str], None] = None) -
     if args_solutions is None:
         solutions = [solution for solution in os.listdir("prog/")
                      if solutions_re.match(solution)]
-        return sorted(solutions, key=get_executable_key)
+        return sorted(solutions, key=lambda solution: get_executable_key(solution, task_id))
     else:
         solutions = []
         for solution in get_files_matching(args_solutions, "prog"):
@@ -132,7 +143,7 @@ def get_solutions(task_id: str, args_solutions: Union[List[str], None] = None) -
             if solutions_re.match(os.path.basename(solution)) is not None:
                 solutions.append(os.path.basename(solution))
 
-        return sorted(solutions, key=get_executable_key)
+        return sorted(solutions, key=lambda solution: get_executable_key(solution, task_id))
 
 
 def get_file_name(file_path):
diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py
index 68c2f4f9..ab0e7167 100644
--- a/src/sinol_make/util.py
+++ b/src/sinol_make/util.py
@@ -9,6 +9,7 @@
 from typing import Union
 
 import sinol_make
+from sinol_make.contest_types import get_contest_type
 
 
 def get_commands():
@@ -286,6 +287,43 @@ def get_file_md5(path):
         return hashlib.md5(f.read()).hexdigest()
 
 
+def make_version_changes():
+    if compare_versions(sinol_make.__version__, "1.5.8") == 1:
+        # In version 1.5.9 we changed the format of sinol_expected_scores.
+        # Now all groups have specified points and status.
+
+        if check_if_package():
+            with open("config.yml", "r") as config_file:
+                config = yaml.load(config_file, Loader=yaml.FullLoader)
+
+            try:
+                new_expected_scores = {}
+                expected_scores = config["sinol_expected_scores"]
+                contest = get_contest_type()
+                groups = []
+                for solution, results in expected_scores.items():
+                    for group in results["expected"].keys():
+                        if group not in groups:
+                            groups.append(int(group))
+
+                scores = contest.assign_scores(groups)
+                for solution, results in expected_scores.items():
+                    new_expected_scores[solution] = {"expected": {}, "points": results["points"]}
+                    for group, result in results["expected"].items():
+                        new_expected_scores[solution]["expected"][group] = {"status": result}
+                        if result == "OK":
+                            new_expected_scores[solution]["expected"][group]["points"] = scores[group]
+                        else:
+                            new_expected_scores[solution]["expected"][group]["points"] = 0
+                config["sinol_expected_scores"] = new_expected_scores
+                save_config(config)
+            except:
+                # If there is an error, we just delete the field.
+                if "sinol_expected_scores" in config:
+                    del config["sinol_expected_scores"]
+                    save_config(config)
+
+
 def color_red(text): return "\033[91m{}\033[00m".format(text)
 def color_green(text): return "\033[92m{}\033[00m".format(text)
 def color_yellow(text): return "\033[93m{}\033[00m".format(text)
diff --git a/tests/commands/run/test_integration.py b/tests/commands/run/test_integration.py
index cc1df96b..84dd9b19 100644
--- a/tests/commands/run/test_integration.py
+++ b/tests/commands/run/test_integration.py
@@ -14,7 +14,7 @@
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
                                             get_checker_package_path(), get_library_package_path(),
                                             get_library_string_args_package_path(), get_limits_package_path(),
-                                            get_override_limits_package_path()],
+                                            get_override_limits_package_path(), get_icpc_package_path()],
                          indirect=True)
 def test_simple(create_package, time_tool):
     """
@@ -33,7 +33,7 @@ def test_simple(create_package, time_tool):
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
                                             get_checker_package_path(), get_library_package_path(),
                                             get_library_string_args_package_path(), get_limits_package_path(),
-                                            get_override_limits_package_path()],
+                                            get_override_limits_package_path(), get_icpc_package_path()],
                          indirect=True)
 def test_no_expected_scores(capsys, create_package, time_tool):
     """
@@ -69,7 +69,7 @@ def test_no_expected_scores(capsys, create_package, time_tool):
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
                                             get_checker_package_path(), get_library_package_path(),
                                             get_library_string_args_package_path(), get_limits_package_path(),
-                                            get_override_limits_package_path()],
+                                            get_override_limits_package_path(), get_icpc_package_path()],
                          indirect=True)
 def test_apply_suggestions(create_package, time_tool):
     """
@@ -110,7 +110,7 @@ def test_incorrect_expected_scores(capsys, create_package, time_tool):
     config_path = os.path.join(package_path, "config.yml")
     with open(config_path, "r") as config_file:
         config = yaml.load(config_file, Loader=yaml.SafeLoader)
-    config["sinol_expected_scores"]["abc.cpp"]["expected"][1] = "WA"
+    config["sinol_expected_scores"]["abc.cpp"]["expected"][1] = {"status": "WA", "points": 0}
     config["sinol_expected_scores"]["abc.cpp"]["points"] = 75
     with open(config_path, "w") as config_file:
         config_file.write(yaml.dump(config))
@@ -130,7 +130,8 @@ def test_incorrect_expected_scores(capsys, create_package, time_tool):
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_checker_package_path(),
-                                            get_library_package_path(), get_library_string_args_package_path()],
+                                            get_library_package_path(), get_library_string_args_package_path(),
+                                            get_icpc_package_path()],
                          indirect=True)
 def test_flag_tests(create_package, time_tool):
     """
@@ -152,68 +153,8 @@ def test_flag_tests(create_package, time_tool):
     assert command.tests == [os.path.join("in", os.path.basename(test))]
 
 
-@pytest.mark.parametrize("create_package", [get_checker_package_path()], indirect=True)
-def test_groups_in_flag_test(capsys, create_package, time_tool):
-    """
-    Test flag --tests with whole and partial groups.
-    """
-    package_path = create_package
-    create_ins_outs(package_path)
-
-    parser = configure_parsers()
-
-    # Test with only one test from group 1.
-    args = parser.parse_args(["run", "--tests", "in/chk1a.in", "--time-tool", time_tool])
-    command = Command()
-    command.run(args)
-    out = capsys.readouterr().out
-    assert "Showing expected scores only for groups with all tests run." in out
-    assert "sinol_expected_scores: {}" in out
-    assert "Expected scores are correct!" in out
-
-    # Test with all tests from group 1.
-    args = parser.parse_args(["run", "--tests", "in/chk1a.in", "in/chk1b.in", "in/chk1c.in", "--time-tool", time_tool])
-    command = Command()
-    command.run(args)
-    out = capsys.readouterr().out
-    assert 'sinol_expected_scores:\n' \
-           '  chk.cpp:\n' \
-           '    expected: {1: OK}\n' \
-           '    points: 50\n' \
-           '  chk1.cpp:\n' \
-           '    expected: {1: WA}\n' \
-           '    points: 0\n' \
-           '  chk2.cpp:\n' \
-           '    expected:\n' \
-           '      1: {points: 25, status: OK}\n' \
-           '    points: 25\n' \
-           '  chk3.cpp:\n' \
-           '    expected: {1: OK}\n' \
-           '    points: 50' in out
-
-    # Test with incorrect expected scores for first group.
-    with open(os.path.join(package_path, "config.yml"), "r") as config_file:
-        correct_config = yaml.load(config_file, Loader=yaml.SafeLoader)
-    config = copy.deepcopy(correct_config)
-    config["sinol_expected_scores"]["chk.cpp"]["expected"][1] = "WA"
-    config["sinol_expected_scores"]["chk.cpp"]["points"] = 50
-    with open(os.path.join(package_path, "config.yml"), "w") as config_file:
-        config_file.write(yaml.dump(config))
-
-    args = parser.parse_args(["run", "--tests", "in/chk1a.in", "in/chk1b.in", "in/chk1c.in", "--time-tool", time_tool,
-                              "--apply-suggestions"])
-    command = Command()
-    command.run(args)
-    out = capsys.readouterr().out
-    sys.stdout.write(out)
-    assert "Solution chk.cpp passed group 1 with status OK while it should pass with status WA." in out
-    with open(os.path.join(package_path, "config.yml"), "r") as config_file:
-        config = yaml.load(config_file, Loader=yaml.SafeLoader)
-    assert config == correct_config
-
-
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path()], indirect=True)
+                                            get_checker_package_path(), get_icpc_package_path()], indirect=True)
 def test_flag_solutions(capsys, create_package, time_tool):
     """
     Test flag --solutions.
@@ -412,7 +353,6 @@ def test_override_limits(create_package, time_tool):
     create_ins_outs(package_path)
     config_file_path = os.path.join(package_path, "config.yml")
 
-
     # With `override_limits` key deleted.
     with open(config_file_path, "r") as config_file:
         original_config = yaml.load(config_file, Loader=yaml.SafeLoader)
@@ -430,7 +370,7 @@ def test_override_limits(create_package, time_tool):
 
     assert config["sinol_expected_scores"] == {
         "ovl.cpp": {
-            "expected": {1: "TL", 2: "TL"},
+            "expected": {1: {"status": "TL", "points": 0}, 2: {"status": "TL", "points": 0}},
             "points": 0
         }
     }
@@ -451,7 +391,7 @@ def test_override_limits(create_package, time_tool):
 
     assert config["sinol_expected_scores"] == {
         "ovl.cpp": {
-            "expected": {1: "ML", 2: "ML"},
+            "expected": {1: {"status": "ML", "points": 0}, 2: {"status": "ML", "points": 0}},
             "points": 0
         }
     }
diff --git a/tests/commands/run/test_unit.py b/tests/commands/run/test_unit.py
index bb9502aa..5e2d2a87 100644
--- a/tests/commands/run/test_unit.py
+++ b/tests/commands/run/test_unit.py
@@ -43,18 +43,6 @@ def test_execution(create_package, time_tool):
     assert result.Status == Status.OK
 
 
-def test_calculate_points():
-    os.chdir(get_simple_package_path())
-    command = get_command()
-    command.scores = command.config["scores"]
-
-    assert command.calculate_points({1: Status.OK, 2: Status.OK, 3: Status.OK, 4: Status.OK}) == 100
-    assert command.calculate_points({1: Status.OK, 2: Status.OK, 3: Status.OK, 4: Status.WA}) == 75
-    assert command.calculate_points({1: Status.OK, 2: Status.OK, 3: Status.TL}) == 50
-    assert command.calculate_points({1: Status.OK}) == 25
-    assert command.calculate_points({1: Status.WA}) == 0
-
-
 def test_run_solutions(create_package, time_tool):
     package_path = create_package
     command = get_command(package_path)
@@ -84,42 +72,16 @@ def flatten_results(results):
     }
 
 
-def test_print_expected_scores(capsys):
-    os.chdir(get_simple_package_path())
-    command = get_command()
-    expected_scores = """sinol_expected_scores:
-  abc.cpp:
-    expected: {1: "OK", 2: "OK", 3: "OK", 4: "OK"}
-    points: 100
-  abc1.cpp:
-    expected: {1: "OK", 2: "OK", 3: "OK", 4: "WA"}
-    points: 75
-  abc2.cpp:
-    expected: {1: "OK", 2: "WA", 3: "WA", 4: "TL"}
-    points: 25
-  abc3.cpp:
-    expected: {1: "OK", 2: "WA", 3: "WA", 4: "ML"}
-    points: 25
-  abc4.cpp:
-    expected: {1: "OK", 2: "OK", 3: "WA", 4: "RE"}
-    points: 50
-
-"""
-
-    expected_scores_dict = yaml.load(expected_scores, Loader=yaml.FullLoader)
-    command.print_expected_scores(expected_scores_dict["sinol_expected_scores"])
-    out = capsys.readouterr().out
-    assert expected_scores.replace('"', '') in out
-
-
 def test_validate_expected_scores_success():
     os.chdir(get_simple_package_path())
     command = get_command()
     command.scores = command.config["scores"]
-    command.tests = package_util.get_tests("abc", None)
+    command.tests = ["in/abc1a.in", "in/abc2a.in", "in/abc3a.in", "in/abc4a.in"]
+    command.groups = command.get_groups(command.tests)
+    command.possible_score = command.contest.get_possible_score(command.groups, command.scores)
 
     # Test with correct expected scores.
-    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
+    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None, print_expected_scores=True)
     results = {
         "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "OK", "points": 25}},
     }
@@ -128,16 +90,16 @@ def test_validate_expected_scores_success():
     assert results.removed_solutions == set()
 
     # Test with incorrect result.
-    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
+    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None, print_expected_scores=True)
     results = {
         "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "WA", "points": 0}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores != results.new_expected_scores
-    assert len(results.changes) == 1
+    assert len(results.changes) == 2
 
     # Test with removed solution.
-    command.args = argparse.Namespace(solutions=None, tests=None)
+    command.args = argparse.Namespace(solutions=None, tests=None, print_expected_scores=True)
     results = {
         "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "OK", "points": 25}},
         "abc1.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "WA", "points": 0}},
@@ -150,7 +112,8 @@ def test_validate_expected_scores_success():
 
     # Test with added solution and added group.
     command.config["scores"][5] = 0
-    command.args = argparse.Namespace(solutions=["prog/abc.cpp", "prog/abc5.cpp"], tests=None)
+    command.args = argparse.Namespace(solutions=["prog/abc.cpp", "prog/abc5.cpp"], tests=None,
+                                      print_expected_scores=True)
     results = {
         "abc.cpp": {1: {"status": "OK", "points": 20}, 2: {"status": "OK", "points": 20}, 3: {"status": "OK", "points": 20}, 4: {"status": "OK", "points": 20}, 5: {"status": "WA", "points": 0}},
         "abc5.cpp": {1: {"status": "OK", "points": 20}, 2: {"status": "OK", "points": 20}, 3: {"status": "OK", "points": 20}, 4: {"status": "OK", "points": 20}, 5: {"status": "WA", "points": 0}},
@@ -161,7 +124,7 @@ def test_validate_expected_scores_success():
     assert len(results.added_groups) == 1
 
     # Test with removed group.
-    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
+    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None, print_expected_scores=True)
     results = {
         "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}},
     }
@@ -170,7 +133,8 @@ def test_validate_expected_scores_success():
     assert len(results.removed_groups) == 1
 
     # Test with correct expected scores and --tests flag.
-    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=["in/abc1a.in", "in/abc2a.in"])
+    command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=["in/abc1a.in", "in/abc2a.in"],
+                                      print_expected_scores=True)
     results = {
         "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}},
     }
@@ -201,6 +165,7 @@ def test_print_expected_scores_diff(capsys, create_package):
     package_path = create_package
     command = get_command(package_path)
     command.args = argparse.Namespace(apply_suggestions=False)
+    command.possible_score = 100
 
     # Test with correct expected scores.
     results = ValidationResult(
@@ -210,10 +175,12 @@ def test_print_expected_scores_diff(capsys, create_package):
         removed_groups=set(),
         changes=[],
         expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         },
         new_expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         }
     )
     command.print_expected_scores_diff(results)
@@ -228,10 +195,12 @@ def test_print_expected_scores_diff(capsys, create_package):
         removed_groups=set(),
         changes=[ResultChange("abc.cpp", 1, "OK", "WA")],
         expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         },
         new_expected_scores={
-            "abc.cpp": {1: "WA", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "WA", "points": 0}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         }
     )
     with pytest.raises(SystemExit) as e:
@@ -249,11 +218,14 @@ def test_print_expected_scores_diff(capsys, create_package):
         removed_groups=set(),
         changes=[],
         expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         },
         new_expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
-            "abc5.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
+            "abc5.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         }
     )
     with pytest.raises(SystemExit) as e:
@@ -271,11 +243,14 @@ def test_print_expected_scores_diff(capsys, create_package):
         removed_groups=set(),
         changes=[],
         expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
-            "abc5.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
+            "abc5.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         },
         new_expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         }
     )
     with pytest.raises(SystemExit) as e:
@@ -293,10 +268,13 @@ def test_print_expected_scores_diff(capsys, create_package):
         removed_groups=set(),
         changes=[],
         expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         },
         new_expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK", 5: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100},
+                        5: {"status": "OK", "points": 100}},
         }
     )
     with pytest.raises(SystemExit) as e:
@@ -314,10 +292,13 @@ def test_print_expected_scores_diff(capsys, create_package):
         removed_groups={5},
         changes=[],
         expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK", 5: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100},
+                        5: {"status": "OK", "points": 100}},
         },
         new_expected_scores={
-            "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+            "abc.cpp": {1: {"status": "OK", "points": 100}, 2: {"status": "OK", "points": 100},
+                        3: {"status": "OK", "points": 100}, 4: {"status": "OK", "points": 100}},
         }
     )
     with pytest.raises(SystemExit) as e:
@@ -339,21 +320,25 @@ def test_print_expected_scores_diff(capsys, create_package):
         changes=[ResultChange("abc.cpp", 1, "OK", "WA")],
         expected_scores={
             "abc.cpp": {
-                "expected": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+                "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25},
+                             3: {"status": "OK", "points": 25}, 4: {"status": "OK", "points": 25}},
                 "points": 100
             },
             "abc4.cpp": {
-                "expected": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+                "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25},
+                             3: {"status": "OK", "points": 25}, 4: {"status": "OK", "points": 25}},
                 "points": 100
             }
         },
         new_expected_scores={
             "abc.cpp": {
-                "expected": {1: "WA", 2: "OK", 3: "OK", 5: "OK"},
+                "expected": {1: {"status": "WA", "points": 0}, 2: {"status": "OK", "points": 25},
+                             3: {"status": "OK", "points": 25}, 5: {"status": "OK", "points": 0}},
                 "points": 50
             },
             "abc5.cpp": {
-                "expected": {1: "OK", 2: "OK", 3: "OK", 5: "OK"},
+                "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25},
+                             3: {"status": "OK", "points": 25}, 5: {"status": "OK", "points": 0}},
                 "points": 75
             }
         }
@@ -371,23 +356,28 @@ def test_print_expected_scores_diff(capsys, create_package):
         config = yaml.load(config_file, Loader=yaml.FullLoader)
     assert config["sinol_expected_scores"] == {
         "abc.cpp": {
-            "expected": {1: "WA", 2: "OK", 3: "OK", 5: "OK"},
+            "expected": {1: {"status": "WA", "points": 0}, 2: {"status": "OK", "points": 25},
+                         3: {"status": "OK", "points": 25}, 5: {"status": "OK", "points": 0}},
             "points": 50
         },
         "abc1.cpp": {
-            "expected": {1: "OK", 2: "OK", 3: "OK"},
+            "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25},
+                         3: {"status": "OK", "points": 25}},
             "points": 75
         },
         "abc2.cpp": {
-            "expected": {1: "OK", 2: "WA", 3: "WA"},
+            "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "WA", "points": 0},
+                         3: {"status": "WA", "points": 0}},
             "points": 25
         },
         "abc3.cpp": {
-            "expected": {1: "OK", 2: "WA", 3: "WA"},
+            "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "WA", "points": 0},
+                         3: {"status": "WA", "points": 0}},
             "points": 25
         },
         "abc5.cpp": {
-            "expected": {1: "OK", 2: "OK", 3: "OK", 5: "OK"},
+            "expected": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25},
+                         3: {"status": "OK", "points": 25}, 5: {"status": "OK", "points": 0}},
             "points": 75
         }
     }
diff --git a/tests/commands/run/util.py b/tests/commands/run/util.py
index d3e063f7..ea66f0f8 100644
--- a/tests/commands/run/util.py
+++ b/tests/commands/run/util.py
@@ -37,4 +37,5 @@ def get_command(path = None):
 def set_default_args(command):
     command.args = argparse.Namespace(
         weak_compilation_flags=False,
+        print_expected_scores=True,
     )
diff --git a/tests/contest_types/test_default.py b/tests/contest_types/test_default.py
index a67a68d3..928107d8 100644
--- a/tests/contest_types/test_default.py
+++ b/tests/contest_types/test_default.py
@@ -16,3 +16,22 @@ def test_get_group_score():
     assert contest.get_group_score([50, 100, 100], 100) == 50
     assert contest.get_group_score([0, 10, 20], 100) == 0
     assert contest.get_group_score([10, 3, 5], 50) == 2
+
+
+def test_assign_scores():
+    contest = contest_types.DefaultContest()
+    assert contest.assign_scores([0, 1, 2, 3]) == {0: 0, 1: 33, 2: 33, 3: 34}
+    assert contest.assign_scores([1, 2, 3]) == {1: 33, 2: 33, 3: 34}
+    assert contest.assign_scores([0, 1, 2]) == {0: 0, 1: 50, 2: 50}
+
+
+def test_get_possible_score():
+    contest = contest_types.DefaultContest()
+    assert contest.get_possible_score([0, 1, 2, 3], {0: 0, 1: 33, 2: 33, 3: 34}) == 100
+    assert contest.get_possible_score([1, 2, 3], {1: 33, 2: 33, 3: 34}) == 100
+    assert contest.get_possible_score([0, 2], {0: 0, 1: 50, 2: 50}) == 50
+
+
+def test_get_global_score():
+    contest = contest_types.DefaultContest()
+    assert contest.get_global_score({1: {"points": 10}, 2: {"points": 20}}, 100) == 30
diff --git a/tests/contest_types/test_icpc.py b/tests/contest_types/test_icpc.py
new file mode 100644
index 00000000..0c7a3e04
--- /dev/null
+++ b/tests/contest_types/test_icpc.py
@@ -0,0 +1,20 @@
+from sinol_make import contest_types
+
+
+def test_assign_scores():
+    contest = contest_types.ICPCContest()
+    assert contest.assign_scores([0, 1, 2, 3]) == {0: 1, 1: 1, 2: 1, 3: 1}
+    assert contest.assign_scores([1, 2, 3]) == {1: 1, 2: 1, 3: 1}
+    assert contest.assign_scores([0, 1, 2]) == {0: 1, 1: 1, 2: 1}
+
+
+def test_get_group_score():
+    contest = contest_types.ICPCContest()
+    assert contest.get_group_score([1, 1, 1, 1], 1) == 1
+    assert contest.get_group_score([1, 0, 1], 1) == 0
+
+
+def test_get_global_score():
+    contest = contest_types.ICPCContest()
+    assert contest.get_global_score({1: {"points": 1}, 2: {"points": 1}}, 1) == 1
+    assert contest.get_global_score({1: {"points": 1}, 2: {"points": 0}}, 1) == 0
diff --git a/tests/helpers/test_package_util.py b/tests/helpers/test_package_util.py
index 44ca3ce4..d821eb17 100644
--- a/tests/helpers/test_package_util.py
+++ b/tests/helpers/test_package_util.py
@@ -231,7 +231,14 @@ def test_validate_files(create_package, capsys):
 
 def test_get_executable_key():
     os.chdir(get_simple_package_path())
-    assert package_util.get_executable_key("abc1.cpp.e") == (0, 1)
+    for task_id in ["abc", "long_task_id", "", "x"]:
+        assert package_util.get_executable_key(f"{task_id}1.cpp.e", task_id) == (0, 1)
+        assert package_util.get_executable_key(f"{task_id}2.cpp.e", task_id) == (0, 2)
+        assert package_util.get_executable_key(f"{task_id}s20.cpp.e", task_id) == (1, 20)
+        assert package_util.get_executable_key(f"{task_id}s21.cpp.e", task_id) == (1, 21)
+        assert package_util.get_executable_key(f"{task_id}b100.cpp.e", task_id) == (2, 100)
+        assert package_util.get_executable_key(f"{task_id}b101.cpp.e", task_id) == (2, 101)
+        assert package_util.get_executable_key(f"{task_id}x1000.cpp.e", task_id) == (0, 0)
 
 
 def test_get_solutions():
diff --git a/tests/packages/abc/config.yml b/tests/packages/abc/config.yml
index 46c73673..e660100e 100644
--- a/tests/packages/abc/config.yml
+++ b/tests/packages/abc/config.yml
@@ -1,26 +1,44 @@
-title: Simple test package
-memory_limit: 16000
-time_limit: 1000
-
-scores:
-  1: 25
-  2: 25
-  3: 25
-  4: 25
-
-sinol_expected_scores:
-  abc.cpp:
-    points: 100
-    expected: {1: "OK", 2: "OK", 3: "OK", 4: "OK"}
-  abc1.cpp:
-    points: 75
-    expected: {1: "OK", 2: "OK", 3: "OK", 4: "WA"}
-  abc2.cpp:
-    points: 25
-    expected: {1: "OK", 2: "WA", 3: "WA", 4: "TL"}
-  abc3.cpp:
-    points: 25
-    expected: {1: "OK", 2: "WA", 3: "WA", 4: "ML"}
-  abc4.cpp:
-    points: 50
-    expected: {1: "OK", 2: "OK", 3: "WA", 4: "RE"}
+title: Simple test package
+memory_limit: 16000
+time_limit: 1000
+scores:
+  1: 25
+  2: 25
+  3: 25
+  4: 25
+sinol_expected_scores:
+  abc.cpp:
+    expected:
+      1: {points: 25, status: OK}
+      2: {points: 25, status: OK}
+      3: {points: 25, status: OK}
+      4: {points: 25, status: OK}
+    points: 100
+  abc1.cpp:
+    expected:
+      1: {points: 25, status: OK}
+      2: {points: 25, status: OK}
+      3: {points: 25, status: OK}
+      4: {points: 0, status: WA}
+    points: 75
+  abc2.cpp:
+    expected:
+      1: {points: 25, status: OK}
+      2: {points: 0, status: WA}
+      3: {points: 0, status: WA}
+      4: {points: 0, status: TL}
+    points: 25
+  abc3.cpp:
+    expected:
+      1: {points: 25, status: OK}
+      2: {points: 0, status: WA}
+      3: {points: 0, status: WA}
+      4: {points: 0, status: ML}
+    points: 25
+  abc4.cpp:
+    expected:
+      1: {points: 25, status: OK}
+      2: {points: 25, status: OK}
+      3: {points: 0, status: WA}
+      4: {points: 0, status: RE}
+    points: 50
diff --git a/tests/packages/chk/config.yml b/tests/packages/chk/config.yml
index fe3d3468..706b92f6 100644
--- a/tests/packages/chk/config.yml
+++ b/tests/packages/chk/config.yml
@@ -4,12 +4,17 @@ time_limit: 1000
 scores:
   1: 50
   2: 50
+
 sinol_expected_scores:
   chk.cpp:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
   chk1.cpp:
-    expected: {1: WA, 2: WA}
+    expected:
+      1: {points: 0, status: WA}
+      2: {points: 0, status: WA}
     points: 0
   chk2.cpp:
     expected:
@@ -17,5 +22,7 @@ sinol_expected_scores:
       2: {points: 25, status: OK}
     points: 50
   chk3.cpp:
-    expected: {1: OK, 2: WA}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 0, status: WA}
     points: 50
diff --git a/tests/packages/example_tests/config.yml b/tests/packages/example_tests/config.yml
index 1ddcd9af..1b480e60 100644
--- a/tests/packages/example_tests/config.yml
+++ b/tests/packages/example_tests/config.yml
@@ -1,8 +1,10 @@
 title: Package with example tests
+sinol_task_id: exa
 memory_limit: 10240
 time_limit: 1000
+
 sinol_expected_scores:
   exa.cpp:
-    expected: {0: OK}
+    expected:
+      0: {points: 0, status: OK}
     points: 0
-sinol_task_id: exa
diff --git a/tests/packages/hwr/config.yml b/tests/packages/hwr/config.yml
index 9b4efe1e..9399952c 100644
--- a/tests/packages/hwr/config.yml
+++ b/tests/packages/hwr/config.yml
@@ -1,13 +1,13 @@
 title: Package with handwritten tests for testing `export` command
-
 memory_limit: 32000
 time_limit: 1
-
 scores:
   1: 50
   2: 50
-
 sinol_expected_scores:
   hwr.cpp:
-    expected: {0: OK, 1: OK, 2: OK}
+    expected:
+      0: {points: 0, status: OK}
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
diff --git a/tests/packages/icpc/config.yml b/tests/packages/icpc/config.yml
new file mode 100644
index 00000000..be144ecb
--- /dev/null
+++ b/tests/packages/icpc/config.yml
@@ -0,0 +1,34 @@
+title: Package for testing ICPC contest
+sinol_task_id: acm
+sinol_contest_type: icpc
+memory_limit: 16000
+time_limit: 1000
+sinol_expected_scores:
+  acm.cpp:
+    expected:
+      1: {points: 1, status: OK}
+      2: {points: 1, status: OK}
+      3: {points: 1, status: OK}
+      4: {points: 1, status: OK}
+    points: 1
+  acm1.cpp:
+    expected:
+      1: {points: 1, status: OK}
+      2: {points: 1, status: OK}
+      3: {points: 1, status: OK}
+      4: {points: 0, status: WA}
+    points: 0
+  acm2.cpp:
+    expected:
+      1: {points: 1, status: OK}
+      2: {points: 0, status: WA}
+      3: {points: 0, status: WA}
+      4: {points: 0, status: TL}
+    points: 0
+  acm3.cpp:
+    expected:
+      1: {points: 1, status: OK}
+      2: {points: 0, status: WA}
+      3: {points: 0, status: WA}
+      4: {points: 0, status: ML}
+    points: 0
diff --git a/tests/packages/icpc/in/.gitkeep b/tests/packages/icpc/in/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/packages/icpc/out/.gitkeep b/tests/packages/icpc/out/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/packages/icpc/prog/abc4.cpp b/tests/packages/icpc/prog/abc4.cpp
new file mode 100644
index 00000000..f339f90d
--- /dev/null
+++ b/tests/packages/icpc/prog/abc4.cpp
@@ -0,0 +1,17 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+	int a, b;
+	cin >> a >> b;
+	if (a == 1 || a == 2)
+		cout << a + b;
+	else if (a == 3)
+		cout << a + b + 2;
+	else if (a == 4) {
+		int c = 0;
+		cout << a + b / c;
+		return 1;
+	}
+}
diff --git a/tests/packages/icpc/prog/acm.cpp b/tests/packages/icpc/prog/acm.cpp
new file mode 100644
index 00000000..9d30bab0
--- /dev/null
+++ b/tests/packages/icpc/prog/acm.cpp
@@ -0,0 +1,9 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+	int a, b;
+	cin >> a >> b;
+	cout << a + b;
+}
diff --git a/tests/packages/icpc/prog/acm1.cpp b/tests/packages/icpc/prog/acm1.cpp
new file mode 100644
index 00000000..c1b5782e
--- /dev/null
+++ b/tests/packages/icpc/prog/acm1.cpp
@@ -0,0 +1,12 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+	int a, b;
+	cin >> a >> b;
+	if (a == 4)
+		cout << a + b + 1;
+	else
+		cout << a + b;
+}
diff --git a/tests/packages/icpc/prog/acm2.cpp b/tests/packages/icpc/prog/acm2.cpp
new file mode 100644
index 00000000..6979f775
--- /dev/null
+++ b/tests/packages/icpc/prog/acm2.cpp
@@ -0,0 +1,20 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+	int a, b;
+	cin >> a >> b;
+	if (a == 1)
+		cout << a + b;
+	else if (a == 2 || a == 3)
+		cout << a + b - 1;
+	else if (a == 4) {
+		time_t start = time(0);
+		int i = 0;
+		while (time(0) - start < 5) {
+			i++;
+		}
+		cout << a + b;
+	}
+}
diff --git a/tests/packages/icpc/prog/acm3.cpp b/tests/packages/icpc/prog/acm3.cpp
new file mode 100644
index 00000000..129e4932
--- /dev/null
+++ b/tests/packages/icpc/prog/acm3.cpp
@@ -0,0 +1,34 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int rnd() {
+	return rand() % 100;
+}
+
+int main() {
+	int a, b;
+	cin >> a >> b;
+	if (a == 1)
+		cout << a + b;
+	else if (a == 2 || a == 3)
+		cout << a + b + 2;
+	else if (a == 4) {
+		vector<int*> v;
+		for (int i = 0; i <= 10000; i++) {
+			int *tmp = new int[1000];
+			for (int j = 0; j < 1000; j++) {
+				tmp[j] = rnd();
+			}
+			v.push_back(tmp);
+		}
+		int s = 0;
+		for (auto i : v) {
+			for (int j = 0; j < 1000; j++) {
+				s = (s + i[j]) % 1000000007;
+			}
+			delete[] i;
+		}
+		cout << a + b;
+	}
+}
diff --git a/tests/packages/icpc/prog/acmingen.cpp b/tests/packages/icpc/prog/acmingen.cpp
new file mode 100644
index 00000000..bdee7ab2
--- /dev/null
+++ b/tests/packages/icpc/prog/acmingen.cpp
@@ -0,0 +1,18 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+	ofstream f("acm1a.in");
+	f << "1 3\n";
+	f.close();
+	f.open("acm2a.in");
+	f << "2 5\n";
+	f.close();
+	f.open("acm3a.in");
+	f << "3 7\n";
+	f.close();
+	f.open("acm4a.in");
+	f << "4 9\n";
+	f.close();
+}
diff --git a/tests/packages/lib/config.yml b/tests/packages/lib/config.yml
index ce668c12..a1131fba 100644
--- a/tests/packages/lib/config.yml
+++ b/tests/packages/lib/config.yml
@@ -9,11 +9,17 @@ extra_compilation_args:
   cpp: [liblib.cpp]
 sinol_expected_scores:
   lib.cpp:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
   lib.py:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
   libs1.cpp:
-    expected: {1: WA, 2: OK}
+    expected:
+      1: {points: 0, status: WA}
+      2: {points: 50, status: OK}
     points: 50
diff --git a/tests/packages/lim/config.yml b/tests/packages/lim/config.yml
index 13000ae9..d4c2f1bc 100644
--- a/tests/packages/lim/config.yml
+++ b/tests/packages/lim/config.yml
@@ -1,27 +1,31 @@
 title: Package with `time_limits` and `memory_limits` set
-
 memory_limit: 15000
 memory_limits:
   2: 60000
-
 time_limit: 1000
 time_limits:
   1: 3000
-
 scores:
   1: 50
   2: 50
-
 sinol_expected_scores:
   lim.cpp:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
   lim2.cpp:
-    expected: {1: OK, 2: TL}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 0, status: TL}
     points: 50
   lim3.cpp:
-    expected: {1: ML, 2: OK}
+    expected:
+      1: {points: 0, status: ML}
+      2: {points: 50, status: OK}
     points: 50
   lim4.cpp:
-    expected: {1: ML, 2: OK}
+    expected:
+      1: {points: 0, status: ML}
+      2: {points: 50, status: OK}
     points: 50
diff --git a/tests/packages/lsa/config.yml b/tests/packages/lsa/config.yml
index b24f8db7..049df992 100644
--- a/tests/packages/lsa/config.yml
+++ b/tests/packages/lsa/config.yml
@@ -5,15 +5,20 @@ scores:
   1: 50
   2: 50
 extra_compilation_files: [lsalib.cpp, lsalib.h, lsalib.py]
-extra_compilation_args:
-  cpp: lsalib.cpp
+extra_compilation_args: {cpp: lsalib.cpp}
 sinol_expected_scores:
   lsa.cpp:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
   lsa.py:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
   lsas1.cpp:
-    expected: {1: WA, 2: OK}
+    expected:
+      1: {points: 0, status: WA}
+      2: {points: 50, status: OK}
     points: 50
diff --git a/tests/packages/ovl/config.yml b/tests/packages/ovl/config.yml
index 95deefb8..99ee09c9 100644
--- a/tests/packages/ovl/config.yml
+++ b/tests/packages/ovl/config.yml
@@ -1,14 +1,13 @@
 title: Package with `override_limits` set
-
 memory_limit: 8192
 time_limit: 100
-
 override_limits:
   cpp:
     memory_limit: 16384
     time_limit: 10000
-
 sinol_expected_scores:
   ovl.cpp:
-    expected: {1: OK, 2: OK}
+    expected:
+      1: {points: 50, status: OK}
+      2: {points: 50, status: OK}
     points: 100
diff --git a/tests/packages/stc/config.yml b/tests/packages/stc/config.yml
index f387bf41..fc312042 100644
--- a/tests/packages/stc/config.yml
+++ b/tests/packages/stc/config.yml
@@ -1,9 +1,8 @@
 title: Package for testing if changing stack size works
-
 memory_limit: 1000
 time_limit: 10000
-
 sinol_expected_scores:
   stc.cpp:
-    expected: {1: OK}
+    expected:
+      1: {points: 100, status: OK}
     points: 100
diff --git a/tests/packages/undocumented_options/config.yml b/tests/packages/undocumented_options/config.yml
index 95aad60b..2387c6d5 100644
--- a/tests/packages/undocumented_options/config.yml
+++ b/tests/packages/undocumented_options/config.yml
@@ -1,18 +1,17 @@
 title: Package with undocumented sinol-make options
 sinol_task_id: und
-
 sinol_undocumented_time_tool: time
 sinol_undocumented_test_limits: true
-
 memory_limit: 20480
 time_limit: 1000
 time_limits:
   1a: 5000
-
 sinol_expected_scores:
   und.cpp:
-    expected: {1: OK}
+    expected:
+      1: {points: 100, status: OK}
     points: 100
   und1.cpp:
-    expected: {1: OK}
+    expected:
+      1: {points: 100, status: OK}
     points: 100
diff --git a/tests/packages/vso/config.yml b/tests/packages/vso/config.yml
index cc2bfdd8..1f9c716a 100644
--- a/tests/packages/vso/config.yml
+++ b/tests/packages/vso/config.yml
@@ -1,32 +1,38 @@
 title: Test package for veryfing correct status order
 memory_limit: 16000
 time_limit: 1000
-
 scores:
   1: 100
-
 sinol_expected_scores:
   vso.cpp:
+    expected:
+      1: {points: 100, status: OK}
     points: 100
-    expected: {1: "OK"}
   vso1.cpp:
+    expected:
+      1: {points: 0, status: WA}
     points: 0
-    expected: {1: "WA"}
   vso2.cpp:
+    expected:
+      1: {points: 0, status: RE}
     points: 0
-    expected: {1: "RE"}
   vso3.cpp:
+    expected:
+      1: {points: 0, status: ML}
     points: 0
-    expected: {1: "ML"}
   vso4.cpp:
+    expected:
+      1: {points: 0, status: TL}
     points: 0
-    expected: {1: "TL"}
   vso5.cpp:
+    expected:
+      1: {points: 0, status: RE}
     points: 0
-    expected: {1: "RE"}
   vso6.cpp:
+    expected:
+      1: {points: 0, status: ML}
     points: 0
-    expected: {1: "ML"}
   vso7.cpp:
+    expected:
+      1: {points: 0, status: TL}
     points: 0
-    expected: {1: "TL"}
diff --git a/tests/packages/wcf/config.yml b/tests/packages/wcf/config.yml
index 05be7e6b..39945cf5 100644
--- a/tests/packages/wcf/config.yml
+++ b/tests/packages/wcf/config.yml
@@ -1,11 +1,10 @@
 title: Package for testing --weak-compilation-flags
 memory_limit: 16000
 time_limit: 1000
-
 scores:
   1: 100
-
 sinol_expected_scores:
   wcf.cpp:
-    expected: {1: OK}
+    expected:
+      1: {points: 100, status: OK}
     points: 100
diff --git a/tests/util.py b/tests/util.py
index 83c86bd6..3034c9dd 100644
--- a/tests/util.py
+++ b/tests/util.py
@@ -115,6 +115,13 @@ def get_example_tests_package_path():
     return os.path.join(os.path.dirname(__file__), "packages", "example_tests")
 
 
+def get_icpc_package_path():
+    """
+    Get path to package with icpc contest type (/tests/packages/icpc)
+    """
+    return os.path.join(os.path.dirname(__file__), "packages", "icpc")
+
+
 def create_ins(package_path, task_id):
     """
     Create .in files for package.
diff --git a/tests/version_changes/__init__.py b/tests/version_changes/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/version_changes/test_expected_scores_format_change.py b/tests/version_changes/test_expected_scores_format_change.py
new file mode 100644
index 00000000..9a721807
--- /dev/null
+++ b/tests/version_changes/test_expected_scores_format_change.py
@@ -0,0 +1,31 @@
+import pytest
+import yaml
+
+from tests import util
+from tests.fixtures import create_package
+from sinol_make.util import make_version_changes
+import sinol_make
+
+
+@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True)
+def test_version_change(create_package):
+    orig_version = sinol_make.__version__
+    sinol_make.__version__ = "1.5.9"
+
+    with open("config.yml", "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.FullLoader)
+    old_expected_scores = config["sinol_expected_scores"]
+    config["sinol_expected_scores"] = {'abc.cpp': {'points': 100, 'expected': {1: 'OK', 2: 'OK', 3: 'OK', 4: 'OK'}},
+                                       'abc1.cpp': {'points': 75, 'expected': {1: 'OK', 2: 'OK', 3: 'OK', 4: 'WA'}},
+                                       'abc2.cpp': {'points': 25, 'expected': {1: 'OK', 2: 'WA', 3: 'WA', 4: 'TL'}},
+                                       'abc3.cpp': {'points': 25, 'expected': {1: 'OK', 2: 'WA', 3: 'WA', 4: 'ML'}},
+                                       'abc4.cpp': {'points': 50, 'expected': {1: 'OK', 2: 'OK', 3: 'WA', 4: 'RE'}}}
+    with open("config.yml", "w") as config_file:
+        yaml.dump(config, config_file)
+
+    make_version_changes()
+
+    with open("config.yml", "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.FullLoader)
+    assert config["sinol_expected_scores"] == old_expected_scores
+    sinol_make.__version__ = orig_version