Merge pull request #229 from sio2project/ingen-inwer-cache-fixes

Remove cache on compilation flags or sanitizers change

src/sinol_make/helpers/cache.py

@@ -35,7 +35,7 @@ def get_cache_file(solution_path: str) -> CacheFile:
         return CacheFile()
 
 
-def check_compiled(file_path: str) -> Union[str, None]:
+def check_compiled(file_path: str, compilation_flags: str, sanitizers: bool) -> Union[str, None]:
     """
     Check if a file is compiled
     :param file_path: Path to the file
@@ -44,7 +44,7 @@ def check_compiled(file_path: str) -> Union[str, None]:
     md5sum = util.get_file_md5(file_path)
     try:
         info = get_cache_file(file_path)
-        if info.md5sum == md5sum:
+        if info.md5sum == md5sum and info.compilation_flags == compilation_flags and info.sanitizers == sanitizers:
             exe_path = info.executable_path
             if os.path.exists(exe_path):
                 return exe_path
@@ -53,19 +53,18 @@ def check_compiled(file_path: str) -> Union[str, None]:
         return None
 
 
-def save_compiled(file_path: str, exe_path: str, is_checker: bool = False):
+def save_compiled(file_path: str, exe_path: str, compilation_flags: str, sanitizers: bool, is_checker: bool = False):
     """
     Save the compiled executable path to cache in `.cache/md5sums/<basename of file_path>`,
     which contains the md5sum of the file and the path to the executable.
     :param file_path: Path to the file
     :param exe_path: Path to the compiled executable
+    :param compilation_flags: Compilation flags used
+    :param sanitizers: Whether -fsanitize=undefined,address was used
     :param is_checker: Whether the compiled file is a checker. If True, all cached tests are removed.
     """
-    info = CacheFile()
-    info.executable_path = exe_path
-    info.md5sum = util.get_file_md5(file_path)
+    info = CacheFile(util.get_file_md5(file_path), exe_path, compilation_flags, sanitizers)
     info.save(file_path)
-
     if is_checker:
         remove_results_cache()
 

src/sinol_make/helpers/compile.py

@@ -37,10 +37,17 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp
     if use_fsanitize and util.is_macos_arm():
         use_fsanitize = False
 
+    if compilation_flags == 'w':
+        compilation_flags = 'weak'
+    elif compilation_flags == 'o':
+        compilation_flags = 'oioioi'
+    elif compilation_flags == 'd':
+        compilation_flags = 'default'
+
     if extra_compilation_files is None:
         extra_compilation_files = []
 
-    compiled_exe = check_compiled(program)
+    compiled_exe = check_compiled(program, compilation_flags, use_fsanitize)
     if compiled_exe is not None:
         if compile_log is not None:
             compile_log.write(f'Using cached executable {compiled_exe}\n')
@@ -53,12 +60,11 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp
         shutil.copy(file, os.path.join(os.path.dirname(output), os.path.basename(file)))
 
     gcc_compilation_flags = ''
-    if compilation_flags == 'weak' or compilation_flags == 'w':
-        compilation_flags = 'weak'
+    if compilation_flags == 'weak':
         gcc_compilation_flags = ''  # Disable all warnings
-    elif compilation_flags == 'oioioi' or compilation_flags == 'o':
+    elif compilation_flags == 'oioioi':
         gcc_compilation_flags = ' -Wall -Wno-unused-result -Werror'  # Same flags as oioioi
-    elif compilation_flags == 'default' or compilation_flags == 'd':
+    elif compilation_flags == 'default':
         gcc_compilation_flags = ' -Werror -Wall -Wextra -Wshadow -Wconversion -Wno-unused-result -Wfloat-equal'
     else:
         util.exit_with_error(f'Unknown compilation flags group: {compilation_flags}')
@@ -108,7 +114,7 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp
     if process.returncode != 0:
         raise CompilationError('Compilation failed')
     else:
-        save_compiled(program, output, is_checker)
+        save_compiled(program, output, compilation_flags, use_fsanitize, is_checker)
         return True
 
 

src/sinol_make/structs/cache_structs.py

@@ -43,20 +43,28 @@ class CacheFile:
     md5sum: str
     # Path to the executable
     executable_path: str
+    # Compilation flags used
+    compilation_flags: str
+    # Whether -fsanitize=undefined,address was used
+    sanitizers: bool
     # Test results
     tests: Dict[str, CacheTest]
 
-    def __init__(self, md5sum="", executable_path="", tests=None):
+    def __init__(self, md5sum="", executable_path="", compilation_flags="default", sanitizers=False, tests=None):
         if tests is None:
             tests = {}
         self.md5sum = md5sum
         self.executable_path = executable_path
+        self.compilation_flags = compilation_flags
+        self.sanitizers = sanitizers
         self.tests = tests
 
     def to_dict(self) -> Dict:
         return {
             "md5sum": self.md5sum,
             "executable_path": self.executable_path,
+            "compilation_flags": self.compilation_flags,
+            "sanitizers": self.sanitizers,
             "tests": {k: v.to_dict() for k, v in self.tests.items()}
         }
 
@@ -65,6 +73,8 @@ def from_dict(dict) -> 'CacheFile':
         return CacheFile(
             md5sum=dict.get("md5sum", ""),
             executable_path=dict.get("executable_path", ""),
+            compilation_flags=dict.get("compilation_flags", "default"),
+            sanitizers=dict.get("sanitizers", False),
             tests={k: CacheTest(
                 time_limit=v["time_limit"],
                 memory_limit=v["memory_limit"],

tests/commands/gen/test_integration.py

@@ -60,7 +60,7 @@ def test_correct_inputs(capsys, create_package):
     """
     task_id = package_util.get_task_id()
     correct_solution = package_util.get_correct_solution(task_id)
-    cache.save_compiled(correct_solution, "exe")
+    cache.save_compiled(correct_solution, "exe", "default", False)
     simple_run()
     md5_sums = get_md5_sums(create_package)
 
@@ -79,7 +79,7 @@ def test_changed_inputs(capsys, create_package):
     """
     task_id = package_util.get_task_id()
     correct_solution = package_util.get_correct_solution(task_id)
-    cache.save_compiled(correct_solution, "exe")
+    cache.save_compiled(correct_solution, "exe", "default", False)
     simple_run()
     md5_sums = get_md5_sums(create_package)
     correct_md5 = md5_sums.copy()
@@ -337,3 +337,21 @@ def test_outgen_cache_cleaning(create_package, capsys):
     simple_run(command="outgen")
     # Run should pass, because output file was regenerated and cache for this test was cleaned.
     RunCommand().run(args)
+
+
+@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True)
+def test_cache_remove_after_flags_change(create_package):
+    """
+    Test if cache for a program is removed if compilation flags change or -fsanitize is disabled.
+    """
+    # Generate cache
+    simple_run(command="gen")
+    cache_file = cache.get_cache_file("abcingen.cpp")
+    cache_dict = cache_file.to_dict()
+    cache_dict["random_key"] = "random_value"
+    with open(paths.get_cache_path("md5sums", "abcingen.cpp"), "w") as f:
+        yaml.dump(cache_dict, f)
+    simple_run(["--compile-mode", "oioioi"], command="gen")
+    with open(paths.get_cache_path("md5sums", "abcingen.cpp"), "r") as f:
+        cache_dict = yaml.load(f, Loader=yaml.FullLoader)
+    assert "random_key" not in cache_dict

tests/helpers/test_cache.py

@@ -13,25 +13,33 @@ def test_compilation_caching():
         program = os.path.join(tmpdir, 'program.cpp')
         open(program, 'w').write('int main() { return 0; }')
 
-        assert cache.check_compiled(program) is None
+        assert cache.check_compiled(program, "default", False) is None
 
         assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None)
-        exe_path = cache.check_compiled(program)
+        exe_path = cache.check_compiled(program, "default", False)
         assert exe_path is not None
 
         assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None)
-        exe_path2 = cache.check_compiled(program)
+        exe_path2 = cache.check_compiled(program, "default", False)
         assert exe_path2 == exe_path
 
         open(program, 'w').write('int main() { return 1; }')
-        assert cache.check_compiled(program) is None
+        assert cache.check_compiled(program, "default", False) is None
         assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None)
-        assert cache.check_compiled(program) is not None
+        assert cache.check_compiled(program, "default", False) is not None
 
         open(program, 'w').write('int main() { return 0; }')
-        assert cache.check_compiled(program) is None
+        assert cache.check_compiled(program, "default", False) is None
         assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None)
-        assert cache.check_compiled(program) is not None
+        assert cache.check_compiled(program, "default", False) is not None
+
+        assert cache.check_compiled(program, "default", True) is None
+        cache.save_compiled(program, exe_path, "default", True)
+        assert cache.check_compiled(program, "default", True) is not None
+
+        assert cache.check_compiled(program, "oioioi", True) is None
+        cache.save_compiled(program, exe_path, "oioioi", True)
+        assert cache.check_compiled(program, "oioioi", True) is not None
 
 
 def test_cache():
@@ -72,7 +80,8 @@ def test_cache():
             f.write("int main() { return 0; }")
         cache_file.save("abc.cpp")
         assert cache.get_cache_file("abc.cpp") == cache_file
-        cache.save_compiled("abc.cpp", "abc.e", is_checker=True)
+        cache.save_compiled("abc.cpp", "abc.e", "default", False,
+                            is_checker=True)
         assert cache.get_cache_file("abc.cpp").tests == {}
 
         # Test that cache is cleared when extra compilation files change

tests/helpers/test_compile.py

@@ -16,10 +16,11 @@ def test_compilation_caching():
         with open(os.path.join(os.getcwd(), "test.e"), "w") as f:
             f.write("")
 
-        assert check_compiled(os.path.join(os.getcwd(), "test.txt")) is None
+        assert check_compiled(os.path.join(os.getcwd(), "test.txt"), "default", False) is None
         save_compiled(os.path.join(os.getcwd(), "test.txt"),
-                      os.path.join(os.getcwd(), "test.e"))
-        assert check_compiled(os.path.join(os.getcwd(), "test.txt")) == os.path.join(os.getcwd(), "test.e")
+                      os.path.join(os.getcwd(), "test.e"), "default", False)
+        assert check_compiled(os.path.join(os.getcwd(), "test.txt"), "default", False) == \
+               os.path.join(os.getcwd(), "test.e")
 
 
 @pytest.mark.parametrize("create_package", [util.get_shell_ingen_pack_path()], indirect=True)