From 9f67ec1f7a0aef6336e5eb52399f304710e69a1d Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Thu, 4 Apr 2024 18:21:15 +0200 Subject: [PATCH 1/4] Remove cache on compilation flags or sanitizers change --- src/sinol_make/helpers/cache.py | 13 ++++++------- src/sinol_make/helpers/compile.py | 18 ++++++++++++------ src/sinol_make/structs/cache_structs.py | 12 +++++++++++- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/sinol_make/helpers/cache.py b/src/sinol_make/helpers/cache.py index 9f405c4f..cca819fd 100644 --- a/src/sinol_make/helpers/cache.py +++ b/src/sinol_make/helpers/cache.py @@ -35,7 +35,7 @@ def get_cache_file(solution_path: str) -> CacheFile: return CacheFile() -def check_compiled(file_path: str) -> Union[str, None]: +def check_compiled(file_path: str, compilation_flags: str, sanitizers: bool) -> Union[str, None]: """ Check if a file is compiled :param file_path: Path to the file @@ -44,7 +44,7 @@ def check_compiled(file_path: str) -> Union[str, None]: md5sum = util.get_file_md5(file_path) try: info = get_cache_file(file_path) - if info.md5sum == md5sum: + if info.md5sum == md5sum and info.compilation_flags == compilation_flags and info.sanitizers == sanitizers: exe_path = info.executable_path if os.path.exists(exe_path): return exe_path @@ -53,19 +53,18 @@ def check_compiled(file_path: str) -> Union[str, None]: return None -def save_compiled(file_path: str, exe_path: str, is_checker: bool = False): +def save_compiled(file_path: str, exe_path: str, compilation_flags: str, sanitizers: bool, is_checker: bool = False): """ Save the compiled executable path to cache in `.cache/md5sums/`, which contains the md5sum of the file and the path to the executable. :param file_path: Path to the file :param exe_path: Path to the compiled executable + :param compilation_flags: Compilation flags used + :param sanitizers: Whether -fsanitize=undefined,address was used :param is_checker: Whether the compiled file is a checker. If True, all cached tests are removed. """ - info = CacheFile() - info.executable_path = exe_path - info.md5sum = util.get_file_md5(file_path) + info = CacheFile(util.get_file_md5(file_path), exe_path, compilation_flags, sanitizers) info.save(file_path) - if is_checker: remove_results_cache() diff --git a/src/sinol_make/helpers/compile.py b/src/sinol_make/helpers/compile.py index 07500090..d3dc8958 100644 --- a/src/sinol_make/helpers/compile.py +++ b/src/sinol_make/helpers/compile.py @@ -37,10 +37,17 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp if use_fsanitize and util.is_macos_arm(): use_fsanitize = False + if compilation_flags == 'w': + compilation_flags = 'weak' + elif compilation_flags == 'o': + compilation_flags = 'oioioi' + elif compilation_flags == 'd': + compilation_flags = 'default' + if extra_compilation_files is None: extra_compilation_files = [] - compiled_exe = check_compiled(program) + compiled_exe = check_compiled(program, compilation_flags, use_fsanitize) if compiled_exe is not None: if compile_log is not None: compile_log.write(f'Using cached executable {compiled_exe}\n') @@ -53,12 +60,11 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp shutil.copy(file, os.path.join(os.path.dirname(output), os.path.basename(file))) gcc_compilation_flags = '' - if compilation_flags == 'weak' or compilation_flags == 'w': - compilation_flags = 'weak' + if compilation_flags == 'weak': gcc_compilation_flags = '' # Disable all warnings - elif compilation_flags == 'oioioi' or compilation_flags == 'o': + elif compilation_flags == 'oioioi': gcc_compilation_flags = ' -Wall -Wno-unused-result -Werror' # Same flags as oioioi - elif compilation_flags == 'default' or compilation_flags == 'd': + elif compilation_flags == 'default': gcc_compilation_flags = ' -Werror -Wall -Wextra -Wshadow -Wconversion -Wno-unused-result -Wfloat-equal' else: util.exit_with_error(f'Unknown compilation flags group: {compilation_flags}') @@ -108,7 +114,7 @@ def compile(program, output, compilers: Compilers = None, compile_log=None, comp if process.returncode != 0: raise CompilationError('Compilation failed') else: - save_compiled(program, output, is_checker) + save_compiled(program, output, compilation_flags, use_fsanitize, is_checker) return True diff --git a/src/sinol_make/structs/cache_structs.py b/src/sinol_make/structs/cache_structs.py index 5d1f3933..249bbdc8 100644 --- a/src/sinol_make/structs/cache_structs.py +++ b/src/sinol_make/structs/cache_structs.py @@ -43,20 +43,28 @@ class CacheFile: md5sum: str # Path to the executable executable_path: str + # Compilation flags used + compilation_flags: str + # Whether -fsanitize=undefined,address was used + sanitizers: bool # Test results tests: Dict[str, CacheTest] - def __init__(self, md5sum="", executable_path="", tests=None): + def __init__(self, md5sum="", executable_path="", compilation_flags="default", sanitizers=False, tests=None): if tests is None: tests = {} self.md5sum = md5sum self.executable_path = executable_path + self.compilation_flags = compilation_flags + self.sanitizers = sanitizers self.tests = tests def to_dict(self) -> Dict: return { "md5sum": self.md5sum, "executable_path": self.executable_path, + "compilation_flags": self.compilation_flags, + "sanitizers": self.sanitizers, "tests": {k: v.to_dict() for k, v in self.tests.items()} } @@ -65,6 +73,8 @@ def from_dict(dict) -> 'CacheFile': return CacheFile( md5sum=dict.get("md5sum", ""), executable_path=dict.get("executable_path", ""), + compilation_flags=dict.get("compilation_flags", "default"), + sanitizers=dict.get("sanitizers", False), tests={k: CacheTest( time_limit=v["time_limit"], memory_limit=v["memory_limit"], From e426fc8be6ae3a84f2e7e1297808c41e34b76950 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Thu, 4 Apr 2024 18:21:20 +0200 Subject: [PATCH 2/4] Add tests --- tests/commands/gen/test_integration.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/commands/gen/test_integration.py b/tests/commands/gen/test_integration.py index da8c5fa7..7bbfa7fa 100644 --- a/tests/commands/gen/test_integration.py +++ b/tests/commands/gen/test_integration.py @@ -337,3 +337,21 @@ def test_outgen_cache_cleaning(create_package, capsys): simple_run(command="outgen") # Run should pass, because output file was regenerated and cache for this test was cleaned. RunCommand().run(args) + + +@pytest.mark.parametrize("create_package", [util.get_simple_package_path()], indirect=True) +def test_cache_remove_after_flags_change(create_package): + """ + Test if cache for a program is removed if compilation flags change or -fsanitize is disabled. + """ + # Generate cache + simple_run(command="gen") + cache_file = cache.get_cache_file("abcingen.cpp") + cache_dict = cache_file.to_dict() + cache_dict["random_key"] = "random_value" + with open(paths.get_cache_path("md5sums", "abcingen.cpp"), "w") as f: + yaml.dump(cache_dict, f) + simple_run(["--compile-mode", "oioioi"], command="gen") + with open(paths.get_cache_path("md5sums", "abcingen.cpp"), "r") as f: + cache_dict = yaml.load(f, Loader=yaml.FullLoader) + assert "random_key" not in cache_dict From 829c8dd6cea22e7feca927d9478966133dd33d2b Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Thu, 4 Apr 2024 18:56:13 +0200 Subject: [PATCH 3/4] Fix tests --- tests/helpers/test_cache.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/helpers/test_cache.py b/tests/helpers/test_cache.py index be2423c7..52c37c77 100644 --- a/tests/helpers/test_cache.py +++ b/tests/helpers/test_cache.py @@ -13,25 +13,33 @@ def test_compilation_caching(): program = os.path.join(tmpdir, 'program.cpp') open(program, 'w').write('int main() { return 0; }') - assert cache.check_compiled(program) is None + assert cache.check_compiled(program, "default", False) is None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - exe_path = cache.check_compiled(program) + exe_path = cache.check_compiled(program, "default", False) assert exe_path is not None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - exe_path2 = cache.check_compiled(program) + exe_path2 = cache.check_compiled(program, "default", False) assert exe_path2 == exe_path open(program, 'w').write('int main() { return 1; }') - assert cache.check_compiled(program) is None + assert cache.check_compiled(program, "default", False) is None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - assert cache.check_compiled(program) is not None + assert cache.check_compiled(program, "default", False) is not None open(program, 'w').write('int main() { return 0; }') - assert cache.check_compiled(program) is None + assert cache.check_compiled(program, "default", False) is None assert compile.compile(program, os.path.join(tmpdir, 'program'), compile_log=None) - assert cache.check_compiled(program) is not None + assert cache.check_compiled(program, "default", False) is not None + + assert cache.check_compiled(program, "default", True) is None + cache.save_compiled(program, exe_path, "default", True) + assert cache.check_compiled(program, "default", True) is not None + + assert cache.check_compiled(program, "oioioi", True) is None + cache.save_compiled(program, exe_path, "oioioi", True) + assert cache.check_compiled(program, "oioioi", True) is not None def test_cache(): @@ -72,7 +80,8 @@ def test_cache(): f.write("int main() { return 0; }") cache_file.save("abc.cpp") assert cache.get_cache_file("abc.cpp") == cache_file - cache.save_compiled("abc.cpp", "abc.e", is_checker=True) + cache.save_compiled("abc.cpp", "abc.e", "default", False, + is_checker=True) assert cache.get_cache_file("abc.cpp").tests == {} # Test that cache is cleared when extra compilation files change From 2268f5cd1e77da468bdfd4cacb845eec91c1534f Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Thu, 4 Apr 2024 19:26:36 +0200 Subject: [PATCH 4/4] Fix --- tests/commands/gen/test_integration.py | 4 ++-- tests/helpers/test_compile.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/commands/gen/test_integration.py b/tests/commands/gen/test_integration.py index 7bbfa7fa..001c9887 100644 --- a/tests/commands/gen/test_integration.py +++ b/tests/commands/gen/test_integration.py @@ -60,7 +60,7 @@ def test_correct_inputs(capsys, create_package): """ task_id = package_util.get_task_id() correct_solution = package_util.get_correct_solution(task_id) - cache.save_compiled(correct_solution, "exe") + cache.save_compiled(correct_solution, "exe", "default", False) simple_run() md5_sums = get_md5_sums(create_package) @@ -79,7 +79,7 @@ def test_changed_inputs(capsys, create_package): """ task_id = package_util.get_task_id() correct_solution = package_util.get_correct_solution(task_id) - cache.save_compiled(correct_solution, "exe") + cache.save_compiled(correct_solution, "exe", "default", False) simple_run() md5_sums = get_md5_sums(create_package) correct_md5 = md5_sums.copy() diff --git a/tests/helpers/test_compile.py b/tests/helpers/test_compile.py index 4447d93c..06703496 100644 --- a/tests/helpers/test_compile.py +++ b/tests/helpers/test_compile.py @@ -16,10 +16,11 @@ def test_compilation_caching(): with open(os.path.join(os.getcwd(), "test.e"), "w") as f: f.write("") - assert check_compiled(os.path.join(os.getcwd(), "test.txt")) is None + assert check_compiled(os.path.join(os.getcwd(), "test.txt"), "default", False) is None save_compiled(os.path.join(os.getcwd(), "test.txt"), - os.path.join(os.getcwd(), "test.e")) - assert check_compiled(os.path.join(os.getcwd(), "test.txt")) == os.path.join(os.getcwd(), "test.e") + os.path.join(os.getcwd(), "test.e"), "default", False) + assert check_compiled(os.path.join(os.getcwd(), "test.txt"), "default", False) == \ + os.path.join(os.getcwd(), "test.e") @pytest.mark.parametrize("create_package", [util.get_shell_ingen_pack_path()], indirect=True)