From 3757dbd4828d715be9bc85819879c9ca9beb0a33 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Sat, 7 Sep 2024 13:45:01 +0200 Subject: [PATCH] Create cache directories on start (#257) * Create all required cache paths on start * Fix tests * Fix conftest --- src/sinol_make/__init__.py | 2 ++ src/sinol_make/commands/chkwer/__init__.py | 1 - src/sinol_make/commands/doc/__init__.py | 1 - src/sinol_make/commands/run/__init__.py | 5 ----- src/sinol_make/commands/verify/__init__.py | 5 +++-- src/sinol_make/helpers/cache.py | 18 ++++++++++++++++-- src/sinol_make/helpers/compile.py | 3 --- src/sinol_make/helpers/package_util.py | 1 - src/sinol_make/structs/cache_structs.py | 1 - src/sinol_make/util.py | 1 - tests/commands/export/test_integration.py | 3 ++- tests/commands/gen/test_integration.py | 1 + tests/commands/verify/test_integration.py | 5 ++++- tests/conftest.py | 7 ++++++- tests/helpers/test_cache.py | 2 ++ tests/helpers/test_compile.py | 3 ++- 16 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/sinol_make/__init__.py b/src/sinol_make/__init__.py index 04de3694..c05b349f 100644 --- a/src/sinol_make/__init__.py +++ b/src/sinol_make/__init__.py @@ -5,6 +5,7 @@ import argcomplete from sinol_make import util, sio2jail +from sinol_make.helpers import cache # Required for side effects from sinol_make.task_type.normal import NormalTaskType # noqa @@ -82,6 +83,7 @@ def main_exn(): if command: if len(arguments) > 1: print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '=')) + cache.create_cache_dirs() command.run(args) else: parser.print_help() diff --git a/src/sinol_make/commands/chkwer/__init__.py b/src/sinol_make/commands/chkwer/__init__.py index 2505acf4..55d3e1ca 100644 --- a/src/sinol_make/commands/chkwer/__init__.py +++ b/src/sinol_make/commands/chkwer/__init__.py @@ -64,7 +64,6 @@ def run_test(self, execution: ChkwerExecution) -> RunResult: return RunResult(execution.in_test_path, ok, int(points), comment) def run_and_print_table(self) -> Dict[str, TestResult]: - os.makedirs(paths.get_chkwer_path(), exist_ok=True) results = {} sorted_tests = sorted(self.tests, key=lambda test: package_util.get_group(test, self.task_id)) executions: List[ChkwerExecution] = [] diff --git a/src/sinol_make/commands/doc/__init__.py b/src/sinol_make/commands/doc/__init__.py index 72deb696..74d0a937 100644 --- a/src/sinol_make/commands/doc/__init__.py +++ b/src/sinol_make/commands/doc/__init__.py @@ -62,7 +62,6 @@ def make_file(self, file_path): def move_logs(self): output_dir = paths.get_cache_path('doc_logs') - os.makedirs(output_dir, exist_ok=True) for pattern in self.LOG_PATTERNS: for file in glob.glob(os.path.join(os.getcwd(), 'doc', pattern)): os.rename(file, os.path.join(output_dir, os.path.basename(file))) diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py index c252192e..f4134026 100644 --- a/src/sinol_make/commands/run/__init__.py +++ b/src/sinol_make/commands/run/__init__.py @@ -330,8 +330,6 @@ def get_groups(self, tests): return sorted(list(set([self.get_group(test) for test in tests]))) def compile_solutions(self, solutions): - os.makedirs(paths.get_compilation_log_path(), exist_ok=True) - os.makedirs(paths.get_executables_path(), exist_ok=True) print("Compiling %d solutions..." % len(solutions)) args = [(solution, None, True, False, None) for solution in solutions] with mp.Pool(self.cpus) as pool: @@ -492,7 +490,6 @@ def compile_and_run(self, solutions): for i in range(len(solutions)): if not compilation_results[i]: self.failed_compilations.append(solutions[i]) - os.makedirs(paths.get_executions_path(), exist_ok=True) executables = [paths.get_executables_path(package_util.get_executable(solution)) for solution in solutions] compiled_commands = zip(solutions, executables, compilation_results) names = solutions @@ -932,8 +929,6 @@ def set_task_type(self, timetool_name, timetool_path): def compile_additional_files(self): additional_files = self.task_type.additional_files_to_compile() - os.makedirs(paths.get_compilation_log_path(), exist_ok=True) - os.makedirs(paths.get_executables_path(), exist_ok=True) for file, dest, name, clear_cache, fail_on_error in additional_files: print(f"Compiling {name}...") success = self.compile(file, dest, False, clear_cache, name) diff --git a/src/sinol_make/commands/verify/__init__.py b/src/sinol_make/commands/verify/__init__.py index 613a7907..c65484d2 100644 --- a/src/sinol_make/commands/verify/__init__.py +++ b/src/sinol_make/commands/verify/__init__.py @@ -5,7 +5,7 @@ import subprocess from sinol_make import util, contest_types -from sinol_make.helpers import parsers, package_util, paths +from sinol_make.helpers import parsers, package_util, paths, cache from sinol_make.interfaces.BaseCommand import BaseCommand from sinol_make.commands.gen import Command as GenCommand from sinol_make.commands.doc import Command as DocCommand @@ -58,11 +58,12 @@ def correct_contest_type(self): def remove_cache(self): """ - Remove whole cache dir + Remove whole cache dir, but keep the directories. """ cache_dir = paths.get_cache_path() if os.path.exists(cache_dir): shutil.rmtree(cache_dir) + cache.create_cache_dirs() def check_extra_files(self): """ diff --git a/src/sinol_make/helpers/cache.py b/src/sinol_make/helpers/cache.py index 0185ba5c..5c3923eb 100644 --- a/src/sinol_make/helpers/cache.py +++ b/src/sinol_make/helpers/cache.py @@ -13,7 +13,6 @@ def get_cache_file(solution_path: str) -> CacheFile: :param solution_path: Path to solution :return: Content of cache file """ - os.makedirs(paths.get_cache_path("md5sums"), exist_ok=True) cache_file_path = paths.get_cache_path("md5sums", os.path.basename(solution_path)) try: with open(cache_file_path, 'r') as cache_file: @@ -143,7 +142,6 @@ def check_can_access_cache(): Checks if user can access cache. """ try: - os.makedirs(paths.get_cache_path(), exist_ok=True) with open(paths.get_cache_path("test"), "w") as f: f.write("test") os.unlink(paths.get_cache_path("test")) @@ -177,3 +175,19 @@ def check_correct_solution(task_id: str): if has_file_changed(solution) and os.path.exists(os.path.join(os.getcwd(), 'in', '.md5sums')): os.unlink(os.path.join(os.getcwd(), 'in', '.md5sums')) + + +def create_cache_dirs(): + """ + Creates all required cache directories. + """ + for dir in [ + paths.get_cache_path(), + paths.get_executables_path(), + paths.get_compilation_log_path(), + paths.get_executions_path(), + paths.get_chkwer_path(), + paths.get_cache_path('md5sums'), + paths.get_cache_path('doc_logs'), + ]: + os.makedirs(dir, exist_ok=True) diff --git a/src/sinol_make/helpers/compile.py b/src/sinol_make/helpers/compile.py index 2e28630e..30f445ba 100644 --- a/src/sinol_make/helpers/compile.py +++ b/src/sinol_make/helpers/compile.py @@ -132,9 +132,6 @@ def compile_file(file_path: str, name: str, compilers: Compilers, compilation_fl :param use_extras: Whether to use extra compilation files and arguments from config :return: Tuple of (executable path or None if compilation failed, log path) """ - os.makedirs(paths.get_executables_path(), exist_ok=True) - os.makedirs(paths.get_compilation_log_path(), exist_ok=True) - config = package_util.get_config() extra_compilation_args = [] diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index 96082899..f9d46d35 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -353,7 +353,6 @@ def save_contest_type_to_cache(contest_type): Saves contest type to cache. :param contest_type: Contest type. """ - os.makedirs(paths.get_cache_path(), exist_ok=True) with open(paths.get_cache_path("contest_type"), "w") as contest_type_file: contest_type_file.write(contest_type) diff --git a/src/sinol_make/structs/cache_structs.py b/src/sinol_make/structs/cache_structs.py index 249bbdc8..41efa19b 100644 --- a/src/sinol_make/structs/cache_structs.py +++ b/src/sinol_make/structs/cache_structs.py @@ -84,6 +84,5 @@ def from_dict(dict) -> 'CacheFile': ) def save(self, solution_path: str): - os.makedirs(paths.get_cache_path("md5sums"), exist_ok=True) with open(paths.get_cache_path("md5sums", os.path.basename(solution_path)), 'w') as cache_file: yaml.dump(self.to_dict(), cache_file) diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py index eab82fd5..9d87ffe4 100644 --- a/src/sinol_make/util.py +++ b/src/sinol_make/util.py @@ -220,7 +220,6 @@ def check_version(): except PermissionError: if find_and_chdir_package(): try: - os.makedirs(paths.get_cache_path(), exist_ok=True) with open(paths.get_cache_path("sinol_make_version"), "w") as f: f.write(latest_version) except PermissionError: diff --git a/tests/commands/export/test_integration.py b/tests/commands/export/test_integration.py index 6f71f012..8f86dd7d 100644 --- a/tests/commands/export/test_integration.py +++ b/tests/commands/export/test_integration.py @@ -10,7 +10,7 @@ from sinol_make import configure_parsers from sinol_make import util as sinol_util from sinol_make.commands.doc import Command as DocCommand -from sinol_make.helpers import paths +from sinol_make.helpers import paths, cache from tests import util from tests.fixtures import create_package from .util import * @@ -54,6 +54,7 @@ def _set_contest_type(contest_type): os.remove(f'{task_id}.tgz') if os.path.exists(paths.get_cache_path()): shutil.rmtree(paths.get_cache_path()) + cache.create_cache_dirs() @pytest.mark.parametrize("create_package", [util.get_simple_package_path(), util.get_library_package_path(), diff --git a/tests/commands/gen/test_integration.py b/tests/commands/gen/test_integration.py index a082ab34..bc47eb1b 100644 --- a/tests/commands/gen/test_integration.py +++ b/tests/commands/gen/test_integration.py @@ -208,6 +208,7 @@ def test_missing_output_files(create_package): assert os.path.exists(outs[0]) shutil.rmtree(paths.get_cache_path()) os.unlink(os.path.join(package_path, "in", ".md5sums")) + cache.create_cache_dirs() @pytest.mark.parametrize("create_package", [util.get_shell_ingen_pack_path(), util.get_simple_package_path()], diff --git a/tests/commands/verify/test_integration.py b/tests/commands/verify/test_integration.py index 8d77ebf7..d6b39d9a 100644 --- a/tests/commands/verify/test_integration.py +++ b/tests/commands/verify/test_integration.py @@ -1,11 +1,13 @@ import os import shutil +from linecache import cache + import pytest from sinol_make import configure_parsers from sinol_make import util as sm_util from sinol_make.commands.verify import Command -from sinol_make.helpers import package_util, paths, func_cache +from sinol_make.helpers import package_util, paths, func_cache, cache from tests import util from tests.fixtures import create_package @@ -112,6 +114,7 @@ def test_scores_not_100(capsys, create_package): for contest_type in ["oi", "oij"]: if os.path.exists(paths.get_cache_path()): shutil.rmtree(paths.get_cache_path()) + cache.create_cache_dirs() config = package_util.get_config() config["sinol_contest_type"] = contest_type sm_util.save_config(config) diff --git a/tests/conftest.py b/tests/conftest.py index 06c3fd16..4998aaf3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ import multiprocessing as mp from sinol_make import util -from sinol_make.helpers import compile, paths +from sinol_make.helpers import compile, paths, cache from sinol_make.interfaces.Errors import CompilationError @@ -48,6 +48,11 @@ def pytest_configure(config): files_to_compile = [] for package in packages: + cwd = os.getcwd() + os.chdir(package) + cache.create_cache_dirs() + os.chdir(cwd) + if os.path.exists(os.path.join(package, "no-precompile")): print(f'Skipping precompilation for {package} due to no-precompile file') continue diff --git a/tests/helpers/test_cache.py b/tests/helpers/test_cache.py index 273ab1bd..b0698821 100644 --- a/tests/helpers/test_cache.py +++ b/tests/helpers/test_cache.py @@ -10,6 +10,7 @@ def test_compilation_caching(): with tempfile.TemporaryDirectory() as tmpdir: os.chdir(tmpdir) + cache.create_cache_dirs() program = os.path.join(tmpdir, 'program.cpp') open(program, 'w').write('int main() { return 0; }') @@ -45,6 +46,7 @@ def test_compilation_caching(): def test_cache(): with tempfile.TemporaryDirectory() as tmpdir: os.chdir(tmpdir) + cache.create_cache_dirs() assert cache.get_cache_file("abc.cpp") == CacheFile() cache_file = CacheFile( diff --git a/tests/helpers/test_compile.py b/tests/helpers/test_compile.py index 06703496..2ce1f0d3 100644 --- a/tests/helpers/test_compile.py +++ b/tests/helpers/test_compile.py @@ -2,7 +2,7 @@ import pytest import tempfile -from sinol_make.helpers.cache import save_compiled, check_compiled +from sinol_make.helpers.cache import save_compiled, check_compiled, create_cache_dirs from tests import util from tests.fixtures import create_package from tests.commands.gen.test_integration import simple_run @@ -11,6 +11,7 @@ def test_compilation_caching(): with tempfile.TemporaryDirectory() as tmpdir: os.chdir(tmpdir) + create_cache_dirs() with open(os.path.join(os.getcwd(), "test.txt"), "w") as f: f.write("Test data") with open(os.path.join(os.getcwd(), "test.e"), "w") as f: