From 158cdb7fc5dc53bd97444e5c87f22c7b090d2a42 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Wed, 3 Apr 2024 21:47:47 +0200 Subject: [PATCH 1/8] Small startup performance improvments --- src/sinol_make/helpers/compiler.py | 13 +++++++++---- src/sinol_make/helpers/parsers.py | 1 - src/sinol_make/util.py | 17 ++++++++++++++--- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/sinol_make/helpers/compiler.py b/src/sinol_make/helpers/compiler.py index 6b11bc36..77a585b3 100644 --- a/src/sinol_make/helpers/compiler.py +++ b/src/sinol_make/helpers/compiler.py @@ -1,14 +1,15 @@ from typing import List - +import subprocess import argparse +import sys import os -import sinol_make.util as util -import sys, subprocess - +from sinol_make import util from sinol_make.structs.compiler_structs import Compilers +from sinol_make.util import cache_result +@cache_result def check_if_installed(compiler): """ Check if a compiler is installed @@ -22,6 +23,7 @@ def check_if_installed(compiler): return True +@cache_result def get_c_compiler_path(): """ Get the C compiler @@ -41,6 +43,7 @@ def get_c_compiler_path(): return None +@cache_result def get_cpp_compiler_path(): """ Get the C++ compiler @@ -62,6 +65,7 @@ def get_cpp_compiler_path(): return None +@cache_result def get_python_interpreter_path(): """ Get the Python interpreter @@ -75,6 +79,7 @@ def get_python_interpreter_path(): return None +@cache_result def get_java_compiler_path(): """ Get the Java compiler diff --git a/src/sinol_make/helpers/parsers.py b/src/sinol_make/helpers/parsers.py index 83f1ae74..e553779a 100644 --- a/src/sinol_make/helpers/parsers.py +++ b/src/sinol_make/helpers/parsers.py @@ -1,5 +1,4 @@ import sys - import argparse from sinol_make.helpers import compiler diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py index a8714b52..e34fe226 100644 --- a/src/sinol_make/util.py +++ b/src/sinol_make/util.py @@ -16,13 +16,25 @@ __cache = {} +def cache_result(func): + """ + Function to cache the result of a function. + """ + def wrapper(*args, **kwargs): + if func.__name__ in __cache: + return __cache[func.__name__] + result = func(*args, **kwargs) + __cache[func.__name__] = result + return result + return wrapper + + +@cache_result def get_commands(): """ Function to get an array of all available commands. """ global __cache - if 'commands' in __cache: - return __cache['commands'] commands_path = glob.glob( os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -34,7 +46,6 @@ def get_commands(): temp = importlib.import_module('sinol_make.commands.' + os.path.basename(path), 'Command') commands.append(temp.Command()) - __cache['commands'] = commands return commands From 72ab0fb27b9af75bd50c14a126e5939a1ed2ebcb Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Wed, 3 Apr 2024 22:05:50 +0200 Subject: [PATCH 2/8] Dict for command names --- src/sinol_make/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/sinol_make/__init__.py b/src/sinol_make/__init__.py index f7aa5f70..408b7358 100644 --- a/src/sinol_make/__init__.py +++ b/src/sinol_make/__init__.py @@ -50,8 +50,10 @@ def main_exn(): parser = configure_parsers() arguments = [] curr_args = [] + commands = util.get_commands() + commands_dict = {command.get_name(): command for command in commands} for arg in sys.argv[1:]: - if arg in util.get_command_names() and not (len(curr_args) > 0 and curr_args[0] == 'init'): + if arg in commands_dict.keys() and not (len(curr_args) > 0 and curr_args[0] == 'init'): if curr_args: arguments.append(curr_args) curr_args = [arg] @@ -59,19 +61,19 @@ def main_exn(): curr_args.append(arg) if curr_args: arguments.append(curr_args) - commands = util.get_commands() + if not arguments: + parser.print_help() + exit(1) check_oiejq() for curr_args in arguments: args = parser.parse_args(curr_args) - command_found = False - for command in commands: - if command.get_name() == args.command: - command_found = True - if len(arguments) > 1: - print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '=')) - command.run(args) - if not command_found: + command = commands_dict.get(args.command, None) + if command: + if len(arguments) > 1: + print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '=')) + command.run(args) + else: parser.print_help() exit(1) From 5750fe63a6f36969f91cb4a84f6e43bd74871b5f Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Wed, 3 Apr 2024 22:16:23 +0200 Subject: [PATCH 3/8] Fix bug and remove java from compilers --- src/sinol_make/helpers/compiler.py | 4 ++-- src/sinol_make/helpers/parsers.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sinol_make/helpers/compiler.py b/src/sinol_make/helpers/compiler.py index 77a585b3..753fd384 100644 --- a/src/sinol_make/helpers/compiler.py +++ b/src/sinol_make/helpers/compiler.py @@ -9,7 +9,6 @@ from sinol_make.util import cache_result -@cache_result def check_if_installed(compiler): """ Check if a compiler is installed @@ -99,7 +98,8 @@ def get_default_compilers(): c_compiler_path=get_c_compiler_path(), cpp_compiler_path=get_cpp_compiler_path(), python_interpreter_path=get_python_interpreter_path(), - java_compiler_path=get_java_compiler_path() + # Java is not currently supported by sinol-make + # java_compiler_path=get_java_compiler_path() ) diff --git a/src/sinol_make/helpers/parsers.py b/src/sinol_make/helpers/parsers.py index e553779a..db233ed1 100644 --- a/src/sinol_make/helpers/parsers.py +++ b/src/sinol_make/helpers/parsers.py @@ -19,8 +19,9 @@ def add_compilation_arguments(parser: argparse.ArgumentParser): parser.add_argument('--python-interpreter-path', dest='python_interpreter_path', type=str, default=compiler.get_python_interpreter_path(), help='Python interpreter to use (default: python3)') - parser.add_argument('--java-compiler-path', dest='java_compiler_path', type=str, - default=compiler.get_java_compiler_path(), help='Java compiler to use (default: javac)') + # Java is not currently supported by sinol-make + # parser.add_argument('--java-compiler-path', dest='java_compiler_path', type=str, + # default=compiler.get_java_compiler_path(), help='Java compiler to use (default: javac)') parser.add_argument('--compile-mode', '-C', dest='compile_mode', choices=['default', 'oioioi', 'weak', 'd', 'o', 'w'], help='Warning flag groups used to compile C/C++ files. Available options:\n' ' default / d - uses default flags: \n' From 813f484109d1b36dfbc3e807dac369fb8cb7c72c Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Thu, 4 Apr 2024 14:12:03 +0200 Subject: [PATCH 4/8] Fix tests --- src/sinol_make/helpers/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinol_make/helpers/compiler.py b/src/sinol_make/helpers/compiler.py index 753fd384..9f5d174f 100644 --- a/src/sinol_make/helpers/compiler.py +++ b/src/sinol_make/helpers/compiler.py @@ -146,5 +146,5 @@ def verify_compilers(args: argparse.Namespace, solutions: List[str]) -> Compiler c_compiler_path=args.c_compiler_path, cpp_compiler_path=args.cpp_compiler_path, python_interpreter_path=args.python_interpreter_path, - java_compiler_path=args.java_compiler_path + java_compiler_path=None ) From f481cb63d340259677d98a35942b4189d8e3eb0c Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Fri, 5 Apr 2024 10:13:31 +0200 Subject: [PATCH 5/8] Refactor --- src/sinol_make/helpers/compiler.py | 2 +- src/sinol_make/helpers/func_cache.py | 14 ++++++++++++++ src/sinol_make/util.py | 19 +------------------ 3 files changed, 16 insertions(+), 19 deletions(-) create mode 100644 src/sinol_make/helpers/func_cache.py diff --git a/src/sinol_make/helpers/compiler.py b/src/sinol_make/helpers/compiler.py index 9f5d174f..e5167ecf 100644 --- a/src/sinol_make/helpers/compiler.py +++ b/src/sinol_make/helpers/compiler.py @@ -6,7 +6,7 @@ from sinol_make import util from sinol_make.structs.compiler_structs import Compilers -from sinol_make.util import cache_result +from sinol_make.helpers.func_cache import cache_result def check_if_installed(compiler): diff --git a/src/sinol_make/helpers/func_cache.py b/src/sinol_make/helpers/func_cache.py new file mode 100644 index 00000000..c329971c --- /dev/null +++ b/src/sinol_make/helpers/func_cache.py @@ -0,0 +1,14 @@ +__cache = {} + + +def cache_result(func): + """ + Function to cache the result of a function. + """ + def wrapper(*args, **kwargs): + if func.__name__ in __cache: + return __cache[func.__name__] + result = func(*args, **kwargs) + __cache[func.__name__] = result + return result + return wrapper diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py index e34fe226..ca145f09 100644 --- a/src/sinol_make/util.py +++ b/src/sinol_make/util.py @@ -1,6 +1,5 @@ import glob, importlib, os, sys, requests, yaml import math -import multiprocessing import platform import tarfile import hashlib @@ -10,31 +9,15 @@ from sinol_make.contest_types import get_contest_type from sinol_make.helpers import paths, cache +from sinol_make.helpers.func_cache import cache_result from sinol_make.structs.status_structs import Status -__cache = {} - - -def cache_result(func): - """ - Function to cache the result of a function. - """ - def wrapper(*args, **kwargs): - if func.__name__ in __cache: - return __cache[func.__name__] - result = func(*args, **kwargs) - __cache[func.__name__] = result - return result - return wrapper - - @cache_result def get_commands(): """ Function to get an array of all available commands. """ - global __cache commands_path = glob.glob( os.path.join( os.path.dirname(os.path.realpath(__file__)), From 1ef0012b8c1ad342c64e85bc2de46654aeb70ea1 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Fri, 5 Apr 2024 10:13:45 +0200 Subject: [PATCH 6/8] Cache `get_task_id()` --- src/sinol_make/helpers/package_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index d155e746..6b32cd83 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -6,10 +6,12 @@ from enum import Enum from typing import List, Union, Dict, Any +from sinol_make.helpers.func_cache import cache_result from sinol_make import util from sinol_make.helpers import paths +@cache_result def get_task_id() -> str: config = get_config() if "sinol_task_id" in config: From 5af811fdfcf5d721e6d20ce965b74328b8a55ba3 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Fri, 5 Apr 2024 10:30:59 +0200 Subject: [PATCH 7/8] Allow caching with cwd --- src/sinol_make/helpers/compiler.py | 8 +++---- src/sinol_make/helpers/func_cache.py | 32 +++++++++++++++++++------- src/sinol_make/helpers/package_util.py | 2 +- src/sinol_make/util.py | 2 +- tests/util.py | 1 + 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/sinol_make/helpers/compiler.py b/src/sinol_make/helpers/compiler.py index e5167ecf..8a1a68a4 100644 --- a/src/sinol_make/helpers/compiler.py +++ b/src/sinol_make/helpers/compiler.py @@ -22,7 +22,7 @@ def check_if_installed(compiler): return True -@cache_result +@cache_result() def get_c_compiler_path(): """ Get the C compiler @@ -42,7 +42,7 @@ def get_c_compiler_path(): return None -@cache_result +@cache_result() def get_cpp_compiler_path(): """ Get the C++ compiler @@ -64,7 +64,7 @@ def get_cpp_compiler_path(): return None -@cache_result +@cache_result() def get_python_interpreter_path(): """ Get the Python interpreter @@ -78,7 +78,7 @@ def get_python_interpreter_path(): return None -@cache_result +@cache_result() def get_java_compiler_path(): """ Get the Java compiler diff --git a/src/sinol_make/helpers/func_cache.py b/src/sinol_make/helpers/func_cache.py index c329971c..6b493489 100644 --- a/src/sinol_make/helpers/func_cache.py +++ b/src/sinol_make/helpers/func_cache.py @@ -1,14 +1,30 @@ +import os + __cache = {} -def cache_result(func): +def cache_result(cwd=False): """ Function to cache the result of a function. """ - def wrapper(*args, **kwargs): - if func.__name__ in __cache: - return __cache[func.__name__] - result = func(*args, **kwargs) - __cache[func.__name__] = result - return result - return wrapper + def decorator(func): + def wrapper(*args, **kwargs): + if cwd: + key = (func.__name__, os.getcwd()) + else: + key = func.__name__ + + if key in __cache: + return __cache[key] + result = func(*args, **kwargs) + __cache[key] = result + return result + return wrapper + return decorator + + +def clear_cache(): + """ + Function to clear the cache. + """ + __cache.clear() diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index 6b32cd83..34286a49 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -11,7 +11,7 @@ from sinol_make.helpers import paths -@cache_result +@cache_result(cwd=True) def get_task_id() -> str: config = get_config() if "sinol_task_id" in config: diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py index ca145f09..86cfa7a0 100644 --- a/src/sinol_make/util.py +++ b/src/sinol_make/util.py @@ -13,7 +13,7 @@ from sinol_make.structs.status_structs import Status -@cache_result +@cache_result() def get_commands(): """ Function to get an array of all available commands. diff --git a/tests/util.py b/tests/util.py index c29d99c3..e1a57782 100644 --- a/tests/util.py +++ b/tests/util.py @@ -202,6 +202,7 @@ def create_ins_outs(package_path): """ os.chdir(package_path) task_id = package_util.get_task_id() + print(task_id) create_ins(package_path, task_id) has_lib = package_util.any_files_matching_pattern(task_id, f"{task_id}lib.*") if not has_lib: From fcf9386acae6e438bcfc19a00fbef6065e582e0b Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Fri, 5 Apr 2024 10:31:06 +0200 Subject: [PATCH 8/8] Fix test --- tests/helpers/test_package_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/test_package_util.py b/tests/helpers/test_package_util.py index d821eb17..3d3538ce 100644 --- a/tests/helpers/test_package_util.py +++ b/tests/helpers/test_package_util.py @@ -3,7 +3,7 @@ from ..commands.run.util import create_ins from ..fixtures import * from tests import util -from sinol_make.helpers import package_util +from sinol_make.helpers import package_util, func_cache @pytest.mark.parametrize("create_package", [util.get_long_name_package_path()], indirect=True) @@ -12,6 +12,7 @@ def test_get_task_id(create_package): assert package_util.get_task_id() == "lpn" with open(os.path.join(package_path, "config.yml"), "w") as config_file: config_file.write("title: Long package name\n") + func_cache.clear_cache() with pytest.raises(SystemExit): package_util.get_task_id()