From 8fdcd8bbcac88295c075daedcf5d878deeaed547 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Tue, 19 Sep 2023 22:54:03 +0200 Subject: [PATCH] Add undocumented options for `sinol-make` (#123) * Proper order of options in config * Add `sinol_undocumented_time_tool` * Add `sinol_undocumented_test_limits` * Add package for tests * Add tests for `undocumented_time_tool` * Fix `undocumented_time_tool` option * Add test for `undocumented_test_limits` option * Refactor * Update src/sinol_make/helpers/package_util.py Co-authored-by: Tomasz Nowak <36604952+tonowak@users.noreply.github.com> * Error handling * Fix tests --------- Co-authored-by: Tomasz Nowak <36604952+tonowak@users.noreply.github.com> --- src/sinol_make/__init__.py | 2 +- src/sinol_make/commands/run/__init__.py | 52 +++++++++++++---- src/sinol_make/helpers/package_util.py | 14 +++-- src/sinol_make/util.py | 4 ++ tests/commands/run/test_integration.py | 58 ++++++++++++++++++- tests/commands/run/test_unit.py | 3 +- .../packages/undocumented_options/config.yml | 18 ++++++ .../packages/undocumented_options/in/.gitkeep | 0 .../undocumented_options/out/.gitkeep | 0 .../undocumented_options/prog/und.cpp | 9 +++ .../undocumented_options/prog/und1.cpp | 14 +++++ .../undocumented_options/prog/undingen.cpp | 13 +++++ tests/util.py | 9 ++- 13 files changed, 176 insertions(+), 20 deletions(-) create mode 100644 tests/packages/undocumented_options/config.yml create mode 100644 tests/packages/undocumented_options/in/.gitkeep create mode 100644 tests/packages/undocumented_options/out/.gitkeep create mode 100644 tests/packages/undocumented_options/prog/und.cpp create mode 100644 tests/packages/undocumented_options/prog/und1.cpp create mode 100644 tests/packages/undocumented_options/prog/undingen.cpp diff --git a/src/sinol_make/__init__.py b/src/sinol_make/__init__.py index 968856fb..d54cc09a 100644 --- a/src/sinol_make/__init__.py +++ b/src/sinol_make/__init__.py @@ -9,7 +9,7 @@ from sinol_make import util, oiejq -__version__ = "1.5.7" +__version__ = "1.5.8" def configure_parsers(): diff --git a/src/sinol_make/commands/run/__init__.py b/src/sinol_make/commands/run/__init__.py index d2e58a04..958d3db3 100644 --- a/src/sinol_make/commands/run/__init__.py +++ b/src/sinol_make/commands/run/__init__.py @@ -254,7 +254,7 @@ def configure_subparser(self, subparser): parser.add_argument('--ml', type=float, help='memory limit for all tests (in MB)') parser.add_argument('--hide-memory', dest='hide_memory', action='store_true', help='hide memory usage in report') - parser.add_argument('-T', '--time-tool', dest='time_tool', choices=['oiejq', 'time'], default=default_timetool, + parser.add_argument('-T', '--time-tool', dest='time_tool', choices=['oiejq', 'time'], help=f'tool to measure time and memory usage (default: {default_timetool})') parser.add_argument('--oiejq-path', dest='oiejq_path', type=str, help='path to oiejq executable (default: `~/.local/bin/oiejq`)') @@ -456,7 +456,10 @@ def sigint_handler(signum, frame): output, lines = process.communicate(timeout=hard_time_limit) except subprocess.TimeoutExpired: timeout = True - os.killpg(os.getpgid(process.pid), signal.SIGTERM) + try: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + except ProcessLookupError: + pass process.communicate() result = ExecutionResult() @@ -531,14 +534,20 @@ def sigint_handler(signum, frame): executable_process = child break if executable_process is not None and executable_process.memory_info().rss > memory_limit * 1024: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) + try: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + except ProcessLookupError: + pass mem_limit_exceeded = True break except psutil.NoSuchProcess: pass if time.time() - start_time > hard_time_limit: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) + try: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + except ProcessLookupError: + pass timeout = True break output, _ = process.communicate() @@ -609,12 +618,12 @@ def run_solution(self, data_for_execution: ExecutionData): result_file = file_no_ext + ".res" hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0) - if self.args.time_tool == 'oiejq': + if self.timetool_name == 'oiejq': command = f'"{timetool_path}" "{executable}"' return self.execute_oiejq(command, name, result_file, test, output_file, self.get_output_file(test), time_limit, memory_limit, hard_time_limit_in_s) - elif self.args.time_tool == 'time': + elif self.timetool_name == 'time': if sys.platform == 'darwin': timeout_name = 'gtimeout' time_name = 'gtime' @@ -997,8 +1006,8 @@ def set_constants(self): def validate_arguments(self, args): compilers = compiler.verify_compilers(args, self.get_solutions(None)) - timetool_path = None - if args.time_tool == 'oiejq': + def use_oiejq(): + timetool_path = None if not util.is_linux(): util.exit_with_error('As `oiejq` works only on Linux-based operating systems,\n' 'we do not recommend using operating systems such as Windows or macOS.\n' @@ -1016,12 +1025,31 @@ def validate_arguments(self, args): timetool_path = oiejq.get_oiejq_path() if timetool_path is None: util.exit_with_error('oiejq is not installed.') - elif args.time_tool == 'time': + return timetool_path, 'oiejq' + def use_time(): if sys.platform == 'win32' or sys.platform == 'cygwin': util.exit_with_error('Measuring with `time` is not supported on Windows.') - timetool_path = 'time' + return 'time', 'time' + + timetool_path, timetool_name = None, None + use_default_timetool = use_oiejq if util.is_linux() else use_time - return compilers, timetool_path + if args.time_tool is None and self.config.get('sinol_undocumented_time_tool', '') != '': + if self.config.get('sinol_undocumented_time_tool', '') == 'oiejq': + timetool_path, timetool_name = use_oiejq() + elif self.config.get('sinol_undocumented_time_tool', '') == 'time': + timetool_path, timetool_name = use_time() + else: + util.exit_with_error('Invalid time tool specified in config.yml.') + elif args.time_tool is None: + timetool_path, timetool_name = use_default_timetool() + elif args.time_tool == 'oiejq': + timetool_path, timetool_name = use_oiejq() + elif args.time_tool == 'time': + timetool_path, timetool_name = use_time() + else: + util.exit_with_error('Invalid time tool specified.') + return compilers, timetool_path, timetool_name def exit(self): if len(self.failed_compilations) > 0: @@ -1147,7 +1175,7 @@ def run(self, args): if not 'title' in self.config.keys(): util.exit_with_error('Title was not defined in config.yml.') - self.compilers, self.timetool_path = self.validate_arguments(args) + self.compilers, self.timetool_path, self.timetool_name = self.validate_arguments(args) title = self.config["title"] print("Task: %s (tag: %s)" % (title, self.ID)) diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index 1721d3de..bf634a73 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -88,7 +88,8 @@ class LimitTypes(Enum): MEMORY_LIMIT = 2 -def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: str, test_group: str, test_path: str): +def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: str, test_group: str, test_path: str, + allow_test_limit: bool = False): if limit_type == LimitTypes.TIME_LIMIT: limit_name = "time_limit" plural_limit_name = "time_limits" @@ -100,7 +101,10 @@ def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: if plural_limit_name in dict: if test_id in dict[plural_limit_name] and test_id != "0": - util.exit_with_error(f'{os.path.basename(test_path)}: Specifying limit for single test is a bad practice and is not supported.') + if allow_test_limit: + return dict[plural_limit_name][test_id] + else: + util.exit_with_error(f'{os.path.basename(test_path)}: Specifying limit for a single test is not allowed in sinol-make.') elif test_group in dict[plural_limit_name]: return dict[plural_limit_name][test_group] if limit_name in dict: @@ -112,9 +116,11 @@ def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: def _get_limit(limit_type: LimitTypes, test_path: str, config: Dict[str, Any], lang: str, task_id: str): test_id = extract_test_id(test_path, task_id) test_group = str(get_group(test_path, task_id)) - global_limit = _get_limit_from_dict(config, limit_type, test_id, test_group, test_path) + allow_test_limit = config.get("sinol_undocumented_test_limits", False) + global_limit = _get_limit_from_dict(config, limit_type, test_id, test_group, test_path, allow_test_limit) override_limits_dict = config.get("override_limits", {}).get(lang, {}) - overriden_limit = _get_limit_from_dict(override_limits_dict, limit_type, test_id, test_group, test_path) + overriden_limit = _get_limit_from_dict(override_limits_dict, limit_type, test_id, test_group, test_path, + allow_test_limit) if overriden_limit is not None: return overriden_limit else: diff --git a/src/sinol_make/util.py b/src/sinol_make/util.py index 8bf300b6..68c2f4f9 100644 --- a/src/sinol_make/util.py +++ b/src/sinol_make/util.py @@ -59,6 +59,10 @@ def save_config(config): "title", "title_pl", "title_en", + "sinol_task_id", + "sinol_contest_type", + "sinol_undocumented_time_tool", + "sinol_undocumented_test_limits", "memory_limit", "memory_limits", "time_limit", diff --git a/tests/commands/run/test_integration.py b/tests/commands/run/test_integration.py index 03e75473..0b846108 100644 --- a/tests/commands/run/test_integration.py +++ b/tests/commands/run/test_integration.py @@ -6,7 +6,7 @@ from ...fixtures import * from .util import * -from sinol_make import configure_parsers +from sinol_make import configure_parsers, util, oiejq @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(), @@ -447,3 +447,59 @@ def test_mem_limit_kill(create_package, time_tool): assert e.value.code == 1 assert end_time - start_time < 5 # The solution runs for 20 seconds, but it immediately exceeds memory limit, # so it should be killed. + + +@pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True) +def test_undocumented_time_tool_option(create_package): + """ + Test if `undocumented_time_tool` option works. + """ + package_path = create_package + create_ins_outs(package_path) + parser = configure_parsers() + args = parser.parse_args(["run"]) + command = Command() + command.run(args) + assert command.timetool_path == "time" + + +@pytest.mark.oiejq +@pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True) +def test_override_undocumented_time_tool_option(create_package): + """ + Test if overriding `undocumented_time_tool` option with --time-tool flag works. + """ + package_path = create_package + create_ins_outs(package_path) + parser = configure_parsers() + args = parser.parse_args(["run", "--time-tool", "oiejq"]) + command = Command() + command.run(args) + assert command.timetool_path == oiejq.get_oiejq_path() + + +@pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True) +def test_undocumented_test_limits_option(create_package, capsys): + """ + Test if `undocumented_test_limits` option works. + """ + package_path = create_package + create_ins_outs(package_path) + parser = configure_parsers() + args = parser.parse_args(["run"]) + command = Command() + command.run(args) + + with open(os.path.join(os.getcwd(), "config.yml")) as config_file: + config = yaml.load(config_file, Loader=yaml.SafeLoader) + del config["sinol_undocumented_test_limits"] + with open(os.path.join(os.getcwd(), "config.yml"), "w") as config_file: + config_file.write(yaml.dump(config)) + + command = Command() + with pytest.raises(SystemExit) as e: + command.run(args) + + assert e.value.code == 1 + out = capsys.readouterr().out + assert "und1a.in: Specifying limit for a single test is not allowed in sinol-make." in out diff --git a/tests/commands/run/test_unit.py b/tests/commands/run/test_unit.py index 2af4d417..989c2883 100644 --- a/tests/commands/run/test_unit.py +++ b/tests/commands/run/test_unit.py @@ -44,6 +44,7 @@ def test_execution(create_package, time_tool): package_path = create_package command = get_command(package_path) command.args.time_tool = time_tool + command.timetool_name = time_tool solution = "abc.cpp" executable = package_util.get_executable(solution) result = command.compile_solutions([solution]) @@ -85,7 +86,7 @@ def test_run_solutions(create_package, time_tool): command.memory_limit = command.config["memory_limit"] command.time_limit = command.config["time_limit"] command.timetool_path = oiejq.get_oiejq_path() - + command.timetool_name = time_tool def flatten_results(results): new_results = {} for solution in results.keys(): diff --git a/tests/packages/undocumented_options/config.yml b/tests/packages/undocumented_options/config.yml new file mode 100644 index 00000000..95aad60b --- /dev/null +++ b/tests/packages/undocumented_options/config.yml @@ -0,0 +1,18 @@ +title: Package with undocumented sinol-make options +sinol_task_id: und + +sinol_undocumented_time_tool: time +sinol_undocumented_test_limits: true + +memory_limit: 20480 +time_limit: 1000 +time_limits: + 1a: 5000 + +sinol_expected_scores: + und.cpp: + expected: {1: OK} + points: 100 + und1.cpp: + expected: {1: OK} + points: 100 diff --git a/tests/packages/undocumented_options/in/.gitkeep b/tests/packages/undocumented_options/in/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/undocumented_options/out/.gitkeep b/tests/packages/undocumented_options/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/undocumented_options/prog/und.cpp b/tests/packages/undocumented_options/prog/und.cpp new file mode 100644 index 00000000..bd410607 --- /dev/null +++ b/tests/packages/undocumented_options/prog/und.cpp @@ -0,0 +1,9 @@ +#include + +using namespace std; + +int main() { + int a, b; + cin >> a >> b; + cout << a + b; +} diff --git a/tests/packages/undocumented_options/prog/und1.cpp b/tests/packages/undocumented_options/prog/und1.cpp new file mode 100644 index 00000000..23c0a790 --- /dev/null +++ b/tests/packages/undocumented_options/prog/und1.cpp @@ -0,0 +1,14 @@ +#include +#include + +using namespace std; +using namespace std::chrono_literals; + +int main() { + int a, b; + cin >> a >> b; + if (a == 1 && b == 1) { + this_thread::sleep_for(3s); + } + cout << a + b << endl; +} diff --git a/tests/packages/undocumented_options/prog/undingen.cpp b/tests/packages/undocumented_options/prog/undingen.cpp new file mode 100644 index 00000000..9c7b7beb --- /dev/null +++ b/tests/packages/undocumented_options/prog/undingen.cpp @@ -0,0 +1,13 @@ +#include + +using namespace std; + +int main() { + ofstream f("und1a.in"); + f << "1 1\n"; + f.close(); + + f.open("und1b.in"); + f << "2 2\n"; + f.close(); +} diff --git a/tests/util.py b/tests/util.py index ca6e2031..74ecf9d1 100644 --- a/tests/util.py +++ b/tests/util.py @@ -94,7 +94,7 @@ def get_doc_package_path(): """ return os.path.join(os.path.dirname(__file__), "packages", "doc") - + def get_long_name_package_path(): """ Get path to package with long name (/test/packages/long_package_name) @@ -102,6 +102,13 @@ def get_long_name_package_path(): return os.path.join(os.path.dirname(__file__), "packages", "long_package_name") +def get_undocumented_options_package_path(): + """ + Get path to package with undocumented options in config.yml (/test/packages/undoc) + """ + return os.path.join(os.path.dirname(__file__), "packages", "undocumented_options") + + def create_ins(package_path, task_id): """ Create .in files for package.