Add undocumented options for sinol-make (#123)

* Proper order of options in config

* Add `sinol_undocumented_time_tool`

* Add `sinol_undocumented_test_limits`

* Add package for tests

* Add tests for `undocumented_time_tool`

* Fix `undocumented_time_tool` option

* Add test for `undocumented_test_limits` option

* Refactor

* Update src/sinol_make/helpers/package_util.py

Co-authored-by: Tomasz Nowak <[email protected]>

* Error handling

* Fix tests

---------

Co-authored-by: Tomasz Nowak <[email protected]>

src/sinol_make/__init__.py

@@ -9,7 +9,7 @@
 from sinol_make import util, oiejq
 
 
-__version__ = "1.5.7"
+__version__ = "1.5.8"
 
 
 def configure_parsers():

src/sinol_make/commands/run/__init__.py

@@ -254,7 +254,7 @@ def configure_subparser(self, subparser):
         parser.add_argument('--ml', type=float, help='memory limit for all tests (in MB)')
         parser.add_argument('--hide-memory', dest='hide_memory', action='store_true',
                             help='hide memory usage in report')
-        parser.add_argument('-T', '--time-tool', dest='time_tool', choices=['oiejq', 'time'], default=default_timetool,
+        parser.add_argument('-T', '--time-tool', dest='time_tool', choices=['oiejq', 'time'],
                             help=f'tool to measure time and memory usage (default: {default_timetool})')
         parser.add_argument('--oiejq-path', dest='oiejq_path', type=str,
                             help='path to oiejq executable (default: `~/.local/bin/oiejq`)')
@@ -456,7 +456,10 @@ def sigint_handler(signum, frame):
                 output, lines = process.communicate(timeout=hard_time_limit)
             except subprocess.TimeoutExpired:
                 timeout = True
-                os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                try:
+                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                except ProcessLookupError:
+                    pass
                 process.communicate()
 
         result = ExecutionResult()
@@ -531,14 +534,20 @@ def sigint_handler(signum, frame):
                             executable_process = child
                             break
                     if executable_process is not None and executable_process.memory_info().rss > memory_limit * 1024:
-                        os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                        try:
+                            os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                        except ProcessLookupError:
+                            pass
                         mem_limit_exceeded = True
                         break
                 except psutil.NoSuchProcess:
                     pass
 
                 if time.time() - start_time > hard_time_limit:
-                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                    try:
+                        os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+                    except ProcessLookupError:
+                        pass
                     timeout = True
                     break
             output, _ = process.communicate()
@@ -609,12 +618,12 @@ def run_solution(self, data_for_execution: ExecutionData):
         result_file = file_no_ext + ".res"
         hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0)
 
-        if self.args.time_tool == 'oiejq':
+        if self.timetool_name == 'oiejq':
             command = f'"{timetool_path}" "{executable}"'
 
             return self.execute_oiejq(command, name, result_file, test, output_file, self.get_output_file(test),
                                       time_limit, memory_limit, hard_time_limit_in_s)
-        elif self.args.time_tool == 'time':
+        elif self.timetool_name == 'time':
             if sys.platform == 'darwin':
                 timeout_name = 'gtimeout'
                 time_name = 'gtime'
@@ -997,8 +1006,8 @@ def set_constants(self):
     def validate_arguments(self, args):
         compilers = compiler.verify_compilers(args, self.get_solutions(None))
 
-        timetool_path = None
-        if args.time_tool == 'oiejq':
+        def use_oiejq():
+            timetool_path = None
             if not util.is_linux():
                 util.exit_with_error('As `oiejq` works only on Linux-based operating systems,\n'
                                      'we do not recommend using operating systems such as Windows or macOS.\n'
@@ -1016,12 +1025,31 @@ def validate_arguments(self, args):
                 timetool_path = oiejq.get_oiejq_path()
             if timetool_path is None:
                 util.exit_with_error('oiejq is not installed.')
-        elif args.time_tool == 'time':
+            return timetool_path, 'oiejq'
+        def use_time():
             if sys.platform == 'win32' or sys.platform == 'cygwin':
                 util.exit_with_error('Measuring with `time` is not supported on Windows.')
-            timetool_path = 'time'
+            return 'time', 'time'
+
+        timetool_path, timetool_name = None, None
+        use_default_timetool = use_oiejq if util.is_linux() else use_time
 
-        return compilers, timetool_path
+        if args.time_tool is None and self.config.get('sinol_undocumented_time_tool', '') != '':
+            if self.config.get('sinol_undocumented_time_tool', '') == 'oiejq':
+                timetool_path, timetool_name = use_oiejq()
+            elif self.config.get('sinol_undocumented_time_tool', '') == 'time':
+                timetool_path, timetool_name = use_time()
+            else:
+                util.exit_with_error('Invalid time tool specified in config.yml.')
+        elif args.time_tool is None:
+            timetool_path, timetool_name = use_default_timetool()
+        elif args.time_tool == 'oiejq':
+            timetool_path, timetool_name = use_oiejq()
+        elif args.time_tool == 'time':
+            timetool_path, timetool_name = use_time()
+        else:
+            util.exit_with_error('Invalid time tool specified.')
+        return compilers, timetool_path, timetool_name
 
     def exit(self):
         if len(self.failed_compilations) > 0:
@@ -1147,7 +1175,7 @@ def run(self, args):
         if not 'title' in self.config.keys():
             util.exit_with_error('Title was not defined in config.yml.')
 
-        self.compilers, self.timetool_path = self.validate_arguments(args)
+        self.compilers, self.timetool_path, self.timetool_name = self.validate_arguments(args)
 
         title = self.config["title"]
         print("Task: %s (tag: %s)" % (title, self.ID))

src/sinol_make/helpers/package_util.py

@@ -88,7 +88,8 @@ class LimitTypes(Enum):
     MEMORY_LIMIT = 2
 
 
-def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: str, test_group: str, test_path: str):
+def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id: str, test_group: str, test_path: str,
+                         allow_test_limit: bool = False):
     if limit_type == LimitTypes.TIME_LIMIT:
         limit_name = "time_limit"
         plural_limit_name = "time_limits"
@@ -100,7 +101,10 @@ def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id:
 
     if plural_limit_name in dict:
         if test_id in dict[plural_limit_name] and test_id != "0":
-            util.exit_with_error(f'{os.path.basename(test_path)}: Specifying limit for single test is a bad practice and is not supported.')
+            if allow_test_limit:
+                return dict[plural_limit_name][test_id]
+            else:
+                util.exit_with_error(f'{os.path.basename(test_path)}: Specifying limit for a single test is not allowed in sinol-make.')
         elif test_group in dict[plural_limit_name]:
             return dict[plural_limit_name][test_group]
     if limit_name in dict:
@@ -112,9 +116,11 @@ def _get_limit_from_dict(dict: Dict[str, Any], limit_type: LimitTypes, test_id:
 def _get_limit(limit_type: LimitTypes, test_path: str, config: Dict[str, Any], lang: str, task_id: str):
     test_id = extract_test_id(test_path, task_id)
     test_group = str(get_group(test_path, task_id))
-    global_limit = _get_limit_from_dict(config, limit_type, test_id, test_group, test_path)
+    allow_test_limit = config.get("sinol_undocumented_test_limits", False)
+    global_limit = _get_limit_from_dict(config, limit_type, test_id, test_group, test_path, allow_test_limit)
     override_limits_dict = config.get("override_limits", {}).get(lang, {})
-    overriden_limit = _get_limit_from_dict(override_limits_dict, limit_type, test_id, test_group, test_path)
+    overriden_limit = _get_limit_from_dict(override_limits_dict, limit_type, test_id, test_group, test_path,
+                                           allow_test_limit)
     if overriden_limit is not None:
         return overriden_limit
     else:

src/sinol_make/util.py

@@ -59,6 +59,10 @@ def save_config(config):
         "title",
         "title_pl",
         "title_en",
+        "sinol_task_id",
+        "sinol_contest_type",
+        "sinol_undocumented_time_tool",
+        "sinol_undocumented_test_limits",
         "memory_limit",
         "memory_limits",
         "time_limit",

tests/commands/run/test_integration.py

@@ -6,7 +6,7 @@
 
 from ...fixtures import *
 from .util import *
-from sinol_make import configure_parsers
+from sinol_make import configure_parsers, util, oiejq
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
@@ -447,3 +447,59 @@ def test_mem_limit_kill(create_package, time_tool):
     assert e.value.code == 1
     assert end_time - start_time < 5  # The solution runs for 20 seconds, but it immediately exceeds memory limit,
                                       # so it should be killed.
+
+
+@pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True)
+def test_undocumented_time_tool_option(create_package):
+    """
+    Test if `undocumented_time_tool` option works.
+    """
+    package_path = create_package
+    create_ins_outs(package_path)
+    parser = configure_parsers()
+    args = parser.parse_args(["run"])
+    command = Command()
+    command.run(args)
+    assert command.timetool_path == "time"
+
+
+@pytest.mark.oiejq
+@pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True)
+def test_override_undocumented_time_tool_option(create_package):
+    """
+    Test if overriding `undocumented_time_tool` option with --time-tool flag works.
+    """
+    package_path = create_package
+    create_ins_outs(package_path)
+    parser = configure_parsers()
+    args = parser.parse_args(["run", "--time-tool", "oiejq"])
+    command = Command()
+    command.run(args)
+    assert command.timetool_path == oiejq.get_oiejq_path()
+
+
+@pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True)
+def test_undocumented_test_limits_option(create_package, capsys):
+    """
+    Test if `undocumented_test_limits` option works.
+    """
+    package_path = create_package
+    create_ins_outs(package_path)
+    parser = configure_parsers()
+    args = parser.parse_args(["run"])
+    command = Command()
+    command.run(args)
+
+    with open(os.path.join(os.getcwd(), "config.yml")) as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
+    del config["sinol_undocumented_test_limits"]
+    with open(os.path.join(os.getcwd(), "config.yml"), "w") as config_file:
+        config_file.write(yaml.dump(config))
+
+    command = Command()
+    with pytest.raises(SystemExit) as e:
+        command.run(args)
+
+    assert e.value.code == 1
+    out = capsys.readouterr().out
+    assert "und1a.in: Specifying limit for a single test is not allowed in sinol-make." in out

tests/commands/run/test_unit.py

@@ -44,6 +44,7 @@ def test_execution(create_package, time_tool):
     package_path = create_package
     command = get_command(package_path)
     command.args.time_tool = time_tool
+    command.timetool_name = time_tool
     solution = "abc.cpp"
     executable = package_util.get_executable(solution)
     result = command.compile_solutions([solution])
@@ -85,7 +86,7 @@ def test_run_solutions(create_package, time_tool):
     command.memory_limit = command.config["memory_limit"]
     command.time_limit = command.config["time_limit"]
     command.timetool_path = oiejq.get_oiejq_path()
-
+    command.timetool_name = time_tool
     def flatten_results(results):
         new_results = {}
         for solution in results.keys():

tests/packages/undocumented_options/config.yml

@@ -0,0 +1,18 @@
+title: Package with undocumented sinol-make options
+sinol_task_id: und
+
+sinol_undocumented_time_tool: time
+sinol_undocumented_test_limits: true
+
+memory_limit: 20480
+time_limit: 1000
+time_limits:
+  1a: 5000
+
+sinol_expected_scores:
+  und.cpp:
+    expected: {1: OK}
+    points: 100
+  und1.cpp:
+    expected: {1: OK}
+    points: 100

tests/packages/undocumented_options/prog/und.cpp

@@ -0,0 +1,9 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b;
+}

tests/packages/undocumented_options/prog/und1.cpp

@@ -0,0 +1,14 @@
+#include <bits/stdc++.h>
+#include <chrono>
+
+using namespace std;
+using namespace std::chrono_literals;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+    if (a == 1 && b == 1) {
+        this_thread::sleep_for(3s);
+    }
+    cout << a + b << endl;
+}

tests/packages/undocumented_options/prog/undingen.cpp

@@ -0,0 +1,13 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    ofstream f("und1a.in");
+    f << "1 1\n";
+    f.close();
+
+    f.open("und1b.in");
+    f << "2 2\n";
+    f.close();
+}

tests/util.py

@@ -94,14 +94,21 @@ def get_doc_package_path():
     """
     return os.path.join(os.path.dirname(__file__), "packages", "doc")
 
- 
+
 def get_long_name_package_path():
     """
     Get path to package with long name (/test/packages/long_package_name)
     """
     return os.path.join(os.path.dirname(__file__), "packages", "long_package_name")
 
 
+def get_undocumented_options_package_path():
+    """
+    Get path to package with undocumented options in config.yml (/test/packages/undoc)
+    """
+    return os.path.join(os.path.dirname(__file__), "packages", "undocumented_options")
+
+
 def create_ins(package_path, task_id):
     """
     Create .in files for package.