diff --git a/setup.py b/setup.py index 3de0116..1664bf9 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,13 @@ 'ping = sio.workers.ping:run', 'compile = sio.compilers.job:run', 'exec = sio.executors.executor:run', + 'interactive-exec = sio.executors.executor:interactive_run', 'sio2jail-exec = sio.executors.sio2jail_exec:run', + 'sio2jail-interactive-exec = sio.executors.sio2jail_exec:interactive_run', 'cpu-exec = sio.executors.executor:run', + 'cpu-interactive-exec = sio.executors.executor:interactive_run', 'unsafe-exec = sio.executors.unsafe_exec:run', + 'unsafe-interactive-exec = sio.executors.unsafe_exec:interactive_run', 'ingen = sio.executors.ingen:run', 'inwer = sio.executors.inwer:run', ], diff --git a/sio/executors/common.py b/sio/executors/common.py index 0da4f35..462d626 100644 --- a/sio/executors/common.py +++ b/sio/executors/common.py @@ -20,6 +20,23 @@ def _populate_environ(renv, environ): environ['out_file'] = renv['out_file'] +def _extract_input_if_zipfile(input_name, zipdir): + if is_zipfile(input_name): + try: + # If not a zip file, will pass it directly to exe + with ZipFile(tempcwd('in'), 'r') as f: + if len(f.namelist()) != 1: + raise Exception("Archive should have only one file.") + + f.extract(f.namelist()[0], zipdir) + input_name = os.path.join(zipdir, f.namelist()[0]) + # zipfile throws some undocumented exceptions + except Exception as e: + raise Exception("Failed to open archive: " + six.text_type(e)) + + return input_name + + @decode_fields(['result_string']) def run(environ, executor, use_sandboxes=True): """ @@ -70,18 +87,7 @@ def _run(environ, executor, use_sandboxes): zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: - if is_zipfile(input_name): - try: - # If not a zip file, will pass it directly to exe - with ZipFile(tempcwd('in'), 'r') as f: - if len(f.namelist()) != 1: - raise Exception("Archive should have only one file.") - - f.extract(f.namelist()[0], zipdir) - input_name = os.path.join(zipdir, f.namelist()[0]) - # zipfile throws some undocumented exceptions - except Exception as e: - raise Exception("Failed to open archive: " + six.text_type(e)) + input_name = _extract_input_if_zipfile(input_name, zipdir) with file_executor as fe: with open(input_name, 'rb') as inf: diff --git a/sio/executors/executor.py b/sio/executors/executor.py index 40ce9af..3f73f11 100644 --- a/sio/executors/executor.py +++ b/sio/executors/executor.py @@ -1,7 +1,10 @@ from __future__ import absolute_import -from sio.executors import common +from sio.executors import common, interactive_common from sio.workers.executors import SupervisedExecutor def run(environ): return common.run(environ, SupervisedExecutor()) + +def interactive_run(environ): + return interactive_common.run(environ, SupervisedExecutor()) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py new file mode 100644 index 0000000..338a658 --- /dev/null +++ b/sio/executors/interactive_common.py @@ -0,0 +1,200 @@ +import os +from shutil import rmtree +from threading import Thread + +from sio.executors.checker import output_to_fraction +from sio.executors.common import _extract_input_if_zipfile, _populate_environ +from sio.workers import ft +from sio.workers.executors import DetailedUnprotectedExecutor +from sio.workers.util import TemporaryCwd, decode_fields, replace_invalid_UTF, tempcwd +from sio.workers.file_runners import get_file_runner + +import signal +import six + +DEFAULT_INTERACTOR_MEM_LIMIT = 256 * 2 ** 10 # in KiB +RESULT_STRING_LENGTH_LIMIT = 1024 # in bytes + + +class InteractorError(Exception): + def __init__(self, message, interactor_out, env, renv, irenv): + super().__init__( + f'{message}\n' + f'Interactor out: {interactor_out}\n' + f'Interactor environ dump: {irenv}\n' + f'Solution environ dump: {renv}\n' + f'Environ dump: {env}' + ) + + +def _limit_length(s): + if len(s) > RESULT_STRING_LENGTH_LIMIT: + suffix = b'[...]' + return s[: max(0, RESULT_STRING_LENGTH_LIMIT - len(suffix))] + suffix + return s + + +@decode_fields(['result_string']) +def run(environ, executor, use_sandboxes=True): + """ + Common code for executors. + + :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` + For all supported options, see the global documentation for + `sio.workers.executors` and prefix them with ``exec_``. + :param: executor Executor instance used for executing commands. + :param: use_sandboxes Enables safe checking output correctness. + See `sio.executors.checkers`. True by default. + """ + + renv = _run(environ, executor, use_sandboxes) + + _populate_environ(renv, environ) + + for key in ('result_code', 'result_string'): + environ[key] = replace_invalid_UTF(environ[key]) + + if 'out_file' in environ: + ft.upload( + environ, + 'out_file', + tempcwd('out'), + to_remote_store=environ.get('upload_out', False), + ) + + return environ + + +def _fill_result(env, renv, irenv, interactor_out): + sol_sig = renv.get('exit_signal', None) + inter_sig = irenv.get('exit_signal', None) + sigpipe = signal.SIGPIPE.value + + if irenv['result_code'] != 'OK' and inter_sig != sigpipe: + renv['result_code'] = 'SE' + raise InteractorError(f'Interactor got {irenv["result_code"]}.', interactor_out, env, renv, irenv) + elif renv['result_code'] != 'OK' and sol_sig != sigpipe: + return + elif len(interactor_out) == 0: + renv['result_code'] = 'SE' + raise InteractorError(f'Empty interactor out.', interactor_out, env, renv, irenv) + elif inter_sig == sigpipe: + renv['result_code'] = 'WA' + renv['result_string'] = 'solution exited prematurely' + else: + renv['result_string'] = '' + if six.ensure_binary(interactor_out[0]) == b'OK': + renv['result_code'] = 'OK' + if interactor_out[1]: + renv['result_string'] = _limit_length(interactor_out[1]) + renv['result_percentage'] = output_to_fraction(interactor_out[2]) + else: + renv['result_code'] = 'WA' + if interactor_out[1]: + renv['result_string'] = _limit_length(interactor_out[1]) + renv['result_percentage'] = (0, 1) + + +def _run(environ, executor, use_sandboxes): + input_name = tempcwd('in') + + file_executor = get_file_runner(executor, environ) + interactor_executor = DetailedUnprotectedExecutor() + exe_filename = file_executor.preferred_filename() + interactor_filename = 'soc' + + ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) + os.chmod(tempcwd(exe_filename), 0o700) + ft.download(environ, 'interactor_file', interactor_filename, add_to_cache=True) + os.chmod(tempcwd(interactor_filename), 0o700) + ft.download(environ, 'in_file', input_name, add_to_cache=True) + + zipdir = tempcwd('in_dir') + os.mkdir(zipdir) + try: + input_name = _extract_input_if_zipfile(input_name, zipdir) + + r1, w1 = os.pipe() + r2, w2 = os.pipe() + for fd in (r1, w1, r2, w2): + os.set_inheritable(fd, True) + + interactor_args = [os.path.basename(input_name), 'out'] + + interactor_time_limit = 2 * environ['exec_time_limit'] + + class ExecutionWrapper(Thread): + def __init__(self, executor, *args, **kwargs): + super(ExecutionWrapper, self).__init__() + self.executor = executor + self.args = args + self.kwargs = kwargs + self.value = None + self.exception = None + + def run(self): + with TemporaryCwd(): + try: + self.value = self.executor(*self.args, **self.kwargs) + except Exception as e: + self.exception = e + + with interactor_executor as ie: + interactor = ExecutionWrapper( + ie, + [tempcwd(interactor_filename)] + interactor_args, + stdin=r2, + stdout=w1, + ignore_errors=True, + environ=environ, + environ_prefix='interactor_', + mem_limit=DEFAULT_INTERACTOR_MEM_LIMIT, + time_limit=interactor_time_limit, + pass_fds=(r2, w1), + close_passed_fd=True, + cwd=tempcwd(), + in_file=environ['in_file'], + ) + + with file_executor as fe: + exe = ExecutionWrapper( + fe, + tempcwd(exe_filename), + [], + stdin=r1, + stdout=w2, + ignore_errors=True, + environ=environ, + environ_prefix='exec_', + pass_fds=(r1, w2), + close_passed_fd=True, + cwd=tempcwd(), + in_file=environ['in_file'], + ) + + exe.start() + interactor.start() + + exe.join() + interactor.join() + + for ew in (exe, interactor): + if ew.exception is not None: + raise ew.exception + + renv = exe.value + irenv = interactor.value + + try: + with open(tempcwd('out'), 'rb') as result_file: + interactor_out = [line.rstrip() for line in result_file.readlines()] + while len(interactor_out) < 3: + interactor_out.append(b'') + except FileNotFoundError: + interactor_out = [] + + _fill_result(environ, renv, irenv, interactor_out) + finally: + rmtree(zipdir) + + return renv diff --git a/sio/executors/sio2jail_exec.py b/sio/executors/sio2jail_exec.py index ac23e4a..d8771e0 100644 --- a/sio/executors/sio2jail_exec.py +++ b/sio/executors/sio2jail_exec.py @@ -1,6 +1,9 @@ -from sio.executors import common +from sio.executors import common, interactive_common from sio.workers.executors import Sio2JailExecutor def run(environ): return common.run(environ, Sio2JailExecutor()) + +def interactive_run(environ): + return interactive_common.run(environ, Sio2JailExecutor()) diff --git a/sio/executors/unsafe_exec.py b/sio/executors/unsafe_exec.py index 3b0ae38..0d1242d 100644 --- a/sio/executors/unsafe_exec.py +++ b/sio/executors/unsafe_exec.py @@ -1,7 +1,10 @@ from __future__ import absolute_import -from sio.executors import common +from sio.executors import common, interactive_common from sio.workers.executors import DetailedUnprotectedExecutor def run(environ): return common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) + +def interactive_run(environ): + return interactive_common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 7119fac..1ef2286 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -79,7 +79,9 @@ def execute_command( real_time_limit=None, ignore_errors=False, extra_ignore_errors=(), - **kwargs + cwd=None, + fds_to_close=(), + **kwargs, ): """Utility function to run arbitrary command. ``stdin`` @@ -123,8 +125,9 @@ def execute_command( devnull = open(os.devnull, 'wb') stdout = stdout or devnull stderr = stderr or devnull - + cwd = cwd or tempcwd() ret_env = {} + if env is not None: for key, value in six.iteritems(env): env[key] = str(value) @@ -139,10 +142,13 @@ def execute_command( close_fds=True, universal_newlines=True, env=env, - cwd=tempcwd(), + cwd=cwd, preexec_fn=os.setpgrp, ) + for fd in fds_to_close: + os.close(fd) + kill_timer = None if real_time_limit: @@ -180,7 +186,6 @@ def oot_killer(): raise ExecError( 'Failed to execute command: %s. Returned with code %s\n' % (command, rc) ) - return ret_env @@ -417,9 +422,8 @@ def _execute(self, command, **kwargs): renv['result_string'] = 'ok' renv['result_code'] = 'OK' elif renv['return_code'] > 128: # os.WIFSIGNALED(1) returns True - renv['result_string'] = 'program exited due to signal %d' % os.WTERMSIG( - renv['return_code'] - ) + renv['exit_signal'] = os.WTERMSIG(renv['return_code']) + renv['result_string'] = 'program exited due to signal %d' % renv['exit_signal'] renv['result_code'] = 'RE' else: renv['result_string'] = 'program exited with code %d' % renv['return_code'] @@ -669,6 +673,9 @@ def _execute(self, command, **kwargs): renv['result_code'] = 'RV' elif renv['result_string'].startswith('process exited due to signal'): renv['result_code'] = 'RE' + renv['exit_signal'] = int( + renv['result_string'][len('process exited due to signal '):] + ) else: raise ExecError( 'Unrecognized Sio2Jail result string: %s' % renv['result_string']