From ba3eb723f0351552e1c862c93dadad4b6ec56496 Mon Sep 17 00:00:00 2001 From: tkwiatkowski Date: Wed, 3 Apr 2024 22:46:39 +0200 Subject: [PATCH 01/14] Temporary changes in sioworkers for interactive problems --- setup.py | 4 + sio/executors/executor.py | 5 +- sio/executors/interactive_common.py | 180 ++++++++++++++++++++++++++++ sio/executors/sio2jail_exec.py | 5 +- sio/executors/unsafe_exec.py | 5 +- sio/workers/executors.py | 11 +- 6 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 sio/executors/interactive_common.py diff --git a/setup.py b/setup.py index eae7f98..1b55441 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,13 @@ 'ping = sio.workers.ping:run', 'compile = sio.compilers.job:run', 'exec = sio.executors.executor:run', + 'interactive-exec = sio.executors.executor:interactive_run', 'sio2jail-exec = sio.executors.sio2jail_exec:run', + 'sio2jail-interactive-exec = sio.executors.sio2jail_exec:interactive_run', 'cpu-exec = sio.executors.executor:run', + 'cpu-interactive-exec = sio.executors.executor:interactive_run', 'unsafe-exec = sio.executors.unsafe_exec:run', + 'unsafe-interactive-exec = sio.executors.unsafe_exec:interactive_run', 'ingen = sio.executors.ingen:run', 'inwer = sio.executors.inwer:run', ], diff --git a/sio/executors/executor.py b/sio/executors/executor.py index 40ce9af..3f73f11 100644 --- a/sio/executors/executor.py +++ b/sio/executors/executor.py @@ -1,7 +1,10 @@ from __future__ import absolute_import -from sio.executors import common +from sio.executors import common, interactive_common from sio.workers.executors import SupervisedExecutor def run(environ): return common.run(environ, SupervisedExecutor()) + +def interactive_run(environ): + return interactive_common.run(environ, SupervisedExecutor()) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py new file mode 100644 index 0000000..15da6fb --- /dev/null +++ b/sio/executors/interactive_common.py @@ -0,0 +1,180 @@ +from __future__ import absolute_import +import os +from shutil import rmtree +from threading import Thread +from zipfile import ZipFile, is_zipfile +from sio.workers import ft +from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd +from sio.workers.file_runners import get_file_runner + +from sio.executors import checker +import signal +import six + + +def _populate_environ(renv, environ): + """Takes interesting fields from renv into environ""" + for key in ('time_used', 'mem_used', 'num_syscalls'): + environ[key] = renv.get(key, 0) + for key in ('result_code', 'result_string'): + environ[key] = renv.get(key, '') + if 'out_file' in renv: + environ['out_file'] = renv['out_file'] + + +@decode_fields(['result_string']) +def run(environ, executor, use_sandboxes=True): + """ + Common code for executors. + + :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` + For all supported options, see the global documentation for + `sio.workers.executors` and prefix them with ``exec_``. + :param: executor Executor instance used for executing commands. + :param: use_sandboxes Enables safe checking output correctness. + See `sio.executors.checkers`. True by default. + """ + + if environ.get('exec_info', {}).get('mode') == 'output-only': + renv = _fake_run_as_exe_is_output_file(environ) + else: + renv = _run(environ, executor, use_sandboxes) + + _populate_environ(renv, environ) + + if environ['result_code'] == 'OK' and environ.get('check_output'): + environ = checker.run(environ, use_sandboxes=use_sandboxes) + + for key in ('result_code', 'result_string'): + environ[key] = replace_invalid_UTF(environ[key]) + + if 'out_file' in environ: + ft.upload( + environ, + 'out_file', + tempcwd('out'), + to_remote_store=environ.get('upload_out', False), + ) + + return environ + + +def _run(environ, executor, use_sandboxes): + input_name = tempcwd('in') + + file_executor = get_file_runner(executor, environ) + interactor_env = environ.copy() + interactor_env['exe_file'] = interactor_env['interactor_file'] + interactor_executor = get_file_runner(executor, interactor_env) + exe_filename = file_executor.preferred_filename() + interactor_filename = 'soc' + + ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) + os.chmod(tempcwd(exe_filename), 0o700) + ft.download(environ, 'interactor_file', interactor_filename, add_to_cache=True) + os.chmod(tempcwd(interactor_filename), 0o700) + ft.download(environ, 'in_file', input_name, add_to_cache=True) + ft.download(environ, 'out_file', 'out', skip_if_exists=True) + ft.download(environ, 'hint_file', 'hint', add_to_cache=True) + + zipdir = tempcwd('in_dir') + os.mkdir(zipdir) + try: + if is_zipfile(input_name): + try: + # If not a zip file, will pass it directly to exe + with ZipFile(tempcwd('in'), 'r') as f: + if len(f.namelist()) != 1: + raise Exception("Archive should have only one file.") + + f.extract(f.namelist()[0], zipdir) + input_name = os.path.join(zipdir, f.namelist()[0]) + # zipfile throws some undocumented exceptions + except Exception as e: + raise Exception("Failed to open archive: " + six.text_type(e)) + + r1, w1 = os.pipe() + r2, w2 = os.pipe() + os.set_inheritable(r1, True) + os.set_inheritable(w1, True) + os.set_inheritable(r2, True) + os.set_inheritable(w2, True) + + irenv = {} + renv = {} + interactor_command = [tempcwd(interactor_filename), input_name, 'out', 'hint'] + with interactor_executor as ie: + with open(tempcwd('out'), 'ab') as outf: + interactor = Thread( + target=ie, + args=( + interactor_command, + [], + ), + kwargs=dict( + stdin=r2, + stdout=w1, + stderr=outf, + ignore_errors=True, + environ=environ, + environ_prefix='exec_', + pass_fds=(r2, w1), + wait=False, + ret_env=irenv, + ) + ) + + with file_executor as fe: + exe = Thread( + target=fe, + args=( + tempcwd(exe_filename), + [], + ), + kwargs=dict( + stdin=r1, + stdout=w2, + ignore_errors=True, + environ=environ, + environ_prefix='exec_', + pass_fds=(r1, w2), + ret_env=renv, + ) + ) + + os.close(r1) + os.close(w1) + os.close(r2) + os.close(w2) + exe.start() + interactor.start() + exe.join() + interactor.join() + + sol_sig = renv.get('exit_signal', None) + inter_sig = irenv.get('exit_signal', None) + sigpipe = signal.SIGPIPE.value + + if sol_sig == sigpipe: + renv['result_code'] = 'SE' + renv['result_string'] = 'Checker exited prematurely. ' + elif inter_sig == sigpipe: + renv['result_code'] = 'SE' + renv['result_string'] = 'Solution exited prematurely' + finally: + rmtree(zipdir) + + return renv + + +def _fake_run_as_exe_is_output_file(environ): + # later code expects 'out' file to be present after compilation + ft.download(environ, 'exe_file', tempcwd('out')) + return { + # copy filetracker id of 'exe_file' as 'out_file' (thanks to that checker will grab it) + 'out_file': environ['exe_file'], + # 'result_code' is left by executor, as executor is not used + # this variable has to be set manually + 'result_code': 'OK', + 'result_string': 'ok', + } diff --git a/sio/executors/sio2jail_exec.py b/sio/executors/sio2jail_exec.py index ac23e4a..d8771e0 100644 --- a/sio/executors/sio2jail_exec.py +++ b/sio/executors/sio2jail_exec.py @@ -1,6 +1,9 @@ -from sio.executors import common +from sio.executors import common, interactive_common from sio.workers.executors import Sio2JailExecutor def run(environ): return common.run(environ, Sio2JailExecutor()) + +def interactive_run(environ): + return interactive_common.run(environ, Sio2JailExecutor()) diff --git a/sio/executors/unsafe_exec.py b/sio/executors/unsafe_exec.py index 3b0ae38..fcd6c61 100644 --- a/sio/executors/unsafe_exec.py +++ b/sio/executors/unsafe_exec.py @@ -1,7 +1,10 @@ from __future__ import absolute_import -from sio.executors import common +from sio.executors import common, interactive_common from sio.workers.executors import DetailedUnprotectedExecutor def run(environ): return common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) + +def run_interactive(environ): + return interactive_common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 7119fac..3385087 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -79,6 +79,7 @@ def execute_command( real_time_limit=None, ignore_errors=False, extra_ignore_errors=(), + ret_env=None, **kwargs ): """Utility function to run arbitrary command. @@ -124,7 +125,7 @@ def execute_command( stdout = stdout or devnull stderr = stderr or devnull - ret_env = {} + ret_env = ret_env or {} if env is not None: for key, value in six.iteritems(env): env[key] = str(value) @@ -417,9 +418,8 @@ def _execute(self, command, **kwargs): renv['result_string'] = 'ok' renv['result_code'] = 'OK' elif renv['return_code'] > 128: # os.WIFSIGNALED(1) returns True - renv['result_string'] = 'program exited due to signal %d' % os.WTERMSIG( - renv['return_code'] - ) + renv['exit_signal'] = os.WTERMSIG(renv['return_code']) + renv['result_string'] = 'program exited due to signal %d' % renv['exit_signal'] renv['result_code'] = 'RE' else: renv['result_string'] = 'program exited with code %d' % renv['return_code'] @@ -669,6 +669,9 @@ def _execute(self, command, **kwargs): renv['result_code'] = 'RV' elif renv['result_string'].startswith('process exited due to signal'): renv['result_code'] = 'RE' + renv['exit_signal'] = int( + renv['result_string'][len('process exited due to signal '):] + ) else: raise ExecError( 'Unrecognized Sio2Jail result string: %s' % renv['result_string'] From 264a057151212dfff220f2f34f6e48d4e5477963 Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Sat, 6 Apr 2024 20:33:52 +0200 Subject: [PATCH 02/14] Some fixes --- sio/executors/interactive_common.py | 3 +-- sio/executors/unsafe_exec.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 15da6fb..5c449bf 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -74,7 +74,6 @@ def _run(environ, executor, use_sandboxes): ft.download(environ, 'interactor_file', interactor_filename, add_to_cache=True) os.chmod(tempcwd(interactor_filename), 0o700) ft.download(environ, 'in_file', input_name, add_to_cache=True) - ft.download(environ, 'out_file', 'out', skip_if_exists=True) ft.download(environ, 'hint_file', 'hint', add_to_cache=True) zipdir = tempcwd('in_dir') @@ -102,7 +101,7 @@ def _run(environ, executor, use_sandboxes): irenv = {} renv = {} - interactor_command = [tempcwd(interactor_filename), input_name, 'out', 'hint'] + interactor_command = [tempcwd(interactor_filename), input_name, 'in', 'hint'] with interactor_executor as ie: with open(tempcwd('out'), 'ab') as outf: interactor = Thread( diff --git a/sio/executors/unsafe_exec.py b/sio/executors/unsafe_exec.py index fcd6c61..0d1242d 100644 --- a/sio/executors/unsafe_exec.py +++ b/sio/executors/unsafe_exec.py @@ -6,5 +6,5 @@ def run(environ): return common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) -def run_interactive(environ): +def interactive_run(environ): return interactive_common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) From b9e37556d3a26982e19204e0a9740293e835a157 Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Sun, 7 Apr 2024 09:53:31 +0200 Subject: [PATCH 03/14] Fixes in interactive_common --- sio/executors/interactive_common.py | 38 +++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 5c449bf..b7845db 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -7,10 +7,11 @@ from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd from sio.workers.file_runners import get_file_runner -from sio.executors import checker import signal import six +RESULT_STRING_LENGTH_LIMIT = 1024 # in bytes + def _populate_environ(renv, environ): """Takes interesting fields from renv into environ""" @@ -22,6 +23,13 @@ def _populate_environ(renv, environ): environ['out_file'] = renv['out_file'] +def _limit_length(s): + if len(s) > RESULT_STRING_LENGTH_LIMIT: + suffix = b'[...]' + return s[: max(0, RESULT_STRING_LENGTH_LIMIT - len(suffix))] + suffix + return s + + @decode_fields(['result_string']) def run(environ, executor, use_sandboxes=True): """ @@ -42,8 +50,8 @@ def run(environ, executor, use_sandboxes=True): _populate_environ(renv, environ) - if environ['result_code'] == 'OK' and environ.get('check_output'): - environ = checker.run(environ, use_sandboxes=use_sandboxes) + # if environ['result_code'] == 'OK' and environ.get('check_output'): + # environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) @@ -74,7 +82,6 @@ def _run(environ, executor, use_sandboxes): ft.download(environ, 'interactor_file', interactor_filename, add_to_cache=True) os.chmod(tempcwd(interactor_filename), 0o700) ft.download(environ, 'in_file', input_name, add_to_cache=True) - ft.download(environ, 'hint_file', 'hint', add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) @@ -101,14 +108,13 @@ def _run(environ, executor, use_sandboxes): irenv = {} renv = {} - interactor_command = [tempcwd(interactor_filename), input_name, 'in', 'hint'] + interactor_command = [tempcwd(interactor_filename), input_name, 'in'] with interactor_executor as ie: with open(tempcwd('out'), 'ab') as outf: interactor = Thread( target=ie, args=( interactor_command, - [], ), kwargs=dict( stdin=r2, @@ -118,7 +124,6 @@ def _run(environ, executor, use_sandboxes): environ=environ, environ_prefix='exec_', pass_fds=(r2, w1), - wait=False, ret_env=irenv, ) ) @@ -128,7 +133,6 @@ def _run(environ, executor, use_sandboxes): target=fe, args=( tempcwd(exe_filename), - [], ), kwargs=dict( stdin=r1, @@ -149,17 +153,31 @@ def _run(environ, executor, use_sandboxes): interactor.start() exe.join() interactor.join() - + + with open(tempcwd('out'), 'ab') as outf: + interactor_out = outf.readlines() + sol_sig = renv.get('exit_signal', None) inter_sig = irenv.get('exit_signal', None) sigpipe = signal.SIGPIPE.value - + if sol_sig == sigpipe: renv['result_code'] = 'SE' renv['result_string'] = 'Checker exited prematurely. ' elif inter_sig == sigpipe: renv['result_code'] = 'SE' renv['result_string'] = 'Solution exited prematurely' + else: + if six.ensure_binary(interactor_out[0]) == b'OK': + environ['result_code'] = 'OK' + if interactor_out[1]: + environ['result_string'] = _limit_length(interactor_out[1]) + environ['result_percentage'] = float(interactor_out[2] or 100) + else: + environ['result_code'] = 'WA' + if interactor_out[1]: + environ['result_string'] = _limit_length(interactor_out[1]) + environ['result_percentage'] = 0 finally: rmtree(zipdir) From 596b862bd5adb7a38e27166f51beebd1dc8ae5db Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Sun, 7 Apr 2024 11:50:09 +0200 Subject: [PATCH 04/14] gaming --- sio/executors/interactive_common.py | 107 ++++++++++++++++++++++------ sio/workers/executors.py | 1 + 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index b7845db..caf17f2 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -4,11 +4,15 @@ from threading import Thread from zipfile import ZipFile, is_zipfile from sio.workers import ft -from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd +from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd, TemporaryCwd from sio.workers.file_runners import get_file_runner import signal import six +import traceback + +import logging +logger = logging.getLogger(__name__) RESULT_STRING_LENGTH_LIMIT = 1024 # in bytes @@ -106,78 +110,135 @@ def _run(environ, executor, use_sandboxes): os.set_inheritable(r2, True) os.set_inheritable(w2, True) - irenv = {} - renv = {} - interactor_command = [tempcwd(interactor_filename), input_name, 'in'] + interactor_res = [] + sol_res = [] + interactor_args = [input_name] + logger.info(str(interactor_executor)) + logger.info(str(file_executor)) + + def thread_wrapper(result, executor, *args, **kwargs): + logger.info("thread_wrapper: " + str(result) + " " + str(executor) + " " + str(args) + " " + str(kwargs)) + try: + with TemporaryCwd(): + res = executor(*args, **kwargs) + logger.info("result: " + str(res)) + result.append(res) + except Exception as e: + logger.error(traceback.format_exc()) + logger.error("Exception in thread_wrapper: " + str(e)) + with interactor_executor as ie: - with open(tempcwd('out'), 'ab') as outf: + logger.info(tempcwd('out')) + with open(tempcwd('out'), 'wb') as outf: interactor = Thread( - target=ie, + target=thread_wrapper, args=( - interactor_command, + interactor_res, + ie, + tempcwd(interactor_filename), + interactor_args, ), - kwargs=dict( + kwargs = dict( stdin=r2, stdout=w1, stderr=outf, - ignore_errors=True, + ignore_errors=False, + forward_stderr=True, environ=environ, environ_prefix='exec_', pass_fds=(r2, w1), - ret_env=irenv, ) ) + # interactor = Thread( + # target=ie, + # args=( + # interactor_command, + # ), + # kwargs=dict( + # stdin=r2, + # stdout=w1, + # stderr=outf, + # ignore_errors=False, + # environ=environ, + # environ_prefix='exec_', + # pass_fds=(r2, w1), + # ) + # ) with file_executor as fe: exe = Thread( - target=fe, + target=thread_wrapper, args=( + sol_res, + fe, tempcwd(exe_filename), + [], ), kwargs=dict( stdin=r1, stdout=w2, - ignore_errors=True, + ignore_errors=False, environ=environ, environ_prefix='exec_', pass_fds=(r1, w2), - ret_env=renv, ) ) + # exe = Thread( + # target=fe, + # args=( + # tempcwd(exe_filename), + # ), + # kwargs=dict( + # stdin=r1, + # stdout=w2, + # ignore_errors=False, + # environ=environ, + # environ_prefix='exec_', + # pass_fds=(r1, w2), + # ret_env=renv, + # ) + # ) + exe.start() + interactor.start() os.close(r1) os.close(w1) os.close(r2) os.close(w2) - exe.start() - interactor.start() exe.join() interactor.join() + irenv = interactor_res[0] + renv = sol_res[0] - with open(tempcwd('out'), 'ab') as outf: + logger.info(tempcwd('out')) + with open(tempcwd('out'), 'rb') as outf: interactor_out = outf.readlines() + logger.info(str(interactor_out)) + logger.info("irenv: " + str(irenv)) + logger.info("renv: " + str(renv)) + sol_sig = renv.get('exit_signal', None) inter_sig = irenv.get('exit_signal', None) sigpipe = signal.SIGPIPE.value if sol_sig == sigpipe: renv['result_code'] = 'SE' - renv['result_string'] = 'Checker exited prematurely. ' + renv['result_string'] = 'Checker exited prematurely. ' elif inter_sig == sigpipe: renv['result_code'] = 'SE' renv['result_string'] = 'Solution exited prematurely' else: if six.ensure_binary(interactor_out[0]) == b'OK': - environ['result_code'] = 'OK' + renv['result_code'] = 'OK' if interactor_out[1]: - environ['result_string'] = _limit_length(interactor_out[1]) - environ['result_percentage'] = float(interactor_out[2] or 100) + renv['result_string'] = _limit_length(interactor_out[1]) + renv['result_percentage'] = float(interactor_out[2] or 100) else: - environ['result_code'] = 'WA' + renv['result_code'] = 'WA' if interactor_out[1]: - environ['result_string'] = _limit_length(interactor_out[1]) - environ['result_percentage'] = 0 + renv['result_string'] = _limit_length(interactor_out[1]) + renv['result_percentage'] = 0 finally: rmtree(zipdir) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 3385087..8c97542 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -131,6 +131,7 @@ def execute_command( env[key] = str(value) perf_timer = util.PerfTimer() + logger.info("stderr") p = subprocess.Popen( command, stdin=stdin, From 4c3f58dba759eccb110f380f39140e2e864012d4 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Sun, 7 Apr 2024 14:14:32 +0200 Subject: [PATCH 05/14] very gaming --- sio/executors/interactive_common.py | 169 ++++++++++++++++------------ sio/workers/executors.py | 18 ++- 2 files changed, 111 insertions(+), 76 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index caf17f2..529d66d 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -105,65 +105,85 @@ def _run(environ, executor, use_sandboxes): r1, w1 = os.pipe() r2, w2 = os.pipe() - os.set_inheritable(r1, True) - os.set_inheritable(w1, True) - os.set_inheritable(r2, True) - os.set_inheritable(w2, True) - - interactor_res = [] - sol_res = [] - interactor_args = [input_name] + for fd in (r1, w1, r2, w2): + os.set_inheritable(fd, True) + + interactor_args = [input_name, tempcwd('out')] logger.info(str(interactor_executor)) logger.info(str(file_executor)) + class InteractiveTaskError(Exception): + def __init__(self, exception, *args, **kwargs): + self.msg = "Interactive task failed: " + str(exception) + "\n" + \ + "args: " + str(args) + "\n" + \ + "kwargs: " + str(kwargs) + "\n" + \ + traceback.format_exc() + + class WrapperResult: + def __init__(self): + self.res = None + self.args = None + self.kwargs = None + self.exception = None + self.process_started = False + + def entry(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def set_result(self, res): + self.res = res + + def get_result(self): + return self.res + + def set_exception(self, e): + self.exception = InteractiveTaskError(e, self.args, self.kwargs) + + def has_exception(self): + return self.exception is not None + + def get_exception(self): + return self.exception + + def set_started(self): + self.process_started = True + + interactor_res = WrapperResult() + sol_res = WrapperResult() + def thread_wrapper(result, executor, *args, **kwargs): - logger.info("thread_wrapper: " + str(result) + " " + str(executor) + " " + str(args) + " " + str(kwargs)) + result.entry(*args, **kwargs) try: - with TemporaryCwd(): - res = executor(*args, **kwargs) - logger.info("result: " + str(res)) - result.append(res) + logger.info("thread_wrapper: " + str(kwargs['in_file']) + " " + str(executor) + " " + str(args) + " " + str(kwargs)) + res = executor(*args, **kwargs) + result.set_result(res) except Exception as e: - logger.error(traceback.format_exc()) - logger.error("Exception in thread_wrapper: " + str(e)) + result.set_exception(e) + logger.error("gaming " + str(kwargs['in_file']) + "\t" +InteractiveTaskError(e, *args, **kwargs).msg) with interactor_executor as ie: logger.info(tempcwd('out')) - with open(tempcwd('out'), 'wb') as outf: - interactor = Thread( - target=thread_wrapper, - args=( - interactor_res, - ie, - tempcwd(interactor_filename), - interactor_args, - ), - kwargs = dict( - stdin=r2, - stdout=w1, - stderr=outf, - ignore_errors=False, - forward_stderr=True, - environ=environ, - environ_prefix='exec_', - pass_fds=(r2, w1), - ) + interactor = Thread( + target=thread_wrapper, + args=( + interactor_res, + ie, + tempcwd(interactor_filename), + interactor_args, + ), + kwargs = dict( + stdin=r2, + stdout=w1, + ignore_errors=False, + environ=environ, + environ_prefix='exec_', + pass_fds=(r2, w1), + cwd=tempcwd(), + process_status=interactor_res, + in_file=environ['in_file'], ) - # interactor = Thread( - # target=ie, - # args=( - # interactor_command, - # ), - # kwargs=dict( - # stdin=r2, - # stdout=w1, - # stderr=outf, - # ignore_errors=False, - # environ=environ, - # environ_prefix='exec_', - # pass_fds=(r2, w1), - # ) - # ) + ) with file_executor as fe: exe = Thread( @@ -181,42 +201,45 @@ def thread_wrapper(result, executor, *args, **kwargs): environ=environ, environ_prefix='exec_', pass_fds=(r1, w2), + cwd=tempcwd(), + process_status=sol_res, + in_file=environ['in_file'], ) ) - # exe = Thread( - # target=fe, - # args=( - # tempcwd(exe_filename), - # ), - # kwargs=dict( - # stdin=r1, - # stdout=w2, - # ignore_errors=False, - # environ=environ, - # environ_prefix='exec_', - # pass_fds=(r1, w2), - # ret_env=renv, - # ) - # ) + logger.info("Starting threads " + environ['in_file']) exe.start() + logger.info("Started exe " + environ['in_file']) interactor.start() - os.close(r1) - os.close(w1) - os.close(r2) - os.close(w2) + logger.info("Started interactor " + environ['in_file']) + + # Very beautiful hack + while not interactor_res.process_started or not sol_res.process_started: + pass + for fd in (r1, w1, r2, w2): + os.close(fd) + logger.info("Closed fds " + environ['in_file']) + exe.join() + logger.info("exe joined " + environ['in_file']) interactor.join() - irenv = interactor_res[0] - renv = sol_res[0] + logger.info("interactor joined " + environ['in_file']) + + if interactor_res.has_exception(): + raise interactor_res.get_exception() + if sol_res.has_exception(): + raise sol_res.get_exception() + + irenv = interactor_res.get_result() + renv = sol_res.get_result() logger.info(tempcwd('out')) with open(tempcwd('out'), 'rb') as outf: interactor_out = outf.readlines() - logger.info(str(interactor_out)) - logger.info("irenv: " + str(irenv)) - logger.info("renv: " + str(renv)) + # logger.info(str(interactor_out)) + # logger.info("irenv: " + str(irenv)) + # logger.info("renv: " + str(renv)) sol_sig = renv.get('exit_signal', None) inter_sig = irenv.get('exit_signal', None) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 8c97542..dbf8993 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -80,7 +80,10 @@ def execute_command( ignore_errors=False, extra_ignore_errors=(), ret_env=None, - **kwargs + pass_fds=(), + cwd=None, + process_status=None, + **kwargs, ): """Utility function to run arbitrary command. ``stdin`` @@ -124,6 +127,7 @@ def execute_command( devnull = open(os.devnull, 'wb') stdout = stdout or devnull stderr = stderr or devnull + cwd = cwd or tempcwd() ret_env = ret_env or {} if env is not None: @@ -131,19 +135,27 @@ def execute_command( env[key] = str(value) perf_timer = util.PerfTimer() - logger.info("stderr") + logger.info("stderr: " + str(forward_stderr) + " " + str(subprocess.STDOUT) + " " + str(stderr) + " " + str(forward_stderr and subprocess.STDOUT or stderr) + "\n" + \ + "stdout: " + str(stdout) + "\n" + \ + "stdin: " + str(stdin) + "\n" ) + logger.info("pass fds: " + str(pass_fds)) + logger.info("cwd: " + cwd) + logger.info(str(os.listdir(cwd))) p = subprocess.Popen( command, stdin=stdin, stdout=stdout, stderr=forward_stderr and subprocess.STDOUT or stderr, + pass_fds=pass_fds, shell=True, close_fds=True, universal_newlines=True, env=env, - cwd=tempcwd(), + cwd=cwd, preexec_fn=os.setpgrp, ) + if process_status: + process_status.set_started() kill_timer = None if real_time_limit: From 1a8a424f1d86b1c679b369b4d593acc8da11b37c Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Sun, 7 Apr 2024 19:53:25 +0200 Subject: [PATCH 06/14] Strip interactor_out and improve sigpipe handling --- sio/executors/interactive_common.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 529d66d..cbc87c2 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -176,6 +176,7 @@ def thread_wrapper(result, executor, *args, **kwargs): stdin=r2, stdout=w1, ignore_errors=False, + extra_ignore_errors=(141,), # SIGPIPE environ=environ, environ_prefix='exec_', pass_fds=(r2, w1), @@ -198,6 +199,7 @@ def thread_wrapper(result, executor, *args, **kwargs): stdin=r1, stdout=w2, ignore_errors=False, + extra_ignore_errors=(141,), # SIGPIPE environ=environ, environ_prefix='exec_', pass_fds=(r1, w2), @@ -234,8 +236,8 @@ def thread_wrapper(result, executor, *args, **kwargs): renv = sol_res.get_result() logger.info(tempcwd('out')) - with open(tempcwd('out'), 'rb') as outf: - interactor_out = outf.readlines() + with open(tempcwd('out'), 'rb') as result_file: + interactor_out = [line.rstrip() for line in result_file.readlines()] # logger.info(str(interactor_out)) # logger.info("irenv: " + str(irenv)) @@ -245,12 +247,12 @@ def thread_wrapper(result, executor, *args, **kwargs): inter_sig = irenv.get('exit_signal', None) sigpipe = signal.SIGPIPE.value - if sol_sig == sigpipe: + if sol_sig == sigpipe and not interactor_out: renv['result_code'] = 'SE' - renv['result_string'] = 'Checker exited prematurely. ' + renv['result_string'] = 'checker exited prematurely' elif inter_sig == sigpipe: - renv['result_code'] = 'SE' - renv['result_string'] = 'Solution exited prematurely' + renv['result_code'] = 'WA' + renv['result_string'] = 'solution exited prematurely' else: if six.ensure_binary(interactor_out[0]) == b'OK': renv['result_code'] = 'OK' From 251dd2eef08180dde5ec94ac67384c6af422826b Mon Sep 17 00:00:00 2001 From: tkwiatkowski Date: Tue, 9 Apr 2024 21:26:10 +0200 Subject: [PATCH 07/14] Get rid of some Hacks --- sio/executors/interactive_common.py | 152 +++++++++------------------- sio/workers/executors.py | 13 ++- 2 files changed, 55 insertions(+), 110 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index cbc87c2..3af5313 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -71,6 +71,39 @@ def run(environ, executor, use_sandboxes=True): return environ +def _fill_result(renv, irenv, interactor_out): + sol_sig = renv.get('exit_signal', None) + inter_sig = irenv.get('exit_signal', None) + sigpipe = signal.SIGPIPE.value + + logger.info(renv) + logger.info(irenv) + + if irenv['result_code'] != 'OK' and inter_sig != sigpipe: + renv['result_code'] = 'SE' + # renv['result_string'] = 'checker exited prematurely' + elif renv['result_code'] != 'OK' and sol_sig != sigpipe: + return + elif len(interactor_out) == 0: + renv['result_code'] = 'SE' + renv['result_string'] = 'invalid interactor output' + elif inter_sig == sigpipe: + renv['result_code'] = 'WA' + renv['result_string'] = 'solution exited prematurely' + else: + renv['result_string'] = '' + if six.ensure_binary(interactor_out[0]) == b'OK': + renv['result_code'] = 'OK' + if interactor_out[1]: + renv['result_string'] = _limit_length(interactor_out[1]) + renv['result_percentage'] = float(interactor_out[2] or 100) + else: + renv['result_code'] = 'WA' + if interactor_out[1]: + renv['result_string'] = _limit_length(interactor_out[1]) + renv['result_percentage'] = 0 + + def _run(environ, executor, use_sandboxes): input_name = tempcwd('in') @@ -112,158 +145,65 @@ def _run(environ, executor, use_sandboxes): logger.info(str(interactor_executor)) logger.info(str(file_executor)) - class InteractiveTaskError(Exception): - def __init__(self, exception, *args, **kwargs): - self.msg = "Interactive task failed: " + str(exception) + "\n" + \ - "args: " + str(args) + "\n" + \ - "kwargs: " + str(kwargs) + "\n" + \ - traceback.format_exc() - - class WrapperResult: - def __init__(self): - self.res = None - self.args = None - self.kwargs = None - self.exception = None - self.process_started = False - - def entry(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - - def set_result(self, res): - self.res = res - - def get_result(self): - return self.res - - def set_exception(self, e): - self.exception = InteractiveTaskError(e, self.args, self.kwargs) - - def has_exception(self): - return self.exception is not None - - def get_exception(self): - return self.exception - - def set_started(self): - self.process_started = True - - interactor_res = WrapperResult() - sol_res = WrapperResult() - - def thread_wrapper(result, executor, *args, **kwargs): - result.entry(*args, **kwargs) - try: - logger.info("thread_wrapper: " + str(kwargs['in_file']) + " " + str(executor) + " " + str(args) + " " + str(kwargs)) - res = executor(*args, **kwargs) - result.set_result(res) - except Exception as e: - result.set_exception(e) - logger.error("gaming " + str(kwargs['in_file']) + "\t" +InteractiveTaskError(e, *args, **kwargs).msg) + irenv = {} + renv = {} with interactor_executor as ie: logger.info(tempcwd('out')) interactor = Thread( - target=thread_wrapper, + target=ie, args=( - interactor_res, - ie, tempcwd(interactor_filename), interactor_args, ), kwargs = dict( stdin=r2, stdout=w1, - ignore_errors=False, - extra_ignore_errors=(141,), # SIGPIPE + ignore_errors=True, environ=environ, environ_prefix='exec_', pass_fds=(r2, w1), + close_passed_fd=True, cwd=tempcwd(), - process_status=interactor_res, in_file=environ['in_file'], + ret_env=irenv, ) ) with file_executor as fe: exe = Thread( - target=thread_wrapper, + target=fe, args=( - sol_res, - fe, tempcwd(exe_filename), [], ), kwargs=dict( stdin=r1, stdout=w2, - ignore_errors=False, - extra_ignore_errors=(141,), # SIGPIPE + ignore_errors=True, environ=environ, environ_prefix='exec_', pass_fds=(r1, w2), + close_passed_fd=True, cwd=tempcwd(), - process_status=sol_res, in_file=environ['in_file'], + ret_env=renv, ) ) - logger.info("Starting threads " + environ['in_file']) exe.start() - logger.info("Started exe " + environ['in_file']) interactor.start() - logger.info("Started interactor " + environ['in_file']) - - # Very beautiful hack - while not interactor_res.process_started or not sol_res.process_started: - pass - for fd in (r1, w1, r2, w2): - os.close(fd) - logger.info("Closed fds " + environ['in_file']) exe.join() - logger.info("exe joined " + environ['in_file']) interactor.join() - logger.info("interactor joined " + environ['in_file']) - - if interactor_res.has_exception(): - raise interactor_res.get_exception() - if sol_res.has_exception(): - raise sol_res.get_exception() - - irenv = interactor_res.get_result() - renv = sol_res.get_result() - logger.info(tempcwd('out')) with open(tempcwd('out'), 'rb') as result_file: interactor_out = [line.rstrip() for line in result_file.readlines()] - # logger.info(str(interactor_out)) - # logger.info("irenv: " + str(irenv)) - # logger.info("renv: " + str(renv)) + while len(interactor_out) < 3: + interactor_out.append('') - sol_sig = renv.get('exit_signal', None) - inter_sig = irenv.get('exit_signal', None) - sigpipe = signal.SIGPIPE.value - - if sol_sig == sigpipe and not interactor_out: - renv['result_code'] = 'SE' - renv['result_string'] = 'checker exited prematurely' - elif inter_sig == sigpipe: - renv['result_code'] = 'WA' - renv['result_string'] = 'solution exited prematurely' - else: - if six.ensure_binary(interactor_out[0]) == b'OK': - renv['result_code'] = 'OK' - if interactor_out[1]: - renv['result_string'] = _limit_length(interactor_out[1]) - renv['result_percentage'] = float(interactor_out[2] or 100) - else: - renv['result_code'] = 'WA' - if interactor_out[1]: - renv['result_string'] = _limit_length(interactor_out[1]) - renv['result_percentage'] = 0 + _fill_result(renv, irenv, interactor_out) finally: rmtree(zipdir) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index dbf8993..6d043b0 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -81,8 +81,8 @@ def execute_command( extra_ignore_errors=(), ret_env=None, pass_fds=(), + close_passed_fd=False, cwd=None, - process_status=None, **kwargs, ): """Utility function to run arbitrary command. @@ -129,7 +129,8 @@ def execute_command( stderr = stderr or devnull cwd = cwd or tempcwd() - ret_env = ret_env or {} + if ret_env is None: + ret_env = {} if env is not None: for key, value in six.iteritems(env): env[key] = str(value) @@ -154,8 +155,10 @@ def execute_command( cwd=cwd, preexec_fn=os.setpgrp, ) - if process_status: - process_status.set_started() + + if close_passed_fd: + for fd in pass_fds: + os.close(fd) kill_timer = None if real_time_limit: @@ -195,6 +198,8 @@ def oot_killer(): 'Failed to execute command: %s. Returned with code %s\n' % (command, rc) ) + logger.info(ret_env) + return ret_env From fa654834153a87af9ebe7bf9c66ca3634e0cbaf5 Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Tue, 9 Apr 2024 23:24:16 +0200 Subject: [PATCH 08/14] Respect mem and time limit for interactor --- sio/executors/common.py | 30 +++++++++++++---------- sio/executors/interactive_common.py | 37 +++++++---------------------- sio/workers/executors.py | 12 ++-------- 3 files changed, 29 insertions(+), 50 deletions(-) diff --git a/sio/executors/common.py b/sio/executors/common.py index 0da4f35..462d626 100644 --- a/sio/executors/common.py +++ b/sio/executors/common.py @@ -20,6 +20,23 @@ def _populate_environ(renv, environ): environ['out_file'] = renv['out_file'] +def _extract_input_if_zipfile(input_name, zipdir): + if is_zipfile(input_name): + try: + # If not a zip file, will pass it directly to exe + with ZipFile(tempcwd('in'), 'r') as f: + if len(f.namelist()) != 1: + raise Exception("Archive should have only one file.") + + f.extract(f.namelist()[0], zipdir) + input_name = os.path.join(zipdir, f.namelist()[0]) + # zipfile throws some undocumented exceptions + except Exception as e: + raise Exception("Failed to open archive: " + six.text_type(e)) + + return input_name + + @decode_fields(['result_string']) def run(environ, executor, use_sandboxes=True): """ @@ -70,18 +87,7 @@ def _run(environ, executor, use_sandboxes): zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: - if is_zipfile(input_name): - try: - # If not a zip file, will pass it directly to exe - with ZipFile(tempcwd('in'), 'r') as f: - if len(f.namelist()) != 1: - raise Exception("Archive should have only one file.") - - f.extract(f.namelist()[0], zipdir) - input_name = os.path.join(zipdir, f.namelist()[0]) - # zipfile throws some undocumented exceptions - except Exception as e: - raise Exception("Failed to open archive: " + six.text_type(e)) + input_name = _extract_input_if_zipfile(input_name, zipdir) with file_executor as fe: with open(input_name, 'rb') as inf: diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 3af5313..37e8679 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -3,6 +3,8 @@ from shutil import rmtree from threading import Thread from zipfile import ZipFile, is_zipfile + +from sio.executors.common import _extract_input_if_zipfile, _populate_environ from sio.workers import ft from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd, TemporaryCwd from sio.workers.file_runners import get_file_runner @@ -14,19 +16,11 @@ import logging logger = logging.getLogger(__name__) +DEFAULT_INTERACTOR_TIME_LIMIT = 30000 # in ms +DEFAULT_INTERACTOR_MEM_LIMIT = 256 * 2 ** 10 # in KiB RESULT_STRING_LENGTH_LIMIT = 1024 # in bytes -def _populate_environ(renv, environ): - """Takes interesting fields from renv into environ""" - for key in ('time_used', 'mem_used', 'num_syscalls'): - environ[key] = renv.get(key, 0) - for key in ('result_code', 'result_string'): - environ[key] = renv.get(key, '') - if 'out_file' in renv: - environ['out_file'] = renv['out_file'] - - def _limit_length(s): if len(s) > RESULT_STRING_LENGTH_LIMIT: suffix = b'[...]' @@ -54,9 +48,6 @@ def run(environ, executor, use_sandboxes=True): _populate_environ(renv, environ) - # if environ['result_code'] == 'OK' and environ.get('check_output'): - # environ = checker.run(environ, use_sandboxes=use_sandboxes) - for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) @@ -81,7 +72,6 @@ def _fill_result(renv, irenv, interactor_out): if irenv['result_code'] != 'OK' and inter_sig != sigpipe: renv['result_code'] = 'SE' - # renv['result_string'] = 'checker exited prematurely' elif renv['result_code'] != 'OK' and sol_sig != sigpipe: return elif len(interactor_out) == 0: @@ -123,18 +113,7 @@ def _run(environ, executor, use_sandboxes): zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: - if is_zipfile(input_name): - try: - # If not a zip file, will pass it directly to exe - with ZipFile(tempcwd('in'), 'r') as f: - if len(f.namelist()) != 1: - raise Exception("Archive should have only one file.") - - f.extract(f.namelist()[0], zipdir) - input_name = os.path.join(zipdir, f.namelist()[0]) - # zipfile throws some undocumented exceptions - except Exception as e: - raise Exception("Failed to open archive: " + six.text_type(e)) + input_name = _extract_input_if_zipfile(input_name, zipdir) r1, w1 = os.pipe() r2, w2 = os.pipe() @@ -161,7 +140,9 @@ def _run(environ, executor, use_sandboxes): stdout=w1, ignore_errors=True, environ=environ, - environ_prefix='exec_', + environ_prefix='interactor_', + mem_limit=DEFAULT_INTERACTOR_MEM_LIMIT, + time_limit=DEFAULT_INTERACTOR_TIME_LIMIT, pass_fds=(r2, w1), close_passed_fd=True, cwd=tempcwd(), @@ -201,7 +182,7 @@ def _run(environ, executor, use_sandboxes): interactor_out = [line.rstrip() for line in result_file.readlines()] while len(interactor_out) < 3: - interactor_out.append('') + interactor_out.append(b'') _fill_result(renv, irenv, interactor_out) finally: diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 6d043b0..4da93e0 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -131,17 +131,12 @@ def execute_command( if ret_env is None: ret_env = {} + if env is not None: for key, value in six.iteritems(env): env[key] = str(value) perf_timer = util.PerfTimer() - logger.info("stderr: " + str(forward_stderr) + " " + str(subprocess.STDOUT) + " " + str(stderr) + " " + str(forward_stderr and subprocess.STDOUT or stderr) + "\n" + \ - "stdout: " + str(stdout) + "\n" + \ - "stdin: " + str(stdin) + "\n" ) - logger.info("pass fds: " + str(pass_fds)) - logger.info("cwd: " + cwd) - logger.info(str(os.listdir(cwd))) p = subprocess.Popen( command, stdin=stdin, @@ -155,7 +150,7 @@ def execute_command( cwd=cwd, preexec_fn=os.setpgrp, ) - + if close_passed_fd: for fd in pass_fds: os.close(fd) @@ -197,9 +192,6 @@ def oot_killer(): raise ExecError( 'Failed to execute command: %s. Returned with code %s\n' % (command, rc) ) - - logger.info(ret_env) - return ret_env From 5c59fbc54ca0f2a4cb689a9825714893d99d5881 Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Tue, 9 Apr 2024 23:31:02 +0200 Subject: [PATCH 09/14] Interactive tasks can't be output-only --- sio/executors/interactive_common.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 37e8679..430a0a1 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -41,10 +41,7 @@ def run(environ, executor, use_sandboxes=True): See `sio.executors.checkers`. True by default. """ - if environ.get('exec_info', {}).get('mode') == 'output-only': - renv = _fake_run_as_exe_is_output_file(environ) - else: - renv = _run(environ, executor, use_sandboxes) + renv = _run(environ, executor, use_sandboxes) _populate_environ(renv, environ) From 00bf8661c33604ea4dbcef5fba502acf846e1fbd Mon Sep 17 00:00:00 2001 From: tkwiatkowski Date: Wed, 17 Apr 2024 20:16:08 +0200 Subject: [PATCH 10/14] Supervisor trolling Co-authored-by: Mateusz Masiarz --- sio/executors/interactive_common.py | 152 +++++++++++++++------------- sio/workers/executors.py | 8 ++ 2 files changed, 87 insertions(+), 73 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 430a0a1..f81edc2 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -2,25 +2,34 @@ import os from shutil import rmtree from threading import Thread -from zipfile import ZipFile, is_zipfile from sio.executors.common import _extract_input_if_zipfile, _populate_environ from sio.workers import ft -from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd, TemporaryCwd +from sio.workers.executors import DetailedUnprotectedExecutor, PRootExecutor +from sio.workers.util import TemporaryCwd, decode_fields, replace_invalid_UTF, tempcwd from sio.workers.file_runners import get_file_runner +import traceback import signal import six -import traceback - import logging logger = logging.getLogger(__name__) -DEFAULT_INTERACTOR_TIME_LIMIT = 30000 # in ms DEFAULT_INTERACTOR_MEM_LIMIT = 256 * 2 ** 10 # in KiB RESULT_STRING_LENGTH_LIMIT = 1024 # in bytes +class InteractorError(Exception): + def __init__(self, message, interactor_out, env, renv, irenv): + super().__init__( + f'{message}\n' + f'Interactor out: {interactor_out}\n' + f'Interactor environ dump: {irenv}\n' + f'Solution environ dump: {renv}\n' + f'Environ dump: {env}' + ) + + def _limit_length(s): if len(s) > RESULT_STRING_LENGTH_LIMIT: suffix = b'[...]' @@ -59,7 +68,7 @@ def run(environ, executor, use_sandboxes=True): return environ -def _fill_result(renv, irenv, interactor_out): +def _fill_result(env, renv, irenv, interactor_out): sol_sig = renv.get('exit_signal', None) inter_sig = irenv.get('exit_signal', None) sigpipe = signal.SIGPIPE.value @@ -69,11 +78,12 @@ def _fill_result(renv, irenv, interactor_out): if irenv['result_code'] != 'OK' and inter_sig != sigpipe: renv['result_code'] = 'SE' + raise InteractorError(f'Interactor got {irenv["result_code"]}.', interactor_out, env, renv, irenv) elif renv['result_code'] != 'OK' and sol_sig != sigpipe: return elif len(interactor_out) == 0: renv['result_code'] = 'SE' - renv['result_string'] = 'invalid interactor output' + raise InteractorError(f'Empty interactor out.', interactor_out, env, renv, irenv) elif inter_sig == sigpipe: renv['result_code'] = 'WA' renv['result_string'] = 'solution exited prematurely' @@ -95,9 +105,7 @@ def _run(environ, executor, use_sandboxes): input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) - interactor_env = environ.copy() - interactor_env['exe_file'] = interactor_env['interactor_file'] - interactor_executor = get_file_runner(executor, interactor_env) + interactor_executor = DetailedUnprotectedExecutor() exe_filename = file_executor.preferred_filename() interactor_filename = 'soc' @@ -117,56 +125,62 @@ def _run(environ, executor, use_sandboxes): for fd in (r1, w1, r2, w2): os.set_inheritable(fd, True) - interactor_args = [input_name, tempcwd('out')] - logger.info(str(interactor_executor)) - logger.info(str(file_executor)) - - irenv = {} - renv = {} + interactor_args = [os.path.basename(input_name), 'out'] + + interactor_time_limit = 2 * environ['exec_time_limit'] + + class ExecutionWrapper(Thread): + def __init__(self, executor, *args, **kwargs): + super(ExecutionWrapper, self).__init__() + self.executor = executor + self.args = args + self.kwargs = kwargs + self.value = None + + def run(self): + with TemporaryCwd(): + logger.info("run" + str(self.args) + str(self.kwargs)) + try: + self.value = self.executor(*self.args, **self.kwargs) + except Exception as e: + logger.info(traceback.format_exc()) + logger.info(str(self.args) + str(e)) + pass + logger.info(str(self.args) + self.value) with interactor_executor as ie: - logger.info(tempcwd('out')) - interactor = Thread( - target=ie, - args=( - tempcwd(interactor_filename), - interactor_args, - ), - kwargs = dict( - stdin=r2, - stdout=w1, - ignore_errors=True, - environ=environ, - environ_prefix='interactor_', - mem_limit=DEFAULT_INTERACTOR_MEM_LIMIT, - time_limit=DEFAULT_INTERACTOR_TIME_LIMIT, - pass_fds=(r2, w1), - close_passed_fd=True, - cwd=tempcwd(), - in_file=environ['in_file'], - ret_env=irenv, - ) + interactor = ExecutionWrapper( + ie, + [tempcwd(interactor_filename)] + interactor_args, + stdin=r2, + stdout=w1, + ignore_errors=True, + environ=environ, + environ_prefix='interactor_', + mem_limit=DEFAULT_INTERACTOR_MEM_LIMIT, + time_limit=interactor_time_limit, + pass_fds=(r2, w1), + close_passed_fd=True, + cwd=tempcwd(), + in_file=environ['in_file'], + # binds=[(input_name, '/', 'ro'), (tempcwd('out'), '/', 'rw')], + # allow_local_open=True, ) with file_executor as fe: - exe = Thread( - target=fe, - args=( - tempcwd(exe_filename), - [], - ), - kwargs=dict( - stdin=r1, - stdout=w2, - ignore_errors=True, - environ=environ, - environ_prefix='exec_', - pass_fds=(r1, w2), - close_passed_fd=True, - cwd=tempcwd(), - in_file=environ['in_file'], - ret_env=renv, - ) + exe = ExecutionWrapper( + fe, + tempcwd(exe_filename), + [], + stdin=r1, + stdout=w2, + ignore_errors=True, + environ=environ, + environ_prefix='exec_', + pass_fds=(r1, w2), + close_passed_fd=True, + cwd=tempcwd(), + in_file=environ['in_file'], ) exe.start() @@ -175,27 +189,19 @@ def _run(environ, executor, use_sandboxes): exe.join() interactor.join() - with open(tempcwd('out'), 'rb') as result_file: - interactor_out = [line.rstrip() for line in result_file.readlines()] + renv = exe.value + irenv = interactor.value - while len(interactor_out) < 3: - interactor_out.append(b'') + try: + with open(tempcwd('out'), 'rb') as result_file: + interactor_out = [line.rstrip() for line in result_file.readlines()] + while len(interactor_out) < 3: + interactor_out.append(b'') + except FileNotFoundError: + interactor_out = [] - _fill_result(renv, irenv, interactor_out) + _fill_result(environ, renv, irenv, interactor_out) finally: rmtree(zipdir) return renv - - -def _fake_run_as_exe_is_output_file(environ): - # later code expects 'out' file to be present after compilation - ft.download(environ, 'exe_file', tempcwd('out')) - return { - # copy filetracker id of 'exe_file' as 'out_file' (thanks to that checker will grab it) - 'out_file': environ['exe_file'], - # 'result_code' is left by executor, as executor is not used - # this variable has to be set manually - 'result_code': 'OK', - 'result_string': 'ok', - } diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 4da93e0..3753894 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -120,6 +120,8 @@ def execute_command( # and it prevents deadlocks. command = shellquote(command) + logger.info(command) + logger.debug('Executing: %s', command) stdout = capture_output and tempfile.TemporaryFile() or stdout @@ -396,11 +398,14 @@ class DetailedUnprotectedExecutor(UnprotectedExecutor): """ def _execute(self, command, **kwargs): + logger.info("DUE") command = ['bash', '-c', [noquote('time')] + command] stderr = tempfile.TemporaryFile() kwargs['stderr'] = stderr kwargs['forward_stderr'] = False + logger.info("AAAAA") renv = super(DetailedUnprotectedExecutor, self)._execute(command, **kwargs) + logger.info("BBBBB") stderr.seek(0) output = stderr.read() stderr.close() @@ -639,6 +644,9 @@ def _execute(self, command, **kwargs): '--output-limit', str(kwargs['output_limit'] or self.DEFAULT_OUTPUT_LIMIT) + 'K', ] + # for (what, where, mode) in kwargs.pop('binds', []): + # logger.info((what, where, mode)) + # options += ['-b', what + ':' + where + ':' + mode] command = [os.path.join(self.rpath, 'sio2jail')] + options + ['--'] + command renv = {} From d9da07dc2fa17f133e323ee2a073eea7a609e6fa Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Mon, 6 May 2024 21:30:17 +0200 Subject: [PATCH 11/14] Supervisor is not trolling Co-authored-by: Mateusz Masiarz --- sio/executors/interactive_common.py | 14 +------------- sio/workers/executors.py | 7 +------ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index f81edc2..5f6038d 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -5,15 +5,12 @@ from sio.executors.common import _extract_input_if_zipfile, _populate_environ from sio.workers import ft -from sio.workers.executors import DetailedUnprotectedExecutor, PRootExecutor +from sio.workers.executors import DetailedUnprotectedExecutor from sio.workers.util import TemporaryCwd, decode_fields, replace_invalid_UTF, tempcwd from sio.workers.file_runners import get_file_runner -import traceback import signal import six -import logging -logger = logging.getLogger(__name__) DEFAULT_INTERACTOR_MEM_LIMIT = 256 * 2 ** 10 # in KiB RESULT_STRING_LENGTH_LIMIT = 1024 # in bytes @@ -73,9 +70,6 @@ def _fill_result(env, renv, irenv, interactor_out): inter_sig = irenv.get('exit_signal', None) sigpipe = signal.SIGPIPE.value - logger.info(renv) - logger.info(irenv) - if irenv['result_code'] != 'OK' and inter_sig != sigpipe: renv['result_code'] = 'SE' raise InteractorError(f'Interactor got {irenv["result_code"]}.', interactor_out, env, renv, irenv) @@ -139,14 +133,10 @@ def __init__(self, executor, *args, **kwargs): def run(self): with TemporaryCwd(): - logger.info("run" + str(self.args) + str(self.kwargs)) try: self.value = self.executor(*self.args, **self.kwargs) except Exception as e: - logger.info(traceback.format_exc()) - logger.info(str(self.args) + str(e)) pass - logger.info(str(self.args) + self.value) with interactor_executor as ie: interactor = ExecutionWrapper( @@ -163,8 +153,6 @@ def run(self): close_passed_fd=True, cwd=tempcwd(), in_file=environ['in_file'], - # binds=[(input_name, '/', 'ro'), (tempcwd('out'), '/', 'rw')], - # allow_local_open=True, ) with file_executor as fe: diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 3753894..daefb97 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -120,8 +120,6 @@ def execute_command( # and it prevents deadlocks. command = shellquote(command) - logger.info(command) - logger.debug('Executing: %s', command) stdout = capture_output and tempfile.TemporaryFile() or stdout @@ -144,7 +142,7 @@ def execute_command( stdin=stdin, stdout=stdout, stderr=forward_stderr and subprocess.STDOUT or stderr, - pass_fds=pass_fds, + # pass_fds=pass_fds, shell=True, close_fds=True, universal_newlines=True, @@ -644,9 +642,6 @@ def _execute(self, command, **kwargs): '--output-limit', str(kwargs['output_limit'] or self.DEFAULT_OUTPUT_LIMIT) + 'K', ] - # for (what, where, mode) in kwargs.pop('binds', []): - # logger.info((what, where, mode)) - # options += ['-b', what + ':' + where + ':' + mode] command = [os.path.join(self.rpath, 'sio2jail')] + options + ['--'] + command renv = {} From 75f6e05d3c439a83d80f309636e0af3742f420fb Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Mon, 6 May 2024 21:34:01 +0200 Subject: [PATCH 12/14] Remove debug --- sio/workers/executors.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index daefb97..9779954 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -396,14 +396,11 @@ class DetailedUnprotectedExecutor(UnprotectedExecutor): """ def _execute(self, command, **kwargs): - logger.info("DUE") command = ['bash', '-c', [noquote('time')] + command] stderr = tempfile.TemporaryFile() kwargs['stderr'] = stderr kwargs['forward_stderr'] = False - logger.info("AAAAA") renv = super(DetailedUnprotectedExecutor, self)._execute(command, **kwargs) - logger.info("BBBBB") stderr.seek(0) output = stderr.read() stderr.close() From 8243cb13d4237aade04ea758032e5fe1b570bf41 Mon Sep 17 00:00:00 2001 From: tkwiatkowski Date: Wed, 15 May 2024 18:21:08 +0200 Subject: [PATCH 13/14] Adapt to new results percentage --- sio/executors/interactive_common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index 5f6038d..a9e5872 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -3,6 +3,7 @@ from shutil import rmtree from threading import Thread +from sio.executors.checker import output_to_fraction from sio.executors.common import _extract_input_if_zipfile, _populate_environ from sio.workers import ft from sio.workers.executors import DetailedUnprotectedExecutor @@ -87,12 +88,12 @@ def _fill_result(env, renv, irenv, interactor_out): renv['result_code'] = 'OK' if interactor_out[1]: renv['result_string'] = _limit_length(interactor_out[1]) - renv['result_percentage'] = float(interactor_out[2] or 100) + renv['result_percentage'] = output_to_fraction(interactor_out[2]) else: renv['result_code'] = 'WA' if interactor_out[1]: renv['result_string'] = _limit_length(interactor_out[1]) - renv['result_percentage'] = 0 + renv['result_percentage'] = (0, 1) def _run(environ, executor, use_sandboxes): From 0b4572da5c1aff7013650a21404316a4c377ede4 Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Wed, 5 Jun 2024 16:33:04 +0200 Subject: [PATCH 14/14] Refactor a bit --- sio/executors/interactive_common.py | 8 ++++++-- sio/workers/executors.py | 14 ++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/sio/executors/interactive_common.py b/sio/executors/interactive_common.py index a9e5872..338a658 100644 --- a/sio/executors/interactive_common.py +++ b/sio/executors/interactive_common.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import os from shutil import rmtree from threading import Thread @@ -131,13 +130,14 @@ def __init__(self, executor, *args, **kwargs): self.args = args self.kwargs = kwargs self.value = None + self.exception = None def run(self): with TemporaryCwd(): try: self.value = self.executor(*self.args, **self.kwargs) except Exception as e: - pass + self.exception = e with interactor_executor as ie: interactor = ExecutionWrapper( @@ -178,6 +178,10 @@ def run(self): exe.join() interactor.join() + for ew in (exe, interactor): + if ew.exception is not None: + raise ew.exception + renv = exe.value irenv = interactor.value diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 9779954..1ef2286 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -79,10 +79,8 @@ def execute_command( real_time_limit=None, ignore_errors=False, extra_ignore_errors=(), - ret_env=None, - pass_fds=(), - close_passed_fd=False, cwd=None, + fds_to_close=(), **kwargs, ): """Utility function to run arbitrary command. @@ -128,9 +126,7 @@ def execute_command( stdout = stdout or devnull stderr = stderr or devnull cwd = cwd or tempcwd() - - if ret_env is None: - ret_env = {} + ret_env = {} if env is not None: for key, value in six.iteritems(env): @@ -142,7 +138,6 @@ def execute_command( stdin=stdin, stdout=stdout, stderr=forward_stderr and subprocess.STDOUT or stderr, - # pass_fds=pass_fds, shell=True, close_fds=True, universal_newlines=True, @@ -151,9 +146,8 @@ def execute_command( preexec_fn=os.setpgrp, ) - if close_passed_fd: - for fd in pass_fds: - os.close(fd) + for fd in fds_to_close: + os.close(fd) kill_timer = None if real_time_limit: