diff --git a/setup.py b/setup.py index eae7f98..8358276 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,13 @@ 'ping = sio.workers.ping:run', 'compile = sio.compilers.job:run', 'exec = sio.executors.executor:run', + 'encdec-exec = sio.executors.executor:encdec_run', 'sio2jail-exec = sio.executors.sio2jail_exec:run', + 'sio2jail-encdec-exec = sio.executors.sio2jail_exec:encdec_run', 'cpu-exec = sio.executors.executor:run', + 'cpu-encdec-exec = sio.executors.executor:encdec_run', 'unsafe-exec = sio.executors.unsafe_exec:run', + 'unsafe-encdec-exec = sio.executors.unsafe_exec:encdec_run', 'ingen = sio.executors.ingen:run', 'inwer = sio.executors.inwer:run', ], diff --git a/sio/compilers/system_gcc.py b/sio/compilers/system_gcc.py index 2fba963..bd9040a 100644 --- a/sio/compilers/system_gcc.py +++ b/sio/compilers/system_gcc.py @@ -28,13 +28,13 @@ def _make_cmdline(self, executor): class CCompiler(CStyleCompiler): compiler = 'gcc' # Without -static as there is no static compilation on Mac - options = ['-O2', '-s', '-lm'] + options = ['-static', '-O2', '-s', '-lm'] class CPPCompiler(CStyleCompiler): lang = 'cpp' compiler = 'g++' - options = ['-std=gnu++0x', '-O2', '-s', '-lm'] + options = ['-std=gnu++0x', '-static', '-O2', '-s', '-lm'] def run_gcc(environ): diff --git a/sio/executors/common.py b/sio/executors/common.py index 0da4f35..2ad222d 100644 --- a/sio/executors/common.py +++ b/sio/executors/common.py @@ -83,28 +83,30 @@ def _run(environ, executor, use_sandboxes): except Exception as e: raise Exception("Failed to open archive: " + six.text_type(e)) - with file_executor as fe: - with open(input_name, 'rb') as inf: + return _run_core(environ, file_executor, input_name, tempcwd('out'), tempcwd(exe_filename), + 'exec_', use_sandboxes) + + finally: + rmtree(zipdir) + + +def _run_core(environ, file_executor, input_name, output_name, exe_filename, environ_prefix, use_sandboxes): + with file_executor as fe: + with open(input_name, 'rb') as inf: # Open output file in append mode to allow appending # only to the end of the output file. Otherwise, # a contestant's program could modify the middle of the file. - with open(tempcwd('out'), 'ab') as outf: - renv = fe( + with open(output_name, 'ab') as outf: + return fe( tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, - environ_prefix='exec_', + environ_prefix=environ_prefix, ) - finally: - rmtree(zipdir) - - return renv - - def _fake_run_as_exe_is_output_file(environ): # later code expects 'out' file to be present after compilation ft.download(environ, 'exe_file', tempcwd('out')) diff --git a/sio/executors/encdec_common.py b/sio/executors/encdec_common.py new file mode 100644 index 0000000..bac00cd --- /dev/null +++ b/sio/executors/encdec_common.py @@ -0,0 +1,267 @@ +from __future__ import absolute_import +import os +import logging +from shutil import copy2, rmtree +import tempfile +from zipfile import ZipFile, is_zipfile +from sio.executors.checker import _limit_length +from sio.executors.common import _run_core +from sio.workers import ft +from sio.workers.executors import ExecError, PRootExecutor, UnprotectedExecutor +from sio.workers.util import decode_fields, replace_invalid_UTF, tempcwd, TemporaryCwd +from sio.workers.file_runners import get_file_runner + +logger = logging.getLogger(__name__) + + +DEFAULT_SUPPLEMENTARY_TIME_LIMIT = 30000 # in ms +DEFAULT_SUPPLEMENTARY_MEM_LIMIT = 268 * 2**10 # in KiB + + +class ChannelError(Exception): + pass + + +class CheckerError(Exception): + pass + + +def _populate_environ(renv, environ, prefix): + """Takes interesting fields from renv into environ""" + for key in ( + "time_used", + "mem_used", + "num_syscalls", + "result_code", + "result_string", + ): + if key in renv: + environ[prefix + key] = renv[key] + + +def _run_supplementary(env, command, executor, environ_prefix, **kwargs): + with executor: + return executor( + command, + capture_output=True, + split_lines=True, + mem_limit=DEFAULT_SUPPLEMENTARY_MEM_LIMIT, + time_limit=DEFAULT_SUPPLEMENTARY_TIME_LIMIT, + environ=env, + environ_prefix=environ_prefix, + **kwargs + ) + + +def _run_encoder(environ, file_executor, exe_filename, use_sandboxes): + ft.download(environ, "in_file", "enc_in", add_to_cache=True) + return _run_core( + environ, + file_executor, + tempcwd("enc_in"), + tempcwd("enc_out"), + tempcwd(exe_filename), + "encoder_", + use_sandboxes, + ) + + +def _run_channel_core(env, result_file, checker_file, use_sandboxes=False): + command = [ + "./chn", + "enc_in", + "enc_out", + "hint", + str(result_file.fileno()), + str(checker_file.fileno()), + ] + + def execute_channel(with_stderr=False, stderr=None): + return _run_supplementary( + env, + command, + PRootExecutor("null-sandbox") + if env.get("untrusted_channel", False) and use_sandboxes + else UnprotectedExecutor(), + "channel_", + ignore_errors=True, + forward_stderr=with_stderr, + stderr=stderr, + pass_fds=(result_file.fileno(), checker_file.fileno()), + ) + + with tempfile.TemporaryFile() as stderr_file: + renv = execute_channel(stderr=stderr_file) + if renv["return_code"] >= 2: + stderr_file.seek(0) + stderr = stderr_file.read() + raise ChannelError( + "Channel returned code(%d) >= 2. Channel stdout: " + '"%s", stderr: "%s". Channel environ dump: %s' + % (renv["return_code"], renv["stdout"], stderr, env) + ) + + return renv["stdout"] + + +def _run_channel(environ, use_sandboxes=False): + ft.download(environ, "hint_file", "hint", add_to_cache=True) + ft.download(environ, "chn_file", "chn", add_to_cache=True) + os.chmod(tempcwd("chn"), 0o700) + result_filename = tempcwd("dec_in") + checker_filename = tempcwd("chn_out") + + try: + with open(result_filename, "wb") as result_file, open( + checker_filename, "wb" + ) as checker_file: + output = _run_channel_core( + environ, result_file, checker_file, use_sandboxes + ) + except (ChannelError, ExecError) as e: + logger.error("Channel failed! %s", e) + logger.error("Environ dump: %s", environ) + raise SystemError(e) + + while len(output) < 3: + output.append(b"") + + if output[0] == b"OK": + environ["channel_result_code"] = "OK" + if output[1]: + environ["channel_result_string"] = _limit_length(output[1]).decode("utf-8") + return True + else: + environ["failed_step"] = "channel" + environ["channel_result_code"] = "WA" + environ["channel_result_string"] = _limit_length(output[1]).decode("utf-8") + return False + + +def _run_decoder(environ, file_executor, exe_filename, use_sandboxes): + return _run_core( + environ, + file_executor, + tempcwd("dec_in"), + tempcwd("dec_out"), + tempcwd(exe_filename), + "decoder_", + use_sandboxes, + ) + + +def _run_checker_core(env, use_sandboxes=False): + command = ["./chk", "enc_in", "hint", "chn_out", "dec_out"] + + def execute_checker(with_stderr=False, stderr=None): + return _run_supplementary( + env, + command, + PRootExecutor("null-sandbox") + if env.get("untrusted_checker", False) and use_sandboxes + else UnprotectedExecutor(), + "checker_", + ignore_errors=True, + forward_stderr=with_stderr, + stderr=stderr, + ) + + with tempfile.TemporaryFile() as stderr_file: + renv = execute_checker(stderr=stderr_file) + if renv["return_code"] >= 2: + stderr_file.seek(0) + stderr = stderr_file.read() + raise CheckerError( + "Checker returned code(%d) >= 2. Checker stdout: " + '"%s", stderr: "%s". Checker environ dump: %s' + % (renv["return_code"], renv["stdout"], stderr, env) + ) + + return renv["stdout"] + + +def _run_checker(environ, use_sandboxes=False): + ft.download(environ, "chk_file", "chk", add_to_cache=True) + os.chmod(tempcwd("chk"), 0o700) + + try: + output = _run_checker_core(environ, use_sandboxes) + except (ChannelError, ExecError) as e: + logger.error("Checker failed! %s", e) + logger.error("Environ dump: %s", environ) + raise SystemError(e) + + while len(output) < 3: + output.append(b"") + + if output[0] == b"OK": + environ["checker_result_code"] = "OK" + if output[1]: + environ["checker_result_string"] = _limit_length(output[1]).decode("utf-8") + environ["checker_result_percentage"] = float(output[2] or 100) + return True + else: + environ["failed_step"] = "checker" + environ["checker_result_code"] = "WA" + environ["checker_result_string"] = _limit_length(output[1]).decode("utf-8") + environ["checker_result_percentage"] = 0 + return False + + +def _run_decoder_hide_files(environ, file_executor, exe_filename, use_sandboxes, orig_dir): + # We now have quite a lot of interes + # be nice if some decoder read them. + with TemporaryCwd() as new_dir: + # Copy the executable and input + for f in 'dec_in', exe_filename: + copy2(os.path.join(orig_dir, f), tempcwd(f)) + + renv = _run_decoder(environ, file_executor, exe_filename, use_sandboxes) + + # Copy the output + for f in 'dec_out',: + copy2(tempcwd(f), os.path.join(orig_dir, f)) + + return renv + + +def run(environ, executor, use_sandboxes=True): + """ + Common code for executors. + + :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` + For all supported options, see the global documentation for + `sio.workers.executors` and prefix them with ``encoder_`` + or ``decoder_``. + :param: executor Executor instance used for executing commands. + :param: use_sandboxes Enables safe checking output correctness. + See `sio.executors.checkers`. True by default. + """ + + file_executor = get_file_runner(executor, environ) + exe_filename = file_executor.preferred_filename() + + ft.download(environ, "exe_file", exe_filename, add_to_cache=True) + os.chmod(tempcwd(exe_filename), 0o700) + + encoder_environ = environ.copy() + renv = _run_encoder(encoder_environ, file_executor, exe_filename, use_sandboxes) + _populate_environ(renv, environ, "encoder_") + + if renv["result_code"] != "OK": + environ["failed_step"] = "encoder" + return environ + + if not _run_channel(environ, use_sandboxes): + return environ + + renv = _run_decoder_hide_files(environ, file_executor, exe_filename, use_sandboxes, tempcwd()) + _populate_environ(renv, environ, "decoder_") + + if renv["result_code"] != "OK": + environ["failed_step"] = "decoder" + return environ + + _run_checker(environ, use_sandboxes) + + return environ diff --git a/sio/executors/executor.py b/sio/executors/executor.py index 40ce9af..11786d4 100644 --- a/sio/executors/executor.py +++ b/sio/executors/executor.py @@ -1,7 +1,11 @@ from __future__ import absolute_import -from sio.executors import common +from sio.executors import common, encdec_common from sio.workers.executors import SupervisedExecutor def run(environ): return common.run(environ, SupervisedExecutor()) + + +def encdec_run(environ): + return encdec_common.run(environ, SupervisedExecutor()) diff --git a/sio/executors/sio2jail_exec.py b/sio/executors/sio2jail_exec.py index ac23e4a..ba05006 100644 --- a/sio/executors/sio2jail_exec.py +++ b/sio/executors/sio2jail_exec.py @@ -1,6 +1,10 @@ -from sio.executors import common +from sio.executors import common, encdec_common from sio.workers.executors import Sio2JailExecutor def run(environ): return common.run(environ, Sio2JailExecutor()) + + +def encdec_run(environ): + return encdec_common.run(environ, Sio2JailExecutor()) diff --git a/sio/executors/test/sources/rle.cpp b/sio/executors/test/sources/rle.cpp new file mode 100644 index 0000000..6ce9ce0 --- /dev/null +++ b/sio/executors/test/sources/rle.cpp @@ -0,0 +1,59 @@ +#include +#include +#include + +void enkoder() { + int last = '\n'; + int ch; + std::size_t count = 0; + + while ((ch = std::getchar()) != '\n') { + if (ch != last) { + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + count = 0; + } + + ++count; + last = ch; + } + + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + std::putchar('\n'); +} + +void dekoder() { + int ch; + + while ((ch = std::getchar()) != '\n') { + std::size_t count; + assert(std::scanf("%zu;", &count) == 1); + while (count--) std::putchar(ch); + } + + std::putchar('\n'); +} + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/sources/rle0a.hint b/sio/executors/test/sources/rle0a.hint new file mode 100644 index 0000000..39aa27c --- /dev/null +++ b/sio/executors/test/sources/rle0a.hint @@ -0,0 +1 @@ +C1;Z1;Y1;M1;P1;I1; diff --git a/sio/executors/test/sources/rle0a.in b/sio/executors/test/sources/rle0a.in new file mode 100644 index 0000000..78abf17 --- /dev/null +++ b/sio/executors/test/sources/rle0a.in @@ -0,0 +1,2 @@ +E +CZYMPI diff --git a/sio/executors/test/sources/rle1.hint b/sio/executors/test/sources/rle1.hint new file mode 100644 index 0000000..d98a846 --- /dev/null +++ b/sio/executors/test/sources/rle1.hint @@ -0,0 +1 @@ +A1;B1;A1;B1;A1;B1;A1;B1;A1;B2;A7;B1;A1;B1;A1;B1;A2;B1;A2;B13;A7;B3;A2;B11;A7; diff --git a/sio/executors/test/sources/rle1.in b/sio/executors/test/sources/rle1.in new file mode 100644 index 0000000..4eb4b84 --- /dev/null +++ b/sio/executors/test/sources/rle1.in @@ -0,0 +1,2 @@ +E +ABABABABABBAAAAAAABABABAABAABBBBBBBBBBBBBAAAAAAABBBAABBBBBBBBBBBAAAAAAA diff --git a/sio/executors/test/sources/rlechk.cpp b/sio/executors/test/sources/rlechk.cpp new file mode 100644 index 0000000..438d6e3 --- /dev/null +++ b/sio/executors/test/sources/rlechk.cpp @@ -0,0 +1,36 @@ +#include +#include +#include + +using namespace std; + +int main(int argc, char **argv) { + assert(argc == 5); + + FILE *in = fopen(argv[1], "r"); + assert(in); + FILE *hint = fopen(argv[2], "r"); + assert(hint); + FILE *channel_out = fopen(argv[3], "r"); + assert(channel_out); + FILE *encoder_out = fopen(argv[4], "r"); + assert(encoder_out); + + assert(fseek(in, 2, SEEK_SET) >= 0); + + int ch; + while ((ch = fgetc(encoder_out)) != EOF) { + if (fgetc(in) != ch) { + puts("WA\nZły wynik\n0"); + return 0; + } + } + if (fgetc(in) != EOF) puts("WA\nZły wynik\n0"); + + puts("OK\nBardzo dobrze\n100"); + + fclose(in); + fclose(hint); + fclose(channel_out); + fclose(encoder_out); +} diff --git a/sio/executors/test/sources/rlechn.cpp b/sio/executors/test/sources/rlechn.cpp new file mode 100644 index 0000000..d16b7c0 --- /dev/null +++ b/sio/executors/test/sources/rlechn.cpp @@ -0,0 +1,43 @@ +#include +#include +#include + +using namespace std; + +int main(int argc, char **argv) { + assert(argc == 6); + + FILE *in = fopen(argv[1], "r"); + assert(in); + FILE *out = fopen(argv[2], "r"); + assert(out); + FILE *hint = fopen(argv[3], "r"); + assert(hint); + FILE *result = fdopen(atoi(argv[4]), "w"); + assert(result); + FILE *checker = fdopen(atoi(argv[5]), "w"); + assert(checker); + + fputs("D\n", result); + + int ch; + while ((ch = fgetc(out)) != EOF) { + if (fgetc(hint) != ch) { + puts("WA\nZły wynik\n0"); + return 0; + } + + assert(fputc(ch, result) != EOF); + } + if (fgetc(hint) != EOF) puts("WA\nZły wynik\n0"); + + assert(fputs("OK\n", checker) != EOF); + + puts("OK\nBardzo dobrze\n100"); + + fclose(in); + fclose(out); + fclose(hint); + fclose(result); + fclose(checker); +} diff --git a/sio/executors/test/sources/rlecrashdec.cpp b/sio/executors/test/sources/rlecrashdec.cpp new file mode 100644 index 0000000..8c34642 --- /dev/null +++ b/sio/executors/test/sources/rlecrashdec.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +void enkoder() { + int last = '\n'; + int ch; + std::size_t count = 0; + + while ((ch = std::getchar()) != '\n') { + if (ch != last) { + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + count = 0; + } + + ++count; + last = ch; + } + + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + std::putchar('\n'); +} + +void dekoder() { std::abort(); } + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/sources/rlecrashenc.cpp b/sio/executors/test/sources/rlecrashenc.cpp new file mode 100644 index 0000000..1cf5620 --- /dev/null +++ b/sio/executors/test/sources/rlecrashenc.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +#include + +void enkoder() { std::abort(); } + +void dekoder() { + int ch; + + while ((ch = std::getchar()) != '\n') { + std::size_t count; + assert(std::scanf("%zu;", &count) == 1); + while (count--) std::putchar(ch); + } + + std::putchar('\n'); +} + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/sources/rleloopdec.cpp b/sio/executors/test/sources/rleloopdec.cpp new file mode 100644 index 0000000..773cfdd --- /dev/null +++ b/sio/executors/test/sources/rleloopdec.cpp @@ -0,0 +1,51 @@ +#include +#include +#include + +void enkoder() { + int last = '\n'; + int ch; + std::size_t count = 0; + + while ((ch = std::getchar()) != '\n') { + if (ch != last) { + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + count = 0; + } + + ++count; + last = ch; + } + + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + std::putchar('\n'); +} + +void dekoder() { + while (true) {} +} + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/sources/rleloopenc.cpp b/sio/executors/test/sources/rleloopenc.cpp new file mode 100644 index 0000000..638cfce --- /dev/null +++ b/sio/executors/test/sources/rleloopenc.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +void enkoder() { + while (true) {} +} + +void dekoder() { +} + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/sources/rlememdec.cpp b/sio/executors/test/sources/rlememdec.cpp new file mode 100644 index 0000000..c132d06 --- /dev/null +++ b/sio/executors/test/sources/rlememdec.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + +void enkoder() { + int last = '\n'; + int ch; + std::size_t count = 0; + + while ((ch = std::getchar()) != '\n') { + if (ch != last) { + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + count = 0; + } + + ++count; + last = ch; + } + + if (last != '\n') { + std::putchar(last); + std::printf("%zu;", count); + } + + std::putchar('\n'); +} + +void dekoder() { + std::vector ps; + while (true) { + ps.push_back(std::malloc(1024 * 1024)); + } +} + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/sources/rlememenc.cpp b/sio/executors/test/sources/rlememenc.cpp new file mode 100644 index 0000000..e028a8d --- /dev/null +++ b/sio/executors/test/sources/rlememenc.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include + +void enkoder() { + std::vector ps; + while (true) { + ps.push_back(std::malloc(1024 * 1024)); + } +} + +void dekoder() { + int ch; + + while ((ch = std::getchar()) != '\n') { + std::size_t count; + assert(std::scanf("%zu;", &count) == 1); + while (count--) std::putchar(ch); + } + + std::putchar('\n'); +} + +// rlelib.cpp +// +#include +#include + +extern void dekoder(); +extern void enkoder(); + +int main() { + int ch = std::getchar(); + + assert(ch == 'D' || ch == 'E'); + assert(std::getchar() == '\n'); + + (ch == 'D' ? dekoder : enkoder)(); +} diff --git a/sio/executors/test/test_encdec.py b/sio/executors/test/test_encdec.py new file mode 100644 index 0000000..a8f046f --- /dev/null +++ b/sio/executors/test/test_encdec.py @@ -0,0 +1,166 @@ +import glob +import os.path + +from filetracker.client.dummy import DummyClient +from sio.assertion_utils import ok_, eq_ +from sio.compilers.job import run as run_compiler +import sio.executors.unsafe_exec +import sio.executors.sio2jail_exec +from sio.testing_utils import str_to_bool +from sio.workers import ft +from sio.workers.util import TemporaryCwd, tempcwd + + + +EXECUTORS = { + 'sio2jail': sio.executors.sio2jail_exec, + 'unsafe': sio.executors.unsafe_exec +} +# Stolen from a sample problem package I used to test the encdec +RLE_TESTS = ['0a', '1'] +SOURCES = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sources') + + +if str_to_bool(os.environ.get('TEST_SIO2JAIL', True)): + import sio.executors.sio2jail_exec + EXECUTORS['sio2jail'] = sio.executors.sio2jail_exec + + +def common_preparations(): + with TemporaryCwd(): + upload_files() + with TemporaryCwd('compile_chn'): + run_compiler({ + 'source_file': '/rlechn.cpp', + 'compiler': 'system-cpp', + 'out_file': '/rlechn.e' + }) + with TemporaryCwd('compile_chk'): + run_compiler({ + 'source_file': '/rlechk.cpp', + 'compiler': 'system-cpp', + 'out_file': '/rlechk.e' + }) + + +def compile_file(file): + with TemporaryCwd('compile_code'): + renv = run_compiler({ + 'source_file': '/%s.cpp' % file, + 'compiler': 'system-cpp', + 'out_file': '/%s.e' % file, + }) + eq_(renv['result_code'], 'OK') + return renv + + +def in_(a, b, msg=None): + """Shorthand for 'assert a in b, "%r not in %r" % (a, b)""" + if a not in b: + raise AssertionError(msg or "%r not in %r" % (a, b)) + + +def make_run_env(compile_renv, file, test, new_settings=None): + result = { + 'chn_file': '/rlechn.e', + 'chk_file': '/rlechk.e', + 'exe_file': '/%s.e' % file, + 'exec_info': compile_renv['exec_info'], + 'hint_file': '/rle%s.hint' % test, + 'in_file': '/rle%s.in' % test, + 'encoder_memory_limit': '65536', + 'encoder_time_limit': 2000, + 'decoder_memory_limit': '65536', + 'decoder_time_limit': 2000, + } + if new_settings: + result.update(new_settings) + return result + + +def not_in_(a, b, msg=None): + """Shorthand for 'assert a not in b, "%r in %r" % (a, b)""" + if a in b: + raise AssertionError(msg or "%r in %r" % (a, b)) + + +def print_env(env): + from pprint import pprint + + pprint(env) + + +def run_all_configurations(file, func, new_settings=None): + compile_renv = compile_file(file) + for execname, executor in EXECUTORS.items(): + for t in RLE_TESTS: + with TemporaryCwd('run_%s_%s' % (execname, t)): + print('Running test %s under executor %s' % (t, execname)) + renv = executor.encdec_run(make_run_env(compile_renv, file, t, new_settings(execname, t) if new_settings else None)) + print_env(renv) + func(execname, t, renv) + + +def upload_files(): + "Uploads all files from SOURCES to a newly created dummy filetracker" + + # DummyClient operates in the working directory. + ft.set_instance(DummyClient()) + for path in glob.glob(os.path.join(SOURCES, '*')): + ft.upload({'path': '/' + os.path.basename(path)}, 'path', path) + + +def test_encdec_run(): + common_preparations() + def check(execname, t, renv): + not_in_('failed_step', renv) + eq_(renv['checker_result_percentage'], 100.) + run_all_configurations('rle', check) + + +def test_encdec_encoder_timeout(): + common_preparations() + def check(execname, t, renv): + eq_(renv['failed_step'], 'encoder') + eq_(renv['encoder_result_code'], 'TLE') + run_all_configurations('rleloopenc', check) + + +def test_encdec_encoder_outofmem(): + common_preparations() + def check(execname, t, renv): + eq_(renv['failed_step'], 'encoder') + in_(renv['encoder_result_code'], ('MLE', 'RE')) + run_all_configurations('rlememenc', check) + + +def test_encdec_encoder_crash(): + common_preparations() + def check(execname, t, renv): + eq_(renv['failed_step'], 'encoder') + in_(renv['encoder_result_code'], ('RE')) + run_all_configurations('rlecrashenc', check) + + +def test_encdec_decoder_timeout(): + common_preparations() + def check(execname, t, renv): + eq_(renv['failed_step'], 'decoder') + eq_(renv['decoder_result_code'], 'TLE') + run_all_configurations('rleloopdec', check) + + +def test_encdec_decoder_outofmem(): + common_preparations() + def check(execname, t, renv): + eq_(renv['failed_step'], 'decoder') + in_(renv['decoder_result_code'], ('MLE', 'RE')) + run_all_configurations('rlememdec', check) + + +def test_encdec_decoder_crash(): + common_preparations() + def check(execname, t, renv): + eq_(renv['failed_step'], 'decoder') + in_(renv['decoder_result_code'], ('RE')) + run_all_configurations('rlecrashdec', check) diff --git a/sio/executors/unsafe_exec.py b/sio/executors/unsafe_exec.py index 3b0ae38..f2c7af6 100644 --- a/sio/executors/unsafe_exec.py +++ b/sio/executors/unsafe_exec.py @@ -1,7 +1,11 @@ from __future__ import absolute_import -from sio.executors import common +from sio.executors import common, encdec_common from sio.workers.executors import DetailedUnprotectedExecutor def run(environ): return common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) + + +def encdec_run(environ): + return encdec_common.run(environ, DetailedUnprotectedExecutor(), use_sandboxes=False) diff --git a/sio/workers/executors.py b/sio/workers/executors.py index 7119fac..5668d40 100644 --- a/sio/workers/executors.py +++ b/sio/workers/executors.py @@ -79,6 +79,7 @@ def execute_command( real_time_limit=None, ignore_errors=False, extra_ignore_errors=(), + pass_fds=(), **kwargs ): """Utility function to run arbitrary command. @@ -111,6 +112,9 @@ def execute_command( ``stdout`` Only when ``capture_output=True``: output of the command + + ``pass_fds`` + Extra file descriptors to pass to the command. """ # Using temporary file is way faster than using subproces.PIPE # and it prevents deadlocks. @@ -141,6 +145,7 @@ def execute_command( env=env, cwd=tempcwd(), preexec_fn=os.setpgrp, + pass_fds=pass_fds, ) kill_timer = None @@ -599,7 +604,7 @@ class Sio2JailExecutor(SandboxExecutor): REAL_TIME_LIMIT_ADDEND = 1000 # (in ms) def __init__(self): - super(Sio2JailExecutor, self).__init__('sio2jail_exec-sandbox') + super(Sio2JailExecutor, self).__init__('sio2jail_exec-sandbox-1.4.4') def _execute(self, command, **kwargs): options = [] @@ -635,6 +640,7 @@ def _execute(self, command, **kwargs): try: result_file = tempfile.NamedTemporaryFile(dir=tempcwd()) kwargs['ignore_errors'] = True + print(command) renv = execute_command( command + [noquote('2>'), result_file.name], **kwargs ) @@ -776,7 +782,7 @@ class PRootExecutor(BaseExecutor): def __init__(self, sandbox): """``sandbox`` has to be a sandbox name.""" self.chroot = get_sandbox(sandbox) - self.proot = SandboxExecutor('proot-sandbox') + self.proot = SandboxExecutor('proot-sandbox_amd64') self.options = [] with self.chroot: