Return result percentage as fraction

sio/executors/checker.py

@@ -3,6 +3,7 @@
 import logging
 import tempfile
 import six
+import re
 
 from sio.workers import ft
 from sio.workers.executors import (
@@ -129,9 +130,42 @@ def run(environ, use_sandboxes=True):
         environ['result_code'] = 'OK'
         if output[1]:
             environ['result_string'] = _limit_length(output[1])
-        environ['result_percentage'] = float(output[2] or 100)
+        if isinstance(output[2], bytes):
+            output[2] = output[2].decode('utf-8')
+        environ['result_percentage'] = output_to_fraction(output[2].strip())
+        if environ['result_percentage'][1] == 0:
+            raise CheckerError(
+                'Checker returned 0 as denominator. Checker stdout: "%s". Checker environ dump: %s'
+                % (output[2], environ)
+            )
     else:
         environ['result_code'] = 'WA'
         environ['result_string'] = _limit_length(output[1])
-        environ['result_percentage'] = 0
+        environ['result_percentage'] = (0, 1)
     return environ
+
+
+def output_to_fraction(output_str):
+    if not output_str:
+        return 100, 1
+    output_is_float = re.match(r"^[0-9]+\.[0-9]*$", output_str)
+    output_is_percent = re.match(r"^[0-9]+$", output_str)
+    output_is_fraction = re.match(r"^[0-9]+ [0-9]+$", output_str)
+    if output_is_float:
+        return float_to_fraction(output_str)
+    elif output_is_percent:
+        return int(output_str), 1
+    elif output_is_fraction:
+        split = output_str.split(' ')
+        return int(split[0]), int(split[1])
+    else:
+        raise CheckerError(
+            'Invalid checker output, expected float, percent or fraction, got "%s"'
+            % output_str
+        )
+
+
+def float_to_fraction(float_str):
+    nominator = int(''.join(filter(str.isdigit, float_str)))
+    denominator = 10 ** (len(float_str) - float_str.find('.') - 1)
+    return nominator, denominator

sio/workers/test/sources/chk-float.c

@@ -0,0 +1,16 @@
+#include <stdio.h>
+/* Simple unsafe checker with buffer overflow */
+
+int main(int argc, char **argv) {
+    char buf[255], buf2[255];
+    FILE* fdi = fopen(argv[1], "r");
+    FILE* fdo = fopen(argv[2], "r");
+    FILE* fdh = fopen(argv[3], "r");
+    fscanf(fdh, "%s", buf);
+    fscanf(fdo, "%s", buf2);
+    if (strcmp(buf, buf2) == 0)
+        puts("OK\nOK\n42.00");
+    else
+        puts("WRONG");
+    return 0;
+}

sio/workers/test/sources/chk-fraction.c

@@ -0,0 +1,16 @@
+#include <stdio.h>
+/* Simple unsafe checker with buffer overflow */
+
+int main(int argc, char **argv) {
+    char buf[255], buf2[255];
+    FILE* fdi = fopen(argv[1], "r");
+    FILE* fdo = fopen(argv[2], "r");
+    FILE* fdh = fopen(argv[3], "r");
+    fscanf(fdh, "%s", buf);
+    fscanf(fdo, "%s", buf2);
+    if (strcmp(buf, buf2) == 0)
+        puts("OK\nOK\n84 2");
+    else
+        puts("WRONG");
+    return 0;
+}

sio/workers/test/test_executors.py

@@ -21,7 +21,11 @@
 from sio.executors.common import run as run_executor
 from sio.executors.ingen import run as run_ingen
 from sio.executors.inwer import run as run_inwer
-from sio.executors.checker import RESULT_STRING_LENGTH_LIMIT
+from sio.executors.checker import (
+    RESULT_STRING_LENGTH_LIMIT,
+    output_to_fraction,
+    CheckerError,
+)
 from sio.workers import ft
 from sio.workers.execute import execute
 from sio.workers.executors import (
@@ -350,7 +354,7 @@ def test_truncating_output():
 def _make_untrusted_checkers_cases():
     def ok_42(env):
         res_ok(env)
-        eq_(42, int(env['result_percentage']))
+        eq_(42, int(env['result_percentage'][0] / env['result_percentage'][1]))
 
     # Test if unprotected execution allows for return code 1
     yield '/chk-rtn1.c', None, False, None
@@ -363,6 +367,10 @@ def ok_42(env):
         yield '/open2.c', res_wa, True, None
         # Wrong model solution
         yield '/chk-rtn2.c', None, True, SystemError
+        # Checker with float result percentage
+        yield '/chk-float.c', ok_42, True, None
+        # Checker with fraction result percentage
+        yield '/chk-fraction.c', ok_42, True, None
 
 
 @pytest.mark.parametrize(
@@ -826,3 +834,16 @@ def test_execute():
         eq_(rc, 0)
         rc, out = execute(['ls', tempcwd()])
         in_(b'spam', out)
+
+
+def test_checker_percentage_parsing():
+    eq_(output_to_fraction('42'), (42, 1))
+    eq_(output_to_fraction('42.123'), (42123, 1000))
+    eq_(output_to_fraction('42 21'), (42, 21))
+
+    with pytest.raises(CheckerError):
+        output_to_fraction('42  2')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42,2')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42 2 1')