Return result percentage as fraction (#24)

* Result percentage changed to a fraction

* Fix typos

* Extend tests

* More tests

* More error checking

* Simplified result percentage string parsing.

* Added division by zero error handling.

* Added and changed tests.

* fix: added fraction package

* fixed attribute error

* fixed type error

* fixed tests

---------

Co-authored-by: Zonkil <[email protected]>

sio/executors/checker.py

@@ -3,6 +3,8 @@
 import logging
 import tempfile
 import six
+import re
+from fractions import Fraction
 
 from sio.workers import ft
 from sio.workers.executors import (
@@ -129,9 +131,28 @@ def run(environ, use_sandboxes=True):
         environ['result_code'] = 'OK'
         if output[1]:
             environ['result_string'] = _limit_length(output[1])
-        environ['result_percentage'] = float(output[2] or 100)
+        environ['result_percentage'] = output_to_fraction(output[2])
     else:
         environ['result_code'] = 'WA'
         environ['result_string'] = _limit_length(output[1])
-        environ['result_percentage'] = 0
+        environ['result_percentage'] = (0, 1)
     return environ
+
+
+def output_to_fraction(output_str):
+    if not output_str:
+        return 100, 1
+    if isinstance(output_str, bytes):
+        output_str = output_str.decode('utf-8')
+    try:
+        frac = Fraction(output_str)
+        return frac.numerator, frac.denominator
+    except ValueError:
+        raise CheckerError(
+            'Invalid checker output, expected float, percent or fraction, got "%s"'
+            % output_str
+        )
+    except ZeroDivisionError:
+        raise CheckerError('Zero division in checker output "%s"' % output_str)
+    except TypeError:
+        raise CheckerError('Invalid checker output "%s"' % output_str)

sio/workers/test/sources/chk-float.c

@@ -0,0 +1,16 @@
+#include <stdio.h>
+/* Simple unsafe checker with buffer overflow */
+
+int main(int argc, char **argv) {
+    char buf[255], buf2[255];
+    FILE* fdi = fopen(argv[1], "r");
+    FILE* fdo = fopen(argv[2], "r");
+    FILE* fdh = fopen(argv[3], "r");
+    fscanf(fdh, "%s", buf);
+    fscanf(fdo, "%s", buf2);
+    if (strcmp(buf, buf2) == 0)
+        puts("OK\nOK\n42.00");
+    else
+        puts("WRONG");
+    return 0;
+}

sio/workers/test/sources/chk-fraction.c

@@ -0,0 +1,16 @@
+#include <stdio.h>
+/* Simple unsafe checker with buffer overflow */
+
+int main(int argc, char **argv) {
+    char buf[255], buf2[255];
+    FILE* fdi = fopen(argv[1], "r");
+    FILE* fdo = fopen(argv[2], "r");
+    FILE* fdh = fopen(argv[3], "r");
+    fscanf(fdh, "%s", buf);
+    fscanf(fdo, "%s", buf2);
+    if (strcmp(buf, buf2) == 0)
+        puts("OK\nOK\n84/2");
+    else
+        puts("WRONG");
+    return 0;
+}

sio/workers/test/test_executors.py

@@ -21,7 +21,11 @@
 from sio.executors.common import run as run_executor
 from sio.executors.ingen import run as run_ingen
 from sio.executors.inwer import run as run_inwer
-from sio.executors.checker import RESULT_STRING_LENGTH_LIMIT
+from sio.executors.checker import (
+    RESULT_STRING_LENGTH_LIMIT,
+    output_to_fraction,
+    CheckerError,
+)
 from sio.workers import ft
 from sio.workers.execute import execute
 from sio.workers.executors import (
@@ -350,7 +354,7 @@ def test_truncating_output():
 def _make_untrusted_checkers_cases():
     def ok_42(env):
         res_ok(env)
-        eq_(42, int(env['result_percentage']))
+        eq_(42, int(env['result_percentage'][0] / env['result_percentage'][1]))
 
     # Test if unprotected execution allows for return code 1
     yield '/chk-rtn1.c', None, False, None
@@ -363,6 +367,10 @@ def ok_42(env):
         yield '/open2.c', res_wa, True, None
         # Wrong model solution
         yield '/chk-rtn2.c', None, True, SystemError
+        # Checker with float result percentage
+        yield '/chk-float.c', ok_42, True, None
+        # Checker with fraction result percentage
+        yield '/chk-fraction.c', ok_42, True, None
 
 
 @pytest.mark.parametrize(
@@ -826,3 +834,38 @@ def test_execute():
         eq_(rc, 0)
         rc, out = execute(['ls', tempcwd()])
         in_(b'spam', out)
+
+
+def test_checker_percentage_parsing():
+    eq_(output_to_fraction('42'), (42, 1))
+    eq_(output_to_fraction('42.123'), (42123, 1000))
+    eq_(output_to_fraction('42/21'), (2, 1))
+    eq_(output_to_fraction('42.'), (42, 1))
+    eq_(output_to_fraction('007'), (7, 1))
+    eq_(output_to_fraction('007/0042'), (1, 6))
+    eq_(output_to_fraction('1e5'), (100000, 1))
+    eq_(output_to_fraction(''), (100, 1))
+
+    with pytest.raises(CheckerError):
+        output_to_fraction('42 2')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42,2')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42 2 1')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42/2/1')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42/2.1')
+
+    with pytest.raises(CheckerError):
+        output_to_fraction('42/')
+    with pytest.raises(CheckerError):
+        output_to_fraction('/42')
+    with pytest.raises(CheckerError):
+        output_to_fraction('/')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42/0')
+    with pytest.raises(CheckerError):
+        output_to_fraction('abc')
+    with pytest.raises(CheckerError):
+        output_to_fraction('42/abc')