Validate test contents (#192)

* Add test contents validation

* Fix tests

* Add tests

* Bump version

* Add checks for carriage return

src/sinol_make/__init__.py

@@ -9,7 +9,7 @@
 from sinol_make import util, oiejq
 
 
-__version__ = "1.5.25"
+__version__ = "1.5.26"
 
 
 def configure_parsers():

src/sinol_make/commands/gen/__init__.py

@@ -35,6 +35,8 @@ def configure_subparser(self, subparser):
                             help=f'number of cpus to use to generate output files '
                                  f'(default: {util.default_cpu_count()})',
                             default=util.default_cpu_count())
+        parser.add_argument('-n', '--no-validate', default=False, action='store_true',
+                            help='do not validate test contents')
         parsers.add_compilation_arguments(parser)
 
     def run(self, args: argparse.Namespace):

src/sinol_make/commands/ingen/__init__.py

@@ -1,4 +1,5 @@
 import argparse
+import glob
 import os
 
 from sinol_make import util
@@ -27,6 +28,8 @@ def configure_subparser(self, subparser):
 
         parser.add_argument('ingen_path', type=str, nargs='?',
                             help='path to ingen source file, for example prog/abcingen.cpp')
+        parser.add_argument('-n', '--no-validate', default=False, action='store_true',
+                            help='do not validate test contents')
         parsers.add_compilation_arguments(parser)
 
     def run(self, args: argparse.Namespace):
@@ -45,3 +48,10 @@ def run(self, args: argparse.Namespace):
             print(util.info('Successfully generated input files.'))
         else:
             util.exit_with_error('Failed to generate input files.')
+
+        if not self.args.no_validate:
+            print(util.info('Validating input test contents.'))
+            tests = sorted(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")))
+            for test in tests:
+                package_util.validate_test(test)
+            print(util.info('Input test contents are valid!'))

src/sinol_make/commands/outgen/__init__.py

@@ -31,6 +31,8 @@ def configure_subparser(self, subparser):
                             help=f'number of cpus to use to generate output files '
                                  f'(default: {util.default_cpu_count()})',
                             default=util.default_cpu_count())
+        parser.add_argument('-n', '--no-validate', default=False, action='store_true',
+                            help='do not validate test contents')
         parsers.add_compilation_arguments(parser)
 
     def generate_outputs(self, outputs_to_generate):
@@ -108,3 +110,9 @@ def run(self, args: argparse.Namespace):
             self.generate_outputs(outputs_to_generate)
             with open(os.path.join(os.getcwd(), 'in', '.md5sums'), 'w') as f:
                 yaml.dump(md5_sums, f)
+
+        if not self.args.no_validate:
+            print(util.info('Validating output test contents.'))
+            for test in sorted(outputs_to_generate):
+                package_util.validate_test(test)
+            print(util.info('Output test contents are valid!'))

src/sinol_make/helpers/package_util.py

@@ -343,3 +343,36 @@ def save_contest_type_to_cache(contest_type):
     os.makedirs(paths.get_cache_path(), exist_ok=True)
     with open(paths.get_cache_path("contest_type"), "w") as contest_type_file:
         contest_type_file.write(contest_type)
+
+
+def validate_test(test_path: str):
+    """
+    Check if test doesn't contain leading/trailing whitespaces,
+    has only one space between tokens and ends with newline.
+    Exits with error if any of the conditions is not met.
+    """
+    basename = os.path.basename(test_path)
+    num_empty = 0
+    with open(test_path, 'br') as file:
+        lines = file.readlines()
+        for i, line in enumerate(lines):
+            line = line.decode('utf-8')
+            if len(line) > 0 and line[0] == ' ':
+                util.exit_with_error(f'Leading whitespace in {basename}:{i + 1}')
+            if len(line) > 0 and (line[-2:] == '\r\n' or line[-2:] == '\n\r' or line[-1] == '\r'):
+                util.exit_with_error(f'Carriage return at the end of {basename}:{i + 1}')
+            if len(line) > 0 and line[-1] != '\n':
+                util.exit_with_error(f'No newline at the end of {basename}')
+            if line == '\n' or line == '':
+                num_empty += 1
+                continue
+            elif i == len(lines) - 1:
+                num_empty = 0
+            if line[-2] == ' ':
+                util.exit_with_error(f'Trailing whitespace in {basename}:{i + 1}')
+            for j in range(len(line) - 1):
+                if line[j] == ' ' and line[j + 1] == ' ':
+                    util.exit_with_error(f'Tokens not separated by one space in {basename}:{i + 1}')
+
+        if num_empty != 0:
+            util.exit_with_error(f'Exactly one empty line expected in {basename}')

tests/commands/gen/test_integration.py

@@ -247,3 +247,48 @@ def test_fsanitize(create_package):
             simple_run([ingen])
         assert e.type == SystemExit
         assert e.value.code == 1
+
+
+@pytest.mark.parametrize("create_package", [util.get_bad_tests_package_path()], indirect=True)
+def test_bad_tests(create_package, capsys):
+    """
+    Test if validation of test contents works.
+    """
+
+    # Gen should fail
+    with pytest.raises(SystemExit) as e:
+        simple_run()
+    assert e.type == SystemExit
+    assert e.value.code == 1
+    out = capsys.readouterr().out
+    assert "Trailing whitespace in bad0.in:1" in out
+
+    # Generate tests without validation
+    simple_run(["--no-validate"], command="ingen")
+
+    # (program, should fail, error message)
+    tests = [
+        ("bad.cpp", False, ""),
+        ("bad1.cpp", True, "Trailing whitespace in bad0.out:1"),
+        ("bad2.cpp", True, "Leading whitespace in bad0.out:1"),
+        ("bad3.cpp", True, "Tokens not separated by one space in bad0.out:1"),
+        ("bad4.cpp", True, "No newline at the end of bad0.out"),
+        ("bad5.cpp", True, "Exactly one empty line expected in bad0.out"),
+    ]
+
+    for program, should_fail, error_message in tests:
+        if program != "bad.cpp":
+            shutil.copyfile(os.path.join(create_package, "prog", program), os.path.join(create_package, "prog", "bad.cpp"))
+        if not should_fail:
+            simple_run(command="outgen")
+        else:
+            with pytest.raises(SystemExit) as e:
+                simple_run(command="outgen")
+            assert e.type == SystemExit
+            assert e.value.code == 1
+            out = capsys.readouterr().out
+            assert error_message in out
+
+        for file in glob.glob(os.path.join(create_package, "out", "*.out")):
+            os.unlink(file)
+        os.unlink(os.path.join(create_package, "in", ".md5sums"))

tests/commands/gen/test_unit.py

@@ -113,3 +113,41 @@ def test_generate_output(create_package):
     run_ingen(ingen_exe)
     assert generate_output(OutputGenerationArguments(correct_sol_exe, "in/abc1a.in", "out/abc1a.out"))
     assert os.path.exists(os.path.join(package_path, "out", "abc1a.out"))
+
+
+@pytest.mark.parametrize("create_package", [util.get_bad_tests_package_path()], indirect=True)
+def test_validate_tests(create_package, capsys):
+    """
+    Test validating test contents.
+    """
+    package_path = create_package
+    task_id = package_util.get_task_id()
+    ingen_path = get_ingen(task_id)
+    args = compiler.get_default_compilers()
+    ingen_exe = compile_ingen(ingen_path, args)
+    run_ingen(ingen_exe)
+
+    with open("in/bad6.in", "w") as f:
+        f.write("1\n\n2 \n")
+    with open("in/bad7.in", "w") as f:
+        f.write("1 1\r\n")
+
+    # (Test, error message)
+    tests = [
+        ("bad0.in", "Trailing whitespace in bad0.in:1"),
+        ("bad1.in", "Leading whitespace in bad1.in:1"),
+        ("bad2.in", "Tokens not separated by one space in bad2.in:1"),
+        ("bad3.in", "Exactly one empty line expected in bad3.in"),
+        ("bad4.in", "Trailing whitespace in bad4.in:2"),
+        ("bad5.in", "No newline at the end of bad5.in"),
+        ("bad6.in", "Trailing whitespace in bad6.in:3"),
+        ("bad7.in", "Carriage return at the end of bad7.in:1"),
+    ]
+
+    for test, error in tests:
+        with pytest.raises(SystemExit) as e:
+            package_util.validate_test(os.path.join(package_path, "in", test))
+        assert e.type == SystemExit
+        assert e.value.code == 1
+        captured = capsys.readouterr()
+        assert error in captured.out, f"Expected error not found in output: {captured.out}"

tests/packages/abc/prog/abc.cpp

@@ -5,5 +5,5 @@ using namespace std;
 int main() {
 	int a, b;
 	cin >> a >> b;
-	cout << a + b;
+	cout << a + b << "\n";
 }

tests/packages/bad_tests/config.yml

@@ -0,0 +1,2 @@
+title: Package with bad tests
+sinol_task_id: bad

tests/packages/bad_tests/prog/bad.cpp

@@ -0,0 +1,7 @@
+#include <bits/stdc++.h>
+using namespace std;
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b << endl;
+}

tests/packages/bad_tests/prog/bad1.cpp

@@ -0,0 +1,7 @@
+#include <bits/stdc++.h>
+using namespace std;
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b << " \n";
+}

tests/packages/bad_tests/prog/bad2.cpp

@@ -0,0 +1,7 @@
+#include <bits/stdc++.h>
+using namespace std;
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << "  " << a + b << endl;
+}

tests/packages/bad_tests/prog/bad3.cpp

@@ -0,0 +1,7 @@
+#include <bits/stdc++.h>
+using namespace std;
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b << "    " << a + b << endl;
+}

tests/packages/bad_tests/prog/bad4.cpp

@@ -0,0 +1,7 @@
+#include <bits/stdc++.h>
+using namespace std;
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b;
+}

tests/packages/bad_tests/prog/bad5.cpp

@@ -0,0 +1,7 @@
+#include <bits/stdc++.h>
+using namespace std;
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b << "\n\n\n";
+}

tests/packages/bad_tests/prog/badingen.cpp

@@ -0,0 +1,24 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    ofstream f("bad0.in");
+    f << "1 1 \n";
+    f.close();
+    f.open("bad1.in");
+    f << "   1 1\n";
+    f.close();
+    f.open("bad2.in");
+    f << "1  1\n";
+    f.close();
+    f.open("bad3.in");
+    f << "1 1\n\n\n";
+    f.close();
+    f.open("bad4.in");
+    f << "1 1\n1 1 \n";
+    f.close();
+    f.open("bad5.in");
+    f << "1 1";
+    f.close();
+}

tests/packages/gen/prog/gen.cpp

@@ -5,5 +5,5 @@ using namespace std;
 int main() {
     int a, b;
     cin >> a >> b;
-    cout << a + b;
+    cout << a + b << "\n";
 }

tests/util.py

@@ -143,6 +143,13 @@ def get_ocen_package_path():
     return os.path.join(os.path.dirname(__file__), "packages", "ocen")
 
 
+def get_bad_tests_package_path():
+    """
+    Get path to package with bad tests (/tests/packages/bad_tests)
+    """
+    return os.path.join(os.path.dirname(__file__), "packages", "bad_tests")
+
+
 def create_ins(package_path, task_id):
     """
     Create .in files for package.