From 0ed77a26199ddfebf70bb33105d221c46cbe9353 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Mon, 19 Feb 2024 22:02:47 +0100 Subject: [PATCH] Validate test contents (#192) * Add test contents validation * Fix tests * Add tests * Bump version * Add checks for carriage return --- src/sinol_make/__init__.py | 2 +- src/sinol_make/commands/gen/__init__.py | 2 + src/sinol_make/commands/ingen/__init__.py | 10 +++++ src/sinol_make/commands/outgen/__init__.py | 8 ++++ src/sinol_make/helpers/package_util.py | 33 ++++++++++++++++ tests/commands/gen/test_integration.py | 45 ++++++++++++++++++++++ tests/commands/gen/test_unit.py | 38 ++++++++++++++++++ tests/packages/abc/prog/abc.cpp | 2 +- tests/packages/bad_tests/config.yml | 2 + tests/packages/bad_tests/in/.gitkeep | 0 tests/packages/bad_tests/out/.gitkeep | 0 tests/packages/bad_tests/prog/bad.cpp | 7 ++++ tests/packages/bad_tests/prog/bad1.cpp | 7 ++++ tests/packages/bad_tests/prog/bad2.cpp | 7 ++++ tests/packages/bad_tests/prog/bad3.cpp | 7 ++++ tests/packages/bad_tests/prog/bad4.cpp | 7 ++++ tests/packages/bad_tests/prog/bad5.cpp | 7 ++++ tests/packages/bad_tests/prog/badingen.cpp | 24 ++++++++++++ tests/packages/gen/prog/gen.cpp | 2 +- tests/util.py | 7 ++++ 20 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 tests/packages/bad_tests/config.yml create mode 100644 tests/packages/bad_tests/in/.gitkeep create mode 100644 tests/packages/bad_tests/out/.gitkeep create mode 100644 tests/packages/bad_tests/prog/bad.cpp create mode 100644 tests/packages/bad_tests/prog/bad1.cpp create mode 100644 tests/packages/bad_tests/prog/bad2.cpp create mode 100644 tests/packages/bad_tests/prog/bad3.cpp create mode 100644 tests/packages/bad_tests/prog/bad4.cpp create mode 100644 tests/packages/bad_tests/prog/bad5.cpp create mode 100644 tests/packages/bad_tests/prog/badingen.cpp diff --git a/src/sinol_make/__init__.py b/src/sinol_make/__init__.py index 5abbb377..33997737 100644 --- a/src/sinol_make/__init__.py +++ b/src/sinol_make/__init__.py @@ -9,7 +9,7 @@ from sinol_make import util, oiejq -__version__ = "1.5.25" +__version__ = "1.5.26" def configure_parsers(): diff --git a/src/sinol_make/commands/gen/__init__.py b/src/sinol_make/commands/gen/__init__.py index c477f265..abfe6f0f 100644 --- a/src/sinol_make/commands/gen/__init__.py +++ b/src/sinol_make/commands/gen/__init__.py @@ -35,6 +35,8 @@ def configure_subparser(self, subparser): help=f'number of cpus to use to generate output files ' f'(default: {util.default_cpu_count()})', default=util.default_cpu_count()) + parser.add_argument('-n', '--no-validate', default=False, action='store_true', + help='do not validate test contents') parsers.add_compilation_arguments(parser) def run(self, args: argparse.Namespace): diff --git a/src/sinol_make/commands/ingen/__init__.py b/src/sinol_make/commands/ingen/__init__.py index ca69c0da..6fff04f4 100644 --- a/src/sinol_make/commands/ingen/__init__.py +++ b/src/sinol_make/commands/ingen/__init__.py @@ -1,4 +1,5 @@ import argparse +import glob import os from sinol_make import util @@ -27,6 +28,8 @@ def configure_subparser(self, subparser): parser.add_argument('ingen_path', type=str, nargs='?', help='path to ingen source file, for example prog/abcingen.cpp') + parser.add_argument('-n', '--no-validate', default=False, action='store_true', + help='do not validate test contents') parsers.add_compilation_arguments(parser) def run(self, args: argparse.Namespace): @@ -45,3 +48,10 @@ def run(self, args: argparse.Namespace): print(util.info('Successfully generated input files.')) else: util.exit_with_error('Failed to generate input files.') + + if not self.args.no_validate: + print(util.info('Validating input test contents.')) + tests = sorted(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in"))) + for test in tests: + package_util.validate_test(test) + print(util.info('Input test contents are valid!')) diff --git a/src/sinol_make/commands/outgen/__init__.py b/src/sinol_make/commands/outgen/__init__.py index dfa6676c..dc8a4a99 100644 --- a/src/sinol_make/commands/outgen/__init__.py +++ b/src/sinol_make/commands/outgen/__init__.py @@ -31,6 +31,8 @@ def configure_subparser(self, subparser): help=f'number of cpus to use to generate output files ' f'(default: {util.default_cpu_count()})', default=util.default_cpu_count()) + parser.add_argument('-n', '--no-validate', default=False, action='store_true', + help='do not validate test contents') parsers.add_compilation_arguments(parser) def generate_outputs(self, outputs_to_generate): @@ -108,3 +110,9 @@ def run(self, args: argparse.Namespace): self.generate_outputs(outputs_to_generate) with open(os.path.join(os.getcwd(), 'in', '.md5sums'), 'w') as f: yaml.dump(md5_sums, f) + + if not self.args.no_validate: + print(util.info('Validating output test contents.')) + for test in sorted(outputs_to_generate): + package_util.validate_test(test) + print(util.info('Output test contents are valid!')) diff --git a/src/sinol_make/helpers/package_util.py b/src/sinol_make/helpers/package_util.py index 2a69d52d..d155e746 100644 --- a/src/sinol_make/helpers/package_util.py +++ b/src/sinol_make/helpers/package_util.py @@ -343,3 +343,36 @@ def save_contest_type_to_cache(contest_type): os.makedirs(paths.get_cache_path(), exist_ok=True) with open(paths.get_cache_path("contest_type"), "w") as contest_type_file: contest_type_file.write(contest_type) + + +def validate_test(test_path: str): + """ + Check if test doesn't contain leading/trailing whitespaces, + has only one space between tokens and ends with newline. + Exits with error if any of the conditions is not met. + """ + basename = os.path.basename(test_path) + num_empty = 0 + with open(test_path, 'br') as file: + lines = file.readlines() + for i, line in enumerate(lines): + line = line.decode('utf-8') + if len(line) > 0 and line[0] == ' ': + util.exit_with_error(f'Leading whitespace in {basename}:{i + 1}') + if len(line) > 0 and (line[-2:] == '\r\n' or line[-2:] == '\n\r' or line[-1] == '\r'): + util.exit_with_error(f'Carriage return at the end of {basename}:{i + 1}') + if len(line) > 0 and line[-1] != '\n': + util.exit_with_error(f'No newline at the end of {basename}') + if line == '\n' or line == '': + num_empty += 1 + continue + elif i == len(lines) - 1: + num_empty = 0 + if line[-2] == ' ': + util.exit_with_error(f'Trailing whitespace in {basename}:{i + 1}') + for j in range(len(line) - 1): + if line[j] == ' ' and line[j + 1] == ' ': + util.exit_with_error(f'Tokens not separated by one space in {basename}:{i + 1}') + + if num_empty != 0: + util.exit_with_error(f'Exactly one empty line expected in {basename}') diff --git a/tests/commands/gen/test_integration.py b/tests/commands/gen/test_integration.py index 2554fe9c..463725b0 100644 --- a/tests/commands/gen/test_integration.py +++ b/tests/commands/gen/test_integration.py @@ -247,3 +247,48 @@ def test_fsanitize(create_package): simple_run([ingen]) assert e.type == SystemExit assert e.value.code == 1 + + +@pytest.mark.parametrize("create_package", [util.get_bad_tests_package_path()], indirect=True) +def test_bad_tests(create_package, capsys): + """ + Test if validation of test contents works. + """ + + # Gen should fail + with pytest.raises(SystemExit) as e: + simple_run() + assert e.type == SystemExit + assert e.value.code == 1 + out = capsys.readouterr().out + assert "Trailing whitespace in bad0.in:1" in out + + # Generate tests without validation + simple_run(["--no-validate"], command="ingen") + + # (program, should fail, error message) + tests = [ + ("bad.cpp", False, ""), + ("bad1.cpp", True, "Trailing whitespace in bad0.out:1"), + ("bad2.cpp", True, "Leading whitespace in bad0.out:1"), + ("bad3.cpp", True, "Tokens not separated by one space in bad0.out:1"), + ("bad4.cpp", True, "No newline at the end of bad0.out"), + ("bad5.cpp", True, "Exactly one empty line expected in bad0.out"), + ] + + for program, should_fail, error_message in tests: + if program != "bad.cpp": + shutil.copyfile(os.path.join(create_package, "prog", program), os.path.join(create_package, "prog", "bad.cpp")) + if not should_fail: + simple_run(command="outgen") + else: + with pytest.raises(SystemExit) as e: + simple_run(command="outgen") + assert e.type == SystemExit + assert e.value.code == 1 + out = capsys.readouterr().out + assert error_message in out + + for file in glob.glob(os.path.join(create_package, "out", "*.out")): + os.unlink(file) + os.unlink(os.path.join(create_package, "in", ".md5sums")) diff --git a/tests/commands/gen/test_unit.py b/tests/commands/gen/test_unit.py index 84107526..662ebeb4 100644 --- a/tests/commands/gen/test_unit.py +++ b/tests/commands/gen/test_unit.py @@ -113,3 +113,41 @@ def test_generate_output(create_package): run_ingen(ingen_exe) assert generate_output(OutputGenerationArguments(correct_sol_exe, "in/abc1a.in", "out/abc1a.out")) assert os.path.exists(os.path.join(package_path, "out", "abc1a.out")) + + +@pytest.mark.parametrize("create_package", [util.get_bad_tests_package_path()], indirect=True) +def test_validate_tests(create_package, capsys): + """ + Test validating test contents. + """ + package_path = create_package + task_id = package_util.get_task_id() + ingen_path = get_ingen(task_id) + args = compiler.get_default_compilers() + ingen_exe = compile_ingen(ingen_path, args) + run_ingen(ingen_exe) + + with open("in/bad6.in", "w") as f: + f.write("1\n\n2 \n") + with open("in/bad7.in", "w") as f: + f.write("1 1\r\n") + + # (Test, error message) + tests = [ + ("bad0.in", "Trailing whitespace in bad0.in:1"), + ("bad1.in", "Leading whitespace in bad1.in:1"), + ("bad2.in", "Tokens not separated by one space in bad2.in:1"), + ("bad3.in", "Exactly one empty line expected in bad3.in"), + ("bad4.in", "Trailing whitespace in bad4.in:2"), + ("bad5.in", "No newline at the end of bad5.in"), + ("bad6.in", "Trailing whitespace in bad6.in:3"), + ("bad7.in", "Carriage return at the end of bad7.in:1"), + ] + + for test, error in tests: + with pytest.raises(SystemExit) as e: + package_util.validate_test(os.path.join(package_path, "in", test)) + assert e.type == SystemExit + assert e.value.code == 1 + captured = capsys.readouterr() + assert error in captured.out, f"Expected error not found in output: {captured.out}" diff --git a/tests/packages/abc/prog/abc.cpp b/tests/packages/abc/prog/abc.cpp index 20d981dc..78de4f8f 100644 --- a/tests/packages/abc/prog/abc.cpp +++ b/tests/packages/abc/prog/abc.cpp @@ -5,5 +5,5 @@ using namespace std; int main() { int a, b; cin >> a >> b; - cout << a + b; + cout << a + b << "\n"; } diff --git a/tests/packages/bad_tests/config.yml b/tests/packages/bad_tests/config.yml new file mode 100644 index 00000000..5e888e3d --- /dev/null +++ b/tests/packages/bad_tests/config.yml @@ -0,0 +1,2 @@ +title: Package with bad tests +sinol_task_id: bad diff --git a/tests/packages/bad_tests/in/.gitkeep b/tests/packages/bad_tests/in/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/bad_tests/out/.gitkeep b/tests/packages/bad_tests/out/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/bad_tests/prog/bad.cpp b/tests/packages/bad_tests/prog/bad.cpp new file mode 100644 index 00000000..b44e7f35 --- /dev/null +++ b/tests/packages/bad_tests/prog/bad.cpp @@ -0,0 +1,7 @@ +#include +using namespace std; +int main() { + int a, b; + cin >> a >> b; + cout << a + b << endl; +} diff --git a/tests/packages/bad_tests/prog/bad1.cpp b/tests/packages/bad_tests/prog/bad1.cpp new file mode 100644 index 00000000..f4890d8f --- /dev/null +++ b/tests/packages/bad_tests/prog/bad1.cpp @@ -0,0 +1,7 @@ +#include +using namespace std; +int main() { + int a, b; + cin >> a >> b; + cout << a + b << " \n"; +} diff --git a/tests/packages/bad_tests/prog/bad2.cpp b/tests/packages/bad_tests/prog/bad2.cpp new file mode 100644 index 00000000..a56e7dc1 --- /dev/null +++ b/tests/packages/bad_tests/prog/bad2.cpp @@ -0,0 +1,7 @@ +#include +using namespace std; +int main() { + int a, b; + cin >> a >> b; + cout << " " << a + b << endl; +} diff --git a/tests/packages/bad_tests/prog/bad3.cpp b/tests/packages/bad_tests/prog/bad3.cpp new file mode 100644 index 00000000..07c1dd4f --- /dev/null +++ b/tests/packages/bad_tests/prog/bad3.cpp @@ -0,0 +1,7 @@ +#include +using namespace std; +int main() { + int a, b; + cin >> a >> b; + cout << a + b << " " << a + b << endl; +} diff --git a/tests/packages/bad_tests/prog/bad4.cpp b/tests/packages/bad_tests/prog/bad4.cpp new file mode 100644 index 00000000..36cc2343 --- /dev/null +++ b/tests/packages/bad_tests/prog/bad4.cpp @@ -0,0 +1,7 @@ +#include +using namespace std; +int main() { + int a, b; + cin >> a >> b; + cout << a + b; +} diff --git a/tests/packages/bad_tests/prog/bad5.cpp b/tests/packages/bad_tests/prog/bad5.cpp new file mode 100644 index 00000000..9c3ad90a --- /dev/null +++ b/tests/packages/bad_tests/prog/bad5.cpp @@ -0,0 +1,7 @@ +#include +using namespace std; +int main() { + int a, b; + cin >> a >> b; + cout << a + b << "\n\n\n"; +} diff --git a/tests/packages/bad_tests/prog/badingen.cpp b/tests/packages/bad_tests/prog/badingen.cpp new file mode 100644 index 00000000..93d0534f --- /dev/null +++ b/tests/packages/bad_tests/prog/badingen.cpp @@ -0,0 +1,24 @@ +#include + +using namespace std; + +int main() { + ofstream f("bad0.in"); + f << "1 1 \n"; + f.close(); + f.open("bad1.in"); + f << " 1 1\n"; + f.close(); + f.open("bad2.in"); + f << "1 1\n"; + f.close(); + f.open("bad3.in"); + f << "1 1\n\n\n"; + f.close(); + f.open("bad4.in"); + f << "1 1\n1 1 \n"; + f.close(); + f.open("bad5.in"); + f << "1 1"; + f.close(); +} diff --git a/tests/packages/gen/prog/gen.cpp b/tests/packages/gen/prog/gen.cpp index bd410607..a40c98d0 100644 --- a/tests/packages/gen/prog/gen.cpp +++ b/tests/packages/gen/prog/gen.cpp @@ -5,5 +5,5 @@ using namespace std; int main() { int a, b; cin >> a >> b; - cout << a + b; + cout << a + b << "\n"; } diff --git a/tests/util.py b/tests/util.py index 22c24ca1..5164849a 100644 --- a/tests/util.py +++ b/tests/util.py @@ -143,6 +143,13 @@ def get_ocen_package_path(): return os.path.join(os.path.dirname(__file__), "packages", "ocen") +def get_bad_tests_package_path(): + """ + Get path to package with bad tests (/tests/packages/bad_tests) + """ + return os.path.join(os.path.dirname(__file__), "packages", "bad_tests") + + def create_ins(package_path, task_id): """ Create .in files for package.