diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 8a133f7..5e60855 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,4 +1,5 @@ import sys +from dataclasses import dataclass from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path @@ -9,37 +10,71 @@ REQUIREMENTS_BASED_TEST_PREFIX = "rbt-" -class TestSetup: - _INPUT_FOLDER_NAME = "input" - _ARGS_FILE_NAME = "args.txt" - _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" - _EXIT_CODE_FILE_NAME = "exit-code.txt" - _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" - _EXPECTED_STDERR_FILE_NAME = "stderr.txt" - - def __init__(self, test_case_path: str): +@dataclass +class FolderStructure: + """This class knows all paths related to a test setup""" + + test_case_path: str + + INPUT_FOLDER_NAME = "input" + ARGS_FILE_NAME = "args.txt" + EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" + TEMPLATE = "-template" + EXIT_CODE_FILE_NAME = "exit-code.txt" + EXPECTED_STDOUT_FILE_NAME = "stdout.txt" + EXPECTED_STDERR_FILE_NAME = "stderr.txt" + + def __post_init__(self): """Constructor :param test_case_path: the test case path. It must contain subdirectories called "input" and "expected-output". """ - self._input_folder = join(test_case_path, self._INPUT_FOLDER_NAME) - self._test_case_path = test_case_path + self._input_folder = join(self.test_case_path, self.INPUT_FOLDER_NAME) + self._name = join( + basename(dirname(self.test_case_path)), + basename(self.test_case_path), + ) + self._expected_output_path = join( + self.test_case_path, + self.EXPECTED_OUTPUT_FOLDER_NAME, + ) + + @property + def expected_output_path(self) -> str: + """Returns the path of the folder containing the expected output, which can be + used to compare against the actual output""" + return self._expected_output_path + + @property + def name(self) -> str: + """Returns a debug name""" + return self._name + + @property + def input_folder(self) -> str: + """Returns the path to the folder containing all input files for the tool + under test""" + return self._input_folder + + +class TestSetup: + def __init__(self, test_case_structure: FolderStructure): + assert isinstance(test_case_structure, FolderStructure) + self._structure = test_case_structure self._args = self._get_args() self._expected_exit_code = self._get_expected_exit_code() - self._name = self._get_name(test_case_path) self._expected_lobster_output_file_name = \ self._get_expected_lobster_output_file_name() + @property + def structure(self) -> FolderStructure: + return self._structure + @property def expected_lobster_output_file_name(self) -> str: return self._expected_lobster_output_file_name - def get_expected_output_path(self) -> str: - """Returns the path of the folder containing the expected output, which can be - used to compare against the actual output""" - return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME) - def _get_expected_lobster_output_file_name(self) -> str: """Retrieves the expected file name of the lobster output file @@ -49,20 +84,14 @@ def _get_expected_lobster_output_file_name(self) -> str: file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing that is currently not supported. """ - for dir_entry in scandir(self.get_expected_output_path()): + for dir_entry in scandir(self._structure.expected_output_path): if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( f"Invalid test setup: No *.lobster file found in " - f"{self.get_expected_output_path()}!", + f"{self._structure.expected_output_path}!", ) - @staticmethod - def _get_name(test_case_path: str) -> str: - """Creates a debug name for the test case, which consists of the RBT folder name - and the test case folder name""" - return join(basename(dirname(test_case_path)), basename(test_case_path)) - @property def args(self) -> List[str]: """Returns the command line arguments which must be used to start the tool under @@ -73,7 +102,7 @@ def args(self) -> List[str]: def name(self) -> str: """Returns the name of the test case, which is equal to the folder name containing the test case""" - return self._name + return self._structure.name @property def expected_exit_code(self) -> int: @@ -83,7 +112,7 @@ def expected_exit_code(self) -> int: def _get_args(self) -> List[str]: """Reads the command line arguments (which must be used to start the tool under test) from the corresponding test setup file""" - file = join(self._input_folder, self._ARGS_FILE_NAME) + file = join(self._structure.input_folder, self._structure.ARGS_FILE_NAME) with open(file, "r", encoding="UTF-8") as file: return [argument.strip() for argument in file.readlines()] @@ -91,8 +120,8 @@ def get_expected_stdout(self) -> str: """Reads the expected command line output (for stdout) from the corresponding test setup file""" cmd_file = join( - self.get_expected_output_path(), - self._EXPECTED_STDOUT_FILE_NAME, + self._structure.expected_output_path, + self._structure.EXPECTED_STDOUT_FILE_NAME, ) with open(cmd_file, "r", encoding="UTF-8") as file: return file.read() @@ -101,8 +130,8 @@ def get_expected_stderr(self) -> str: """Reads the expected command line output (for stderr) from the corresponding test setup file""" errout_file = join( - self.get_expected_output_path(), - self._EXPECTED_STDERR_FILE_NAME, + self._structure.expected_output_path, + self._structure.EXPECTED_STDERR_FILE_NAME, ) with open(errout_file, "r", encoding="UTF-8") as file: return file.read() @@ -110,14 +139,14 @@ def get_expected_stderr(self) -> str: @property def input_folder(self) -> str: """Returns the path containing the input data for the test""" - return self._input_folder + return self._structure.input_folder def _get_expected_exit_code(self) -> int: """Returns the expectation of the tool exit code""" expected_exit_code_file = join( - self._test_case_path, - self._EXPECTED_OUTPUT_FOLDER_NAME, - self._EXIT_CODE_FILE_NAME, + self._structure.test_case_path, + self._structure.EXPECTED_OUTPUT_FOLDER_NAME, + self._structure.EXIT_CODE_FILE_NAME, ) with open(expected_exit_code_file, "r", encoding="UTF-8") as file: return int(file.readline()) @@ -178,7 +207,7 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess): ), "Command line output for STDERR is different!" expected = join( - setup.get_expected_output_path(), + setup.structure.expected_output_path, setup.expected_lobster_output_file_name, ) actual = join(setup.input_folder, setup.expected_lobster_output_file_name) @@ -225,6 +254,21 @@ def _delete_generated_files(setup: TestSetup): remove(generated) +def _prepare_test(structure: FolderStructure): + system_test_root = Path(".").absolute().resolve().as_posix() + try: + for dir_entry in scandir(structure.expected_output_path + structure.TEMPLATE): + if dir_entry.is_file: + print(f"Preparing file based on template {dir_entry.path}") + with open(dir_entry.path, "r", encoding="UTF-8") as file: + content = file.read().replace("TESTS-SYSTEM", system_test_root) + target_path = dir_entry.path.replace(structure.TEMPLATE, "") + with open(target_path, "w", encoding="UTF-8") as target_file: + target_file.write(content) + except FileNotFoundError: + pass + + def _run_tests(directory: str, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. @@ -240,7 +284,9 @@ def _run_tests(directory: str, tool: str) -> int: counter = 0 for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX): for test_case_dir_entry in _get_directories(rbt_dir_entry.path): - test_setup = TestSetup(test_case_dir_entry.path) + structure = FolderStructure(test_case_dir_entry.path) + _prepare_test(structure) + test_setup = TestSetup(structure) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) _delete_generated_files(test_setup)