diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 5e60855..51e6577 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,5 +1,5 @@ +from posixpath import normpath import sys -from dataclasses import dataclass from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path @@ -10,71 +10,37 @@ REQUIREMENTS_BASED_TEST_PREFIX = "rbt-" -@dataclass -class FolderStructure: - """This class knows all paths related to a test setup""" - - test_case_path: str - - INPUT_FOLDER_NAME = "input" - ARGS_FILE_NAME = "args.txt" - EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" - TEMPLATE = "-template" - EXIT_CODE_FILE_NAME = "exit-code.txt" - EXPECTED_STDOUT_FILE_NAME = "stdout.txt" - EXPECTED_STDERR_FILE_NAME = "stderr.txt" - - def __post_init__(self): +class TestSetup: + _INPUT_FOLDER_NAME = "input" + _ARGS_FILE_NAME = "args.txt" + _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" + _EXIT_CODE_FILE_NAME = "exit-code.txt" + _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" + _EXPECTED_STDERR_FILE_NAME = "stderr.txt" + + def __init__(self, test_case_path: str): """Constructor :param test_case_path: the test case path. It must contain subdirectories called "input" and "expected-output". """ - self._input_folder = join(self.test_case_path, self.INPUT_FOLDER_NAME) - self._name = join( - basename(dirname(self.test_case_path)), - basename(self.test_case_path), - ) - self._expected_output_path = join( - self.test_case_path, - self.EXPECTED_OUTPUT_FOLDER_NAME, - ) - - @property - def expected_output_path(self) -> str: - """Returns the path of the folder containing the expected output, which can be - used to compare against the actual output""" - return self._expected_output_path - - @property - def name(self) -> str: - """Returns a debug name""" - return self._name - - @property - def input_folder(self) -> str: - """Returns the path to the folder containing all input files for the tool - under test""" - return self._input_folder - - -class TestSetup: - def __init__(self, test_case_structure: FolderStructure): - assert isinstance(test_case_structure, FolderStructure) - self._structure = test_case_structure + self._input_folder = join(test_case_path, self._INPUT_FOLDER_NAME) + self._test_case_path = test_case_path self._args = self._get_args() self._expected_exit_code = self._get_expected_exit_code() + self._name = self._get_name(test_case_path) self._expected_lobster_output_file_name = \ self._get_expected_lobster_output_file_name() - @property - def structure(self) -> FolderStructure: - return self._structure - @property def expected_lobster_output_file_name(self) -> str: return self._expected_lobster_output_file_name + def get_expected_output_path(self) -> str: + """Returns the path of the folder containing the expected output, which can be + used to compare against the actual output""" + return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME) + def _get_expected_lobster_output_file_name(self) -> str: """Retrieves the expected file name of the lobster output file @@ -84,14 +50,20 @@ def _get_expected_lobster_output_file_name(self) -> str: file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing that is currently not supported. """ - for dir_entry in scandir(self._structure.expected_output_path): + for dir_entry in scandir(self.get_expected_output_path()): if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( f"Invalid test setup: No *.lobster file found in " - f"{self._structure.expected_output_path}!", + f"{self.get_expected_output_path()}!", ) + @staticmethod + def _get_name(test_case_path: str) -> str: + """Creates a debug name for the test case, which consists of the RBT folder name + and the test case folder name""" + return join(basename(dirname(test_case_path)), basename(test_case_path)) + @property def args(self) -> List[str]: """Returns the command line arguments which must be used to start the tool under @@ -102,7 +74,7 @@ def args(self) -> List[str]: def name(self) -> str: """Returns the name of the test case, which is equal to the folder name containing the test case""" - return self._structure.name + return self._name @property def expected_exit_code(self) -> int: @@ -112,7 +84,7 @@ def expected_exit_code(self) -> int: def _get_args(self) -> List[str]: """Reads the command line arguments (which must be used to start the tool under test) from the corresponding test setup file""" - file = join(self._structure.input_folder, self._structure.ARGS_FILE_NAME) + file = join(self._input_folder, self._ARGS_FILE_NAME) with open(file, "r", encoding="UTF-8") as file: return [argument.strip() for argument in file.readlines()] @@ -120,8 +92,8 @@ def get_expected_stdout(self) -> str: """Reads the expected command line output (for stdout) from the corresponding test setup file""" cmd_file = join( - self._structure.expected_output_path, - self._structure.EXPECTED_STDOUT_FILE_NAME, + self.get_expected_output_path(), + self._EXPECTED_STDOUT_FILE_NAME, ) with open(cmd_file, "r", encoding="UTF-8") as file: return file.read() @@ -130,8 +102,8 @@ def get_expected_stderr(self) -> str: """Reads the expected command line output (for stderr) from the corresponding test setup file""" errout_file = join( - self._structure.expected_output_path, - self._structure.EXPECTED_STDERR_FILE_NAME, + self.get_expected_output_path(), + self._EXPECTED_STDERR_FILE_NAME, ) with open(errout_file, "r", encoding="UTF-8") as file: return file.read() @@ -139,14 +111,14 @@ def get_expected_stderr(self) -> str: @property def input_folder(self) -> str: """Returns the path containing the input data for the test""" - return self._structure.input_folder + return self._input_folder def _get_expected_exit_code(self) -> int: """Returns the expectation of the tool exit code""" expected_exit_code_file = join( - self._structure.test_case_path, - self._structure.EXPECTED_OUTPUT_FOLDER_NAME, - self._structure.EXIT_CODE_FILE_NAME, + self._test_case_path, + self._EXPECTED_OUTPUT_FOLDER_NAME, + self._EXIT_CODE_FILE_NAME, ) with open(expected_exit_code_file, "r", encoding="UTF-8") as file: return int(file.readline()) @@ -207,25 +179,32 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess): ), "Command line output for STDERR is different!" expected = join( - setup.structure.expected_output_path, + setup.get_expected_output_path(), setup.expected_lobster_output_file_name, ) actual = join(setup.input_folder, setup.expected_lobster_output_file_name) with open(expected, "r", encoding="UTF-8") as expected_lobster_file: try: with open(actual, "r", encoding="UTF-8") as actual_lobster_file: - # Note: we replace Windows-like slashes \\ with one / in order to be - # able to compare the actual output on all OS against the expected - # output on Linux + # Before comparing the actual text with the expected text, we do the + # following replacements: + # a) Replace Windows-like slashes \\ with / in order to be able to + # compare the actual output on all OS against the expected output on + # Linux + # b) Replace the fixed string TEST_CASE_PATH with the absolute path to + # the current test case directory. This is necessary for tools like + # lobster-cpptest which write absolute paths into their *.lobster + # output files. + print(setup._test_case_path) assert actual_lobster_file.read().replace("\\\\", "/") \ - == expected_lobster_file.read(), \ + == expected_lobster_file.read().replace("TEST_CASE_PATH", setup._test_case_path), \ "Actual *.lobster file differs from expectation!" except FileNotFoundError as ex: assert True, f"File {ex.filename} was not generated by the tool under test!" def _get_directories( - start_directory: str, + start_directory: Path, startswith: Optional[str] = None, ) -> Iterator[DirEntry]: """Returns DirEntry instances for each subdirectory found in the given start @@ -254,22 +233,7 @@ def _delete_generated_files(setup: TestSetup): remove(generated) -def _prepare_test(structure: FolderStructure): - system_test_root = Path(".").absolute().resolve().as_posix() - try: - for dir_entry in scandir(structure.expected_output_path + structure.TEMPLATE): - if dir_entry.is_file: - print(f"Preparing file based on template {dir_entry.path}") - with open(dir_entry.path, "r", encoding="UTF-8") as file: - content = file.read().replace("TESTS-SYSTEM", system_test_root) - target_path = dir_entry.path.replace(structure.TEMPLATE, "") - with open(target_path, "w", encoding="UTF-8") as target_file: - target_file.write(content) - except FileNotFoundError: - pass - - -def _run_tests(directory: str, tool: str) -> int: +def _run_tests(directory: Path, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. :param directory: the path to the directory containing all test cases @@ -284,9 +248,7 @@ def _run_tests(directory: str, tool: str) -> int: counter = 0 for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX): for test_case_dir_entry in _get_directories(rbt_dir_entry.path): - structure = FolderStructure(test_case_dir_entry.path) - _prepare_test(structure) - test_setup = TestSetup(structure) + test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) _delete_generated_files(test_setup) @@ -310,7 +272,7 @@ def _get_tool(test_dir: str) -> str: to the tool name :param test_dir: The path containing the requirements-based tests """ - return join("../../../../../", basename(test_dir)) + return normpath(Path(join("../", basename(test_dir))).absolute()) if __name__ == "__main__":