simplification

tests-system/run_tool_tests.py

@@ -1,5 +1,5 @@
+from posixpath import normpath
 import sys
-from dataclasses import dataclass
 from os import scandir, DirEntry, remove
 from os.path import basename, dirname, join
 from pathlib import Path
@@ -10,71 +10,37 @@
 REQUIREMENTS_BASED_TEST_PREFIX = "rbt-"
 
 
-@dataclass
-class FolderStructure:
-    """This class knows all paths related to a test setup"""
-
-    test_case_path: str
-
-    INPUT_FOLDER_NAME = "input"
-    ARGS_FILE_NAME = "args.txt"
-    EXPECTED_OUTPUT_FOLDER_NAME = "expected-output"
-    TEMPLATE = "-template"
-    EXIT_CODE_FILE_NAME = "exit-code.txt"
-    EXPECTED_STDOUT_FILE_NAME = "stdout.txt"
-    EXPECTED_STDERR_FILE_NAME = "stderr.txt"
-
-    def __post_init__(self):
+class TestSetup:
+    _INPUT_FOLDER_NAME = "input"
+    _ARGS_FILE_NAME = "args.txt"
+    _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output"
+    _EXIT_CODE_FILE_NAME = "exit-code.txt"
+    _EXPECTED_STDOUT_FILE_NAME = "stdout.txt"
+    _EXPECTED_STDERR_FILE_NAME = "stderr.txt"
+
+    def __init__(self, test_case_path: str):
         """Constructor
 
         :param test_case_path: the test case path. It must contain subdirectories
         called "input" and "expected-output".
         """
-        self._input_folder = join(self.test_case_path, self.INPUT_FOLDER_NAME)
-        self._name = join(
-            basename(dirname(self.test_case_path)),
-            basename(self.test_case_path),
-        )
-        self._expected_output_path = join(
-            self.test_case_path,
-            self.EXPECTED_OUTPUT_FOLDER_NAME,
-        )
-
-    @property
-    def expected_output_path(self) -> str:
-        """Returns the path of the folder containing the expected output, which can be
-        used to compare against the actual output"""
-        return self._expected_output_path
-
-    @property
-    def name(self) -> str:
-        """Returns a debug name"""
-        return self._name
-
-    @property
-    def input_folder(self) -> str:
-        """Returns the path to the folder containing all input files for the tool
-        under test"""
-        return self._input_folder
-
-
-class TestSetup:
-    def __init__(self, test_case_structure: FolderStructure):
-        assert isinstance(test_case_structure, FolderStructure)
-        self._structure = test_case_structure
+        self._input_folder = join(test_case_path, self._INPUT_FOLDER_NAME)
+        self._test_case_path = test_case_path
         self._args = self._get_args()
         self._expected_exit_code = self._get_expected_exit_code()
+        self._name = self._get_name(test_case_path)
         self._expected_lobster_output_file_name = \
             self._get_expected_lobster_output_file_name()
 
-    @property
-    def structure(self) -> FolderStructure:
-        return self._structure
-
     @property
     def expected_lobster_output_file_name(self) -> str:
         return self._expected_lobster_output_file_name
 
+    def get_expected_output_path(self) -> str:
+        """Returns the path of the folder containing the expected output, which can be
+        used to compare against the actual output"""
+        return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME)
+
     def _get_expected_lobster_output_file_name(self) -> str:
         """Retrieves the expected file name of the lobster output file
 
@@ -84,14 +50,20 @@ def _get_expected_lobster_output_file_name(self) -> str:
         file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing
         that is currently not supported.
         """
-        for dir_entry in scandir(self._structure.expected_output_path):
+        for dir_entry in scandir(self.get_expected_output_path()):
             if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"):
                 return dir_entry.name
         raise ValueError(
             f"Invalid test setup: No *.lobster file found in "
-            f"{self._structure.expected_output_path}!",
+            f"{self.get_expected_output_path()}!",
         )
 
+    @staticmethod
+    def _get_name(test_case_path: str) -> str:
+        """Creates a debug name for the test case, which consists of the RBT folder name
+        and the test case folder name"""
+        return join(basename(dirname(test_case_path)), basename(test_case_path))
+
     @property
     def args(self) -> List[str]:
         """Returns the command line arguments which must be used to start the tool under
@@ -102,7 +74,7 @@ def args(self) -> List[str]:
     def name(self) -> str:
         """Returns the name of the test case, which is equal to the folder name
         containing the test case"""
-        return self._structure.name
+        return self._name
 
     @property
     def expected_exit_code(self) -> int:
@@ -112,16 +84,16 @@ def expected_exit_code(self) -> int:
     def _get_args(self) -> List[str]:
         """Reads the command line arguments (which must be used to start the tool under
         test) from the corresponding test setup file"""
-        file = join(self._structure.input_folder, self._structure.ARGS_FILE_NAME)
+        file = join(self._input_folder, self._ARGS_FILE_NAME)
         with open(file, "r", encoding="UTF-8") as file:
             return [argument.strip() for argument in file.readlines()]
 
     def get_expected_stdout(self) -> str:
         """Reads the expected command line output (for stdout) from the corresponding
         test setup file"""
         cmd_file = join(
-            self._structure.expected_output_path,
-            self._structure.EXPECTED_STDOUT_FILE_NAME,
+            self.get_expected_output_path(),
+            self._EXPECTED_STDOUT_FILE_NAME,
         )
         with open(cmd_file, "r", encoding="UTF-8") as file:
             return file.read()
@@ -130,23 +102,23 @@ def get_expected_stderr(self) -> str:
         """Reads the expected command line output (for stderr) from the corresponding
         test setup file"""
         errout_file = join(
-            self._structure.expected_output_path,
-            self._structure.EXPECTED_STDERR_FILE_NAME,
+            self.get_expected_output_path(),
+            self._EXPECTED_STDERR_FILE_NAME,
         )
         with open(errout_file, "r", encoding="UTF-8") as file:
             return file.read()
 
     @property
     def input_folder(self) -> str:
         """Returns the path containing the input data for the test"""
-        return self._structure.input_folder
+        return self._input_folder
 
     def _get_expected_exit_code(self) -> int:
         """Returns the expectation of the tool exit code"""
         expected_exit_code_file = join(
-            self._structure.test_case_path,
-            self._structure.EXPECTED_OUTPUT_FOLDER_NAME,
-            self._structure.EXIT_CODE_FILE_NAME,
+            self._test_case_path,
+            self._EXPECTED_OUTPUT_FOLDER_NAME,
+            self._EXIT_CODE_FILE_NAME,
         )
         with open(expected_exit_code_file, "r", encoding="UTF-8") as file:
             return int(file.readline())
@@ -207,25 +179,32 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess):
     ), "Command line output for STDERR is different!"
 
     expected = join(
-        setup.structure.expected_output_path,
+        setup.get_expected_output_path(),
         setup.expected_lobster_output_file_name,
     )
     actual = join(setup.input_folder, setup.expected_lobster_output_file_name)
     with open(expected, "r", encoding="UTF-8") as expected_lobster_file:
         try:
             with open(actual, "r", encoding="UTF-8") as actual_lobster_file:
-                # Note: we replace Windows-like slashes \\ with one / in order to be
-                # able to compare the actual output on all OS against the expected
-                # output on Linux
+                # Before comparing the actual text with the expected text, we do the
+                # following replacements:
+                # a) Replace Windows-like slashes \\ with / in order to be able to
+                #    compare the actual output on all OS against the expected output on
+                #    Linux
+                # b) Replace the fixed string TEST_CASE_PATH with the absolute path to
+                #    the current test case directory. This is necessary for tools like
+                #    lobster-cpptest which write absolute paths into their *.lobster
+                #    output files.
+                print(setup._test_case_path)
                 assert actual_lobster_file.read().replace("\\\\", "/") \
-                    == expected_lobster_file.read(), \
+                    == expected_lobster_file.read().replace("TEST_CASE_PATH", setup._test_case_path), \
                     "Actual *.lobster file differs from expectation!"
         except FileNotFoundError as ex:
             assert True, f"File {ex.filename} was not generated by the tool under test!"
 
 
 def _get_directories(
-        start_directory: str,
+        start_directory: Path,
         startswith: Optional[str] = None,
     ) -> Iterator[DirEntry]:
     """Returns DirEntry instances for each subdirectory found in the given start
@@ -254,22 +233,7 @@ def _delete_generated_files(setup: TestSetup):
     remove(generated)
 
 
-def _prepare_test(structure: FolderStructure):
-    system_test_root = Path(".").absolute().resolve().as_posix()
-    try:
-        for dir_entry in scandir(structure.expected_output_path + structure.TEMPLATE):
-            if dir_entry.is_file:
-                print(f"Preparing file based on template {dir_entry.path}")
-                with open(dir_entry.path, "r", encoding="UTF-8") as file:
-                    content = file.read().replace("TESTS-SYSTEM", system_test_root)
-                target_path = dir_entry.path.replace(structure.TEMPLATE, "")
-                with open(target_path, "w", encoding="UTF-8") as target_file:
-                    target_file.write(content)
-    except FileNotFoundError:
-        pass
-
-
-def _run_tests(directory: str, tool: str) -> int:
+def _run_tests(directory: Path, tool: str) -> int:
     """Runs all system tests in the given folder for the specified tool.
 
     :param directory: the path to the directory containing all test cases
@@ -284,9 +248,7 @@ def _run_tests(directory: str, tool: str) -> int:
     counter = 0
     for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX):
         for test_case_dir_entry in _get_directories(rbt_dir_entry.path):
-            structure = FolderStructure(test_case_dir_entry.path)
-            _prepare_test(structure)
-            test_setup = TestSetup(structure)
+            test_setup = TestSetup(test_case_dir_entry.path)
             completed_process = _run_test(test_setup, tool)
             _compare_results(test_setup, completed_process)
             _delete_generated_files(test_setup)
@@ -310,7 +272,7 @@ def _get_tool(test_dir: str) -> str:
     to the tool name
     :param test_dir: The path containing the requirements-based tests
     """
-    return join("../../../../../", basename(test_dir))
+    return normpath(Path(join("../", basename(test_dir))).absolute())
 
 
 if __name__ == "__main__":