Support system tests with absolute paths

Add feature to the system test framework to generate absolute paths in output files
based on template files.

It works like this:
If a folder called `expected-output-template` exists, then
1. it takes each file from that folder
2. creates a copy in folder `expected-output`
3. and replaces the string `TESTS-SYSTEM` with an absolute path to the
   `tests-system` folder.

This way it is possible to compare *.lobster files that contain absolute paths
against their expected content. `lobster-cpptest` needs this feature.

tests-system/run_tool_tests.py

@@ -1,4 +1,5 @@
 import sys
+from dataclasses import dataclass
 from os import scandir, DirEntry, remove
 from os.path import basename, dirname, join
 from pathlib import Path
@@ -9,37 +10,71 @@
 REQUIREMENTS_BASED_TEST_PREFIX = "rbt-"
 
 
-class TestSetup:
-    _INPUT_FOLDER_NAME = "input"
-    _ARGS_FILE_NAME = "args.txt"
-    _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output"
-    _EXIT_CODE_FILE_NAME = "exit-code.txt"
-    _EXPECTED_STDOUT_FILE_NAME = "stdout.txt"
-    _EXPECTED_STDERR_FILE_NAME = "stderr.txt"
-
-    def __init__(self, test_case_path: str):
+@dataclass
+class FolderStructure:
+    """This class knows all paths related to a test setup"""
+
+    test_case_path: str
+
+    INPUT_FOLDER_NAME = "input"
+    ARGS_FILE_NAME = "args.txt"
+    EXPECTED_OUTPUT_FOLDER_NAME = "expected-output"
+    TEMPLATE = "-template"
+    EXIT_CODE_FILE_NAME = "exit-code.txt"
+    EXPECTED_STDOUT_FILE_NAME = "stdout.txt"
+    EXPECTED_STDERR_FILE_NAME = "stderr.txt"
+
+    def __post_init__(self):
         """Constructor
 
         :param test_case_path: the test case path. It must contain subdirectories
         called "input" and "expected-output".
         """
-        self._input_folder = join(test_case_path, self._INPUT_FOLDER_NAME)
-        self._test_case_path = test_case_path
+        self._input_folder = join(self.test_case_path, self.INPUT_FOLDER_NAME)
+        self._name = join(
+            basename(dirname(self.test_case_path)),
+            basename(self.test_case_path),
+        )
+        self._expected_output_path = join(
+            self.test_case_path,
+            self.EXPECTED_OUTPUT_FOLDER_NAME,
+        )
+
+    @property
+    def expected_output_path(self) -> str:
+        """Returns the path of the folder containing the expected output, which can be
+        used to compare against the actual output"""
+        return self._expected_output_path
+
+    @property
+    def name(self) -> str:
+        """Returns a debug name"""
+        return self._name
+
+    @property
+    def input_folder(self) -> str:
+        """Returns the path to the folder containing all input files for the tool
+        under test"""
+        return self._input_folder
+
+
+class TestSetup:
+    def __init__(self, test_case_structure: FolderStructure):
+        assert isinstance(test_case_structure, FolderStructure)
+        self._structure = test_case_structure
         self._args = self._get_args()
         self._expected_exit_code = self._get_expected_exit_code()
-        self._name = self._get_name(test_case_path)
         self._expected_lobster_output_file_name = \
             self._get_expected_lobster_output_file_name()
 
+    @property
+    def structure(self) -> FolderStructure:
+        return self._structure
+
     @property
     def expected_lobster_output_file_name(self) -> str:
         return self._expected_lobster_output_file_name
 
-    def get_expected_output_path(self) -> str:
-        """Returns the path of the folder containing the expected output, which can be
-        used to compare against the actual output"""
-        return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME)
-
     def _get_expected_lobster_output_file_name(self) -> str:
         """Retrieves the expected file name of the lobster output file
 
@@ -49,20 +84,14 @@ def _get_expected_lobster_output_file_name(self) -> str:
         file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing
         that is currently not supported.
         """
-        for dir_entry in scandir(self.get_expected_output_path()):
+        for dir_entry in scandir(self._structure.expected_output_path):
             if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"):
                 return dir_entry.name
         raise ValueError(
             f"Invalid test setup: No *.lobster file found in "
-            f"{self.get_expected_output_path()}!",
+            f"{self._structure.expected_output_path}!",
         )
 
-    @staticmethod
-    def _get_name(test_case_path: str) -> str:
-        """Creates a debug name for the test case, which consists of the RBT folder name
-        and the test case folder name"""
-        return join(basename(dirname(test_case_path)), basename(test_case_path))
-
     @property
     def args(self) -> List[str]:
         """Returns the command line arguments which must be used to start the tool under
@@ -73,7 +102,7 @@ def args(self) -> List[str]:
     def name(self) -> str:
         """Returns the name of the test case, which is equal to the folder name
         containing the test case"""
-        return self._name
+        return self._structure.name
 
     @property
     def expected_exit_code(self) -> int:
@@ -83,16 +112,16 @@ def expected_exit_code(self) -> int:
     def _get_args(self) -> List[str]:
         """Reads the command line arguments (which must be used to start the tool under
         test) from the corresponding test setup file"""
-        file = join(self._input_folder, self._ARGS_FILE_NAME)
+        file = join(self._structure.input_folder, self._structure.ARGS_FILE_NAME)
         with open(file, "r", encoding="UTF-8") as file:
             return [argument.strip() for argument in file.readlines()]
 
     def get_expected_stdout(self) -> str:
         """Reads the expected command line output (for stdout) from the corresponding
         test setup file"""
         cmd_file = join(
-            self.get_expected_output_path(),
-            self._EXPECTED_STDOUT_FILE_NAME,
+            self._structure.expected_output_path,
+            self._structure.EXPECTED_STDOUT_FILE_NAME,
         )
         with open(cmd_file, "r", encoding="UTF-8") as file:
             return file.read()
@@ -101,23 +130,23 @@ def get_expected_stderr(self) -> str:
         """Reads the expected command line output (for stderr) from the corresponding
         test setup file"""
         errout_file = join(
-            self.get_expected_output_path(),
-            self._EXPECTED_STDERR_FILE_NAME,
+            self._structure.expected_output_path,
+            self._structure.EXPECTED_STDERR_FILE_NAME,
         )
         with open(errout_file, "r", encoding="UTF-8") as file:
             return file.read()
 
     @property
     def input_folder(self) -> str:
         """Returns the path containing the input data for the test"""
-        return self._input_folder
+        return self._structure.input_folder
 
     def _get_expected_exit_code(self) -> int:
         """Returns the expectation of the tool exit code"""
         expected_exit_code_file = join(
-            self._test_case_path,
-            self._EXPECTED_OUTPUT_FOLDER_NAME,
-            self._EXIT_CODE_FILE_NAME,
+            self._structure.test_case_path,
+            self._structure.EXPECTED_OUTPUT_FOLDER_NAME,
+            self._structure.EXIT_CODE_FILE_NAME,
         )
         with open(expected_exit_code_file, "r", encoding="UTF-8") as file:
             return int(file.readline())
@@ -178,7 +207,7 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess):
     ), "Command line output for STDERR is different!"
 
     expected = join(
-        setup.get_expected_output_path(),
+        setup.structure.expected_output_path,
         setup.expected_lobster_output_file_name,
     )
     actual = join(setup.input_folder, setup.expected_lobster_output_file_name)
@@ -225,6 +254,21 @@ def _delete_generated_files(setup: TestSetup):
     remove(generated)
 
 
+def _prepare_test(structure: FolderStructure):
+    system_test_root = Path(".").absolute().resolve().as_posix()
+    try:
+        for dir_entry in scandir(structure.expected_output_path + structure.TEMPLATE):
+            if dir_entry.is_file:
+                print(f"Preparing file based on template {dir_entry.path}")
+                with open(dir_entry.path, "r", encoding="UTF-8") as file:
+                    content = file.read().replace("TESTS-SYSTEM", system_test_root)
+                target_path = dir_entry.path.replace(structure.TEMPLATE, "")
+                with open(target_path, "w", encoding="UTF-8") as target_file:
+                    target_file.write(content)
+    except FileNotFoundError:
+        pass
+
+
 def _run_tests(directory: str, tool: str) -> int:
     """Runs all system tests in the given folder for the specified tool.
 
@@ -240,7 +284,9 @@ def _run_tests(directory: str, tool: str) -> int:
     counter = 0
     for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX):
         for test_case_dir_entry in _get_directories(rbt_dir_entry.path):
-            test_setup = TestSetup(test_case_dir_entry.path)
+            structure = FolderStructure(test_case_dir_entry.path)
+            _prepare_test(structure)
+            test_setup = TestSetup(structure)
             completed_process = _run_test(test_setup, tool)
             _compare_results(test_setup, completed_process)
             _delete_generated_files(test_setup)