Support system tests with arbitrary many *.lobster files

tests-system/README.md

@@ -0,0 +1,45 @@
+The script `run_tool_tests.py` executes system tests.
+It can execute system tests for all lobster tools.
+
+The one and only command line argument to the script must be the name of the tool under
+test.
+For example, to run system tests for `lobster-trlc`, the tool must be started like this:
+```
+> python3 run_tool_tests.py lobster-trlc
+```
+
+System test cases always consist of the following folder structure:
+```
+test-case/
+├─ input/
+│  ├─ args.txt
+expected-output/
+├─ exit-code.txt
+├─ stderr.txt
+├─ stdout.txt
+├─ *.lobster
+
+```
+
+The target `make system-tests` in the root `~Makefile` sets the current working
+directory of the tool under test to the `input` folder.
+
+The files must contain the following piece of information:
+- `args.txt`:
+  This file shall contain the command line arguments that must be used to start the tool
+  under test. Each line represents a separate argument.
+- `exit-code.txt`: This file shall contain the expected exit code of the tool under test.
+- `stderr.txt`: This file shall contain the expected standard error stream of the tool
+  under test.
+- `stdout.txt`: This file shall contain the expected standard output stream of the tool
+  under test.
+- `*.lobster`: There can be zero to infinite many `*.lobster` files given. The tool
+  under test is expected to generate all these files inside the current working directory
+  during the execution of the test case.
+
+Any additional files needed by the tool under test must be located in the `input`
+folder.
+
+The expected values will all be compared against their actual values, and test cases
+only count as "passed" if the values match.
+

tests-system/run_tool_tests.py

@@ -17,6 +17,7 @@ class TestSetup:
     _EXIT_CODE_FILE_NAME = "exit-code.txt"
     _EXPECTED_STDOUT_FILE_NAME = "stdout.txt"
     _EXPECTED_STDERR_FILE_NAME = "stderr.txt"
+    _LOBSTER_FILE_ENDING = ".lobster"
 
     def __init__(self, test_case_path: str):
         """Constructor
@@ -29,39 +30,36 @@ def __init__(self, test_case_path: str):
         self._args = self._get_args()
         self._expected_exit_code = self._get_expected_exit_code()
         self._name = self._get_name(test_case_path)
-        self._expected_lobster_output_file_name = \
-            self._get_expected_lobster_output_file_name()
+        self._expected_lobster_output_file_names = \
+            self._get_expected_lobster_output_file_names()
 
     @property
     def test_case_path(self) -> str:
         """Returns the path to the directory which contains the test case"""
         return self._test_case_path
 
     @property
-    def expected_lobster_output_file_name(self) -> str:
-        return self._expected_lobster_output_file_name
+    def expected_lobster_output_file_names(self) -> List[str]:
+        """Returns a list of all file names of all expected lobster output files.
+
+        The tool under test is expected to generate files with these names in the
+        "input" folder.
+        """
+        return self._expected_lobster_output_file_names
 
     def get_expected_output_path(self) -> str:
         """Returns the path of the folder containing the expected output, which can be
         used to compare against the actual output"""
         return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME)
 
-    def _get_expected_lobster_output_file_name(self) -> str:
-        """Retrieves the expected file name of the lobster output file
-
-        Note: The system test must always be prepared such that the tool under test
-        generates the lobster file in the "input" folder. Other test setups are not
-        supported. Furthermore, the tool under test must generate exactly one output
-        file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing
-        that is currently not supported.
+    def _get_expected_lobster_output_file_names(self) -> List[str]:
+        """Retrieves the expected file names of all lobster output files
         """
-        for dir_entry in scandir(self.get_expected_output_path()):
-            if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"):
-                return dir_entry.name
-        raise ValueError(
-            f"Invalid test setup: No *.lobster file found in "
-            f"{self.get_expected_output_path()}!",
-        )
+        return [
+            dir_entry.name for dir_entry in scandir(self.get_expected_output_path())
+            if (dir_entry.is_file()) \
+                and dir_entry.name.endswith(self._LOBSTER_FILE_ENDING)
+        ]
 
     @staticmethod
     def _get_name(test_case_path: str) -> str:
@@ -183,31 +181,43 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess):
         actual=completed_process.stderr,
     ), "Command line output for STDERR is different!"
 
-    expected = join(
-        setup.get_expected_output_path(),
-        setup.expected_lobster_output_file_name,
-    )
-    actual = join(setup.input_folder, setup.expected_lobster_output_file_name)
-    with open(expected, "r", encoding="UTF-8") as expected_lobster_file:
-        try:
-            with open(actual, "r", encoding="UTF-8") as actual_lobster_file:
-                # Before comparing the actual text with the expected text, we do the
-                # following replacements:
-                # a) Replace Windows-like slashes \\ with / in order to be able to
-                #    compare the actual output on all OS against the expected output on
-                #    Linux
-                # b) Replace the fixed string TEST_CASE_PATH with the absolute path to
-                #    the current test case directory. This is necessary for tools like
-                #    lobster-cpptest which write absolute paths into their *.lobster
-                #    output files.
-                modified_actual = actual_lobster_file.read().replace("\\\\", "/")
-                modified_expected = expected_lobster_file.read().replace(
-                    "TEST_CASE_PATH", setup.test_case_path
-                )
-                assert modified_actual == modified_expected, \
-                    "Actual *.lobster file differs from expectation!"
-        except FileNotFoundError as ex:
-            assert True, f"File {ex.filename} was not generated by the tool under test!"
+    for expected_lobster_output_file_name in setup.expected_lobster_output_file_names:
+        expected = join(
+            setup.get_expected_output_path(),
+            expected_lobster_output_file_name,
+        )
+        actual = join(setup.input_folder, expected_lobster_output_file_name)
+        _compare_lobster_files(expected, actual, setup.test_case_path)
+
+def _compare_lobster_files(expected: str, actual: str, test_case_path: str):
+        """Compares an actual LOBSTER file with an expected LOBSTER file
+
+        Before comparing the actual text with the expected text, we do the
+        following replacements:
+        a) Replace Windows-like slashes \\ with / in order to be able to
+           compare the actual output on all OS against the expected output on
+           Linux
+        b) Replace the fixed string TEST_CASE_PATH with the absolute path to
+           the current test case directory. This is necessary for tools like
+           lobster-cpptest which write absolute paths into their *.lobster
+           output files.
+
+        :param expected: The path to the file containing the expected output
+        :param actual: The path to the file containing the actual output
+        :param test_case_path: The path containing the test case files
+        """
+        with open(expected, "r", encoding="UTF-8") as expected_lobster_file:
+            try:
+                with open(actual, "r", encoding="UTF-8") as actual_lobster_file:
+                    modified_actual = actual_lobster_file.read().replace("\\\\", "/")
+                    modified_expected = expected_lobster_file.read().replace(
+                        "TEST_CASE_PATH", test_case_path
+                    )
+                    assert modified_actual == modified_expected, \
+                        f"Actual *.lobster file differs from expectation {expected}!"
+            except FileNotFoundError as ex:
+                assert True, \
+                    f"File {ex.filename} was not generated by the tool under test!"
 
 
 def _get_directories(
@@ -232,12 +242,16 @@ def _get_directories(
 
 def _delete_generated_files(setup: TestSetup):
     """Deletes the *.lobster file that has been generated by the test"""
-    generated = join(
-        setup.input_folder,
-        setup.expected_lobster_output_file_name,
-    )
-    print(f"DELETING {generated}")
-    remove(generated)
+    for expected_lobster_output_file_name in setup.expected_lobster_output_file_names:
+        generated = join(
+            setup.input_folder,
+            expected_lobster_output_file_name,
+        )
+        print(f"DELETING {generated}")
+        try:
+            remove(generated)
+        except FileNotFoundError:
+            pass
 
 
 def _run_tests(directory: Path, tool: str) -> int: