From 7f344f16454e9f0f41008b99bb3bff2af034162e Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:34:08 +0100 Subject: [PATCH 1/2] Support system tests with arbitrary many *.lobster files (#162) Add `README.md` to explain how system tests work. --- tests-system/README.md | 45 +++++++++++++ tests-system/run_tool_tests.py | 114 ++++++++++++++++++--------------- 2 files changed, 109 insertions(+), 50 deletions(-) create mode 100644 tests-system/README.md diff --git a/tests-system/README.md b/tests-system/README.md new file mode 100644 index 0000000..3fa14ee --- /dev/null +++ b/tests-system/README.md @@ -0,0 +1,45 @@ +The script `run_tool_tests.py` executes system tests. +It can execute system tests for all lobster tools. + +The one and only command line argument to the script must be the name of the tool under +test. +For example, to run system tests for `lobster-trlc`, the tool must be started like this: +``` +> python3 run_tool_tests.py lobster-trlc +``` + +System test cases always consist of the following folder structure: +``` +test-case/ +├─ input/ +│ ├─ args.txt +expected-output/ +├─ exit-code.txt +├─ stderr.txt +├─ stdout.txt +├─ *.lobster + +``` + +The target `make system-tests` in the root `~Makefile` sets the current working +directory of the tool under test to the `input` folder. + +The files must contain the following piece of information: +- `args.txt`: + This file shall contain the command line arguments that must be used to start the tool + under test. Each line represents a separate argument. +- `exit-code.txt`: This file shall contain the expected exit code of the tool under test. +- `stderr.txt`: This file shall contain the expected standard error stream of the tool + under test. +- `stdout.txt`: This file shall contain the expected standard output stream of the tool + under test. +- `*.lobster`: There can be zero to infinite many `*.lobster` files given. The tool + under test is expected to generate all these files inside the current working directory + during the execution of the test case. + +Any additional files needed by the tool under test must be located in the `input` +folder. + +The expected values will all be compared against their actual values, and test cases +only count as "passed" if the values match. + diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 064125d..70e9950 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -17,6 +17,7 @@ class TestSetup: _EXIT_CODE_FILE_NAME = "exit-code.txt" _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" _EXPECTED_STDERR_FILE_NAME = "stderr.txt" + _LOBSTER_FILE_ENDING = ".lobster" def __init__(self, test_case_path: str): """Constructor @@ -29,8 +30,8 @@ def __init__(self, test_case_path: str): self._args = self._get_args() self._expected_exit_code = self._get_expected_exit_code() self._name = self._get_name(test_case_path) - self._expected_lobster_output_file_name = \ - self._get_expected_lobster_output_file_name() + self._expected_lobster_output_file_names = \ + self._get_expected_lobster_output_file_names() @property def test_case_path(self) -> str: @@ -38,30 +39,27 @@ def test_case_path(self) -> str: return self._test_case_path @property - def expected_lobster_output_file_name(self) -> str: - return self._expected_lobster_output_file_name + def expected_lobster_output_file_names(self) -> List[str]: + """Returns a list of all file names of all expected lobster output files. + + The tool under test is expected to generate files with these names in the + "input" folder. + """ + return self._expected_lobster_output_file_names def get_expected_output_path(self) -> str: """Returns the path of the folder containing the expected output, which can be used to compare against the actual output""" return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME) - def _get_expected_lobster_output_file_name(self) -> str: - """Retrieves the expected file name of the lobster output file - - Note: The system test must always be prepared such that the tool under test - generates the lobster file in the "input" folder. Other test setups are not - supported. Furthermore, the tool under test must generate exactly one output - file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing - that is currently not supported. + def _get_expected_lobster_output_file_names(self) -> List[str]: + """Retrieves the expected file names of all lobster output files """ - for dir_entry in scandir(self.get_expected_output_path()): - if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): - return dir_entry.name - raise ValueError( - f"Invalid test setup: No *.lobster file found in " - f"{self.get_expected_output_path()}!", - ) + return [ + dir_entry.name for dir_entry in scandir(self.get_expected_output_path()) + if (dir_entry.is_file()) \ + and dir_entry.name.endswith(self._LOBSTER_FILE_ENDING) + ] @staticmethod def _get_name(test_case_path: str) -> str: @@ -183,31 +181,43 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess): actual=completed_process.stderr, ), "Command line output for STDERR is different!" - expected = join( - setup.get_expected_output_path(), - setup.expected_lobster_output_file_name, - ) - actual = join(setup.input_folder, setup.expected_lobster_output_file_name) - with open(expected, "r", encoding="UTF-8") as expected_lobster_file: - try: - with open(actual, "r", encoding="UTF-8") as actual_lobster_file: - # Before comparing the actual text with the expected text, we do the - # following replacements: - # a) Replace Windows-like slashes \\ with / in order to be able to - # compare the actual output on all OS against the expected output on - # Linux - # b) Replace the fixed string TEST_CASE_PATH with the absolute path to - # the current test case directory. This is necessary for tools like - # lobster-cpptest which write absolute paths into their *.lobster - # output files. - modified_actual = actual_lobster_file.read().replace("\\\\", "/") - modified_expected = expected_lobster_file.read().replace( - "TEST_CASE_PATH", setup.test_case_path - ) - assert modified_actual == modified_expected, \ - "Actual *.lobster file differs from expectation!" - except FileNotFoundError as ex: - assert True, f"File {ex.filename} was not generated by the tool under test!" + for expected_lobster_output_file_name in setup.expected_lobster_output_file_names: + expected = join( + setup.get_expected_output_path(), + expected_lobster_output_file_name, + ) + actual = join(setup.input_folder, expected_lobster_output_file_name) + _compare_lobster_files(expected, actual, setup.test_case_path) + +def _compare_lobster_files(expected: str, actual: str, test_case_path: str): + """Compares an actual LOBSTER file with an expected LOBSTER file + + Before comparing the actual text with the expected text, we do the + following replacements: + a) Replace Windows-like slashes \\ with / in order to be able to + compare the actual output on all OS against the expected output on + Linux + b) Replace the fixed string TEST_CASE_PATH with the absolute path to + the current test case directory. This is necessary for tools like + lobster-cpptest which write absolute paths into their *.lobster + output files. + + :param expected: The path to the file containing the expected output + :param actual: The path to the file containing the actual output + :param test_case_path: The path containing the test case files + """ + with open(expected, "r", encoding="UTF-8") as expected_lobster_file: + try: + with open(actual, "r", encoding="UTF-8") as actual_lobster_file: + modified_actual = actual_lobster_file.read().replace("\\\\", "/") + modified_expected = expected_lobster_file.read().replace( + "TEST_CASE_PATH", test_case_path + ) + assert modified_actual == modified_expected, \ + f"Actual *.lobster file differs from expectation {expected}!" + except FileNotFoundError as ex: + assert True, \ + f"File {ex.filename} was not generated by the tool under test!" def _get_directories( @@ -232,12 +242,16 @@ def _get_directories( def _delete_generated_files(setup: TestSetup): """Deletes the *.lobster file that has been generated by the test""" - generated = join( - setup.input_folder, - setup.expected_lobster_output_file_name, - ) - print(f"DELETING {generated}") - remove(generated) + for expected_lobster_output_file_name in setup.expected_lobster_output_file_names: + generated = join( + setup.input_folder, + expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + try: + remove(generated) + except FileNotFoundError: + pass def _run_tests(directory: Path, tool: str) -> int: From 0cbcc6aba05af7d44f6e9c6f6482257f2d51c271 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Wed, 18 Dec 2024 05:24:07 +0100 Subject: [PATCH 2/2] Fix typo (#163) Fixed a typo in a `README.md` file. --- tests-system/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-system/README.md b/tests-system/README.md index 3fa14ee..e98bd8a 100644 --- a/tests-system/README.md +++ b/tests-system/README.md @@ -21,7 +21,7 @@ expected-output/ ``` -The target `make system-tests` in the root `~Makefile` sets the current working +The target `make system-tests` in the root `Makefile` sets the current working directory of the tool under test to the `input` folder. The files must contain the following piece of information: