diff --git a/tests-system/README.md b/tests-system/README.md new file mode 100644 index 0000000..3fa14ee --- /dev/null +++ b/tests-system/README.md @@ -0,0 +1,45 @@ +The script `run_tool_tests.py` executes system tests. +It can execute system tests for all lobster tools. + +The one and only command line argument to the script must be the name of the tool under +test. +For example, to run system tests for `lobster-trlc`, the tool must be started like this: +``` +> python3 run_tool_tests.py lobster-trlc +``` + +System test cases always consist of the following folder structure: +``` +test-case/ +├─ input/ +│ ├─ args.txt +expected-output/ +├─ exit-code.txt +├─ stderr.txt +├─ stdout.txt +├─ *.lobster + +``` + +The target `make system-tests` in the root `~Makefile` sets the current working +directory of the tool under test to the `input` folder. + +The files must contain the following piece of information: +- `args.txt`: + This file shall contain the command line arguments that must be used to start the tool + under test. Each line represents a separate argument. +- `exit-code.txt`: This file shall contain the expected exit code of the tool under test. +- `stderr.txt`: This file shall contain the expected standard error stream of the tool + under test. +- `stdout.txt`: This file shall contain the expected standard output stream of the tool + under test. +- `*.lobster`: There can be zero to infinite many `*.lobster` files given. The tool + under test is expected to generate all these files inside the current working directory + during the execution of the test case. + +Any additional files needed by the tool under test must be located in the `input` +folder. + +The expected values will all be compared against their actual values, and test cases +only count as "passed" if the values match. + diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 064125d..4a00425 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -17,6 +17,7 @@ class TestSetup: _EXIT_CODE_FILE_NAME = "exit-code.txt" _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" _EXPECTED_STDERR_FILE_NAME = "stderr.txt" + _LOBSTER_FILE_ENDING = ".lobster" def __init__(self, test_case_path: str): """Constructor @@ -29,8 +30,8 @@ def __init__(self, test_case_path: str): self._args = self._get_args() self._expected_exit_code = self._get_expected_exit_code() self._name = self._get_name(test_case_path) - self._expected_lobster_output_file_name = \ - self._get_expected_lobster_output_file_name() + self._expected_lobster_output_file_names = \ + self._get_expected_lobster_output_file_names() @property def test_case_path(self) -> str: @@ -38,30 +39,25 @@ def test_case_path(self) -> str: return self._test_case_path @property - def expected_lobster_output_file_name(self) -> str: - return self._expected_lobster_output_file_name + def expected_lobster_output_file_names(self) -> List[str]: + return self._expected_lobster_output_file_names def get_expected_output_path(self) -> str: """Returns the path of the folder containing the expected output, which can be used to compare against the actual output""" return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME) - def _get_expected_lobster_output_file_name(self) -> str: - """Retrieves the expected file name of the lobster output file + def _get_expected_lobster_output_file_names(self) -> List[str]: + """Retrieves the expected file names of all lobster output file Note: The system test must always be prepared such that the tool under test generates the lobster file in the "input" folder. Other test setups are not - supported. Furthermore, the tool under test must generate exactly one output - file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing - that is currently not supported. + supported. """ - for dir_entry in scandir(self.get_expected_output_path()): - if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): - return dir_entry.name - raise ValueError( - f"Invalid test setup: No *.lobster file found in " - f"{self.get_expected_output_path()}!", - ) + return [ + dir_entry.name for dir_entry in scandir(self.get_expected_output_path()) + if (dir_entry.is_file()) and dir_entry.name.endswith(self._LOBSTER_FILE_ENDING) + ] @staticmethod def _get_name(test_case_path: str) -> str: @@ -183,31 +179,32 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess): actual=completed_process.stderr, ), "Command line output for STDERR is different!" - expected = join( - setup.get_expected_output_path(), - setup.expected_lobster_output_file_name, - ) - actual = join(setup.input_folder, setup.expected_lobster_output_file_name) - with open(expected, "r", encoding="UTF-8") as expected_lobster_file: - try: - with open(actual, "r", encoding="UTF-8") as actual_lobster_file: - # Before comparing the actual text with the expected text, we do the - # following replacements: - # a) Replace Windows-like slashes \\ with / in order to be able to - # compare the actual output on all OS against the expected output on - # Linux - # b) Replace the fixed string TEST_CASE_PATH with the absolute path to - # the current test case directory. This is necessary for tools like - # lobster-cpptest which write absolute paths into their *.lobster - # output files. - modified_actual = actual_lobster_file.read().replace("\\\\", "/") - modified_expected = expected_lobster_file.read().replace( - "TEST_CASE_PATH", setup.test_case_path - ) - assert modified_actual == modified_expected, \ - "Actual *.lobster file differs from expectation!" - except FileNotFoundError as ex: - assert True, f"File {ex.filename} was not generated by the tool under test!" + for expected_lobster_output_file_name in setup.expected_lobster_output_file_names: + expected = join( + setup.get_expected_output_path(), + expected_lobster_output_file_name, + ) + actual = join(setup.input_folder, expected_lobster_output_file_name) + with open(expected, "r", encoding="UTF-8") as expected_lobster_file: + try: + with open(actual, "r", encoding="UTF-8") as actual_lobster_file: + # Before comparing the actual text with the expected text, we do the + # following replacements: + # a) Replace Windows-like slashes \\ with / in order to be able to + # compare the actual output on all OS against the expected output on + # Linux + # b) Replace the fixed string TEST_CASE_PATH with the absolute path to + # the current test case directory. This is necessary for tools like + # lobster-cpptest which write absolute paths into their *.lobster + # output files. + modified_actual = actual_lobster_file.read().replace("\\\\", "/") + modified_expected = expected_lobster_file.read().replace( + "TEST_CASE_PATH", setup.test_case_path + ) + assert modified_actual == modified_expected, \ + "Actual *.lobster file differs from expectation!" + except FileNotFoundError as ex: + assert True, f"File {ex.filename} was not generated by the tool under test!" def _get_directories( @@ -232,12 +229,16 @@ def _get_directories( def _delete_generated_files(setup: TestSetup): """Deletes the *.lobster file that has been generated by the test""" - generated = join( - setup.input_folder, - setup.expected_lobster_output_file_name, - ) - print(f"DELETING {generated}") - remove(generated) + for expected_lobster_output_file_name in setup.expected_lobster_output_file_names: + generated = join( + setup.input_folder, + expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + try: + remove(generated) + except FileNotFoundError: + pass def _run_tests(directory: Path, tool: str) -> int: