Skip to content

Commit

Permalink
Support system tests with arbitrary many *.lobster files
Browse files Browse the repository at this point in the history
Add support for system test cases which are expected to generate zero *.lobster output files,
or more than one file.

Add `README.md` to explain how system tests work.
  • Loading branch information
phiwuu committed Dec 17, 2024
1 parent eb9dade commit 8d2b1fb
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 50 deletions.
45 changes: 45 additions & 0 deletions tests-system/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
The script `run_tool_tests.py` executes system tests.
It can execute system tests for all lobster tools.

The one and only command line argument to the script must be the name of the tool under
test.
For example, to run system tests for `lobster-trlc`, the tool must be started like this:
```
> python3 run_tool_tests.py lobster-trlc
```

System test cases always consist of the following folder structure:
```
test-case/
├─ input/
│ ├─ args.txt
expected-output/
├─ exit-code.txt
├─ stderr.txt
├─ stdout.txt
├─ *.lobster
```

The target `make system-tests` in the root `~Makefile` sets the current working
directory of the tool under test to the `input` folder.

The files must contain the following piece of information:
- `args.txt`:
This file shall contain the command line arguments that must be used to start the tool
under test. Each line represents a separate argument.
- `exit-code.txt`: This file shall contain the expected exit code of the tool under test.
- `stderr.txt`: This file shall contain the expected standard error stream of the tool
under test.
- `stdout.txt`: This file shall contain the expected standard output stream of the tool
under test.
- `*.lobster`: There can be zero to infinite many `*.lobster` files given. The tool
under test is expected to generate all these files inside the current working directory
during the execution of the test case.

Any additional files needed by the tool under test must be located in the `input`
folder.

The expected values will all be compared against their actual values, and test cases
only count as "passed" if the values match.

114 changes: 64 additions & 50 deletions tests-system/run_tool_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ class TestSetup:
_EXIT_CODE_FILE_NAME = "exit-code.txt"
_EXPECTED_STDOUT_FILE_NAME = "stdout.txt"
_EXPECTED_STDERR_FILE_NAME = "stderr.txt"
_LOBSTER_FILE_ENDING = ".lobster"

def __init__(self, test_case_path: str):
"""Constructor
Expand All @@ -29,39 +30,36 @@ def __init__(self, test_case_path: str):
self._args = self._get_args()
self._expected_exit_code = self._get_expected_exit_code()
self._name = self._get_name(test_case_path)
self._expected_lobster_output_file_name = \
self._get_expected_lobster_output_file_name()
self._expected_lobster_output_file_names = \
self._get_expected_lobster_output_file_names()

@property
def test_case_path(self) -> str:
"""Returns the path to the directory which contains the test case"""
return self._test_case_path

@property
def expected_lobster_output_file_name(self) -> str:
return self._expected_lobster_output_file_name
def expected_lobster_output_file_names(self) -> List[str]:
"""Returns a list of all file names of all expected lobster output files.
The tool under test is expected to generate files with these names in the
"input" folder.
"""
return self._expected_lobster_output_file_names

def get_expected_output_path(self) -> str:
"""Returns the path of the folder containing the expected output, which can be
used to compare against the actual output"""
return join(self._test_case_path, self._EXPECTED_OUTPUT_FOLDER_NAME)

def _get_expected_lobster_output_file_name(self) -> str:
"""Retrieves the expected file name of the lobster output file
Note: The system test must always be prepared such that the tool under test
generates the lobster file in the "input" folder. Other test setups are not
supported. Furthermore, the tool under test must generate exactly one output
file. Tools like 'lobster-cpptest' are able to generate multiple files. Testing
that is currently not supported.
def _get_expected_lobster_output_file_names(self) -> List[str]:
"""Retrieves the expected file names of all lobster output files
"""
for dir_entry in scandir(self.get_expected_output_path()):
if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"):
return dir_entry.name
raise ValueError(
f"Invalid test setup: No *.lobster file found in "
f"{self.get_expected_output_path()}!",
)
return [
dir_entry.name for dir_entry in scandir(self.get_expected_output_path())
if (dir_entry.is_file()) \
and dir_entry.name.endswith(self._LOBSTER_FILE_ENDING)
]

@staticmethod
def _get_name(test_case_path: str) -> str:
Expand Down Expand Up @@ -183,31 +181,43 @@ def _compare_results(setup: TestSetup, completed_process: CompletedProcess):
actual=completed_process.stderr,
), "Command line output for STDERR is different!"

expected = join(
setup.get_expected_output_path(),
setup.expected_lobster_output_file_name,
)
actual = join(setup.input_folder, setup.expected_lobster_output_file_name)
with open(expected, "r", encoding="UTF-8") as expected_lobster_file:
try:
with open(actual, "r", encoding="UTF-8") as actual_lobster_file:
# Before comparing the actual text with the expected text, we do the
# following replacements:
# a) Replace Windows-like slashes \\ with / in order to be able to
# compare the actual output on all OS against the expected output on
# Linux
# b) Replace the fixed string TEST_CASE_PATH with the absolute path to
# the current test case directory. This is necessary for tools like
# lobster-cpptest which write absolute paths into their *.lobster
# output files.
modified_actual = actual_lobster_file.read().replace("\\\\", "/")
modified_expected = expected_lobster_file.read().replace(
"TEST_CASE_PATH", setup.test_case_path
)
assert modified_actual == modified_expected, \
"Actual *.lobster file differs from expectation!"
except FileNotFoundError as ex:
assert True, f"File {ex.filename} was not generated by the tool under test!"
for expected_lobster_output_file_name in setup.expected_lobster_output_file_names:
expected = join(
setup.get_expected_output_path(),
expected_lobster_output_file_name,
)
actual = join(setup.input_folder, expected_lobster_output_file_name)
_compare_lobster_files(expected, actual, setup.test_case_path)

def _compare_lobster_files(expected: str, actual: str, test_case_path: str):
"""Compares an actual LOBSTER file with an expected LOBSTER file
Before comparing the actual text with the expected text, we do the
following replacements:
a) Replace Windows-like slashes \\ with / in order to be able to
compare the actual output on all OS against the expected output on
Linux
b) Replace the fixed string TEST_CASE_PATH with the absolute path to
the current test case directory. This is necessary for tools like
lobster-cpptest which write absolute paths into their *.lobster
output files.
:param expected: The path to the file containing the expected output
:param actual: The path to the file containing the actual output
:param test_case_path: The path containing the test case files
"""
with open(expected, "r", encoding="UTF-8") as expected_lobster_file:
try:
with open(actual, "r", encoding="UTF-8") as actual_lobster_file:
modified_actual = actual_lobster_file.read().replace("\\\\", "/")
modified_expected = expected_lobster_file.read().replace(
"TEST_CASE_PATH", test_case_path
)
assert modified_actual == modified_expected, \
f"Actual *.lobster file differs from expectation {expected}!"
except FileNotFoundError as ex:
assert True, \
f"File {ex.filename} was not generated by the tool under test!"


def _get_directories(
Expand All @@ -232,12 +242,16 @@ def _get_directories(

def _delete_generated_files(setup: TestSetup):
"""Deletes the *.lobster file that has been generated by the test"""
generated = join(
setup.input_folder,
setup.expected_lobster_output_file_name,
)
print(f"DELETING {generated}")
remove(generated)
for expected_lobster_output_file_name in setup.expected_lobster_output_file_names:
generated = join(
setup.input_folder,
expected_lobster_output_file_name,
)
print(f"DELETING {generated}")
try:
remove(generated)
except FileNotFoundError:
pass


def _run_tests(directory: Path, tool: str) -> int:
Expand Down

0 comments on commit 8d2b1fb

Please sign in to comment.