Skip to content

Commit

Permalink
Revert gitignore for expected-output:
Browse files Browse the repository at this point in the history
All files in any folder called `expected-output` inside `tests-system`
are exempted from all gitignore rules.

Improve readability of assertion messages for system tests:
If there is a difference between the expected and actual output
of either stdout or stderr, then these values are fully printed.
This helps developers to spot errors faster.
  • Loading branch information
phiwuu committed Dec 10, 2024
1 parent 3105b8e commit c9c22e5
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 7 deletions.
1 change: 1 addition & 0 deletions tests-system/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
!**/expected-output/*
28 changes: 21 additions & 7 deletions tests-system/run_tool_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,17 +151,31 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess:
return completed_process


def _compare_cmd_output(name: str, expected: str, actual: str) -> bool:
if expected != actual:
print(f"Actual {name} is (length: {len(actual)} chars):\n{actual}")
print(f"Expected {name} is (length: {len(expected)} chars):\n{expected}")
return False
return True


def _compare_results(setup: TestSetup, completed_process: CompletedProcess):
if setup.expected_exit_code != completed_process.returncode:
print(f"STDOUT is: {completed_process.stdout}")
print(f"STDERR is: {completed_process.stderr}")
assert setup.expected_exit_code == completed_process.returncode, \
f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \
f"actual is {completed_process.returncode}!"
assert setup.get_expected_stdout() == completed_process.stdout, \
"Command line output for stdout is different!"
assert setup.get_expected_stderr() == completed_process.stderr, \
"Command line output for stderr is different!"

assert _compare_cmd_output(
name="STDOUT",
expected=setup.get_expected_stdout(),
actual=completed_process.stdout,
), "Command line output for STDOUT is different!"

assert _compare_cmd_output(
name="STDERR",
expected=setup.get_expected_stderr(),
actual=completed_process.stderr,
), "Command line output for STDERR is different!"

expected = join(
setup.get_expected_output_path(),
setup.expected_lobster_output_file_name,
Expand Down

0 comments on commit c9c22e5

Please sign in to comment.