From b6f3095a6f9a4ebb5d2bfb5e4bb903dd5eb55c09 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:09:42 +0100 Subject: [PATCH] Migrate "basic" system tests of `lobster-json` (#153) Revert gitignore for expected-output: All files in any folder called `expected-output` inside `tests-system` are exempted from all gitignore rules. Improve readability of assertion messages for system tests: If there is a difference between the expected and actual output of either stdout or stderr, then these values are fully printed. This helps developers to spot errors faster. Migrate "basic" system tests of lobster-json: Migrate the "basic" system test so that they can be executed with the `run_tool_tests.py` tool. Write requirements for these tests. `run_tool_tests.py` now deletes *.lobster files that were generated during test execution. --- Makefile | 2 +- lobster/tools/json/requirements.trlc | 16 ++- tests-system/.gitignore | 1 + tests-system/lobster-json/Makefile | 15 --- .../expected-output/exit-code.txt | 1 + .../expected-output/output.lobster} | 2 - .../expected-output/stderr.txt} | 0 .../expected-output/stdout.txt | 1 + .../attribute-given/input/args.txt | 4 + .../attribute-given/input/basic.json} | 1 - .../rbt-synthetic-tag-name/README.md | 22 ++++ .../expected-output/exit-code.txt | 1 + .../expected-output/output.lobster} | 38 +++--- .../expected-output/stderr.txt | 0 .../expected-output/stdout.txt | 1 + .../flat-input-structure/input/args.txt | 3 + .../flat-input-structure/input/basic.json} | 1 - .../expected-output/exit-code.txt | 1 + .../expected-output/output.lobster | 119 ++++++++++++++++++ .../expected-output/stderr.txt | 0 .../expected-output/stdout.txt | 1 + .../nested-input-structure/input/args.txt | 3 + .../input/one/two/basic.json | 38 ++++++ tests-system/run_tool_tests.py | 44 +++++-- 24 files changed, 260 insertions(+), 55 deletions(-) create mode 100644 tests-system/.gitignore delete mode 100644 tests-system/lobster-json/Makefile create mode 100644 tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt rename tests-system/lobster-json/{basic.output => rbt-name-attribute/attribute-given/expected-output/output.lobster} (97%) rename tests-system/lobster-json/{rbt-dummy/.gitkeep => rbt-name-attribute/attribute-given/expected-output/stderr.txt} (100%) create mode 100644 tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt create mode 100644 tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt rename tests-system/lobster-json/{basic2.input => rbt-name-attribute/attribute-given/input/basic.json} (95%) create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/README.md create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt rename tests-system/lobster-json/{basic2.output => rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster} (72%) create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt rename tests-system/lobster-json/{basic.input => rbt-synthetic-tag-name/flat-input-structure/input/basic.json} (92%) create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json diff --git a/Makefile b/Makefile index 127eefd3..7ec0e6de 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,9 @@ integration-tests: packages system-tests: mkdir -p docs + make -B -C tests-system TOOL=lobster-json make -B -C tests-system TOOL=lobster-trlc make -B -C tests-system TOOL=lobster-python - make -B -C tests-system/lobster-json unit-tests: coverage run -p \ diff --git a/lobster/tools/json/requirements.trlc b/lobster/tools/json/requirements.trlc index 309de1f3..0ee8b944 100644 --- a/lobster/tools/json/requirements.trlc +++ b/lobster/tools/json/requirements.trlc @@ -1,16 +1,20 @@ package json_req import req -req.Software_Requirement Dummy_Requirement { +req.Software_Requirement Synthetic_Tag_Name { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is not given, then the tool shall create + a synthetic tag name based on the path of the JSON input file and an item counter. + + Note: The item counter does not need to be unique across files. + It shall be unique at least for each input file separately. ''' } -req.Software_Requirement Dummy_Requirement_Unit_Test { +req.Software_Requirement Name_Attribute { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is given, then the tool shall + - use the value of the command line argument as JSON key of the JSON item + - and use that obtained value as LOBSTER item tag name. ''' } diff --git a/tests-system/.gitignore b/tests-system/.gitignore new file mode 100644 index 00000000..78f221c8 --- /dev/null +++ b/tests-system/.gitignore @@ -0,0 +1 @@ +!**/expected-output/* diff --git a/tests-system/lobster-json/Makefile b/tests-system/lobster-json/Makefile deleted file mode 100644 index 55f3e314..00000000 --- a/tests-system/lobster-json/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -TOOL=../../lobster-json - -TARGETS=$(addsuffix .output, $(basename $(wildcard **/*.input) $(wildcard *.input))) - -all: $(TARGETS) - -%.output: %.input - @tail -n +2 $< > $*.json - @touch $*.lobster - -@coverage run -p --rcfile=../../coverage.cfg --branch \ - --data-file ../../.coverage \ - $(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 - @echo "==========" >> $@ - @cat $*.lobster >> $@ - @rm $*.json $*.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic.output b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster similarity index 97% rename from tests-system/lobster-json/basic.output rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster index 3157c93e..79d8d7f3 100644 --- a/tests-system/lobster-json/basic.output +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster @@ -1,5 +1,3 @@ -lobster-json: wrote 6 items to basic.lobster -========== { "data": [ { diff --git a/tests-system/lobster-json/rbt-dummy/.gitkeep b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-json/rbt-dummy/.gitkeep rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt new file mode 100644 index 00000000..44e3c250 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt @@ -0,0 +1,4 @@ +--single +--tag-attribute=tags +--name-attribute=name +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.input b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json similarity index 95% rename from tests-system/lobster-json/basic2.input rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json index f57ea48e..7bb52fe8 100644 --- a/tests-system/lobster-json/basic2.input +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/README.md b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md new file mode 100644 index 00000000..944ef261 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md @@ -0,0 +1,22 @@ +If the `--name-attribute` option is not specified, +then the tag name must be constructed based on the following two values: +- path of the json file +- item counter + +# Test case "flat-input-structure" +- No files or directories are specified. + The tool is expected to search for input files in the current working directory. +- `--name-attribute` is not given. + The tool is expected to construct LOBSTER item tags by taking the file name and + appending an integer counter, starting at 1. + This is implemented inside function `syn_test_name`, and it must handle all file + paths as relative paths, because no input files were given explicitly. +- `--out` is specified. + The tool is expected to use that path name for the output file. +- `--single` is specified to make the tool output predictable. + +# Test case "nested-input-structure" +This is the same as the above test, except that its input file is nested in a +sub-sub-directory. +The tool is expected to take the names of the sub-directories into account when +generating te tag name. diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.output b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster similarity index 72% rename from tests-system/lobster-json/basic2.output rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster index 287d5a13..d0331f77 100644 --- a/tests-system/lobster-json/basic2.output +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster @@ -1,16 +1,14 @@ -lobster-json: wrote 6 items to basic2.lobster -========== { "data": [ { - "tag": "json ./basic2.json:basic2.1", + "tag": "json ./basic.json:basic.1", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.1", + "name": "./basic.json:basic.1", "messages": [], "just_up": [], "just_down": [], @@ -23,14 +21,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.2", + "tag": "json ./basic.json:basic.2", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.2", + "name": "./basic.json:basic.2", "messages": [], "just_up": [], "just_down": [], @@ -40,14 +38,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.3", + "tag": "json ./basic.json:basic.3", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.3", + "name": "./basic.json:basic.3", "messages": [], "just_up": [], "just_down": [], @@ -61,14 +59,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.4", + "tag": "json ./basic.json:basic.4", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.4", + "name": "./basic.json:basic.4", "messages": [], "just_up": [], "just_down": [], @@ -78,14 +76,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.5", + "tag": "json ./basic.json:basic.5", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.5", + "name": "./basic.json:basic.5", "messages": [], "just_up": [], "just_down": [], @@ -98,14 +96,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.6", + "tag": "json ./basic.json:basic.6", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.6", + "name": "./basic.json:basic.6", "messages": [], "just_up": [], "just_down": [], diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic.input b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json similarity index 92% rename from tests-system/lobster-json/basic.input rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json index fa82b8a4..7bb52fe8 100644 --- a/tests-system/lobster-json/basic.input +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags --name-attribute=name [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster new file mode 100644 index 00000000..8571ad42 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster @@ -0,0 +1,119 @@ +{ + "data": [ + { + "tag": "json ./one/two/basic.json:one.two.basic.1", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.1", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req1" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.2", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.2", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.3", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.3", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req2", + "req example.req3" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.4", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.4", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.5", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.5", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req4" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.6", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.6", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + } + ], + "generator": "lobster-json", + "schema": "lobster-act-trace", + "version": 3 +} diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json new file mode 100644 index 00000000..7bb52fe8 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json @@ -0,0 +1,38 @@ +[ + { + "name" : "Test_1", + "tags" : ["example.req1"], + "inputs" : [1, 2], + "expect" : 3 + }, + { + "name" : "Test_2", + "tags" : [], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_3", + "tags" : ["example.req2", + "example.req3"], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_4", + "tags" : null, + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_5", + "tags" : "example.req4", + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_6", + "inputs" : [1, 0], + "expect" : 1 + } +] diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 8e8d34f9..8a133f76 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,5 +1,5 @@ import sys -from os import scandir, DirEntry +from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path from subprocess import CompletedProcess, PIPE, run @@ -53,7 +53,8 @@ def _get_expected_lobster_output_file_name(self) -> str: if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( - "Invalid test setup: No *.lobster file found in expected output folder!", + f"Invalid test setup: No *.lobster file found in " + f"{self.get_expected_output_path()}!", ) @staticmethod @@ -151,17 +152,31 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: return completed_process +def _compare_cmd_output(name: str, expected: str, actual: str) -> bool: + if expected != actual: + print(f"Actual {name} is (length: {len(actual)} chars):\n{actual}") + print(f"Expected {name} is (length: {len(expected)} chars):\n{expected}") + return False + return True + + def _compare_results(setup: TestSetup, completed_process: CompletedProcess): - if setup.expected_exit_code != completed_process.returncode: - print(f"STDOUT is: {completed_process.stdout}") - print(f"STDERR is: {completed_process.stderr}") assert setup.expected_exit_code == completed_process.returncode, \ f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \ f"actual is {completed_process.returncode}!" - assert setup.get_expected_stdout() == completed_process.stdout, \ - "Command line output for stdout is different!" - assert setup.get_expected_stderr() == completed_process.stderr, \ - "Command line output for stderr is different!" + + assert _compare_cmd_output( + name="STDOUT", + expected=setup.get_expected_stdout(), + actual=completed_process.stdout, + ), "Command line output for STDOUT is different!" + + assert _compare_cmd_output( + name="STDERR", + expected=setup.get_expected_stderr(), + actual=completed_process.stderr, + ), "Command line output for STDERR is different!" + expected = join( setup.get_expected_output_path(), setup.expected_lobster_output_file_name, @@ -200,6 +215,16 @@ def _get_directories( yield dir_entry +def _delete_generated_files(setup: TestSetup): + """Deletes the *.lobster file that has been generated by the test""" + generated = join( + setup.input_folder, + setup.expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + remove(generated) + + def _run_tests(directory: str, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. @@ -218,6 +243,7 @@ def _run_tests(directory: str, tool: str) -> int: test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) + _delete_generated_files(test_setup) counter += 1 print(f"{counter} system tests finished successfully for {tool}.")