diff --git a/Makefile b/Makefile index 127eefd3..7ec0e6de 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,9 @@ integration-tests: packages system-tests: mkdir -p docs + make -B -C tests-system TOOL=lobster-json make -B -C tests-system TOOL=lobster-trlc make -B -C tests-system TOOL=lobster-python - make -B -C tests-system/lobster-json unit-tests: coverage run -p \ diff --git a/lobster/tools/json/requirements.trlc b/lobster/tools/json/requirements.trlc index 309de1f3..0ee8b944 100644 --- a/lobster/tools/json/requirements.trlc +++ b/lobster/tools/json/requirements.trlc @@ -1,16 +1,20 @@ package json_req import req -req.Software_Requirement Dummy_Requirement { +req.Software_Requirement Synthetic_Tag_Name { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is not given, then the tool shall create + a synthetic tag name based on the path of the JSON input file and an item counter. + + Note: The item counter does not need to be unique across files. + It shall be unique at least for each input file separately. ''' } -req.Software_Requirement Dummy_Requirement_Unit_Test { +req.Software_Requirement Name_Attribute { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is given, then the tool shall + - use the value of the command line argument as JSON key of the JSON item + - and use that obtained value as LOBSTER item tag name. ''' } diff --git a/tests-system/.gitignore b/tests-system/.gitignore new file mode 100644 index 00000000..78f221c8 --- /dev/null +++ b/tests-system/.gitignore @@ -0,0 +1 @@ +!**/expected-output/* diff --git a/tests-system/lobster-json/Makefile b/tests-system/lobster-json/Makefile deleted file mode 100644 index 55f3e314..00000000 --- a/tests-system/lobster-json/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -TOOL=../../lobster-json - -TARGETS=$(addsuffix .output, $(basename $(wildcard **/*.input) $(wildcard *.input))) - -all: $(TARGETS) - -%.output: %.input - @tail -n +2 $< > $*.json - @touch $*.lobster - -@coverage run -p --rcfile=../../coverage.cfg --branch \ - --data-file ../../.coverage \ - $(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 - @echo "==========" >> $@ - @cat $*.lobster >> $@ - @rm $*.json $*.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic.output b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster similarity index 97% rename from tests-system/lobster-json/basic.output rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster index 3157c93e..79d8d7f3 100644 --- a/tests-system/lobster-json/basic.output +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster @@ -1,5 +1,3 @@ -lobster-json: wrote 6 items to basic.lobster -========== { "data": [ { diff --git a/tests-system/lobster-json/rbt-dummy/.gitkeep b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-json/rbt-dummy/.gitkeep rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt new file mode 100644 index 00000000..44e3c250 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt @@ -0,0 +1,4 @@ +--single +--tag-attribute=tags +--name-attribute=name +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.input b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json similarity index 95% rename from tests-system/lobster-json/basic2.input rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json index f57ea48e..7bb52fe8 100644 --- a/tests-system/lobster-json/basic2.input +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/README.md b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md new file mode 100644 index 00000000..944ef261 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md @@ -0,0 +1,22 @@ +If the `--name-attribute` option is not specified, +then the tag name must be constructed based on the following two values: +- path of the json file +- item counter + +# Test case "flat-input-structure" +- No files or directories are specified. + The tool is expected to search for input files in the current working directory. +- `--name-attribute` is not given. + The tool is expected to construct LOBSTER item tags by taking the file name and + appending an integer counter, starting at 1. + This is implemented inside function `syn_test_name`, and it must handle all file + paths as relative paths, because no input files were given explicitly. +- `--out` is specified. + The tool is expected to use that path name for the output file. +- `--single` is specified to make the tool output predictable. + +# Test case "nested-input-structure" +This is the same as the above test, except that its input file is nested in a +sub-sub-directory. +The tool is expected to take the names of the sub-directories into account when +generating te tag name. diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.output b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster similarity index 72% rename from tests-system/lobster-json/basic2.output rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster index 287d5a13..d0331f77 100644 --- a/tests-system/lobster-json/basic2.output +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster @@ -1,16 +1,14 @@ -lobster-json: wrote 6 items to basic2.lobster -========== { "data": [ { - "tag": "json ./basic2.json:basic2.1", + "tag": "json ./basic.json:basic.1", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.1", + "name": "./basic.json:basic.1", "messages": [], "just_up": [], "just_down": [], @@ -23,14 +21,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.2", + "tag": "json ./basic.json:basic.2", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.2", + "name": "./basic.json:basic.2", "messages": [], "just_up": [], "just_down": [], @@ -40,14 +38,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.3", + "tag": "json ./basic.json:basic.3", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.3", + "name": "./basic.json:basic.3", "messages": [], "just_up": [], "just_down": [], @@ -61,14 +59,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.4", + "tag": "json ./basic.json:basic.4", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.4", + "name": "./basic.json:basic.4", "messages": [], "just_up": [], "just_down": [], @@ -78,14 +76,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.5", + "tag": "json ./basic.json:basic.5", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.5", + "name": "./basic.json:basic.5", "messages": [], "just_up": [], "just_down": [], @@ -98,14 +96,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.6", + "tag": "json ./basic.json:basic.6", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.6", + "name": "./basic.json:basic.6", "messages": [], "just_up": [], "just_down": [], diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic.input b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json similarity index 92% rename from tests-system/lobster-json/basic.input rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json index fa82b8a4..7bb52fe8 100644 --- a/tests-system/lobster-json/basic.input +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags --name-attribute=name [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster new file mode 100644 index 00000000..8571ad42 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster @@ -0,0 +1,119 @@ +{ + "data": [ + { + "tag": "json ./one/two/basic.json:one.two.basic.1", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.1", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req1" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.2", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.2", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.3", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.3", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req2", + "req example.req3" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.4", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.4", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.5", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.5", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req4" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.6", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.6", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + } + ], + "generator": "lobster-json", + "schema": "lobster-act-trace", + "version": 3 +} diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json new file mode 100644 index 00000000..7bb52fe8 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json @@ -0,0 +1,38 @@ +[ + { + "name" : "Test_1", + "tags" : ["example.req1"], + "inputs" : [1, 2], + "expect" : 3 + }, + { + "name" : "Test_2", + "tags" : [], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_3", + "tags" : ["example.req2", + "example.req3"], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_4", + "tags" : null, + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_5", + "tags" : "example.req4", + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_6", + "inputs" : [1, 0], + "expect" : 1 + } +] diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 8e8d34f9..8a133f76 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,5 +1,5 @@ import sys -from os import scandir, DirEntry +from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path from subprocess import CompletedProcess, PIPE, run @@ -53,7 +53,8 @@ def _get_expected_lobster_output_file_name(self) -> str: if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( - "Invalid test setup: No *.lobster file found in expected output folder!", + f"Invalid test setup: No *.lobster file found in " + f"{self.get_expected_output_path()}!", ) @staticmethod @@ -151,17 +152,31 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: return completed_process +def _compare_cmd_output(name: str, expected: str, actual: str) -> bool: + if expected != actual: + print(f"Actual {name} is (length: {len(actual)} chars):\n{actual}") + print(f"Expected {name} is (length: {len(expected)} chars):\n{expected}") + return False + return True + + def _compare_results(setup: TestSetup, completed_process: CompletedProcess): - if setup.expected_exit_code != completed_process.returncode: - print(f"STDOUT is: {completed_process.stdout}") - print(f"STDERR is: {completed_process.stderr}") assert setup.expected_exit_code == completed_process.returncode, \ f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \ f"actual is {completed_process.returncode}!" - assert setup.get_expected_stdout() == completed_process.stdout, \ - "Command line output for stdout is different!" - assert setup.get_expected_stderr() == completed_process.stderr, \ - "Command line output for stderr is different!" + + assert _compare_cmd_output( + name="STDOUT", + expected=setup.get_expected_stdout(), + actual=completed_process.stdout, + ), "Command line output for STDOUT is different!" + + assert _compare_cmd_output( + name="STDERR", + expected=setup.get_expected_stderr(), + actual=completed_process.stderr, + ), "Command line output for STDERR is different!" + expected = join( setup.get_expected_output_path(), setup.expected_lobster_output_file_name, @@ -200,6 +215,16 @@ def _get_directories( yield dir_entry +def _delete_generated_files(setup: TestSetup): + """Deletes the *.lobster file that has been generated by the test""" + generated = join( + setup.input_folder, + setup.expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + remove(generated) + + def _run_tests(directory: str, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. @@ -218,6 +243,7 @@ def _run_tests(directory: str, tool: str) -> int: test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) + _delete_generated_files(test_setup) counter += 1 print(f"{counter} system tests finished successfully for {tool}.")