Migrate "basic" system tests of lobster-json (#153)

Revert gitignore for expected-output:
All files in any folder called `expected-output` inside `tests-system`
are exempted from all gitignore rules.

Improve readability of assertion messages for system tests:
If there is a difference between the expected and actual output of
either stdout or stderr, then these values are fully printed. This helps
developers to spot errors faster.

Migrate "basic" system tests of lobster-json:
Migrate the "basic" system test so that they can be executed with the
`run_tool_tests.py` tool.

Write requirements for these tests.

`run_tool_tests.py` now deletes *.lobster files that were generated
during test execution.

Makefile

@@ -58,9 +58,9 @@ integration-tests: packages
 
 system-tests:
 	mkdir -p docs
+	make -B -C tests-system TOOL=lobster-json
 	make -B -C tests-system TOOL=lobster-trlc
 	make -B -C tests-system TOOL=lobster-python
-	make -B -C tests-system/lobster-json
 
 unit-tests:
 	coverage run -p \

lobster/tools/json/requirements.trlc

@@ -1,16 +1,20 @@
 package json_req
 import req
 
-req.Software_Requirement Dummy_Requirement {
+req.Software_Requirement Synthetic_Tag_Name {
   description = '''
-    This is not really a requirement. It will be used only to generate a minimal tracing report for each tool.
-    It can be deleted as soon as all the tools get their real requirements.
+    If the command line option --name-attribute is not given, then the tool shall create
+    a synthetic tag name based on the path of the JSON input file and an item counter.
+
+    Note: The item counter does not need to be unique across files.
+    It shall be unique at least for each input file separately.
   '''
 }
 
-req.Software_Requirement Dummy_Requirement_Unit_Test {
+req.Software_Requirement Name_Attribute {
   description = '''
-    This is not really a requirement. It will be used only to generate a minimal tracing report for each tool.
-    It can be deleted as soon as all the tools get their real requirements.
+    If the command line option --name-attribute is given, then the tool shall
+    - use the value of the command line argument as JSON key of the JSON item
+    - and use that obtained value as LOBSTER item tag name.
   '''
 }

tests-system/.gitignore

@@ -0,0 +1 @@
+!**/expected-output/*

tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt

@@ -0,0 +1 @@
+0

tests-system/lobster-json/basic.output → tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster

@@ -1,5 +1,3 @@
-lobster-json: wrote 6 items to basic.lobster
-==========
 {
   "data": [
     {

tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt

@@ -0,0 +1 @@
+lobster-json: wrote 6 items to output.lobster

tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt

@@ -0,0 +1,4 @@
+--single
+--tag-attribute=tags
+--name-attribute=name
+--out=output.lobster

tests-system/lobster-json/basic2.input → tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json

@@ -1,4 +1,3 @@
-// --tag-attribute=tags
 [
     {
 	"name"   : "Test_1",

tests-system/lobster-json/rbt-synthetic-tag-name/README.md

@@ -0,0 +1,22 @@
+If the `--name-attribute` option is not specified,
+then the tag name must be constructed based on the following two values:
+- path of the json file
+- item counter
+
+# Test case "flat-input-structure"
+- No files or directories are specified.
+  The tool is expected to search for input files in the current working directory.
+- `--name-attribute` is not given.
+  The tool is expected to construct LOBSTER item tags by taking the file name and
+  appending an integer counter, starting at 1.
+  This is implemented inside function `syn_test_name`, and it must handle all file
+  paths as relative paths, because no input files were given explicitly.
+- `--out` is specified.
+  The tool is expected to use that path name for the output file.
+- `--single` is specified to make the tool output predictable.
+
+# Test case "nested-input-structure"
+This is the same as the above test, except that its input file is nested in a
+sub-sub-directory.
+The tool is expected to take the names of the sub-directories into account when
+generating te tag name.

tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt

@@ -0,0 +1 @@
+0

tests-system/lobster-json/basic2.output → tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster

@@ -1,16 +1,14 @@
-lobster-json: wrote 6 items to basic2.lobster
-==========
 {
   "data": [
     {
-      "tag": "json ./basic2.json:basic2.1",
+      "tag": "json ./basic.json:basic.1",
       "location": {
         "kind": "file",
-        "file": "./basic2.json",
+        "file": "./basic.json",
         "line": null,
         "column": null
       },
-      "name": "./basic2.json:basic2.1",
+      "name": "./basic.json:basic.1",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -23,14 +21,14 @@ lobster-json: wrote 6 items to basic2.lobster
       "status": null
     },
     {
-      "tag": "json ./basic2.json:basic2.2",
+      "tag": "json ./basic.json:basic.2",
       "location": {
         "kind": "file",
-        "file": "./basic2.json",
+        "file": "./basic.json",
         "line": null,
         "column": null
       },
-      "name": "./basic2.json:basic2.2",
+      "name": "./basic.json:basic.2",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -40,14 +38,14 @@ lobster-json: wrote 6 items to basic2.lobster
       "status": null
     },
     {
-      "tag": "json ./basic2.json:basic2.3",
+      "tag": "json ./basic.json:basic.3",
       "location": {
         "kind": "file",
-        "file": "./basic2.json",
+        "file": "./basic.json",
         "line": null,
         "column": null
       },
-      "name": "./basic2.json:basic2.3",
+      "name": "./basic.json:basic.3",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -61,14 +59,14 @@ lobster-json: wrote 6 items to basic2.lobster
       "status": null
     },
     {
-      "tag": "json ./basic2.json:basic2.4",
+      "tag": "json ./basic.json:basic.4",
       "location": {
         "kind": "file",
-        "file": "./basic2.json",
+        "file": "./basic.json",
         "line": null,
         "column": null
       },
-      "name": "./basic2.json:basic2.4",
+      "name": "./basic.json:basic.4",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -78,14 +76,14 @@ lobster-json: wrote 6 items to basic2.lobster
       "status": null
     },
     {
-      "tag": "json ./basic2.json:basic2.5",
+      "tag": "json ./basic.json:basic.5",
       "location": {
         "kind": "file",
-        "file": "./basic2.json",
+        "file": "./basic.json",
         "line": null,
         "column": null
       },
-      "name": "./basic2.json:basic2.5",
+      "name": "./basic.json:basic.5",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -98,14 +96,14 @@ lobster-json: wrote 6 items to basic2.lobster
       "status": null
     },
     {
-      "tag": "json ./basic2.json:basic2.6",
+      "tag": "json ./basic.json:basic.6",
       "location": {
         "kind": "file",
-        "file": "./basic2.json",
+        "file": "./basic.json",
         "line": null,
         "column": null
       },
-      "name": "./basic2.json:basic2.6",
+      "name": "./basic.json:basic.6",
       "messages": [],
       "just_up": [],
       "just_down": [],

tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt

@@ -0,0 +1 @@
+lobster-json: wrote 6 items to output.lobster

tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt

@@ -0,0 +1,3 @@
+--single
+--tag-attribute=tags
+--out=output.lobster

tests-system/lobster-json/basic.input → tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json

@@ -1,4 +1,3 @@
-// --tag-attribute=tags --name-attribute=name
 [
     {
 	"name"   : "Test_1",

tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt

@@ -0,0 +1 @@
+0

tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster

@@ -0,0 +1,119 @@
+{
+  "data": [
+    {
+      "tag": "json ./one/two/basic.json:one.two.basic.1",
+      "location": {
+        "kind": "file",
+        "file": "./one/two/basic.json",
+        "line": null,
+        "column": null
+      },
+      "name": "./one/two/basic.json:one.two.basic.1",
+      "messages": [],
+      "just_up": [],
+      "just_down": [],
+      "just_global": [],
+      "refs": [
+        "req example.req1"
+      ],
+      "framework": "JSON",
+      "kind": "Test Vector",
+      "status": null
+    },
+    {
+      "tag": "json ./one/two/basic.json:one.two.basic.2",
+      "location": {
+        "kind": "file",
+        "file": "./one/two/basic.json",
+        "line": null,
+        "column": null
+      },
+      "name": "./one/two/basic.json:one.two.basic.2",
+      "messages": [],
+      "just_up": [],
+      "just_down": [],
+      "just_global": [],
+      "framework": "JSON",
+      "kind": "Test Vector",
+      "status": null
+    },
+    {
+      "tag": "json ./one/two/basic.json:one.two.basic.3",
+      "location": {
+        "kind": "file",
+        "file": "./one/two/basic.json",
+        "line": null,
+        "column": null
+      },
+      "name": "./one/two/basic.json:one.two.basic.3",
+      "messages": [],
+      "just_up": [],
+      "just_down": [],
+      "just_global": [],
+      "refs": [
+        "req example.req2",
+        "req example.req3"
+      ],
+      "framework": "JSON",
+      "kind": "Test Vector",
+      "status": null
+    },
+    {
+      "tag": "json ./one/two/basic.json:one.two.basic.4",
+      "location": {
+        "kind": "file",
+        "file": "./one/two/basic.json",
+        "line": null,
+        "column": null
+      },
+      "name": "./one/two/basic.json:one.two.basic.4",
+      "messages": [],
+      "just_up": [],
+      "just_down": [],
+      "just_global": [],
+      "framework": "JSON",
+      "kind": "Test Vector",
+      "status": null
+    },
+    {
+      "tag": "json ./one/two/basic.json:one.two.basic.5",
+      "location": {
+        "kind": "file",
+        "file": "./one/two/basic.json",
+        "line": null,
+        "column": null
+      },
+      "name": "./one/two/basic.json:one.two.basic.5",
+      "messages": [],
+      "just_up": [],
+      "just_down": [],
+      "just_global": [],
+      "refs": [
+        "req example.req4"
+      ],
+      "framework": "JSON",
+      "kind": "Test Vector",
+      "status": null
+    },
+    {
+      "tag": "json ./one/two/basic.json:one.two.basic.6",
+      "location": {
+        "kind": "file",
+        "file": "./one/two/basic.json",
+        "line": null,
+        "column": null
+      },
+      "name": "./one/two/basic.json:one.two.basic.6",
+      "messages": [],
+      "just_up": [],
+      "just_down": [],
+      "just_global": [],
+      "framework": "JSON",
+      "kind": "Test Vector",
+      "status": null
+    }
+  ],
+  "generator": "lobster-json",
+  "schema": "lobster-act-trace",
+  "version": 3
+}

tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt

@@ -0,0 +1 @@
+lobster-json: wrote 6 items to output.lobster

tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt

@@ -0,0 +1,3 @@
+--single
+--tag-attribute=tags
+--out=output.lobster

tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json

@@ -0,0 +1,38 @@
+[
+    {
+	"name"   : "Test_1",
+	"tags"   : ["example.req1"],
+	"inputs" : [1, 2],
+	"expect" : 3
+    },
+    {
+	"name"   : "Test_2",
+	"tags"   : [],
+	"inputs" : [1, 0],
+	"expect" : 1
+    },
+    {
+	"name"   : "Test_3",
+	"tags"   : ["example.req2",
+		    "example.req3"],
+	"inputs" : [1, 0],
+	"expect" : 1
+    },
+    {
+	"name"   : "Test_4",
+	"tags"   : null,
+	"inputs" : [1, 0],
+	"expect" : 1
+    },
+    {
+	"name"   : "Test_5",
+	"tags"   : "example.req4",
+	"inputs" : [1, 0],
+	"expect" : 1
+    },
+    {
+	"name"   : "Test_6",
+	"inputs" : [1, 0],
+	"expect" : 1
+    }
+]