diff --git a/.gitignore b/.gitignore index f7f60576..5d7900cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *~ *.pyc bazel-* +launch.json *.egg-info build diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7e3327..d1a66fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,23 @@ ### 0.9.21-dev +* `lobster-html-report` has the following updates. + * Filter items by status (Ok, Missing, Partial, Warning, Justified) + * Hide/Unhide Issues. + * Search in issues and detailed report. +* Add support to view version for lobster tools for following tools: + - `lobster-ci-report` + - `lobster-codebeamer` + - `lobster-cpp` + - `lobster-cpptest` + - `lobster-gtest` + - `lobster-html-report` + - `lobster-json` + - `lobster-online-report` + - `lobster-python` + - `lobster-report` + - `lobster-trlc` ### 0.9.20 @@ -24,19 +40,6 @@ * `lobster-codebeamer` * `lobster-report` -* Add support to view version for lobster tools for following tools: - - `lobster-ci-report` - - `lobster-codebeamer` - - `lobster-cpp` - - `lobster-cpptest` - - `lobster-gtest` - - `lobster-html-report` - - `lobster-json` - - `lobster-online-report` - - `lobster-python` - - `lobster-report` - - `lobster-trlc` - * `lobster-gtest` accepts XML nodes other than `testcase`, but ignores them. ### 0.9.19 diff --git a/Makefile b/Makefile index 9bf253bb..7ec0e6de 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,9 @@ integration-tests: packages system-tests: mkdir -p docs - make -B -C tests-system/lobster-trlc - make -B -C tests-system/lobster-json - make -B -C tests-system/lobster-python + make -B -C tests-system TOOL=lobster-json + make -B -C tests-system TOOL=lobster-trlc + make -B -C tests-system TOOL=lobster-python unit-tests: coverage run -p \ @@ -92,9 +92,9 @@ full-release: coverage: coverage combine -q coverage html --rcfile=coverage.cfg - coverage report --rcfile=coverage.cfg --fail-under=66 + coverage report --rcfile=coverage.cfg --fail-under=62 -test: system-tests unit-tests +test: clean-coverage system-tests unit-tests make coverage util/check_local_modifications.sh @@ -149,3 +149,9 @@ unit-tests.lobster-%: system-tests.lobster-%: $(eval TOOL_PATH := $(subst -,/,$*)) python3 tests-system/lobster-trlc-system-test.py $(TOOL_PATH); + +clean-coverage: + @rm -rf htmlcov + @find . -name '.coverage*' -type f -delete + @find . -name '*.pyc' -type f -delete + @echo "All .coverage, .coverage.* and *.pyc files deleted." diff --git a/lobster/html/htmldoc.py b/lobster/html/htmldoc.py index 071065f0..fbdea7dc 100644 --- a/lobster/html/htmldoc.py +++ b/lobster/html/htmldoc.py @@ -247,6 +247,7 @@ def __init__(self, title, subtitle): } self.scripts = [] self.body = [] + self.css_files = [] def add_line(self, line): assert isinstance(line, str) @@ -293,6 +294,10 @@ def render(self): rv.append(" %s: %s;" % (attr, value)) rv.append("}") rv.append("") + + # add css files that are appended to self.files + for css_file in self.css_files: + rv.append(f"") rv.append("") rv.append("") diff --git a/lobster/tools/core/html_report/assets/html_report.css b/lobster/tools/core/html_report/assets/html_report.css new file mode 100644 index 00000000..a505901f --- /dev/null +++ b/lobster/tools/core/html_report/assets/html_report.css @@ -0,0 +1,139 @@ +.button { + background-color: #818589; + border: none; + border-radius: 5px; + color: white; + padding: 12px 25px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 14px; + margin: 4px 2px; + cursor: pointer +} + +.button active:before { + content: ; + position: absolute; + left: 0; + top: 0; + display: inline-block; + width: 0; + height: 0; + border-style: solid; + border-width: 15px 15px 0 0; + border-color: #333 transparent transparent transparent +} + +.buttonActive.button { + text-decoration: none; + border: 5px solid #000000 +} + +.buttonOK { + background-color: #04AA6D; + color: white; + border: 2px solid #04AA6D; + border-radius: 5px +} + +.buttonOK:hover { + background-color: #026641; + color: white; + border: 2px solid #026641 +} + +.buttonActive.buttonOK { + text-decoration: none; + border: 5px solid #026641 +} + +.buttonPartial { + background-color: #17a2b8; + color: white; + border: 2px solid #17a2b8; + border-radius: 5px +} + +.buttonPartial:hover { + background-color: #0e616e; + color: white; + border: 2px solid #0e616e +} + +.buttonActive.buttonPartial { + text-decoration: none; + border: 5px solid #0e616e +} + +.buttonMissing { + background-color: #f44336; + color: white; + border: 2px solid #f44336; + border-radius: 5px +} + +.buttonMissing:hover { + background-color: #a91409; + color: white; + border: 2px solid #a91409 +} + +.buttonActive.buttonMissing { + text-decoration: none; + border: 5px solid #a91409 +} + +.buttonJustified { + background-color: #6c757d; + color: white; + border: 2px solid #6c757d; + border-radius: 5px +} + +.buttonJustified:hover { + background-color: #41464b; + color: white; + border: 2px solid #41464b +} + +.buttonActive.buttonJustified { + text-decoration: none; + border: 5px solid #41464b +} + +.buttonWarning { + background-color: #ffbf00; + color: white; + border: 2px solid #ffbf00; + border-radius: 5px +} + +.buttonWarning:hover { + background-color: #997300; + color: white; + border: 2px solid #997300 +} + +.buttonActive.buttonWarning { + text-decoration: none; + border: 5px solid #997300 +} + +.buttonBlue { + background-color: #0000ff; + color: white; + border: 2px solid #0000ff; + border-radius: 5px +} + +.buttonBlue:hover { + background-color: #000099; + color: white; + border: 2px solid #000099 +} + +.buttonActive.buttonBlue { + text-decoration: none; + border: 5px solid #000099 +} diff --git a/lobster/tools/core/html_report/assets/html_report.js b/lobster/tools/core/html_report/assets/html_report.js new file mode 100644 index 00000000..b2f5a874 --- /dev/null +++ b/lobster/tools/core/html_report/assets/html_report.js @@ -0,0 +1,87 @@ +function buttonFilter(filter) { + var elms = document.getElementsByTagName("div"); + var issue_elms = document.getElementsByClassName("issue"); + for (i = 0; i < elms.length; i++) { + if (elms[i].id.startsWith("item-")) { + console.log("elms[i].className ", elms[i].className) + if (filter == "all") { + elms[i].style.display = "block"; + } else if (elms[i].className == "item-" + filter) { + elms[i].style.display = "block"; + } else { + elms[i].style.display = "none"; + } + } + } + // filter the issues list based on the issue filter button clicked + for (i = 0; i < issue_elms.length; i++) { + console.log("log ", issue_elms[i].className) + if (filter == "all") { + issue_elms[i].style.display = "list-item"; + } else if (issue_elms[i].className == "issue issue-" + filter) { + issue_elms[i].style.display = "list-item"; + } else { + issue_elms[i].style.display = "none"; + } + } + activeButton(filter); + //call the search filering which could have been overwritten by the current filtering + searchItem(); +} + + +function activeButton(filter) { + var elms = document.getElementsByTagName("button"); + console.log("the click buitton is " + filter); + for (i = 0; i < elms.length; i++) { + if (elms[i].className.includes("buttonActive")) { + console.log("elem active found : " + elms[i].className); + elms[i].className = elms[i].className.replace("buttonActive", ""); + } else if (elms[i].className.toLowerCase().includes("button" + filter.toLowerCase())) { + console.log("elem to be activated found : " + elms[i].className); + elms[i].className = elms[i].className + " buttonActive"; + } + } +} + + +function ToggleIssues() { + var div_issue = document.getElementById("issues-section"); + if (div_issue.style.display == "block" || div_issue.style.display == "") { + div_issue.style.display = "none"; + document.getElementById("BtnToggleIssue").innerHTML = "Show Issues"; + document.getElementById("BtnToggleIssue").className = document.getElementById("BtnToggleIssue").className + " buttonActive"; + } else { + div_issue.style = 'display: block; flex-direction: column; height: 200px;' + + 'overflow:auto;'; + document.getElementById("BtnToggleIssue").innerHTML = "Hide Issues"; + document.getElementById("BtnToggleIssue").className = document.getElementById("BtnToggleIssue").className.replace("buttonActive", ""); + } +} + + +function searchItem() { + var input = document.getElementById('search').value + input = input.toLowerCase(); + + var divs = document.getElementsByClassName('item-name'); + for (i = 0; i < divs.length; i++) { + var title = divs[i].parentNode.getAttribute("title"); + // get requirement name: 2nd part when we cut the long string with /svg + var reqname = divs[i].innerHTML.toLowerCase().split("").pop(); + reqname = reqname.split(" ").pop(); + if (reqname.includes(input)) { + // the search pattern has been found, if this elem has the title "hidden-not-matching", put it back to diplayed + if (title) { + if (title.startsWith("hidden-not-matching")) { + divs[i].parentNode.style.display = "block"; + } + } + divs[i].parentNode.setAttribute("title", "matching-" + input) + } else { + // not maching, we hide + divs[i].parentNode.setAttribute("title", "hidden-not-matching") + divs[i].parentNode.style.display = "none"; + } + } +} diff --git a/lobster/tools/core/html_report/html_report.py b/lobster/tools/core/html_report/html_report.py index 2bf7c141..c40a9501 100755 --- a/lobster/tools/core/html_report/html_report.py +++ b/lobster/tools/core/html_report/html_report.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU Affero General Public # License along with this program. If not, see # . - import os.path import argparse import html @@ -374,8 +373,43 @@ def write_html(fd, report, dot, high_contrast): print("> please install Graphviz (https://graphviz.org)") doc.add_line('') + ### Filtering + doc.add_heading(2, "Filtering", "filtering-options") + doc.add_heading(3, "Item Filters") + doc.add_line('
') + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + doc.add_line("
") + + doc.add_heading(3, "Show Issues") + doc.add_line('
') + doc.add_line('') + doc.add_line('
') + + doc.add_heading(3, "Search", "search") + doc.add_line('') + doc.add_line('") ### Report file_heading = None @@ -447,6 +483,19 @@ def write_html(fd, report, dot, high_contrast): write_item_box_end(doc) else: doc.add_line("No items recorded at this level.") + # Closing tag for id #search-sec-id + doc.add_line("") + + # Add the css from assets + dir_path = os.path.dirname(os.path.abspath(__file__)) + file_path = dir_path + "/assets/html_report.css" + doc.css_files.append(file_path) + + # Add javascript from assets/html_report.js file + dir_path = os.path.dirname(os.path.abspath(__file__)) + file_path = dir_path + "/assets/html_report.js" + with open(file_path, "r", encoding="UTF-8") as scripts: + doc.scripts.append("".join(scripts.readlines())) ### STM # doc.add_heading(2, "Software traceability matrix", "matrix") diff --git a/lobster/tools/json/requirements.trlc b/lobster/tools/json/requirements.trlc index 309de1f3..0ee8b944 100644 --- a/lobster/tools/json/requirements.trlc +++ b/lobster/tools/json/requirements.trlc @@ -1,16 +1,20 @@ package json_req import req -req.Software_Requirement Dummy_Requirement { +req.Software_Requirement Synthetic_Tag_Name { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is not given, then the tool shall create + a synthetic tag name based on the path of the JSON input file and an item counter. + + Note: The item counter does not need to be unique across files. + It shall be unique at least for each input file separately. ''' } -req.Software_Requirement Dummy_Requirement_Unit_Test { +req.Software_Requirement Name_Attribute { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is given, then the tool shall + - use the value of the command line argument as JSON key of the JSON item + - and use that obtained value as LOBSTER item tag name. ''' } diff --git a/packages/lobster-core/setup.py b/packages/lobster-core/setup.py index af0db774..9df68841 100644 --- a/packages/lobster-core/setup.py +++ b/packages/lobster-core/setup.py @@ -52,6 +52,9 @@ "lobster.tools.core.html_report", "lobster.tools.core.online_report", "lobster.tools.core.report"], + package_data={ + "lobster.tools.core.html_report":["assets/*"] + }, install_requires=[], python_requires=">=3.7, <4", classifiers=[ diff --git a/packages/lobster-monolithic/setup.py b/packages/lobster-monolithic/setup.py index 4960fead..2b161414 100644 --- a/packages/lobster-monolithic/setup.py +++ b/packages/lobster-monolithic/setup.py @@ -48,6 +48,9 @@ project_urls=project_urls, license="GNU Affero General Public License v3", packages=setuptools.find_packages(), + package_data={ + "lobster.tools.core.html_report":["assets/*"] + }, install_requires=[ "miss-hit>=0.9.42", "requests>=2.22", diff --git a/tests-system/.gitignore b/tests-system/.gitignore new file mode 100644 index 00000000..78f221c8 --- /dev/null +++ b/tests-system/.gitignore @@ -0,0 +1 @@ +!**/expected-output/* diff --git a/tests-system/Makefile b/tests-system/Makefile new file mode 100644 index 00000000..a99ec8cb --- /dev/null +++ b/tests-system/Makefile @@ -0,0 +1,4 @@ +PYTHON = python + +run-tool-tests: + @$(PYTHON) ./run_tool_tests.py ./$(TOOL) diff --git a/tests-system/lobster-json/Makefile b/tests-system/lobster-json/Makefile deleted file mode 100644 index 55f3e314..00000000 --- a/tests-system/lobster-json/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -TOOL=../../lobster-json - -TARGETS=$(addsuffix .output, $(basename $(wildcard **/*.input) $(wildcard *.input))) - -all: $(TARGETS) - -%.output: %.input - @tail -n +2 $< > $*.json - @touch $*.lobster - -@coverage run -p --rcfile=../../coverage.cfg --branch \ - --data-file ../../.coverage \ - $(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 - @echo "==========" >> $@ - @cat $*.lobster >> $@ - @rm $*.json $*.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic.output b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster similarity index 97% rename from tests-system/lobster-json/basic.output rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster index 3157c93e..79d8d7f3 100644 --- a/tests-system/lobster-json/basic.output +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster @@ -1,5 +1,3 @@ -lobster-json: wrote 6 items to basic.lobster -========== { "data": [ { diff --git a/tests-system/lobster-json/rbt-output-file/.gitkeep b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-json/rbt-output-file/.gitkeep rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt new file mode 100644 index 00000000..44e3c250 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt @@ -0,0 +1,4 @@ +--single +--tag-attribute=tags +--name-attribute=name +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.input b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json similarity index 95% rename from tests-system/lobster-json/basic2.input rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json index f57ea48e..7bb52fe8 100644 --- a/tests-system/lobster-json/basic2.input +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/README.md b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md new file mode 100644 index 00000000..944ef261 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md @@ -0,0 +1,22 @@ +If the `--name-attribute` option is not specified, +then the tag name must be constructed based on the following two values: +- path of the json file +- item counter + +# Test case "flat-input-structure" +- No files or directories are specified. + The tool is expected to search for input files in the current working directory. +- `--name-attribute` is not given. + The tool is expected to construct LOBSTER item tags by taking the file name and + appending an integer counter, starting at 1. + This is implemented inside function `syn_test_name`, and it must handle all file + paths as relative paths, because no input files were given explicitly. +- `--out` is specified. + The tool is expected to use that path name for the output file. +- `--single` is specified to make the tool output predictable. + +# Test case "nested-input-structure" +This is the same as the above test, except that its input file is nested in a +sub-sub-directory. +The tool is expected to take the names of the sub-directories into account when +generating te tag name. diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.output b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster similarity index 72% rename from tests-system/lobster-json/basic2.output rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster index 287d5a13..d0331f77 100644 --- a/tests-system/lobster-json/basic2.output +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster @@ -1,16 +1,14 @@ -lobster-json: wrote 6 items to basic2.lobster -========== { "data": [ { - "tag": "json ./basic2.json:basic2.1", + "tag": "json ./basic.json:basic.1", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.1", + "name": "./basic.json:basic.1", "messages": [], "just_up": [], "just_down": [], @@ -23,14 +21,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.2", + "tag": "json ./basic.json:basic.2", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.2", + "name": "./basic.json:basic.2", "messages": [], "just_up": [], "just_down": [], @@ -40,14 +38,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.3", + "tag": "json ./basic.json:basic.3", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.3", + "name": "./basic.json:basic.3", "messages": [], "just_up": [], "just_down": [], @@ -61,14 +59,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.4", + "tag": "json ./basic.json:basic.4", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.4", + "name": "./basic.json:basic.4", "messages": [], "just_up": [], "just_down": [], @@ -78,14 +76,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.5", + "tag": "json ./basic.json:basic.5", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.5", + "name": "./basic.json:basic.5", "messages": [], "just_up": [], "just_down": [], @@ -98,14 +96,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.6", + "tag": "json ./basic.json:basic.6", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.6", + "name": "./basic.json:basic.6", "messages": [], "just_up": [], "just_down": [], diff --git a/tests-system/lobster-python/rbt-output-file/.gitkeep b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-python/rbt-output-file/.gitkeep rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic.input b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json similarity index 92% rename from tests-system/lobster-json/basic.input rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json index fa82b8a4..7bb52fe8 100644 --- a/tests-system/lobster-json/basic.input +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags --name-attribute=name [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster new file mode 100644 index 00000000..8571ad42 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster @@ -0,0 +1,119 @@ +{ + "data": [ + { + "tag": "json ./one/two/basic.json:one.two.basic.1", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.1", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req1" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.2", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.2", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.3", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.3", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req2", + "req example.req3" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.4", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.4", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.5", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.5", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req4" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.6", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.6", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + } + ], + "generator": "lobster-json", + "schema": "lobster-act-trace", + "version": 3 +} diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json new file mode 100644 index 00000000..7bb52fe8 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json @@ -0,0 +1,38 @@ +[ + { + "name" : "Test_1", + "tags" : ["example.req1"], + "inputs" : [1, 2], + "expect" : 3 + }, + { + "name" : "Test_2", + "tags" : [], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_3", + "tags" : ["example.req2", + "example.req3"], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_4", + "tags" : null, + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_5", + "tags" : "example.req4", + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_6", + "inputs" : [1, 0], + "expect" : 1 + } +] diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster new file mode 100644 index 00000000..98ad4f23 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster @@ -0,0 +1,63 @@ +{ + "data": [ + { + "tag": "python basic.trlc_reference", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 3, + "column": null + }, + "name": "basic.trlc_reference", + "messages": [], + "just_up": [ + "helper function" + ], + "just_down": [], + "just_global": [], + "language": "Python", + "kind": "Function" + }, + { + "tag": "python basic.Example.helper_function", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 11, + "column": null + }, + "name": "basic.Example.helper_function", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req_nor" + ], + "language": "Python", + "kind": "Method" + }, + { + "tag": "python basic.Example.nor", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 15, + "column": null + }, + "name": "basic.Example.nor", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req_nor" + ], + "language": "Python", + "kind": "Method" + } + ], + "generator": "lobster_python", + "schema": "lobster-imp-trace", + "version": 3 +} diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stderr.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt new file mode 100644 index 00000000..31e0cb57 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt @@ -0,0 +1 @@ +Written output for 3 items to basic.lobster diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt new file mode 100644 index 00000000..3903b871 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt @@ -0,0 +1,5 @@ +. +--out=basic.lobster +--parse-decorator +trlc_reference +requirement diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py b/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py new file mode 100644 index 00000000..6d3ff093 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py @@ -0,0 +1,20 @@ +import potatolib + +def trlc_reference(requirement): + # lobster-exclude: helper function + def decorator(obj): + return obj + return decorator + +class Example: + @trlc_reference(requirement="example.req_nor") + def helper_function(a, b): + # potato + return a or b + + def nor(a, b): + # lobster-trace: example.req_nor + assert isinstance(a, bool) + assert isinstance(b, bool) + + return not helper_function(a, b) diff --git a/tests-system/lobster-trlc/Makefile b/tests-system/lobster-trlc/Makefile deleted file mode 100644 index 54a7a3fc..00000000 --- a/tests-system/lobster-trlc/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -PYTHON = python - -lobster-trlc-system-tests: - @$(PYTHON) ../run_tool_tests.py . diff --git a/tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt b/tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 1d1cf843..8a133f76 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,5 +1,5 @@ import sys -from os import scandir, DirEntry +from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path from subprocess import CompletedProcess, PIPE, run @@ -15,6 +15,7 @@ class TestSetup: _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" _EXIT_CODE_FILE_NAME = "exit-code.txt" _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" + _EXPECTED_STDERR_FILE_NAME = "stderr.txt" def __init__(self, test_case_path: str): """Constructor @@ -52,7 +53,8 @@ def _get_expected_lobster_output_file_name(self) -> str: if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( - "Invalid test setup: No *.lobster file found in expected output folder!", + f"Invalid test setup: No *.lobster file found in " + f"{self.get_expected_output_path()}!", ) @staticmethod @@ -83,9 +85,9 @@ def _get_args(self) -> List[str]: test) from the corresponding test setup file""" file = join(self._input_folder, self._ARGS_FILE_NAME) with open(file, "r", encoding="UTF-8") as file: - return file.readlines() + return [argument.strip() for argument in file.readlines()] - def get_expected_cmd_output(self) -> str: + def get_expected_stdout(self) -> str: """Reads the expected command line output (for stdout) from the corresponding test setup file""" cmd_file = join( @@ -95,6 +97,16 @@ def get_expected_cmd_output(self) -> str: with open(cmd_file, "r", encoding="UTF-8") as file: return file.read() + def get_expected_stderr(self) -> str: + """Reads the expected command line output (for stderr) from the corresponding + test setup file""" + errout_file = join( + self.get_expected_output_path(), + self._EXPECTED_STDERR_FILE_NAME, + ) + with open(errout_file, "r", encoding="UTF-8") as file: + return file.read() + @property def input_folder(self) -> str: """Returns the path containing the input data for the test""" @@ -112,13 +124,25 @@ def _get_expected_exit_code(self) -> int: def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: - """Runs the tool system test. + """Runs the tool system test using the coverage command. The tool will be executed such that its current working directory is equal to the "input" folder.""" print(f"Starting system test '{setup.name}' with arguments {setup.args} " \ - f"for tool '{tool}'.") + f"for tool '{tool}' with coverage.") + root_directory = Path(__file__).resolve().parents[1] + coverage_config_path = root_directory / "coverage.cfg" + coverage_data_path = root_directory / ".coverage" + + coverage_command = [ + "coverage", "run", "-p", + f"--rcfile={coverage_config_path}", + "--branch", + f"--data-file={coverage_data_path}", + tool, *setup.args + ] + completed_process = run( - [sys.executable, tool, *setup.args], + coverage_command, stdout=PIPE, stderr=PIPE, encoding="UTF-8", @@ -128,12 +152,31 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: return completed_process +def _compare_cmd_output(name: str, expected: str, actual: str) -> bool: + if expected != actual: + print(f"Actual {name} is (length: {len(actual)} chars):\n{actual}") + print(f"Expected {name} is (length: {len(expected)} chars):\n{expected}") + return False + return True + + def _compare_results(setup: TestSetup, completed_process: CompletedProcess): assert setup.expected_exit_code == completed_process.returncode, \ f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \ f"actual is {completed_process.returncode}!" - assert setup.get_expected_cmd_output() == completed_process.stdout, \ - "Command line output is different!" + + assert _compare_cmd_output( + name="STDOUT", + expected=setup.get_expected_stdout(), + actual=completed_process.stdout, + ), "Command line output for STDOUT is different!" + + assert _compare_cmd_output( + name="STDERR", + expected=setup.get_expected_stderr(), + actual=completed_process.stderr, + ), "Command line output for STDERR is different!" + expected = join( setup.get_expected_output_path(), setup.expected_lobster_output_file_name, @@ -172,6 +215,16 @@ def _get_directories( yield dir_entry +def _delete_generated_files(setup: TestSetup): + """Deletes the *.lobster file that has been generated by the test""" + generated = join( + setup.input_folder, + setup.expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + remove(generated) + + def _run_tests(directory: str, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. @@ -190,6 +243,7 @@ def _run_tests(directory: str, tool: str) -> int: test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) + _delete_generated_files(test_setup) counter += 1 print(f"{counter} system tests finished successfully for {tool}.")