From 68a2c2ff8e2f5ba3eedaa61b649cba8795b1c451 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:22:11 +0100 Subject: [PATCH 01/16] Print all "duplicate definition" errors instead of only the first (#131) Function `lobster_read` prints all "duplicate definition" errors, and calls `mh.error` (with `fatal=True`) only for the last duplicate definition. This helps the users to get an overview of all the duplicate definitions in their input files. Previously the function raised an error already on the first occurrence, and the user had to fix that one, and call the L.O.B.S.T.E.R tools over and over again, until all errors were fixed. **Note:** `lobster-codebeamer` is affected when the parameter `--import-tagged` is used, as the tool then reads a lobster file to figure out which items to download from codebeamer. Apart from that, only `lobster-report` is affected by the change. --- CHANGELOG.md | 7 ++++++ lobster/io.py | 23 +++++++++++++------ lobster/report.py | 2 +- lobster/tools/core/report/requirements.trlc | 6 ++--- .../expected-output/exit-code.txt | 1 + .../expected-output/stdout.txt | 4 ++++ .../multiple-duplicates/input/args.txt | 1 + .../multiple-duplicates/input/lobster.config | 3 +++ .../expected-output/exit-code.txt | 1 + .../no-duplicates/expected-output/stdout.txt} | 0 .../no-duplicates/input/args.txt | 1 + .../no-duplicates/input/lobster.config | 3 +++ 12 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt rename tests-system/lobster-core/report/{rbt-output-file/.gitkeep => rbt-report-input-duplicate-definition/no-duplicates/expected-output/stdout.txt} (100%) create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt create mode 100644 tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config diff --git a/CHANGELOG.md b/CHANGELOG.md index 0584631f..656d0a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ ### 0.9.20-dev +* If a `*.lobster` file contains a tag more than once, then an error message + ("duplicated definition") is printed for each consecutive entry with the same tag, + instead of printing it just for the first entry. + The following tools are affected: + * `lobster-codebeamer` + * `lobster-report` + * `lobster-gtest` accepts XML nodes other than `testcase`, but ignores them. ### 0.9.19 diff --git a/lobster/io.py b/lobster/io.py index 22eb7292..66b0d1b8 100644 --- a/lobster/io.py +++ b/lobster/io.py @@ -101,6 +101,7 @@ def lobster_read(mh, filename, level, items, source_info=None): "version %u for schema %s is not supported" % (data["version"], data["schema"])) + duplicate_items = [] # Convert to items, and integrate into symbol table for raw in data["data"]: if data["schema"] == "lobster-req-trace": @@ -124,10 +125,18 @@ def lobster_read(mh, filename, level, items, source_info=None): if all(filter_conditions): if item.tag.key() in items: - mh.error(item.location, - "duplicate definition of %s, " - "previously defined at %s" % - (item.tag.key(), - items[item.tag.key()].location.to_string())) - - items[item.tag.key()] = item + # 'duplicate definition' errors are fatal, but the user wants to see all + # of them. So store the affected items in a list first, and create + # errors later. + duplicate_items.append(item) + else: + items[item.tag.key()] = item + + if duplicate_items: + for counter, item in enumerate(duplicate_items, start=1): + mh.error( + item.location, + f"duplicate definition of {item.tag.key()}, " + f"previously defined at {items[item.tag.key()].location.to_string()}", + fatal=(counter == len(duplicate_items)), + ) diff --git a/lobster/report.py b/lobster/report.py index c27e3510..037cd563 100644 --- a/lobster/report.py +++ b/lobster/report.py @@ -174,7 +174,7 @@ def load_report(self, filename): def compute_items_and_coverage_for_items(self, data): """ - Function calcuates items and coverage for the items + Function calculates items and coverage for the items Parameters ---------- data - contents of lobster json file. diff --git a/lobster/tools/core/report/requirements.trlc b/lobster/tools/core/report/requirements.trlc index 220f6e16..1de78715 100644 --- a/lobster/tools/core/report/requirements.trlc +++ b/lobster/tools/core/report/requirements.trlc @@ -1,10 +1,10 @@ package core_report_req import req -req.Software_Requirement Dummy_Requirement { +req.System_Requirement Report_Input_Duplicate_Definition { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If any of the source files given in the lobster configuration file contains a tag + more than once, then the tool must exit with code 1. ''' } diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt new file mode 100644 index 00000000..17c7c77d --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt @@ -0,0 +1,4 @@ +DuplicateDefinitionTest.py:2: lobster error: duplicate definition of hello DuplicateDefinitionTest, previously defined at DuplicateDefinitionTest.py:1 +DuplicateDefinitionTest.py:3: lobster error: duplicate definition of hello DuplicateDefinitionTest, previously defined at DuplicateDefinitionTest.py:1 +DuplicateDefinitionTest.py:4: lobster error: duplicate definition of hello DuplicateDefinitionTest, previously defined at DuplicateDefinitionTest.py:1 +lobster: aborting due to earlier errors. \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt new file mode 100644 index 00000000..d2688e44 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt @@ -0,0 +1 @@ +--lobster-config="lobster.config" \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config new file mode 100644 index 00000000..518720d1 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config @@ -0,0 +1,3 @@ +implementation "Something" { + source: "file.lobster"; +} diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-output-file/.gitkeep b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/stdout.txt similarity index 100% rename from tests-system/lobster-core/report/rbt-output-file/.gitkeep rename to tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/stdout.txt diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt new file mode 100644 index 00000000..d2688e44 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt @@ -0,0 +1 @@ +--lobster-config="lobster.config" \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config new file mode 100644 index 00000000..518720d1 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config @@ -0,0 +1,3 @@ +implementation "Something" { + source: "file.lobster"; +} From d465039dbd5bdc02c7365db22f43da31be6b1420 Mon Sep 17 00:00:00 2001 From: "Thomas Deboben (Vispiron)" Date: Tue, 3 Dec 2024 13:51:49 +0100 Subject: [PATCH 02/16] Lobster-cpptest shall output absolute paths (#136) Lobster-cpptest shall output absolute paths Lobster-cpptest implementation ant unit tests have been adapted. Resolves https://github.com/bmw-software-engineering/lobster/issues/135 --- CHANGELOG.md | 3 +++ lobster/tools/cpptest/cpptest.py | 3 +-- tests-unit/lobster-cpptest/test_cpptest.py | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 656d0a4e..570987c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ ### 0.9.20-dev +* `lobster-cpptest` writes absolute paths into its `*.lobster` output files instead of + paths relative to the current working directory. + * If a `*.lobster` file contains a tag more than once, then an error message ("duplicated definition") is printed for each consecutive entry with the same tag, instead of printing it just for the first entry. diff --git a/lobster/tools/cpptest/cpptest.py b/lobster/tools/cpptest/cpptest.py index 47d444fb..4dfb10b0 100644 --- a/lobster/tools/cpptest/cpptest.py +++ b/lobster/tools/cpptest/cpptest.py @@ -220,7 +220,6 @@ def create_lobster_items_output_dict_from_test_cases( The lobster items dictionary for the given test cases grouped by configured output. """ - prefix = os.getcwd() lobster_items_output_dict = {} no_marker_output_file_name = '' @@ -239,7 +238,7 @@ def create_lobster_items_output_dict_from_test_cases( for test_case in test_case_list: function_name: str = test_case.suite_name - file_name = os.path.relpath(test_case.file_name, prefix) + file_name = os.path.abspath(test_case.file_name) line_nr = int(test_case.docu_start_line) function_uid = "%s:%s:%u" % (os.path.basename(file_name), function_name, diff --git a/tests-unit/lobster-cpptest/test_cpptest.py b/tests-unit/lobster-cpptest/test_cpptest.py index 9b7736ef..373082ce 100644 --- a/tests-unit/lobster-cpptest/test_cpptest.py +++ b/tests-unit/lobster-cpptest/test_cpptest.py @@ -1,4 +1,5 @@ import os +import json import unittest from os.path import dirname from pathlib import Path @@ -240,6 +241,15 @@ def test_single_file(self): file_exists = os.path.exists(self.output_file_name) self.assertTrue(file_exists) + # Open and read the JSON output file to validate all file paths are absolute + with open(self.output_file_name, 'r') as output_file: + data = json.load(output_file) + tag_list = data.get('data') + + for tag in tag_list: + file_name = tag.get('location').get('file') + self.assertTrue(os.path.isabs(file_name)) + def test_single_directory(self): file_dir_list = [self.test_data_dir] @@ -333,6 +343,8 @@ def test_separate_output_config(self): for lobster_item in unit_test_lobster_items: self.assertIsNotNone(lobster_item) self.assertIsInstance(lobster_item, dict) + file_name = lobster_item.get('location').get('file') + self.assertTrue(os.path.isabs(file_name)) tag = lobster_item.get('tag') refs = lobster_item.get('refs') self.assertIsInstance(refs, list) @@ -361,6 +373,8 @@ def test_separate_output_config(self): for lobster_item in component_test_lobster_items: self.assertIsNotNone(lobster_item) self.assertIsInstance(lobster_item, dict) + file_name = lobster_item.get('location').get('file') + self.assertTrue(os.path.isabs(file_name)) tag = lobster_item.get('tag') refs = lobster_item.get('refs') self.assertIsInstance(refs, list) From de48ba4bc91a708a56287fec2ab2260a54563549 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:33:44 +0100 Subject: [PATCH 03/16] Cosmetic fix of string in `lobster-cpp` (#139) Added 'r' prefix to string to have the same coding style throughout all related strings. --- lobster/tools/cpp/cpp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lobster/tools/cpp/cpp.py b/lobster/tools/cpp/cpp.py index 511d49a2..b09aa546 100755 --- a/lobster/tools/cpp/cpp.py +++ b/lobster/tools/cpp/cpp.py @@ -35,8 +35,8 @@ SUFFIX = r"\[lobster-tracing\]$" RE_NOTAGS = (PREFIX + " " + - "%s %s has no tracing tags" % (KIND_PATTERN, - NAME_PATTERN) + + r"%s %s has no tracing tags" % (KIND_PATTERN, + NAME_PATTERN) + " " + SUFFIX) RE_TAGS = (PREFIX + " " + r"%s %s traces to +(.+) +" % (KIND_PATTERN, From bb708cb4156a6237569ac57636aebcfccdf0242c Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:46:43 +0100 Subject: [PATCH 04/16] Add option to pass parameters to clang tidy (#138) Add new command line argument `--compile-commands` to `lobster-cpp`. Its value will be forwarded to the call of clang tidy. --- CHANGELOG.md | 5 ++++ lobster/tools/cpp/cpp.py | 50 +++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 570987c8..b3bca2f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ ### 0.9.20-dev +* Add `--compile-commands` flag to `lobster-cpp`. This allows to specify a path to the + compile command database and is effectively a wrapper around the `-p` argument of + `clang tidy`. See the official documentation of `clang tidy` for more details on this + parameter. + * `lobster-cpptest` writes absolute paths into its `*.lobster` output files instead of paths relative to the current working directory. diff --git a/lobster/tools/cpp/cpp.py b/lobster/tools/cpp/cpp.py index b09aa546..93e22f6f 100755 --- a/lobster/tools/cpp/cpp.py +++ b/lobster/tools/cpp/cpp.py @@ -59,6 +59,14 @@ def main(): metavar="FILE", help=("use the specified clang-tidy; by default we" " pick the one on PATH")) + ap.add_argument("--compile-commands", + metavar="FILE", + default=None, + help=("Path to the compile command database for all targets for " + "'clang tidy', or none to use the default behavior of " + "'clang tidy'. This is equal to calling 'clang tidy' " + "directly with its '-p' option. Refer to its official " + "documentation for more details.")) ap.add_argument("--out", default=None, help=("write output to this file; otherwise output to" @@ -83,13 +91,17 @@ def main(): # Test if the clang-tidy can be used - rv = subprocess.run([os.path.expanduser(options.clang_tidy), - "-checks=-*,lobster-tracing", - "--list-checks"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="UTF-8", - check=False) + rv = subprocess.run( + [ + os.path.expanduser(options.clang_tidy), + "-checks=-*,lobster-tracing", + "--list-checks", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", + check=False, + ) if "No checks enabled." in rv.stderr: print("The provided clang-tidy does include the lobster-tracing check") @@ -99,13 +111,23 @@ def main(): "correct binary using the --clang-tidy flag") return 1 - rv = subprocess.run([os.path.expanduser(options.clang_tidy), - "-checks=-*,lobster-tracing"] + - file_list, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="UTF-8", - check=False) + subprocess_args = [ + os.path.expanduser(options.clang_tidy), + "-checks=-*,lobster-tracing", + ] + if options.compile_commands: + subprocess_args.append("-p") + subprocess_args.append(options.compile_commands) + + subprocess_args += file_list + + rv = subprocess.run( + subprocess_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", + check=False, + ) if rv.returncode != 0: found_reason = False From 84537cc66bfa1d54178869761dd6890de6425047 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:10:21 +0100 Subject: [PATCH 05/16] LOBSTER Release 0.9.20 (#141) --- CHANGELOG.md | 2 +- lobster/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3bca2f8..bee4066d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Changelog -### 0.9.20-dev +### 0.9.20 * Add `--compile-commands` flag to `lobster-cpp`. This allows to specify a path to the compile command database and is effectively a wrapper around the `-p` argument of diff --git a/lobster/version.py b/lobster/version.py index 4080b3a0..d5701c52 100644 --- a/lobster/version.py +++ b/lobster/version.py @@ -18,7 +18,7 @@ # . VERSION_TUPLE = (0, 9, 20) -VERSION_SUFFIX = "dev" +VERSION_SUFFIX = "" LOBSTER_VERSION = ("%u.%u.%u" % VERSION_TUPLE) + ( "-%s" % VERSION_SUFFIX if VERSION_SUFFIX else "" From 1dc3de9f0423f834d08a6b31cbaf8616cae1c1c6 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:18:56 +0100 Subject: [PATCH 06/16] Bump version to 0.9.21-dev after release (#142) --- CHANGELOG.md | 4 ++++ lobster/version.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bee4066d..21fecb20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## Changelog +### 0.9.21-dev + + + ### 0.9.20 * Add `--compile-commands` flag to `lobster-cpp`. This allows to specify a path to the diff --git a/lobster/version.py b/lobster/version.py index d5701c52..bf577149 100644 --- a/lobster/version.py +++ b/lobster/version.py @@ -17,8 +17,8 @@ # License along with this program. If not, see # . -VERSION_TUPLE = (0, 9, 20) -VERSION_SUFFIX = "" +VERSION_TUPLE = (0, 9, 21) +VERSION_SUFFIX = "dev" LOBSTER_VERSION = ("%u.%u.%u" % VERSION_TUPLE) + ( "-%s" % VERSION_SUFFIX if VERSION_SUFFIX else "" From 6791c44a38f78b99f542f40c08dccc5180e235ef Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:29:46 +0100 Subject: [PATCH 07/16] Move `run_tool_tests.py` one level up (#143) Moved the tool to its parent directory so that it can be used to execute system tests for all lobster tools in the future. --- Makefile | 2 +- tests-system/lobster-trlc/Makefile | 3 +-- tests-system/{lobster-trlc => }/run_tool_tests.py | 8 ++++++-- 3 files changed, 8 insertions(+), 5 deletions(-) rename tests-system/{lobster-trlc => }/run_tool_tests.py (97%) diff --git a/Makefile b/Makefile index 56d8a3df..9bf253bb 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: style python3 -m pylint --rcfile=pylint3.cfg \ --reports=no \ --ignore=assets.py \ - lobster util tests-system/lobster-trlc/run_tool_tests.py + lobster util tests-system/run_tool_tests.py style: @python3 -m pycodestyle lobster \ diff --git a/tests-system/lobster-trlc/Makefile b/tests-system/lobster-trlc/Makefile index 243eecff..54a7a3fc 100644 --- a/tests-system/lobster-trlc/Makefile +++ b/tests-system/lobster-trlc/Makefile @@ -1,5 +1,4 @@ PYTHON = python lobster-trlc-system-tests: - @echo "Running lobster-trlc system tests..." - @$(PYTHON) run_tool_tests.py + @$(PYTHON) ../run_tool_tests.py . diff --git a/tests-system/lobster-trlc/run_tool_tests.py b/tests-system/run_tool_tests.py similarity index 97% rename from tests-system/lobster-trlc/run_tool_tests.py rename to tests-system/run_tool_tests.py index fb525e8b..1d1cf843 100644 --- a/tests-system/lobster-trlc/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -184,12 +184,14 @@ def _run_tests(directory: str, tool: str) -> int: if not tool: raise ValueError("No tool specified!") + counter = 0 for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX): for test_case_dir_entry in _get_directories(rbt_dir_entry.path): test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) - print(f"All system tests finished successfully for {tool}.") + counter += 1 + print(f"{counter} system tests finished successfully for {tool}.") # TODO: the current implementation is not consistent with respect to return codes. # The tests use assertion statements to indicate failures, but here we use an @@ -212,7 +214,9 @@ def _get_tool(test_dir: str) -> str: if __name__ == "__main__": - test_directory = dirname(Path(__file__).resolve()) + test_directory = Path(sys.argv[1]).resolve() + print(f"Starting system tests on folder '{test_directory}'") + sys.exit( _run_tests( test_directory, From 72fe2f1650372a953750937f177029e0b9050286 Mon Sep 17 00:00:00 2001 From: Diego Fernandez <110383200+DiFerMa@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:37:28 +0100 Subject: [PATCH 08/16] Documentation/improve documentation (#127) Co-authored-by: Philipp Wullstein-Kammler --- README.md | 114 +++++++++++++++++++++++---- documentation/user-manual.md | 38 +++++++-- packages/lobster-tool-cpp/README.md | 2 + packages/lobster-tool-trlc/README.md | 4 - 4 files changed, 132 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b37146b1..acc104db 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,15 @@ and requirements coverage, which is essential for meeting standards such as ISO 26262. This repository contains the prototype for LOBSTER, which is a key -ingredient to make TRCL more useful. +ingredient to make TRLC and other supported tools more useful. -It has tools to extract tracing tags from a variety of sources combine -them and produce a tracing report. The [TRLC tracing +It has tools to extract tracing tags from a variety of sources to +combine them and produce a tracing report. The [TRLC tracing report](https://bmw-software-engineering.github.io/trlc/tracing.html) from the [TRLC Project](https://github.com/bmw-software-engineering/trlc/) is a -reasonable example of what is possible. +reasonable example of what is possible when lobster is used in combination +with TRLC or any of the other supported tools. ## Installing @@ -23,12 +24,19 @@ convenient meta-package `bmw-lobster` which installs everything. ``` $ pip3 install bmw-lobster ``` +For the HTML Report `graphviz` is also used to generate the tracing policy diagram. More on that on the user [manual](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/user-manual.md). + +``` +$ sudo apt-get install -y graphviz +``` + +The `lobster-cpp` converter tool needs a specific version of `clang-tidy`. Please see [here](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/user-manual.md#clang-tidy-file-generation) to create it. ## Supported inputs The following requirements frameworks are supported: -* [TRLC](work-in-progress) (only some use cases supported right now) +* [TRLC](https://github.com/bmw-software-engineering/trlc/) (only some use cases supported right now) * [Codebeamer](packages/lobster-tool-codebeamer/README.md) (only some use cases supported right now) @@ -56,17 +64,18 @@ The following verification and miscellaneous frameworks are supported: ## Installing individual packages -The individual packages that `bmw-lobster` depends on are: +The individual PyPI packages that `bmw-lobster` depends on are: * `bmw-lobster-core` the core API and various report generators. All - other tools depend on this. -* `bmw-lobster-tool-cpp` (for C/C++ code) -* `bmw-lobster-tool-cpptest` (for C/C++ code) -* `bmw-lobster-tool-gtest` (for GoogleTest tests) -* `bmw-lobster-tool-python` (for Python3 code) -* `bmw-lobster-tool-beamer` (for requirements in Codebeamer) -* `bmw-lobster-tool-json` (for activities in JSON) -* `miss_hit` (for MATLAB/Octave code or Simulink models) + other tools depend on this [Link](https://pypi.org/project/bmw-lobster-core) +* `bmw-lobster-tool-codebeamer` (for requirements in Codebeamer) [Link](https://pypi.org/project/bmw-lobster-tool-codebeamer) +* `bmw-lobster-tool-cpp` (for C/C++ code) [Link](https://pypi.org/project/bmw-lobster-tool-cpp) +* `bmw-lobster-tool-cpptest` (for C/C++ code) [Link](https://pypi.org/project/bmw-lobster-tool-cpp) +* `bmw-lobster-tool-gtest` (for GoogleTest tests) [Link](https://pypi.org/project/bmw-lobster-tool-gtest) +* `bmw-lobster-tool-json` (for activities in JSON) [Link](https://pypi.org/project/bmw-lobster-tool-json) +* `bmw-lobster-tool-python` (for Python3 code) [Link](https://pypi.org/project/bmw-lobster-tool-python) +* `bmw-lobster-tool-trlc` (for TRLC code) [Link](https://pypi.org/project/bmw-lobster-tool-trlc) +* `miss_hit` (for MATLAB/Octave code or Simulink models) [Link](https://pypi.org/project/miss_hit) ### For LOBSTER developers @@ -89,15 +98,86 @@ Here are the links to the individual html requirements coverage reports: * [Requirement Coverage Report Core Report](https://bmw-software-engineering.github.io/lobster/tracing-core_report.html) * [Requirement Coverage Report Codebeamer](https://bmw-software-engineering.github.io/lobster/tracing-codebeamer.html) +### Simple lobster-demo + +* A simple example can be found in the repository: [lobster-demo](https://github.com/bmw-software-engineering/lobster-demo) + +## Workflow of LOBSTER + +The lobster tool uses several steps to accomplish a fully modular software traceability +and requirements coverage report. You can consider lobster as a set of 1) conversion tools, 2) a common interchange format, 3) the report creation tool and 4) a renderer for the tracing report. + +For a more detailed description please read our [user guide](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/config_files.md). + +These steps are in the following diagram and go from left to right side: + +```mermaid +graph LR + subgraph "Converter-Tools" + direction TB + A1[Lobster-python] + A2[Lobster-trlc] + A3[Lobster-json] + A4[Lobster-cpp] + A5[Lobster-codebeamer] + A6[Lobster-gtest] + end + + subgraph "Common inter. format" + direction TB + B1[Python.lobster] + B2[Trlc.lobster] + B3[Json.lobster] + B4[Cpp.lobster] + B5[Codebeamer.lobster] + B6[Gtest.lobster] + end + + subgraph "Generate lobster report" + direction TB + D1[Lobster-online-report] + D1 ---> D2 + D2[Lobster-report -> report.lobster] + D3[Tracing policy -> lobster.conf] + D3 ---> D2 + end + + subgraph "Renderer" + direction TB + C1[html] + C2[CI] + C3["..."] + end + + %% Main connections + A1 ---> B1 + A2 ---> B2 + A3 ---> B3 + A4 ---> B4 + A5 ---> B5 + A6 ---> B6 + + + %% Connect all schema elements to Lobster-report -> report.lobster + B1 ----> D2 + B2 ----> D2 + B3 ----> D2 + B4 ----> D2 + B5 ----> D2 + B6 ----> D2 + + %% Connect Lobster-report -> report.lobster to renderers + D2 ---> C1 + D2 ---> C2 + D2 ---> C3 + ``` + ## Planned inputs The following inputs are planned but not implemeted yet: * `lobster-java`: Java code * `lobster-kotlin`: Kotlin code -* `lobster-ada`: Ada and SPARK code (via libadalang) -* `lobster-latex`: Requirements written in LaTeX -* `lobster-markdown`: Requirements written in Markdown ## Copyright & License information diff --git a/documentation/user-manual.md b/documentation/user-manual.md index 67773ce1..dc66adf8 100644 --- a/documentation/user-manual.md +++ b/documentation/user-manual.md @@ -8,11 +8,11 @@ of any safety related software developments, and often involves duplicate maintenance and tedious manual processes. LOBSTER aims to simplify this. The basic setup is as follows: -1. Configure LOBSTER -2. Annotate code / tests / requirements -3. Setup build system to use lobster tools (e.g. `lobster_cpp`) to - extract information -4. Build report by calling `lobster_report` +1. Configure LOBSTER via the tracing policy. Usually called `lobster.conf`. More on that [here](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/config_files.md). +2. Annotate traces onto your code, your tests and/or your requirements. Please notice each one of the supported languages has their own way to annotate the tracing tags. +3. Extract the requirement traces with the corresponding lobster conversion tools (e.g. `lobster-cpp`). This converts those traces into a *.lobster file (into a "common unified interchange format"). +4. Build report by calling `lobster-report` +5. Render your report by calling the core losbter tools. Usually a local HTML report is desired via `lobster-html-report` or even `lobster-ci-report` for your CI. The basic idea is that you have a number of artefacts that you wish to relate to each other; stored in different "databases". Sometimes this @@ -105,6 +105,10 @@ as integration tests, HIL tests, etc.) * lobster_trlc: DOCUMENTATION TODO +Note: We are providing a full traceability report, using LOBSTER itself and [TRLC](https://github.com/bmw-software-engineering/trlc), for each tool maintained in this repository. You are able to produce these reports yourself when using the target `make tracing`. + +All its final html outputs are to be seen [here](https://github.com/bmw-software-engineering/lobster?tab=readme-ov-file#requirements-coverage). + ### codeBeamer * [lobster_codebeamer](manual-lobster_codebeamer.md): for tracing to @@ -116,6 +120,30 @@ as integration tests, HIL tests, etc.) * [lobster_gtest](manual-lobster_gtest.md): for tracing tags in (executed) googletests. +> **Note:** The lobster-cpp tool needs a clang-tidy file. Don't forget to generate it to be able to use this conversion tool. + +#### Clang-tidy File Generation + +You can just run the `clang-tidy` Makefile target in root folder. + +If you prefer to do it manually or are Windows user, install Ninja and cmake and then please follow the next steps: + +* To generate the clang-tidy file, which is needed for the cpp tool, make sure that your apt is working well on WSL (Windows Subsystem for Linux) or use a Linux environment. + +* Clone this repository - `https://github.com/bmw-software-engineering/llvm-project` + +* Below 2 dependencies required for clang-tidy creation. + 1. `sudo apt install cmake` + 2. `sudo apt install ninja-build` + +* Below 2 commands need to execute to generate the build folder. + 1. `cmake -S llvm -B build -G Ninja -DLLVM_ENABLE_PROJECTS='clang;clang-tools-extra' -DCMAKE_BUILD_TYPE=Release` + 2. `cmake --build build --target clang-tidy` + +* Once you generate the build folder you can see the clang-tidy file in `./build/bin` folder. + +* To generate the cpp.lobster file, you need to make sure that your llvm-project and lobster-demo project should be in same directory. + ### Python * lobster_python: DOCUMENTATION TODO diff --git a/packages/lobster-tool-cpp/README.md b/packages/lobster-tool-cpp/README.md index b7b1fa27..59c3f0b2 100644 --- a/packages/lobster-tool-cpp/README.md +++ b/packages/lobster-tool-cpp/README.md @@ -11,6 +11,8 @@ this to work you need to build [our clang-tidy fork](https://github.com/bmw-software-engineering/llvm-project) and place the `clang-tidy` binary somewhere on your PATH. +Instructions for clang-tidy [here](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/user-manual.md#clang-tidy-file-generation). + This tool works using a custom clang-tidy checker `lobster-trace` which emits tracing information as clang checks. diff --git a/packages/lobster-tool-trlc/README.md b/packages/lobster-tool-trlc/README.md index 35a8f5ab..d64d14d6 100644 --- a/packages/lobster-tool-trlc/README.md +++ b/packages/lobster-tool-trlc/README.md @@ -5,10 +5,6 @@ The **L**ightweight **O**pen **B**MW **S**oftware **T**raceability and requirements coverage, which is essential for meeting standards such as ISO 26262. -This package contains a tool to interface with the proprietary -requirements management tool -[Codebeamer](https://intland.com/codebeamer). - ## Configuration This tool is a bit more complex and you need to supply a config file, From 5eb0928296abe07ccb3b53c3d620fe66fa02c524 Mon Sep 17 00:00:00 2001 From: Tannaz Vahidi Date: Fri, 6 Dec 2024 09:44:53 +0100 Subject: [PATCH 09/16] Lobster-cpptest documentation (#116) Lobster-cpptest documentation has been adopted. Resolves https://github.com/bmw-software-engineering/lobster/issues/57 --- documentation/manual-lobster_cpptest.md | 134 ++++++++++++++++++++++++ documentation/manual-lobster_gtest.md | 8 +- documentation/user-manual.md | 12 +++ packages/lobster-tool-cpptest/README.md | 34 ++++-- 4 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 documentation/manual-lobster_cpptest.md diff --git a/documentation/manual-lobster_cpptest.md b/documentation/manual-lobster_cpptest.md new file mode 100644 index 00000000..201bc13b --- /dev/null +++ b/documentation/manual-lobster_cpptest.md @@ -0,0 +1,134 @@ +# Tracing to C/C++ tests + +## Setup and requirements + +You will need a C/C++ file or a directory containing these files. +* Note: The tool also supports a combination of files and folders. + +C/C++ files should have one of these extensions to be evaluated by this tool: `.cpp`, `.cc`, `.c` or `.h` + +## Preparing C/C++ test documentation with requirements +The test functions should be specified with one the following `macros`: + `TEST`, + `TEST_P`, + `TEST_F`, + `TYPED_TEST`, + `TYPED_TEST_P`, + `TYPED_TEST_SUITE`, + `TEST_P_INSTANCE` or + `TEST_F_INSTANCE` + +In your test you need to also add documentation. For example: + +```C++ +/** + * @requirement CB-#0815, CB-#0816, + * CB-#0817 + * @requirement CB-#0818 CB-#0819 + * CB-#0820 + */ +TEST(ImplicationTest, BasicTest) {} +``` + +* Each test can have multiple test types defined in the documentation part. +* Test types can be used multiple times in the test documentation and can be written on multiple lines + +* These can be simply defined with `/** Test types */` or `/// Test types` format. + +* Test-tags can be separated by commas or spaces. + +* Note: The `markers` shall be exactly close to the test functions (without any empty lines) + + +```C++ +/** + * @requirement CB-#0815, CB-#0816 + */ +TEST(ImplicationTest, BasicTest) {} +``` + +```C++ +/// +/// @requiredby FOO0::BAR0 +/// +TEST(ImplicationTest, BasicTest) {} +``` +The regex used for each test-tag is as follows: + +@requirement +: ```r"(CB-#\d+)"``` +: ```r"({provided codebeamer-url in config-file}(?P\d+))"``` + +@requiredby +: ```r"(\w*::\w+)"``` + +@defect +: ```r"(CB-#\d+)|(OCT-#\d+)"``` + + +## Preparing cpptest config-file + +You have to provide a config-file which determines which `markers` should be extracted in which output-files. +The expected `kind` for each output-file should also be specified. + +In addition, you have to provide the `codebeamer-url`: + +```cpp.config +{ + "output": { + "unit_tests.lobster" : + { + "markers": ["@requirement"], + "kind": "req" + }, + "components_tests.lobster" : + { + "markers": ["@requiredby", "@requirement"], + "kind": "imp" + } + }, + "codebeamer_url": "https://codebeamer.example.com/test" +} + ``` + +* Note: If you want to extract the other tests with other `markers`, + you can use an empty list as `markers` value. Be aware in this case the tests do not have any references. + +```cpp.config +{ + "output": { + "tests.lobster" : + { + "markers": [], + "kind": "req" + } + }, + "codebeamer_url": "https://codebeamer.example.com/test" +} + ``` + + +## Creating lobster files + +Run the `lobster_cpptest` tool, pointing it to one or more C/C++ files, or a directory containing one or more C/C++ files. + +For example `lobster_cpptest .` should find all your C/C++ files in the root directory. + +You have to also provide a `--config-file` file to configure the behaviour of the tool. +A more complete command line might look like: + +```sh +$ lobster-cpptest . --config-file cpp.config +``` + +## Example + +The LOBSTER unit tests contains a working example: + +* Test [test_case.cpp](../test-unit/lobster-cpptest/data/test_case.cpp) containing requirement tags + +## Notes & Caveats +* This tool supports these `markers`: '@requirement', '@requiredby' and '@defect' +* This tool supports these `kind`: 'req', 'imp' and 'act' + + diff --git a/documentation/manual-lobster_gtest.md b/documentation/manual-lobster_gtest.md index 03b3c75e..2e96afaf 100644 --- a/documentation/manual-lobster_gtest.md +++ b/documentation/manual-lobster_gtest.md @@ -55,11 +55,11 @@ $ lobster_gtest . --out gtests.lobster The LOBSTER testsuite contains a working example: -* Bazel [BUILD](../tests/projects/basic/BUILD) file to set up -* Test [test.cpp](../tests/projects/basic/test.cpp) containing tracing tags -* Requrements [potato.trlc](../tests/projects/basic/potato.trlc) +* Bazel [BUILD](../integration-tests/projects/basic/BUILD) file to set up +* Test [test.cpp](../integration-tests/projects/basic/test.cpp) containing tracing tags +* Requrements [potato.trlc](../integration-tests/projects/basic/potato.trlc) containing tracing the requirements mentioned by the test -* [Makefile](../tests/projects/basic/Makefile) gluing everything +* [Makefile](../integration-tests/projects/basic/Makefile) gluing everything together ## Notes & Caveats diff --git a/documentation/user-manual.md b/documentation/user-manual.md index dc66adf8..c2b48411 100644 --- a/documentation/user-manual.md +++ b/documentation/user-manual.md @@ -56,6 +56,17 @@ void main() } ``` +For C++ tests requirements will be referenced inside the documentation. + +The syntax for Codebeamer requirements looks like this: + +```C++ TEST +/** + * @requirement CB-#0815 CB-#0816 + */ +TEST(RequirementTagTest1, RequirementAsComments) {} +``` + A central idea in LOBSTER is to reduce duplication of information, so tracing links are *only ever added to artefacts lower in the tracing hierarchy*. So in the first example above, we'd add tags to the tests, @@ -117,6 +128,7 @@ All its final html outputs are to be seen [here](https://github.com/bmw-software ### C / C++ * lobster_cpp: DOCUMENTATION TODO +* [lobster_cpptest](manual-lobster_cpptest.md): for tracing requirements and/or defects from C++ tests. * [lobster_gtest](manual-lobster_gtest.md): for tracing tags in (executed) googletests. diff --git a/packages/lobster-tool-cpptest/README.md b/packages/lobster-tool-cpptest/README.md index a4124362..bd32c6e7 100644 --- a/packages/lobster-tool-cpptest/README.md +++ b/packages/lobster-tool-cpptest/README.md @@ -6,19 +6,21 @@ and requirements coverage, which is essential for meeting standards such as ISO 26262. This package contains a tool extract tracing tags from ISO C or C++ -source code. This tool is also extracting configurable markers/ test-types -from the provided comments in cpp files +source code. The tracing tags are identified by searching for configurable +markers in the comments of the source code. ## Tools -* `lobster-cpptest`: Extract requirements with dynamic refrences - from comments. +* `lobster-cpptest`: Extract requirements with specific references + from tests. ## Usage This tool supports C/C++ code. -For this you can provide some cpp file with these comments: +For this you have to provide a C/C++ test documentation with `markers`: + +`Markers` can be either `@requirement`, `@requiredby` or `@defect`. ```cpp /** @@ -29,9 +31,23 @@ For this you can provide some cpp file with these comments: */ TEST(RequirementTagTest1, RequirementsAsMultipleComments) {} ``` -You can also provide a config-file which determines which markers -should be extracted in which files. In addition you have to provide -the codebeamer-url: +You have to provide a config-file which determines which `markers` should be extracted in which output-files. +The expected `kind` for each output-file should also be specified. + +* Note: If you want to extract the other tests with other `markers`, + you can use an empty list as `markers` value. Be aware in this case the tests do not have any references. + + +```config +{ + "markers": [], + "kind": "req" +} +``` + +In addition, you have to provide the codebeamer-url: + +`Kind` can be either `req`, `imp` or `act`. ```config { @@ -51,6 +67,8 @@ the codebeamer-url: } ``` +For more information about how to setup cpp and config files take a look at [manual-lobster_cpptest](../../documentation/manual-lobster_cpptest.md) + ## Copyright & License information From 0f8095e291cb6ac963109811ed56948877e419bf Mon Sep 17 00:00:00 2001 From: Tannaz Vahidi Date: Mon, 9 Dec 2024 09:07:12 +0100 Subject: [PATCH 10/16] Lobster-cpptest documentation (#145) Lobster-cpptest documentation has been adopted. The wrong terms have been corrected. Resolves https://github.com/bmw-software-engineering/lobster/issues/57 --- documentation/manual-lobster_cpptest.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/manual-lobster_cpptest.md b/documentation/manual-lobster_cpptest.md index 201bc13b..7b7ee6bc 100644 --- a/documentation/manual-lobster_cpptest.md +++ b/documentation/manual-lobster_cpptest.md @@ -30,10 +30,10 @@ In your test you need to also add documentation. For example: TEST(ImplicationTest, BasicTest) {} ``` -* Each test can have multiple test types defined in the documentation part. -* Test types can be used multiple times in the test documentation and can be written on multiple lines +* Each test can have multiple test-tags defined in the documentation part. +* Test-tags can be used multiple times in the test documentation and can be written on multiple lines -* These can be simply defined with `/** Test types */` or `/// Test types` format. +* These can be simply defined with `/** Test-tags */` or `/// Test-tags` format. * Test-tags can be separated by commas or spaces. From e026b8a12efb9b328fafab5ff3985de06f54693c Mon Sep 17 00:00:00 2001 From: kedarnn Date: Tue, 10 Dec 2024 08:21:27 +0100 Subject: [PATCH 11/16] Feature/lobster version flag for tools (#129) Co-authored-by: Kedar Navare --- CHANGELOG.md | 13 ++++++ lobster/tool.py | 3 +- lobster/tools/codebeamer/codebeamer.py | 7 ++- lobster/tools/core/ci_report/ci_report.py | 6 ++- lobster/tools/core/html_report/html_report.py | 6 ++- .../tools/core/online_report/online_report.py | 6 ++- lobster/tools/core/report/report.py | 6 ++- lobster/tools/cpp/cpp.py | 7 ++- lobster/tools/cpptest/cpptest.py | 6 ++- lobster/tools/gtest/gtest.py | 5 ++- lobster/tools/python/python.py | 6 ++- lobster/tools/trlc/trlc.py | 6 ++- lobster/version.py | 45 +++++++++++++++++++ 13 files changed, 109 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21fecb20..1d7e3327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,19 @@ * `lobster-codebeamer` * `lobster-report` +* Add support to view version for lobster tools for following tools: + - `lobster-ci-report` + - `lobster-codebeamer` + - `lobster-cpp` + - `lobster-cpptest` + - `lobster-gtest` + - `lobster-html-report` + - `lobster-json` + - `lobster-online-report` + - `lobster-python` + - `lobster-report` + - `lobster-trlc` + * `lobster-gtest` accepts XML nodes other than `testcase`, but ignores them. ### 0.9.19 diff --git a/lobster/tool.py b/lobster/tool.py index a9e42869..435f3811 100644 --- a/lobster/tool.py +++ b/lobster/tool.py @@ -25,7 +25,7 @@ from abc import ABCMeta, abstractmethod from functools import partial -from lobster.version import FULL_NAME +from lobster.version import FULL_NAME, get_version from lobster.errors import Message_Handler from lobster.location import File_Reference from lobster.items import Requirement, Implementation, Activity @@ -94,6 +94,7 @@ def __init__(self, name, description, extensions, official): self.add_argument = self.g_tool.add_argument + @get_version def process_commandline_options(self): options = self.ap.parse_args() diff --git a/lobster/tools/codebeamer/codebeamer.py b/lobster/tools/codebeamer/codebeamer.py index dfca5949..fd2ad674 100755 --- a/lobster/tools/codebeamer/codebeamer.py +++ b/lobster/tools/codebeamer/codebeamer.py @@ -49,6 +49,7 @@ from lobster.location import Codebeamer_Reference from lobster.errors import Message_Handler, LOBSTER_Error from lobster.io import lobster_read, lobster_write +from lobster.version import get_version TOKEN = 'token' REFERENCES = 'references' @@ -403,10 +404,12 @@ def parse_cb_config(file_name): return json_config +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: codebeamer_req.Dummy_Requirement - ap = argparse.ArgumentParser() - modes = ap.add_mutually_exclusive_group(required=True) modes.add_argument("--import-tagged", metavar="LOBSTER_FILE", diff --git a/lobster/tools/core/ci_report/ci_report.py b/lobster/tools/core/ci_report/ci_report.py index 996e1c23..1d951efd 100755 --- a/lobster/tools/core/ci_report/ci_report.py +++ b/lobster/tools/core/ci_report/ci_report.py @@ -23,11 +23,15 @@ from lobster.report import Report from lobster.items import Tracing_Status +from lobster.version import get_version +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_ci_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("lobster_report", nargs="?", default="report.lobster") diff --git a/lobster/tools/core/html_report/html_report.py b/lobster/tools/core/html_report/html_report.py index 400e0c6b..2bf7c141 100755 --- a/lobster/tools/core/html_report/html_report.py +++ b/lobster/tools/core/html_report/html_report.py @@ -32,6 +32,7 @@ Codebeamer_Reference) from lobster.items import (Tracing_Status, Item, Requirement, Implementation, Activity) +from lobster.version import get_version LOBSTER_GH = "https://github.com/bmw-software-engineering/lobster" @@ -454,9 +455,12 @@ def write_html(fd, report, dot, high_contrast): fd.write(doc.render() + "\n") +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_html_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("lobster_report", nargs="?", default="report.lobster") diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index 10c99ccd..dcd278c6 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -26,6 +26,7 @@ from lobster.report import Report from lobster.location import File_Reference, Github_Reference +from lobster.version import get_version class Parse_Error(Exception): @@ -141,9 +142,12 @@ def parse_git_root(cfg): return gh_root +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_online_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("lobster_report", nargs="?", default="report.lobster") diff --git a/lobster/tools/core/report/report.py b/lobster/tools/core/report/report.py index 96618683..48f0b943 100755 --- a/lobster/tools/core/report/report.py +++ b/lobster/tools/core/report/report.py @@ -24,11 +24,15 @@ from lobster.exceptions import LOBSTER_Exception from lobster.errors import LOBSTER_Error from lobster.report import Report +from lobster.version import get_version +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("--lobster-config", metavar="FILE", default="lobster.conf") diff --git a/lobster/tools/cpp/cpp.py b/lobster/tools/cpp/cpp.py index 93e22f6f..4820bf96 100755 --- a/lobster/tools/cpp/cpp.py +++ b/lobster/tools/cpp/cpp.py @@ -26,7 +26,7 @@ from lobster.items import Tracing_Tag, Implementation from lobster.location import File_Reference from lobster.io import lobster_write - +from lobster.version import get_version FILE_LINE_PATTERN = r"(.*):(\d+):\d+:" KIND_PATTERN = r"(function|main function|method)" @@ -48,9 +48,12 @@ SUFFIX) +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: cpp_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") diff --git a/lobster/tools/cpptest/cpptest.py b/lobster/tools/cpptest/cpptest.py index 4dfb10b0..5b2bc30a 100644 --- a/lobster/tools/cpptest/cpptest.py +++ b/lobster/tools/cpptest/cpptest.py @@ -30,6 +30,7 @@ from lobster.tools.cpptest.parser.constants import Constants from lobster.tools.cpptest.parser.requirements_parser import \ ParserForRequirements +from lobster.version import get_version OUTPUT = "output" CODEBEAMER_URL = "codebeamer_url" @@ -366,13 +367,16 @@ def lobster_cpptest(file_dir_list: list, config_dict: dict): ) +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): """ Main function to parse arguments, read configuration and launch lobster_cpptest. """ # lobster-trace: cpptest_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") diff --git a/lobster/tools/gtest/gtest.py b/lobster/tools/gtest/gtest.py index 5bbbe3c5..66b29316 100755 --- a/lobster/tools/gtest/gtest.py +++ b/lobster/tools/gtest/gtest.py @@ -25,11 +25,14 @@ from lobster.items import Tracing_Tag, Activity from lobster.location import Void_Reference, File_Reference from lobster.io import lobster_write +from lobster.version import get_version +ap = argparse.ArgumentParser() + +@get_version(ap) def main(): # lobster-trace: gtest_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") diff --git a/lobster/tools/python/python.py b/lobster/tools/python/python.py index 5d6cb123..06d4bc07 100755 --- a/lobster/tools/python/python.py +++ b/lobster/tools/python/python.py @@ -30,6 +30,7 @@ from lobster.items import Tracing_Tag, Implementation, Activity from lobster.location import File_Reference from lobster.io import lobster_write +from lobster.version import get_version LOBSTER_TRACE_PREFIX = "# lobster-trace: " LOBSTER_JUST_PREFIX = "# lobster-exclude: " @@ -443,9 +444,12 @@ def process_file(file_name, options): raise +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: python_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") diff --git a/lobster/tools/trlc/trlc.py b/lobster/tools/trlc/trlc.py index 6b00d8c9..8f72d869 100644 --- a/lobster/tools/trlc/trlc.py +++ b/lobster/tools/trlc/trlc.py @@ -33,6 +33,7 @@ from lobster.items import Tracing_Tag, Requirement from lobster.location import File_Reference from lobster.io import lobster_write +from lobster.version import get_version class Config_Parser(Parser_Base): @@ -362,8 +363,11 @@ def parse_directive(self): self.parse_tuple_type(n_typ) +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): - ap = argparse.ArgumentParser() ap.add_argument("--config-file", help=("name of lobster-trlc config file, " "by default %(default)s"), diff --git a/lobster/version.py b/lobster/version.py index bf577149..b3584d2a 100644 --- a/lobster/version.py +++ b/lobster/version.py @@ -16,6 +16,8 @@ # You should have received a copy of the GNU Affero General Public # License along with this program. If not, see # . +import sys +from argparse import ArgumentParser VERSION_TUPLE = (0, 9, 21) VERSION_SUFFIX = "dev" @@ -25,3 +27,46 @@ ) FULL_NAME = "LOBSTER %s" % LOBSTER_VERSION + + +def get_version(obj): + """ + This decorator function is used on function wherever we are parsing + the command line arguments which then adds a version argument to the function. + If a version flag is passed to the command line arguments then the LOBSTER version + is printed. + Parameters + ---------- + obj - obj can be an ArgumentParser object or a Function object. + + Returns - Nothing + ------- + + """ + if isinstance(obj, ArgumentParser): + obj.add_argument("-v, --version", action="store_true", + default=None, + help="Get version for the tool") + + def version(func): + def execution(): + if (len(sys.argv) > 1 and + (sys.argv[1] == "--version" or sys.argv[1] == "-v")): + print(FULL_NAME) + return sys.exit(0) + else: + return func() + return execution + return version + else: + def version(func): + if not isinstance(obj, ArgumentParser): + func.ap.add_argument("-v, --version", action="store_true", + default=None, + help="Get version for the tool") + if (len(sys.argv) > 1 and + (sys.argv[1] == "--version" or sys.argv[1] == "-v")): + print(FULL_NAME) + return sys.exit(0) + return obj(func) + return version From 95db70b5f05f63446ff6d11240010b125e431e22 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:37:55 +0100 Subject: [PATCH 12/16] Supporting Bazel 8 in CI (#147) The CI uses Bazel 8, but we are still using WORKSPACES in our setup. Bazel 8 disables WORKSPACES by default. Hence adding `--enable_workspace` to restore the behavior of Bazel 7. --- tests-integration/projects/basic/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-integration/projects/basic/Makefile b/tests-integration/projects/basic/Makefile index e15f644e..a6f3f8ab 100644 --- a/tests-integration/projects/basic/Makefile +++ b/tests-integration/projects/basic/Makefile @@ -21,7 +21,7 @@ cppcode.lobster: foo.h foo.cpp --out="cppcode.lobster" --clang-tidy $(CLANG_TIDY) gtests.lobster: foo.h foo.cpp test.cpp - @bazel test foo_test --cxxopt='-std=c++14' + @bazel test foo_test --cxxopt='-std=c++14' --enable_workspace @lobster-gtest $(LOBSTER_ROOT)/bazel-out/*/testlogs/$(THIS_TEST) \ --out="gtests.lobster" sed -i s/$(THIS_TEST_ESCAPED)\\///g gtests.lobster From 3105b8ee9dadda26ffcdf1aadfdab07de7d4bdcb Mon Sep 17 00:00:00 2001 From: Diego Fernandez <110383200+DiFerMa@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:29:00 +0100 Subject: [PATCH 13/16] Feature/use run tool tests py for all system tests (#149) - CMD errout.txt is compared against expected-output - Coverage command is run inside the tool - lobster-python/system-tests was added to run like lobster-trlc Co-authored-by: Diego Fernandez --- .gitignore | 1 + Makefile | 14 +++-- tests-system/Makefile | 4 ++ .../{rbt-output-file => rbt-dummy}/.gitkeep | 0 .../expected-output/basic.lobster | 63 +++++++++++++++++++ .../expected-output/exit-code.txt | 1 + .../expected-output/stderr.txt} | 0 .../expected-output/stdout.txt | 1 + .../rbt-dummy/default-scenario/input/args.txt | 5 ++ .../rbt-dummy/default-scenario/input/basic.py | 20 ++++++ tests-system/lobster-trlc/Makefile | 4 -- .../expected-output/stderr.txt | 0 tests-system/run_tool_tests.py | 42 ++++++++++--- 13 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 tests-system/Makefile rename tests-system/lobster-json/{rbt-output-file => rbt-dummy}/.gitkeep (100%) create mode 100644 tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster create mode 100644 tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt rename tests-system/lobster-python/{rbt-output-file/.gitkeep => rbt-dummy/default-scenario/expected-output/stderr.txt} (100%) create mode 100644 tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt create mode 100644 tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt create mode 100644 tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py delete mode 100644 tests-system/lobster-trlc/Makefile create mode 100644 tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt diff --git a/.gitignore b/.gitignore index f7f60576..5d7900cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *~ *.pyc bazel-* +launch.json *.egg-info build diff --git a/Makefile b/Makefile index 9bf253bb..127eefd3 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,9 @@ integration-tests: packages system-tests: mkdir -p docs - make -B -C tests-system/lobster-trlc + make -B -C tests-system TOOL=lobster-trlc + make -B -C tests-system TOOL=lobster-python make -B -C tests-system/lobster-json - make -B -C tests-system/lobster-python unit-tests: coverage run -p \ @@ -92,9 +92,9 @@ full-release: coverage: coverage combine -q coverage html --rcfile=coverage.cfg - coverage report --rcfile=coverage.cfg --fail-under=66 + coverage report --rcfile=coverage.cfg --fail-under=62 -test: system-tests unit-tests +test: clean-coverage system-tests unit-tests make coverage util/check_local_modifications.sh @@ -149,3 +149,9 @@ unit-tests.lobster-%: system-tests.lobster-%: $(eval TOOL_PATH := $(subst -,/,$*)) python3 tests-system/lobster-trlc-system-test.py $(TOOL_PATH); + +clean-coverage: + @rm -rf htmlcov + @find . -name '.coverage*' -type f -delete + @find . -name '*.pyc' -type f -delete + @echo "All .coverage, .coverage.* and *.pyc files deleted." diff --git a/tests-system/Makefile b/tests-system/Makefile new file mode 100644 index 00000000..a99ec8cb --- /dev/null +++ b/tests-system/Makefile @@ -0,0 +1,4 @@ +PYTHON = python + +run-tool-tests: + @$(PYTHON) ./run_tool_tests.py ./$(TOOL) diff --git a/tests-system/lobster-json/rbt-output-file/.gitkeep b/tests-system/lobster-json/rbt-dummy/.gitkeep similarity index 100% rename from tests-system/lobster-json/rbt-output-file/.gitkeep rename to tests-system/lobster-json/rbt-dummy/.gitkeep diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster new file mode 100644 index 00000000..98ad4f23 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster @@ -0,0 +1,63 @@ +{ + "data": [ + { + "tag": "python basic.trlc_reference", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 3, + "column": null + }, + "name": "basic.trlc_reference", + "messages": [], + "just_up": [ + "helper function" + ], + "just_down": [], + "just_global": [], + "language": "Python", + "kind": "Function" + }, + { + "tag": "python basic.Example.helper_function", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 11, + "column": null + }, + "name": "basic.Example.helper_function", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req_nor" + ], + "language": "Python", + "kind": "Method" + }, + { + "tag": "python basic.Example.nor", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 15, + "column": null + }, + "name": "basic.Example.nor", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req_nor" + ], + "language": "Python", + "kind": "Method" + } + ], + "generator": "lobster_python", + "schema": "lobster-imp-trace", + "version": 3 +} diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-python/rbt-output-file/.gitkeep b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-python/rbt-output-file/.gitkeep rename to tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stderr.txt diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt new file mode 100644 index 00000000..31e0cb57 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt @@ -0,0 +1 @@ +Written output for 3 items to basic.lobster diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt new file mode 100644 index 00000000..3903b871 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt @@ -0,0 +1,5 @@ +. +--out=basic.lobster +--parse-decorator +trlc_reference +requirement diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py b/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py new file mode 100644 index 00000000..6d3ff093 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py @@ -0,0 +1,20 @@ +import potatolib + +def trlc_reference(requirement): + # lobster-exclude: helper function + def decorator(obj): + return obj + return decorator + +class Example: + @trlc_reference(requirement="example.req_nor") + def helper_function(a, b): + # potato + return a or b + + def nor(a, b): + # lobster-trace: example.req_nor + assert isinstance(a, bool) + assert isinstance(b, bool) + + return not helper_function(a, b) diff --git a/tests-system/lobster-trlc/Makefile b/tests-system/lobster-trlc/Makefile deleted file mode 100644 index 54a7a3fc..00000000 --- a/tests-system/lobster-trlc/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -PYTHON = python - -lobster-trlc-system-tests: - @$(PYTHON) ../run_tool_tests.py . diff --git a/tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt b/tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 1d1cf843..8e8d34f9 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -15,6 +15,7 @@ class TestSetup: _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" _EXIT_CODE_FILE_NAME = "exit-code.txt" _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" + _EXPECTED_STDERR_FILE_NAME = "stderr.txt" def __init__(self, test_case_path: str): """Constructor @@ -83,9 +84,9 @@ def _get_args(self) -> List[str]: test) from the corresponding test setup file""" file = join(self._input_folder, self._ARGS_FILE_NAME) with open(file, "r", encoding="UTF-8") as file: - return file.readlines() + return [argument.strip() for argument in file.readlines()] - def get_expected_cmd_output(self) -> str: + def get_expected_stdout(self) -> str: """Reads the expected command line output (for stdout) from the corresponding test setup file""" cmd_file = join( @@ -95,6 +96,16 @@ def get_expected_cmd_output(self) -> str: with open(cmd_file, "r", encoding="UTF-8") as file: return file.read() + def get_expected_stderr(self) -> str: + """Reads the expected command line output (for stderr) from the corresponding + test setup file""" + errout_file = join( + self.get_expected_output_path(), + self._EXPECTED_STDERR_FILE_NAME, + ) + with open(errout_file, "r", encoding="UTF-8") as file: + return file.read() + @property def input_folder(self) -> str: """Returns the path containing the input data for the test""" @@ -112,13 +123,25 @@ def _get_expected_exit_code(self) -> int: def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: - """Runs the tool system test. + """Runs the tool system test using the coverage command. The tool will be executed such that its current working directory is equal to the "input" folder.""" print(f"Starting system test '{setup.name}' with arguments {setup.args} " \ - f"for tool '{tool}'.") + f"for tool '{tool}' with coverage.") + root_directory = Path(__file__).resolve().parents[1] + coverage_config_path = root_directory / "coverage.cfg" + coverage_data_path = root_directory / ".coverage" + + coverage_command = [ + "coverage", "run", "-p", + f"--rcfile={coverage_config_path}", + "--branch", + f"--data-file={coverage_data_path}", + tool, *setup.args + ] + completed_process = run( - [sys.executable, tool, *setup.args], + coverage_command, stdout=PIPE, stderr=PIPE, encoding="UTF-8", @@ -129,11 +152,16 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: def _compare_results(setup: TestSetup, completed_process: CompletedProcess): + if setup.expected_exit_code != completed_process.returncode: + print(f"STDOUT is: {completed_process.stdout}") + print(f"STDERR is: {completed_process.stderr}") assert setup.expected_exit_code == completed_process.returncode, \ f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \ f"actual is {completed_process.returncode}!" - assert setup.get_expected_cmd_output() == completed_process.stdout, \ - "Command line output is different!" + assert setup.get_expected_stdout() == completed_process.stdout, \ + "Command line output for stdout is different!" + assert setup.get_expected_stderr() == completed_process.stderr, \ + "Command line output for stderr is different!" expected = join( setup.get_expected_output_path(), setup.expected_lobster_output_file_name, From e2439ad23e5043423bee84d27eb01c0d1ecba56a Mon Sep 17 00:00:00 2001 From: kedarnn Date: Wed, 11 Dec 2024 13:48:52 +0100 Subject: [PATCH 14/16] Feature/html report UI upgrade (#134) This feature adds the following updates to the HTML report. 1. Filter items by status (Ok, Missing, Partial, Warning, Justified) 2. Hide/Unhide Issues. 3. Search in issues. --- CHANGELOG.md | 29 ++-- lobster/html/htmldoc.py | 5 + .../core/html_report/assets/html_report.css | 139 ++++++++++++++++++ .../core/html_report/assets/html_report.js | 87 +++++++++++ lobster/tools/core/html_report/html_report.py | 55 ++++++- packages/lobster-core/setup.py | 3 + packages/lobster-monolithic/setup.py | 3 + 7 files changed, 305 insertions(+), 16 deletions(-) create mode 100644 lobster/tools/core/html_report/assets/html_report.css create mode 100644 lobster/tools/core/html_report/assets/html_report.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7e3327..d1a66fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,23 @@ ### 0.9.21-dev +* `lobster-html-report` has the following updates. + * Filter items by status (Ok, Missing, Partial, Warning, Justified) + * Hide/Unhide Issues. + * Search in issues and detailed report. +* Add support to view version for lobster tools for following tools: + - `lobster-ci-report` + - `lobster-codebeamer` + - `lobster-cpp` + - `lobster-cpptest` + - `lobster-gtest` + - `lobster-html-report` + - `lobster-json` + - `lobster-online-report` + - `lobster-python` + - `lobster-report` + - `lobster-trlc` ### 0.9.20 @@ -24,19 +40,6 @@ * `lobster-codebeamer` * `lobster-report` -* Add support to view version for lobster tools for following tools: - - `lobster-ci-report` - - `lobster-codebeamer` - - `lobster-cpp` - - `lobster-cpptest` - - `lobster-gtest` - - `lobster-html-report` - - `lobster-json` - - `lobster-online-report` - - `lobster-python` - - `lobster-report` - - `lobster-trlc` - * `lobster-gtest` accepts XML nodes other than `testcase`, but ignores them. ### 0.9.19 diff --git a/lobster/html/htmldoc.py b/lobster/html/htmldoc.py index 071065f0..fbdea7dc 100644 --- a/lobster/html/htmldoc.py +++ b/lobster/html/htmldoc.py @@ -247,6 +247,7 @@ def __init__(self, title, subtitle): } self.scripts = [] self.body = [] + self.css_files = [] def add_line(self, line): assert isinstance(line, str) @@ -293,6 +294,10 @@ def render(self): rv.append(" %s: %s;" % (attr, value)) rv.append("}") rv.append("") + + # add css files that are appended to self.files + for css_file in self.css_files: + rv.append(f"") rv.append("") rv.append("") diff --git a/lobster/tools/core/html_report/assets/html_report.css b/lobster/tools/core/html_report/assets/html_report.css new file mode 100644 index 00000000..a505901f --- /dev/null +++ b/lobster/tools/core/html_report/assets/html_report.css @@ -0,0 +1,139 @@ +.button { + background-color: #818589; + border: none; + border-radius: 5px; + color: white; + padding: 12px 25px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 14px; + margin: 4px 2px; + cursor: pointer +} + +.button active:before { + content: ; + position: absolute; + left: 0; + top: 0; + display: inline-block; + width: 0; + height: 0; + border-style: solid; + border-width: 15px 15px 0 0; + border-color: #333 transparent transparent transparent +} + +.buttonActive.button { + text-decoration: none; + border: 5px solid #000000 +} + +.buttonOK { + background-color: #04AA6D; + color: white; + border: 2px solid #04AA6D; + border-radius: 5px +} + +.buttonOK:hover { + background-color: #026641; + color: white; + border: 2px solid #026641 +} + +.buttonActive.buttonOK { + text-decoration: none; + border: 5px solid #026641 +} + +.buttonPartial { + background-color: #17a2b8; + color: white; + border: 2px solid #17a2b8; + border-radius: 5px +} + +.buttonPartial:hover { + background-color: #0e616e; + color: white; + border: 2px solid #0e616e +} + +.buttonActive.buttonPartial { + text-decoration: none; + border: 5px solid #0e616e +} + +.buttonMissing { + background-color: #f44336; + color: white; + border: 2px solid #f44336; + border-radius: 5px +} + +.buttonMissing:hover { + background-color: #a91409; + color: white; + border: 2px solid #a91409 +} + +.buttonActive.buttonMissing { + text-decoration: none; + border: 5px solid #a91409 +} + +.buttonJustified { + background-color: #6c757d; + color: white; + border: 2px solid #6c757d; + border-radius: 5px +} + +.buttonJustified:hover { + background-color: #41464b; + color: white; + border: 2px solid #41464b +} + +.buttonActive.buttonJustified { + text-decoration: none; + border: 5px solid #41464b +} + +.buttonWarning { + background-color: #ffbf00; + color: white; + border: 2px solid #ffbf00; + border-radius: 5px +} + +.buttonWarning:hover { + background-color: #997300; + color: white; + border: 2px solid #997300 +} + +.buttonActive.buttonWarning { + text-decoration: none; + border: 5px solid #997300 +} + +.buttonBlue { + background-color: #0000ff; + color: white; + border: 2px solid #0000ff; + border-radius: 5px +} + +.buttonBlue:hover { + background-color: #000099; + color: white; + border: 2px solid #000099 +} + +.buttonActive.buttonBlue { + text-decoration: none; + border: 5px solid #000099 +} diff --git a/lobster/tools/core/html_report/assets/html_report.js b/lobster/tools/core/html_report/assets/html_report.js new file mode 100644 index 00000000..b2f5a874 --- /dev/null +++ b/lobster/tools/core/html_report/assets/html_report.js @@ -0,0 +1,87 @@ +function buttonFilter(filter) { + var elms = document.getElementsByTagName("div"); + var issue_elms = document.getElementsByClassName("issue"); + for (i = 0; i < elms.length; i++) { + if (elms[i].id.startsWith("item-")) { + console.log("elms[i].className ", elms[i].className) + if (filter == "all") { + elms[i].style.display = "block"; + } else if (elms[i].className == "item-" + filter) { + elms[i].style.display = "block"; + } else { + elms[i].style.display = "none"; + } + } + } + // filter the issues list based on the issue filter button clicked + for (i = 0; i < issue_elms.length; i++) { + console.log("log ", issue_elms[i].className) + if (filter == "all") { + issue_elms[i].style.display = "list-item"; + } else if (issue_elms[i].className == "issue issue-" + filter) { + issue_elms[i].style.display = "list-item"; + } else { + issue_elms[i].style.display = "none"; + } + } + activeButton(filter); + //call the search filering which could have been overwritten by the current filtering + searchItem(); +} + + +function activeButton(filter) { + var elms = document.getElementsByTagName("button"); + console.log("the click buitton is " + filter); + for (i = 0; i < elms.length; i++) { + if (elms[i].className.includes("buttonActive")) { + console.log("elem active found : " + elms[i].className); + elms[i].className = elms[i].className.replace("buttonActive", ""); + } else if (elms[i].className.toLowerCase().includes("button" + filter.toLowerCase())) { + console.log("elem to be activated found : " + elms[i].className); + elms[i].className = elms[i].className + " buttonActive"; + } + } +} + + +function ToggleIssues() { + var div_issue = document.getElementById("issues-section"); + if (div_issue.style.display == "block" || div_issue.style.display == "") { + div_issue.style.display = "none"; + document.getElementById("BtnToggleIssue").innerHTML = "Show Issues"; + document.getElementById("BtnToggleIssue").className = document.getElementById("BtnToggleIssue").className + " buttonActive"; + } else { + div_issue.style = 'display: block; flex-direction: column; height: 200px;' + + 'overflow:auto;'; + document.getElementById("BtnToggleIssue").innerHTML = "Hide Issues"; + document.getElementById("BtnToggleIssue").className = document.getElementById("BtnToggleIssue").className.replace("buttonActive", ""); + } +} + + +function searchItem() { + var input = document.getElementById('search').value + input = input.toLowerCase(); + + var divs = document.getElementsByClassName('item-name'); + for (i = 0; i < divs.length; i++) { + var title = divs[i].parentNode.getAttribute("title"); + // get requirement name: 2nd part when we cut the long string with /svg + var reqname = divs[i].innerHTML.toLowerCase().split("").pop(); + reqname = reqname.split(" ").pop(); + if (reqname.includes(input)) { + // the search pattern has been found, if this elem has the title "hidden-not-matching", put it back to diplayed + if (title) { + if (title.startsWith("hidden-not-matching")) { + divs[i].parentNode.style.display = "block"; + } + } + divs[i].parentNode.setAttribute("title", "matching-" + input) + } else { + // not maching, we hide + divs[i].parentNode.setAttribute("title", "hidden-not-matching") + divs[i].parentNode.style.display = "none"; + } + } +} diff --git a/lobster/tools/core/html_report/html_report.py b/lobster/tools/core/html_report/html_report.py index 2bf7c141..c40a9501 100755 --- a/lobster/tools/core/html_report/html_report.py +++ b/lobster/tools/core/html_report/html_report.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU Affero General Public # License along with this program. If not, see # . - import os.path import argparse import html @@ -374,8 +373,43 @@ def write_html(fd, report, dot, high_contrast): print("> please install Graphviz (https://graphviz.org)") doc.add_line('') + ### Filtering + doc.add_heading(2, "Filtering", "filtering-options") + doc.add_heading(3, "Item Filters") + doc.add_line('
') + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + doc.add_line("
") + + doc.add_heading(3, "Show Issues") + doc.add_line('
') + doc.add_line('') + doc.add_line('
') + + doc.add_heading(3, "Search", "search") + doc.add_line('') + doc.add_line('") ### Report file_heading = None @@ -447,6 +483,19 @@ def write_html(fd, report, dot, high_contrast): write_item_box_end(doc) else: doc.add_line("No items recorded at this level.") + # Closing tag for id #search-sec-id + doc.add_line("") + + # Add the css from assets + dir_path = os.path.dirname(os.path.abspath(__file__)) + file_path = dir_path + "/assets/html_report.css" + doc.css_files.append(file_path) + + # Add javascript from assets/html_report.js file + dir_path = os.path.dirname(os.path.abspath(__file__)) + file_path = dir_path + "/assets/html_report.js" + with open(file_path, "r", encoding="UTF-8") as scripts: + doc.scripts.append("".join(scripts.readlines())) ### STM # doc.add_heading(2, "Software traceability matrix", "matrix") diff --git a/packages/lobster-core/setup.py b/packages/lobster-core/setup.py index af0db774..9df68841 100644 --- a/packages/lobster-core/setup.py +++ b/packages/lobster-core/setup.py @@ -52,6 +52,9 @@ "lobster.tools.core.html_report", "lobster.tools.core.online_report", "lobster.tools.core.report"], + package_data={ + "lobster.tools.core.html_report":["assets/*"] + }, install_requires=[], python_requires=">=3.7, <4", classifiers=[ diff --git a/packages/lobster-monolithic/setup.py b/packages/lobster-monolithic/setup.py index 4960fead..2b161414 100644 --- a/packages/lobster-monolithic/setup.py +++ b/packages/lobster-monolithic/setup.py @@ -48,6 +48,9 @@ project_urls=project_urls, license="GNU Affero General Public License v3", packages=setuptools.find_packages(), + package_data={ + "lobster.tools.core.html_report":["assets/*"] + }, install_requires=[ "miss-hit>=0.9.42", "requests>=2.22", From b6f3095a6f9a4ebb5d2bfb5e4bb903dd5eb55c09 Mon Sep 17 00:00:00 2001 From: Philipp Wullstein-Kammler <111539239+phiwuu@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:09:42 +0100 Subject: [PATCH 15/16] Migrate "basic" system tests of `lobster-json` (#153) Revert gitignore for expected-output: All files in any folder called `expected-output` inside `tests-system` are exempted from all gitignore rules. Improve readability of assertion messages for system tests: If there is a difference between the expected and actual output of either stdout or stderr, then these values are fully printed. This helps developers to spot errors faster. Migrate "basic" system tests of lobster-json: Migrate the "basic" system test so that they can be executed with the `run_tool_tests.py` tool. Write requirements for these tests. `run_tool_tests.py` now deletes *.lobster files that were generated during test execution. --- Makefile | 2 +- lobster/tools/json/requirements.trlc | 16 ++- tests-system/.gitignore | 1 + tests-system/lobster-json/Makefile | 15 --- .../expected-output/exit-code.txt | 1 + .../expected-output/output.lobster} | 2 - .../expected-output/stderr.txt} | 0 .../expected-output/stdout.txt | 1 + .../attribute-given/input/args.txt | 4 + .../attribute-given/input/basic.json} | 1 - .../rbt-synthetic-tag-name/README.md | 22 ++++ .../expected-output/exit-code.txt | 1 + .../expected-output/output.lobster} | 38 +++--- .../expected-output/stderr.txt | 0 .../expected-output/stdout.txt | 1 + .../flat-input-structure/input/args.txt | 3 + .../flat-input-structure/input/basic.json} | 1 - .../expected-output/exit-code.txt | 1 + .../expected-output/output.lobster | 119 ++++++++++++++++++ .../expected-output/stderr.txt | 0 .../expected-output/stdout.txt | 1 + .../nested-input-structure/input/args.txt | 3 + .../input/one/two/basic.json | 38 ++++++ tests-system/run_tool_tests.py | 44 +++++-- 24 files changed, 260 insertions(+), 55 deletions(-) create mode 100644 tests-system/.gitignore delete mode 100644 tests-system/lobster-json/Makefile create mode 100644 tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt rename tests-system/lobster-json/{basic.output => rbt-name-attribute/attribute-given/expected-output/output.lobster} (97%) rename tests-system/lobster-json/{rbt-dummy/.gitkeep => rbt-name-attribute/attribute-given/expected-output/stderr.txt} (100%) create mode 100644 tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt create mode 100644 tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt rename tests-system/lobster-json/{basic2.input => rbt-name-attribute/attribute-given/input/basic.json} (95%) create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/README.md create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt rename tests-system/lobster-json/{basic2.output => rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster} (72%) create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt rename tests-system/lobster-json/{basic.input => rbt-synthetic-tag-name/flat-input-structure/input/basic.json} (92%) create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt create mode 100644 tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json diff --git a/Makefile b/Makefile index 127eefd3..7ec0e6de 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,9 @@ integration-tests: packages system-tests: mkdir -p docs + make -B -C tests-system TOOL=lobster-json make -B -C tests-system TOOL=lobster-trlc make -B -C tests-system TOOL=lobster-python - make -B -C tests-system/lobster-json unit-tests: coverage run -p \ diff --git a/lobster/tools/json/requirements.trlc b/lobster/tools/json/requirements.trlc index 309de1f3..0ee8b944 100644 --- a/lobster/tools/json/requirements.trlc +++ b/lobster/tools/json/requirements.trlc @@ -1,16 +1,20 @@ package json_req import req -req.Software_Requirement Dummy_Requirement { +req.Software_Requirement Synthetic_Tag_Name { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is not given, then the tool shall create + a synthetic tag name based on the path of the JSON input file and an item counter. + + Note: The item counter does not need to be unique across files. + It shall be unique at least for each input file separately. ''' } -req.Software_Requirement Dummy_Requirement_Unit_Test { +req.Software_Requirement Name_Attribute { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is given, then the tool shall + - use the value of the command line argument as JSON key of the JSON item + - and use that obtained value as LOBSTER item tag name. ''' } diff --git a/tests-system/.gitignore b/tests-system/.gitignore new file mode 100644 index 00000000..78f221c8 --- /dev/null +++ b/tests-system/.gitignore @@ -0,0 +1 @@ +!**/expected-output/* diff --git a/tests-system/lobster-json/Makefile b/tests-system/lobster-json/Makefile deleted file mode 100644 index 55f3e314..00000000 --- a/tests-system/lobster-json/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -TOOL=../../lobster-json - -TARGETS=$(addsuffix .output, $(basename $(wildcard **/*.input) $(wildcard *.input))) - -all: $(TARGETS) - -%.output: %.input - @tail -n +2 $< > $*.json - @touch $*.lobster - -@coverage run -p --rcfile=../../coverage.cfg --branch \ - --data-file ../../.coverage \ - $(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 - @echo "==========" >> $@ - @cat $*.lobster >> $@ - @rm $*.json $*.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic.output b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster similarity index 97% rename from tests-system/lobster-json/basic.output rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster index 3157c93e..79d8d7f3 100644 --- a/tests-system/lobster-json/basic.output +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster @@ -1,5 +1,3 @@ -lobster-json: wrote 6 items to basic.lobster -========== { "data": [ { diff --git a/tests-system/lobster-json/rbt-dummy/.gitkeep b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-json/rbt-dummy/.gitkeep rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt new file mode 100644 index 00000000..44e3c250 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt @@ -0,0 +1,4 @@ +--single +--tag-attribute=tags +--name-attribute=name +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.input b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json similarity index 95% rename from tests-system/lobster-json/basic2.input rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json index f57ea48e..7bb52fe8 100644 --- a/tests-system/lobster-json/basic2.input +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/README.md b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md new file mode 100644 index 00000000..944ef261 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md @@ -0,0 +1,22 @@ +If the `--name-attribute` option is not specified, +then the tag name must be constructed based on the following two values: +- path of the json file +- item counter + +# Test case "flat-input-structure" +- No files or directories are specified. + The tool is expected to search for input files in the current working directory. +- `--name-attribute` is not given. + The tool is expected to construct LOBSTER item tags by taking the file name and + appending an integer counter, starting at 1. + This is implemented inside function `syn_test_name`, and it must handle all file + paths as relative paths, because no input files were given explicitly. +- `--out` is specified. + The tool is expected to use that path name for the output file. +- `--single` is specified to make the tool output predictable. + +# Test case "nested-input-structure" +This is the same as the above test, except that its input file is nested in a +sub-sub-directory. +The tool is expected to take the names of the sub-directories into account when +generating te tag name. diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.output b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster similarity index 72% rename from tests-system/lobster-json/basic2.output rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster index 287d5a13..d0331f77 100644 --- a/tests-system/lobster-json/basic2.output +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster @@ -1,16 +1,14 @@ -lobster-json: wrote 6 items to basic2.lobster -========== { "data": [ { - "tag": "json ./basic2.json:basic2.1", + "tag": "json ./basic.json:basic.1", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.1", + "name": "./basic.json:basic.1", "messages": [], "just_up": [], "just_down": [], @@ -23,14 +21,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.2", + "tag": "json ./basic.json:basic.2", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.2", + "name": "./basic.json:basic.2", "messages": [], "just_up": [], "just_down": [], @@ -40,14 +38,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.3", + "tag": "json ./basic.json:basic.3", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.3", + "name": "./basic.json:basic.3", "messages": [], "just_up": [], "just_down": [], @@ -61,14 +59,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.4", + "tag": "json ./basic.json:basic.4", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.4", + "name": "./basic.json:basic.4", "messages": [], "just_up": [], "just_down": [], @@ -78,14 +76,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.5", + "tag": "json ./basic.json:basic.5", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.5", + "name": "./basic.json:basic.5", "messages": [], "just_up": [], "just_down": [], @@ -98,14 +96,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.6", + "tag": "json ./basic.json:basic.6", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.6", + "name": "./basic.json:basic.6", "messages": [], "just_up": [], "just_down": [], diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic.input b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json similarity index 92% rename from tests-system/lobster-json/basic.input rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json index fa82b8a4..7bb52fe8 100644 --- a/tests-system/lobster-json/basic.input +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags --name-attribute=name [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster new file mode 100644 index 00000000..8571ad42 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster @@ -0,0 +1,119 @@ +{ + "data": [ + { + "tag": "json ./one/two/basic.json:one.two.basic.1", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.1", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req1" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.2", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.2", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.3", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.3", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req2", + "req example.req3" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.4", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.4", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.5", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.5", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req4" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.6", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.6", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + } + ], + "generator": "lobster-json", + "schema": "lobster-act-trace", + "version": 3 +} diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json new file mode 100644 index 00000000..7bb52fe8 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json @@ -0,0 +1,38 @@ +[ + { + "name" : "Test_1", + "tags" : ["example.req1"], + "inputs" : [1, 2], + "expect" : 3 + }, + { + "name" : "Test_2", + "tags" : [], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_3", + "tags" : ["example.req2", + "example.req3"], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_4", + "tags" : null, + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_5", + "tags" : "example.req4", + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_6", + "inputs" : [1, 0], + "expect" : 1 + } +] diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 8e8d34f9..8a133f76 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,5 +1,5 @@ import sys -from os import scandir, DirEntry +from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path from subprocess import CompletedProcess, PIPE, run @@ -53,7 +53,8 @@ def _get_expected_lobster_output_file_name(self) -> str: if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( - "Invalid test setup: No *.lobster file found in expected output folder!", + f"Invalid test setup: No *.lobster file found in " + f"{self.get_expected_output_path()}!", ) @staticmethod @@ -151,17 +152,31 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: return completed_process +def _compare_cmd_output(name: str, expected: str, actual: str) -> bool: + if expected != actual: + print(f"Actual {name} is (length: {len(actual)} chars):\n{actual}") + print(f"Expected {name} is (length: {len(expected)} chars):\n{expected}") + return False + return True + + def _compare_results(setup: TestSetup, completed_process: CompletedProcess): - if setup.expected_exit_code != completed_process.returncode: - print(f"STDOUT is: {completed_process.stdout}") - print(f"STDERR is: {completed_process.stderr}") assert setup.expected_exit_code == completed_process.returncode, \ f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \ f"actual is {completed_process.returncode}!" - assert setup.get_expected_stdout() == completed_process.stdout, \ - "Command line output for stdout is different!" - assert setup.get_expected_stderr() == completed_process.stderr, \ - "Command line output for stderr is different!" + + assert _compare_cmd_output( + name="STDOUT", + expected=setup.get_expected_stdout(), + actual=completed_process.stdout, + ), "Command line output for STDOUT is different!" + + assert _compare_cmd_output( + name="STDERR", + expected=setup.get_expected_stderr(), + actual=completed_process.stderr, + ), "Command line output for STDERR is different!" + expected = join( setup.get_expected_output_path(), setup.expected_lobster_output_file_name, @@ -200,6 +215,16 @@ def _get_directories( yield dir_entry +def _delete_generated_files(setup: TestSetup): + """Deletes the *.lobster file that has been generated by the test""" + generated = join( + setup.input_folder, + setup.expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + remove(generated) + + def _run_tests(directory: str, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. @@ -218,6 +243,7 @@ def _run_tests(directory: str, tool: str) -> int: test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) + _delete_generated_files(test_setup) counter += 1 print(f"{counter} system tests finished successfully for {tool}.") From 0d854ef1ddfefc0f0de005ac491db401a9397e7b Mon Sep 17 00:00:00 2001 From: "Tannaz Vahidi (ext.)" Date: Wed, 11 Dec 2024 15:40:31 +0100 Subject: [PATCH 16/16] Tests adaptation --- .../expected-output/output.lobster} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests-system/lobster-cpptest/{rbt-output-file/.gitkeep => rbt-invalid-source-input-file-paths/invalid-directory-input-scenario/expected-output/output.lobster} (100%) diff --git a/tests-system/lobster-cpptest/rbt-output-file/.gitkeep b/tests-system/lobster-cpptest/rbt-invalid-source-input-file-paths/invalid-directory-input-scenario/expected-output/output.lobster similarity index 100% rename from tests-system/lobster-cpptest/rbt-output-file/.gitkeep rename to tests-system/lobster-cpptest/rbt-invalid-source-input-file-paths/invalid-directory-input-scenario/expected-output/output.lobster