diff --git a/.gitignore b/.gitignore index f7f60576..5d7900cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *~ *.pyc bazel-* +launch.json *.egg-info build diff --git a/CHANGELOG.md b/CHANGELOG.md index 0584631f..d1a66fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,42 @@ ## Changelog -### 0.9.20-dev +### 0.9.21-dev + +* `lobster-html-report` has the following updates. + * Filter items by status (Ok, Missing, Partial, Warning, Justified) + * Hide/Unhide Issues. + * Search in issues and detailed report. + +* Add support to view version for lobster tools for following tools: + - `lobster-ci-report` + - `lobster-codebeamer` + - `lobster-cpp` + - `lobster-cpptest` + - `lobster-gtest` + - `lobster-html-report` + - `lobster-json` + - `lobster-online-report` + - `lobster-python` + - `lobster-report` + - `lobster-trlc` + +### 0.9.20 + +* Add `--compile-commands` flag to `lobster-cpp`. This allows to specify a path to the + compile command database and is effectively a wrapper around the `-p` argument of + `clang tidy`. See the official documentation of `clang tidy` for more details on this + parameter. + +* `lobster-cpptest` writes absolute paths into its `*.lobster` output files instead of + paths relative to the current working directory. + +* If a `*.lobster` file contains a tag more than once, then an error message + ("duplicated definition") is printed for each consecutive entry with the same tag, + instead of printing it just for the first entry. + The following tools are affected: + * `lobster-codebeamer` + * `lobster-report` * `lobster-gtest` accepts XML nodes other than `testcase`, but ignores them. diff --git a/Makefile b/Makefile index 56d8a3df..7ec0e6de 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: style python3 -m pylint --rcfile=pylint3.cfg \ --reports=no \ --ignore=assets.py \ - lobster util tests-system/lobster-trlc/run_tool_tests.py + lobster util tests-system/run_tool_tests.py style: @python3 -m pycodestyle lobster \ @@ -58,9 +58,9 @@ integration-tests: packages system-tests: mkdir -p docs - make -B -C tests-system/lobster-trlc - make -B -C tests-system/lobster-json - make -B -C tests-system/lobster-python + make -B -C tests-system TOOL=lobster-json + make -B -C tests-system TOOL=lobster-trlc + make -B -C tests-system TOOL=lobster-python unit-tests: coverage run -p \ @@ -92,9 +92,9 @@ full-release: coverage: coverage combine -q coverage html --rcfile=coverage.cfg - coverage report --rcfile=coverage.cfg --fail-under=66 + coverage report --rcfile=coverage.cfg --fail-under=62 -test: system-tests unit-tests +test: clean-coverage system-tests unit-tests make coverage util/check_local_modifications.sh @@ -149,3 +149,9 @@ unit-tests.lobster-%: system-tests.lobster-%: $(eval TOOL_PATH := $(subst -,/,$*)) python3 tests-system/lobster-trlc-system-test.py $(TOOL_PATH); + +clean-coverage: + @rm -rf htmlcov + @find . -name '.coverage*' -type f -delete + @find . -name '*.pyc' -type f -delete + @echo "All .coverage, .coverage.* and *.pyc files deleted." diff --git a/README.md b/README.md index b37146b1..acc104db 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,15 @@ and requirements coverage, which is essential for meeting standards such as ISO 26262. This repository contains the prototype for LOBSTER, which is a key -ingredient to make TRCL more useful. +ingredient to make TRLC and other supported tools more useful. -It has tools to extract tracing tags from a variety of sources combine -them and produce a tracing report. The [TRLC tracing +It has tools to extract tracing tags from a variety of sources to +combine them and produce a tracing report. The [TRLC tracing report](https://bmw-software-engineering.github.io/trlc/tracing.html) from the [TRLC Project](https://github.com/bmw-software-engineering/trlc/) is a -reasonable example of what is possible. +reasonable example of what is possible when lobster is used in combination +with TRLC or any of the other supported tools. ## Installing @@ -23,12 +24,19 @@ convenient meta-package `bmw-lobster` which installs everything. ``` $ pip3 install bmw-lobster ``` +For the HTML Report `graphviz` is also used to generate the tracing policy diagram. More on that on the user [manual](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/user-manual.md). + +``` +$ sudo apt-get install -y graphviz +``` + +The `lobster-cpp` converter tool needs a specific version of `clang-tidy`. Please see [here](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/user-manual.md#clang-tidy-file-generation) to create it. ## Supported inputs The following requirements frameworks are supported: -* [TRLC](work-in-progress) (only some use cases supported right now) +* [TRLC](https://github.com/bmw-software-engineering/trlc/) (only some use cases supported right now) * [Codebeamer](packages/lobster-tool-codebeamer/README.md) (only some use cases supported right now) @@ -56,17 +64,18 @@ The following verification and miscellaneous frameworks are supported: ## Installing individual packages -The individual packages that `bmw-lobster` depends on are: +The individual PyPI packages that `bmw-lobster` depends on are: * `bmw-lobster-core` the core API and various report generators. All - other tools depend on this. -* `bmw-lobster-tool-cpp` (for C/C++ code) -* `bmw-lobster-tool-cpptest` (for C/C++ code) -* `bmw-lobster-tool-gtest` (for GoogleTest tests) -* `bmw-lobster-tool-python` (for Python3 code) -* `bmw-lobster-tool-beamer` (for requirements in Codebeamer) -* `bmw-lobster-tool-json` (for activities in JSON) -* `miss_hit` (for MATLAB/Octave code or Simulink models) + other tools depend on this [Link](https://pypi.org/project/bmw-lobster-core) +* `bmw-lobster-tool-codebeamer` (for requirements in Codebeamer) [Link](https://pypi.org/project/bmw-lobster-tool-codebeamer) +* `bmw-lobster-tool-cpp` (for C/C++ code) [Link](https://pypi.org/project/bmw-lobster-tool-cpp) +* `bmw-lobster-tool-cpptest` (for C/C++ code) [Link](https://pypi.org/project/bmw-lobster-tool-cpp) +* `bmw-lobster-tool-gtest` (for GoogleTest tests) [Link](https://pypi.org/project/bmw-lobster-tool-gtest) +* `bmw-lobster-tool-json` (for activities in JSON) [Link](https://pypi.org/project/bmw-lobster-tool-json) +* `bmw-lobster-tool-python` (for Python3 code) [Link](https://pypi.org/project/bmw-lobster-tool-python) +* `bmw-lobster-tool-trlc` (for TRLC code) [Link](https://pypi.org/project/bmw-lobster-tool-trlc) +* `miss_hit` (for MATLAB/Octave code or Simulink models) [Link](https://pypi.org/project/miss_hit) ### For LOBSTER developers @@ -89,15 +98,86 @@ Here are the links to the individual html requirements coverage reports: * [Requirement Coverage Report Core Report](https://bmw-software-engineering.github.io/lobster/tracing-core_report.html) * [Requirement Coverage Report Codebeamer](https://bmw-software-engineering.github.io/lobster/tracing-codebeamer.html) +### Simple lobster-demo + +* A simple example can be found in the repository: [lobster-demo](https://github.com/bmw-software-engineering/lobster-demo) + +## Workflow of LOBSTER + +The lobster tool uses several steps to accomplish a fully modular software traceability +and requirements coverage report. You can consider lobster as a set of 1) conversion tools, 2) a common interchange format, 3) the report creation tool and 4) a renderer for the tracing report. + +For a more detailed description please read our [user guide](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/config_files.md). + +These steps are in the following diagram and go from left to right side: + +```mermaid +graph LR + subgraph "Converter-Tools" + direction TB + A1[Lobster-python] + A2[Lobster-trlc] + A3[Lobster-json] + A4[Lobster-cpp] + A5[Lobster-codebeamer] + A6[Lobster-gtest] + end + + subgraph "Common inter. format" + direction TB + B1[Python.lobster] + B2[Trlc.lobster] + B3[Json.lobster] + B4[Cpp.lobster] + B5[Codebeamer.lobster] + B6[Gtest.lobster] + end + + subgraph "Generate lobster report" + direction TB + D1[Lobster-online-report] + D1 ---> D2 + D2[Lobster-report -> report.lobster] + D3[Tracing policy -> lobster.conf] + D3 ---> D2 + end + + subgraph "Renderer" + direction TB + C1[html] + C2[CI] + C3["..."] + end + + %% Main connections + A1 ---> B1 + A2 ---> B2 + A3 ---> B3 + A4 ---> B4 + A5 ---> B5 + A6 ---> B6 + + + %% Connect all schema elements to Lobster-report -> report.lobster + B1 ----> D2 + B2 ----> D2 + B3 ----> D2 + B4 ----> D2 + B5 ----> D2 + B6 ----> D2 + + %% Connect Lobster-report -> report.lobster to renderers + D2 ---> C1 + D2 ---> C2 + D2 ---> C3 + ``` + ## Planned inputs The following inputs are planned but not implemeted yet: * `lobster-java`: Java code * `lobster-kotlin`: Kotlin code -* `lobster-ada`: Ada and SPARK code (via libadalang) -* `lobster-latex`: Requirements written in LaTeX -* `lobster-markdown`: Requirements written in Markdown ## Copyright & License information diff --git a/documentation/manual-lobster_cpptest.md b/documentation/manual-lobster_cpptest.md new file mode 100644 index 00000000..7b7ee6bc --- /dev/null +++ b/documentation/manual-lobster_cpptest.md @@ -0,0 +1,134 @@ +# Tracing to C/C++ tests + +## Setup and requirements + +You will need a C/C++ file or a directory containing these files. +* Note: The tool also supports a combination of files and folders. + +C/C++ files should have one of these extensions to be evaluated by this tool: `.cpp`, `.cc`, `.c` or `.h` + +## Preparing C/C++ test documentation with requirements +The test functions should be specified with one the following `macros`: + `TEST`, + `TEST_P`, + `TEST_F`, + `TYPED_TEST`, + `TYPED_TEST_P`, + `TYPED_TEST_SUITE`, + `TEST_P_INSTANCE` or + `TEST_F_INSTANCE` + +In your test you need to also add documentation. For example: + +```C++ +/** + * @requirement CB-#0815, CB-#0816, + * CB-#0817 + * @requirement CB-#0818 CB-#0819 + * CB-#0820 + */ +TEST(ImplicationTest, BasicTest) {} +``` + +* Each test can have multiple test-tags defined in the documentation part. +* Test-tags can be used multiple times in the test documentation and can be written on multiple lines + +* These can be simply defined with `/** Test-tags */` or `/// Test-tags` format. + +* Test-tags can be separated by commas or spaces. + +* Note: The `markers` shall be exactly close to the test functions (without any empty lines) + + +```C++ +/** + * @requirement CB-#0815, CB-#0816 + */ +TEST(ImplicationTest, BasicTest) {} +``` + +```C++ +/// +/// @requiredby FOO0::BAR0 +/// +TEST(ImplicationTest, BasicTest) {} +``` +The regex used for each test-tag is as follows: + +@requirement +: ```r"(CB-#\d+)"``` +: ```r"({provided codebeamer-url in config-file}(?P\d+))"``` + +@requiredby +: ```r"(\w*::\w+)"``` + +@defect +: ```r"(CB-#\d+)|(OCT-#\d+)"``` + + +## Preparing cpptest config-file + +You have to provide a config-file which determines which `markers` should be extracted in which output-files. +The expected `kind` for each output-file should also be specified. + +In addition, you have to provide the `codebeamer-url`: + +```cpp.config +{ + "output": { + "unit_tests.lobster" : + { + "markers": ["@requirement"], + "kind": "req" + }, + "components_tests.lobster" : + { + "markers": ["@requiredby", "@requirement"], + "kind": "imp" + } + }, + "codebeamer_url": "https://codebeamer.example.com/test" +} + ``` + +* Note: If you want to extract the other tests with other `markers`, + you can use an empty list as `markers` value. Be aware in this case the tests do not have any references. + +```cpp.config +{ + "output": { + "tests.lobster" : + { + "markers": [], + "kind": "req" + } + }, + "codebeamer_url": "https://codebeamer.example.com/test" +} + ``` + + +## Creating lobster files + +Run the `lobster_cpptest` tool, pointing it to one or more C/C++ files, or a directory containing one or more C/C++ files. + +For example `lobster_cpptest .` should find all your C/C++ files in the root directory. + +You have to also provide a `--config-file` file to configure the behaviour of the tool. +A more complete command line might look like: + +```sh +$ lobster-cpptest . --config-file cpp.config +``` + +## Example + +The LOBSTER unit tests contains a working example: + +* Test [test_case.cpp](../test-unit/lobster-cpptest/data/test_case.cpp) containing requirement tags + +## Notes & Caveats +* This tool supports these `markers`: '@requirement', '@requiredby' and '@defect' +* This tool supports these `kind`: 'req', 'imp' and 'act' + + diff --git a/documentation/manual-lobster_gtest.md b/documentation/manual-lobster_gtest.md index 03b3c75e..2e96afaf 100644 --- a/documentation/manual-lobster_gtest.md +++ b/documentation/manual-lobster_gtest.md @@ -55,11 +55,11 @@ $ lobster_gtest . --out gtests.lobster The LOBSTER testsuite contains a working example: -* Bazel [BUILD](../tests/projects/basic/BUILD) file to set up -* Test [test.cpp](../tests/projects/basic/test.cpp) containing tracing tags -* Requrements [potato.trlc](../tests/projects/basic/potato.trlc) +* Bazel [BUILD](../integration-tests/projects/basic/BUILD) file to set up +* Test [test.cpp](../integration-tests/projects/basic/test.cpp) containing tracing tags +* Requrements [potato.trlc](../integration-tests/projects/basic/potato.trlc) containing tracing the requirements mentioned by the test -* [Makefile](../tests/projects/basic/Makefile) gluing everything +* [Makefile](../integration-tests/projects/basic/Makefile) gluing everything together ## Notes & Caveats diff --git a/documentation/user-manual.md b/documentation/user-manual.md index 67773ce1..c2b48411 100644 --- a/documentation/user-manual.md +++ b/documentation/user-manual.md @@ -8,11 +8,11 @@ of any safety related software developments, and often involves duplicate maintenance and tedious manual processes. LOBSTER aims to simplify this. The basic setup is as follows: -1. Configure LOBSTER -2. Annotate code / tests / requirements -3. Setup build system to use lobster tools (e.g. `lobster_cpp`) to - extract information -4. Build report by calling `lobster_report` +1. Configure LOBSTER via the tracing policy. Usually called `lobster.conf`. More on that [here](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/config_files.md). +2. Annotate traces onto your code, your tests and/or your requirements. Please notice each one of the supported languages has their own way to annotate the tracing tags. +3. Extract the requirement traces with the corresponding lobster conversion tools (e.g. `lobster-cpp`). This converts those traces into a *.lobster file (into a "common unified interchange format"). +4. Build report by calling `lobster-report` +5. Render your report by calling the core losbter tools. Usually a local HTML report is desired via `lobster-html-report` or even `lobster-ci-report` for your CI. The basic idea is that you have a number of artefacts that you wish to relate to each other; stored in different "databases". Sometimes this @@ -56,6 +56,17 @@ void main() } ``` +For C++ tests requirements will be referenced inside the documentation. + +The syntax for Codebeamer requirements looks like this: + +```C++ TEST +/** + * @requirement CB-#0815 CB-#0816 + */ +TEST(RequirementTagTest1, RequirementAsComments) {} +``` + A central idea in LOBSTER is to reduce duplication of information, so tracing links are *only ever added to artefacts lower in the tracing hierarchy*. So in the first example above, we'd add tags to the tests, @@ -105,6 +116,10 @@ as integration tests, HIL tests, etc.) * lobster_trlc: DOCUMENTATION TODO +Note: We are providing a full traceability report, using LOBSTER itself and [TRLC](https://github.com/bmw-software-engineering/trlc), for each tool maintained in this repository. You are able to produce these reports yourself when using the target `make tracing`. + +All its final html outputs are to be seen [here](https://github.com/bmw-software-engineering/lobster?tab=readme-ov-file#requirements-coverage). + ### codeBeamer * [lobster_codebeamer](manual-lobster_codebeamer.md): for tracing to @@ -113,9 +128,34 @@ as integration tests, HIL tests, etc.) ### C / C++ * lobster_cpp: DOCUMENTATION TODO +* [lobster_cpptest](manual-lobster_cpptest.md): for tracing requirements and/or defects from C++ tests. * [lobster_gtest](manual-lobster_gtest.md): for tracing tags in (executed) googletests. +> **Note:** The lobster-cpp tool needs a clang-tidy file. Don't forget to generate it to be able to use this conversion tool. + +#### Clang-tidy File Generation + +You can just run the `clang-tidy` Makefile target in root folder. + +If you prefer to do it manually or are Windows user, install Ninja and cmake and then please follow the next steps: + +* To generate the clang-tidy file, which is needed for the cpp tool, make sure that your apt is working well on WSL (Windows Subsystem for Linux) or use a Linux environment. + +* Clone this repository - `https://github.com/bmw-software-engineering/llvm-project` + +* Below 2 dependencies required for clang-tidy creation. + 1. `sudo apt install cmake` + 2. `sudo apt install ninja-build` + +* Below 2 commands need to execute to generate the build folder. + 1. `cmake -S llvm -B build -G Ninja -DLLVM_ENABLE_PROJECTS='clang;clang-tools-extra' -DCMAKE_BUILD_TYPE=Release` + 2. `cmake --build build --target clang-tidy` + +* Once you generate the build folder you can see the clang-tidy file in `./build/bin` folder. + +* To generate the cpp.lobster file, you need to make sure that your llvm-project and lobster-demo project should be in same directory. + ### Python * lobster_python: DOCUMENTATION TODO diff --git a/lobster/html/htmldoc.py b/lobster/html/htmldoc.py index 071065f0..fbdea7dc 100644 --- a/lobster/html/htmldoc.py +++ b/lobster/html/htmldoc.py @@ -247,6 +247,7 @@ def __init__(self, title, subtitle): } self.scripts = [] self.body = [] + self.css_files = [] def add_line(self, line): assert isinstance(line, str) @@ -293,6 +294,10 @@ def render(self): rv.append(" %s: %s;" % (attr, value)) rv.append("}") rv.append("") + + # add css files that are appended to self.files + for css_file in self.css_files: + rv.append(f"") rv.append("") rv.append("") diff --git a/lobster/io.py b/lobster/io.py index 22eb7292..66b0d1b8 100644 --- a/lobster/io.py +++ b/lobster/io.py @@ -101,6 +101,7 @@ def lobster_read(mh, filename, level, items, source_info=None): "version %u for schema %s is not supported" % (data["version"], data["schema"])) + duplicate_items = [] # Convert to items, and integrate into symbol table for raw in data["data"]: if data["schema"] == "lobster-req-trace": @@ -124,10 +125,18 @@ def lobster_read(mh, filename, level, items, source_info=None): if all(filter_conditions): if item.tag.key() in items: - mh.error(item.location, - "duplicate definition of %s, " - "previously defined at %s" % - (item.tag.key(), - items[item.tag.key()].location.to_string())) - - items[item.tag.key()] = item + # 'duplicate definition' errors are fatal, but the user wants to see all + # of them. So store the affected items in a list first, and create + # errors later. + duplicate_items.append(item) + else: + items[item.tag.key()] = item + + if duplicate_items: + for counter, item in enumerate(duplicate_items, start=1): + mh.error( + item.location, + f"duplicate definition of {item.tag.key()}, " + f"previously defined at {items[item.tag.key()].location.to_string()}", + fatal=(counter == len(duplicate_items)), + ) diff --git a/lobster/report.py b/lobster/report.py index c27e3510..037cd563 100644 --- a/lobster/report.py +++ b/lobster/report.py @@ -174,7 +174,7 @@ def load_report(self, filename): def compute_items_and_coverage_for_items(self, data): """ - Function calcuates items and coverage for the items + Function calculates items and coverage for the items Parameters ---------- data - contents of lobster json file. diff --git a/lobster/tool.py b/lobster/tool.py index a9e42869..435f3811 100644 --- a/lobster/tool.py +++ b/lobster/tool.py @@ -25,7 +25,7 @@ from abc import ABCMeta, abstractmethod from functools import partial -from lobster.version import FULL_NAME +from lobster.version import FULL_NAME, get_version from lobster.errors import Message_Handler from lobster.location import File_Reference from lobster.items import Requirement, Implementation, Activity @@ -94,6 +94,7 @@ def __init__(self, name, description, extensions, official): self.add_argument = self.g_tool.add_argument + @get_version def process_commandline_options(self): options = self.ap.parse_args() diff --git a/lobster/tools/codebeamer/codebeamer.py b/lobster/tools/codebeamer/codebeamer.py index dfca5949..fd2ad674 100755 --- a/lobster/tools/codebeamer/codebeamer.py +++ b/lobster/tools/codebeamer/codebeamer.py @@ -49,6 +49,7 @@ from lobster.location import Codebeamer_Reference from lobster.errors import Message_Handler, LOBSTER_Error from lobster.io import lobster_read, lobster_write +from lobster.version import get_version TOKEN = 'token' REFERENCES = 'references' @@ -403,10 +404,12 @@ def parse_cb_config(file_name): return json_config +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: codebeamer_req.Dummy_Requirement - ap = argparse.ArgumentParser() - modes = ap.add_mutually_exclusive_group(required=True) modes.add_argument("--import-tagged", metavar="LOBSTER_FILE", diff --git a/lobster/tools/core/ci_report/ci_report.py b/lobster/tools/core/ci_report/ci_report.py index 996e1c23..1d951efd 100755 --- a/lobster/tools/core/ci_report/ci_report.py +++ b/lobster/tools/core/ci_report/ci_report.py @@ -23,11 +23,15 @@ from lobster.report import Report from lobster.items import Tracing_Status +from lobster.version import get_version +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_ci_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("lobster_report", nargs="?", default="report.lobster") diff --git a/lobster/tools/core/html_report/assets/html_report.css b/lobster/tools/core/html_report/assets/html_report.css new file mode 100644 index 00000000..a505901f --- /dev/null +++ b/lobster/tools/core/html_report/assets/html_report.css @@ -0,0 +1,139 @@ +.button { + background-color: #818589; + border: none; + border-radius: 5px; + color: white; + padding: 12px 25px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 14px; + margin: 4px 2px; + cursor: pointer +} + +.button active:before { + content: ; + position: absolute; + left: 0; + top: 0; + display: inline-block; + width: 0; + height: 0; + border-style: solid; + border-width: 15px 15px 0 0; + border-color: #333 transparent transparent transparent +} + +.buttonActive.button { + text-decoration: none; + border: 5px solid #000000 +} + +.buttonOK { + background-color: #04AA6D; + color: white; + border: 2px solid #04AA6D; + border-radius: 5px +} + +.buttonOK:hover { + background-color: #026641; + color: white; + border: 2px solid #026641 +} + +.buttonActive.buttonOK { + text-decoration: none; + border: 5px solid #026641 +} + +.buttonPartial { + background-color: #17a2b8; + color: white; + border: 2px solid #17a2b8; + border-radius: 5px +} + +.buttonPartial:hover { + background-color: #0e616e; + color: white; + border: 2px solid #0e616e +} + +.buttonActive.buttonPartial { + text-decoration: none; + border: 5px solid #0e616e +} + +.buttonMissing { + background-color: #f44336; + color: white; + border: 2px solid #f44336; + border-radius: 5px +} + +.buttonMissing:hover { + background-color: #a91409; + color: white; + border: 2px solid #a91409 +} + +.buttonActive.buttonMissing { + text-decoration: none; + border: 5px solid #a91409 +} + +.buttonJustified { + background-color: #6c757d; + color: white; + border: 2px solid #6c757d; + border-radius: 5px +} + +.buttonJustified:hover { + background-color: #41464b; + color: white; + border: 2px solid #41464b +} + +.buttonActive.buttonJustified { + text-decoration: none; + border: 5px solid #41464b +} + +.buttonWarning { + background-color: #ffbf00; + color: white; + border: 2px solid #ffbf00; + border-radius: 5px +} + +.buttonWarning:hover { + background-color: #997300; + color: white; + border: 2px solid #997300 +} + +.buttonActive.buttonWarning { + text-decoration: none; + border: 5px solid #997300 +} + +.buttonBlue { + background-color: #0000ff; + color: white; + border: 2px solid #0000ff; + border-radius: 5px +} + +.buttonBlue:hover { + background-color: #000099; + color: white; + border: 2px solid #000099 +} + +.buttonActive.buttonBlue { + text-decoration: none; + border: 5px solid #000099 +} diff --git a/lobster/tools/core/html_report/assets/html_report.js b/lobster/tools/core/html_report/assets/html_report.js new file mode 100644 index 00000000..b2f5a874 --- /dev/null +++ b/lobster/tools/core/html_report/assets/html_report.js @@ -0,0 +1,87 @@ +function buttonFilter(filter) { + var elms = document.getElementsByTagName("div"); + var issue_elms = document.getElementsByClassName("issue"); + for (i = 0; i < elms.length; i++) { + if (elms[i].id.startsWith("item-")) { + console.log("elms[i].className ", elms[i].className) + if (filter == "all") { + elms[i].style.display = "block"; + } else if (elms[i].className == "item-" + filter) { + elms[i].style.display = "block"; + } else { + elms[i].style.display = "none"; + } + } + } + // filter the issues list based on the issue filter button clicked + for (i = 0; i < issue_elms.length; i++) { + console.log("log ", issue_elms[i].className) + if (filter == "all") { + issue_elms[i].style.display = "list-item"; + } else if (issue_elms[i].className == "issue issue-" + filter) { + issue_elms[i].style.display = "list-item"; + } else { + issue_elms[i].style.display = "none"; + } + } + activeButton(filter); + //call the search filering which could have been overwritten by the current filtering + searchItem(); +} + + +function activeButton(filter) { + var elms = document.getElementsByTagName("button"); + console.log("the click buitton is " + filter); + for (i = 0; i < elms.length; i++) { + if (elms[i].className.includes("buttonActive")) { + console.log("elem active found : " + elms[i].className); + elms[i].className = elms[i].className.replace("buttonActive", ""); + } else if (elms[i].className.toLowerCase().includes("button" + filter.toLowerCase())) { + console.log("elem to be activated found : " + elms[i].className); + elms[i].className = elms[i].className + " buttonActive"; + } + } +} + + +function ToggleIssues() { + var div_issue = document.getElementById("issues-section"); + if (div_issue.style.display == "block" || div_issue.style.display == "") { + div_issue.style.display = "none"; + document.getElementById("BtnToggleIssue").innerHTML = "Show Issues"; + document.getElementById("BtnToggleIssue").className = document.getElementById("BtnToggleIssue").className + " buttonActive"; + } else { + div_issue.style = 'display: block; flex-direction: column; height: 200px;' + + 'overflow:auto;'; + document.getElementById("BtnToggleIssue").innerHTML = "Hide Issues"; + document.getElementById("BtnToggleIssue").className = document.getElementById("BtnToggleIssue").className.replace("buttonActive", ""); + } +} + + +function searchItem() { + var input = document.getElementById('search').value + input = input.toLowerCase(); + + var divs = document.getElementsByClassName('item-name'); + for (i = 0; i < divs.length; i++) { + var title = divs[i].parentNode.getAttribute("title"); + // get requirement name: 2nd part when we cut the long string with /svg + var reqname = divs[i].innerHTML.toLowerCase().split("").pop(); + reqname = reqname.split(" ").pop(); + if (reqname.includes(input)) { + // the search pattern has been found, if this elem has the title "hidden-not-matching", put it back to diplayed + if (title) { + if (title.startsWith("hidden-not-matching")) { + divs[i].parentNode.style.display = "block"; + } + } + divs[i].parentNode.setAttribute("title", "matching-" + input) + } else { + // not maching, we hide + divs[i].parentNode.setAttribute("title", "hidden-not-matching") + divs[i].parentNode.style.display = "none"; + } + } +} diff --git a/lobster/tools/core/html_report/html_report.py b/lobster/tools/core/html_report/html_report.py index 400e0c6b..c40a9501 100755 --- a/lobster/tools/core/html_report/html_report.py +++ b/lobster/tools/core/html_report/html_report.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU Affero General Public # License along with this program. If not, see # . - import os.path import argparse import html @@ -32,6 +31,7 @@ Codebeamer_Reference) from lobster.items import (Tracing_Status, Item, Requirement, Implementation, Activity) +from lobster.version import get_version LOBSTER_GH = "https://github.com/bmw-software-engineering/lobster" @@ -373,8 +373,43 @@ def write_html(fd, report, dot, high_contrast): print("> please install Graphviz (https://graphviz.org)") doc.add_line('') + ### Filtering + doc.add_heading(2, "Filtering", "filtering-options") + doc.add_heading(3, "Item Filters") + doc.add_line('
') + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + + doc.add_line('') + doc.add_line("
") + + doc.add_heading(3, "Show Issues") + doc.add_line('
') + doc.add_line('') + doc.add_line('
') + + doc.add_heading(3, "Search", "search") + doc.add_line('') + doc.add_line('") ### Report file_heading = None @@ -446,6 +483,19 @@ def write_html(fd, report, dot, high_contrast): write_item_box_end(doc) else: doc.add_line("No items recorded at this level.") + # Closing tag for id #search-sec-id + doc.add_line("") + + # Add the css from assets + dir_path = os.path.dirname(os.path.abspath(__file__)) + file_path = dir_path + "/assets/html_report.css" + doc.css_files.append(file_path) + + # Add javascript from assets/html_report.js file + dir_path = os.path.dirname(os.path.abspath(__file__)) + file_path = dir_path + "/assets/html_report.js" + with open(file_path, "r", encoding="UTF-8") as scripts: + doc.scripts.append("".join(scripts.readlines())) ### STM # doc.add_heading(2, "Software traceability matrix", "matrix") @@ -454,9 +504,12 @@ def write_html(fd, report, dot, high_contrast): fd.write(doc.render() + "\n") +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_html_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("lobster_report", nargs="?", default="report.lobster") diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index 10c99ccd..dcd278c6 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -26,6 +26,7 @@ from lobster.report import Report from lobster.location import File_Reference, Github_Reference +from lobster.version import get_version class Parse_Error(Exception): @@ -141,9 +142,12 @@ def parse_git_root(cfg): return gh_root +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_online_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("lobster_report", nargs="?", default="report.lobster") diff --git a/lobster/tools/core/report/report.py b/lobster/tools/core/report/report.py index 96618683..48f0b943 100755 --- a/lobster/tools/core/report/report.py +++ b/lobster/tools/core/report/report.py @@ -24,11 +24,15 @@ from lobster.exceptions import LOBSTER_Exception from lobster.errors import LOBSTER_Error from lobster.report import Report +from lobster.version import get_version +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: core_report_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("--lobster-config", metavar="FILE", default="lobster.conf") diff --git a/lobster/tools/core/report/requirements.trlc b/lobster/tools/core/report/requirements.trlc index 220f6e16..1de78715 100644 --- a/lobster/tools/core/report/requirements.trlc +++ b/lobster/tools/core/report/requirements.trlc @@ -1,10 +1,10 @@ package core_report_req import req -req.Software_Requirement Dummy_Requirement { +req.System_Requirement Report_Input_Duplicate_Definition { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If any of the source files given in the lobster configuration file contains a tag + more than once, then the tool must exit with code 1. ''' } diff --git a/lobster/tools/cpp/cpp.py b/lobster/tools/cpp/cpp.py index 511d49a2..4820bf96 100755 --- a/lobster/tools/cpp/cpp.py +++ b/lobster/tools/cpp/cpp.py @@ -26,7 +26,7 @@ from lobster.items import Tracing_Tag, Implementation from lobster.location import File_Reference from lobster.io import lobster_write - +from lobster.version import get_version FILE_LINE_PATTERN = r"(.*):(\d+):\d+:" KIND_PATTERN = r"(function|main function|method)" @@ -35,8 +35,8 @@ SUFFIX = r"\[lobster-tracing\]$" RE_NOTAGS = (PREFIX + " " + - "%s %s has no tracing tags" % (KIND_PATTERN, - NAME_PATTERN) + + r"%s %s has no tracing tags" % (KIND_PATTERN, + NAME_PATTERN) + " " + SUFFIX) RE_TAGS = (PREFIX + " " + r"%s %s traces to +(.+) +" % (KIND_PATTERN, @@ -48,9 +48,12 @@ SUFFIX) +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: cpp_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") @@ -59,6 +62,14 @@ def main(): metavar="FILE", help=("use the specified clang-tidy; by default we" " pick the one on PATH")) + ap.add_argument("--compile-commands", + metavar="FILE", + default=None, + help=("Path to the compile command database for all targets for " + "'clang tidy', or none to use the default behavior of " + "'clang tidy'. This is equal to calling 'clang tidy' " + "directly with its '-p' option. Refer to its official " + "documentation for more details.")) ap.add_argument("--out", default=None, help=("write output to this file; otherwise output to" @@ -83,13 +94,17 @@ def main(): # Test if the clang-tidy can be used - rv = subprocess.run([os.path.expanduser(options.clang_tidy), - "-checks=-*,lobster-tracing", - "--list-checks"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="UTF-8", - check=False) + rv = subprocess.run( + [ + os.path.expanduser(options.clang_tidy), + "-checks=-*,lobster-tracing", + "--list-checks", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", + check=False, + ) if "No checks enabled." in rv.stderr: print("The provided clang-tidy does include the lobster-tracing check") @@ -99,13 +114,23 @@ def main(): "correct binary using the --clang-tidy flag") return 1 - rv = subprocess.run([os.path.expanduser(options.clang_tidy), - "-checks=-*,lobster-tracing"] + - file_list, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="UTF-8", - check=False) + subprocess_args = [ + os.path.expanduser(options.clang_tidy), + "-checks=-*,lobster-tracing", + ] + if options.compile_commands: + subprocess_args.append("-p") + subprocess_args.append(options.compile_commands) + + subprocess_args += file_list + + rv = subprocess.run( + subprocess_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", + check=False, + ) if rv.returncode != 0: found_reason = False diff --git a/lobster/tools/cpptest/cpptest.py b/lobster/tools/cpptest/cpptest.py index e14b209d..da292a6f 100644 --- a/lobster/tools/cpptest/cpptest.py +++ b/lobster/tools/cpptest/cpptest.py @@ -30,6 +30,7 @@ from lobster.tools.cpptest.parser.constants import Constants from lobster.tools.cpptest.parser.requirements_parser import \ ParserForRequirements +from lobster.version import get_version OUTPUT = "output" CODEBEAMER_URL = "codebeamer_url" @@ -220,7 +221,6 @@ def create_lobster_items_output_dict_from_test_cases( The lobster items dictionary for the given test cases grouped by configured output. """ - prefix = os.getcwd() lobster_items_output_dict = {} no_marker_output_file_name = '' @@ -239,7 +239,7 @@ def create_lobster_items_output_dict_from_test_cases( for test_case in test_case_list: function_name: str = test_case.suite_name - file_name = os.path.relpath(test_case.file_name, prefix) + file_name = os.path.abspath(test_case.file_name) line_nr = int(test_case.docu_start_line) function_uid = "%s:%s:%u" % (os.path.basename(file_name), function_name, @@ -367,6 +367,10 @@ def lobster_cpptest(file_dir_list: list, config_dict: dict): ) +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): """ Main function to parse arguments, read configuration diff --git a/lobster/tools/gtest/gtest.py b/lobster/tools/gtest/gtest.py index 5bbbe3c5..66b29316 100755 --- a/lobster/tools/gtest/gtest.py +++ b/lobster/tools/gtest/gtest.py @@ -25,11 +25,14 @@ from lobster.items import Tracing_Tag, Activity from lobster.location import Void_Reference, File_Reference from lobster.io import lobster_write +from lobster.version import get_version +ap = argparse.ArgumentParser() + +@get_version(ap) def main(): # lobster-trace: gtest_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") diff --git a/lobster/tools/json/requirements.trlc b/lobster/tools/json/requirements.trlc index 309de1f3..0ee8b944 100644 --- a/lobster/tools/json/requirements.trlc +++ b/lobster/tools/json/requirements.trlc @@ -1,16 +1,20 @@ package json_req import req -req.Software_Requirement Dummy_Requirement { +req.Software_Requirement Synthetic_Tag_Name { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is not given, then the tool shall create + a synthetic tag name based on the path of the JSON input file and an item counter. + + Note: The item counter does not need to be unique across files. + It shall be unique at least for each input file separately. ''' } -req.Software_Requirement Dummy_Requirement_Unit_Test { +req.Software_Requirement Name_Attribute { description = ''' - This is not really a requirement. It will be used only to generate a minimal tracing report for each tool. - It can be deleted as soon as all the tools get their real requirements. + If the command line option --name-attribute is given, then the tool shall + - use the value of the command line argument as JSON key of the JSON item + - and use that obtained value as LOBSTER item tag name. ''' } diff --git a/lobster/tools/python/python.py b/lobster/tools/python/python.py index 5d6cb123..06d4bc07 100755 --- a/lobster/tools/python/python.py +++ b/lobster/tools/python/python.py @@ -30,6 +30,7 @@ from lobster.items import Tracing_Tag, Implementation, Activity from lobster.location import File_Reference from lobster.io import lobster_write +from lobster.version import get_version LOBSTER_TRACE_PREFIX = "# lobster-trace: " LOBSTER_JUST_PREFIX = "# lobster-exclude: " @@ -443,9 +444,12 @@ def process_file(file_name, options): raise +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): # lobster-trace: python_req.Dummy_Requirement - ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+", metavar="FILE|DIR") diff --git a/lobster/tools/trlc/trlc.py b/lobster/tools/trlc/trlc.py index 6b00d8c9..8f72d869 100644 --- a/lobster/tools/trlc/trlc.py +++ b/lobster/tools/trlc/trlc.py @@ -33,6 +33,7 @@ from lobster.items import Tracing_Tag, Requirement from lobster.location import File_Reference from lobster.io import lobster_write +from lobster.version import get_version class Config_Parser(Parser_Base): @@ -362,8 +363,11 @@ def parse_directive(self): self.parse_tuple_type(n_typ) +ap = argparse.ArgumentParser() + + +@get_version(ap) def main(): - ap = argparse.ArgumentParser() ap.add_argument("--config-file", help=("name of lobster-trlc config file, " "by default %(default)s"), diff --git a/lobster/version.py b/lobster/version.py index 4080b3a0..b3584d2a 100644 --- a/lobster/version.py +++ b/lobster/version.py @@ -16,8 +16,10 @@ # You should have received a copy of the GNU Affero General Public # License along with this program. If not, see # . +import sys +from argparse import ArgumentParser -VERSION_TUPLE = (0, 9, 20) +VERSION_TUPLE = (0, 9, 21) VERSION_SUFFIX = "dev" LOBSTER_VERSION = ("%u.%u.%u" % VERSION_TUPLE) + ( @@ -25,3 +27,46 @@ ) FULL_NAME = "LOBSTER %s" % LOBSTER_VERSION + + +def get_version(obj): + """ + This decorator function is used on function wherever we are parsing + the command line arguments which then adds a version argument to the function. + If a version flag is passed to the command line arguments then the LOBSTER version + is printed. + Parameters + ---------- + obj - obj can be an ArgumentParser object or a Function object. + + Returns - Nothing + ------- + + """ + if isinstance(obj, ArgumentParser): + obj.add_argument("-v, --version", action="store_true", + default=None, + help="Get version for the tool") + + def version(func): + def execution(): + if (len(sys.argv) > 1 and + (sys.argv[1] == "--version" or sys.argv[1] == "-v")): + print(FULL_NAME) + return sys.exit(0) + else: + return func() + return execution + return version + else: + def version(func): + if not isinstance(obj, ArgumentParser): + func.ap.add_argument("-v, --version", action="store_true", + default=None, + help="Get version for the tool") + if (len(sys.argv) > 1 and + (sys.argv[1] == "--version" or sys.argv[1] == "-v")): + print(FULL_NAME) + return sys.exit(0) + return obj(func) + return version diff --git a/packages/lobster-core/setup.py b/packages/lobster-core/setup.py index af0db774..9df68841 100644 --- a/packages/lobster-core/setup.py +++ b/packages/lobster-core/setup.py @@ -52,6 +52,9 @@ "lobster.tools.core.html_report", "lobster.tools.core.online_report", "lobster.tools.core.report"], + package_data={ + "lobster.tools.core.html_report":["assets/*"] + }, install_requires=[], python_requires=">=3.7, <4", classifiers=[ diff --git a/packages/lobster-monolithic/setup.py b/packages/lobster-monolithic/setup.py index 4960fead..2b161414 100644 --- a/packages/lobster-monolithic/setup.py +++ b/packages/lobster-monolithic/setup.py @@ -48,6 +48,9 @@ project_urls=project_urls, license="GNU Affero General Public License v3", packages=setuptools.find_packages(), + package_data={ + "lobster.tools.core.html_report":["assets/*"] + }, install_requires=[ "miss-hit>=0.9.42", "requests>=2.22", diff --git a/packages/lobster-tool-cpp/README.md b/packages/lobster-tool-cpp/README.md index b7b1fa27..59c3f0b2 100644 --- a/packages/lobster-tool-cpp/README.md +++ b/packages/lobster-tool-cpp/README.md @@ -11,6 +11,8 @@ this to work you need to build [our clang-tidy fork](https://github.com/bmw-software-engineering/llvm-project) and place the `clang-tidy` binary somewhere on your PATH. +Instructions for clang-tidy [here](https://github.com/bmw-software-engineering/lobster/blob/main/documentation/user-manual.md#clang-tidy-file-generation). + This tool works using a custom clang-tidy checker `lobster-trace` which emits tracing information as clang checks. diff --git a/packages/lobster-tool-cpptest/README.md b/packages/lobster-tool-cpptest/README.md index a4124362..bd32c6e7 100644 --- a/packages/lobster-tool-cpptest/README.md +++ b/packages/lobster-tool-cpptest/README.md @@ -6,19 +6,21 @@ and requirements coverage, which is essential for meeting standards such as ISO 26262. This package contains a tool extract tracing tags from ISO C or C++ -source code. This tool is also extracting configurable markers/ test-types -from the provided comments in cpp files +source code. The tracing tags are identified by searching for configurable +markers in the comments of the source code. ## Tools -* `lobster-cpptest`: Extract requirements with dynamic refrences - from comments. +* `lobster-cpptest`: Extract requirements with specific references + from tests. ## Usage This tool supports C/C++ code. -For this you can provide some cpp file with these comments: +For this you have to provide a C/C++ test documentation with `markers`: + +`Markers` can be either `@requirement`, `@requiredby` or `@defect`. ```cpp /** @@ -29,9 +31,23 @@ For this you can provide some cpp file with these comments: */ TEST(RequirementTagTest1, RequirementsAsMultipleComments) {} ``` -You can also provide a config-file which determines which markers -should be extracted in which files. In addition you have to provide -the codebeamer-url: +You have to provide a config-file which determines which `markers` should be extracted in which output-files. +The expected `kind` for each output-file should also be specified. + +* Note: If you want to extract the other tests with other `markers`, + you can use an empty list as `markers` value. Be aware in this case the tests do not have any references. + + +```config +{ + "markers": [], + "kind": "req" +} +``` + +In addition, you have to provide the codebeamer-url: + +`Kind` can be either `req`, `imp` or `act`. ```config { @@ -51,6 +67,8 @@ the codebeamer-url: } ``` +For more information about how to setup cpp and config files take a look at [manual-lobster_cpptest](../../documentation/manual-lobster_cpptest.md) + ## Copyright & License information diff --git a/packages/lobster-tool-trlc/README.md b/packages/lobster-tool-trlc/README.md index 35a8f5ab..d64d14d6 100644 --- a/packages/lobster-tool-trlc/README.md +++ b/packages/lobster-tool-trlc/README.md @@ -5,10 +5,6 @@ The **L**ightweight **O**pen **B**MW **S**oftware **T**raceability and requirements coverage, which is essential for meeting standards such as ISO 26262. -This package contains a tool to interface with the proprietary -requirements management tool -[Codebeamer](https://intland.com/codebeamer). - ## Configuration This tool is a bit more complex and you need to supply a config file, diff --git a/tests-integration/projects/basic/Makefile b/tests-integration/projects/basic/Makefile index e15f644e..a6f3f8ab 100644 --- a/tests-integration/projects/basic/Makefile +++ b/tests-integration/projects/basic/Makefile @@ -21,7 +21,7 @@ cppcode.lobster: foo.h foo.cpp --out="cppcode.lobster" --clang-tidy $(CLANG_TIDY) gtests.lobster: foo.h foo.cpp test.cpp - @bazel test foo_test --cxxopt='-std=c++14' + @bazel test foo_test --cxxopt='-std=c++14' --enable_workspace @lobster-gtest $(LOBSTER_ROOT)/bazel-out/*/testlogs/$(THIS_TEST) \ --out="gtests.lobster" sed -i s/$(THIS_TEST_ESCAPED)\\///g gtests.lobster diff --git a/tests-system/.gitignore b/tests-system/.gitignore new file mode 100644 index 00000000..78f221c8 --- /dev/null +++ b/tests-system/.gitignore @@ -0,0 +1 @@ +!**/expected-output/* diff --git a/tests-system/Makefile b/tests-system/Makefile new file mode 100644 index 00000000..a99ec8cb --- /dev/null +++ b/tests-system/Makefile @@ -0,0 +1,4 @@ +PYTHON = python + +run-tool-tests: + @$(PYTHON) ./run_tool_tests.py ./$(TOOL) diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/exit-code.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt new file mode 100644 index 00000000..17c7c77d --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/expected-output/stdout.txt @@ -0,0 +1,4 @@ +DuplicateDefinitionTest.py:2: lobster error: duplicate definition of hello DuplicateDefinitionTest, previously defined at DuplicateDefinitionTest.py:1 +DuplicateDefinitionTest.py:3: lobster error: duplicate definition of hello DuplicateDefinitionTest, previously defined at DuplicateDefinitionTest.py:1 +DuplicateDefinitionTest.py:4: lobster error: duplicate definition of hello DuplicateDefinitionTest, previously defined at DuplicateDefinitionTest.py:1 +lobster: aborting due to earlier errors. \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt new file mode 100644 index 00000000..d2688e44 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/args.txt @@ -0,0 +1 @@ +--lobster-config="lobster.config" \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config new file mode 100644 index 00000000..518720d1 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/multiple-duplicates/input/lobster.config @@ -0,0 +1,3 @@ +implementation "Something" { + source: "file.lobster"; +} diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-output-file/.gitkeep b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/stdout.txt similarity index 100% rename from tests-system/lobster-core/report/rbt-output-file/.gitkeep rename to tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/expected-output/stdout.txt diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt new file mode 100644 index 00000000..d2688e44 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/args.txt @@ -0,0 +1 @@ +--lobster-config="lobster.config" \ No newline at end of file diff --git a/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config new file mode 100644 index 00000000..518720d1 --- /dev/null +++ b/tests-system/lobster-core/report/rbt-report-input-duplicate-definition/no-duplicates/input/lobster.config @@ -0,0 +1,3 @@ +implementation "Something" { + source: "file.lobster"; +} diff --git a/tests-system/lobster-json/Makefile b/tests-system/lobster-json/Makefile deleted file mode 100644 index 55f3e314..00000000 --- a/tests-system/lobster-json/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -TOOL=../../lobster-json - -TARGETS=$(addsuffix .output, $(basename $(wildcard **/*.input) $(wildcard *.input))) - -all: $(TARGETS) - -%.output: %.input - @tail -n +2 $< > $*.json - @touch $*.lobster - -@coverage run -p --rcfile=../../coverage.cfg --branch \ - --data-file ../../.coverage \ - $(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 - @echo "==========" >> $@ - @cat $*.lobster >> $@ - @rm $*.json $*.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic.output b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster similarity index 97% rename from tests-system/lobster-json/basic.output rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster index 3157c93e..79d8d7f3 100644 --- a/tests-system/lobster-json/basic.output +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/output.lobster @@ -1,5 +1,3 @@ -lobster-json: wrote 6 items to basic.lobster -========== { "data": [ { diff --git a/tests-system/lobster-json/rbt-output-file/.gitkeep b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-json/rbt-output-file/.gitkeep rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt new file mode 100644 index 00000000..44e3c250 --- /dev/null +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/args.txt @@ -0,0 +1,4 @@ +--single +--tag-attribute=tags +--name-attribute=name +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.input b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json similarity index 95% rename from tests-system/lobster-json/basic2.input rename to tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json index f57ea48e..7bb52fe8 100644 --- a/tests-system/lobster-json/basic2.input +++ b/tests-system/lobster-json/rbt-name-attribute/attribute-given/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/README.md b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md new file mode 100644 index 00000000..944ef261 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/README.md @@ -0,0 +1,22 @@ +If the `--name-attribute` option is not specified, +then the tag name must be constructed based on the following two values: +- path of the json file +- item counter + +# Test case "flat-input-structure" +- No files or directories are specified. + The tool is expected to search for input files in the current working directory. +- `--name-attribute` is not given. + The tool is expected to construct LOBSTER item tags by taking the file name and + appending an integer counter, starting at 1. + This is implemented inside function `syn_test_name`, and it must handle all file + paths as relative paths, because no input files were given explicitly. +- `--out` is specified. + The tool is expected to use that path name for the output file. +- `--single` is specified to make the tool output predictable. + +# Test case "nested-input-structure" +This is the same as the above test, except that its input file is nested in a +sub-sub-directory. +The tool is expected to take the names of the sub-directories into account when +generating te tag name. diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/basic2.output b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster similarity index 72% rename from tests-system/lobster-json/basic2.output rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster index 287d5a13..d0331f77 100644 --- a/tests-system/lobster-json/basic2.output +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/output.lobster @@ -1,16 +1,14 @@ -lobster-json: wrote 6 items to basic2.lobster -========== { "data": [ { - "tag": "json ./basic2.json:basic2.1", + "tag": "json ./basic.json:basic.1", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.1", + "name": "./basic.json:basic.1", "messages": [], "just_up": [], "just_down": [], @@ -23,14 +21,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.2", + "tag": "json ./basic.json:basic.2", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.2", + "name": "./basic.json:basic.2", "messages": [], "just_up": [], "just_down": [], @@ -40,14 +38,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.3", + "tag": "json ./basic.json:basic.3", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.3", + "name": "./basic.json:basic.3", "messages": [], "just_up": [], "just_down": [], @@ -61,14 +59,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.4", + "tag": "json ./basic.json:basic.4", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.4", + "name": "./basic.json:basic.4", "messages": [], "just_up": [], "just_down": [], @@ -78,14 +76,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.5", + "tag": "json ./basic.json:basic.5", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.5", + "name": "./basic.json:basic.5", "messages": [], "just_up": [], "just_down": [], @@ -98,14 +96,14 @@ lobster-json: wrote 6 items to basic2.lobster "status": null }, { - "tag": "json ./basic2.json:basic2.6", + "tag": "json ./basic.json:basic.6", "location": { "kind": "file", - "file": "./basic2.json", + "file": "./basic.json", "line": null, "column": null }, - "name": "./basic2.json:basic2.6", + "name": "./basic.json:basic.6", "messages": [], "just_up": [], "just_down": [], diff --git a/tests-system/lobster-python/rbt-output-file/.gitkeep b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-python/rbt-output-file/.gitkeep rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stderr.txt diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/basic.input b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json similarity index 92% rename from tests-system/lobster-json/basic.input rename to tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json index fa82b8a4..7bb52fe8 100644 --- a/tests-system/lobster-json/basic.input +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/flat-input-structure/input/basic.json @@ -1,4 +1,3 @@ -// --tag-attribute=tags --name-attribute=name [ { "name" : "Test_1", diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster new file mode 100644 index 00000000..8571ad42 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/output.lobster @@ -0,0 +1,119 @@ +{ + "data": [ + { + "tag": "json ./one/two/basic.json:one.two.basic.1", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.1", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req1" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.2", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.2", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.3", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.3", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req2", + "req example.req3" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.4", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.4", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.5", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.5", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req4" + ], + "framework": "JSON", + "kind": "Test Vector", + "status": null + }, + { + "tag": "json ./one/two/basic.json:one.two.basic.6", + "location": { + "kind": "file", + "file": "./one/two/basic.json", + "line": null, + "column": null + }, + "name": "./one/two/basic.json:one.two.basic.6", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "JSON", + "kind": "Test Vector", + "status": null + } + ], + "generator": "lobster-json", + "schema": "lobster-act-trace", + "version": 3 +} diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt new file mode 100644 index 00000000..bec63021 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/expected-output/stdout.txt @@ -0,0 +1 @@ +lobster-json: wrote 6 items to output.lobster diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt new file mode 100644 index 00000000..93216391 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/args.txt @@ -0,0 +1,3 @@ +--single +--tag-attribute=tags +--out=output.lobster \ No newline at end of file diff --git a/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json new file mode 100644 index 00000000..7bb52fe8 --- /dev/null +++ b/tests-system/lobster-json/rbt-synthetic-tag-name/nested-input-structure/input/one/two/basic.json @@ -0,0 +1,38 @@ +[ + { + "name" : "Test_1", + "tags" : ["example.req1"], + "inputs" : [1, 2], + "expect" : 3 + }, + { + "name" : "Test_2", + "tags" : [], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_3", + "tags" : ["example.req2", + "example.req3"], + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_4", + "tags" : null, + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_5", + "tags" : "example.req4", + "inputs" : [1, 0], + "expect" : 1 + }, + { + "name" : "Test_6", + "inputs" : [1, 0], + "expect" : 1 + } +] diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster new file mode 100644 index 00000000..98ad4f23 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/basic.lobster @@ -0,0 +1,63 @@ +{ + "data": [ + { + "tag": "python basic.trlc_reference", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 3, + "column": null + }, + "name": "basic.trlc_reference", + "messages": [], + "just_up": [ + "helper function" + ], + "just_down": [], + "just_global": [], + "language": "Python", + "kind": "Function" + }, + { + "tag": "python basic.Example.helper_function", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 11, + "column": null + }, + "name": "basic.Example.helper_function", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req_nor" + ], + "language": "Python", + "kind": "Method" + }, + { + "tag": "python basic.Example.nor", + "location": { + "kind": "file", + "file": "./basic.py", + "line": 15, + "column": null + }, + "name": "basic.Example.nor", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [ + "req example.req_nor" + ], + "language": "Python", + "kind": "Method" + } + ], + "generator": "lobster_python", + "schema": "lobster-imp-trace", + "version": 3 +} diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stderr.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt new file mode 100644 index 00000000..31e0cb57 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/expected-output/stdout.txt @@ -0,0 +1 @@ +Written output for 3 items to basic.lobster diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt b/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt new file mode 100644 index 00000000..3903b871 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/input/args.txt @@ -0,0 +1,5 @@ +. +--out=basic.lobster +--parse-decorator +trlc_reference +requirement diff --git a/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py b/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py new file mode 100644 index 00000000..6d3ff093 --- /dev/null +++ b/tests-system/lobster-python/rbt-dummy/default-scenario/input/basic.py @@ -0,0 +1,20 @@ +import potatolib + +def trlc_reference(requirement): + # lobster-exclude: helper function + def decorator(obj): + return obj + return decorator + +class Example: + @trlc_reference(requirement="example.req_nor") + def helper_function(a, b): + # potato + return a or b + + def nor(a, b): + # lobster-trace: example.req_nor + assert isinstance(a, bool) + assert isinstance(b, bool) + + return not helper_function(a, b) diff --git a/tests-system/lobster-trlc/Makefile b/tests-system/lobster-trlc/Makefile deleted file mode 100644 index 243eecff..00000000 --- a/tests-system/lobster-trlc/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -PYTHON = python - -lobster-trlc-system-tests: - @echo "Running lobster-trlc system tests..." - @$(PYTHON) run_tool_tests.py diff --git a/tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt b/tests-system/lobster-trlc/rbt-output-file/default-scenario/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-trlc/run_tool_tests.py b/tests-system/run_tool_tests.py similarity index 77% rename from tests-system/lobster-trlc/run_tool_tests.py rename to tests-system/run_tool_tests.py index fb525e8b..8a133f76 100644 --- a/tests-system/lobster-trlc/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -1,5 +1,5 @@ import sys -from os import scandir, DirEntry +from os import scandir, DirEntry, remove from os.path import basename, dirname, join from pathlib import Path from subprocess import CompletedProcess, PIPE, run @@ -15,6 +15,7 @@ class TestSetup: _EXPECTED_OUTPUT_FOLDER_NAME = "expected-output" _EXIT_CODE_FILE_NAME = "exit-code.txt" _EXPECTED_STDOUT_FILE_NAME = "stdout.txt" + _EXPECTED_STDERR_FILE_NAME = "stderr.txt" def __init__(self, test_case_path: str): """Constructor @@ -52,7 +53,8 @@ def _get_expected_lobster_output_file_name(self) -> str: if (not dir_entry.is_dir()) and dir_entry.name.endswith(".lobster"): return dir_entry.name raise ValueError( - "Invalid test setup: No *.lobster file found in expected output folder!", + f"Invalid test setup: No *.lobster file found in " + f"{self.get_expected_output_path()}!", ) @staticmethod @@ -83,9 +85,9 @@ def _get_args(self) -> List[str]: test) from the corresponding test setup file""" file = join(self._input_folder, self._ARGS_FILE_NAME) with open(file, "r", encoding="UTF-8") as file: - return file.readlines() + return [argument.strip() for argument in file.readlines()] - def get_expected_cmd_output(self) -> str: + def get_expected_stdout(self) -> str: """Reads the expected command line output (for stdout) from the corresponding test setup file""" cmd_file = join( @@ -95,6 +97,16 @@ def get_expected_cmd_output(self) -> str: with open(cmd_file, "r", encoding="UTF-8") as file: return file.read() + def get_expected_stderr(self) -> str: + """Reads the expected command line output (for stderr) from the corresponding + test setup file""" + errout_file = join( + self.get_expected_output_path(), + self._EXPECTED_STDERR_FILE_NAME, + ) + with open(errout_file, "r", encoding="UTF-8") as file: + return file.read() + @property def input_folder(self) -> str: """Returns the path containing the input data for the test""" @@ -112,13 +124,25 @@ def _get_expected_exit_code(self) -> int: def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: - """Runs the tool system test. + """Runs the tool system test using the coverage command. The tool will be executed such that its current working directory is equal to the "input" folder.""" print(f"Starting system test '{setup.name}' with arguments {setup.args} " \ - f"for tool '{tool}'.") + f"for tool '{tool}' with coverage.") + root_directory = Path(__file__).resolve().parents[1] + coverage_config_path = root_directory / "coverage.cfg" + coverage_data_path = root_directory / ".coverage" + + coverage_command = [ + "coverage", "run", "-p", + f"--rcfile={coverage_config_path}", + "--branch", + f"--data-file={coverage_data_path}", + tool, *setup.args + ] + completed_process = run( - [sys.executable, tool, *setup.args], + coverage_command, stdout=PIPE, stderr=PIPE, encoding="UTF-8", @@ -128,12 +152,31 @@ def _run_test(setup: TestSetup, tool: str) -> CompletedProcess: return completed_process +def _compare_cmd_output(name: str, expected: str, actual: str) -> bool: + if expected != actual: + print(f"Actual {name} is (length: {len(actual)} chars):\n{actual}") + print(f"Expected {name} is (length: {len(expected)} chars):\n{expected}") + return False + return True + + def _compare_results(setup: TestSetup, completed_process: CompletedProcess): assert setup.expected_exit_code == completed_process.returncode, \ f"{setup.name}: Expected exit code is {setup.expected_exit_code}, " \ f"actual is {completed_process.returncode}!" - assert setup.get_expected_cmd_output() == completed_process.stdout, \ - "Command line output is different!" + + assert _compare_cmd_output( + name="STDOUT", + expected=setup.get_expected_stdout(), + actual=completed_process.stdout, + ), "Command line output for STDOUT is different!" + + assert _compare_cmd_output( + name="STDERR", + expected=setup.get_expected_stderr(), + actual=completed_process.stderr, + ), "Command line output for STDERR is different!" + expected = join( setup.get_expected_output_path(), setup.expected_lobster_output_file_name, @@ -172,6 +215,16 @@ def _get_directories( yield dir_entry +def _delete_generated_files(setup: TestSetup): + """Deletes the *.lobster file that has been generated by the test""" + generated = join( + setup.input_folder, + setup.expected_lobster_output_file_name, + ) + print(f"DELETING {generated}") + remove(generated) + + def _run_tests(directory: str, tool: str) -> int: """Runs all system tests in the given folder for the specified tool. @@ -184,12 +237,15 @@ def _run_tests(directory: str, tool: str) -> int: if not tool: raise ValueError("No tool specified!") + counter = 0 for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX): for test_case_dir_entry in _get_directories(rbt_dir_entry.path): test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) _compare_results(test_setup, completed_process) - print(f"All system tests finished successfully for {tool}.") + _delete_generated_files(test_setup) + counter += 1 + print(f"{counter} system tests finished successfully for {tool}.") # TODO: the current implementation is not consistent with respect to return codes. # The tests use assertion statements to indicate failures, but here we use an @@ -212,7 +268,9 @@ def _get_tool(test_dir: str) -> str: if __name__ == "__main__": - test_directory = dirname(Path(__file__).resolve()) + test_directory = Path(sys.argv[1]).resolve() + print(f"Starting system tests on folder '{test_directory}'") + sys.exit( _run_tests( test_directory, diff --git a/tests-unit/lobster-cpptest/test_cpptest.py b/tests-unit/lobster-cpptest/test_cpptest.py index 9b7736ef..373082ce 100644 --- a/tests-unit/lobster-cpptest/test_cpptest.py +++ b/tests-unit/lobster-cpptest/test_cpptest.py @@ -1,4 +1,5 @@ import os +import json import unittest from os.path import dirname from pathlib import Path @@ -240,6 +241,15 @@ def test_single_file(self): file_exists = os.path.exists(self.output_file_name) self.assertTrue(file_exists) + # Open and read the JSON output file to validate all file paths are absolute + with open(self.output_file_name, 'r') as output_file: + data = json.load(output_file) + tag_list = data.get('data') + + for tag in tag_list: + file_name = tag.get('location').get('file') + self.assertTrue(os.path.isabs(file_name)) + def test_single_directory(self): file_dir_list = [self.test_data_dir] @@ -333,6 +343,8 @@ def test_separate_output_config(self): for lobster_item in unit_test_lobster_items: self.assertIsNotNone(lobster_item) self.assertIsInstance(lobster_item, dict) + file_name = lobster_item.get('location').get('file') + self.assertTrue(os.path.isabs(file_name)) tag = lobster_item.get('tag') refs = lobster_item.get('refs') self.assertIsInstance(refs, list) @@ -361,6 +373,8 @@ def test_separate_output_config(self): for lobster_item in component_test_lobster_items: self.assertIsNotNone(lobster_item) self.assertIsInstance(lobster_item, dict) + file_name = lobster_item.get('location').get('file') + self.assertTrue(os.path.isabs(file_name)) tag = lobster_item.get('tag') refs = lobster_item.get('refs') self.assertIsInstance(refs, list)