diff --git a/CHANGELOG.md b/CHANGELOG.md index 9946023b..a9284e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ ### 0.9.17-dev +* Adds a simplified version of the `lobster-cpp` tool, called + `lobster-cpp_raw`. This tool does not have any dependendies unlike + `lobster-cpp`. It is also less accurate, but might be useful in + some cases. ### 0.9.16 diff --git a/Makefile b/Makefile index fa4eccc7..0e5b51af 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ packages: make -C packages/lobster-tool-trlc make -C packages/lobster-tool-codebeamer make -C packages/lobster-tool-cpp + make -C packages/lobster-tool-cpp_raw make -C packages/lobster-tool-gtest make -C packages/lobster-tool-json make -C packages/lobster-tool-python diff --git a/README.md b/README.md index bb39cad8..5ca852cf 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ $ pip3 install bmw-lobster The following requirements frameworks are supported: -* [TRLC](work-in-progress) (only some use cases supported right now) +* [TRLC](packages/lobster-tool-trlc/README.md) (only some use cases supported right now) * [Codebeamer](packages/lobster-tool-codebeamer/README.md) (only some use cases supported right now) @@ -61,6 +61,7 @@ The individual packages that `bmw-lobster` depends on are: * `bmw-lobster-core` the core API and various report generators. All other tools depend on this. * `bmw-lobster-tool-cpp` (for C/C++ code) +* `bmw-lobster-tool-cpp_raw` (for C/C++ code) -> simple version, without the clang-tidy hack * `bmw-lobster-tool-gtest` (for GoogleTest tests) * `bmw-lobster-tool-python` (for Python3 code) * `bmw-lobster-tool-beamer` (for requirements in Codebeamer) diff --git a/lobster/tools/core/online_report.py b/lobster/tools/core/online_report.py index 0b6afdd0..24ce9dca 100755 --- a/lobster/tools/core/online_report.py +++ b/lobster/tools/core/online_report.py @@ -122,6 +122,7 @@ def main(): rel_path_from_root = os.path.relpath(item.location.filename, repo_root) + # pylint: disable=possibly-used-before-assignment actual_repo = gh_root actual_sha = options.commit actual_path = rel_path_from_root diff --git a/lobster/tools/cpp/cpp.py b/lobster/tools/cpp/cpp.py index cb2f87da..2adc66d0 100755 --- a/lobster/tools/cpp/cpp.py +++ b/lobster/tools/cpp/cpp.py @@ -164,7 +164,7 @@ def main(): kind = kind, name = function_name) - db[tag.key].just_up.append(reason) + db[tag.key()].just_up.append(reason) continue diff --git a/lobster/tools/cpp_raw/__init__.py b/lobster/tools/cpp_raw/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lobster/tools/cpp_raw/cpp_raw.py b/lobster/tools/cpp_raw/cpp_raw.py new file mode 100755 index 00000000..de2f3511 --- /dev/null +++ b/lobster/tools/cpp_raw/cpp_raw.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# +# lobster_cpp_raw - Extract C/C++ tracing tags for LOBSTER +# Copyright (C) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . + +import os.path +import re +import sys + +from argparse import ArgumentParser, Namespace +from io import TextIOWrapper +from typing import Callable, Dict, List, Tuple +from lobster.io import lobster_write +from lobster.items import Tracing_Tag, Implementation +from lobster.location import File_Reference + + +class MisplacedTagException(Exception): + def __init__(self, message: str) -> None: + super().__init__(message) + + +class CppRawArgumentParser(ArgumentParser): + def __init__(self) -> None: + super().__init__() + self.add_argument("files", + nargs="+", + metavar="FILE|DIR") + self.add_argument("--only-tagged-functions", + default=False, + action="store_true", + help="only trace functions with tags") + self.add_argument("--out", + default=None, + help=("write output to this file; otherwise output " + "to to stdout")) + + +def get_valid_files(file_dir: List[str], + argument_parser: CppRawArgumentParser) -> List[str]: + file_list = [] + for item in file_dir: + if os.path.isfile(item): + file_list.append(item) + elif os.path.isdir(item): + for path, _, files in os.walk(item): + for filename in files: + _, ext = os.path.splitext(filename) + if ext in (".cpp", ".cc", ".c", ".h"): + file_list.append(os.path.join(path, filename)) + else: + argument_parser.error("%s is not a file or directory" % item) + return file_list + + +def extract_function_name(all_lines: List[str], trace_line: int) -> str: + if trace_line < len(all_lines): + function_name_match = re.search(r'\b(\w+)\(', + all_lines[trace_line + 1]) + if function_name_match: + return function_name_match.group(1) + else: + return None + + +def write_to_file(options: Namespace, data: Dict[str, Implementation]) -> None: + with open(options.out, "w", encoding="UTF-8") as file: + lobster_write(file, Implementation, "lobster-cpp_raw", data.values()) + print("Written output for %u items to %s" % (len(data), options.out)) + + +def find_lobster_traces(all_lines: List[str]) -> List[Tuple[List[str], int]]: + matches = [] + i = 0 + while i < len(all_lines): + line = all_lines[i] + match = re.search(r'lobster-trace:\s*(.*)', line) + if match: + items = match.group(1).split(',') + matches.append(([item.strip() for item in items], i)) + i += 1 + return matches + + +def find_lobster_excludes(all_lines: List[str]) -> List[Tuple[re.Match, int]]: + matches = [] + i = 0 + while i < len(all_lines): + line = all_lines[i] + match = re.search(r'lobster-exclude:\s*(.*)', line) + if match: + matches.append((match, i)) + i += 1 + return matches + + +def find_all_function_decl(file_content: str) -> List[Tuple[re.Match, int]]: + function_matches = [] + matches = re.finditer(r'(\w+)\s*\([^;]*\)\s*\{', file_content) + for match in matches: + start_pos = match.start() + line_number = file_content.count('\n', 0, start_pos) + 1 + function_matches.append((match, line_number)) + return function_matches + + +def create_raw_entry(data: Dict[str, Implementation], file_name: str, + name: str, line_number: int) -> Tracing_Tag: + tag = Tracing_Tag("cpp", f"{file_name}:{name}:{line_number}") + loc = File_Reference(file_name, line_number) + data[tag.key()] = Implementation( + tag = tag, + location = loc, + language = "C/C++", + kind = "function", + name = name) + return tag + + +def create_entry(match: Tuple[List[str], int], + data: Dict[str, Implementation], + all_lines: List[str], file: TextIOWrapper, + tag_type: str) -> Tracing_Tag: + line = match[1] + function_line = line + 1 + name = extract_function_name(all_lines, line) + if name is None: + raise MisplacedTagException("ERROR: No function declaration " + "found following the " + f"'{tag_type}' in line {line + 1}") + return create_raw_entry(data, file.name, name, function_line + 1) + + +def create_trace_entry(match: Tuple[List[str], int], + data: Dict[str, Implementation], + all_lines: List[str], file: TextIOWrapper) -> None: + tag = create_entry(match, data, all_lines, file, "lobster-trace") + for req in match[0]: + data[tag.key()].add_tracing_target(Tracing_Tag("req", req)) + + +def create_exclude_entry(match: Tuple[re.Match, int], + data: Dict[str, Implementation], + all_lines: List[str], file: TextIOWrapper) -> None: + tag = create_entry(match, data, all_lines, file, "lobster-exclude") + data[tag.key()].just_up.append(match[0].group(1)) + + +def create_remaining_function_decl(data: Dict[str, Implementation], + file_content: str, + file: TextIOWrapper) -> None: + function_matches = find_all_function_decl(file_content) + for match in function_matches: + create_raw_entry(data, file.name, match[0].group(1), match[1]) + + +def create_lobster_entries(data: Dict[str, Implementation], + all_lines: List[str], + file: TextIOWrapper, find_function: Callable, + create_type_entry: Callable) -> List[int]: + tracing_matches = find_function(all_lines) + lines_of_creation = [] + for match in tracing_matches: + lines_of_creation.append(match[1]) + create_type_entry(match, data, all_lines, file) + return lines_of_creation + + +def main() -> int: + argument_parser = CppRawArgumentParser() + options = argument_parser.parse_args() + file_list = get_valid_files(options.files, argument_parser) + data = {} + + for file_path in file_list: + with open(file_path, 'r', encoding="UTF-8") as file: + file_content = file.read() + all_lines = file_content.split('\n') + try: + lines_to_remove = create_lobster_entries(data, all_lines, file, + find_lobster_traces, + create_trace_entry) + if options.only_tagged_functions: + continue + lines_to_remove.extend( + create_lobster_entries(data, all_lines, file, + find_lobster_excludes, + create_exclude_entry)) + for line in lines_to_remove: + all_lines[line] = "" + all_lines[line + 1] = "" + remaining_content = '\n'.join(all_lines) + create_remaining_function_decl(data, remaining_content, file) + except MisplacedTagException as e: + print(f"{e}\nERROR: Misplaced LOBSTER tag " + f"in file {file.name}") + return 1 + + if options.out: + write_to_file(options, data) + else: + lobster_write(sys.stdout, Implementation, "lobster-cpp_raw", + data.values()) + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/lobster/tools/trlc/trlc.py b/lobster/tools/trlc/trlc.py index ef9905c2..801a411c 100644 --- a/lobster/tools/trlc/trlc.py +++ b/lobster/tools/trlc/trlc.py @@ -410,6 +410,7 @@ def main(): ok = False if ok: stab = sm.process() + # pylint: disable=possibly-used-before-assignment if not ok or stab is None: print("lobster-trlc: aborting due to earlier error") return 1 diff --git a/packages/lobster-metapackage/README.md b/packages/lobster-metapackage/README.md index 2b2038e3..3de27b4c 100644 --- a/packages/lobster-metapackage/README.md +++ b/packages/lobster-metapackage/README.md @@ -11,6 +11,7 @@ LOBSTER packages as a convenience: * [bmw-lobster-core](https://pypi.org/project/bmw-lobster-core) * [bmw-lobster-tool-codebeamer](https://pypi.org/project/bmw-lobster-tool-codebeamer) * [bmw-lobster-tool-cpp](https://pypi.org/project/bmw-lobster-tool-cpp) +* [bmw-lobster-tool-cpp_raw](https://pypi.org/project/bmw-lobster-tool-cpp_raw) * [bmw-lobster-tool-gtest](https://pypi.org/project/bmw-lobster-tool-gtest) * [bmw-lobster-tool-json](https://pypi.org/project/bmw-lobster-tool-json) * [bmw-lobster-tool-python](https://pypi.org/project/bmw-lobster-tool-python) diff --git a/packages/lobster-metapackage/setup.py b/packages/lobster-metapackage/setup.py index ce6f8a78..4494383b 100644 --- a/packages/lobster-metapackage/setup.py +++ b/packages/lobster-metapackage/setup.py @@ -2,7 +2,6 @@ import os import re -import sys import setuptools import glob diff --git a/packages/lobster-monolithic/setup.py b/packages/lobster-monolithic/setup.py index cb71e2fa..09a76e20 100644 --- a/packages/lobster-monolithic/setup.py +++ b/packages/lobster-monolithic/setup.py @@ -2,9 +2,7 @@ import os import re -import sys import setuptools -import glob from lobster import version diff --git a/packages/lobster-tool-codebeamer/setup.py b/packages/lobster-tool-codebeamer/setup.py index 1d55e4d2..c9292c93 100644 --- a/packages/lobster-tool-codebeamer/setup.py +++ b/packages/lobster-tool-codebeamer/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import re -import sys import setuptools from lobster import version diff --git a/packages/lobster-tool-cpp/setup.py b/packages/lobster-tool-cpp/setup.py index e386b090..f063f80e 100644 --- a/packages/lobster-tool-cpp/setup.py +++ b/packages/lobster-tool-cpp/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import re -import sys import setuptools from lobster import version diff --git a/packages/lobster-tool-cpp_raw/Makefile b/packages/lobster-tool-cpp_raw/Makefile new file mode 100644 index 00000000..4fb8a462 --- /dev/null +++ b/packages/lobster-tool-cpp_raw/Makefile @@ -0,0 +1,5 @@ +package: + rm -rf lobster + mkdir -p lobster/tools + cp -Rv $(LOBSTER_ROOT)/lobster/tools/cpp_raw lobster/tools + @python3 setup.py sdist bdist_wheel diff --git a/packages/lobster-tool-cpp_raw/README.md b/packages/lobster-tool-cpp_raw/README.md new file mode 100644 index 00000000..091697ff --- /dev/null +++ b/packages/lobster-tool-cpp_raw/README.md @@ -0,0 +1,28 @@ +# LOBSTER + +The **L**ightweight **O**pen **B**MW **S**oftware **T**raceability +**E**vidence **R**eport allows you to demonstrate software traceability +and requirements coverage, which is essential for meeting standards +such as ISO 26262. + +This package contains a tool extract tracing tags from ISO C or C++ +source code. + +This tool searches for tracing tags `lobster-trace` +and extracts the name information of the function occurrence. +To be able to extract the information, the tracing tag has to be +placed in a comment line directly above the function declaration. + +## Tools + +* `lobster-cpp_raw`: Extract requirements from C/C++ code using + a simple text extracting algorithm and a regex to identify + function declarations. Use this tool if you are not able to + provide the clang tidy version needed for the superior `lobster-cpp` tool. + +## Copyright & License information + +The copyright holder of LOBSTER is the Bayerische Motoren Werke +Aktiengesellschaft (BMW AG), and LOBSTER is published under the [GNU +Affero General Public License, Version +3](https://github.com/bmw-software-engineering/lobster/blob/main/LICENSE.md). diff --git a/packages/lobster-tool-cpp_raw/entrypoints b/packages/lobster-tool-cpp_raw/entrypoints new file mode 100644 index 00000000..cc1a94da --- /dev/null +++ b/packages/lobster-tool-cpp_raw/entrypoints @@ -0,0 +1 @@ +lobster-cpp_raw = lobster.tools.cpp_raw.cpp_raw:main \ No newline at end of file diff --git a/packages/lobster-tool-cpp_raw/requirements b/packages/lobster-tool-cpp_raw/requirements new file mode 100644 index 00000000..e69de29b diff --git a/packages/lobster-tool-cpp_raw/setup.py b/packages/lobster-tool-cpp_raw/setup.py new file mode 100644 index 00000000..717455ee --- /dev/null +++ b/packages/lobster-tool-cpp_raw/setup.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import re +import setuptools + +from lobster import version + +gh_root = "https://github.com" +gh_project = "bmw-software-engineering/lobster" + +with open("README.md", "r") as fd: + long_description = fd.read() + +with open("requirements", "r") as fd: + package_requirements = [line + for line in fd.read().splitlines() + if line.strip()] +package_requirements.append("bmw-lobster-core>=%s" % version.LOBSTER_VERSION) +with open("entrypoints", "r") as fd: + entrypoints = [line + for line in fd.read().splitlines() + if line.strip()] + +# For the readme to look right on PyPI we need to translate any +# relative links to absolute links to github. +fixes = [] +for match in re.finditer(r"\[(.*)\]\((.*)\)", long_description): + if not match.group(2).startswith("http"): + fixes.append((match.span(0)[0], match.span(0)[1], + "[%s](%s/%s/blob/main/%s)" % (match.group(1), + gh_root, + gh_project, + match.group(2)))) + +for begin, end, text in reversed(fixes): + long_description = (long_description[:begin] + + text + + long_description[end:]) + +project_urls = { + "Bug Tracker" : "%s/%s/issues" % (gh_root, gh_project), + "Documentation" : "%s/pages/%s/" % (gh_root, gh_project), + "Source Code" : "%s/%s" % (gh_root, gh_project), +} + +setuptools.setup( + name="bmw-lobster-tool-cpp_raw", + version=version.LOBSTER_VERSION, + author="Bayerische Motoren Werke Aktiengesellschaft (BMW AG)", + author_email="philipp.wullstein-kammler@bmw.de", + description="LOBSTER Tool for ISO C/C++", + long_description=long_description, + long_description_content_type="text/markdown", + url=project_urls["Source Code"], + project_urls=project_urls, + license="GNU Affero General Public License v3", + packages=["lobster.tools.cpp_raw"], + install_requires=package_requirements, + python_requires=">=3.7, <4", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Topic :: Documentation", + "Topic :: Software Development", + ], + entry_points={ + "console_scripts": entrypoints, + }, +) diff --git a/packages/lobster-tool-json/setup.py b/packages/lobster-tool-json/setup.py index 62734202..7ba52010 100644 --- a/packages/lobster-tool-json/setup.py +++ b/packages/lobster-tool-json/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import re -import sys import setuptools from lobster import version diff --git a/packages/lobster-tool-python/setup.py b/packages/lobster-tool-python/setup.py index e81b46fe..155e4ce6 100644 --- a/packages/lobster-tool-python/setup.py +++ b/packages/lobster-tool-python/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import re -import sys import setuptools from lobster import version diff --git a/packages/lobster-tool-trlc/setup.py b/packages/lobster-tool-trlc/setup.py index 4812ed94..3e34e29b 100644 --- a/packages/lobster-tool-trlc/setup.py +++ b/packages/lobster-tool-trlc/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import re -import sys import setuptools from lobster import version diff --git a/test-unit/test_json.py b/test-unit/test_json.py index ddf4d1a8..f77cd305 100644 --- a/test-unit/test_json.py +++ b/test-unit/test_json.py @@ -1,5 +1,4 @@ import unittest -import re from pathlib import PurePosixPath, PureWindowsPath from lobster.tools.json import json