Merge branch 'main' into lobster-coda

.github/CODEOWNERS

@@ -3,6 +3,6 @@
 
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence,
-# @bmw-software-engineering/maintainers will be requested for
+# @bmw-software-engineering/trlc-lobster-maintainers will be requested for
 # review when someone opens a pull request.
-*       @bmw-software-engineering/maintainers
+*       @bmw-software-engineering/trlc-lobster-maintainers

.github/workflows/ci.yml

@@ -18,8 +18,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          python3 -m pip install --upgrade pip
-          python3 -m pip install -r requirements_dev.txt
+          python3 -m pip install --upgrade pip --break-system-packages
+          python3 -m pip install -r requirements_dev.txt --break-system-packages
           make lobster/html/assets.py
       - name: Executing linter
         run: |
@@ -30,7 +30,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13, macos-14]
-        py-version: ["3.8", "3.9", "3.10", "3.11"]
+        py-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         include:
           - os: macos-13
             brew: "/usr/local"
@@ -49,8 +49,7 @@ jobs:
         run: |
           brew install python@${{ matrix.py-version }}
           echo "${{ matrix.brew }}/opt/python@${{ matrix.py-version }}/libexec/bin" >> $GITHUB_PATH
-          curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-          python${{ matrix.py-version }} get-pip.py
+          python${{ matrix.py-version }} -m ensurepip
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -66,3 +65,10 @@ jobs:
       - name: Executing system tests
         run: |
           make system-tests
+      - name: Coverage analysis
+        run: |
+          make coverage
+      - name: Check output files
+        if: always()
+        run: |
+          util/check_local_modifications.sh

.github/workflows/docs.yml

@@ -0,0 +1,60 @@
+name: Deploy static content to Pages
+
+on:
+  push:
+    branches: ["main"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+      - name: Generate docs
+        run: |
+          make docs
+        env:
+          PYTHONPATH: "."
+      - name: Building code coverage report
+        run: |
+          make system-tests unit-tests
+          make coverage
+          mv htmlcov docs
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: 'docs'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -9,3 +9,8 @@ meta_dist
 
 test_install
 test_install_monolithic
+
+# Project ignores
+.coverage*
+*.lobster
+*.html

CHANGELOG.md

@@ -4,6 +4,12 @@
 
 ### 0.9.18-dev
 
+* The `lobster-python` tool adds the counter logic to the function
+  identifier. This improves the situations where different functions have
+  the same name. Line numbers are no longer used in the identifier.
+
+* The `lobster-codebeamer` tool now supports `refs` as an upstream reference
+
 * The `lobster-online-report` tool now works with config files located in
   main- and submodules of a repository. This feature needs `git 1.7.8` or higher.
 
@@ -16,6 +22,9 @@
   resulting lobster file will now contain "Unset" as status information,
   too, instead of `Null`.
 
+* The `lobster-html-report` tool now supports argument `--high-contrast` to use
+  a color palette with a higher contrast for easier visualization.
+
 ### 0.9.17
 
 * The `lobster-python` tool now adds the line number to the function

Makefile

@@ -45,11 +45,15 @@ integration-tests: packages
 	(cd integration-tests/projects/filter; make)
 
 system-tests:
+	mkdir -p docs
 	make -B -C test-system/lobster-json
 	make -B -C test-system/lobster-python
 
 unit-tests:
-	python3 -m unittest discover -s test-unit -v
+	coverage run -p \
+			--branch --rcfile=coverage.cfg \
+			--data-file .coverage \
+			-m unittest discover -s test-unit -v
 
 test: integration-tests system-tests unit-tests
 
@@ -74,3 +78,15 @@ full-release:
 	make github-release
 	make bump
 	git push
+
+coverage:
+	coverage combine -q
+	coverage html --rcfile=coverage.cfg
+	coverage report --rcfile=coverage.cfg --fail-under=57
+
+test-ci: system-tests unit-tests coverage
+	util/check_local_modifications.sh
+
+docs:
+	rm -rf docs
+	mkdir docs

README.md

@@ -47,10 +47,10 @@ The following verification and miscellaneous frameworks are supported:
 
 ## Documentation
 
-* Writing [configuration files](docs/config_files.md) for LOBSTER.
+* Writing [configuration files](documentation/config_files.md) for LOBSTER.
 * It is easy to expand the languages and activities supported by
   LOBSTER by adding new tracing tools, as long as they create data in
-  the [common interchange format](docs/schemas.md) for LOBSTER.
+  the [common interchange format](documentation/schemas.md) for LOBSTER.
 
 (More to come...)
 
@@ -67,6 +67,10 @@ The individual packages that `bmw-lobster` depends on are:
 * `bmw-lobster-tool-json` (for activities in JSON)
 * `miss_hit` (for MATLAB/Octave code or Simulink models)
 
+### For LOBSTER developers
+
+* [Code Coverage Report](https://bmw-software-engineering.github.io/lobster/htmlcov/index.html)
+
 ## Planned inputs
 
 The following inputs are planned but not implemeted yet:

coverage.cfg

@@ -0,0 +1,16 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    ice_loc
+    @abstractmethod
+    if DEBUG_
+    assert False
+    def __repr__(self)
+    def sanity_test()
+    if __name__ == "__main__"
+
+[run]
+omit =
+    /usr/*
+    */site-packages/*
+    test-unit/*

integration-tests/projects/basic/Makefile

@@ -14,7 +14,7 @@ html_report.html: cppcode.lobster gtests.lobster mcode.lobster system-requiremen
 	@lobster-online-report
 	@cp report.lobster report.reference_output
 	@lobster-html-report
-	@cp lobster_report.html ../../../docs/example_report.html
+	@cp lobster_report.html ../../../documentation/example_report.html
 	@lobster-ci-report | tee ci_report.reference_output
 
 cppcode.lobster: foo.h foo.cpp

lobster/io.py

@@ -125,9 +125,9 @@ def lobster_read(mh, filename, level, items, source_info=None):
         if all(filter_conditions):
             if item.tag.key() in items:
                 mh.error(item.location,
-                            "duplicate definition of %s, "
-                            "previously defined at %s" %
-                            (item.tag.key(),
-                                items[item.tag.key()].location.to_string()))
+                          "duplicate definition of %s, "
+                          "previously defined at %s" %
+                          (item.tag.key(),
+                           items[item.tag.key()].location.to_string()))
 
             items[item.tag.key()] = item

lobster/tools/codebeamer/codebeamer.py

@@ -41,6 +41,8 @@
 import argparse
 import netrc
 from urllib.parse import quote
+from enum import Enum
+import json
 import requests
 
 from lobster.items import Tracing_Tag, Requirement
@@ -49,6 +51,26 @@
 from lobster.io import lobster_read, lobster_write
 
 
+class References(Enum):
+    REFS = "refs"
+
+
+SUPPORTED_REFERENCES = [References.REFS.value]
+
+
+def add_refs_refrences(req, flat_values_list):
+    # refs
+    for value in flat_values_list:
+        if value.get("id"):
+            ref_id = value.get("id")
+            req.add_tracing_target(Tracing_Tag("req", str(ref_id)))
+
+
+map_reference_name_to_function = {
+    References.REFS.value: add_refs_refrences
+}
+
+
 def query_cb_single(cb_config, url):
     assert isinstance(cb_config, dict)
     assert isinstance(url, str)
@@ -193,6 +215,30 @@ def to_lobster(cb_config, cb_item):
         text      = None,
         status    = status)
 
+    if cb_config.get('references'):
+        for reference_name, displayed_chosen_names in (
+                cb_config['references'].items()):
+            if reference_name not in map_reference_name_to_function:
+                continue
+
+            for displayed_name in displayed_chosen_names:
+                if cb_item.get(displayed_name):
+                    flat_values_list = cb_item.get(displayed_name) if (
+                        isinstance(cb_item.get(displayed_name), list)) \
+                        else [cb_item.get(displayed_name)]
+                else:
+                    flat_values_list = (
+                        list(value for custom_field
+                             in cb_item["customFields"]
+                             if custom_field["name"] == displayed_name and
+                             custom_field.get("values")
+                             for value in custom_field["values"]))
+                if not flat_values_list:
+                    continue
+
+                (map_reference_name_to_function[reference_name]
+                 (req, flat_values_list))
+
     return req
 
 
@@ -210,17 +256,54 @@ def import_tagged(mh, cb_config, items_to_import):
     return rv
 
 
+def ensure_array_of_strings(instance):
+    if (isinstance(instance, list) and
+            all(isinstance(item, str)
+                for item in instance)):
+        return instance
+    else:
+        return [str(instance)]
+
+
+def parse_cb_config(file_name):
+    assert isinstance(file_name, str)
+    assert os.path.isfile(file_name)
+
+    with open(file_name, "r", encoding='utf-8') as file:
+        data = json.loads(file.read())
+
+    provided_config_keys = set(data.keys())
+    supported_references = set(SUPPORTED_REFERENCES)
+
+    if not provided_config_keys.issubset(supported_references):
+        raise KeyError("The provided references are not supported! "
+                        "supported referenes: '%s'" %
+                        ', '.join(SUPPORTED_REFERENCES))
+
+    json_config = {}
+    for key, value in data.items():
+        json_config[key] = ensure_array_of_strings(value)
+    return json_config
+
+
 def main():
     ap = argparse.ArgumentParser()
 
     modes = ap.add_mutually_exclusive_group(required=True)
     modes.add_argument("--import-tagged",
                        metavar="LOBSTER_FILE",
                        default=None)
+
     modes.add_argument("--import-query",
                        metavar="CB_QUERY_ID",
                        default=None)
 
+    ap.add_argument("--config",
+                    help=("name of codebeamer "
+                          "config file, supported references: '%s'" %
+                          ', '.join(SUPPORTED_REFERENCES)),
+                    default=None)
+
     ap.add_argument("--ignore-ssl-errors",
                     action="store_true",
                     default=False,
@@ -254,6 +337,12 @@ def main():
         "timeout"    : options.timeout,
     }
 
+    if options.config:
+        if os.path.isfile(options.config):
+            cb_config["references"] = parse_cb_config(options.config)
+        else:
+            ap.error("cannot open config file '%s'" % options.config)
+
     if cb_config["root"] is None:
         ap.error("please set CB_ROOT or use --cb-root")