From 47b9f5063b3fef7ac883eaebfcfdc6f6a384511b Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Sat, 15 Jul 2023 21:32:21 +0200 Subject: [PATCH 1/3] refactor(project CreateBom): build CycloneDX BOM directly The old code used to create a legacy BOM which was then converted into a CycloneDX BOM using LegacySupport.legacy_component_to_cdx(). While the old code was a bit shorter and avoided some code duplication to legacy_component_to_cdx(), this approach was quite error prone (see previous commit, 404fa53c, e9a349f5 and 66e4780e). The new code also allows to add multiple source/binary attachments as external references easily, see #30. --- capycli/common/capycli_bom_support.py | 28 +++++++-- capycli/project/create_bom.py | 90 +++++++++++++++------------ 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/capycli/common/capycli_bom_support.py b/capycli/common/capycli_bom_support.py index b8b5247..c5b8e8d 100644 --- a/capycli/common/capycli_bom_support.py +++ b/capycli/common/capycli_bom_support.py @@ -298,6 +298,11 @@ def get_property(comp: Component, name: str) -> Any: return None + @staticmethod + def set_property(comp: Component, name: str, value: str) -> None: + """Sets the property with the given name.""" + comp.properties.add(Property(name=name, value=value)) + @staticmethod def update_or_set_property(comp: Component, name: str, value: str) -> None: """Returns the property with the given name.""" @@ -310,7 +315,7 @@ def update_or_set_property(comp: Component, name: str, value: str) -> None: if prop: prop.value = value else: - comp.properties.add(Property(name=name, value=value)) + CycloneDxSupport.set_property(comp, name, value) @staticmethod def remove_property(comp: Component, name: str) -> None: @@ -337,6 +342,21 @@ def get_ext_ref(comp: Component, type: ExternalReferenceType, comment: str) -> O return None + @staticmethod + def set_ext_ref(comp: Component, type: ExternalReferenceType, comment: str, value: str, + hash_algo: str = None, hash: str = None) -> None: + ext_ref = ExternalReference( + reference_type=type, + url=value, + comment=comment) + + if hash_algo and hash: + ext_ref.hashes.add(HashType( + algorithm=HashAlgorithm.SHA_1, + hash_value=hash)) + + comp.external_references.add(ext_ref) + @staticmethod def update_or_set_ext_ref(comp: Component, type: ExternalReferenceType, comment: str, value: str) -> None: ext_ref = None @@ -348,11 +368,7 @@ def update_or_set_ext_ref(comp: Component, type: ExternalReferenceType, comment: if ext_ref: ext_ref.url = value else: - ext_ref = ExternalReference( - reference_type=type, - url=value, - comment=comment) - comp.external_references.add(ext_ref) + CycloneDxSupport.set_ext_ref(comp, type, comment, value) @staticmethod def get_ext_ref_by_comment(comp: Component, comment: str) -> Any: diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index d46f6a5..6cb3313 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -10,12 +10,14 @@ import sys import sw360 +from cyclonedx.model import ExternalReferenceType, HashAlgorithm from cyclonedx.model.bom import Bom +from cyclonedx.model.component import Component import capycli.common.script_base from capycli import get_logger -from capycli.bom.legacy import LegacySupport -from capycli.common.capycli_bom_support import CaPyCliBom, SbomCreator +from capycli.common.capycli_bom_support import CaPyCliBom, SbomCreator, CycloneDxSupport + from capycli.common.print import print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode @@ -66,46 +68,57 @@ def create_project_bom(self, project) -> list: href = release["_links"]["self"]["href"] state = self.get_clearing_state(project, href) - rel_item = {} - rel_item["Name"] = release["name"] - rel_item["Version"] = release["version"] - rel_item["ProjectClearingState"] = state - rel_item["Id"] = self.client.get_id_from_href(href) - rel_item["Sw360Id"] = rel_item["Id"] - rel_item["Url"] = ( + rel_item = Component(name=release["name"], version=release["version"]) + if state: + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state) + + sw360_id = self.client.get_id_from_href(href) + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id) + + CycloneDxSupport.set_property( + rel_item, + CycloneDxSupport.CDX_PROP_SW360_URL, self.sw360_url + "group/guest/components/-/component/release/detailRelease/" - + self.client.get_id_from_href(href)) + + sw360_id) try: release_details = self.client.get_release_by_url(href) - # capycli.common.json_support.print_json(release_details) - rel_item["ClearingState"] = release_details["clearingState"] - rel_item["ReleaseMainlineState"] = release_details.get("mainlineState", "") - - rel_item["Language"] = self.list_to_string(release_details.get("languages", "")) - rel_item["SourceFileUrl"] = release_details.get("sourceCodeDownloadurl", "") - rel_item["BinaryFileUrl"] = release_details.get("binaryDownloadurl", "") - rel_item["RepositoryId"] = self.get_external_id("package-url", release_details) - if not rel_item["RepositoryId"]: + purl = self.get_external_id("package-url", release_details) + if not purl: # try another id name - rel_item["RepositoryId"] = self.get_external_id("purl", release_details) - if rel_item["RepositoryId"]: - rel_item["RepositoryType"] = "package-url" - - if "repository" in release_details: - rel_item["RepositoryUrl"] = release_details["repository"].get("url", "") - - source_attachment = self.get_attachment("SOURCE", release_details) - if source_attachment: - rel_item["SourceFile"] = source_attachment.get("filename", "") - rel_item["SourceFileHash"] = source_attachment.get("sha1", "") - - binary_attachment = self.get_attachment("BINARY", release_details) - if binary_attachment: - rel_item["BinaryFile"] = binary_attachment.get("filename", "") - rel_item["BinaryFileHash"] = binary_attachment.get("sha1", "") + purl = self.get_external_id("purl", release_details) + if purl: + rel_item.purl = purl + + for key, property in (("clearingState", CycloneDxSupport.CDX_PROP_CLEARING_STATE), + ("mainlineState", CycloneDxSupport.CDX_PROP_REL_STATE)): + if key in release_details and release_details[key]: + CycloneDxSupport.set_property(rel_item, property, release_details[key]) + + if "languages" in release_details and release_details["languages"]: + languages = self.list_to_string(release_details["languages"]) + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_LANGUAGE, languages) + + for key, comment in (("sourceCodeDownloadurl", CaPyCliBom.SOURCE_URL_COMMENT), + ("binaryDownloadurl", CaPyCliBom.BINARY_URL_COMMENT)): + if key in release_details and release_details[key]: + # add hash from attachment (see below) also here if same filename? + CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.DISTRIBUTION, + comment, release_details[key]) + + if "repository" in release_details and "url" in release_details["repository"]: + CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.VCS, comment=None, + value=release_details["repository"]["url"]) + + for at_type, comment in (("SOURCE", CaPyCliBom.SOURCE_FILE_COMMENT), + ("BINARY", CaPyCliBom.BINARY_FILE_COMMENT)): + attachment = self.get_attachment(at_type, release_details) + if attachment: + CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.DISTRIBUTION, + comment, attachment["filename"], + HashAlgorithm.SHA_1, attachment.get("sha1")) except sw360.SW360Error as swex: print_red(" ERROR: unable to access project:" + repr(swex)) @@ -126,12 +139,7 @@ def create_project_cdx_bom(self, project_id) -> Bom: print_text(" Project name: " + project["name"] + ", " + project["version"]) - bom = self.create_project_bom(project) - - cdx_components = [] - for item in bom: - cx_comp = LegacySupport.legacy_component_to_cdx(item) - cdx_components.append(cx_comp) + cdx_components = self.create_project_bom(project) creator = SbomCreator() sbom = creator.create(cdx_components, addlicense=True, addprofile=True, addtools=True, From 9dba53868caa624d97a486caac723bdaf242f4d1 Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Sun, 16 Jul 2023 20:23:05 +0200 Subject: [PATCH 2/3] refactor(project CreateBom): create Component with purl It seems we can only create a new CycloneDX Component() object when we know the purl as we need to specify the purl as bom-ref on object creation. --- capycli/project/create_bom.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index 6cb3313..7156d3e 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -66,21 +66,6 @@ def create_project_bom(self, project) -> list: for release in releases: print_text(" ", release["name"], release["version"]) href = release["_links"]["self"]["href"] - state = self.get_clearing_state(project, href) - - rel_item = Component(name=release["name"], version=release["version"]) - if state: - CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state) - - sw360_id = self.client.get_id_from_href(href) - CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id) - - CycloneDxSupport.set_property( - rel_item, - CycloneDxSupport.CDX_PROP_SW360_URL, - self.sw360_url - + "group/guest/components/-/component/release/detailRelease/" - + sw360_id) try: release_details = self.client.get_release_by_url(href) @@ -89,8 +74,11 @@ def create_project_bom(self, project) -> list: if not purl: # try another id name purl = self.get_external_id("purl", release_details) + if purl: - rel_item.purl = purl + rel_item = Component(name=release["name"], version=release["version"], purl=purl, bom_ref=purl) + else: + rel_item = Component(name=release["name"], version=release["version"]) for key, property in (("clearingState", CycloneDxSupport.CDX_PROP_CLEARING_STATE), ("mainlineState", CycloneDxSupport.CDX_PROP_REL_STATE)): @@ -124,6 +112,20 @@ def create_project_bom(self, project) -> list: print_red(" ERROR: unable to access project:" + repr(swex)) sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) + state = self.get_clearing_state(project, href) + if state: + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state) + + sw360_id = self.client.get_id_from_href(href) + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id) + + CycloneDxSupport.set_property( + rel_item, + CycloneDxSupport.CDX_PROP_SW360_URL, + self.sw360_url + + "group/guest/components/-/component/release/detailRelease/" + + sw360_id) + bom.append(rel_item) # sub-projects are not handled at the moment From 13af3219ffee4b65e6683e762126e6958862ed16 Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Sun, 16 Jul 2023 21:01:38 +0200 Subject: [PATCH 3/3] feat(project CreateBom): multiple and self made attachments Add support for multiple attachments in one release and also include SOURCE_SELF and BINARY_SELF attachments. Closes #30. --- ChangeLog.md | 2 +- capycli/project/create_bom.py | 17 ++++++++++------- tests/test_create_bom.py | 13 ++++++++++++- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 9a32132..137b905 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -11,7 +11,7 @@ * The `-o` parameter of the command `project GetLicenseInfo` is now optional. But you still need this output when you want to create a Readme. * `project createbom` add purl, source and repository url from SW360 if available -* `project createbom` adds first SW360 source attachment of each releases as external reference to SBOM. +* `project createbom` add SW360 source and binary attachments as external reference to SBOM. * `project createbom` adds SW360 project name, version and description to SBOM. ## 2.0.0 (2023-06-02) diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index 7156d3e..7102ac9 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -9,6 +9,8 @@ import logging import sys +from typing import List, Tuple + import sw360 from cyclonedx.model import ExternalReferenceType, HashAlgorithm from cyclonedx.model.bom import Bom @@ -34,20 +36,21 @@ def get_external_id(self, name: str, release_details: dict): return release_details["externalIds"].get(name, "") - def get_attachment(self, att_type: str, release_details: dict) -> dict: - """Returns the first attachment with the given type or None.""" + def get_attachments(self, att_types: Tuple[str], release_details: dict) -> List[dict]: + """Returns the attachments with the given types or empty list.""" if "_embedded" not in release_details: return None if "sw360:attachments" not in release_details["_embedded"]: return None + found = [] attachments = release_details["_embedded"]["sw360:attachments"] for attachment in attachments: - if attachment["attachmentType"] == att_type: - return attachment + if attachment["attachmentType"] in att_types: + found.append(attachment) - return None + return found def get_clearing_state(self, proj, href) -> str: """Returns the clearing state of the given component/release""" @@ -102,8 +105,8 @@ def create_project_bom(self, project) -> list: for at_type, comment in (("SOURCE", CaPyCliBom.SOURCE_FILE_COMMENT), ("BINARY", CaPyCliBom.BINARY_FILE_COMMENT)): - attachment = self.get_attachment(at_type, release_details) - if attachment: + attachments = self.get_attachments((at_type, at_type + "_SELF"), release_details) + for attachment in attachments: CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.DISTRIBUTION, comment, attachment["filename"], HashAlgorithm.SHA_1, attachment.get("sha1")) diff --git a/tests/test_create_bom.py b/tests/test_create_bom.py index b0849de..b23011c 100644 --- a/tests/test_create_bom.py +++ b/tests/test_create_bom.py @@ -158,6 +158,17 @@ def test_project_by_id(self): release = self.get_release_cli_for_test() # use a specific purl release["externalIds"]["package-url"] = "pkg:deb/debian/cli-support@1.3-1" + # add a SOURCE_SELF attachment + release["_embedded"]["sw360:attachments"].append({ + "filename": "clipython-repacked-for-fun.zip", + "sha1": "face4b90d134e2a2bcf9464c50ea086f849a9b82", + "attachmentType": "SOURCE_SELF", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/attachments/r002a002" + } + } + }) responses.add( responses.GET, url=self.MYURL + "resource/api/releases/r002", @@ -177,7 +188,7 @@ def test_project_by_id(self): self.assertEqual(ext_refs_src_url[0].type, ExternalReferenceType.DISTRIBUTION) ext_refs_src_file = [e for e in cx_comp.external_references if e.comment == CaPyCliBom.SOURCE_FILE_COMMENT] - self.assertEqual(len(ext_refs_src_file), 1) + self.assertEqual(len(ext_refs_src_file), 2) self.assertEqual(ext_refs_src_file[0].url, release["_embedded"]["sw360:attachments"][0]["filename"]) self.assertEqual(ext_refs_src_file[0].type, ExternalReferenceType.DISTRIBUTION) self.assertEqual(ext_refs_src_file[0].hashes[0].alg, "SHA-1")