From 8051ec755cdb862027d35d0081388fab64605b7c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 1 Feb 2024 22:13:00 +0530 Subject: [PATCH 01/15] Add add-on pipeline for collecting dwarfs from elfs Signed-off-by: Tushar Goel --- scanpipe/models.py | 17 +++++++++++++ scanpipe/pipelines/get_dwarfs_from_elfs.py | 28 ++++++++++++++++++++++ setup.cfg | 4 +++- 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 scanpipe/pipelines/get_dwarfs_from_elfs.py diff --git a/scanpipe/models.py b/scanpipe/models.py index bcf7eadd2..c43060da1 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1946,6 +1946,23 @@ def has_directory_content_fingerprint(self): and ~Q(extra_data__directory_content__in=IGNORED_DIRECTORY_FINGERPRINTS) ) + def elfs(self): + """ + Resources that are ``files`` and their filetype startswith `elf` and contains any of thes + `executable`, `relocatable`, `shared object`. + """ + return ( + self.files() + .filter( + file_type__istartswith="elf", + ) + .filter( + Q(file_type__icontains="executable") + | Q(file_type__icontains="relocatable") + | Q(file_type__icontains="shared object") + ) + ) + class ScanFieldsModelMixin(models.Model): """Fields returned by the ScanCode-toolkit scans.""" diff --git a/scanpipe/pipelines/get_dwarfs_from_elfs.py b/scanpipe/pipelines/get_dwarfs_from_elfs.py new file mode 100644 index 000000000..76ee6e472 --- /dev/null +++ b/scanpipe/pipelines/get_dwarfs_from_elfs.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from elf_inspector.dwarf import get_dwarf_paths + +from scanpipe.models import CodebaseResource +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import purldb +from scanpipe.pipes import scancode + + +class GetDwarfsFromElfs(Pipeline): + """Get dwarfs from elfs.""" + + download_inputs = False + is_addon = True + + @classmethod + def steps(cls): + return (cls.get_dwarfs_from_elfs,) + + def get_dwarfs_from_elfs(self): + """ + Update ``extra_data`` of project with + dwarf data extracted from elf files. + """ + for elf in self.project.codebaseresources.elfs(): + data = get_dwarf_paths(Path(self.project.codebase_path / elf.path)) + self.project.update_extra_data({elf.path: data}) diff --git a/setup.cfg b/setup.cfg index b2c041292..39a5700ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,7 +76,8 @@ install_requires = # FetchCode fetchcode-container==1.2.3.210512; sys_platform == "linux" # Inspectors - python-inspector==0.11.0 + python-inspector==0.10.0 + elf-inspector==0.0.1 aboutcode-toolkit==10.1.0 # Utilities XlsxWriter==3.1.9 @@ -126,6 +127,7 @@ scancodeio_pipelines = analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities + get_dwarfs_from_elfs = scanpipe.pipelines.get_dwarfs_from_elfs:GetDwarfsFromElfs inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages load_inventory = scanpipe.pipelines.load_inventory:LoadInventory load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM From e6cc9b72ca9df962d425848f0344d2de471884df Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 1 Feb 2024 22:14:50 +0530 Subject: [PATCH 02/15] Add headers Signed-off-by: Tushar Goel --- scanpipe/pipelines/get_dwarfs_from_elfs.py | 25 +++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipelines/get_dwarfs_from_elfs.py b/scanpipe/pipelines/get_dwarfs_from_elfs.py index 76ee6e472..7aa1d9b40 100644 --- a/scanpipe/pipelines/get_dwarfs_from_elfs.py +++ b/scanpipe/pipelines/get_dwarfs_from_elfs.py @@ -1,11 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + from pathlib import Path from elf_inspector.dwarf import get_dwarf_paths -from scanpipe.models import CodebaseResource from scanpipe.pipelines import Pipeline -from scanpipe.pipes import purldb -from scanpipe.pipes import scancode class GetDwarfsFromElfs(Pipeline): From 04f412250c31a85d9d89a726dd63590714288baf Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 1 Feb 2024 22:46:56 +0530 Subject: [PATCH 03/15] Add tests for models Signed-off-by: Tushar Goel --- scanpipe/models.py | 4 ++-- scanpipe/tests/test_models.py | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index c43060da1..fd307a029 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1948,8 +1948,8 @@ def has_directory_content_fingerprint(self): def elfs(self): """ - Resources that are ``files`` and their filetype startswith `elf` and contains any of thes - `executable`, `relocatable`, `shared object`. + Resources that are ``files`` and their filetype startswith `elf` and + contains any of these `executable`, `relocatable`, `shared object`. """ return ( self.files() diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 436f04de5..98e21a6fb 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2110,6 +2110,45 @@ def test_scanpipe_codebase_resource_queryset_has_directory_content_fingerprint( results = self.project1.codebaseresources.has_directory_content_fingerprint() self.assertQuerySetEqual(expected, results, ordered=False) + def test_scanpipe_codebase_resource_elfs(self): + project = Project.objects.create(name="Test") + CodebaseResource.objects.create( + file_type="""ELF 32-bit LSB executable, ARM, version 1 (ARM), statically + linked, with debug_info, not stripped""", + project=project, + path="a", + type=CodebaseResource.Type.FILE, + ) + CodebaseResource.objects.create( + file_type="""32-bit LSB executable, ARM, version 1 (ARM), statically + linked, with debug_info, not stripped""", + project=project, + path="b", + type=CodebaseResource.Type.FILE, + ) + CodebaseResource.objects.create( + file_type="""ELF 32-bit LSB resourcable, ARM, version 1 (ARM), statically + linked, with debug_info, not stripped""", + project=project, + path="c", + type=CodebaseResource.Type.FILE, + ) + CodebaseResource.objects.create( + file_type="""32-bit LSB relocatable, ARM, version 1 (ARM), statically + linked, with debug_info, not stripped""", + project=project, + path="d", + type=CodebaseResource.Type.FILE, + ) + CodebaseResource.objects.create( + file_type="""ELF 32-bit LSB relocatable, ARM, version 1 (ARM), statically + linked, with debug_info, not stripped""", + project=project, + path="e", + type=CodebaseResource.Type.FILE, + ) + self.assertEqual(2, project.codebaseresources.elfs().count()) + class ScanPipeModelsTransactionTest(TransactionTestCase): """ From 4ca303ebc481d9c29deff98ac5cc5dc348279003 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sun, 11 Feb 2024 22:50:31 +0530 Subject: [PATCH 04/15] Address review comments Signed-off-by: Tushar Goel --- scanpipe/models.py | 2 ++ scanpipe/pipelines/get_dwarfs_from_elfs.py | 12 ++++++------ scanpipe/tests/test_models.py | 16 +++++++++------- setup.cfg | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index fd307a029..8b43ebfe1 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1950,6 +1950,8 @@ def elfs(self): """ Resources that are ``files`` and their filetype startswith `elf` and contains any of these `executable`, `relocatable`, `shared object`. + Keep sync with contenttype implementation: + https://github.com/nexB/typecode/blob/92feb7be3a87c1b541e7034c3f9797c96bc52305/src/typecode/contenttype.py#L733 """ return ( self.files() diff --git a/scanpipe/pipelines/get_dwarfs_from_elfs.py b/scanpipe/pipelines/get_dwarfs_from_elfs.py index 7aa1d9b40..d31c449e6 100644 --- a/scanpipe/pipelines/get_dwarfs_from_elfs.py +++ b/scanpipe/pipelines/get_dwarfs_from_elfs.py @@ -27,21 +27,21 @@ from scanpipe.pipelines import Pipeline -class GetDwarfsFromElfs(Pipeline): - """Get dwarfs from elfs.""" +class InspectElfBinaries(Pipeline): + """Inspect ELF binaries and collect DWARF paths.""" download_inputs = False is_addon = True @classmethod def steps(cls): - return (cls.get_dwarfs_from_elfs,) + return (cls.collect_dwarf_source_path_references,) - def get_dwarfs_from_elfs(self): + def collect_dwarf_source_path_references(self): """ Update ``extra_data`` of project with dwarf data extracted from elf files. """ for elf in self.project.codebaseresources.elfs(): - data = get_dwarf_paths(Path(self.project.codebase_path / elf.path)) - self.project.update_extra_data({elf.path: data}) + dwarf_paths = get_dwarf_paths(Path(self.project.codebase_path / elf.path)) + elf.update_extra_data(dwarf_paths) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 98e21a6fb..dfd8af5dc 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2110,44 +2110,46 @@ def test_scanpipe_codebase_resource_queryset_has_directory_content_fingerprint( results = self.project1.codebaseresources.has_directory_content_fingerprint() self.assertQuerySetEqual(expected, results, ordered=False) - def test_scanpipe_codebase_resource_elfs(self): + def test_scanpipe_codebase_resource_queryset_elfs(self): project = Project.objects.create(name="Test") - CodebaseResource.objects.create( + resource_starting_with_elf_and_executable_in_file_type = CodebaseResource.objects.create( file_type="""ELF 32-bit LSB executable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="a", type=CodebaseResource.Type.FILE, ) - CodebaseResource.objects.create( + resource_with_executable_in_file_type = CodebaseResource.objects.create( file_type="""32-bit LSB executable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="b", type=CodebaseResource.Type.FILE, ) - CodebaseResource.objects.create( + resource_starting_with_elf_in_file_type = CodebaseResource.objects.create( file_type="""ELF 32-bit LSB resourcable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="c", type=CodebaseResource.Type.FILE, ) - CodebaseResource.objects.create( + resource = CodebaseResource.objects.create( file_type="""32-bit LSB relocatable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="d", type=CodebaseResource.Type.FILE, ) - CodebaseResource.objects.create( + resource_starting_with_elf_and_relocatable_in_file_type = CodebaseResource.objects.create( file_type="""ELF 32-bit LSB relocatable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="e", type=CodebaseResource.Type.FILE, ) - self.assertEqual(2, project.codebaseresources.elfs().count()) + paths = [str(resource.path) for resource in project.codebaseresources.elfs()] + assert "e" in paths + assert "a" in paths class ScanPipeModelsTransactionTest(TransactionTestCase): diff --git a/setup.cfg b/setup.cfg index 39a5700ca..182694058 100644 --- a/setup.cfg +++ b/setup.cfg @@ -127,7 +127,7 @@ scancodeio_pipelines = analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities - get_dwarfs_from_elfs = scanpipe.pipelines.get_dwarfs_from_elfs:GetDwarfsFromElfs + get_dwarfs_from_elfs = scanpipe.pipelines.get_dwarfs_from_elfs:InspectElfBinaries inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages load_inventory = scanpipe.pipelines.load_inventory:LoadInventory load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM From 31eae95aeef4450c743a1704b3f5ee77795baad6 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sun, 11 Feb 2024 23:13:42 +0530 Subject: [PATCH 05/15] Address review comments Signed-off-by: Tushar Goel --- CHANGELOG.rst | 3 +++ ...s_from_elfs.py => inspect_elf_binaries.py} | 2 +- scanpipe/tests/test_models.py | 19 ++++++++++++------- setup.cfg | 2 +- 4 files changed, 17 insertions(+), 9 deletions(-) rename scanpipe/pipelines/{get_dwarfs_from_elfs.py => inspect_elf_binaries.py} (97%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b96f596ba..e68406af4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -68,6 +68,9 @@ v33.1.0 (2024-02-02) `upload_file`. https://github.com/nexB/scancode.io/issues/708 +- Add an add-on pipeline for collecting DWARF debug symbol compilation + unit paths when available from elfs https://github.com/nexB/purldb/issues/260. + v33.0.0 (2024-01-16) -------------------- diff --git a/scanpipe/pipelines/get_dwarfs_from_elfs.py b/scanpipe/pipelines/inspect_elf_binaries.py similarity index 97% rename from scanpipe/pipelines/get_dwarfs_from_elfs.py rename to scanpipe/pipelines/inspect_elf_binaries.py index d31c449e6..dc3c14077 100644 --- a/scanpipe/pipelines/get_dwarfs_from_elfs.py +++ b/scanpipe/pipelines/inspect_elf_binaries.py @@ -39,7 +39,7 @@ def steps(cls): def collect_dwarf_source_path_references(self): """ - Update ``extra_data`` of project with + Update ``extra_data`` of elf files with dwarf data extracted from elf files. """ for elf in self.project.codebaseresources.elfs(): diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index dfd8af5dc..0ec953142 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2112,44 +2112,49 @@ def test_scanpipe_codebase_resource_queryset_has_directory_content_fingerprint( def test_scanpipe_codebase_resource_queryset_elfs(self): project = Project.objects.create(name="Test") - resource_starting_with_elf_and_executable_in_file_type = CodebaseResource.objects.create( + resource_starting_with_elf_and_executable_in_file_type = CodebaseResource( file_type="""ELF 32-bit LSB executable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="a", type=CodebaseResource.Type.FILE, ) - resource_with_executable_in_file_type = CodebaseResource.objects.create( + resource_starting_with_elf_and_executable_in_file_type.save() + resource_with_executable_in_file_type = CodebaseResource( file_type="""32-bit LSB executable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="b", type=CodebaseResource.Type.FILE, ) - resource_starting_with_elf_in_file_type = CodebaseResource.objects.create( + resource_with_executable_in_file_type.save() + resource_starting_with_elf_in_file_type = CodebaseResource( file_type="""ELF 32-bit LSB resourcable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="c", type=CodebaseResource.Type.FILE, ) - resource = CodebaseResource.objects.create( + resource_starting_with_elf_in_file_type.save() + resource = CodebaseResource( file_type="""32-bit LSB relocatable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="d", type=CodebaseResource.Type.FILE, ) - resource_starting_with_elf_and_relocatable_in_file_type = CodebaseResource.objects.create( + resource.save() + resource_starting_with_elf_and_relocatable_in_file_type = CodebaseResource( file_type="""ELF 32-bit LSB relocatable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", project=project, path="e", type=CodebaseResource.Type.FILE, ) + resource_starting_with_elf_and_relocatable_in_file_type.save() paths = [str(resource.path) for resource in project.codebaseresources.elfs()] - assert "e" in paths - assert "a" in paths + self.assertTrue("e" in paths) + self.assertTrue("a" in paths) class ScanPipeModelsTransactionTest(TransactionTestCase): diff --git a/setup.cfg b/setup.cfg index 182694058..323b67beb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -127,7 +127,7 @@ scancodeio_pipelines = analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities - get_dwarfs_from_elfs = scanpipe.pipelines.get_dwarfs_from_elfs:InspectElfBinaries + inspect_elf_binaries = scanpipe.pipelines.inspect_elf_binaries:InspectElfBinaries inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages load_inventory = scanpipe.pipelines.load_inventory:LoadInventory load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM From 00ffa8efb3422e7f764067f1346e85d2ac47ba3a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sun, 11 Feb 2024 23:57:27 +0530 Subject: [PATCH 06/15] Add documentation for pipeline Signed-off-by: Tushar Goel --- docs/built-in-pipelines.rst | 8 ++++++++ docs/faq.rst | 2 ++ 2 files changed, 10 insertions(+) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index f2e8a9810..f3d44ea7b 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -58,6 +58,14 @@ Find Vulnerabilities (addon) .. _pipeline_inspect_packages: +Inspect Elf Binaries +--------------------- +.. autoclass:: scanpipe.pipelines.inspect_elf_binaries.InspectElfBinaries() + :members: + :member-order: bysource + +.. _pipeline_inspect_elf: + Inspect Packages ---------------- .. autoclass:: scanpipe.pipelines.inspect_packages.InspectPackages() diff --git a/docs/faq.rst b/docs/faq.rst index 1b854b341..ac00f8b0b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -48,6 +48,8 @@ Here are some general guidelines based on different input scenarios: :ref:`inspect_packages ` pipeline. - For scenarios involving both a **development and deployment codebase**, consider using the :ref:`map_deploy_to_develop ` pipeline. +- For getting the DWARF debug symbol compilation unit paths when available from an elf binary. + use the :ref:`inspect_elf_binaries ` pipeline. These pipelines will automatically execute the necessary steps to scan and create the packages, dependencies, and resources for your project based on the input data provided. From 4a3a0f8e74fae37dc70b988370e1645b2d5376bc Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 17 Feb 2024 02:06:05 +0530 Subject: [PATCH 07/15] Fix built-in pipelines file Signed-off-by: Tushar Goel --- docs/built-in-pipelines.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index f3d44ea7b..4b2d5871e 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -56,7 +56,7 @@ Find Vulnerabilities (addon) :members: :member-order: bysource -.. _pipeline_inspect_packages: +.. _pipeline_inspect_elf: Inspect Elf Binaries --------------------- @@ -64,7 +64,7 @@ Inspect Elf Binaries :members: :member-order: bysource -.. _pipeline_inspect_elf: +.. _pipeline_inspect_packages: Inspect Packages ---------------- From 43ce108d5c271f6a10c70af9ced96a84c11ebb0a Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:08:28 +0100 Subject: [PATCH 08/15] Update built-in-pipelines.rst --- docs/built-in-pipelines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 4b2d5871e..796263746 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -59,7 +59,7 @@ Find Vulnerabilities (addon) .. _pipeline_inspect_elf: Inspect Elf Binaries ---------------------- +-------------------- .. autoclass:: scanpipe.pipelines.inspect_elf_binaries.InspectElfBinaries() :members: :member-order: bysource From add51cf1c6cb082766a878b8fbcb83500f562dda Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:10:26 +0100 Subject: [PATCH 09/15] Update models.py --- scanpipe/models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 8b43ebfe1..d800e3975 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1948,10 +1948,9 @@ def has_directory_content_fingerprint(self): def elfs(self): """ - Resources that are ``files`` and their filetype startswith `elf` and - contains any of these `executable`, `relocatable`, `shared object`. - Keep sync with contenttype implementation: - https://github.com/nexB/typecode/blob/92feb7be3a87c1b541e7034c3f9797c96bc52305/src/typecode/contenttype.py#L733 + Resources that are ``files`` and their filetype starts with "elf" and + contains any of these "executable", "relocatable", "shared object". + Keep sync with the content type implementation at ``typecode.contenttype``. """ return ( self.files() From a698b6c51106be1e519e41ee4b696a7ec0fbe2f8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 19 Feb 2024 17:31:48 +0530 Subject: [PATCH 10/15] Revert python-inspector version Signed-off-by: Tushar Goel --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 323b67beb..e2b0d54cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,7 +76,7 @@ install_requires = # FetchCode fetchcode-container==1.2.3.210512; sys_platform == "linux" # Inspectors - python-inspector==0.10.0 + python-inspector==0.11.0 elf-inspector==0.0.1 aboutcode-toolkit==10.1.0 # Utilities From 83f1c1fa333c03890c72c0962f92433c27dc3eb1 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 19 Feb 2024 18:01:22 +0530 Subject: [PATCH 11/15] Use ELF instead of elf Signed-off-by: Tushar Goel --- docs/built-in-pipelines.rst | 2 +- scanpipe/pipelines/inspect_elf_binaries.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 796263746..0179ba7da 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -58,7 +58,7 @@ Find Vulnerabilities (addon) .. _pipeline_inspect_elf: -Inspect Elf Binaries +Inspect ELF Binaries -------------------- .. autoclass:: scanpipe.pipelines.inspect_elf_binaries.InspectElfBinaries() :members: diff --git a/scanpipe/pipelines/inspect_elf_binaries.py b/scanpipe/pipelines/inspect_elf_binaries.py index dc3c14077..e32068540 100644 --- a/scanpipe/pipelines/inspect_elf_binaries.py +++ b/scanpipe/pipelines/inspect_elf_binaries.py @@ -39,8 +39,8 @@ def steps(cls): def collect_dwarf_source_path_references(self): """ - Update ``extra_data`` of elf files with - dwarf data extracted from elf files. + Update ``extra_data`` of ELF files with + dwarf data extracted from ELF files. """ for elf in self.project.codebaseresources.elfs(): dwarf_paths = get_dwarf_paths(Path(self.project.codebase_path / elf.path)) From 9b4dd7726caa3bfc0a3bf7fdf8b42dea03c85bb5 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 19 Feb 2024 20:58:07 +0530 Subject: [PATCH 12/15] Address review comments Signed-off-by: Tushar Goel --- scanpipe/models.py | 2 +- scanpipe/pipelines/inspect_elf_binaries.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index d800e3975..e9997e784 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1948,7 +1948,7 @@ def has_directory_content_fingerprint(self): def elfs(self): """ - Resources that are ``files`` and their filetype starts with "elf" and + Resources that are ``files`` and their filetype starts with "ELF" and contains any of these "executable", "relocatable", "shared object". Keep sync with the content type implementation at ``typecode.contenttype``. """ diff --git a/scanpipe/pipelines/inspect_elf_binaries.py b/scanpipe/pipelines/inspect_elf_binaries.py index e32068540..1d109a612 100644 --- a/scanpipe/pipelines/inspect_elf_binaries.py +++ b/scanpipe/pipelines/inspect_elf_binaries.py @@ -27,7 +27,7 @@ from scanpipe.pipelines import Pipeline -class InspectElfBinaries(Pipeline): +class InspectELFBinaries(Pipeline): """Inspect ELF binaries and collect DWARF paths.""" download_inputs = False From 5d0d4dff8bb09457b6f866134652ad916b791b4a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 19 Feb 2024 23:22:09 +0530 Subject: [PATCH 13/15] Fix tests Signed-off-by: Tushar Goel --- scanpipe/models.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index e9997e784..d55f08d91 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1955,7 +1955,7 @@ def elfs(self): return ( self.files() .filter( - file_type__istartswith="elf", + file_type__istartswith="ELF", ) .filter( Q(file_type__icontains="executable") diff --git a/setup.cfg b/setup.cfg index e2b0d54cc..9cd03e843 100644 --- a/setup.cfg +++ b/setup.cfg @@ -127,7 +127,7 @@ scancodeio_pipelines = analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities - inspect_elf_binaries = scanpipe.pipelines.inspect_elf_binaries:InspectElfBinaries + inspect_elf_binaries = scanpipe.pipelines.inspect_elf_binaries:InspectELFBinaries inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages load_inventory = scanpipe.pipelines.load_inventory:LoadInventory load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM From b1416a458a3c146141543fd3e4ee214237b8f84d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 19 Feb 2024 23:25:58 +0530 Subject: [PATCH 14/15] Correct reference in builtin-pipelines.rst Signed-off-by: Tushar Goel --- docs/built-in-pipelines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 0179ba7da..7cbe75458 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -60,7 +60,7 @@ Find Vulnerabilities (addon) Inspect ELF Binaries -------------------- -.. autoclass:: scanpipe.pipelines.inspect_elf_binaries.InspectElfBinaries() +.. autoclass:: scanpipe.pipelines.inspect_elf_binaries.InspectELFBinaries() :members: :member-order: bysource From 52817aecd379be736d97708e3f5deee0abf3fc0b Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 19 Feb 2024 19:05:00 +0100 Subject: [PATCH 15/15] Update CHANGELOG.rst --- CHANGELOG.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e68406af4..6f861e6b2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,6 +34,10 @@ v33.2.0 (unreleased) - Remove "packageFileName" entry from SPDX output. https://github.com/nexB/scancode.io/issues/1076 +- Add an add-on pipeline for collecting DWARF debug symbol compilation + unit paths when available from elfs. + https://github.com/nexB/purldb/issues/260 + v33.1.0 (2024-02-02) -------------------- @@ -68,9 +72,6 @@ v33.1.0 (2024-02-02) `upload_file`. https://github.com/nexB/scancode.io/issues/708 -- Add an add-on pipeline for collecting DWARF debug symbol compilation - unit paths when available from elfs https://github.com/nexB/purldb/issues/260. - v33.0.0 (2024-01-16) --------------------