diff --git a/scancodeio/settings.py b/scancodeio/settings.py
index 9ac7b43e7..ef932f5e7 100644
--- a/scancodeio/settings.py
+++ b/scancodeio/settings.py
@@ -110,6 +110,7 @@
"resource": 100,
"package": 100,
"dependency": 100,
+ "license": 100,
"relation": 100,
},
)
diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py
index fe7c1f4ea..63cfca2ea 100644
--- a/scanpipe/api/serializers.py
+++ b/scanpipe/api/serializers.py
@@ -31,6 +31,7 @@
from scanpipe.models import CodebaseRelation
from scanpipe.models import CodebaseResource
from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
from scanpipe.models import DiscoveredPackage
from scanpipe.models import InputSource
from scanpipe.models import Project
@@ -466,6 +467,20 @@ class Meta:
]
+class DiscoveredLicenseSerializer(serializers.ModelSerializer):
+ compliance_alert = serializers.CharField()
+
+ class Meta:
+ model = DiscoveredLicense
+ fields = [
+ "detection_count",
+ "identifier",
+ "license_expression",
+ "license_expression_spdx",
+ "compliance_alert",
+ ]
+
+
class CodebaseRelationSerializer(serializers.ModelSerializer):
from_resource = serializers.ReadOnlyField(source="from_resource.path")
to_resource = serializers.ReadOnlyField(source="to_resource.path")
@@ -524,6 +539,7 @@ def get_model_serializer(model_class):
CodebaseResource: CodebaseResourceSerializer,
DiscoveredPackage: DiscoveredPackageSerializer,
DiscoveredDependency: DiscoveredDependencySerializer,
+ DiscoveredLicense: DiscoveredLicenseSerializer,
CodebaseRelation: CodebaseRelationSerializer,
ProjectMessage: ProjectMessageSerializer,
}.get(model_class, None)
diff --git a/scanpipe/filters.py b/scanpipe/filters.py
index 2f6cf7195..c032fbd9e 100644
--- a/scanpipe/filters.py
+++ b/scanpipe/filters.py
@@ -39,6 +39,7 @@
from scanpipe.models import CodebaseRelation
from scanpipe.models import CodebaseResource
from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
from scanpipe.models import DiscoveredPackage
from scanpipe.models import Project
from scanpipe.models import ProjectMessage
@@ -529,6 +530,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
"related_from__from_resource__path",
],
)
+
compliance_alert = django_filters.ChoiceFilter(
choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
)
@@ -587,8 +589,8 @@ class Meta:
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- license_expression_filer = self.filters["detected_license_expression"]
- license_expression_filer.extra["widget"] = HasValueDropdownWidget()
+ license_expression_filter = self.filters["detected_license_expression"]
+ license_expression_filter.extra["widget"] = HasValueDropdownWidget()
class IsVulnerable(django_filters.ChoiceFilter):
@@ -626,6 +628,19 @@ def filter(self, qs, value):
return qs.filter(lookups)
+class DiscoveredLicenseSearchFilter(QuerySearchFilter):
+ def filter(self, qs, value):
+ if not value:
+ return qs
+
+ search_fields = ["license_expression", "license_expression_spdx"]
+ lookups = Q()
+ for field_names in search_fields:
+ lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value})
+
+ return qs.filter(lookups)
+
+
class GroupOrderingFilter(django_filters.OrderingFilter):
"""Add the ability to provide a group a fields to order by."""
@@ -801,6 +816,44 @@ class Meta:
]
+class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+ dropdown_widget_fields = [
+ "compliance_alert",
+ "license_expression",
+ "license_expression_spdx",
+ ]
+
+ search = DiscoveredLicenseSearchFilter(
+ label="Search", field_name="name", lookup_expr="icontains"
+ )
+ sort = GroupOrderingFilter(
+ label="Sort",
+ fields=[
+ "detection_count",
+ "identifier",
+ "license_expression",
+ "license_expression_spdx",
+ "compliance_alert",
+ ],
+ )
+ license_expression = django_filters.AllValuesFilter()
+ license_expression_spdx = django_filters.AllValuesFilter()
+ compliance_alert = django_filters.ChoiceFilter(
+ choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
+ )
+
+ class Meta:
+ model = DiscoveredLicense
+ fields = [
+ "search",
+ "identifier",
+ "detection_count",
+ "license_expression",
+ "license_expression_spdx",
+ "compliance_alert",
+ ]
+
+
class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
search = QuerySearchFilter(
label="Search", field_name="description", lookup_expr="icontains"
diff --git a/scanpipe/migrations/0070_discovered_license_models.py b/scanpipe/migrations/0070_discovered_license_models.py
new file mode 100644
index 000000000..88f4ac3e6
--- /dev/null
+++ b/scanpipe/migrations/0070_discovered_license_models.py
@@ -0,0 +1,143 @@
+# Generated by Django 5.0.2 on 2024-03-17 12:38
+
+import django.db.models.deletion
+import scanpipe.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("scanpipe", "0069_project_purl"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="DiscoveredLicense",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "compliance_alert",
+ models.CharField(
+ blank=True,
+ choices=[
+ ("ok", "Ok"),
+ ("warning", "Warning"),
+ ("error", "Error"),
+ ("missing", "Missing"),
+ ],
+ editable=False,
+ help_text="Indicates how the license expression complies with provided policies.",
+ max_length=10,
+ ),
+ ),
+ (
+ "license_expression",
+ models.TextField(
+ blank=True,
+ help_text="A license expression string using the SPDX license expression syntax and ScanCode license keys, the effective license expression for this license detection.",
+ ),
+ ),
+ (
+ "license_expression_spdx",
+ models.TextField(
+ blank=True,
+ help_text="SPDX license expression string with SPDX ids.",
+ ),
+ ),
+ (
+ "matches",
+ models.JSONField(
+ blank=True,
+ default=list,
+ help_text="List of license matches combined in this detection.",
+ verbose_name="Reference Matches",
+ ),
+ ),
+ (
+ "detection_log",
+ models.JSONField(
+ blank=True,
+ default=list,
+ help_text="A list of detection DetectionRule explaining how this detection was created.",
+ ),
+ ),
+ (
+ "identifier",
+ models.CharField(
+ blank=True,
+ help_text="An identifier unique for a license detection, containing the license expression and a UUID crafted from the match contents.",
+ max_length=1024,
+ ),
+ ),
+ (
+ "detection_count",
+ models.BigIntegerField(
+ blank=True,
+ help_text="Total number of this license detection discovered.",
+ null=True,
+ ),
+ ),
+ (
+ "file_regions",
+ models.JSONField(
+ blank=True,
+ default=list,
+ help_text="A list of file regions with resource path, start and end line details for each place this license detection was discovered at. Also contains whether this license was discovered from a file or from package metadata.",
+ verbose_name="Detection Locations",
+ ),
+ ),
+ (
+ "project",
+ models.ForeignKey(
+ editable=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="%(class)ss",
+ to="scanpipe.project",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["detection_count", "identifier"],
+ "indexes": [
+ models.Index(
+ fields=["identifier"], name="scanpipe_di_identif_b533f3_idx"
+ ),
+ models.Index(
+ fields=["license_expression"],
+ name="scanpipe_di_license_33d11a_idx",
+ ),
+ models.Index(
+ fields=["license_expression_spdx"],
+ name="scanpipe_di_license_eb5e9d_idx",
+ ),
+ models.Index(
+ fields=["detection_count"],
+ name="scanpipe_di_detecti_d87ff1_idx",
+ ),
+ ],
+ },
+ bases=(
+ scanpipe.models.UpdateMixin,
+ scanpipe.models.SaveProjectMessageMixin,
+ scanpipe.models.UpdateFromDataMixin,
+ models.Model,
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="discoveredlicense",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("identifier", ""), _negated=True),
+ fields=("project", "identifier"),
+ name="scanpipe_discoveredlicense_unique_license_id_within_project",
+ ),
+ ),
+ ]
diff --git a/scanpipe/models.py b/scanpipe/models.py
index 0afce189a..5a5a4914e 100644
--- a/scanpipe/models.py
+++ b/scanpipe/models.py
@@ -653,6 +653,7 @@ def delete_related_objects(self):
self.projectmessages,
self.codebaserelations,
self.discovereddependencies,
+ self.discoveredlicenses,
self.codebaseresources,
self.runs,
self.inputsources,
@@ -1406,6 +1407,35 @@ def dependency_count(self):
"""Return the number of dependencies related to this project."""
return self.discovereddependencies.count()
+ @cached_property
+ def license_detections_count(self):
+ """Return the number of license detections in this project."""
+ return self.discoveredlicenses.count()
+
+ @cached_property
+ def package_compliance_alert_count(self):
+ """
+ Return the number of packages related to this project which have
+ a license compliance error alert.
+ """
+ return self.discoveredpackages.has_compliance_alert().count()
+
+ @cached_property
+ def license_compliance_alert_count(self):
+ """
+ Return the number of license detections related to this project
+ which have a license compliance error alert.
+ """
+ return self.discoveredlicenses.has_compliance_alert().count()
+
+ @cached_property
+ def resource_compliance_alert_count(self):
+ """
+ Return the number of codebase resources related to this project which have
+ a license compliance error alert.
+ """
+ return self.codebaseresources.has_compliance_alert().count()
+
@cached_property
def message_count(self):
"""Return the number of messages related to this project."""
@@ -2143,6 +2173,9 @@ def compliance_issues(self, severity):
return self.filter(compliance_alert__in=severity_mapping[severity])
+ def has_compliance_alert(self):
+ return self.filter(Q(compliance_alert__exact=CodebaseResource.Compliance.ERROR))
+
def convert_glob_to_django_regex(glob_pattern):
"""
@@ -2450,6 +2483,17 @@ class Compliance(models.TextChoices):
class Meta:
abstract = True
+ @property
+ def has_compliance_alert(self):
+ """
+ Returns True if this instance has a compliance alert of `ERROR`
+ for it's respective license_expression fields.
+ """
+ if self.compliance_alert == self.Compliance.ERROR:
+ return True
+
+ return False
+
@classmethod
def from_db(cls, db, field_names, values):
"""
@@ -3072,6 +3116,12 @@ def with_resources_count(self):
)
return self.annotate(resources_count=count_subquery)
+ def has_license_detections(self):
+ return self.filter(~Q(license_detections=[]) | ~Q(other_license_detections=[]))
+
+ def has_no_license_detections(self):
+ return self.filter(Q(license_detections=[]) & Q(other_license_detections=[]))
+
def only_package_url_fields(self):
"""
Only select and return the UUID and PURL fields.
@@ -3964,6 +4014,180 @@ def as_spdx(self):
)
+class DiscoveredLicenseQuerySet(
+ ComplianceAlertQuerySetMixin,
+ ProjectRelatedQuerySet,
+):
+ def order_by_count_and_expression(self):
+ """Order by detection count and license expression (identifer) fields."""
+ return self.order_by("-detection_count", "identifier")
+
+
+class AbstractLicenseDetection(models.Model):
+ """
+ These fields should be kept in line with
+ `licensedcode.detection.LicenseDetection`.
+ """
+
+ license_expression = models.TextField(
+ blank=True,
+ help_text=_(
+ "A license expression string using the SPDX license expression"
+ " syntax and ScanCode license keys, the effective license expression"
+ " for this license detection."
+ ),
+ )
+
+ license_expression_spdx = models.TextField(
+ blank=True,
+ help_text=_("SPDX license expression string with SPDX ids."),
+ )
+
+ matches = models.JSONField(
+ _("Reference Matches"),
+ default=list,
+ blank=True,
+ help_text=_("List of license matches combined in this detection."),
+ )
+
+ detection_log = models.JSONField(
+ default=list,
+ blank=True,
+ help_text=_(
+ "A list of detection DetectionRule explaining how "
+ "this detection was created."
+ ),
+ )
+
+ identifier = models.CharField(
+ max_length=1024,
+ blank=True,
+ help_text=_(
+ "An identifier unique for a license detection, containing the license "
+ "expression and a UUID crafted from the match contents."
+ ),
+ )
+
+ class Meta:
+ abstract = True
+
+
+class DiscoveredLicense(
+ ProjectRelatedModel,
+ SaveProjectMessageMixin,
+ UpdateFromDataMixin,
+ ComplianceAlertMixin,
+ AbstractLicenseDetection,
+):
+ """
+ A project's Discovered Licenses are the unique License Detection objects
+ discovered in the code under analysis.
+ """
+
+ license_expression_field = "license_expression"
+
+ # If this license was discovered in a extracted license statement
+ # this is True, and False if this was discovered in a file.
+ from_package = None
+
+ detection_count = models.BigIntegerField(
+ blank=True,
+ null=True,
+ help_text=_("Total number of this license detection discovered."),
+ )
+
+ file_regions = models.JSONField(
+ _("Detection Locations"),
+ default=list,
+ blank=True,
+ help_text=_(
+ "A list of file regions with resource path, start and end line "
+ "details for each place this license detection was discovered at. "
+ "Also contains whether this license was discovered from a file or "
+ "from package metadata."
+ ),
+ )
+
+ objects = DiscoveredLicenseQuerySet.as_manager()
+
+ class Meta:
+ ordering = ["detection_count", "identifier"]
+ indexes = [
+ models.Index(fields=["identifier"]),
+ models.Index(fields=["license_expression"]),
+ models.Index(fields=["license_expression_spdx"]),
+ models.Index(fields=["detection_count"]),
+ ]
+ constraints = [
+ models.UniqueConstraint(
+ fields=["project", "identifier"],
+ condition=~Q(identifier=""),
+ name="%(app_label)s_%(class)s_unique_license_id_within_project",
+ ),
+ ]
+
+ def __str__(self):
+ return self.identifier
+
+ @classmethod
+ def create_from_data(cls, project, detection_data):
+ """
+ Create and returns a DiscoveredLicense for a `project` from the
+ `detection_data`. If one of the values of the required fields is not
+ available, a "ProjectMessage" is created instead of a new
+ DiscoveredLicense instance.
+ """
+ detection_data = detection_data.copy()
+ required_fields = ["license_expression", "identifier", "matches"]
+ missing_values = [
+ field_name
+ for field_name in required_fields
+ if not detection_data.get(field_name)
+ ]
+
+ if missing_values:
+ message = (
+ f"No values for the following required fields: "
+ f"{', '.join(missing_values)}"
+ )
+
+ project.add_warning(
+ description=message,
+ model=cls,
+ details=detection_data,
+ )
+ return
+
+ cleaned_data = {
+ field_name: value
+ for field_name, value in detection_data.items()
+ if field_name in cls.model_fields() and value not in EMPTY_VALUES
+ }
+
+ discovered_license = cls(project=project, **cleaned_data)
+ # Using save_error=False to not capture potential errors at this level but
+ # rather in the CodebaseResource.create_and_add_license_data method so
+ # resource data can be injected in the ProjectMessage record.
+ discovered_license.save(save_error=False, capture_exception=False)
+ return discovered_license
+
+ def update_with_file_region(self, file_region, count_detection):
+ """
+ If the `file_region` is a new file region, include it in the
+ `file_regions` list and increase the `detection_count` by 1.
+ """
+ file_region_data = file_region.to_dict()
+ if file_region_data not in self.file_regions:
+ self.file_regions.append(file_region_data)
+ if count_detection:
+ if not self.detection_count:
+ self.detection_count = 1
+ else:
+ self.detection_count += 1
+
+ self.save(update_fields=["detection_count", "file_regions"])
+
+
def normalize_package_url_data(purl_mapping, ignore_nulls=False):
"""
Normalize a mapping of purl data so database queries with
diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py
index ea74f93e8..1dae9662d 100644
--- a/scanpipe/pipelines/deploy_to_develop.py
+++ b/scanpipe/pipelines/deploy_to_develop.py
@@ -85,6 +85,7 @@ def steps(cls):
cls.remove_packages_without_resources,
cls.scan_unmapped_to_files,
cls.scan_mapped_from_for_files,
+ cls.collect_and_create_license_detections,
cls.flag_deployed_from_resources_with_missing_license,
cls.create_local_files_packages,
)
@@ -298,6 +299,13 @@ def scan_mapped_from_for_files(self):
scan_files = d2d.get_from_files_for_scanning(self.project.codebaseresources)
scancode.scan_for_files(self.project, scan_files, progress_logger=self.log)
+ def collect_and_create_license_detections(self):
+ """
+ Collect and create unique license detections from resources and
+ package data.
+ """
+ scancode.collect_and_create_license_detections(project=self.project)
+
def create_local_files_packages(self):
"""Create local-files packages for codebase resources not part of a package."""
d2d.create_local_files_packages(self.project)
diff --git a/scanpipe/pipelines/docker.py b/scanpipe/pipelines/docker.py
index 5bf338458..877548597 100644
--- a/scanpipe/pipelines/docker.py
+++ b/scanpipe/pipelines/docker.py
@@ -42,6 +42,7 @@ def steps(cls):
cls.flag_ignored_resources,
cls.scan_for_application_packages,
cls.scan_for_files,
+ cls.collect_and_create_license_detections,
cls.analyze_scanned_files,
cls.flag_not_analyzed_codebase_resources,
)
diff --git a/scanpipe/pipelines/docker_windows.py b/scanpipe/pipelines/docker_windows.py
index 98684da13..768f279c0 100644
--- a/scanpipe/pipelines/docker_windows.py
+++ b/scanpipe/pipelines/docker_windows.py
@@ -45,6 +45,7 @@ def steps(cls):
cls.flag_ignored_resources,
cls.scan_for_application_packages,
cls.scan_for_files,
+ cls.collect_and_create_license_detections,
cls.analyze_scanned_files,
cls.flag_data_files_with_no_clues,
cls.flag_not_analyzed_codebase_resources,
diff --git a/scanpipe/pipelines/root_filesystem.py b/scanpipe/pipelines/root_filesystem.py
index 76478ce6d..9a0fa9ef8 100644
--- a/scanpipe/pipelines/root_filesystem.py
+++ b/scanpipe/pipelines/root_filesystem.py
@@ -45,6 +45,7 @@ def steps(cls):
cls.scan_for_application_packages,
cls.match_not_analyzed_to_system_packages,
cls.scan_for_files,
+ cls.collect_and_create_license_detections,
cls.analyze_scanned_files,
cls.flag_not_analyzed_codebase_resources,
)
@@ -122,6 +123,13 @@ def scan_for_files(self):
"""Scan unknown resources for copyrights, licenses, emails, and urls."""
scancode.scan_for_files(self.project, progress_logger=self.log)
+ def collect_and_create_license_detections(self):
+ """
+ Collect and create unique license detections from resources and
+ package data.
+ """
+ scancode.collect_and_create_license_detections(project=self.project)
+
def analyze_scanned_files(self):
"""Analyze single file scan results for completeness."""
flag.analyze_scanned_files(self.project)
diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py
index d5bbe992c..fd6580e45 100644
--- a/scanpipe/pipelines/scan_codebase.py
+++ b/scanpipe/pipelines/scan_codebase.py
@@ -45,6 +45,7 @@ def steps(cls):
cls.flag_ignored_resources,
cls.scan_for_application_packages,
cls.scan_for_files,
+ cls.collect_and_create_license_detections,
)
def copy_inputs_to_codebase_directory(self):
@@ -65,3 +66,10 @@ def scan_for_application_packages(self):
def scan_for_files(self):
"""Scan unknown resources for copyrights, licenses, emails, and urls."""
scancode.scan_for_files(self.project, progress_logger=self.log)
+
+ def collect_and_create_license_detections(self):
+ """
+ Collect and create unique license detections from resources and
+ package data.
+ """
+ scancode.collect_and_create_license_detections(project=self.project)
diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py
index df1b5ecc3..3ed78f27d 100644
--- a/scanpipe/pipes/__init__.py
+++ b/scanpipe/pipes/__init__.py
@@ -36,6 +36,7 @@
from scanpipe.models import CodebaseRelation
from scanpipe.models import CodebaseResource
from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
from scanpipe.models import DiscoveredPackage
from scanpipe.pipes import scancode
@@ -307,6 +308,78 @@ def update_or_create_dependency(
return dependency
+def update_or_create_license_detection(
+ project,
+ detection_data,
+ resource_path=None,
+ from_package=False,
+ count_detection=True,
+):
+ """
+ Get, update or create a DiscoveredLicense object then return it.
+ Use the `project` and `detection_data` mapping to lookup and creates the
+ DiscoveredLicense using its detection identifier as a unique key.
+
+ Additonally if `resource_path` is passed, add the file region where
+ the license was detected to the DiscoveredLicense object, if not present
+ already. `from_package` is True if the license detection was in a
+ `extracted_license_statement` from a package metadata.
+ """
+ detection_identifier = detection_data["identifier"]
+
+ license_detection = project.discoveredlicenses.get_or_none(
+ identifier=detection_identifier,
+ )
+ detection_data = _clean_license_detection_data(detection_data)
+
+ if license_detection:
+ license_detection.update_from_data(detection_data)
+ else:
+ license_detection = DiscoveredLicense.create_from_data(
+ project,
+ detection_data,
+ )
+
+ if not license_detection:
+ project.add_error(
+ model="update_or_create_license_detection",
+ details=detection_data,
+ resource=resource_path,
+ )
+ return
+
+ if resource_path:
+ file_region = scancode.get_file_region(
+ detection_data=detection_data,
+ resource_path=resource_path,
+ )
+ license_detection.update_with_file_region(
+ file_region=file_region,
+ count_detection=count_detection,
+ )
+
+ license_detection.from_package = from_package
+ return license_detection
+
+
+def _clean_license_detection_data(detection_data):
+ detection_data = detection_data.copy()
+ if "reference_matches" in detection_data:
+ matches = detection_data.pop("reference_matches")
+ detection_data["matches"] = matches
+
+ updated_matches = []
+ for match_data in detection_data["matches"]:
+ from_file_path = match_data["from_file"]
+ if from_file_path:
+ match_data["from_file"] = from_file_path.removeprefix("codebase/")
+
+ updated_matches.append(match_data)
+
+ detection_data["matches"] = updated_matches
+ return detection_data
+
+
def get_dependencies(project, dependency_data):
"""
Given a `dependency_data` mapping, get a list of DiscoveredDependency objects
diff --git a/scanpipe/pipes/input.py b/scanpipe/pipes/input.py
index f57e7eafd..0a1535c37 100644
--- a/scanpipe/pipes/input.py
+++ b/scanpipe/pipes/input.py
@@ -34,6 +34,7 @@
from scanpipe.models import CodebaseRelation
from scanpipe.models import CodebaseResource
from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
from scanpipe.models import DiscoveredPackage
from scanpipe.pipes import scancode
from scanpipe.pipes.output import mappings_key_by_fieldname
@@ -85,10 +86,11 @@ def is_archive(location):
def load_inventory_from_toolkit_scan(project, input_location):
"""
- Create packages, dependencies, and resources loaded from the ScanCode-toolkit scan
- results located at ``input_location``.
+ Create license detections, packages, dependencies, and resources
+ loaded from the ScanCode-toolkit scan results located at ``input_location``.
"""
scanned_codebase = scancode.get_virtual_codebase(project, input_location)
+ scancode.create_discovered_licenses(project, scanned_codebase)
scancode.create_discovered_packages(project, scanned_codebase)
scancode.create_codebase_resources(project, scanned_codebase)
scancode.create_discovered_dependencies(
@@ -98,9 +100,12 @@ def load_inventory_from_toolkit_scan(project, input_location):
def load_inventory_from_scanpipe(project, scan_data):
"""
- Create packages, dependencies, resources, and relations loaded from a ScanCode.io
- JSON output provided as ``scan_data``.
+ Create license detections, packages, dependencies, resources, and relations
+ loaded from a ScanCode.io JSON output provided as ``scan_data``.
"""
+ for detection_data in scan_data.get("license_detections", []):
+ pipes.update_or_create_license_detection(project, detection_data)
+
for package_data in scan_data.get("packages", []):
pipes.update_or_create_package(project, package_data)
@@ -117,12 +122,14 @@ def load_inventory_from_scanpipe(project, scan_data):
model_to_object_maker_func = {
DiscoveredPackage: pipes.update_or_create_package,
DiscoveredDependency: pipes.update_or_create_dependency,
+ DiscoveredLicense: pipes.update_or_create_license_detection,
CodebaseResource: pipes.update_or_create_resource,
CodebaseRelation: pipes.get_or_create_relation,
}
worksheet_name_to_model = {
"PACKAGES": DiscoveredPackage,
+ "LICENSE_DETECTIONS": DiscoveredLicense,
"RESOURCES": CodebaseResource,
"DEPENDENCIES": DiscoveredDependency,
"RELATIONS": CodebaseRelation,
diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py
index c9812bc97..d5d34477f 100644
--- a/scanpipe/pipes/scancode.py
+++ b/scanpipe/pipes/scancode.py
@@ -38,6 +38,7 @@
from commoncode import fileutils
from commoncode.resource import VirtualCodebase
from extractcode import api as extractcode_api
+from licensedcode.detection import FileRegion
from packagedcode import get_package_handler
from packagedcode import models as packagedcode_models
from scancode import Scanner
@@ -435,6 +436,62 @@ def add_resource_to_package(package_uid, resource, project):
resource.discovered_packages.add(package)
+def collect_and_create_license_detections(project):
+ """
+ Create instances of DiscoveredLicense for `project` from the parsed
+ license detections present in the CodebaseResources and
+ DiscoveredPackages of `project`.
+ """
+ logger.info(f"Project {project} collect_license_detections:")
+
+ for resource in project.codebaseresources.has_license_detections():
+ logger.info(f" Processing: {resource.path} for licenses")
+
+ for detection_data in resource.license_detections:
+ pipes.update_or_create_license_detection(
+ project=project,
+ detection_data=detection_data,
+ resource_path=resource.path,
+ )
+
+ for resource in project.codebaseresources.has_package_data():
+ for package_mapping in resource.package_data:
+ package_data = packagedcode_models.PackageData.from_dict(
+ mapping=package_mapping,
+ )
+
+ for detection in package_data.license_detections:
+ pipes.update_or_create_license_detection(
+ project=project,
+ detection_data=detection,
+ resource_path=resource.path,
+ from_package=True,
+ )
+
+ for detection in package_data.other_license_detections:
+ pipes.update_or_create_license_detection(
+ project=project,
+ detection_data=detection,
+ resource_path=resource.path,
+ from_package=True,
+ )
+
+
+def get_file_region(detection_data, resource_path):
+ """
+ From a LicenseDetection mapping `detection_data`, create a FileRegion
+ object containing information about where this license was detected
+ exactly in a codebase, with `resource_path`, with start and end lines.
+ """
+ start_line = min([match["start_line"] for match in detection_data["matches"]])
+ end_line = max([match["end_line"] for match in detection_data["matches"]])
+ return FileRegion(
+ path=resource_path,
+ start_line=start_line,
+ end_line=end_line,
+ )
+
+
def assemble_packages(project):
"""
Create instances of DiscoveredPackage and DiscoveredDependency for `project`
@@ -795,6 +852,32 @@ def create_codebase_resources(project, scanned_codebase):
discovered_package=package,
)
+ license_detections = getattr(scanned_resource, "license_detections", [])
+ for detection_data in license_detections:
+ detection_identifier = detection_data.get("identifier")
+ pipes.update_or_create_license_detection(
+ project=project,
+ detection_data=detection_data,
+ resource_path=resource_path,
+ count_detection=False,
+ )
+ logger.debug(f"Add {codebase_resource} to {detection_identifier}")
+
+ packages = getattr(scanned_resource, "package_data", [])
+ for package_data in packages:
+ license_detections = package_data.get("license_detections", [])
+ license_detections.extend(package_data.get("other_license_detections", []))
+ for detection_data in license_detections:
+ detection_identifier = detection_data.get("identifier")
+ pipes.update_or_create_license_detection(
+ project=project,
+ detection_data=detection_data,
+ resource_path=resource_path,
+ count_detection=False,
+ from_package=True,
+ )
+ logger.debug(f"Add {codebase_resource} to {detection_identifier}")
+
def create_discovered_packages(project, scanned_codebase):
"""
@@ -804,6 +887,16 @@ def create_discovered_packages(project, scanned_codebase):
if hasattr(scanned_codebase.attributes, "packages"):
for package_data in scanned_codebase.attributes.packages:
pipes.update_or_create_package(project, package_data)
+ license_detections = package_data.get("license_detections", [])
+ license_detections.extend(package_data.get("other_license_detections", []))
+
+ for license_detection in license_detections:
+ pipes.update_or_create_license_detection(
+ project=project,
+ detection_data=license_detection,
+ from_package=True,
+ count_detection=False,
+ )
def create_discovered_dependencies(
@@ -829,6 +922,17 @@ def create_discovered_dependencies(
)
+def create_discovered_licenses(project, scanned_codebase):
+ """
+ Save the license detections of a ScanCode `scanned_codebase`
+ scancode.resource.Codebase object to the database as a DiscoveredLicense of
+ `project`.
+ """
+ if hasattr(scanned_codebase.attributes, "license_detections"):
+ for detection_data in scanned_codebase.attributes.license_detections:
+ pipes.update_or_create_license_detection(project, detection_data)
+
+
def set_codebase_resource_for_package(codebase_resource, discovered_package):
"""
Assign the `discovered_package` to the `codebase_resource` and set its
diff --git a/scanpipe/templates/scanpipe/includes/project_summary_level.html b/scanpipe/templates/scanpipe/includes/project_summary_level.html
index 2dad4e586..e5c36665b 100644
--- a/scanpipe/templates/scanpipe/includes/project_summary_level.html
+++ b/scanpipe/templates/scanpipe/includes/project_summary_level.html
@@ -14,6 +14,12 @@
{% endif %}
+ {% if project.package_compliance_alert_count %}
+
+ {{ project.package_compliance_alert_count|intcomma }}
+
+
+ {% endif %}
{% else %}
0
{% endif %}
@@ -43,11 +49,17 @@
Resources
-
+
{% if project.resource_count %}
{{ project.resource_count|intcomma }}
+ {% if project.resource_compliance_alert_count %}
+
+ {{ project.resource_compliance_alert_count|intcomma }}
+
+
+ {% endif %}
{% else %}
0
{% endif %}
diff --git a/scanpipe/templates/scanpipe/license_detection_detail.html b/scanpipe/templates/scanpipe/license_detection_detail.html
new file mode 100644
index 000000000..eec34ad59
--- /dev/null
+++ b/scanpipe/templates/scanpipe/license_detection_detail.html
@@ -0,0 +1,13 @@
+{% extends "scanpipe/base.html" %}
+{% load static %}
+
+{% block title %}ScanCode.io: {{ project.name }} - {{ object.name }}{% endblock %}
+
+{% block content %}
+
+ {% include 'scanpipe/includes/navbar_header.html' %}
+
{% include 'scanpipe/includes/messages.html' %}
+ {% include 'scanpipe/includes/breadcrumb_detail_view.html' with object_title=object.identifier url_name="project_licenses" %}
+ {% include 'scanpipe/tabset/tabset.html' %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/license_detection_list.html b/scanpipe/templates/scanpipe/license_detection_list.html
new file mode 100644
index 000000000..6b5ced789
--- /dev/null
+++ b/scanpipe/templates/scanpipe/license_detection_list.html
@@ -0,0 +1,64 @@
+{% extends "scanpipe/base.html" %}
+
+{% block title %}ScanCode.io: {{ project.name }} - License Detections{% endblock %}
+
+{% block content %}
+
+
+
+
+
+ {% if is_paginated %}
+ {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %}
+ {% endif %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html
index ee415673f..95b2ae922 100644
--- a/scanpipe/templates/scanpipe/package_list.html
+++ b/scanpipe/templates/scanpipe/package_list.html
@@ -30,6 +30,11 @@
{% endif %}
+ {% if package.has_compliance_alert %}
+
+
+
+ {% endif %}
diff --git a/scanpipe/templates/scanpipe/panels/license_detections_summary.html b/scanpipe/templates/scanpipe/panels/license_detections_summary.html
new file mode 100644
index 000000000..91bba8086
--- /dev/null
+++ b/scanpipe/templates/scanpipe/panels/license_detections_summary.html
@@ -0,0 +1,31 @@
+{% load humanize %}
+{% if license_detection_summary %}
+
+{% endif %}
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/panels/resource_license_summary.html b/scanpipe/templates/scanpipe/panels/resource_license_summary.html
deleted file mode 100644
index 7a0f72388..000000000
--- a/scanpipe/templates/scanpipe/panels/resource_license_summary.html
+++ /dev/null
@@ -1,16 +0,0 @@
-{% load humanize %}
-{% if resource_license_summary %}
-
-{% endif %}
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html
index 52c11c1a2..1def8b4be 100644
--- a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html
+++ b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html
@@ -6,15 +6,80 @@
- {% for field_label, values in scan_summary.items %}
-
-
- {{ field_label }}
- |
-
-
- {% for entry in values %}
- {% if entry.value %}
+
+
+ Declared license
+ |
+
+
+ {% for entry in scan_summary.declared_license_expression %}
+ {% if entry.value %}
+ -
+ {{ entry.value }}
+ {% if entry.count %}
+
+ {{ entry.count|intcomma }}
+
+ {% endif %}
+
+ {% endif %}
+ {% endfor %}
+
+ |
+
+
+
+ Declared holder
+ |
+
+
+ {% for entry in scan_summary.declared_holder %}
+ {% if entry.value %}
+ -
+ {{ entry.value }}
+ {% if entry.count %}
+
+ {{ entry.count|intcomma }}
+
+ {% endif %}
+
+ {% endif %}
+ {% endfor %}
+
+ |
+
+
+
+ Primary language
+ |
+
+
+ |
+
+
+
+ Other licenses
+ |
+
+
- |
-
- {% endfor %}
+
+ {% endif %}
+ {% endfor %}
+
+ |
+
+
+
+ Other holders
+ |
+
+
+ {% for entry in scan_summary.other_holders %}
+ {% if entry.value %}
+ -
+ {{ entry.value }}
+ {% if entry.count %}
+
+ {{ entry.count|intcomma }}
+
+ {% endif %}
+
+ {% endif %}
+ {% endfor %}
+
+ |
+
+
+
+ Other languages
+ |
+
+
+ |
+
+
+
+ Key Files
+ |
+
+
+ |
+
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html
index 63a3ff54c..3a0ebe48b 100644
--- a/scanpipe/templates/scanpipe/project_detail.html
+++ b/scanpipe/templates/scanpipe/project_detail.html
@@ -104,7 +104,7 @@
{% if project.policies_enabled %}
diff --git a/scanpipe/templates/scanpipe/tabset/tab_license_detections.html b/scanpipe/templates/scanpipe/tabset/tab_license_detections.html
new file mode 100644
index 000000000..6903978b9
--- /dev/null
+++ b/scanpipe/templates/scanpipe/tabset/tab_license_detections.html
@@ -0,0 +1,99 @@
+
+
+
+
+ License expression |
+ Origin resource path |
+ Matched text |
+ Rule URL |
+ Score |
+ Matcher |
+ Match length |
+ Match coverage |
+ Rule relevance |
+
+
+
+ {% for match in tab_data.fields.matches.value %}
+
+
+ {{ match.license_expression }}
+ |
+
+ {{ match.from_file }}
+ |
+
+ {{ match.matched_text }}
+ |
+
+ {% if match.rule_url %}
+
+ {{ match.rule_identifier }}
+
+
+ {% else %}
+ {{ match.rule_identifier }}
+ {% endif %}
+ |
+
+ {{ match.score }}
+ |
+
+ {{ match.matcher }}
+ |
+
+ {{ match.matched_length }}
+ |
+
+ {{ match.match_coverage }}
+ |
+
+ {{ match.rule_relevance }}
+ |
+
+ {% endfor %}
+
+
+ {% if tab_data.fields.detection_log.value %}
+
+
+
+ Detection log |
+
+
+
+ {% for log_entry in tab_data.fields.detection_log.value %}
+
+
+ {{ log_entry }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+
+
+
+ Resource path |
+ Start line |
+ End line |
+
+
+
+ {% for file_region in tab_data.fields.file_regions.value %}
+
+
+ {{ file_region.path }}
+ |
+
+ {{ file_region.start_line }}
+ |
+
+ {{ file_region.end_line }}
+ |
+
+ {% endfor %}
+
+
+
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/tabset/tab_package_detections.html b/scanpipe/templates/scanpipe/tabset/tab_package_detections.html
new file mode 100644
index 000000000..c033a74a6
--- /dev/null
+++ b/scanpipe/templates/scanpipe/tabset/tab_package_detections.html
@@ -0,0 +1,87 @@
+
+
+
+
+ Datafile paths |
+
+
+
+ {% for path in tab_data.fields.datafile_paths.value %}
+
+
+ {{ path }}
+ |
+
+ {% endfor %}
+
+
+
+
+
+ Datasource IDs |
+
+
+
+ {% for id in tab_data.fields.datasource_ids.value %}
+
+
+ {{ id }}
+ |
+
+ {% endfor %}
+
+
+ {% if tab_data.fields.license_detections.value %}
+
+
+
+ License detections |
+ License expression |
+ License expression SPDX |
+
+
+
+ {% for detection in tab_data.fields.license_detections.value %}
+
+
+ {{ detection.identifier }}
+ |
+
+ {{ detection.license_expression }}
+ |
+
+ {{ detection.license_expression_spdx }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+ {% if tab_data.fields.other_license_detections.value %}
+
+
+
+ License detections |
+ License expression |
+ License expression SPDX |
+
+
+
+ {% for detection in tab_data.fields.other_license_detections.value %}
+
+
+ {{ detection.identifier }}
+
+ |
+
+ {{ detection.license_expression }}
+ |
+
+ {{ detection.license_expression_spdx }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+
\ No newline at end of file
diff --git a/scanpipe/templates/scanpipe/tabset/tab_packages.html b/scanpipe/templates/scanpipe/tabset/tab_packages.html
index 1370d741d..3231d4d86 100644
--- a/scanpipe/templates/scanpipe/tabset/tab_packages.html
+++ b/scanpipe/templates/scanpipe/tabset/tab_packages.html
@@ -18,6 +18,11 @@
{% endif %}
+ {% if package.has_compliance_alert %}
+
+
+
+ {% endif %}
|
{{ package.declared_license_expression }}
diff --git a/scanpipe/templates/scanpipe/tabset/tab_resource_detections.html b/scanpipe/templates/scanpipe/tabset/tab_resource_detections.html
new file mode 100644
index 000000000..d5165a3f2
--- /dev/null
+++ b/scanpipe/templates/scanpipe/tabset/tab_resource_detections.html
@@ -0,0 +1,102 @@
+
+ {% if tab_data.fields.license_detections.value %}
+
+
+
+ License detections |
+ License expression |
+ License expression SPDX |
+
+
+
+ {% for detection in tab_data.fields.license_detections.value %}
+
+
+ {{ detection.identifier }}
+ |
+
+ {{ detection.license_expression }}
+ |
+
+ {{ detection.license_expression_spdx }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+ {% if tab_data.fields.license_clues.value %}
+
+
+
+ License expression |
+ License clue detials |
+
+
+
+ {% for clue in tab_data.fields.license_clues.value %}
+
+
+ {{ clue.license_expression }}
+ |
+
+ {{ clue }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+ {% if tab_data.fields.emails.value %}
+
+
+
+ Email |
+ Start line |
+ End line |
+
+
+
+ {% for email in tab_data.fields.emails.value %}
+
+
+ {{ email.email }}
+ |
+
+ {{ email.start_line }}
+ |
+
+ {{ email.end_line }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+ {% if tab_data.fields.urls.value %}
+
+
+
+ URL |
+ Start line |
+ End line |
+
+
+
+ {% for url in tab_data.fields.urls.value %}
+
+
+ {{ url.url }}
+ |
+
+ {{ url.start_line }}
+ |
+
+ {{ url.end_line }}
+ |
+
+ {% endfor %}
+
+
+ {% endif %}
+
\ No newline at end of file
diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py
index 9c365fde9..23b1fe540 100644
--- a/scanpipe/tests/test_models.py
+++ b/scanpipe/tests/test_models.py
@@ -174,6 +174,7 @@ def test_scanpipe_project_model_delete_related_objects(self):
"scanpipe.CodebaseRelation": 0,
"scanpipe.CodebaseResource": 1,
"scanpipe.DiscoveredDependency": 0,
+ "scanpipe.DiscoveredLicense": 0,
"scanpipe.DiscoveredPackage": 1,
"scanpipe.DiscoveredPackage_codebase_resources": 1,
"scanpipe.InputSource": 0,
diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py
index bf27e9b6c..0aeafcfe4 100644
--- a/scanpipe/tests/test_views.py
+++ b/scanpipe/tests/test_views.py
@@ -477,16 +477,17 @@ def test_scanpipe_views_project_details_get_scan_summary_data(self):
scan_summary = self.data / "scancode" / "is-npm-1.0.0_scan_package_summary.json"
scan_summary_json = json.loads(scan_summary.read_text())
- scan_summary_data = get_scan_summary_data(scan_summary_json)
+ scan_summary_data = get_scan_summary_data(self.project1, scan_summary_json)
- self.assertEqual(6, len(scan_summary_data))
+ self.assertEqual(7, len(scan_summary_data))
expected = [
- "Declared license",
- "Declared holder",
- "Primary language",
- "Other licenses",
- "Other holders",
- "Other languages",
+ "declared_license_expression",
+ "declared_holder",
+ "primary_language",
+ "other_license_expressions",
+ "other_holders",
+ "other_languages",
+ "key_file_licenses",
]
self.assertEqual(expected, list(scan_summary_data.keys()))
@@ -921,8 +922,10 @@ def test_scanpipe_views_codebase_resource_details_view_tabset(self):
self.assertContains(response, 'id="tab-others"')
self.assertContains(response, 'data-target="tab-viewer"')
self.assertContains(response, 'id="tab-viewer"')
- self.assertNotContains(response, 'data-target="tab-detection"')
- self.assertNotContains(response, 'id="tab-detection"')
+ self.assertNotContains(response, 'data-target="tab-terms"')
+ self.assertNotContains(response, 'id="tab-terms"')
+ self.assertNotContains(response, 'data-target="tab-resource-detection"')
+ self.assertNotContains(response, 'id="tab-resource-detection"')
self.assertNotContains(response, 'data-target="tab-packages"')
self.assertNotContains(response, 'id="tab-packages"')
self.assertNotContains(response, 'data-target="tab-relations"')
@@ -940,10 +943,8 @@ def test_scanpipe_views_codebase_resource_details_view_tabset(self):
map_type="path",
)
response = self.client.get(resource1.get_absolute_url())
- self.assertContains(response, 'data-target="tab-detection"')
- self.assertContains(response, 'id="tab-detection"')
- self.assertContains(response, 'data-target="tab-packages"')
- self.assertContains(response, 'id="tab-packages"')
+ self.assertContains(response, 'data-target="tab-terms"')
+ self.assertContains(response, 'id="tab-terms"')
self.assertContains(response, 'data-target="tab-relations"')
self.assertContains(response, 'id="tab-relations"')
self.assertContains(response, 'data-target="tab-extra_data"')
@@ -1011,7 +1012,7 @@ def test_scanpipe_views_codebase_resource_views(self):
with self.assertNumQueries(8):
self.client.get(url)
- with self.assertNumQueries(7):
+ with self.assertNumQueries(8):
self.client.get(resource1.get_absolute_url())
def test_scanpipe_views_discovered_package_views(self):
diff --git a/scanpipe/urls.py b/scanpipe/urls.py
index 96dd2be22..f3cf8d63e 100644
--- a/scanpipe/urls.py
+++ b/scanpipe/urls.py
@@ -56,6 +56,11 @@
views.DiscoveredPackageDetailsView.as_view(),
name="package_detail",
),
+ path(
+ "project//license_detections//",
+ views.DiscoveredLicenseDetailsView.as_view(),
+ name="license_detail",
+ ),
path(
"project//dependencies//",
views.DiscoveredDependencyDetailsView.as_view(),
@@ -66,6 +71,11 @@
views.DiscoveredPackageListView.as_view(),
name="project_packages",
),
+ path(
+ "project//license_detections/",
+ views.DiscoveredLicenseListView.as_view(),
+ name="project_licenses",
+ ),
path(
"project//dependencies/",
views.DiscoveredDependencyListView.as_view(),
@@ -202,9 +212,9 @@
name="project_resource_status_summary",
),
path(
- "project//resource_license_summary/",
- views.ProjectResourceLicenseSummaryView.as_view(),
- name="project_resource_license_summary",
+ "project//license_detection_summary/",
+ views.ProjectLicenseDetectionSummaryView.as_view(),
+ name="project_license_detection_summary",
),
path(
"project//compliance_panel/",
diff --git a/scanpipe/views.py b/scanpipe/views.py
index 153542c21..673f99e2a 100644
--- a/scanpipe/views.py
+++ b/scanpipe/views.py
@@ -66,6 +66,7 @@
from scanpipe.api.serializers import DiscoveredDependencySerializer
from scanpipe.filters import PAGE_VAR
from scanpipe.filters import DependencyFilterSet
+from scanpipe.filters import LicenseFilterSet
from scanpipe.filters import PackageFilterSet
from scanpipe.filters import ProjectFilterSet
from scanpipe.filters import ProjectMessageFilterSet
@@ -83,6 +84,7 @@
from scanpipe.models import CodebaseRelation
from scanpipe.models import CodebaseResource
from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
from scanpipe.models import DiscoveredPackage
from scanpipe.models import InputSource
from scanpipe.models import Project
@@ -172,12 +174,12 @@
SCAN_SUMMARY_FIELDS = [
- ("Declared license", "declared_license_expression"),
- ("Declared holder", "declared_holder"),
- ("Primary language", "primary_language"),
- ("Other licenses", "other_license_expressions"),
- ("Other holders", "other_holders"),
- ("Other languages", "other_languages"),
+ "declared_license_expression",
+ "declared_holder",
+ "primary_language",
+ "other_license_expressions",
+ "other_holders",
+ "other_languages",
]
@@ -316,7 +318,9 @@ def get_field_value(self, field_name, render_func=None):
"""
Return the formatted value of the specified `field_name` from the object.
- By default, JSON types (list and dict) are rendered as YAML.
+ By default, JSON types (list and dict) are rendered as YAML,
+ except some fields which are used for a more complex tabular
+ representation with links to other views.
If a `render_func` is provided, it will take precedence and be used for
rendering the value.
"""
@@ -328,9 +332,23 @@ def get_field_value(self, field_name, render_func=None):
if isinstance(field_value, Manager):
return list(field_value.all())
+ # We need these as mappings
+ detection_fields = [
+ "license_detections",
+ "other_license_detections",
+ "license_clues",
+ "matches",
+ "file_regions",
+ "urls",
+ "emails",
+ "datafile_paths",
+ "datasource_ids",
+ ]
+
if isinstance(field_value, list | dict):
- with suppress(Exception):
- field_value = render_as_yaml(field_value)
+ if field_name not in detection_fields:
+ with suppress(Exception):
+ field_value = render_as_yaml(field_value)
return field_value
@@ -637,11 +655,12 @@ def get_license_clarity_data(scan_summary_json):
]
@staticmethod
- def get_scan_summary_data(scan_summary_json):
+ def get_scan_summary_data(project, scan_summary_json):
summary_data = {}
- for field_label, field_name in SCAN_SUMMARY_FIELDS:
- field_data = scan_summary_json.get(field_name)
+ for field_name, field_data in scan_summary_json.items():
+ if field_name not in SCAN_SUMMARY_FIELDS:
+ continue
if type(field_data) is list:
# Do not include `None` entries
@@ -650,7 +669,13 @@ def get_scan_summary_data(scan_summary_json):
# Converts single value type into common data-structure
values = [{"value": field_data}]
- summary_data[field_label] = values
+ summary_data[field_name] = values
+
+ key_files = project.codebaseresources.filter(is_key_file=True)
+ summary_data["key_file_licenses"] = {
+ key_file.path: key_file.detected_license_expression
+ for key_file in key_files
+ }
return summary_data
@@ -708,7 +733,7 @@ def get_context_data(self, **kwargs):
with suppress(json.decoder.JSONDecodeError):
scan_summary_json = json.loads(scan_summary_file.read_text())
license_clarity = self.get_license_clarity_data(scan_summary_json)
- scan_summary = self.get_scan_summary_data(scan_summary_json)
+ scan_summary = self.get_scan_summary_data(project, scan_summary_json)
codebase_root = sorted(
project.codebase_path.glob("*"),
@@ -962,42 +987,57 @@ def get_context_data(self, **kwargs):
return context
-class ProjectResourceLicenseSummaryView(ConditionalLoginRequired, generic.DetailView):
+class ProjectLicenseDetectionSummaryView(ConditionalLoginRequired, generic.DetailView):
model = Project
- template_name = "scanpipe/panels/resource_license_summary.html"
+ template_name = "scanpipe/panels/license_detections_summary.html"
@staticmethod
- def get_resource_license_summary(project, limit=10):
+ def get_license_detection_summary(project, limit=10):
license_counter = count_group_by(
- project.codebaseresources.files(), "detected_license_expression"
+ project.discoveredlicenses, "license_expression"
)
if list(license_counter.keys()) == [""]:
- return
+ return None, None, None
# Order the license list by the number of detections, higher first
sorted_by_count = dict(
sorted(license_counter.items(), key=operator.itemgetter(1), reverse=True)
)
- # Remove the "no licenses" entry from the top list
- no_licenses = sorted_by_count.pop("", None)
-
# Keep the top entries
top_licenses = dict(list(sorted_by_count.items())[:limit])
- # Add the "no licenses" entry at the end
- if no_licenses:
- top_licenses[""] = no_licenses
+ # Also get count for detections with
+ expressions_with_compliance_alert = []
+ for license_expression in top_licenses.keys():
+ has_compliance_alert = (
+ project.discoveredlicenses.filter(license_expression=license_expression)
+ .has_compliance_alert()
+ .exists()
+ )
+ if has_compliance_alert:
+ expressions_with_compliance_alert.append(license_expression)
+
+ total_counts = {
+ "with_compliance_error": (
+ project.discoveredlicenses.has_compliance_alert().count()
+ ),
+ "all": project.discoveredlicenses.count(),
+ }
- return top_licenses
+ return top_licenses, expressions_with_compliance_alert, total_counts
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
- summary = self.get_resource_license_summary(project=self.object)
- context["resource_license_summary"] = summary
- context["project_resources_url"] = reverse(
- "project_resources", args=[self.object.slug]
+ summary, expressions, counts = self.get_license_detection_summary(
+ project=self.object
+ )
+ context["license_detection_summary"] = summary
+ context["expressions_with_compliance_alert"] = expressions
+ context["total_counts"] = counts
+ context["project_licenses_url"] = reverse(
+ "project_licenses", args=[self.object.slug]
)
return context
@@ -1585,6 +1625,53 @@ def get_queryset(self):
return super().get_queryset().order_by("dependency_uid")
+class DiscoveredLicenseListView(
+ ConditionalLoginRequired,
+ ProjectRelatedViewMixin,
+ TableColumnsMixin,
+ ExportXLSXMixin,
+ PaginatedFilterView,
+):
+ model = DiscoveredLicense
+ filterset_class = LicenseFilterSet
+ template_name = "scanpipe/license_detection_list.html"
+ paginate_by = settings.SCANCODEIO_PAGINATE_BY.get("license", 10)
+ table_columns = [
+ "identifier",
+ {
+ "field_name": "license_expression",
+ "filter_fieldname": "license_expression",
+ },
+ {
+ "field_name": "license_expression_spdx",
+ "filter_fieldname": "license_expression_spdx",
+ },
+ "detection_count",
+ {
+ "field_name": "compliance_alert",
+ "filter_fieldname": "compliance_alert",
+ },
+ ]
+
+ def get_queryset(self):
+ return (
+ super()
+ .get_queryset()
+ .only(
+ "detection_count",
+ "license_expression",
+ "license_expression_spdx",
+ "compliance_alert",
+ )
+ .order_by_count_and_expression()
+ )
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+ context["display_compliance_alert"] = self.get_project().policies_enabled
+ return context
+
+
class ProjectMessageListView(
ConditionalLoginRequired,
ProjectRelatedViewMixin,
@@ -1731,23 +1818,24 @@ class CodebaseResourceDetailsView(
"disable_condition": do_not_disable,
"display_condition": is_displayable_image_type,
},
- "detection": {
+ "terms": {
"fields": [
"detected_license_expression",
{
"field_name": "detected_license_expression_spdx",
"label": "Detected license expression (SPDX)",
},
- "license_detections",
- "license_clues",
"percentage_of_license_text",
- "copyrights",
- "holders",
- "authors",
- "emails",
- "urls",
+ {"field_name": "copyrights", "render_func": render_as_yaml},
+ {"field_name": "holders", "render_func": render_as_yaml},
+ {"field_name": "authors", "render_func": render_as_yaml},
],
+ "icon_class": "fa-solid fa-file-contract",
+ },
+ "detection": {
+ "fields": ["license_detections", "license_clues", "emails", "urls"],
"icon_class": "fa-solid fa-search",
+ "template": "scanpipe/tabset/tab_resource_detections.html",
},
"packages": {
"fields": ["discovered_packages"],
@@ -1940,10 +2028,18 @@ class DiscoveredPackageDetailsView(
"copyright",
"holder",
"notice_text",
+ ],
+ "icon_class": "fa-solid fa-file-contract",
+ },
+ "detection": {
+ "fields": [
+ "datasource_ids",
+ "datafile_paths",
"license_detections",
"other_license_detections",
],
- "icon_class": "fa-solid fa-file-contract",
+ "icon_class": "fa-solid fa-search",
+ "template": "scanpipe/tabset/tab_package_detections.html",
},
"resources": {
"fields": ["codebase_resources"],
@@ -2104,6 +2200,35 @@ def get_context_data(self, **kwargs):
return context
+class DiscoveredLicenseDetailsView(
+ ConditionalLoginRequired,
+ ProjectRelatedViewMixin,
+ TabSetMixin,
+ generic.DetailView,
+):
+ model = DiscoveredLicense
+ model_label = "license_detections"
+ slug_field = "identifier"
+ slug_url_kwarg = "identifier"
+ template_name = "scanpipe/license_detection_detail.html"
+ tabset = {
+ "essentials": {
+ "fields": [
+ "license_expression",
+ "license_expression_spdx",
+ "identifier",
+ "detection_count",
+ ],
+ "icon_class": "fa-solid fa-info-circle",
+ },
+ "detection": {
+ "fields": ["matches", "detection_log", "file_regions"],
+ "icon_class": "fa-solid fa-search",
+ "template": "scanpipe/tabset/tab_license_detections.html",
+ },
+ }
+
+
@conditional_login_required
def run_detail_view(request, uuid):
template = "scanpipe/modals/run_modal_content.html"
|