diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 9ac7b43e7..ef932f5e7 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -110,6 +110,7 @@ "resource": 100, "package": 100, "dependency": 100, + "license": 100, "relation": 100, }, ) diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index fe7c1f4ea..63cfca2ea 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -31,6 +31,7 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredLicense from scanpipe.models import DiscoveredPackage from scanpipe.models import InputSource from scanpipe.models import Project @@ -466,6 +467,20 @@ class Meta: ] +class DiscoveredLicenseSerializer(serializers.ModelSerializer): + compliance_alert = serializers.CharField() + + class Meta: + model = DiscoveredLicense + fields = [ + "detection_count", + "identifier", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ] + + class CodebaseRelationSerializer(serializers.ModelSerializer): from_resource = serializers.ReadOnlyField(source="from_resource.path") to_resource = serializers.ReadOnlyField(source="to_resource.path") @@ -524,6 +539,7 @@ def get_model_serializer(model_class): CodebaseResource: CodebaseResourceSerializer, DiscoveredPackage: DiscoveredPackageSerializer, DiscoveredDependency: DiscoveredDependencySerializer, + DiscoveredLicense: DiscoveredLicenseSerializer, CodebaseRelation: CodebaseRelationSerializer, ProjectMessage: ProjectMessageSerializer, }.get(model_class, None) diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 2f6cf7195..c032fbd9e 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -39,6 +39,7 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredLicense from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import ProjectMessage @@ -529,6 +530,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "related_from__from_resource__path", ], ) + compliance_alert = django_filters.ChoiceFilter( choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices, ) @@ -587,8 +589,8 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - license_expression_filer = self.filters["detected_license_expression"] - license_expression_filer.extra["widget"] = HasValueDropdownWidget() + license_expression_filter = self.filters["detected_license_expression"] + license_expression_filter.extra["widget"] = HasValueDropdownWidget() class IsVulnerable(django_filters.ChoiceFilter): @@ -626,6 +628,19 @@ def filter(self, qs, value): return qs.filter(lookups) +class DiscoveredLicenseSearchFilter(QuerySearchFilter): + def filter(self, qs, value): + if not value: + return qs + + search_fields = ["license_expression", "license_expression_spdx"] + lookups = Q() + for field_names in search_fields: + lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value}) + + return qs.filter(lookups) + + class GroupOrderingFilter(django_filters.OrderingFilter): """Add the ability to provide a group a fields to order by.""" @@ -801,6 +816,44 @@ class Meta: ] +class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): + dropdown_widget_fields = [ + "compliance_alert", + "license_expression", + "license_expression_spdx", + ] + + search = DiscoveredLicenseSearchFilter( + label="Search", field_name="name", lookup_expr="icontains" + ) + sort = GroupOrderingFilter( + label="Sort", + fields=[ + "detection_count", + "identifier", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ], + ) + license_expression = django_filters.AllValuesFilter() + license_expression_spdx = django_filters.AllValuesFilter() + compliance_alert = django_filters.ChoiceFilter( + choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices, + ) + + class Meta: + model = DiscoveredLicense + fields = [ + "search", + "identifier", + "detection_count", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ] + + class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): search = QuerySearchFilter( label="Search", field_name="description", lookup_expr="icontains" diff --git a/scanpipe/migrations/0070_discovered_license_models.py b/scanpipe/migrations/0070_discovered_license_models.py new file mode 100644 index 000000000..88f4ac3e6 --- /dev/null +++ b/scanpipe/migrations/0070_discovered_license_models.py @@ -0,0 +1,143 @@ +# Generated by Django 5.0.2 on 2024-03-17 12:38 + +import django.db.models.deletion +import scanpipe.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0069_project_purl"), + ] + + operations = [ + migrations.CreateModel( + name="DiscoveredLicense", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "compliance_alert", + models.CharField( + blank=True, + choices=[ + ("ok", "Ok"), + ("warning", "Warning"), + ("error", "Error"), + ("missing", "Missing"), + ], + editable=False, + help_text="Indicates how the license expression complies with provided policies.", + max_length=10, + ), + ), + ( + "license_expression", + models.TextField( + blank=True, + help_text="A license expression string using the SPDX license expression syntax and ScanCode license keys, the effective license expression for this license detection.", + ), + ), + ( + "license_expression_spdx", + models.TextField( + blank=True, + help_text="SPDX license expression string with SPDX ids.", + ), + ), + ( + "matches", + models.JSONField( + blank=True, + default=list, + help_text="List of license matches combined in this detection.", + verbose_name="Reference Matches", + ), + ), + ( + "detection_log", + models.JSONField( + blank=True, + default=list, + help_text="A list of detection DetectionRule explaining how this detection was created.", + ), + ), + ( + "identifier", + models.CharField( + blank=True, + help_text="An identifier unique for a license detection, containing the license expression and a UUID crafted from the match contents.", + max_length=1024, + ), + ), + ( + "detection_count", + models.BigIntegerField( + blank=True, + help_text="Total number of this license detection discovered.", + null=True, + ), + ), + ( + "file_regions", + models.JSONField( + blank=True, + default=list, + help_text="A list of file regions with resource path, start and end line details for each place this license detection was discovered at. Also contains whether this license was discovered from a file or from package metadata.", + verbose_name="Detection Locations", + ), + ), + ( + "project", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)ss", + to="scanpipe.project", + ), + ), + ], + options={ + "ordering": ["detection_count", "identifier"], + "indexes": [ + models.Index( + fields=["identifier"], name="scanpipe_di_identif_b533f3_idx" + ), + models.Index( + fields=["license_expression"], + name="scanpipe_di_license_33d11a_idx", + ), + models.Index( + fields=["license_expression_spdx"], + name="scanpipe_di_license_eb5e9d_idx", + ), + models.Index( + fields=["detection_count"], + name="scanpipe_di_detecti_d87ff1_idx", + ), + ], + }, + bases=( + scanpipe.models.UpdateMixin, + scanpipe.models.SaveProjectMessageMixin, + scanpipe.models.UpdateFromDataMixin, + models.Model, + ), + ), + migrations.AddConstraint( + model_name="discoveredlicense", + constraint=models.UniqueConstraint( + condition=models.Q(("identifier", ""), _negated=True), + fields=("project", "identifier"), + name="scanpipe_discoveredlicense_unique_license_id_within_project", + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 0afce189a..5a5a4914e 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -653,6 +653,7 @@ def delete_related_objects(self): self.projectmessages, self.codebaserelations, self.discovereddependencies, + self.discoveredlicenses, self.codebaseresources, self.runs, self.inputsources, @@ -1406,6 +1407,35 @@ def dependency_count(self): """Return the number of dependencies related to this project.""" return self.discovereddependencies.count() + @cached_property + def license_detections_count(self): + """Return the number of license detections in this project.""" + return self.discoveredlicenses.count() + + @cached_property + def package_compliance_alert_count(self): + """ + Return the number of packages related to this project which have + a license compliance error alert. + """ + return self.discoveredpackages.has_compliance_alert().count() + + @cached_property + def license_compliance_alert_count(self): + """ + Return the number of license detections related to this project + which have a license compliance error alert. + """ + return self.discoveredlicenses.has_compliance_alert().count() + + @cached_property + def resource_compliance_alert_count(self): + """ + Return the number of codebase resources related to this project which have + a license compliance error alert. + """ + return self.codebaseresources.has_compliance_alert().count() + @cached_property def message_count(self): """Return the number of messages related to this project.""" @@ -2143,6 +2173,9 @@ def compliance_issues(self, severity): return self.filter(compliance_alert__in=severity_mapping[severity]) + def has_compliance_alert(self): + return self.filter(Q(compliance_alert__exact=CodebaseResource.Compliance.ERROR)) + def convert_glob_to_django_regex(glob_pattern): """ @@ -2450,6 +2483,17 @@ class Compliance(models.TextChoices): class Meta: abstract = True + @property + def has_compliance_alert(self): + """ + Returns True if this instance has a compliance alert of `ERROR` + for it's respective license_expression fields. + """ + if self.compliance_alert == self.Compliance.ERROR: + return True + + return False + @classmethod def from_db(cls, db, field_names, values): """ @@ -3072,6 +3116,12 @@ def with_resources_count(self): ) return self.annotate(resources_count=count_subquery) + def has_license_detections(self): + return self.filter(~Q(license_detections=[]) | ~Q(other_license_detections=[])) + + def has_no_license_detections(self): + return self.filter(Q(license_detections=[]) & Q(other_license_detections=[])) + def only_package_url_fields(self): """ Only select and return the UUID and PURL fields. @@ -3964,6 +4014,180 @@ def as_spdx(self): ) +class DiscoveredLicenseQuerySet( + ComplianceAlertQuerySetMixin, + ProjectRelatedQuerySet, +): + def order_by_count_and_expression(self): + """Order by detection count and license expression (identifer) fields.""" + return self.order_by("-detection_count", "identifier") + + +class AbstractLicenseDetection(models.Model): + """ + These fields should be kept in line with + `licensedcode.detection.LicenseDetection`. + """ + + license_expression = models.TextField( + blank=True, + help_text=_( + "A license expression string using the SPDX license expression" + " syntax and ScanCode license keys, the effective license expression" + " for this license detection." + ), + ) + + license_expression_spdx = models.TextField( + blank=True, + help_text=_("SPDX license expression string with SPDX ids."), + ) + + matches = models.JSONField( + _("Reference Matches"), + default=list, + blank=True, + help_text=_("List of license matches combined in this detection."), + ) + + detection_log = models.JSONField( + default=list, + blank=True, + help_text=_( + "A list of detection DetectionRule explaining how " + "this detection was created." + ), + ) + + identifier = models.CharField( + max_length=1024, + blank=True, + help_text=_( + "An identifier unique for a license detection, containing the license " + "expression and a UUID crafted from the match contents." + ), + ) + + class Meta: + abstract = True + + +class DiscoveredLicense( + ProjectRelatedModel, + SaveProjectMessageMixin, + UpdateFromDataMixin, + ComplianceAlertMixin, + AbstractLicenseDetection, +): + """ + A project's Discovered Licenses are the unique License Detection objects + discovered in the code under analysis. + """ + + license_expression_field = "license_expression" + + # If this license was discovered in a extracted license statement + # this is True, and False if this was discovered in a file. + from_package = None + + detection_count = models.BigIntegerField( + blank=True, + null=True, + help_text=_("Total number of this license detection discovered."), + ) + + file_regions = models.JSONField( + _("Detection Locations"), + default=list, + blank=True, + help_text=_( + "A list of file regions with resource path, start and end line " + "details for each place this license detection was discovered at. " + "Also contains whether this license was discovered from a file or " + "from package metadata." + ), + ) + + objects = DiscoveredLicenseQuerySet.as_manager() + + class Meta: + ordering = ["detection_count", "identifier"] + indexes = [ + models.Index(fields=["identifier"]), + models.Index(fields=["license_expression"]), + models.Index(fields=["license_expression_spdx"]), + models.Index(fields=["detection_count"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["project", "identifier"], + condition=~Q(identifier=""), + name="%(app_label)s_%(class)s_unique_license_id_within_project", + ), + ] + + def __str__(self): + return self.identifier + + @classmethod + def create_from_data(cls, project, detection_data): + """ + Create and returns a DiscoveredLicense for a `project` from the + `detection_data`. If one of the values of the required fields is not + available, a "ProjectMessage" is created instead of a new + DiscoveredLicense instance. + """ + detection_data = detection_data.copy() + required_fields = ["license_expression", "identifier", "matches"] + missing_values = [ + field_name + for field_name in required_fields + if not detection_data.get(field_name) + ] + + if missing_values: + message = ( + f"No values for the following required fields: " + f"{', '.join(missing_values)}" + ) + + project.add_warning( + description=message, + model=cls, + details=detection_data, + ) + return + + cleaned_data = { + field_name: value + for field_name, value in detection_data.items() + if field_name in cls.model_fields() and value not in EMPTY_VALUES + } + + discovered_license = cls(project=project, **cleaned_data) + # Using save_error=False to not capture potential errors at this level but + # rather in the CodebaseResource.create_and_add_license_data method so + # resource data can be injected in the ProjectMessage record. + discovered_license.save(save_error=False, capture_exception=False) + return discovered_license + + def update_with_file_region(self, file_region, count_detection): + """ + If the `file_region` is a new file region, include it in the + `file_regions` list and increase the `detection_count` by 1. + """ + file_region_data = file_region.to_dict() + if file_region_data not in self.file_regions: + self.file_regions.append(file_region_data) + if count_detection: + if not self.detection_count: + self.detection_count = 1 + else: + self.detection_count += 1 + + self.save(update_fields=["detection_count", "file_regions"]) + + def normalize_package_url_data(purl_mapping, ignore_nulls=False): """ Normalize a mapping of purl data so database queries with diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index ea74f93e8..1dae9662d 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -85,6 +85,7 @@ def steps(cls): cls.remove_packages_without_resources, cls.scan_unmapped_to_files, cls.scan_mapped_from_for_files, + cls.collect_and_create_license_detections, cls.flag_deployed_from_resources_with_missing_license, cls.create_local_files_packages, ) @@ -298,6 +299,13 @@ def scan_mapped_from_for_files(self): scan_files = d2d.get_from_files_for_scanning(self.project.codebaseresources) scancode.scan_for_files(self.project, scan_files, progress_logger=self.log) + def collect_and_create_license_detections(self): + """ + Collect and create unique license detections from resources and + package data. + """ + scancode.collect_and_create_license_detections(project=self.project) + def create_local_files_packages(self): """Create local-files packages for codebase resources not part of a package.""" d2d.create_local_files_packages(self.project) diff --git a/scanpipe/pipelines/docker.py b/scanpipe/pipelines/docker.py index 5bf338458..877548597 100644 --- a/scanpipe/pipelines/docker.py +++ b/scanpipe/pipelines/docker.py @@ -42,6 +42,7 @@ def steps(cls): cls.flag_ignored_resources, cls.scan_for_application_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, cls.analyze_scanned_files, cls.flag_not_analyzed_codebase_resources, ) diff --git a/scanpipe/pipelines/docker_windows.py b/scanpipe/pipelines/docker_windows.py index 98684da13..768f279c0 100644 --- a/scanpipe/pipelines/docker_windows.py +++ b/scanpipe/pipelines/docker_windows.py @@ -45,6 +45,7 @@ def steps(cls): cls.flag_ignored_resources, cls.scan_for_application_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, cls.analyze_scanned_files, cls.flag_data_files_with_no_clues, cls.flag_not_analyzed_codebase_resources, diff --git a/scanpipe/pipelines/root_filesystem.py b/scanpipe/pipelines/root_filesystem.py index 76478ce6d..9a0fa9ef8 100644 --- a/scanpipe/pipelines/root_filesystem.py +++ b/scanpipe/pipelines/root_filesystem.py @@ -45,6 +45,7 @@ def steps(cls): cls.scan_for_application_packages, cls.match_not_analyzed_to_system_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, cls.analyze_scanned_files, cls.flag_not_analyzed_codebase_resources, ) @@ -122,6 +123,13 @@ def scan_for_files(self): """Scan unknown resources for copyrights, licenses, emails, and urls.""" scancode.scan_for_files(self.project, progress_logger=self.log) + def collect_and_create_license_detections(self): + """ + Collect and create unique license detections from resources and + package data. + """ + scancode.collect_and_create_license_detections(project=self.project) + def analyze_scanned_files(self): """Analyze single file scan results for completeness.""" flag.analyze_scanned_files(self.project) diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py index d5bbe992c..fd6580e45 100644 --- a/scanpipe/pipelines/scan_codebase.py +++ b/scanpipe/pipelines/scan_codebase.py @@ -45,6 +45,7 @@ def steps(cls): cls.flag_ignored_resources, cls.scan_for_application_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, ) def copy_inputs_to_codebase_directory(self): @@ -65,3 +66,10 @@ def scan_for_application_packages(self): def scan_for_files(self): """Scan unknown resources for copyrights, licenses, emails, and urls.""" scancode.scan_for_files(self.project, progress_logger=self.log) + + def collect_and_create_license_detections(self): + """ + Collect and create unique license detections from resources and + package data. + """ + scancode.collect_and_create_license_detections(project=self.project) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index df1b5ecc3..3ed78f27d 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -36,6 +36,7 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredLicense from scanpipe.models import DiscoveredPackage from scanpipe.pipes import scancode @@ -307,6 +308,78 @@ def update_or_create_dependency( return dependency +def update_or_create_license_detection( + project, + detection_data, + resource_path=None, + from_package=False, + count_detection=True, +): + """ + Get, update or create a DiscoveredLicense object then return it. + Use the `project` and `detection_data` mapping to lookup and creates the + DiscoveredLicense using its detection identifier as a unique key. + + Additonally if `resource_path` is passed, add the file region where + the license was detected to the DiscoveredLicense object, if not present + already. `from_package` is True if the license detection was in a + `extracted_license_statement` from a package metadata. + """ + detection_identifier = detection_data["identifier"] + + license_detection = project.discoveredlicenses.get_or_none( + identifier=detection_identifier, + ) + detection_data = _clean_license_detection_data(detection_data) + + if license_detection: + license_detection.update_from_data(detection_data) + else: + license_detection = DiscoveredLicense.create_from_data( + project, + detection_data, + ) + + if not license_detection: + project.add_error( + model="update_or_create_license_detection", + details=detection_data, + resource=resource_path, + ) + return + + if resource_path: + file_region = scancode.get_file_region( + detection_data=detection_data, + resource_path=resource_path, + ) + license_detection.update_with_file_region( + file_region=file_region, + count_detection=count_detection, + ) + + license_detection.from_package = from_package + return license_detection + + +def _clean_license_detection_data(detection_data): + detection_data = detection_data.copy() + if "reference_matches" in detection_data: + matches = detection_data.pop("reference_matches") + detection_data["matches"] = matches + + updated_matches = [] + for match_data in detection_data["matches"]: + from_file_path = match_data["from_file"] + if from_file_path: + match_data["from_file"] = from_file_path.removeprefix("codebase/") + + updated_matches.append(match_data) + + detection_data["matches"] = updated_matches + return detection_data + + def get_dependencies(project, dependency_data): """ Given a `dependency_data` mapping, get a list of DiscoveredDependency objects diff --git a/scanpipe/pipes/input.py b/scanpipe/pipes/input.py index f57e7eafd..0a1535c37 100644 --- a/scanpipe/pipes/input.py +++ b/scanpipe/pipes/input.py @@ -34,6 +34,7 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredLicense from scanpipe.models import DiscoveredPackage from scanpipe.pipes import scancode from scanpipe.pipes.output import mappings_key_by_fieldname @@ -85,10 +86,11 @@ def is_archive(location): def load_inventory_from_toolkit_scan(project, input_location): """ - Create packages, dependencies, and resources loaded from the ScanCode-toolkit scan - results located at ``input_location``. + Create license detections, packages, dependencies, and resources + loaded from the ScanCode-toolkit scan results located at ``input_location``. """ scanned_codebase = scancode.get_virtual_codebase(project, input_location) + scancode.create_discovered_licenses(project, scanned_codebase) scancode.create_discovered_packages(project, scanned_codebase) scancode.create_codebase_resources(project, scanned_codebase) scancode.create_discovered_dependencies( @@ -98,9 +100,12 @@ def load_inventory_from_toolkit_scan(project, input_location): def load_inventory_from_scanpipe(project, scan_data): """ - Create packages, dependencies, resources, and relations loaded from a ScanCode.io - JSON output provided as ``scan_data``. + Create license detections, packages, dependencies, resources, and relations + loaded from a ScanCode.io JSON output provided as ``scan_data``. """ + for detection_data in scan_data.get("license_detections", []): + pipes.update_or_create_license_detection(project, detection_data) + for package_data in scan_data.get("packages", []): pipes.update_or_create_package(project, package_data) @@ -117,12 +122,14 @@ def load_inventory_from_scanpipe(project, scan_data): model_to_object_maker_func = { DiscoveredPackage: pipes.update_or_create_package, DiscoveredDependency: pipes.update_or_create_dependency, + DiscoveredLicense: pipes.update_or_create_license_detection, CodebaseResource: pipes.update_or_create_resource, CodebaseRelation: pipes.get_or_create_relation, } worksheet_name_to_model = { "PACKAGES": DiscoveredPackage, + "LICENSE_DETECTIONS": DiscoveredLicense, "RESOURCES": CodebaseResource, "DEPENDENCIES": DiscoveredDependency, "RELATIONS": CodebaseRelation, diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index c9812bc97..d5d34477f 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -38,6 +38,7 @@ from commoncode import fileutils from commoncode.resource import VirtualCodebase from extractcode import api as extractcode_api +from licensedcode.detection import FileRegion from packagedcode import get_package_handler from packagedcode import models as packagedcode_models from scancode import Scanner @@ -435,6 +436,62 @@ def add_resource_to_package(package_uid, resource, project): resource.discovered_packages.add(package) +def collect_and_create_license_detections(project): + """ + Create instances of DiscoveredLicense for `project` from the parsed + license detections present in the CodebaseResources and + DiscoveredPackages of `project`. + """ + logger.info(f"Project {project} collect_license_detections:") + + for resource in project.codebaseresources.has_license_detections(): + logger.info(f" Processing: {resource.path} for licenses") + + for detection_data in resource.license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=detection_data, + resource_path=resource.path, + ) + + for resource in project.codebaseresources.has_package_data(): + for package_mapping in resource.package_data: + package_data = packagedcode_models.PackageData.from_dict( + mapping=package_mapping, + ) + + for detection in package_data.license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=detection, + resource_path=resource.path, + from_package=True, + ) + + for detection in package_data.other_license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=detection, + resource_path=resource.path, + from_package=True, + ) + + +def get_file_region(detection_data, resource_path): + """ + From a LicenseDetection mapping `detection_data`, create a FileRegion + object containing information about where this license was detected + exactly in a codebase, with `resource_path`, with start and end lines. + """ + start_line = min([match["start_line"] for match in detection_data["matches"]]) + end_line = max([match["end_line"] for match in detection_data["matches"]]) + return FileRegion( + path=resource_path, + start_line=start_line, + end_line=end_line, + ) + + def assemble_packages(project): """ Create instances of DiscoveredPackage and DiscoveredDependency for `project` @@ -795,6 +852,32 @@ def create_codebase_resources(project, scanned_codebase): discovered_package=package, ) + license_detections = getattr(scanned_resource, "license_detections", []) + for detection_data in license_detections: + detection_identifier = detection_data.get("identifier") + pipes.update_or_create_license_detection( + project=project, + detection_data=detection_data, + resource_path=resource_path, + count_detection=False, + ) + logger.debug(f"Add {codebase_resource} to {detection_identifier}") + + packages = getattr(scanned_resource, "package_data", []) + for package_data in packages: + license_detections = package_data.get("license_detections", []) + license_detections.extend(package_data.get("other_license_detections", [])) + for detection_data in license_detections: + detection_identifier = detection_data.get("identifier") + pipes.update_or_create_license_detection( + project=project, + detection_data=detection_data, + resource_path=resource_path, + count_detection=False, + from_package=True, + ) + logger.debug(f"Add {codebase_resource} to {detection_identifier}") + def create_discovered_packages(project, scanned_codebase): """ @@ -804,6 +887,16 @@ def create_discovered_packages(project, scanned_codebase): if hasattr(scanned_codebase.attributes, "packages"): for package_data in scanned_codebase.attributes.packages: pipes.update_or_create_package(project, package_data) + license_detections = package_data.get("license_detections", []) + license_detections.extend(package_data.get("other_license_detections", [])) + + for license_detection in license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=license_detection, + from_package=True, + count_detection=False, + ) def create_discovered_dependencies( @@ -829,6 +922,17 @@ def create_discovered_dependencies( ) +def create_discovered_licenses(project, scanned_codebase): + """ + Save the license detections of a ScanCode `scanned_codebase` + scancode.resource.Codebase object to the database as a DiscoveredLicense of + `project`. + """ + if hasattr(scanned_codebase.attributes, "license_detections"): + for detection_data in scanned_codebase.attributes.license_detections: + pipes.update_or_create_license_detection(project, detection_data) + + def set_codebase_resource_for_package(codebase_resource, discovered_package): """ Assign the `discovered_package` to the `codebase_resource` and set its diff --git a/scanpipe/templates/scanpipe/includes/project_summary_level.html b/scanpipe/templates/scanpipe/includes/project_summary_level.html index 2dad4e586..e5c36665b 100644 --- a/scanpipe/templates/scanpipe/includes/project_summary_level.html +++ b/scanpipe/templates/scanpipe/includes/project_summary_level.html @@ -14,6 +14,12 @@ {% endif %} + {% if project.package_compliance_alert_count %} + + {{ project.package_compliance_alert_count|intcomma }} + + + {% endif %} {% else %} 0 {% endif %} @@ -43,11 +49,17 @@

Resources

-

+

{% if project.resource_count %} {{ project.resource_count|intcomma }} + {% if project.resource_compliance_alert_count %} + + {{ project.resource_compliance_alert_count|intcomma }} + + + {% endif %} {% else %} 0 {% endif %} diff --git a/scanpipe/templates/scanpipe/license_detection_detail.html b/scanpipe/templates/scanpipe/license_detection_detail.html new file mode 100644 index 000000000..eec34ad59 --- /dev/null +++ b/scanpipe/templates/scanpipe/license_detection_detail.html @@ -0,0 +1,13 @@ +{% extends "scanpipe/base.html" %} +{% load static %} + +{% block title %}ScanCode.io: {{ project.name }} - {{ object.name }}{% endblock %} + +{% block content %} +

+ {% include 'scanpipe/includes/navbar_header.html' %} +
{% include 'scanpipe/includes/messages.html' %}
+ {% include 'scanpipe/includes/breadcrumb_detail_view.html' with object_title=object.identifier url_name="project_licenses" %} + {% include 'scanpipe/tabset/tabset.html' %} +
+{% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/license_detection_list.html b/scanpipe/templates/scanpipe/license_detection_list.html new file mode 100644 index 000000000..6b5ced789 --- /dev/null +++ b/scanpipe/templates/scanpipe/license_detection_list.html @@ -0,0 +1,64 @@ +{% extends "scanpipe/base.html" %} + +{% block title %}ScanCode.io: {{ project.name }} - License Detections{% endblock %} + +{% block content %} +
+ {% include 'scanpipe/includes/navbar_header.html' %} +
+
+ {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="License Detections" %} + {% include 'scanpipe/includes/search_field.html' with extra_class="is-small" %} +
+ {% include 'scanpipe/includes/pagination_header.html' %} + {% include 'scanpipe/includes/filters_breadcrumb.html' with filterset=filter only %} +
+
+ +
+ + {% include 'scanpipe/includes/list_view_thead.html' %} + + {% for license_detection in object_list %} + + + + + + {% if display_compliance_alert %} + + {% endif %} + + {% empty %} + + + + {% endfor %} + +
+ {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #} + {{ license_detection.identifier }} + {% if license_detection.has_compliance_alert %} + + + + {% endif %} + + {{ license_detection.license_expression }} + + {{ license_detection.license_expression_spdx }} + + {{ license_detection.detection_count }} + + + {{ license_detection.compliance_alert }} + +
+ No licenses detected. Clear search and filters +
+ + {% if is_paginated %} + {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+{% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index ee415673f..95b2ae922 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -30,6 +30,11 @@ {% endif %} + {% if package.has_compliance_alert %} + + + + {% endif %} diff --git a/scanpipe/templates/scanpipe/panels/license_detections_summary.html b/scanpipe/templates/scanpipe/panels/license_detections_summary.html new file mode 100644 index 000000000..91bba8086 --- /dev/null +++ b/scanpipe/templates/scanpipe/panels/license_detections_summary.html @@ -0,0 +1,31 @@ +{% load humanize %} +{% if license_detection_summary %} + +{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/panels/resource_license_summary.html b/scanpipe/templates/scanpipe/panels/resource_license_summary.html deleted file mode 100644 index 7a0f72388..000000000 --- a/scanpipe/templates/scanpipe/panels/resource_license_summary.html +++ /dev/null @@ -1,16 +0,0 @@ -{% load humanize %} -{% if resource_license_summary %} -
- -
-{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html index 52c11c1a2..1def8b4be 100644 --- a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html +++ b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html @@ -6,15 +6,80 @@
- {% for field_label, values in scan_summary.items %} - - - + + + + + + + + + + + + + + - - {% endfor %} + + {% endif %} + {% endfor %} + + + + + + + + + + + + + + +
- {{ field_label }} - -
    - {% for entry in values %} - {% if entry.value %} +
+ Declared license + +
    + {% for entry in scan_summary.declared_license_expression %} + {% if entry.value %} +
  • + {{ entry.value }} + {% if entry.count %} + + {{ entry.count|intcomma }} + + {% endif %} +
  • + {% endif %} + {% endfor %} +
+
+ Declared holder + +
    + {% for entry in scan_summary.declared_holder %} + {% if entry.value %} +
  • + {{ entry.value }} + {% if entry.count %} + + {{ entry.count|intcomma }} + + {% endif %} +
  • + {% endif %} + {% endfor %} +
+
+ Primary language + + +
+ Other licenses + + -
+ Other holders + +
    + {% for entry in scan_summary.other_holders %} + {% if entry.value %} +
  • + {{ entry.value }} + {% if entry.count %} + + {{ entry.count|intcomma }} + + {% endif %} +
  • + {% endif %} + {% endfor %} +
+
+ Other languages + + +
+ Key Files + + +
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index 63a3ff54c..3a0ebe48b 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -104,7 +104,7 @@
-
+
{% if project.policies_enabled %} diff --git a/scanpipe/templates/scanpipe/tabset/tab_license_detections.html b/scanpipe/templates/scanpipe/tabset/tab_license_detections.html new file mode 100644 index 000000000..6903978b9 --- /dev/null +++ b/scanpipe/templates/scanpipe/tabset/tab_license_detections.html @@ -0,0 +1,99 @@ +
+ + + + + + + + + + + + + + + + {% for match in tab_data.fields.matches.value %} + + + + + + + + + + + + {% endfor %} + +
License expressionOrigin resource pathMatched textRule URLScoreMatcherMatch lengthMatch coverageRule relevance
+ {{ match.license_expression }} + + {{ match.from_file }} + + {{ match.matched_text }} + + {% if match.rule_url %} + + {{ match.rule_identifier }} + + + {% else %} + {{ match.rule_identifier }} + {% endif %} + + {{ match.score }} + + {{ match.matcher }} + + {{ match.matched_length }} + + {{ match.match_coverage }} + + {{ match.rule_relevance }} +
+ {% if tab_data.fields.detection_log.value %} + + + + + + + + {% for log_entry in tab_data.fields.detection_log.value %} + + + + {% endfor %} + +
Detection log
+ {{ log_entry }} +
+ {% endif %} + + + + + + + + + + {% for file_region in tab_data.fields.file_regions.value %} + + + + + + {% endfor %} + +
Resource pathStart lineEnd line
+ {{ file_region.path }} + + {{ file_region.start_line }} + + {{ file_region.end_line }} +
+
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tabset/tab_package_detections.html b/scanpipe/templates/scanpipe/tabset/tab_package_detections.html new file mode 100644 index 000000000..c033a74a6 --- /dev/null +++ b/scanpipe/templates/scanpipe/tabset/tab_package_detections.html @@ -0,0 +1,87 @@ +
+ + + + + + + + {% for path in tab_data.fields.datafile_paths.value %} + + + + {% endfor %} + +
Datafile paths
+ {{ path }} +
+ + + + + + + + {% for id in tab_data.fields.datasource_ids.value %} + + + + {% endfor %} + +
Datasource IDs
+ {{ id }} +
+ {% if tab_data.fields.license_detections.value %} + + + + + + + + + + {% for detection in tab_data.fields.license_detections.value %} + + + + + + {% endfor %} + +
License detectionsLicense expressionLicense expression SPDX
+ {{ detection.identifier }} + + {{ detection.license_expression }} + + {{ detection.license_expression_spdx }} +
+ {% endif %} + {% if tab_data.fields.other_license_detections.value %} + + + + + + + + + + {% for detection in tab_data.fields.other_license_detections.value %} + + + + + + {% endfor %} + +
License detectionsLicense expressionLicense expression SPDX
+ {{ detection.identifier }} + + + {{ detection.license_expression }} + + {{ detection.license_expression_spdx }} +
+ {% endif %} +
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tabset/tab_packages.html b/scanpipe/templates/scanpipe/tabset/tab_packages.html index 1370d741d..3231d4d86 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_packages.html +++ b/scanpipe/templates/scanpipe/tabset/tab_packages.html @@ -18,6 +18,11 @@ {% endif %} + {% if package.has_compliance_alert %} + + + + {% endif %} {{ package.declared_license_expression }} diff --git a/scanpipe/templates/scanpipe/tabset/tab_resource_detections.html b/scanpipe/templates/scanpipe/tabset/tab_resource_detections.html new file mode 100644 index 000000000..d5165a3f2 --- /dev/null +++ b/scanpipe/templates/scanpipe/tabset/tab_resource_detections.html @@ -0,0 +1,102 @@ +
+ {% if tab_data.fields.license_detections.value %} + + + + + + + + + + {% for detection in tab_data.fields.license_detections.value %} + + + + + + {% endfor %} + +
License detectionsLicense expressionLicense expression SPDX
+ {{ detection.identifier }} + + {{ detection.license_expression }} + + {{ detection.license_expression_spdx }} +
+ {% endif %} + {% if tab_data.fields.license_clues.value %} + + + + + + + + + {% for clue in tab_data.fields.license_clues.value %} + + + + + {% endfor %} + +
License expressionLicense clue detials
+ {{ clue.license_expression }} + + {{ clue }} +
+ {% endif %} + {% if tab_data.fields.emails.value %} + + + + + + + + + + {% for email in tab_data.fields.emails.value %} + + + + + + {% endfor %} + +
EmailStart lineEnd line
+ {{ email.email }} + + {{ email.start_line }} + + {{ email.end_line }} +
+ {% endif %} + {% if tab_data.fields.urls.value %} + + + + + + + + + + {% for url in tab_data.fields.urls.value %} + + + + + + {% endfor %} + +
URLStart lineEnd line
+ {{ url.url }} + + {{ url.start_line }} + + {{ url.end_line }} +
+ {% endif %} +
\ No newline at end of file diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 9c365fde9..23b1fe540 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -174,6 +174,7 @@ def test_scanpipe_project_model_delete_related_objects(self): "scanpipe.CodebaseRelation": 0, "scanpipe.CodebaseResource": 1, "scanpipe.DiscoveredDependency": 0, + "scanpipe.DiscoveredLicense": 0, "scanpipe.DiscoveredPackage": 1, "scanpipe.DiscoveredPackage_codebase_resources": 1, "scanpipe.InputSource": 0, diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index bf27e9b6c..0aeafcfe4 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -477,16 +477,17 @@ def test_scanpipe_views_project_details_get_scan_summary_data(self): scan_summary = self.data / "scancode" / "is-npm-1.0.0_scan_package_summary.json" scan_summary_json = json.loads(scan_summary.read_text()) - scan_summary_data = get_scan_summary_data(scan_summary_json) + scan_summary_data = get_scan_summary_data(self.project1, scan_summary_json) - self.assertEqual(6, len(scan_summary_data)) + self.assertEqual(7, len(scan_summary_data)) expected = [ - "Declared license", - "Declared holder", - "Primary language", - "Other licenses", - "Other holders", - "Other languages", + "declared_license_expression", + "declared_holder", + "primary_language", + "other_license_expressions", + "other_holders", + "other_languages", + "key_file_licenses", ] self.assertEqual(expected, list(scan_summary_data.keys())) @@ -921,8 +922,10 @@ def test_scanpipe_views_codebase_resource_details_view_tabset(self): self.assertContains(response, 'id="tab-others"') self.assertContains(response, 'data-target="tab-viewer"') self.assertContains(response, 'id="tab-viewer"') - self.assertNotContains(response, 'data-target="tab-detection"') - self.assertNotContains(response, 'id="tab-detection"') + self.assertNotContains(response, 'data-target="tab-terms"') + self.assertNotContains(response, 'id="tab-terms"') + self.assertNotContains(response, 'data-target="tab-resource-detection"') + self.assertNotContains(response, 'id="tab-resource-detection"') self.assertNotContains(response, 'data-target="tab-packages"') self.assertNotContains(response, 'id="tab-packages"') self.assertNotContains(response, 'data-target="tab-relations"') @@ -940,10 +943,8 @@ def test_scanpipe_views_codebase_resource_details_view_tabset(self): map_type="path", ) response = self.client.get(resource1.get_absolute_url()) - self.assertContains(response, 'data-target="tab-detection"') - self.assertContains(response, 'id="tab-detection"') - self.assertContains(response, 'data-target="tab-packages"') - self.assertContains(response, 'id="tab-packages"') + self.assertContains(response, 'data-target="tab-terms"') + self.assertContains(response, 'id="tab-terms"') self.assertContains(response, 'data-target="tab-relations"') self.assertContains(response, 'id="tab-relations"') self.assertContains(response, 'data-target="tab-extra_data"') @@ -1011,7 +1012,7 @@ def test_scanpipe_views_codebase_resource_views(self): with self.assertNumQueries(8): self.client.get(url) - with self.assertNumQueries(7): + with self.assertNumQueries(8): self.client.get(resource1.get_absolute_url()) def test_scanpipe_views_discovered_package_views(self): diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 96dd2be22..f3cf8d63e 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -56,6 +56,11 @@ views.DiscoveredPackageDetailsView.as_view(), name="package_detail", ), + path( + "project//license_detections//", + views.DiscoveredLicenseDetailsView.as_view(), + name="license_detail", + ), path( "project//dependencies//", views.DiscoveredDependencyDetailsView.as_view(), @@ -66,6 +71,11 @@ views.DiscoveredPackageListView.as_view(), name="project_packages", ), + path( + "project//license_detections/", + views.DiscoveredLicenseListView.as_view(), + name="project_licenses", + ), path( "project//dependencies/", views.DiscoveredDependencyListView.as_view(), @@ -202,9 +212,9 @@ name="project_resource_status_summary", ), path( - "project//resource_license_summary/", - views.ProjectResourceLicenseSummaryView.as_view(), - name="project_resource_license_summary", + "project//license_detection_summary/", + views.ProjectLicenseDetectionSummaryView.as_view(), + name="project_license_detection_summary", ), path( "project//compliance_panel/", diff --git a/scanpipe/views.py b/scanpipe/views.py index 153542c21..673f99e2a 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -66,6 +66,7 @@ from scanpipe.api.serializers import DiscoveredDependencySerializer from scanpipe.filters import PAGE_VAR from scanpipe.filters import DependencyFilterSet +from scanpipe.filters import LicenseFilterSet from scanpipe.filters import PackageFilterSet from scanpipe.filters import ProjectFilterSet from scanpipe.filters import ProjectMessageFilterSet @@ -83,6 +84,7 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredLicense from scanpipe.models import DiscoveredPackage from scanpipe.models import InputSource from scanpipe.models import Project @@ -172,12 +174,12 @@ SCAN_SUMMARY_FIELDS = [ - ("Declared license", "declared_license_expression"), - ("Declared holder", "declared_holder"), - ("Primary language", "primary_language"), - ("Other licenses", "other_license_expressions"), - ("Other holders", "other_holders"), - ("Other languages", "other_languages"), + "declared_license_expression", + "declared_holder", + "primary_language", + "other_license_expressions", + "other_holders", + "other_languages", ] @@ -316,7 +318,9 @@ def get_field_value(self, field_name, render_func=None): """ Return the formatted value of the specified `field_name` from the object. - By default, JSON types (list and dict) are rendered as YAML. + By default, JSON types (list and dict) are rendered as YAML, + except some fields which are used for a more complex tabular + representation with links to other views. If a `render_func` is provided, it will take precedence and be used for rendering the value. """ @@ -328,9 +332,23 @@ def get_field_value(self, field_name, render_func=None): if isinstance(field_value, Manager): return list(field_value.all()) + # We need these as mappings + detection_fields = [ + "license_detections", + "other_license_detections", + "license_clues", + "matches", + "file_regions", + "urls", + "emails", + "datafile_paths", + "datasource_ids", + ] + if isinstance(field_value, list | dict): - with suppress(Exception): - field_value = render_as_yaml(field_value) + if field_name not in detection_fields: + with suppress(Exception): + field_value = render_as_yaml(field_value) return field_value @@ -637,11 +655,12 @@ def get_license_clarity_data(scan_summary_json): ] @staticmethod - def get_scan_summary_data(scan_summary_json): + def get_scan_summary_data(project, scan_summary_json): summary_data = {} - for field_label, field_name in SCAN_SUMMARY_FIELDS: - field_data = scan_summary_json.get(field_name) + for field_name, field_data in scan_summary_json.items(): + if field_name not in SCAN_SUMMARY_FIELDS: + continue if type(field_data) is list: # Do not include `None` entries @@ -650,7 +669,13 @@ def get_scan_summary_data(scan_summary_json): # Converts single value type into common data-structure values = [{"value": field_data}] - summary_data[field_label] = values + summary_data[field_name] = values + + key_files = project.codebaseresources.filter(is_key_file=True) + summary_data["key_file_licenses"] = { + key_file.path: key_file.detected_license_expression + for key_file in key_files + } return summary_data @@ -708,7 +733,7 @@ def get_context_data(self, **kwargs): with suppress(json.decoder.JSONDecodeError): scan_summary_json = json.loads(scan_summary_file.read_text()) license_clarity = self.get_license_clarity_data(scan_summary_json) - scan_summary = self.get_scan_summary_data(scan_summary_json) + scan_summary = self.get_scan_summary_data(project, scan_summary_json) codebase_root = sorted( project.codebase_path.glob("*"), @@ -962,42 +987,57 @@ def get_context_data(self, **kwargs): return context -class ProjectResourceLicenseSummaryView(ConditionalLoginRequired, generic.DetailView): +class ProjectLicenseDetectionSummaryView(ConditionalLoginRequired, generic.DetailView): model = Project - template_name = "scanpipe/panels/resource_license_summary.html" + template_name = "scanpipe/panels/license_detections_summary.html" @staticmethod - def get_resource_license_summary(project, limit=10): + def get_license_detection_summary(project, limit=10): license_counter = count_group_by( - project.codebaseresources.files(), "detected_license_expression" + project.discoveredlicenses, "license_expression" ) if list(license_counter.keys()) == [""]: - return + return None, None, None # Order the license list by the number of detections, higher first sorted_by_count = dict( sorted(license_counter.items(), key=operator.itemgetter(1), reverse=True) ) - # Remove the "no licenses" entry from the top list - no_licenses = sorted_by_count.pop("", None) - # Keep the top entries top_licenses = dict(list(sorted_by_count.items())[:limit]) - # Add the "no licenses" entry at the end - if no_licenses: - top_licenses[""] = no_licenses + # Also get count for detections with + expressions_with_compliance_alert = [] + for license_expression in top_licenses.keys(): + has_compliance_alert = ( + project.discoveredlicenses.filter(license_expression=license_expression) + .has_compliance_alert() + .exists() + ) + if has_compliance_alert: + expressions_with_compliance_alert.append(license_expression) + + total_counts = { + "with_compliance_error": ( + project.discoveredlicenses.has_compliance_alert().count() + ), + "all": project.discoveredlicenses.count(), + } - return top_licenses + return top_licenses, expressions_with_compliance_alert, total_counts def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - summary = self.get_resource_license_summary(project=self.object) - context["resource_license_summary"] = summary - context["project_resources_url"] = reverse( - "project_resources", args=[self.object.slug] + summary, expressions, counts = self.get_license_detection_summary( + project=self.object + ) + context["license_detection_summary"] = summary + context["expressions_with_compliance_alert"] = expressions + context["total_counts"] = counts + context["project_licenses_url"] = reverse( + "project_licenses", args=[self.object.slug] ) return context @@ -1585,6 +1625,53 @@ def get_queryset(self): return super().get_queryset().order_by("dependency_uid") +class DiscoveredLicenseListView( + ConditionalLoginRequired, + ProjectRelatedViewMixin, + TableColumnsMixin, + ExportXLSXMixin, + PaginatedFilterView, +): + model = DiscoveredLicense + filterset_class = LicenseFilterSet + template_name = "scanpipe/license_detection_list.html" + paginate_by = settings.SCANCODEIO_PAGINATE_BY.get("license", 10) + table_columns = [ + "identifier", + { + "field_name": "license_expression", + "filter_fieldname": "license_expression", + }, + { + "field_name": "license_expression_spdx", + "filter_fieldname": "license_expression_spdx", + }, + "detection_count", + { + "field_name": "compliance_alert", + "filter_fieldname": "compliance_alert", + }, + ] + + def get_queryset(self): + return ( + super() + .get_queryset() + .only( + "detection_count", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ) + .order_by_count_and_expression() + ) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["display_compliance_alert"] = self.get_project().policies_enabled + return context + + class ProjectMessageListView( ConditionalLoginRequired, ProjectRelatedViewMixin, @@ -1731,23 +1818,24 @@ class CodebaseResourceDetailsView( "disable_condition": do_not_disable, "display_condition": is_displayable_image_type, }, - "detection": { + "terms": { "fields": [ "detected_license_expression", { "field_name": "detected_license_expression_spdx", "label": "Detected license expression (SPDX)", }, - "license_detections", - "license_clues", "percentage_of_license_text", - "copyrights", - "holders", - "authors", - "emails", - "urls", + {"field_name": "copyrights", "render_func": render_as_yaml}, + {"field_name": "holders", "render_func": render_as_yaml}, + {"field_name": "authors", "render_func": render_as_yaml}, ], + "icon_class": "fa-solid fa-file-contract", + }, + "detection": { + "fields": ["license_detections", "license_clues", "emails", "urls"], "icon_class": "fa-solid fa-search", + "template": "scanpipe/tabset/tab_resource_detections.html", }, "packages": { "fields": ["discovered_packages"], @@ -1940,10 +2028,18 @@ class DiscoveredPackageDetailsView( "copyright", "holder", "notice_text", + ], + "icon_class": "fa-solid fa-file-contract", + }, + "detection": { + "fields": [ + "datasource_ids", + "datafile_paths", "license_detections", "other_license_detections", ], - "icon_class": "fa-solid fa-file-contract", + "icon_class": "fa-solid fa-search", + "template": "scanpipe/tabset/tab_package_detections.html", }, "resources": { "fields": ["codebase_resources"], @@ -2104,6 +2200,35 @@ def get_context_data(self, **kwargs): return context +class DiscoveredLicenseDetailsView( + ConditionalLoginRequired, + ProjectRelatedViewMixin, + TabSetMixin, + generic.DetailView, +): + model = DiscoveredLicense + model_label = "license_detections" + slug_field = "identifier" + slug_url_kwarg = "identifier" + template_name = "scanpipe/license_detection_detail.html" + tabset = { + "essentials": { + "fields": [ + "license_expression", + "license_expression_spdx", + "identifier", + "detection_count", + ], + "icon_class": "fa-solid fa-info-circle", + }, + "detection": { + "fields": ["matches", "detection_log", "file_regions"], + "icon_class": "fa-solid fa-search", + "template": "scanpipe/tabset/tab_license_detections.html", + }, + } + + @conditional_login_required def run_detail_view(request, uuid): template = "scanpipe/modals/run_modal_content.html"