Support SCTK License detection models

scancodeio/settings.py

@@ -110,6 +110,7 @@
         "resource": 100,
         "package": 100,
         "dependency": 100,
+        "license": 100,
         "relation": 100,
     },
 )

scanpipe/api/serializers.py

@@ -31,6 +31,7 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import InputSource
 from scanpipe.models import Project
@@ -466,6 +467,20 @@ class Meta:
         ]
 
 
+class DiscoveredLicenseSerializer(serializers.ModelSerializer):
+    compliance_alert = serializers.CharField()
+
+    class Meta:
+        model = DiscoveredLicense
+        fields = [
+            "detection_count",
+            "identifier",
+            "license_expression",
+            "license_expression_spdx",
+            "compliance_alert",
+        ]
+
+
 class CodebaseRelationSerializer(serializers.ModelSerializer):
     from_resource = serializers.ReadOnlyField(source="from_resource.path")
     to_resource = serializers.ReadOnlyField(source="to_resource.path")
@@ -524,6 +539,7 @@ def get_model_serializer(model_class):
         CodebaseResource: CodebaseResourceSerializer,
         DiscoveredPackage: DiscoveredPackageSerializer,
         DiscoveredDependency: DiscoveredDependencySerializer,
+        DiscoveredLicense: DiscoveredLicenseSerializer,
         CodebaseRelation: CodebaseRelationSerializer,
         ProjectMessage: ProjectMessageSerializer,
     }.get(model_class, None)

scanpipe/filters.py

@@ -39,6 +39,7 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
@@ -529,6 +530,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "related_from__from_resource__path",
         ],
     )
+
     compliance_alert = django_filters.ChoiceFilter(
         choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
     )
@@ -587,8 +589,8 @@ class Meta:
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        license_expression_filer = self.filters["detected_license_expression"]
-        license_expression_filer.extra["widget"] = HasValueDropdownWidget()
+        license_expression_filter = self.filters["detected_license_expression"]
+        license_expression_filter.extra["widget"] = HasValueDropdownWidget()
 
 
 class IsVulnerable(django_filters.ChoiceFilter):
@@ -626,6 +628,19 @@ def filter(self, qs, value):
         return qs.filter(lookups)
 
 
+class DiscoveredLicenseSearchFilter(QuerySearchFilter):
+    def filter(self, qs, value):
+        if not value:
+            return qs
+
+        search_fields = ["license_expression", "license_expression_spdx"]
+        lookups = Q()
+        for field_names in search_fields:
+            lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value})
+
+        return qs.filter(lookups)
+
+
 class GroupOrderingFilter(django_filters.OrderingFilter):
     """Add the ability to provide a group a fields to order by."""
 
@@ -801,6 +816,44 @@ class Meta:
         ]
 
 
+class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+    dropdown_widget_fields = [
+        "compliance_alert",
+        "license_expression",
+        "license_expression_spdx",
+    ]
+
+    search = DiscoveredLicenseSearchFilter(
+        label="Search", field_name="name", lookup_expr="icontains"
+    )
+    sort = GroupOrderingFilter(
+        label="Sort",
+        fields=[
+            "detection_count",
+            "identifier",
+            "license_expression",
+            "license_expression_spdx",
+            "compliance_alert",
+        ],
+    )
+    license_expression = django_filters.AllValuesFilter()
+    license_expression_spdx = django_filters.AllValuesFilter()
+    compliance_alert = django_filters.ChoiceFilter(
+        choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
+    )
+
+    class Meta:
+        model = DiscoveredLicense
+        fields = [
+            "search",
+            "identifier",
+            "detection_count",
+            "license_expression",
+            "license_expression_spdx",
+            "compliance_alert",
+        ]
+
+
 class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     search = QuerySearchFilter(
         label="Search", field_name="description", lookup_expr="icontains"

scanpipe/migrations/0070_discovered_license_models.py

@@ -0,0 +1,143 @@
+# Generated by Django 5.0.2 on 2024-03-17 12:38
+
+import django.db.models.deletion
+import scanpipe.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scanpipe", "0069_project_purl"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DiscoveredLicense",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "compliance_alert",
+                    models.CharField(
+                        blank=True,
+                        choices=[
+                            ("ok", "Ok"),
+                            ("warning", "Warning"),
+                            ("error", "Error"),
+                            ("missing", "Missing"),
+                        ],
+                        editable=False,
+                        help_text="Indicates how the license expression complies with provided policies.",
+                        max_length=10,
+                    ),
+                ),
+                (
+                    "license_expression",
+                    models.TextField(
+                        blank=True,
+                        help_text="A license expression string using the SPDX license expression syntax and ScanCode license keys, the effective license expression for this license detection.",
+                    ),
+                ),
+                (
+                    "license_expression_spdx",
+                    models.TextField(
+                        blank=True,
+                        help_text="SPDX license expression string with SPDX ids.",
+                    ),
+                ),
+                (
+                    "matches",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text="List of license matches combined in this detection.",
+                        verbose_name="Reference Matches",
+                    ),
+                ),
+                (
+                    "detection_log",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text="A list of detection DetectionRule explaining how this detection was created.",
+                    ),
+                ),
+                (
+                    "identifier",
+                    models.CharField(
+                        blank=True,
+                        help_text="An identifier unique for a license detection, containing the license expression and a UUID crafted from the match contents.",
+                        max_length=1024,
+                    ),
+                ),
+                (
+                    "detection_count",
+                    models.BigIntegerField(
+                        blank=True,
+                        help_text="Total number of this license detection discovered.",
+                        null=True,
+                    ),
+                ),
+                (
+                    "file_regions",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text="A list of file regions with resource path, start and end line details for each place this license detection was discovered at. Also contains whether this license was discovered from a file or from package metadata.",
+                        verbose_name="Detection Locations",
+                    ),
+                ),
+                (
+                    "project",
+                    models.ForeignKey(
+                        editable=False,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="%(class)ss",
+                        to="scanpipe.project",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["detection_count", "identifier"],
+                "indexes": [
+                    models.Index(
+                        fields=["identifier"], name="scanpipe_di_identif_b533f3_idx"
+                    ),
+                    models.Index(
+                        fields=["license_expression"],
+                        name="scanpipe_di_license_33d11a_idx",
+                    ),
+                    models.Index(
+                        fields=["license_expression_spdx"],
+                        name="scanpipe_di_license_eb5e9d_idx",
+                    ),
+                    models.Index(
+                        fields=["detection_count"],
+                        name="scanpipe_di_detecti_d87ff1_idx",
+                    ),
+                ],
+            },
+            bases=(
+                scanpipe.models.UpdateMixin,
+                scanpipe.models.SaveProjectMessageMixin,
+                scanpipe.models.UpdateFromDataMixin,
+                models.Model,
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="discoveredlicense",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("identifier", ""), _negated=True),
+                fields=("project", "identifier"),
+                name="scanpipe_discoveredlicense_unique_license_id_within_project",
+            ),
+        ),
+    ]