Process deleting projects as background task

1. Fix Default Filtering in ProjectFilterSet:
   - Modified the `ProjectFilterSet` to include `is_marked_for_deletion` in the default filtering when `is_archived` is not provided.
   - This ensures that projects marked for deletion are excluded from default views.

2. Migration for `is_marked_for_deletion`:
   - Added a migration (`0052_project_is_marked_for_deletion.py`) to introduce the `is_marked_for_deletion` field in the `Project` model.

3. Updated `Project` Model:
   - Added `is_marked_for_deletion` field to the `Project` model with a default value of `False`.

4. Modified `ProjectListView` to Exclude Marked Projects:
   - Modified the `get_queryset` method in `ProjectListView` to only fetch projects with `is_marked_for_deletion=False`.

5. Introduced `mark_for_deletion` Method:
   - Added a `mark_for_deletion` method in the `Project` model to set the `is_marked_for_deletion` flag and save the model.

6. Add `delete_in_background` Method in `Project` Model:
   - Introduced a new `delete_in_background` method that marks the project for deletion and enqueues a background deletion task.

7. Background Deletion Task:
   - Created a background deletion task using `django_rq` to handle project deletion asynchronously.
   - Checks if the project is still marked for deletion before proceeding.
   - If an error occurs during deletion, updates project status and logs an error.

8. Updated `ProjectActionView` Success Message:
   - Modified the success message in `ProjectActionView` for the "delete" action to provide a more informative message based on the count.

Fixes: #1002
Signed-off-by: Jayanth Kumar <[email protected]>

scanpipe/filters.py

@@ -349,9 +349,13 @@ def __init__(self, data=None, *args, **kwargs):
 
         # Default filtering by "Active" projects.
         if not data or data.get("is_archived", "") == "":
-            self.queryset = self.queryset.filter(is_archived=False)
+            self.queryset = self.queryset.filter(
+                is_archived=False, is_marked_for_deletion=False
+            )
 
-        active_count = Project.objects.filter(is_archived=False).count()
+        active_count = Project.objects.filter(
+            is_archived=False, is_marked_for_deletion=False
+        ).count()
         archived_count = Project.objects.filter(is_archived=True).count()
         self.filters["is_archived"].extra["widget"] = BulmaLinkWidget(
             choices=[

scanpipe/migrations/0052_project_is_marked_for_deletion.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.1 on 2024-01-26 12:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0051_rename_pipelines_data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='is_marked_for_deletion',
+            field=models.BooleanField(default=False),
+        ),
+    ]

scanpipe/models.py

@@ -76,6 +76,7 @@
 from packageurl.contrib.django.models import PackageURLMixin
 from packageurl.contrib.django.models import PackageURLQuerySetMixin
 from rest_framework.authtoken.models import Token
+from rq import Queue
 from rq.command import send_stop_job_command
 from rq.exceptions import NoSuchJobError
 from rq.job import Job
@@ -534,6 +535,7 @@ class Project(UUIDPKModel, ExtraDataFieldMixin, UpdateMixin, models.Model):
     labels = TaggableManager(through=UUIDTaggedItem)
 
     objects = ProjectQuerySet.as_manager()
+    is_marked_for_deletion = models.BooleanField(default=False)
 
     class Meta:
         ordering = ["-created_date"]
@@ -633,6 +635,16 @@ def delete(self, *args, **kwargs):
 
         return super().delete(*args, **kwargs)
 
+    def mark_for_deletion(self):
+        self.is_marked_for_deletion = True
+        self.save()
+
+    def delete_in_background(self):
+        # Mark the project for deletion and enqueue background deletion task
+        self.mark_for_deletion()
+        q = Queue("default", connection=redis.Redis())
+        job = q.enqueue(tasks.background_delete_task, self)
+
     def reset(self, keep_input=True):
         """
         Reset the project by deleting all related database objects and all work

scanpipe/tasks.py

@@ -24,6 +24,8 @@
 
 from django.apps import apps
 
+from django_rq import job
+
 logger = logging.getLogger(__name__)
 
 
@@ -76,3 +78,19 @@ def execute_pipeline_task(run_pk):
         project.clear_tmp_directory()
         if next_run := project.get_next_run():
             next_run.start()
+
+
+@job
+def background_delete_task(project):
+    # Check if the project is still marked for deletion
+    if not project.is_marked_for_deletion:
+        return
+
+    # Perform the deletion process
+    try:
+        project.delete()
+    except Exception as e:
+        # Handle errors and update project errors or display a banner
+        project.is_marked_for_deletion = False
+        project.save()
+        project.add_error(description=f"Deletion failed: {str(e)}")

scanpipe/tests/test_views.py

@@ -183,11 +183,9 @@ def test_scanpipe_views_project_actions_view(self):
         }
         response = self.client.post(url, data=data, follow=True)
         self.assertRedirects(response, reverse("project_list"))
-        expected = '<div class="message-body">1 projects have been delete.</div>'
+        expected = '<div class="message-body">1 project is being deleted in the background.</div>'
         self.assertContains(response, expected, html=True)
-        expected = (
-            f'<div class="message-body">Project {random_uuid} does not exist.</div>'
-        )
+        expected = f"1 project is being deleted in the background."
         self.assertContains(response, expected, html=True)
 
     def test_scanpipe_views_project_details_is_archived(self):

scanpipe/views.py

@@ -1076,7 +1076,10 @@ def perform_action(self, action, project_uuid, action_kwargs=None):
 
         try:
             project = Project.objects.get(pk=project_uuid)
-            getattr(project, action)(**action_kwargs)
+            if action == "delete":
+                project.delete_in_background()
+            else:
+                getattr(project, action)(**action_kwargs)
             return True
         except Project.DoesNotExist:
             messages.error(self.request, f"Project {project_uuid} does not exist.")
@@ -1086,6 +1089,8 @@ def perform_action(self, action, project_uuid, action_kwargs=None):
             raise Http404
 
     def get_success_message(self, action, count):
+        if action == "delete":
+            return f"{count} project{'s' if count != 1 else ''} {'is' if count == 1 else 'are'} being deleted in the background."
         return f"{count} projects have been {action}."