ENA Status Sync commands (#280)

ENA Sync commands for samples, studies and assemblies. Ported from MGnify legacy production system.

.github/workflows/test.yml

@@ -37,8 +37,6 @@ jobs:
         pip install -U git+https://github.com/EBI-Metagenomics/ena-api-handler.git
         pip install -U -r requirements.txt
         pip install -U -r requirements-test.txt
-        python setup.py sdist
-        pip install -U .
         pip freeze
     - name: 🧪 - Testing
       run: |

emgapi/migrations/0040_auto_20220721_0958.py

@@ -22,8 +22,8 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name='analysisjob',
-            name='suppresion_reason',
-            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='REASON', null=True),
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
         ),
         migrations.AddField(
             model_name='analysisjob',
@@ -42,8 +42,8 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name='assembly',
-            name='suppresion_reason',
-            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='REASON', null=True),
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
         ),
         migrations.AddField(
             model_name='assembly',
@@ -62,8 +62,8 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name='run',
-            name='suppresion_reason',
-            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='REASON', null=True),
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
         ),
         migrations.AddField(
             model_name='run',
@@ -82,8 +82,8 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name='sample',
-            name='suppresion_reason',
-            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='REASON', null=True),
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
         ),
         migrations.AddField(
             model_name='sample',
@@ -112,7 +112,7 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name='study',
-            name='suppresion_reason',
-            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='REASON', null=True),
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
         ),
     ]

emgapi/migrations/0041_migrate_status_to_suppressed_and_private.py

@@ -44,7 +44,7 @@ def migrate_the_status_field(apps, schema_editor):
             CANCELLED,
         ]:
             model.objects.filter(status_id=suppression_code).update(
-                **dict(suppress, **{"suppresion_reason": suppression_code})
+                **dict(suppress, **{"suppression_reason": suppression_code})
             )
 
     AnalysisJob = apps.get_model("emgapi", "AnalysisJob")
@@ -58,7 +58,7 @@ def migrate_the_status_field(apps, schema_editor):
         CANCELLED,
     ]:
         AnalysisJob.objects.filter(run_status_id=suppression_code).update(
-            **dict(suppress, **{"suppresion_reason": suppression_code})
+            **dict(suppress, **{"suppression_reason": suppression_code})
         )
 
 

emgapi/migrations/0043_auto_20220811_1308.py

@@ -0,0 +1,38 @@
+# Generated by Django 3.2.12 on 2022-08-11 13:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('emgapi', '0042_auto_20220722_0745'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='analysisjob',
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (5, 'Suppressed'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
+        ),
+        migrations.AlterField(
+            model_name='assembly',
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (5, 'Suppressed'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
+        ),
+        migrations.AlterField(
+            model_name='run',
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (5, 'Suppressed'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
+        ),
+        migrations.AlterField(
+            model_name='sample',
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (5, 'Suppressed'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
+        ),
+        migrations.AlterField(
+            model_name='study',
+            name='suppression_reason',
+            field=models.IntegerField(blank=True, choices=[(1, 'Draft'), (3, 'Cancelled'), (5, 'Suppressed'), (6, 'Killed'), (7, 'Temporary Suppressed'), (8, 'Temporary Killed')], db_column='SUPPRESSION_REASON', null=True),
+        ),
+    ]

emgapi/models.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
 from django.db import models
 from django.db.models import (CharField, Count, OuterRef, Prefetch, Q,
                               Subquery, Value)
@@ -26,6 +28,11 @@
 
 from emgapi.validators import validate_ena_study_accession
 
+from emgena.models import Status as ENAStatus
+
+
+logger = logging.getLogger(__name__)
+
 
 class Resource(object):
     def __init__(self, **kwargs):
@@ -47,8 +54,10 @@ class Meta:
 
 class SuppressQuerySet(models.QuerySet):
     def suppress(self, reason):
-        return self.update(is_suppressed=True, suppressed_at=timezone.now(), reason=reason)
+        return self.update(is_suppressed=True, suppressed_at=timezone.now(), suppression_reason=reason)
 
+    def unsuppress(self, reason):
+        return self.update(is_suppressed=False, suppressed_at=None, suppression_reason=None)
 
 class SuppressManager(models.Manager):
     def get_queryset(self):
@@ -60,20 +69,86 @@ class SuppressibleModel(models.Model):
     class Reason(models.IntegerChoices):
         DRAFT = 1
         CANCELLED = 3
+        SUPPRESSED = 5
         KILLED = 6
         TEMPORARY_SUPPRESSED = 7
         TEMPORARY_KILLED = 8
 
     is_suppressed = models.BooleanField(db_column='IS_SUPPRESSED', default=False)
     suppressed_at = models.DateTimeField(db_column='SUPPRESSED_AT', blank=True, null=True)
-    suppresion_reason = models.IntegerField(db_column='REASON', blank=True, null=True, choices=Reason.choices)
+    suppression_reason = models.IntegerField(db_column='SUPPRESSION_REASON', blank=True, null=True, choices=Reason.choices)
 
-    def suppress(self, reason=None):
+    def suppress(self, suppression_reason=None, save=True):
         self.is_suppressed = True
         self.suppressed_at = timezone.now()
-        if reason:
-            self.suppresion_reason = reason
-        self.save()
+        self.suppression_reason = suppression_reason
+        if save:
+            self.save()
+        return self
+
+    def unsuppress(self, suppression_reason=None, save=True):
+        self.is_suppressed = False
+        self.suppressed_at = None
+        self.suppression_reason = None
+        if save:
+            self.save()
+        return self
+
+    class Meta:
+        abstract = True
+
+
+class ENASyncableModel(SuppressibleModel, PrivacyControlledModel):
+
+    def sync_with_ena_status(self, ena_model_status: ENAStatus):
+        """Sync the model with the ENA status accordingly.
+        Fields that are updated: is_supppressed, suppressed_at, reason and is_private
+        """
+        if ena_model_status == ENAStatus.PRIVATE and not self.is_private:
+            self.is_private = True
+            logging.info(f"{self} marked as private")
+        if ena_model_status == ENAStatus.PUBLIC and self.is_private:
+            self.is_private = False
+            logging.info(f"{self} marked as public")
+
+        if ena_model_status == ENAStatus.DRAFT:
+            logging.warning(
+                f"{study} will not be updated due to the study status being 'draft'"
+            )
+
+        if (
+            ena_model_status
+            in [
+                ENAStatus.SUPPRESSED,
+                ENAStatus.KILLED,
+                ENAStatus.TEMPORARY_SUPPRESSED,
+                ENAStatus.TEMPORARY_KILLED,
+                ENAStatus.CANCELLED,
+            ]
+            and not self.is_suppressed
+        ):
+            reason = None
+            if ena_model_status == ENAStatus.SUPPRESSED:
+                reason = SuppressibleModel.Reason.SUPPRESSED
+            if ena_model_status == ENAStatus.KILLED:
+                reason = SuppressibleModel.Reason.KILLED
+            elif ena_model_status == ENAStatus.CANCELLED:
+                reason = SuppressibleModel.Reason.CANCELLED
+            elif ena_model_status == ENAStatus.TEMPORARY_SUPPRESSED:
+                reason = (
+                    SuppressibleModel.Reason.TEMPORARY_SUPPRESSED
+                )
+            elif ena_model_status == ENAStatus.TEMPORARY_KILLED:
+                reason = SuppressibleModel.Reason.TEMPORARY_KILLED
+            elif ena_model_status == ENAStatus.CANCELLED:
+                reason = SuppressibleModel.Reason.CANCELLED
+
+            self.suppress(suppression_reason=reason, save=False)
+
+            logging.info(
+                f"{self} was suppressed, status on ENA {ena_model_status}"
+            )
+
         return self
 
     class Meta:
@@ -675,7 +750,7 @@ def mydata(self, request):
         return self.get_queryset().mydata(request)
 
 
-class Study(SuppressibleModel, PrivacyControlledModel):
+class Study(ENASyncableModel):
 
     def __init__(self, *args, **kwargs):
         super(Study, self).__init__(*args, **kwargs)
@@ -906,7 +981,7 @@ def available(self, request, prefetch=False):
         return queryset
 
 
-class Sample(SuppressibleModel, PrivacyControlledModel):
+class Sample(ENASyncableModel):
     sample_id = models.AutoField(
         db_column='SAMPLE_ID', primary_key=True)
     accession = models.CharField(
@@ -1112,7 +1187,7 @@ def available(self, request):
         return self.get_queryset().available(request)
 
 
-class Run(SuppressibleModel, PrivacyControlledModel):
+class Run(ENASyncableModel):
     run_id = models.BigAutoField(
         db_column='RUN_ID', primary_key=True)
     accession = models.CharField(
@@ -1174,7 +1249,7 @@ def available(self, request):
         )
 
 
-class Assembly(SuppressibleModel, PrivacyControlledModel):
+class Assembly(ENASyncableModel):
 
     assembly_id = models.BigAutoField(
         db_column='ASSEMBLY_ID', primary_key=True)
@@ -1212,7 +1287,7 @@ class Meta:
         verbose_name_plural = 'assemblies'
 
     def __str__(self):
-        return self.accession
+        return self.accession or str(self.assembly_id)
 
 
 class AssemblyRun(models.Model):

emgapi/serializers.py

@@ -508,7 +508,7 @@ class Meta:
         model = emg_models.Run
         exclude = (
             'is_suppressed',
-            'suppresion_reason',
+            'suppression_reason',
             'suppressed_at',
         )
 
@@ -1063,7 +1063,7 @@ class Meta:
             'primary_accession',
             'is_private',
             'is_suppressed',
-            'suppresion_reason',
+            'suppression_reason',
             'suppressed_at',
             'metadata_received',
             'sequencedata_received',
@@ -1318,7 +1318,7 @@ class Meta:
             'author_email',
             'author_name',
             'is_suppressed',
-            'suppresion_reason',
+            'suppression_reason',
             'suppressed_at',
         )