fixes

ci/configuration.yaml

@@ -19,5 +19,4 @@ emg:
     celery_backend: 'redis://localhost:6379/1'
   results_production_dir: '/dummy/path/results'
   # metagenomics exchange
-  me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api'
-  me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014'
+  me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api'

config/local-tests.yml

@@ -16,5 +16,4 @@ emg:
     celery_backend: 'redis://localhost:6379/1'
   results_production_dir: '/dummy/path/results'
   # metagenomics exchange
-  me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api'
-  me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014'
+  me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api'

emgapi/management/commands/populate_metagenomics_exchange.py

@@ -60,17 +60,6 @@ def add_arguments(self, parser):
             help="Dry mode, no population of ME",
         )
 
-    def generate_metadata(self, mgya, run_accession, status):
-        return {
-            "confidence": "full",
-            "endPoint": f"https://www.ebi.ac.uk/metagenomics/analyses/{mgya}",
-            "method": ["other_metadata"],
-            "sourceID": mgya,
-            "sequenceID": run_accession,
-            "status": status,
-            "brokerID": settings.METAGENOMICS_EXCHANGE_MGNIFY_BROKER,
-        }
-
     def handle(self, *args, **options):
         self.study_accession = options.get("study")
         self.dry_run = options.get("dry_run")
@@ -81,7 +70,7 @@ def handle(self, *args, **options):
         # never indexed or updated after indexed
         analyses_to_index_and_update = AnalysisJob.objects_for_mgx_indexing.to_add()
         # suppressed only
-        analyses_to_delete = AnalysisJob.objects_for_mgx_indexing.get_suppressed()
+        analyses_to_delete = AnalysisJob.objects_for_mgx_indexing.to_delete()
 
         if self.study_accession:
             analyses_to_index_and_update = analyses_to_index_and_update.filter(
@@ -104,15 +93,13 @@ def handle(self, *args, **options):
 
         logging.info("Done")
 
-
     def process_to_index_and_update_records(self, analyses_to_index_and_update):
         logging.info(f"Indexing {len(analyses_to_index_and_update)} new analyses")
 
         for page in Paginator(analyses_to_index_and_update, 100):
             jobs_to_update = []
-
             for ajob in page:
-                metadata = self.generate_metadata(
+                metadata = self.mgx_api.generate_metadata(
                     mgya=ajob.accession,
                     run_accession=ajob.run,
                     status="public" if not ajob.is_private else "private",
@@ -157,7 +144,7 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update):
                             logging.info(f"Analysis {ajob} updated successfully")
                             # Just to be safe, update the MGX accession
                             ajob.mgx_accession = registry_id
-                            ajob.last_mgx_indexed = timezone.now()
+                            ajob.last_mgx_indexed = timezone.now() + timedelta(minutes=1)
                             jobs_to_update.append(ajob)
                         else:
                             logging.error(f"Analysis {ajob} update failed")
@@ -178,7 +165,7 @@ def process_to_delete_records(self, analyses_to_delete):
             jobs_to_update = []
 
             for ajob in page:
-                metadata = self.generate_metadata(
+                metadata = self.mgx_api.generate_metadata(
                     mgya=ajob.accession,
                     run_accession=ajob.run,
                     status="public" if not ajob.is_private else "private",

emgapi/metagenomics_exchange.py

@@ -73,16 +73,19 @@ def patch_request(self, endpoint: str, data: dict):
         response.raise_for_status()
         return response
 
-    def add_analysis(self, mgya: str, run_accession: str, public: bool):
-        data = {
+    def generate_metadata(self, mgya, run_accession, status):
+        return {
             "confidence": "full",
             "endPoint": f"https://www.ebi.ac.uk/metagenomics/analyses/{mgya}",
             "method": ["other_metadata"],
             "sourceID": mgya,
             "sequenceID": run_accession,
-            "status": "public" if public else "private",
+            "status": status,
             "brokerID": self.broker,
         }
+
+    def add_analysis(self, mgya: str, run_accession: str, public: bool):
+        data = self.generate_metadata(mgya, run_accession, public)
         response = self.post_request(endpoint="datasets", data=data)
         return response
 

emgapi/models.py

@@ -331,16 +331,6 @@ def to_add(self):
 
         return self.filter(never_indexed | updated_after_indexing, not_suppressed, not_private)
 
-    def get_suppressed(self):
-        try:
-            self.model._meta.get_field("suppressed_at")
-        except FieldDoesNotExist:
-            return Q()
-        else:
-            return self.filter(
-                Q(suppressed_at__gte=F(self.index_field))
-            )
-
 
 class EBISearchIndexQueryset(IndexableModelQueryset):
 
@@ -366,6 +356,16 @@ class MetagenomicsExchangeQueryset(IndexableModelQueryset):
 
     index_field = "last_mgx_indexed"
 
+    def to_delete(self):
+        try:
+            self.model._meta.get_field("suppressed_at")
+        except FieldDoesNotExist:
+            return Q()
+        else:
+            return self.filter(
+                Q(suppressed_at__gte=F(self.index_field))
+            )
+
 
 class MetagenomicsExchangeIndexedModel(models.Model):
     """Model to track Metagenomics Exchange indexation of analysis jobs
@@ -1687,49 +1687,6 @@ def __str__(self):
         return 'Assembly:{} - Sample:{}'.format(self.assembly, self.sample)
 
 
-class MetagenomicsExchangeQueryset(models.QuerySet):
-    """
-    to_delete: Objects that have been suppressed since they were last populated,
-    or that have been added but updated since.
-
-    to_add: Objects that have never been added,
-    or that have been added but updated since.
-    """
-    def to_delete(self):
-        updated_after_populating = Q(last_updated_me__gte=F("last_populated_me"), last_populated_me__isnull=False)
-
-        try:
-            self.model._meta.get_field("suppressed_at")
-        except FieldDoesNotExist:
-            return self.filter(
-                updated_after_populating
-            )
-        else:
-            return self.filter(
-                Q(suppressed_at__gte=F("last_populated_me"))
-            )
-
-    def to_add(self):
-        updated_after_populating = Q(last_updated_me__gte=F("last_populated_me"), last_populated_me__isnull=False)
-        never_populated = Q(last_populated_me__isnull=True)
-
-        try:
-            self.model._meta.get_field("is_suppressed")
-        except FieldDoesNotExist:
-            not_suppressed = Q()
-        else:
-            not_suppressed = Q(is_suppressed=False)
-
-        try:
-            self.model._meta.get_field("is_private")
-        except FieldDoesNotExist:
-            not_private = Q()
-        else:
-            not_private = Q(is_private=False)
-
-        return self.filter(never_populated | updated_after_populating, not_suppressed, not_private)
-
-
 class MetagenomicsExchangeModel(models.Model):
     """Model to track Metagenomics Exchange population
     https://www.ebi.ac.uk/ena/registry/metagenome/api/

emgcli/settings.py

@@ -689,6 +689,9 @@ def create_secret_key(var_dir):
 METAGENOMICS_EXCHANGE_API_TOKEN = ""
 try:
     METAGENOMICS_EXCHANGE_API = EMG_CONF['emg']['me_api']
-    METAGENOMICS_EXCHANGE_API_TOKEN = EMG_CONF['emg']['me_api_token']
+    if EMG_CONF['emg'].get('me_api_token'):
+        METAGENOMICS_EXCHANGE_API_TOKEN = EMG_CONF['emg']['me_api_token']
+    else:
+        METAGENOMICS_EXCHANGE_API_TOKEN = os.getenv('METAGENOMICS_EXCHANGE_API_TOKEN')
 except KeyError:
     warnings.warn("The metagenomics exchange API and Token are not configured properly")