From 6a55c40bfa0ad390fe619d0b54cfb918f1ecccb9 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Thu, 1 Feb 2024 11:52:00 +0000 Subject: [PATCH] fixes --- ci/configuration.yaml | 3 +- config/local-tests.yml | 3 +- .../populate_metagenomics_exchange.py | 21 ++----- emgapi/metagenomics_exchange.py | 9 ++- emgapi/models.py | 63 +++---------------- emgcli/settings.py | 5 +- 6 files changed, 26 insertions(+), 78 deletions(-) diff --git a/ci/configuration.yaml b/ci/configuration.yaml index 33f780ee9..1102dc73e 100644 --- a/ci/configuration.yaml +++ b/ci/configuration.yaml @@ -19,5 +19,4 @@ emg: celery_backend: 'redis://localhost:6379/1' results_production_dir: '/dummy/path/results' # metagenomics exchange - me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api' - me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014' \ No newline at end of file + me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api' \ No newline at end of file diff --git a/config/local-tests.yml b/config/local-tests.yml index 52636a862..db4297888 100644 --- a/config/local-tests.yml +++ b/config/local-tests.yml @@ -16,5 +16,4 @@ emg: celery_backend: 'redis://localhost:6379/1' results_production_dir: '/dummy/path/results' # metagenomics exchange - me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api' - me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014' \ No newline at end of file + me_api: 'https://wwwdev.ebi.ac.uk/ena/registry/metagenome/api' \ No newline at end of file diff --git a/emgapi/management/commands/populate_metagenomics_exchange.py b/emgapi/management/commands/populate_metagenomics_exchange.py index bd0b80f74..1d188d678 100644 --- a/emgapi/management/commands/populate_metagenomics_exchange.py +++ b/emgapi/management/commands/populate_metagenomics_exchange.py @@ -60,17 +60,6 @@ def add_arguments(self, parser): help="Dry mode, no population of ME", ) - def generate_metadata(self, mgya, run_accession, status): - return { - "confidence": "full", - "endPoint": f"https://www.ebi.ac.uk/metagenomics/analyses/{mgya}", - "method": ["other_metadata"], - "sourceID": mgya, - "sequenceID": run_accession, - "status": status, - "brokerID": settings.METAGENOMICS_EXCHANGE_MGNIFY_BROKER, - } - def handle(self, *args, **options): self.study_accession = options.get("study") self.dry_run = options.get("dry_run") @@ -81,7 +70,7 @@ def handle(self, *args, **options): # never indexed or updated after indexed analyses_to_index_and_update = AnalysisJob.objects_for_mgx_indexing.to_add() # suppressed only - analyses_to_delete = AnalysisJob.objects_for_mgx_indexing.get_suppressed() + analyses_to_delete = AnalysisJob.objects_for_mgx_indexing.to_delete() if self.study_accession: analyses_to_index_and_update = analyses_to_index_and_update.filter( @@ -104,15 +93,13 @@ def handle(self, *args, **options): logging.info("Done") - def process_to_index_and_update_records(self, analyses_to_index_and_update): logging.info(f"Indexing {len(analyses_to_index_and_update)} new analyses") for page in Paginator(analyses_to_index_and_update, 100): jobs_to_update = [] - for ajob in page: - metadata = self.generate_metadata( + metadata = self.mgx_api.generate_metadata( mgya=ajob.accession, run_accession=ajob.run, status="public" if not ajob.is_private else "private", @@ -157,7 +144,7 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update): logging.info(f"Analysis {ajob} updated successfully") # Just to be safe, update the MGX accession ajob.mgx_accession = registry_id - ajob.last_mgx_indexed = timezone.now() + ajob.last_mgx_indexed = timezone.now() + timedelta(minutes=1) jobs_to_update.append(ajob) else: logging.error(f"Analysis {ajob} update failed") @@ -178,7 +165,7 @@ def process_to_delete_records(self, analyses_to_delete): jobs_to_update = [] for ajob in page: - metadata = self.generate_metadata( + metadata = self.mgx_api.generate_metadata( mgya=ajob.accession, run_accession=ajob.run, status="public" if not ajob.is_private else "private", diff --git a/emgapi/metagenomics_exchange.py b/emgapi/metagenomics_exchange.py index a5942bc75..bcbab1f88 100644 --- a/emgapi/metagenomics_exchange.py +++ b/emgapi/metagenomics_exchange.py @@ -73,16 +73,19 @@ def patch_request(self, endpoint: str, data: dict): response.raise_for_status() return response - def add_analysis(self, mgya: str, run_accession: str, public: bool): - data = { + def generate_metadata(self, mgya, run_accession, status): + return { "confidence": "full", "endPoint": f"https://www.ebi.ac.uk/metagenomics/analyses/{mgya}", "method": ["other_metadata"], "sourceID": mgya, "sequenceID": run_accession, - "status": "public" if public else "private", + "status": status, "brokerID": self.broker, } + + def add_analysis(self, mgya: str, run_accession: str, public: bool): + data = self.generate_metadata(mgya, run_accession, public) response = self.post_request(endpoint="datasets", data=data) return response diff --git a/emgapi/models.py b/emgapi/models.py index cf6ab08f8..a7d36220a 100644 --- a/emgapi/models.py +++ b/emgapi/models.py @@ -331,16 +331,6 @@ def to_add(self): return self.filter(never_indexed | updated_after_indexing, not_suppressed, not_private) - def get_suppressed(self): - try: - self.model._meta.get_field("suppressed_at") - except FieldDoesNotExist: - return Q() - else: - return self.filter( - Q(suppressed_at__gte=F(self.index_field)) - ) - class EBISearchIndexQueryset(IndexableModelQueryset): @@ -366,6 +356,16 @@ class MetagenomicsExchangeQueryset(IndexableModelQueryset): index_field = "last_mgx_indexed" + def to_delete(self): + try: + self.model._meta.get_field("suppressed_at") + except FieldDoesNotExist: + return Q() + else: + return self.filter( + Q(suppressed_at__gte=F(self.index_field)) + ) + class MetagenomicsExchangeIndexedModel(models.Model): """Model to track Metagenomics Exchange indexation of analysis jobs @@ -1687,49 +1687,6 @@ def __str__(self): return 'Assembly:{} - Sample:{}'.format(self.assembly, self.sample) -class MetagenomicsExchangeQueryset(models.QuerySet): - """ - to_delete: Objects that have been suppressed since they were last populated, - or that have been added but updated since. - - to_add: Objects that have never been added, - or that have been added but updated since. - """ - def to_delete(self): - updated_after_populating = Q(last_updated_me__gte=F("last_populated_me"), last_populated_me__isnull=False) - - try: - self.model._meta.get_field("suppressed_at") - except FieldDoesNotExist: - return self.filter( - updated_after_populating - ) - else: - return self.filter( - Q(suppressed_at__gte=F("last_populated_me")) - ) - - def to_add(self): - updated_after_populating = Q(last_updated_me__gte=F("last_populated_me"), last_populated_me__isnull=False) - never_populated = Q(last_populated_me__isnull=True) - - try: - self.model._meta.get_field("is_suppressed") - except FieldDoesNotExist: - not_suppressed = Q() - else: - not_suppressed = Q(is_suppressed=False) - - try: - self.model._meta.get_field("is_private") - except FieldDoesNotExist: - not_private = Q() - else: - not_private = Q(is_private=False) - - return self.filter(never_populated | updated_after_populating, not_suppressed, not_private) - - class MetagenomicsExchangeModel(models.Model): """Model to track Metagenomics Exchange population https://www.ebi.ac.uk/ena/registry/metagenome/api/ diff --git a/emgcli/settings.py b/emgcli/settings.py index 27fc146eb..0d55f1e94 100644 --- a/emgcli/settings.py +++ b/emgcli/settings.py @@ -689,6 +689,9 @@ def create_secret_key(var_dir): METAGENOMICS_EXCHANGE_API_TOKEN = "" try: METAGENOMICS_EXCHANGE_API = EMG_CONF['emg']['me_api'] - METAGENOMICS_EXCHANGE_API_TOKEN = EMG_CONF['emg']['me_api_token'] + if EMG_CONF['emg'].get('me_api_token'): + METAGENOMICS_EXCHANGE_API_TOKEN = EMG_CONF['emg']['me_api_token'] + else: + METAGENOMICS_EXCHANGE_API_TOKEN = os.getenv('METAGENOMICS_EXCHANGE_API_TOKEN') except KeyError: warnings.warn("The metagenomics exchange API and Token are not configured properly")