diff --git a/emgapi/management/commands/populate_metagenomics_exchange.py b/emgapi/management/commands/populate_metagenomics_exchange.py index 1d188d678..8707cd8e9 100644 --- a/emgapi/management/commands/populate_metagenomics_exchange.py +++ b/emgapi/management/commands/populate_metagenomics_exchange.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2017-2022 EMBL - European Bioinformatics Institute +# Copyright 2017-2024 EMBL - European Bioinformatics Institute # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -96,7 +96,7 @@ def handle(self, *args, **options): def process_to_index_and_update_records(self, analyses_to_index_and_update): logging.info(f"Indexing {len(analyses_to_index_and_update)} new analyses") - for page in Paginator(analyses_to_index_and_update, 100): + for page in Paginator(analyses_to_index_and_update, settings.METAGENOMICS_EXCHANGE_PAGINATOR_NUMBER): jobs_to_update = [] for ajob in page: metadata = self.mgx_api.generate_metadata( @@ -152,7 +152,7 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update): logging.debug(f"No edit for {ajob}, metadata is correct") AnalysisJob.objects.bulk_update( - jobs_to_update, ["last_mgx_indexed", "mgx_accession"], batch_size=100 + jobs_to_update, ["last_mgx_indexed", "mgx_accession"], batch_size=settings.METAGENOMICS_EXCHANGE_PAGINATOR_NUMBER ) def process_to_delete_records(self, analyses_to_delete): @@ -161,7 +161,7 @@ def process_to_delete_records(self, analyses_to_delete): """ logging.info(f"Processing {len(analyses_to_delete)} analyses to remove") - for page in Paginator(analyses_to_delete, 100): + for page in Paginator(analyses_to_delete, settings.METAGENOMICS_EXCHANGE_PAGINATOR_NUMBER): jobs_to_update = [] for ajob in page: @@ -192,5 +192,5 @@ def process_to_delete_records(self, analyses_to_delete): # BULK UPDATE # AnalysisJob.objects.bulk_update( - jobs_to_update, ["last_mgx_indexed"], batch_size=100 + jobs_to_update, ["last_mgx_indexed"], batch_size=settings.METAGENOMICS_EXCHANGE_PAGINATOR_NUMBER ) diff --git a/emgapi/metagenomics_exchange.py b/emgapi/metagenomics_exchange.py index bcbab1f88..6c3015154 100644 --- a/emgapi/metagenomics_exchange.py +++ b/emgapi/metagenomics_exchange.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2018-2023 EMBL - European Bioinformatics Institute +# Copyright 2018-2024 EMBL - European Bioinformatics Institute # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ class MetagenomicsExchangeAPI: """Metagenomics Exchange API Client""" def __init__(self, base_url=None): - self.base_url = base_url if base_url else settings.METAGENOMICS_EXCHANGE_API + self.base_url = base_url or settings.METAGENOMICS_EXCHANGE_API self.__token = settings.METAGENOMICS_EXCHANGE_API_TOKEN self.broker = settings.METAGENOMICS_EXCHANGE_MGNIFY_BROKER diff --git a/emgapi/models.py b/emgapi/models.py index a7d36220a..31db68bb3 100644 --- a/emgapi/models.py +++ b/emgapi/models.py @@ -1687,27 +1687,6 @@ def __str__(self): return 'Assembly:{} - Sample:{}'.format(self.assembly, self.sample) -class MetagenomicsExchangeModel(models.Model): - """Model to track Metagenomics Exchange population - https://www.ebi.ac.uk/ena/registry/metagenome/api/ - """ - last_updated_me = models.DateTimeField( - db_column='LAST_UPDATED_ME', - auto_now=True - ) - last_populated_me = models.DateTimeField( - db_column='LAST_POPULATED_ME', - null=True, - blank=True, - help_text="Date at which this model was last appeared in Metagenomics Exchange" - ) - - objects_for_population = MetagenomicsExchangeQueryset.as_manager() - - class Meta: - abstract = True - - class AnalysisJobQuerySet(BaseQuerySet, MySQLQuerySet, SuppressQuerySet, SelectRelatedOnlyQuerySetMixin): def __init__(self, *args, **kwargs): diff --git a/emgcli/settings.py b/emgcli/settings.py index 0d55f1e94..c778d1f5a 100644 --- a/emgcli/settings.py +++ b/emgcli/settings.py @@ -687,11 +687,11 @@ def create_secret_key(var_dir): METAGENOMICS_EXCHANGE_MGNIFY_BROKER = "EMG" METAGENOMICS_EXCHANGE_API = "" METAGENOMICS_EXCHANGE_API_TOKEN = "" +METAGENOMICS_EXCHANGE_PAGINATOR_NUMBER = 100 try: METAGENOMICS_EXCHANGE_API = EMG_CONF['emg']['me_api'] - if EMG_CONF['emg'].get('me_api_token'): + METAGENOMICS_EXCHANGE_API_TOKEN = os.getenv('METAGENOMICS_EXCHANGE_API_TOKEN') + if not METAGENOMICS_EXCHANGE_API_TOKEN: METAGENOMICS_EXCHANGE_API_TOKEN = EMG_CONF['emg']['me_api_token'] - else: - METAGENOMICS_EXCHANGE_API_TOKEN = os.getenv('METAGENOMICS_EXCHANGE_API_TOKEN') except KeyError: - warnings.warn("The metagenomics exchange API and Token are not configured properly") + warnings.warn("The metagenomics exchange API and Token are not configured properly") \ No newline at end of file