From fcd9b450c6093848f13cb7326c407ecf4afcc8a8 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Fri, 17 Nov 2023 16:26:40 +0000 Subject: [PATCH] add population scripts and tests for dry-run --- ci/configuration.yaml | 4 +- config/local-tests.yml | 4 +- emgapi/management/commands/mgx_api.py | 123 -------------- .../commands/populate_existing_me.py | 74 --------- .../populate_metagenomics_exchange.py | 151 ++++++++++++++++++ emgapi/metagenomics_exchange.py | 39 +++-- emgapi/migrations/0012_auto_20231115_1448.py | 37 +++++ emgapi/models.py | 12 +- emgcli/settings.py | 8 + tests/me/test_metagenomics_exchange.py | 10 +- tests/me/test_mgx_api.py | 99 ------------ .../me/test_populate_metagenomics_exchange.py | 66 ++++++++ tests/test_utils/emg_fixtures.py | 34 ++-- 13 files changed, 331 insertions(+), 330 deletions(-) delete mode 100644 emgapi/management/commands/mgx_api.py delete mode 100644 emgapi/management/commands/populate_existing_me.py create mode 100644 emgapi/management/commands/populate_metagenomics_exchange.py create mode 100644 emgapi/migrations/0012_auto_20231115_1448.py delete mode 100644 tests/me/test_mgx_api.py create mode 100644 tests/me/test_populate_metagenomics_exchange.py diff --git a/ci/configuration.yaml b/ci/configuration.yaml index 8d99cd8f6..a05127d10 100644 --- a/ci/configuration.yaml +++ b/ci/configuration.yaml @@ -20,4 +20,6 @@ emg: results_production_dir: '/dummy/path/results' # metagenomics exchange me_api: 'http://wp-np2-5c.ebi.ac.uk:8080/ena/registry/metagenome/api' - me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014 ' \ No newline at end of file + me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014' + me_api_dev: 'http://wp-np2-5c.ebi.ac.uk:8080/ena/registry/metagenome/api' + me_api_token_dev: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014' \ No newline at end of file diff --git a/config/local-tests.yml b/config/local-tests.yml index 4f24fada9..f37eaff5d 100644 --- a/config/local-tests.yml +++ b/config/local-tests.yml @@ -17,4 +17,6 @@ emg: results_production_dir: '/dummy/path/results' # metagenomics exchange me_api: 'http://wp-np2-5c.ebi.ac.uk:8080/ena/registry/metagenome/api' - me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014 ' \ No newline at end of file + me_api_token: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014' + me_api_dev: 'http://wp-np2-5c.ebi.ac.uk:8080/ena/registry/metagenome/api' + me_api_token_dev: 'mgx 871cd915-2826-46bb-94ed-8e6a3d9b6014' \ No newline at end of file diff --git a/emgapi/management/commands/mgx_api.py b/emgapi/management/commands/mgx_api.py deleted file mode 100644 index 984f8fe6e..000000000 --- a/emgapi/management/commands/mgx_api.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2017-2022 EMBL - European Bioinformatics Institute -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from django.core.management import BaseCommand - -from emgapi.models import AnalysisJob, MetagenomicsExchange, ME_Broker -from emgapi.metagenomics_exchange import MetagenomicsExchangeAPI - -logger = logging.getLogger(__name__) - -RETRY_COUNT = 5 - -class Command(BaseCommand): - help = "Check and populate metagenomics exchange." - def add_arguments(self, parser): - super(Command, self).add_arguments(parser) - parser.add_argument( - "-s", - "--study", - action="store", - required=False, - type=str, - help="Study accession (rather than all)", - ) - parser.add_argument( - "-p", - "--pipeline", - help="Pipeline version (rather than all). Not applicable to Genomes.", - action="store", - dest="pipeline", - choices=[1.0, 2.0, 3.0, 4.0, 4.1, 5.0], - required=False, - type=float, - ) - parser.add_argument( - "-z", - "--assembly", - action="store", - required=False, - type=str, - help="Assembly accession (rather than all)", - nargs="+" - ) - parser.add_argument( - "-r", - "--run", - action="store", - required=False, - type=str, - help="Run accession (rather than all)", - nargs="+" - ) - parser.add_argument( - "--dev", - action="store_true", - required=False, - help="Populate dev API", - ) - parser.add_argument( - "-b", - "--broker", - action="store", - required=False, - type=str, - default='EMG', - help="Broker name", - choices=["EMG", "MAR"], - ) - - def _filtering_analyses(self): - analyses = AnalysisJob.objects.all() - if self.study_accession: - analyses = analyses.filter(study__secondary_accession=self.study_accession) - if self.run_accession: - analyses = analyses.filter(run__accession=self.run_accession) - if self.assembly_accession: - analyses = analyses.filter(assembly__accession=self.assembly_accession) - if self.pipeline_version: - analyses = analyses.filter(pipeline__pipeline_id=self.pipeline_version) - return analyses - - def handle(self, *args, **options): - self.study_accession = options.get("study") - self.pipeline_version = options.get("pipeline") - self.assembly_accession = options.get("assembly") - self.run_accession = options.get("run") - - broker = options.get("broker") - broker_obj = ME_Broker.objects.get_or_create(broker) - ME = MetagenomicsExchangeAPI(broker=broker) - analyses = self._filtering_analyses() - - for analysis in analyses: - MGYA = analysis.accession - public = not analysis.is_private - # check is MGYA in ME - if ME.check_analysis(source_id=MGYA, public=public): - logging.info(f"{MGYA} exists in ME") - else: - logging.info(f"{MGYA} does not exist in ME") - response = ME.add_record(mgya=MGYA, run_accession=analysis.run, public=public) - if response.status_code == 201: - logging.info(f"Populating MetagenomicsExchange table with {MGYA}") - MetagenomicsExchange.objects.get_or_create(analysis, broker=broker_obj) - else: - logging.error(f"{MGYA} response exit code: {response.status_code}. No population to DB.") - - diff --git a/emgapi/management/commands/populate_existing_me.py b/emgapi/management/commands/populate_existing_me.py deleted file mode 100644 index 9a031092e..000000000 --- a/emgapi/management/commands/populate_existing_me.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2017-2022 EMBL - European Bioinformatics Institute -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import requests - -from django.db.models import Count -from django.core.management import BaseCommand -from django.conf import settings - -from emgapi.models import Assembly, AnalysisJob - -logger = logging.getLogger(__name__) -API = { - 'real': 'https://www.ebi.ac.uk/ena/registry/metagenome/api/', - 'dev': 'http://wp-np2-5c.ebi.ac.uk:8080/ena/registry/metagenome/api/' -} -TOKEN = 'mgx 3D70473ED7C443CA9E97749F62FFCC5D' - -RETRY_COUNT = 5 - -class Command(BaseCommand): - help = "Populate DB with existing metagenomics exchange" - def add_arguments(self, parser): - super(Command, self).add_arguments(parser) - - parser.add_argument( - "--dev", - action="store_true", - required=False, - help="Populate dev API", - ) - parser.add_argument( - "-b", - "--broker", - action="store", - required=False, - type=str, - default='EMG', - help="Broker name", - choices=["EMG", "MAR"], - ) - def get_request(self, url, params): - if self.auth: - response = requests.get(url, params=params, headers=self.auth) - else: - logging.warning( - "Not authenticated to get private data." - # noqa: E501 - ) - response = requests.get(url, params=params) - return response - - def handle(self, *args, **options): - self.auth = None - self.url = API['dev'] - self.broker = options.get("broker") - self.check_url = self.url + f'brokers/{self.broker}/datasets' - - diff --git a/emgapi/management/commands/populate_metagenomics_exchange.py b/emgapi/management/commands/populate_metagenomics_exchange.py new file mode 100644 index 000000000..2c4dff5d8 --- /dev/null +++ b/emgapi/management/commands/populate_metagenomics_exchange.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2017-2022 EMBL - European Bioinformatics Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import responses + +from django.conf import settings +from django.core.management import BaseCommand + +from emgapi.models import AnalysisJob +from emgapi.metagenomics_exchange import MetagenomicsExchangeAPI + +logger = logging.getLogger(__name__) + +RETRY_COUNT = 5 + +class Command(BaseCommand): + help = "Check and populate metagenomics exchange (ME)." + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser.add_argument( + "-s", + "--study", + required=False, + type=str, + help="Study accession list (rather than all)", + nargs='+', + ) + parser.add_argument( + "-p", + "--pipeline", + help="Pipeline version (rather than all). Not applicable to Genomes.", + action="store", + dest="pipeline", + choices=[1.0, 2.0, 3.0, 4.0, 4.1, 5.0], + required=False, + type=float, + ) + parser.add_argument( + "--dev", + action="store_true", + required=False, + help="Populate dev API", + ) + parser.add_argument( + "--dry-run", + action="store_true", + required=False, + help="Dry mode, no population of ME", + ) + # TODO: do I need it? + parser.add_argument( + "--full", + action="store_true", + help="Do a full check of DB", + ) + + def handle(self, *args, **options): + self.study_accession = options.get("study") + self.dry_run = options.get("dry_run") + self.pipeline_version = options.get("pipeline") + if options.get("dev"): + base_url = settings.ME_API_DEV + else: + base_url = settings.ME_API + ME = MetagenomicsExchangeAPI(base_url=base_url) + + new_analyses = AnalysisJob.objects_for_population.to_add() + removals = AnalysisJob.objects_for_population.to_delete() + if self.study_accession: + new_analyses = new_analyses.filter(study__secondary_accession__in=self.study_accession) + removals = removals.filter(study__secondary_accession__in=self.study_accession) + if self.pipeline_version: + new_analyses = new_analyses.filter(pipeline__pipeline_id=self.pipeline_version) + removals = removals.filter(pipeline__pipeline_id=self.pipeline_version) + logging.info(f"Processing {len(new_analyses)} new analyses") + for ajob in new_analyses: + metadata = { + "confidence": "full", + "endPoint": f"https://www.ebi.ac.uk/metagenomics/analyses/{ajob.accession}", + "method": ["other_metadata"], + "sourceID": ajob.accession, + "sequenceID": ajob.run, + "status": "public" if not ajob.is_private else "private", + "brokerID": settings.MGNIFY_BROKER, + } + registryID, metadata_match = ME.check_analysis(source_id=ajob.accession, metadata=metadata) + if not registryID: + logging.debug(f"Add new {ajob}") + if not self.dry_run: + response = ME.add_analysis(mgya=ajob.accession, run_accession=ajob.run, public=not ajob.is_private) + if response.ok: + logging.debug(f"Added {ajob}") + else: + logging.debug(f"Error adding {ajob}: {response.message}") + else: + logging.info(f"Dry-mode run: no addition to real ME for {ajob}") + else: + if not metadata_match: + logging.debug(f"Patch existing {ajob}") + data = { + "confidence": "full", + "endPoint": f"https://www.ebi.ac.uk/metagenomics/analyses/{ajob.accession}", + "method": ["other_metadata"], + "sourceID": ajob.accession, + "sequenceID": ajob.run, + "status": "public" if not ajob.is_private else "private", + "brokerID": settings.MGNIFY_BROKER, + } + if not self.dry_run: + if ME.patch_analysis(registry_id=registryID, data=data): + logging.info(f"Analysis {ajob} updated successfully") + else: + logging.info(f"Analysis {ajob} update failed") + else: + logging.info(f"Dry-mode run: no patch to real ME for {ajob}") + else: + logging.debug(f"No edit for {ajob}, metadata is correct") + + logging.info(f"Processing {len(removals)} analyses to remove") + for ajob in removals: + registryID, _ = ME.check_analysis(source_id=ajob.accession) + if registryID: + if not self.dry_run: + if ME.delete_analysis(registryID): + logging.info(f"{ajob} successfully deleted") + else: + logging.info(f"{ajob} failed on delete") + else: + logging.info(f"Dry-mode run: no delete from real ME for {ajob}") + else: + logging.info(f"No {ajob} in ME, nothing to delete") + logging.info("Done") + + + + + diff --git a/emgapi/metagenomics_exchange.py b/emgapi/metagenomics_exchange.py index 1a8d92654..b0245b83e 100644 --- a/emgapi/metagenomics_exchange.py +++ b/emgapi/metagenomics_exchange.py @@ -24,10 +24,10 @@ class MetagenomicsExchangeAPI: """Metagenomics Exchange API Client""" - def __init__(self, broker="EMG"): - self.base_url = settings.ME_API + def __init__(self, base_url=None): + self.base_url = base_url if base_url else settings.ME_API self.__token = settings.ME_API_TOKEN - self.broker = broker + self.broker = settings.MGNIFY_BROKER def get_request(self, endpoint: str, params: dict): """Make a GET request, returns the response""" @@ -86,22 +86,39 @@ def add_analysis(self, mgya: str, run_accession: str, public: bool): response = self.post_request(endpoint="datasets", data=data) return response - def check_analysis(self, source_id: str, public: bool) -> bool: + + def check_analysis(self, source_id: str, public=None, metadata=None) -> [str, bool]: logging.info(f"Check {source_id}") - params = { - "status": "public" if public else "private", - } + params = {} + if public: + params = { + "status": "public" if public else "private", + } endpoint = f"brokers/{self.broker}/datasets" response = self.get_request(endpoint=endpoint, params=params) + analysis_registryID = "" + metadata_match = True if response.ok: data = response.json() datasets = data.get("datasets") for item in datasets: if item.get("sourceID") == source_id: - logging.info(f"{source_id} exists") - return True - logging.info(f"{source_id} does not exist") - return False + logging.info(f"{source_id} exists in ME") + analysis_registryID = item.get("registryID") + if metadata: + for metadata_record in metadata: + if not(metadata_record in item): + metadata_match = False + return analysis_registryID, metadata_match + else: + if metadata[metadata_record] != item[metadata_record]: + metadata_match = False + logging.info(f"Incorrect field {metadata[metadata_record]} in ME ({item[metadata_record]})") + return analysis_registryID, metadata_match + return analysis_registryID, metadata_match + else: + logging.info(f"{source_id} does not exist in ME") + return analysis_registryID, metadata_match def delete_analysis(self, registry_id: str): response = self.delete_request(endpoint=f"datasets/{registry_id}") diff --git a/emgapi/migrations/0012_auto_20231115_1448.py b/emgapi/migrations/0012_auto_20231115_1448.py new file mode 100644 index 000000000..cb27bcb04 --- /dev/null +++ b/emgapi/migrations/0012_auto_20231115_1448.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.18 on 2023-11-15 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('emgapi', '0011_auto_20230912_1346'), + ] + + operations = [ + migrations.RemoveField( + model_name='metagenomicsexchange', + name='analysis_id', + ), + migrations.RemoveField( + model_name='metagenomicsexchange', + name='broker_id', + ), + migrations.AddField( + model_name='analysisjob', + name='last_populated_me', + field=models.DateTimeField(blank=True, db_column='LAST_POPULATED_ME', help_text='Date at which this model was last appeared in Metagenomics Exchange', null=True), + ), + migrations.AddField( + model_name='analysisjob', + name='last_updated_me', + field=models.DateTimeField(auto_now=True, db_column='LAST_UPDATED_ME'), + ), + migrations.DeleteModel( + name='ME_Broker', + ), + migrations.DeleteModel( + name='MetagenomicsExchange', + ), + ] diff --git a/emgapi/models.py b/emgapi/models.py index 46e2d699a..360bcb31e 100644 --- a/emgapi/models.py +++ b/emgapi/models.py @@ -1406,7 +1406,7 @@ class MetagenomicsExchangeQueryset(models.QuerySet): or that have been added but updated since. """ def to_delete(self): - updated_after_populating = Q(last_update_me__gte=F("last_indexed"), last_populated_me__isnull=False) + updated_after_populating = Q(last_updated_me__gte=F("last_populated_me"), last_populated_me__isnull=False) try: self.model._meta.get_field("suppressed_at") @@ -1416,11 +1416,11 @@ def to_delete(self): ) else: return self.filter( - Q(suppressed_at__gte=F("last_indexed")) | updated_after_populating + Q(suppressed_at__gte=F("last_populated_me")) ) def to_add(self): - updated_after_populating = Q(last_update_me__gte=F("last_indexed"), last_populated_me__isnull=False) + updated_after_populating = Q(last_updated_me__gte=F("last_populated_me"), last_populated_me__isnull=False) never_populated = Q(last_populated_me__isnull=True) try: @@ -1444,8 +1444,8 @@ class MetagenomicsExchangeModel(models.Model): """Model to track Metagenomics Exchange population https://www.ebi.ac.uk/ena/registry/metagenome/api/ """ - last_update_me = models.DateTimeField( - db_column='LAST_UPDATE_ME', + last_updated_me = models.DateTimeField( + db_column='LAST_UPDATED_ME', auto_now=True ) last_populated_me = models.DateTimeField( @@ -1455,7 +1455,7 @@ class MetagenomicsExchangeModel(models.Model): help_text="Date at which this model was last appeared in Metagenomics Exchange" ) - objects_for_indexing = MetagenomicsExchangeQueryset.as_manager() + objects_for_population = MetagenomicsExchangeQueryset.as_manager() class Meta: abstract = True diff --git a/emgcli/settings.py b/emgcli/settings.py index 4d3ad4610..575ed5ff4 100644 --- a/emgcli/settings.py +++ b/emgcli/settings.py @@ -643,6 +643,7 @@ def create_secret_key(var_dir): os.environ['ENA_API_PASSWORD'] = EMG_CONF['emg']['ena_api_password'] # Metagenomics Exchange +MGNIFY_BROKER = "EMG" try: ME_API = EMG_CONF['emg']['me_api'] ME_API_TOKEN = EMG_CONF['emg']['me_api_token'] @@ -650,3 +651,10 @@ def create_secret_key(var_dir): ME_API = "" ME_API_TOKEN = "" warnings.warn("The metagenomics exchange API and Token are not configured properly") +try: + ME_API_DEV = EMG_CONF['emg']['me_api_dev'] + ME_API_TOKEN = EMG_CONF['emg']['me_api_token_dev'] +except KeyError: + ME_API_DEV = "" + ME_API_TOKEN = "" + warnings.warn("The metagenomics exchange DEV API and Token are not configured properly") diff --git a/tests/me/test_metagenomics_exchange.py b/tests/me/test_metagenomics_exchange.py index 2621b1e90..c6bd36a0d 100644 --- a/tests/me/test_metagenomics_exchange.py +++ b/tests/me/test_metagenomics_exchange.py @@ -15,12 +15,14 @@ class TestME: def test_check_existing_analysis_me(self): me_api = MetagenomicsExchangeAPI() source_id = "MGYA00293719" - assert me_api.check_analysis(source_id, True) + return_values = me_api.check_analysis(source_id, True) + assert return_values[0] def test_check_not_existing_analysis_me(self): - me_api = MetagenomicsExchangeAPI(broker="MAR") - source_id = "MGYA00293719" - assert not me_api.check_analysis(source_id, True) + me_api = MetagenomicsExchangeAPI() + source_id = "MGYA10293719" + return_values = me_api.check_analysis(source_id, True) + assert not return_values[0] def test_post_existing_analysis_me(self): me_api = MetagenomicsExchangeAPI() diff --git a/tests/me/test_mgx_api.py b/tests/me/test_mgx_api.py deleted file mode 100644 index 9a75acea0..000000000 --- a/tests/me/test_mgx_api.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2020 EMBL - European Bioinformatics Institute -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -import pytest - -from unittest.mock import patch - -from django.core.management import call_command - -from test_utils.emg_fixtures import * # noqa - -from emgapi.metagenomics_exchange import MetagenomicsExchangeAPI -from emgapi.models import AnalysisJob, MetagenomicsExchange, ME_Broker - -@pytest.mark.django_db -class TestMeAPI: - @patch("emgapi.metagenomics_exchange.MetagenomicsExchangeAPI.post_request") - @pytest.mark.usefixtures( - "me_broker", - "metagenomics_exchange", - ) - def test_new_analysis_population( - self, - mock_post_request - ): - class MockResponse: - def __init__(self, ok, status_code): - self.ok = ok - self.status_code = status_code - - mock_post_request.return_value = MockResponse(True, 201) - test_run = "ABC01234" - test_pipeline_version = 5.0 - test_broker = 'EMG' - - call_command( - "mgx_api", - run=test_run, - pipeline=test_pipeline_version, - broker=test_broker - ) - analysis = AnalysisJob.objects.filter(run__accession=test_run).filter(pipeline__pipeline_id=test_pipeline_version).first() - assert ME_Broker.objects.filter(brokerID=test_broker).count() == 1 - broker_id = ME_Broker.objects.filter(brokerID=test_broker).first().id - assert MetagenomicsExchange.objects.filter(analysis=analysis).filter(broker=broker_id).count() == 1 - - @pytest.mark.usefixtures( - "me_broker", - "metagenomics_exchange", - ) - @patch("emgapi.metagenomics_exchange.MetagenomicsExchangeAPI.get_request") - def test_check_existing_analysis( - self, - mock_get_request, - run_multiple_analysis, - me_broker, - metagenomics_exchange, - ): - class MockResponse: - def __init__(self, ok, status_code, json_data): - self.ok = ok - self.status_code = status_code - self.json_data = json_data - - def json(self): - return self.json_data - - test_run = "ABC01234" - test_pipeline_version = 1.0 - test_broker = 'MAR' - analysis = AnalysisJob.objects.filter(run__accession=test_run).filter( - pipeline__pipeline_id=test_pipeline_version).first() - json_data = {'datasets': [{'sourceID': analysis.accession}] } - mock_get_request.return_value = MockResponse(True, 200, json_data) - - call_command( - "mgx_api", - run=test_run, - pipeline=test_pipeline_version, - broker=test_broker - ) - - assert ME_Broker.objects.filter(brokerID=test_broker).count() == 1 - broker_id = ME_Broker.objects.filter(brokerID=test_broker).first().id - assert MetagenomicsExchange.objects.filter(analysis=analysis, broker=broker_id).count() == 1 -""" \ No newline at end of file diff --git a/tests/me/test_populate_metagenomics_exchange.py b/tests/me/test_populate_metagenomics_exchange.py new file mode 100644 index 000000000..9093051ce --- /dev/null +++ b/tests/me/test_populate_metagenomics_exchange.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2020 EMBL - European Bioinformatics Institute +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from unittest.mock import patch + +from django.core.management import call_command + +from test_utils.emg_fixtures import * # noqa + + +@pytest.mark.django_db +class TestMeAPI: + @pytest.mark.usefixtures('run_multiple_analysis') + def test_population_dry_mode( + self, + caplog + ): + call_command( + "populate_metagenomics_exchange", + dev=True, + dry_run=True, + ) + assert "Dry-mode run: no addition to real ME for MGYA00001234" in caplog.text + assert "Dry-mode run: no addition to real ME for MGYA00005678" in caplog.text + assert "Dry-mode run: no addition to real ME for MGYA00466090" in caplog.text + assert "Processing 0 analyses to remove" in caplog.text + + @pytest.mark.usefixtures('suppressed_analysis_jobs') + def test_removals_dry_mode( + self, + caplog + ): + call_command( + "populate_metagenomics_exchange", + dev=True, + dry_run=True, + ) + + assert "No MGYA00000002 in ME, nothing to delete" in caplog.text + assert "Processing 0 new analyses" in caplog.text + + @pytest.mark.usefixtures('analysis_existed_in_me') + def test_update_dry_mode(self, caplog): + call_command( + "populate_metagenomics_exchange", + dev=True, + dry_run=True, + ) + assert "Incorrect field None in ME (ERR1806500)" in caplog.text + assert "Dry-mode run: no patch to real ME for MGYA00147343" in caplog.text + assert "Processing 0 analyses to remove" in caplog.text diff --git a/tests/test_utils/emg_fixtures.py b/tests/test_utils/emg_fixtures.py index 32417f066..977121a21 100644 --- a/tests/test_utils/emg_fixtures.py +++ b/tests/test_utils/emg_fixtures.py @@ -36,7 +36,7 @@ 'ena_private_studies', 'ena_suppressed_studies', 'ena_public_runs', 'ena_private_runs', 'ena_suppressed_runs', 'ena_public_samples', 'ena_private_samples', 'ena_suppressed_samples', 'ena_public_assemblies', 'ena_private_assemblies', 'ena_suppressed_assemblies', - 'assembly_extra_annotation', 'me_broker', 'metagenomics_exchange', + 'assembly_extra_annotation', 'suppressed_analysis_jobs', 'analysis_existed_in_me', ] @@ -920,13 +920,25 @@ def ena_suppressed_assemblies(): ) return assemblies -@pytest.fixture -def me_broker(): - broker = baker.make(emg_models.ME_Broker, brokerID="MAR") - return broker - -@pytest.fixture -def metagenomics_exchange(me_broker, run_multiple_analysis): - analysis = run_multiple_analysis[0] - me = baker.make(emg_models.MetagenomicsExchange, broker=me_broker, analysis=analysis) - return me \ No newline at end of file +def make_suppressed_analysis_jobs(quantity, emg_props=None): + emg_props = emg_props or {} + analyses = baker.make(emg_models.AnalysisJob, _quantity=quantity, **emg_props) + return analyses + +@pytest.fixture +def suppressed_analysis_jobs(ena_suppressed_runs): + suppressed_analysisjobs = make_suppressed_analysis_jobs(quantity=5, + emg_props={"is_suppressed": True, + 'last_populated_me': '1970-01-01 00:00:00'}) + return suppressed_analysisjobs + +@pytest.fixture +def analysis_existed_in_me(): + emg_props = { + "job_id": 147343, + "last_populated_me": '1970-01-01 00:00:00', + "last_updated_me": '1980-01-01 00:00:00', + "is_suppressed": False, + "is_private": False + } + return baker.make(emg_models.AnalysisJob, **emg_props) \ No newline at end of file