From ec096dd06b18ada369741542b01d4887e17bc093 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Thu, 6 Jun 2024 11:07:13 +0100 Subject: [PATCH 1/2] Add check for missing records in DB (by some reason?) --- .../commands/populate_metagenomics_exchange.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/emgapi/management/commands/populate_metagenomics_exchange.py b/emgapi/management/commands/populate_metagenomics_exchange.py index 269addca9..db47f5680 100644 --- a/emgapi/management/commands/populate_metagenomics_exchange.py +++ b/emgapi/management/commands/populate_metagenomics_exchange.py @@ -174,9 +174,19 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update): else: logging.error(f"Analysis {annotation_job} update failed") else: - logging.debug( - f"No edit for {annotation_job}, metadata is correct" - ) + if annotation_job.mgx_accession and annotation_job.last_mgx_indexed: + logging.info( + f"No edit for {annotation_job}, metadata is correct" + ) + else: + logging.info( + f"Metadata is correct but {annotation_job} is missing in DB. Adding." + ) + annotation_job.mgx_accession = registry_id + annotation_job.last_mgx_indexed = ( + timezone.now() + timedelta(minutes=1) + ) + jobs_to_update.append(annotation_job) AnalysisJob.objects.bulk_update( jobs_to_update, From db8a6eab7755798bfb2eb161a5ca93d09e044095 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Thu, 6 Jun 2024 13:28:53 +0100 Subject: [PATCH 2/2] add retries to add_analysis --- emgapi/metagenomics_exchange.py | 45 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/emgapi/metagenomics_exchange.py b/emgapi/metagenomics_exchange.py index 500ce133d..13c69caaf 100644 --- a/emgapi/metagenomics_exchange.py +++ b/emgapi/metagenomics_exchange.py @@ -17,6 +17,7 @@ import logging import requests +import time from django.conf import settings from requests.exceptions import HTTPError, JSONDecodeError @@ -106,22 +107,38 @@ def add_analysis(self, mgya: str, sequence_accession: str): The response object from the API request. """ data = self.generate_metadata(mgya, sequence_accession) - try: - response = self.post_request(endpoint="datasets", data=data) - response.raise_for_status() # Ensure we raise for HTTP errors - return response - except HTTPError as http_error: + max_retries = 3 + wait_time = 2 # seconds + + for attempt in range(max_retries): try: - response_json = http_error.response.json() - logging.error(f"API response content: {response_json}") - except JSONDecodeError: # Catch JSON decoding errors - logging.error(f"Failed to decode JSON from response: {http_error.response.text}") + response = self.post_request(endpoint="datasets", data=data) + response.raise_for_status() # Ensure we raise for HTTP errors + return response + except HTTPError as http_error: + if http_error.response.status_code == 500: + logging.error(f"HTTP 500 error on attempt {attempt + 1} of {max_retries}: {http_error}") + if attempt < max_retries - 1: + time.sleep(wait_time) # Wait before retrying + continue + else: + logging.error("Max retries reached. Failing the request.") + else: + # For other HTTP errors, do not retry and log the error + try: + response_json = http_error.response.json() + logging.error(f"API response content: {response_json}") + except JSONDecodeError: # Catch JSON decoding errors + logging.error(f"Failed to decode JSON from response: {http_error.response.text}") + except Exception as e: + logging.error(f"Unexpected error: {e}") + # Log the HTTP status code and the error message + logging.error(f"HTTPError occurred: {http_error}") + return None except Exception as e: - logging.error(f"Unexpected error: {e}") - - # Log the HTTP status code and the error message - logging.error(f"HTTPError occurred: {http_error}") - return None + logging.error(f"An unexpected error occurred: {e}") + return None + return None def check_analysis(self, mgya: str, sequence_accession: str, metadata=None): """Check if a sequence exists in the M. Exchange