Merge pull request #361 from EBI-Metagenomics/fix/me-missing-analysis

Some tiny fixes for ME scripts
EBI-Metagenomics · Jun 10, 2024 · 069f04d · 069f04d
2 parents 0201c09 + db8a6ea
commit 069f04d
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 17 deletions.
diff --git a/emgapi/management/commands/populate_metagenomics_exchange.py b/emgapi/management/commands/populate_metagenomics_exchange.py
@@ -174,9 +174,19 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update):
                         else:
                             logging.error(f"Analysis {annotation_job} update failed")
                     else:
-                        logging.debug(
-                            f"No edit for {annotation_job}, metadata is correct"
-                        )
+                        if annotation_job.mgx_accession and annotation_job.last_mgx_indexed:
+                            logging.info(
+                                f"No edit for {annotation_job}, metadata is correct"
+                            )
+                        else:
+                            logging.info(
+                                f"Metadata is correct but {annotation_job} is missing in DB. Adding."
+                            )
+                            annotation_job.mgx_accession = registry_id
+                            annotation_job.last_mgx_indexed = (
+                                    timezone.now() + timedelta(minutes=1)
+                            )
+                            jobs_to_update.append(annotation_job)
 
             AnalysisJob.objects.bulk_update(
                 jobs_to_update,

diff --git a/emgapi/metagenomics_exchange.py b/emgapi/metagenomics_exchange.py
@@ -17,6 +17,7 @@
 import logging
 
 import requests
+import time
 from django.conf import settings
 from requests.exceptions import HTTPError, JSONDecodeError
 
@@ -106,22 +107,38 @@ def add_analysis(self, mgya: str, sequence_accession: str):
             The response object from the API request.
         """
         data = self.generate_metadata(mgya, sequence_accession)
-        try:
-            response = self.post_request(endpoint="datasets", data=data)
-            response.raise_for_status()  # Ensure we raise for HTTP errors
-            return response
-        except HTTPError as http_error:
+        max_retries = 3
+        wait_time = 2  # seconds
+
+        for attempt in range(max_retries):
             try:
-                response_json = http_error.response.json()
-                logging.error(f"API response content: {response_json}")
-            except JSONDecodeError:  # Catch JSON decoding errors
-                logging.error(f"Failed to decode JSON from response: {http_error.response.text}")
+                response = self.post_request(endpoint="datasets", data=data)
+                response.raise_for_status()  # Ensure we raise for HTTP errors
+                return response
+            except HTTPError as http_error:
+                if http_error.response.status_code == 500:
+                    logging.error(f"HTTP 500 error on attempt {attempt + 1} of {max_retries}: {http_error}")
+                    if attempt < max_retries - 1:
+                        time.sleep(wait_time)  # Wait before retrying
+                        continue
+                    else:
+                        logging.error("Max retries reached. Failing the request.")
+                else:
+                    # For other HTTP errors, do not retry and log the error
+                    try:
+                        response_json = http_error.response.json()
+                        logging.error(f"API response content: {response_json}")
+                    except JSONDecodeError:  # Catch JSON decoding errors
+                        logging.error(f"Failed to decode JSON from response: {http_error.response.text}")
+                    except Exception as e:
+                        logging.error(f"Unexpected error: {e}")
+                    # Log the HTTP status code and the error message
+                    logging.error(f"HTTPError occurred: {http_error}")
+                    return None
             except Exception as e:
-                logging.error(f"Unexpected error: {e}")
-
-            # Log the HTTP status code and the error message
-            logging.error(f"HTTPError occurred: {http_error}")
-            return None
+                logging.error(f"An unexpected error occurred: {e}")
+                return None
+        return None
 
     def check_analysis(self, mgya: str, sequence_accession: str, metadata=None):
         """Check if a sequence exists in the M. Exchange