download jobs add retry (#353)

deepmodeling · Jul 17, 2023 · bf66cc3 · bf66cc3
2 parents 8ee05a7 + 5d4357d
commit bf66cc3
Showing 1 changed file with 23 additions and 1 deletion.
diff --git a/dpdispatcher/submission.py b/dpdispatcher/submission.py
@@ -258,12 +258,34 @@ def run_submission(
             finally:
                 pass
         self.handle_unexpected_submission_state()
-        self.download_jobs()
+        self.try_download_result()
         self.submission_to_json()
         if clean:
             self.clean_jobs()
         return self.serialize()
 
+    def try_download_result(self):
+        start_time = time.time()
+        retry_interval = 60  # 每1分钟重试一次
+        success = False
+        while not success:
+            try:
+                self.download_jobs()
+                success = True
+            except (EOFError, Exception) as e:
+                dlog.exception(e)
+                elapsed_time = time.time() - start_time
+                if elapsed_time < 3600:  # 1小时内
+                    dlog.info("Retrying in 1 minute...")
+                    time.sleep(retry_interval)
+                elif elapsed_time < 86400:  # 1小时后，但在24小时内
+                    retry_interval = 600  # 每10分钟重试一次
+                    dlog.info("Retrying in 10 minutes...")
+                    time.sleep(retry_interval)
+                else:  # 超过24小时
+                    dlog.info("Maximum retries time reached. Exiting.")
+                    break
+
     async def async_run_submission(self, **kwargs):
         """Async interface of run_submission.