From 32007c3aca91729a6c0f01649055d6532eeef262 Mon Sep 17 00:00:00 2001
From: wangxiangfei <wangxf01@dp.tech>
Date: Thu, 13 Jul 2023 17:37:33 +0800
Subject: [PATCH 1/3] add retry

---
 dpdispatcher/submission.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/dpdispatcher/submission.py b/dpdispatcher/submission.py
index a776cf37..6fa75a52 100644
--- a/dpdispatcher/submission.py
+++ b/dpdispatcher/submission.py
@@ -258,12 +258,34 @@ def run_submission(
             finally:
                 pass
         self.handle_unexpected_submission_state()
-        self.download_jobs()
+        self.try_download_result()  
         self.submission_to_json()
         if clean:
             self.clean_jobs()
         return self.serialize()
 
+    def try_download_result(self):
+        start_time = time.time()
+        retry_interval = 60  # 每1分钟重试一次
+        success = False
+        while not success:
+            try:
+                self.download_jobs()
+                success = True
+            except RuntimeError("Failed to run") as e:
+                dlog.exception(e)
+                elapsed_time = time.time() - start_time
+                if elapsed_time < 3600:  # 1小时内
+                    dlog.info("Retrying in 1 minute...")
+                    time.sleep(retry_interval)
+                elif elapsed_time < 86400:  # 1小时后，但在24小时内
+                    retry_interval = 600  # 每10分钟重试一次
+                    dlog.info("Retrying in 10 minutes...")
+                    time.sleep(retry_interval)
+                else:  # 超过24小时
+                    dlog.info("Maximum retries time reached. Exiting.")
+                    break
+
     async def async_run_submission(self, **kwargs):
         """Async interface of run_submission.
 

From 5bf379a4148c40909be7e7d913d60286928329c7 Mon Sep 17 00:00:00 2001
From: wangxiangfei <wangxf01@dp.tech>
Date: Mon, 17 Jul 2023 15:17:06 +0800
Subject: [PATCH 2/3] edit except err type

---
 dpdispatcher/submission.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dpdispatcher/submission.py b/dpdispatcher/submission.py
index 6fa75a52..6aa4370e 100644
--- a/dpdispatcher/submission.py
+++ b/dpdispatcher/submission.py
@@ -272,7 +272,7 @@ def try_download_result(self):
             try:
                 self.download_jobs()
                 success = True
-            except RuntimeError("Failed to run") as e:
+            except (EOFError, Exception) as e:
                 dlog.exception(e)
                 elapsed_time = time.time() - start_time
                 if elapsed_time < 3600:  # 1小时内

From 5d4357dd2bc161dfc1fedb6fad0fcf9beffa1420 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 17 Jul 2023 07:20:14 +0000
Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 dpdispatcher/submission.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dpdispatcher/submission.py b/dpdispatcher/submission.py
index 6aa4370e..104cbbe6 100644
--- a/dpdispatcher/submission.py
+++ b/dpdispatcher/submission.py
@@ -258,7 +258,7 @@ def run_submission(
             finally:
                 pass
         self.handle_unexpected_submission_state()
-        self.try_download_result()  
+        self.try_download_result()
         self.submission_to_json()
         if clean:
             self.clean_jobs()