check if app is alive

supervisely-ecosystem · Apr 1, 2024 · 622951e · 622951e
1 parent ba37334
commit 622951e
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/train/src/callbacks.py b/train/src/callbacks.py
@@ -1,4 +1,3 @@
 def on_train_batch_end(trainer):
-    print("train batch end...")
     with open("train_batches.txt", "w") as file:
         file.write("train batch end")
diff --git a/train/src/main.py b/train/src/main.py
@@ -1303,11 +1303,16 @@ def train_batch_watcher_disable():
         threading.Thread(target=train_batch_watcher_func, daemon=True).start()
 
     def stop_on_batch_end_if_needed(trainer_validator, *args, **kwargs):
-        print("Checking if app is stopped")
-        try:
-            app_is_stopped = app.is_stopped()
-        except:
-            app_is_stopped = True
+        app_is_stopped = app.is_stopped()
+        print(f"App is stopped: {app_is_stopped}")
+        if not app_is_stopped:
+            try:
+                app_is_stopped = not app.is_running()
+            except:
+                app_is_stopped = True
+        print(f"App is stopped: {app_is_stopped}")
+        if not app_is_stopped:
+            app_is_stopped = not api.app.is_ready_for_api_calls(g.app_session_id)
         print(f"App is stopped: {app_is_stopped}")
         if app_is_stopped:
             trainer_validator.stop = True