Skip to content

Commit

Permalink
add debug logs
Browse files Browse the repository at this point in the history
  • Loading branch information
almazgimaev committed Apr 1, 2024
1 parent 8fd2532 commit ba37334
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 0 deletions.
1 change: 1 addition & 0 deletions train/src/callbacks.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
def on_train_batch_end(trainer):
print("train batch end...")
with open("train_batches.txt", "w") as file:
file.write("train batch end")
2 changes: 2 additions & 0 deletions train/src/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -1303,10 +1303,12 @@ def train_batch_watcher_disable():
threading.Thread(target=train_batch_watcher_func, daemon=True).start()

def stop_on_batch_end_if_needed(trainer_validator, *args, **kwargs):
print("Checking if app is stopped")
try:
app_is_stopped = app.is_stopped()
except:
app_is_stopped = True
print(f"App is stopped: {app_is_stopped}")
if app_is_stopped:
trainer_validator.stop = True
raise app.StopException("This error is expected.")
Expand Down

0 comments on commit ba37334

Please sign in to comment.