V2 stop (#18)

* disable whatcher on shutdown

* upd sdk

* added logger to whatcher

* start train in thread

* added stop callback on train

* added stop event on validation

* beautify

* rename

* added checkpoints dir

train/.gitignore

@@ -1,3 +1,4 @@
 src/__pycache__/
 local.env
-tempfiles/
+tempfiles/
+runs/

train/requirements.txt

@@ -1,4 +1,5 @@
-supervisely==6.72.110
+supervisely==6.72.182
+# git+https://github.com/supervisely/supervisely.git@v2-stop
 ultralytics==8.0.112
 --extra-index-url https://download.pytorch.org/whl/cu113
 torch==1.10.1+cu113

train/src/globals.py

@@ -13,8 +13,11 @@
 
 if sly.is_production():
     app_session_id = sly.io.env.task_id()
+    root_model_checkpoint_dir = sly.app.get_synced_data_dir()
 else:
     app_session_id = 777  # for debug
+    root_model_checkpoint_dir = os.path.join(app_root_directory, "runs")
+
 
 det_models_data_path = os.path.join(root_source_path, "models", "det_models_data.json")
 seg_models_data_path = os.path.join(root_source_path, "models", "seg_models_data.json")
@@ -29,4 +32,4 @@
     train_params_filepath = "training_params.yml"  # for debug
 train_counter, val_counter = 0, 0
 center_matches = {}
-keypoints_template = None
+keypoints_template = None