Merge pull request #32 from saaresearch/develop

Develop

.github/workflows/ml-test.yaml

@@ -21,9 +21,9 @@ jobs:
         pip install -r requirements.txt
     - name: Download dataset
       run: |
-        cd user_datasets/yolo
-        mkdir aeral
-        cd aeral
-        curl -L "https://public.roboflow.com/ds/et6MNsOwMJ?key=GMSDIzpYnW" > roboflow.zip; unzip roboflow.zip; rm roboflow.zip
+        cd user_datasets
+        git clone https://github.com/AIRI-Institute/WaRP
+        mkdir WaRP/Warp-D/valid
+        cp -r WaRP/Warp-D/test/* WaRP/Warp-D/valid/        
     - name: Ml predictions
       run: python ODRS/ml_utils/ml_model_optimizer.py

.github/workflows/model-test.yaml

@@ -21,10 +21,10 @@ jobs:
         pip install -r requirements.txt
     - name: Download dataset
       run: |
-        cd user_datasets/yolo
-        mkdir aeral
-        cd aeral
-        curl -L "https://public.roboflow.com/ds/et6MNsOwMJ?key=GMSDIzpYnW" > roboflow.zip; unzip roboflow.zip; rm roboflow.zip
+        cd user_datasets
+        git clone https://github.com/AIRI-Institute/WaRP
+        mkdir WaRP/Warp-D/valid
+        cp -r WaRP/Warp-D/test/* WaRP/Warp-D/valid/
     - name: AI training
       run: |
         python ODRS/train_utils/custom_train_all.py

.github/workflows/pep8-test.yaml

@@ -21,4 +21,4 @@ jobs:
         pip install flake8
     - name: Run PEP8 test
       run: |
-        flake8 --max-line-length=130 --exclude=ODRS/train_utils/train_model/models/*,ODRS/ml_utils/*,ODRS/train_utils/custom_train_all.py,ODRS/train_utils/train_model/scripts/yolov7_train.py,tests/*,docs/* .
+        flake8 --max-line-length=130 --exclude=ODRS/train_utils/train_model/models/*,ODRS/ml_utils/*,ODRS/train_utils/custom_train_all.py,ODRS/train_utils/train_model/scripts/yolov7_train.py,tests/*,docs/*,ODRS/utils/dataset_info.py .

.github/workflows/unit-test.yaml

@@ -21,10 +21,10 @@ jobs:
         pip install -r requirements.txt
     - name: Download dataset
       run: |
-        cd user_datasets/yolo
-        mkdir aeral
-        cd aeral
-        curl -L "https://public.roboflow.com/ds/et6MNsOwMJ?key=GMSDIzpYnW" > roboflow.zip; unzip roboflow.zip; rm roboflow.zip
+        cd user_datasets
+        git clone https://github.com/AIRI-Institute/WaRP
+        mkdir WaRP/Warp-D/valid
+        cp -r WaRP/Warp-D/test/* WaRP/Warp-D/valid/
     - name: Test
       run: |
         python tests/unit_test.py 

ODRS/data_utils/create_config.py

@@ -20,6 +20,7 @@ def delete_cache(data_path):
             if file.endswith(tuple(extensions_to_delete)):
                 os.remove(os.path.join(root, file))
 
+
 def createRunDirectory(model):
     current_file_path = Path(__file__).resolve()
 
@@ -36,7 +37,6 @@ def create_config_data(train_path, val_path, classname_file, config_path, arch,
     current_file_path = Path(__file__).resolve()
 
     runs_path = createRunDirectory(model)
-
     class_file_path = Path(current_file_path.parents[2]) / classname_file
 
     config_path = runs_path / config_path

ODRS/data_utils/prepare_ssd.py

@@ -1,7 +1,5 @@
 import os
 import json
-import glob
-from PIL import Image
 import xml.etree.ElementTree as ET
 from tqdm import tqdm
 from pathlib import Path
@@ -92,4 +90,4 @@ def create_ssd_json(path_folder, txt_path):
                 except Exception as e:
                     print(e)
 
-        save_as_json(Path(os.path.dirname(path_folder)) / f'{path_folder.name}.json', dataset)
+        save_as_json(Path(os.path.dirname(path_folder)) / f'{path_folder.name}.json', dataset)

ODRS/data_utils/resize_image.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 from PIL import Image
 
+
 def resize_images_and_annotations(data_path, img_size):
     if isinstance(img_size, int):
         width = height = img_size
@@ -55,6 +56,3 @@ def resize_images_and_annotations(data_path, img_size):
                                     file.write(f"{parts[0]} {x_center} {y_center} {box_width} {box_height}\n")
 
                     img.save(image_path)
-
-
-# resize_images_and_annotations('/media/space/ssd_1_tb_evo_sumsung/Work/Warp-D', (640, 480))

ODRS/data_utils/split_dataset.py

@@ -6,6 +6,7 @@
 from loguru import logger
 from pathlib import Path
 
+
 def sorted_files(image_files, label_files):
     new_label_files = list()
     new_image_files = list()
@@ -27,20 +28,19 @@ def split_data(datapath, split_train_value, split_valid_value):
     test_path = os.path.join(datapath, 'test')
     val_path = os.path.join(datapath, 'valid')
 
-    if os.path.exists(train_path) and (os.path.exists(val_path)
-                                        or os.path.exists(os.path.join(datapath, 'val'))):
+    if os.path.exists(train_path) and (os.path.exists(val_path) or
+                                       os.path.exists(os.path.join(datapath, 'val'))):
         logger.info("Dataset is ready")
         return train_path, val_path if os.path.exists(val_path) else os.path.join(datapath, 'val')
-    if os.path.exists(train_path) and not (os.path.exists(val_path)
-                                        or os.path.exists(os.path.join(datapath, 'val'))):
+    if os.path.exists(train_path) and not (os.path.exists(val_path) or
+                                           os.path.exists(os.path.join(datapath, 'val'))):
         logger.error("Dataset has no validation sample")
         sys.exit()
-    if not os.path.exists(train_path) and (os.path.exists(val_path)
-                                        or os.path.exists(os.path.join(datapath, 'val'))):
+    if not os.path.exists(train_path) and (os.path.exists(val_path) or
+                                           os.path.exists(os.path.join(datapath, 'val'))):
         logger.error("Dataset has no training sample")
         sys.exit()
 
-
     images_path = os.path.join(datapath, 'images')
     labels_path = os.path.join(datapath, 'labels')
 

ODRS/ml_utils/config/ml_config.yaml

@@ -1,5 +1,5 @@
 GPU: true
 accuracy: 10
 classes_path: classes.txt
-dataset_path: /media/space/ssd_1_tb_evo_sumsung/exp-train/datasets/Monitoring_photo
+dataset_path: /home/runner/work/ODRS/ODRS/user_datasets/WaRP/Warp-D
 speed: 1