Fix split

aimclub · Jan 9, 2024 · 601144f · 601144f
1 parent a137857
commit 601144f
Show file tree

Hide file tree

Showing 27 changed files with 225 additions and 10 deletions.
diff --git a/ODRS/data_utils/__pycache__/__init__.cpython-38.pyc b/ODRS/data_utils/__pycache__/__init__.cpython-38.pyc
diff --git a/ODRS/data_utils/__pycache__/create_config.cpython-38.pyc b/ODRS/data_utils/__pycache__/create_config.cpython-38.pyc
diff --git a/ODRS/data_utils/__pycache__/prepare_ssd.cpython-38.pyc b/ODRS/data_utils/__pycache__/prepare_ssd.cpython-38.pyc
diff --git a/ODRS/data_utils/__pycache__/split_dataset.cpython-38.pyc b/ODRS/data_utils/__pycache__/split_dataset.cpython-38.pyc
diff --git a/ODRS/data_utils/split_dataset.py b/ODRS/data_utils/split_dataset.py
@@ -4,6 +4,21 @@
 import sys
 from tqdm import tqdm
 from loguru import logger
+from pathlib import Path
+
+def sorted_files(image_files, label_files):
+    new_label_files = list()
+    new_image_files = list()
+    for image_path in tqdm(image_files, desc="Sorting"):
+        image_stem = Path(image_path).stem
+        for label_path in label_files:
+            label_stem = Path(label_path).stem
+            if label_stem == image_stem:
+                new_image_files.append(image_path)
+                new_label_files.append(label_path)
+                break
+    return new_image_files, new_label_files
+
 
 def split_data(datapath, split_train_value, split_valid_value):
     selected_folders = ['test', 'train', 'valid']
@@ -40,8 +55,7 @@ def split_data(datapath, split_train_value, split_valid_value):
                       glob.glob(os.path.join(datapath, '*.png'))
         label_files = glob.glob(os.path.join(datapath, '*.txt'))
 
-    image_files.sort()
-    label_files.sort()
+    image_files, label_files = sorted_files(image_files, label_files)
 
     total_files = len(image_files) + len(label_files)
 
@@ -55,17 +69,17 @@ def split_data(datapath, split_train_value, split_valid_value):
     logger.info(f'Total number of labels:{len(label_files)}')
 
     train_images = image_files[:train_split]
-    train_labels = label_files[:train_split]
+    train_labels = label_files[:len(train_images)]
     logger.info(f'Number train images:{len(train_images)}')
     logger.info(f'Number train labels:{len(train_labels)}')
 
-    val_images = image_files[train_split:train_split+val_split]
-    val_labels = label_files[train_split:train_split+val_split]
+    val_images = image_files[len(train_images):len(train_images)+val_split]
+    val_labels = label_files[len(train_images):len(train_images)+val_split]
     logger.info(f'Number valid images:{len(val_images)}')
     logger.info(f'Number valid labels:{len(val_labels)}')
 
-    test_images = image_files[train_split+val_split:]
-    test_labels = label_files[train_split+val_split:]
+    test_images = image_files[len(train_images)+len(val_images):]
+    test_labels = label_files[len(train_images)+len(val_images):]
     logger.info(f'Number test images:{len(test_images)}')
     logger.info(f'Number test labels:{len(test_labels)}')
 

diff --git a/ODRS/ml_utils/config/ml_config.yaml b/ODRS/ml_utils/config/ml_config.yaml
@@ -1,5 +1,5 @@
 GPU: true
 accuracy: 10
 classes_path: classes.txt
-dataset_path: vending_machine_24_11
+dataset_path: /media/space/ssd_1_tb_evo_sumsung/exp-train/datasets/Monitoring_photo
 speed: 1
diff --git a/ODRS/ml_utils/ml_model_optimizer.py b/ODRS/ml_utils/ml_model_optimizer.py
@@ -35,7 +35,7 @@ def predict(mode, classes_path, dataset_path, speed, accuracy):
 
     dataset_path_new = getDataPath(ROOT, dataset_path)
 
-    split_data(dataset_path_new, split_train_value=0.6, split_valid_value=0.35)
+    split_data(dataset_path_new, split_train_value=0.75, split_valid_value=0.15)
 
     dataset_data = dataset_info(dataset_path_new, Path(file.parents[2]) / classes_path, run_path)
 

diff --git a/ODRS/utils/__pycache__/dataset_info.cpython-38.pyc b/ODRS/utils/__pycache__/dataset_info.cpython-38.pyc
diff --git a/ODRS/utils/__pycache__/ml_plot.cpython-38.pyc b/ODRS/utils/__pycache__/ml_plot.cpython-38.pyc
diff --git a/ODRS/utils/__pycache__/ml_utils.cpython-38.pyc b/ODRS/utils/__pycache__/ml_utils.cpython-38.pyc
diff --git a/ODRS/utils/__pycache__/utils.cpython-38.pyc b/ODRS/utils/__pycache__/utils.cpython-38.pyc
diff --git a/ODRS/utils/utils.py b/ODRS/utils/utils.py
@@ -76,6 +76,7 @@ def getDataPath(ROOT, folder_name):
             target_path = DATA_PATH / FOLDER_PATH.name
             logger.info(f"Copying a set of images to {DATA_PATH}")
             shutil.copytree(FOLDER_PATH, target_path, dirs_exist_ok=True)
+            FOLDER_PATH = target_path
 
     except Exception as e:
         logger.error(f"An error has occurred: {e}")

diff --git a/classes.txt b/classes.txt
@@ -25,4 +25,4 @@ bottle-blue5l
 bottle-blue5l-full
 glass-transp
 glass-dark
-glass-green
+glass-green
diff --git a/runs/2024-01-09_14-36-41_ml/Classes_balance.png b/runs/2024-01-09_14-36-41_ml/Classes_balance.png
diff --git a/runs/2024-01-09_14-36-41_ml/class_counts.csv b/runs/2024-01-09_14-36-41_ml/class_counts.csv
@@ -0,0 +1,29 @@
+Class_name,Value
+juice-cardboard,246
+detergent-transparent,88
+glass-transp,30
+bottle-blue,170
+bottle-transp,385
+cans,406
+bottle-milk,114
+detergent-white,99
+milk-cardboard,383
+bottle-oil,48
+detergent-color,77
+bottle-dark,76
+bottle-yogurt,182
+bottle-green,91
+bottle-blue-full,4
+bottle-multicolor,17
+detergent-box,6
+bottle-green-full,3
+glass-green,6
+bottle-blue5l,9
+bottle-transp-full,8
+canister,4
+bottle-multicolorv-full,2
+bottle-milk-full,1
+glass-dark,1
+bottle-dark-full,0
+bottle-oil-full,0
+bottle-blue5l-full,0
diff --git a/runs/2024-01-09_14-36-41_ml/results.yaml b/runs/2024-01-09_14-36-41_ml/results.yaml
@@ -0,0 +1,13 @@
+GPU: true
+Gini_Coefficient: 89.0
+Number_of_classes: 25.0
+Number_of_images: 1715
+Top_1: yolov7
+Top_2: yolov8x6
+Top_3: yolov7x
+accuracy: 10
+classes_path: classes.txt
+dataset_path: /media/space/ssd_1_tb_evo_sumsung/exp-train/ODRS/user_datasets/Monitoring_photo
+image_Height: 1080.0
+image_Width: 1920.0
+speed: 1
diff --git a/runs/2024-01-09_15-07-33_ml/Classes_balance.png b/runs/2024-01-09_15-07-33_ml/Classes_balance.png
diff --git a/runs/2024-01-09_15-07-33_ml/class_counts.csv b/runs/2024-01-09_15-07-33_ml/class_counts.csv
@@ -0,0 +1,29 @@
+Class_name,Value
+bottle-blue,0
+bottle-green,0
+bottle-dark,0
+bottle-milk,0
+bottle-transp,0
+bottle-multicolor,0
+bottle-yogurt,0
+bottle-oil,0
+cans,0
+juice-cardboard,0
+milk-cardboard,0
+detergent-color,0
+detergent-transparent,0
+detergent-box,0
+canister,0
+bottle-blue-full,0
+bottle-transp-full,0
+bottle-dark-full,0
+bottle-green-full,0
+bottle-multicolorv-full,0
+bottle-milk-full,0
+bottle-oil-full,0
+detergent-white,0
+bottle-blue5l,0
+bottle-blue5l-full,0
+glass-transp,0
+glass-dark,0
+glass-green,0
diff --git a/runs/2024-01-09_15-12-31_ml/Classes_balance.png b/runs/2024-01-09_15-12-31_ml/Classes_balance.png
diff --git a/runs/2024-01-09_15-12-31_ml/class_counts.csv b/runs/2024-01-09_15-12-31_ml/class_counts.csv
@@ -0,0 +1,29 @@
+Class_name,Value
+bottle-blue,0
+bottle-green,0
+bottle-dark,0
+bottle-milk,0
+bottle-transp,0
+bottle-multicolor,0
+bottle-yogurt,0
+bottle-oil,0
+cans,0
+juice-cardboard,0
+milk-cardboard,0
+detergent-color,0
+detergent-transparent,0
+detergent-box,0
+canister,0
+bottle-blue-full,0
+bottle-transp-full,0
+bottle-dark-full,0
+bottle-green-full,0
+bottle-multicolorv-full,0
+bottle-milk-full,0
+bottle-oil-full,0
+detergent-white,0
+bottle-blue5l,0
+bottle-blue5l-full,0
+glass-transp,0
+glass-dark,0
+glass-green,0
diff --git a/runs/2024-01-09_15-14-10_ml/Classes_balance.png b/runs/2024-01-09_15-14-10_ml/Classes_balance.png
diff --git a/runs/2024-01-09_15-14-10_ml/class_counts.csv b/runs/2024-01-09_15-14-10_ml/class_counts.csv
@@ -0,0 +1,29 @@
+Class_name,Value
+bottle-blue,0
+bottle-green,0
+bottle-dark,0
+bottle-milk,0
+bottle-transp,0
+bottle-multicolor,0
+bottle-yogurt,0
+bottle-oil,0
+cans,0
+juice-cardboard,0
+milk-cardboard,0
+detergent-color,0
+detergent-transparent,0
+detergent-box,0
+canister,0
+bottle-blue-full,0
+bottle-transp-full,0
+bottle-dark-full,0
+bottle-green-full,0
+bottle-multicolorv-full,0
+bottle-milk-full,0
+bottle-oil-full,0
+detergent-white,0
+bottle-blue5l,0
+bottle-blue5l-full,0
+glass-transp,0
+glass-dark,0
+glass-green,0
diff --git a/runs/2024-01-09_15-24-45_ml/Classes_balance.png b/runs/2024-01-09_15-24-45_ml/Classes_balance.png
diff --git a/runs/2024-01-09_15-24-45_ml/class_counts.csv b/runs/2024-01-09_15-24-45_ml/class_counts.csv
@@ -0,0 +1,29 @@
+Class_name,Value
+bottle-blue,0
+bottle-green,0
+bottle-dark,0
+bottle-milk,0
+bottle-transp,0
+bottle-multicolor,0
+bottle-yogurt,0
+bottle-oil,0
+cans,0
+juice-cardboard,0
+milk-cardboard,0
+detergent-color,0
+detergent-transparent,0
+detergent-box,0
+canister,0
+bottle-blue-full,0
+bottle-transp-full,0
+bottle-dark-full,0
+bottle-green-full,0
+bottle-multicolorv-full,0
+bottle-milk-full,0
+bottle-oil-full,0
+detergent-white,0
+bottle-blue5l,0
+bottle-blue5l-full,0
+glass-transp,0
+glass-dark,0
+glass-green,0
diff --git a/runs/2024-01-09_15-25-25_ml/Classes_balance.png b/runs/2024-01-09_15-25-25_ml/Classes_balance.png
diff --git a/runs/2024-01-09_15-25-25_ml/class_counts.csv b/runs/2024-01-09_15-25-25_ml/class_counts.csv
@@ -0,0 +1,29 @@
+Class_name,Value
+juice-cardboard,246
+detergent-transparent,88
+glass-transp,30
+bottle-blue,170
+bottle-transp,385
+cans,406
+bottle-milk,114
+detergent-white,99
+milk-cardboard,383
+bottle-oil,48
+detergent-color,77
+bottle-dark,76
+bottle-yogurt,182
+bottle-green,91
+bottle-blue-full,4
+bottle-multicolor,17
+detergent-box,6
+bottle-green-full,3
+glass-green,6
+bottle-blue5l,9
+bottle-transp-full,8
+canister,4
+bottle-multicolorv-full,2
+bottle-milk-full,1
+glass-dark,1
+bottle-dark-full,0
+bottle-oil-full,0
+bottle-blue5l-full,0
diff --git a/runs/2024-01-09_15-25-25_ml/results.yaml b/runs/2024-01-09_15-25-25_ml/results.yaml
@@ -0,0 +1,13 @@
+GPU: true
+Gini_Coefficient: 89.0
+Number_of_classes: 25.0
+Number_of_images: 1715
+Top_1: yolov7
+Top_2: yolov8x6
+Top_3: yolov7x
+accuracy: 10
+classes_path: classes.txt
+dataset_path: Monitoring_photo
+image_Height: 1080.0
+image_Width: 1920.0
+speed: 1