data_preparation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "data_preparation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# 1. Cek Raw Dataset"
      ],
      "metadata": {
        "id": "UrUmJXgWXtAV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "metadata": {
        "id": "wMSnAiLa5qM2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa223690-abb0-4b56-e9a8-fecdd48b7f02"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "# Global Variables\n",
        "RAW_DATASET_PATH = \"/content/drive/MyDrive/Product_Based_Capstone/Team_ML/New_Raw_Dataset\"\n",
        "CLEAN_DATASET_PATH = \"/content/drive/MyDrive/Product_Based_Capstone/Team_ML/Clean_Dataset\"\n",
        "TRAIN_PATH = os.path.join(CLEAN_DATASET_PATH, 'train')\n",
        "DEV_PATH = os.path.join(CLEAN_DATASET_PATH, 'dev')\n",
        "# TEST_PATH = os.path.join(DATASET_SPLIT_PATH, 'test')\n",
        "PLANTS_LIST = os.listdir(RAW_DATASET_PATH)\n",
        "NUM_OF_PLANTS = len(PLANTS_LIST)\n",
        "\n",
        "print('Total jenis tanaman herbal:', NUM_OF_PLANTS)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T6t6-j7JX9Ct",
        "outputId": "eba17320-ccf1-48d3-df8a-f086a49e1249"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total jenis tanaman herbal: 18\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# List gambar per tanaman\n",
        "def CountPlantImages(data_path):\n",
        "  index = 1\n",
        "  sum = 0\n",
        "  print('-= Banyak gambar per tanaman =-')\n",
        "  for plant in PLANTS_LIST:\n",
        "    plant_path = os.path.join(data_path, plant)\n",
        "    print('{:2}. {:16} : {}'.format(index, plant, len(os.listdir(plant_path))))\n",
        "    index += 1\n",
        "    sum += len(os.listdir(plant_path))\n",
        "  print('\\nTotal gambar:', sum)\n",
        "\n",
        "CountPlantImages(RAW_DATASET_PATH)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D_nS63pAYxK0",
        "outputId": "2a5e5b62-f6b0-47fa-9500-7f3d6222433b"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-= Banyak gambar per tanaman =-\n",
            " 1. Andong           : 70\n",
            " 2. Meniran          : 83\n",
            " 3. Cincau Hijau     : 93\n",
            " 4. Bayam Duri       : 66\n",
            " 5. Binahong         : 103\n",
            " 6. Kumis Kucing     : 127\n",
            " 7. Kelor            : 66\n",
            " 8. Kemangi          : 88\n",
            " 9. Jeruk Nipis      : 68\n",
            "10. Pandan           : 69\n",
            "11. Mint             : 84\n",
            "12. Singkong         : 64\n",
            "13. Sambiloto        : 80\n",
            "14. Serai            : 69\n",
            "15. Sembung          : 94\n",
            "16. Talas            : 89\n",
            "17. Pepaya           : 109\n",
            "18. Sirih            : 129\n",
            "\n",
            "Total gambar: 1551\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2. Preparasi Clean Dataset (Run kalo mau update ke Raw Dataset terbaru)"
      ],
      "metadata": {
        "id": "ZHCTxJA6ZHNS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Kosongin directory Train-Dev (Train-Dev-Test nanti aja kalo dataset udah agak gede)\n",
        "from shutil import rmtree\n",
        "\n",
        "def EmptyCleanDatasetDirectory(clean_data_path, train_path, dev_path): # minus test_path parameter\n",
        "  if os.path.exists(clean_data_path): # Remove old directory\n",
        "    rmtree(clean_data_path)\n",
        "\n",
        "  # Make new empty directory\n",
        "  for plant in PLANTS_LIST:\n",
        "    train_plant_path = os.path.join(train_path, plant)\n",
        "    os.makedirs(train_plant_path)\n",
        "    \n",
        "    dev_plant_path = os.path.join(dev_path, plant)\n",
        "    os.makedirs(dev_plant_path)\n",
        "\n",
        "    # test_plant_path = os.path.join(test_path, plant)\n",
        "    # os.makedirs(test_plant_path)\n",
        "  \n",
        "  print('Finished emptying old data.')"
      ],
      "metadata": {
        "id": "QYwp7tMuZahn"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Resize data and save on temporary folder\n",
        "from PIL import Image\n",
        "\n",
        "def ResizeAndRenameData(source, dest, plant_name, image_size):\n",
        "  os.makedirs(dest, exist_ok=True)\n",
        "  count = 0\n",
        "  zeros_padding = 4\n",
        "\n",
        "  for plant_image in os.listdir(source):\n",
        "    im = Image.open(os.path.join(source, plant_image)).convert('RGB')\n",
        "    jpeg_name = plant_name + str(count).zfill(zeros_padding) + \".jpeg\"\n",
        "    image_dest_path = os.path.join(dest, jpeg_name)\n",
        "    im.resize(image_size).save(image_dest_path)\n",
        "    count += 1"
      ],
      "metadata": {
        "id": "HpZ-ZakAaQZ9"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Data Splitting Function (masih Train-Dev Split jadi masih pake SPLIT_SIZE)\n",
        "import random\n",
        "from shutil import move\n",
        "\n",
        "def SplitData(SOURCE, TRAINING, DEVELOPMENT, SPLIT_SIZE, PLANT_NAME): # nanti pake dev_size / test_size\n",
        "\n",
        "  dir_list = os.listdir(SOURCE)\n",
        "  randomized_dir_list = random.sample(dir_list, len(dir_list))\n",
        "\n",
        "  # Remove 0 size images\n",
        "  final_list = []\n",
        "  for filename in randomized_dir_list:\n",
        "    fullpath = os.path.join(SOURCE, filename)\n",
        "    if os.path.getsize(fullpath) != 0:\n",
        "      final_list.append(filename)\n",
        "    else:\n",
        "      print(\"{} is zero length, so ignoring.\".format(filename))\n",
        "\n",
        "  # Start Splitting (train-dev split)\n",
        "  index_split = round(SPLIT_SIZE * len(final_list))\n",
        "  for filename in final_list[:index_split]:\n",
        "    source = os.path.join(SOURCE, filename)\n",
        "    dest = os.path.join(TRAINING, filename)\n",
        "    move(source, dest)\n",
        "\n",
        "  for filename in final_list[index_split:]:\n",
        "    source = os.path.join(SOURCE, filename)\n",
        "    dest = os.path.join(DEVELOPMENT, filename)\n",
        "    move(source, dest)\n",
        "  \n",
        "  print('Finished splitting data of:', PLANT_NAME)"
      ],
      "metadata": {
        "id": "1ZZPy4ssZgIX"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Copy file ke Train-Dev Directory (Train-Dev-Test nanti aja kalo dataset udah gede)\n",
        "\n",
        "def MakeCleanDataset(data_path, train_path, dev_path, image_size, split_size): # minus test_path parameter\n",
        "  temp_folder_path = '/tmp/convert_images'\n",
        "\n",
        "  for plant in PLANTS_LIST:\n",
        "    source_path = os.path.join(data_path, plant)\n",
        "    train_plant_path = os.path.join(train_path, plant)\n",
        "    dev_plant_path = os.path.join(dev_path, plant)\n",
        "    # test_plant_path = os.path.join(test_path, plant)\n",
        "    \n",
        "    ResizeAndRenameData(source_path, temp_folder_path, plant, image_size)\n",
        "    SplitData(temp_folder_path, train_plant_path, dev_plant_path, split_size, plant)"
      ],
      "metadata": {
        "id": "v9yFDIPCZh6k"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "IMAGE_SIZE = (384, 384)\n",
        "SPLIT_SIZE = 0.7\n",
        "\n",
        "EmptyCleanDatasetDirectory(CLEAN_DATASET_PATH, TRAIN_PATH, DEV_PATH)\n",
        "MakeCleanDataset(RAW_DATASET_PATH, TRAIN_PATH, DEV_PATH, IMAGE_SIZE, SPLIT_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1HzxEQrH5XFb",
        "outputId": "73b4446b-5609-4fe2-93d7-70a6355dd9c9"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished emptying old data.\n",
            "Finished splitting data of: Andong\n",
            "Finished splitting data of: Meniran\n",
            "Finished splitting data of: Cincau Hijau\n",
            "Finished splitting data of: Bayam Duri\n",
            "Finished splitting data of: Binahong\n",
            "Finished splitting data of: Kumis Kucing\n",
            "Finished splitting data of: Kelor\n",
            "Finished splitting data of: Kemangi\n",
            "Finished splitting data of: Jeruk Nipis\n",
            "Finished splitting data of: Pandan\n",
            "Finished splitting data of: Mint\n",
            "Finished splitting data of: Singkong\n",
            "Finished splitting data of: Sambiloto\n",
            "Finished splitting data of: Serai\n",
            "Finished splitting data of: Sembung\n",
            "Finished splitting data of: Talas\n",
            "Finished splitting data of: Pepaya\n",
            "Finished splitting data of: Sirih\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def CekTotalGambar(folder_name, data_path):\n",
        "  sum = 0\n",
        "  for rootdir, dirs, files in os.walk(data_path):\n",
        "    for subdir in dirs:\n",
        "      path = os.path.join(rootdir, subdir)\n",
        "      sum += len(os.listdir(path))\n",
        "  print('Total Gambar {}: {}'.format(folder_name, sum))\n",
        "  return sum\n",
        "\n",
        "train_count = CekTotalGambar('Train', TRAIN_PATH)\n",
        "dev_count = CekTotalGambar('Dev', DEV_PATH)\n",
        "total_count = train_count + dev_count\n",
        "ratio_train = round(train_count/total_count, 4)\n",
        "ratio_dev = round(dev_count/total_count, 4)\n",
        "print('Total Gambar Clean Dataset: {}\\nRasio Train: {}\\nRasio Dev: {}'.format(total_count, ratio_train, ratio_dev))"
      ],
      "metadata": {
        "id": "f1zZnCCYZr9Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca074c32-4911-42fe-eabb-1a84af14db8e"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Gambar Train: 1085\n",
            "Total Gambar Dev: 466\n",
            "Total Gambar Clean Dataset: 1551\n",
            "Rasio Train: 0.6995\n",
            "Rasio Dev: 0.3005\n"
          ]
        }
      ]
    }
  ]
}