train-cnn14decisionlevel.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python385jvsc74a57bd07c1e26b787ea84c0136f54dc33c70b3c054a8e0cf94179623b14c46360f5a1e6",
   "display_name": "Python 3.8.5 64-bit ('asg': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# Train your model"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import os\n",
    "import random\n",
    "import time\n",
    "import warnings\n",
    "\n",
    "import librosa\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as data\n",
    "from catalyst.dl import SupervisedRunner, CallbackOrder, Callback, CheckpointCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from csrc.utils import TrainingDirs as TD\n",
    "from csrc.configurations import DatasetConfig as DC\n",
    "from csrc.configurations import ModelConfig as MC\n",
    "from csrc.utils import seed_dataset, seed_all "
   ]
  },
  {
   "source": [
    "## Train configurations"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For better debugging.\n",
    "\n",
    "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Training clip length (sencods): 2\n"
     ]
    }
   ],
   "source": [
    "### The folder name of your dataset.\n",
    "DATASET = \"standard-p2-32khz\"\n",
    "# DATASET = \"test-p2\"\n",
    "\n",
    "### Whether you have split your dataset.\n",
    "### If False then the test dataset will be generated as configured in TrainParams and choose the split ratio.\n",
    "BUILD_TEST = False\n",
    "PREBUILD_TEST = False\n",
    "TEST_RATIO = 5\n",
    "\n",
    "### The ratio to split your train/validaion dataset.\n",
    "VALID_RATIO = 5\n",
    "### Whether to shuffle the dataset.\n",
    "SHUFFLE = True\n",
    "\n",
    "### Clip length that will be used for training.\n",
    "### Default to be the same as the audio clip length in the dataset.\n",
    "PERIOD = DC.dataset_clip_time\n",
    "print(f\"Training clip length (sencods): {PERIOD}\")\n",
    "\n",
    "### Batch size for training. For example: 8gb GPU for 5s clips - batch size 32.\n",
    "BS = 64\n",
    "\n",
    "### Training epochs.\n",
    "EPOCHS = 30\n",
    "\n",
    "### Weights file path used for training.\n",
    "### Default under weights folder.\n",
    "WEIGHTS_PATH = \"./weights/Cnn14_DecisionLevelAtt_mAP0.425.pth\"\n",
    "\n",
    "### Default path to store your model.\n",
    "LOG_DIR = \"./train/logs/sp2-32000hz/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random seeding.\n",
    "# Change seed will change your validation set randomly picked from the dataset.\n",
    "\n",
    "SEED = 42\n",
    "seed_all(SEED)\n",
    "seed_dataset(SEED)"
   ]
  },
  {
   "source": [
    "## Process"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Working with dataset under D:\\Dev\\asfg\\data\\standard-p2-32khz.\nFOLDER_FOR_TRAINING: D:\\Dev\\asfg\\data\\standard-p2-32khz\nFOLDER_FOR_TEST: D:\\Dev\\asfg\\data\\standard-p2-32khz\n"
     ]
    }
   ],
   "source": [
    "# Set up working folder for training.\n",
    "\n",
    "dirs = TD(DATASET, PREBUILD_TEST)\n",
    "DATASET_FOLDER = dirs.dataset_folder\n",
    "TRAIN_FOLDER = dirs.train_folder\n",
    "TEST_FOLDER = dirs.test_folder\n",
    "\n",
    "### Currently we are training so we set up the training folder as the working folder.\n",
    "TRAIN_WORKING_FOLDER = TRAIN_FOLDER\n",
    "TEST_WORKING_FOLDER = TEST_FOLDER if TEST_FOLDER else TRAIN_FOLDER\n",
    "\n",
    "print(f\"FOLDER_FOR_TRAINING: {TRAIN_WORKING_FOLDER}\")\n",
    "print(f\"FOLDER_FOR_TEST: {TEST_WORKING_FOLDER}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Files for training: 45736\nFiles for testing: 11434\n"
     ]
    }
   ],
   "source": [
    "# Train/Test split. If the test folder has not been manually selected, then split the test folder.\n",
    "\n",
    "def sort_index(x):\n",
    "    return int(x.split(\"-\")[0])\n",
    "\n",
    "if not TEST_FOLDER:\n",
    "    all_files = os.listdir(TRAIN_FOLDER)\n",
    "    all_files.sort(key=sort_index)\n",
    "    test_index = len(all_files) // TEST_RATIO\n",
    "    test_files = all_files[-test_index:]\n",
    "    train_files = all_files[:-test_index]\n",
    "else:\n",
    "    train_files = os.listdir(TRAIN_FOLDER)\n",
    "    test_files = os.listdir(TEST_FOLDER)\n",
    "\n",
    "print(f\"Files for training: {len(train_files)}\")\n",
    "print(f\"Files for testing: {len(test_files)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Files for training: 45736\nFiles for validation: 11434\nValidation file samples: ['3081-the-kingdom-of-heaven-eng-0.wav', '3081-the-kings-speech-eng-0.wav', '3082-american-beauty-eng-0.wav', '3082-dallas-buyers-club-eng-1.wav', '3082-src-fkdsc-0.wav']\n"
     ]
    }
   ],
   "source": [
    "# Train/Validation split\n",
    "\n",
    "if SHUFFLE:\n",
    "    random.shuffle(train_files)\n",
    "\n",
    "if not BUILD_TEST:\n",
    "    train_files.extend(test_files)\n",
    "\n",
    "valid_idx = len(train_files) // VALID_RATIO\n",
    "valid_files = train_files[-valid_idx:]\n",
    "train_files = train_files[:-valid_idx]\n",
    "\n",
    "print(f\"Files for training: {len(train_files)}\")\n",
    "print(f\"Files for validation: {len(valid_files)}\")\n",
    "print(f\"Validation file samples: {valid_files[:5]}\")"
   ]
  },
  {
   "source": [
    "## Dataset"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from csrc.dataset import PANNsDataset"
   ]
  },
  {
   "source": [
    "## Transformer"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from csrc.transformers import BaseAug"
   ]
  },
  {
   "source": [
    "## Set up dataloader "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaders = {\n",
    "    \"train\": data.DataLoader(PANNsDataset(train_files, training_folder=TRAIN_WORKING_FOLDER, test_folder=TEST_WORKING_FOLDER, waveform_transforms=BaseAug), # Build training set\n",
    "                            batch_size=BS,\n",
    "                            shuffle=True,\n",
    "                            num_workers=0, # 0 for windows system.\n",
    "                            pin_memory=True,\n",
    "                            drop_last=True),\n",
    "    \"valid\": data.DataLoader(PANNsDataset(valid_files, training_folder=TRAIN_WORKING_FOLDER, test_folder=TEST_WORKING_FOLDER, waveform_transforms=None), # Build training set.\\n\",\n",
    "                             batch_size=BS,\n",
    "                             shuffle=False,\n",
    "                             num_workers=0,\n",
    "                             pin_memory=True,\n",
    "                             drop_last=False)\n",
    "}"
   ]
  },
  {
   "source": [
    "## Model"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from csrc.models import AttBlock, PANNsCNN14Att"
   ]
  },
  {
   "source": [
    "## Loss"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from csrc.losses import ImprovedPANNsLoss"
   ]
  },
  {
   "source": [
    "## Callbacks"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from csrc.callbacks import F1Callback, mAPCallback, PrecisionCallback\n",
    "from catalyst import dl"
   ]
  },
  {
   "source": [
    "## Training Configurations"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # device\n",
    "# device = torch.device(\"cuda:0\")\n",
    "\n",
    "# # model\n",
    "# model = PANNsCNN14Att(**MC.sed_model_config)\n",
    "# weights = torch.load(WEIGHTS_PATH)\n",
    "# model.load_state_dict(weights[\"model\"])\n",
    "# model.att_block = AttBlock(2048, 2, activation=\"sigmoid\")\n",
    "# model.att_block.init_weights()\n",
    "# model.to(device)\n",
    "\n",
    "# # optimizer\n",
    "# optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# # scheduler\n",
    "# scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n",
    "\n",
    "# # loss\n",
    "# loss = ImprovedPANNsLoss().to(device)\n",
    "\n",
    "# # callbacks\n",
    "# callbacks = [\n",
    "#     F1Callback(),\n",
    "#     mAPCallback(),\n",
    "#     PrecisionCallback(),\n",
    "#     CheckpointCallback(save_n_best=3),\n",
    "# ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "### FOR catalyst 21\n",
    "\n",
    "# device\n",
    "device = torch.device(\"cuda:0\")\n",
    "\n",
    "# model\n",
    "model = PANNsCNN14Att(**MC.sed_model_config)\n",
    "weights = torch.load(WEIGHTS_PATH)\n",
    "model.load_state_dict(weights[\"model\"])\n",
    "model.att_block = AttBlock(2048, 2, activation=\"sigmoid\")\n",
    "model.att_block.init_weights()\n",
    "model.to(device)\n",
    "\n",
    "# optimizer\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# scheduler\n",
    "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n",
    "\n",
    "# loss\n",
    "loss = ImprovedPANNsLoss().to(device)\n",
    "\n",
    "# callbacks\n",
    "callbacks = [\n",
    "    F1Callback(),\n",
    "    mAPCallback(),\n",
    "    PrecisionCallback(),\n",
    "    CheckpointCallback(save_n_best=3, logdir=LOG_DIR + 'checkpoints/', loader_key=\"valid\", metric_key=\"precision\", minimize=True),\n",
    "]\n",
    "\n",
    "# callbacks = [\n",
    "#     dl.MAPCallback(\n",
    "#         input_key=\"logits\", \n",
    "#         target_key=\"targets\", \n",
    "#         topk_args=(1, 3, 5), \n",
    "#         prefix=\"mAP\")\n",
    "# ]"
   ]
  },
  {
   "source": [
    "## Training"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# warnings.simplefilter(\"ignore\")\n",
    "\n",
    "# runner = SupervisedRunner(\n",
    "#     device=device,\n",
    "#     input_key=\"waveform\",\n",
    "#     input_target_key=\"targets\")\n",
    "\n",
    "# runner.train(\n",
    "#     model=model,\n",
    "#     criterion=loss,\n",
    "#     loaders=loaders,\n",
    "#     optimizer=optimizer,\n",
    "#     scheduler=scheduler,\n",
    "#     num_epochs=EPOCHS,\n",
    "#     verbose=True,\n",
    "#     callbacks=callbacks,\n",
    "#     logdir=LOG_DIR,\n",
    "#     main_metric=\"epoch_precision\",\n",
    "#     minimize_metric=True,\n",
    "#     amp=True\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "1/30 * Epoch (train):  15%|█▌        | 108/714 [04:22<24:33,  2.43s/it, loss=1.156, lr=1.000e-03, mAP=0.846, macro_f1=0.797, momentum=0.900, precision=0.897]\n",
      "Keyboard Interrupt\n"
     ]
    }
   ],
   "source": [
    "### For catalyst 21\n",
    "warnings.simplefilter(\"ignore\")\n",
    "\n",
    "runner = SupervisedRunner(\n",
    "    input_key=\"waveform\",\n",
    "    target_key=\"targets\"\n",
    ")\n",
    "\n",
    "runner.train(\n",
    "    model=model,\n",
    "    criterion=loss,\n",
    "    loaders=loaders,\n",
    "    optimizer=optimizer,\n",
    "    scheduler=scheduler,\n",
    "    num_epochs=EPOCHS,\n",
    "    verbose=True,\n",
    "    callbacks=callbacks,\n",
    "    valid_loader=\"valid\",\n",
    "    valid_metric=\"precision\",\n",
    "    logdir=LOG_DIR,\n",
    "    minimize_valid_metric=True,\n",
    "    # amp=True # Using fixed-precision for training can suit this task pretty well but I can't handle the nan/inf problem for both pytorch.amp and nvidia apex.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}