Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "15945d26",
   "metadata": {},
   "source": [
    "# Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6eab989",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "import pprint\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchsummary import summary\n",
    "import IPython.display as ipd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import librosa\n",
    "import librosa.display\n",
    "import tqdm.notebook as tq\n",
    "import utils\n",
    "import Datasets, Models\n",
    "from pydub import AudioSegment\n",
    "from tkinter import Tcl # file sorting by name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c01547d",
   "metadata": {},
   "source": [
    "# Train and Test functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e38e904",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to test the model against a dataset.\n",
    "# Be careful to pass the correct parameters to the test function to indicate\n",
    "# if you are training an ensamble (is_ensable=True), if you are using an RGB dataset \n",
    "# (RGB=True) and if you want the confusion matrix of the validation dataset \n",
    "# to be normalized (cm_normalized=True).\n",
    "\n",
    "def test(model, validation_dataset, Y_validation, RGB = False, is_ensamble = False, normalized_cm = False):\n",
    "    # Stop parameters learning\n",
    "    model.eval()\n",
    "\n",
    "    validation_loader = torch.utils.data.DataLoader(validation_dataset)\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    total_loss = 0\n",
    "    confusion_matrix = np.zeros((8, 8), dtype=int)\n",
    "    i=0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for inputs, label in validation_loader:\n",
    "            if(i%100==0):\n",
    "                print(i,\"/\",(len(validation_dataset)))\n",
    "            \n",
    "            if(RGB==False and is_ensamble == False):\n",
    "                inputs=inputs.unsqueeze(1)\n",
    "                \n",
    "            #predict label\n",
    "            output = model(inputs)\n",
    "           \n",
    "            \n",
    "            # Compute loss\n",
    "            loss = criterion(output, label)\n",
    "            total_loss += loss.item()\n",
    "\n",
    "            max_index = torch.argmax(output).item()  # The index with maximum probability\n",
    "\n",
    "            confusion_matrix[label][max_index] += 1\n",
    "\n",
    "            correct += (max_index == label)\n",
    "            i+=1\n",
    "\n",
    "    fig, ax = plt.subplots(dpi=500)\n",
    "    if(normalized_cm == True):\n",
    "        confusion_matrix = (confusion_matrix*8)/len(Y_validation)\n",
    "    cm=ConfusionMatrixDisplay(confusion_matrix=confusion_matrix)\n",
    "    cm.plot(ax=ax)\n",
    "    accuracy = 100 * correct / len(Y_validation)\n",
    "    average_loss = total_loss / len(Y_validation)\n",
    "\n",
    "    model.train()\n",
    "    return accuracy, average_loss, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7781caaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to test the model against a dataset using majority voting of 20 samples.\n",
    "# Be careful to pass the correct parameters to the vote_test function to indicate\n",
    "# if you are training an ensamble (is_ensable=True), if you are using an RGB dataset \n",
    "# (RGB=True) and if you want the confusion matrix of the validation dataset \n",
    "# to be normalized (cm_normalized=True).\n",
    "\n",
    "def vote_test(model, validation_dataset, Y_validation, RGB = False, is_ensamble = False, normalized_cm=False):\n",
    "    #Stop parameters learning\n",
    "    model.eval()\n",
    "    \n",
    "    validation_loader = torch.utils.data.DataLoader(validation_dataset, batch_size=20)\n",
    "\n",
    "    # Crea una funzione di perdita con pesi\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    \n",
    "    #criterion = nn.CrossEntropyLoss()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    total_loss = 0\n",
    "    confusion_matrix = np.zeros((8,8 ), dtype=int)\n",
    "\n",
    "    correct_maj=0\n",
    "    i = 0\n",
    "    \n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in validation_loader:\n",
    "            if(i%100==0):\n",
    "                print(i*20,\"/\",(len(validation_dataset)))\n",
    "            if(RGB==False and is_ensamble == False):\n",
    "                inputs=inputs.unsqueeze(1)\n",
    "            \n",
    "            #predict label\n",
    "            outputs = model(inputs)\n",
    "                \n",
    "            #print(\"Outputs:\",outputs,\"size:\",outputs.size())\n",
    "            #compute loss\n",
    "            voting=outputs.mean(dim=0)\n",
    "            voting=voting.unsqueeze(0)\n",
    "            label=labels[0].unsqueeze(0)\n",
    "            loss = criterion(voting, label)\n",
    "            total_loss += loss.item()\n",
    "            #print(\"Voting:\",voting,\"size:\",voting.size())\n",
    "            predicted= torch.argmax(voting)\n",
    "            #print(\"winning class\",predicted)\n",
    "            correct += (predicted == labels[0])\n",
    "            confusion_matrix[label][predicted]+=1\n",
    "            \n",
    "            votes=[0,0,0,0,0,0,0,0]\n",
    "            i+=1\n",
    "       \n",
    "            \n",
    "        \n",
    "    if(normalized_cm == True):\n",
    "        confusion_matrix = (confusion_matrix*8)/(len(Y_validation)/20)\n",
    "    cm=ConfusionMatrixDisplay(confusion_matrix=confusion_matrix)\n",
    "    fig, ax = plt.subplots(dpi=500)\n",
    "    cm.plot(ax=ax)\n",
    "    cm.plot()    \n",
    "    accuracy = 100*correct / (len(Y_validation)/20) \n",
    "    average_loss = total_loss / (len(Y_validation)/20)\n",
    "\n",
    "    model.train()\n",
    "    return accuracy, average_loss, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97555bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function used to train a model.\n",
    "# Be careful to pass the correct parameters to the train functions to indicate\n",
    "# if you are training an ensamble (is_ensable=True), if you are using an RGB dataset \n",
    "# (RGB=True) and if you want the confusion matrix of the validation set after each epoch \n",
    "# to be normalized (cm_normalized=True).\n",
    "\n",
    "def train(model, dataset, validation_dataset, batch_size, num_epochs, learning_rate, verbose = False,\n",
    "          reg=1e-5, RGB = False, is_ensamble = False, normalized_cm=False):\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    model.to(device)\n",
    "    val_loss_list=[]\n",
    "    val_acc_list=[]\n",
    "    train_loss_list=[]\n",
    "    train_acc_list=[]\n",
    "    counted_labels=[0,0,0,0,0,0,0,0]\n",
    "    \n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=reg)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    if not isinstance(dataset, Dataset):\n",
    "        raise ValueError(\"The dataset parameter should be an instance of torch.utils.data.Dataset.\")\n",
    "\n",
    "    data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n",
    "    num_batches = len(data_loader)\n",
    "    \n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0 \n",
    "        running_accuracy = 0.0\n",
    "        #initialize correctly predicted samples\n",
    "        \n",
    "        # Initialize the progress bar\n",
    "        progress_bar = tq.tqdm(total=num_batches, unit=\"batch\")\n",
    "    \n",
    "        # Initialize the progress bar description\n",
    "        progress_bar.set_description(f\"Epoch {epoch+1}/{num_epochs}\")\n",
    "        start_time = time.time()\n",
    "        \n",
    "        for batch_idx, batch in enumerate(data_loader):\n",
    "            \n",
    "            correct = 0 # reset train accuracy each batch\n",
    "            \n",
    "            inputs,labels = batch[0],batch[1]\n",
    "            if(verbose == True):\n",
    "                print(\"\\ninputs shape:\",inputs.size(),\", dtype:\",inputs.dtype,\" content: \",inputs)\n",
    "                print(\"min value:\",torch.min(inputs))\n",
    "                print(\"max value:\",torch.max(inputs))\n",
    "                print(\"\\nlabels shape:\",labels.size(),\",dtype:\",labels.dtype,\", content: \",labels)\n",
    "          \n",
    "            \n",
    "            # Extract the inputs and targets\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            if(RGB==False and is_ensamble == False):\n",
    "                inputs=inputs.unsqueeze(1)\n",
    "            \n",
    "            outputs = model(inputs)\n",
    "                        \n",
    "            if(verbose == True):\n",
    "                print(\"\\noutputs size:\",outputs.size(),\"content:\",outputs)\n",
    "                print(\"List of labels until now:\",counted_labels)\n",
    "\n",
    "            loss = criterion(outputs, labels) #labels need to be a vector of class indexes (0-7) of dim (batch_size)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "            \n",
    "            #calculate train accuracy\n",
    "            for index, output in enumerate(outputs):\n",
    "                max_index = torch.argmax(output).item() #the index with maximum probability\n",
    "                counted_labels[labels[index].item()]+=1\n",
    "                if(labels[index].item() == max_index):\n",
    "                    correct += 1\n",
    "            \n",
    "                if(verbose==True):\n",
    "                    print(\"considering output at index {}:\".format(index,output))\n",
    "                    print(\"max output index = {}\",max_index)\n",
    "                    if(labels[index].item() == max_index):\n",
    "                        print(\"correct! in fact labels[index] = {}, max_index = {}\".format(labels[index].item(),max_index))\n",
    "                    else:\n",
    "                        print(\"NOT correct! in fact labels[index] = {}, max_index = {}\".format(labels[index].item(),max_index))\n",
    "\n",
    "            \n",
    "            accuracy = 100 * correct / batch_size\n",
    "            running_accuracy += accuracy #epoch running_accuracy\n",
    "            \n",
    "            # Update the progress bar description and calculate bps\n",
    "            #progress_bar.set_postfix({\"Loss\": running_loss / (batch_idx + 1)})\n",
    "            average_accuracy = running_accuracy / (batch_idx + 1)\n",
    "            average_loss = running_loss / (batch_idx + 1)\n",
    "            progress_bar.set_postfix({\"avg_loss\": average_loss, \"acc\": accuracy, \"avg_acc\": average_accuracy})\n",
    "\n",
    "            # Update the progress bar\n",
    "            progress_bar.update(1)\n",
    "            # Evaluate the model on the validation dataset\n",
    "        \n",
    "        #calculate train loss and accuracy\n",
    "        average_loss = running_loss / len(data_loader)\n",
    "        average_accuracy = running_accuracy / len(data_loader)\n",
    "        train_loss_list.append(average_loss)\n",
    "        train_acc_list.append(average_accuracy)\n",
    "        \n",
    "        #calculate validation loss and accuracy\n",
    "        val_acc, val_loss,_ = test(model, validation_dataset, Y_validation, RGB = RGB, is_ensamble = is_ensamble, normalized_cm=False)\n",
    "        val_loss_list.append(val_loss)\n",
    "        val_acc_list.append(val_acc)\n",
    "        \n",
    "        \n",
    "        print(f\"Epoch [{epoch+1}/{num_epochs}],Train Loss: {average_loss:.4f}. Train Accuracy: {average_accuracy} Val Loss: {val_loss} Val Accuracy: {val_acc}\")\n",
    "        progress_bar.close()\n",
    "    return train_loss_list, train_acc_list, val_loss_list, val_acc_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "888c3e14",
   "metadata": {},
   "source": [
    "# Load STFT Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ed8ded2",
   "metadata": {},
   "source": [
    "### Dictionary creation for the classes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e70c4013",
   "metadata": {},
   "source": [
    "We want a dictionary indicating a numbeer for each genre:\n",
    "\n",
    "{0: 'Hip-Hop', 1: 'Pop', 2: 'Folk', 3: 'Rock', 4: 'Experimental', 5: 'International', 6: 'Electronic', 7: 'Instrumental'}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "029e985c",
   "metadata": {},
   "source": [
    "### Creation of the labels vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0be9951a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "folder_path=\"data/fma_small_stft_transposed_22050_overlapped\"\n",
    "Y_train, Y_validation, Y_test = Datasets.create_dataset_splitted(folder_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a20175a8",
   "metadata": {},
   "source": [
    "# Dataset Class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "590fe67f",
   "metadata": {},
   "source": [
    "Class to load the STFT from files. Each file has a (128,513) matrix containing the STFT of a 3 seconds audio clip."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cae70891",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "folder_path=\"data/fma_small_stft_transposed_22050_overlapped\"\n",
    "\n",
    "stft_train_folder = os.path.join(folder_path,'train') # concatenate train folder to path\n",
    "stft_validation_folder = os.path.join(folder_path,'validation') # concatenate train folder to path\n",
    "stft_test_folder = os.path.join(folder_path,'test') # concatenate train folder to path\n",
    "\n",
    "stft_train_file_paths, _ = Datasets.get_sorted_file_paths(stft_train_folder)\n",
    "stft_train_dataset = Datasets.DatasetSTFT(stft_train_file_paths, Y_train)\n",
    "print(\"len of train dataset: \",len(stft_train_dataset))\n",
    "\n",
    "stft_validation_file_paths, _ = Datasets.get_sorted_file_paths(stft_validation_folder)\n",
    "stft_validation_dataset = Datasets.DatasetSTFT(stft_validation_file_paths, Y_validation)\n",
    "print(\"len of validation dataset: \",len(stft_validation_dataset))\n",
    "\n",
    "stft_test_file_paths, _ = Datasets.get_sorted_file_paths(stft_test_folder)\n",
    "stft_test_dataset = Datasets.DatasetSTFT(stft_test_file_paths, Y_test)\n",
    "print(\"len of test dataset: \",len(stft_test_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1bd9c68",
   "metadata": {},
   "source": [
    "# Data normalization\n",
    "We will use Z-Score to normalize the training, validation and test set by calculating the mean and the std deviation on the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f571ada5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "save_filename = './data/fma_small_stft_transposed_22050_overlapped/train_mean'\n",
    "std_save_filename = './data/fma_small_stft_transposed_22050_overlapped/train_std_deviation'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "677263cf",
   "metadata": {},
   "source": [
    "## Calculation of mean and standard deviation (Long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5795abf2",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "batch_size=1\n",
    "total_n_batches = len(stft_train_dataset)/batch_size\n",
    "train_loader = torch.utils.data.DataLoader(stft_train_dataset, batch_size=batch_size)\n",
    "current_sum=0\n",
    "\n",
    "#iter all the training set by batches and calculate the sum of all the sample values (513*128 values for each sample)\n",
    "for batch_idx, batch in enumerate(train_loader):\n",
    "    #print(\"batch\",batch_idx,\"/\",total_n_batches,\"current_sum:\",current_sum)\n",
    "    inputs = batch[0]\n",
    "    labels = batch[1]\n",
    "    #print(\"inputs: shape:\",inputs.shape,\"content:\",inputs)\n",
    "    #print(\"labels:\",labels)\n",
    "    for sample in inputs:\n",
    "        #print(\"sample: shape\",sample.shape,\"content:\",sample)\n",
    "        current_sum += torch.sum(sample)\n",
    "        #print(\"current_sum:\",current_sum)\n",
    "print(\"final sum:\",current_sum)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c794bf2c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mean = current_sum/(len(stft_train_dataset)*513*128) #divide the sum for the total number of values considerated\n",
    "print(\"mean of training set:\",mean)\n",
    "\n",
    "#save the mean to a file\n",
    "print(\"Saving the mean in file:\",save_filename) \n",
    "np.save(save_filename,mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ec52eb5",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#now let's calculate the standard deviation (squared root of the variance)\n",
    "batch_size=1\n",
    "total_n_batches = len(stft_train_dataset)/batch_size\n",
    "train_loader = torch.utils.data.DataLoader(stft_train_dataset, batch_size=batch_size)\n",
    "current_sum_of_squares = 0\n",
    "\n",
    "for batch_idx, batch in enumerate(train_loader):\n",
    "    print(\"batch\",batch_idx,\"/\",total_n_batches,\"current_sum_of_squares:\",current_sum_of_squares)\n",
    "    inputs = batch[0]\n",
    "    labels = batch[1]\n",
    "    #print(\"inputs: shape:\",inputs.shape,\"content:\",inputs)\n",
    "    #print(\"labels:\",labels)\n",
    "    for sample in inputs:\n",
    "        #print(\"sample shape\",sample.shape)\n",
    "        for row in sample:\n",
    "            #print(\"row shape:\",row.shape)\n",
    "            for elem in row:\n",
    "                #print(\"elem: shape\",elem.shape,\"content:\",elem)\n",
    "                difference = elem - mean\n",
    "                difference_squared = difference**2\n",
    "                current_sum_of_squares += difference_squared\n",
    "                #print(\"current_sum:\",current_sum)\n",
    "print(\"final sum of squares:\",current_sum_of_squares)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cecfb0b5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "variance = current_sum_of_squares/((len(stft_train_dataset) * 513 * 128)-1)\n",
    "std_deviation = math.sqrt(variance)\n",
    "\n",
    "print(\"variance:\",variance)\n",
    "print(\"std_deviation:\",std_deviation)\n",
    "\n",
    "#save the std deviation to a file\n",
    "print(\"Saving the std_deviation in file:\",std_save_filename) \n",
    "np.save(std_save_filename,std_deviation)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "941fc2ab",
   "metadata": {},
   "source": [
    "## Load calculated mean and std deviation from file (fast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "977c1ac9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "loaded_mean = np.load(save_filename+'.npy')\n",
    "print(\"loaded mean:\",loaded_mean)\n",
    "\n",
    "loaded_std = np.load(std_save_filename+'.npy')\n",
    "print(\"loaded std:\",loaded_std)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c2d4804",
   "metadata": {},
   "source": [
    "## Create the normalized dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "092efbff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "\n",
    "batch_size = 1\n",
    "\n",
    "#transform used to normalize the dataset using the mean and the standard deviation computed\n",
    "transform = transforms.Compose([\n",
    "    transforms.Normalize(mean= loaded_mean, std= loaded_std)\n",
    "])\n",
    "\n",
    "stft_train_dataset = Datasets.DatasetSTFT(stft_train_file_paths, Y_train,  transform = transform)\n",
    "stft_validation_dataset = Datasets.DatasetSTFT(stft_validation_file_paths, Y_validation,  transform = transform)\n",
    "stft_test_dataset = Datasets.DatasetSTFT(stft_test_file_paths, Y_test,  transform = transform)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8985c8f",
   "metadata": {},
   "source": [
    "## Plot some STFT spectrograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eab2553b",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=1\n",
    "train_loader = torch.utils.data.DataLoader(stft_train_dataset, batch_size=batch_size)\n",
    "\n",
    "#iter all the training set by batches and calculate the sum of all the sample values (513*128 values for each sample)\n",
    "for batch_idx, batch in enumerate(train_loader):\n",
    "    inputs = batch[0]\n",
    "    labels = batch[1]\n",
    "    #print(\"inputs: shape:\",inputs.shape,\"content:\",inputs)\n",
    "    #print(\"labels:\",labels)\n",
    "    for sample in inputs:\n",
    "        print(\"sample: shape\",sample.shape)\n",
    "        fig, ax = plt.subplots(dpi=500)\n",
    "        img = librosa.display.specshow(librosa.amplitude_to_db(sample.T,ref=np.max), y_axis='log', x_axis='time', ax=ax)\n",
    "        ax.set_xlabel('Time (s)')\n",
    "        ax.set_ylabel('Frequency (Hz)')\n",
    "\n",
    "        fig.colorbar(img, ax=ax, format=\"%+2.0f dB\")\n",
    "    if(batch_idx==4):\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f8a52dc",
   "metadata": {},
   "source": [
    "# STFT Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89378ba6",
   "metadata": {},
   "source": [
    "### NNet1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65294264",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_NNet1 = Models.NNet1()\n",
    "summary(model_NNet1, (1, 128, 513))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d966cb85",
   "metadata": {},
   "source": [
    "### NNet2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05f7b904",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_NNet2 = Models.NNet2()\n",
    "summary(model_NNet2, (1, 128, 513))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72bcfe30",
   "metadata": {},
   "source": [
    "### NNet1_Small"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ead3547",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_NNet1_Small = Models.NNet1_Small()\n",
    "summary(model_NNet1_Small, (1, 128, 513))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9845a53e",
   "metadata": {},
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d034d9f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE=32\n",
    "EPOCHS=10\n",
    "\n",
    "learning_rate_list = [0.001,0.0001,0.00001]\n",
    "reg_list=[0.001,0.0001,0.00001]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9ef7fab",
   "metadata": {},
   "source": [
    "# Grid Search on NNet1_Small\n",
    "\n",
    "Here we show an example of code for performing grid search on hyperparameters. In this case we are doing it for NNet1_Small, but the same can be done to other STFT models. Be careful to pass the correct parameters to the train functions to indicate if you are training an ensamble (is_ensable=True), if you are using an RGB dataset (RGB=True) and if you want the confusion matrix of the validation set after each epoch to be normalized (cm_normalized=True).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac93d69c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "save_directory=\"./results/NNet1_Small/\"\n",
    "\n",
    "\n",
    "for i in learning_rate_list:\n",
    "    for j in reg_list:\n",
    "        if(j!=1e-5 and j!=1e-6):\n",
    "            filename=save_directory+\"lr_\"+\"0\"+str(i).split(\".\")[1]+\"_reg_\"+\"0\"+str(j).split(\".\")[1]\n",
    "        elif(j==1e-5):\n",
    "            filename=save_directory+\"lr_\"+\"0\"+str(i).split(\".\")[1]+\"_reg_\"+\"00001\"\n",
    "        else:\n",
    "            filename=save_directory+\"lr_\"+\"0\"+str(i).split(\".\")[1]+\"_reg_\"+\"000001\"\n",
    "        print(filename)\n",
    "        model = Models.NNet1_Small()\n",
    "        train_loss_list, train_acc_list, val_loss_list, val_acc_list =train(model, stft_train_dataset,stft_validation_dataset, batch_size=128, num_epochs=10, learning_rate=i, reg=j)\n",
    "        print(\"Trained with learning rate=\",i,\" and with regularization term=\",j)\n",
    "        print(\"Loss:\",val_loss_list)\n",
    "        print(\"Accuracy:\",val_acc_list)\n",
    "        save_values=[val_loss_list,val_acc_list]\n",
    "        np.savetxt(filename,save_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e7112c8",
   "metadata": {},
   "source": [
    "## Create dataset for raw audio\n",
    "\n",
    "The labels are the same as the STFT dataset (in the same order)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45f5f209",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "raw_folder_path=\"data/fma_small_raw_array_22050_overlapped\"\n",
    "    \n",
    "Y_train, Y_validation, Y_test = Datasets.create_dataset_splitted(raw_folder_path)\n",
    "raw_train_folder = os.path.join(raw_folder_path,'train') # concatenate train folder to path\n",
    "raw_validation_folder = os.path.join(raw_folder_path,'validation') # concatenate train folder to path\n",
    "raw_test_folder = os.path.join(raw_folder_path,'test') # concatenate train folder to path\n",
    "\n",
    "raw_train_file_paths, _ = Datasets.get_sorted_file_paths(raw_train_folder)\n",
    "raw_train_dataset = Datasets.DatasetRaw(raw_train_file_paths, Y_train)\n",
    "print(\"len of train dataset: \",len(raw_train_dataset))\n",
    "\n",
    "raw_validation_file_paths, _ = Datasets.get_sorted_file_paths(raw_validation_folder)\n",
    "raw_validation_dataset = Datasets.DatasetRaw(raw_validation_file_paths, Y_validation)\n",
    "print(\"len of validation dataset: \",len(raw_validation_dataset))\n",
    "\n",
    "raw_test_file_paths, _ = Datasets.get_sorted_file_paths(raw_test_folder)\n",
    "raw_test_dataset = Datasets.DatasetRaw(raw_test_file_paths, Y_test)\n",
    "print(\"len of test dataset: \",len(raw_test_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31de69e8",
   "metadata": {},
   "source": [
    "## Normalization of raw audio"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d390039",
   "metadata": {},
   "source": [
    "As for the STFT dataset, also for raw audio we calculate the mean and std deviation of the entire dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1ec8db0",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_save_filename_raw = './data/fma_small_raw_array_22050_overlapped/train_mean'\n",
    "std_save_filename_raw = './data/fma_small_raw_array_22050_overlapped/train_std_deviation'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "639b38ea",
   "metadata": {},
   "source": [
    "## Calculation of mean raw (Long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba507d2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_train_dataset = Datasets.DatasetRaw(raw_file_paths_train, Y_train)\n",
    "batch_size=1\n",
    "total_n_batches = len(raw_train_dataset)/batch_size\n",
    "train_loader = torch.utils.data.DataLoader(raw_train_dataset, batch_size=batch_size)\n",
    "current_sum=0\n",
    "\n",
    "#iter all the training set by batches and calculate the sum of all the sample values (513*128 values for each sample)\n",
    "for batch_idx, batch in enumerate(train_loader):\n",
    "    if(batch_idx%1000==0):\n",
    "        print(\"batch\",batch_idx,\"/\",total_n_batches,\"(\",round((batch_idx/len(train_dataset)*100)),\"%), current_sum:\",current_sum)\n",
    "    \n",
    "    inputs = batch[0]\n",
    "    labels = batch[1]\n",
    "    #print(\"inputs: shape:\",inputs.shape,\"content:\",inputs)\n",
    "    #print(\"labels:\",labels)\n",
    "    for sample in inputs:\n",
    "        #print(\"sample: shape\",sample.shape,\"content:\",sample)\n",
    "        current_sum += torch.sum(sample)\n",
    "       \n",
    "        #print(\"type of current_sum:\",current_sum.dtype)\n",
    "        #print(\"current_sum:\",current_sum)\n",
    "print(\"final sum:\",current_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e9a7842",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"current_sum\",current_sum)\n",
    "mean_raw = current_sum/(len(raw_train_dataset)*66150) #divide the sum for the total number of values considerated\n",
    "print(\"mean of training set:\",mean_raw)\n",
    "\n",
    "print(\"Saving the mean in file:\",mean_save_filename_raw) \n",
    "np.save(mean_save_filename_raw,mean_raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fcee895",
   "metadata": {},
   "source": [
    "# Calculation of std deviation raw (Long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6525b631",
   "metadata": {},
   "outputs": [],
   "source": [
    "#now let's calculate the standard deviation (squared root of the variance)\n",
    "\n",
    "batch_size=1\n",
    "total_n_batches = len(raw_train_dataset)/batch_size\n",
    "train_loader = torch.utils.data.DataLoader(raw_train_dataset, batch_size=batch_size)\n",
    "current_sum_of_squares = 0\n",
    "\n",
    "for batch_idx, batch in enumerate(train_loader):\n",
    "    if(batch_idx%1000==0):\n",
    "        print(\"batch\",batch_idx,\"/\",total_n_batches,round((batch_idx/len(train_dataset)*100)),\"%, current_sum_of_squares:\",current_sum_of_squares)\n",
    "    inputs = batch[0]\n",
    "    labels = batch[1]\n",
    "    #print(\"inputs: shape:\",inputs.shape,\"content:\\n\",inputs)\n",
    "    #print(\"labels:\",labels)\n",
    "    for elem in inputs:\n",
    "        elem = elem.double() #convert to float64 for precise calculations\n",
    "        #print(\"elem: shape\",elem.shape,\"content:\\n\",elem)\n",
    "        difference = elem - mean_raw\n",
    "        #print(\"difference: shape:\",difference.shape,\"content:\\n\", difference)\n",
    "        difference_squared = difference**2\n",
    "        #print(\"difference_squared: shape:\",difference_squared.shape,\"content:\\n\", difference_squared)\n",
    "        current_sum_of_squares += torch.sum(difference_squared)\n",
    "        #print(\"current_sum_of_squares:\",current_sum_of_squares)\n",
    "print(\"final sum of squares:\",current_sum_of_squares)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6603f3f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "variance_raw = current_sum_of_squares/((len(train_dataset)*66150)-1)\n",
    "std_deviation_raw = math.sqrt(variance)\n",
    "\n",
    "print(\"variance raw:\",variance)\n",
    "print(\"std_deviation_raw:\",std_deviation_raw)\n",
    "\n",
    "print(\"Saving the std_deviation_raw in file:\",std_save_filename_raw) \n",
    "np.save(std_save_filename_raw,std_deviation_raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "460cb8ee",
   "metadata": {},
   "source": [
    "# Load mean and std from file (fast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5d39f61",
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_mean_raw = np.load(mean_save_filename_raw+'.npy')\n",
    "print(\"loaded mean:\",loaded_mean_raw)\n",
    "\n",
    "loaded_std_raw = np.load(std_save_filename_raw+'.npy')\n",
    "print(\"loaded std:\",loaded_std_raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4912205",
   "metadata": {},
   "source": [
    "# Raw audio model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3b5c086",
   "metadata": {},
   "outputs": [],
   "source": [
    "RawModel=Models.NNet_Raw()\n",
    "summary(RawModel, (1,66150))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "469b75a5",
   "metadata": {},
   "source": [
    "## Plot some raw audio samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b6ffe6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import Tensor\n",
    "batch_size=1\n",
    "train_loader = torch.utils.data.DataLoader(raw_train_dataset, batch_size=batch_size)\n",
    "\n",
    "#iter all the training set by batches and calculate the sum of all the sample values (513*128 values for each sample)\n",
    "for batch_idx, batch in enumerate(train_loader):\n",
    "    inputs = batch[0]\n",
    "    labels = batch[1]\n",
    "    print(\"inputs: shape:\",inputs.shape)\n",
    "    print(\"labels:\",labels)\n",
    "    for sample in inputs:\n",
    "        fig, ax = plt.subplots(dpi=300)\n",
    "        librosa.display.waveshow(Tensor.numpy(sample.float()), sr=22150, ax=ax)\n",
    "        ax.set(title='Envelope view, mono')\n",
    "        ax.set_ylabel('Amplitude')\n",
    "        ax.set_xlabel('Time (s)')\n",
    "        ax.label_outer()\n",
    "    if(batch_idx==4):\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33724630",
   "metadata": {},
   "source": [
    "# Grid Search for NNet_Raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab3f0168",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#TODO: Add batch_size optimization loop\n",
    "save_directory=\"./results/NNet_Raw/\"\n",
    "\n",
    "for i in learning_rate_list:\n",
    "    for j in reg_list:\n",
    "        if(j!=1e-5 and j!=1e-6):\n",
    "            filename=save_directory+\"lr_\"+\"0\"+str(i).split(\".\")[1]+\"_reg_\"+\"0\"+str(j).split(\".\")[1]\n",
    "        elif(j==1e-5):\n",
    "            filename=save_directory+\"lr_\"+\"0\"+str(i).split(\".\")[1]+\"_reg_\"+\"00001\"\n",
    "        else:\n",
    "            filename=save_directory+\"lr_\"+\"0\"+str(i).split(\".\")[1]+\"_reg_\"+\"000001\"\n",
    "        print(filename)\n",
    "        model = Models.NNet_Raw()\n",
    "        train_loss_list, train_acc_list, val_loss_list, val_acc_list =train(model, raw_train_dataset, raw_validation_dataset, batch_size=128, num_epochs=10, learning_rate=i, reg=j)\n",
    "        print(\"Trained with learning rate=\",i,\" and with regularization term=\",j)\n",
    "        print(\"Loss:\",val_loss_list)\n",
    "        print(\"Accuracy:\",val_acc_list)\n",
    "        save_values=[val_loss_list,val_acc_list]\n",
    "        np.savetxt(filename,save_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed65d637",
   "metadata": {},
   "source": [
    "# Ensamble of NNet1_Small and NNet_Raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4f2b06b",
   "metadata": {},
   "outputs": [],
   "source": [
    "stft_train_file_paths, _ = Datasets.get_sorted_file_paths(stft_train_folder)\n",
    "raw_train_file_paths, _ = Datasets.get_sorted_file_paths(raw_train_folder)\n",
    "ens_train_dataset = Datasets.DatasetEnsemble(stft_train_file_paths,raw_train_file_paths, Y_train)\n",
    "print(\"len of train dataset: \",len(ens_train_dataset))\n",
    "\n",
    "stft_validation_file_paths, _ = Datasets.get_sorted_file_paths(stft_validation_folder)\n",
    "raw_validation_file_paths, _ = Datasets.get_sorted_file_paths(raw_validation_folder)\n",
    "ens_validation_dataset = Datasets.DatasetEnsemble(stft_validation_file_paths,raw_validation_file_paths, Y_validation)\n",
    "print(\"len of validation dataset: \",len(ens_validation_dataset))\n",
    "\n",
    "stft_test_file_paths, _ = Datasets.get_sorted_file_paths(stft_test_folder)\n",
    "raw_test_file_paths, _ = Datasets.get_sorted_file_paths(raw_test_folder)\n",
    "ens_test_dataset = Datasets.DatasetEnsemble(stft_test_file_paths,raw_test_file_paths, Y_test)\n",
    "print(\"len of test dataset: \",len(ens_test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b40ba6ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "#With the following parameter it can be established if the ensamble should use pre-trained weights or not\n",
    "weights=True\n",
    "model_ensamble_weights=Models.Ensemble(load_weights=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fb50b87",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss_list, train_acc_list, val_loss_list, val_acc_list =train(model_ensamble_weights, ens_train_dataset, ens_validation_dataset, batch_size=64, num_epochs=10, learning_rate=0.001,verbose=False, RGB=False,is_ensamble=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "641166af",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "# NN on echonest features \n",
    "\n",
    "As you can see below, unfortunately we cannot perform a train using echonest features (danceability, tempo, acousticness, ...) because there are only ►1300 tracks with echonest features which are not NaN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25e87d69",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"opening csvs...\")\n",
    "\n",
    "tracks = utils.load('data/fma_metadata/tracks.csv')\n",
    "\n",
    "echonest = utils.load('data/fma_metadata/echonest.csv')\n",
    "echonest = echonest['echonest', 'audio_features']\n",
    "small = tracks[tracks['set', 'subset'] <= 'small']\n",
    "\n",
    "print(\"small dataset shape:\",small.shape)\n",
    "print(\"echonest csv shape (only audio features):\",echonest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8b18347",
   "metadata": {},
   "outputs": [],
   "source": [
    "#select small dataset from echonest csv\n",
    "\n",
    "track_ids = small.index.values.tolist()\n",
    "print(\"Track ids shape:\",len(track_ids),\"content:\",track_ids[:10],\"...\")\n",
    "echonest_small = pd.DataFrame(echonest,index=track_ids)\n",
    "\n",
    "ipd.display(echonest_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12714b32",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_echonest = echonest_small.to_numpy(dtype=np.float16)\n",
    "print(\"X_train_echonest: shape:\",X_train_echonest.shape)\n",
    "\n",
    "nan_rows = np.argwhere(np.isnan(X_train_echonest).all(axis=1))\n",
    "\n",
    "print(\"There are\",len(nan_rows),\"rows containing NaN only as data\")\n",
    "#nan_rows = nan_rows.squeeze()\n",
    "#for i in nan_rows:\n",
    "#    print(\"row:\",i,\":\",X_train_echonest[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e917bc8e",
   "metadata": {},
   "source": [
    "# ResNet\n",
    "We try to use transfer learning using a ResNet18 by torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b7bc56f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Choose the model to use by changing the value of the following variable\n",
    "fine_tuning=True\n",
    "\n",
    "ResNetModel=Models.Model_ResNet18(pretrained=fine_tuning)\n",
    "summary(ResNetModel, (3,128,513))\n",
    "print(ResNetModel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c24605b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "transform_ResNet18 = transforms.Compose([\n",
    "    transforms.Normalize(mean= [loaded_mean,loaded_mean,loaded_mean], std=[loaded_std,loaded_std,loaded_std]) #our values\n",
    "    #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) #ResNet18 specific values\n",
    "])\n",
    "\n",
    "rgb_train_dataset = Datasets.DatasetRGB(stft_train_file_paths, Y_train,  transform = transform)\n",
    "rgb_validation_dataset = Datasets.DatasetRGB(stft_validation_file_paths, Y_validation,  transform = transform)\n",
    "rgb_test_dataset = Datasets.DatasetRGB(stft_test_file_paths, Y_test,  transform = transform)\n",
    "\n",
    "train_data_loader = DataLoader(rgb_train_dataset, batch_size = 10, shuffle=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f17e750f",
   "metadata": {},
   "source": [
    "# Training ResNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f3747da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train the model\n",
    "train_loss_list, train_acc_list, val_loss_list, val_acc_list =train(ResNetModel, rgb_train_dataset,rgb_validation_dataset, batch_size=64, num_epochs=10, learning_rate=0.001,verbose=False, RGB=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d622477a",
   "metadata": {},
   "source": [
    "# Example of \"Best Model\" Training\n",
    "Here we train a model using the hyperparameters found with grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b43d6a9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_directory=\"./best_models/\"\n",
    "\n",
    "learning_rate = 0.001\n",
    "reg=0.0001\n",
    "epochs=9\n",
    "\n",
    "\n",
    "filename=save_directory+\"results/NNet1\"        \n",
    "print(filename)\n",
    "model = Models.NNet1()\n",
    "train_loss_list, train_acc_list, val_loss_list, val_acc_list =train(model, stft_train_dataset,stft_validation_dataset, batch_size=64, num_epochs=epochs, learning_rate=learning_rate, reg=reg)\n",
    "print(\"Trained with learning rate=\",i,\" and with regularization term=\",j)\n",
    "torch.save(model.state_dict(), save_directory+\"best_models/NNet_Raw\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e16b6f4",
   "metadata": {},
   "source": [
    "# Example of Model Testing\n",
    "Here we test a model using majority voting on the test dataset. The same procedure can be done to all the models we want."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c13ed2db",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path=\"./best_models/best_models/\"\n",
    "model_name=\"NNet1\"\n",
    "\n",
    "test_model=Models.NNet1()\n",
    "test_model.load_state_dict(torch.load(model_path+model_name), strict=False)\n",
    "print(\"Model loaded: testing...\")\n",
    "test_results = vote_test(test_model,stft_test_dataset, Y_test, RGB=False, is_ensamble=False, normalized_cm=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71f82462",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "# Results Loader\n",
    "These lines of code are used to find the best models among the ones trained with different hyperparameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdead8f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_value(values,loss):\n",
    "    if loss==True:\n",
    "        max_v=100\n",
    "    else:\n",
    "        max_v=0\n",
    "    index=-1\n",
    "    for i in range(len(values)):\n",
    "        if loss==True:\n",
    "            if values[i]<max_v:\n",
    "                max_v=values[i]\n",
    "                index=i+1\n",
    "        else:\n",
    "            if values[i]>max_v:\n",
    "                max_v=values[i]\n",
    "                index=i+1\n",
    "    return max_v,index\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4a9f9ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_directory=\"./results/\"\n",
    "models=os.listdir(res_directory) \n",
    "for i in models:\n",
    "    print(i)\n",
    "    best_loss=100\n",
    "    best_acc=0\n",
    "    best_loss_ep=0\n",
    "    best_acc_ep=0\n",
    "    model_folder=res_directory+i\n",
    "    trials=os.listdir(model_folder)\n",
    "    if(len(trials)==0):\n",
    "        continue\n",
    "    best_trials=[(best_loss,best_loss_ep),(best_acc,best_acc_ep)]\n",
    "    best_trials_names=['','']\n",
    "    for j in trials:\n",
    "        print(j)\n",
    "        res=np.loadtxt(model_folder+\"/\"+j)\n",
    "        loss,epoch_l=find_best_value(res[0],True)\n",
    "        accuracy,epoch_a=find_best_value(res[1],False)\n",
    "        if(loss<best_loss):\n",
    "            best_trials_names[0]=j\n",
    "            best_loss=loss\n",
    "            best_loss_ep=epoch_l\n",
    "            best_trials[0]=(best_loss,best_loss_ep)\n",
    "        if(accuracy>best_acc):\n",
    "            best_trials_names[1]=j\n",
    "            best_acc=accuracy\n",
    "            best_acc_ep=epoch_a\n",
    "            best_trials[1]=(best_acc,best_acc_ep)\n",
    "            \n",
    "    print(\"Model:\",i)\n",
    "    print(\"Best Model for accuracy:\",best_trials_names[1])\n",
    "    print(\"Value:\",best_trials[1][0],\"Epoch:\",best_trials[1][1])\n",
    "    print(\"Best Model for loss:\",best_trials_names[0])\n",
    "    print(\"Value:\",best_trials[0][0],\"Epoch:\",best_trials[0][1])\n",
    "    print(\"\")\n",
    "    \n",
    "print(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f2c292f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#plot the first three best models train vs val loss in 10 epochs (ResNet f.t., Ensamble l.w., NNet1_Small) (in best_models/results)\n",
    "\n",
    "data_ResNet = np.loadtxt('./best_models/results/ResNet18_Reduced_FineTuning')\n",
    "print(\"\\nResNet18_Reduced_FineTuning:\\n\",data_ResNet)\n",
    "\n",
    "data_Ensemble_Weights = np.loadtxt('./best_models/results/Ensemble_Weights')\n",
    "print(\"\\nEnsemble_Weights:\\n\",data_Ensemble_Weights)\n",
    "\n",
    "data_NNet1_Small = np.loadtxt('./best_models/results/NNet1_Small')\n",
    "print(\"\\nNNet1_Small:\\n\",data_NNet1_Small)\n",
    "\n",
    "print(\"asdasd\\n\",data_ResNet[0])\n",
    "\n",
    "# Epochs (assuming you have 10 epochs)\n",
    "epochs = np.arange(1, 11)\n",
    "\n",
    "# Create a figure and axis\n",
    "fig, ax = plt.subplots(dpi=500)\n",
    "\n",
    "# Plot train loss and validation accuracy for each model\n",
    "ax.plot(epochs, data_ResNet[0], label='ResNet (f.t) Train Loss', color='blue', linestyle='dashed')\n",
    "ax.plot(epochs, data_ResNet[2], label='ResNet (f.t.) Validation Loss', color='blue', linestyle='solid')\n",
    "\n",
    "ax.plot(epochs, data_Ensemble_Weights[0], label='Ensemble Weights Train Loss', color='green', linestyle='dashed')\n",
    "ax.plot(epochs, data_Ensemble_Weights[2], label='Ensemble Weights Validation Loss', color='green', linestyle='solid')\n",
    "\n",
    "ax.plot(epochs, data_NNet1_Small[0], label='NNet1 Small Train Loss', color='red', linestyle='dashed')\n",
    "ax.plot(epochs, data_NNet1_Small[2], label='NNet1 Small Validation Loss', color='red', linestyle='solid')\n",
    "\n",
    "# Add labels and title\n",
    "ax.set_xlabel('Epochs')\n",
    "ax.set_ylabel('Loss')\n",
    "ax.grid(True)\n",
    "\n",
    "# Place the legend below the plot\n",
    "ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), fancybox=True, shadow=True, ncol=2)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec16cbd4",
   "metadata": {},
   "source": [
    "# F1-Score\n",
    "Here is the f1_score function which returns the f1 score for each class and the average f1 score. Below also a test using NNet1 on its f1 score results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "185f4311",
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1_score(confusion_matrix):\n",
    "    # Calculate F1 score for each class, given a confusion matrix (rows= true label, column = predicted label)\n",
    "    f1_scores = []\n",
    "    for i in range(len(confusion_matrix)):\n",
    "        true_positives = confusion_matrix[i][i]\n",
    "        false_positives = sum(confusion_matrix[j][i] for j in range(len(confusion_matrix)) if j != i)\n",
    "        false_negatives = sum(confusion_matrix[i][j] for j in range(len(confusion_matrix)) if j != i)\n",
    "\n",
    "        precision = true_positives / (true_positives + false_positives)\n",
    "        recall = true_positives / (true_positives + false_negatives)\n",
    "\n",
    "        f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
    "        f1_scores.append(f1)\n",
    "\n",
    "    # Calculate the average F1 score\n",
    "    average_f1_score = sum(f1_scores) / len(f1_scores)\n",
    "\n",
    "    # Print F1 scores for each class\n",
    "    for i, f1 in enumerate(f1_scores):\n",
    "        print(f\"F1 Score for Class {i+1}: {f1:.4f}\")\n",
    "\n",
    "    print(f\"Average F1 Score: {average_f1_score:.4f}\")\n",
    "    \n",
    "    return average_f1_score, f1_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b69d407",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path=\"./best_models/best_models/\"\n",
    "model_name=\"NNet1\"\n",
    "\n",
    "test_model=Models.NNet1()\n",
    "test_model.load_state_dict(torch.load(model_path+model_name), strict=False)\n",
    "print(\"Model loaded: testing...\")\n",
    "test_results = vote_test(test_model,stft_test_dataset, Y_test, RGB=False, is_ensamble=False, normalized_cm=False)\n",
    "\n",
    "confusion_matrix = test_results[2]\n",
    "print(\"confusion matrix for Ensemble using weights:\\n\",confusion_matrix)\n",
    "\n",
    "# Define your confusion matrix\n",
    "confusion_matrix = np.array(confusion_matrix)\n",
    "\n",
    "f1_average_score, f1_scores = f1_score(confusion_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5f1aa8f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}