inspect_cnn_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspect prediction results from a CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sn\n",
    "import pickle\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, recall_score\n",
    "import read_settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set input and output directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read settings\n",
    "global_settings = read_settings.check_global()\n",
    "instrument = global_settings['input_data']['instrument']\n",
    "cnn_settings = read_settings.check_cnn()\n",
    "use_weights = cnn_settings['data']['use_weights']\n",
    "\n",
    "# Directory for input data\n",
    "data_dir = os.path.join('data', instrument)\n",
    "\n",
    "# Directory for training outputs\n",
    "output_dirs = glob.glob(os.path.join('output', '_'.join(['cnn', 'w' if use_weights else 'nw', instrument, '*'])))\n",
    "output_dirs.sort()\n",
    "output_dirs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choose output directory to inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = output_dirs[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Glimpse at settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(output_dir, 'settings.pickle'),'rb') as settings_file:\n",
    "    settings = pickle.load(settings_file)\n",
    "settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_comp = pd.read_csv(os.path.join(output_dir, 'df_comp.csv')).set_index('classif_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "df_comp.plot.bar(stacked=True, figsize=(16,8), fontsize = 14)\n",
    "plt.xlabel(\"Classes\", fontsize = 14)\n",
    "plt.ylabel(\"Image number\", fontsize = 14)\n",
    "plt.legend(loc=\"best\")\n",
    "plt.title(\"Dataset composition for CNN\", fontsize = 16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read training file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(output_dir, 'train_results.pickle'),'rb') as results_file:\n",
    "    train_results = pickle.load(results_file)\n",
    "    \n",
    "    train_acc = train_results.get('accuracy')\n",
    "    val_acc = train_results.get('val_accuracy')\n",
    "    \n",
    "    train_loss = train_results.get('loss')\n",
    "    val_loss = train_results.get('val_loss')\n",
    "    \n",
    "    lr = train_results.get('lr')\n",
    "    \n",
    "# Compute number of training epochs\n",
    "epochs = len(train_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Look for best epoch, i.e. where validation loss is smaller"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_epoch = np.argmin(val_loss) + 1 # Add 1 because vector indexing starts from 0 but epochs start from 1\n",
    "print(f'Best epoch is number {best_epoch}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot training evolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 15))\n",
    "plt.subplot(3, 1, 1)\n",
    "plt.plot(list(range(1, epochs+1)), train_acc, label='Training Accuracy')\n",
    "plt.plot(list(range(1, epochs+1)), val_acc, label='Validation Accuracy')\n",
    "plt.axvline(best_epoch, color = 'k', ls='dotted', label='best epoch')\n",
    "plt.legend(loc='best')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.ylim([min(plt.ylim()),1])\n",
    "plt.title('Training and Validation Accuracy')\n",
    "\n",
    "plt.subplot(3, 1, 2)\n",
    "plt.plot(list(range(1, epochs+1)), train_loss, label='Training Loss')\n",
    "plt.plot(list(range(1, epochs+1)), val_loss, label='Validation Loss')\n",
    "plt.axvline(best_epoch, color = 'k', ls='dotted', label='best epoch')\n",
    "plt.legend(loc='best')\n",
    "plt.ylabel('Cross Entropy')\n",
    "plt.title('Training and Validation Loss')\n",
    "\n",
    "plt.subplot(3, 1, 3)\n",
    "plt.plot(list(range(1, epochs+1)), lr)\n",
    "plt.ylabel('Learning rate')\n",
    "plt.title('Learning rate evolution')\n",
    "\n",
    "plt.xlabel('epoch')\n",
    "plt.suptitle('CNN training', fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read test file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(output_dir, 'test_results.pickle'),'rb') as results_file:\n",
    "    test_results = pickle.load(results_file)\n",
    "\n",
    "    classes = test_results.get('classes')\n",
    "    classes_g = test_results.get('classes_g')\n",
    "    plankton_classes = test_results.get('plankton_classes')\n",
    "    plankton_classes_g = test_results.get('plankton_classes_g')\n",
    "    \n",
    "    true_classes = test_results.get('true_classes')\n",
    "    predicted_classes = test_results.get('predicted_classes')\n",
    "    true_classes_g = test_results.get('true_classes_g')\n",
    "    predicted_classes_g = test_results.get('predicted_classes_g')\n",
    "    \n",
    "    accuracy = test_results.get('accuracy')\n",
    "    balanced_accuracy = test_results.get('balanced_accuracy')\n",
    "    plankton_precision = test_results.get('plankton_precision')\n",
    "    plankton_recall = test_results.get('plankton_recall')\n",
    "    \n",
    "    accuracy_g = test_results.get('accuracy_g')\n",
    "    balanced_accuracy_g = test_results.get('balanced_accuracy_g')\n",
    "    plankton_precision_g = test_results.get('plankton_precision_g')\n",
    "    plankton_recall_g = test_results.get('plankton_recall_g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy, precision and recall scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Accuracy score is {accuracy}')\n",
    "print(f'Balanced accuracy score is {balanced_accuracy}')\n",
    "print(f'Weighted plankton precision score is {plankton_precision}')\n",
    "print(f'Weighted plankton recall is {plankton_recall}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot a confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create confution matrix\n",
    "cm = confusion_matrix(true_classes, predicted_classes, normalize='true')\n",
    "\n",
    "# Plot it\n",
    "plt.figure(figsize=(20,20))\n",
    "plt.imshow(cm, cmap='Greys')\n",
    "plt.colorbar()\n",
    "plt.clim(0,1)\n",
    "tick_marks = np.arange(len(classes))\n",
    "plt.xticks(tick_marks, classes, rotation=90, fontsize=14)\n",
    "plt.yticks(tick_marks, classes, fontsize=14)\n",
    "plt.ylabel('True label', fontsize=14)\n",
    "plt.xlabel('Predicted label', fontsize=14)\n",
    "plt.title(\"Confusion matrix for CNN\", fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot a classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create classification report\n",
    "report = classification_report(true_classes, predicted_classes, output_dict=True)\n",
    "\n",
    "# List annotations for figure\n",
    "annot = [str(x) for x in classes]\n",
    "annot.extend((\"accuracy\", \"macro avg\", \"weighted avg\"))\n",
    "\n",
    "# Convert report to dataframe\n",
    "df_report = pd.DataFrame(report).transpose().drop('support', axis=1)\n",
    "\n",
    "# Plot figure\n",
    "plt.figure(figsize = (8,15))\n",
    "sn.heatmap(df_report, annot=True, vmin=0, vmax=1.0,  yticklabels = annot, cmap=\"Greys\")\n",
    "plt.title(\"Classification report for CNN\", fontsize=16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy, precision and recall scores after regrouping classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Grouped accuracy score is {accuracy_g}')\n",
    "print(f'Grouped balanced accuracy score is {balanced_accuracy_g}')\n",
    "print(f'Grouped weighted plankton precision score is {plankton_precision_g}')\n",
    "print(f'Grouped weighted plankton recall is {plankton_recall_g}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot a confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create confution matrix\n",
    "cm_g = confusion_matrix(true_classes_g, predicted_classes_g, normalize='true')\n",
    "\n",
    "# Plot it\n",
    "plt.figure(figsize=(20,20))\n",
    "plt.imshow(cm_g, cmap='Greys')\n",
    "plt.colorbar()\n",
    "plt.clim(0,1)\n",
    "tick_marks = np.arange(len(classes_g))\n",
    "plt.xticks(tick_marks, classes_g, rotation=90, fontsize=14)\n",
    "plt.yticks(tick_marks, classes_g, fontsize=14)\n",
    "plt.ylabel('True label', fontsize=14)\n",
    "plt.xlabel('Predicted label', fontsize=14)\n",
    "plt.title(\"Confusion matrix for CNN after grouping ecological classes\", fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot a classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create classification report\n",
    "report = classification_report(true_classes_g, predicted_classes_g, output_dict=True)\n",
    "\n",
    "# List annotations for figure\n",
    "annot = [str(x) for x in classes_g]\n",
    "annot.extend((\"accuracy\", \"macro avg\", \"weighted avg\"))\n",
    "\n",
    "# Convert report to dataframe\n",
    "df_report = pd.DataFrame(report).transpose().drop('support', axis=1)\n",
    "\n",
    "# Plot figure\n",
    "plt.figure(figsize = (8,15))\n",
    "sn.heatmap(df_report, annot=True, vmin=0, vmax=1.0,  yticklabels = annot, cmap=\"Greys\")\n",
    "plt.title(\"Classification report for CNN after grouping ecological classes\", fontsize=16)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}