whale_app.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/controll-ST/lib/python3.6/site-packages/matplotlib/__init__.py:1003: UserWarning: Duplicate key in file \"/home/ubuntu/.config/matplotlib/matplotlibrc\", line #2\n",
      "  (fname, cnt))\n",
      "/home/ubuntu/anaconda3/envs/controll-ST/lib/python3.6/site-packages/matplotlib/__init__.py:1003: UserWarning: Duplicate key in file \"/home/ubuntu/.config/matplotlib/matplotlibrc\", line #3\n",
      "  (fname, cnt))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import os\n",
    "\n",
    "from matplotlib.pyplot import imshow\n",
    "from PIL import Image\n",
    "\n",
    "# Ignore warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "plt.ion()   # interactive mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_train_path = os.path.abspath('data/train/')\n",
    "img_test_path = os.path.abspath('data/test/')\n",
    "csv_train_path = os.path.abspath('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Image</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0000e88ab.jpg</th>\n",
       "      <td>w_f48451c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0001f9222.jpg</th>\n",
       "      <td>w_c3d896a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>00029d126.jpg</th>\n",
       "      <td>w_20df2c5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>00050a15a.jpg</th>\n",
       "      <td>new_whale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0005c1ef8.jpg</th>\n",
       "      <td>new_whale</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Id\n",
       "Image                   \n",
       "0000e88ab.jpg  w_f48451c\n",
       "0001f9222.jpg  w_c3d896a\n",
       "00029d126.jpg  w_20df2c5\n",
       "00050a15a.jpg  new_whale\n",
       "0005c1ef8.jpg  new_whale"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(csv_train_path).set_index('Image')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIR = 'train_images/'\n",
    "\n",
    "def plot_images_for_filenames(filenames, labels, rows=5):\n",
    "    imgs = [plt.imread(INPUT_DIR + str(filename)) for filename in filenames]\n",
    "    \n",
    "    return plot_images(imgs, labels, rows)\n",
    "    \n",
    "        \n",
    "def plot_images(imgs, labels, rows=5):\n",
    "    # Set figure to 13 inches x 8 inches\n",
    "    figure = plt.figure(figsize=(13, 8))\n",
    "\n",
    "    cols = len(imgs) // rows + 1\n",
    "\n",
    "    for i in range(len(imgs)):\n",
    "        subplot = figure.add_subplot(rows, cols, i + 1)\n",
    "        subplot.axis('Off')\n",
    "        if labels:\n",
    "            subplot.set_title(labels[i], fontsize=16)\n",
    "        plt.imshow(imgs[i])\n",
    "\n",
    "        \n",
    "def show_Whale(imgs, labels, per_row=2):\n",
    "    n         = len(imgs)\n",
    "    rows      = (n + per_row - 1)//per_row\n",
    "    cols      = min(per_row, n)\n",
    "    fig, axes = plt.subplots(rows,cols, figsize=(24//per_row*cols,24//per_row*rows))\n",
    "    for ax in axes.flatten(): \n",
    "        ax.axis('off')\n",
    "    for i,(img,ax) in enumerate(zip(imgs, axes.flatten())): \n",
    "        ax.set_title(labels[i], fontsize=20)\n",
    "        ax.imshow(img.convert('RGB'))\n",
    "        \n",
    "#Now show some of the whales\n",
    "from PIL import Image as pil_image\n",
    "\n",
    "rand_rows = df.sample(frac=1.)[:20]\n",
    "imgs_list = list(rand_rows['Image'])\n",
    "labels = list(rand_rows['Id'])\n",
    "\n",
    "imgs = [pil_image.open(INPUT_DIR + str(p)) for p in imgs_list]\n",
    "\n",
    "show_Whale(imgs, labels)\n",
    "# plot_images_for_filenames(imgs_list, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-6df28902806f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mnew_whale_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mId\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"new_whale\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# only new_whale dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mtrain_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mId\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"new_whale\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# no new_whale dataset, used for training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0munique_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mId\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "new_whale_df = df[df.Id == \"new_whale\"] # only new_whale dataset\n",
    "train_df = df[~(df.Id == \"new_whale\")] # no new_whale dataset, used for training\n",
    "unique_labels = np.unique(train_df.Id.values)\n",
    "\n",
    "labels_dict = dict()\n",
    "labels_list = []\n",
    "for i in range(len(unique_labels)):\n",
    "    labels_dict[unique_labels[i]] = i\n",
    "    labels_list.append(unique_labels[i])\n",
    "train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])\n",
    "split_ratio = 0.2\n",
    "train_new_df, test_new_df = train_test_split(train_df, test_size=split_ratio)\n",
    "\n",
    "train_new_df.to_csv('./data/train_new.csv', sep=',', encoding='utf-8', header=False)\n",
    "test_new_df.to_csv('./data/test_new.csv', sep=',', encoding='utf-8', header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25361, 1)\n",
      "5004\n",
      "20288 5073\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "print(len(unique_labels)) #total no. of final classes\n",
    "\n",
    "train_data_size = int(df.shape[0]*(1-split_ratio))\n",
    "valid_data_size = df.shape[0] - train_data_size\n",
    "print(train_data_size, valid_data_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Image</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>603fd4218.jpg</th>\n",
       "      <td>2044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ad0b56920.jpg</th>\n",
       "      <td>4474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e9afe2a62.jpg</th>\n",
       "      <td>3212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b75579156.jpg</th>\n",
       "      <td>1833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79b102480.jpg</th>\n",
       "      <td>4353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Id\n",
       "Image              \n",
       "603fd4218.jpg  2044\n",
       "ad0b56920.jpg  4474\n",
       "e9afe2a62.jpg  3212\n",
       "b75579156.jpg  1833\n",
       "79b102480.jpg  4353"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_new_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference taken from https://github.com/pytorch/vision/issues/81 to load CSV file as a data loader in pytorch\n",
    "# and https://pytorch.org/tutorials/beginner/data_loading_tutorial.html\n",
    "import torch.utils.data as data\n",
    "\n",
    "from PIL import Image\n",
    "import os\n",
    "import os.path\n",
    "\n",
    "def default_loader(path):\n",
    "    return Image.open(path).convert('RGB')\n",
    "\n",
    "def default_flist_reader(flist):\n",
    "    \"\"\"\n",
    "    flist format: impath,label\\nimpath,label\\n ...(same to caffe's filelist)\n",
    "    \"\"\"\n",
    "    imlist = []\n",
    "    with open(flist, 'r') as rf:\n",
    "        for line in rf.readlines():\n",
    "            impath, imlabel = line.split(',')\n",
    "            imlist.append( (impath, int(imlabel)) )\n",
    "\n",
    "    return imlist\n",
    "\n",
    "class ImageFilelist(data.Dataset):\n",
    "    def __init__(self, root, flist, transform=None, target_transform=None,\n",
    "            flist_reader=default_flist_reader, loader=default_loader):\n",
    "        self.root   = root\n",
    "        self.imlist = flist_reader(flist)\n",
    "        self.transform = transform\n",
    "        self.target_transform = target_transform\n",
    "        self.loader = loader\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        impath, target = self.imlist[index]\n",
    "        img = self.loader(os.path.join(self.root,impath))\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        if self.target_transform is not None:\n",
    "            target = self.target_transform(target)\n",
    "\n",
    "        return img, target\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.imlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<__main__.ImageFilelist object at 0x7efc273d56a0>\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import torch\n",
    "import torchvision\n",
    "from torchvision import datasets\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import os\n",
    "import torch\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils\n",
    "from skimage import io, transform\n",
    "\n",
    "normalize = transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "\n",
    "# preprocess_train = transforms.Compose([transforms.RandomResizedCrop(224),\n",
    "#                                     transforms.RandomHorizontalFlip(),\n",
    "#                                     transforms.ToTensor(),\n",
    "#                                     normalize])\n",
    "\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "         ImageFilelist(root=\"./data/train/\", flist=\"./data/train_new.csv\",\n",
    "             transform=transforms.Compose([transforms.RandomSizedCrop(224),\n",
    "                 transforms.RandomHorizontalFlip(),\n",
    "                 transforms.ToTensor(), normalize,\n",
    "         ])),\n",
    "         batch_size=4, shuffle=True,\n",
    "         num_workers=4, pin_memory=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "         ImageFilelist(root=\"./data/train/\", flist=\"./data/test_new.csv\",\n",
    "             transform=transforms.Compose([transforms.RandomSizedCrop(224),\n",
    "                 transforms.ToTensor(), normalize,\n",
    "         ])),\n",
    "         batch_size=4, shuffle=True,\n",
    "         num_workers=4, pin_memory=True)\n",
    "\n",
    "\n",
    "loaders_transfer = {\n",
    "        \"train\" : train_loader,\n",
    "        \"test\"  : test_loader\n",
    "}\n",
    "\n",
    "print(loaders_transfer['train'].dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2048\n"
     ]
    }
   ],
   "source": [
    "import torchvision.models as models\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.autograd import Variable\n",
    "\n",
    "model_transfer = models.resnet50(pretrained=True)\n",
    "\n",
    "for param in model_transfer.parameters():\n",
    "    param.requires_grad = False\n",
    "    \n",
    "num_ftrs = model_transfer.fc.in_features\n",
    "print(num_ftrs)\n",
    "model_transfer.fc = nn.Linear(num_ftrs, len(unique_labels))\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "# if GPU is available, move the model to GPU\n",
    "if use_cuda:\n",
    "    model_transfer.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def train(n_epochs, loaders, model, optimizer, criterion, use_cuda, save_path):\n",
    "    valid_loss_min = np.Inf \n",
    "    \n",
    "    for epoch in range(1, n_epochs+1):\n",
    "        train_loss = 0.0\n",
    "        valid_loss = 0.0\n",
    "        \n",
    "        model.train() #put model in training mode        \n",
    "        for batch_idx, (data, target) in enumerate(loaders.get('train')):\n",
    "#             print(batch_idx)\n",
    "#             print(batch_idx)\n",
    "\n",
    "#             data = data.unsqueeze(0)\n",
    "#             target = torch.tensor(target)\n",
    "#             target = target.unsqueeze(0)\n",
    "#             print(data)\n",
    "#             print(target)\n",
    "             # move to GPU\n",
    "            if use_cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            ## find the loss and update the model parameters accordingly\n",
    "            # clear the gradients of all optimized variables\n",
    "            optimizer.zero_grad()\n",
    "            # forward pass: compute predicted outputs by passing inputs to the model\n",
    "            output = model(data)\n",
    "            # calculate the batch loss\n",
    "            loss = criterion(output, target)\n",
    "            # backward pass: compute gradient of the loss with respect to model parameters\n",
    "            loss.backward()\n",
    "            # perform a single optimization step (parameter update)\n",
    "            optimizer.step()\n",
    "            # update training loss\n",
    "            ## record the average training loss, using something like\n",
    "            train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.data - train_loss))\n",
    "            \n",
    "        print('validate the model')\n",
    "            \n",
    "        ######################    \n",
    "        # validate the model #\n",
    "        ######################\n",
    "        model.eval() #put model in eval mode\n",
    "        for batch_idx, (data, target) in enumerate(loaders.get('test')):\n",
    "#             data = data.unsqueeze(0)\n",
    "#             target = torch.tensor(target)\n",
    "#             target = target.unsqueeze(0)\n",
    "            # move to GPU\n",
    "            if use_cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            ## update the average validation loss\n",
    "             # forward pass: compute predicted outputs by passing inputs to the model\n",
    "            output = model(data)\n",
    "            # calculate the batch loss\n",
    "            loss = criterion(output, target)\n",
    "            # update average validation loss \n",
    "            valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (loss.data - valid_loss))\n",
    "        # calculate average losses\n",
    "        train_loss = train_loss/train_data_size\n",
    "        valid_loss = valid_loss/valid_data_size\n",
    "        # print training/validation statistics \n",
    "        print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(\n",
    "            epoch, \n",
    "            train_loss,\n",
    "            valid_loss\n",
    "            ))\n",
    "        \n",
    "        ## save the model if validation loss has decreased\n",
    "        if valid_loss <= valid_loss_min:\n",
    "            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
    "            valid_loss_min,\n",
    "            valid_loss))\n",
    "            torch.save(model.state_dict(), save_path)\n",
    "            valid_loss_min = valid_loss    \n",
    "    # return trained model\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    1. I took the orignial pre trained ResNet50 network on the imageNet dataset and changed the final layer to include the classes(5004 classes) rather the classes from the original ImageNet dataset.\n",
    "\n",
    "    2. I think the the pre trained network would be useful because it's layer has learned to extract the fetures from the real workd objects whih includes animals and dogs as well.\n",
    "\n",
    "    3. So it'd easily take out the outlines as well as the curves, shapes and features related to the whale classes and would train on that based on the changes done in the final fully connected layer.\n",
    "\n",
    "    4. We will only train the network for the coneections in the final layer while freezing the previously learned network.\n",
    "\n",
    "    5. Here since the new data set is small and similar to the original training data hence we slice off the end of the neural network and add a new fully connected layer that matches the number of classes(5004) in the new whale data set. We randomize the weights of the new fully connected layer; freeze all the weights from the pre-trained network. Now we train the network to update the weights of the new fully connected layer.\n",
    "\n",
    "    6. To avoid overfitting on the small data set, the weights of the original network will be held constant rather than re-training the weights.\n",
    "\n",
    "    7. Since the data sets are similar, images from each data set will have similar higher level features. Therefore most or all of the pre-trained neural network layers already contain relevant information about the new data set and should be kept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "# learning_rate = 1e-4\n",
    "criterion_transfer = nn.CrossEntropyLoss().cuda() if use_cuda else nn.CrossEntropyLoss()\n",
    "optimizer_transfer = torch.optim.Adam(model_transfer.fc.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validate the model\n",
      "Epoch: 1 \tTraining Loss: 0.000546 \tValidation Loss: 0.002686\n",
      "Validation loss decreased (inf --> 0.002686).  Saving model ...\n",
      "validate the model\n",
      "Epoch: 2 \tTraining Loss: 0.000490 \tValidation Loss: 0.003093\n",
      "validate the model\n",
      "validate the model\n",
      "Epoch: 4 \tTraining Loss: 0.000483 \tValidation Loss: 0.003218\n",
      "validate the model\n",
      "Epoch: 5 \tTraining Loss: 0.000479 \tValidation Loss: 0.003224\n",
      "validate the model\n",
      "Epoch: 6 \tTraining Loss: 0.000473 \tValidation Loss: 0.003276\n",
      "validate the model\n",
      "Epoch: 7 \tTraining Loss: 0.000471 \tValidation Loss: 0.003561\n",
      "validate the model\n",
      "Epoch: 8 \tTraining Loss: 0.000470 \tValidation Loss: 0.003612\n",
      "validate the model\n",
      "Epoch: 9 \tTraining Loss: 0.000463 \tValidation Loss: 0.003558\n",
      "validate the model\n",
      "Epoch: 10 \tTraining Loss: 0.000463 \tValidation Loss: 0.003580\n",
      "validate the model\n",
      "Epoch: 11 \tTraining Loss: 0.000460 \tValidation Loss: 0.003797\n",
      "validate the model\n",
      "Epoch: 12 \tTraining Loss: 0.000455 \tValidation Loss: 0.003745\n",
      "validate the model\n",
      "Epoch: 13 \tTraining Loss: 0.000455 \tValidation Loss: 0.003874\n",
      "validate the model\n",
      "Epoch: 14 \tTraining Loss: 0.000454 \tValidation Loss: 0.003888\n",
      "validate the model\n",
      "Epoch: 15 \tTraining Loss: 0.000455 \tValidation Loss: 0.004044\n",
      "validate the model\n",
      "Epoch: 16 \tTraining Loss: 0.000450 \tValidation Loss: 0.003986\n",
      "validate the model\n",
      "Epoch: 17 \tTraining Loss: 0.000449 \tValidation Loss: 0.004237\n",
      "validate the model\n",
      "Epoch: 18 \tTraining Loss: 0.000449 \tValidation Loss: 0.004342\n",
      "validate the model\n",
      "Epoch: 19 \tTraining Loss: 0.000445 \tValidation Loss: 0.004340\n",
      "validate the model\n",
      "Epoch: 20 \tTraining Loss: 0.000444 \tValidation Loss: 0.004482\n",
      "validate the model\n",
      "Epoch: 21 \tTraining Loss: 0.000444 \tValidation Loss: 0.004499\n",
      "validate the model\n",
      "Epoch: 22 \tTraining Loss: 0.000443 \tValidation Loss: 0.004592\n",
      "validate the model\n",
      "Epoch: 23 \tTraining Loss: 0.000440 \tValidation Loss: 0.004801\n",
      "validate the model\n",
      "Epoch: 24 \tTraining Loss: 0.000436 \tValidation Loss: 0.004774\n",
      "validate the model\n",
      "Epoch: 25 \tTraining Loss: 0.000435 \tValidation Loss: 0.004775\n",
      "validate the model\n",
      "Epoch: 26 \tTraining Loss: 0.000440 \tValidation Loss: 0.004844\n",
      "validate the model\n",
      "Epoch: 27 \tTraining Loss: 0.000434 \tValidation Loss: 0.005065\n",
      "validate the model\n",
      "Epoch: 28 \tTraining Loss: 0.000433 \tValidation Loss: 0.005054\n",
      "validate the model\n",
      "Epoch: 29 \tTraining Loss: 0.000431 \tValidation Loss: 0.005260\n",
      "validate the model\n",
      "Epoch: 30 \tTraining Loss: 0.000434 \tValidation Loss: 0.005235\n",
      "validate the model\n",
      "Epoch: 31 \tTraining Loss: 0.000431 \tValidation Loss: 0.005232\n",
      "validate the model\n",
      "Epoch: 32 \tTraining Loss: 0.000423 \tValidation Loss: 0.005586\n",
      "validate the model\n",
      "Epoch: 33 \tTraining Loss: 0.000428 \tValidation Loss: 0.005486\n",
      "validate the model\n",
      "Epoch: 34 \tTraining Loss: 0.000430 \tValidation Loss: 0.005514\n",
      "validate the model\n",
      "Epoch: 35 \tTraining Loss: 0.000429 \tValidation Loss: 0.005772\n",
      "validate the model\n",
      "Epoch: 36 \tTraining Loss: 0.000426 \tValidation Loss: 0.005745\n",
      "validate the model\n",
      "Epoch: 37 \tTraining Loss: 0.000422 \tValidation Loss: 0.005825\n",
      "validate the model\n",
      "Epoch: 38 \tTraining Loss: 0.000423 \tValidation Loss: 0.005852\n",
      "validate the model\n",
      "Epoch: 39 \tTraining Loss: 0.000423 \tValidation Loss: 0.005977\n",
      "validate the model\n",
      "Epoch: 40 \tTraining Loss: 0.000419 \tValidation Loss: 0.005899\n",
      "validate the model\n",
      "Epoch: 41 \tTraining Loss: 0.000417 \tValidation Loss: 0.006250\n",
      "validate the model\n",
      "Epoch: 42 \tTraining Loss: 0.000425 \tValidation Loss: 0.006428\n",
      "validate the model\n",
      "Epoch: 43 \tTraining Loss: 0.000423 \tValidation Loss: 0.006180\n",
      "validate the model\n",
      "Epoch: 44 \tTraining Loss: 0.000422 \tValidation Loss: 0.006314\n",
      "validate the model\n",
      "Epoch: 45 \tTraining Loss: 0.000416 \tValidation Loss: 0.006429\n",
      "validate the model\n",
      "Epoch: 46 \tTraining Loss: 0.000416 \tValidation Loss: 0.006834\n",
      "validate the model\n",
      "Epoch: 47 \tTraining Loss: 0.000415 \tValidation Loss: 0.006653\n",
      "validate the model\n",
      "Epoch: 48 \tTraining Loss: 0.000417 \tValidation Loss: 0.006576\n",
      "validate the model\n",
      "Epoch: 49 \tTraining Loss: 0.000418 \tValidation Loss: 0.006838\n",
      "validate the model\n",
      "Epoch: 50 \tTraining Loss: 0.000412 \tValidation Loss: 0.006787\n",
      "validate the model\n",
      "Epoch: 51 \tTraining Loss: 0.000418 \tValidation Loss: 0.006841\n",
      "validate the model\n",
      "Epoch: 52 \tTraining Loss: 0.000411 \tValidation Loss: 0.006846\n",
      "validate the model\n",
      "Epoch: 53 \tTraining Loss: 0.000412 \tValidation Loss: 0.006884\n",
      "validate the model\n",
      "Epoch: 54 \tTraining Loss: 0.000409 \tValidation Loss: 0.007031\n",
      "validate the model\n",
      "Epoch: 55 \tTraining Loss: 0.000417 \tValidation Loss: 0.007019\n",
      "validate the model\n",
      "Epoch: 56 \tTraining Loss: 0.000413 \tValidation Loss: 0.007285\n",
      "validate the model\n",
      "Epoch: 57 \tTraining Loss: 0.000404 \tValidation Loss: 0.007509\n",
      "validate the model\n",
      "Epoch: 58 \tTraining Loss: 0.000413 \tValidation Loss: 0.007450\n",
      "validate the model\n",
      "Epoch: 59 \tTraining Loss: 0.000415 \tValidation Loss: 0.007697\n",
      "validate the model\n",
      "Epoch: 60 \tTraining Loss: 0.000412 \tValidation Loss: 0.007578\n",
      "validate the model\n",
      "Epoch: 61 \tTraining Loss: 0.000403 \tValidation Loss: 0.007705\n",
      "validate the model\n",
      "Epoch: 62 \tTraining Loss: 0.000407 \tValidation Loss: 0.007784\n",
      "validate the model\n",
      "Epoch: 63 \tTraining Loss: 0.000405 \tValidation Loss: 0.008039\n",
      "validate the model\n",
      "Epoch: 64 \tTraining Loss: 0.000402 \tValidation Loss: 0.008160\n",
      "validate the model\n",
      "Epoch: 65 \tTraining Loss: 0.000406 \tValidation Loss: 0.008006\n",
      "validate the model\n",
      "Epoch: 66 \tTraining Loss: 0.000403 \tValidation Loss: 0.007979\n",
      "validate the model\n",
      "Epoch: 67 \tTraining Loss: 0.000407 \tValidation Loss: 0.008134\n",
      "validate the model\n",
      "Epoch: 68 \tTraining Loss: 0.000410 \tValidation Loss: 0.008394\n",
      "validate the model\n",
      "Epoch: 69 \tTraining Loss: 0.000408 \tValidation Loss: 0.008108\n",
      "validate the model\n",
      "Epoch: 70 \tTraining Loss: 0.000399 \tValidation Loss: 0.008545\n",
      "validate the model\n",
      "Epoch: 71 \tTraining Loss: 0.000407 \tValidation Loss: 0.008484\n",
      "validate the model\n",
      "Epoch: 72 \tTraining Loss: 0.000405 \tValidation Loss: 0.008169\n",
      "validate the model\n",
      "Epoch: 73 \tTraining Loss: 0.000397 \tValidation Loss: 0.008521\n",
      "validate the model\n",
      "Epoch: 74 \tTraining Loss: 0.000407 \tValidation Loss: 0.008556\n",
      "validate the model\n",
      "Epoch: 75 \tTraining Loss: 0.000400 \tValidation Loss: 0.008804\n",
      "validate the model\n",
      "Epoch: 76 \tTraining Loss: 0.000400 \tValidation Loss: 0.008625\n",
      "validate the model\n",
      "Epoch: 77 \tTraining Loss: 0.000400 \tValidation Loss: 0.008876\n",
      "validate the model\n",
      "Epoch: 78 \tTraining Loss: 0.000398 \tValidation Loss: 0.008809\n",
      "validate the model\n",
      "Epoch: 79 \tTraining Loss: 0.000399 \tValidation Loss: 0.008780\n",
      "validate the model\n",
      "Epoch: 80 \tTraining Loss: 0.000397 \tValidation Loss: 0.009273\n",
      "validate the model\n",
      "Epoch: 81 \tTraining Loss: 0.000396 \tValidation Loss: 0.009231\n",
      "validate the model\n",
      "Epoch: 82 \tTraining Loss: 0.000397 \tValidation Loss: 0.009096\n",
      "validate the model\n",
      "Epoch: 83 \tTraining Loss: 0.000400 \tValidation Loss: 0.009145\n",
      "validate the model\n",
      "Epoch: 84 \tTraining Loss: 0.000396 \tValidation Loss: 0.009427\n",
      "validate the model\n",
      "Epoch: 85 \tTraining Loss: 0.000395 \tValidation Loss: 0.009173\n",
      "validate the model\n",
      "Epoch: 86 \tTraining Loss: 0.000399 \tValidation Loss: 0.009406\n",
      "validate the model\n",
      "Epoch: 87 \tTraining Loss: 0.000396 \tValidation Loss: 0.009598\n",
      "validate the model\n",
      "Epoch: 88 \tTraining Loss: 0.000398 \tValidation Loss: 0.009511\n",
      "validate the model\n",
      "Epoch: 89 \tTraining Loss: 0.000396 \tValidation Loss: 0.009410\n",
      "validate the model\n",
      "Epoch: 90 \tTraining Loss: 0.000395 \tValidation Loss: 0.009709\n",
      "validate the model\n",
      "Epoch: 91 \tTraining Loss: 0.000397 \tValidation Loss: 0.009563\n",
      "validate the model\n",
      "Epoch: 92 \tTraining Loss: 0.000389 \tValidation Loss: 0.009673\n",
      "validate the model\n",
      "Epoch: 93 \tTraining Loss: 0.000394 \tValidation Loss: 0.009837\n",
      "validate the model\n",
      "Epoch: 94 \tTraining Loss: 0.000396 \tValidation Loss: 0.009937\n",
      "validate the model\n",
      "Epoch: 95 \tTraining Loss: 0.000398 \tValidation Loss: 0.010000\n",
      "validate the model\n",
      "Epoch: 96 \tTraining Loss: 0.000395 \tValidation Loss: 0.009818\n",
      "validate the model\n",
      "Epoch: 97 \tTraining Loss: 0.000390 \tValidation Loss: 0.010207\n",
      "validate the model\n",
      "Epoch: 98 \tTraining Loss: 0.000393 \tValidation Loss: 0.010255\n",
      "validate the model\n",
      "Epoch: 99 \tTraining Loss: 0.000393 \tValidation Loss: 0.009923\n",
      "validate the model\n",
      "Epoch: 100 \tTraining Loss: 0.000389 \tValidation Loss: 0.010139\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "model_transfer = train(n_epochs, loaders_transfer, model_transfer, optimizer_transfer, criterion_transfer, use_cuda, 'model_transfer.pt')\n",
    "\n",
    "# load the model that got the best validation accuracy (uncomment the line below)\n",
    "model_transfer.load_state_dict(torch.load('model_transfer.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}