ex7_cnn_customdata_vgg_finetune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training VGG model on your custom data \n",
    "### Goals\n",
    "* Learn how to use a pre-trained model and fine-tune the model with your custom dataset\n",
    "\n",
    "### Contents\n",
    "* Reading the parameters of the pre-trained VGG model\n",
    "* Extracting deep features for the given images\n",
    "* Training fully connected layers with the custom dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python2.7/dist-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import scipy.misc\n",
    "import scipy.io\n",
    "\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['trainlabel', 'trainimg', 'testimg', 'testlabel']\n",
      "69 train images loaded\n",
      "18 test images loaded\n",
      "37632 dimensional input\n",
      "2 classes\n"
     ]
    }
   ],
   "source": [
    "# Load your custom data\n",
    "loadpath = \"data/data4vgg.npz\"\n",
    "l = np.load(loadpath)\n",
    "\n",
    "# Show Files\n",
    "print (l.files)\n",
    "\n",
    "# Parse data\n",
    "trainimg   = l['trainimg']\n",
    "trainlabel = l['trainlabel']\n",
    "testimg    = l['testimg']\n",
    "testlabel  = l['testlabel']\n",
    "ntrain     = trainimg.shape[0]\n",
    "nclass     = trainlabel.shape[1]\n",
    "dim        = trainimg.shape[1]\n",
    "ntest      = testimg.shape[0]\n",
    "\n",
    "print (\"%d train images loaded\" % (ntrain))\n",
    "print (\"%d test images loaded\"  % (ntest))\n",
    "print (\"%d dimensional input\"   % (dim))\n",
    "print (\"%d classes\"             % (nclass))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of trainimg_tensor is (69, 112, 112, 3)\n",
      "shape of trainimg_tensor is (18, 112, 112, 3)\n"
     ]
    }
   ],
   "source": [
    "# Reshape data\n",
    "trainimg_tensor = np.ndarray((ntrain, 112, 112, 3))\n",
    "for i in range(ntrain):\n",
    "    currimg = trainimg[i, :]\n",
    "    currimg = np.reshape(currimg, [112, 112, 3])\n",
    "    trainimg_tensor[i, :, :, :] = currimg \n",
    "print (\"shape of trainimg_tensor is %s\" % (trainimg_tensor.shape,))\n",
    "    \n",
    "testimg_tensor = np.ndarray((ntest, 112, 112, 3))\n",
    "for i in range(ntest):\n",
    "    currimg = testimg[i, :]\n",
    "    currimg = np.reshape(currimg, [112, 112, 3])\n",
    "    testimg_tensor[i, :, :, :] = currimg \n",
    "print (\"shape of trainimg_tensor is %s\" % (testimg_tensor.shape,))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define VGG-19 model \n",
    "# Paper: \"Very Deep Convolutional Networks for Large-Scale Image Recognition\" by Simonyan and Zisserman\n",
    "# https://arxiv.org/pdf/1409.1556.pdf\n",
    "\n",
    "def net(data_path, input_image):\n",
    "    layers = (\n",
    "        'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1',\n",
    "\n",
    "        'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',\n",
    "\n",
    "        'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', \n",
    "        'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4', 'pool3',\n",
    "\n",
    "        'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', \n",
    "        'conv4_3', 'relu4_3', 'conv4_4', 'relu4_4', 'pool4',\n",
    "\n",
    "        'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', \n",
    "        'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4'\n",
    "    )\n",
    "\n",
    "    # Load mat file and save weights and mean values\n",
    "    data = scipy.io.loadmat(data_path)\n",
    "    mean = data['normalization'][0][0][0]\n",
    "    mean_pixel = np.mean(mean, axis=(0, 1))\n",
    "    weights = data['layers'][0]\n",
    "\n",
    "    net = {}\n",
    "    current = input_image\n",
    "    for i, name in enumerate(layers):\n",
    "        kind = name[:4]\n",
    "        if kind == 'conv':\n",
    "            kernels, bias = weights[i][0][0][0][0]\n",
    "            # matconvnet: weights are [width, height, in_channels, out_channels]\n",
    "            # tensorflow: weights are [height, width, in_channels, out_channels]\n",
    "            kernels = np.transpose(kernels, (1, 0, 2, 3))\n",
    "            bias = bias.reshape(-1)\n",
    "            current = _conv_layer(current, kernels, bias)\n",
    "        elif kind == 'relu':\n",
    "            current = tf.nn.relu(current)\n",
    "        elif kind == 'pool':\n",
    "            current = _pool_layer(current)\n",
    "        net[name] = current\n",
    "\n",
    "    assert len(net) == len(layers)\n",
    "    return net, mean_pixel\n",
    "\n",
    "def _conv_layer(input, weights, bias):\n",
    "    conv = tf.nn.conv2d(input, tf.constant(weights), strides=(1, 1, 1, 1),\n",
    "            padding='SAME')\n",
    "    return tf.nn.bias_add(conv, bias)\n",
    "def _pool_layer(input):\n",
    "    return tf.nn.max_pool(input, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1),\n",
    "            padding='SAME')\n",
    "def preprocess(image, mean_pixel):\n",
    "    return image - mean_pixel\n",
    "def unprocess(image, mean_pixel):\n",
    "    return image + mean_pixel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Extracting deep features with the pre-trained VGG weights\n",
    "VGG_PATH = \"data/imagenet-vgg-verydeep-19.mat\"\n",
    "with tf.Graph().as_default(), tf.Session() as sess:\n",
    "    img_placeholder = tf.placeholder(tf.float32, shape=(None, 112, 112, 3))\n",
    "    net, mean_pixel = net(VGG_PATH, img_placeholder)\n",
    "    train_features = net['relu5_4'].eval(feed_dict={img_placeholder: trainimg_tensor})\n",
    "    test_features = net['relu5_4'].eval(feed_dict={img_placeholder: testimg_tensor})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(69, 7, 7, 512)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Your Exercise\n",
    "* Build the rest fully connected layers of the VGG-19 model, which\n",
    "   * takes the 4-d Tensor (n_data, height, width, depth),\n",
    "   * vectorizes to 2-d Tensor (n_data, height &ast; width &ast; depth),\n",
    "   * has two fully connected layers with weight and bias variables, and\n",
    "   * outputs logits with 2-d Tensor (n_data, 10)\n",
    "   \n",
    "   \n",
    "* Define \n",
    "   * a cost function with softmax_cross_entropy_with_logits,\n",
    "   * an optimizer, and\n",
    "   * an accuracy measure \n",
    "   \n",
    "   \n",
    "* Train your model with the extracted train_features and labels\n",
    "* Test your model with the extracted test_features while training\n",
    "* Save the trained model\n",
    "* Write a code to predict cat/dog given images by using your trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}