ex3_mnist_cnn_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST Convolutional Neural Networks\n",
    "### Goals\n",
    "* TensorFlow (TF) CNNs\n",
    "* Training / valication curves\n",
    "\n",
    "### Contents\n",
    "* How to use TF operators\n",
    "* How to build a neural network model with TF\n",
    "* How to check the model is overfitted while training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Model parameters\n",
    "# input\n",
    "n_input_width = 28\n",
    "n_input_height = 28\n",
    "n_input_channel = 1\n",
    "\n",
    "# 1st convolutional layer\n",
    "n_conv1_patch_size = 3 \n",
    "n_conv1_filter = 64\n",
    "\n",
    "# 2nd convolutional layer\n",
    "n_conv2_patch_size = 3 \n",
    "n_conv2_filter = 64\n",
    "\n",
    "# fully connected output layer\n",
    "n_output  = 10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# input and output placeholders\n",
    "x = tf.placeholder(tf.float32, [None, n_input_width * n_input_height * n_input_channel])\n",
    "y = tf.placeholder(tf.float32, [None, n_output])\n",
    "\n",
    "# 1st convolutional layer variables\n",
    "wc1 = tf.Variable(tf.random_normal([n_conv1_patch_size, n_conv1_patch_size, n_input_channel, n_conv1_filter], stddev=0.1))\n",
    "bc1 = tf.Variable(tf.random_normal([n_conv1_filter], stddev=0.1))\n",
    "\n",
    "# 2nd convolutional layer variables\n",
    "wc2 = tf.Variable(tf.random_normal([n_conv2_patch_size, n_conv2_patch_size, n_conv1_filter, n_conv2_filter], stddev=0.1))\n",
    "bc2 = tf.Variable(tf.random_normal([n_conv2_filter], stddev=0.1))\n",
    "\n",
    "# fully connected output layer\n",
    "# size reduced by four (2 pooling layers)\n",
    "wf = tf.Variable(tf.random_normal([(n_input_width/4)*(n_input_height/4)*n_conv2_filter, n_output], stddev=0.1))\n",
    "bf = tf.Variable(tf.random_normal([n_output], stddev=0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reshape input: one hot vector to w x w x c tensor\n",
    "input_r = tf.reshape(x, shape=[-1, n_input_width, n_input_width, n_input_channel])\n",
    "\n",
    "# 1st Convolution layer: conv + bias + relu + pooling\n",
    "# TensorFlow operators for CNNs\n",
    "# https://www.tensorflow.org/api_guides/python/nn#Convolution\n",
    "conv1 = tf.nn.conv2d(input_r, wc1, strides=[1, 1, 1, 1], padding='SAME')\n",
    "bias1 = tf.nn.bias_add(conv1, bc1)\n",
    "relu1 = tf.nn.relu(bias1)\n",
    "pool1  = tf.nn.max_pool(relu1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "# 2nd Convolution layer\n",
    "conv2 = tf.nn.conv2d(pool1, wc2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "bias2 = tf.nn.bias_add(conv2, bc2)\n",
    "relu2 = tf.nn.relu(bias2)\n",
    "pool2  = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "# Vectorize\n",
    "dense = tf.reshape(pool2, [-1, wf.get_shape().as_list()[0]])\n",
    "\n",
    "# Fully-connected layer\n",
    "logits = tf.add(tf.matmul(dense, wf), bf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cost function and accuracy\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "corr = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))    \n",
    "accr = tf.reduce_mean(tf.cast(corr, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Optimizer\n",
    "learn_rate   = 0.001\n",
    "optm = tf.train.AdamOptimizer(learn_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Load dataset\n",
    "mnist = input_data.read_data_sets('data/', one_hot=True)\n",
    "trainImg   = mnist.train.images\n",
    "trainLabel = mnist.train.labels\n",
    "testImg    = mnist.test.images\n",
    "testLabel  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize the graph\n",
    "init = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "\n",
    "# Training cycle\n",
    "training_epochs = 20\n",
    "batch_size      = 100\n",
    "\n",
    "# Saver \n",
    "save_step = 10;\n",
    "saver = tf.train.Saver(max_to_keep=training_epochs) \n",
    "save_cost_train = []\n",
    "save_cost_test = []\n",
    "save_accr_train = []\n",
    "save_accr_test = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/020\n",
      " Train cost: 0.254956580 accuracy: 0.923\n",
      " Test  cost: 0.072711943 accuracy: 0.977\n",
      "Epoch: 001/020\n",
      " Train cost: 0.064100152 accuracy: 0.980\n",
      " Test  cost: 0.047569367 accuracy: 0.985\n",
      "Epoch: 002/020\n",
      " Train cost: 0.043949587 accuracy: 0.986\n",
      " Test  cost: 0.041038073 accuracy: 0.987\n",
      "Epoch: 003/020\n",
      " Train cost: 0.033633826 accuracy: 0.990\n",
      " Test  cost: 0.043713845 accuracy: 0.986\n",
      "Epoch: 004/020\n",
      " Train cost: 0.027839697 accuracy: 0.992\n",
      " Test  cost: 0.036438632 accuracy: 0.987\n",
      "Epoch: 005/020\n",
      " Train cost: 0.021209063 accuracy: 0.994\n",
      " Test  cost: 0.032638622 accuracy: 0.989\n",
      "Epoch: 006/020\n",
      " Train cost: 0.017363110 accuracy: 0.995\n",
      " Test  cost: 0.036684049 accuracy: 0.988\n",
      "Epoch: 007/020\n",
      " Train cost: 0.014316047 accuracy: 0.996\n",
      " Test  cost: 0.031956062 accuracy: 0.990\n",
      "Epoch: 008/020\n",
      " Train cost: 0.011357366 accuracy: 0.997\n",
      " Test  cost: 0.037013591 accuracy: 0.989\n",
      "Epoch: 009/020\n",
      " Train cost: 0.009555685 accuracy: 0.997\n",
      " Test  cost: 0.034631296 accuracy: 0.990\n",
      "Epoch: 010/020\n",
      " Train cost: 0.007523726 accuracy: 0.998\n",
      " Test  cost: 0.030733857 accuracy: 0.990\n",
      "Epoch: 011/020\n",
      " Train cost: 0.005631273 accuracy: 0.999\n",
      " Test  cost: 0.036084500 accuracy: 0.990\n",
      "Epoch: 012/020\n",
      " Train cost: 0.004624245 accuracy: 0.999\n",
      " Test  cost: 0.039150852 accuracy: 0.989\n",
      "Epoch: 013/020\n",
      " Train cost: 0.003534207 accuracy: 0.999\n",
      " Test  cost: 0.040463964 accuracy: 0.989\n",
      "Epoch: 014/020\n",
      " Train cost: 0.003276651 accuracy: 0.999\n",
      " Test  cost: 0.039266982 accuracy: 0.990\n",
      "Epoch: 015/020\n",
      " Train cost: 0.002261054 accuracy: 1.000\n",
      " Test  cost: 0.041060310 accuracy: 0.990\n",
      "Epoch: 016/020\n",
      " Train cost: 0.002747197 accuracy: 1.000\n",
      " Test  cost: 0.036859668 accuracy: 0.990\n",
      "Epoch: 017/020\n",
      " Train cost: 0.002162712 accuracy: 1.000\n",
      " Test  cost: 0.041475889 accuracy: 0.989\n",
      "Epoch: 018/020\n",
      " Train cost: 0.000863466 accuracy: 1.000\n",
      " Test  cost: 0.040447767 accuracy: 0.991\n",
      "Epoch: 019/020\n",
      " Train cost: 0.001478301 accuracy: 1.000\n",
      " Test  cost: 0.039657075 accuracy: 0.991\n",
      "Optimization Finished.\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "for epoch in range(training_epochs):    \n",
    "    cost_train_avg = 0.\n",
    "    accr_train_avg = 0.\n",
    "    num_batch = int(mnist.train.num_examples/batch_size)\n",
    "    # Loop over all batches\n",
    "    for i in range(num_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)        \n",
    "        # Fit training using batch data\n",
    "        sess.run(optm, feed_dict={x: batch_xs, y: batch_ys})\n",
    "        # Training cost\n",
    "        cost_train_avg += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "        accr_train_avg += sess.run(accr, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "        \n",
    "    # Display logs per epoch step \n",
    "    save_cost_train.append(cost_train_avg)\n",
    "    save_accr_train.append(accr_train_avg)\n",
    "    print (\"Epoch: %03d/%03d\" % (epoch, training_epochs))\n",
    "    print (\" Train cost: %.9f accuracy: %.3f\" % (cost_train_avg, accr_train_avg))\n",
    "    \n",
    "    # Test\n",
    "    num_batch = int(mnist.test.num_examples/batch_size)\n",
    "    cost_test_avg = 0.\n",
    "    accr_test_avg = 0.\n",
    "    for i in range(num_batch):\n",
    "        batch_xs, batch_ys = mnist.test.next_batch(batch_size)\n",
    "        cost_test_avg += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "        accr_test_avg += sess.run(accr, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "    \n",
    "    save_cost_test.append(cost_test_avg)\n",
    "    save_accr_test.append(accr_test_avg)\n",
    "    print (\" Test  cost: %.9f accuracy: %.3f\" % (cost_test_avg, accr_test_avg))\n",
    "\n",
    "    # Save Net\n",
    "    if epoch % save_step == 0:\n",
    "        saver.save(sess, \"net/cnn_mnist.ckpt-\" + str(epoch))\n",
    "\n",
    "print (\"Optimization Finished.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fe52a6d5750>,\n",
       " <matplotlib.lines.Line2D at 0x7fe52a6d59d0>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEACAYAAABS29YJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl0Y/V99/G3LVle5H0Z27Mwi2emDAnDlrKEoXHZCjxJ\n8/D0PAHaPCGQ0rQwIUmTJqGHBJPShiQkISE5gTY0aQlNe55QcqBP2GEgwwAzwGyF2fB4ZpixPeN9\nk21Zlp4/flquZNm6siVLsj+vc+65v3t1r/SzLH3v1e/+7vcHIiIiIiIiIiIiIiIiIiIiIiIiIiKS\nQlcB+4FDwFfjPP5xYDewE3gLuDSJfUVEJAs4gPeAVUABsAvYELON21I+M7i93X1FRGQe5Cd4/HxM\nwD4CTAD/jjmTtxqxlEuB7iT2FRGReZAo2C8D3rcsHw+ui/U/gX3AU8DtSe4rIiJplijYB2w+z28w\nTTQfAx4B8uZSKRERSS1ngsdPACssyyswZ+jT+V3wOauD2yXct6mpKdDa2mqrsiIiEtYKrE3VkzmD\nT7gKcBH/ImsTkTP5c4Pb290XICCpc9ddd2W6CguK3s/U0XuZWthveQkH5Jn4gM3AM5jeNQ9j2uY/\nG3z8IeBPgE9hLsIOA9cn2FdEROZZomAP5qLrUzHrHrKUvxOc7O4rIiLzLNEFWskxzc3Nma7CgqL3\nM3X0XmZWNvSaCTY/iYiIXXl5eZBEDNeZvYjIIqBgLyKyCCjYi4gsAgr2IiKLgIK9iMgioGAvIrII\n2An2iQYg+TPM4CV7gFeBjZbHjgTX7wS2z6WiIiIye4n6aDqAA8DlmKRoO4AbiE57cBHwLjCAOTC0\nABcGH2sDzgN6Z3gN9bMXEUlSqvvZ2xmA5DVMoAd4A1geWye7lRERkfRI1eAlIZ8BfmtZDgDPA28C\nt0y3k8eToBYiIjInqRq8BOAPgZuJbte/GDgHuBq4Dbgk3o5btybxKiIikrRUDV6yEfgnTJt9n2V9\nR3DeBTyOaRb6XezO3/xmC9u2mXJzc7MSJomIxNiyZQtbtmyZ9f6J2tOdmAu0lwHtmB41sRdoTwNe\nBD4JvG5ZX4K5wDsEuIFngbuDc6vAhz4UYMeOWf4FIiKLULIXaFMxeMk3gCrgp8F1E5gz+AbgPy2v\n8yhTAz0A+/dDXx9UVdmttoiIJCMbesoErrwywK23wsdj+/mIiEhcOZni+NJL4YUXMl0LEZGFK2uC\n/YsvZroWIiILV1Y04/h8AWprYd8+aGjIdHVERLJfTjbjOBzwkY/ASy9luiYiIgtTVgR7gMsuU1OO\niEi6ZE2w10VaEZH0yZpgf8YZJkdOW1umayIisvBkTbDPy1OvHBGRdEn34CWJ9o2iYC8ikh7pHLzE\nzr5gGbykrQ0+/GFobzdn+iIiEl82DV5iZ98oq1dDUZHpby8iIqmTzsFLkt0XUFOOiEg6JMp6OZvB\nSy5Odt+WlpZwubGxmRdfbGbz5iReWURkgUt3PvsLMW3wVwWX7wD8wLdjttuISWd8FabpJpl9owYc\n7+iAD3wAurrMnbUiIjJVqtvs3wTWAasAF3Ad8ETMNqdhAv0niQR6u/tO0dhopl27EldeRETsSRTs\nrYOXvAv8B5HBS0IDmFgHL9mJGc1qpn0T0t20IiKplQ0dHKOacQAefxweegiefjpDNRIRyXLJNuNk\nZbDv64OVK6G7G1yuDNVKRCSL5WSK41hVVbB+PbzxRqZrIiKyMGRlsAelPBYRSaWsDfa6SCsikjpZ\n2WYPMDIC9fVw8iS43RmolYhIFlsQbfZgAvy558Krr2a6JiIiuS9rgz2oKUdEJFWyOtjrIq2ISGqk\nYvCS0zFpjseAL8U8dgQzqIn1zlrbLrgA9u83/e5FRGT2EgV7B/BjTMA/AzP4yIaYbXqAzwH3xdk/\nADQD52Dy2yfF5TKDmbz8crJ7ioiIVSoGL+nCJD2bmOY55tTjR005IiJzl+rBS2IFgOcxB4Nbkqua\noYu0IiJzl8rBS+K5GOgA6oDnMG3/v4vdyDp4SXNzM83NzeHlc84xOe47O6GhYY61ERHJUdkyeAnA\nXcAw8L1pnmu6x+PeVGV17bXwiU/ADTckqK2IyCKRicFLwq8ds1wClAXLbuBKYK/dilmpKUdEZG7s\nHBWuBu7H9Mx5GPgWkYFLHgIagB1AOeasfwjTc2cJZgQrMM1Fjwb3jZXwzP7dd+GjH4XDh23UVkRk\nEVgQ+eynbgBLl8K2bbB69TzVSkQkiy2Y3DhWeXmmKUddMEVEZicngj0o2IuIzEVONOMAtLWZu2nb\n282ZvojIYrYgm3HAtNUXF8O+fZmuiYhI7smZYA/qgikiMls5F+zVbi8ikrxsaP221WYPJmXCGWdA\nVxc4HGmulYhIFluwbfZgcuM0NsLOnZmuiYhIbkn34CWJ9k2aUh6LiCQvnYOX2Nk3abpIKyKSvHQO\nXmJn36R95CMmbYLXO9dnEhFZPNI5eMlcBz6Jq6oKTj8d3nhjrs8kIrJ4pHPwEtv7zjR4STyhppxL\nLplt1UREcks2D15id1/bXS9Dnn0W7rkHXnklqd1ERBaMbBq8JJl9k3LxxfD22zAykopnExFZ+BIF\nex+wGXgGeBf4D2AfZvCS0AAmDZi2+S8CdwLHgNIZ9p0ztxvOPRe2bk3Fs4mILHw5dQet1d13g8cD\n347XoCQissAt6DtordTfXkTEvpw9s/d6obYWjh413TFFRBaTRXNm73KZwUxefjnTNRERyX45G+xB\nTTkiInZlRbBv2dJC/1h/0vspKZqIiD1ZEeyPDhxl3QPruOeVexgcH7S939lnQ0eHmUREZHpZEex/\n/vGf8+rNr3Kg5wBrf7SWe7fey7B3OOF+Dgc0N8NLL6W/jiIiuSwrgj3A+pr1PHLtI7z86ZfZfXI3\nTT9q4r5t9+GZ8My433XXwebNcNttsHfvPFVWRCTHpGLwEoAfBR/fDZxjWX8E2APsBLbbqdCGug38\n6k9+xQufeoE3TrxB04+auP/1+xmdGI27/XXXwZ49UFcHV10FmzbBL38JY2N2Xk1EZHFI1EfTARwA\nLgdOADswg5BY0x5cg0mLcA1wAfBDTBI0gDbgPKB3hteYsZ/97s7dtLzcwvYT27lj0x3ccu4tFDoL\n427r88GTT8KDD5rcOTfeCJ/9LKxbl+CvFBHJManuZ29nAJI/Bv4lWH4DqATqrXWyW5l4zmo4i8ev\ne5wnrn+CZ1qfYd0D63jozYfwTk4dvcTphGuvhWeegddfN236F18MV1wBjz0GE7HDq4iILBKpGLxk\npm0CwPOYDJi3zL6acN7S83jyhif59Sd+zW8O/Ib1D6zn4bcfZmIyfgRvajJ5c95/H266CX74Q1i5\nEr7xDTh2bC41ERHJPakavGS6s/dNQDtQBzyHafv/XexGyQxecv6y83nqz55i2/vbuGvLXfzD1n/g\n02d9mrXVa1lTtYbVVaupK6kL/cShsBD+9E/N9M47ponn7LNN2/5f/iX80R+ZXwAiItksGwYveRDY\ngmniARPQPwKcjHmu2MFNQmaVGyfklaOv8NtDv6Wtv43DfYc53HeYcd84a6rWhKfVlavD5VWVq/B7\ni/n3fzeBv6sL/uIv4GMfgw0bTFOQHT6/j6HxIQbHBxnyDuGZ8HBaxWnUu+vDBxoRkXRJts0+0YZO\nzAXayzBn6NuZ+QLthcD9wXkJ5gLvEOAGngXuDs6t5hTs4xkYG4gK/of7DoeXj/Yfpbq4Ohz8i0bX\ncGj7avbvz6d3ZJDGVYPUrxiisn6QkqpBAq5BBscHw0E9VB7zjVFeWE6Zq4zywnKKnEUcGzjGmG+M\ndTXrWFe9jvU161lfsz5cripWxjZJP5/fh3fSy7hvnPHJccZ94+Tl5eFyuKKmgvyClJ2Y+Pw+RidG\nGfWNhueeCU/UOp/fR01JDXUldSxxL6GquIr8vPT1/g4EAvSP9dM53MnJkZN0DnfSOdzJuG/c/P2O\nAgryC6LKBY6C8HszU7kgvwBnvjNqKnCYdY48x7yc8KU62ANcjQngDuBh4FtEBi55KDj/MebsfwS4\nCXgbWAP8Z/BxJ/BocN9YKQ/2M5n0T9I+1B51MGjrbwOgkHKGe8vo6yin82g5779XzsRwOaevLues\n08s478xyPnxuOWtXlFNSUBL3H9o32seh3kMc6jnEwZ6DHOw9GC67HK4pB4D1NetZW70Wt8s9b++B\nZN64b5z+sX4GxgfoH+uPmgbGLOvG+xkaHwoHbe+kN1yebh1AoaOQQmchLoeLQofpvead9Ian8clx\nfH7flANAaPspBwZHAROTEyaAB4O3tTwZmKTYWUxxQTHFzmJKCkrC5eICs+zIc9A72supkVOcGjnF\nkHeImuIalriXhKfQgWCJewl17rqo9eWF5eTl5THiHYkK3qHp5PBJOkeil4ucRTSUNtBQ2kB9aT31\n7nqKnEVMTE7gnfQy4Z9gwh8sT04tR21nKfv8vqhpYjKybjIwiSPPMeVgYD0grChfwSs3zW1c1XQE\n+3Sb12CfrI4O2LHDTNu3m3lpKfz+75vp/PPhvPOgomLm5wkEApwcORk5CPQc5FCvKbf2tVJTXBO+\n3lBbUktNcQ21JbWmXGIpF9dQUVSR9BmRd9JLt6ebrpEuujxd4XnUOk8XPZ4e3C43jaWN4S9JqNxY\nFlnncrhm9X4GAgF6R3tpH2qnfaidE0MnwmXrNDA+gLvATVlhGaWuUkpdpZS54pdLXaVxt6ssqqS2\npHZeDqQTkxN0DHdwfPA4xwePc2LwBMcHj9M+3E7faN+UoO7z+6gqqqKyqJLKokoqiipMubAyvC40\nlbpKKXIWmUDsLAwH8lBQtq5zOVw48+21RfoD/nAAsx4ErMvWyeVwTRvQXQ5X0mezE5MTdHu6w8G/\ny9MVLscud410MT5pzsh9fl/4c9hQ2kCDuyEqoIfL7nqKC4pn8++ck0AgwGRgctqDgc/vA2B11eo5\nvY6CfZoFAnD4cCTw79gBO3fCsmWmB9Bpp02dli2DgoLpn3PSP8nxweO09bfR7emmx9NDt6fblEd7\noubdnm48Ex6qi6ujDwjBMhAO3NbAPuwdDv+ErnPXmbm1HJzXlNQw4h2hc7iTjuEOMx/qCJ8xdQx1\ncGrkFGWFZdMeEKqLq+n2dE8J4CeGTtAx1EFxQTHLypaxtGxp3GlZ2TLKC8vxTHgY9g4z5B1i2Dts\nyuOWsnea8vgQQ94h+sf66RrpIj8vP/w+WafQgTVqnbuOmuIaChyRf5hnwhMO3ieGTkQCuqXc7emm\n3l3P8vLlLCtfxvIyM19atpTq4uopAbzYWaxrO0kanRjFO+kNn+Evdgr2GeDzwYEDcOSI6dYZO3V0\nmDt84x0IQlN1Ndj9/HonvfSO9kYdGHpGe+ga6QKYEsDr3HVUFlWmrH3UH/DT4+mJOiBYDwo9nh5q\nS2rDgdsayBvLGikpKElJPewIBAJ4JjzhA2VoCv2qiTf1jPbgLnBTXVzNwPgAngkPy8qWmSBevpxl\nZWZuLdeX1ts+oxZJBQX7LOTzmYAf70AQmrxecx/AmjWRqanJzFevhpL5i4+Lnj/gZ2BsgJ7RHiqL\nKqkprtGZpGQdBfscNThohlhsa4PWVtNUdPiwKR85YoZeDAX/2INBQ4P9XwUisjAo2C9Afj+0t0eC\nf+hAEFoeHjZn/01NsHy5GZvXOtXURMolJTowiCwECvaL0NBQ5BdBezt0d0dPPT1m3mWa9KccDGIP\nDFVVUFkZmSoqoKgos3+jiERTsJcZeTzTHwxCB4T+fjMNDETKeXlTDwDW5dh1FRWR5YoK011VvyhE\nUkfBXlIuEDDjA1iDf+zBwLrc12fKoam/3+xfVhb/QBBvubzcNDkVF5sptlxQoIOHLG7pCPZXEbmD\n9mdE58UJ+RHmTlsP8GnMYCV291WwT6EtW7bMmEguU3w+cxE6dFAIHQSmKw8OwuhoZPJ4ost+f/yD\nQKgcWna7I3Nr2c68uBheeSU7389clK2fzVyVbLBP1DHYgUmFYB285Amm5sZZC6zDDF7yU0xuHDv7\nSopl6xfK6TT3ElRXp+b5fL74B4HY8siIKYfmHR2R5djHYudjY+BwbKG8vJmSEsJT6EBiZ7m42PwK\ncbnMPLZsd3kh/IrJ1s/mYpEo2FsHL4HI4CXWgB1v8JIGYLWNfUVmxek0zUJlZel7Db8fvv51+OIX\nTfAPHUBC5djlULmvD06ciCxPTJj7KCYmIlOyy05n9IHA5Youx1tnLTsc5oCRn2/mocm6nKjsdM7u\nQBUqHz8Or70Gk5PmvbXO462LNw8EzN9iZ3I6468vKDCpz10uMw9NLpf5WxeqRME+3sAkF9jYZhmw\n1Ma+IlkrP98EhtrazNYjEDC/ZEIHAOt8urJ1nddrgmUgYKZQ0Iy3PFPZ54s878iIvQOVtXzkiBlT\nIj/fBN3Yebx1sXOIHBwSTT5f/PUTEzA+buo2Ph5ddjqnHgjila0HqZleb7p1jY2wf//8fo7SPXiJ\nHa15eXlNc9hfYtx9992ZrsKCovczddrbs/u99PnMNDKS3tcZHExJ01xrMhsnCvYngBWW5RWYM/SZ\ntlke3KbAxr5g2vtFRCSDnJijxyrABewCNsRscw3w22D5QuD1JPYVEZEscTVmtKr3MMMSghm85LOW\nbX4cfHw3cG6CfUVEREREZKG5CjNA+SHgqxmuy0JwBNiDualte2arknP+GTgJ7LWsqwaeAw5ixk6u\nzEC9clW897MFc91uZ3C6av6rlbNWAC8B7wD/DdweXJ8Tn1EHpnlnFeZirtr0564N88+X5F0CnEN0\ncPoO8JVg+avAvfNdqRwW7/28C/jrzFQn5zUAZwfLpZjm8Q3kyGf0IuBpy/LXgpPMXhtQk+lK5LBV\nRAen/UB9sNwQXBb7VjE12H8pM1VZcH6DyU5g+zOayfvFprsZS2YvADwPvAnckuG6LAT1mKYIgvP6\nGbYVez6H6cjxMFna5JADVmF+Nb1BEp/RTAZ7ZT9LvYsxH4KrgdswP6UlNQLoMztXP8WkUTkb6AC+\nl9nq5KRS4DHg88BQzGMzfkYzGezt3LAlyekIzruAxzG5jWT2TmJ+GgM0AqcyWJeF4BSRgPQz9PlM\nVgEm0D+CacaBJD6jmQz2b2IyZa7C3HR1HSYrpsxOCRBKC+YGriS6vVSS9wRwY7B8I5EvmMxOo6V8\nLfp8JiMP0/T1LiZtfEjOfEZ101XqrMb0aNqF6Zql9zM5vwLaAS/mWtJNmJ5Nz5Pl3dqyVOz7eTPw\nr5iuwbsxQUnXQOzbBPgx329r11V9RkVEREREREREFp94tz3H+hEm5cFuTNe/EKVDEBHJEfFue7ay\npji+gEiKY6VDEBHJEna6Xv4O6Jvh8enGoLWOXztBZAxaERGZZ6noZ5/MGLRKhyAikgGJhiW0a9aj\nKTY1NQVaW5MaSlFERMxIgLaHdU1FsJ/TGLStra0EAko5kiotLS20tLRkuho5LRAwA0739cG997Zw\n000teDxm3cgI4fJ0c2t5dNQMYD05GZnHTtOtDwTA4QCnEwoLweUyU7xyonX5+eb5AgHw++PPZ3rM\n7zfPV1gIRUWRKXY53rrQ8oMPtnDrrS34fDAxYaZQ2e46vx8KCsx7Eppbp9h1M21TUDB9OT9/7gOC\nBwLR/1/rHGDJkrk9f15eXlMy26ci2D8BbMa0yV8I9GN67/QQSYfQjkmHcEMKXk/EFp/PBOy+Pujt\nNVO8crx1TidUVcH4OLz+OpSUgNsdmVvLFRWwdGn0utC8uNg8VyhoOxxTp+nWpyLgZJPaWjjjjEzX\nYv7k5UUOMNnATjV+BXwEqMW0wd+FOWsHeAjTE+cazMXYEcxt5gA+zEHgGUzPnIeBfamquOSu0VHo\n6oqeurujlwcGTLCOd1Y03TzeuspKE7Srq80UKldVwcqVcM450etC86IiU9eWFjOJ5Do7wd7O2fjm\nadY/FZxknjQ3N8/7a46MwKlTZurqiszjTd3d5ud4Xd3UqbYWPvQhU66sND+nrWe+yc4LC83Z8Vxk\n4v1cqPReZlY2/EgMqM0+u3i9JiiHAnjsFAroocnvh/p60wa5ZEn8IG5dLi1dWM0TIpmQZ75Etr9J\n2fCVU7CfBz6fCcwnT0Jnp5nilU+ehKEhE6BDwTveVFcXKbvdCt4i803BfhEaGYFDh+DgQTh2LH4w\n7++HmhpoaDBTff3Ucn29maqr5978ISLppWC/QE1MwJEjJqAfPAgHDkTKPT2wdi2sX28uOjY2Tg3m\ntbWmHVtEFoZ0BPurMCOjODBDiX075vEqTLK0NcAYZpCCd4KPfR748+Dr/BPwwzjPr2AfFAhAe3sk\niFuno0dh2TIT0GOnFSt0Ji6y2KQ62DswI0ldjrl5agemd461C+V3gUHg74DfA34S3P6DmG6bv4/J\njfM08JeYu76sFlWwDwTMxc9DhyJNL6HyoUPm4mW8gN7UZHqXiEhuCgQCjPpGGfYO4530srx8+Zye\nL9lgn6jrpTWZGUSSmVmD/Qbg3mD5AOYmqiXB9W9gzvYBXgb+F+bgsOD190cHcWtQBxPA160z07XX\nRsoVFZmtt4hE8wf89I720jncycnhk5waOcXg+CDD3mFGJkYY9g5PLXunrh/xjlDoLKTUVcra6rW8\n9pnX5vXvSBTs4yU5uyBmm92YIL4Vc3BYGdxvL3APZozEMeB/ANvnXuXsMjkJb70FL70E+/dHgrrH\nEwng69fDlVfCbbeZck3Nwum94g/4aetrY8/JPew+uZs9J/fwXu971LnrWFmx0kyVkfny8uW4HK60\n1cc76aXb003XSBf9Y/3TfgkTfUlHJkYY8Y6Qn5dPkbMoPBU6C6OXHTMvFzmLqCiqoLKokorC4Nyy\nXOoqDZ2hZYVAIIB30suobxTPhAfPhIfRCUvZzvppthnzjTEZmMQf8EdNgUBgyrrpJme+k6riKqqL\nq6kqCs6Lq6guqo5aH7tNZVEljvzIRavYAH5y5GRU2brc7emmrLCMenc9DaUNLHEvoaKwArfLTamr\nlNqSWlZWrKTUVRpeV+oqxV1gKbvcuAvcUXWYb4mCvZ32lXsxbfE7MQF+JzCJGbTk25hBcEeC6/3x\nnsCay6W5uTnrb75oa4PnnoNnn4UXXzS3yl92GXz4w3DjjSagNzamP6B7J7209rayr3sf+7r2sa97\nH95JLyvKV7CiYkXUvL60nvy8uTXsD40PsffUXnZ37g4H972n9lJVVMVZDWexcclGrv/g9ayrXke3\np5ujA0c52n+UF9pe4Gj/UY4OHKVjqIMl7iWRA0DMwWBlxUrcLnfU3xgK3qdGTtHl6Youe4LlEVMe\nGh+itqSWOncdlUWVlLnKIl/AgsiXsc5dN+XLGPslLSkowR/wMz45zphvLDyN+6KXx3xjU7YJbdc/\n1s+xgWP0j/czMDZA/1g/A+PB+dgAY74xygvLpxwEKosqw+WSghIC2A+I8SbvpNd2wHbkOSgpKKG4\noJiSghJTdlrKofXO6G2qiqrC5Xj7FzmLcOY7yc/LD095eXlRy4mmickJ+sb66B3tpW80OLcs7x3c\nG1ke6wtvMzg+SKmrlKriKsZ943R5uigvLKfeXU99qQni9e566t31nF57OvWl9eHHlriXpPUExa4t\nW7awZcuWWe+fKBxdCLRgLtIC3IEJ2LEXaa3agDOB4Zj1/wAcAx6MWZ/1bfYDA+bM/dlnTZAfHITL\nLzdn65dfbi6cptPQ+BD7u/ezr3tfeL6vax9H+o+womIFG2o3sKF2A6fXnk6Rs4j3B9/n/YH3zTxY\nHhgfYGnZUk6rOM0cBOIcEKqLq8nLywufrYfO1EPzzuFOzqg7g7Pqz2Jj/UbOqj+LM+vPpLq42vbf\n4vP7ODF4InwgiJoPHOXYwDHcBW6qiqvo9nRHBe+6kjqWuJdQV1JHnTt+uaq4as4Htfk0MTnB4Pjg\nlIOAddkz4UkqIMZOeeRR6CyMH7DjBHJnfpYkc0khf8DPwNgAfWN9uByurAngc5HqC7ROTDv8ZZhk\nZtuZeoG2AhgFvMAtwMXAp4OPLQFOAadhcuRcgLmYa5V1wd7ng+3bI8F992646CIT3K+4AjZuTH3v\nl0AgwKmRU1Fn6aHA3jvay/qa9Zxee3o4sG+o28Da6rUUOYtsPf+Yb4zjg8cjB4GB9zk2cCzqgOCd\n9LKsfBmdw51UFVWFA/pZDSa4r6tel/afoaH3oX+sn9qS2pwL3iLzJR1dL68m0vXyYeBbwGeDjz0E\nXAT8AtPk89/AZ4CB4OOvADWY3jhfBF6K8/zzGuxDwaS1r5XW3lZa+1pp62tjeDifgY4aOlpraXun\nhiVltVy4sYZLL6rhyk21LKuqpsBRkPgFYvgDfrpGuugY7qB9qJ2OoQ46hjsi8+D6zuFOylxlkYBe\ntyFcXlm5cl4C3tD4ECeGTrDEvSSps3URmX+6qQrTVHBs4Fg4mIfnfa0c7juMy+GiqaqJVRVNjBxv\nYueLa/CMBGg6s4fGpm7KG3rwBHro9nTTM9pDj6eH3tFeSl2l1JTUUFNcQ21JLTUlNdQWm3lFYQU9\noz3hIN4+1E7HcAddI11UFFXQWNpIY1kjS8uWmnJwubHUrGsobaC4oDil74OILFyLLtgf6D7Akwef\njAroxweP01DaQFNVk5mqzXxN1Rqaqpvweyp58EF44AE4+2z48pfh0ktnvqAaavOzHgCs5f6xfmpK\naqYE8frS+pxvGxSR7LNogv3g+CDffPmb/GLXL7jhgzewvmZ9OKivqlxFoXPqHUhHjsAPfgCPPAJ/\n/MfwpS/BmWem4C8QEZlnqb6pKusEAgF+ueeXfO2Fr3Fl05W8c+s71JfWz7jPW2/BffeZC66f+Qzs\n3Zv+HjQiItnETrCfS26cO4BPYrpr7sWMYjU+28ru7NjJ5qc245308tgnHuPC5RdOu20gAE8/Dd/9\nrrnJ6QtfgIcegvLy2b66iEjuSmdunFXAi5i0CePAf2CGMPyXmNdI2IzT4+nhzhfv5PH9j3PPpfdw\n8zk3T9s7xeuFf/s3cybvcMDf/A1cd50Z9UhEZKFIthknUX8+a26cCSK5caw2EOlSGcqNU4c5AEwA\nJZhfECUP6do7AAAKz0lEQVSYA4Ztk/5Jfrrjp2z4yQac+U723baPPz/3z+MG+v5++M53YM0aePRR\n+P73Ydcu+OQnFehFRNKVG2c5Jj3C9zB3zY5ibqp63m7FXj32Kpuf2kx5YTnPf+p5NtZvjLtdby/8\n/d/Dz38O11wD//VfpoeNiIhEpDM3ThPwBcyZ/gDwf4E/Ax6d6ck6hjr4yvNf4aW2l/juFd/l+g9e\nP2OiqO9/37TJ79oFp51mo7YiIotQomB/AlhhWV6BObu3GsJclA1pAw5jslxuA3qC6/8T+DBxgn1L\nSwuT/kleP/462wu2c+v/vpX9m/dT6ipN+Ads3Qp33KFALyILW7oToc0lN87ZwC8xg5eMYVIqbMdc\nwLUKPPPeM9z+1O2sqVrD/Vfdz/qa9bYq7/Wa8VJPnFAeeBFZXFLdz94HbMa0t4dy4+wjOjfOGUzN\njQOwC/hX4E1M18u3gX+M9yJ/9f/+ivv/6H4+uv6jSeX23rnTjOCkQC8iMrOsuIN2dGLUdvZGq+9/\nH1pb4SexvxVERBa4VHe9nBezCfQAr74KF1+c4sqIiCxAWXFmP5vcOIGAGQ3qjTdg5co01EpEJIvl\n5Jn9bBw+DE6neuGIiNiRs8F+61bThJNFYzWLiGQtO8H+Kszg4YeAr8Z5vAp4HHMn7RvAB4Lrfw9z\ng1VoGgBun2N9w9ReLyJiXzoToVnlB/c/n+j0CzDLNvsPfMDkpT/33KR3FRHJedmUCM3qcqCVqYF+\nVnp74f33zcDfIiKSWKJgHy8RWuywH6FEaBCdCM3qeuDfZlnHKV57Dc4/31ygFRGRxNKZCC3EBXyM\n+O39gMmNE9Lc3Exzc/OML6j2ehFZbNKdG+dCoAVzkRbMyFN+po5WZdUGnAkMB5c/DvyV5TliJd1m\n/wd/AHfeCVdemdRuIiILRqrb7N8E1mHa4V3AdcATMdtUBB8DkwjtZSKBHswF3V/ZrVAiXi+8/TZc\nOP2IhCIiEiOdidAA3JiLs7ekqsJvvw3r1mksWRGRZGTDLUlJNeN873vQ1gY//nEaayQikuUWfLoE\nXZwVEUleTp3ZBwLQ0AA7dignjogsbgv6zP6998DlUqAXEUlWTgV7NeGIiMxOOhOhAVQCv8b04HkX\n029/1l59FTZtmssziIgsTomCvQP4MSbgn4HpM78hZpu/xYwvexbwKczdtCE/BH4b3Gcj0QnUkqYz\nexGR2UlnIrQK4BLgn4OP+TBpjmeltxdOnIAzz5ztM4iILF7pTIS2GugCfo458/8noGS2Fd22TcnP\nRERmK52J0FzAuZg7cHcA9wNfA74R+wR2EqGFRqYSEVmMsjkRWinwGuYMH2ATJth/NGZ7W/3sL7kE\nvvENuOKKhJuKiCx42ZQIrRPTBLQ++NjlwDt2K2Y1Pg47dyr5mYjIbKU7EdrngEcxB4NW4KbZVPLt\nt2H9eigrm83eIiKSE+kS7rsPjh6FBx6YpxqJiGS5BZkuQRdnRUTmJuvP7AMBqK+Ht96CFSvmsVYi\nIllswZ3ZHzoERUUK9CIic5Hu3DhHgD2YvvfbZ1NB5cMREZm7RL1xQrlxLgdOYG6OeoLoHDeh3DjX\nAr8H/CS4PZgeOs1A72wrqHw4IiJzl87cOCFzui6gYC8iMnfpzI0D5sz+eczNWUkPOt7dDe3tSn4m\nIjJX6cyNAyZFQjvmTP85TNv/7+xWbts2uOACcDjs7iEiIvEkCvYnAGs/mBWYs3urIeBmy3IbcDhY\nbg/OuzAXcc8nTrCfLhGamnBERIx0J0JzYtrhL8ME7u2YAUysF2grgFHAi2mquRj4NCadsQNzMHAD\nzwJ3B+dW0/az37QJ7r4bLrvM9t8jIrIoJNvPPp25ceoxZ/Oh13mUqYF+WuPjsGuXacYREZG5ydo7\naLdtg899ztw5KyIi0RbMHbTKhyMikjpZG+x1cVZEJHWyshknEIAlS8yAJcuXT7OXiMgitiCacQ4e\nBLdbgV5EJFXSnQgNTC+encCTdiulJhwRkdRKFOxDidCuwnSxvAGTC8cqlAjtLOBTmLtprT4PvIu9\nu3EBBXsRkVRLdyK05cA1wM9Iom1JPXFERFIr3YnQfgD8DeC3W6GuLujshA9+0O4eIiKSSLoSofmB\njwKngsvNMz2BNTeO09nMRRc1K/mZiIhFunPjXAi0YNrsAe7ABPJvz7BPG7AxuO3/waRcKALKgccw\n7fpWUV0vv/IVKCuDr3/d3h8gIrIYpbrr5ZvAOkw7vAu4DjNSlVVF8DEwidBexiQ/+1tMlszVwPXA\ni0wN9FPo4qyISOqlMxFarIRNQmNjSn4mIpIOWXUH7dat8IUvwJtvZrhGIiJZLqfvoFUTjohIemRd\nsN+0KdO1EBFZeLKmGScQgLo62LMHli7NdJVERLJbzjbjHDhgulwq0IuIpF46E6EVBZd3YXLjfGum\nF1GKBBGR9ElnIrQx4A+BszE3Wf0hMG2LvC7OioikT7oToXmCcxfmwNE73Qsp2IuIpE+6E6E5MM04\nJzEHhHfjvcipU2b6QGwmfBERSYl0JUKbDD42iWnGqcDchdsMbIl9gttvb6G2Fv7u76C5uZnm5mY7\ndRcRWTSyNRHamcBwzPqvA6PAfTHrA1/+coCKCrjzTjtVFhGRbEmENgzUApXB9cXAFZiz/inUE0dE\nJL3SmQitEfgXzAElH3gEeCHei+zZA+efP9s/QUREEsmKO2g/9KEAO3ZkuhoiIrkjJ++gVT4cEZH0\nyopgr/Z6EZH0yopmnPb2AI2Nma6GiEjuSFczzmzz46zA3Ez1Dubi7e3xnlyBXkQkvewE+7nkx5kA\nvogJ/hcCt8XZV1JoLjddyFR6P1NH72Vm2Qn2c8mP04lJlwCm7/0+QEmM00hfqNTS+5k6ei8zy06w\nn2t+nJBVwDmYZh4REZlHdoK93fw4lZg7ZDcTnR8HoBT4NfB5pqZREBGRNLNzJXeu+XEKgP8CngLu\nj7Pte0CTveqKiEhQK7A2lU/oDD7pKkwOnF1Mvcgamx/nF8FyHvCvwA9SWSEREUmPqzEXXt/DnNmD\nyY8TypFzUfDx/Zjmmorg+k2YXwG7ME07O4n8QhARERERkYUi0c1akpwjwB7ML6jtma1KzvlnzIhq\ney3rqoHngIPAs0RSdkti8d7PFkxvPv3KT950N6jmxGfUgWkWWoW5iBvvWoAkpw3zz5fkXYLpGmwN\nTt8BvhIsfxXT60zsifd+3gX8dWaqk/MaMKP+gendeAATL3PiM3oR8LRl+WvBSWavDajJdCVy2Cqi\ng9N+oD5Ybggui32rmBrsv5SZqiw4vwEuJ4nPaCazXtq5WUuSEwCex4wwdkuG67IQ1GOaIgjO62fY\nVuz5HOYmzIfJ0iaHHLCKyA2qtj+jmQz2dm7WkuRcjPkQXI3JQ3RJZquzoATQZ3aufgqsxjRHdADf\ny2x1clIp8BjmBtWhmMdm/IxmMtifwFx0CFmBObuX2esIzrswWUg12OPcnMT8NAYzzOapDNZlIThF\nJCD9DH0+k1WACfSPYJpxIInPaCaDvZ3BzMW+EqAsWHYDVxLdXirJewK4MVi+kcgXTGbHmsz8WvT5\nTEYepunrXaIzEeTMZzTezVoyO6sxPZp2Ybpm6f1Mzq+AdsCLuZZ0E6Zn0/Nkebe2LBX7ft6MuZt+\nD6bN/jfoGkgyprtBVZ9REREREREREREREREREREREREREREREREREZF4/j8qUk5oN/MfkgAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe59851aad0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Traininig curve\n",
    "epochs = range(training_epochs)\n",
    "plt.subplot(211)\n",
    "plt.plot(epochs, save_cost_train, epochs, save_cost_test)\n",
    "plt.subplot(212)\n",
    "plt.plot(epochs, save_accr_train, epochs, save_accr_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmin(save_cost_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}