ex2_mnist_lr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST Logistic Regression\n",
    "### Goals\n",
    "* TensorFlow (TF) basics\n",
    "\n",
    "### Contents\n",
    "* How to use TF operators\n",
    "* How to build a neural network model with TF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(2), Dimension(2)])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What is a tensor?\n",
    "# https://www.tensorflow.org/api_docs/python/tf/Tensor\n",
    "c = tf.constant([[1.0, 2.0], [3.0, 4.0]])\n",
    "c?\n",
    "c.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# What is tf operators?\n",
    "# https://www.tensorflow.org/api_guides/python/math_ops\n",
    "d = tf.constant([[1.0, 1.0], [0.0, 1.0]])\n",
    "e = tf.matmul(c, d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.  3.]\n",
      " [ 3.  7.]]\n"
     ]
    }
   ],
   "source": [
    "# Evaluation\n",
    "sess = tf.Session()\n",
    "result = sess.run(e)    # result is ndarray\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a static graph for logistic regression\n",
    "# Input and output as tf.placeholder,\n",
    "# and weight variables as tf.Variable\n",
    "x = tf.placeholder(\"float\", [None, 784])\n",
    "y = tf.placeholder(\"float\", [None, 10])\n",
    "\n",
    "# You can initialize the tf variable with other tf tensors\n",
    "W = tf.Variable(tf.zeros([784, 10]))  \n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "logits = tf.matmul(x, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cost function\n",
    "# https://www.tensorflow.org/api_guides/python/nn#Losses\n",
    "cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = logits, labels = y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Optimizer\n",
    "# https://www.tensorflow.org/api_guides/python/train#Optimizers\n",
    "learning_rate = 0.01\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a static graph for logistic regression\n",
    "# Input and output as tf.placeholder,\n",
    "# and weight variables as tf.Variable\n",
    "x = tf.placeholder(\"float\", [None, 784])\n",
    "y = tf.placeholder(\"float\", [None, 10])\n",
    "\n",
    "# You can initialize the tf variable with other tf tensors\n",
    "W = tf.Variable(tf.zeros([784, 10]))  \n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "logits = tf.matmul(x, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cost function\n",
    "# https://www.tensorflow.org/api_guides/python/nn#Losses\n",
    "cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = logits, labels = y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Optimizer\n",
    "# https://www.tensorflow.org/api_guides/python/train#Optimizers\n",
    "learn_rate = 0.01\n",
    "optm = tf.train.GradientDescentOptimizer(learn_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Accruracy evaluation\n",
    "corr = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))    \n",
    "accr = tf.reduce_mean(tf.cast(corr, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Load dataset\n",
    "mnist = input_data.read_data_sets('data/', one_hot=True)\n",
    "trainImg   = mnist.train.images\n",
    "trainLabel = mnist.train.labels\n",
    "testImg    = mnist.test.images\n",
    "testLabel  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize the graph\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Training cycle\n",
    "training_epochs = 50\n",
    "batch_size      = 100\n",
    "\n",
    "# save \n",
    "save_cost = []\n",
    "save_accr = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/050 cost: 0.365334638 train_acc: 0.726\n",
      "Epoch: 001/050 cost: 0.275513337 train_acc: 0.760\n",
      "Epoch: 002/050 cost: 0.245153714 train_acc: 0.780\n",
      "Epoch: 003/050 cost: 0.223635283 train_acc: 0.796\n",
      "Epoch: 004/050 cost: 0.207221926 train_acc: 0.804\n",
      "Epoch: 005/050 cost: 0.194422008 train_acc: 0.814\n",
      "Epoch: 006/050 cost: 0.184142347 train_acc: 0.822\n",
      "Epoch: 007/050 cost: 0.175629972 train_acc: 0.827\n",
      "Epoch: 008/050 cost: 0.169683297 train_acc: 0.825\n",
      "Epoch: 009/050 cost: 0.163862658 train_acc: 0.831\n",
      "Epoch: 010/050 cost: 0.159016688 train_acc: 0.834\n",
      "Epoch: 011/050 cost: 0.154744371 train_acc: 0.835\n",
      "Epoch: 012/050 cost: 0.151101166 train_acc: 0.837\n",
      "Epoch: 013/050 cost: 0.147892638 train_acc: 0.842\n",
      "Epoch: 014/050 cost: 0.144450042 train_acc: 0.843\n",
      "Epoch: 015/050 cost: 0.141542632 train_acc: 0.845\n",
      "Epoch: 016/050 cost: 0.139417501 train_acc: 0.845\n",
      "Epoch: 017/050 cost: 0.136743886 train_acc: 0.846\n",
      "Epoch: 018/050 cost: 0.134629381 train_acc: 0.849\n",
      "Epoch: 019/050 cost: 0.132584783 train_acc: 0.853\n",
      "Epoch: 020/050 cost: 0.130676738 train_acc: 0.852\n",
      "Epoch: 021/050 cost: 0.129064180 train_acc: 0.855\n",
      "Epoch: 022/050 cost: 0.127027679 train_acc: 0.857\n",
      "Epoch: 023/050 cost: 0.126548907 train_acc: 0.856\n",
      "Epoch: 024/050 cost: 0.124979931 train_acc: 0.856\n",
      "Epoch: 025/050 cost: 0.123698823 train_acc: 0.856\n",
      "Epoch: 026/050 cost: 0.123136672 train_acc: 0.856\n",
      "Epoch: 027/050 cost: 0.121811311 train_acc: 0.857\n",
      "Epoch: 028/050 cost: 0.120347084 train_acc: 0.858\n",
      "Epoch: 029/050 cost: 0.119222399 train_acc: 0.862\n",
      "Epoch: 030/050 cost: 0.118442496 train_acc: 0.861\n",
      "Epoch: 031/050 cost: 0.116488903 train_acc: 0.864\n",
      "Epoch: 032/050 cost: 0.116252425 train_acc: 0.863\n",
      "Epoch: 033/050 cost: 0.115235569 train_acc: 0.865\n",
      "Epoch: 034/050 cost: 0.113997772 train_acc: 0.867\n",
      "Epoch: 035/050 cost: 0.113078813 train_acc: 0.865\n",
      "Epoch: 036/050 cost: 0.112846801 train_acc: 0.867\n",
      "Epoch: 037/050 cost: 0.112126685 train_acc: 0.866\n",
      "Epoch: 038/050 cost: 0.112212507 train_acc: 0.866\n",
      "Epoch: 039/050 cost: 0.111309893 train_acc: 0.868\n",
      "Epoch: 040/050 cost: 0.110588047 train_acc: 0.867\n",
      "Epoch: 041/050 cost: 0.110132159 train_acc: 0.867\n",
      "Epoch: 042/050 cost: 0.108858201 train_acc: 0.870\n",
      "Epoch: 043/050 cost: 0.109161790 train_acc: 0.870\n",
      "Epoch: 044/050 cost: 0.108314116 train_acc: 0.869\n",
      "Epoch: 045/050 cost: 0.107370763 train_acc: 0.870\n",
      "Epoch: 046/050 cost: 0.107722828 train_acc: 0.869\n",
      "Epoch: 047/050 cost: 0.106445233 train_acc: 0.871\n",
      "Epoch: 048/050 cost: 0.106244062 train_acc: 0.871\n",
      "Epoch: 049/050 cost: 0.105300859 train_acc: 0.872\n",
      "Optimization Finished!\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "with tf.Session() as sess:   \n",
    "    sess.run(init)\n",
    "    for epoch in range(training_epochs):\n",
    "        avg_cost = 0.\n",
    "        avg_train_acc = 0.\n",
    "        num_batch = int(mnist.train.num_examples/batch_size)\n",
    "        # Loop over all batches\n",
    "        for i in range(num_batch): \n",
    "            if 0: # Using tensorflow API\n",
    "                batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            else: # Random batch sampling \n",
    "                randidx = np.random.randint(trainImg.shape[0], size=batch_size)\n",
    "                batch_xs = trainImg[randidx, :]\n",
    "                batch_ys = trainLabel[randidx, :]                \n",
    "\n",
    "            # Fit training using batch data\n",
    "            sess.run(optm, feed_dict={x: batch_xs, y: batch_ys})\n",
    "            \n",
    "            # Compute average loss\n",
    "            avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "            avg_train_acc += sess.run(accr, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "        \n",
    "        save_cost.append(avg_cost)\n",
    "        save_accr.append(avg_train_acc)\n",
    "        # Display logs per epoch step\n",
    "        print (\"Epoch: %03d/%03d cost: %.9f train_acc: %.3f\" \n",
    "                   % (epoch, training_epochs, avg_cost, avg_train_acc))\n",
    "\n",
    "print (\"Optimization Finished!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fc9ec84c7d0>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEACAYAAABS29YJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUVOWd//F3bwjdLLI3CAhBEERUFhFwazWKCG5JJpqf\nMYnJaGaSuIRolDgjrZlIzKY/Y2aOEsQlmhgdNYAi4AKiIosCsio0KFvTLM3WgHTTVfPH997c20V1\nV1V3VVd138/rnOfcquq6VZfncL731vd57vcBERERERERERERERERERERERERERFJosuBdcB64O46\n3nc2cAz4ej32FRGRNMoBNgC9gTxgOTCwlve9DczEC/bx7isiIimWHePvI7CA/TlQBfwNuDrK+24F\nXgJ21WNfERFJsVjB/iRgi+/5Vue1yPdcDfyP8zycwL4iItIIYgX7cIy/AzwC3OO8N8tp8e4rIiKN\nIDfG37cBPX3Pe2JX6H7DsBQNQCdgLJa2iWdf+vbtGy4pKUngkEVEBCgBTknWh+U6H9gbaEHsQdZp\nwNcS3DcsZtKkSek+hIyhvvCoLzzqCw8JZk9iXdkfA34CzMZm10wF1gI/dP7+eD32FRGRRhYr2APM\ncppfbUH+pjj2FRGRRhZrgFYaUVFRUboPIWOoLzzqC4/6ov6yYr8l5Zz0k4iIxCsrKwsSiOG6shcR\nCQAFexGRAFCwFxEJAAV7EZEAULAXEQkABXsRkQCIJ9jHWoDkamAFsAz4CLjY97fPgU+cvy2u7QtK\nS+M7WBERqZ9YczRzgE+Br2KFzZYA36Jm2YMC4JDzeDDwCl5xnk1YobTyOr4j/PLLYa69NrEDFxEJ\nsmTPs49nAZJDvsetgd2RxxTrIBYujPUOERFpiGQsXgJwDXa1Pwu4zfd6GHgTWArcXNuXfPhhPIcq\nIiL1FasQWrx1DF512vnAs8CpzuvnAqVAZ2AulvtfELnzxx9DVRXk5cX5bSIikpBkLF7it8D5zI7A\nHizQg61N+wqWFjou2OfnF/PjH0P37lboSMWORERqmjdvHvPmzav3/rHy6bnYAO0lwHZsRk3kAG1f\nYCP2K2Ao8KLzWj42wHsQG8SdA9zvbP3CP/hBmCFD4Mc/rve/Q0QkUJI9QOtfgGQN8ALe4iXuAiZf\nB1Zi0yv/P3C983ohdhW/HFgEzOT4QA/AqFEapBURSaWMKHG8alWYq6+GDRvSfSgiIk1Dolf2GRHs\nq6vDdOgA69dD587pPhwRkczXJOvZZ2fDiBGagikikioZEewBRo5UsBcRSZWMCfYapBURSZ2MyNmH\nw2HKy6F3b9i7F3Jy0n1IIiKZrUnm7AE6dLCbqlavTveRiIg0PxkT7MHy9krliIgkX8YFew3Siogk\nX6oXL4m1bw0apBURSY1ULl4Sz77gDNACHDsG7dvDF19YDl9ERKLLpMVL4tm3htxcOPtsWFzrAoYi\nIlIfqVy8JN59a9AgrYhI8qVq8ZIBiRxEcXHxPx+3bl3E/PlFiewuItLspbqe/UigGBtoBZgIhICH\n6tinBEvh9Itz33/m7AF27oT+/aG83GrmiIjI8ZKds1+KBe3eQAvgOmB6xHv6+r5wqLPdE+e+x+nS\nBTp2hHXrYh+8iIjEJ1Yax794SQ4wFW/xEoDHscVLvoMNwlbgLV5S274xjRpl8+1POy3uf4eIiNQh\nY2rj+D32GKxYAVOmpOmIREQyXJOtjePnXtmLiEhyZOSVfVWV3Vy1bRu0a5emoxIRyWDN4so+Lw+G\nDIElS9J9JCIizUNGBnuA0aNh9ux0H4WISPOQkWkcgC1bYOhQmD9fs3JERCI1izQOQM+e8Mtfwve/\nD9XV6T4aEZGmLWODPcAtt0DLlvDoo+k+EhGRpi1j0ziuDRusONqiRdC3byMelYhIBktFGifWAiQ3\nYIuXfAK8D5zh+9vnzuvLgHoVLj7lFJg4EW6+Geo4J4iISB2SsXjJKGANsB87MRRjBdQANgHDgPI6\nvqPOK3uwnP3o0fCDH1hqR0Qk6NKxeMlCLNADLAJ6RB5TvAdTm5wcePJJuPde2Lq1oZ8mIhI8yVq8\nxPUD4HXf8zDwJlYB8+b6HKBr0CC49Vb4t39TOkdEJFHJWrwE4CLg+8C5vtfOBUqBzsBcLPe/IHJH\n/+IlRUVFFBUVRf2Ce+6B4cPh+efhhhsSODIRkSYuUxYvOQN42Xnfhlo+axJWAvn3Ea/HzNn7LV0K\n48bBsmXQvXvcu4mINCvpWLykFxbov03NQJ8PtHEeFwCXASvjPbDaDB8OEybAmDGwd29DP01EJBiS\nsXjJfUB74H+c16qwgd1C7CTgfs9zwJxkHPTPfw5lZTB+PMydC/n5yfhUEZHmK+NvqqpNKAQ33QS7\ndsE//mGVMkVEgiLRNE6TDfZgde+/9jWref/MM1qgXESCo9kUQotHXh78/e+weTPccYemZIqI1KZJ\nB3uAVq1g+nR49134r/9K99GIiGSmWAO0TcKJJ8Ibb8B550HHjvCjH6X7iEREMkuzCPYAhYUwZw5c\nfDHs2AHFxcrhi4i4mvQAbTRlZTZo260bPP00FBQk7aNFRDJGoAZoo+naFd5+G1q3trTOli2x9xER\nae6aXbAHOOEEmDbN6ueccw4sXJjuIxIRSa9UL14Sa9+UycqCO++EJ56Aq66CZ59tzG8XEcksqVy8\nJJ59Ick5+2hWr4Yrr7T24IPK44tI05dJi5fEs2+jGDQIliyB3bvhjDMspy8iEiSpXLwk0X1TqmNH\neO45ePRR+N734F//FfbtS9fRiIg0rlQuXhL3vvEuXpIM48bBqlW2iPmgQfDYY3DttSn7OhGRpMjk\nxUvi3TflOfvaLFhgV/iDB8PDD0PPnmk5DBGRhGXS4iXx7JtW558PK1bAaafBWWdZMbWysnQflYhI\n8sUK9v7FS9YAL+AtXuIuYOJfvGQZsDjGvhmlZUt44AGbsRMOW+D/xS+0CpaINC/NrlxCQ23eDL/8\nJbz6Ktx+u7U2bWLvJyLSmAJfLqGhevWCKVPggw9g7Vo45RT4j/+AbdvSfWQiIvWnYF+Lfv1squaC\nBXDggA3iXn+9nQQy6IeIiEhclMaJ04EDVm/nj3+0+vm33QbXXWd1eEREGlug1qBNh1AIZs2ym7MW\nL4YxY6wMw9ix0KFDuo9ORIJCwb4RlZbCa6/BjBnwzjswZAiMH2/B/9RTrRibiEgqKNinyZEjVnNn\nxgyYOdOmdI4bZ8H/gguU7hGR5FKwzwDhsN2sNXOmXfmvWWPLJY4bB1dcAd27p/sIRaSpU7DPQLt2\n2YLoM2faOrmFhXDhhdYuuABOSlt5OBFpqlIR7C8HHsHq0/+Z42vbDACmAUOAe4Hf+/72OXAAqMbK\nHI+I8vnNPtj7VVfbVf/8+fDuu9bat7fAf/75MHIk9O+vxdJFpG7JDvbxLEDSGTgZuAbYS81gvwkY\nBpTX8R2BCvaRQiFL88yfD++9B4sWWamGESNsSUW3deqU7iMVkUyS7GA/CpiEV7nyHmf76yjvnQRU\ncHywHw7sqeM7Ah3so9m504L+okXw4Ye28Er79nDmmV476yzo00e/AESCKtFgH6uefbQFSM5J4HjC\nwJtYGudxYEoC+wZWly7eMopgV/8lJZb+WbECnn4afvpTW3xl8GCryz9woBVxGzjQSjVr2qeI+CVz\n8ZJozgVKsVTPXGzx8QWRb2rMxUuaouxsK9/Qrx984xve6+Xl8MknlgZau9YGgNeutbt93eB/+unW\nBg+2gWCdBESapkxZvASip3Hi+bvSOEm2b58F/dWrbVWuVatg5UqorPSC/6BBdvLo39+Kv+XkpPuo\nRSQRyc7Z52IDtJcA27Fa9ZEDtK5i4CBeMM/HBngPAgXAHOB+Z+unYN9Idu70Av+6dfDZZ7B+vb3+\nla94wf/kk+1XQI8etu3aVScDkUyTiqmXY/GmXk4FJuMtXPI4UIjN0mmLXfUfBE4DumArWIGdNJ5z\n9o2kYJ9mhw/bmIAb/DdvtpLOW7fatrzcAn6PHnZS6NvX2/bta/cNaKBYpHHppipJuspKqwO0ZQts\n3GgnBndbUgIHDx7/ayDycefO+nUgkkwK9tLoKirgiy/sV4D/F4H7ePt2u3ega1crFXHSSd62sBC6\ndbNWWGgnBf1KEIlNwV4yUmUl7NhhgX/bNm9bWmqvl5Za27/fAn7XrrZ1W6dO3uPCQjtZdOsGLVqk\n+18mkh4K9tKkVVZCWZm1Xbus7d7tPd61yztplJXZQjLur4Ru3aCgwE4AbjvhBNu2agUdO3onDrfl\n5aX7XyxSPwr2EhjV1Rb8/b8SjhyxE4a/HT1qg9B79ngnjt277XlBgS060769nTgit/n5dsI44QQr\nW+0+btXKe1/79tCuncYkpHEp2IvEKRSytFF5uY0p7NtXc7t3r508jh619uWX3uPDh2u+r6IC2rSx\nwN+hg/2KiGzuSaVtWzs5uNs2bXSikMQp2IukQXW1nTj27rWTx5490dv+/V47cMC2FRX2S6GgwH5J\nuFu3tW3rDWK7YxVuc08UujM6eBTsRZqYUMgC/uHD0du+fd44hTuQ7baKCjvRZGVZ0Hdbbq6dNNq0\nOb61bWspqGitbVto3dprGtPIXAr2IgEUClnQr66GY8esHTpk90AcPGi/IvyP9++3k4i/7d1rf6+o\n8FpOjgX9ggIL/P4Titvy8mw8Iz/ffqG4LT/f/hYOWwuFvMdg+xQUeCcW97H7yybys1q1sn3y8vRL\nBjJv8ZJY+4KC/T/NmzdPReAc6gtPuvoiHLbxCTfwHzvmnVD8rarKxjMOH7YxDrcdPmx/y8624Oxv\nUPOzDx2qeZLxf477WbadRyhURF5ezVlXLVrYiSLaQHvbtvZ97knQ37KyvHRZQUHN5v+F06aNbfPz\njz/RuCfYqip7XlDQOCejZJc4zgEeo+biJdOpWRtnD3ArtnhJovuKjwKcR33hSVdfZGXZlXTLlpmz\neE5x8Tzuu6+IqqrjZ10dPFjzV4q73bHDTji5uV7Lz7dtKGQnku3b7YRz+LBtI08+FRX2+UeP2r6h\nUM0An5dnnwf2Hjct5j/p5OfbL6HsbO9Xkfs4P7/mycU9wbRrB6NHJ6fvYgX7EcAGbHlBgL8BV1Mz\nYO9y2rh67CsikpDsbG8KbGOrrrYTQXa2F+AjZ1JVVdU86bgnnsOHbX835eZPvR05YicUdxzGbdnZ\n8NpryTn2VC5e0tCFT0REMkpOjpcWqk1enne3d1PydWquLvVt4I+1vHcS8LN67LsBWyRFTU1NTS3+\ntoEExLqy3wb09D3viV2hxyPefU+J8/NERCRFcoESoDfQAlgODKzlvcXUvLJPZF8REUmzsdhqVRuw\nZQnBFi9xFzApxHLz+4G9wGagdR37ioiIiIhIc3M5sA5YD9yd5mNpbE8CZcBK32sdgLnAZ9havSem\n4bjSoSfwDrAaWAXc5rwexP5oCSzC0p5r8JbyDGJfgN2vswyY4TwPaj+ATWP/BOuPxc5rTaI/crD0\nTm8gj+Dl9M/H7jr2B/vfAD93Ht8N/LqxDypNCoGznMetsdTfQILbH/nONhf4EDiP4PbFBGz96unO\n86D2A8AmLLj7NYn+GAW84Xt+j9OCpDc1g/06oKvzuNB5HkSvYndeB70/8rE7zwcRzL7oAbwJXIR3\nZR/EfnBtAjpGvBZ3f6Rztc9oN12dlKZjyRRdsdQOzrZrHe9trnpjv3gWEdz+yMZ+6ZbhpbeC2BcP\nA3cBId9rQewHVxg7+S0FbnZei7s/Ys2zT6VwGr+7KXBvnAiS1sD/ArcDByP+FqT+CGFprXbAbOzK\n1i8IfTEe2Inlp4tqeU8Q+sHvXKAU6Izl6SOv4uvsj3Re2Tfkhq3mqgz7KQbQDfvPHhR5WKB/Fkvj\nQLD7A2w682vAMILXF6OBq7DUxV+Bi7H/G0HrB79SZ7sLeAWrPxZ3f6Qz2C8F+uHddHUd3iBMUE0H\nvus8/i5e0GvusoCp2OyTR3yvB7E/OuHNqGgFXIpd3QatL36BXQD2Aa4H3gZuJHj94MoH2jiPC4DL\nsPG+JtMfQb7p6q/AdqASG7u4CRtpf5MMn0aVAudhqYvlWGBbhk3LDWJ/DAY+xvriEyxnDcHsC9eF\neBeCQe2HPtj/ieXY9GQ3Xga1P0REREREREREgixWSYNO2M1Rbi7pe76/TcTmCK8EngfSsLaMiIjE\nEk9Jg2K8+h2dsDVpc519NuIF+BfwRo1FRKQRxZp66V9HtgpvHVm/UsBdqKstFuyPAQecffKx4J+P\nza0XEZFGFivYx1PSYApWu2M7sAK7+xGgHPg9Vt9+O7APmyIkIiKNLFawj+dW5F9g6Z3u2C3ef8Ju\ne+8L3IGlc7o7r90QufOZZ56Z7nUc1dTU1JpiW04CYgX7eEoajAZedB6XYLc3D8Ru8f4AL63zsvPe\nGlasWEE4HFYLh5k0aVLajyFTmvpCfaG+qLsBZyYz2MdT0mAdVo4WrOLaqU7Q/xQYid3yneW8Z00i\nByciIskRq+rlMeAnWOW9HKx+yVq89WcfBx4EpmH5+myskH65057BThgh7BbwJ5J7+CIiEo94ShzP\ncprf477Hu4Era9n3N06TOBQVFaX7EDKG+sKjvvAEoS9CIThwAPbuhUOH4PTTk/O5Wcn5mAYJO/kn\nEZFmpawM3nwT5s6FL77wXs/K8rbhMFRUQHm5Bfj9+yE/H9q3h169YMGC6J+dZR8SdwxXsBeRZufo\nUQuSr78O8+dbUG3d+vjWqhXk5EB2ds2WkwPHjsGRI/Dll9bcx5WV0KkT9OwJPXrY1n2clWXfO3eu\ntc8/h6IiuPRSGDjQC+7gbcGOpX17ayeeCHl5sf+NCvYiUi/bt1sK4aSTvCvPxrR/P3zyCaxfD23a\nQMeOFlQ7drTWsmXd+2/dCrNmwWuvwTvvwGmnwRVXwCWXWPCsqLB26JD3+PBhC7rV1fZvd1t1NeTm\n2ne2amVb93FeHuzaBVu22Hdu2eI9Bhg2DC67zAL8iBH2OamgYC8iMR07ZoH1gw+svf++BcFsZ37e\n0KHWhg2zbe/edkW7cSN89lnNtm0bFBRAu3bQtq23dVubNl5r3dq2BQWW1li+HFassO3OnZafPvVU\nO5bdu2HPHm/booXtG+1EFArZv2nMGAvwY8bYiaIxhcPWRyc0UgWwVAT7y7HVg3KAPwMPRfy9E/AX\nbGmsXOB3wFPO30509hmE3QTwfeDDiP0V7EXqobzcUgtdutT9s//QIVizBlatsrZsGSxdaqmH0aOt\nnXsu9Otn79++HT7+GD76yLYff2wDhpWVcPLJ9r7+/b3Wo4ddIR84YFfn+/d7jw8csCvogwetuY8r\nKuz7zzwTzjrLtqecYumTaNy89sHIlYl9unatff/mKNnBPgebL/9V7AarJcC3sOmXrmKs2NlELPB/\nis23PwY8DcwHnsROBAXYupp+CvYicaiuhiVL4I03LF2xbp1dIe/aZXnebt2gsNBahw6WL165EkpL\n7Wr59NOtnXEGjBxp+eF4lZfbVXU8uWRpHIkG+1jZJH8hNPAKofmDfSlwhvPYXwitHXA+XqXLYxwf\n6EUCq7oa3noLnnoKZsyw9EePHl5zB/2OHrUAP2eOBfTLL4fJk+G88yy1UV1tqY4dO7y2e7f9ffBg\n6Nu34XnjDh2S8k+WNIr1XyBaIbRzIt4zBVsMeDu2IO43ndf7YKugT8Nu6/0IK5J2uGGHLNK0bdhg\nAf6ZZ6BzZ7jpJnj4YQvqW7d6g35bt8KHTtLzssvgoYfsBBApJ8dSGF27WjpEJJpYwT6e/IpbCK0I\nK342FwvuucBQ7A7cJVje/x7gvsgPKC4u/ufjoqKiQNw4Ic1fOGxX2Js3W/DetAleeQU+/RRuuAFm\nzrSUil+vXuk5Vsl88+bNY968efXeP1a+ZySWk7/ceT4RK33gH6R9HfgV8L7z/C1sRautwELsCh/g\nPCzYj4/4DuXspV4OH7bAuXatDUCWlFi+esAAr3XpkvxphKGQXYXv2GGDmaWltnUf+6fj5efb1Xiv\nXra99FKbLdKiRXKPSYIn2Tl7fyG07VghtG9FvMcthPY+XiG0jVhtnC1Af+Az5z2r4z0wkUibNsFz\nz9lUwbVr7e7EU06xm1UGDLAgWlZmg5jPPmvvCYftb716WW776FFrlZXetrISqqqit+rq4xtYsC4s\nhO7dLY/evbu1U0+1eepucC8oSG+fibjiOSuMxZt6ORVbgtBfCK0TlpfvhRVCm4ytNwuWzvkzVjGz\nBLgJzcYR7Op48WLLM598sje/O9K+ffDii17wvu46m0M9cCD06RN7qt3u3TZrZfNmm0nSooXNgz7h\nBO9xXl7tLSfn+FbbsYo0Jt1UJRnt0CEbnHzkEZshUlFh9UAGDLA7HgcNsm0oZFfxs2db6uPGG2Hs\nWKU/RFwK9pKRtm2Dxx6DKVPgggtgwgS7kScry26+cfPua9bA6tVWg+T66+Gb30xsPrhIUCjYS8ao\nqLD8+dSpVpDqxhvhttts3reINIyCvaRFVZXdir94sdc2brQ7Nr/+dbjlFrvLU0SSI9Nq4+DstxSb\nihltkRMF+wzizljxl2F1W2WlTSf8/HNrmzZ5j0tKrFjWiBFeGzxYOXaRVMm02jgAE7DFx9sAV0X5\nDgX7NAmHLVAvWuS15cvtdf/iCm7Ly7PphL1712x9+lhqpk2btP1TRAInk2rjAPQArsBuupoQ70FJ\ncoRCVsCqrKxm3ZQdO2w64qJFNiPmnHOs/epXMHy4grZIc5TK2jgADwN3YScBaSSrV8Pvfgd/+5st\ntuBWQuza1Xv83e/Cf/+3t7qOiDRvqayNcyGwE1jm/E1SKBy25dB++1ubAfOTn9ht+x07pvvIRCQT\nxAr22wB/nb2e2NW932gsTQN2l+wmYIDz+lVYGqcldnX/DPCdyC9RIbT6q66Gf/wDfvMbW83nzjvh\n73+3K3oRaT5SXQgtFxtwvQRL0yzm+AHaP2AlEO7HBmY/wnL45b73XAjciWbjNFhVlS3jtnChlb99\n912ryXL33XD11cFaqUckyJI9QHsMK1E8G682zlpq1sZ5EKuNswKrjfNzagZ6lyJ6DEeOWC0Yd2k3\nf/v0Uwvuy5bZ7JdRo2wh5Xvv9VatFxGpTSaEiEBf2W/cCC+9ZMW+Vq60G4/atfOau4DzV75iAf7s\ns+01EQk23UHbBJSUWHB/8UUbRL32WviXf4ELL2z48nEiEgwK9hkqFLJVin79a7sL9WtfswB//vkK\n8CKSuGTn7KWB3CB///1WO/2++2yRDQ2kikhjUrCvh1DI8uwLFljt9dNPt+YvxRsKwcsvW5Bv2RIm\nT7Ygr4FUEUmHeENPfYuh9cTm1nfBZuM8ATwasW+TSeOEw3aVPmmSzWP/xjdg/Xqr9rhqlQ2kDh5s\ns2PmzrUgX1ysIC8iyZeKnH1DiqF1wk4Ay4HW2Bz8ayL2zfhgHw7DzJkW5AEeeADGjasZwEMhW/pu\n5UorVzB4sIK8iKROKnL2DSmGtsNpABXOPt0j9s1oc+bAf/6nzYG//3645proATw726sCeWW0W8dE\nRNIonmDf0GJort7AEGBRwkeZBkeOwE9/aumYyZMtZaOFpkWkqYon2DekGNpB5++tgZeA27Er/Boy\nrTbO+vU2LbJ/f7tjVTcxiUi6pbo2DsBILCd/ufN8IhCi5iDt61gxtPed528Bd2MrVOUBM4FZ2CBv\npIzK2b/wglWMvP9++Pd/V85dRDJTKnL2S4F+WBpmO3AdNkDrtw4bwH0fG5g9FdjoHMhUYA3RA33G\n+PJLmDDBcvSzZ8PQoek+IhGR5IknC+0vhrYGeAGvGJpbEO1BYDhWDO1NvGJo5wLfBi7C6tovw/uF\nkDFKSmD0aNi5Ez76SIFeRJqfTEhSpDWNs2SJzZ65915L3yhtIyJNgcolJOCtt+Bb34KpUzVdUkSa\nt8BOJnz5ZQv0L72kQC8izV8gg/3UqZaymT0bLrgg3UcjIpJ6gUvj/Pa38Kc/wbx5No9eRCQI4rmy\nvxybWrkemzsfqRPwBnZT1Srgewns22jCYVun9amn4L33FOhFJFhijeQ2pAhaOI59oRFm41RUwM03\n2xKAr78OHTum9OtERFIu0dk4sa7s/UXQqvCKoPmVYsXPoGYRtHj2Tbm1a2HECMjPt9SNAr2IBFGs\nYB+tCNpJEe+ZAgzC7q5dgdW/iXfflHrhBRuA/dnPbFC2VavG/HYRkcwRa4C2IUXQ4pbsQmiVlXDX\nXVaDfs4cGDKkQR8nIpJ2qS6E1pAiaLlx7AtJztlv3Qrf/CZ06gRPP11zqUARkeYi2Tl7fxG0FlgR\ntOkR73GLoEHNImjx7JtUn39u+fmrroJXX1WgFxFxxXNWGIu3/uxUYDJeAbTHsRk404Be2MljMvB8\nHftGSsqVfTgMl10GF18MEyc2+ONERDJaKtagTbWkBPspU+CJJ2DhQsgN3K1iIhI0gQz2mzfDsGHw\nzjtw+ulJOioRkQyW7Jx9xguH4ZZb4PbbFehFRGrT5IP9U0/ZoiN3p7UYg4hIZktGbZw78VahWond\nPXui87eJwGrn9eexsgpJs22bBflp0yAvL5mfLCLSvCSjNo7feOAO5/29gbeBgcBRbDnD14GnI/ap\nV84+HLY69MOHg++eLBGRQEj2SlX++jbg1bepLdj/P+CvzuMDWE2cfKDa2W6L98Bi+ctfYMsWW4RE\nRETqlozaOK58YAzwv87zcuD3wGasbs4+bDHyBisttXo306ZBixbJ+EQRkeYtVrBPJL9yJfAeFtTB\n6uTcgaVzugOtgRsSPL6obr0VfvhDGDo0GZ8mItL8xUrjbAN6+p73xK7uo7keL4UDMBz4ACt5DPAy\nMBp4LnLHRAqhvfceLF1qaRwRkaBIdSG0XGyA9hIsFbOY6AO07bB6OD2AI85rZ2KB/WzgS+ApZ/8/\nRewb9wBtOAyjR8OPfgQ33hjXLiIizVKyB2iPAT8BZuPVt1lLzdo4ANc47zni23cF8AxWEC0EfAw8\nEe+BRfPKK3DkCNyQlGSQiEhwNJlyCVVVdofso4/CmDGNcFQiIhms2ZZLmDoVeva0ypYiIpKYJnFl\nX1EB/fu53HDdAAAGF0lEQVTDjBlW8ExEJOia5ZX9H/4ARUUK9CIi9ZXxV/Y7d8Jpp8GSJdCnTyMe\nlYhIBkvFlX1DCqGdCLyEzeBZg61pm5AHHoBvf1uBXkSkIVJZCA2s6Nl84ElsmmcBsD9in1qv7Nev\nh1GjYN06W0BcRERMsq/s/YXQqvAKodXGXwitHXA+FujBrvgjA32d7r0XJkxQoBcRaahUFkLrA+zC\nFiP/GJjivCcuixfD++/bClQiItIwse6gbUghtFxgKHYH7hLgEeAe4L7IHaPVxnn8catsWVCQwBGI\niDRTqa6NMxIoxgZpwVaeCgEPRXnvK9gCJX9znhcCC7ErfIDzsGA/PmK/43L21dXQvTt8+KEGZkVE\nokl2zn4p0A8rU9wCuA6YHuV97YALgH/4XtuBpYD6O8+/ii1RGNPixdCliwK9iEiypLIQGsCtWOXL\nFkAJcFM8BzV9Olx1VTzvFBGReGTkTVWDBsGTT8I556TpiEREMlyTL5ewYQPs2QNnn53uIxERaT4y\nLtjPmAFXXgnZGXdkIiJNV8aFVOXrRUSSL6Ny9uXl0Ls37NgB+XHffiUiEjyZVggNbBbPMmBGrC+a\nNQsuukiBXkQk2WIF+xzgMSzgn4YVQRsY8Z7fAUOcNhGYh3cXLcDtWMXLmHfjzpihFI6ISCqkshAa\nQA/gCuDPxPi5UVkJs2fD+Mj7a0VEpMFSWQgN4GHgLqzEQp3efRcGDICuXWO9U0REEpXKQmjjgZ1Y\nvr6orh2Li4uZNQtat4Z586wQmoiIeDK5ENqDwI3YgG1LoC121f+diP3CoVCYPn3gtdfs7lkREalb\norNxYr0xF1up6hJgO7CY6CtVtQM2Yjn6yPo4ABdis3aujPK38IoVYa691u6ezcqEyaAiIhku0WCf\n6kJofrWmhNwbqRToRURSIxPCa/jss8M89JDNsRcRkdiSncZpDOH27cOUlUFeXroPRUSkaWiSVS/H\njlWgFxFJpYwI9ldGG7YVEZGkiTfY17c+Tk/gHWw5wlXAbVE//PJor4qISLLEE+wbUh+nCvgpMAib\ns//jKPty4omRrwRTQ26YaG7UFx71hUd9UX/xBPuG1MfZASx3Hldg0za71+dAg0D/kT3qC4/6wqO+\nqL94gn1D6+O4emNX/osSOD4REUmCeIJ9Q+rjuFoDL2HljisS+DwREWkkI4E3fM8nEn2QFqw+zvUR\nr+Vhd9feUcs+G7ATipqamppa/G0DSZYLlGBpmBZYDv64QVasPs4eoJXvtSzgGazUsYiIZLixWEG0\nDdiVPVh9nB/63vNd4PmI/c7DqmQux5uaqYmWIiIiIiLNTaybtZqzJ4Ey7CY0VwdgLvAZMIeaC7c3\nZ7XdfBfE/miJzVhbjq3dPNl5PYh9AXafzzJghvM8qP0ANv39E6w/FjuvNYn+yMHSQr2xQdzaxgKa\nq/Oxqaj+YP8b4OfO47uBXzf2QaVJIXCW87g1ljIcSHD7I9/Z5gIfYunQoPbFBOA5YLrzPKj9ALAJ\nC+5+TaI/RlFzls89TguS3tQM9usAdxXeQud5EL0KfBX1Rz6wBLsDPYh90QN4E7gI78o+iP3g2gR0\njHgt7v5IZyG0RG7WCoquWGoHZxvE5dd74918F9T+yMZ+6ZbhpbeC2BcPA3dhkzxcQewHVxg7+S0F\nbnZei7s/Yq1UlUrhNH53U+DOpQ2S1tjd17cDByP+FqT+CGFprXbYPSqRy/oEoS/GAzux/HRRLe8J\nQj/4nQuUAp2xPH3kVXyd/ZHOK/tt2MCcqyd2dR9kZdhPMYBu2H/2oMjDAv2zWBoHgt0fAPuB14Bh\nBK8vRgNXYamLvwIXY/83gtYPfqXOdhd2A+sIEuiPdAb7pUA/vJu1rsMbhAmq6dj9CjjbV+t4b3OS\nha1vvAZ4xPd6EPujE96MilbApdjVbdD64hfYBWAf7K78t4EbCV4/uPKBNs7jAuAybLyvyfRHtJu1\nguKvwHagEhu7uAkbaX+TDJ9GlQK13XwXxP4YDHyM9cUnWM4agtkXrgvxLgSD2g99sP8Ty7HpyW68\nDGp/iIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIjU7f8AgR+hHRzKPR0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc9ec851550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Traininig curve\n",
    "epochs = range(training_epochs)\n",
    "plt.subplot(211)\n",
    "plt.plot(epochs, save_cost)\n",
    "plt.subplot(212)\n",
    "plt.plot(epochs, save_accr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}