Logistic Regression and Multi-layered Perceptron.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Model Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_weights(shape, name):\n",
    "    return tf.Variable(tf.random_normal(shape, stddev=0.01), name=name)\n",
    "\n",
    "def logistic_model(X, w):\n",
    "    return tf.matmul(X, w)\n",
    "\n",
    "def mlp_model(X, w_h, w_o):\n",
    "    h = tf.nn.sigmoid(tf.matmul(X, w_h))\n",
    "    return tf.matmul(h, w_o)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels\n",
    "# or batch_X, batch_Y = mnist.train.next_batch(64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(\"float\", [None, 784], name=\"X\")\n",
    "Y = tf.placeholder(\"float\", [None, 10], name=\"Y\")\n",
    "\n",
    "model = 'Logistic' \n",
    "model = 'MLP'\n",
    "\n",
    "if model == 'Logistic':\n",
    "    w = init_weights([784, 10], \"w\")\n",
    "    py_x = logistic_model(X, w)\n",
    "    tf.summary.histogram(\"w_summary\", w)\n",
    "else :\n",
    "    w_h = init_weights([784, 625], \"w_h\")\n",
    "    w_o = init_weights([625, 10], \"w_o\")\n",
    "    py_x = mlp_model(X, w_h, w_o)\n",
    "    tf.summary.histogram(\"w_h_summary\", w_h)\n",
    "    tf.summary.histogram(\"w_o_summary\", w_o)\n",
    "    \n",
    "with tf.name_scope(\"cost\"):\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y))\n",
    "    train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost)\n",
    "    tf.summary.scalar(\"cost\", cost)\n",
    "    \n",
    "with tf.name_scope(\"accuracy\"):\n",
    "    correct_pred = tf.equal(tf.argmax(Y, 1), tf.argmax(py_x, 1))\n",
    "    acc_op = tf.reduce_mean(tf.cast(correct_pred, \"float\"))\n",
    "    tf.summary.scalar(\"accuracy\", acc_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration : 1 , Test Accuracy : 0.9057\n",
      "Iteration : 2 , Test Accuracy : 0.9293\n",
      "Model saved in : ./models/\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "model_path = './models/'\n",
    "if not os.path.exists(model_path):\n",
    "    os.makedirs(model_path)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    writer = tf.summary.FileWriter(\"./logs/nn_logs\", sess.graph)\n",
    "    merged = tf.summary.merge_all()\n",
    "    \n",
    "    tf.global_variables_initializer().run()\n",
    "\n",
    "    for i in range(2):\n",
    "        for start, end in zip(range(0, len(trX), 128), range(128, len(trX)+1, 128)):\n",
    "            sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})\n",
    "\n",
    "        summary, acc = sess.run([merged, acc_op], feed_dict={X: teX, Y: teY})\n",
    "        writer.add_summary(summary, i)\n",
    "        \n",
    "        print(\"Iteration :\", i+1, \", Test Accuracy :\", acc)\n",
    "    save_path = saver.save(sess, model_path)\n",
    "    print(\"Model saved in :\", save_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keep Training by Loading Existing Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model loaded from : ./models/\n",
      "Iteration : 1 , Test Accuracy : 0.9429\n",
      "Iteration : 2 , Test Accuracy : 0.9547\n",
      "Iteration : 3 , Test Accuracy : 0.9609\n",
      "Iteration : 4 , Test Accuracy : 0.9653\n",
      "Iteration : 5 , Test Accuracy : 0.9683\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "       \n",
    "    tf.global_variables_initializer().run()\n",
    "    saver.restore(sess, model_path)\n",
    "    print(\"Model loaded from :\", save_path)\n",
    "    \n",
    "    for i in range(5):\n",
    "        for start, end in zip(range(0, len(trX), 128), range(128, len(trX)+1, 128)):\n",
    "            sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})\n",
    "\n",
    "        summary, acc = sess.run([merged, acc_op], feed_dict={X: teX, Y: teY})\n",
    "        writer.add_summary(summary, i)\n",
    "        \n",
    "        save_path = saver.save(sess, model_path)\n",
    "        \n",
    "        print(\"Iteration :\", i+1, \", Test Accuracy :\", acc)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}