deep_learning/keras_solve_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solving MNIST with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from keras.datasets import mnist\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAD8CAYAAADub8g7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF3JJREFUeJzt3XtsFdX2B/DvEsUXASmaWgEBk4qpv6D4QPQioIBB1IBv\niUqJxJoIBg0a0ItG46s+E0BQUXkpAa9BBDVEuLVAjNgAPu7lIRRNQLCCiAiIykXX74+O29ljT3se\nc2bmnP39JE3Xnt3TWZcu1533iKqCiMglR8SdABFR1Nj4iMg5bHxE5Bw2PiJyDhsfETmHjY+InMPG\nR0TOyanxichgEdkkIltEZEJYSRHFjbVd3CTbC5hFpBWAzQAGAdgOYDWA4aq6Ibz0iKLH2i5+R+bw\n2V4Atqjq1wAgIvMBDAWQsjhEhLeJJMduVT0p7iQSKqPaZl0nSlp1ncuubkcA3/jG271lVBi2xp1A\ngrG2C1dadZ3LFl9aRKQKQFW+10MUJdZ1Ycul8e0A0Nk37uQts6jqdADTAe4SUMFosbZZ14Utl13d\n1QDKRaSbiLQGcBOAxeGkRRQr1naRy3qLT1UPi8gYAB8AaAVghqquDy0zopiwtotf1pezZLUy7hIk\nyVpVPS/uJIoB6zpR0qpr3rlBRM5h4yMi57DxEZFz2PiIyDlsfETkHDY+InIOGx8ROSfv9+oSUeE5\n99xzrfGYMWNMPGLECGtuzpw5Jp4yZYo19+mnn+Yhu9xxi4+InMPGR0TOYeMjIufwXt0mtGrVyhq3\na9cu7c/6j4Ucd9xx1lz37t1NPHr0aGvu2WefNfHw4cOtuV9//dXE1dXV1twjjzySdm4BvFc3JIVS\n1805++yzrfGHH35ojdu2bZvW7/npp5+scYcOHXJLLHO8V5eIqClsfETknKK+nOXUU0+1xq1btzbx\nRRddZM316dPHxCeccII1d+2114aSz/bt2008efJka+7qq6828f79+625L774wsQrVqwIJReiXr16\nmXjBggXWXPDwjv+QWLA+Dx06ZOLgrm3v3r1NHLy0xf+5qHGLj4icw8ZHRM5h4yMi5xTd5Sz+0/LB\nU/KZXJYShj/++MMa33bbbSY+cOBAys81NDRY4x9//NHEmzZtCik7Xs4SliRfzuK/pOqcc86x5t54\n4w0Td+rUyZoTEWvs7xPBY3VPP/20iefPn5/y90ycONGae/LJJ5vNPUu8nIWIqClsfETknKK7nGXb\ntm0m/uGHH6y5MHZ16+rqrPHevXut8SWXXGLi4On6119/Pef1E2Xi5ZdfNnHwjqBsBXeZ27RpY+Lg\n5Vb9+/c3cY8ePUJZfxi4xUdEzmHjIyLnsPERkXOK7hjfnj17THzfffdZc1deeaWJP/vsM2sueAuZ\n3+eff27iQYMGWXM///yzNT7zzDNNPHbs2DQyJgpP8MnJV1xxhYmDl6j4BY/Nvfvuu9bY//Sgb7/9\n1prz/7fkv/QKAC699NK01h81bvERkXNabHwiMkNEdonIOt+yEhFZJiL13vf2+U2TKHysbXe1eOeG\niPQFcADAHFX9P2/Z0wD2qGq1iEwA0F5Vx7e4spivcPc/TDH4hAn/af9Ro0ZZc7fccouJ582bl6fs\nIuf8nRth1Xbcdd3c3UrNPUB0yZIlJg5e6tKvXz9r7L8U5dVXX7Xmvv/++5Tr+P3330188ODBlOsI\n8aVE4dy5oaorAewJLB4KYLYXzwYwLOP0iGLG2nZXtic3SlX1zxtKvwNQmuoHRaQKQFWW6yGKWlq1\nzboubDmf1VVVbW5TX1WnA5gOxL9LQJSJ5mqbdV3Ysm18O0WkTFUbRKQMwK4wk8qXffv2pZwLviTF\n7/bbbzfxm2++ac0Fn8BCBS/xtX366adbY/9lW8HbMnfv3m3i4FN/Zs+ebeLg04Lef//9ZsfZOPbY\nY63xuHHjTHzzzTfn/Pszke3lLIsBVHpxJYBF4aRDFDvWtgPSuZxlHoBVALqLyHYRGQWgGsAgEakH\nMNAbExUU1ra7iu5BpNk6/vjjTRy8at1/2v3yyy+35pYuXZrfxPLH+ctZwhJFXR999NEmfuutt6y5\nIUOGmDi4y3rjjTeaeM2aNdacf9fT/yKsMPkvZwn2mlWrVpn44osvDmuVfBApEVFT2PiIyDlsfETk\nnKJ7Oku2/E9Z8V++Ati307zyyivWXG1trTX2H0eZOnWqNRfl8VQqLj179jSx/5he0NChQ60xX0Df\nNG7xEZFz2PiIyDnc1W3CV199ZY1Hjhxp4pkzZ1pzt956a8qx/xIZAJgzZ46Jg1fREzXn+eefN3Hw\ngZ7+3dmk7doeccRf21ZJusuJW3xE5Bw2PiJyDhsfETmHx/jSsHDhQhPX19dbc/5jLwAwYMAAEz/x\nxBPWXJcuXUz8+OOPW3M7duzIOU8qHv4XYwH2U5aDl0UtXrw4kpyy4T+uF8zb/xKvqHGLj4icw8ZH\nRM5h4yMi5/AYX4bWrVtnjW+44QZrfNVVV5k4eM3fHXfcYeLy8nJrLviicnJb8GnFrVu3NvGuXfZD\noYNPBY+a/5FZDz/8cMqfC74B7v77789XSi3iFh8ROYeNj4icw13dHO3du9cav/766yYOvnj5yCP/\n+ufu27evNde/f38TL1++PLwEqej89ttv1jjq2x/9u7YAMHHiRBP7X3wE2E92fu6556y54NOio8Qt\nPiJyDhsfETmHjY+InMNjfBnq0aOHNb7uuuus8fnnn29i/zG9oA0bNljjlStXhpAduSCOW9T8t8wF\nj+P53+S2aJH9GuJrr702v4lliVt8ROQcNj4icg53dZvQvXt3azxmzBgTX3PNNdbcySefnPbv9b9c\nOXgJQpKeTkvxCz5l2T8eNmyYNTd27NjQ13/PPfdY4wcffNDE7dq1s+bmzp1r4hEjRoSeSz5wi4+I\nnNNi4xORziJSKyIbRGS9iIz1lpeIyDIRqfe+t89/ukThYW27K50tvsMAxqlqBYDeAEaLSAWACQBq\nVLUcQI03JiokrG1HtXiMT1UbADR48X4R2QigI4ChAPp7PzYbwHIA4/OSZR4Ej80NHz7cxP5jegDQ\ntWvXrNbhf7k4YD91OclPzXVFkms7+LRi/zhYu5MnTzbxjBkzrLkffvjBxL1797bm/G8EPOuss6y5\nTp06WeNt27aZ+IMPPrDmpk2b9vf/AQmX0TE+EekKoCeAOgClXuEAwHcASkPNjChCrG23pH1WV0Ta\nAFgA4G5V3ec/y6SqKiKa4nNVAKpyTZQoX7KpbdZ1YUur8YnIUWgsjLmq+ra3eKeIlKlqg4iUAdjV\n1GdVdTqA6d7vabI55ktpqf1/1BUVFSZ+4YUXrLkzzjgjq3XU1dVZ42eeecbEwavYeclK8mRb23HW\ndatWrazxnXfeaeLgnRL79u0zcfDht835+OOPrXFtba2JH3roobR/T1Klc1ZXALwGYKOq+l8pthhA\npRdXAlgU/CxRkrG23ZXOFt8/ANwK4L8i8uf74B4AUA3gXyIyCsBWADek+DxRUrG2HZXOWd2PAEiK\n6QEplhMlHmvbXQV/y1pJSYk1fvnll03sf6IEAJx22mlZrcN/vCP4FNngqf1ffvklq3UQ+a1atcoa\nr1692sT+JwAFBS91CR7n9vNf6jJ//nxrLh+3wSUJb1kjIuew8RGRcyR4hXheV5blaf8LLrjAGvsf\nhNirVy9rrmPHjtmsAgcPHjSx/0p4AHjiiSdM/PPPP2f1+xNoraqeF3cSxSCKy1nKyspM7H8/M2C/\n7Cf4VBf/f9+TJk2y5l588UUTb9myJZQ8EyCtuuYWHxE5h42PiJzDxkdEzimIY3zV1dXWOPiyk1SC\nL/R57733THz48GFrzn+ZSvAl4UWKx/hCEvUta9QsHuMjImoKGx8ROacgdnUpL7irGxLWdaJwV5eI\nqClsfETkHDY+InIOGx8ROYeNj4icw8ZHRM5h4yMi57DxEZFz2PiIyDlsfETknKhfNrQbja/rO9GL\nk8DVXLpEtB4XJLGugWTlE1UuadV1pPfqmpWKrEnKfaLMhcKStL9fkvJJUi4Ad3WJyEFsfETknLga\n3/SY1tsU5kJhSdrfL0n5JCmXeI7xERHFibu6ROQcNj4ick6kjU9EBovIJhHZIiIToly3t/4ZIrJL\nRNb5lpWIyDIRqfe+t48ol84iUisiG0RkvYiMjTMfyk2ctc26zlxkjU9EWgGYCuByABUAhotIRVTr\n98wCMDiwbAKAGlUtB1DjjaNwGMA4Va0A0BvAaO/fI658KEsJqO1ZYF1nJMotvl4Atqjq16p6CMB8\nAEMjXD9UdSWAPYHFQwHM9uLZAIZFlEuDqn7qxfsBbATQMa58KCex1jbrOnNRNr6OAL7xjbd7y+JW\nqqoNXvwdgNKoExCRrgB6AqhLQj6UsSTWdux1lOS65skNH228tifS63tEpA2ABQDuVtV9cedDxYd1\n/XdRNr4dADr7xp28ZXHbKSJlAOB93xXVikXkKDQWx1xVfTvufChrSaxt1nUzomx8qwGUi0g3EWkN\n4CYAiyNcfyqLAVR6cSWARVGsVEQEwGsANqrq83HnQzlJYm2zrpujqpF9ARgCYDOArwD8M8p1e+uf\nB6ABwP/QeBxmFIAOaDzLVA/g3wBKIsqlDxo39/8D4HPva0hc+fAr579nbLXNus78i7esEZFzeHKD\niJyTU+OL+04MonxhbRe3rHd1vavVNwMYhMbjCqsBDFfVDeGlRxQ91nbxy+WdG+ZqdQAQkT+vVk9Z\nHCLCA4rJsVtVT4o7iYTKqLZZ14mSVl3nsqubxKvVKX1b404gwVjbhSutus77W9ZEpApAVb7XQxQl\n1nVhy6XxpXW1uqpOh/fYae4SUIFosbZZ14Utl13dJF6tThQG1naRy3qLT1UPi8gYAB8AaAVghqqu\nDy0zopiwtotfpHducJcgUdZqgl7wXMhY14mSVl3zzg0icg4bHxE5h42PiJzDxkdEzmHjIyLnsPER\nkXPY+IjIOWx8ROQcNj4icg4bHxE5h42PiJyT9+fxUXoGDBhg4rlz51pz/fr1M/GmTZsiy4koHRMn\nTjTxI488Ys0dccRf21b9+/e35lasWJHXvJrDLT4icg4bHxE5pyB2dfv27WuNO3ToYOKFCxdGnU5e\nnH/++SZevXp1jJkQNW/kyJHWePz48Sb+448/Un4uykfgtYRbfETkHDY+InIOGx8ROacgjvEFT4OX\nl5ebuFCP8flP8wNAt27dTNylSxdrTkQiyYkoHcH6POaYY2LKJHvc4iMi57DxEZFzCmJXd8SIEdZ4\n1apVMWUSnrKyMmt8++23m/iNN96w5r788stIciJKZeDAgSa+6667Uv5csFavvPJKE+/cuTP8xLLE\nLT4icg4bHxE5h42PiJxTEMf4gpd+FINXX3015Vx9fX2EmRD9XZ8+fazxzJkzTdyuXbuUn3vmmWes\n8datW8NNLCQtdhQRmSEiu0RknW9ZiYgsE5F673v7/KZJFD7WtrvS2ZSaBWBwYNkEADWqWg6gxhsT\nFZpZYG07qcVdXVVdKSJdA4uHAujvxbMBLAcwHiHq0aOHiUtLS8P81YnQ3O7CsmXLIszEXXHVdiGo\nrKy0xqecckrKn12+fLmJ58yZk6+UQpXtwbNSVW3w4u8AFF9nIlexth2Q88kNVVURSfmgLRGpAlCV\n63qIotZcbbOuC1u2W3w7RaQMALzvu1L9oKpOV9XzVPW8LNdFFKW0apt1Xdiy3eJbDKASQLX3fVFo\nGXmGDBli4mOPPTbsXx8L/7FK/9NYgnbs2BFFOtS0vNd2Ep144onW+LbbbrPG/icr792715p77LHH\n8pdYnqRzOcs8AKsAdBeR7SIyCo1FMUhE6gEM9MZEBYW17a50zuoOTzE1IMVyooLA2nZXYu/c6N69\ne8q59evXR5hJeJ599lkTBy/R2bx5s4n3798fWU7krq5du5p4wYIFaX9uypQp1ri2tjaslCJTfPeC\nERG1gI2PiJzDxkdEzknsMb7mJOmF223btrXGgwf/devnLbfcYs1ddtllKX/Po48+auLg5QJE+eCv\nVf8tok2pqakx8aRJk/KWU1S4xUdEzmHjIyLnFOSubklJSVafO+uss0wcfFet/2UqnTp1suZat25t\n4ptvvtmaCz4k9ZdffjFxXV2dNffbb7+Z+Mgj7X/6tWvXNps7Ua6GDRtmjaurU1+b/dFHH1lj/9Na\nfvrpp3ATiwG3+IjIOWx8ROQcNj4ick5ij/H5j5Wp2o9Ee+mll0z8wAMPpP07/afsg8f4Dh8+bOKD\nBw9acxs2bDDxjBkzrLk1a9ZY4xUrVpg4+ALl7du3mzj4xBm+NJzyIdvb0r7++mtrnKSXgYeBW3xE\n5Bw2PiJyDhsfETknscf47rzzThMHX0p80UUXZfU7t23bZuJ33nnHmtu4caOJP/nkk6x+f1BVlf1K\nhpNOOsnEwWMoRPkwfvxfL4jzP0W5Jc1d41cMuMVHRM5h4yMi5yR2V9fvqaeeijuFrAwYkPoJ5plc\nWkCUrrPPPtsaN/dEIL9Fi+x3Km3atCm0nJKIW3xE5Bw2PiJyDhsfETmnII7xFaOFCxfGnQIVoaVL\nl1rj9u3bp/xZ/2VbI0eOzFdKicQtPiJyDhsfETmHu7pERaRDhw7WuLm7NaZNm2biAwcO5C2nJOIW\nHxE5p8XGJyKdRaRWRDaIyHoRGestLxGRZSJS731PfRSVKIFY2+5KZ4vvMIBxqloBoDeA0SJSAWAC\ngBpVLQdQ442JCglr21EtHuNT1QYADV68X0Q2AugIYCiA/t6PzQawHMD4Jn4FefxPfT799NOtubCe\nCEPpK5banjlzpomDb/1rzscff5yPdApCRic3RKQrgJ4A6gCUeoUDAN8BKE3xmSoAVU3NESVFprXN\nui5saf/fg4i0AbAAwN2qus8/p40vxdCmPqeq01X1PFU9L6dMifIkm9pmXRe2tLb4ROQoNBbGXFV9\n21u8U0TKVLVBRMoA7MpXksXC/9KkTHZJKH8KsbaDT2AZOHCgiYOXrxw6dMjEU6dOteaK7QVCmUjn\nrK4AeA3ARlV93je1GMCfr1evBLAo+FmiJGNtuyudLb5/ALgVwH9F5HNv2QMAqgH8S0RGAdgK4Ib8\npEiUN6xtR6VzVvcjAJJiOvWTNokSjrXtLt6yFpMLL7zQGs+aNSueRKjgnHDCCdb45JNPTvmzO3bs\nMPG9996bt5wKDY+wE5Fz2PiIyDnc1Y2Q/84NIooPt/iIyDlsfETkHDY+InIOj/Hl0ZIlS6zx9ddf\nH1MmVEy+/PJLa+x/ykqfPn2iTqcgcYuPiJzDxkdEzhH/E0PyvjKR6FZGLVnLRyqFg3WdKGnVNbf4\niMg5bHxE5Bw2PiJyDhsfETmHjY+InMPGR0TOYeMjIuew8RGRc9j4iMg5bHxE5Jyon86yG42v6zvR\ni5PA1Vy6RLQeFySxroFk5RNVLmnVdaT36pqViqxJyn2izIXCkrS/X5LySVIuAHd1ichBbHxE5Jy4\nGt/0mNbbFOZCYUna3y9J+SQpl3iO8RERxYm7ukTknEgbn4gMFpFNIrJFRCZEuW5v/TNEZJeIrPMt\nKxGRZSJS731vH1EunUWkVkQ2iMh6ERkbZz6Umzhrm3Wducgan4i0AjAVwOUAKgAMF5GKqNbvmQVg\ncGDZBAA1qloOoMYbR+EwgHGqWgGgN4DR3r9HXPlQlhJQ27PAus5IlFt8vQBsUdWvVfUQgPkAhka4\nfqjqSgB7AouHApjtxbMBDIsolwZV/dSL9wPYCKBjXPlQTmKtbdZ15qJsfB0BfOMbb/eWxa1UVRu8\n+DsApVEnICJdAfQEUJeEfChjSazt2OsoyXXNkxs+2niKO9LT3CLSBsACAHer6r6486Hiw7r+uygb\n3w4AnX3jTt6yuO0UkTIA8L7vimrFInIUGotjrqq+HXc+lLUk1jbruhlRNr7VAMpFpJuItAZwE4DF\nEa4/lcUAKr24EsCiKFYqIgLgNQAbVfX5uPOhnCSxtlnXzVHVyL4ADAGwGcBXAP4Z5bq99c8D0ADg\nf2g8DjMKQAc0nmWqB/BvACUR5dIHjZv7/wHwufc1JK58+JXz3zO22mZdZ/7FOzeIyDk8uUFEzmHj\nIyLnsPERkXPY+IjIOWx8ROQcNj4icg4bHxE5h42PiJzz/39p+s2eXr60AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x102f79828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "# plot 4 images as gray scale\n",
    "plt.subplot(221)\n",
    "plt.imshow(X_train[0], cmap=plt.get_cmap('gray'))\n",
    "plt.subplot(222)\n",
    "plt.imshow(X_train[1], cmap=plt.get_cmap('gray'))\n",
    "plt.subplot(223)\n",
    "plt.imshow(X_train[2], cmap=plt.get_cmap('gray'))\n",
    "plt.subplot(224)\n",
    "plt.imshow(X_train[3], cmap=plt.get_cmap('gray'))\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline Model with Multi-Layer Perceptrons\n",
    "Do we really need a complex model like a convolutional neural network to get the best results with MNIST?\n",
    "\n",
    "You can get very good results using a very simple neural network model with a single hidden layer. In this section we will create a simple multi-layer perceptron model that achieves an error rate of 1.74%. We will use this as a baseline for comparing more complex convolutional neural network models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The training dataset is structured as a 3-dimensional array of instance, image width and image height. For a multi-layer perceptron model we must reduce the images down into a vector of pixels. In this case the 28×28 sized images will be 784 pixel input values.\n",
    "\n",
    "We can do this transform easily using the reshape() function on the NumPy array. We can also reduce our memory requirements by forcing the precision of the pixel values to be 32 bit, the default precision used by Keras anyway."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flatten 28*28 images to a 784 vector for each image\n",
    "num_pixels = X_train.shape[1] * X_train.shape[2]\n",
    "X_train = X_train.reshape(X_train.shape[0], num_pixels).astype('float32')\n",
    "X_test = X_test.reshape(X_test.shape[0], num_pixels).astype('float32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The pixel values are gray scale between 0 and 255. It is almost always a good idea to perform some scaling of input values when using neural network models. Because the scale is well known and well behaved, we can very quickly normalize the pixel values to the range 0 and 1 by dividing each value by the maximum of 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# normalize inputs from 0-255 to 0-1\n",
    "X_train = X_train / 255\n",
    "X_test = X_test / 255"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, the output variable is an integer from 0 to 9. This is a multi-class classification problem. As such, it is good practice to use a one hot encoding of the class values, transforming the vector of class integers into a binary matrix.\n",
    "\n",
    "We can easily do this using the built-in np_utils.to_categorical() helper function in Keras."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)\n",
    "num_classes = y_test.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are now ready to create our simple neural network model. We will define our model in a function. This is handy if you want to extend the example later and try and get a better score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define baseline model\n",
    "def baseline_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(num_pixels, input_dim=num_pixels, init='normal', activation='relu'))\n",
    "    model.add(Dense(num_classes, init='normal', activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model is a simple neural network with one hidden layer with the same number of neurons as there are inputs (784). A rectifier activation function is used for the neurons in the hidden layer.\n",
    "\n",
    "A softmax activation function is used on the output layer to turn the outputs into probability-like values and allow one class of the 10 to be selected as the model’s output prediction. Logarithmic loss is used as the loss function (called categorical_crossentropy in Keras) and the efficient ADAM gradient descent algorithm is used to learn the weights.\n",
    "\n",
    "We can now fit and evaluate the model. The model is fit over 10 epochs with updates every 200 images. The test data is used as the validation dataset, allowing you to see the skill of the model as it trains. A verbose value of 2 is used to reduce the output to one line for each training epoch.\n",
    "\n",
    "Finally, the test dataset is used to evaluate the model and a classification error rate is printed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "5s - loss: 0.2811 - acc: 0.9199 - val_loss: 0.1403 - val_acc: 0.9578\n",
      "Epoch 2/10\n",
      "2s - loss: 0.1115 - acc: 0.9674 - val_loss: 0.0913 - val_acc: 0.9723\n",
      "Epoch 3/10\n",
      "2s - loss: 0.0712 - acc: 0.9797 - val_loss: 0.0792 - val_acc: 0.9760\n",
      "Epoch 4/10\n",
      "2s - loss: 0.0506 - acc: 0.9851 - val_loss: 0.0714 - val_acc: 0.9782\n",
      "Epoch 5/10\n",
      "2s - loss: 0.0369 - acc: 0.9897 - val_loss: 0.0670 - val_acc: 0.9791\n",
      "Epoch 6/10\n",
      "2s - loss: 0.0267 - acc: 0.9932 - val_loss: 0.0643 - val_acc: 0.9801\n",
      "Epoch 7/10\n",
      "2s - loss: 0.0204 - acc: 0.9948 - val_loss: 0.0628 - val_acc: 0.9809\n",
      "Epoch 8/10\n",
      "2s - loss: 0.0134 - acc: 0.9973 - val_loss: 0.0601 - val_acc: 0.9819\n",
      "Epoch 9/10\n",
      "3s - loss: 0.0106 - acc: 0.9979 - val_loss: 0.0567 - val_acc: 0.9821\n",
      "Epoch 10/10\n",
      "3s - loss: 0.0081 - acc: 0.9985 - val_loss: 0.0577 - val_acc: 0.9819\n",
      "Baseline Error:  1.81\n"
     ]
    }
   ],
   "source": [
    "# build the model\n",
    "model = baseline_model()\n",
    "# Fit the model\n",
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), nb_epoch=10, batch_size=200, verbose=2)\n",
    "# Final evaluation of the model\n",
    "scores = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"Baseline Error: \", (100-scores[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Convolutional Neural Network for MNIST\n",
    "\n",
    "In this section we will create a simple CNN for MNIST that demonstrates how to use all of the aspects of a modern CNN implementation, including Convolutional layers, Pooling layers and Dropout layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Flatten\n",
    "from keras.layers.convolutional import Convolution2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K\n",
    "K.set_image_dim_ordering('th')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we need to load the MNIST dataset and reshape it so that it is suitable for use training a CNN. In Keras, the layers used for two-dimensional convolutions expect pixel values with the dimensions [pixels][width][height].\n",
    "\n",
    "In the case of RGB, the first dimension pixels would be 3 for the red, green and blue components and it would be like having 3 image inputs for every color image. In the case of MNIST where the pixel values are gray scale, the pixel dimension is set to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "# reshape to be [samples][pixels][width][height]\n",
    "X_train = X_train.reshape(X_train.shape[0], 1, 28, 28).astype('float32')\n",
    "X_test = X_test.reshape(X_test.shape[0], 1, 28, 28).astype('float32')\n",
    "\n",
    "# normalize inputs from 0-255 to 0-1\n",
    "X_train = X_train / 255\n",
    "X_test = X_test / 255\n",
    "# one hot encode outputs\n",
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)\n",
    "num_classes = y_test.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convolutional neural networks are more complex than standard multi-layer perceptrons, so we will start by using a simple structure to begin with that uses all of the elements for state of the art results. Below summarizes the network architecture.\n",
    "\n",
    "- The first hidden layer is a convolutional layer called a Convolution2D. The layer has 32 feature maps, which with the size of 5×5 and a rectifier activation function. This is the input layer, expecting images with the structure outline above [pixels][width][height].\n",
    "- Next we define a pooling layer that takes the max called MaxPooling2D. It is configured with a pool size of 2×2.\n",
    "- The next layer is a regularization layer using dropout called Dropout. It is configured to randomly exclude 20% of neurons in the layer in order to reduce overfitting.\n",
    "- Next is a layer that converts the 2D matrix data to a vector called Flatten. It allows the output to be processed by standard fully connected layers.\n",
    "- Next a fully connected layer with 128 neurons and rectifier activation function.\n",
    "Finally, the output layer has 10 neurons for the 10 classes and a softmax activation function to output probability-like predictions for each class.\n",
    "\n",
    "As before, the model is trained using logarithmic loss and the ADAM gradient descent algorithm.\n",
    "\n",
    "## Dropout Tips\n",
    "- Generally use a small dropout value of 20%-50% of neurons with 20% providing a good starting point. A probability too low has minimal effect and a value too high results in under-learning by the network.\n",
    "- Use a larger network. You are likely to get better performance when dropout is used on a larger network, giving the model more of an opportunity to learn independent representations.\n",
    "- Use dropout on incoming (visible) as well as hidden units. Application of dropout at each layer of the network has shown good results.\n",
    "- Use a large learning rate with decay and a large momentum. Increase your learning rate by a factor of 10 to 100 and use a high momentum value of 0.9 or 0.99.\n",
    "- Constrain the size of network weights. A large learning rate can result in very large network weights. Imposing a constraint on the size of network weights such as max-norm regularization with a size of 4 or 5 has been shown to improve results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def baseline_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Convolution2D(32, 5, 5, border_mode='valid', input_shape=(1, 28, 28), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    # Compile model\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "51s - loss: 0.2500 - acc: 0.9285 - val_loss: 0.0852 - val_acc: 0.9737\n",
      "Epoch 2/10\n",
      "50s - loss: 0.0752 - acc: 0.9781 - val_loss: 0.0625 - val_acc: 0.9795\n",
      "Epoch 3/10\n",
      "49s - loss: 0.0529 - acc: 0.9839 - val_loss: 0.0445 - val_acc: 0.9859\n",
      "Epoch 4/10\n",
      "50s - loss: 0.0415 - acc: 0.9871 - val_loss: 0.0405 - val_acc: 0.9867\n",
      "Epoch 5/10\n",
      "51s - loss: 0.0335 - acc: 0.9901 - val_loss: 0.0350 - val_acc: 0.9882\n",
      "Epoch 6/10\n",
      "50s - loss: 0.0272 - acc: 0.9919 - val_loss: 0.0360 - val_acc: 0.9885\n",
      "Epoch 7/10\n",
      "50s - loss: 0.0239 - acc: 0.9920 - val_loss: 0.0318 - val_acc: 0.9895\n",
      "Epoch 8/10\n",
      "50s - loss: 0.0192 - acc: 0.9939 - val_loss: 0.0327 - val_acc: 0.9897\n",
      "Epoch 9/10\n",
      "51s - loss: 0.0180 - acc: 0.9943 - val_loss: 0.0376 - val_acc: 0.9875\n",
      "Epoch 10/10\n",
      "51s - loss: 0.0147 - acc: 0.9950 - val_loss: 0.0331 - val_acc: 0.9895\n",
      "Baseline Error: 1.05%\n"
     ]
    }
   ],
   "source": [
    "# build the model\n",
    "model = baseline_model()\n",
    "# Fit the model\n",
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), nb_epoch=10, batch_size=200, verbose=2)\n",
    "# Final evaluation of the model\n",
    "scores = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"Baseline Error: %.2f%%\" % (100-scores[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running the example, the accuracy on the training and validation test is printed each epoch and at the end of the classification error rate is printed.\n",
    "\n",
    "Epochs may take 60 to 90 seconds to run on the CPU, or about 15 minutes in total depending on your hardware. You can see that the network achieves an error rate of 1.10, which is better than our simple multi-layer perceptron model above."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Larger Convolutional Neural Network for MNIST\n",
    "\n",
    "Now that we have seen how to create a simple CNN, let’s take a look at a model capable of close to state of the art results.\n",
    "\n",
    "We import classes and function then load and prepare the data the same as in the previous CNN example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Flatten\n",
    "from keras.layers.convolutional import Convolution2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K\n",
    "K.set_image_dim_ordering('th')\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# load data\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "# reshape to be [samples][pixels][width][height]\n",
    "X_train = X_train.reshape(X_train.shape[0], 1, 28, 28).astype('float32')\n",
    "X_test = X_test.reshape(X_test.shape[0], 1, 28, 28).astype('float32')\n",
    "\n",
    "# normalize inputs from 0-255 to 0-1\n",
    "X_train = X_train / 255\n",
    "X_test = X_test / 255\n",
    "# one hot encode outputs\n",
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)\n",
    "num_classes = y_test.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This time we define a large CNN architecture with additional convolutional, max pooling layers and fully connected layers. The network topology can be summarized as follows.\n",
    "\n",
    "- Convolutional layer with 30 feature maps of size 5×5.\n",
    "- Pooling layer taking the max over 2*2 patches.\n",
    "- Convolutional layer with 15 feature maps of size 3×3.\n",
    "- Pooling layer taking the max over 2*2 patches.\n",
    "- Dropout layer with a probability of 20%.\n",
    "- Flatten layer.\n",
    "- Fully connected layer with 128 neurons and rectifier activation.\n",
    "- Fully connected layer with 50 neurons and rectifier activation.\n",
    "- Output layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def larger_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Convolution2D(30, 5, 5, border_mode='valid', input_shape=(1, 28, 28), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Convolution2D(15, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(50, activation='relu'))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    # Compile model\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "53s - loss: 0.4190 - acc: 0.8697 - val_loss: 0.0930 - val_acc: 0.9723\n",
      "Epoch 2/10\n",
      "52s - loss: 0.1073 - acc: 0.9674 - val_loss: 0.0560 - val_acc: 0.9815\n",
      "Epoch 3/10\n",
      "53s - loss: 0.0772 - acc: 0.9762 - val_loss: 0.0432 - val_acc: 0.9847\n",
      "Epoch 4/10\n",
      "54s - loss: 0.0598 - acc: 0.9816 - val_loss: 0.0353 - val_acc: 0.9890\n",
      "Epoch 5/10\n",
      "55s - loss: 0.0510 - acc: 0.9845 - val_loss: 0.0318 - val_acc: 0.9892\n",
      "Epoch 6/10\n",
      "52s - loss: 0.0441 - acc: 0.9863 - val_loss: 0.0303 - val_acc: 0.9910\n",
      "Epoch 7/10\n",
      "51s - loss: 0.0403 - acc: 0.9873 - val_loss: 0.0305 - val_acc: 0.9897\n",
      "Epoch 8/10\n",
      "51s - loss: 0.0365 - acc: 0.9886 - val_loss: 0.0267 - val_acc: 0.9908\n",
      "Epoch 9/10\n",
      "52s - loss: 0.0327 - acc: 0.9897 - val_loss: 0.0300 - val_acc: 0.9898\n",
      "Epoch 10/10\n",
      "51s - loss: 0.0308 - acc: 0.9901 - val_loss: 0.0296 - val_acc: 0.9907\n",
      "Baseline Error: 0.93%\n"
     ]
    }
   ],
   "source": [
    "# build the model\n",
    "model = larger_model()\n",
    "# Fit the model\n",
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), nb_epoch=10, batch_size=200, verbose=2)\n",
    "# Final evaluation of the model\n",
    "scores = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"Baseline Error: %.2f%%\" % (100-scores[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running the example prints accuracy on the training and validation datasets each epoch and a final classification error rate.\n",
    "\n",
    "The model takes about 100 seconds to run per epoch. This slightly larger model achieves the respectable classification error rate of 0.89%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}