Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import math\n",
    "\n",
    "from sklearn.datasets import fetch_openml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import math\n",
    "from sklearn.datasets import fetch_openml\n",
    "\n",
    "def relu(X):\n",
    "    return np.maximum(X, 0)\n",
    "\n",
    "def relu_derivative(X):\n",
    "    return 1. * (X > 0)\n",
    "\n",
    "def build_model(X,hidden_nodes,output_dim=2):\n",
    "    model = {}\n",
    "    input_dim = X.shape[1]\n",
    "    model['W1'] = np.random.randn(input_dim, hidden_nodes) / np.sqrt(input_dim)\n",
    "    model['b1'] = np.zeros((1, hidden_nodes))\n",
    "    model['W2'] = np.random.randn(hidden_nodes, hidden_nodes) / np.sqrt(hidden_nodes)\n",
    "    model['b2'] = np.zeros((1, hidden_nodes))\n",
    "    model['W3'] = np.random.randn(hidden_nodes, output_dim) / np.sqrt(hidden_nodes)\n",
    "    model['b3'] = np.zeros((1, output_dim))\n",
    "    return model\n",
    "\n",
    "def feed_forward(model, x):\n",
    "    W1, b1, W2, b2, W3, b3 = model['W1'], model['b1'], model['W2'], model['b2'], model['W3'], model['b3']\n",
    "    # Forward propagation\n",
    "    z1 = x.dot(W1) + b1\n",
    "    #a1 = np.tanh(z1)\n",
    "    a1 = relu(z1)\n",
    "    z2 = a1.dot(W2) + b2\n",
    "    a2 = relu(z2)\n",
    "    z3 = a2.dot(W3) + b3\n",
    "    exp_scores = np.exp(z3)\n",
    "    out = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)\n",
    "    return z1, a1, z2, a2, z3, out\n",
    "\n",
    "def calculate_loss(model,X,y,reg_lambda):\n",
    "    num_examples = X.shape[0]\n",
    "    W1, b1, W2, b2, W3, b3 = model['W1'], model['b1'], model['W2'], model['b2'], model['W3'], model['b3']\n",
    "    # Forward propagation to calculate our predictions\n",
    "    z1, a1, z2, a2, z3, out = feed_forward(model, X)\n",
    "    probs = out / np.sum(out, axis=1, keepdims=True)\n",
    "    # Calculating the loss\n",
    "    corect_logprobs = -np.log(probs[range(num_examples), y])\n",
    "    loss = np.sum(corect_logprobs)\n",
    "    # Add regulatization term to loss (optional)\n",
    "    loss += reg_lambda/2 * (np.sum(np.square(W1)) + np.sum(np.square(W2)) + np.sum(np.square(W3)))\n",
    "    return 1./num_examples * loss\n",
    "\n",
    "def backprop(X,y,model,z1,a1,z2,a2,z3,output,reg_lambda):\n",
    "    delta3 = output\n",
    "    delta3[range(X.shape[0]), y] -= 1  #yhat - y\n",
    "    dW3 = (a2.T).dot(delta3)\n",
    "    db3 = np.sum(delta3, axis=0, keepdims=True)\n",
    "    delta2 = delta3.dot(model['W3'].T) * relu_derivative(a2) #if ReLU\n",
    "    dW2 = np.dot(a1.T, delta2)\n",
    "    db2 = np.sum(delta2, axis=0)\n",
    "    #delta2 = delta3.dot(model['W2'].T) * (1 - np.power(a1, 2)) #if tanh\n",
    "    delta1 = delta2.dot(model['W2'].T) * relu_derivative(a1) #if ReLU\n",
    "    dW1 = np.dot(X.T, delta1)\n",
    "    db1 = np.sum(delta1, axis=0)\n",
    "    # Add regularization terms\n",
    "    dW3 += reg_lambda * model['W3']\n",
    "    dW2 += reg_lambda * model['W2']\n",
    "    dW1 += reg_lambda * model['W1']\n",
    "    return dW1, dW2, dW3, db1, db2, db3\n",
    "\n",
    "\n",
    "def train(model, X, y, num_passes=10000, reg_lambda = .1, learning_rate=0.1):\n",
    "    # Batch gradient descent\n",
    "    done = False\n",
    "    previous_loss = float('inf')\n",
    "    i = 0\n",
    "    losses = []\n",
    "    while done == False:  #comment out while performance testing\n",
    "    #while i < 1500:\n",
    "        #feed forward\n",
    "        z1,a1,z2,a2,z3,output = feed_forward(model, X)\n",
    "        #backpropagation\n",
    "        dW1, dW2, dW3, db1, db2, db3 = backprop(X,y,model,z1,a1,z2,a2,z3,output,reg_lambda)\n",
    "        #update weights and biases\n",
    "        model['W1'] -= learning_rate * dW1\n",
    "        model['b1'] -= learning_rate * db1\n",
    "        model['W2'] -= learning_rate * dW2\n",
    "        model['b2'] -= learning_rate * db2\n",
    "        model['W3'] -= learning_rate * dW3\n",
    "        model['b3'] -= learning_rate * db3\n",
    "        if i % 1000 == 0:\n",
    "            loss = calculate_loss(model, X, y, reg_lambda)\n",
    "            losses.append(loss)\n",
    "            print \"Loss after iteration %i: %f\" %(i, loss)  #uncomment once testing finished, return mod val to 1000\n",
    "            if (previous_loss-loss)/previous_loss < 0.01:\n",
    "                done = True\n",
    "                #print i\n",
    "            previous_loss = loss\n",
    "        i += 1\n",
    "    return model, losses\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "IOError",
     "evalue": "[Errno 2] No such file or directory: 'samples/train-labels-idx1-ubyte'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mIOError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-d0dea23fc677>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmndata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMNIST\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'samples'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmndata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;31m# or\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmndata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_testing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/mnist/loader.pyc\u001b[0m in \u001b[0;36mload_training\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    124\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mload_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         ims, labels = self.load(os.path.join(self.path, self.train_img_fname),\n\u001b[0;32m--> 126\u001b[0;31m                                 os.path.join(self.path, self.train_lbl_fname))\n\u001b[0m\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/mnist/loader.pyc\u001b[0m in \u001b[0;36mload\u001b[0;34m(self, path_img, path_lbl, batch)\u001b[0m\n\u001b[1;32m    245\u001b[0m                                  '(start_point, batch_size)')\n\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_lbl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    248\u001b[0m             \u001b[0mmagic\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\">II\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    249\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmagic\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m2049\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/mnist/loader.pyc\u001b[0m in \u001b[0;36mopener\u001b[0;34m(self, path_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m    237\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mgzip\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_fn\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.gz'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 239\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath_img\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath_lbl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIOError\u001b[0m: [Errno 2] No such file or directory: 'samples/train-labels-idx1-ubyte'"
     ]
    }
   ],
   "source": [
    "#toy dataset\n",
    "from mnist import MNIST\n",
    "\n",
    "mndata = MNIST('samples')\n",
    "\n",
    "X, y = mndata.load_training()\n",
    "# or\n",
    "images, labels = mndata.load_testing()\n",
    "\n",
    "num_examples = len(X) # training set size\n",
    "nn_input_dim = 2 # input layer dimensionality\n",
    "nn_output_dim = 2 # output layer dimensionality \n",
    "learning_rate = 0.01 # learning rate for gradient descent\n",
    "reg_lambda = 0.01 # regularization strength\n",
    "model = build_model(X,20,2)\n",
    "model, losses = train(model,X, y, reg_lambda=reg_lambda, learning_rate=learning_rate)\n",
    "output = feed_forward(model, X)\n",
    "preds = np.argmax(output[3], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named google.protobuf",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mImportError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-fcf8fef2ec03>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/keras/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m//\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \"\"\"\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdistribute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/keras/distribute/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistribute\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msidecar_evaluator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/keras/distribute/sidecar_evaluator.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\"\"\"Python module for evaluation loop.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv2\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;31m# isort: off\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/tensorflow/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m \u001b[0;31m# We still need all the names that are toplevel on tensorflow_core\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_core\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;31m# These should not be visible in the main tf module.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/tensorflow_core/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtools\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmodule_util\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_module_util\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0m_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0maudio\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0m_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mautodiff\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0m_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mautograph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/tensorflow_core/_api/v2/audio/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_sys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen_audio_ops\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdecode_wav\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen_audio_ops\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mencode_wav\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/tensorflow_core/python/ops/gen_audio_ops.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpywrap_tensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_pywrap_tensorflow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meager\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcontext\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meager\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcore\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meager\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexecute\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_execute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/tensorflow_core/python/eager/context.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotobuf\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconfig_pb2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotobuf\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrewriter_config_pb2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nao/.local/lib/python2.7/site-packages/tensorflow_core/core/protobuf/config_pb2.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0m_b\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0;36m3\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'latin1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotobuf\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdescriptor\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_descriptor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotobuf\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_message\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotobuf\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mreflection\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_reflection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named google.protobuf"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.datasets import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_X, train_y), (test_X, test_y) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0  12  56 140 126 175 200  96   2\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0  35 166 238 254 246 242 253 246 254  67\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0 184 182 146 127  70  30  45  36 215 175\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0  30   0   0   0   0   0   0   0 207 246\n",
      "   14   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  55 251 169\n",
      "    1   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0  11 215 232  20\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0  20 190 250  61   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0  24 118 206 254 248 142 108\n",
      "   18   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0  63 223 254 254 254 254 254 254\n",
      "  209   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0  52 174 129  95  16  16  16 106\n",
      "  249 125   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "  179 239   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "   80 239   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "   80 244  20   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "  100 239   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "  234 239   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   4 140   5   0   0   0   0   0   0   3 150\n",
      "  254 129   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0  64 254 181  38   0   0   0   0  34 188 254\n",
      "  209  20   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0  12 226 255 223  88  68 128 157 242 254 207\n",
      "   23   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0  45 210 254 254 254 254 255 254 187  49\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0  41 129 239 229 179  91  16   3   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGdCAYAAAC7EMwUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAavElEQVR4nO3df3DUdZ7n8VeThBa5Tq85TLojMZtyYPSApW6AAVIogR1y5kYWjFOFujsVamc4fwTuuGhZwzBVUHN1xGIGlrlCmRvLQlhhYHYXgTpYMR4kjIvMRhZPjrG4cIQhHsmmiNgdInYI+dwfHH02xOC36eadTj8fVd8q0/190x+/fvHJl+5843POOQEAYGCE9QIAANmLCAEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADO51gu4UX9/v86fP69AICCfz2e9HACAR845dXd3q7i4WCNGDH6tM+QidP78eZWUlFgvAwBwm9ra2jR27NhB9xlyEQoEApKkWfq3ylWe8WoAAF716Yre0/74/88Hk7YIvfrqq/rZz36m9vZ2TZgwQRs2bNDDDz98y7nrfwWXqzzl+ogQAGSc/3dH0q/zlkpaPpiwc+dOLV++XCtXrtTx48f18MMPq6qqSufOnUvHywEAMlRaIrR+/Xr94Ac/0A9/+EM99NBD2rBhg0pKSrRp06Z0vBwAIEOlPEK9vb06duyYKisrEx6vrKzUkSNHbto/FospGo0mbACA7JDyCF24cEFXr15VUVFRwuNFRUXq6Oi4af/6+noFg8H4xifjACB7pO2bVW98Q8o5N+CbVCtWrFAkEolvbW1t6VoSAGCISfmn48aMGaOcnJybrno6OztvujqSJL/fL7/fn+plAAAyQMqvhEaOHKkpU6aooaEh4fGGhgaVl5en+uUAABksLd8nVFdXp+9///uaOnWqZs6cqV/96lc6d+6cnn322XS8HAAgQ6UlQosWLVJXV5d++tOfqr29XRMnTtT+/ftVWlqajpcDAGQon3POWS/iy6LRqILBoCq0gDsmAEAG6nNX1Kg9ikQiys/PH3RffpQDAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJghQgAAM0QIAGCGCAEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJghQgAAM0QIAGCGCAEAzBAhAIAZIgQAMJNrvQDgVnLuucfzzOXp30jqtc5We5/J6c7xPDN2UofnmSn/8pznmXffnOF5RpJCv/id96H+q0m9FrIbV0IAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBluYIqkjZj8kOeZf/5P/Z5n/vpP3vA882Ce3/OMJHX1X/Y809PvPM+MzR3leeZi/xeeZ9a++IHnGUma3fac55nRf5vETU+R9bgSAgCYIUIAADMpj9Dq1avl8/kStlAolOqXAQAMA2l5T2jChAl6991341/n5Hj/oV8AgOEvLRHKzc3l6gcAcEtpeU+opaVFxcXFKisr05NPPqkzZ8585b6xWEzRaDRhAwBkh5RHaPr06dq6dasOHDig1157TR0dHSovL1dXV9eA+9fX1ysYDMa3kpKSVC8JADBEpTxCVVVVeuKJJzRp0iR95zvf0b59+yRJW7ZsGXD/FStWKBKJxLe2trZULwkAMESl/ZtVR48erUmTJqmlpWXA5/1+v/z+5L6xEACQ2dL+fUKxWEwff/yxwuFwul8KAJBhUh6hF198UU1NTWptbdXvfvc7fe9731M0GlVNTU2qXwoAkOFS/tdxn3zyiZ566ilduHBB9957r2bMmKGjR4+qtLQ01S8FAMhwKY/Qjh07Uv1LYojy/5dPPc8UO+8X3/Mbl3qe8X2a53lGkoqSuAfnH3008Cc/B9NXMNrzTE5Pr+eZBdubPM9IUu6/+2fvQ3+b1Eshy3HvOACAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADATNp/qB2Gr95n8z3PXP144B9uOJhxavc8cyddTWLGl8RMfxIzF/oCSUxJv3lom+eZxWP+zPPM1Qveb/6K4YUrIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJjhLtpIWjJ3xEbyev/NVM8zdQWvJvVaFf9jseeZe7pOJ/VayG5cCQEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADBDhAAAZriBKWAg54+CnmcW/dXfe5453pvcb/F7n/nc80yfc0m9FrIbV0IAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBluYArcptyx93meuec3PZ5n/iL/f3ue+e5z/97zjCTd9ck/JjUHeMWVEADADBECAJjxHKHDhw9r/vz5Ki4uls/n0+7duxOed85p9erVKi4u1qhRo1RRUaGTJ0+mar0AgGHEc4R6eno0efJkbdy4ccDn165dq/Xr12vjxo1qbm5WKBTSvHnz1N3dfduLBQAML54/mFBVVaWqqqoBn3POacOGDVq5cqWqq6slSVu2bFFRUZG2b9+uZ5555vZWCwAYVlL6nlBra6s6OjpUWVkZf8zv92v27Nk6cuTIgDOxWEzRaDRhAwBkh5RGqKOjQ5JUVFSU8HhRUVH8uRvV19crGAzGt5KSklQuCQAwhKXl03E+ny/ha+fcTY9dt2LFCkUikfjW1taWjiUBAIaglH6zaigUknTtiigcDscf7+zsvOnq6Dq/3y+/35/KZQAAMkRKr4TKysoUCoXU0NAQf6y3t1dNTU0qLy9P5UsBAIYBz1dCly5d0unTp+Nft7a26sMPP1RBQYHuv/9+LV++XGvWrNG4ceM0btw4rVmzRnfffbeefvrplC4cAJD5PEfogw8+0Jw5c+Jf19XVSZJqamr0xhtv6KWXXtLly5f1/PPP6+LFi5o+fbreeecdBQKB1K0aADAs+JxzznoRXxaNRhUMBlWhBcr15VkvBxkqt6w0qbmWJcWeZ/78u02eZ34y5n96non2f+F55lsHkruB6aizIz3PlL1+xvNMX/vAn5pFZutzV9SoPYpEIsrPzx90X+4dBwAwQ4QAAGaIEADADBECAJghQgAAM0QIAGCGCAEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADMp/cmqQDpcXvhtzzP/Ye2OpF5r4ejPkpq7E/JH3OV55nTVr9KwkoH9fNE3Pc8cnDQ6DStBJuFKCABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwww1MMeTldV/1PPOL1j9N6rVe+rjI88y/OOv9z3L3/fq055k76Q8/+IbnmSPPr/M889rP/6PnmQdePOp5BkMXV0IAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBmfc85ZL+LLotGogsGgKrRAub486+UA+Lr++1jPIxse+I3nmeV/XO55BndWn7uiRu1RJBJRfn7+oPtyJQQAMEOEAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmMm1XgCA4eHTN0u8D61K/TqQWbgSAgCYIUIAADOeI3T48GHNnz9fxcXF8vl82r17d8Lzixcvls/nS9hmzJiRqvUCAIYRzxHq6enR5MmTtXHjxq/c59FHH1V7e3t8279//20tEgAwPHn+YEJVVZWqqqoG3cfv9ysUCiW9KABAdkjLe0KNjY0qLCzU+PHjtWTJEnV2dn7lvrFYTNFoNGEDAGSHlEeoqqpK27Zt08GDB7Vu3To1Nzdr7ty5isViA+5fX1+vYDAY30pKkviYJwAgI6X8+4QWLVoU/+eJEydq6tSpKi0t1b59+1RdXX3T/itWrFBdXV3862g0SogAIEuk/ZtVw+GwSktL1dLSMuDzfr9ffr8/3csAAAxBaf8+oa6uLrW1tSkcDqf7pQAAGcbzldClS5d0+vTp+Netra368MMPVVBQoIKCAq1evVpPPPGEwuGwzp49qx//+McaM2aMHn/88ZQuHACQ+TxH6IMPPtCcOXPiX19/P6empkabNm3SiRMntHXrVn322WcKh8OaM2eOdu7cqUAgkLpVAwCGBc8RqqiokHPuK58/cODAbS0IQPYIjOj3PJM79j7PM32f/B/PM7gzuHccAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJghQgAAM0QIAGCGCAEAzKT9J6sCyA5fjPF5nunu9/7nYO6IPbxwJQQAMEOEAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmOEGpgBS4vXnf2G9BGQgroQAAGaIEADADBECAJghQgAAM0QIAGCGCAEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADPcwBTy5Y1Mau7UK5M9z3xz2UeeZ1ws5nkG1/hyk/st3vLGJM8zU0b+k+eZ8X+zzPPMN3TU8wyGLq6EAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAz3MAU6nnsXyc1d/q7mzzPzB/3mOeZ/hfv8Tzjjp30PDPUjfiTBz3PBDd1JvVa/+uPX/c88/NPv+l55sGft3me6fM8gaGMKyEAgBkiBAAw4ylC9fX1mjZtmgKBgAoLC7Vw4UKdOnUqYR/nnFavXq3i4mKNGjVKFRUVOnly+P3VCADg9nmKUFNTk2pra3X06FE1NDSor69PlZWV6unpie+zdu1arV+/Xhs3blRzc7NCoZDmzZun7u7ulC8eAJDZPH0w4e233074evPmzSosLNSxY8f0yCOPyDmnDRs2aOXKlaqurpYkbdmyRUVFRdq+fbueeeaZ1K0cAJDxbus9oUgkIkkqKCiQJLW2tqqjo0OVlZXxffx+v2bPnq0jR44M+GvEYjFFo9GEDQCQHZKOkHNOdXV1mjVrliZOnChJ6ujokCQVFRUl7FtUVBR/7kb19fUKBoPxraSkJNklAQAyTNIRWrp0qT766CP9+te/vuk5n8+X8LVz7qbHrluxYoUikUh8a2vz/n0DAIDMlNQ3qy5btkx79+7V4cOHNXbs2PjjoVBI0rUronA4HH+8s7Pzpquj6/x+v/x+fzLLAABkOE9XQs45LV26VLt27dLBgwdVVlaW8HxZWZlCoZAaGhrij/X29qqpqUnl5eWpWTEAYNjwdCVUW1ur7du3a8+ePQoEAvH3eYLBoEaNGiWfz6fly5drzZo1GjdunMaNG6c1a9bo7rvv1tNPP52WfwEAQObyFKFNm67dK6yioiLh8c2bN2vx4sWSpJdeekmXL1/W888/r4sXL2r69Ol65513FAgEUrJgAMDw4XPOOetFfFk0GlUwGFSFFijXl2e9nKyQc4/3G4RKUm3z+55nvnv3F55n/u5SvueZ//xXf+55RpJGXej3PNNRPvCHbgaTd1/PrXe6wd9P937D2Ptz7/Y8I0n1Xf/K88z7fzbe80zf2XOeZzD09bkratQeRSIR5ecP/vuXe8cBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADATFI/WRXDy9WLF5Oa+0XNk55nfv9ff+t5pu6eFs8zC3/yiueZOynH5/3Pf1ed9ztiP9U6z/OMJH36k1LPMzln/ymp10J240oIAGCGCAEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADDDDUyRNN8/fOh55t2/LPc888unvd+E878tXO95RpLOXw14nll7tsrzTOdb93ueCf/Nac8z/V2fep6RpJw+bkaKO4MrIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADAjM8556wX8WXRaFTBYFAVWqBcX571cgAAHvW5K2rUHkUiEeXn5w+6L1dCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJghQgAAM0QIAGCGCAEAzBAhAIAZIgQAMEOEAABmiBAAwIynCNXX12vatGkKBAIqLCzUwoULderUqYR9Fi9eLJ/Pl7DNmDEjpYsGAAwPniLU1NSk2tpaHT16VA0NDerr61NlZaV6enoS9nv00UfV3t4e3/bv35/SRQMAhodcLzu//fbbCV9v3rxZhYWFOnbsmB555JH4436/X6FQKDUrBAAMW7f1nlAkEpEkFRQUJDze2NiowsJCjR8/XkuWLFFnZ+dX/hqxWEzRaDRhAwBkh6Qj5JxTXV2dZs2apYkTJ8Yfr6qq0rZt23Tw4EGtW7dOzc3Nmjt3rmKx2IC/Tn19vYLBYHwrKSlJdkkAgAzjc865ZAZra2u1b98+vffeexo7duxX7tfe3q7S0lLt2LFD1dXVNz0fi8USAhWNRlVSUqIKLVCuLy+ZpQEADPW5K2rUHkUiEeXn5w+6r6f3hK5btmyZ9u7dq8OHDw8aIEkKh8MqLS1VS0vLgM/7/X75/f5klgEAyHCeIuSc07Jly/TWW2+psbFRZWVlt5zp6upSW1ubwuFw0osEAAxPnt4Tqq2t1Ztvvqnt27crEAioo6NDHR0dunz5siTp0qVLevHFF/X+++/r7Nmzamxs1Pz58zVmzBg9/vjjafkXAABkLk9XQps2bZIkVVRUJDy+efNmLV68WDk5OTpx4oS2bt2qzz77TOFwWHPmzNHOnTsVCARStmgAwPDg+a/jBjNq1CgdOHDgthYEAMge3DsOAGCGCAEAzBAhAIAZIgQAMEOEAABmiBAAwAwRAgCYIUIAADNECABghggBAMwQIQCAGSIEADBDhAAAZogQAMAMEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJghQgAAM0QIAGAm13oBN3LOSZL6dEVyxosBAHjWpyuS/v//zwcz5CLU3d0tSXpP+41XAgC4Hd3d3QoGg4Pu43NfJ1V3UH9/v86fP69AICCfz5fwXDQaVUlJidra2pSfn2+0Qnsch2s4DtdwHK7hOFwzFI6Dc07d3d0qLi7WiBGDv+sz5K6ERowYobFjxw66T35+flafZNdxHK7hOFzDcbiG43CN9XG41RXQdXwwAQBghggBAMxkVIT8fr9WrVolv99vvRRTHIdrOA7XcByu4Thck2nHYch9MAEAkD0y6koIADC8ECEAgBkiBAAwQ4QAAGYyKkKvvvqqysrKdNddd2nKlCn67W9/a72kO2r16tXy+XwJWygUsl5W2h0+fFjz589XcXGxfD6fdu/enfC8c06rV69WcXGxRo0apYqKCp08edJmsWl0q+OwePHim86PGTNm2Cw2Terr6zVt2jQFAgEVFhZq4cKFOnXqVMI+2XA+fJ3jkCnnQ8ZEaOfOnVq+fLlWrlyp48eP6+GHH1ZVVZXOnTtnvbQ7asKECWpvb49vJ06csF5S2vX09Gjy5MnauHHjgM+vXbtW69ev18aNG9Xc3KxQKKR58+bF70M4XNzqOEjSo48+mnB+7N8/vO7B2NTUpNraWh09elQNDQ3q6+tTZWWlenp64vtkw/nwdY6DlCHng8sQ3/72t92zzz6b8NiDDz7ofvSjHxmt6M5btWqVmzx5svUyTElyb731Vvzr/v5+FwqF3Msvvxx/7IsvvnDBYND98pe/NFjhnXHjcXDOuZqaGrdgwQKT9Vjp7Ox0klxTU5NzLnvPhxuPg3OZcz5kxJVQb2+vjh07psrKyoTHKysrdeTIEaNV2WhpaVFxcbHKysr05JNP6syZM9ZLMtXa2qqOjo6Ec8Pv92v27NlZd25IUmNjowoLCzV+/HgtWbJEnZ2d1ktKq0gkIkkqKCiQlL3nw43H4bpMOB8yIkIXLlzQ1atXVVRUlPB4UVGROjo6jFZ1502fPl1bt27VgQMH9Nprr6mjo0Pl5eXq6uqyXpqZ6//9s/3ckKSqqipt27ZNBw8e1Lp169Tc3Ky5c+cqFotZLy0tnHOqq6vTrFmzNHHiREnZeT4MdBykzDkfhtxdtAdz4492cM7d9NhwVlVVFf/nSZMmaebMmXrggQe0ZcsW1dXVGa7MXrafG5K0aNGi+D9PnDhRU6dOVWlpqfbt26fq6mrDlaXH0qVL9dFHH+m999676blsOh++6jhkyvmQEVdCY8aMUU5Ozk1/kuns7LzpTzzZZPTo0Zo0aZJaWlqsl2Lm+qcDOTduFg6HVVpaOizPj2XLlmnv3r06dOhQwo9+ybbz4auOw0CG6vmQEREaOXKkpkyZooaGhoTHGxoaVF5ebrQqe7FYTB9//LHC4bD1UsyUlZUpFAolnBu9vb1qamrK6nNDkrq6utTW1jaszg/nnJYuXapdu3bp4MGDKisrS3g+W86HWx2HgQzZ88HwQxGe7Nixw+Xl5bnXX3/d/f73v3fLly93o0ePdmfPnrVe2h3zwgsvuMbGRnfmzBl39OhR99hjj7lAIDDsj0F3d7c7fvy4O378uJPk1q9f744fP+7+8Ic/OOece/nll10wGHS7du1yJ06ccE899ZQLh8MuGo0arzy1BjsO3d3d7oUXXnBHjhxxra2t7tChQ27mzJnuvvvuG1bH4bnnnnPBYNA1Nja69vb2+Pb555/H98mG8+FWxyGTzoeMiZBzzr3yyiuutLTUjRw50n3rW99K+DhiNli0aJELh8MuLy/PFRcXu+rqanfy5EnrZaXdoUOHnKSbtpqaGufctY/lrlq1yoVCIef3+90jjzziTpw4YbvoNBjsOHz++eeusrLS3XvvvS4vL8/df//9rqamxp07d8562Sk10L+/JLd58+b4PtlwPtzqOGTS+cCPcgAAmMmI94QAAMMTEQIAmCFCAAAzRAgAYIYIAQDMECEAgBkiBAAwQ4QAAGaIEADADBECAJghQgAAM0QIAGDm/wJqcvE8eMUPRwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_X[50])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand=np.arange(60000)\n",
    "np.random.shuffle(rand)\n",
    "train_no=rand[:50000]\n",
    "\n",
    "val_no=np.setdiff1d(rand,train_no)\n",
    "\n",
    "X_train,X_val = train_X[train_no,:,:], train_X[val_no,:,:]\n",
    "Y_train,Y_val = train_y[train_no], train_y[val_no]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init(x,y):\n",
    "    \n",
    "    layer=np.random.uniform(-1.,1.,size=(x,y))/np.sqrt(x*y)\n",
    "    return layer.astype(np.float32)\n",
    "\n",
    "np.random.seed(42)\n",
    "l1=init(28*28,128)\n",
    "l2=init(128,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sigmoid funstion\n",
    "def sigmoid(x):\n",
    "    return 1/(np.exp(-x)+1)    \n",
    "\n",
    "#derivative of sigmoid\n",
    "def d_sigmoid(x):\n",
    "    return (np.exp(-x))/((np.exp(-x)+1)**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def softmax(x):\n",
    "#    exponents=np.exp(x)\n",
    "#    return exponents/np.sum(exponents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Softmax\n",
    "def softmax(x):\n",
    "    exp_element=np.exp(x-x.max())\n",
    "    return exp_element/np.sum(exp_element,axis=0)\n",
    "\n",
    "#derivative of softmax\n",
    "def d_softmax(x):\n",
    "    exp_element=np.exp(x-x.max())\n",
    "    return exp_element/np.sum(exp_element,axis=0)*(1-exp_element/np.sum(exp_element,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#forward and backward pass\n",
    "def forward_backward_pass(x,y):\n",
    "    targets = np.zeros((len(y),10), np.float32)\n",
    "    targets[range(targets.shape[0]),y] = 1\n",
    " \n",
    "    \n",
    "    x_l1=x.dot(l1)\n",
    "    x_sigmoid=sigmoid(x_l1)\n",
    "    x_l2=x_sigmoid.dot(l2)\n",
    "    out=softmax(x_l2)\n",
    "   \n",
    " \n",
    "    error=2*(out-targets)/out.shape[0]*d_softmax(x_l2)\n",
    "    update_l2=x_sigmoid.T@error\n",
    "    \n",
    "    \n",
    "    error=((l2).dot(error.T)).T*d_sigmoid(x_l1)\n",
    "    update_l1=x.T@error\n",
    "\n",
    "    return out,update_l1,update_l2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For 0th epoch: train accuracy: 0.266 | validation accuracy:0.305\n",
      "For 10th epoch: train accuracy: 0.445 | validation accuracy:0.475\n",
      "For 20th epoch: train accuracy: 0.609 | validation accuracy:0.571\n",
      "For 30th epoch: train accuracy: 0.633 | validation accuracy:0.633\n",
      "For 40th epoch: train accuracy: 0.641 | validation accuracy:0.676\n",
      "For 50th epoch: train accuracy: 0.664 | validation accuracy:0.701\n",
      "For 60th epoch: train accuracy: 0.703 | validation accuracy:0.719\n",
      "For 70th epoch: train accuracy: 0.758 | validation accuracy:0.730\n",
      "For 80th epoch: train accuracy: 0.750 | validation accuracy:0.742\n",
      "For 90th epoch: train accuracy: 0.766 | validation accuracy:0.755\n",
      "For 100th epoch: train accuracy: 0.758 | validation accuracy:0.757\n",
      "For 110th epoch: train accuracy: 0.766 | validation accuracy:0.763\n",
      "For 120th epoch: train accuracy: 0.773 | validation accuracy:0.771\n",
      "For 130th epoch: train accuracy: 0.750 | validation accuracy:0.776\n",
      "For 140th epoch: train accuracy: 0.789 | validation accuracy:0.782\n",
      "For 150th epoch: train accuracy: 0.773 | validation accuracy:0.782\n",
      "For 160th epoch: train accuracy: 0.750 | validation accuracy:0.788\n",
      "For 170th epoch: train accuracy: 0.789 | validation accuracy:0.789\n",
      "For 180th epoch: train accuracy: 0.766 | validation accuracy:0.788\n",
      "For 190th epoch: train accuracy: 0.852 | validation accuracy:0.792\n",
      "For 200th epoch: train accuracy: 0.758 | validation accuracy:0.794\n",
      "For 210th epoch: train accuracy: 0.773 | validation accuracy:0.799\n",
      "For 220th epoch: train accuracy: 0.758 | validation accuracy:0.803\n",
      "For 230th epoch: train accuracy: 0.766 | validation accuracy:0.804\n",
      "For 240th epoch: train accuracy: 0.836 | validation accuracy:0.804\n",
      "For 250th epoch: train accuracy: 0.789 | validation accuracy:0.806\n",
      "For 260th epoch: train accuracy: 0.742 | validation accuracy:0.808\n",
      "For 270th epoch: train accuracy: 0.828 | validation accuracy:0.809\n",
      "For 280th epoch: train accuracy: 0.820 | validation accuracy:0.810\n",
      "For 290th epoch: train accuracy: 0.758 | validation accuracy:0.810\n"
     ]
    }
   ],
   "source": [
    "epochs = 300\n",
    "lr = 0.01\n",
    "batch = 128\n",
    "\n",
    "losses,accuracies,val_accuracies=[],[],[]\n",
    "\n",
    "for i in range(epochs):\n",
    "    sample=np.random.randint(0,X_train.shape[0],size=(batch))\n",
    "    x=X_train[sample].reshape((-1,28*28))\n",
    "    y=Y_train[sample]\n",
    " \n",
    "\n",
    "    out,update_l1,update_l2=forward_backward_pass(x,y)\n",
    "  \n",
    "    category=np.argmax(out,axis=1)\n",
    "    accuracy=(category==y).mean()\n",
    "    accuracies.append(accuracy)\n",
    "    \n",
    "    loss=((category-y)**2).mean()\n",
    "    losses.append(loss.item())\n",
    "    \n",
    "    l1=l1-lr*update_l1\n",
    "    l2=l2-lr*update_l2\n",
    "    \n",
    "    if(i%2==0):    \n",
    "        X_val=X_val.reshape((-1,28*28))\n",
    "        val_out=np.argmax(softmax(sigmoid(X_val.dot(l1)).dot(l2)),axis=1)\n",
    "        val_acc=(val_out==Y_val).mean()\n",
    "        val_accuracies.append(val_acc.item())\n",
    "    if(i%10==0): print(f'For {i}th epoch: train accuracy: {accuracy:.3f} | validation accuracy:{val_acc:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2.7.12 64-bit",
   "name": "python2712jvsc74a57bd0767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  },
  "metadata": {
   "interpreter": {
    "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}