net-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(0)\n",
    "torch.cuda.manual_seed(0)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import torchvision.datasets\n",
    "MNIST_train = torchvision.datasets.MNIST('./', download=True, train=True)\n",
    "MNIST_test = torchvision.datasets.MNIST('./', download=True, train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\programs\\anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:58: UserWarning: train_data has been renamed data\n",
      "  warnings.warn(\"train_data has been renamed data\")\n",
      "D:\\programs\\anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:48: UserWarning: train_labels has been renamed targets\n",
      "  warnings.warn(\"train_labels has been renamed targets\")\n",
      "D:\\programs\\anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:63: UserWarning: test_data has been renamed data\n",
      "  warnings.warn(\"test_data has been renamed data\")\n",
      "D:\\programs\\anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:53: UserWarning: test_labels has been renamed targets\n",
      "  warnings.warn(\"test_labels has been renamed targets\")\n"
     ]
    }
   ],
   "source": [
    "X_train = MNIST_train.train_data\n",
    "y_train = MNIST_train.train_labels\n",
    "X_test = MNIST_test.test_data\n",
    "y_test = MNIST_test.test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.uint8, torch.int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.dtype, y_train.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X_train = X_train.float()\n",
    "X_test = X_test.float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([60000, 28, 28]), torch.Size([10000, 28, 28]))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([60000]), torch.Size([10000]))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAOX0lEQVR4nO3dbYxc5XnG8euKbUwxJvHGseMQFxzjFAg0Jl0ZkBFQoVCCIgGKCLGiiFBapwlOQutKUFoVWtHKrRIiSimSKS6m4iWQgPAHmsSyECRqcFmoAROHN+MS4+0aswIDIfZ6fffDjqsFdp5dZs68eO//T1rNzLnnzLk1cPmcmeeceRwRAjD5faDTDQBoD8IOJEHYgSQIO5AEYQeSmNrOjR3i6XGoZrRzk0Aqv9Fb2ht7PFatqbDbPkfS9ZKmSPrXiFhVev6hmqGTfVYzmwRQsDE21K01fBhve4qkGyV9TtLxkpbZPr7R1wPQWs18Zl8i6fmI2BoReyXdJem8atoCULVmwn6kpF+Nery9tuwdbC+33We7b0h7mtgcgGY0E/axvgR4z7m3EbE6InojoneapjexOQDNaCbs2yXNH/X445J2NNcOgFZpJuyPSlpke4HtQyR9SdK6atoCULWGh94iYp/tFZJ+rJGhtzUR8XRlnQGoVFPj7BHxgKQHKuoFQAtxuiyQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJNDWLK7qfp5b/E0/5yOyWbv+ZPz+6bm34sP3FdY9auLNYP+wbLtb/97pD6tYe7/1+cd1dw28V6yffs7JYP+bPHinWO6GpsNveJukNScOS9kVEbxVNAaheFXv234+IXRW8DoAW4jM7kESzYQ9JP7H9mO3lYz3B9nLbfbb7hrSnyc0BaFSzh/FLI2KH7TmS1tv+ZUQ8PPoJEbFa0mpJOsI90eT2ADSoqT17ROyo3e6UdJ+kJVU0BaB6DYfd9gzbMw/cl3S2pM1VNQagWs0cxs+VdJ/tA69zR0T8qJKuJpkpxy0q1mP6tGJ9xxkfKtbfPqX+mHDPB8vjxT/9dHm8uZP+49czi/V/+OdzivWNJ95Rt/bi0NvFdVcNfLZY/9hPD75PpA2HPSK2Svp0hb0AaCGG3oAkCDuQBGEHkiDsQBKEHUiCS1wrMHzmZ4r16269sVj/5LT6l2JOZkMxXKz/9Q1fLdanvlUe/jr1nhV1azNf3ldcd/qu8tDcYX0bi/VuxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnL0C05/ZUaw/9pv5xfonpw1U2U6lVvafUqxvfbP8U9S3LvxB3drr+8vj5HP/6T+L9VY6+C5gHR97diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IwhHtG1E8wj1xss9q2/a6xeAlpxbru88p/9zzlCcPL9af+MYN77unA67d9bvF+qNnlMfRh197vViPU+v/APG2bxVX1YJlT5SfgPfYGBu0OwbHnMuaPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMvvDxfrwq4PF+ot31B8rf/r0NcV1l/z9N4v1OTd27ppyvH9NjbPbXmN7p+3No5b12F5v+7na7awqGwZQvYkcxt8q6d2z3l8paUNELJK0ofYYQBcbN+wR8bCkdx9Hnidpbe3+WknnV9sWgKo1+gXd3Ijol6Ta7Zx6T7S93Haf7b4h7WlwcwCa1fJv4yNidUT0RkTvNE1v9eYA1NFo2Adsz5Ok2u3O6loC0AqNhn2dpItr9y+WdH817QBolXF/N972nZLOlDTb9nZJV0taJelu25dKeknSha1scrIb3vVqU+sP7W58fvdPffkXxforN00pv8D+8hzr6B7jhj0iltUpcXYMcBDhdFkgCcIOJEHYgSQIO5AEYQeSYMrmSeC4K56tW7vkxPKgyb8dtaFYP+PCy4r1md9/pFhH92DPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM4+CZSmTX7168cV131p3dvF+pXX3las/8UXLyjW478/WLc2/+9+XlxXbfyZ8wzYswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEkzZnNzgH55arN9+9XeK9QVTD21425+6bUWxvujm/mJ939ZtDW97smpqymYAkwNhB5Ig7EAShB1IgrADSRB2IAnCDiTBODuKYuniYv2IVduL9Ts/8eOGt33sg39UrP/O39S/jl+Shp/b2vC2D1ZNjbPbXmN7p+3No5ZdY/tl25tqf+dW2TCA6k3kMP5WSeeMsfx7EbG49vdAtW0BqNq4YY+IhyUNtqEXAC3UzBd0K2w/WTvMn1XvSbaX2+6z3TekPU1sDkAzGg37TZIWSlosqV/Sd+s9MSJWR0RvRPRO0/QGNwegWQ2FPSIGImI4IvZLulnSkmrbAlC1hsJue96ohxdI2lzvuQC6w7jj7LbvlHSmpNmSBiRdXXu8WFJI2ibpaxFRvvhYjLNPRlPmzinWd1x0TN3axiuuL677gXH2RV9+8exi/fXTXi3WJ6PSOPu4k0RExLIxFt/SdFcA2orTZYEkCDuQBGEHkiDsQBKEHUiCS1zRMXdvL0/ZfJgPKdZ/HXuL9c9/8/L6r33fxuK6Byt+ShoAYQeyIOxAEoQdSIKwA0kQdiAJwg4kMe5Vb8ht/2mLi/UXLixP2XzC4m11a+ONo4/nhsGTivXD7u9r6vUnG/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE4+yTnHtPKNaf/VZ5rPvmpWuL9dMPLV9T3ow9MVSsPzK4oPwC+8f9dfNU2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsx8Epi44qlh/4ZKP1a1dc9FdxXW/cPiuhnqqwlUDvcX6Q9efUqzPWlv+3Xm807h7dtvzbT9oe4vtp21/u7a8x/Z628/Vbme1vl0AjZrIYfw+SSsj4jhJp0i6zPbxkq6UtCEiFknaUHsMoEuNG/aI6I+Ix2v335C0RdKRks6TdOBcyrWSzm9RjwAq8L6+oLN9tKSTJG2UNDci+qWRfxAkzamzznLbfbb7hrSnyXYBNGrCYbd9uKQfSro8InZPdL2IWB0RvRHRO03TG+kRQAUmFHbb0zQS9Nsj4t7a4gHb82r1eZJ2tqZFAFUYd+jNtiXdImlLRFw3qrRO0sWSVtVu729Jh5PA1KN/u1h//ffmFesX/e2PivU/+dC9xXorrewvD4/9/F/qD6/13PpfxXVn7WdorUoTGWdfKukrkp6yvam27CqNhPxu25dKeknShS3pEEAlxg17RPxM0piTu0s6q9p2ALQKp8sCSRB2IAnCDiRB2IEkCDuQBJe4TtDUeR+tWxtcM6O47tcXPFSsL5s50FBPVVjx8mnF+uM3LS7WZ/9gc7He8wZj5d2CPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJJFmnH3vH5R/tnjvnw4W61cd80Dd2tm/9VZDPVVlYPjturXT160srnvsX/2yWO95rTxOvr9YRTdhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSaQZZ992fvnftWdPvKdl277xtYXF+vUPnV2se7jej/uOOPbaF+vWFg1sLK47XKxiMmHPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJOCLKT7DnS7pN0kc1cvny6oi43vY1kv5Y0iu1p14VEfUv+pZ0hHviZDPxK9AqG2ODdsfgmCdmTOSkmn2SVkbE47ZnSnrM9vpa7XsR8Z2qGgXQOhOZn71fUn/t/hu2t0g6stWNAajW+/rMbvtoSSdJOnAO5grbT9peY3tWnXWW2+6z3TekPc11C6BhEw677cMl/VDS5RGxW9JNkhZKWqyRPf93x1ovIlZHRG9E9E7T9OY7BtCQCYXd9jSNBP32iLhXkiJiICKGI2K/pJslLWldmwCaNW7YbVvSLZK2RMR1o5bPG/W0CySVp/ME0FET+TZ+qaSvSHrK9qbasqskLbO9WFJI2ibpay3oD0BFJvJt/M8kjTVuVxxTB9BdOIMOSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQxLg/JV3pxuxXJP3PqEWzJe1qWwPvT7f21q19SfTWqCp7OyoiPjJWoa1hf8/G7b6I6O1YAwXd2lu39iXRW6Pa1RuH8UAShB1IotNhX93h7Zd0a2/d2pdEb41qS28d/cwOoH06vWcH0CaEHUiiI2G3fY7tZ2w/b/vKTvRQj+1ttp+yvcl2X4d7WWN7p+3No5b12F5v+7na7Zhz7HWot2tsv1x77zbZPrdDvc23/aDtLbaftv3t2vKOvneFvtryvrX9M7vtKZKelfRZSdslPSppWUT8oq2N1GF7m6TeiOj4CRi2T5f0pqTbIuKE2rJ/lDQYEatq/1DOiogruqS3ayS92elpvGuzFc0bPc24pPMlfVUdfO8KfX1RbXjfOrFnXyLp+YjYGhF7Jd0l6bwO9NH1IuJhSYPvWnyepLW1+2s18j9L29XprStERH9EPF67/4akA9OMd/S9K/TVFp0I+5GSfjXq8XZ113zvIeknth+zvbzTzYxhbkT0SyP/80ia0+F+3m3cabzb6V3TjHfNe9fI9OfN6kTYx5pKqpvG/5ZGxGckfU7SZbXDVUzMhKbxbpcxphnvCo1Of96sToR9u6T5ox5/XNKODvQxpojYUbvdKek+dd9U1AMHZtCt3e7scD//r5um8R5rmnF1wXvXyenPOxH2RyUtsr3A9iGSviRpXQf6eA/bM2pfnMj2DElnq/umol4n6eLa/Ysl3d/BXt6hW6bxrjfNuDr83nV8+vOIaPufpHM18o38C5L+shM91OnrE5KeqP093eneJN2pkcO6IY0cEV0q6cOSNkh6rnbb00W9/bukpyQ9qZFgzetQb6dp5KPhk5I21f7O7fR7V+irLe8bp8sCSXAGHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4k8X+zhHFo7nUhhwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(5)\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(X_train[0, :, :])\n",
    "plt.show()\n",
    "print(y_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape([-1, 28 * 28])\n",
    "X_test = X_test.reshape([-1, 28 * 28])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class MNISTNet(torch.nn.Module):\n",
    "    def __init__(self, n_hidden_neurons):\n",
    "        super(MNISTNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(28 * 28, n_hidden_neurons)\n",
    "        self.ac1 = torch.nn.Sigmoid()\n",
    "        self.fc2 = torch.nn.Linear(n_hidden_neurons, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = self.ac1(x)\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "mnist_net = MNISTNet(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\"nvidia-smi\" ­Ґ пў«пҐвбп ў­гваҐ­­Ґ© Ё«Ё ў­Ґи­Ґ©\n",
      "Є®¬ ­¤®©, ЁбЇ®«­пҐ¬®© Їа®Ја ¬¬®© Ё«Ё Ї ЄҐв­л¬ д ©«®¬.\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "mnist_net = mnist_net.to(device)\n",
    "#list(mnist_net.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "loss = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(mnist_net.parameters(), lr=1.0e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.9624, device='cuda:0')\n",
      "tensor(0.9626, device='cuda:0')\n",
      "tensor(0.9627, device='cuda:0')\n",
      "tensor(0.9614, device='cuda:0')\n",
      "tensor(0.9622, device='cuda:0')\n",
      "tensor(0.9620, device='cuda:0')\n",
      "tensor(0.9616, device='cuda:0')\n",
      "tensor(0.9643, device='cuda:0')\n",
      "tensor(0.9633, device='cuda:0')\n",
      "tensor(0.9607, device='cuda:0')\n",
      "tensor(0.9604, device='cuda:0')\n",
      "tensor(0.9620, device='cuda:0')\n",
      "tensor(0.9636, device='cuda:0')\n",
      "tensor(0.9621, device='cuda:0')\n",
      "tensor(0.9630, device='cuda:0')\n",
      "tensor(0.9593, device='cuda:0')\n",
      "tensor(0.9623, device='cuda:0')\n",
      "tensor(0.9634, device='cuda:0')\n",
      "tensor(0.9617, device='cuda:0')\n",
      "tensor(0.9621, device='cuda:0')\n",
      "tensor(0.9628, device='cuda:0')\n",
      "tensor(0.9634, device='cuda:0')\n",
      "tensor(0.9638, device='cuda:0')\n",
      "tensor(0.9631, device='cuda:0')\n",
      "tensor(0.9614, device='cuda:0')\n",
      "tensor(0.9644, device='cuda:0')\n",
      "tensor(0.9661, device='cuda:0')\n",
      "tensor(0.9634, device='cuda:0')\n",
      "tensor(0.9654, device='cuda:0')\n",
      "tensor(0.9632, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "\n",
    "test_accuracy_history = []\n",
    "test_loss_history = []\n",
    "\n",
    "X_test = X_test.to(device)\n",
    "y_test = y_test.to(device)\n",
    "\n",
    "for epoch in range(30):\n",
    "    order = np.random.permutation(len(X_train))\n",
    "\n",
    "    for start_index in range(0, len(X_train), batch_size):\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        batch_indexes = order[start_index:start_index+batch_size]\n",
    "\n",
    "        X_batch = X_train[batch_indexes].to(device)\n",
    "        y_batch = y_train[batch_indexes].to(device)\n",
    "\n",
    "        preds = mnist_net.forward(X_batch)\n",
    "\n",
    "        loss_value = loss(preds, y_batch)\n",
    "        loss_value.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "    test_preds = mnist_net.forward(X_test)\n",
    "    test_loss_history.append(loss(test_preds, y_test))\n",
    "\n",
    "    accuracy = (test_preds.argmax(dim=1) == y_test).float().mean()\n",
    "    test_accuracy_history.append(accuracy)\n",
    "\n",
    "    print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2128085ee50>]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD6CAYAAABebNdxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXU0lEQVR4nO3de4xc513G8ed3zpnZqy+xvc7FdmIXOU3SkqTtNq0EhVTQJmmFQlFBSRC9CBQiNaj8gZQICVqokABRVKFeogBRWgkaVWraBhSaFkEpqBS8LmkSJ00wiRNvnMTr2M56r3P78cd75rLj2d1ZZ+z1vPl+pKP33Gbmfc/lOe85u7Nr7i4AQByS9a4AAKB3CHUAiAihDgARIdQBICKEOgBEhFAHgIisGupmdp+ZHTWzJ5ZZbmb2V2Z20MweM7O3976aAIBuZF2sc7+kz0v6yjLLb5K0Nx/eJelLebmibdu2+e7du7uqJAAg2L9//zF3H1tu+aqh7u7fN7PdK6xys6SvePgW0w/NbLOZXezuL630vrt379bExMRqHw8AaGFmz6+0vBfP1HdIOtwyPZnP61SZ281swswmpqamevDRAIBWvQh16zCv498ecPd73X3c3cfHxpa9ewAAnKFehPqkpF0t0zslHenB+wIA1qgXof6QpI/kvwXzbkmvrfY8HQBwdqz6g1Iz+6qk6yVtM7NJSZ+SVJAkd79H0sOSPiDpoKQ5SR8/W5UFAKysm99+uXWV5S7pEz2rEQDgjPGNUgCISDdfPkKfqdZcMwsV1dxVyBJliamYJkqSTr+oBPS/xUpVr82XNbdY1UKlqvlSVfPlqhbLNc2Xq1ooh+n5UlWLlZoqVdfoYKYNg5k2Dha0cTDTxqGCNgxm2jAYykK6tM9bq7kWKlXNLob3mS1VNFeqai4vK1XX8ECq0YFMw8V6mWl0INNgIZHZuTn/+i7U/+eFE7r/B4fkHn5v0t3z8bxsHVeYrv+GZf2fPNVf11wSfi+zkCYaKKQqpokGCkkoszAUs0QDWapilshMKlddlWpNlZqrUnVVazWVa23z8s8wSYmZzMJ4feeGaVNiUpKE5YmZUgvzzCxMJ81xVwjs6YWyTi1UND0fylMt82YWKx23XZpYI+DrYV/I21g/wDcMLD2wm0OYrl8c6vWrt63RvrwN7uFEq59YC+Vwsi3Ux/OTbTE/4RbKtXx5+7Lm/ErVVcwSDRbSsF/ycrClHMzCvpOkStVVqoYTuFKrLdln5WqYNkmbh4vaOlLUltG8zIetIwPaOhrGBwupJGmhXNX0fFmvzZd1cr6s1+bCeOsws1hRqVJTuVpTqVJTqVprfF5jfj4vS8L2HyqmGiqEYTAfhoqJBrNUQ8VwTLrCBdvdVXOp5uEYcw+BUx+v1lyVWjgmQ5lPVzvPr9VLD8dtzcOyav6elWrzM8Nn5OP5a7xel5rLJWWJNY61NEma0+nS+WGfJaHdxVSDWarBYn0bJI1tkZhpeqG5fafnT9/mC+VaryKmYaiQasNgppp7Ht7VM36vxKSRYqbhgVQjA5luu+5S/dZ73tTD2jb1XaifnCvr0cMnG+HYGpShDEFZvygmLQG6pGxZxyTVXI2TcLExVBvT3SikpixJlKXWOJAlNS4w9RPAPRz8aptfc8+H5rxOssRaehWhp7F727A2Dha0YbCgjUMhhNP84lMPtnK1GSj16VI1tG9moaJTC2UdnV5sXBiWuzj0UjFLGidxCOUwPlBItXm4uGR+llpjfyyUq41yer7cmF4sV7WQ76/6/iikpixtXsSyNJSDhUTu0uSJOT02eVLHZ0uq1Dpv9JFiqkrNVz0WNgyEfVLMOwKFNAz1dm7Ml9XnV2ve6EkulKuaXihrvtS8+M3nQ+uxUL9w1i+s9U5A/cKapUlLeLaWyZJwTS2UA4VEQ9ZcL7GwTpLPS5L6Z5iSpHkRr3dGkpbPl6lxkVjpYlKuuhYr4YJ/bKa0ZBvUe9ntu2J0INOmoYI2DhW0aSjTnm0j2jRUWDKMDGThgti4ODYvmI2LZSFVYtLsYtje0wtlTc+H4/9Uhw5TmpqGC6mG8x74SDHVUDHLyxDSQ4VUhTTRXKmi2cXQi59drGi2VA3lYj5/saLZUkXbRgde13mzkr4L9fdesV3vvWL7Of1Md2+E42I5XK3rIZGlzZPlbHxurTXsa+GEHsjOza1cteaaWWwe7KcWKipXa0suQPU7o1otvxCpeRc0kKUhMFpOqMFCkgd36F2fT4+E3F3TCxUdny3p+Oyijs2U8vGSXp0pKUutESqb28JkU36RzdLe/5jKPYRia5DHrn7OzZerqtZcG8/Ctt00nGjTcKGn73k+6LtQXw9mpmJmKmaJRgfO3SYLvTAp7fil3bMvTawRWG8EZs327tk2st7VaTAzFdL4g7xV6zmHtWGLAUBECHUAiAihDgARIdQBICKEOgBEhFAHgIgQ6gAQEUIdACJCqANARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABEh1AEgIoQ6AESEUAeAiBDqABARQh0AIkKoA0BECHUAiAihDgARIdQBICKEOgBEhFAHgIgQ6gAQEUIdACLSVaib2Y1m9rSZHTSzuzss32Rm/2BmPzazA2b28d5XFQCwmlVD3cxSSV+QdJOkqyTdamZXta32CUlPuvs1kq6X9FkzK/a4rgCAVXTTU79O0kF3f9bdS5IekHRz2zouaYOZmaRRScclVXpaUwDAqroJ9R2SDrdMT+bzWn1e0pWSjkh6XNIn3b3W/kZmdruZTZjZxNTU1BlWGQCwnG5C3TrM87bpGyQ9KukSSddK+ryZbTztRe73uvu4u4+PjY2tsaoAgNV0E+qTkna1TO9U6JG3+rikBz04KOk5SVf0pooAgG51E+r7JO01sz35Dz9vkfRQ2zovSPoFSTKzCyW9WdKzvawoAGB12WoruHvFzO6U9IikVNJ97n7AzO7Il98j6TOS7jezxxUe19zl7sfOYr0BAB2sGuqS5O4PS3q4bd49LeNHJL2/t1UDAKwV3ygFgIgQ6gAQEUIdACJCqANARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABEh1AEgIoQ6AESEUAeAiBDqABARQh0AIkKoA0BECHUAiAihDgARIdQBICKEOgBEhFAHgIgQ6gAQEUIdACJCqANARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABHpKtTN7EYze9rMDprZ3cusc72ZPWpmB8zs33pbTQBAN7LVVjCzVNIXJL1P0qSkfWb2kLs/2bLOZklflHSju79gZtvPUn0BACvopqd+naSD7v6su5ckPSDp5rZ1bpP0oLu/IEnufrS31QQAdKObUN8h6XDL9GQ+r9Xlki4ws++Z2X4z+0inNzKz281swswmpqamzqzGAIBldRPq1mGet01nkt4h6YOSbpD0B2Z2+Wkvcr/X3cfdfXxsbGzNlQUArGzVZ+oKPfNdLdM7JR3psM4xd5+VNGtm35d0jaRnelJLAEBXuump75O018z2mFlR0i2SHmpb51uS3mNmmZkNS3qXpKd6W1UAwGpW7am7e8XM7pT0iKRU0n3ufsDM7siX3+PuT5nZtyU9Jqkm6W/c/YmzWXEAwOnMvf3x+LkxPj7uExMT6/LZANCvzGy/u48vt5xvlAJARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABEh1AEgIoQ6AESEUAeAiBDqABARQh0AIkKoA0BECHUAiAihDgARIdQBICKEOgBEhFAHgIgQ6gAQEUIdACJCqANARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABEh1AEgIoQ6AESEUAeAiHQV6mZ2o5k9bWYHzezuFdZ7p5lVzezDvasiAKBbq4a6maWSviDpJklXSbrVzK5aZr0/k/RIrysJAOhONz316yQddPdn3b0k6QFJN3dY73ckfV3S0R7WDwCwBt2E+g5Jh1umJ/N5DWa2Q9KHJN2z0huZ2e1mNmFmE1NTU2utKwBgFd2EunWY523Tn5N0l7tXV3ojd7/X3cfdfXxsbKzLKgIAupV1sc6kpF0t0zslHWlbZ1zSA2YmSdskfcDMKu7+zV5UEgDQnW5CfZ+kvWa2R9KLkm6RdFvrCu6+pz5uZvdL+kcCHQDOvVVD3d0rZnanwm+1pJLuc/cDZnZHvnzF5+gAgHOnm5663P1hSQ+3zesY5u7+sddfLQDAmeAbpQAQEUIdACJCqANARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABEh1AEgIoQ6AESEUAeAiBDqABARQh0AIkKoA0BECHUAiAihDgARIdQBICKEOgBEhFAHgIgQ6gAQEUIdACJCqANARAh1AIgIoQ4AESHUASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCICKEOABHpKtTN7EYze9rMDprZ3R2W/7qZPZYPPzCza3pfVQDAalYNdTNLJX1B0k2SrpJ0q5ld1bbac5J+3t2vlvQZSff2uqIAgNV101O/TtJBd3/W3UuSHpB0c+sK7v4Ddz+RT/5Q0s7eVhMA0I1uQn2HpMMt05P5vOX8pqR/6rTAzG43swkzm5iamuq+lgCArnQT6tZhnndc0ey9CqF+V6fl7n6vu4+7+/jY2Fj3tQQAdCXrYp1JSbtapndKOtK+kpldLelvJN3k7q/2pnoAgLXopqe+T9JeM9tjZkVJt0h6qHUFM7tU0oOSfsPdn+l9NQEA3Vi1p+7uFTO7U9IjklJJ97n7ATO7I19+j6Q/lLRV0hfNTJIq7j5+9qoNAOjE3Ds+Hj/rxsfHfWJiYl0+GwD6lZntX6nTzDdKASAihDoARIRQB4CIEOoAEBFCHQAiQqgDQEQIdQCISDd/JgB4Y6uWpRf3S9mgdOFbpLSw3jUClkWoo3ulWckSqTC03jU5++aOSwf/WXrm26FceC3Mz4akS94m7XqntDMfNly0vnUFWvRfqC+ekmaPSZsvkxKeHvWUuzRzVDrxnHT8OenEoZbx56TZ/M8lF0elkbGWYZs0ur05PrJdGt4qDYyGdQvDUjYgWac/+HmecJemng4h/swj0uEfSl4Lbbnyl6S97w899skJaXKf9J9flGrl8NpNu6Sd482Qv/ia0F5gHfRfqP/fv0hf+0gIirE3S2NXStuvkLZfJY1dIW3a2X14uIeLxNyroWe2cCL0yOZPhnLhZOfpaiV8hiXLD0kiyVrq0jYuLZ32qlQthfeulkJgVFuGWjnMl6TRi6SNl+TDjtPHN1zUfETQ3sa5V6X5vKwPM1MhtE8ckspzLRvIwntu2SNdfqN0we5Q59ljIfxnp8JrJvdJc8dCCC7HUqk40hwKwyHwiyNScThMF4bycrjzvMKglGRhsDQfT9qm0zDU2+61Zqn6dK05f/ao9Mx3QpiffD687qKrpff8XmjzJW9b2nn46Q+Hsrwgvfx4aPvkvhD2B77R3G6j26UNF4dh48Wdx4cuCNvTPezb8lx43/KcVFmQyvPNoVYJ6w9vkYa2hPG0/05fuUulmeadT1oMx2paDEOSnT8X/8VT0snDoXOycUfzuDrP9d/ffjl5OAT71E+ko09KR38izbzcXF7cEMJ++5VhyAak2XqAHQvlbMt4PSg7SYvS4GZpcFMYhvLxtNgWDm1B0TothXn1P0Hf2N6+dFmSSUkhP8ALzQO8cdAXmp976mVp+kVp+kgYlgSxFELlwvC+c8ebPcp2loaQGBkLdz5b9kgX7AnhvWWPtPnS7nuctao0fyIE/exUCP7yXHhkU5rJy7nm+JJlc3l4zTWHcykblN50vXT5DdLeG6RNK/0PmBWcejmE+8uPS6eOSNMvSafyYa7DX6POBsM+qMyvfEFczuCmEPDDW8KdUX28MJQfS9nS4yrJmmV9eT1MlwyF08tapWUf5Ree8nyoe+vFpzQrLU7nnaAOw+L06m09rR4DoU3FYamQdwKKI83xwnCzozCwIYTwwMZ8PB+Ko6Fs/XlIeUF67bB04nnp5KG8fCFc3E88Hzo/dUkW7sguuCycH5vz8oLLwjlTv0BLUqUU8mX2WF6+2jZ9TLrig9K1t619n2v1v/3Sf6HeydzxPOSfapZHnwobsG5gkzSyNRz8w9uWjg/n40ObmyE+tDk/6c6TXsNy3MMdRD3gG2H/YrhjqLet0caWEBjYeH4+wnIPPdXS3NKgLy+EcPFqKGu1tulqPlTC+yy5e7Lm3ZVa7rKKI9Kud4VwOJsqiyHcW4P+1EuhrYWhcKw17kzyIWsZtyRcNOdPhOO9cbfVOn4ijJfnwzZZL8XRZkdocFM4zlqnBzeFgLWk5Y60lA/ltrIUtl2jEzAnlWebx0a9g7BS56xVNpR/tkkzryxdlhZDR2bzpXloXxaCvDQTQv7EoWbgt2aLFDqTI1vDPlh8rfNnN87HbdI7Pia9+461btnwNm+IUF/O7LH8tnWLlBXP7mcB55P6Ba9WDmW10gzQWiWfV2p5xNceqm3jaWHpRea0C0/+eCwbWp/HQtVKCPvFmfDYpDQT7goWT3UeapVmz7se4KMXdd/JWTwVevUnDjUDf/54yJqRbWEYbisHN/ekE7VaqPfhQ7k1GNm23jUA1keSSElR0hukM5NmUprfBZwLAxvCr7de+JZz83lrcB7eewMAzhShDgARIdQBICKEOgBEhFAHgIgQ6gAQEUIdACJCqANARNbtG6VmNiXp+TN8+TZJx1Zdq7/E1qbY2iPF16bY2iPF16ZO7bnM3ceWe8G6hfrrYWYTK31Nth/F1qbY2iPF16bY2iPF16YzaQ+PXwAgIoQ6AESkX0P93vWuwFkQW5tia48UX5tia48UX5vW3J6+fKYOAOisX3vqAIAOCHUAiEjfhbqZ3WhmT5vZQTO7e73r0wtmdsjMHjezR83sLP87qN4zs/vM7KiZPdEyb4uZfdfM/jcvL1jPOq7VMm36tJm9mO+nR83sA+tZx7Uws11m9q9m9pSZHTCzT+bz+3I/rdCeft5Hg2b232b247xNf5TPX9M+6qtn6maWSnpG0vskTUraJ+lWd39yXSv2OpnZIUnj7t6XX5ows5+TNCPpK+7+1nzen0s67u5/ml98L3D3u9aznmuxTJs+LWnG3f9iPet2JszsYkkXu/uPzGyDpP2SflnSx9SH+2mF9vya+ncfmaQRd58xs4Kk/5D0SUm/ojXso37rqV8n6aC7P+vuJUkPSLp5nev0hufu35d0vG32zZK+nI9/WeGE6xvLtKlvuftL7v6jfPyUpKck7VCf7qcV2tO3PJjJJwv54FrjPuq3UN8h6XDL9KT6fEfmXNJ3zGy/md2+3pXpkQvd/SUpnICStq9zfXrlTjN7LH880xePKtqZ2W5Jb5P0X4pgP7W1R+rjfWRmqZk9KumopO+6+5r3Ub+FunWY1z/Pj5b3M+7+dkk3SfpEfuuP88+XJP2UpGslvSTps+tamzNgZqOSvi7pd919er3r83p1aE9f7yN3r7r7tZJ2SrrOzN661vfot1CflLSrZXqnpCPrVJeecfcjeXlU0jcUHjP1u1fy5571559H17k+r5u7v5KfdDVJf60+20/5c9qvS/o7d38wn923+6lTe/p9H9W5+0lJ35N0o9a4j/ot1PdJ2mtme8ysKOkWSQ+tc51eFzMbyX/QIzMbkfR+SU+s/Kq+8JCkj+bjH5X0rXWsS0/UT6zch9RH+yn/IdzfSnrK3f+yZVFf7qfl2tPn+2jMzDbn40OSflHST7TGfdRXv/0iSfmvKH1OUirpPnf/k/Wt0etjZm9S6J1LUibp7/utTWb2VUnXK/yZ0FckfUrSNyV9TdKlkl6Q9Kvu3jc/eFymTdcr3Na7pEOSfrv+rPN8Z2Y/K+nfJT0uqZbP/n2F59B9t59WaM+t6t99dLXCD0JThQ7319z9j81sq9awj/ou1AEAy+u3xy8AgBUQ6gAQEUIdACJCqANARAh1AIgIoQ4AESHUASAi/w/FksWAhP+3/QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(test_accuracy_history)\n",
    "plt.plot(test_loss_history)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}