Model_Optimization_LeNet_5_MNIST_TensorFlow_Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "e1I-KzFCHOGc"
   },
   "outputs": [],
   "source": [
    "!pip uninstall tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "emg7IXZHHPHd"
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow-gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n28jSoZQHUvN"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ZafjCP0iHXhR",
    "outputId": "ba8d7e45-aca9-4e04-fa80-806d5e581b6d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0'"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "_Dzsnw65Had-",
    "outputId": "7a108b3e-b895-4e19-bd23-1b6c7d953810"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Your GPU is available to TensorFlow\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if tf.test.is_gpu_available():\n",
    "    print(\"\\nYour GPU is available to TensorFlow\\n\")\n",
    "else:\n",
    "    print(\"\\nYour GPU is NOT available to TensorFlow\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "VHgLaFSCHfuD",
    "outputId": "54cf98a2-b134-4338-c5f0-4eee7019882c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is your GPU available for use?\n",
      "Yes, your GPU is available: True\n"
     ]
    }
   ],
   "source": [
    "print(('Is your GPU available for use?\\n{0}').format(\n",
    "    'Yes, your GPU is available: True' if tf.test.is_gpu_available() == True else 'No, your GPU is NOT available: False'\n",
    "))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5o-3nSFkHixk"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.layers import AveragePooling2D, Conv2D\n",
    "from tensorflow.keras import models, layers\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.initializers import RandomNormal\n",
    "# import math\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "x_ofAJhWIWPZ"
   },
   "outputs": [],
   "source": [
    "# Load and prepare the MNIST dataset-\n",
    "mnist = tf.keras.datasets.mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "TyPr2p7ZIY5q",
    "outputId": "5c6a3bea-9bd5-4edf-e410-14f9c546afe6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# Load data from TensorFlow 2.0-\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "# Normalize dataset-\n",
    "X_train, X_test = X_train / 255.0, X_test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zakhvTrvIbWL"
   },
   "outputs": [],
   "source": [
    "# Cast dataset to floating type-\n",
    "X_train = tf.cast(X_train, dtype=tf.float32)\n",
    "X_test = tf.cast(X_test, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bymti4lmId2P"
   },
   "outputs": [],
   "source": [
    "# Perform one-hot encoding for multi-class classification-\n",
    "n_classes = 10\n",
    "\n",
    "y_train_ohe = tf.keras.utils.to_categorical(y_train, n_classes)\n",
    "y_test_ohe = tf.keras.utils.to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "a_ikqJGJIi8u",
    "outputId": "3bae7cea-ab24-43d5-b590-bb35728afdcb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "X_train.shape = (60000, 28, 28, 1) & y_train_ohe.shape = (60000, 10)\n",
      "X_test.shape = (10000, 28, 28, 1) & y_test_ohe.shape = (10000, 10)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Reshape training and testing sets-\n",
    "X_train = tf.reshape(X_train, shape=(X_train.shape[0], 28, 28, 1))\n",
    "X_test = tf.reshape(X_test, shape = (X_test.shape[0], 28, 28, 1))\n",
    "\n",
    "print(\"\\nX_train.shape = {0} & y_train_ohe.shape = {1}\".format(X_train.shape, y_train_ohe.shape))\n",
    "print(\"X_test.shape = {0} & y_test_ohe.shape = {1}\\n\".format(X_test.shape, y_test_ohe.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yoPrY4l3InUP"
   },
   "outputs": [],
   "source": [
    "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train_ohe))\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test_ohe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SV8xU9vkIt9I"
   },
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.shuffle(20000).batch(128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0MUdIPGaI571"
   },
   "source": [
    "Create CNN for LeNet-5 architecture for MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "njXs4-40IxzS"
   },
   "outputs": [],
   "source": [
    "# Define the LeNet-5 CNN model for MNIST classification:\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "# Convolutional layer-1\n",
    "model.add(Conv2D(filters = 6, kernel_size = (5, 5),\n",
    "    strides = (1, 1), activation = 'relu', kernel_initializer=tf.initializers.glorot_normal),\n",
    "    input_shape = (28, 28, 1)))\n",
    "\n",
    "# Average pooling layer-1\n",
    "model.add(AveragePooling2D(pool_size = (2, 2), strides = (2, 2)))\n",
    "\n",
    "# Convolutional layer-2\n",
    "model.add(Conv2D(filters = 16, kernel_size = (5, 5),\n",
    "    strides = (1, 1), activation = 'relu',\n",
    "    kernel_initializer=tf.initializers.glorot_normal)))\n",
    "\n",
    "# Average pooling layer-2\n",
    "model.add(AveragePooling2D(pool_size = (2, 2), strides = (2, 2)))\n",
    "\n",
    "# Flatten the output-\n",
    "model.add(layers.Flatten())\n",
    "\n",
    "# Fully connected layer-\n",
    "model.add(layers.Dense(units = 120, activation = 'relu',\n",
    "  kernel_initializer=tf.initializers.glorot_normal)))\n",
    "\n",
    "# Another flattening of the previous layer-\n",
    "model.add(layers.Flatten())\n",
    "\n",
    "# Fully connected layer-\n",
    "model.add(layers.Dense(units = 84, activation = 'relu'\n",
    "  kernel_initializer=tf.initializers.glorot_normal)))\n",
    "\n",
    "# Output layer-\n",
    "model.add(layers.Dense(units = 10, activation = 'softmax'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gUmVim0MJBI9"
   },
   "outputs": [],
   "source": [
    "# Define an optimizer for the model to choose-\n",
    "optimizer_SGD = tf.keras.optimizers.SGD(lr = 0.001, momentum=0.0, decay = 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qLmp-8LJJaAH"
   },
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    optimizer=optimizer_SGD,\n",
    "    loss=tf.keras.losses.categorical_crossentropy,\n",
    "    metrics = ['accuracy']\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bUQ6PYOlLNV8"
   },
   "outputs": [],
   "source": [
    "# Use early stopping-\n",
    "callback = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='val_loss',\n",
    "    patience = 3,\n",
    "    restore_best_weights = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "KBY19OnMJEKa",
    "outputId": "b86c2b0c-a52a-4de3-b1be-c904d2939830"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "60000/60000 [==============================] - 15s 245us/sample - loss: 2.2729 - accuracy: 0.1347 - val_loss: 2.2170 - val_accuracy: 0.2529\n",
      "Epoch 2/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 1.6955 - accuracy: 0.5599 - val_loss: 0.7774 - val_accuracy: 0.8057\n",
      "Epoch 3/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.5702 - accuracy: 0.8370 - val_loss: 0.4338 - val_accuracy: 0.8690\n",
      "Epoch 4/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.4120 - accuracy: 0.8777 - val_loss: 0.3589 - val_accuracy: 0.8933\n",
      "Epoch 5/100\n",
      "60000/60000 [==============================] - 8s 135us/sample - loss: 0.3538 - accuracy: 0.8938 - val_loss: 0.3125 - val_accuracy: 0.9082\n",
      "Epoch 6/100\n",
      "60000/60000 [==============================] - 8s 141us/sample - loss: 0.3173 - accuracy: 0.9049 - val_loss: 0.2863 - val_accuracy: 0.9103\n",
      "Epoch 7/100\n",
      "60000/60000 [==============================] - 8s 135us/sample - loss: 0.2899 - accuracy: 0.9122 - val_loss: 0.2581 - val_accuracy: 0.9204\n",
      "Epoch 8/100\n",
      "60000/60000 [==============================] - 8s 135us/sample - loss: 0.2684 - accuracy: 0.9195 - val_loss: 0.2485 - val_accuracy: 0.9276\n",
      "Epoch 9/100\n",
      "60000/60000 [==============================] - 8s 137us/sample - loss: 0.2507 - accuracy: 0.9245 - val_loss: 0.2269 - val_accuracy: 0.9305\n",
      "Epoch 10/100\n",
      "60000/60000 [==============================] - 8s 136us/sample - loss: 0.2351 - accuracy: 0.9283 - val_loss: 0.2148 - val_accuracy: 0.9376\n",
      "Epoch 11/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.2225 - accuracy: 0.9327 - val_loss: 0.1976 - val_accuracy: 0.9409\n",
      "Epoch 12/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.2110 - accuracy: 0.9360 - val_loss: 0.1912 - val_accuracy: 0.9412\n",
      "Epoch 13/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.2008 - accuracy: 0.9390 - val_loss: 0.1811 - val_accuracy: 0.9457\n",
      "Epoch 14/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1909 - accuracy: 0.9424 - val_loss: 0.1749 - val_accuracy: 0.9460\n",
      "Epoch 15/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1828 - accuracy: 0.9444 - val_loss: 0.1651 - val_accuracy: 0.9511\n",
      "Epoch 16/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.1755 - accuracy: 0.9470 - val_loss: 0.1630 - val_accuracy: 0.9474\n",
      "Epoch 17/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.1683 - accuracy: 0.9492 - val_loss: 0.1520 - val_accuracy: 0.9510\n",
      "Epoch 18/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.1616 - accuracy: 0.9514 - val_loss: 0.1554 - val_accuracy: 0.9542\n",
      "Epoch 19/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.1554 - accuracy: 0.9536 - val_loss: 0.1497 - val_accuracy: 0.9545\n",
      "Epoch 20/100\n",
      "60000/60000 [==============================] - 8s 135us/sample - loss: 0.1496 - accuracy: 0.9549 - val_loss: 0.1370 - val_accuracy: 0.9580\n",
      "Epoch 21/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.1444 - accuracy: 0.9558 - val_loss: 0.1340 - val_accuracy: 0.9601\n",
      "Epoch 22/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1395 - accuracy: 0.9581 - val_loss: 0.1332 - val_accuracy: 0.9604\n",
      "Epoch 23/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1349 - accuracy: 0.9592 - val_loss: 0.1275 - val_accuracy: 0.9608\n",
      "Epoch 24/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1307 - accuracy: 0.9602 - val_loss: 0.1223 - val_accuracy: 0.9639\n",
      "Epoch 25/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.1264 - accuracy: 0.9619 - val_loss: 0.1177 - val_accuracy: 0.9637\n",
      "Epoch 26/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1239 - accuracy: 0.9625 - val_loss: 0.1200 - val_accuracy: 0.9632\n",
      "Epoch 27/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1198 - accuracy: 0.9638 - val_loss: 0.1135 - val_accuracy: 0.9663\n",
      "Epoch 28/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.1170 - accuracy: 0.9646 - val_loss: 0.1090 - val_accuracy: 0.9651\n",
      "Epoch 29/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1141 - accuracy: 0.9655 - val_loss: 0.1122 - val_accuracy: 0.9667\n",
      "Epoch 30/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1114 - accuracy: 0.9663 - val_loss: 0.1066 - val_accuracy: 0.9667\n",
      "Epoch 31/100\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1082 - accuracy: 0.9671 - val_loss: 0.1027 - val_accuracy: 0.9689\n",
      "Epoch 32/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.1058 - accuracy: 0.9676 - val_loss: 0.1011 - val_accuracy: 0.9686\n",
      "Epoch 33/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.1033 - accuracy: 0.9687 - val_loss: 0.0997 - val_accuracy: 0.9694\n",
      "Epoch 34/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.1012 - accuracy: 0.9692 - val_loss: 0.1028 - val_accuracy: 0.9703\n",
      "Epoch 35/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0993 - accuracy: 0.9707 - val_loss: 0.1026 - val_accuracy: 0.9684\n",
      "Epoch 36/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0969 - accuracy: 0.9701 - val_loss: 0.0995 - val_accuracy: 0.9705\n",
      "Epoch 37/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0954 - accuracy: 0.9706 - val_loss: 0.0923 - val_accuracy: 0.9720\n",
      "Epoch 38/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0932 - accuracy: 0.9716 - val_loss: 0.0910 - val_accuracy: 0.9720\n",
      "Epoch 39/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0916 - accuracy: 0.9720 - val_loss: 0.0919 - val_accuracy: 0.9715\n",
      "Epoch 40/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0900 - accuracy: 0.9726 - val_loss: 0.0904 - val_accuracy: 0.9709\n",
      "Epoch 41/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0879 - accuracy: 0.9736 - val_loss: 0.0969 - val_accuracy: 0.9704\n",
      "Epoch 42/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0869 - accuracy: 0.9736 - val_loss: 0.0863 - val_accuracy: 0.9740\n",
      "Epoch 43/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0853 - accuracy: 0.9742 - val_loss: 0.0900 - val_accuracy: 0.9719\n",
      "Epoch 44/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.0841 - accuracy: 0.9748 - val_loss: 0.0914 - val_accuracy: 0.9709\n",
      "Epoch 45/100\n",
      "60000/60000 [==============================] - 8s 138us/sample - loss: 0.0824 - accuracy: 0.9750 - val_loss: 0.0856 - val_accuracy: 0.9731\n",
      "Epoch 46/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0813 - accuracy: 0.9751 - val_loss: 0.0824 - val_accuracy: 0.9743\n",
      "Epoch 47/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0803 - accuracy: 0.9762 - val_loss: 0.0833 - val_accuracy: 0.9746\n",
      "Epoch 48/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.0787 - accuracy: 0.9760 - val_loss: 0.0827 - val_accuracy: 0.9737\n",
      "Epoch 49/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0775 - accuracy: 0.9761 - val_loss: 0.0801 - val_accuracy: 0.9760\n",
      "Epoch 50/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0765 - accuracy: 0.9766 - val_loss: 0.0809 - val_accuracy: 0.9748\n",
      "Epoch 51/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0751 - accuracy: 0.9771 - val_loss: 0.0799 - val_accuracy: 0.9760\n",
      "Epoch 52/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0741 - accuracy: 0.9773 - val_loss: 0.0802 - val_accuracy: 0.9748\n",
      "Epoch 53/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0730 - accuracy: 0.9782 - val_loss: 0.0771 - val_accuracy: 0.9759\n",
      "Epoch 54/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0722 - accuracy: 0.9783 - val_loss: 0.0735 - val_accuracy: 0.9766\n",
      "Epoch 55/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0714 - accuracy: 0.9780 - val_loss: 0.0729 - val_accuracy: 0.9775\n",
      "Epoch 56/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0703 - accuracy: 0.9790 - val_loss: 0.0735 - val_accuracy: 0.9765\n",
      "Epoch 57/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0688 - accuracy: 0.9790 - val_loss: 0.0750 - val_accuracy: 0.9766\n",
      "Epoch 58/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0682 - accuracy: 0.9789 - val_loss: 0.0725 - val_accuracy: 0.9771\n",
      "Epoch 59/100\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0678 - accuracy: 0.9794 - val_loss: 0.0707 - val_accuracy: 0.9782\n",
      "Epoch 60/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0666 - accuracy: 0.9799 - val_loss: 0.0711 - val_accuracy: 0.9774\n",
      "Epoch 61/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0659 - accuracy: 0.9804 - val_loss: 0.0760 - val_accuracy: 0.9758\n",
      "Epoch 62/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0653 - accuracy: 0.9801 - val_loss: 0.0699 - val_accuracy: 0.9771\n",
      "Epoch 63/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0641 - accuracy: 0.9811 - val_loss: 0.0683 - val_accuracy: 0.9780\n",
      "Epoch 64/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0635 - accuracy: 0.9814 - val_loss: 0.0716 - val_accuracy: 0.9764\n",
      "Epoch 65/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0626 - accuracy: 0.9814 - val_loss: 0.0705 - val_accuracy: 0.9786\n",
      "Epoch 66/100\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.0622 - accuracy: 0.9816 - val_loss: 0.0660 - val_accuracy: 0.9795\n",
      "Epoch 67/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0612 - accuracy: 0.9816 - val_loss: 0.0699 - val_accuracy: 0.9775\n",
      "Epoch 68/100\n",
      "60000/60000 [==============================] - 8s 129us/sample - loss: 0.0610 - accuracy: 0.9811 - val_loss: 0.0672 - val_accuracy: 0.9794\n",
      "Epoch 69/100\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.0600 - accuracy: 0.9819 - val_loss: 0.0668 - val_accuracy: 0.9789\n"
     ]
    }
   ],
   "source": [
    "# Train defined model-\n",
    "history = model.fit(\n",
    "\tx = X_train, y = y_train_ohe,\n",
    "\tepochs = 100, callbacks=[callback],\n",
    "  validation_data = [X_test, y_test_ohe]\n",
    "\t)\n",
    "\n",
    "# https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping?version=stable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_zQT_ZM1Q_LD"
   },
   "outputs": [],
   "source": [
    "score = model.evaluate(X_test, y_test_ohe, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "d0u3-nI-scih",
    "outputId": "af504251-990c-4562-f996-f23f4d4139f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "LeNet-5 CNN MNIST original model metrics on test dataset are:\n",
      "loss = 0.0660 & accuracy = 0.9795\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nLeNet-5 CNN MNIST original model metrics on test dataset are:\")\n",
    "print(\"loss = {0:.4f} & accuracy = {1:.4f}\\n\".format(score[0], score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 527
    },
    "colab_type": "code",
    "id": "Py_LLvc5soqv",
    "outputId": "5a8afc34-5c53-4833-f566-a27d911e84cc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Original LenNet-5 CNN on MNIST dataset is:\n",
      "\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 24, 24, 6)         156       \n",
      "_________________________________________________________________\n",
      "average_pooling2d (AveragePo (None, 12, 12, 6)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 8, 8, 16)          2416      \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 4, 4, 16)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 120)               30840     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 120)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 84)                10164     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                850       \n",
      "=================================================================\n",
      "Total params: 44,426\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nOriginal LenNet-5 CNN on MNIST dataset is:\\n\")\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G_8n_Nh9RqTF"
   },
   "outputs": [],
   "source": [
    "# Make predictions using trained model-\n",
    "y_pred = model.predict_classes(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "7DHWXeEKRtB8",
    "outputId": "4c83f372-fd11-4677-9867-2ec3ee936576"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XVOy0m9kRuep"
   },
   "outputs": [],
   "source": [
    "# Get trained model metrics-\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "precision = precision_score(y_test, y_pred, average = 'macro')\n",
    "recall = recall_score(y_test, y_pred, average = 'macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "UyxAVMGAR30-",
    "outputId": "82ac1f17-ba55-40e3-a44f-d28706881566"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Trained LeNet-5 CNN model metrics are:\n",
      "accuracy = 0.9795, precision = 0.9794 & recall = 0.9793\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nTrained LeNet-5 CNN model metrics are:\")\n",
    "print(\"accuracy = {0:.4f}, precision = {1:.4f} & recall = {2:.4f}\\n\".format(accuracy, precision, recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qE-RsG0GtCdt",
    "outputId": "be1ab4bf-4350-4714-d0bc-ad029c713310"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(history.history['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "o5A4uPz0tl0g"
   },
   "source": [
    "The CNN was trained for 69 epochs after which earlystopping stopped training as 'val_loss' stopped increasing with a patience of 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ssbKsdrLtVPX",
    "outputId": "dae3a9bb-b7d8-4562-9368-21d0c617ccf7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy'])"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 746
    },
    "colab_type": "code",
    "id": "rvaNsLbwKIWw",
    "outputId": "3dab1662-e1dc-457b-d2d8-01aed2e63e18"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA5sAAALZCAYAAAA5jDXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde3yT9f3//2eSK2laKG0pB2EIOCqF\nAm0nIFQpIqci6mTMKQoqbHIYY+CEDd0EwSnTjwwEpj8YOpwymBsiMA9TRFAQxOnQryfKYSJWOVPK\noQ1tDr8/0kYOSRugV0LSx/126y00uXLlRctu23Ov9/v1tvh8Pp8AAAAAAKhF1mgXAAAAAACIP4RN\nAAAAAECtI2wCAAAAAGodYRMAAAAAUOsImwAAAACAWkfYBAAAAADUOsImAAAAAKDWETYBAKjjnn32\nWbVo0SLaZQAA4gxhEwBw0erVq5ceeOCB83rvrl27ZLFYdOmll6q0tDTwvNvtlsVi0bp168K+V+vW\nrfX000/XeN20adNks9lUv379wNdtt91W4/tWrVqlPn36KC0tTSkpKWrXrp0mTpyoPXv2SJKGDx8u\ni8WiefPmnfa+Bx54QL169Qp8H+51AABEAmETABDXXC6XHnvssYh9Xl5eno4fPx74Wrp0abXXP/ro\noxo2bJh+/OMfq7CwUCUlJXrjjTdUr149vf3224HrGjVqpGnTpunw4cPV3i/c6wAAMBthEwAQs159\n9VV169ZNaWlpuvzyyzV37tyzrvn973+vxx9/XF9//XXI+2zdulU33HCDmjZtqu9973saO3asTpw4\nIUm67rrrtHv3bo0bN07169dXhw4daq3+3bt3a8qUKZo9e7bGjh2rJk2aSJJatmyphx56SEOGDAlc\nW1BQoKysLE2dOrXae4Z7XXVcLpcmT56syy67TGlpacrPz9fmzZsDr3/88ce65pprlJqaqrS0NHXu\n3FmFhYWSpLVr16pLly5KSUlRenq6rr76ahUXF593LQCA2EXYBADEpLVr1+r222/XjBkzdOjQIb30\n0kt6/PHH9be//e206/r27av+/fvr17/+ddD7HDx4UPn5+erTp492796tjz/+WNu2bdM999wjSXrt\ntdfUsmVL/elPf9Lx48f12WefVVvXli1b1LhxY7Vq1Uq33367vvzyy5DXvv766/J6vRo2bFhYf+c5\nc+Zo4cKFNdYQ7nWh/PrXv9arr76q1atXa9++fRo0aJD69u2roqIiSdLYsWPVp08fHTx4UAcOHNAz\nzzyj1NRUSdKwYcP0i1/8QkeOHNGePXs0c+ZMORyO86oDABDbCJsAgJg0e/Zs/fznP1efPn1ktVrV\nsWNHjRkzRosWLTrr2pkzZ2rFihXasGHDWa8999xzysjI0K9+9SslJCSoUaNGmj59up577jl5PJ5z\nqunmm2/W559/rv3792vjxo2yWCzq27evjh8/HvT6/fv3q1GjRkpISAjr/ldccYWGDRsWCMIXel0w\nXq9XzzzzjB5++GFlZGTI4XBo4sSJ+v73v6/FixdLkhwOh3bv3q2vvvpKhmEoNzdXTZs2Dby2c+dO\nffvtt3I4HMrLy1O9evXOuQ4AQOwjbAIAYtL27ds1Z84cpaamBr4effTRwFCdU2VkZGjChAmaMGGC\nvF7vWff58MMPT7vPwIEDZbFYtHfv3qCfvXv37tOGAFV1Uzt27KhWrVrJYrHoe9/7nv7yl7/om2++\n0caNG4Pep0mTJjp48KBOnjwZ9t97xowZev/997Vy5cpaue5MBw8eVFlZmdq0aXPa8xkZGdq9e7ck\n//Rai8Wi3r17q0WLFrrnnnsCgXrVqlX63//+p86dOysjI0MPPvig3G73OdUAAIgPhE0AQEy65JJL\ndN999+nIkSOBr2PHjoVcOvrAAw/om2++Oavzeckll6hHjx6n3aekpEQul0vf+973JElW6+n/ddmy\nZcvThgANHTo06GdaLBZZLBb5fL6grxcUFMhqtZ619Lc6TZs21QMPPKBJkyapvLz8gq87U6NGjeR0\nOrVz587Tnt+5c6datmwpSWrVqpUWLlyor776SuvWrdPq1av1hz/8QZLUqVMnLVmyRHv37tWyZcs0\nf/78oN1mAED8I2wCAC5qHo9HLpfrtC+v16sJEyZo3rx5WrNmjdxut9xutz799FO98847Qe+TnJys\nGTNmnHWUyogRI7RlyxY99dRTKi0tlc/n09dff60VK1YErrnkkksCA3Cq849//EMHDx6UJO3bt093\n3323mjZtqquuuiro9VWDgH71q19pwYIFgfd+8803mj59uv7+978Hfd+ECRMkqcYQF+51p7Jarfrp\nT3+qqVOn6n//+5/Ky8s1e/Zs7dixIxCqn332WRUVFcnn86lBgwYyDEOGYai8vFyLFi3SgQMHJEkp\nKSmy2WwyDCPszwcAxA/CJgDgovboo48qMTHxtK+33npLgwYN0vPPP6+pU6eqSZMmatKkie6+++5A\nYAtmxIgRatWq1WnPtWzZUps2bdLq1avVpk0bpaamqqCgQJ988kngmqlTp2rlypVKTU1VdnZ2yPsv\nXrxY7du3V1JSkq644gpVVFTozTffVHJycsj33H///Xruuef0wgsvqE2bNkpJSVGfPn10/PjxkGdj\nOhwO/fGPf6z273ou151p5syZ6t+/v6699lo1adJEL774olavXq1LL71Ukn8405VXXqn69esrJydH\neXl5mjx5siRp2bJl6tChg+rVq6drrrlGw4cP11133XVOnw8AiA8WX6i1PQAAAAAAnCc6mwAAAACA\nWkfYBAAAAADUOsImAAAAAKDWETYBAAAAALWOsAkAAAAAqHUxf/BVQkKCGjduHO0yAAAAAKDOOXDg\ngE6ePBn0tZgPm40bN1ZRUVG0ywAAAACAOqdFixYhX2MZLQAAAACg1hE2AQAAAAC1jrAJAAAAAKh1\nMb9nEwAAAEDt8Hq98vl80S4DFxGLxSKr9fx6lIRNAAAAoI4rLy/X7t27VVFREe1ScBGy2+1q2bKl\nHA7HOb2PsAkAAADUcbt371ZycrLS09NlsViiXQ4uIj6fT4cOHdLu3buVkZFxTu8lbAIAAAB1mNfr\nVUVFhdLT02UYxAOcLT09XYcPH5bX6z2nJbUMCAIAAADqsKo9mnQ0EUrVv41z3c9L2AQAAAAA1DrC\nJgAAAICLyrRp0+Ryuc7rvd9++63y8/PDunbgwIEqLCw8r88JxWKx6MiRI7V6z1hF2AQAAABwUZk+\nfXrIsOl2u6t9b/PmzbV+/fqwPufVV19VZmbmOdeH8LADGAAAAEDA3X/9j746VGra/VulJ+npu7qG\nfH3MmDGSpPz8fNlsNr3xxhv6zW9+I6vVqh07dmj//v3aunWrhg4dqsLCQpWXl+vSSy/VM888o0su\nuUS7du1Sbm5uoLtosVj0yCOPaMWKFTpw4ICmTp2qESNGSJJat26tFStWKDc3V7169VKXLl20efNm\nffvtt+rXr5/mz58vSdqzZ4/uuusuFRUVqUWLFmrYsKHatWunadOmVft3/eCDDzR+/HgdP35cTqdT\ns2fP1tVXX60DBw5o6NCh2rNnjywWizp37qxFixbpvffe0y9+8Qt5PB653W794he/0M9//vNa+KlH\nB2ETAAAAwEVj/vz5WrBggdavX6/U1NTA8x9++KE2bNig5ORkSdITTzyhxo0bS5IeffRRTZs2LRAO\nz5SQkKD3339fW7duVdeuXXXHHXcEnby7c+dOrV27VhUVFcrKytKmTZuUl5en8ePHKy8vT9OnT9fe\nvXuVm5urdu3aVfv3KC8v1+DBg7Vw4UIVFBRow4YN+vGPf6wdO3Zo8eLFuuyyy/TGG29Ikg4fPixJ\n+sMf/qBJkybptttukyQVFxef40/v4kLYBAAAABBQXdcxmn7yk58EgqYkLVmyRM8//7xcLpdcLpca\nNWoU8r1Dhw6VJLVr106GYWjv3r1q0aLFWdfdeuutMgxDhmEoNzdXO3fuVF5entasWaOZM2dKki65\n5BLdcMMNNdZbWFgoq9WqgoICSVKPHj3UtGlTffTRR+revbtmz56tiRMnqmfPnhowYIAk6dprr9Xv\nf/97bd++Xb1791aPHj3C/wFdhNizCQAAAOCiV79+/cCfN2zYoLlz5+rVV1/Vp59+qlmzZlU7UMjp\ndAb+bLPZQu77DPe68z0mpup9eXl5+uijj9StWzctX75cXbt2lcfj0T333KNXXnlFzZo1029/+1uN\nHTv2vD7nYkHYBAAAAHBRSU5OVklJScjXi4uLlZycrPT0dJWXl2vBggWm1tO7d289++yzkqR9+/bp\n5ZdfrvE9mZmZ8nq9Wr16tSRp48aNgSW4X375perXr69bbrlF8+bN07Zt23T8+HEVFhbqsssu08iR\nI/Xb3/5W7733npl/LdOxjBYAAADARWXixInq16+fkpKSAvsaTzVgwAAtXrxYmZmZSk9PV9++ffXN\nN9+YVs+cOXN01113KSsrS82bN1e3bt1O208ajMPh0PLlyzV+/HhNnDhRTqdTy5YtU/369fXPf/5T\ns2bNCnRPH3/8caWkpOiBBx7QW2+9JYfDIZvNpj/+8Y+m/Z0iweLz+XzRLuJCtGjRQkVFRdEuAwAA\nAIhJHo9H27ZtU9u2bWWz2aJdzkWprKxMdrtdhmHo0KFD6t69uxYvXqxu3bpFu7SIqO7fSHV5jM4m\nAAAAAFRj+/btuvPOO+Xz+VReXq6xY8fWmaB5IQibAAAAAFCN7OxsffTRR9EuI+YwIAgAAAAAUOsI\nmwAAAACAWkfYBAAAAADUOsImAAAAAKDWETYBAAAAXFSmTZsml8tVq/eYOnWq/va3v11oaafp1auX\nVqxYUav3jCeETQAAAAAXlenTp19w2DzzHg899JCGDh16oaXhHBA2AQAAAFw0xowZI0nKz89Xbm6u\n9u/fr2PHjmnkyJG68sorlZ2drVGjRqm8vFyS9PDDD6t9+/bKzc1Vbm6uvvrqq6D3GD58uJ544glJ\n/q7nrbfeqhtvvFFZWVnq3bu3Dh8+LEmqqKjQ2LFj1bZtW3Xv3l0TJ05Ur169aqx7//79Gjx4sDp1\n6qSOHTtqwYIFkiSv16tx48apffv2ysnJUefOneVyuXTgwAH1799fnTp1UnZ2tkaMGFHbP8qo45xN\nk+w/6tK2fcfV6XspSkmyR7scAAAAIDxLhkjFX5p3/7TLpNv/HvLl+fPna8GCBVq/fr1SU1MlSaNG\njVJ+fr4WLlwon8+nkSNHas6cObr77rs1c+ZM7dmzR4mJiSotLZXVag16jzNt3rxZH374odLT0zVk\nyBAtWLBA999/v/785z9r+/bt+uyzzyRJAwcODOuv9ctf/lKZmZlavny59u/fr86dOysnJ0cJCQla\ns2aNPvvsM1mtVpWUlMjhcGjx4sW67LLL9MYbb0hSIOzGEzqbJvn3Z3s17JnN+mxPSbRLAQAAAGLa\nihUr9Pjjjys3N1c/+MEPtH79eu3YsUMNGjTQ5ZdfrmHDhmnBggU6fPiwnE5nWPccMGCA0tPTJUl5\neXnauXOnJGnNmjUaNmyY7Ha77Ha77rrrrrDu9+abb2r06NGSpCZNmmjw4MF688039f3vf19ut1s/\n/elP9de//lUVFRWyWq3q3r27XnvtNU2cOFErV65UvXr1zuMnc3Gjs2mS1CSHJOlIaUWUKwEAAADO\nQTVdx2jx+Xx68cUX1bZt27Nee++997Rx40atW7dO3bt319KlS5Wfn1/jPU8NpTabTW63O+h1Fovl\nvGquel9KSoo+/fRTvf3221q7dq3uv/9+vfPOO8rLy9NHH32kN998U8uXL9eUKVO0ZcsW2Wy28/q8\nixGdTZOkVS6dLS4tj3IlAAAAQGxJTk5WScl3KwQHDRqkxx57LBAIi4uLtWPHDh07dkz79u1Tfn6+\npkyZoh49emjLli1B7xGu3r17a8mSJaqoqFBFRYWee+65sN7Xt29fLVy4UJJ04MABLV++XP369dOB\nAwd04sQJ9e/fXzNmzFDr1q31+eef68svv1T9+vV1yy23aN68edq2bZuOHz9+zvVezAibJkmjswkA\nAACcl4kTJ6pfv36B4T6zZ89WYmKicnNzlZ2drT59+mjXrl0qKSkJDOXJzs5WRUVFYNnrmfcI1+jR\no9W6dWtlZWXp6quvVps2bULu+zzV3Llz9cUXX6hTp0669tpr9bvf/U7dunXT119/rX79+ik7O1sd\nO3ZUx44ddd1112ndunXq3LmzcnNzddVVV+nxxx9XSkrKef/MLkYWn8/ni3YRF6JFixYqKiqKdhln\nKSouVY/H1uruHpfpgRuyol0OAAAAEJTH49G2bdvUtm3buFrCeSGOHTum5ORkVVRUaOjQoercubMm\nT54c7bKiprp/I9XlMfZsmqRqz2YxnU0AAAAgpvTt21cnT56Uy+VSjx49NH78+GiXFJMImyap57DJ\nbrOopIw9mwAAAEAs2bx5c7RLiAvs2TSJxWJRapKDziYAAACAOomwaaK0JDvTaAEAAADUSYRNE6Um\nOZhGCwAAAKBOImyaKC3JriOl5fJ6Y3rgLwAAAACcM8KmiVITHfL6pGMud7RLAQAAAICIImyaKLWe\nXZLYtwkAAACcg2nTpsnlcp3Xe7/99lvl5+eHde3AgQNVWFh4Xp+DmhE2TZRWedbmkTL2bQIAAADh\nmj59esiw6XZXv2qwefPmWr9+fVif8+qrryozM/Oc67vY1PQziRbCpll8PjV0SlZ56WwCAAAAYRoz\nZowkKT8/X7m5udq/f7+GDx+un/70p+rZs6c6duwoSRo6dKi6dOmi7OxsXX/99dq7d68kadeuXUpN\nTQ3cz2KxaMaMGbryyit12WWXadGiRYHXWrdurY8++kiS1KtXL02aNEn5+flq06ZNoA5J2rNnj/r3\n76+srCz1799fQ4YM0bRp04LWH6ouSXrllVfUtWtX5eTkKDc3N3Ce56ZNm9SjRw/l5OQoOztbK1eu\nPKs+SerSpYvWrVsXqHf8+PHKy8tT//795Xa7VVBQoC5duqhDhw66/fbbdeLEicB7Fy1apNzcXOXk\n5KhLly7atWuXxo0bpxkzZgSuKSws1KWXXlpr4dWolbvgbP95Wre8NknLrQ/oSOkPol0NAAAAEJZf\nrvmlvj72tWn3vzT5Us3rMy/k6/Pnz9eCBQu0fv3600Ljhx9+qA0bNig5OVmS9MQTT6hx48aSpEcf\nfVTTpk3T/Pnzg94zISFB77//vrZu3aquXbvqjjvukGGcHYV27typtWvXqqKiQllZWdq0aZPy8vIC\noW769Onau3evcnNz1a5du6CfFaqubdu2acSIEXrnnXfUrl07VVRUqLS0VIcPH9agQYO0bNky5efn\ny+v16siRI2H9LLdt26Z33nlHdrtdPp9PS5YsUXp6unw+n8aOHat58+bpvvvu07p16/TQQw9p48aN\natasmUpLSyVJv/zlL1VQUKDJkyfLZrPpqaee0qhRo4L+bM4HYdMsRoIkyaEKFZ9gGS0AAABwIX7y\nk58EgqYkLVmyRM8//7xcLpdcLpcaNWoU8r1Dhw6VJLVr106GYWjv3r1q0aLFWdfdeuutMgxDhmEo\nNzdXO3fuVF5entasWaOZM2dKki655BLdcMMNIT8rVF2rV6/WgAEDAiHVbrcrJSVFr7zyijIzMwP7\nTK1Wqxo2bBjWz2TYsGGy2/1zYnw+n2bPnq1XXnlFbrdbJSUluuqqqyT5O6p33HGHmjVrJklKSkqS\nJGVmZiorK0srV65UQUGBli5dqk8++SSszw4HYdMsNn/YTFCFjrCMFgAAADGiuq5jNNWvXz/w5w0b\nNmju3LnatGmTmjRpolWrVmnq1Kkh3+t0OgN/ttlsIZeJhnudxWIJ+vy51lUTwzDk8XgC35+5j/XU\nn8mSJUv01ltv6e2331aDBg00d+5cvfXWWzV+xoQJE/TYY4/pwIED6tevn5o2bXre9Z6JPZtmCXQ2\n3SoupbMJAAAAhCs5OVklJSUhXy8uLlZycrLS09NVXl6uBQsWmFpP79699eyzz0qS9u3bp5dffvmc\n6yooKNDrr7+urVu3SpIqKioC3cft27cHhhp5vV4dPnxYkpSRkRHY1/n+++9XOzm3uLhYjRo1UoMG\nDXTs2LFAvZJ04403avHixdqzZ48kqbS0NLCUtn///tq7d68efvhhjRs37jx+OqERNs1iVHU2yxkQ\nBAAAAJyDiRMnql+/foEBQWcaMGCAMjMzA8tPc3NzTa1nzpw5Wr9+vbKysjR06FB169bttP2k4dSV\nkZGhRYsWadiwYcrJyVG3bt1UWFiotLQ0vfTSS7rvvvuUnZ2tK664Qu+++64k6eGHH9aTTz6pnJwc\n/eUvf1GHDh1C1njnnXeqtLRUmZmZuu666047/qVnz5568MEHVVBQoJycHF1zzTU6cOCAJH+X9mc/\n+5maNGmivLy82vqR+e/t8/l8tXrHCGvRooWKioqiXcbZdr4lPf8jTfON0s6WN+v5n3WLdkUAAADA\nWTwej7Zt26a2bdvKZrNFu5yLUllZmex2uwzD0KFDh9S9e3ctXrxY3brFx//Gv+GGG3Trrbfqjjvu\nCPp6df9GqstjdDbNUrlnM8XB0ScAAABALNu+fbu6dOminJwc9ejRQ2PHjo2LoPnBBx8oIyNDVqtV\nt99+e63fnwFBZjH8m4tT7F6m0QIAAAAxLDs7+7TzLuNFly5dtGPHDtPuT2fTLIZDktTA8DCNFgAA\nAECdQ9g0S2VnM9nw6ES5R+Vub5QLAgAAAIDIIWyapXIabT3Dfy4O3U0AAAAAdQlh0yyVA4Lq2SrD\nZhn7NgEAAADUHYRNs1R2NpOs/pBZfILOJgAAABCOadOmyeVy1eo9pk6dqr/97W8XWhrOAWHTLJVh\nM9Hi72wWl9LZBAAAAMIxffr0Cw6bZ97joYce0tChQy+0tKjzer3yemNjHgxh0yyVy2idFn/IZM8m\nAAAAULMxY8ZIkvLz85Wbm6v9+/fr2LFjGjlypK688kplZ2dr1KhRKi/3/+/rhx9+WO3bt1dubq5y\nc3P11VdfBb3H8OHD9cQTT0jydz1vvfVW3XjjjcrKylLv3r11+PBhSVJFRYXGjh2rtm3bqnv37po4\ncaJ69eoVtNZZs2apa9euys3NVdeuXbVp06bAa1988YUKCgqUnZ2t7OxszZ8/X5L0zTff6Oabb1an\nTp2UnZ2tKVOmSNJp9UnSpEmTNG3atEC9P/7xj1VQUKCOHTtqz549mjRpUuCze/bsqcLCwsB7N23a\npB49eignJ0fZ2dlauXKlli1bpv79+weu8Xg8atWqlT7//PPz/l3VhHM2zWK1Sla7EixuSXQ2AQAA\nEBu+/vlYlX+927T7Oy5tqUv/v6dCvj5//nwtWLBA69evV2pqqiRp1KhRys/P18KFC+Xz+TRy5EjN\nmTNHd999t2bOnKk9e/YoMTFRpaWlslqtQe9xps2bN+vDDz9Uenq6hgwZogULFuj+++/Xn//8Z23f\nvl2fffaZJGngwIEha73jjjt07733SpLee+89DR8+XFu3bpXb7dZNN92k6dOn67bbbpMkHTx4UJI0\nbNgw9e/fX8uWLZMkHThwIKyf26ZNm7RlyxY1bdpUkjR58mTNnDlTkvT3v/9dEyZM0L///W8dPnxY\ngwYN0rJly5Sfny+v16sjR44oJSVFkyZNUmFhoTIzM7Vq1SplZGQoKysrrM8/H4RNMxlOOeT/f1zo\nbAIAAADnZ8WKFdq0aZNmzZolSSorK5PNZlODBg10+eWXBwLc9ddfrxYtWoR1zwEDBig9PV2SlJeX\np08++USStGbNGg0bNkx2u12SdNddd+npp58Oeo8tW7bokUce0aFDh2QYhgoLC1VWVqb//e9/crlc\ngaApSY0aNdLx48e1YcMGvf7664HnGzduHFa9AwcODARNSVq9erXmzZunY8eOyev1BjqzmzZtUmZm\npvLz8yVJVqtVDRs2lCSNHTtWTz75pObOnasnn3xS48aNC+uzzxdh00yGQ3Zf5YAgwiYAAABiQHVd\nx2jx+Xx68cUX1bZt27Nee++997Rx40atW7dO3bt319KlSwNBqzpOpzPwZ5vNJrfbHfQ6i8US9Pny\n8nINHjxYa9euVdeuXXX06FGlpKTo5MmTYf6tTmcYhjweT+B7l8ul+vXrB74/9c+7d+/WuHHj9J//\n/Edt2rTR//t//089e/as8TNGjhyprKws3XnnndqxY4d++MMfnlet4WLPppkMp2zectmsFh1hGS0A\nAAAQluTkZJWUlAS+HzRokB577LFAICwuLtaOHTt07Ngx7du3T/n5+ZoyZYp69OihLVu2BL1HuHr3\n7q0lS5aooqJCFRUVeu6554Je53K5VF5erpYtW0qS5s2bF3gtMzNTSUlJWrp0aeC5gwcPqn79+urZ\ns6f++Mc/Bp6vWkabkZGh999/X5J06NAhvfrqqyFrLCkpkd1uV7NmzeTz+fSnP/0p8NpVV12l7du3\na/369ZJ0WtczLS1NN910k370ox9p9OjRstls5/SzOVeETTMZCbJ4Tio10U7YBAAAAMI0ceJE9evX\nLzDcZ/bs2UpMTFRubq6ys7PVp08f7dq1SyUlJRo8eHBg2E5FRYXuuuuuoPcI1+jRo9W6dWtlZWXp\n6quvVps2bYLu+2zQoIEefvhhXXnllercubMcDkfgNcMwtHLlSi1atEidOnVSTk6OXnzxRUnS888/\nrw8++EAdOnRQbm5uICiOGjVKBw4cUPv27XXnnXeqe/fuIWvs1KmThgwZog4dOqhr166BwCv5A+VL\nL72k++67T9nZ2briiiv07rvvBl4fOXKkDhw4oJEjR4b9MzlfFp/P5zP9U0zUokULFRUVRbuM4J7s\nLvk86nPycVktFq2+95poVwQAAACcxuPxaNu2bWrbtq3pna5YcezYMSUnJ6uiokJDhw5V586dNXny\n5GiXVStmzpypL774Qs8880zY76nu30h1eYw9m2YyEqSyw0pLcmjXodJoVwMAAAAgDH379tXJkyfl\ncrnUo0cPjR8/Ptol1YoOHTrIYrHo3//+d0Q+j7BpJiNBcpcrNc2hI18fkc/nC7nBGAAAAMDFYfPm\nzdEuwRRVx7lECns2zWQkSDIdtf4AACAASURBVG6XUpPscnt9On4y+IQrAAAAAIg3hE0z2RIkT7nS\nkvxn9DAkCAAAABebqpV3MT7KBSaq+rdxrqs0WUZrpkBn0z+Z6khphS5tGOWaAAAAgFNYrVbZ7XYd\nOnRI6enpbPvCaXw+nw4dOiS73S6r9dx6lYRNMxlOyedVQ6f/l1JcWh7lggAAAICztWzZUrt37w6c\nxwicym63n3a8SrgIm2YyEiRJ6U6vJMImAAAALk4Oh0MZGRnyer0sp8VpLBbLOXc0qxA2zVQZNtMc\n/v/AsmcTAAAAF7PzDRVAMPxrMpOtMmwm0NkEAAAAULcQNs1U2dlMsdPZBAAAAFC3EDbNVBk2G9g9\nkuhsAgAAAKg7CJtmqgybCapQot1GZxMAAABAnUHYNFPlnk15ypWWZNcROpsAAAAA6gjCppkqO5ty\nu5Sa5FAxnU0AAAAAdQRh00yG0//oLldaPTt7NgEAAADUGYRNM53a2Ux06JjLLbfHG92aAAAAACAC\nCJtmOm0ZrV2SdKSMpbQAAAAA4h9h00ynDQhySOKsTQAAAAB1A2HTTME6m+zbBAAAAFAHEDbNFAib\n33U2mUgLAAAAoC4gbJrplM5mWj1/Z5OJtAAAAADqAsKmmaqOPvGUKzWwZ5OwCQAAACD+ETbNZDv1\n6JOqzibLaAEAAADEP8KmmQLLaE+eMo2WziYAAACA+EfYNNMpYbNBol0WC0efAAAAAKgbCJtmOmVA\nkM1qUUqinQFBAAAAAOoEwqaZqvZsevwBMy3JQWcTAAAAQJ1A2DTTKZ1NSUpNorMJAAAAoG4gbJrJ\nYpFsDsn9XWezuLRCPp8vyoUBAAAAgLkIm2YznN91NhPtKnd7VVbhiXJRAAAAAGAuwqbZjITAns3U\nyuNPOGsTAAAAQLwjbJrNlhDobKYl2SVx1iYAAACA+EfYNJuRILlPSpJS6/k7m0ykBQAAABDvCJtm\nOyVsVnU2mUgLAAAAIN4RNs12WthkzyYAAACAusHUsOlyuTRo0CC1bdtWOTk56tevn3bs2BH02pdf\nflnt2rXT5ZdfrsGDB+vo0aNmlhY5tgTJ4w+bKYmVezZP0NkEAAAAEN9M72yOGjVKhYWF+vjjj3XT\nTTfp7rvvPuua48eP62c/+5lWrFih7du3q3nz5vr9739vdmmRYZwyIKgenU0AAAAAdYOpYdPpdGrg\nwIGyWCySpO7du2vXrl1nXffaa6/pBz/4gdq1aydJGjt2rJYuXWpmaZFjJEhufyeTabQAAAAA6oqI\n7tmcM2eObrrpprOe3717t1q1ahX4vnXr1tqzZ4/cbnckyzNHVWfT51Oi3SaHYdWRMjqbAAAAAOKb\nEakPmjFjhnbs2KE1a9Zc0H1mzZqlWbNmBb4/fvz4hZZmLsMpySd53bLY7EpLsjONFgAAAEDci0hn\nc+bMmVq+fLlee+01JSUlnfV6y5Yt9dVXXwW+37Vrl5o1aybDODsL33vvvSoqKgp81a9f39TaL5gt\nwf9YtW8zycE5mwAAAADinulhc9asWVq6dKlWr16t1NTUoNcMGDBA//3vf7V161ZJ0lNPPaUhQ4aY\nXVpkGFVh0z+RNpXOJgAAAIA6wNRltEVFRZo4caK+//3v69prr5UkJSQkaPPmzZo6daqaN2+uMWPG\nKDk5WU8//bQGDRokt9utjh076q9//auZpUXOmWEz0aGSsgp5vD7ZrJYoFgYAAAAA5jE1bLZo0UI+\nny/oaw899NBp3//whz/UD3/4QzPLiQ7jjGW09ezy+aSjZRWBo1AAAAAAIN5EdBptnVS1Z9PjXzqb\nmlR11iZLaQEAAADEL8Km2c7sbFadtcnxJwAAAADiGGHTbIbT/+g+vbN5hM4mAAAAgDhG2DSbUbkv\n85SjTySp+ASdTQAAAADxi7BptqrOZuWezapltOzZBAAAABDPCJtms52+ZzO1as9mKZ1NAAAAAPGL\nsGm2M8/ZZBotAAAAgDqAsGm2M8NmItNoAQAAAMQ/wqbZzjj6xLBZlew0mEYLAAAAIK4RNs1WtWfT\n8124TEtyMI0WAAAAQFwjbJrtjM6m5J9IS2cTAAAAQDwjbJqt6ugT93fhMiXJoWKm0QIAAACIY4RN\ns4XobJZVeOSq8ESpKAAAAAAwF2HTbGdMo5X8ezYlztoEAAAAEL8Im2YLDAj6LmymJlUdf8K+TQAA\nAADxibBptqDLaP2dTSbSAgAAAIhXhE2zBcLmd13MQGeTibQAAAAA4hRh02w2fxczaGeTPZsAAAAA\n4hRh02wWi3/fpufszmYxnU0AAAAAcYqwGQmGM2hnk2W0AAAAAOIVYTMSjISgezZZRgsAAAAgXhE2\nI8FIOK2zWT/BkGG1cM4mAAAAgLhF2IwEI0Fyf3fOpsViUWqSg2W0AAAAAOIWYTMSbAmS5+RpT6Ul\n2RkQBAAAACBuETYj4YzOpuQfEsQyWgAAAADxirAZCUHCZkqSXUfKKuTz+aJUFAAAAACYh7AZCUE7\nm3Z5vD4ddbmjVBQAAAAAmIewGQmGM8ieTc7aBAAAABC/CJuRYHP4jz45ZclsaiBssm8TAAAAQPwh\nbEaC4fQ/er7rYqYl2SWJibQAAAAA4hJhMxIMfxfz1H2bdDYBAAAAxDPCZiRUdTZPC5t0NgEAAADE\nL8JmJNgS/I+nDAmqGhBUTGcTAAAAQBwibEaCURk23aeGTX9nk2m0AAAAAOIRYTMSgoRN9mwCAAAA\niGeEzUgIhE1X4CmHYVU9h409mwAAAADiEmEzEoIcfSL5u5t0NgEAAADEI8JmJNiqjj5xnfZ0Wj07\nnU0AAAAAcYmwGQlBjj6RpNREOpsAAAAA4hNhMxKCDAiS/GdtHj/pVrnbG4WiAAAAAMA8hM1ICBE2\nq87aPFLGUloAAAAA8YWwGQm2yrDpOTNs+s/aLGEpLQAAAIA4Q9iMhCBHn0jfnbVZTNgEAAAAEGcI\nm5EQCJunL5dNq+fvbDKRFgAAAEC8IWxGQg2dzSOETQAAAABxhrAZCVVHn3hOD5WpiVWdTZbRAgAA\nAIgvhM1IsPk7mGd2NtMCezbpbAIAAACIL4TNSKjqbIY6+uQEnU0AAAAA8YWwGQkhztlMdhqyWjhn\nEwAAAED8IWxGQoiwabValJrkYM8mAAAAgLhD2IwEW2XY9Jw866XUJDvTaAEAAADEHcJmJNjskixn\nDQiS/BNp6WwCAAAAiDeEzUiwWPxDgtxndzDTkhw6Uloun88XhcIAAAAAwByEzUgxHME7m0kOVXh8\nOlHuiUJRAAAAAGAOwmakGM6zBgRJUlqSXZJUfIJ9mwAAAADiB2EzUmwJQQcEpdXzn7VZUsa+TQAA\nAADxg7AZKUZC0M5malVnk4m0AAAAAOIIYTNSQoTNtCR/Z5OJtAAAAADiCWEzUkJ1NhP9nU3O2gQA\nAAAQTwibkRJiz2ZqVWfzBJ1NAAAAAPGDsBkpoZbR1mPPJgAAAID4Q9iMlJBHn/g7myyjBQAAABBP\nCJuRYjj8y2h9vtOedtptctqtOsLRJwAAAADiCGEzUgyn/zFEd5NptAAAAADiCWEzUmz+5bKhhgSx\njBYAAABAPCFsRko1nc3URLuKTxA2AQAAAMQPwmakGAn+xxATaY+63HJ7vBEuCgAAAADMQdiMlGrC\nZtVZmyUMCQIAAAAQJwibkWKrDJtB9mymJfnP2mQiLQAAAIB4QdiMlEBn03XWS5y1CQAAACDeEDYj\npboBQZVhs/gEnU0AAAAA8YGwGSlG5dEnIabRSlIxnU0AAAAAcYKwGSnVdDbT6lXu2SylswkAAAAg\nPhA2I8VW2dkMMiAosIyWziYAAACAOEHYjJTqOpuBsElnEwAAAEB8IGxGSjXnbKYk2mWxSCVldDYB\nAAAAxAfCZqRUc/SJzWpRA6edabQAAAAA4gZhM1KqltF6gncv05Ls7NkEAAAAEDcIm5FSNSAoSGdT\nkhok2nXM5Y5gQQAAAABgHsJmpFQzIEiSnHabXBWeCBYEAAAAAOYhbEaKUdXZDB02ywibAAAAAOIE\nYTNSauhsJtqtclV45PP5IlgUAAAAAJiDsBkptspptJ5QYdMmr08q93gjWBQAAAAAmIOwGSnVnLMp\n+ZfRSpKrnLAJAAAAIPYRNiMl3LDpZt8mAAAAgNhH2IwUqyFZrCGPPkl0+MNmWTlhEwAAAEDsI2xG\nisXiHxLkKQ/6stOoDJtMpAUAAAAQBwibkWRzVNPZ9P8qOGsTAAAAQDwgbEaS4azm6BM6mwAAAADi\nB2EzkgxHyLCZUDUgiLAJAAAAIA4QNiMpjM6mq4KjTwAAAADEPsJmJNkSJE8Ny2iZRgsAAAAgDhA2\nI8lIqPGcTfZsAgAAAIgHhM1IqiZsMo0WAAAAQDwhbEZSGJ1NwiYAAACAeEDYjCTDGfKcTZbRAgAA\nAIgnhM1Isjkkb4XkPXviLNNoAQAAAMQTwmYkGU7/Y5CJtIl0NgEAAADEEcJmJBkO/2OQfZuBPZsc\nfQIAAAAgDhA2I6mqsxkkbCYYldNo3YRNAAAAALGPsBlJtsrOZpBltFarRU67VWV0NgEAAADEAcJm\nJFXT2ZT8+zbZswkAAAAgHhA2I6mGsOm025hGCwAAACAuEDYjqZoBQZK/s+miswkAAAAgDhA2IynQ\n2XQFfdnJMloAAAAAcYKwGUnVDAiSJKfdSmcTAAAAQFwgbEZSTQOCHDam0QIAAACIC4TNSAprzyYD\nggAAAADEPsJmJNXQ2Uyw21Tu8crj9UWwKAAAAACofYTNSLIl+B9D7NlMtNskiX2bAAAAAGIeYTOS\njMqwWc0yWklMpAUAAAAQ8wibkVTj0Sf+XwedTQAAAACxjrAZSWEMCJIImwAAAABiH2EzkmoYEOR0\nVC6jLWciLQAAAIDYRtiMJFtlZzPEgCCnUdnZdNPZBAAAABDbCJuRVENnMzHQ2SRsAgAAAIhthM1I\nYhotAAAAgDqCsBlJNYRNptECAAAAiBeEzUiyVYbNUHs2mUYLAAAAIE4QNiPJZkgWW83LaNmzCQAA\nACDGETYjzXBKblfQl5yBPZscfQIAAAAgthE2I81w1DiNlmW0AAAAAGIdYTPSDGeNy2gJmwAAAABi\nHWEz0myOkAOCEiqn0XL0CQAAAIBYR9iMNDqbAAAAAOoAwmakVbNnkwFBAAAAAOIFYTPSquls2m1W\nGVYLR58AAAAAiHmEzUgznCH3bEr+pbQn3YRNAAAAALHN9LA5fvx4tW7dWhaLRR999FHQa9atW6fE\nxETl5uYGvsrKyswuLTpsoZfRSpLTYaOzCQAAACDmGWZ/wM0336zf/OY36tGjR7XXZWZmhgyjccVw\nSm5XyJeddivTaAEAAADEPNPDZs+ePc3+iNhiOCSvW/J6JKvtrJcT7Tam0QIAAACIeRfNns2dO3fq\niiuuUNeuXfXUU09FuxzzGE7/YzXHn7iYRgsAAAAgxpne2QzHFVdcoaKiIqWkpKioqEgDBw5Uo0aN\ndMstt5x17axZszRr1qzA98ePH49kqRfO5vA/ek5KSjrr5QS7jWW0AAAAAGLeRdHZbNCggVJSUiRJ\nLVq00G233ab169cHvfbee+9VUVFR4Kt+/fqRLPXChdXZJGwCAAAAiG0XRdjcs2ePvF7/0tFjx47p\n5Zdf1g9+8IMoV2USo7KzWU3YLKvwyOfzRbAoAAAAAKhdpofN0aNHq0WLFioqKlJBQYEyMjIkSXff\nfbdWrVolSXrxxRfVqVMn5eTkqHv37urXr59GjBhhdmnRUUNn02m3yueTTrrZtwkAAAAgdpm+Z3PB\nggVBn3/66acDfx43bpzGjRtndikXByPB/xji+JNEh39C7ckKr5z2s6fVAgAAAEAsuCiW0dYptsqw\n6SkP+nJVwGRIEAAAAIBYRtiMtMAy2uCdTcImAAAAgHhA2Iy0MAYESWIiLQAAAICYRtiMtDCOPpHo\nbAIAAACIbYTNSLNVdjY9oafRSpKrnLAJAAAAIHYRNiOtxqNPKpfRugmbAAAAAGIXYTPSAkefhFhG\nW3n0SVk552wCAAAAiF2EzUirIWw6DfZsAgAAAIh9hM1Iq+Hok6rOJtNoAQAAAMQywmak1TggiLAJ\nAAAAIPYRNiOtxgFB/l9JGdNoAQAAAMQwwmakGZWdzRrO2WQaLQAAAIBYRtiMtBo6m0yjBQAAABAP\nCJuRVtOeTabRAgAAAIgDhM1IC7OzeZKwCQAAACCGETYjLXDOZvCjTxKMygFBhE0AAAAAMYywGWlW\nm2Q1JHd50JctFoucdithEwAAAEBMI2xGg+EM2dmU/BNpOWcTAAAAQCwjbEaDzSF5gnc2JX/YLKtg\nGi0AAACA2EXYjIYaOptOu02ucjqbAAAAAGIXYTMaDEfIabRSZdh0EzYBAAAAxC7CZjQYzmrDZqLD\npjI6mwAAAABiGGEzGmwOyVNdZ5NptAAAAABiG2EzGmrqbNptOsmAIAAAAAAxjLAZDUZCjQOCyj1e\nuT0ETgAAAACxibAZDUaC5A599InTbpMkudyETQAAAACxibAZDTUcfZJYFTbZtwkAAAAgRhE2o8Hm\nkHweyeMO+nKiwx82mUgLAAAAIFYRNqPBcPofQ0ykdRr+XwudTQAAAACxirAZDYbD/xhiIq3TUbWM\nlj2bAAAAAGITYTMaqjqbIcJm1Z5NztoEAAAAEKsIm9FgJPgfQy2jJWwCAAAAiHGEzWiwVYbNGjqb\n7NkEAAAAEKsIm9FQ1dkMcfyJk7AJAAAAIMYRNqMhEDbLg77stPt/LRx9AgAAACBWETajITAgKHhn\nk2W0AAAAAGIdYTMabJVHn4QYEJToqBoQxNEnAAAAAGITYTMaajj6hGm0AAAAAGIdYTMajMrOZg3T\naE8SNgEAAADEKMJmNNDZBAAAABDnCJvRUOPRJ0yjBQAAABDbCJvRYKsMm55QR59UTqN1MyAIAAAA\nQGwibEZDDUef2G1W2W0WOpsAAAAAYhZhMxoCA4KCdzYlyWnYOGcTAAAAQMwibEZDDZ1NSXI6bAwI\nAgAAABCzCJvRYKvsbHqCT6OV/Mef0NkEAAAAEKsIm9FQw9Enkn8iLZ1NAAAAALGKsBkNgT2bNXQ2\nGRAEAAAAIEYRNqMhrM6mjaNPAAAAAMQswmY0VJ2zWd2AILuNo08AAAAAxKyww+aRI0fMrKNusVol\nq13yhD76JNHun0br8/kiWBgAAAAA1I6ww+bll1+ukSNH6uOPPzaznrrDcFbb2Ux02CRJJ1lKCwAA\nACAGhR02d+zYoQ4dOuiWW25Rfn6+/vGPf8jjYZnneTMckjt0Z9Np9/9qOP4EAAAAQCwKO2ympKTo\nnnvuUWFhoX73u99p0qRJatmypR555BGdOHHCzBrjUw2dTafd39nk+BMAAAAAseicBgQdPXpUTzzx\nhCZMmKAOHTpo3rx52rt3rwoKCsyqL37ZHJKn+qNPJMlVwTJaAAAAALHHCPfC0aNHa+XKlbr55pu1\natUqZWZmSpIGDx6s9u3bm1Zg3DKcNR59IomJtAAAAABiUthhs23btiosLFRKSspZr7311lu1WlSd\nYCRIZYdDvpzIMloAAAAAMSzsZbQ9e/aU1frd5UePHtUHH3wgSWrWrFntVxbvjITqBwRVTaMlbAIA\nAACIQWGHzdGjRyspKSnwfVJSksaMGWNKUXWCkVD9gCDD/6uhswkAAAAgFoUdNr1er2w2W+B7wzDk\ndrtNKapOsCVIntCdzapzNgmbAAAAAGJR2GHT4XBo+/btge+3bdsmu91uSlF1Qg2dTabRAgAAAIhl\nYQ8IevDBB9WjRw9dd911kqTXX39dixYtMq2wuGckSD6v5HFLtrN/DZyzCQAAACCWhR02r7/+eq1f\nv15vvvmmJGnKlClq06aNaYXFPcPpf3S7JFv9s16uCpsujj4BAAAAEIPCDpuS//iTtm3bmlVL3WJz\n+B9D7Nv8bhktYRMAAABA7Ak7bO7fv18PPvigPv74Y7lc3+01/O9//2tKYXHv1M5mEE4702gBAAAA\nxK6wBwT97Gc/U+vWrXXw4EFNnz5dzZs31/XXX29mbfHNSPA/hgibTKMFAAAAEMvCDptff/21Jk+e\nrISEBN14441avnx5YP8mzkMgbNa0jJZptAAAAABizzkdfSJJTqdThw4dkmEYOnjwoGmFxb0aOptO\n9mwCAAAAiGFh79ls27atDh06pGHDhqlbt25q0KCBOnfubGZt8c1WGTZDDAhKMKyyWKQyptECAAAA\niEFhh83FixdLkiZMmKAuXbqouLhYAwYMMK2wuFdDZ9Nischp2ORyEzYBAAAAxJ6wwqbH41GnTp30\n+eefS5KuvvpqU4uqE2rYsyn5hwTR2QQAAAAQi8Las2mz2dS4cWOVlpaaXU/dUcPRJ5LkNKzs2QQA\nAAAQk8JeRpuRkaGrr75aP/nJT1S/fv3A8+PHjzelsLhn8w9ckudkyEucDhvTaAEAAADEpLDDptfr\nVW5urrZv3x54zmKxmFJUnRDobIYOm4l2m4653BEqCAAAAABqT9hhc9GiRWbWUffUMCBI8h9/sv9Y\n6DAKAAAAABersMPmc889F/T5O++8s9aKqVPCGRBkt7FnEwAAAEBMCjts/utf/wr82eVyacOGDere\nvTth83yF2dkkbAIAAACIRWGHzX/+85+nff/ll1/qd7/7Xa0XVGfYKsOmJ3Rn02m3qsLjk9vjlWEL\na3AwAAAAAFwUzjvBXHbZZfrss89qs5a6JYyjTxLtNkmSy81EWgAAAACxJezO5qpVqwJ/9ng82rx5\nsxISEkwpqk4wKo8+qW4arcMfNsvKPaqfEPavCgAAAACiLuwEM3v27O/eZBjKyMjQCy+8YEpRdUIY\nR584qzqb7NsEAAAAEGPCDptr1641s466J8wBQRJhEwAAAEDsCXvP5p///GcdPnw48P2hQ4e0cOFC\nU4qqE8IYEFS1Z7OMsAkAAAAgxoQdNp966ik1bNgw8H16erqeeuopU4qqE8LqbPp/PWXlhE0AAAAA\nsSXssOnz+c56zuMhBJ03i0WyOSR3zZ1NptECAAAAiDVhh81mzZrpH//4R+D7F154Qc2aNTOlqDrD\ncFZ/9Mkp02gBAAAAIJac0zTaQYMG6Te/+Y0kKSkpSStXrjStsDrB5qh2z2aCwYAgAAAAALEp7LDZ\nvn17ff755yosLJQkZWZmymazmVZYnRBmZ5OwCQAAACDWhL2M9l//+peOHTumrKwsZWVl6ejRo3rl\nlVfMrC3+GeHt2WQaLQAAAIBYE3bYnDJlilJTUwPfp6amasqUKaYUVWfU0NkMTKMlbAIAAACIMWGH\nzTNZLBam0V4oI0Fynwz5cmAabQXTaAEAAADElrDDZnJysjZu3Bj4/t1331VycrIpRdUZtgTJEzps\nOu3s2QQAAAAQm8IeEPR///d/+tGPfqR27dpJkrZv366XXnrJtMLqBCOhhmW0HH0CAAAAIDaFHTbz\n8vL0xRdfaNOmTZKkq6666rQ9nDgPRkL1A4KYRgsAAAAgRoUdNiUpLS1NAwcONKuWuqeqs+nzSRbL\nWS87DQYEAQAAAIhNYe/Z3L59u6677jo1b95cDRs2DHzhAtgSJPkkrzvoy4bNKrvNQmcTAAAAQMwJ\nO2yOHDlSw4cPV1pamt5++23dfPPNmjRpkpm1xT/D6X+sYd8m02gBAAAAxJqww+bRo0d16623ymq1\nqlOnTlqwYIFWrFhhZm3xz3D4H2s4/oRltAAAAABiTdhh0263S/IfgbJr1y6dPHlSBw8eNK2wOiHQ\n2az++BOm0QIAAACINWEPCOrZs6cOHTqkcePGqXPnznI4HBoyZIiZtcU/I8H/WM0y2kS7TS43YRMA\nAABAbKkxbLrdbhmGoccff1ySdPvttys/P18lJSXq2LGj6QXGNVtl2PSEPv7E6bDp2NGKCBUEAAAA\nALWjxmW0l19+uYYOHaply5bpxIkTkqRLL72UoFkbwuhsOg0rezYBAAAAxJwaw+b27ds1YsQIrVu3\nTjk5Obrxxhv1zDPPsF+zNgTCZujOZqKDabQAAAAAYk+Ny2gNw1Dfvn3Vt29fSdJ//vMfvfTSS+rV\nq5caNmyoQYMG6d577zW90LgU5p7NsgqPfD6fLBZLhAoDAAAAgAsT9jTaKl27dtWMGTP06aefauHC\nhaqoYD/heQtnz6bdJkk66aa7CQAAACB2hB02FyxYoJKSEknSuHHj1KVLF+3bt0+TJ082rbi4Fzj6\npJo9m5Vhk+NPAAAAAMSSsMPmk08+qZSUFL377rv65JNP9Mgjj2jSpElm1hb/AstoQ5+zmVgZNjn+\nBAAAAEAsCTtsGoZ/e+dbb72lO++8UwUFBXK73aYVVieEETaddv+viM4mAAAAgFgSdti0Wq164YUX\n9MILLwSGBZWXh95riDCEOSBIEsefAAAAAIgpYYfNP/3pT1q6dKlGjhypVq1aadu2berdu7eZtcW/\nMAYEJToql9Fy/AkAAACAGFLj0SdVunfvrhUrVkiSfD6fmjVrprlz55pWWJ0QRmczoWrPJp1N/P/s\n3XmYZGV99//32WrptXqmu2ffmBkGVLZRFhkEAQFFTBTUaCR5TNjUhxjFKC7xSfzFqAmRqIkLSJSA\nBI2AisSVRdkEBAEJ+4zTs890z3RVV3fXduqc8/vjVFXvPT1LLT3zeV1XXafq1Kmqu6dHrM987/t7\ni4iIiIjMIjOubF5yySWkUikKhQLHH3888+bN42tf+1o1x3boq4TNaSqb6kYrIiIiIiKz0IzD5hNP\nPEEikeBnP/sZJ5xwAjt37uQb3/hGNcd26JvB1ifqRisiIiIiIrPRjMNmEAQAPPDAA1xwwQW0tbVh\nWVbVBnZYsCLhcZo1m+pGKyIiIiIis9GMw+b8+fN5//vfz/e//33e8IY34LounqcAdED2pbKpNZsi\nIiIiIjKLzDhs3nLLLaxZs4bvfve7JBIJtm3bxlVXXVXNsR36ZtAgKKZutCIiIiIiMgvNOGx2dnZy\n+eWX4/s+Dz/8MN3df2ODowAAIABJREFU3bz3ve+t4tAOAzNoEBSztc+miIiIiIjMPjPe+uThhx/m\noosuYv78+QDs2rWL22+/nde+9rVVG9whr7xmc7pptBGFTRERERERmX1mHDavuuoqbrvtNtatWweE\n4fPDH/4wjzzySNUGd8gzDLCi0zYI0ppNERERERGZjWY8jTabzVaCJsCpp55KLjd1RU5myI5Nv2az\n1I1WYVNERERERGaTGYfNlpYW7r777srje+65h+bm5qoM6rBiR6Zfs1mqbGrrExERERERmU1mPI32\ny1/+MhdddFFlb03f97njjjuqNrDDxl4qm1HbxDDUjVZERERERGaXGYfN17zmNaxfv54XX3wRgDVr\n1uA4TtUGdtiwIlDMT/m0YRjEbEsNgkREREREZFbZa9hMp9NjHi9duhQI13Bms1na2tqqM7LDhR0D\nb+qwCWFHWoVNERERERGZTfYaNhOJBIZhEARB5Vz5sWEYeJ5C0AGxo5BPT3tJ3LHIK2yKiIiIiMgs\nstew6ftaK1hVdnTaabQAUcdUZVNERERERGaVGXejlSqZQdiMO5pGKyIiIiIis0vVw+YHP/hBli9f\njmEYPPXUU1Ne9x//8R+sXr2alStXctlll+G6brWH1his6N7XbDqWutGKiIiIiMisUvWw+fa3v50H\nH3yQZcuWTXnNxo0b+fSnP80DDzzA+vXr2bVrF9dff321h9YY7Gi49cmoNbHjxRyLnPbZFBERERGR\nWaTqYfP0009n8eLF015z22238Ud/9EfMnz8fwzB43/vex6233lrtoTUGOxoevakruTFNoxURERER\nkVmmIdZsbt68eUzlc/ny5WzevLmOI6qhctgs5qa8JB6xKPoBrqeptCIiIiIiMjs0RNjcF9deey2L\nFy+u3IaGhuo9pANjx8LjNE2CYnb4a8qpuikiIiIiIrNEQ4TNpUuXsmnTpsrjnp4eli5dOum1V111\nFVu3bq3cWlpaajXM6rDK02inDpvxiAWgqbQiIiIiIjJrNETYvOiii7jzzjvZuXMnQRDwjW98g3e9\n6131HlZtVKbRThM2nTBs5tWRVkREREREZomqh80rrriCxYsXs3XrVs477zxWrVoFwKWXXsqdd94J\nwBFHHMFnPvMZ1q1bx6pVq+jq6uKKK66o9tAawwzCZtRRZVNERERERGYXu9ofcN111016/oYbbhjz\n+LLLLuOyyy6r9nAaz0waBJXDprY/EREREWkIQRBQDIq4nls5BgRjngcq58Y/LjMwMAyjcpxwrnTf\nD3y8wCMIArzAqzye8rwfHsvnin6x8pznl84FxbHv4098Xz/wJ9wfP2YTc+xjwxzz8xT9Iq7vjvmz\nKp8bfyz/LH7g4+NPej8IgsrjffylgedhegGm52MWfYzS0fR8jGLpfOlWtE0KUZNCZOQYmMakb20Y\nBpZhVY6mYVaOdmASKQTEXIgWfCIu4HkU8XENDxefIh7u+Jvh4QYeBYr89Wkf5+yV5+3bz9sAqh42\nZS8qazYLU14Sd9QgSEREpNEFQTAhWMDYcDEmaATg+i4Fr0DBL4THSe67nkvBL+D5HhhhEKl8mR/1\npd80zMqXe9MwpxxT4BVhOIeRyWBkchiZLEa2gGcZeI6Fb5sUHZPAsShaBp5t4jkmnmVQtCAwwPM9\nPL+I77rgugSFAoHrQr4AhSKG6xK4LkbBBbeIZ4JXel/PAs+xKNpGeLMMPMekaIFPUAlxRbeA5+ah\nEL5XUAg/i0IB3CK4RQy3iOH7mD6Yno/lgeEHWD6YfhCe9wMsb+S+7YHlg+2D7YHtG1jeqPNegOWB\n5YV/Xq4V4JrhrWAGuJZP3vRxDZ9i6c+kfAsAo/S7pXwfMEb92o1Rz5k+mMHoYxAeg4nP2T7YxXDM\nTulmeeCUxu54I8/bPvhGOCbPHHWzoGgalfvl864FBQcKNuQdKNhGeBxzbuQ5yw9wiqVxFCEy6v7I\nuZFrrNKfteVB3IfW0n3bB6v8OymP24S8beA6UHAMXNsIj075aOI6Bm4k/DsZcQNi+dKtMHKM5gPi\n+YBoISCW98NzU+80OGMFxyAfMShETPIRg3zUpBAJ/x44hYBIwSdS8IkWAiKFgKgbECke+OcOffJZ\nUNiUfTaDymZM02hFRGQaQRCA6+Ln8wS5XOUYFKf/hlOuZJSrCj4+vmkQGAa+yagj+JaBD/gW+IZB\nYIJngGcGeEaAb1CpmExWOZmssuH6o27jHpcZfkBkuEAsnSc2mCc6mB97fzCHnS9SiFq4UYtCxCIf\nCysQ+ahJPmqQi5SPkC19aS5YPgXTJ2+Ujx4F08MrjXn0zxIEAUHgh18c8z7xcV9mY4WAWAHiBYgW\ngkoQsIsQKR1Hn3O88Eu47YdBwiiFCyMIA4gRjD1nAvFSQPHMUrCpBIfS0TImnHOK4Zji+YB4HpoK\nECtA7AC/cBcswAh/loO9Hsu1wpBke2DtY9HqYPNLSdGcdhz1G6TnWASWie9Y+I5FYFv4EYvAscCy\nMPwgrNp5PoYXYLg+Rs7DKFXzDM8Lj8XJvl9W+eeyLHAcDNsGx8ZwHEzHwYg64Hn4+RxBKoefy8Fe\n/js2GcNxMFtaMJubMdubw2NLM2YsjjHucw07fFw+b5TOY9kEhQJ+JlO6DVfuB8OZUecz+P0ZgkIB\nIx7HbGrCbC0d43GMpvL9ppFz8TiGZYHvEXj+xKPnEfjlowd+wPITzqnCL6L6FDbrbSYNgkrdaFXZ\nFJHDnV8o4A8N4Q8Pj71lMuEFhgmmgWGaYBhglO6bZvicAYZpEvg+RbdA0c1TLOTCm5unWMjjFcPz\nfqGA5xbw3QJ+0SUoFsPKTbFI4BbDL0DFsLpC0Ss9Dr8YFCMWbsQkHwn/xTsXMcjZPlkHMo5PxvbI\nWB5DdhGCgLhrEMtTCS3hMQw2kYIfHvM+kbyHVfCwCx6W62G7fvjY9ffyhfjgMwi/RJS/SPiEFQl/\nVBXFN0rnjNLj8rlR5yc7FxjQkoX2DLRmp/+y7xtheDzQAFXmWUZ4s018KwzekYKHk/fGVKf2671t\ns1Il9G0TzzLBKv39NA0Mw6z8nQ3/3oZT8sLHVlglK3oYpRulm5EtlkJDEYpheDCCgMA08Zui+E1R\nvPZYeIzHSDdFCZpi+E3hOT8ehVgknFpY9DFdD7PoYRY8jGIR0/Uw3GLlseUWwywSjRBEIhhOBCPi\nYESjGJEIZiQSHqMxzEgEMxIFz8MoFKFcAXVdKD8ulKuVLkG+AL6PGYliRBzMSASc8Fj5HCd8f8Nx\nwlvECcOV7WDYVvgl3rLD+7Y9cr983nEqgaIcMiiHjVHnDCv8/hV4XlhZHX+rVFwLlXMVBpUqM6OP\nhjHqAsLPMC0My5x4tKzw70H5aNuVP1tse+T9D1AQBOH4c2G4C3I5/GyOID/umMvi5/IEuWw4ltLv\n24hEMaKR8HE0Gj6ORDCjkcrfiQl/1vsw9qD8D2nZ7MixPM58AbMpHobJ0s1qbg7/jKQhKGzW20z2\n2XTKYVPdaEVk/wWeh5dO46VS4W1gYML9IJcvfXmIgGMTRBxw7PBfzm0L37EJHAvfMfFsC8/N42Wz\neLkMfjYb3vJ5gmz4RSDI5yGXx8gXwqlwpSlyARCU1tsEjEz1CwggCK8x3CJO1sXOFXHyRZxcsTKt\nrdrM0m1f/k/SK01ZCwxodQ9uxccHcpHwVrAh4xi4MXBbw2llRdsJp5U5JkXHoOiE0xV9y8QybSyz\nvHbICo+miWWYWIZdOobPmRgjU/lGTeEzgpHHRmman+EFmMHI1ETDD0pHStMaA6zSeaPol96j9HjU\nDT/A8P3wWPShdN9sb8VaOgdzTgfWnDmYHR2l+x2Yc+ZgdSQw58zBbG/DsMKxWzkXsjkYzo5UIkr/\nGBH+w0R4HB0OxoaHiecpFjGbmzCbW8Z8oQ2rJk0jX3BLVRQjFhv1Jbx0i0bDL9sHKRzMROB5YJo1\n/cxDlWGVgmosVu+hVIVhGOF/8yMRrLa2eg9nAsNxsBwHZvt2h4cphc16m0FlU9NoReojCILwi28Q\nhNNZSrfJ7o8/5w8OTgx0qdH3R57zh4bACCsoIwt8Rh4HhAGG0jEwDAJzklt5yuPo+4CTKRAZyhPJ\nugdcmZlMOZhNJ2+Da4/8eDPhWpCNQDYKuZbSlMiYSb48XTJmUYjZuDEbL2pjmCZmYGBhYmFgBeE6\nNpvwvGmMnLMMszJ1ynQiWE4E03ZGjpEIphPBjkSxnCim7YShwbbDqV6l68yIg2lHMB0Hy3JKIc7C\nsWJYRSOsSLoBkbyHU/Axs6VprpkMfiYMRBiM/Gt8UxNWOcw0jQQZIxYLK7JBoPAwnQjQeN+V66Jc\nlRMRqSeFzXqrNAja+z6b6kYrMiLwvDCw9fdT3NOPl0ripdOVysVIJWPc/fLjbDZcC1EOlOXASFhp\nwa/eTIKiBcNxk6G4wVArDM8tV/TGrtmCkemDI+eDcA3XhKYSk5wrvV86BkOdBkNxg2yzTbbJJt/k\nkGtxKLREKbbEcFtj+K1NEI0S9U0ivknUM4n4Bk4Rop6B4xmVxg+2B04xCKe4RaOYsRhGLI4Vj2HG\n45ixOHa8CSvehBNrwrYcbNPGMcOjZVhYpoVt2OFjM6y42aaNbYSPbdOuPFbACunPQUREZhOFzXrb\nh61PcpMu4BZpXIHrjqvspcKGJb5P4AcQjK4aho89r0i+mCPv5nBzGYr9e/CSKYJkiiA1gJEaxBwY\nxBzMYsyw5blvGmEVrFINMym0GbgGFAMfF49i4IVdEI3ymjGjVEUMq4Pl9WTlc/6oW+Vceb2ZaVCI\n2mSbbXLNNvlmh3xzBLclhtsSJYhFcewIETOCbYUBLG7HidtxYlYsPNqxyuOYHRvzOGJFwpbqpYBm\nGia2YU88Z4bnolaUiBVRaBMREZGaUtist0rYnHrrk1h56xNVNmUGgiDASyZxt23H3bGd4o4dYBhY\nc+diz52LNWcOdmcnVnv7Pk2zCoIAf2CAYn8Sr39PWE1M9lPs78ft7yef3I3bvwcvNYA/MAADg5jD\n2QP+ecpNSHwDBuOQboJ0AtILDdJxI3zcFB6HY5CNGJW1bblIOA2zaBsjVbNSdc0xHZqcJpqdZlqc\nFpqd5spt9OOWSAvNdjNxO45jOUSsCI7pEDEjlfvjz1umpq+JiIiIKGzWm7Y+ERg7lXPUOsExj8v3\ngwA/k6W4Yzvu9u24O3aEwbJ8f8cOguwMQp5pYibaoSOB39FKsb2ZfFuMXJONP5CG1ABmagh7YBgn\nnSU2mMf0p68k5hwYioehcLDLYHCpMfI4bjAUC9ftBQZYpk3EjhF1YpVjrHSMOvHwFo1Doh2zI4HV\n1k4kEiNqRemwIiwoVfiiVpSoFQ0DnxkZM1WzfCxX+ERERESkdhQ2662yZnPqyubI1ifqRjsb+ZkM\n7q5dFHf1UuzdNXJ/1y7c3tL9vj7wDuwfE/yoQ6Grndya+QzPbWKoI0aqwyGZsMi6OUgOYA8M4Qxk\niKZzNA+6tGWStPUnSWyFpnz4H4TmUe+ZiYSVxJ0tkO42yLQ4ZFsj5NviFNvieIkWgrY2jI42rESC\npubEmGrhktL9MTc7PDqWc0A/r4iIiIg0NoXNelNlsyEEnheuLUwmw1sqRTGZxEumRs6l0wResVRx\nLFcdy2sNJ1YhveEhirt68QcHJ/9Q08Tq7MTomot55PJwk/GgSN4vUPBd8kGBnF8g75ePeXJ+Hg8f\n14I9bQZ97dDXZrC7HQbjPhgpIDXho+yITVtHGy1OC62RebRGWiu38FwrbcRJZA1acgbxjk6aOucz\npyXBklJIjNtxVQdFREREZMYUNuttJvts2uEXfIXNA1Ps7ye/fj359esprF9Pfv0Gir29lSDJ3prN\nWKXNocv7lpU3jS8dRzeOCQgoRm1ySxMMJeaRbrPpbzXY3eKzs6nAtniOrZEhhvx+oH/aj212mumI\ndjAnNoeOWAcdsQ7aI+0sc5o52mkibsdpcpposptGjqX75aYzUSuqxjAiIiIiUlMKm/U2g302bcsk\nYpnkFTZnpJhMkn/55VKo3FAJmF7/2FBnNjfjLFxIdM0arI4OrI4EViKB3dGB1dGB2xpnIO7THy3S\nG8nR56fpLyRJ5VIk80mSuSSpfIpkLkm6kJ5iNAOVewYGbdE22iJttEfmcWx0Ne2R9sq5cpCcEx0J\nlR2xDqLlqdYiIiIiIrOIwma9zSBsAkQd87CsbPrDwxSTKbyBFH46HU51HQiPfnpgzGMvnQ4rlXv2\njHkPs7mZyKqVtJz5eqKrVuEvX8TQog762wz6crvpy/TRl+ljV2YXfdnN9GX66B3sJZPMTDku27Dp\niHWQiCU4as5RJKKJSjhMRBN0RMNje3QkTLZGWjUNVUREREQOGwqb9VZpEDR92Iw7FtlDeOsTP5Mh\nv2ED+ZfXV6qS+Zdfprhz595fbBiYra0Erc0U5yXIrl1JalEbvfOibO0y2BLLsCfXz57cb9mT/RmF\nzQXYPPlbtUZamdc0j+O6jqOrqYvupm664uGxM97J3NhcErEELU6LpqWKiIiIiExDYbPeLAcwpm0Q\nBGFH2kOhG61fKFDYsCEMlKOCpbt165jrjEiEyMqVNL3mNdhdneSbIwxGA1IRl91Ojp3WMNvNATYH\n/fR4vewu9APlSuSmkTfaA7ZpMyc2h7mxuaxMrKyExrnxuXTFu8JQGe+ms6mTuB2v2Z+FiIiIiMih\nTGGz3gwjnEpbnHrrE4CYbZGbhdNovcFBsk89RebxJ8g88Ti53z9DUBj1szoO0eXLaTv/TdirVpJe\n2MamzoAX4inWp//AhtQLbB/aTsEf9Ro3vBkYdMW7WNi+mFc3n8iC5gXMa5oXhsn43EqgbIu0qQop\nIiIiIlJjCpuNwI7utbIZi1ik026NBrT/in19ZJ54ohQunyD/4ovhViCEayebTjqJ2DGvYnhJJ1u6\nDF5qTvPS0B9Yn1rPxoH7KA4UKz11TMNkaetSTlpwEguaF7CgeQHzm+eH91sW0B3v1l6NIiIiIiIN\nSmGzEVhR8KavbMYds66VzcD38TMZ/OFh/OHycRg/k6G4u4/sk0+ReeIJ3M0jiyGtuXOJnPk6Ukct\n4A/LYzzTnubl9B/YMHAL2cEsjNp+clHLItYtXMeqxCpWJlayumM1K9pXqBOriIiIiMgspbDZCOzY\n3iubjlX1brT+8DCD995H+uc/w922HT8zEiyDbHavrw8WzSN19glsXB7jifkZfmtvJVV4KHyytJ3k\n3Nhcju06ltWJ1azuWF0Jl81Oc1V/NhERERERqS2FzUZgR/e69UncCRsEBUFwUNcfBoUCQw8+RPqu\nuxi8774wVFoWzoIFmM3N2HM7MZuaMJubw1vp/pBdZFOxl/WFbTxf2Mxv5yRJtewBwm1Hmq1mVrWv\n4g2lQHlkx5GsTKxkTmzOQRu7iIiIiIg0LoXNRmBHwZ2+chh3LADyRZ9Y6f7+CnyfzG8fJ33XXaR/\n8Qv8gXCRZHztWtoueDNt552HPXfumNfsye7hsZ2P8eiOR3ls5y/YMril8tzK9pWsm7uuUq1cnVjN\n/Ob5asojIiIiInIYU9hsBHYUsqlpL4mWAma24O1X2AyCgNxzz5G+639I/+QnFHftCt93zRraLr2E\n9vPPx1m0qHJ9upDmiZ1P8OjOR3l0x6OsT62vPLekdQkXrb6IkxeczInzT6Qz3rnP4xERERERkUOb\nwmYjsKLg7X0aLUDW9ejYh7cOikVS3/8+/f95E4WeHgCcxYuZ+74raH/zm4muXl25drAwyM96fsaP\nN/yYp/uexg/CLrJd8S4uOOICTpp/EicvOJmFLQv36ccTEREREZHDj8JmI5jJms2ICbBPHWmHHnqI\n3i98gfzL67HmzKHj4otpv+DNxI47rjLF1Q98Ht/5OD9Y/wPu3nQ3OS9H3I5z5pIzOXnByZy84GRW\ntK3QlFgREREREdknCpuNYAZhM2aPVDb3Jr9xI73/fA1D992HEY0y9/3vo/PSSzGbRzq+bh/azo/W\n/4gfbfgR24a2AXB81/G8bfXbOG/5eeoOKyIiIiIiB0RhsxHYpWm0QQBTVBDjkTBsTlfZ9NJpdn/1\na/TfcgsUi7Sdfz7dH7mqshYzV8xxz+Z7+MH6H/DYjscICOiKd/GXr/pL3rrqraxoX3HwfzYRERER\nETksKWw2AisaHot5cGKTXlJuCpRz/QnPlddl9n35K3ipFLFXvYp5n/wETWvXArBtaBvfeuZb/HTj\nTxl0B7FNmzcsewNvXfVWTl14KrapvwYiIiIiInJwKWU0ArsUNr29h81sYWxlc/jhh9n1+S+Qf/ll\n7K4uFnzh87T/0R9hmOEaz5eTL3PZLy5jT24PR3YcyQdWfYA3H/FmOmL70mZIRERERERk3yhsNgK7\nFDCnWbc5uhstQKGnh13/9M/Trst8bs9zXPHLKxhyh7jmjGs4b9l5avQjIiIiIiI1obDZCOxR02in\nUO5Gm3U9ss88w6b3XExQKNB2/pvo/shHxuyRCfBU71O8/+734/ouXz7zy5y++PSqDV9ERERERGQ8\nhc1GMIOwWZ5GW8hk2f53nyAIApbeeCPNp5w84dpHdzzKX937VwB89eyvcvKCideIiIiIiIhUk1nv\nAQgjDYK8vYfN7ttvorBhA11XXjlp0Lx/6/184O4PYBkW159zvYKmiIiIiIjUhSqbjaBS2cxNeUnc\nsTiqv4fFD/yA2DHHMPeSv5xwzS83/ZKP3f8xWpwWrjvnOl4x9xXVGrGIiIiIiMi0VNlsBJWwWZjy\nkpjv8pHffQ/fslj4+c9h2GP/neDHG37M3/z6b0hEE3zrvG8paIqIiIiISF0pbDaCGVQ2g//4BouH\n+njm3HcRXbVqzHP//eJ/86kHP0V3Uzc3vvFGVnesruZoRURERERE9kphsxHsZeuTzBNPkL/1Fp7v\nWMbvX3v+mOduevYm/uGRf2Bx62L+843/ybK2ZdUerYiIiIiIyF4pbDYCKxIeJ2kQ5GcybP/kJzEi\nEb649k/IegEAQRBw3dPXcc3j13BE+xHc+MYbWdiysJajFhERERERmZLCZiOYprLZ+69fwt20ma4P\nfYjtbd1kXQ+Arzz5Ff79qX/nqDlH8e03fpvupu5ajlhERERERGRaCpuNYIp9Nocfe4zkzTcTf/Wr\nmfPnf0bcsci6Pi/2v8gNz9zAMZ3HcMO5NzAnNqcOgxYREREREZmawmYjmKRBkD88zI5PfgojFmPh\n5/4Rw7KIOxY51+Ol5EsAvPeV76U92l6PEYuIiIiIiExLYbMRWKWw6Y1sfdL7xS/ibt1K90c+QmRZ\n2PQnVgqbGwc2ArCifUXNhyoiIiIiIjITCpuNYFxlc/iRR0j+1600nXgiHe/508plMcckW/DoSfdg\nYLC0bWk9RisiIiIiIrJXCpuNYNSaTW+oNH22qYkFn/tHDHPkVxSPWOSKYWVzYctCouWKqIiIiIiI\nSINR2GwEo8Jm7zXX4G7fzryP/g2RJUvGXBazLTKFIpvTm1nevrz24xQREREREZkhu94DECpbnwz9\n7xZS3/sNTa89hcSf/MmEy+IRi3ywG8MvsKJN6zVFRERERKRxqbLZCKwoXsFgx/eexGxqYuFnPztm\n+mxZzLEomDsBNQcSEREREZHGprDZCOwofb9vo5jK0f3xq3EWLZr0sphjEdh9ACxvW17DAYqIiIiI\niOwbhc1GYEcZ3hXBmRMj8Y53THlZ3DExo2HYVGVTREREREQamcJmIzBtvLyF02pjGMaUl8UdCzPS\nR5PdTGe8s4YDFBERERER2TcKmw0g8H0818CKT//riJXC5qLmpdOGUhERERERkXpT2GwAXjoNgYEV\nm/4608pjOoPMb1oy/YUiIiIiIiJ1prDZALxkCgArFkx7Xd7YBcC82NKqj0lERERERORAKGw2AC9V\nCpuR6cPmkL8NgLnRxVUfk4iIiIiIyIFQ2GwAXioJgBXxpr1u0NsBQMJZWPUxiYiIiIiIHAiFzQbg\nJcOwaUfcaa9LudsIAoN2e0EthiUiIiIiIrLfFDYbQGUard8PwdRTafcUthK4CYqeXauhiYiIiIiI\n7BeFzQZQrmxaVhaGd096jR/47M5twy90kS34tRyeiIiIiIjIPlPYbADFctiM+jCwedJrdgzvwA0K\n+IVOcu70aztFRERERETqTWGzAXjJFFgmphNAavKw2TPQA4Cf7yarsCkiIiIiIg1OYbMBeKkUVlsL\nhsGUYXPjwEYAVTZFRERERGRWUNhsAF4yid0xJ3yQ2jLpNT3pHgD8QpfCpoiIiIiINDyFzQbgJZNY\nczrBikw7jTZuNxEU2zSNVkREREREGp7CZp0FnoeXTmN1dED74mmn0S5rXQYY6kYrIiIiIiINT2Gz\nzrx0Gnw/DJuJpTCwZcJem8PuML3ZXla0rwAgV1RlU0REREREGpvCZp15yRQAViIRhs3CEGSTY64p\nr9dckVhOxDLJFRQ2RURERESksSls1pmXKoXNjg5oXxqeTG0ac01525MV7SuIOabWbIqIiIiISMNT\n2KwzLxVWMa2OUmUTJqzbLG97sqJtBfGIpW60IiIiIiLS8BQ268xLlsJmYnTYHLv9SXka7dK2pcQc\ni6yrBkEiIiIiItLY7HoP4HBXnkZrd3RAYm54clxls2eghwXNC4jbceKOKpsiIiIiItL4VNmss0pl\ns6MDWheAaY8Jm37gsym9qdKJNqawKSIiIiIis4DCZp0VR4dN0wr32hwYmUa7c3gnOS/H8rblAGoQ\nJCIiIiIis4LCZp15qQGwbcyWlvBE+5Kwslnaa7PciXZ5+3IA4o5FVlufiIiIiIhIg1PYrDMvmcRK\nJDAMIzyRWAb5NOTCtZwb06VOtKVptPGIRb7o4/tBXcYrIiIiIiIyEwqbdeYlk9gdiZET4zrSlrc9\nqUyjtS0A8kV1pBURERERkcalsFlnXjKJ1T46bC4Jj6UmQT3pHuJ2nHlN8wCIRcKwqXWbIiIiIiLS\nyBQ26yjwPLyd1fM8AAAgAElEQVR0OmwOVFapbJbC5kAPy9uWV6bZxp0wbKojrYiIiIiINDKFzTry\n0mnw/cnD5sAWMm6GXZldleZAEHajBVU2RURERESksSls1pGXDJsAWYlR02hbF4JhQWozPekeAFa0\nrag8Xa5sqiOtiIiIiIg0MoXNOvJSpbA5urJp2dC2CFKbJmx7AhBzyg2CFDZFRERERKRxKWzWkZdK\nAmCN7kYL4VTa1JZKZbPciRZGwma2oG60IiIiIiLSuBQ268hLlsJmYnzYXAK5FBv7XwJgWduyylOV\nabRasykiIiIiIg1MYbOOytNo7dHTaKHSJKgntZ75zfNpcpoqT8Uj6kYrIiIiIiKNT2GzjiqVzUnC\npg9sGt4+ZgotqButiIiIiIjMDgqbdVScahpt+xJ2WRZZ32VF+4oxT8W0z6aIiIiIiMwCCpt15KUG\nwLIwW1vHPpFYykbHAZhQ2YwrbIqIiIiIyCygsFlHXjKJ1dGBYRhjn2hbRE+kFDZHbXsC6kYrIiIi\nIiKzg8JmHXnJJFaifeITdoSepnBq7Yq2sdNo1Y1WRERERERmA4XNOvJSKexEx6TPbYzFiPsB85rn\njTmvbrQiIiIiIjIbKGzWSeB5eAMDEzvRlvSYActcF7OQGXM+ZitsioiIiIhI41PYrBMvnQbfnzRs\nZtwMO4MCy10XBraMeS4W0dYnIiIiIiLS+BQ268RLpoBJtj0BNg9uBmCFW4TU5jHPRSwT01BlU0RE\nREREGpvCZp14qVLYnKSyuXFgI0BY2RwXNg3DIOZYZF11oxURERERkcalsFknXioJgNUxsbLZM9AD\nTB42IexImyuosikiIiIiIo1LYbNOvGQpbE4yjXZjulzZnDiNFihVNhU2RURERESkcdn1HsDhqjyN\n1p5kGm3PQA/zmubR1DKxQRBAzDG1ZlNERERERBqaKpt1UqlsjgubQRDQk+5heftySCydfBptRJVN\nERERERFpbAqbdVKcYhrtrswussUsy9uWQ/sSGO6DcXttxh1LlU0REREREWloCpt14qUGwLIwW1vH\nnO9J9wCwon1FWNkEGNg65pqYY5FTN1oREREREWlgCpt14iWTWB0dGIYx5nx525MVbaPC5riptOUG\nQUEQ1GSsIiIiIiIi+0oNgurESyaxEu0Tzle2PWlfDtnS9NnUpjHXxB0Lzw9wvYCIPTasioiIiIiI\nNAJVNuvES6WwE5N0ok33ELNizG+eD4ll4clxHWljTvhryxW1blNERERERBqTwmYdBJ6HNzAwoRMt\nhNNol7UtwzRMaF8cnhw3jTbuWADkCgqbIiIiIiLSmBQ268BLp8H3J3SizRaz7BjeEU6hBXDi0Nw9\ncc1mJAyb2v5EREREREQalcJmHXipFDBxj83N6TBULm9bPnIysRRS46bR2qXKpjrSioiIiIhIg1LY\nrAMvOXnY3JgOO9FWKpsQhs2hneDmKqfiqmyKiIiIiEiDU9isAy+VBJgwjbay7Un7ipGTiSXhcdRe\nm+U1m1mt2RQRERERkQalsFkHXrIUNjvGhs3Ktifjp9ECDIys21Q3WhERERERaXQKm3VQXrNpj5tG\n25PuoTveTbPTPHKyvP1JanTYVDdaERERERFpbAqbdTBS2RwJm0EQ0DPQM3YKLUB7aRrtqLBZmUar\nNZsiIiIiItKgFDbroJicuGazN9NLppgZ2xwIRtZsjupIW6lsqhutiIiIiIg0KIXNOvBSA2BZmK2t\nlXM96R5g3HpNgEgzNHWOrWyqG62IiIiIiDQ4hc068JJJrI4ODMOonKs0Bxpf2YSwujkqbHY0OQDs\nSGWrOUwREREREZH9prBZB14yiZVoH3OuvMfmhDWbEHakHdwBxQIAR3S20NkS4cH1u6s+VhERERER\nkf2hsFkHXiqFnRjXiXagh6gVZUHzgokvSCwFAkiHe22apsG6VZ28sHOQvsF8DUYsIiIiIiKybxQ2\nayzwPLyBgTGdaAG2DW1jUcsiTGOSX0l7aa/NUVNpT1vVCcDDG1TdFBERERGRxqOwWWNeOg2+P6YT\nLUBfto/upu7JX5Qoh82RjrSnrQ7D5gMvK2yKiIiIiEjjUdisMS+VAsbusZlxMwy7w3TFuyZ/UWJi\nZXNBe5yVXc08+PJugiCo2nhFRERERET2h8JmjXnJiWGzL9sHQFfTVGGzvNfm5jGnX7e6i53pHBv6\nhg7+QEVERERERA6AwmaNeakkwJhptL2ZXoCpK5vRVoh3wMCWMafL6zY1lVZERERERBqNwmaNjVQ2\nR8JmX2YvlU0Ip9KOq2yesnIutmnwkLZAERERERGRBqOwWWPlyqY9yTTaKRsEAbQvgfQ28NzKqZao\nzQlLEzzyh35cz6/OgEVERERERPaDwmaNecmJ02jLlc3OeOfUL0wsg8CH9PYxp9et6mQoX+SpLamD\nP1gREREREZH9pLBZY8Vy2JysQdBUazZh0o60AK/TFigiIiIiItKAFDZrzEsNgGVhtrZWzvVl+2iN\ntBKzY1O/cIqOtMctTtAatXnw5b5qDFdERERERGS/KGzWmJdMYiUSGIZROdeX6aM7Ps16TRipbI7r\nSGtbJqesnMvTWwdI59xJXigiIiIiIlJ7Cps15iWTYzrRQljZnLYTLYQNgmBCZRPCqbSeH/DIhj0H\na5giIiIiIiIHRGGzxrxUCjsxsl4z42YYdoenX68JEE9AtH3SsLmutN/mg9oCRUREREREGkTVw+bL\nL7/MqaeeypFHHsmJJ57Is88+O+GaX/3qV8TjcY4//vjKLZvNVntoNRd4Ht7AwNg9NrMz2GOzbJK9\nNgGO6GxmYXuMB9UkSEREREREGkTVw+YVV1zB5ZdfzksvvcTVV1/Ne9/73kmvW7NmDU899VTlFo/H\nqz20mvPSafB9rFGVzd5ML7CXTrRliaWlvTaLY04bhsFpqzv5w+5htqUOvZAuIiIiIiKzT1XDZm9v\nL48//jgXX3wxABdddBFbtmxh/fr11fzYhuWlwr0wx2x7ktmXyuYS8IswuGPCU6etDl//kKqbIiIi\nIiLSAKoaNrds2cKCBQuwbRsIK3BLly5l8+aJU0E3bNjA2rVrOfHEE/na17425Xtee+21LF68uHIb\nGhqq2vgPNi85SdgsTaPtbtpLN1qYsiMtwLqVcwF4QOs2RURERESkAdj1HgDA2rVr2bp1K+3t7Wzd\nupXzzz+fzs5O3vnOd0649qqrruKqq66qPF68eHEth3pAvFQSACsxas1mqbLZGe/c+xuUw2ZqMyw7\ndcxTc1uivGJBGw+t343vB5imMckbiIiIiIiI1EZVK5tLlixhx44dFIvhGsMgCNi8eTNLly4dc11b\nWxvt7e1AGB7f/e5388ADD1RzaHUxUtmcpEHQTNZsTrP9CYRboPQPF3huR/rABioiIiIiInKAqho2\nu7u7Wbt2Ld/5zncAuP3221m8eDGrVq0ac92OHTvwfR+AwcFB7rrrLk444YRqDq0uypVNe9w02tZI\nKzE7tvc3GF3ZnMRpq7UFioiIiIiINIaqd6O97rrruO666zjyyCP5whe+wLe//W0ALr30Uu68804g\nDKHHHHMMxx13HKeccgrnnHMOf/EXf1HtodWcl5x8Gm13fAbrNQHiHRBpnTJsnrh8DhHb1BYoIiIi\nIiJSd1Vfs7lmzRp+85vfTDh/ww03VO5feeWVXHnlldUeSt0VJ+tGm+3jmM5jZvYGhjHlXpsAMcfi\npOVzeKynn5zrEXOsAx6ziIiIiIjI/qh6ZVNGeMkUWBZmaysAGTfDsDs8s/WaZYklMLAVStOOx1u3\nqpNC0efxnuTBGLKIiIiIiMh+UdisIS+ZxEokMIywU2ylOdBM9tgsSywF34WhnZM+/brSus0H1vcd\n2GBFREREREQOgMJmDXnJ5JhOtL2ZXmCGnWjL9tIk6BUL2pjTHNG6TRERERERqSuFzRryUinsxMh6\nzd3ZMBDuU2Wzsv3JlkmfNk2DU1fO5dntafYM5fd7rCIiIiIiIgdCYbNGAs/DGxiYtLLZ3TTDbrQw\nqrK5acpLylNpH96wZ98HKiIiIiIichAobNaIl06D72ONqmz2ZcJ1lZ3xzpm/UWJZeJxiGi2ETYIA\nTaUVEREREZG6UdisEW+KbU9gH9dsNs0BpwkGJp9GC7C4o4kVnc08uH43QRDs34BFREREREQOgMJm\njXjJUthMjEyj7cv20RppJWbHZv5Ge9lrs+y0VZ1sS2XZuHt4v8YrIiIiIiJyIBQ2a8RLhftejqls\nZvroju/Des2yxNKwQdAUe20CnFZat/ngek2lFRERERGR2lPYrJFKZbNjbGVznzrRlrUvAS8Pw1Pv\npfnalXMxDa3bFBERERGR+lDYrJFyZdMuTaPNuBmG3eF9W69Ztpe9NgHaYg7HLUnwmw17KHpTV0BF\nRERERESqQWGzRrzk2Gm0leZA+1PZnMH2JwCvW9XJYL7I01sH9v0zREREREREDoDCZo0Ux3WjLe+x\nuV+VzY7l4XHzb6a97LTV4XtrKq2IiIiIiNSawmaNeMkUWBZmaysAu7NhANyvyuaC42HhWnjiRtj9\n8pSXnbA0QXPE4sH1U6/tFBERERERqQaFzRrxkkmsRALDMICRymZ30350ozVNeOMXwC/CLz495WWO\nZXLKEXN5cnOKoXxxv8YtIiIiIiKyPxQ2a8RLpcZ2os2E1cbOeOf+veHSk+GVF8JLP4UN90152bpV\nnRT9gEf/sGf/PkdERERERGQ/KGzWiJdMYidG7bFZbhC0P2s2y875DFhR+PmnwPcmveR1pf02H9C6\nTRERERERqSGFzRoIPA9vYGDCHputkVZidmz/3zixFE69Enqfhd/dNOklq7pbmNcW5cH1CpsiIiIi\nIlI7Cps14KXT4PtYoyubmT664/uxXnO80z4MLfPg3s9CbuIWJ4ZhcPrqLtb3DnHfi70H/nkiIiIi\nIiIzoLBZA964bU8grGzuVyfa8aKtcNanIbMbHvjipJd88OzVtMZs/ua/n6Y3nTvwzxQREREREdkL\nhc0a8JKlsJkIp9Fm3AzD7vCBrdcc7fg/hfnHwCNfh/6NE55eMqeJL1x4LHuGC1z130/j+8HB+VwR\nEREREZEpKGzWgJdKAiOVzUpzoINR2QQwLTjv8+AV4Jf/b9JL3nzsAt590hIeXL+b6+7/w8H5XBER\nERERkSkobNZApbJZahBU3mPzoFU2AVa8Do66AJ6/E3oemvSS/3fBK1nd3cIXf/EiT25OHrzPFhER\nERERGUdhswbKlU27NI12dzbsDHvQKptl5/x/YDrw80+A7094Oh6x+Lc/PQHLNPirW58knXMP7ueL\niIiIiIiUKGzWgJccO422XNnsbjoI3WhHm7sSTnkf7Hganr510kuOmt/G317wCrYms3zyjmcIAq3f\nFBERERGRg09hswaK47rRliubnfHOg/9hp38UmubCPZ+B/NCkl1x88lLOe+U87vr9Dr7/+NaDPwYR\nERERETnsKWzWgJdMgWVhtrYCVVqzWRZrhzM/BUO74KEvTXqJYRj800XHsrA9xt/d+SzrewcP/jhE\nREREROSwprBZA14yiZVIYBgGEHajbY20ErNj1fnAtf8Huo6Gh/8NUlsmvSTRFOFL7zqBfNHjyv96\nkpzrVWcsIiIiIiJyWFLYrAEvlap0ogXoy/TRHT/I6zVHs2w47x+hmIO7/37Ky05aMYe/PvtIXtg5\nyOd/8nz1xiMiIiIiIocdhc0aKFc2y/qyfQe/E+14q86G1efB/94GWx6b8rIrz1rFySvm8J+/2cQv\nn9tV3TGJiIiIiMhhQ2GzygLfxxsYwC41B8q4GYbd4eqs1xzv3M+CacPPJt8KBcAyDb70ruNJNDl8\n9Lan2TGQrf64RERERETkkKewWWV+Og2+j5UIw2Zftg+owh6bk+k6Ek68FLY9Dv97+5SXLWiP888X\nHUsq4/Kh7z6F52s7FBEREREROTAKm1VWnGKPzZpUNgHOuBpiCbj772B495SXnfvK+fyf1y7j0Y39\n/Pu962szNhEREREROWQpbFaZlyztsVlas1neY7MmlU2Apjlw7j9Aehvc8nbIT73NySfOP5qjF7Tx\n5Xte4tE/7KnN+ERERERE5JCksFllXmryymZ3UxW70Y639s/h9I/C9ifhexdDMT/pZTHH4t/efQJR\n2+Ivbvwt9zyvhkEiIiIiIrJ/FDarrFLZ7Bhb2eyMd9Z2IGd+Cl79F/CHX8Edl4M/+b6aq7pbuPmS\nk4jaJpfd9Dg3PrSxtuMUEREREZFDgsJmlZUrm3ZpGm3N12yWGQa8+Yvwij+G534IP/koBJM3AnrN\n8jn84APrWD63mb//8XP8/Z3PqmmQiIiIiIjsE4XNKvPGNQjqy/bRGmklZsdqPxjTggu/CSvOgMf/\nA371+SkvXd7ZzB0fOJWTV8zhxod7uPymxxnOF2s4WBERERERmc0UNqusmCpPoy2FzUwf3fEartcc\nz47Cu26BBcfDr/8JHr1+yksTTRFuvuRkLly7iHte6OWd1/2GnQO5Gg5WRERERERmK4XNKvOSKbAs\nzNZWIKxs1qwT7VSirXDx7TB3Ffz0Y/DMbVNeGrFNvviO47jqnCN5dnuat371IZ7dPlDDwYqIiIiI\nyGyksFllXjKJlUhgGAYZN8OwO1z79ZqTae6EP/sBtC6AH1wB6++e8lLDMPjg2av58ruOp3+4wDu+\n8Rt1qhURERERkWkpbFaZl0pVOtH2ZfuAGu6xuTeJpfBnd0CkBb73Z7D18Wkv/+PjF3HLZSerU62I\niIiIiOyVwmaVlSubEK7XhDp0op1O99Hwnu+H9295O/S9OO3lJ5Y61S5Tp1oREREREZmGwmYVBb6P\nNzCAPaoTLTRQZbNsyUnwzpshPwg3vw1SW6a9fHlnM3e8/1ROGtWpdiDj1miwIiIiIiIyGyhsVpGf\nToPvYyXCsFneY7O7qY7daKey+g3w1m9AelsYOIf3THt5R3OEmy85iQtPCDvVnn3tr/npMztqNFgR\nEREREWl0CptVVCzvsVmaRrs7uxuAznhn3cY0rWPfAW/8J9jzMlx/Brzwk2kvj9oWX3zncfzLO47D\n9Xzef8vvuOLmx+lNa3sUEREREZHDncJmFXnJsXtsliubDbVmc7xT3gdv+TLkBuC774Zb/3TaabWG\nYfD2Vy/m7qvO4M3HLODnz+7i7Gt/zXcf20wQaC2niIiIiMjhSmGzirzU2LC5O7ub1kgrMTtWz2Ht\n3avfC1f+Fl51Ebz4P/DVk+Chr4A39brMrtYoX33PWq7/s1fTFLH4+B3P8J4bHmXTnuHajVtERERE\nRBqGwmYVeZVptO1AWNnsjjfges3JtM6Ht38LLr4DWubBLz8N150Bmx+d9mXnvnI+v/jwGbz7pKU8\nvGEP533pfq6/fwNFz6/RwEVEREREpBEobFaRlwrD5uhutA3XiXZvVp0NH3gEzrg6XMv5rXPhzg9C\npn/Kl7THHT5/4THcetkpzG+L8bmfvMCFX3+Y53ekazhwERERERGpJ4XNKqpUNjs6yLgZht3hxl6v\nORUnBmd+Et7/MKw4HX73n/Dvr4GnboVp1mW+duVcfvah07nijCN4dnuat/zbg/zLz18k53o1HLyI\niIiIiNSDwmYVFUet2WzYPTb3Redq+PM74cJvgmHCD98HN14AfS9O+ZKYY/GJNx3Nj/7vOo6c18q/\n37ee8750P7c/sVVTa0VEREREDmEKm1XkJVNgWZitrfRlSmFzNlY2RzMMOPadYQOh1/wlbHoIvr4O\nfv3P0zYQetWidn505To+/qaj6B8u8JHvP825/3o/P3xyG56vrrUiIiIiIocahc0q8pJJrEQCwzAO\njcrmaPEOuOBf4ZJfhhXP+/4RvnkW7Hxmypc4lsn7zljJg1efxV+fvZq+wTwf+t5TnPuvv+bOp7fj\nK3SKiIiIiBwyFDaryEulsDoSwMgem91Ns6Qb7UwtOREu/xWc/lHY9Sxc/3r41RegWJjyJe1xhw+f\ncyQPXH0mf3XWKnYO5PjgrU/yxi/fz//8fodCp4iIiIjIIUBhs4rKlU0I99gE6Ix31nNI1WFH4ay/\nhcvuhc418KvPh1XOHb+f9mWJpggfOXcND159Fh94/Uq2JrP83//6Hed/5QF+9r8KnSIiIiIis5nC\nZpUEvo83MFDZ9qRc2Zz1azans/D4sMp5xtXQ9zx880y473PTVjkBOpojfOyNR/HAx87kijOOYNOe\nDO/7zu+44N8e5BfP7iSYpuOtiIiIiIg0JoXNKvHTafB9rEQYNndnd9MaaSVmx+o8siqzI+E2KZfd\nC11Hw6//KQyd25/a60vntkT5xJuO5v6Pncmlp61gQ98Ql9/8BGdf+2u+/dBGBrJTNyASEREREZHG\norBZJcXyHpuJkTWb3fFDbL3mdBYcFwbO138C+l4Ip9Xe+1ko5vf60q7WKH97wSt44GNncvnpR9A/\nXOAzP36OUz53D5+44/c8u32gBj+AiIiIiIgcCIXNKvGSI3tsAvRl+w6dTrQzZUfg9R+Hy+6D7lfA\n/deEDYQ23Atudq8v726L8cnzj+aRT5zNNW8/liPntXDrY1t481ce5KKvP8wPn9xGvuhV/+cQERER\nEZF9Ztd7AIcqLzUSNjNuhmF3+NBerzmdBcfC5ffBA9eGgfPmt4HphOeXnAyLTwyP7YsmfXnMsXjH\na5bwjtcs4ektKW5+ZBM/fno7H/reU/zDXRHeeeIS3nPyUhZ3NNX4BxMRERERkakobFaJV5lG237o\n7bG5PywHXn81vPKt8MJdsOWx8LbtiZFr2haHW6ksORmWnATzjw1fN8pxSxIctyTBp84/mtue2Mp3\nHt3E13+1get+vYGzjurmPacs4/TVXVimUeMfUERERERERlPYrBKztYX4CSfgzJ9PX6YUNg/XyuZo\nXWvCG0AQwJ4NsOXR8Lb1t/DsD+HZH4TP2zFYuBZOuBiO/ROwRv66djRHuOz0I7jktBXc/3If33lk\nE/e80Mvdz/fS3RrlbScs4sK1i1kzv7UOP6SIiIiIiBjBLN9XYvHixWzdurXew5jWTzf+lI/d/zH+\n5Yx/4bzl59V7OI0tm4Jtj49UPrc8Bu4wzFkZrv981UVgWpO+dEt/htue2ModT25lS3+4JvSVC9u4\ncO1i/vj4hXS2RGv5k4iIiIiIHPKmy2MKmzVw07M3cc3j13DTm27ihO4T6j2c2SWbgke+Do98DfJp\n6DwyDJ2veBuYk/e3CoKAxzclueN3W7nr6R0M5otYpsHrj+ziwrWLOfvobmLO5IFVRERERERmTmGz\nzr74+Be58dkb+cmFP2FJ65J6D2d2yvTDb74Kj34DCkNhd9vXfxyOesuUoRMg53r88rld3PG7rdz/\n8m48P6A1ZnPBsQu5aO0iXr2sA8PQ+k4RERERkf2hsFlnV99/NT/Z+BN++57fErNj9R7O7Da8Bx7+\nCjx2PbgZmHcMnPkJWHM+7CU09g7muPOp7dz+u208vyMNwIL2GGcd1c1ZR3Vz6spO4hFVPEVERERE\nZkphs84u+fklPN//PA+/++F6D+XQMdQHD30JfnsDFHOw4Hg481Ow+py9hk6A57an+eFT2/jlc7vY\nuHsYgKhtsm5VZyV8LkzEq/1TiIiIiIjMagqbdfaWH7wFy7D44Vt/WO+hHHoGd8KDX4LHvwVeHhYc\nF06tXf0GmH/ctFNsy/7QN8S9L/Ry7wu9PLaxn6If/k/iqPmtnH10N2cdNY/jlyS0nYqIiIiIyDgK\nm3X22v96La/qfBXfPPeb9R7KoSu9HR64Fp6+NVzTCdDcBSvPhlVvgJVnQfPcvb9NzuWBl3Zz7wu9\n3PdiL/3DBQDmNEd4/ZFdnH5kF+tWddLVqs62IiIiIiIKm3WUcTOc/F8n85Yj3sLnXve5eg/n0Fcs\nhHt2rv8lrL8Hdv1v6QkDFq2FVeeE4XPR2im3UCnz/P+/vXsPjqs87D7+PefsXdrVanXxTZJlYxuI\nwTYXB2NCKSFNyIQk9OXNNG0Jl5BAkkk7tEnLvJOmTWcyNJ3wMk3bmZekL2USkjBJSkIzpYSXhCSY\nUi4mGGKuvuhmS5atu1Z7Pec87x9ntZJsAwZLXiT/PjPPnGfPnt19pOMj+afnOc9jeP7AGI++fJhf\nvHK4ep8nwNkrUly6vpn3rGvm3Wsymt1WRERERE5LCps11DPRw1U/uYpPnvNJ/uyCP6t1c04/E/1B\n6Nz7COz7FRTHg/3xxqC3811Xw5kfBCf8pm91aLzA43uHeHzPER7fO8RQNuj1jIRs3t2Z4T3rm7l0\nfTNnL09ha8itiIiIiJwGFDZraOehndz48I3ctvU2rn3XtbVuzunNc+HAM7D350EZ2BXsr18O518X\nlPSJLU3j+4ZXDk2yoxI8n+4aoej6ADTVRbhkXdDruXVNhs6mhJZXEREREZElSWGzhh7qeoi/fOwv\nueOyO/hA5wdq3RyZbWIAdn0Pnv02jPeCZcP6D8CFn4R1V7zpMNvZCmWPZ7pHeHzPEDv2DPHSrCG3\nLcko7+7MsLWzkXevaeLM5UlNNiQiIiIiS8Ib5bHQKW7LaedI7ggArYnWGrdEjpFaAb/zRXjPnwVD\nbXf+K+x5GF57CBra4YLr4bzrILnsTd8qFna4dH0Ll65v4X8BQ9kiT+0f4ZnuEZ7qGuE/dw/w4G8H\nAEjGQmztzLC1M8O712Q4d1UDkdCbz5orIiIiIrKYKGwusCP5IGw2x5tr3BJ5XbYDG94flLE++M13\ngvLoV+FXX4OzPhT0dnb+zgktpQLQXB/lQ5tW8KFNKwAYz5f5Tc8oT3UFAXTHniM8+sphAGJhmy3t\nabZ2ZriwM8N5HWlSsTe/h1RERERE5J1MYXOBHc4FgaIl3lLjlsgJSbfDe78El/0lvPpQ0Nv50r8H\nJb06mMW2aR00ra9sz4B4+k3ftiEe5vKzWrn8rKCHO1/y2NU3xtOV8PlszyhP7h8BwLLgzGVJLuxs\nZGtnhmZ9EcUAACAASURBVAtWN7IqHdd9nyIiIiKyqOiezQV208M38fLIyzzxh0/Uuinydg3vg998\nG178CYz1Hvt8orkSPCvhs2kdNK+HzFoIndh6nGXP5+WBCXZ2j/JszyjPdI9weLJYfX55KsYFnY1s\nXd3IhZ0ZzlqeJORo6K2IiIiI1JYmCKqhD//kwziWwwNXP1Drpsh8KOVgtAuG98LQniCIDu8NSn5k\n7rFOFFZdAKsvho7t0P5uiKVO6GOMMRwYzbOzZ6QaQF8dnGT6ao2HHc5ekWTjygY2rkyxcWUDG5bX\nEw1pvU8REREROXU0QVANDeWHOKf5nFo3Q+ZLJAHLNgblaLmRWeFzD/Tvgr6nofcJ4H8Hs90uOwdW\nXzITQOuPP7zasizaMwnaMwl+/7w2oHLfZ+8oz3aPsqtvjBf7x/lN71j1NSHbYv2yZCV8BgH0XStT\n1Ed1mYuIiIjIqaeezQWUK+e46PsX8eG1H+b2S2+vdXOkFjwXBndDzxNB6Oz5b8gNzTzftA46LobV\n24MAmzkDovUn9NbGGPrHC7x4cJwX+yd4sT/YDowXqsdYFnQ21XH2iiQbls2UzqaEhuGKiIiIyElT\nz2aNTM9E25LQ5ECnLScEK7cE5eLPgTFBz2fPE9D739DzX/DcvUGZVr88uPczs3bmPtDMGZBZA+F4\n9TDLsliVjrMqHef9G5dX9w9ni7w0MMHug0EAfal/god2H+I/f3uoekzEsTmjtZ4zl9WzflmSM5cl\nOXN5klXpOLbWABURERGReaCwuYCm19jUTLRSZVnB5EHN64N1PAHGD0Lfk3PvAR3cHQTRuS+GhrYg\nhLacCW3vho6LgjVBZ81U21Qfra75OS1f8th3JMurhyZ5bXC6ZHlgV/+cT0hEHNa31rOhEj6ng+iy\nVFSz4YqIiIjIW6KwuYDUsyknpGEVNFwzd58xlXtA98LIvpkQOrIPDuyErl/D098Kjk2uDCYf6tgG\n7RfB8k1Bj+os8YjDOasaOGdVw5z9E4UyewazvDY4yauHJtlzONg+f2B8znHJWIgzl02HzyCMblie\npLn+xGbbFREREZHTj8LmApru2WxNtNa4JbLoWBbUNQWl46K5zxkDo93Q9xT0PhlMQvTSv8NLlRmP\nw4lgFtyObdC+Ddq3QqzhmI8ASMXCXLC6kQtWN87ZPzJVmtUDOslrh7K8OjjJzp7ROcdl6iKsb63n\njNZ6zmip54yWOs5oqddwXBERERFR2FxI0z2bzfHmGrdElhTLCu7fzKyBzR8P9uXHgh7PvieDAHrw\nWejeMf0CqGuGSB1EksE2Wg+RSqnW6yCahFiaTNuFbFvTyba1TdWPNcZwZLLIa4NB8NwzOMmrg5O8\n1D/BU11zl32JhmzWNNcdE0LXttSRiOjHjoiIiMjpQP/rW0DVYbS6Z1MWWjwN698XFACvDId+G/R6\n9j0Fk4egNAnFLGQHoZSFcu6N37OhA9b8TqVcipVaSWsqRmsqxnvWz/wBxRjDkWyRfYen2Hcky74j\nWfYfCer/+dsBjp7vekVDjM6mOjqb61jbHGzXNNfRkUkQCWmGXBEREZGlQkufLKCbHr6Jl0de5ok/\nfKLWTRE5lu9BaSoInsVssC1lIXs4mJyo67HgPtFpTetmwmfnpUFv6ZvIF126Dw1xsL+Pw4P9jAwd\n5rXJCE+NpRgsRuYca1vQ1pgIwmdTgjWVILq6qY5V6biCqIiIiMg7kJY+qZHDucO0xnW/prxD2Q7E\nUkE52rn/M9iOHwyG43btCCYl2vmvQQFYdk4QOjNrIT8KueFZZQhyI8Rzw5ztFjj76Pe3wG9sIlfX\nxkhkJQes5ewtNbM718jTXSkefy2Nz0y4tC1YmY6zuilBRyZBR6auWl/dlCAZCy/It0hERERE3j6F\nzQU0lB/inOZzat0MkbevYVVwX+jmj89MTNT12Ex56v8c+5pIEhIZqF8Gre+CRFPQC5rIBBMVTQ3D\naBf2aDf1o93UDz1PB7B9+vUOmEiEQmIVQ/HVvBjfyq/N+eyaDLOrd4z/2jt8zEdm6iJ0ZBK0ZxLB\n2qONcdoa47RV6rpPVEREROTU0//AFkiunCNbzup+TVk6Zk9MdMH1Qfg88ipkDwWBMlEJlKG3uBxK\nKQdjvTDaFYTZ0W6skS7io920Dz1Ou/8rrgRYdi7msg8w3nEF+8Jn0juap2c4R+9wjp6RHD3DOXb1\njR33IxoTYVY1xoMgmk7Q1hivPl6ZjtOYCGsdUREREZF5prC5QIbyQ4DW2JQlzLKg9aygnIxI4vXf\npzAO+x6F1x6G1x7G2nEHae7ggroWLlj/AdjwAbj08mAWXSBf8jg4lg/KaJ6DY7nKNnj8Uv8E/nHu\nUo+FbVamK+GzIQigK9Oxahhd3hAjFnZO7usUEREROc0obC6Qw7nDgGaiFTkpsQbY+PtB8b1geZfX\nHoJXfwa7vhsUJwKd74ENHyS+YjPr4o2sW5mGtcshNHcSorLnc2i8UA2fA+N5Do4V6B/L0z+W59me\nUXaUho7blOb6CMsbYixPxWZt4zOPG2LUR/UjVURERGSa/me0QKrLnqhnU2R+2A50XBSU930lGHL7\n2v8Lwmf340EP6NEi9RBLQ7wR4mnC8TTt8Uba441BkI1FYVUYOsJghzF2iLxnM1QwDOcNQzmfIzmf\nwaxHXy7Mb7NpfjUwiXu87lEgGQ2xrCHGioYYLckorckYrckoram5dd1DKiIiIqcD/Y9ngYwUgkXu\nWxOajVZkQTR2wkU3B6U4Cft/HQTQ/GhQCmMz9fwYjPcFw3J5/dWeLCABdFTK8Zh0M25DB1OJdoYj\nKxl0ltPjt7Kn3MyrU3UMTJZ5tmeUXMl73c+pj4ZoTUZpqZRlqRjLUtPboCxPxYhHNHRXREREFi+t\ns7mAcuUcYSdM2NayDCLvCL4XBM7CWBBAvTL4ZfBK4LmVehl8N9h6pco+F/IjMNpTmcSoC7KDx76/\nE4F0B6axEzfZxkRsJSOhZRyyW+n1mugt1nM4W+bwZIHDE0UOTxYZz5dft7nJWIjllfDZmopW65m6\nCMlYiGQsTKqyTcZCJCKOJjoSERGRU0rrbNZIIpyodRNEZDbbCWbMTWRO/r1KU5VZdLthZGYmXUa7\nsbp2EPaKNAFNwPrp1zgRaGiHdDuc0Q7p1ZSTqxgNL2PQauKgm2ZgynBoIgijh8YLDE4W2NU3Rrbo\nvmmTHNuiPhqqBtFkLEQqFqYlGaElOT20N1rdNtdHNfGRiIiILBj1bIqIzDffh6nDMNYH471BKB3r\nC4byTtfLU8d/baIJUishuTLYplZBagX5+DKOWE0MeI2MFKGQHaU0NU45N46bn8AvTEBhEkqTOOVJ\nnPIUETeL5RY4aJrY769kn1lBl1lBjlj14xri4TkhtKkuSkM8TEM8RCoertSDMv1YAVVERESmqWdT\nRORUsm1ILg9K+9ZjnzcmuJd0rGcmhE70w+RAsJ3oD9Yw9UrVl8R543tJX9dxcmE22sqRSDsHnHb2\n+St4udzK8wdb+O9CGoP9pm8ZCdmkYkEgTSciNMTDpONhGhLhan16f0Oi8lylhJw3f38RERFZGhQ2\nRURONcuaGc678rzjH+P7kBuGyf6ZADpdjB+sLTqnpI6/zwkHvanDe2AoKPXDe6gfeoU15We5dNZH\nmvoYbqqDYmIZuWgr2UgLo04zw3YTh8kw4Dcy4CYZK3iM58uM58v0juQYz5UpeX7wpeHTwBSNVpZG\nJklbWRrJkrYmiVJm2GllNLqSbHwVbryVVCJMKhb0mqbiwT2owTZMOhGmMRGhMRGE10hIQVVERGQx\n0TBaEZHTkTFBT+rQnkoQ3RtsR7thYuD1h/naIahfDqkVQc+tV8bkRjC5EciNYBVGsd5gxt/ZCkTo\n81voMa30mVZ652xbyM8a7gvBLL7TAXR6m6mbqSdjoco9q+Fqvb6y1dBfERGRhfFGeUxhU0RE5jIG\nihNB6JzsP2o7MDPkN3s4CJ+JTHCvaTwDicbKNnPs1onA+IE5kymZ0W4Y68Xyj52VtxBuIBvKMOFk\nGLHSHDENHPJTHCwn6S3W01dOMWQaGCaF/ybDfyOOXQ2esydRCmbznQmoM9u5s/3Wx0Ikwg62rdl+\nRUREZtM9myIicuIsC2INQWk96/WP8z2w7OD4E7Xq/LkfNf0+kwNzQiij3cQmBohlB2me2svawvjc\n97GBaFA1lk052ojrJHDtKCUrStGKUiRKgTA5EyHnh8n6EbJemMlsmIkxh4myQ96EmTBhjhCmSJgi\nEYpmuh6UkgkTt4qkybIsnKc1nKPFyZGxp8jYU6TJkiJL0kxS500S86eCr6vyvbEqJajblWJh2ZXn\nwolZwbxxplQfZ+bWI5rpXEREFgeFTREReXvseRqaajvQ0BaUzvcc/5hyIZjhN3skWOM0Oxj0rE4d\nxsoOEpkaIlKagnIe3Ako54LXlHPwesN63+5vQK9SZpk0ccap46CpJ0sDxlhYlsFidgEbHwsqQ40N\nNoY6a4pGa5AUWUJHv/FxFOLLyDduoNR4Jm7TmZiWs7BazyJa10As7BAN2YQX60RMbilYx3asD1rO\nDJYJEhGRRUthU0RE3vnCMUh3BOWtMAbcIrj5IIhOF7dQ2V+YVYqvv43Uze11rJYMxNPEcPCLHnbJ\nJVp0yZU8pkouuWKwzZc8pkoeuaJLrhxsp0oeuZJLtugxWSiTzZfxCpM4xVGi5XHS1lR1kqU0WRqt\nLE3WBGdM9bMu9ySN/TvmfKkHTDMv+G28ZtrZSxvddgdDoRWEognC0TjJWJi6qENdZRhxXSREXXTm\n3ta6aDBUOB5xiIUd4mGHWNgmHpmuB0HWeis92a93TrKDMLy3cs/w3pl7h0d7wMwK3OkOWP0e6Lwk\n+ENEevVb60kXEZGa0j2bIiIi7zCeb8gW3SCEFl0mC0F9suBSLPuUyiXCE73Uj++lIbuHxqn9NOX2\n01zsIWTc475nkfDcIcImTIm5jy0MYcsjhEcYlzAeIdygXtkfqdQdfEpWhJIVpWxFcO0orh3Ft6N4\nThTfieGHYphQDEJx6vwJ0rkeklM9hN3snLYZJ4JpXIPVvB6reX2wvuzgbuh+PAij01JtQfBcXQmf\nmbVvHD59HwpjMDUEU0cgV9kWs7P+CFE4zrZSyvkgHKfboXENZCqlcU0QhJ3wfJxuEZFFTRMEiYiI\nnA68Mozsh8Mvw5FXgjVc3dKcnlrfLeKXC5hKwS2CV8D2ihhsPCuMZzl4VgjPCuESokwIF4cyIUrG\noWwcysYi5JcImyJhv0jElIhQIlYpcat0TPMGTZr9/kr2mxXsNyvYZ1aw36zkoGnGw8GxLRIRh7pI\niGg4GA7cao1xnv8im9zfsrG8mza3t/p+Y6Fm9ic2MRzvJGmypLwxkt4YCXeURHmUaGkU27z50OQ5\nnGjQkx6KQ6hyY/DEQfCPCvGWc1QIXRvUG1bN3GcbqVdPrIgseQqbIiIickqUPZ982aNQcink8xQL\nUxTzOXImyoSJM1VymSoGQ4inKsOMp4qVMmtocdn1KXvTxVTrSW+Mzf6LXGhe5CL7Fc6y++Z8/oSJ\nM2JSDJNi2KQYMilGKvVh08AwSbImToEIhcqEUNN140QJh0JEQjYRxyYSsomGbOrCsMoeodM6xCpz\niBXeAK1uP83lfjLFA4T94nG/F74dxo2k8WKNeLEMfnXodSNWXRN2ohHHtnFMGce42L4LXikItl4p\n+OPB7LrxgpBrh4J7nS0HbDt4bDmz9lVKpD5Yb3d6wq/ZJVKnICwi80Kz0YqIiMgpEXaCHslULAyp\nOJBZkM8xxuD5hvzkEO5ID/lIhlwoTc4PB2G37GGXPJJlj1DZI1nyWFb2yJU8Sq4fFM8LhiV7/qx9\nPsVKvej6FF2PI1M+veV6Hi2tIV8++r5hQwtjrLYG6bQHaWWURiu4xzbtTdLoZknnJmm0+kkzhW29\nM/7GbywHP5rCxBqwoimIJrGNj2XcmZDrT9fLwazRs+sQhFwnEgwndsJghyuPK/vt8MxzEAxJNj5Q\n2VYfM/exHYJ4embppOrszMephyI1+f6JyIlR2BQREZFFx7IsQo5FKN0K6VaSp+hzjTEUXZ98ySNf\nrpRK/eggO+z6DLg+Jc9Qcn3K5TJWcZxQcZRwcYxQaQLXN5SMTdGEKPg2Bd+h5DvkjUPRsyn6Nnnf\nIe9b5MsWrlumVC4H72V8HMvHxieEh42Pg08IHwePhFUkxRQpK0eKHElylXpln5sjlZsixQB11n48\nbNzKkGkfJxhOTTCk2rdi+NZ0PYRjQdj3glJ2K/f5FnGYImRcHFxCxsU2LrZfrvSiVoo1XezKY7u6\nNBCWheW7WMVxrOkg+kYi9cE6v3XNkGiGuhaoa5pVb648X6mH4wv8L4QgjE9PRPZGW+MFPc/RFMSC\nwE80FXxN9jtkRulyIRhGPnEQxg/CxAGYPBR8TzNrK+WM4A8A6imX41DYFBERETlBlmURq8zO21jD\ndhhjKHk+hZJ/TOgtVIJv0fXm9NiWXJ+C6zPhHtWbW3l8vGHL0/WSe+zjQtmj6Pq4/vz31lr4pMiR\ntrJkrCzNzhTN9hRN9hSNVpaMnSXNJA1+lvTkBA0T3TT4zxPi+BNkTStZUTw7HPTsWg4GB2M7GKtS\nKkOSg22wLq6Nh208bDws4wd142HNLr4X9Aq7RSy/fNJf/UwAnbWN1kM4EQyBnt6+Xj0cr/QW+0H4\nnVP3jqobKIzD+IG5oXL8YDCp1omINgT3LjedMTeEZtYGIX8+gmg5D7nhWWUE8qNBT3goGvSmh6LB\nfdehyNytEwnq0VTQI65gfMoobIqIiIgsMpZlEQ05REMODdR2Vly3MvS4UPYouD7Fskeh7FNwg2HK\nBXf6HtyZoDo73JYqvb9lz6c867li5TUl16PsGfpcn32zQnLZC451PYPr+7iuT9SfosEfJ+WPkTYT\npJmgiXGarEky1gSNZCt9twanMsOyPas3OESx2lPsWD4WBs/Yld7e2dswHjGCCGrjmWBbJEyBKCUr\nQrkyQ7NnR/GcGL4TxXfimMoszSHHIW7y1JkpEiZHwkwR96eIe1PE/CwxN0e0OEF0dIComyVkjp10\nayEYy4HUCqymdbD2smB26Ia2ynYV1C8PQujI/pkyXNkO7Dr2DUOxmSAcjldK4qhtpR6KBZOWHR0q\nc0OVdZPnQSgGyeWQXAmpFZBcAamVc7fJFUE4NQaKk8G6zrPXeJ611nO1Xpw8/j3S8fTx75sOxSv3\nWVvH3nM9fT/27Mfx9KnpmZ9nmiBIRERERJYk3zd4xuB6hrI/K5hWe2/nPnb9yrayv+TO7dkteXN7\nimeeM9V7fIvl6eAdhO5iZVuohPBipUe45J3AMOFZQrgkKBKnSMIqkqBIggIJK9hXR4G4FeyLU8LH\nwsMOZpmu1P1K8eZsLaZMjAHTRL9p4ghpfGzCjoVjW4Rtm5Bj4dgz+0K2hW1bOFbweLrEKNLmH2KF\nGWClN8AKr58m7wgxSkRNkShFoqZAxC8S9guE/QI2x/8+uE6CUjRNOdqIG83gVibacmNN+PEMfjwD\n0QZClh/Mme2XcUyJkCkHQ7n9Eo5fxvZL2H4JyysGPbgTAzA5ABP9kB95/W94PFMZ9px//WOcKNQv\ng/rWYBh0cTL4jMJ4sOySN49/IPgf/xc2fWz+3m8eaYIgERERETnt2LaFjUXYgThOrZszx3QQ9nyD\n6xu8SsD1fEP56MeewTcGY8A3plKC4dT+rH3GBOv0esZUe4mPHkpdPOpxyfVxPJ+Eb1hdCdzVUD67\n7gXtnK770213g32+D54P+/1l+KYVz9+E7wchv+T6HH+0tSGMR5wiMUokrAJFE2GEJEUiMHUi30kb\niFbK6wvNCsWOZWFZELfKLLfHaGWEZYyyzBqhxQzTygjNxVEKdpyxSCPjTobJUIZsqJHJcBO5cBO5\nSBNuOEm4Mnu1Y1vYqeB97cow3bApkfAmiftZ4l6WmD9JzM0S9yaJUCJkQcgyOJZfCc0GxzKELL/S\n++5X7qD2iSdW13To/tulsCkiIiIicorNDsKnA7fSMzx7BuiZmZ+9agCeDrWeH/QYV8N4Jfx6swJw\n9XnPpzy99WZ6p2f3Zk8f6/mmGs493+D7TYyYtQwZw2+rfwAI/hgw/TrX8ynlp9+3MrzbywLZt/Ad\nqKuUt+cbZ6/io2/71bWjsCkiIiIiIgsq5NiEHJvEElmtZnr5pXIl1JZdH0NlBR+CXubpnmhDEF6B\naq+058+eiCsYsh2E25nh29XnPMOmtnRNv963S2FTRERERETkLaguv/QOHKL9TvIOWcRHRERERERE\nlhKFTREREREREZl3CpsiIiIiIiIy7xQ2RUREREREZN4pbIqIiIiIiMi8U9gUERERERGReaewKSIi\nIiIiIvNOYVNERERERETmncKmiIiIiIiIzDuFTREREREREZl3Cx429+zZw/bt29mwYQNbt27lxRdf\nPO5xd999N+vXr+eMM87g05/+NOVyeaGbJiIiIiIiIgtkwcPmLbfcws0338xrr73Gbbfdxg033HDM\nMV1dXXz5y19mx44d7N27l8HBQb71rW8tdNNERERERERkgSxo2Dx8+DA7d+7k2muvBeCaa66hr6+P\nvXv3zjnu3/7t3/jIRz7C8uXLsSyLz3zmM9x3330L2TQRERERERFZQAsaNvv6+lixYgWhUAgAy7Lo\n6Oigt7d3znG9vb2sXr26+rizs/OYY0RERERERGTxWHQTBN155520tbVVSzabrXWTRERERERE5CgL\nGjbb29sZGBjAdV0AjDH09vbS0dEx57iOjg56enqqj7u7u485Ztqf//mfc+DAgWqpr69fuC9ARERE\nRERE3pYFDZutra2cf/75fPe73wXg/vvvp62tjXXr1s057pprruGnP/0phw4dwhjDXXfdxcc//vGF\nbJqIiIiIiIgsoAUfRvvNb36Tb37zm2zYsIGvfe1r3HPPPQB86lOf4qc//SkAa9eu5W//9m+55JJL\nWLduHS0tLdxyyy0L3TQRERERERFZIJYxxtS6ESejra2NAwcO1LoZIiIiIiIip503ymOLboIgERER\nEREReedT2BQREREREZF5p7ApIiIiIiIi805hU0REREREROadwqaIiIiIiIjMO4VNERERERERmXcK\nmyIiIiIiIjLvFDZFRERERERk3ilsioiIiIiIyLyzjDGm1o04GdFolJaWllo347iy2Sz19fW1bobM\nE53PpUfndGnR+VxadD6XFp3PpUXnc2k52fN55MgRisXicZ9b9GHznaytrY0DBw7UuhkyT3Q+lx6d\n06VF53Np0flcWnQ+lxadz6VlIc+nhtGKiIiIiIjIvFPYFBERERERkXnnfOUrX/lKrRuxlF188cW1\nboLMI53PpUfndGnR+VxadD6XFp3PpUXnc2lZqPOpezZFRERERERk3mkYrYiIiIiIiMw7hU0RERER\nERGZdwqbC2TPnj1s376dDRs2sHXrVl588cVaN0negj/90z+ls7MTy7LYtWtXdb/O6+JTKBS4+uqr\n2bBhA5s3b+b3fu/32Lt3LwCHDx/myiuvZP369Zxzzjk89thjNW6tnIj3v//9bNq0iS1btnDppZfy\n3HPPAbo+F7t77rkHy7J44IEHAF2fi1VnZydnnnkmW7ZsYcuWLfzgBz8AdH0uVsVikc9//vOsX7+e\nc889l2uvvRbQ+VyshoeHq9fmli1b2LBhA6FQiJGRkYX7mWtkQVx++eXmnnvuMcYY86Mf/chceOGF\ntW2QvCW//vWvTV9fn1m9erV57rnnqvt1XheffD5vHnzwQeP7vjHGmH/6p38yl112mTHGmBtvvNH8\nzd/8jTHGmKefftqsWrXKlEqlGrVUTtTo6Gi1/uMf/9hs2rTJGKPrczHr6uoyF198sdm2bZv5yU9+\nYozR9blYHf17c5quz8Xp1ltvNZ///Oerv0MHBgaMMTqfS8XXv/51c9VVVxljFu5nrsLmAhgcHDTJ\nZNKUy2VjjDG+75tly5aZPXv21Lhl8lbN/qWp87o0PPPMM2b16tXGGGPq6uqqvziNMWbr1q3mkUce\nqVHL5O245557zObNm3V9LmKe55krrrjC7Ny501x22WXVsKnrc3E6XtjU9bk4ZbNZk0wmzfj4+Jz9\nOp9Lx1lnnbXgP3M1jHYB9PX1sWLFCkKhEACWZdHR0UFvb2+NWyYnQ+d1afjGN77BRz/6UYaHhymX\nyyxfvrz6XGdnp87nInHdddfR3t7Ol7/8Ze69915dn4vYnXfeySWXXMIFF1xQ3afrc3G77rrrOPfc\nc7nppps4cuSIrs9Fat++fWQyGW6//XYuvPBCLr30Un7xi1/ofC4RTzzxBKOjo1x11VUL+jNXYVNE\nThu33347e/fu5e/+7u9q3RQ5Sd/5znfo6+vjq1/9KrfddlutmyNv0+7du7n//vv5q7/6q1o3RebJ\nY489xgsvvMBvfvMbmpubuf7662vdJHmbXNelp6eHd73rXezcuZN//Md/5A/+4A9wXbfWTZN5cPfd\nd3PddddV/2iwUBQ2F0B7ezsDAwPVi9EYQ29vLx0dHTVumZwMndfF7Y477uDHP/4xDz30EIlEgqam\nJkKhEIcOHaoe093drfO5yFx//fX88pe/pK2tTdfnIrRjxw66u7tZv349nZ2dPPnkk9x888388Ic/\n1PW5SE2fo3A4zK233sqOHTv0+3OR6ujowLZt/viP/xiA8847jzVr1tDT06Pzuchls1l++MMf8slP\nfhJgQf9PpLC5AFpbWzn//PP57ne/C8D9999PW1sb69atq3HL5GTovC5ed955J/fddx+PPPII6XS6\nuv9jH/sYd911FwDPPPMMBw8e5LLLLqtVM+UEjI2N0d/fX338wAMP0NTUpOtzkfrsZz/LwMAA3d3d\ndHd3s23bNr71rW/x2c9+VtfnIjQ1NcXY2Fj18X333cd5552n63ORam5u5oorruDhhx8GoKuri66u\nPDMQzQAABDhJREFULi655BKdz0XuBz/4AZs3b+ass86q7luon7mWMcac9LvIMV599VVuuOEGhoeH\nSaVS3HPPPZx77rm1bpacoFtuuYUHH3yQQ4cO0dTURDKZZO/evTqvi9CBAwdob29n7dq1JJNJAKLR\nKE899RSDg4N84hOfoKuri0gkwj//8z9z+eWX17jF8kZ6enr42Mc+Rj6fx7ZtWlpauOOOO9iyZYuu\nzyXgd3/3d7n11lu5+uqrdX0uQvv37+eaa67B8zyMMaxdu5ZvfOMbdHZ26vpcpPbv389NN93E0NAQ\ntm3z13/911xzzTU6n4vc9u3b+fSnP82NN95Y3bdQP3MVNkVERERERGTeaRitiIiIiIiIzDuFTRER\nEREREZl3CpsiIiIiIiIy7xQ2RUREREREZN4pbIqIiIiIiMi8U9gUERFZBH71q1+xZcuWWjdDRETk\nhClsioiIiIiIyLxT2BQRETlJzzzzDO9973u58MILOe+88/jRj35Ed3c36XSaL37xi2zatImNGzfy\n85//vPqae++9l02bNrFp0yY+9KEPcfDgwepzf//3f8+5557L5s2b2bZtG7lcDgDXdfnc5z7H5s2b\n2bhxIzt37jzlX6uIiMiJUtgUERE5CWNjY9x8881873vfY+fOnTzyyCN84Qtf4ODBg4yPj3P22Wfz\nwgsvcPfdd/NHf/RHTE5Osnv3bv7iL/6Chx56iBdeeIHt27fzqU99CoBvf/vb3H///Tz++OM8//zz\nPPTQQ0SjUQBeeeUVrr/+ep5//nn+5E/+hC996Uu1/NJFRETekMKmiIjISXjiiSfYv38/H/zgB9my\nZQvve9/7AHj11VcJhULccMMNAGzbto2VK1fy3HPP8ctf/pIrr7ySVatWAfC5z32ORx99FM/z+I//\n+A8+85nP0NDQAEBjYyOO4wCwbt06LrroIgAuvvhi9u3bd4q/WhERkRMXqnUDREREFjNjDBs3buSJ\nJ56Ys7+7u/u4x1uWdUL7jicWi1XrjuPguu6JN1REROQUU8+miIjISdi+fTtdXV1z7sfctWsXpVIJ\n13W59957AXj66afp7+9ny5YtXH755fzsZz+jv78fgLvuuosrrrgCx3H4yEc+wl133cX4+DgQDNP1\nPO/Uf2EiIiInST2bIiIiJ6GxsZEHH3yQL37xi3zhC1+gXC7T0dHBP/zDP9DQ0MDu3bvZvHkzruvy\n/e9/n2QyyTnnnMPXv/51rrzySgDa29v5l3/5FwA+8YlP0N/fz/bt2wmFQtTV1c0JsiIiIouFZYwx\ntW6EiIjIUtPd3c2WLVsYGxurdVNERERqQsNoRUREREREZN6pZ1NERERERETmnXo2RUREREREZN4p\nbIqIiIiIiMi8U9gUERERERGReaewKSIiIiIiIvNOYVNERERERETmncKmiIiIiIiIzDuFTRERERER\nEZl3/x9N3JyfFg9NfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1120x880 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# We can visualize the loss over epochs to get an overview-\n",
    "fig=plt.figure(figsize=(14, 11), dpi= 80, facecolor='w', edgecolor='k')\n",
    "\n",
    "plt.plot(history.history['loss'], label = 'training loss')\n",
    "plt.plot(history.history['val_loss'], label = 'testing loss')\n",
    "plt.plot(history.history['accuracy'], label = 'training accuracy')\n",
    "plt.plot(history.history['val_accuracy'], label = 'testing accuracy')\n",
    "\n",
    "plt.title('LeNet-5 CNN loss')\n",
    "plt.ylabel('loss/accuracy')\n",
    "plt.xlabel('epoch')\n",
    "# plt.legend(['train', 'test'], loc='upper right')\n",
    "plt.legend(loc = 'upper right')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l9PqxbVyNMHY"
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow-model-optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7pJOpPX8NOXN"
   },
   "outputs": [],
   "source": [
    "import tensorflow_model_optimization as tfmot\n",
    "from tensorflow_model_optimization.sparsity import keras as sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "E49uzA0NOF0K",
    "outputId": "96e82c53-1ae9-4ea4-a4f3-7d1b0d3a500c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Original CNN trained for 69 epochs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "epochs = len(history.history['accuracy'])\n",
    "\n",
    "print(\"\\nOriginal CNN trained for {0} epochs\\n\".format(epochs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "-1pqmDaiN54U",
    "outputId": "d9a2f53b-094e-41f4-e0db-8d2f79632c6f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32361.0"
      ]
     },
     "execution_count": 46,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# end_step = np.ceil(1.0 * X_train / batch_size).astype(np.int32) * epochs\n",
    "end_step = np.ceil(X_train.shape[0] / 128) * epochs\n",
    "end_step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nRHUiAu6NTnF"
   },
   "outputs": [],
   "source": [
    "pruning_schedule = tfmot.sparsity.keras.PolynomialDecay(\n",
    "  initial_sparsity = 0.0, final_sparsity = 0.9,\n",
    "  begin_step = 2000, end_step = end_step,\n",
    "  frequency = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "colab_type": "code",
    "id": "q6LX7CNdOpk1",
    "outputId": "6fe67af8-e1ef-47df-9265-bb4526a4ec8a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_model_optimization/python/core/sparsity/keras/pruning_wrapper.py:183: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n"
     ]
    }
   ],
   "source": [
    "model_pruning = tfmot.sparsity.keras.prune_low_magnitude(model, pruning_schedule= pruning_schedule)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Nqmdt7fEPezu",
    "outputId": "6d753cde-e7b3-4f81-e25d-71a04ee1aaf8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\npruned_model.compile(\\n    loss=tf.keras.losses.categorical_crossentropy,\\n    optimizer='adam',\\n    metrics=['accuracy'])\\n\""
      ]
     },
     "execution_count": 49,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compile the defined pruned model-\n",
    "model_pruning.compile(optimizer=optimizer_SGD,\n",
    "                      loss=tf.keras.losses.categorical_crossentropy,\n",
    "                      metrics = ['accuracy'])\n",
    "\n",
    "\"\"\"\n",
    "pruned_model.compile(\n",
    "    loss=tf.keras.losses.categorical_crossentropy,\n",
    "    optimizer='adam',\n",
    "    metrics=['accuracy'])\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_Cu65UrRQYVQ"
   },
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "    sparsity.UpdatePruningStep()\n",
    "    # sparsity.PruningSummaries(log_dir=logdir, profile_batch=0)\n",
    "]\n",
    "\n",
    "# callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "22gulLyMO9Vw",
    "outputId": "26611661-44cf-44aa-b26e-514db40ab2da"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.0572 - accuracy: 0.9828 - val_loss: 0.0652 - val_accuracy: 0.9797\n",
      "Epoch 2/100\n",
      "60000/60000 [==============================] - 7s 110us/sample - loss: 0.0566 - accuracy: 0.9833 - val_loss: 0.0647 - val_accuracy: 0.9801\n",
      "Epoch 3/100\n",
      "60000/60000 [==============================] - 7s 114us/sample - loss: 0.0563 - accuracy: 0.9833 - val_loss: 0.0666 - val_accuracy: 0.9797\n",
      "Epoch 4/100\n",
      "60000/60000 [==============================] - 7s 114us/sample - loss: 0.0563 - accuracy: 0.9835 - val_loss: 0.0647 - val_accuracy: 0.9801\n",
      "Epoch 5/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0558 - accuracy: 0.9833 - val_loss: 0.0663 - val_accuracy: 0.9794\n",
      "Epoch 6/100\n",
      "60000/60000 [==============================] - 6s 108us/sample - loss: 0.0558 - accuracy: 0.9834 - val_loss: 0.0654 - val_accuracy: 0.9796\n",
      "Epoch 7/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0556 - accuracy: 0.9836 - val_loss: 0.0650 - val_accuracy: 0.9798\n",
      "Epoch 8/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0553 - accuracy: 0.9836 - val_loss: 0.0661 - val_accuracy: 0.9789\n",
      "Epoch 9/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0552 - accuracy: 0.9835 - val_loss: 0.0639 - val_accuracy: 0.9802\n",
      "Epoch 10/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0551 - accuracy: 0.9836 - val_loss: 0.0654 - val_accuracy: 0.9794\n",
      "Epoch 11/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0549 - accuracy: 0.9838 - val_loss: 0.0647 - val_accuracy: 0.9797\n",
      "Epoch 12/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0548 - accuracy: 0.9837 - val_loss: 0.0640 - val_accuracy: 0.9804\n",
      "Epoch 13/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0546 - accuracy: 0.9836 - val_loss: 0.0645 - val_accuracy: 0.9800\n",
      "Epoch 14/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0544 - accuracy: 0.9842 - val_loss: 0.0644 - val_accuracy: 0.9798\n",
      "Epoch 15/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0542 - accuracy: 0.9839 - val_loss: 0.0641 - val_accuracy: 0.9798\n",
      "Epoch 16/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0540 - accuracy: 0.9839 - val_loss: 0.0656 - val_accuracy: 0.9795\n",
      "Epoch 17/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0541 - accuracy: 0.9836 - val_loss: 0.0646 - val_accuracy: 0.9804\n",
      "Epoch 18/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0539 - accuracy: 0.9839 - val_loss: 0.0624 - val_accuracy: 0.9810\n",
      "Epoch 19/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0537 - accuracy: 0.9840 - val_loss: 0.0628 - val_accuracy: 0.9805\n",
      "Epoch 20/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0535 - accuracy: 0.9837 - val_loss: 0.0631 - val_accuracy: 0.9794\n",
      "Epoch 21/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0533 - accuracy: 0.9844 - val_loss: 0.0635 - val_accuracy: 0.9799\n",
      "Epoch 22/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0531 - accuracy: 0.9843 - val_loss: 0.0635 - val_accuracy: 0.9801\n",
      "Epoch 23/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0530 - accuracy: 0.9843 - val_loss: 0.0631 - val_accuracy: 0.9806\n",
      "Epoch 24/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0530 - accuracy: 0.9843 - val_loss: 0.0642 - val_accuracy: 0.9799\n",
      "Epoch 25/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0526 - accuracy: 0.9845 - val_loss: 0.0640 - val_accuracy: 0.9797\n",
      "Epoch 26/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0527 - accuracy: 0.9844 - val_loss: 0.0631 - val_accuracy: 0.9803\n",
      "Epoch 27/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0523 - accuracy: 0.9842 - val_loss: 0.0639 - val_accuracy: 0.9801\n",
      "Epoch 28/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0523 - accuracy: 0.9844 - val_loss: 0.0659 - val_accuracy: 0.9796\n",
      "Epoch 29/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0521 - accuracy: 0.9844 - val_loss: 0.0644 - val_accuracy: 0.9795\n",
      "Epoch 30/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0520 - accuracy: 0.9843 - val_loss: 0.0644 - val_accuracy: 0.9793\n",
      "Epoch 31/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0519 - accuracy: 0.9843 - val_loss: 0.0624 - val_accuracy: 0.9800\n",
      "Epoch 32/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0516 - accuracy: 0.9847 - val_loss: 0.0621 - val_accuracy: 0.9813\n",
      "Epoch 33/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0515 - accuracy: 0.9847 - val_loss: 0.0636 - val_accuracy: 0.9801\n",
      "Epoch 34/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0515 - accuracy: 0.9848 - val_loss: 0.0617 - val_accuracy: 0.9807\n",
      "Epoch 35/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0513 - accuracy: 0.9850 - val_loss: 0.0635 - val_accuracy: 0.9798\n",
      "Epoch 36/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0510 - accuracy: 0.9851 - val_loss: 0.0641 - val_accuracy: 0.9802\n",
      "Epoch 37/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0510 - accuracy: 0.9845 - val_loss: 0.0619 - val_accuracy: 0.9805\n",
      "Epoch 38/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0509 - accuracy: 0.9849 - val_loss: 0.0621 - val_accuracy: 0.9805\n",
      "Epoch 39/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0506 - accuracy: 0.9849 - val_loss: 0.0620 - val_accuracy: 0.9803\n",
      "Epoch 40/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0506 - accuracy: 0.9851 - val_loss: 0.0627 - val_accuracy: 0.9795\n",
      "Epoch 41/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0506 - accuracy: 0.9850 - val_loss: 0.0613 - val_accuracy: 0.9811\n",
      "Epoch 42/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0503 - accuracy: 0.9848 - val_loss: 0.0615 - val_accuracy: 0.9807\n",
      "Epoch 43/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0501 - accuracy: 0.9850 - val_loss: 0.0631 - val_accuracy: 0.9802\n",
      "Epoch 44/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0498 - accuracy: 0.9853 - val_loss: 0.0627 - val_accuracy: 0.9806\n",
      "Epoch 45/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0498 - accuracy: 0.9851 - val_loss: 0.0620 - val_accuracy: 0.9804\n",
      "Epoch 46/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0497 - accuracy: 0.9852 - val_loss: 0.0622 - val_accuracy: 0.9806\n",
      "Epoch 47/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0497 - accuracy: 0.9852 - val_loss: 0.0615 - val_accuracy: 0.9809\n",
      "Epoch 48/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0494 - accuracy: 0.9851 - val_loss: 0.0609 - val_accuracy: 0.9808\n",
      "Epoch 49/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0495 - accuracy: 0.9856 - val_loss: 0.0619 - val_accuracy: 0.9804\n",
      "Epoch 50/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0492 - accuracy: 0.9857 - val_loss: 0.0609 - val_accuracy: 0.9811\n",
      "Epoch 51/100\n",
      "60000/60000 [==============================] - 7s 110us/sample - loss: 0.0491 - accuracy: 0.9852 - val_loss: 0.0616 - val_accuracy: 0.9812\n",
      "Epoch 52/100\n",
      "60000/60000 [==============================] - 7s 115us/sample - loss: 0.0489 - accuracy: 0.9853 - val_loss: 0.0607 - val_accuracy: 0.9809\n",
      "Epoch 53/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0488 - accuracy: 0.9856 - val_loss: 0.0608 - val_accuracy: 0.9809\n",
      "Epoch 54/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0487 - accuracy: 0.9857 - val_loss: 0.0618 - val_accuracy: 0.9809\n",
      "Epoch 55/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0487 - accuracy: 0.9856 - val_loss: 0.0602 - val_accuracy: 0.9815\n",
      "Epoch 56/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0485 - accuracy: 0.9854 - val_loss: 0.0618 - val_accuracy: 0.9804\n",
      "Epoch 57/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0485 - accuracy: 0.9855 - val_loss: 0.0618 - val_accuracy: 0.9817\n",
      "Epoch 58/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0483 - accuracy: 0.9855 - val_loss: 0.0602 - val_accuracy: 0.9807\n",
      "Epoch 59/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0480 - accuracy: 0.9858 - val_loss: 0.0608 - val_accuracy: 0.9813\n",
      "Epoch 60/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0479 - accuracy: 0.9857 - val_loss: 0.0607 - val_accuracy: 0.9809\n",
      "Epoch 61/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0479 - accuracy: 0.9858 - val_loss: 0.0606 - val_accuracy: 0.9809\n",
      "Epoch 62/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0476 - accuracy: 0.9861 - val_loss: 0.0612 - val_accuracy: 0.9812\n",
      "Epoch 63/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0475 - accuracy: 0.9862 - val_loss: 0.0611 - val_accuracy: 0.9806\n",
      "Epoch 64/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0474 - accuracy: 0.9862 - val_loss: 0.0613 - val_accuracy: 0.9805\n",
      "Epoch 65/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0474 - accuracy: 0.9858 - val_loss: 0.0605 - val_accuracy: 0.9811\n",
      "Epoch 66/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0473 - accuracy: 0.9861 - val_loss: 0.0603 - val_accuracy: 0.9807\n",
      "Epoch 67/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0471 - accuracy: 0.9865 - val_loss: 0.0623 - val_accuracy: 0.9802\n",
      "Epoch 68/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0471 - accuracy: 0.9860 - val_loss: 0.0608 - val_accuracy: 0.9810\n",
      "Epoch 69/100\n",
      "60000/60000 [==============================] - 6s 104us/sample - loss: 0.0468 - accuracy: 0.9865 - val_loss: 0.0605 - val_accuracy: 0.9806\n",
      "Epoch 70/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0465 - accuracy: 0.9865 - val_loss: 0.0610 - val_accuracy: 0.9809\n",
      "Epoch 71/100\n",
      "60000/60000 [==============================] - 6s 104us/sample - loss: 0.0466 - accuracy: 0.9864 - val_loss: 0.0603 - val_accuracy: 0.9807\n",
      "Epoch 72/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0464 - accuracy: 0.9864 - val_loss: 0.0604 - val_accuracy: 0.9817\n",
      "Epoch 73/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0465 - accuracy: 0.9863 - val_loss: 0.0592 - val_accuracy: 0.9818\n",
      "Epoch 74/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0463 - accuracy: 0.9862 - val_loss: 0.0599 - val_accuracy: 0.9817\n",
      "Epoch 75/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0461 - accuracy: 0.9862 - val_loss: 0.0600 - val_accuracy: 0.9815\n",
      "Epoch 76/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0461 - accuracy: 0.9861 - val_loss: 0.0593 - val_accuracy: 0.9811\n",
      "Epoch 77/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0459 - accuracy: 0.9863 - val_loss: 0.0597 - val_accuracy: 0.9814\n",
      "Epoch 78/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0458 - accuracy: 0.9865 - val_loss: 0.0600 - val_accuracy: 0.9809\n",
      "Epoch 79/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0456 - accuracy: 0.9867 - val_loss: 0.0609 - val_accuracy: 0.9806\n",
      "Epoch 80/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0455 - accuracy: 0.9863 - val_loss: 0.0605 - val_accuracy: 0.9812\n",
      "Epoch 81/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0455 - accuracy: 0.9864 - val_loss: 0.0603 - val_accuracy: 0.9807\n",
      "Epoch 82/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0452 - accuracy: 0.9865 - val_loss: 0.0597 - val_accuracy: 0.9811\n",
      "Epoch 83/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0453 - accuracy: 0.9865 - val_loss: 0.0609 - val_accuracy: 0.9808\n",
      "Epoch 84/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0450 - accuracy: 0.9867 - val_loss: 0.0602 - val_accuracy: 0.9808\n",
      "Epoch 85/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0451 - accuracy: 0.9867 - val_loss: 0.0584 - val_accuracy: 0.9809\n",
      "Epoch 86/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0450 - accuracy: 0.9868 - val_loss: 0.0587 - val_accuracy: 0.9810\n",
      "Epoch 87/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0448 - accuracy: 0.9869 - val_loss: 0.0583 - val_accuracy: 0.9819\n",
      "Epoch 88/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0446 - accuracy: 0.9867 - val_loss: 0.0603 - val_accuracy: 0.9809\n",
      "Epoch 89/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0445 - accuracy: 0.9868 - val_loss: 0.0586 - val_accuracy: 0.9814\n",
      "Epoch 90/100\n",
      "60000/60000 [==============================] - 6s 105us/sample - loss: 0.0444 - accuracy: 0.9867 - val_loss: 0.0584 - val_accuracy: 0.9816\n",
      "Epoch 91/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0444 - accuracy: 0.9869 - val_loss: 0.0592 - val_accuracy: 0.9820\n",
      "Epoch 92/100\n",
      "60000/60000 [==============================] - 6s 107us/sample - loss: 0.0441 - accuracy: 0.9869 - val_loss: 0.0585 - val_accuracy: 0.9816\n",
      "Epoch 93/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0441 - accuracy: 0.9870 - val_loss: 0.0586 - val_accuracy: 0.9820\n",
      "Epoch 94/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0441 - accuracy: 0.9870 - val_loss: 0.0593 - val_accuracy: 0.9817\n",
      "Epoch 95/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0439 - accuracy: 0.9869 - val_loss: 0.0591 - val_accuracy: 0.9822\n",
      "Epoch 96/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0436 - accuracy: 0.9872 - val_loss: 0.0581 - val_accuracy: 0.9817\n",
      "Epoch 97/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0437 - accuracy: 0.9874 - val_loss: 0.0582 - val_accuracy: 0.9824\n",
      "Epoch 98/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0434 - accuracy: 0.9870 - val_loss: 0.0581 - val_accuracy: 0.9819\n",
      "Epoch 99/100\n",
      "60000/60000 [==============================] - 6s 106us/sample - loss: 0.0436 - accuracy: 0.9869 - val_loss: 0.0578 - val_accuracy: 0.9817\n",
      "Epoch 100/100\n",
      "60000/60000 [==============================] - 7s 115us/sample - loss: 0.0435 - accuracy: 0.9872 - val_loss: 0.0579 - val_accuracy: 0.9830\n"
     ]
    }
   ],
   "source": [
    "# Train defined model-\n",
    "history_pruned = model_pruning.fit(\n",
    "    x = X_train, y = y_train_ohe,\n",
    "    batch_size = batch_size,\n",
    "    epochs = 100, callbacks = callbacks,\n",
    "    validation_data = [X_test, y_test_ohe]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bqAriBqyPV-7"
   },
   "outputs": [],
   "source": [
    "score = model_pruning.evaluate(X_test, y_test_ohe, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "EnO540UpWhA5",
    "outputId": "d4a561b3-2e1b-4788-c6ce-f35d65cc1003"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "LeNet-5 CNN MNIST pruned model metrics on test dataset are:\n",
      "loss = 0.0579 & accuracy = 0.9830\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nLeNet-5 CNN MNIST pruned model metrics on test dataset are:\")\n",
    "print(\"loss = {0:.4f} & accuracy = {1:.4f}\\n\".format(score[0], score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 527
    },
    "colab_type": "code",
    "id": "NOp5N59PvGfL",
    "outputId": "37d3526d-bf9c-426b-ddb0-323d04bd2a52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Pruned LenNet-5 CNN on MNIST dataset is:\n",
      "\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "prune_low_magnitude_conv2d ( (None, 24, 24, 6)         308       \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_average_ (None, 12, 12, 6)         1         \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_conv2d_1 (None, 8, 8, 16)          4818      \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_average_ (None, 4, 4, 16)          1         \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_flatten  (None, 256)               1         \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_dense (P (None, 120)               61562     \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_flatten_ (None, 120)               1         \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_dense_1  (None, 84)                20246     \n",
      "_________________________________________________________________\n",
      "prune_low_magnitude_dense_2  (None, 10)                1692      \n",
      "=================================================================\n",
      "Total params: 88,630\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 44,204\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nPruned LenNet-5 CNN on MNIST dataset is:\\n\")\n",
    "print(model_pruning.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 527
    },
    "colab_type": "code",
    "id": "y4jiBVx6xs4b",
    "outputId": "f3d54b38-8658-4733-ad67-5b5aa54fadda"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Original LenNet-5 CNN on MNIST dataset is:\n",
      "\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 24, 24, 6)         156       \n",
      "_________________________________________________________________\n",
      "average_pooling2d (AveragePo (None, 12, 12, 6)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 8, 8, 16)          2416      \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 4, 4, 16)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 120)               30840     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 120)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 84)                10164     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                850       \n",
      "=================================================================\n",
      "Total params: 44,426\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nOriginal LenNet-5 CNN on MNIST dataset is:\\n\")\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "_MGGgGxmzTYZ",
    "outputId": "c6c40f03-5a50-4358-e943-8585e2917198"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "F7Af09hjzpsq"
   },
   "source": [
    "https://medium.com/@ml_kid/how-to-save-our-model-to-google-drive-and-reuse-it-2c1028058cb2\n",
    "\n",
    "https://medium.com/@rajath95/how-to-save-and-upload-deep-learning-machine-learning-models-in-google-colab-using-google-drive-eb2901937ae\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eSEMhZ0EzUI9"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rt2L9cI5wRq3"
   },
   "outputs": [],
   "source": [
    "# Save the entire model to a HDF5 file.\n",
    "# The '.h5' extension indicates that the model shuold be saved to HDF5.\n",
    "model.save('LeNet-5_CNN_Original_model_MNIST.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wYZl2sW1yaYR"
   },
   "outputs": [],
   "source": [
    "model.save_weights(\"LeNet-5_CNN_Original_model_MNIST_weights.h5\", overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uzLTJAbjwXwS"
   },
   "outputs": [],
   "source": [
    "model_pruning.save('LeNet-5_CNN_Pruned_model_MNIST.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ymi9wEMQwlUP"
   },
   "outputs": [],
   "source": [
    "# Recreate the model from saved file-\n",
    "\n",
    "# Recreate the exact same model, including its weights and the optimizer\n",
    "orig_model = tf.keras.models.load_model('LeNet-5_CNN_Original_model_MNIST.h5')\n",
    "\n",
    "# Show the model architecture\n",
    "orig_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yNK6hdtnwzMd"
   },
   "outputs": [],
   "source": [
    "# Check accuracy and loss of restored/loaded model-\n",
    "loss, acc = orig_model.evaluate(X_test,  y_test_ohe, verbose=2)\n",
    "print('Restored model, accuracy: {:5.2f}%'.format(100*acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8VQ-SE7RxGDl"
   },
   "source": [
    "This technique saves everything:\n",
    "\n",
    "    The weight values\n",
    "    The model's configuration(architecture)\n",
    "    The optimizer configuration\n",
    "\n",
    "Keras saves models by inspecting the architecture. Currently, it is not able to save TensorFlow optimizers (from tf.train). When using those you will need to re-compile the model after loading, and you will lose the state of the optimizer.\n",
    "\n",
    "https://www.tensorflow.org/tutorials/keras/save_and_load#save_the_entire_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "colab_type": "code",
    "id": "Tj3JcOkOWjsu",
    "outputId": "329eba11-30f9-439b-b1a6-98899525c507"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer 1 has shape = (5, 5, 1, 6)\n",
      "layer 2 has shape = (6,)\n",
      "layer 3 has shape = (5, 5, 6, 16)\n",
      "layer 4 has shape = (16,)\n",
      "layer 5 has shape = (256, 120)\n",
      "layer 6 has shape = (120,)\n",
      "layer 7 has shape = (120, 84)\n",
      "layer 8 has shape = (84,)\n",
      "layer 9 has shape = (84, 10)\n",
      "layer 10 has shape = (10,)\n"
     ]
    }
   ],
   "source": [
    "for n, layer in enumerate(model.trainable_variables):\n",
    "  print(\"layer {0} has shape = {1}\".format(n + 1, layer.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "colab_type": "code",
    "id": "Fnw2psyaX8ad",
    "outputId": "27db9d31-8dc7-48ae-eb97-c04b62025b02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer 1 has shape = (5, 5, 1, 6)\n",
      "layer 2 has shape = (6,)\n",
      "layer 3 has shape = (5, 5, 6, 16)\n",
      "layer 4 has shape = (16,)\n",
      "layer 5 has shape = (256, 120)\n",
      "layer 6 has shape = (120,)\n",
      "layer 7 has shape = (120, 84)\n",
      "layer 8 has shape = (84,)\n",
      "layer 9 has shape = (84, 10)\n",
      "layer 10 has shape = (10,)\n"
     ]
    }
   ],
   "source": [
    "for n, layer in enumerate(model_pruning.trainable_variables):\n",
    "  print(\"layer {0} has shape = {1}\".format(n + 1, layer.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "O-i5wYbGYHeU",
    "outputId": "fe26af95-e6ad-4d97-d9ce-e607c32315dd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.02855234  0.05989161 -0.0607013  -0.00576936 -0.00045072  0.1610433 ]\n"
     ]
    }
   ],
   "source": [
    "print(model.trainable_variables[1].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "DpgjVlK4YSkQ",
    "outputId": "e40bf06d-aebc-4073-821e-9f840eb0535e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.02855234  0.05989161 -0.0607013  -0.00576936 -0.00045072  0.1610433 ]\n"
     ]
    }
   ],
   "source": [
    "print(model_pruning.trainable_variables[1].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "L95VwY35YVLV",
    "outputId": "077c5342-308d-4c05-8042-0c6d6731aaf4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.06838122  0.11613953 -0.0261348  -0.07213172  0.01736846  0.00848707\n",
      "  0.0143916   0.0469756  -0.04419191  0.00747634]\n"
     ]
    }
   ],
   "source": [
    "print(model.trainable_variables[9].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "Mn_iyid5Yahx",
    "outputId": "d25edf6b-703a-4e8c-fc60-5bd693790f59"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.06838122  0.11613953 -0.0261348  -0.07213172  0.01736846  0.00848707\n",
      "  0.0143916   0.0469756  -0.04419191  0.00747634]\n"
     ]
    }
   ],
   "source": [
    "print(model_pruning.trainable_variables[9].numpy())"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Model_Optimization_LeNet_5_MNIST_TensorFlow_Example.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}