VGG18_Dropout_PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jZOD2mGIeyEJ"
   },
   "source": [
    "# VGG-18 CNN: PyTorch & CIFAR-10\n",
    "\n",
    "End-to-end programming tutorial including:\n",
    "\n",
    "1. Progress bar - training model\n",
    "1. Train model with _early stopping criterion_\n",
    "1. Learning rate scheduler\n",
    "1. Compare between learning rate scheduler and early stopping criterion\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "7hpnDY7Mfb8b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=2\n"
     ]
    }
   ],
   "source": [
    "# Specify GPU to be used-\n",
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "cM6Ih3nVq0Gh"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "\n",
    "from tqdm import tqdm\n",
    "from tqdm import trange\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3DvYMsqNezdm"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ha-v5IiWezqk"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_H9KbTonq0Q9",
    "outputId": "41b72a44-64e1-44c2-a917-615e9adafb76"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "currently available device: cuda\n"
     ]
    }
   ],
   "source": [
    "# Device configuration-\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(f\"currently available device: {device}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "f2CJ2XyNq9Nd",
    "outputId": "3e5961db-ccb3-4be4-9207-21d280f44beb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version: 1.8.0\n"
     ]
    }
   ],
   "source": [
    "print(f\"PyTorch version: {torch.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k6rTKbcZrBN0"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "uV1m3vD0rBee"
   },
   "outputs": [],
   "source": [
    "# Hyper-parameters-\n",
    "num_epochs = 65\n",
    "batch_size = 128\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6QGc3oYHrDzL"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "9R6Sc5v6rD8M"
   },
   "outputs": [],
   "source": [
    "# Define transformations for training and test sets-\n",
    "transform_train = transforms.Compose(\n",
    "    [\n",
    "      transforms.RandomCrop(32, padding = 4),\n",
    "      transforms.RandomHorizontalFlip(),\n",
    "      transforms.ToTensor(),\n",
    "      transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "     ]\n",
    "     )\n",
    "\n",
    "transform_test = transforms.Compose(\n",
    "    [\n",
    "      transforms.ToTensor(),\n",
    "      transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "     ]\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "id": "4ddOxdkirHPN",
    "outputId": "764e40cc-2f3c-482f-bbb1-542c39d0d1f9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# Load dataset-\n",
    "train_dataset = torchvision.datasets.CIFAR10(\n",
    "        root = './data', train = True,\n",
    "        download = True, transform = transform_train\n",
    "        )\n",
    "\n",
    "test_dataset = torchvision.datasets.CIFAR10(\n",
    "        root = './data', train = False,\n",
    "        download = True, transform = transform_test\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FW8h3ztJrU5_",
    "outputId": "643be603-0775-412b-9a3d-7dfbe5df1887"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(train_dataset) = 50000 & len(test_dataset) = 10000\n"
     ]
    }
   ],
   "source": [
    "print(f\"len(train_dataset) = {len(train_dataset)} & len(test_dataset) = {len(test_dataset)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "cydrwaW7rbnV"
   },
   "outputs": [],
   "source": [
    "# Create training and testing loaders-\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "        train_dataset, batch_size = batch_size,\n",
    "        shuffle = True\n",
    "        )\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "        test_dataset, batch_size = batch_size,\n",
    "        shuffle = False\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "62AgWl9jrfzW",
    "outputId": "0972fead-2556-4cea-9ff4-8f50eabf6cef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(train_loader) = 391 & len(test_loader) = 79\n"
     ]
    }
   ],
   "source": [
    "print(f\"len(train_loader) = {len(train_loader)} & len(test_loader) = {len(test_loader)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rjGADCMfrhAl",
    "outputId": "e3bb3177-720d-48ce-ef5a-b0e61c5ac9bb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(390.625, 78.125)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check-\n",
    "len(train_dataset) / batch_size, len(test_dataset) / batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XEBunYxErhh8"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jhwdHTu0rsf8",
    "outputId": "c84eaa78-265f-4af7-dbdf-f45370c2bb4c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([128, 3, 32, 32]), torch.Size([128]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check-\n",
    "len(train_dataset) / batch_size, len(test_dataset) / batch_size\n",
    "\n",
    "# Sanity check-\n",
    "images, labels = next(iter(train_loader))\n",
    "\n",
    "images.size(), labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "hoRuYT9srtH1"
   },
   "outputs": [],
   "source": [
    "del images, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DOgym6Ffrtds"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ln5CjwJhrtqc"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zdWaY-5NruCM"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zlFoyg5Hruqn"
   },
   "source": [
    "### Define _VGG_ architecture:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "xqxULQxKrxSM"
   },
   "outputs": [],
   "source": [
    "class VGGBlock(nn.Module):\n",
    "    '''\n",
    "    VGG block within a VGG-* CNN model\n",
    "    '''\n",
    "    def __init__(self, input_channels, num_channels, padding_size = 1,\n",
    "                 stride_size = 1, use_extra_conv = False, dropout = 0.2):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(\n",
    "            in_channels = input_channels, out_channels = num_channels,\n",
    "            kernel_size = 3, padding = padding_size,\n",
    "            stride = stride_size, bias = False\n",
    "        )\n",
    "        \n",
    "        self.dropout = nn.Dropout(p = dropout)\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(\n",
    "            in_channels = num_channels, out_channels = num_channels,\n",
    "            kernel_size = 3, padding = padding_size,\n",
    "            stride = stride_size, bias = False\n",
    "        )\n",
    "        \n",
    "        if use_extra_conv:\n",
    "            self.conv3 = nn.Conv2d(\n",
    "                in_channels = num_channels, out_channels = num_channels,\n",
    "                kernel_size = 3, padding = padding_size,\n",
    "                stride = stride_size, bias = False\n",
    "            )\n",
    "            \n",
    "            self.conv4 = nn.Conv2d(\n",
    "                in_channels = num_channels, out_channels = num_channels,\n",
    "                kernel_size = 3, padding = padding_size,\n",
    "                stride = stride_size, bias = False\n",
    "            )\n",
    "        else:\n",
    "            self.conv3 = False\n",
    "            self.conv4 = False\n",
    "        \n",
    "        self.pool = nn.MaxPool2d(\n",
    "            kernel_size = 2, stride = 2)\n",
    "            # padding = 1)\n",
    "        \n",
    "        self.bn1 = nn.BatchNorm2d(num_features = num_channels)\n",
    "        self.bn2 = nn.BatchNorm2d(num_features = num_channels)\n",
    "\n",
    "        if use_extra_conv:\n",
    "            self.bn3 = nn.BatchNorm2d(num_features = num_channels)\n",
    "            self.bn4 = nn.BatchNorm2d(num_features = num_channels)\n",
    "        \n",
    "        self.initialize_weights()\n",
    "    \n",
    "    \n",
    "    def forward(self, X):\n",
    "        Y = self.dropout(F.relu(self.bn1(self.conv1(X))))\n",
    "        Y = self.dropout(F.relu(self.bn2(self.conv2(Y))))\n",
    "        \n",
    "        if self.conv3 and self.conv4:\n",
    "            Y = self.dropout(F.relu(self.bn3(self.conv3(Y))))\n",
    "            Y = self.dropout(F.relu(self.bn4(self.conv4(Y))))\n",
    "            Y = self.pool(Y)\n",
    "        else:\n",
    "            Y = self.pool(Y)\n",
    "        \n",
    "        return Y\n",
    "    \n",
    "    \n",
    "    def shape_computation(self, X):\n",
    "        Y = self.bn1(self.conv1(X))\n",
    "        print(f\"self.bn(conv1(X)).shape: {Y.shape}\")\n",
    "        \n",
    "        Y = self.bn2(self.conv2(Y))\n",
    "        print(f\"self.bn(conv2(Y)).shape: {Y.shape}\")\n",
    "        \n",
    "        if self.conv3 and self.conv4:\n",
    "            Y = self.bn3(self.conv3(Y))\n",
    "            print(f\"self.bn(conv3(Y)).shape: {Y.shape}\")\n",
    "            \n",
    "            Y = self.bn4(self.conv4(Y))\n",
    "            print(f\"self.bn(conv4(Y)).shape: {Y.shape}\")\n",
    "            \n",
    "            Y = self.pool(Y)\n",
    "            print(f\"self.pool(Y).shape: {Y.shape}\")\n",
    "        else:\n",
    "            Y = self.pool(Y)\n",
    "            print(f\"self.pool(Y).shape: {Y.shape}\")\n",
    "        \n",
    "\n",
    "    def initialize_weights(self):\n",
    "        for m in self.modules():\n",
    "            # print(m)\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                nn.init.kaiming_uniform_(m.weight)\n",
    "\n",
    "                '''\n",
    "                # Do not initialize bias (due to batchnorm)-\n",
    "                if m.bias is not None:\n",
    "                    nn.init.constant_(m.bias, 0)\n",
    "                '''\n",
    "            \n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                # Standard initialization for batch normalization-\n",
    "                nn.init.constant_(m.weight, 1)\n",
    "                nn.init.constant_(m.bias, 0)\n",
    "\n",
    "            elif isinstance(m, nn.Linear):\n",
    "                nn.init.kaiming_normal_(m.weight)\n",
    "                nn.init.constant_(m.bias, 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "blQAhpclr2HN"
   },
   "outputs": [],
   "source": [
    "class VGG18(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.block1 = VGGBlock(input_channels = 3, num_channels = 64, dropout = 0)\n",
    "        self.block2 = VGGBlock(input_channels = 64, num_channels = 128)\n",
    "        self.block3 = VGGBlock(input_channels = 128, num_channels = 256,\n",
    "                use_extra_conv = True, dropout = 0.3)\n",
    "        self.block4 = VGGBlock(input_channels = 256, num_channels = 512,\n",
    "                use_extra_conv = True, dropout = 0.3)\n",
    "        self.block5 = VGGBlock(input_channels = 512, num_channels = 512,\n",
    "                use_extra_conv = True, dropout = 0.3)\n",
    "\n",
    "        # self.pool = nn.MaxPool2d(kernel_size = 1, stride = 1)\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.dense1 = nn.Linear(in_features = 512, out_features = 256)\n",
    "        self.dense2 = nn.Linear(in_features = 512, out_features = 10)\n",
    "        \n",
    "        self.dropout = nn.Dropout(p = 0.3)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.block1(x)\n",
    "        x = self.block2(x)\n",
    "        x = self.block3(x)\n",
    "        x = self.block4(x)\n",
    "        x = self.block5(x)\n",
    "        # x = self.pool(x)\n",
    "        # print(f\"After block5, x.shape: {x.shape}\")\n",
    "        # After block5, x.shape: torch.Size([128, 512, 1, 1]) \n",
    "        # x = torch.flatten(x)\n",
    "        x = self.flatten(x)\n",
    "        # print(f\"After flattening, x.shape: {x.shape}\")\n",
    "        # After flattening, x.shape: torch.Size([128, 512]) \n",
    "        x = F.relu(self.dropout(self.dense1(x)))\n",
    "        x = F.relu(self.dense2(x))\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "DH9-QD9cr72F"
   },
   "outputs": [],
   "source": [
    "# model = VGG18()\n",
    "# model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "qZ6d8fD8bh4n"
   },
   "outputs": [],
   "source": [
    "# images = images.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "v_NeRxjgbOGE"
   },
   "outputs": [],
   "source": [
    "# Sanity check-\n",
    "# model(images).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dUg4P5nMb2U5"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WZuNmcwYr8Z8"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "RMcYwZYBr8io"
   },
   "outputs": [],
   "source": [
    "# Initialize VGG-18 CNN model-\n",
    "model = nn.Sequential(\n",
    "    VGGBlock(input_channels = 3, num_channels = 64, dropout = 0.0),\n",
    "    VGGBlock(input_channels = 64, num_channels = 128, dropout = 0.2),\n",
    "    VGGBlock(input_channels = 128, num_channels = 256, use_extra_conv = True, dropout = 0.3),\n",
    "    VGGBlock(input_channels = 256, num_channels = 512, use_extra_conv = True, dropout = 0.3),\n",
    "    VGGBlock(input_channels = 512, num_channels = 512, use_extra_conv = True, dropout = 0.3),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(in_features = 512, out_features = 256),\n",
    "    nn.Dropout(p = 0.3),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(in_features = 256, out_features = 10)\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "q9aSBydxsD27",
    "outputId": "79ba1ecd-97d6-4c18-fea7-2acb2cbe66f8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): VGGBlock(\n",
       "    (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.0, inplace=False)\n",
       "    (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (1): VGGBlock(\n",
       "    (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.2, inplace=False)\n",
       "    (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (2): VGGBlock(\n",
       "    (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.3, inplace=False)\n",
       "    (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (3): VGGBlock(\n",
       "    (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.3, inplace=False)\n",
       "    (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (4): VGGBlock(\n",
       "    (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.3, inplace=False)\n",
       "    (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (5): Flatten(start_dim=1, end_dim=-1)\n",
       "  (6): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (7): Dropout(p=0.3, inplace=False)\n",
       "  (8): ReLU()\n",
       "  (9): Linear(in_features=256, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Place model on GPU-\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1SnL7fjksJLh"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ORmSWVi7cAPh"
   },
   "outputs": [],
   "source": [
    "images = images.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "W4XCUqDusFZs",
    "outputId": "755b0505-9d2c-41ed-ca55-73265efcf1cb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 10])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check-\n",
    "model(images).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-7cIEuAusMAs"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9A-AfIXwcE6u"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "q8zux0jjsHq0",
    "outputId": "e986b7ad-171b-474c-cb60-2097fdecc112"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer.shape = torch.Size([64, 3, 3, 3]) has 1728 parameters\n",
      "layer.shape = torch.Size([64, 64, 3, 3]) has 36864 parameters\n",
      "layer.shape = torch.Size([64]) has 64 parameters\n",
      "layer.shape = torch.Size([64]) has 64 parameters\n",
      "layer.shape = torch.Size([64]) has 64 parameters\n",
      "layer.shape = torch.Size([64]) has 64 parameters\n",
      "layer.shape = torch.Size([128, 64, 3, 3]) has 73728 parameters\n",
      "layer.shape = torch.Size([128, 128, 3, 3]) has 147456 parameters\n",
      "layer.shape = torch.Size([128]) has 128 parameters\n",
      "layer.shape = torch.Size([128]) has 128 parameters\n",
      "layer.shape = torch.Size([128]) has 128 parameters\n",
      "layer.shape = torch.Size([128]) has 128 parameters\n",
      "layer.shape = torch.Size([256, 128, 3, 3]) has 294912 parameters\n",
      "layer.shape = torch.Size([256, 256, 3, 3]) has 589824 parameters\n",
      "layer.shape = torch.Size([256, 256, 3, 3]) has 589824 parameters\n",
      "layer.shape = torch.Size([256, 256, 3, 3]) has 589824 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([512, 256, 3, 3]) has 1179648 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512, 512, 3, 3]) has 2359296 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([512]) has 512 parameters\n",
      "layer.shape = torch.Size([256, 512]) has 131072 parameters\n",
      "layer.shape = torch.Size([256]) has 256 parameters\n",
      "layer.shape = torch.Size([10, 256]) has 2560 parameters\n",
      "layer.shape = torch.Size([10]) has 10 parameters\n"
     ]
    }
   ],
   "source": [
    "# Count number of layer-wise parameters and total parameters-\n",
    "tot_params = 0\n",
    "for param in model.parameters():\n",
    "    print(f\"layer.shape = {param.shape} has {param.nelement()} parameters\")\n",
    "    tot_params += param.nelement()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Bp9X8qnOsY1t",
    "outputId": "161ec2b1-37d8-4676-dcf8-aa96cf9bfc3f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of parameters in VGG-18 CNN = 20163786\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total number of parameters in VGG-18 CNN = {tot_params}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5WdKiWsLsZYN"
   },
   "outputs": [],
   "source": [
    "'''\n",
    "# Print layer names-\n",
    "for layer in model.state_dict().keys():\n",
    "    print(f\"{layer} has dimension = {model.state_dict()[layer].shape}\")\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Y1jFfuJiscmN"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "O_WUBfzFscrc"
   },
   "outputs": [],
   "source": [
    "# Save random initial weights-\n",
    "torch.save(model.state_dict(), 'VGG18_random_weights_dropouts.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gdgada5J5mIR",
    "outputId": "4ad9e79d-a499-4831-c030-219544802967"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load randomly initialised weights-\n",
    "# model.load_state_dict(torch.load('VGG18_random_weights_dropouts.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FGUR1m04stvU"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "OozQ7SF4st4U"
   },
   "outputs": [],
   "source": [
    "# Define loss function and optimizer-\n",
    "loss = nn.CrossEntropyLoss()\n",
    "\n",
    "# optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = 0.01, momentum = 0.9, weight_decay = 5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1NXRjN5is4eF"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "stjVD5uus0xW"
   },
   "outputs": [],
   "source": [
    "def train_step(model, train_loader):\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0.0\n",
    "\n",
    "    for batch, (images, labels) in enumerate(train_loader):\n",
    "        # Attempt to push to GPU if available\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # print(f\"batch # = {batch}\")\n",
    "\n",
    "        model.train()\n",
    "\n",
    "        # Forward pass-\n",
    "        outputs = model(images)\n",
    "\n",
    "        # Compute loss-\n",
    "        J = loss(outputs, labels)\n",
    "\n",
    "        # Backward pass-\n",
    "        optimizer.zero_grad()   # empty accumulated gradients\n",
    "\n",
    "        J.backward()    # perform backpropagation\n",
    "\n",
    "        # Updates parameters-\n",
    "        optimizer.step()\n",
    "        \n",
    "        # Compute model's performance statistics-\n",
    "        running_loss += J.item() * images.size(0)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        running_corrects += torch.sum(predicted == labels.data)\n",
    "\n",
    "        '''\n",
    "        # Print information every 100 steps-\n",
    "        if (batch + 1) % 100 == 0:\n",
    "            print(f\"epoch {epoch + 1}/{num_epochs}, step {batch + 1}/{num_training_steps}, loss = {J.item():.4f}\")\n",
    "        '''\n",
    "\n",
    "    epoch_loss = running_loss / len(train_dataset)\n",
    "    epoch_acc = (running_corrects.double() / len(train_dataset)) * 100\n",
    "    \n",
    "    return epoch_loss, epoch_acc.cpu().numpy()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "0DyTwwRhcOeY"
   },
   "outputs": [],
   "source": [
    "def train_model_progress(model, train_loader):\n",
    "    '''\n",
    "    Function to perform one epoch of training by using 'train_loader'.\n",
    "    Returns loss and number of correct predictions for this epoch.\n",
    "    '''\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0.0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    with tqdm(train_loader, unit = 'batch') as tepoch:\n",
    "        for images, labels in tepoch:\n",
    "            tepoch.set_description(f\"Training: \")\n",
    "            \n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "            \n",
    "            # Get model predictions-\n",
    "            outputs = model(images)\n",
    "            \n",
    "            # Compute loss-\n",
    "            J = loss(outputs, labels)\n",
    "            \n",
    "            # Empty accumulated gradients-\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            # Perform backprop-\n",
    "            J.backward()\n",
    "            \n",
    "            # Update parameters-\n",
    "            optimizer.step()\n",
    "            \n",
    "            # Compute model's performance statistics-\n",
    "            running_loss += J.item() * images.size(0)\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            running_corrects += torch.sum(predicted == labels.data)\n",
    "            \n",
    "            tepoch.set_postfix(\n",
    "                loss = running_loss / len(train_dataset),\n",
    "                accuracy = (running_corrects.double().cpu().numpy() / len(train_dataset)) * 100\n",
    "            )\n",
    "            \n",
    "    \n",
    "    train_loss = running_loss / len(train_dataset)\n",
    "    train_acc = (running_corrects.double() / len(train_dataset)) * 100\n",
    "    \n",
    "\n",
    "    # return running_loss, running_corrects\n",
    "    return train_loss, train_acc.cpu().numpy()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uCq8FAdOckD6"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lHUDSkX8s8Od"
   },
   "outputs": [],
   "source": [
    "def validate_step(model, test_loader):\n",
    "    total, correct = 0, 0\n",
    "    running_loss_val = 0.0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for images, labels in test_loader:\n",
    "\n",
    "            # Place features (images) and targets (labels) to GPU-\n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Set model to evaluation mode-\n",
    "            model.eval()\n",
    "    \n",
    "            # Make predictions using trained model-\n",
    "            outputs = model(images)\n",
    "            _, y_pred = torch.max(outputs, 1)\n",
    "\n",
    "            # Compute validation loss-\n",
    "            J_val = loss(outputs, labels)\n",
    "\n",
    "            running_loss_val += J_val.item() * labels.size(0)\n",
    "    \n",
    "            # Total number of labels-\n",
    "            total += labels.size(0)\n",
    "\n",
    "            # Total number of correct predictions-\n",
    "            correct += (y_pred == labels).sum()\n",
    "\n",
    "    epoch_val_loss = running_loss_val / len(test_dataset)\n",
    "    val_acc = 100 * (correct / total)\n",
    "\n",
    "    return epoch_val_loss, val_acc.cpu().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "YymBc1UtcnEA"
   },
   "outputs": [],
   "source": [
    "def test_model_progress(model, test_loader):\n",
    "    total = 0.0\n",
    "    correct = 0.0\n",
    "    running_loss_val = 0.0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        with tqdm(test_loader, unit = 'batch') as tepoch:\n",
    "            for images, labels in tepoch:\n",
    "                tepoch.set_description(f\"Validation: \")\n",
    "                \n",
    "                images = images.to(device)\n",
    "                labels = labels.to(device)\n",
    "                \n",
    "                # Set model to evaluation mode-\n",
    "                model.eval()\n",
    "            \n",
    "                # Predict using trained model-\n",
    "                outputs = model(images)\n",
    "                _, y_pred = torch.max(outputs, 1)\n",
    "                \n",
    "                # Compute validation loss-\n",
    "                J_val = loss(outputs, labels)\n",
    "                \n",
    "                running_loss_val += J_val.item() * labels.size(0)\n",
    "    \n",
    "                # Total number of labels-\n",
    "                total += labels.size(0)\n",
    "\n",
    "                # Total number of correct predictions-\n",
    "                correct += (y_pred == labels).sum()\n",
    "                \n",
    "                tepoch.set_postfix(\n",
    "                    val_loss = running_loss_val / len(test_dataset),\n",
    "                    val_acc = 100 * (correct.cpu().numpy() / total)\n",
    "                )\n",
    "            \n",
    "        \n",
    "    # return (running_loss_val, correct, total)\n",
    "    val_loss = running_loss_val / len(test_dataset)\n",
    "    val_acc = (correct / total) * 100\n",
    "\n",
    "    return val_loss, val_acc.cpu().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "3BGHQY9YcuqJ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "n5OVpgCicu1i",
    "outputId": "20d63dcf-26b5-4958-96f7-c134ef794420"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [04:08<00:00,  1.57batch/s, accuracy=45.5, loss=1.47]\n"
     ]
    }
   ],
   "source": [
    "# Sanity check-\n",
    "train_loss, train_acc = train_model_progress(model, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AP1HjJC6c7nS",
    "outputId": "44c7dcd7-c693-4904-9328-d6cf916ab75f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Validation: : 100%|██████████| 79/79 [00:08<00:00,  9.06batch/s, val_acc=56.6, val_loss=1.21]\n"
     ]
    }
   ],
   "source": [
    "# Sanity check-\n",
    "val_loss, val_acc = test_model_progress(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RgTblttjdFXu"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2gw4-9h2vibj"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gDIK4_3W8L95"
   },
   "source": [
    "### Train model _without_ learning rate scheduler, using early-stopping criterion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "NOytz_v9tPxV"
   },
   "outputs": [],
   "source": [
    "# Initialize parameters for Early Stopping manual implementation-\n",
    "best_val_loss = 100\n",
    "loc_patience = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "SbXa8wiR6OHv"
   },
   "outputs": [],
   "source": [
    "# User input parameters for Early Stopping in manual implementation-\n",
    "minimum_delta = 0.001\n",
    "patience = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "K1oqH6x25YFS"
   },
   "outputs": [],
   "source": [
    "training_history = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "qYVpzJ0_h97Z"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "G7E2MwUJh-Dv",
    "outputId": "e29a407f-e25c-4b77-9ee2-3bd839eb7952"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.50batch/s, accuracy=19.4, loss=1.98]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.44batch/s, val_acc=10, val_loss=3.27]   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 1 training loss = 1.9801, training accuracy = 19.40%, val_loss = 3.2728, val_accuracy = 10.00% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 3.2728\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.50batch/s, accuracy=27.8, loss=1.78]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.41batch/s, val_acc=10, val_loss=4.1]    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 2 training loss = 1.7840, training accuracy = 27.78%, val_loss = 4.0970, val_accuracy = 10.00% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=38.1, loss=1.57]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.63batch/s, val_acc=10.2, val_loss=3.16] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 3 training loss = 1.5700, training accuracy = 38.14%, val_loss = 3.1569, val_accuracy = 10.25% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 3.1569\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.36batch/s, accuracy=45.9, loss=1.41]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.53batch/s, val_acc=18.6, val_loss=2.44] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 4 training loss = 1.4121, training accuracy = 45.88%, val_loss = 2.4424, val_accuracy = 18.59% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 2.4424\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=53.8, loss=1.26]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.71batch/s, val_acc=24.5, val_loss=2.1]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 5 training loss = 1.2565, training accuracy = 53.83%, val_loss = 2.0994, val_accuracy = 24.52% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 2.0994\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.41batch/s, accuracy=59.9, loss=1.13]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.24batch/s, val_acc=42.7, val_loss=1.63] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 6 training loss = 1.1289, training accuracy = 59.86%, val_loss = 1.6303, val_accuracy = 42.67% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 1.6303\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=64.2, loss=1.02]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.73batch/s, val_acc=45.6, val_loss=1.68] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 7 training loss = 1.0215, training accuracy = 64.21%, val_loss = 1.6820, val_accuracy = 45.61% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.37batch/s, accuracy=67.6, loss=0.939] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.23batch/s, val_acc=50.8, val_loss=1.45] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 8 training loss = 0.9390, training accuracy = 67.64%, val_loss = 1.4533, val_accuracy = 50.79% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 1.4533\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.42batch/s, accuracy=70, loss=0.877]   \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.12batch/s, val_acc=52.2, val_loss=1.44] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 9 training loss = 0.8771, training accuracy = 70.02%, val_loss = 1.4436, val_accuracy = 52.24% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 1.4436\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=72.5, loss=0.805] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.34batch/s, val_acc=55.2, val_loss=1.35] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 10 training loss = 0.8048, training accuracy = 72.45%, val_loss = 1.3493, val_accuracy = 55.21% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 1.3493\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.42batch/s, accuracy=74.5, loss=0.76]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.05batch/s, val_acc=65.1, val_loss=1.05] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 11 training loss = 0.7601, training accuracy = 74.50%, val_loss = 1.0531, val_accuracy = 65.15% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 1.0531\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.44batch/s, accuracy=76.3, loss=0.715] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.33batch/s, val_acc=71.8, val_loss=0.909]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 12 training loss = 0.7153, training accuracy = 76.27%, val_loss = 0.9085, val_accuracy = 71.79% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.9085\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.41batch/s, accuracy=77.2, loss=0.686] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.44batch/s, val_acc=72.1, val_loss=0.888]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 13 training loss = 0.6856, training accuracy = 77.18%, val_loss = 0.8877, val_accuracy = 72.10% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.8877\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.42batch/s, accuracy=78.5, loss=0.649] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.86batch/s, val_acc=71.6, val_loss=0.884]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 14 training loss = 0.6492, training accuracy = 78.51%, val_loss = 0.8837, val_accuracy = 71.61% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.8837\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=79.6, loss=0.615] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.75batch/s, val_acc=77.6, val_loss=0.714]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 15 training loss = 0.6154, training accuracy = 79.61%, val_loss = 0.7138, val_accuracy = 77.58% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.7138\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=80.2, loss=0.605] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.23batch/s, val_acc=76.5, val_loss=0.764]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 16 training loss = 0.6053, training accuracy = 80.20%, val_loss = 0.7636, val_accuracy = 76.47% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.44batch/s, accuracy=81, loss=0.578]   \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.80batch/s, val_acc=80.5, val_loss=0.669] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 17 training loss = 0.5781, training accuracy = 80.95%, val_loss = 0.6687, val_accuracy = 80.52% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.6687\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=81.9, loss=0.552] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.40batch/s, val_acc=76.6, val_loss=0.761]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 18 training loss = 0.5520, training accuracy = 81.92%, val_loss = 0.7614, val_accuracy = 76.60% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.42batch/s, accuracy=82.4, loss=0.535] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.86batch/s, val_acc=79.2, val_loss=0.717]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 19 training loss = 0.5347, training accuracy = 82.36%, val_loss = 0.7166, val_accuracy = 79.17% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.39batch/s, accuracy=83.2, loss=0.516] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.00batch/s, val_acc=76.2, val_loss=0.789]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 20 training loss = 0.5157, training accuracy = 83.21%, val_loss = 0.7891, val_accuracy = 76.22% & LR = 0.0100\n",
      "\n",
      "\n",
      "'EarlyStopping' called!\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(num_epochs):\n",
    "\n",
    "    if loc_patience >= patience:\n",
    "        print(\"\\n'EarlyStopping' called!\\n\")\n",
    "        break\n",
    "\n",
    "    train_loss, train_acc = train_model_progress(model, train_loader)\n",
    "    val_loss, val_acc = test_model_progress(model, test_loader)\n",
    "    \n",
    "    print(f\"\\nepoch: {epoch + 1} training loss = {train_loss:.4f}, \"\n",
    "          f\"training accuracy = {train_acc:.2f}%, val_loss = {val_loss:.4f}\"\n",
    "          f\", val_accuracy = {val_acc:.2f}% & \"\n",
    "          f\"LR = {optimizer.param_groups[0]['lr']:.4f}\\n\")\n",
    "    \n",
    "    training_history[epoch + 1] = {\n",
    "        'loss': train_loss, 'acc': train_acc,\n",
    "        'val_loss': val_loss, 'val_acc': val_acc,\n",
    "        'lr': optimizer.param_groups[0]['lr']\n",
    "    }\n",
    "\n",
    "    '''\n",
    "    # Save best weights achieved until now-\n",
    "    if (val_loss < best_val_loss):    \n",
    "        # update 'best_val_loss' variable to lowest loss encountered so far-\n",
    "        best_val_loss = val_loss\n",
    "\n",
    "        print(f\"Saving model with lowest val_loss = {val_loss:.4f}\\n\")\n",
    "        \n",
    "        # Save trained model with 'best' validation accuracy-\n",
    "        torch.save(model.state_dict(), \"VGG18_best_model.pth\")\n",
    "    '''\n",
    "\n",
    "    # Code for manual Early Stopping:\n",
    "    # if np.abs(val_loss < best_val_loss) >= minimum_delta:\n",
    "    if (val_loss < best_val_loss) and \\\n",
    "    (np.abs(val_loss - best_val_loss) >= minimum_delta):\n",
    "\n",
    "        # update 'best_val_loss' variable to lowest loss encountered so far-\n",
    "        best_val_loss = val_loss\n",
    "        \n",
    "        # reset 'loc_patience' variable-\n",
    "        loc_patience = 0\n",
    "\n",
    "        print(f\"Saving model with lowest val_loss = {val_loss:.4f}\\n\")\n",
    "        \n",
    "        # Save trained model with 'best' validation accuracy-\n",
    "        torch.save(model.state_dict(), \"VGG18_dropout_best_model.pth\")\n",
    "        \n",
    "    else:  # there is no improvement in monitored metric 'val_loss'\n",
    "        loc_patience += 1  # number of epochs without any improvement\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HW6eI7f-4W9R"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "X3dcgzOwii1g"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qXesVlcAtj_U",
    "outputId": "fa906693-19de-4aa6-fe1c-b4d197346f26"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "COrZUx-_P6Zx",
    "outputId": "66e63461-5f0b-436b-b4fd-5615ef5c769b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['loss', 'acc', 'val_loss', 'val_acc', 'lr'])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_history[2].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "D0ZrNvFoP_fZ",
    "outputId": "d1441b8f-507c-4044-93ae-c4a775c61fb7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(9.999999, dtype=float32), array(27.784))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_history[2]['val_acc'], training_history[2]['acc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAG5CAYAAAC6Fv9NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABdMklEQVR4nO3dd3hUVf7H8fc3jRBCCQm9NwHpRQFRRMFeULE37N1VtyiWVVdddXddf66ruy5WVOyoiJ0iYEEFlA5CQocQIEBIIf38/rgDBkwgQCZ3ZvJ5PU+embn3zsznzmQy35x7zrnmnENEREQkFET5HUBERERkFxUmIiIiEjJUmIiIiEjIUGEiIiIiIUOFiYiIiIQMFSYiIiISMlSYiNQAZvaZmY2q6m2DwcwuMbMv97F+qJmtq85Mez3/NDO7xq/nrwpm9pyZ/dnvHCLlUWEiEc3MvjCzh8pZPsLMNppZTBCe82EzW2BmxWb2YDnrbzWzlWa2w8xmm9nRFTxOTpmfUjPbWeb2JQeSyTl3inNubFVvGwzOuXHOuRN33TYzZ2YdD/bxAoVE/l6v58SqSXtAOeLM7J9mti6QYaWZ/V+Z9avMbHh1ZHHO3eCce7g6nkvkQKkwkUj3CnCZmdleyy8DxjnnioPwnKnAncAne68wswHA48C5QH3gReADM4vee1vnXOKuH2ANcEaZZePKPGaVF1cR6Jayr6dz7owDfQDzHMrfzLuB/sCRQF3gOODnQ3g8kYikwkQi3YdAQ+CYXQvMLAk4HXg1cDvZzCYGWjBmmdkjZvZNme1PNLNfzCzLzP5jZtP31ZTvnBvrnPsMyC5ndVtgkXNujvOmXX4VSAEaV3aHdh3KMLO7zGwj8LKZJZnZx2a22cy2Ba63LHOf3YcfzOwKM/vGzJ4IbLvSzE45yG3bmdkMM8s2s8lm9qyZvV5B7ulmNjJw/ehAS8ipgdvDzWxu2ecMXJ8RuPu8QCvDBWUe7w9mtsnM0s3sysq+fntlqszr9lcz+xbIA9qXWVfLzLaaWY8yyxoHWrYalfN0RwAfOOc2OM8q59yu38HXgNbAxMB+3hlYfqaZLTKz7YEsXcs81yozu9vMFgeyv2xm8YF1u35H7jGzLYFtLylz31fM7JG9ti339dzf50OkqqkwkYjmnNsJvANcXmbx+cBS59y8wO1ngVygKTAq8AOAmaUA7+H9t5sM/AIcdQiRPgOizWxAoJXkKmAusPEAH6cpXsHVBrgO77P8cuB2a2An8Mw+7j8Ab19SgL8DL5r9plWpMtu+AfyI99o8iNcSVZHpwNDA9SHACuDYMren730H59yQwNVegZaOtwO3m+K1OLUArgaeDRScB6oyr9tleK9xXWB1mWwFwFvApWW2vQiY7JzbXM5zfQ/83sxuMrMeZV9v59xl7Nkq9nczOwx4E7gdaAR8ile4xJV5zEuAk4AOwGHAfWXWNcV7z1rg/U6PMbPOFbwO+3o9K/x8iASDChOpCcYC55lZ7cDtywPLCBQHI4EHnHN5zrnFu9YFnIrXwvF+4LDP0xx4EVFWNjAe+AYoAB4ArnMHftKq0kDmAufcTudcpnNufGAfsoG/8uuXfnlWO+eed86V4O1vM6DJgWxrZq3xWgHud84VOue+AT7ax3NOZ89C5LEyt4+lnMJkH4qAh5xzRc65T4EcoKIvXYCnA60Ou34eBqjk6/aKc26Rc67YOVe017qxwMX26yGey4DXKsjwGPA3vGJiNrDe9t3J+ALgE+fcpMDzPgHUZs/C+Bnn3Frn3NZA9ov2eow/B35HpuMdWjy/gucq9/WsxOdDpMqpMJGIF/jC3AyMMLP2eF+mbwRWNwJigLVl7lL2evOytwMFxO4RIYFm9l0dKo9h/67BayXpBsTh/bf9sZk1P8Dd2uycyy+TI8HM/mdmq81sBzADaGDl9F0J2F1cOefyAlcTD3Db5sDWMstgz9dubzOBw8ysCdAb7zBWq0Cr1JGBzJWVuVf/oLx95Af4nXOuQZmfP0OlX7cK98k59wNea8KxZtYF6EgFxZlzrsQ596xzbjDQAK+QeKns4Zm9NGfPFprSQJYWFWRbHbjPLtucc7n7WF9WRa/n/j4fIlVOhYnUFK/itZRcBnzpnMsILN8MFAMty2zbqsz19LLrAs3vu28757qV6VD5dSVy9AImOueWOedKnXOfB57jQA8P7d3C8ge8FoMBzrl6eC0SABUdnqkK6UBDM0sos6xVRRsHCpg5wG3AQudcIfAd8HsgzTm3JYhZK1KZ121/rVlj8QrMy4D3yhaMFQm0cj0LbAMOr+B5NuAdYvICeb97rYD1ZbYp+3q3DtxnlyQzq7OP9ZWxv8+HSJVTYSI1xavAcOBayjRFBw5PvA88GPjvuQt79kf5BOhhZmeZN/rlZrxj7RUys9hAJ8QoIMbM4sv8Bz4LOM3M2pvnBLy+AQsPcf/q4vWP2G5mDfEOEQWVc2413iGJB80bCjsI2N9ol+nALfx62GbaXrfLk0GZTqdVrCpet9eAs/GKk1cr2sjMbg90NK1tZjGBwzh1+XVkzt77+Q7e78owM4vFK6IK8Iq5XW42s5aB7PcAb7OnvwTem2PwOny/eyA7VonPh0iVU2EiNYJzbhXeH/Q6/Lap/Ra8jn8b8b5k3sT7AiDwX/x5eJ0+M/H+u529a30Fnsf7srsIuDdwfVen0FfxOkxOA3bg9Vm53jm39BB2D+ApvP4HW/A6WX5+iI9XWZcAg/Bem0fwvhj39dpMx/synlHB7fI8CIwN9A2pqI/E/jxje85jMiew/CkO8XVzzq0DfsJr8dhXq9lO4J94v2db8Irckc65FYH1jwH3Bfbzj865X/CKnX8Htj8Dr3NsYZnHfAP4Eq8j8Qq892CXjXgtMhuAccANB/l7VuHnQyQY7MD73IlENjP7G9DUOfebjomBTo7rgEucc19Ve7gQZ2Zv4414CnqLTSgxs5eADc65+/a7cdU95yrgGufc5HLWDQVed8613HtdFTxvhZ8PkaqgFhOp8cysi5n1DBxaORJvuOQHZdafZGYNzKwWXnO54f13XeOZ2RFm1sHMoszsZGAE3twxNYaZtQXOwZssL+Ls7/MhUtU0Y6SIdyjhTbwRC5vwmtsnlFk/CK/JPA5YDJwVmB9FvP427+PNY7IOuNE5V2NmMw0MO74DeMw5t9LvPEGyv8+HSJXSoRwREREJGTqUIyIiIiEjLA7lpKSkuLZt2/odQ0RERKrAnDlztjjnyjunVHgUJm3btmX27Nl+xxAREZEqYGarK1qnQzkiIiISMlSYiIiISMhQYSIiIiIhIyz6mJSnqKiIdevWkZ+/3/NlyV7i4+Np2bIlsbGxfkcRERHZQ9gWJuvWraNu3bq0bdsW76SbUhnOOTIzM1m3bh3t2rXzO46IiMgewvZQTn5+PsnJySpKDpCZkZycrJYmEREJSWFbmAAqSg6SXjcREQlVYV2YiIiISGRRYXKQtm/fzn/+85+Duu+pp57K9u3bqzaQiIhIBFBhcpD2VZiUlJTs876ffvopDRo0CEIqERGR8KbC5CCNHj2atLQ0evfuzZ/+9CemTZvGcccdx8UXX0yPHj0AOOuss+jXrx/dunVjzJgxu+/btm1btmzZwqpVq+jatSvXXnst3bp148QTT2Tnzp2/ea6JEycyYMAA+vTpw/Dhw8nIyAAgJyeHK6+8kh49etCzZ0/Gjx8PwOeff07fvn3p1asXw4YNq4ZXQ0REpGqE7XDhsv4ycRGLN+yo0sc8vHk9HjijW4XrH3/8cRYuXMjcuXMBmDZtGj/++CMLFy7cPQz3pZdeomHDhuzcuZMjjjiCkSNHkpycvMfjLF++nDfffJPnn3+e888/n/Hjx3PppZfusc3RRx/N999/j5nxwgsv8Pe//51//vOfPPzww9SvX58FCxYAsG3bNjZv3sy1117LjBkzaNeuHVu3bq3CV0VERCS4IqIwCRVHHnnkHnODPP3003zwwQcArF27luXLl/+mMGnXrh29e/cGoF+/fqxateo3j7tu3TouuOAC0tPTKSws3P0ckydP5q233tq9XVJSEhMnTmTIkCG7t2nYsGFV7qKIiEhQRURhsq+WjepUp06d3denTZvG5MmTmTlzJgkJCQwdOrTcuUNq1aq1+3p0dHS5h3JuvfVWfv/733PmmWcybdo0HnzwQcCbLG3vob/lLRMREQkX6mNykOrWrUt2dnaF67OyskhKSiIhIYGlS5fy/fffH/RzZWVl0aJFCwDGjh27e/mJJ57IM888s/v2tm3bGDRoENOnT2flypUAOpQjIiIHLDu/iHlrt/vy3BHRYuKH5ORkBg8eTPfu3TnllFM47bTT9lh/8skn89xzz9GzZ086d+7MwIEDD/q5HnzwQc477zxatGjBwIEDdxcd9913HzfffDPdu3cnOjqaBx54gHPOOYcxY8ZwzjnnUFpaSuPGjZk0adIh7auIiESmnIJilmdkszwjh2UZ2SzflMPyjGw2ZHkt/HPvP4EGCXHVmsmcc9X6hAejf//+bvbs2XssW7JkCV27dvUpUfjT6yciUnPkFhSTuunX4mNZoBhZv/3X7gNxMVF0bJTIYU0S6dSkLoc1qcvRHVOoHRdd5XnMbI5zrn9569RiIiIiEiHyCncVIDks3/RrS8i6bWUKkOgo2jeqQ782SVx0ZKvdRUjrhglER/nfR1GFiYiISBgpKXVszyskPSufZRnZXhGSkc2yTV4BsutAyK4CpE/rJC7ov6sASaR1wwRiokO3i6kKExERER8558gtLCEzp4AtOYVk5hSQmVv46+3A9cycQjJzC9iaW0hpmV4YsdFG+5REerVswLl9W+0+FNM2ObQLkIqoMBEREalipaWOzTkFbClTUGTmFO5ReOxatyWngILi0nIfp258DCmJtUiuE0fblAT6tU0ipU4cyYm1aFy3Fp2aJNImuQ6xYViAVESFiYiIyCEoLC5l+aZsFm3YweINO1i0IYsl6dnkFBT/Ztu46ChSEr3CIjkxjk6N6wZux5Fcx1uWEljXsE4ctWKqvuNpqFNhIiIiUknZ+UUsSc9m8YYsFm3YwaINO1i+KZuiEu/YSkJcNF2b1eOcvi3o1KQujRJr7S5EUhLjSKwVo0kw90OFSTVKTEwkJyfH7xgiIlIJm7Lzd7eC7GoJWZWZt3t9SmIchzevz5DDGtGteT26Na9Hm+Q6ITGyJZypMBERkRqttNSxZmteoAUki8XpXkvI5uyC3du0bphAt+b1GNm3Jd1a1KNb8/o0rltLrR9BoMLkIN111120adOGm266CfBmZ61bty7XX389I0aMYNu2bRQVFfHII48wYsSIfT7WWWedxdq1a8nPz+e2227juuuuA+Dzzz/nnnvuoaSkhJSUFKZMmUJOTg633nors2fPxsx44IEHGDlyZND3V0Qk3JWWOjKy81m5OZcVW3JJ3ZTzm/4gMVFGx8aJDOn0aytI1+b1qBcf63P6miMyCpPPRsPGBVX7mE17wCmPV7j6wgsv5Pbbb99dmLzzzjt8/vnnxMfH88EHH1CvXj22bNnCwIEDOfPMM/dZVb/00ks0bNiQnTt3csQRRzBy5EhKS0u59tprmTFjBu3atdt9zpuHH36Y+vXrs2CBt7/btm2rwp0WEQlvzjm25hayckvub35WZeaSX/Tr6JeEuGgOD/QH8YqQ+nRqklgjO5yGksgoTHzQp08fNm3axIYNG9i8eTNJSUm0bt2aoqIi7rnnHmbMmEFUVBTr168nIyODpk2bVvhYTz/9NB988AEAa9euZfny5WzevJkhQ4bQrl07ABo2bAjA5MmTeeutt3bfNykpKYh7KSISmrLzi1i1JY8VW3K8oqNMAbIj/9fRMDFRRuvkBNol1+Hojim0a1SHdsl16BiTQcqOJUR1PxGiImeobSQIamFiZncA1wAOWABcCSQAbwNtgVXA+c65Q/u3fx8tG8F07rnn8t5777Fx40YuvPBCAMaNG8fmzZuZM2cOsbGxtG3blvz8/AofY9q0aUyePJmZM2eSkJDA0KFDyc/PxzlXbitLRctFRCJNflEJqzPzWLklh5Vbdl3msnJLHltyfu3/YQbN69emfaM6jOjdgnYpdXYXIC2Tav92krGln8A710NhNix8D87+L9TWP3mhImiFiZm1AH4HHO6c22lm7wAXAocDU5xzj5vZaGA0cFewcgTThRdeyLXXXsuWLVuYPn06AFlZWTRu3JjY2Fi++uorVq9evc/HyMrKIikpiYSEBJYuXcr3338PwKBBg7j55ptZuXLl7kM5DRs25MQTT+SZZ57hqaeeArxDOWo1EZFws+uQy4bt+azfvpMN23eSnrVzj9ubcwooe57ZlMRatE+pw7AujWmbUod2KXVo36gOrRsmEB9bicMvpaUw7TGY8Xdo3ge6nO7d/t+xcMFr0KxX8HZYKi3Yh3JigNpmVoTXUrIBuBsYGlg/FphGmBYm3bp1Izs7mxYtWtCsWTMALrnkEs444wz69+9P79696dKlyz4f4+STT+a5556jZ8+edO7cmYEDBwLQqFEjxowZwznnnENpaSmNGzdm0qRJ3Hfffdx88810796d6OhoHnjgAc4555yg76uIyIHIKyxmw/b8QLGxk/Xb8/coPjZs3/mb2U5rxUTRokFtmjeozbGHNaJlUgJtUxJon5JI25QE6h5KB9Sd2+H962D5F9D7EjjtSYiNh3ZD4J1R8MIJcNoT0PfyQ9txOWTmypajVf3gZrcBfwV2Al865y4xs+3OuQZlttnmnPvNv/xmdh1wHUDr1q377d3ysGTJErp27Rq07JFOr5+IHIqsnUWkbsreXWTsKj52FSLb8or22N4MmtSNp1mDeJo3qO0VIPXjabbreoPaJCXEBudQ9aYl8NYlsH01nPw4HHGNF2iX3C0w/mpYMQ36XAqnPgGxtas+h+xmZnOcc/3LWxfMQzlJwAigHbAdeNfMLq3s/Z1zY4AxAP379w9e9SQiIvtUXFLK0o3ZzF27nZ/XbGfu2m2kbc7dY5u68TG7C4w+rRvQrP6vBUez+vE0rR/vz/lcFk+AD26EuDow6mNoM+i329RJgUvfDxzm+Qekz4PzX4WG7as/rwT1UM5wYKVzbjOAmb0PHAVkmFkz51y6mTUDNgUxg4iIHADnHOlZ+cxduz1QiGxjwfqs3cNsk+vE0ad1A87u04LDm9ejZVICzerHH9phlmAoLYGpj8A3T0KL/l4fknrNK94+KhqOvw9aHuEd8vnfUK9TbJfTqi3yAdl1tCMCB0MEszBZAww0swS8QznDgNlALjAKeDxwOeFgn0AjVA5OMA/fiUh4yS0oZv66rEAhso25a7eTscMb8RIXHUW3FvW46MjW9GmdRJ9WDWiZVDv0/+7mbYXx10DaFOg7Ck79B8TUqtx9DzsJrp8B71wOb10Mg2+H4/8M0SEyu4ZzkDrFa93ZuACOuhWO+b3XIhQhgvZKO+d+MLP3gJ+AYuBnvEMzicA7ZnY1XvFy3sE8fnx8PJmZmSQnJ4f+hySEOOfIzMwkPj7e7ygiUs1KSx2pm3OYu2Y7P6/dxs9rtrMsI5vSwP8qbZMTGNQ+md6tGtCndRJdm9UjLibM5vjYuBDevgSy1sPpT0H/Kw/8MZLawFVfwOej4dunYP0cOPclSGxc1WkrzzlY8RV89Ris+xHqt4KOw+HrJ2DuG3DCQ9Dj3IhoQQlq59eq0r9/fzd79uw9lhUVFbFu3bp9zhEi5YuPj6dly5bExoZY06uIVKnN2QW7W0J+XrOd+euydk+9Xi8+hl6BAqRPqwb0atWAhnXifE58iBaOhwm3QK163qGbVkce+mPOfRM+vgPi68N5r5TfRyWYnIOVM+CrR2Ht91CvBQz5I/S+FGLiYM338NmdXr+YVgO8zr0t+lZvxoOwr86vYVuYiIjInrblFjJzRSbfpm7hu7RMVm7xOqjGRBldmtX1WkJaJdG7dQPaJdchKlLOgltSDFMehO/+Da0GwvljoW7Fs20fsIxF8PZlsG2V1zIx6ObqaZlY9Y1XkKz+Fuo2g2P+4A1n3vuwVGkJzB0HUx7yRhj1uQSGPeBvC89+qDAREYlAeYXFzFq1je9St/Bt2hYWbdiBc1AnLpoB7ZMZ2L4hfVon0b15fWrHRej5X3Iz4b0rYeV0bxjwSY95LQlVLT8LJtwMSyZC1zNhxLMQX6/qnwdg9XdeQbLqa0hs6vUh6TvKm3dlfxmn/x1+eA5iasOxd8KAG4LzehwiFSYiIhGgqKSU+eu2883yTL5N28LPa7ZRVOKIjTb6tE5icIcUju6UTM+WDfwZmlvd0ufBW5dCTgac/qQ3B0kwOQczn4FJD0BSW+9wUZNuVff4a36AaY9686nUaQxH3+H1kTnQOVW2LIcv7oHlX0LDDnDyY16n3hCiwkREJAyVljp+ycjefWjmhxWZ5BaWYAbdmtdjcIcUjuqYwhFtk0iIC5FRI9Vl3tsw8XeQkOwVCC36Vd9zr/4O3r3Sa6E44ynodeGhPd7aWV5BkjYVElICBclVEJdwaI+77Ev44m7ITIWOJ8BJj0Kjww7tMauIChMRkTCxdmse36Zu4du0TL5L3UJmbiEA7VLqcFSHZAZ3TGFQ+2SS/O6oumEu/PIZNO3hzf1Rt0n1PG9JEUy6H77/D7Q52uuQmtioep67rOwMeO8qWP2NV0Sc/HjlhyTvsn6ON8omdZJXYA2+zTscVZVDf4sL4ccxMP1vUJTnHdo59k6vM6+PVJiIiISoLTkFfBcoQr5N28LarTsBaFy3FoM7puwuRpo3CKEp0jcvgxeHey0Gu9RvDS37e0VKyyOgWc8D/6Len5zN8O4VXjEw4EY48WGI9nF0YUkxTH3YG1LcvA+cN9Ybarw/G36GaY/Dss+9sxof9Ts48jqolRi8rDmbvKw/veYVQcPu9w59RfnT90iFiYhICNiRX8Syjdks2ZjN0vQdzFm9jaUbswFvSveB7ZMZHChEOjZODM05mnK3wAvDoDAXrvgUdm6DdbMCP7Nhxzpvu+g4aNqzTLHSHxq0OfjRLOt/grcvhbxMOONp6HVB1e3ToVr6iTftvRmc8zwcdmL526XP8wqSXz6F+AZw1C1w5PXB60Rbng0/w2ejvaHHzXrByX+r/iHQqDAREalWJaWOVZm5LE3PZunGHSwJXK7btnP3NvXiY+jRsj5HdUhhcMcUujevR0yod1gtyodXR0D6XLjiE6/Y2NuOdFg/+9dCZf1PUBzY7zqNfi1SWh7htTLUqrv/5/15nDeXSGITrz9J895VuVdVY+sKePtyyFgAQ+6EoaN/bY3YuNCbqXXpx94hlEG3wIDr/Tuc4pw358uk+2HHeuh+LpzwF6jfstoiqDAREQmSrbmFLN24Y3cRsnRjNr9szKag2Du3THSU0T6lDl2a1aNL07p0bVaXLk3r0ax+fGi2iFTEOXj/Wljwrtevo9vZlbtfSRFsWvxrobJuNmQu99ZZFDTquuchoJTDICpQoBUXeqNLZj0P7Y6Fc1+GOslB2b0qUbQTPvkjzH0d2h/n9eX44TnvRIK16sHAm2DgjVC7gd9JPYW58M1T8N3TgHmdbgf/rlrOrKzCRETkEBUWl7JiSw5L07NZUqYQ2XVeGYCUxDi6NPUKkF2FSMfGicTHRsAcItMe9/7rH3a/N9HXocjb6rWk7DoEtH72r/1VatXzRti07O9NMLZmpnc+mGEPhs75avbnp1e9AqWkAOLqwsAbvEnZaif5nax821Z7rSeLP/T6Cp34MBw+IqiTyKkwERE5AM455q3L4ocVmSzdmM2S9B2kbc6hqMT7exkXHUWnJol0aVpvdwtI56Z1aVS3ijt7hor573itJb0v8SYWq+ovrNJSb0jrriJl3SxvttXoWjDiGe8cMOEmfb43H0mfSyGhod9pKmfl1975gTIWQttjvJFGTbsH5alUmIiIVMIvG7P5aN56Js5LZ83WPACa1Y/fowXk8Gb1aJtSp2ZMYAbeuVjGngEtj4TLPqi+WUQLcsCVVm/HUPFGGv00FqY+Avnb4ZbZkNyhyp9mX4VJmLSLiYgEx5rMPCbO38BHczfwS0Y2UQaDO6Zwy/EdGd61Sfif2O5QbF0Bb13sncn2gteqd2rzYA6dlYpFx8ARV3t9iH75NChFyf6oMBGRGidjRz4fz0/no3kbmLd2OwD92yTx0IhunNqjGSmJEXpI5kDs3AZvXOC1WlzybvgcjpCqkdAw+FP8V0CFiYjUCNtyC/l80UY+mruB71dm4pw3rfvdp3ThtJ7NaJl0iNN/R5KSInjncti6Ei6f4Mt/zVJzqTARkYiVW1DMpMUZfDRvAzOWbaa41NE+pQ6/O74TZ/RqTsfGOlzwG855c4asnAFnPQdtB/udSGoYFSYiElHyi0qY9stmJs7bwJSlGeQXldK8fjxXH92OM3o1p1vzeuE1f0h1++5p+Pk1GPIn6H2R32mkBlJhIiJhr7iklG/TMpk4bwNfLNxIdkExyXXiOK9fK87s3Zx+rZOIilIxsl+LP4JJD0C3c2DoPX6nkRpKhYmIhKXSUsecNdv4aO4GPl2QTmZuIXVrxXBS96ac2as5R3VIDv0p3kPJ+p/g/eu8ic3O+s+vs6+KVDMVJiISVjZnF/DunLW89eNa1mzNo1ZMFMO7NuGMXs0Z2rlRZMyyWt2y1sGbF0JiI7jwjWqZklykIipMRCTkOeeYmZbJuB/X8OWijRSVOAa0a8jtwztxYremJNbSn7KDVpDtDQsu2umNwEls7HciqeH0aRaRkLU1t5Dxc9bx5o9rWLEll/q1Y7lsYFsuHtCKjo0rcVZa2beSYnjvKti0xJurpHFXvxOJqDARkdDinGPWqm2M+2E1ny3YSGFJKf3aJPHk8R05tUczHaqpSl/cA8u/hNP/DzoO8zuNCKDCRERCRFZeEeN/WscbP64hdVMOdWvFcNGRrbh4QBs6N1XrSJX74X/w4/9g0C3Q/yq/04jspsJERHzjnOOnNdt544c1fDx/AwXFpfRq1YC/j+zJ6b2akRCnP1FBsexL7yyynU+FEx7yO43IHvSpF5FqtyO/iAk/r2fcD2tYujGbOnHRjOzXkouPbE33FvX9jhfZNi6E966EJt3hnOchSofGJLSoMBGRajN/3XbGfb+Gj+ZtYGdRCd2a1+PRs3twZu/mGllTHbI3eiNwatWFi9/WGXwlJOkvgYgEVW5BMRPmbuCNH1ezcP0OasdGc2av5lw8oDU9W9bX9PDVpTDPm6tk51a46nOo19zvRCLlUmEiIkHxy8ZsXp25iglzN5BTUEyXpnV5aEQ3zurTgnrxsX7Hq1lKS+GD62DDXG8CtWa9/E4kUiEVJiJSpX5es41nv0pj8pIMasVEcXpPr3Wkb+sGah3xy5S/wJKJcNKj0OVUv9OI7JMKExE5ZM45vkvL5NmvUvkuLZMGCbHcMfwwLh/UhqQ6cX7Hqz7OQVEe5O+A/Czvp6DM9b2XOQeJTaBuE0hsGrgMXE9oCFVRyP30Knz7lDckeOBNh/54IkGmwkREDlppqWPK0k08+1Uqc9dup3HdWtx7alcuHtCaOuHembUwD9bP2auwCFwvyCpnWeCytHjfjxsVC/H1vR8c5GyCwpzyt0tsHChcmgauN/21iNlV0NRpDDEVFH8rpsPHd0CH4+GUv1dNoSMSZGH+l0NE/FBcUsonC9L5z1dp/JKRTauGtfnr2d0Z2bdlZMzMWpQPL58M6fN+uy4uEWrV+7W4SGwMKZ32XBZf5nqtvZbFxP+2QCjIgZwM7yd7o1es5GyE7MCy7Wtg7Y+Qt6X8vLUbBoqXMkVMQjJ8/U9I7gjnvQLR6tcj4UGFiYhUWkFxCePnrOe56Wms2ZrHYU0SeeqC3pzesxkx0VF+x6s6n9/lFSWn/x8071umyKgH0UH4s1kr0ftJ7rDv7UqKIHdz+cXLrqImM81bXlLotaxc/E6gdUYkPKgwEZH9yi0o5s0f1/D81yvI2FFAr5b1ue+0fgzv2oSoqAg7PDD3TZjzChx9R+hN1R4d6w3z3d9QX+dg5zaITYDY+OrJJlJFVJiISIWy8ooYO3MVL3+7km15RQxqn8w/z+vN4I7JkTnCJmOR1yejzdFw3H1+pzl4Zl7nWZEwpMJERH5jU3Y+L36zktdnria3sIThXRtz49CO9GuT5He04CnIhncu9/qCnPtScA7ZiMh+6ZMnIrut3ZrHmBkreHv2WopLSjm9Z3NuHNqBrs3q+R0tuJyDj26FrStg1ERvtIuI+CJohYmZdQbeLrOoPXA/8GpgeVtgFXC+c25bsHKIyP6lbsrmP9PSmDB3A1EG5/ZryfVDOtA2pY7f0arHj2Ng0Qcw/EFoe7TfaURqtKAVJs65X4DeAGYWDawHPgBGA1Occ4+b2ejA7buClUNEKrZgXRb/mZbK54s2UismilGD2nLtkHY0q1/b72jVZ+0s+OJeOOwUOOo2v9OI1HjVdShnGJDmnFttZiOAoYHlY4FpqDARqTbOOb5fsZX/Tk9jxrLN1I2P4ZbjOnLFUW1JTqzld7zqlZsJ714B9ZrB2f+FqAga8iwSpqqrMLkQeDNwvYlzLh3AOZduZo3Lu4OZXQdcB9C6detqCSkSyUpLHZOWZPDfaWnMXbudlMRa3HlyZy4b2Ia6NfGkeqWl8P61kLsJrv4Sakdwx16RMBL0wsTM4oAzgbsP5H7OuTHAGID+/fu7IEQTqREKi0uZMNebFC1tcy6tGybwyFndObdfhMzSerC+fgLSpsDpT0HzPn6nEZGA6mgxOQX4yTmXEbidYWbNAq0lzYBN1ZBBpMbJLSjmrVlreeHrFaRn5dO1WT2evqgPp3ZvGlmztB6MtKnw1aPQ80Lod4XfaUSkjOooTC7i18M4AB8Bo4DHA5cTqiGDSI2xNbeQsd+tYuzMVWzPK2JAu4Y8dk4Pjj2sUWROinagstbD+GugURc4/Umd2E4kxAS1MDGzBOAE4Poyix8H3jGzq4E1wHnBzCBSU6zfvpMXvl7BWz+uZWdRCScc3oQbju0Q2ZOiHaiSIq+za3EBXPAaxNWQ4dAiYSSohYlzLg9I3mtZJt4oHRGpAsszsnlu+gomzF0PwIjeLbjh2PZ0alLX52QhaNIDsO5Hb2bXlE5+pxGRcmjmV5Ew9dOabfx3WhqTFmdQOzaaywa14Zpj2tOiQQ2ag+RALJ4A3z8LR14P3Uf6nUZEKqDCRCSMOOeYvmwz/52Wxg8rt9IgIZbbhnVi1FFtaVgnzu94oSszDT68GVr0hxMf8TuNiOyDChORMFBcUsqnCzfy32lpLEnfQbP68fz59MO58IhW1Kmlj/E+FeZ5J+eLjoHzXoEYFXAioUx/0URCWH5RCe/NWceYGStYszWPDo3q8I9zezKidwviYmr4kN/K+vRPkLEILnkPGrTyO42I7IcKE5EQVFLqeHXmKp79Ko0tOQX0atWAe0/rygldmxAVpeGtlfbTazD3dRhyJ3Qa7ncaEakEFSYiIWbVllz+9N48Zq3axuCOyfz7uD4MbN9Qc5AcqPT58Okfod2xMHS032lEpJJUmIiEiNJSx2vfr+bxz5YSE208eX4vzu7TQgXJwcjP8vqV1E6CkS9CVA2eel8kzKgwEQkBa7fmcdf4+XyXlsmxhzXibyN70rR+vN+xwpNz8OFNsH0NXPkpJDbyO5GIHAAVJiI+cs7x1qy1PPLxYsyMx8/pwQVHtFIryaGY+Sws/RhO/Cu0Huh3GhE5QCpMRHySnrWT0eMXMH3ZZo7qkMzfz+1Jy6QEv2OFt9UzYdL90PUMGHSz32lE5CCoMBGpZs453v9pPQ9OXERxieOhEd24dEAbjbY5VDmb4b0rIakNjHhWJ+cTCVMqTESq0absfO55fyGTl2RwRNsk/nFuL9qm6ERyh6y0BMZfDTu3wSXvQnx9vxOJyEFSYSJSTSbO28CfJyxkZ2EJ953WlSsHtyNarSRVY9rjsHK611LStIffaUTkEKgwEQmyzJwC7p+wiE8WpNO7VQOeOK8XHRsn+h0rciyfBDP+Dn0u9X5EJKypMBEJos8XbuS+DxewY2cxd57cmeuOaU9MtKaSrzLb18L710KT7nDqE36nEZEqoMJEJAiy8op44KOFfDh3A92a12PcNb3p3LSu37EiS3EhvDvK619y/qsQW9vvRCJSBVSYiFSxr5Zu4q7x89maW8jtwztx83EdiVUrSdWb+jCsnwPnvwbJHfxOIyJVRIWJSBXZkV/EIx8v5p3Z6+jcpC4vXXEE3VtodEhQbF8LPzzn9Sk5/Ey/04hIFVJhIlIFvl6+mbvem8/GHfncNLQDtw3vRK0YnZ8laL4O9Cc5VifnE4k0KkxEDkFuQTGPfrqEcT+soX2jOoy/8Sj6tE7yO1Zk27oSfn4d+l0JDVr5nUZEqpgKE5GDNGvVVn7/zlzWbdvJtce04w8ndiY+Vq0kQTfjHxAVA8f8we8kIhIEKkxEDsLbs9Zw7wcLaZFUm3euH8QRbRv6Halm2JIK896EATdCvWZ+pxGRIFBhInIASksdf//iF56bnsYxnVJ49pK+1IuP9TtWzTH9cYiJh6Pv8DuJiASJChORStpZWMLv35nLZws3cvGA1vzlzG4aBlydNi2BBe/B4NsgsZHfaUQkSFSYiFTCpux8rn11DvPXbefeU7tyzTHtMJ29tnpNewziEr3CREQilgoTkf1YlpHNlS/PYmtuIc9d2o+TujX1O1LNkz4fFk+AIXdCgvrziEQyFSYi+zBj2WZuHvcT8XHRvHP9IHq01IRpvpj2GMTXh0E3+51ERIJMB8hFKjDuh9Vc+cosWiTVZsLNg1WU+GX9T/DLpzDoVqjdwO80IhJkajER2UtJqePxz5bw/NcrGdq5Ec9c3JfEWvqo+OarR6F2Qxh4g99JRKQa6K+tSBl5hcXc/tZcvlycweWD2nD/6YcTo5E3/ln7I6ROguF/gVo6O7NITaDCRCQgY0c+14ydzcINWTxwxuFcObid35Fk6iNQpxEcea3fSUSkmqgwEQGWpO/gqldmkbWziOcv68/ww5v4HUlWfQMrp8NJj0FcHb/TiEg1UWEiNd5Xv2zilnE/kRgfwzvXD6J7C3Vy9Z1zMPWvULcZ9L/S7zQiUo1UmEiN9urMVTz40SK6NqvHi6OOoGn9eL8jCcCKr2DNd3DqExBb2+80IlKNVJhIjVRS6njkk8W8/O0qhndtzL8u7EMdjbwJDbtaS+q1hL6X+51GRKqZ/hJLjZNbUMxtb/3M5CWbuGpwO+49rSvRUZpePmQs/xLWz4Yz/gUxtfxOIyLVTIWJ1Cgbs/K5euwslqTv4OER3bhsUFu/I0lZzsFXf4WkttD7Er/TiIgPVJhIjbFoQxZXvzKb7PwiXrziCI7r3NjvSLK3pR9D+jw4678QHet3GhHxQVBnjjKzBmb2npktNbMlZjbIzBqa2SQzWx64TApmBhGAKUsyOO+5mUQZvHfjUSpKQlFpqTfLa3JH6HG+32lExCfBntLyX8DnzrkuQC9gCTAamOKc6wRMCdwWCZqXv13Jta/OpkOjRD68eTBdm9XzO5KUZ/EHsGkxDL0botWYK1JTBe3Tb2b1gCHAFQDOuUKg0MxGAEMDm40FpgF3BSuH1FzFJaU8/PFixs5czYmHN+GpC3uTEKcvvJBUWgLTHodGXaHbOX6nEREfBfOvdHtgM/CymfUC5gC3AU2cc+kAzrl0Myu3Td3MrgOuA2jdunUQY0okKil13DTuJ75cnMF1Q9oz+uQuRGnkTeha8C5sWQbnvwpROjeRSE0WzL8AMUBf4L/OuT5ALgdw2MY5N8Y51985179Ro0bByigR6q+fLOHLxRn8+fTDuefUripKQllJkdda0rQHdDnD7zQi4rNgFibrgHXOuR8Ct9/DK1QyzKwZQOByUxAzSA007ofVvPTtSq4a3I6rj9aJ+ELevDdh20o47l61lohI8AoT59xGYK2ZdQ4sGgYsBj4CRgWWjQImBCuD1DzfLN/C/RMWcXyXxtx7Wle/48j+FBfC9H9Ai35w2Ml+pxGREBDsnoC3AuPMLA5YAVyJVwy9Y2ZXA2uA84KcQWqI1E053DhuDh0bJfKvC3trNtdw8POrkLUGzvg/ML1fIhLkwsQ5NxfoX86qYcF8Xql5tuUWcvXYWdSKieKFUf2pG6/JuUJeUT7M+Ce0Gggd9CdBRDwaOylhr7C4lOtfn0N6Vj5vXjuQVg0T/I4klTHnZcjeAOf8T60lIrKbeppJWHPOce8HC/hx5Vb+cW5P+rXRRMJhoTAPvn4S2h4D7Yb4nUZEQohaTCSsjZmxgnfnrOO2YZ0Y0buF33GksmY9D7mb4ILX/E4iIiFGLSYStr5YtJHHP1/KGb2ac/vwTn7HkcoqyIZvnvL6lbQe6HcaEQkxKkwkLC1cn8Xtb82lV8sG/OPcnpj6KISPH56DnVu9eUtERPaiwkTCTsaOfK4ZO5ukhFjGXN6P+NhovyNJZe3cDt/9Gw47BVr28zuNiIQg9TGRsLKzsIRrxs4mO7+I9248isZ14/2OJAfi+/9AfhYcd4/fSUQkRKkwkbBRWur4w7tzWbghixcu70/XZvX8jiQHIm8rzPwPdD0TmvX0O42IhCgdypGw8eSkZXy6YCP3ntqVYV2b+B1HDtR3T0NhjlpLRGSfVJhIWHj/p3U881UqFx3ZSifmC0c5m+GH/0H3kdBY5zASkYqpMJGQN2vVVkaPX8BRHZJ5aER3jcAJR98+BcX5MHS030lEJMSpMJGQtiYzj+tfm0PLpNr895J+xEbrVzbs7EiHWS9AzwshRfPNiMi+6a+8hKwd+UVcNXYWJaWOF684gvoJOjFfWPrmSSgthmPv9DuJiIQBFSYSkopLSrnljZ9ZtSWX5y7tR7uUOn5HkoOxfS3MeQV6XwIN1TdIRPZPw4UlJD388WJmLNvM30b2YFCHZL/jyMH6+glwDob80e8kIhIm1GIiIWfsd6sYO3M11w1pzwVHtPY7jhysrSvh59eh3yhooPdRRCpHhYmElGm/bOIvExcxvGsT7jq5i99x5GBtXQkTbgaLhmP+4HcaEQkjOpQjIWNZRja3vvEznZvW418X9iY6SsOCw05BjtfZ9btnICoGTnsC6jX3O5WIhBEVJhISMnMKuHrsLOLjonlxVH/q1NKvZlhxDha8B5Puh+wN0ON8OOEvKkpE5IDpr7/4rqC4hOtfm8OmHQW8c/0gmjeo7XckORDp8+Czu2DNTGjWC857GVoP9DuViIQpFSbiK+ccd49fwOzV23j24r70atXA70hSWblbYOrDMGcsJCTDGU9Dn0shKtrvZCISxlSYiK/+My2N939ezx9OOIzTejbzO45URkkRzHoRpj3q9SkZeCMcexfUbuB3MhGJACpMxDefLkjnH1/8wlm9m3PL8R39jiOVsWIafDYaNi+B9kPh5L9BY42eEpGqo8JEfJGetZM735tP39YNeHxkT52YL9RtWw1f3gtLJkKDNnDBOOhyGuh9E5EqpsJEqp1zjj9/uJCSUse/LuxDfKz6JISswjz45v/gu6fBouD4P8OgWyA23u9kIhKhVJhItft0wUYmL9nEfad1pVXDBL/jSHmcg0Xvw5f3w4510P1cOOEhqN/C72QiEuFUmEi12p5XyAMfLaRny/pccVRbv+NIeTYu8Ib/rv4WmvaAkc9Dm6P8TiUiNYQKE6lWj366hG15Rbx61QBionVGhJCStxWmPgJzXob4BnD6U9D3cg3/FZFqpcJEqs13qVt4Z/Y6bhzagcOb1/M7juxSUuwVI1MfgYJsOPI6GDoaaif5nUxEaiAVJlIt8otKuPuDBbRNTuC2YZ38jiO7rPzaO2yzaRG0OxZO+Rs07up3KhGpwQ6oMDGzeCDOObcjSHkkQj01eTmrM/N449oBGoXjJ+dgyzJYPgmWfQ6rvoYGreH816DrGRr+KyK+q3RhYmbXAJcBUWb2tXPunuDFkkiycH0Wz3+9gguPaMVRHVL8jlPzFOTAyhmQOgmWT4asNd7yRl0Cw39vhlidn0hEQkOFhYmZneGcm1hm0XDn3LGBdfMAFSayX8UlpYx+fz4N68Rx9yk6RFAtnIPNvwQKkUneyfVKCiEu0Ttcc8wd0PEEaNDK76QiIr+xrxaTXoFWkvudc/OA+WY2DnDAompJJ2HvpW9XsnD9Dv5zSV/qJ8T6HSdyFeTAyuleIZI6GbLWessbdYUB13uFSOtBEBPnb04Rkf2osDBxzj1iZk2BhwLThd8PJAIJzrn51ZRPwtiazDyenLSMEw5vwindm/odJ7I4B5uXBgqRSbB6JpQWea0i7YfCMX+AjsPVKiIiYWd/fUxygduBTsAYYBbwjyBnkgjgnOOeDxYQGxXFwyO661w4VaEgG1ZM9wqR1Cm/too0Ptw7w2+nE6DVQLWKiEhY21cfk0eAIUAs8LZz7kwzOxP4xMxecc69Vl0hJfyM/2k936Ru4ZGzutO0vs6rclCcg01LyvQV+T7QKlIX2h8LQ/7otYrUb+l3UhGRKrOvFpPTnXO9zftXdw7wlHPuIzP7FLi5euJJONqcXcDDHy/miLZJXHxka7/jhKeCbHjjAm9aeIDG3WDQTV5fkVYD1CoiIhFrX4XJQjN7DagNTN+10DlXDPyrMg9uZquAbKAEKHbO9TezhsDbQFtgFXC+c27bwYSX0PTQx4vZWVjCY+f0ICpKh3AOWEEOjDsP1v4IJz0Kh5+lk+eJSI2xr86vl5pZD6DIObf0EJ7jOOfcljK3RwNTnHOPm9nowO27DuHxJYRMXZrBxHkb+P0Jh9GxcV2/44Sfwjx480JY+wOMfBG6n+N3IhGRarXPs6g55xYcYlFSnhHA2MD1scBZVfz44pOcgmLu+2AhhzVJ5IZjO/gdJ/wU7fSKktXfwtljVJSISI0U7NO7OuBLM5tjZtcFljVxzqUDBC4bl3dHM7vOzGab2ezNmzcHOaZUhSe++IX0Hfk8PrIncTE6c/ABKcqHty72Zmgd8R/oeZ7fiUREfBHsk/gNds5tMLPGwCQzq3Tri3NuDN4QZfr37++CFVCqxpzV2xg7cxWjBrWlb2udlfaAFBfAO5dB2lQ48xnofZHfiUREfLPff2vNbLyZnWZmB/wvsHNuQ+ByE/ABcCSQYWbNAo/dDNh0oI8roaWwuJS7359Ps3rx/PGkzn7HCS/FhfDuFbD8Szj9Keh7md+JRER8VZli47/AxcByM3vczLpU5oHNrI6Z1d11HTgRWAh8BIwKbDYKmHDAqSWkPDc9jWUZOTxydncSawW7ES6ClBTBe1fCL5/CqU9A/yv9TiQi4rv9fos45yYDk82sPnAR3iGZtcDzwOvOuaIK7toE+CAw42cM8IZz7nMzmwW8Y2ZXA2sAHUwPY6mbsnlmaipn9mrO8V2a+B0nfJQUw/hrYOnHcPLf4Mhr/U4kIhISKvXvrZklA5cClwE/A+OAo/FaPIaWdx/n3AqgVznLM4FhBxdXQklpqWP0+AUk1Irm/jMO9ztO+CgtgQ9vgMUfwomPwMAb/E4kIhIy9luYmNn7QBfgNeCMXSNqgLfNbHYww0loG/fjGmav3sYT5/UiJbGW33HCQ2kJfHgTLHgXhj8IR93qdyIRkZBSmRaTZ5xzU8tb4ZzrX8V5JEykZ+3kb58t5eiOKYzsq1lJK6W0FD76Hcx/C467D46+w+9EIiIhpzKdX7uaWYNdN8wsycxuCl4kCXXOOf784SKKS0t59OweOnNwZZSWwid3wNzX4di74Ng/+Z1IRCQkVaYwudY5t33XjcB5bdRTrwb7bOFGJi/J4A8ndKZ1coLfcUKfc/DZn2DOK3DMH2Do3X4nEhEJWZUpTKKszL/EZhYN6NSmNVRWXhH3T1hEjxb1uXJwW7/jhD7n4PPRMOsFOOp3cPyfQS1MIiIVqkwfky/whvc+hzfF/A3A50FNJSHr0U+XsC2vkFeuPIKYaE07v0/OwZf3wQ/PwcCb4ISHVJSIiOxHZQqTu4DrgRsBA74EXghmKAlN36Vu4e3Za7nh2A50b1Hf7zihzTmY/CDMfAaOvA5OelRFiYhIJVRmgrVSvNlf/xv8OBKq8otKuPuDBbRJTuD24Z38jhP6vvorfPsU9L8KTvm7ihIRkUqqzDwmnYDHgMOB+F3LnXPtg5hLQsy/pixndWYeb1w7gPjYaL/jhLZpf4MZ/4C+l8Op/1RRIiJyACrTSeBlvNaSYuA44FW8ydakhli0IYsxM1Zwfv+WHNUhxe84oe3rf8K0R6HXxXD6vyBK/XBERA5EZf5q1nbOTQHMObfaOfcgcHxwY0moKC4pZfT4BSQlxHHPqV39jhPavv0XTHkIepwPI55RUSIichAq0/k138yi8M4ufAuwHmgc3FgSKl7+dhUL1mfx7MV9aZCgUeIVmvksTLofuo+Es/4LUTrcJSJyMCrzL93tQALwO6Af3sn8RgUxk4SINZl5/HPSLwzv2oRTezT1O07o+mEMfHEPdD0Tzh4D0ZU6N6aIiJRjn39BA5Opne+c+xOQA1xZLanEd8457v1wATFRUTx8VjdNO1+RWS96s7p2OR3OfUlFiYjIIdpni4lzrgToZ/pWqnHenb2Or5dv4a6TO9Osfm2/44Se0lL4/jn45Pdw2Mlw7ssQHet3KhGRsFeZf+9+BiaY2btA7q6Fzrn3g5ZKfLUxK5+HP1nMgHYNuWRAG7/jhJ61s+CzO2HDT9DpJDj/VYhR/xsRkapQmcKkIZDJniNxHKDCJAI557j3gwUUlZTyt5E9iYpSY9luOzZ4s7nOfxvqNvP6k/Q4T6NvRESqUGVmflW/khpkwtwNTFm6iftO60rblDp+xwkNRTu9qeW/fhJKS+CYP8LRd0CtRL+TiYhEnMrM/PoyXgvJHpxzVwUlkfhmc3YBD05cRN/WDbhycDu/4/jPOVjykXcivu1rvFE3Jz4MSW39TiYiErEqcyjn4zLX44GzgQ3BiSN+un/CQvIKS/j7ub2IrumHcDYugM/vhlVfQ+NuMGoitBvidyoRkYhXmUM548veNrM3gclBSyS++HRBOp8t3MidJ3emY+MafIgiNxO+egTmvALxDeC0J6HvKA0DFhGpJgfz17YT0Lqqg4h/tuYWcv+EhfRoUZ/rjqmh52YsKYJZL8C0x6AgB468HobeBbWT/E4mIlKjVKaPSTZ79jHZCNwVtERS7R6auIisnUW8fs0AYqJr4AiT1MneYZsty6DD8XDSY9C4i9+pRERqpMocyqlbHUHEH5MXZ/Dh3A3cPrwTXZrW8ztO9dqSCl/eC8s+h4bt4aK34bCTQPMJioj4pjItJmcDU51zWYHbDYChzrkPgxtNgi1rZxH3fLCALk3rctPQjn7HqT75WTDjH97MrTHxcMLDMOB6iKnldzIRkRqvMn1MHnDOfbDrhnNuu5k9AHwYtFRSLf76yWIycwt5cdQRxMXUgEM4pSUwdxxMeQhyt0CfS2DYA5Cok2WLiISKyhQm5X1jaYhCmJuxbDPvzF7HTUM70KNlfb/jBN/qmfD5XZA+D1oNgIvfgRZ9/U4lIiJ7qUyBMdvMngSexesEeyswJ6ipJKhyCoq5+/0FdGhUh98N6+R3nODavhYmPwALx0O9FjDyReg+Uv1IRERCVGUKk1uBPwNvB25/CdwXtEQSdI9/toQNWTt574ajiI+N9jtO8Mx+CT6/B3Bw7GgY/DuI0zT7IiKhrDKjcnKB0dWQRarBzLRMXv9+DVcf3Y5+bSJ4jo7iQvjyfmjeG84ZAw009Y6ISDjYb49HM5sUGImz63aSmX0R1FQSFHmFxdw1fj5tkhP444md/Y4TXOt+hMJsGHSzihIRkTBSmaEYKc657btuOOe2ARrGEIae+GIZa7bm8beRPakdF8GHcABSp0BUjM5vIyISZipTmJSa2e5/Oc2sDeWcbVhC25zVW3n5u5VcNrANA9sn+x0n+FIne6Nv4mvAiCMRkQhSmc6v9wLfmNn0wO0hwHXBiyRVLb+ohD+9N5/m9Wtz1yk1YKr1nE2wcT4c/2e/k4iIyAGqTOfXz82sLzAQMOAO59yWoCeTKvOvKctZsTmXV686ksRaNWAKmrSp3mXH4f7mEBGRA1bZb6kSYBMQDxxuZjjnZgQvllSV+eu2M2bGCi7o34ohhzXyO071SJ0MdRpB055+JxERkQNUmXPlXAPcBrQE5uK1nMwEjg9qMjlkhcWl/Ond+aQkxnHPaV39jlM9Sku8jq+dToSoGjDNvohIhKnMX+7bgCOA1c6544A+wOagppIq8exXqfySkc2jZ/egfu1Yv+NUj/S5sHMrdBzmdxIRETkIlSlM8p1z+QBmVss5txSI8Ekwwt+S9B08+1UqZ/VuzrCuTfyOU31SpwIGHdSgJyISjipTmKwLTLD2ITDJzCYAGyr7BGYWbWY/m9nHgdsNA5O2LQ9cRvD0o/4oKinlT+/No0FCLA+c0c3vONUrdbI322udFL+TiIjIQdhvYeKcO9s5t9059yDeOXNeBM46gOe4DVhS5vZoYIpzrhMwBU13X+XGzFjBwvU7eHhEd5LqxPkdp/rs3ObN+NpBh3FERMLVAfUOdM5Nd8595JwrrMz2ZtYSOA14ocziEcDYwPWxHFiRI/uRuimbf01ezqk9mnJKj2Z+x6leK6aDK9UwYRGRMBbsYQtPAXcCpWWWNXHOpQMELsud3t7MrjOz2WY2e/Nm9bWtjJJSx5/em09CrWj+cmZ3v+NUv7QpUKs+tDzC7yQiInKQglaYmNnpwCbn3JyDub9zboxzrr9zrn+jRjVk/o1D9PK3K/l5zXYePKMbjerW8jtO9XLOGybc/liIrgGTyImIRKhgtpgMBs40s1XAW8DxZvY6kGFmzQACl5uCmKHGWLkll3988QvDuzZmRO/mfsepfpuXwo71GiYsIhLmglaYOOfuds61dM61BS4EpjrnLgU+AkYFNhsFTAhWhpqitNRx1/j5xMVE8chZPTAzvyNVv9Qp3qU6voqIhDU/psZ8HDjBzJYDJwRuyyEY98Nqfly5lT+fdjhN68f7HccfqZOhURdo0MrvJCIicgiq5WC8c24aMC1wPRPQv7VVZO3WPB77bCnHdErhvP4t/Y7jj8JcWP0tHKmTXouIhDudTCSMOee4+/0FGPDYOTX0EA7Aqm+hpFCzvYqIRAAVJmHsndlr+SZ1C6NP7UrLpAS/4/gnbQrE1IY2g/1OIiIih0iFSZjakV/Eo58uZUC7hlxyZGu/4/grdTK0PRpia2j/GhGRCKLCJEy9+t0qsnYWcd9phxMVVUMP4QBsWwWZqZrtVUQkQqgwCUM5BcW88M1Kju/SmB4t6/sdx1+7hglr/hIRkYigwiQMvf79arbnFXHr8R39juK/1CnQoDUk67UQEYkEKkzCTF5hMc/PWMExnVLo0zrJ7zj+Ki6EldO9wzg1dUSSiEiEUWESZt74YQ2ZuYXcNqyT31H8t/YHKMzRbK8iIhFEhUkYyS8q4X8zVjCofTL92zb0O47/0qZAVAy0G+J3EhERqSIqTMLIWz+uYXN2Ab9Ta4kndTK0Ggjx9fxOIiIiVUSFSZgoKC7huekrOLJtQwa2V2sJ2Rth4wKNxhERiTAqTMLEu7PXsXFHPr8b1qnmTj1fVtpU71KFiYhIRFFhEgYKi0v577Q0+rRuwOCOyX7HCQ2pU6BOY2jSw+8kIiJShVSYhIH3f1rH+u071VqyS2mJ12LScRhE6VdYRCSS6K96iCsqKeXZaan0bFmfoYc18jtOaNgwF3Zu1TT0IiIRSIVJiJswdwNrt+7k1uPVWrJb2hTAoP1xficREZEqpsIkhJWUOp79KpWuzeoxvGtjv+OEjtTJ0LwP1FF/GxGRSKPCJIR9PH8DK7fkctuwjmot2WXnNlg3S4dxREQilAqTEFVS6vj31FQ6N6nLiYc39TtO6FgxDVypChMRkQilwiREfbYwndRNOdxyfEeiotRaslvqFKhVH1r08zuJiIgEgQqTEFRa6nhmaiodGtXh1B7N/I4TOpzzCpMOQyE6xu80IiISBCpMQtCXizNYujGbW4/vRLRaS361aQlkb9BhHBGRCKbCJMQ45/j31OW0TU7g9J5qLdlD6mTvsoOmoRcRiVQqTELM1KWbWLRhBzcf15GYaL09e0ibAo26Qv0WficREZEg0TdfCHHO8fSU5bRqWJuz+ujLdw+FubD6O520T0QkwqkwCSHTl21m3rosbhrakVi1luxp1TdQUqj+JSIiEU7ffiFiV2tJ8/rxjOzb0u84oSd1CsTUhtaD/E4iIiJBpMIkRHyXlslPa7Zz43EdiYvR2/IbqZOh3TEQG+93EhERCSJ9A4aIf01ZTpN6tTivn1pLfmPrCtiapsM4IiI1gAqTEPD9ikx+XLmVG47tQHxstN9xQk/qFO9ShYmISMRTYRIC/j11OSmJtbjoyNZ+RwlNaVOhQRto2N7vJCIiEmQqTHw2e9VWvk3N5IZj26u1pDzFhbBiutdaojMsi4hEPBUmPnt6aioN68Rx8QC1lpRr7fdQlKvDOCIiNYQKEx/NXbudGcs2c+0x7UmI00npypU6BaJivRE5IiIS8VSY+OjfU5bTICGWywa18TtK6EqdAq0HQq26ficREZFqoMLEJwvXZzFl6SauHtyOxFpqLSnXjnTIWKBp6EVEahAVJj7599Tl1I2PYdTgtn5HCV1pU71L9S8REakxVJj4YEn6Dr5YlMFVg9tRLz7W7zihK20KJDaBJt39TiIiItUkaIWJmcWb2Y9mNs/MFpnZXwLLG5rZJDNbHrhMClaGUPXM1FQSa8Vw1eB2fkcJXaUlXotJh2EaJiwiUoMEs8WkADjeOdcL6A2cbGYDgdHAFOdcJ2BK4HaNsTwjm08XpjPqqDbUT1BrSYU2/Aw7t6l/iYhIDRO0wsR5cgI3YwM/DhgBjA0sHwucFawMoeiZr1KpHRvN1UdrFtN9Sp0MGHQ43u8kIiJSjYLax8TMos1sLrAJmOSc+wFo4pxLBwhcNq7gvteZ2Wwzm7158+Zgxqw2aZtzmDhvA5cNakPDOnF+xwltqVOgRV9IaOh3EhERqUZBLUyccyXOud5AS+BIM6t0L0bn3BjnXH/nXP9GjRoFLWN1evarVOJiorj2GLWW7FPeVlg/W6NxRERqoGoZleOc2w5MA04GMsysGUDgclN1ZPDb6sxcJszdwCUD2pCSWMvvOKFtxTRwpSpMRERqoGCOymlkZg0C12sDw4GlwEfAqMBmo4AJwcoQSv7zVRrRUcb1Q9Rasl9pUyC+ATTv63cSERGpZsGccrQZMNbMovEKoHeccx+b2UzgHTO7GlgDnBfEDCFh7dY8xv+0jksGtKZxvXi/44Q257z+Je2HQrRmxBURqWmC9pffOTcf6FPO8kygRo0BfW56GlFm3DC0g99RQt+mxZCdrsM4IiI1lGZ+DbIN23fyzuy1nNe/Jc3q1/Y7TuhLnexdav4SEZEaSYVJkP1vehrOwY1qLamc1CnQuBvUa+53EhER8YEKkyDatCOfN2etZWTflrRMSvA7TugryIE1M6GjJlUTEampVJgE0f9mrKCk1HHTcWotqZRV30BJofqXiIjUYCpMgmTt1jxe/341Z/VuQZvkOn7HCQ+pkyE2AVoP8juJiIj4RIVJkDz40SJioow/ndTZ7yjhI20KtBsCMZqATkSkplJhEgSTFmcwZekmbh9+GE3ra96SSslMg60roING44iI1GQqTKrYzsISHvxoEYc1SeSKwW39jhM+0qZ6lxomLCJSo2lqzSr2zFfLWb99J+9cP4jYaNV9lZY6BZLaQbI6CouI1GT65qxCaZtzGDNjBef0bcGR7Rr6HSd8FBfAyhlqLRERERUmVcU5xwMTFhEfG83dp3T1O054WfM9FOVqmLCIiKgwqSofz0/nm9Qt/OmkzjSqq1ElByR1MkTFQttj/E4iIiI+U2FSBXIKinnkk8V0b1GPSwa08TtO+EmbCm0GQa1Ev5OIiIjPVJhUgacmLWNTdgGPnNWD6CjzO0542ZEOGQs1TFhERAAVJods6cYdvPzdKi48ojW9WzXwO074SZviXap/iYiIoMLkkDjn+POHC6lfO5Y7NcPrwUmdAolNoUk3v5OIiEgIUGFyCMb/tJ5Zq7Yx+uQuJNWJ8ztO+Ckt8fqXdBwOpkNgIiKiwuSgZeUV8dinS+jbugHn9mvpd5zwtP4nyN8OHY/3O4mIiIQIzfx6kP7x5VK25RXy6tVHEqUOrwcndTJYFLQ/zu8kIiISItRichDmr9vOuB/WMOqotnRrXt/vOOGpuBCWTIQW/SBBs+SKiIhHhckBKil13PfhQlISa3HHCYf5HSc8OQcf3wGbFsHAG/1OIyIiIUSFyQF688c1zF+XxX2ndaVefKzfccLT1/+Eua/DsaOh+0i/04iISAhRYXIAtuQU8I8vfmFQ+2TO7NXc7zjhaeF4mPow9Dgfho72O42IiIQYFSYH4PHPlpJXWMzDZ3XDNLz1wK35AT64EVoPghHPaIiwiIj8hgqTSpq9aivvzVnH1Ue3p2Pjun7HCT9bV8BbF0H9FnDhGxCjEx2KiMhvqTCphOKSUu77cCHN68fzu2Ed/Y4TfnZug3HngyuFS97TKBwREamQ5jGphFe+W8XSjdk8d2k/EuL0kh2Q4kJ4+zLYtgounwDJHfxOJCIiIUzfsvuRsSOfpyYvZ2jnRpzUrYnfccKLc/Dx7bDqazh7DLQd7HciEREJcTqUsx+PfLKEwpJS/nKmOrwesK+fgLnjvGHBvS7wO42IiIQBFSb78G3qFibO28BNQzvQJrmO33HCy4L3YOoj0PMCDQsWEZFKU2FSgYLiEv48YSFtkhO44Vj1izgga36AD2+C1kfBmf/WsGAREak09TGpwAtfr2TF5lxevvII4mOj/Y4TPnYPC24JF47TsGARETkgajEpx9qtefx76nJO7taU4zo39jtO+NhjWPC7GhYsIiIHTC0m5Xjo48UYxv1nHO53lPCxa1jw9tUaFiwiIgdNLSZ7mbIkg0mLM/jdsE40b1Db7zjhwTmYeJs3LPjMZ6DNUX4nEhGRMKXCpIz8ohIenLiIjo0Tufrodn7HCR8znoB5b8DQuzUsWEREDokO5ZTxn69SWbt1J29cO4C4GNVslbLgPfgqMCz42Lv8TiMiImFO374BK7fk8tz0FYzo3ZyjOqT4HSc8rPkePrwR2gzWsGAREakSQStMzKyVmX1lZkvMbJGZ3RZY3tDMJpnZ8sBlUrAyVJZzjvsnLKRWTBT3ntrV7zjhITMN3rwI6reCC17XsGAREakSwWwxKQb+4JzrCgwEbjazw4HRwBTnXCdgSuC2rz5buJGvl2/h9yceRuN68X7HCX15W+GN8wGnYcEiIlKlglaYOOfSnXM/Ba5nA0uAFsAIYGxgs7HAWcHKUBk5BcU8NHExhzerx2UD2/gZJTzsHha8Bi58Q8OCRUSkSlVLHxMzawv0AX4Amjjn0sErXoByZzAzs+vMbLaZzd68eXPQsj09ZTkbd+Tz8FndiYlWl5t9cg4m/g5WfwMjntWwYBERqXJB/yY2s0RgPHC7c25HZe/nnBvjnOvvnOvfqFGjoGRblpHNS9+s5IL+rejXxveuLqFvxhMw700Yeg/0PN/vNCIiEoGCWpiYWSxeUTLOOfd+YHGGmTULrG8GbApmhoo457jvw4Ukxsdw1yld/IgQXnYPC74Qjr3T7zQiIhKhgjkqx4AXgSXOuSfLrPoIGBW4PgqYEKwM+7I9r4jC4lLuPKkLDevE+REhfKyeWWZY8NMaFiwiIkFjzrngPLDZ0cDXwAKgNLD4Hrx+Ju8ArYE1wHnOua37eqz+/fu72bNnV3nG0lJv36Oi9EVbocw0eGE41E6CayZrBI6IiBwyM5vjnOtf3rqgzfzqnPsGqOgbf1iwnvdAqCDZj93DgtGwYBERqRaakl7KV1oK717hDQu+/CMNCxYRkWqhwkTKN+9NWDkdTv8/aDPI7zQiIlJDaOIO+a28rTDpz9BqAPS9wu80IiJSg6gwkd+a8hDs3A6nPQlR+hUREZHqo28d2dO62TDnFRhwAzTt7ncaERGpYVSYyK9KS+DjO6BuUxjq+7kVRUSkBlLnV/nVrBdh43w492WIr+d3GhERqYHUYiKe7AyY+jB0OB66ne13GhERqaFUmIjny3uhOB9OfUJTzouIiG9UmAismA4L3oWj79BEaiIi4isVJjVdcSF88gdIausVJiIiIj5S59eabua/IXM5XPIexNb2O42IiNRwajGpybathun/gK5nQKcT/E4jIiKiwqRG+3w0WBSc/LjfSURERAAVJjXX0k/hl09h6F1Qv6XfaURERAAVJjVTYR58dhc06gIDb/I7jYiIyG7q/FoTff0EZK2BKz6F6Fi/04iIiOymFpOaZvMy+PZp6HURtB3sdxoREZE9qDCpSZyDT/8AcQlwwsN+pxEREfkNHcqpSRaOh5Uz4LQnIbGR32lERER+Qy0mNUV+FnxxDzTvC/2u8DuNiIhIudRiUlNM/SvkbIKL34aoaL/TiIiIlEstJjXBhrkw63k44hpo3sfvNCIiIhVSYRLpSkvhk99DQjIcf5/faURERPZJh3Ii3U9jYf0cOHsM1G7gdxoREZF9UotJJMvdApMfhDZHQ8/z/U4jIiKyXypMItmkB6AwB077J5j5nUZERGS/VJhEqtUzYe7rMOgWaNzF7zQiIiKVosIkEpUUeR1e67eCY+/0O42IiEilqfNrJPrhf7BpMVz4BsTV8TuNiIhIpanFJNJkrYdpj8FhJ0PnU/1OIyIickBUmESaL+6B0mI45W/q8CoiImFHhUkkSZ0Miz+EIX+EpLZ+pxERETlgKkwiRVE+fPonSO4IR/3O7zQiIiIHRZ1fI8W3T8HWFXDZhxBTy+80IiIiB0UtJpEgMw2+fhK6j4QOx/mdRkRE5KCpMAl3znmHcKLj4MS/+p1GRETkkKgwCXdLPoK0KXD8vVCvmd9pREREDknQChMze8nMNpnZwjLLGprZJDNbHrhMCtbz1wgF2fDZaGjSA4641u80IiIihyyYnV9fAZ4BXi2zbDQwxTn3uJmNDty+K4gZwltJkVd8FOZAYS4U5ASuB24v/xKyN8D5YyFa/ZhFRCT8Be3bzDk3w8za7rV4BDA0cH0sMA0/C5NV3wIueI/vHBTne4VEQaCYKMwucz2nzLpyio+Swv0/x8CboNWRwdsHERGRalTd/2Y3cc6lAzjn0s2scTU//55eO6tyX/5VLTrOO4dNXF2olRi4ngiJTaBWXe96XJ3AusBP2eu719WFxEbVn19ERCRIQrb938yuA64DaN26dXCe5NL3CWqLCUBM7d8WGTFxwX1OERGRMFXdhUmGmTULtJY0AzZVtKFzbgwwBqB///7BqR7aHROUhxUREZGDU93DhT8CRgWujwImVPPzi4iISAgL5nDhN4GZQGczW2dmVwOPAyeY2XLghMBtERERESC4o3IuqmDVsGA9p4iIiIQ3zfwqIiIiIUOFiYiIiIQMFSYiIiISMlSYiIiISMhQYSIiIiIhQ4WJiIiIhAwVJiIiIhIyVJiIiIhIyFBhIiIiIiFDhYmIiIiEDBUmIiIiEjJUmIiIiEjIMOec3xn2y8w2A6v9zlFNUoAtfoeoRtrfyKb9jWza38gWzP1t45xrVN6KsChMahIzm+2c6+93juqi/Y1s2t/Ipv2NbH7trw7liIiISMhQYSIiIiIhQ4VJ6Bnjd4Bqpv2NbNrfyKb9jWy+7K/6mIiIiEjIUIuJiIiIhAwVJiIiIhIyVJj4wMxamdlXZrbEzBaZ2W3lbDPUzLLMbG7g534/slYVM1tlZgsC+zK7nPVmZk+bWaqZzTezvn7krApm1rnM+zbXzHaY2e17bRPW76+ZvWRmm8xsYZllDc1skpktD1wmVXDfk83sl8B7Pbr6Uh+8Cvb3H2a2NPD7+oGZNajgvvv83Q9FFezvg2a2vszv7KkV3DdS3t+3y+zrKjObW8F9w/H9Lfc7KGQ+w845/VTzD9AM6Bu4XhdYBhy+1zZDgY/9zlqF+7wKSNnH+lOBzwADBgI/+J25ivY7GtiIN5lQxLy/wBCgL7CwzLK/A6MD10cDf6vg9UgD2gNxwLy9f/dD8aeC/T0RiAlc/1t5+xtYt8/f/VD8qWB/HwT+uJ/7Rcz7u9f6fwL3R9D7W+53UKh8htVi4gPnXLpz7qfA9WxgCdDC31S+GwG86jzfAw3MrJnfoarAMCDNORdRMxc752YAW/daPAIYG7g+FjirnLseCaQ651Y45wqBtwL3C2nl7a9z7kvnXHHg5vdAy2oPFiQVvL+VETHv7y5mZsD5wJvVGiqI9vEdFBKfYRUmPjOztkAf4IdyVg8ys3lm9pmZdaveZFXOAV+a2Rwzu66c9S2AtWVuryMyirULqfgPWiS9vwBNnHPp4P3hAxqXs02kvs9X4bX4lWd/v/vh5JbAoauXKmjmj8T39xggwzm3vIL1Yf3+7vUdFBKfYRUmPjKzRGA8cLtzbsdeq3/Ca/7vBfwb+LCa41W1wc65vsApwM1mNmSv9VbOfcJ6LLuZxQFnAu+WszrS3t/KisT3+V6gGBhXwSb7+90PF/8FOgC9gXS8wxt7i7j3F7iIfbeWhO37u5/voArvVs6yKn2PVZj4xMxi8X4hxjnn3t97vXNuh3MuJ3D9UyDWzFKqOWaVcc5tCFxuAj7Aaw4sax3QqsztlsCG6kkXNKcAPznnMvZeEWnvb0DGrsNvgctN5WwTUe+zmY0CTgcucYED8HurxO9+WHDOZTjnSpxzpcDzlL8fkfb+xgDnAG9XtE24vr8VfAeFxGdYhYkPAscsXwSWOOeerGCbpoHtMLMj8d6rzOpLWXXMrI6Z1d11Ha/T4MK9NvsIuNw8A4GsXU2KYazC/7Qi6f0t4yNgVOD6KGBCOdvMAjqZWbtAi9KFgfuFHTM7GbgLONM5l1fBNpX53Q8Le/X5Opvy9yNi3t+A4cBS59y68laG6/u7j++g0PgM+907uCb+AEfjNX3NB+YGfk4FbgBuCGxzC7AIr8fz98BRfuc+hP1tH9iPeYF9ujewvOz+GvAsXm/vBUB/v3Mf4j4n4BUa9cssi5j3F6/gSgeK8P6DuhpIBqYAywOXDQPbNgc+LXPfU/FGAaTt+l0I9Z8K9jcV71j7rs/wc3vvb0W/+6H+U8H+vhb4bM7H+yJqFsnvb2D5K7s+s2W2jYT3t6LvoJD4DGtKehEREQkZOpQjIiIiIUOFiYiIiIQMFSYiIiISMlSYiIiISMhQYSIiIiIhQ4WJiIQ8887G/LHfOUQk+FSYiIiISMhQYSIiVcbMLjWzH81srpn9z8yizSzHzP5pZj+Z2RQzaxTYtreZfR84KdwHu04KZ2YdzWxy4ASHP5lZh8DDJ5rZe2a21MzGlZk593EzWxx4nCd82nURqSIqTESkSphZV+ACvJOa9QZKgEuAOnjnDOoLTAceCNzlVeAu51xPvBlFdy0fBzzrvBMcHoU3Iyd4Z0C9HTgcb8bNwWbWEG969G6Bx3kkmPsoIsGnwkREqsowoB8wy8zmBm63B0r59SRorwNHm1l9oIFzbnpg+VhgSOC8Iy2ccx8AOOfy3a/nofnRObfOeSeRmwu0BXYA+cALZnYOUO45a0QkfKgwEZGqYsBY51zvwE9n59yD5Wy3r/NglHdK9V0KylwvAWKcc8V4Z3MdD5wFfH5gkUUk1KgwEZGqMgU418waA5hZQzNrg/d35tzANhcD3zjnsoBtZnZMYPllwHTn3A5gnZmdFXiMWmaWUNETmlki3okSP8U7zNO7yvdKRKpVjN8BRCQyOOcWm9l9wJdmFoV3ptabgVygm5nNAbLw+qGAd1r15wKFxwrgysDyy4D/mdlDgcc4bx9PWxeYYGbxeK0td1TxbolINdPZhUUkqMwsxzmX6HcOEQkPOpQjIiIiIUMtJiIiIhIy1GIiIiIiIUOFiYiIiIQMFSYiIiISMlSYiIiISMhQYSIiIiIh4/8B5vOtNS1vIYMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (9, 7))\n",
    "plt.plot(list(training_history.keys()), [training_history[k]['acc'] for k in training_history.keys()], label = 'train acc')\n",
    "plt.plot(list(training_history.keys()), [training_history[k]['val_acc'] for k in training_history.keys()], label = 'val acc')\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"accuracy %\")\n",
    "plt.legend(loc = 'best')\n",
    "plt.title(\"Vgg-18 Training with Early Stopping\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAG5CAYAAABLHaTAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABVLElEQVR4nO3dd3yV5f3/8dcne0AChEDCDFE2CCoiigLWCW6lVevW6letbW37a7VDa7Xfb621y1W1da9qXXWAqyqIggoIsveeIZBBBlnX74/7BEJIIIFzcp/xfj4e53HGfZ/7fO5zkpx3ruu6r9ucc4iIiIiEmzi/CxARERFpikKKiIiIhCWFFBEREQlLCikiIiISlhRSREREJCwppIiIiEhYUkgRiTFmNtnMrgz2uqFgZpea2fv7WT7OzNa3ZU2NXv8TM/ueX68fDGb2iJnd7ncdIk1RSJGYYWbvmdldTTx+rpltNrOEELzm3WY2z8xqzOzOJpb/wMxWmVmJmc00sxOa2c7OBpc6M6tocP/S1tTknBvvnHs62OuGgnPueefcafX3zcyZ2eEHu71AqKhs9H6+FZxqW1VHkpn9yczWB2pYZWZ/abB8tZmd0ha1OOducM7d3RavJdJaCikSS54CLjcza/T45cDzzrmaELzmcuDnwDuNF5jZscA9wEQgE3gceN3M4huv65xrV38B1gJnN3js+QbbDHrQikI3N3w/nXNnt3YD5jmUv5+/AEYAI4H2wEnA14ewPZGopJAiseQNoBNwYv0DZtYROAt4JnA/y8zeCrRsfGVmvzOzaQ3WP83MlphZsZk9bGZT9tfc75x72jk3GShtYnEesMA5N8t5Uz8/A3QGurR0h+q7O8zsVjPbDDxpZh3N7G0zKzCzHYHbPRo8Z3cXhZldZWbTzOy+wLqrzGz8Qa7bx8ymmlmpmX1oZg+Z2XPN1D3FzC4M3D4h0EIyIXD/FDOb0/A1A7enBp4+N9D6cFGD7f3UzLaa2SYzu7ql71+jmlryvv2vmX0GlAP5DZYlm9l2Mxva4LEugRav7CZe7hjgdefcRudZ7Zyr/xl8FugFvBXYz58HHj/HzBaYWVGgloENXmu1mf3CzBYGan/SzFICy+p/Rn5pZtsC617a4LlPmdnvGq3b5Pt5oN8PkWBTSJGY4ZyrAF4Grmjw8HeAxc65uYH7DwFlQA5wZeACgJl1Bl7B+y84C1gCHH8IJU0G4s3s2EDryTXAHGBzK7eTgxe+egPX4/1ePxm43wuoAB7cz/OPxduXzsC9wONm+7Q2tWTdF4Av8d6bO/FaqJozBRgXuD0GWAmMbXB/SuMnOOfGBG4OC7SAvBS4n4PXEtUduBZ4KBA+W6sl79vleO9xe2BNg9p2Af8CLmuw7iXAh865giZeawbwEzO7ycyGNny/nXOXs3dr2b1m1g94EbgFyAYm4YWYpAbbvBQ4HTgM6Af8usGyHLzPrDvez/RjZta/mfdhf+9ns78fIqGgkCKx5mng22aWGrh/ReAxAkHhQuA3zrly59zC+mUBE/BaPl4LdA3dT+sDRUOlwKvANGAX8Bvgetf6E2rVBWre5ZyrcM4VOudeDexDKfC/7AkATVnjnPuHc64Wb39zga6tWdfMeuG1DtzhnKtyzk0D3tzPa05h71Dy+wb3x9JESNmPauAu51y1c24SsBNo7gsY4P5Aa0T95W6AFr5vTznnFjjnapxz1Y2WPQ181/Z0A10OPNtMDb8H/oAXLGYCG2z/A5QvAt5xzn0QeN37gFT2DskPOufWOee2B2q/pNE2bg/8jEzB6378TjOv1eT72YLfD5GgU0iRmBL48iwAzjWzfLwv1hcCi7OBBGBdg6c0vN2t4f1AmNh9ZEmgKb5+MOaJHNj38FpPBgNJeP+Fv21m3Vq5WwXOucoGdaSZ2aNmtsbMSoCpQAdrYqxLwO6g5ZwrD9xs18p1uwHbGzwGe793jU0H+plZV2A4XldXz0Br1chAzS1V2Gg8Ufl+6gf4oXOuQ4PL7dDi963ZfXLOfYHXyjDWzAYAh9NMUHPO1TrnHnLOjQY64IWKJxp24TTSjb1bbuoCtXRvprY1gefU2+GcK9vP8oaaez8P9PshEnQKKRKLnsFrQbkceN85tyXweAFQA/RosG7PBrc3NVwWaKLffd85N7jBYMxPW1DHMOAt59xS51ydc+7dwGu0tgupccvLT/FaEo51zmXgtVQANNeFEwybgE5mltbgsZ7NrRwIM7OAHwHznXNVwOfAT4AVzrltIay1OS153w7UyvU0Xti8HHilYXhsTqD16yFgBzComdfZiNcN5RXk/ez1BDY0WKfh+90r8Jx6Hc0sfT/LW+JAvx8iQaeQIrHoGeAU4DoaNFcHujBeA+4M/Fc9gL3Hr7wDDDWz88w7iub7eH3zzTKzxMAAxjggwcxSGvxn/hVwppnlm+dUvLEE8w9x/9rjjacoMrNOeN1IIeWcW4PXbXGneYfXHgcc6KiZKcDN7Ona+aTR/aZsocGA1SALxvv2LHA+XlB5prmVzOyWwCDVVDNLCHT1tGfPET6N9/NlvJ+Vk80sES9Q7cILdvW+b2Y9ArX/EniJvf028NmciDdY/N+t2bEW/H6IBJ1CisQc59xqvD/u6ezbHH8z3qDBzXhfOC/ifRkQ+O/+23gDRgvx/uudWb+8Gf/A++K7BPhV4Hb9gNJn8AZbfgKU4I1x+R/n3OJD2D2Av+KNV9iGN0Dz3UPcXktdChyH9978Du9Lcn/vzRS8L+apzdxvyp3A04GxJM2NqTiQB23veVJmBR7/K4f4vjnn1gOz8VpC9teaVgH8Ce/nbBte4L3QObcysPz3wK8D+/n/nHNL8ILPA4H1z8YbWFvVYJsvAO/jDUJeifcZ1NuM11KzEXgeuOEgf86a/f0QCQVr/Rg9kdhhZn8Acpxz+wxqDAyQXA9c6pz7uM2LC3Nm9hLekVMhb8kJJ2b2BLDROffrA64cvNdcDXzPOfdhE8vGAc8553o0XhaE123290MkGNSSItKAmQ0wsyMC3S8j8Q7BfL3B8tPNrIOZJeM1qRvef90xz8yOMbPDzCzOzM4AzsWbmyZmmFkecAHexHxR50C/HyLBptkpRfbWHq8JuxuwFa9J/j8Nlh+H16yeBCwEzgvMvyLe+JzX8OZJWQ/c6JyLmVlUA4cy/xj4vXNuld/1hMiBfj9EgkrdPSIiIhKW1N0jIiIiYSniuns6d+7s8vLy/C5DREREgmDWrFnbnHNNneMq8kJKXl4eM2fO9LsMERERCQIzW9PcMnX3iIiISFhSSBEREZGwpJAiIiIiYSnixqSIiIj4obq6mvXr11NZecDzRkoTUlJS6NGjB4mJiS1+jkKKiIhIC6xfv5727duTl5eHdyJqaSnnHIWFhaxfv54+ffq0+Hnq7hEREWmByspKsrKyFFAOgpmRlZXV6lYohRQREZEWUkA5eAfz3imkiIiISFhSSBEREYkARUVFPPzwwwf13AkTJlBUVNTi9e+8807uu+++g3qtYFJIERERiQD7Cym1tbX7fe6kSZPo0KFDCKoKLYUUERGRCHDbbbexYsUKhg8fzs9+9jM++eQTTjrpJL773e8ydOhQAM477zyOPvpoBg8ezGOPPbb7uXl5eWzbto3Vq1czcOBArrvuOgYPHsxpp51GRUXFfl93zpw5jBo1iiOOOILzzz+fHTt2AHD//fczaNAgjjjiCC6++GIApkyZwvDhwxk+fDhHHnkkpaWlh7TPOgRZRESklX771gIWbiwJ6jYHdcvgN2cPbnb5Pffcw/z585kzZw4An3zyCV9++SXz58/ffVjvE088QadOnaioqOCYY47hwgsvJCsra6/tLFu2jBdffJF//OMffOc73+HVV1/lsssua/Z1r7jiCh544AHGjh3LHXfcwW9/+1v++te/cs8997Bq1SqSk5N3dyXdd999PPTQQ4wePZqdO3eSkpJySO+JWlJEREQi1MiRI/ead+T+++9n2LBhjBo1inXr1rFs2bJ9ntOnTx+GDx8OwNFHH83q1aub3X5xcTFFRUWMHTsWgCuvvJKpU6cCcMQRR3DppZfy3HPPkZDgtXmMHj2an/zkJ9x///0UFRXtfvxgqSVFRESklfbX4tGW0tPTd9/+5JNP+PDDD5k+fTppaWmMGzeuyXlJkpOTd9+Oj48/YHdPc9555x2mTp3Km2++yd13382CBQu47bbbOPPMM5k0aRKjRo3iww8/ZMCAAQe1fVBLStuproCqMr+rEBGRCNW+ffv9jvEoLi6mY8eOpKWlsXjxYmbMmHHIr5mZmUnHjh359NNPAXj22WcZO3YsdXV1rFu3jpNOOol7772XoqIidu7cyYoVKxg6dCi33norI0aMYPHixYf0+mpJaSuv/w+UboZr3/e7EhERiUBZWVmMHj2aIUOGMH78eM4888y9lp9xxhk88sgjHHHEEfTv359Ro0YF5XWffvppbrjhBsrLy8nPz+fJJ5+ktraWyy67jOLiYpxz/PjHP6ZDhw7cfvvtfPzxx8THxzNo0CDGjx9/SK9tzrmg7ERbGTFihJs5c6bfZbROVRn8oQ/U7oKfrYD0zn5XJCIirbRo0SIGDhzodxkRran30MxmOedGNLW+unvawspPvIACsGqKr6WIiIhECoWUtrBkEiRnepeVn/hdjYiISETQmJRQq6uDpe9D31OgZpdCioiISAupJSXUNs6Gsq3Qbzzkj4OitbB9ld9ViYiIhD21pITakslg8V5Lys4C77GVn0CnPvt9moiISKwLeUuKmcWb2ddm9nYTy8zM7jez5Wb2jZkdFep62tySydDrOEjtCJ37Qvtu6vIRERFpgbbo7vkRsKiZZeOBvoHL9cDf26CetlO0FrYugP6B48TNvC6fVVO9sSoiIiIh1K5du1Y9Hm5CGlLMrAdwJvDPZlY5F3jGeWYAHcwsN5Q1takl73rX/RtMZpM/Fiq2w5Z5/tQkIiISIULdkvJX4OdAc80G3YF1De6vDzy2FzO73sxmmtnMgoKCoBcZMksnQ1ZfyDpsz2N9vJM0qctHRERa49Zbb+Xhhx/eff/OO+/kT3/6Ezt37uTkk0/mqKOOYujQofznP/9p8Tadc/zsZz9jyJAhDB06lJdeegmATZs2MWbMGIYPH86QIUP49NNPqa2t5aqrrtq97l/+8peg72NjIRs4a2ZnAVudc7PMbFxzqzXx2D5T4DrnHgMeA2/G2WDVGFKVJbDqUxh1496PZ+RC9gAvpIz+kS+liYjIIZp8G2wOcot4zlAYf0+ziy+++GJuueUWbrrpJgBefvll3n33XVJSUnj99dfJyMhg27ZtjBo1inPOOQezpr5i9/baa68xZ84c5s6dy7Zt2zjmmGMYM2YML7zwAqeffjq/+tWvqK2tpby8nDlz5rBhwwbmz58PQFFRUVB2e39CeXTPaOAcM5sApAAZZvacc+6yBuusB3o2uN8D2BjCmtrOio+grnrvrp56+eNg1tPevCkJyfsuFxERaeTII49k69atbNy4kYKCAjp27EivXr2orq7ml7/8JVOnTiUuLo4NGzawZcsWcnJyDrjNadOmcckllxAfH0/Xrl0ZO3YsX331FccccwzXXHMN1dXVnHfeeQwfPpz8/HxWrlzJD37wA84880xOO+20kO9zyEKKc+4XwC8AAi0p/69RQAF4E7jZzP4FHAsUO+c2haqmNrX0Xe+Inh4j913WZyx88Qis+xL6nNj2tYmIyKHZT4tHKE2cOJFXXnmFzZs3c/HFFwPw/PPPU1BQwKxZs0hMTCQvL4/KysoWba+58/eNGTOGqVOn8s4773D55Zfzs5/9jCuuuIK5c+fy3nvv8dBDD/Hyyy/zxBNPBG3fmtLmk7mZ2Q1mdkPg7iRgJbAc+AdwU1vXExJ1tbD0Peh7GsQ3kQPzRntzp2hcioiItMLFF1/Mv/71L1555RUmTpwIQHFxMV26dCExMZGPP/6YNWvWtHh7Y8aM4aWXXqK2tpaCggKmTp3KyJEjWbNmDV26dOG6667j2muvZfbs2Wzbto26ujouvPBC7r77bmbPnh2q3dytTSZzc859AnwSuP1Ig8cd8P22qKFNrfvSO4Knqa4egJRM6H60F1JOvr1NSxMRkcg1ePBgSktL6d69O7m53sGwl156KWeffTYjRoxg+PDhDBgwoMXbO//885k+fTrDhg3DzLj33nvJycnh6aef5o9//COJiYm0a9eOZ555hg0bNnD11VdTF5hC4/e//31I9rEha66pJ1yNGDHCzZw50+8y9u+DO2D6w/DzlZCS0fQ6H/0vfHof3LraCy0iIhLWFi1axMCBA/0uI6I19R6a2Szn3Iim1te5e0Jhybtel05zAQW8+VJcHaye1nZ1iYiIRBCFlGArXAHblngnFNyfHsdAYprGpYiIiDRDISXYltbPMnvG/tdLSIbexyukiIhEkEgbIhFODua9U0gJtiWTocsg6Jh34HXzx8G2pVASHVPDiIhEs5SUFAoLCxVUDoJzjsLCQlJSUlr1vDY5uidmVOyANZ+3fCbZ3VPkT4Hhl4SuLhEROWQ9evRg/fr1RNTpWcJISkoKPXr0aNVzFFKCafl/wdVC/wktW7/rEEjL8rp8FFJERMJaYmIiffr08buMmKLunmBaMhnSs705UFoiLs5rTVk1BdR8KCIisheFlGCprYblH0Df073w0VL546B0kzc2RURERHZTSAmWtTOgsvjAR/U0ll8/LuWToJckIiISyRRSgmXJZIhPhvyTWve8jnneRSFFRERkLwopweAcLJ0MfcZAcrvWPz9/nDfzbG1N0EsTERGJVAopwbBtGWxf2fqunnr542BXCWz8OqhliYiIRDKFlGBYMsm77neQISVvjHetLh8REZHdFFKCYem7kHMEZLZukprd0rO85yukiIiI7KaQcqjKt8O6L6D/AU4oeCD542D9l1BVFpSyREREIp1CyqFa9j64uoPv6qmXPw5qq2Dt9KCUJSIiEukUUg7VkknQLgdyhx/adnodB/FJ6vIREREJUEg5FDVVsPwj76ie1swy25SkNOh5rEKKiIhIgELKoVgzDapKod8hjkeplz8WNs+DssLgbE9ERCSCKaQciiXvQkLqnqntD1X9bLWrpgRneyIiIhFMIeVgOedNhX/YSZCYGpxt5g6H5Ax1+YiIiKCQcvC2LoTitYd+VE9D8QmQd6JaUkRERFBIOXhLJnvX/U4P7nbzx8GO1bB9VXC3KyIiEmEUUg7WksnQ7ShonxPc7eaP867VmiIiIjFOIeVg7NwKG2ZB/wnB33bnvtA+V+NSREQk5imkHIyl7wHu4M96vD9mXmvKqqlQVxf87YuIiEQIhZSDsfRdyOgBXYeEZvv546C8ELbMD832RUREIoBCSmtVV8KKwCyzZqF5jT6BeVfU5SMiIjFMIaW1Vk2F6vJDP+vx/mTkQuf+CikiIhLTFFJaa+lkSGrnzWcSSvnjvDMi1+wK7euIiIiEKYWU1nDOmwr/sJMgITm0r5U/zmuxWf9VaF9HREQkTCmktMamuVC6MTSHHjeWNxosTl0+IiISsxRSWmPpu4BB39NC/1opmdD9aIUUERGJWQoprbFkMvQcCemd2+b18sfBhtlQWdw2ryciIhJGFFJaqmQjbJoT3BMKHkj+OHC1sPqztntNERGRMKGQ0lJL3/Wu22I8Sr0ex0BCqrp8REQkJimktNSSd6FjHmT3b7vXTEiG3sfrZIMiIhKTFFJaoqrMCwr9xodultnm5I+DgsVQsqltX1dERMRnIQspZpZiZl+a2VwzW2Bmv21inXFmVmxmcwKXO0JVzyFZ+QnUVIbmhIIHkj/Ou1ZrioiIxJiEEG57F/At59xOM0sEppnZZOfcjEbrfeqcOyuEdRy6JZMhORN6j2771+46BNKyvKA07OK2f30RERGfhCykOOccsDNwNzFwcaF6vZCpq4Ol78HhJ0N8Ytu/flwc9BkDK6d4M962dXeTiIiIT0I6JsXM4s1sDrAV+MA590UTqx0X6BKabGaDm9nO9WY208xmFhQUhLLkfW38Gsq2hvaEggeSP86b6XbbMv9qEBERaWMhDSnOuVrn3HCgBzDSzIY0WmU20Ns5Nwx4AHijme085pwb4ZwbkZ2dHcqS97VkElg8HH5K275uQ/XjUnQosoiIxJA2ObrHOVcEfAKc0ejxEufczsDtSUCimbXRdK4ttPRd6HUcpHXyr4aOedCht0KKiIjElFAe3ZNtZh0Ct1OBU4DFjdbJMfMGWZjZyEA9haGqqdWK1sKW+f4c1dNY/jhYPQ1qa/yuREREpE2EsiUlF/jYzL4BvsIbk/K2md1gZjcE1pkIzDezucD9wMWBAbfhYUlgltl+Po5HqZc/DnYVe1Pzi4iIxIBQHt3zDXBkE48/0uD2g8CDoarhkC2dDFl9ofPhflcCfcZ61ys/hh4j/K1FRESkDWjG2ebsKvW6V8KhqwcgPQtyhnqHIouIiMQAhZTmrPgIaqvCo6unXv44WPcFVJX7XYmIiEjIKaQ0Z8lkSOkAPY/1u5I98sd5wWntdL8rERERCTmFlKbU1cKy96Hf6RAfyjMHtFKv4yA+SYcii4hITFBIacr6r6C8EPqFyXiUeknp0GOkTjYoIiIxQSGlKUsmQ1yCd76ecJM/DjZ9A2XhM52MiIhIKCikNGXJZO+MxymZfleyr/xxgIPVU/2uREREJKQUUhrbvhK2LYH+E/yupGndjoTkDI1LERGRqKeQ0lj9LLPhMj9KY/EJkHeC5ksREZGop5DS2NLJkD3QO6lfuMofBztWwY7VflciIiISMgopDVUUwZrPoX8YTeDWlPxx3rVaU0REJIoppDS0/EOoqwn/kNK5H7TP1bgUERGJagopDS19F9I6Q/ej/a5k/8y8Ew6umgp1dX5XIyIiEhIKKfVqq/fMMhsX73c1B5Y/Dsq3wdYFflciIiISEgop9dbOgMri8O/qqZc/1rtWl4+IiEQphZR6S9/1zouTf5LflbRMRjfo3F8hRUREopZCSr2iNd44j+R2flfScvljvaORaqr8rkRERCToFFLqXfQcXPy831W0Tv44qC73TogoIiISZRRSGkpI9ruC1sk7ASxOXT4iIhKVFFIiWUqmd7i0QoqIiEQhhZRI12csbJgFlSV+VyIiIhJUCimRLn8cuFpY85nflYiIiASVQkqk6zkSElLV5SMiIlFHISXSJSRD7+N1skEREYk6CinRIH8sFCyC0s1+VyIiIhI0CinRIH+cd63WFBERiSIKKdGg61BI7aRxKSIiElUUUqJBXJzXmrL8A6it8bsaERGRoFBIiRaDz4OyAlj9qd+ViIiIBIVCSrToexoktYf5r/hdiYiISFAopESLxFQYeBYsfAtqdvldjYiIyCFTSIkmQyfCrmJY/qHflYiIiBwyhZRo0mccpHWGef/2uxIREZFDppASTeITvAG0S96FXTv9rkZEROSQKKREmyEToaYClkzyuxIREZFDopASbXoeCxk9YJ6O8hERkcimkBJt4uJgyAWw4r9Qvt3vakRERA6aQko0GjoR6mpg4X/8rkREROSgKaREo5wjIKsvzH/V70pEREQOWshCipmlmNmXZjbXzBaY2W+bWMfM7H4zW25m35jZUaGqJ6aYwdBvw+ppULLR72pEREQOSihbUnYB33LODQOGA2eY2ahG64wH+gYu1wN/D2E9sWXoRMDB/Nf8rkREROSghCykOE/9ZB2JgYtrtNq5wDOBdWcAHcwsN1Q1xZSswyB3uM7lIyIiESukY1LMLN7M5gBbgQ+cc180WqU7sK7B/fWBxyQYhk6EjV9D4Qq/KxEREWm1kIYU51ytc2440AMYaWZDGq1iTT2t8QNmdr2ZzTSzmQUFBSGoNEoNvgAwDaAVEZGI1CZH9zjnioBPgDMaLVoP9Gxwvwewz0hP59xjzrkRzrkR2dnZoSoz+mR2h97HexO7uX2yn4iISFgL5dE92WbWIXA7FTgFWNxotTeBKwJH+YwCip1zm0JVU0waciFsWwJb5vtdiYiISKuEsiUlF/jYzL4BvsIbk/K2md1gZjcE1pkErASWA/8AbgphPbFp0HkQl6Bp8kVEJOIkhGrDzrlvgCObePyRBrcd8P1Q1SBAehYc9i1vXMrJv/GmzRcREYkA+saKBUMmQvE6WP+l35WIiIi0mEJKLBgwARJS1OUjIiIRRSElFiS3h35nwMI3oLbG72pERERaRCElVgydCGUFsGqK35WIiIi0iEJKrDj8VEjO0MRuIiISMRRSYkViCgw8Gxa9BdWVflcjIiJyQAopsWToRNhVAsve97sSERGRA1JIiSV5YyA9W2dGFhGRiKCQEkviE2Dw+bD0Pags8bsaERGR/VJIiTVDJkJNJSyZ5HclIiIi+6WQEmt6joTMXprYTUREwp5CSqwxgyEXwMqPoazQ72pERESapZASi4ZOhLoabwZaERGRMKWQEou6DoHsAeryERGRsKaQEovMvAG0az+H4vV+VyMiItIkhZRYNeQC73r+a/7WISIi0gyFlFiVdRh0O0oTu4mISNhSSIllQyfCprmwbbnflYiIiOxDISWWDb4AMLWmiIhIWFJIiWUZuZB3gneUj3N+VyMiIrIXhZRYN+RCKFwGm7/xuxIREZG9KKTEukHnQlwizPu335WIiIjsRSEl1qV1gsNP9g5FrqvzuxoREZHdFFLEm9itZAOsm+F3JSIiIrsppAj0Hw8JqZomX0REwopCikByOy+oLHwDaqv9rkZERARQSJF6QydCeSGsnOJ3JSIiIoBCitQ7/BRIydTEbiIiEjYUUsSTkAwDz4FFb0F1hd/ViIiIKKRIA0MnQtVOWPqe35WIiIgopEgDeSdCu67q8hERkbCgkCJ7xMXD4PNh6ftQWex3NSIiEuMUUmRvQyZC7S5Y/I7flYiISIxTSJG99RgBHXprYjcREfGdQorszcw7M/LKT2Bngd/ViIhIDFNIkX0NnQiu1puBVkRExCcKKbKvroOhyyB1+YiIiK8UUqRpQy70zopctM7vSkREJEYppEjThlzoXc9/1d86REQkZimkSNM69YHuIzSxm4iI+CZkIcXMeprZx2a2yMwWmNmPmlhnnJkVm9mcwOWOUNUjB2HoRNg8DwqW+l2JiIjEoFC2pNQAP3XODQRGAd83s0FNrPepc2544HJXCOuR1hp8PlicWlNERMQXIQspzrlNzrnZgdulwCKge6heT0KgfQ7knQDz/g3O+V2NiIjEmDYZk2JmecCRwBdNLD7OzOaa2WQzG9zM8683s5lmNrOgQBOMtamh34btK2Hj135XIiIiMSbkIcXM2gGvArc450oaLZ4N9HbODQMeAN5oahvOuceccyOccyOys7NDWq80MvBsiEvUUT4iItLmQhpSzCwRL6A875x7rfFy51yJc25n4PYkINHMOoeyJmml1I7Q91SY/xrU1fldjYiIxJBQHt1jwOPAIufcn5tZJyewHmY2MlBPYahqkoM05EIo3QhrP/e7EhERiSEJIdz2aOByYJ6ZzQk89kugF4Bz7hFgInCjmdUAFcDFzmmEZtjpPx4S07xp8vNO8LsaERGJESELKc65aYAdYJ0HgQdDVYMESVI6DDwH5v4LRt0E2f38rkhERGKAZpyVljnlTkhKg1eugepKv6sREZEYoJAiLZORC+c+DFvmwYd3+l2NiIjEAIUUabn+Z8CxN8AXf4el7/ldjYiIRDmFFGmdU34LXYfCGzdC6Wa/qxERkSimkCKtk5gCEx+HqnJ47XrNnSIiIiGjkCKtl90fxv8BVk2Bz//mdzUiIhKlWhRSzOxHZpZhnsfNbLaZnRbq4iSMHXUFDDoPPvodrJ/ldzUiIhKFWtqSck3gvDunAdnA1cA9IatKwp8ZnP03aJ8Lr14DlY1PyyQiInJoWhpS6idlmwA86ZybywEmapMYkNoBLvwnFK2Fd37qdzUiIhJlWhpSZpnZ+3gh5T0zaw9oxKRAr1Ew9jaY97I3I62IiEiQtDSkXAvcBhzjnCsHEvG6fERgzP+D3qO91pTCFX5XIyIiUaKlIeU4YIlzrsjMLgN+DRSHriyJKHHxcMFjEJfgTZtfU+V3RSIiEgVaGlL+DpSb2TDg58Aa4JmQVSWRJ7MHnPsgbJoDH93tdzUiIhIFWhpSapxzDjgX+Jtz7m9A+9CVJRFp4Nkw4hr4/H5Y/l+/qxERkQjX0pBSama/AC4H3jGzeLxxKSJ7O/3/IHsgvH4D7CzwuxoREYlgLQ0pFwG78OZL2Qx0B/4YsqokciWmetPmVxbDGzdo2nwRETloLQopgWDyPJBpZmcBlc45jUmRpnUdDKf/Lyz/0DtjsoiIyEFo6bT43wG+BL4NfAf4wswmhrIwiXDHfA/6nwkf/AY2zvG7GhERiUAt7e75Fd4cKVc6564ARgK3h64siXhm3tE+6dnw6rWwa6ffFYmISIRpaUiJc85tbXC/sBXPlViV1smbP6VwBUy+1e9qREQkwrQ0aLxrZu+Z2VVmdhXwDjApdGVJ1OhzIpz4U5jzHMx7xe9qREQkgrR04OzPgMeAI4BhwGPOOf1rLC0z7jboMRLe/jHsWO13NSIiEiFa3GXjnHvVOfcT59yPnXOvh7IoiTLxid7ZkgFe/R7UVvtbj4iIRIT9hhQzKzWzkiYupWZW0lZFShTo2BvO/ius/wo+ucfvakREJAIk7G+hc05T30vwDLkQVnwEn/4J8sdCnzF+VyQiImFMR+hI2xp/L2QdBq9dD2WFflcjIiJhTCFF2lZSOkx8AsoL4c2bwTm/KxIRkTClkCJtL3cYnPJbWDIJvvqn39WIiEiYUkgRf4y6EQ4/Fd77FWye73c1IiIShhRSxB9mcN7fISXTmza/qtzvikREJMwopIh/2mXDBY9CwWJ475d+VyMiImFGIUX8ddi34PgfwqwnYeGbflcjIiJhRCFF/Pet26HbkfD2LVCpOQJFRMSjkCL+S0iCM//sHZY8/SG/qxERkTChkCLhoftRMOhcmP4g7CzwuxoREQkDCikSPr51O1RXeNPmi4hIzFNIkfDRuS8ceSnMfByK1vpdjYiI+EwhRcLL2NsA05mSRUREIUXCTGZ3GHkdzH0Rti7yuxoREfFRyEKKmfU0s4/NbJGZLTCzHzWxjpnZ/Wa23My+MbOjQlWPRJATfwpJ7eCj3/ldiYiI+CiULSk1wE+dcwOBUcD3zWxQo3XGA30Dl+uBv4ewHokUaZ28Cd4Wvw3rvvK7GhER8UnIQopzbpNzbnbgdimwCOjeaLVzgWecZwbQwcxyQ1WTRJBRN0J6Nvz3t+Cc39WIiIgP2mRMipnlAUcCXzRa1B1Y1+D+evYNMpjZ9WY208xmFhRoDo2YkNwOxvwMVn8KKz7yuxoREfFByEOKmbUDXgVucc41nvPcmnjKPv82O+cec86NcM6NyM7ODkWZ/O3DZdwzeTG1dfqvPWwcfRV06OW1ptTV+V2NiIi0sZCGFDNLxAsozzvnXmtilfVAzwb3ewAbQ1lTU5xzFOys5JEpK7jyiS/ZXlbV1iVIUxKS4aRfwaa5sPANv6sREZE2Fsqjewx4HFjknPtzM6u9CVwROMpnFFDsnNsUqpqaY2b87ryh3HvhEXy5ejtnPzCNeeuL27oMacrQb0P2QO9In9pqv6sREZE2FMqWlNHA5cC3zGxO4DLBzG4wsxsC60wCVgLLgX8AN4WwngP6zjE9eeWG4wC48JHP+ffMdQd4hoRcXDycfAdsXwFznve7GhERaUPmIuzIiREjRriZM2eG9DUKd+7ih//6ms+WF3LZqF7ccdZgkhI0751vnIPHT4PidfDDryEx1e+KREQkSMxslnNuRFPL9M3bhKx2yTx99Uj+Z2w+z81Yy0WPTWdzcaXfZcUuMzjlTijdBF8+5nc1IiLSRhRSmpEQH8cvxg/k4UuPYunmUs564FO+WFnod1mxK280HH4qfPpnqCjyuxoREWkDCikHMGFoLm98fzQZKYl8959f8MS0VURaF1nUOPl2qCyCz+/3uxIREWkDCikt0Ldre964eTTfGtCFu95eyC0vzaG8qsbvsmJP7jAYciHM+DuUbvG7GhERCTGFlBbKSEnk0cuO5men9+fNuRu54OHPWVNY5ndZseekX0FtFUz9o9+ViIhIiCmktEJcnPH9kw7nqatHsrmkkrMfmMbHi7f6XVZsyToMjroCZj0J21f5XY2IiISQQspBGNsvm7duPoEeHdO45umv+NuHy6jTdPptZ8zPIS4RPv4/vysREZEQUkg5SD07pfHqjcdz/vDu/OXDpVz3zEyKKzQjapvIyIVj/wfm/Rs2z/e7GhERCRGFlEOQmhTPn74zjLvOHcyUpQWc++A0lmwu9bus2HDCLZCSAR/d7XclIiISIgoph8jMuOK4PP51/SjKqmo576HPeGtum58jMfakdoTRt8DSd2HNdL+rERGREFBICZIReZ145wcnMLhbBj948Wt+9/ZCamrr/C4ruh17A7TrCh/e6U2dLyIiUUUhJYi6ZKTwwnWjuOr4PP45bRWXPf4F23bu8rus6JWUBmN/DutmwLL3/a5GRESCTCElyJIS4rjznMH85aJhzFlXxNkPTOPrtTv8Lit6HXUldOwD/70L6tRyJSISTRRSQuT8I3vw6o3HEx9nXPToDF78cq3fJUWn+ET41q9hy3yY/6rf1YiISBAppITQ4G6ZvP2DExh1WBa/eG0ev3jtG6o1TiX4Bl8AXYfCx7+Dmiq/qxERkSBRSAmxDmlJPHnVMXz/pMN48ct1XPXkl5pPJdji4uDkO2DHapj9tN/ViIhIkCiktIH4OONnpw/gvm8P48tV25n4989Zt73c77KiS99Todfx3jl9qnROJRGRaKCQ0oYmHt2Dp68ZyZaSSs5/+HPmrCvyu6ToYQan/AZ2boEvHvG7GhERCQKFlDZ2/GGdee2m40lNiuPix6bz7vzNfpcUPXqNgn7jYdrfoHy739WIiMghUkjxweFd2vP6TaMZmJvBjc/P4h9TV+I0GVlwnHw77CqBz/7qdyUiInKIFFJ80rldMi9eN4oJQ3L530mL+PUb8zVDbTB0HQxHfAe+eBRKdHoCEZFIppDio5TEeB645EhuHHcYz3+xlmufnklppY78OWTjfgF1tTDlXr8rERGRQ6CQ4rO4OOPWMwbw+wuGMm35Nr79yHQ2FlX4XVZk69QHRlwNs5+BwhV+VyMiIgdJISVMXDKyF09dfQwbdlRw3kOfMX9Dsd8lRbYxP4OEZPjod35XIiIiB0khJYyc2DebV248nsT4OL79yHQ+XLjF75IiV7suMOomWPAabJwT/O3X1cG2ZTD3XzDp5/DPU+DpczQORkQkiCzSjioZMWKEmzlzpt9lhNTWkkq+98xM5m8o5vazBnH16D5+lxSZKovhb8Og+9Fw2SGc18c5KNkAG2bBhtmwcbYXfHaVeMsT0yB3OGyeB6kd4Yo3IOuwIOyAiEj0M7NZzrkRTS5TSAlP5VU13PKvOby/cAtXHZ/H7WcNIj7O/C4r8nx2P3xwO1z5NvQ5sWXPKd/uhZENs7xAsmE2lG31lsUlQNch0P0o6HaUd925P8QnwMav4bkLAYPLX4PcYSHbLRGRaKGQEqFq6xy/n7SIf05bxckDunD/JUeSnpzgd1mRpboC7j8KMrvDtR94M9M2tGsnbJq7dyApWhNYaNC5396BpOsQSExp/vW2LYdnz4OKIrjkxZYHIxGRGKWQEuGenb6a37y5gIG5GTxx1TF0zdjPl6Tsa9ZT8NaP4NtPQYfegUDytRdIti0BF5ifJrPn3oEkdzikZLT+9Yo3wHMXwPZVMPEJGHhWEHdGRCS6KKREgY8Xb+XmF2aTkZrI41cew6BuB/HlGatqa+DhY6Fw+Z7H0rK8sSr1gaTbUdAuO3ivWb4dnv+21zpzzgNw5GXB27aISBRRSIkSCzYWc+1T3oRvD156FCf17+J3SZFj/UxY/A7kHuEFkg699u36CbZdO+Hly2HFR3DqXTD6R6F9PRGRCKSQEkU2F1dyzVNfsWRLKXeeM5jLR/X2uyTZn5oqeP1/vEOhj/+hF1ZCHY5ERCLI/kKK5kmJMDmZKbx8w3GM7ZfN7W/M53dvL6S2LrKCZkxJSIIL/wnHfA8+vx/+c7PX/SQiIgekkBKB2iUn8NjlR3Plcb3557RV3PT8LCqqav0uS5oTFw8T7oOxt8Gc5+DlK6C60u+qRETCnkJKhEqIj+O35w7hjrMG8f7CLVz82HS2luqLL2yZwUm/gPF/hCXvePOpVOrUByIi+6OQEuGuOaEPj152NEu37OTM+6cxad4mIm2cUUw59nq48HFYNwOeOgt2bvW7IhGRsKWQEgVOG5zDqzceT5f2ydz0/Gyue2aWzqQczoZOhEte8s7988TpsGPNgZ8jIhKDFFKixKBuGfzn+6P51YSBTFtewKl/nsLTn6/WoNpw1fcUuPJNbz6Vx0+DLQv9rkhEJOwopESRhPg4rhuTzwc/HstRvTvymzcXMPGRz1m8ucTv0qQpPUfC1ZO98SpPngFrv/C7IhGRsBKykGJmT5jZVjOb38zycWZWbGZzApc7QlVLrOnZKY1nrhnJXy8azprCcs66fxr3vbeEymodARR2ug6Ca97zZsB95lxY9oHfFYmIhI1QtqQ8BZxxgHU+dc4ND1zuCmEtMcfMOO/I7nz4k7GcM7wbD368nPF/+5TpKwr9Lk0a69jbCyqd+8KLF8M3//a7IhGRsBCykOKcmwpsD9X2pWU6pSfx5+8M57lrj6W2znHJP2Zw6yvfUFRe5Xdp0lC7LnDV29BzFLz2PfjiUb8rEhHxnd9jUo4zs7lmNtnMBje3kpldb2YzzWxmQUFBW9YXNU7o25n3bhnDDWMP45XZ6znlz1N4a+5GHa4cTlIy4bJXYcBZMPnn8PH/gT4fEYlhIT13j5nlAW8754Y0sSwDqHPO7TSzCcDfnHN9D7TNWD93TzAs3FjCL177hrnri/nWgC7cde5genRM87ssqVdbA2/9yJuddsS1MOGP3qy1IiJRKCzP3eOcK3HO7QzcngQkmllnv+qJJYO6ZfDaTaO5/axBzFhZyGl/mcrj01bpcOVwEZ8A5z7onZBw5uPw6ve8ExWKiMQY30KKmeWYeaeDNbORgVo0qrONxMcZ157Qh/d/PIZj+3Ti7rcXcv7Dn7Fwow5XDgtmcNrd3lmTF7wGL14Eu3b6XZWISJsKWXePmb0IjAM6A1uA3wCJAM65R8zsZuBGoAaoAH7inPv8QNtVd0/wOed4+5tN/PatBewor+a6E/O55ZS+pCSqiyEszH4W3vohZHSHzB6QmAqJaYHrhrf391gzy+IT/d47EYlx++vuCemYlFBQSAmdovIq/m/SIl6euZ7eWWn873lDOaGveuDCwtL3YOYTUFUG1RWBS/net+uqW7/duIQ9oaV9DgyZCEdcBO27Bn8fRESaoJAirfL5im386vX5rNpWxoVH9eDXZw6kY3qS32XJgdRW7xtgaiqaCDWNrwO3tyyA9V+BxUPf0+DIS6Hv6ZCgz15EQkchRVqtsrqWBz5axqNTVpKRmsgdZw3i3OHdCAwjkmhVsBTmPA9zX4SdW7yZcI+4CIZfCjn7HKQnInLIFFLkoC3eXMJtr85jzroixvTL5nfnDqFXlg5Xjnq1NbDiv/D1c7BksteVlDsMhl/mncU5rZPfFYpIlFBIkUNSW+d4bsYa/vjeEqpr6/jRKX257sR8EuP9ngtQ2kT5dpj3by+wbP4G4pOg/wQ48jI47Fuaw0VEDolCigTFpuIKfvvmQt5dsJn+XdvzfxcM4eje+o86pmyeB18/D9+8BBXboX0uDLvYa2HpfLjf1YlIBFJIkaD6cOEW7vjPfDYWV/LdY3tx6+kDyEzToawxpaYKlk6GOS94Z252tdDzWG/syuDzISXD7wpFJEIopEjQle2q4S8fLOWJz1bRKT2ZO84exNlH5GpgbSwq3Qxz/+UNuN221DukeeA53tFBvU+AOHULikjzFFIkZOZvKOaXr8/jm/XFGlgb65yD9TO9cw7Nfw12lUCH3jD8uzDsEujY2+8KRSQMKaRISNXWOZ6dvpr73l+qgbXiqSqHxe94gWXlFMBBvzPgwschuZ3f1YlIGFFIkTahgbXSpKK13mDbqfdCv/Fw0XPqAhKR3cLyLMgSfXIzU3nk8qP5xxUjKK2s5sK/T+eXr8+juPwgpmuX6NGhF5z0CzjjHljyDvz3Tr8rEpEIoZAiQXfqoK588JOxfO+EPvzry7Wc/OcpvDl3I5HWaidBNvJ6GHEtfPY3r2VFROQAFFIkJNKTE/j1WYN48+YT6NYhhR+++DVXPvkVawvL/S5N/GIG4/8A+ePgrR/B6s/8rkhEwpxCioTUkO6ZvH7TaO48exCzVm/n1L9M4eFPllNdW+d3aeKH+ET49tPQMQ9eugy2r/S7IhEJYwopEnLxccZVo/vw4U/HMq5/Nve+u4Sz7p/GrDU7/C5N/JDaAb77EuDghYuhstjvikQkTCmkSJvJzUzl0ctHNBhY+zm/en0exRUaWBtzsg6D7zwL21fAv6/yTmgoItKIQoq0ufqBtdee0IcXv1zLyX+awlsaWBt7+pwIZ/0FVnwE7/3C72pEJAwppIgv0pMTuD0wsDY3M4UfvPg1lz/+Jd+sL/K7NGlLR10Bx90MXz4GX/7D72pEJMwopIivhnTP5I3vewNrF2ws5pwHP+O6Z2ayaFOJ36VJWzn1Lm822sm3eq0qIiIBmnFWwkZpZTVPfbaaxz5dSWllDWcOzeWWU/rSt2t7v0uTUNtVCo+fDsXr4XsfQnY/vysSkTaiafElohSXV/PPaSt5YtoqyqtrOW94d350cl/yOqf7XZqEUtFa+Me3IKkdXPcRpOmUCiKxQCFFItL2sioenbKCp6evprrWceFR3fnBt/rSs5POshy11n0JT50FPUbA5W9AQpLfFYlIiCmkSETbWlrJwx+v4IUv1uJwXHRMT24+qS85mSl+lyah8M3L8Np1cORlcM6D3ky1IhK1FFIkKmwqruDBj5bz0lfriIszLju2NzeOO4zs9sl+lybB9tHvYOof4dS7YfQP/a5GREJIIUWiyrrt5dz/32W89vUGkuLjuOL43tww5jA6pqtrIGrU1cErV8HCN+HiF2DABL8rEpEQUUiRqLSyYCd/++8y3py7kfSkBK4Znce1J+aTmZrod2kSDFXl8NQEKFgK174HOUP9rkhEQkAhRaLa0i2l/PXDpUyat5mMlASuH5PPVaP70C45we/S5FCVbPKO+LE474if9l39rkhEgkwhRWLCgo3F/OWDpXy4aCsd0xK5YexhXHFcHqlJ8X6XJodi4xx4cjx0GQRXvQ2JqX5XJCJBtL+QohlnJWoM7pbJP688hje+P5oh3TP5/eTFnHjvxzz52Soqq2v9Lk8OVrfhcP6jsGEm/OdmiLB/rETk4CmkSNQZ3rMDz157LP++4TgOy07nt28t5KT7PuHZ6aspr9LZdiPSoHPg5Dtg/isw5V6/qxGRNqLuHolqzjk+X1HIn95fwuy1RWSkJHDJyF5cflxvenTUpHARxTl440aY+yJMfBKGXOB3RSISBBqTIjHPOcfMNTt46rPVvLtgM845ThuUw1Wj8zi2TydME4ZFhppd8PQ5sGkOXDUJehztd0UicogUUkQa2FBUwXMz1vDil2spKq9mYG4GVx+fxznDu5GSqEG2Ya9sG/zjJC+wXPcRZPbwuyIROQQKKSJNqKiq5Y05G3jqs9Us2VJKp/QkvjuyF5eN6q0p98Pd1kXwz1OhUx5c/S4kt/O7IhE5SAopIvvhnGP6ikKe/Hw1Hy7aQrwZ44fmctXxeRzVq4O6gsLVsg/ghe9A/wnwnWchTscBiEQihRSRFlpbWM4z01fz0sx1lFbWMKxHJleP7sOEobkkJehLMOzM+Du8exuMvgVO/a3f1YjIQVBIEWmlsl01vDp7PU99tpqV28rIbp/MZcf25rvH9tIJDcOJc/D2j2HWk3Duw3DkpX5XJCKtpJAicpDq6hxTlxXw1Oer+WRJAUnxcZw1LJerj+/D0B6ZfpcnALXV8NyFsPpTGHENnPQrSOvkd1Ui0kIKKSJBsKJgJ898vpp/z1pPeVUtI3p35OrRfTh9cFcS4tUV5KvKEvjobvjqcUhu7wWVEddAvM7fJBLuFFJEgqikspqXv1rH09NXs257BbmZKVx+XG8uPqYXndKT/C4vtm1ZCO/eCqumeuf6OeMeyB/rd1Uish++hBQzewI4C9jqnBvSxHID/gZMAMqBq5xzsw+0XYUUCRe1dY6PFm/lqc9X8dnyQhLjjdGHd2bC0FxOH5RDZlqi3yXGJudg8dvw3i+haC0MPBtO+x10zPO7MhFpgl8hZQywE3immZAyAfgBXkg5Fvibc+7YA21XIUXC0ZLNpbw6ez3vfLOJDUUVJMR5geXMobmcNrgrHdLUwtLmqith+gPw6Z+hrhaO/wGc+BNISve7MhFpwLfuHjPLA95uJqQ8CnzinHsxcH8JMM45t2l/21RIkXDmnOOb9cVMmreJd+ZtYv0OL7Acf3hnzhyaw2mDcuioLqG2VbwBPvwNzPs3tO8Gp94FQyeC5r8RCQvhGlLeBu5xzk0L3P8vcKtzbp8EYmbXA9cD9OrV6+g1a9aErGaRYHHOMW9DMe/M28SkeZtYt72C+Djj+MOyvC6hwTkaw9KW1s6Aybd65/3peSyM/wN0O9LvqkRiXriGlHeA3zcKKT93zs3a3zbVkiKRyDnHgo0luwPLmsJy4uOM4/LrA0tXstpp/pWQq6uDOc/Bf+/yzgF05GVw8m+gXbbflYnErHANKerukZhUH1gmBQLL6kBgGZXfaXcLS2cFltCqLIYp98IXj0BiGoz9OYz8H0hQy5ZIWwvXkHImcDN7Bs7e75wbeaBtKqRINHHOsXBTfWDZzKptZcQZjAq0sJwxRIElpLYtg3d/Acs/gKy+cMbvoe+pflclElP8OrrnRWAc0BnYAvwGSARwzj0SOAT5QeAMvEOQr25qPEpjCikSrZxzLNpUuruFZWUgsBzbJ4sJR+Ry+qCudMnQ2ZlDYul7XljZvgL6ng6n/x90PtzvqkRigiZzE4kwzjmWbCll0jfeUUIrCsoAOLxLO0bld2JUfhbH9snSeYSCqabK6/6Zci/UVMKoG2DMzyElw+/KRKKaQopIBHPOsXTLTj5avJUvVhXy1artlFXVAnBYdroXWPKzGNWnk1pagqF0izewds5zkN4FTvkNDPsuxOnUByKhoJAiEkVqauuYv7GEL1YWMmNlIV+t3sHOXTUA5NeHlj5ea0tXhZaDt2GWd8jy+q+8Q5VPuROyB0BaFsRrNmGRYFFIEYliNbV1LNxUwoyVhcxYuZ2vVm2ntD60dE7n2AbdQzmZCi2tUlfnTQL34W+gtMGBhykdIL0zpGd7oSU9u9H9+tudA6FGJzoUaY5CikgMqa1zLNxYH1oK+XL1dkorvdCSl5XGqPysQBdRJ3IzU32uNkLs2gkrPoKyrVBWCGUFUL7Nm2ulbJt3u7wQXF3Tz0/tuCe0pAcuaYEgkx4IOakdIT4J4hIgLj5wndjofsKe+5oxV6KEQopIDKutcyza1CC0rNpOSSC09M5KY1SfLEYd5rW2KLQcgrpaqChqFGAKvPBSVhAIM41ucwh/fxuHlv2FmvgELwR96w7ocXSw9lgkKBRSRGS3hqHli1Xb+WJl4e7QMiCnPScN6MJJ/btwVK8OJMRrsGjI1NVCxY49oaViB9TVNHGphdrqve/X1UBddaP7NYH1apvezqa5sHOrd5LFMT/XxHUSNhRSRKRZtXWOxZtLmLZsGx8v2crM1TuoqXNkpCQwpl82J/Xvwtj+2ZpULtJVFsPk22DuC5AzFM5/FLoO9rsqEYUUEWm5kspqPlu2jY8Wb+WTpQUUlO7CDI7onrm7lWVo90zi4jQmIiItfgfe+pEXWk76JRz/Q697SMQnCikiclDq6rxp+z9evJWPl2zl63VFOAed2yUxtl8XThqQzYl9s8lM1SG5EaVsG7z9Y1j0pndG6PP+DlmH+V2VxCiFFBEJiu1lVUxdWsDHS7YyZWkBReXVxMcZR/fuyEn9u/CtAV3o17UdpiNPwp9z3uHVk/6fN5bl1LtgxLWatE7anEKKiARdbZ1jzrodfLy4gI8Wb2XhphIAumWmMC7QLTT68CzSkjRHSFgr2Qj/uRlW/Bfyx8G5D0FmD7+rkhiikCIiIbe5uJIpS7fy0eKtTFu2jbKqWpLi4zg2vxMn9e/CmH7Z9OmcTrzGsoQf52DmE/D+7d4hy+P/AMMu1lws0iYUUkSkTVXV1DFz9XY+XrKVj5cUsHzrTgBSE+Ppl9OegTntGZDTngG5GQzIaU+HNB0OGxa2r4Q3boK102HAWXDWX6Fdtt9VSZRTSBERX60tLGfGqkIWbypl8eYSFm0qYUd59e7luZkpe4WWgbkZ9OmcTqLmaWl7dbUw/SH46G5IzoCz/woDz/a7KvHLzgJY/gEsex/G3gpdBgb9JfYXUtRZLCIh1ysrjV5ZabvvO+coKN3Fos2lLN5UwuLNpSzaVMK05duorvX+cUqKj+PwLu0YkNuegTkZDMhtz4CcDLLba76WkIqLh9E/hL6nwuv/Ay9dBkdcBOPvhdQOflcnoVZXB5vmeKFk6Xuw8WvAQbuuULQ2JCFlf9SSIiJho6qmjpXbdrJ4UymLNpfsbnnZUrJr9zqd2yUxICdjr5aXw7u0IyVRc30EXW01TL0Ppv7R+5I690E4/OS2r6O6wjvBY8c+GicTCpXF3rmpln3gXcq2AgY9RkDf07xLzhEhO/JL3T0iEtG2l1WxuEFoWby5lCWbS9lV453QLz7OyO+c3qC7yOsyyslI0eHQwbBhNrx+A2xb4h2mfOpdkNwuNK9VXQFbFnj/wW+c4/1Xv3URuFrIOhyOvAyOuBgyckPz+sFUWwMrP4Y5L3hHT2X29Gb53X0ZCu26tH3wcg4KlsCy97xQsna6d+qElEw4/BToe7oXRtM7t0k5CikiEnVq6xyrC8v2GueyaFMpG4oqdq+TmZq4e4xLfctL/67tSU1Sq0urVVfAR7/zxqt0zPMmgOt93KFvs7lAApCWBbnDodtwryVnweveF6rFweGneoGl3xnhdx6irYu8YPLNy7Bzs3dyx37jvRNPblkAJRv2rJvWORBYhnjXOUOgc39ITAluTdUVsOrTQDB53+u6Ae91+57qBZMex3gno2xjCikiEjNKKqtZEhjr0nDMS3mV98VnBn2y0nePcakPMT06pqrVpSVWfwZv3Oh9yR3/AzjpVy37Qq2uhC3zvUCyaQ5snAtbFzYdSOqvM3vu28qwbTnMeR7mvuh1AaVleWNmhl/qfcH7pXw7zHvFOzfSxq+9Q7n7ngbDLoF+p0NC8t7rblkQuMz3rrcugppAwLZ46Nx37xaXroMho1vrWl12rPECybL3YdVUqKmExDRvPpy+p3r1hcGcOAopIhLT6uoc63aUs6i+uyhwvbqwfPc67ZITAq0tXngZmNue/jkZtEvW8QX72FUK7/8aZj0F2QPhgkchd9ie5dWVgRaS2XsCScEir0sBILWTF0K6Hbn/QLI/9V0pXz8Liyd5Z4XOHe61rgyd6LVehFpttdddMvcFWPKuV0POUBj2XRj67dYdvl1X6x0CXh9a6gNMfYsHQEqHvVtcug723v+ktD31rJ2xJ5gULPYe79jHC0p9T4Peo4PfSnOIFFJERJpQtquGJVtK94x1CQzYLa2s2b1Oz06pDMjxuol6dEyle8dUunVIpXuHVA3WXfaBN1tt+TY45jqoKm0+kOQO90LJwQSSAykr9Kb4//pZ74s9PhkGnuUFlj5jg38Cxc3z9nTnlG/zumyOuAiGX+KFlGCqLIYtCxuEl/ne/eqywArmnXcpsydsmAW7SiAuEfJGBwa9ng6dDw9uTUGmkCIi0kLOOTYWV3rdRQ26jFZtK6Ou0Z/LrPQkL7Rk7h1eegRud0xLjP4upPLtMPnnXkjYK5AErjv0aruBoc7Bprled9A3L0NlEWT0gOHf9S6d+hz8tncWwLyXYc6LsGWeFwT6j/e2e/gpEN+GJ9msq4Oi1XtaXDbP87p2uh/pBZP8cZDcvu3qOUQKKSIih6i6to7NxZVsLKpgQ1HF7usNRZVs2FHOhqIKKqvr9npOamI83Tqk0L1jGt07pNC9w54g061DKjmZKdEzYV1lsTf5W7iEsupKWDIJvn7OO7wWB3kneq0rA8/Z00WyPzW7vLlC5rzgTWhWV+O1Bg2/FIZcCGmdQr4bsUAhRUQkxJxz7CivZmNRBet3VOwTZjYWVbBtZ9Vez4kz6JrhhZfuHVPJy0qnT2fvktc5nczUNvzvPJoVr/cG2n79HOxYDUntYcgFcOTl3lwgDYOVc97A1zkvwPxXoGIHtMuBYRd5Y026DPBtN6KVQoqISBiorK7dE1wCQWZ94P667RVsLK6g4Z/krPQk8jo3CC5Z9QEmTWeXPhh1dbD2c/j6eVj4BlSXQ+d+XuvK4afA8g+9cFKweM+4lmHf9bpPfDg0N1YopIiIRIDK6lrWbS9n5bYyVm8rY3VhGSsLvOuGs+4C5GSkkNc5jT6d29GnwXXPTmkkJ8T4gN6WqCzx5l2Z8zys+2LP4z1GeuNMBp+v0wC0EYUUEZEIV7arhtWFZawKBJhV28pZtW0nqwvL2V62pxspztjddZQf6Daqb4np1iE1esbABFPBUlj9qXckUJgfCRONdIJBEZEIl56cwOBumQzulrnPsuLyalYVeuFl5e4QU8ZrszdQumvP4dTxcUb3Dqn06uSd8LF3pzR6Z6XRq1M6vbLSYndOmOx+3kXCToz+RIqIRI/MtESGp3VgeM8Oez3unGPbzqrdLTBrC8tZs72ctdvLmTxvEzvKq/daPys9aXd46ZWVTq9AiOndKY3s9snRfzi1hB2FFBGRKGVmZLdPJrt9Msfk7Xu4bElltRdcCstZs90LMWu3l/PV6h28OXfjXvPCpCbG06uTN+ald1Z9C0wavbPS6d4hlaQEdSNJ8CmkiIjEqIyURIZ0z2RI9327kKpq6li/I9DyEggvawrLWbu9jGnLC/aaE6b+UOp2yQmkJsWTmhhPalI8aUnxpCR699N2P55AamIcaUkJpCTFkxZYd/fzAuvWL0vQGJqYppAiIiL7SEqIIz+7HfnZ7fZZ5pxja+muPcGlsIwNRZVUVNdQXlVLRVUt28uq2LCjlvKqWiqra6kIXFp7rEZivJGaGE96cgJdMlLIzUghJzOF3Mz661RyM1PokpGso5qikEKKiIi0ipnRNSOFrhkpTXYjNcc5x66aOi/IVNdSUVVDRVUd5VU1gft7wkxFIOyUB27v3FXDlpJKVhTs5LPl2/YaEFyvc7skcjJTyMlIbRBi9oSZnIwUUpMUZCKJQoqIiLQJMyMlMT4oJ2YsraxmS0klm4q9y+bd1xWs31HOzDXbKWo0MBigQ1oiORn14WVPmMkJtNB0zUghIyVBg4TDhEKKiIhEnPYpibRPSeTwLs2fSK+iqpbNJZVsKq5oEGIC1yUVzNtQvM+pCsAbJNw1I5muGXsCTH3LUU6m93iX9ikaLNwGFFJERCQqpSbF757Irjm7amrZWrKLjUUVbCndxZbiSjaXVLIlcJm9dgdbSnZRVVO3z3Oz0pN2B5muGfVhJpmu9S0zGSl0iIUzYYeQQoqIiMSs5IR4egYOrW6Oc46i8mo2lwQCTHElW0p27Q4zm4sr+WZ9UZOtMkkJcV5waZ9Cx/QkOqQm0iEtkczURDLT9tzvkJrkPZ6WSPtkdTfVU0gRERHZDzOjY3oSHdOTGJib0ex6VTV1bC2tb4XZxebiPS0yW0p2sX5HBQs2FFNUUU15VW2z24mPMzJSEuiQlkTm7hCTuPv+7sfSEskMhJsOgcej7ZBthRQREZEgSEqIo0fHNHp0bL5Vpt6umlqKK6opLq+mqKKaovJqiiuqKSqvClzXP17F9rIqVhaUUVReRemumv0exp2RkkCn9CQ6pCUFrhPplOYFrI5pSXRMS9xzOz2RjmlJYX0+p5CGFDM7A/gbEA/80zl3T6Pl44D/AKsCD73mnLsrlDWJiIj4LTkhni7t4+nSPqVVz6utc5RW7h1iiiuq2VFWRVHgekd5NTvKq9haWsmSzaXsKK/ab8tN++QEOqR7YaZxuOmQnuSFnLREBuRm0Ck96VB3vVVCFlLMLB54CDgVWA98ZWZvOucWNlr1U+fcWaGqQ0REJFrExxkdAmGiNSqraykqr2Z7WZXXOlMeCDNlVewor9or3KzctpMdZdXsbDQXzd8vPYrxQ3ODuTsHFMqWlJHAcufcSgAz+xdwLtA4pIiIiEgIpSTGk5MZT05my1tuqmrqKAqEme1lVfTruu/sw6EWypDSHVjX4P564Ngm1jvOzOYCG4H/55xb0HgFM7seuB6gV69eIShVREREGkpKiKNLRgpdMlrXJRVMoRwt09TxU42H+8wGejvnhgEPAG80tSHn3GPOuRHOuRHZ2dnBrVJERETCUihDynqgZ4P7PfBaS3ZzzpU453YGbk8CEs2scwhrEhERkQgRypDyFdDXzPqYWRJwMfBmwxXMLMcCM9aY2chAPYUhrElEREQiRMjGpDjnaszsZuA9vEOQn3DOLTCzGwLLHwEmAjeaWQ1QAVzsXGtP5C0iIiLRyCItE4wYMcLNnDnT7zJEREQkCMxslnNuRFPLwneaOREREYlpCikiIiISlhRSREREJCwppIiIiEhYUkgRERGRsKSQIiIiImFJIUVERETCkkKKiIiIhCWFFBEREQlLCikiIiISliJuWnwzKwDW+F1HG+kMbPO7iDak/Y1u2t/oFmv7C7G3z6Ha397OueymFkRcSIklZjazufMZRCPtb3TT/ka3WNtfiL199mN/1d0jIiIiYUkhRURERMKSQkp4e8zvAtqY9je6aX+jW6ztL8TePrf5/mpMioiIiIQltaSIiIhIWFJIERERkbCkkOIzM+tpZh+b2SIzW2BmP2pinXFmVmxmcwKXO/yoNVjMbLWZzQvsy8wmlpuZ3W9my83sGzM7yo86g8HM+jf43OaYWYmZ3dJonYj+fM3sCTPbambzGzzWycw+MLNlgeuOzTz3DDNbEvisb2u7qg9eM/v7RzNbHPh5fd3MOjTz3P3+7IejZvb3TjPb0OBndkIzz42Wz/elBvu62szmNPPcSPx8m/wOCpvfYeecLj5egFzgqMDt9sBSYFCjdcYBb/tdaxD3eTXQeT/LJwCTAQNGAV/4XXOQ9jse2Iw3cVHUfL7AGOAoYH6Dx+4Fbgvcvg34QzPvxwogH0gC5jb+2Q/HSzP7exqQELj9h6b2N7Bsvz/74XhpZn/vBP7fAZ4XNZ9vo+V/Au6Ios+3ye+gcPkdVkuKz5xzm5xzswO3S4FFQHd/q/LducAzzjMD6GBmuX4XFQQnAyucc1E1Y7JzbiqwvdHD5wJPB24/DZzXxFNHAsudcyudc1XAvwLPC2tN7a9z7n3nXE3g7gygR5sXFiLNfL4tETWfbz0zM+A7wIttWlQI7ec7KCx+hxVSwoiZ5QFHAl80sfg4M5trZpPNbHDbVhZ0DnjfzGaZ2fVNLO8OrGtwfz3REdwupvk/btH0+QJ0dc5tAu+PINCliXWi9XO+Bq8lsCkH+tmPJDcHureeaKYrIBo/3xOBLc65Zc0sj+jPt9F3UFj8DiukhAkzawe8CtzinCtptHg2XhfBMOAB4I02Li/YRjvnjgLGA983szGNllsTz4noY+XNLAk4B/h3E4uj7fNtqWj8nH8F1ADPN7PKgX72I8XfgcOA4cAmvC6QxqLu8wUuYf+tKBH7+R7gO6jZpzXxWFA/Y4WUMGBmiXg/HM87515rvNw5V+Kc2xm4PQlINLPObVxm0DjnNgautwKv4zUZNrQe6Nngfg9gY9tUFzLjgdnOuS2NF0Tb5xuwpb6LLnC9tYl1oupzNrMrgbOAS12gw76xFvzsRwTn3BbnXK1zrg74B03vR7R9vgnABcBLza0TqZ9vM99BYfE7rJDis0Af5+PAIufcn5tZJyewHmY2Eu9zK2y7KoPHzNLNrH39bbwBh/MbrfYmcIV5RgHF9c2OEazZ/8Ci6fNt4E3gysDtK4H/NLHOV0BfM+sTaGm6OPC8iGNmZwC3Auc458qbWaclP/sRodEYsfNpej+i5vMNOAVY7Jxb39TCSP189/MdFB6/w36PLI71C3ACXvPYN8CcwGUCcANwQ2Cdm4EFeCOnZwDH+133IexvfmA/5gb26VeBxxvurwEP4Y0anweM8LvuQ9znNLzQkdngsaj5fPHC1yagGu8/q2uBLOC/wLLAdafAut2ASQ2eOwHvaIIV9T8L4X5pZn+X4/XN1/8OP9J4f5v72Q/3SzP7+2zgd/MbvC+l3Gj+fAOPP1X/O9tg3Wj4fJv7DgqL32FNiy8iIiJhSd09IiIiEpYUUkRERCQsKaSIiIhIWFJIERERkbCkkCIiIiJhSSFFRCKKeWeNftvvOkQk9BRSREREJCwppIhISJjZZWb2pZnNMbNHzSzezHaa2Z/MbLaZ/dfMsgPrDjezGYET1r1ef8I6MzvczD4MnHxxtpkdFth8OzN7xcwWm9nzDWbsvcfMFga2c59Puy4iQaKQIiJBZ2YDgYvwTrg2HKgFLgXS8c5hdBQwBfhN4CnPALc6547Am8m0/vHngYecd/LF4/FmAgXvTK23AIPwZvocbWad8KZoHxzYzu9CuY8iEnoKKSISCicDRwNfmdmcwP18oI49J2h7DjjBzDKBDs65KYHHnwbGBM6D0t059zqAc67S7TkvzpfOufXOO8HdHCAPKAEqgX+a2QVAk+fQEZHIoZAiIqFgwNPOueGBS3/n3J1NrLe/83I0dRr4ersa3K4FEpxzNXhnnX0VOA94t3Uli0i4UUgRkVD4LzDRzLoAmFknM+uN9zdnYmCd7wLTnHPFwA4zOzHw+OXAFOdcCbDezM4LbCPZzNKae0Eza4d3EsdJeF1Bw4O+VyLSphL8LkBEoo9zbqGZ/Rp438zi8M4o+32gDBhsZrOAYrxxK+CdCv6RQAhZCVwdePxy4FEzuyuwjW/v52XbA/8xsxS8VpgfB3m3RKSN6SzIItJmzGync66d33WISGRQd4+IiIiEJbWkiIiISFhSS4qIiIiEJYUUERERCUsKKSIiIhKWFFJEREQkLCmkiIiISFj6/7OTeHU2hQ5tAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (9, 7))\n",
    "plt.plot(list(training_history.keys()), [training_history[k]['loss'] for k in training_history.keys()], label = 'train loss')\n",
    "plt.plot(list(training_history.keys()), [training_history[k]['val_loss'] for k in training_history.keys()], label = 'val loss')\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend(loc = 'best')\n",
    "plt.title(\"Vgg-18 Training with Early Stopping\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "yJkajjh9QGKK"
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "0G1TaNnnQHDI"
   },
   "outputs": [],
   "source": [
    "with open(\"VGG18_dropout_earlystopping_training_history.pkl\", \"wb\") as file:\n",
    "    pickle.dump(training_history, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DiAv9vgBQigh"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MuBmbE6CQuPO"
   },
   "outputs": [],
   "source": [
    "# Save trained weights-\n",
    "# torch.save(model.state_dict(), 'VGG18_trained_weights.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "tSOrFC5gQz9b"
   },
   "outputs": [],
   "source": [
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "h0Fbk-V3Q0H8"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WC2enWfbT5wt"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model with _learning rate scheduler_\n",
    "\n",
    "- Training dataset = 50000, batch size = 128, number of training steps/iterations = 50000 / 128 = 391\n",
    "\n",
    "- Initial learning rate warmup: 391 x 10 = 3910 steps or, 10 epochs at LR = 0.1\n",
    "\n",
    "- Until 25th epoch or, 9775 steps use LR = 0.1\n",
    "\n",
    "- From 26th epoch until 40th epoch or, 15640 steps use LR = 0.01\n",
    "\n",
    "- From 41st epoch until 50th epoch or, 19550 steps use LR = 0.001\n",
    "\n",
    "- From 51st epoch until 60th epoch use LR = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize VGG-18 CNN model-\n",
    "model = nn.Sequential(\n",
    "    VGGBlock(input_channels = 3, num_channels = 64, dropout = 0.0),\n",
    "    VGGBlock(input_channels = 64, num_channels = 128, dropout = 0.2),\n",
    "    VGGBlock(input_channels = 128, num_channels = 256, use_extra_conv = True, dropout = 0.3),\n",
    "    VGGBlock(input_channels = 256, num_channels = 512, use_extra_conv = True, dropout = 0.3),\n",
    "    VGGBlock(input_channels = 512, num_channels = 512, use_extra_conv = True, dropout = 0.3),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(in_features = 512, out_features = 256),\n",
    "    nn.Dropout(p = 0.3),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(in_features = 256, out_features = 10)\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): VGGBlock(\n",
       "    (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.0, inplace=False)\n",
       "    (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (1): VGGBlock(\n",
       "    (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.2, inplace=False)\n",
       "    (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (2): VGGBlock(\n",
       "    (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.3, inplace=False)\n",
       "    (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (3): VGGBlock(\n",
       "    (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.3, inplace=False)\n",
       "    (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (4): VGGBlock(\n",
       "    (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (dropout): Dropout(p=0.3, inplace=False)\n",
       "    (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (conv4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (bn4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (5): Flatten(start_dim=1, end_dim=-1)\n",
       "  (6): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (7): Dropout(p=0.3, inplace=False)\n",
       "  (8): ReLU()\n",
       "  (9): Linear(in_features=256, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Place model on GPU-\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load randomly initialised weights-\n",
    "model.load_state_dict(torch.load('VGG18_random_weights_dropouts.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "boundaries = [9775, 15640, 19550]\n",
    "values = [0.1, 0.01, 0.001, 0.0001]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "OozQ7SF4st4U"
   },
   "outputs": [],
   "source": [
    "# Define loss function and optimizer-\n",
    "loss = nn.CrossEntropyLoss()\n",
    "\n",
    "# optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = 0.0, momentum = 0.9, weight_decay = 5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check-\n",
    "optimizer.param_groups[0]['lr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decay_function(step, boundaries = [9775, 15640, 19550], values = [0.1, 0.01, 0.001, 0.0001]):\n",
    "    '''\n",
    "    1 epoch has 391 steps/iterations using batch size used above.\n",
    "    \n",
    "    Until 25th epochs, or 25 x 391 = 9775 steps, use lr = 0.1\n",
    "    From 26th epoch until 40th epoch, or 15640 steps use LR = 0.01\n",
    "    From 41st epoch until 50th epoch or, 19550 steps use LR = 0.001\n",
    "    From 51st epoch until 60th epoch use LR = 0.0001\n",
    "    '''\n",
    "    \n",
    "    for idx, bound in enumerate(boundaries):\n",
    "        if step < bound:\n",
    "            return values[idx]\n",
    "\n",
    "    return values[-1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "def decay_function(step, boundaries = [15640, 19550], values = [0.01, 0.001, 0.0001]):\n",
    "    '''\n",
    "    1 epoch has 391 steps/iterations using batch size used above.\n",
    "    \n",
    "    Until 40 epochs, or 40 x 391 = 15640 steps, use lr = 0.01\n",
    "    Until 50 epochs, or 50 x 391 = 19550 steps, use lr = 0.001\n",
    "    \n",
    "    For any remaining steps, use lr = 0.0001\n",
    "    '''\n",
    "    \n",
    "    for idx, bound in enumerate(boundaries):\n",
    "        if step < bound:\n",
    "            return values[idx]\n",
    "\n",
    "    return values[-1]\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decay_function(step = 2, boundaries = [0, 2, 4, 6, 8, 10], values = [10, 15, 20, 30, 40, 50, 60])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class schedule():\n",
    "\n",
    "    def __init__(self, initial_learning_rate = 0.1, warmup_steps = 1000, decay_func = None):\n",
    "        self.initial_learning_rate = initial_learning_rate\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.decay_func = decay_func\n",
    "        self.warmup_step_size = initial_learning_rate/warmup_steps\n",
    "        self.current_lr = 0\n",
    "\n",
    "    def get_lr(self, step):\n",
    "        if step == 0:\n",
    "            return self.current_lr\n",
    "        elif step <= self.warmup_steps:\n",
    "            self.current_lr+=self.warmup_step_size\n",
    "            return self.current_lr\n",
    "        elif step > self.warmup_steps:\n",
    "            if self.decay_func:\n",
    "                return self.decay_func(step)\n",
    "        else:\n",
    "            return self.current_lr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 391 x 10 = 3910 steps (or, 10 epochs) is learning rate warmup\n",
    "custom_lr_scheduler = schedule(\n",
    "    initial_learning_rate = 0.1, warmup_steps = 3910,\n",
    "    decay_func = decay_function\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "step = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "0DyTwwRhcOeY"
   },
   "outputs": [],
   "source": [
    "def train_model_progress(model, train_loader):\n",
    "    '''\n",
    "    Function to perform one epoch of training by using 'train_loader'.\n",
    "    Returns loss and number of correct predictions for this epoch.\n",
    "    '''\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0.0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    with tqdm(train_loader, unit = 'batch') as tepoch:\n",
    "        for images, labels in tepoch:\n",
    "            tepoch.set_description(f\"Training: \")\n",
    "            \n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "            \n",
    "            # Get model predictions-\n",
    "            outputs = model(images)\n",
    "            \n",
    "            # Compute loss-\n",
    "            J = loss(outputs, labels)\n",
    "            \n",
    "            # Empty accumulated gradients-\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            # Perform backprop-\n",
    "            J.backward()\n",
    "            \n",
    "            # Update parameters-\n",
    "            optimizer.step()\n",
    "            \n",
    "            global step\n",
    "            optimizer.param_groups[0]['lr'] = custom_lr_scheduler.get_lr(step)\n",
    "\n",
    "            step += 1\n",
    "            \n",
    "            # Compute model's performance statistics-\n",
    "            running_loss += J.item() * images.size(0)\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            running_corrects += torch.sum(predicted == labels.data)\n",
    "            \n",
    "            tepoch.set_postfix(\n",
    "                loss = running_loss / len(train_dataset),\n",
    "                accuracy = (running_corrects.double().cpu().numpy() / len(train_dataset)) * 100\n",
    "            )\n",
    "            \n",
    "    \n",
    "    train_loss = running_loss / len(train_dataset)\n",
    "    train_acc = (running_corrects.double() / len(train_dataset)) * 100\n",
    "    \n",
    "\n",
    "    # return running_loss, running_corrects\n",
    "    return train_loss, train_acc.cpu().numpy()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uCq8FAdOckD6"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "YymBc1UtcnEA"
   },
   "outputs": [],
   "source": [
    "def test_model_progress(model, test_loader):\n",
    "    total = 0.0\n",
    "    correct = 0.0\n",
    "    running_loss_val = 0.0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        with tqdm(test_loader, unit = 'batch') as tepoch:\n",
    "            for images, labels in tepoch:\n",
    "                tepoch.set_description(f\"Validation: \")\n",
    "                \n",
    "                images = images.to(device)\n",
    "                labels = labels.to(device)\n",
    "                \n",
    "                # Set model to evaluation mode-\n",
    "                model.eval()\n",
    "            \n",
    "                # Predict using trained model-\n",
    "                outputs = model(images)\n",
    "                _, y_pred = torch.max(outputs, 1)\n",
    "                \n",
    "                # Compute validation loss-\n",
    "                J_val = loss(outputs, labels)\n",
    "                \n",
    "                running_loss_val += J_val.item() * labels.size(0)\n",
    "    \n",
    "                # Total number of labels-\n",
    "                total += labels.size(0)\n",
    "\n",
    "                # Total number of correct predictions-\n",
    "                correct += (y_pred == labels).sum()\n",
    "                \n",
    "                tepoch.set_postfix(\n",
    "                    val_loss = running_loss_val / len(test_dataset),\n",
    "                    val_acc = 100 * (correct.cpu().numpy() / total)\n",
    "                )\n",
    "            \n",
    "        \n",
    "    # return (running_loss_val, correct, total)\n",
    "    val_loss = running_loss_val / len(test_dataset)\n",
    "    val_acc = (correct / total) * 100\n",
    "\n",
    "    return val_loss, val_acc.cpu().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_history_lr_scheduler = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "NOytz_v9tPxV"
   },
   "outputs": [],
   "source": [
    "# Initialize parameters for Early Stopping manual implementation-\n",
    "best_val_loss = 100\n",
    "loc_patience = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:44<00:00,  8.75batch/s, accuracy=16.8, loss=2.1]   \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.32batch/s, val_acc=10, val_loss=3.21]   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 1 training loss = 2.0969, training accuracy = 16.79%, val_loss = 3.2113, val_accuracy = 10.00% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 3.2113\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.63batch/s, accuracy=22.2, loss=1.88]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.17batch/s, val_acc=10, val_loss=4.37]   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 2 training loss = 1.8845, training accuracy = 22.17%, val_loss = 4.3712, val_accuracy = 10.00% & LR = 0.0200\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.45batch/s, accuracy=33.8, loss=1.67]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.37batch/s, val_acc=11.2, val_loss=2.77] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 3 training loss = 1.6718, training accuracy = 33.76%, val_loss = 2.7710, val_accuracy = 11.19% & LR = 0.0300\n",
      "\n",
      "Saving model with lowest val_loss = 2.7710\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=45.7, loss=1.45]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.95batch/s, val_acc=24.3, val_loss=2.14] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 4 training loss = 1.4486, training accuracy = 45.67%, val_loss = 2.1377, val_accuracy = 24.28% & LR = 0.0400\n",
      "\n",
      "Saving model with lowest val_loss = 2.1377\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.38batch/s, accuracy=56.8, loss=1.24]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.93batch/s, val_acc=54.6, val_loss=1.38] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 5 training loss = 1.2442, training accuracy = 56.84%, val_loss = 1.3811, val_accuracy = 54.56% & LR = 0.0500\n",
      "\n",
      "Saving model with lowest val_loss = 1.3811\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.37batch/s, accuracy=63.4, loss=1.09]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.60batch/s, val_acc=58.9, val_loss=1.29] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 6 training loss = 1.0901, training accuracy = 63.39%, val_loss = 1.2909, val_accuracy = 58.89% & LR = 0.0600\n",
      "\n",
      "Saving model with lowest val_loss = 1.2909\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.41batch/s, accuracy=67.5, loss=0.987] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.00batch/s, val_acc=63.6, val_loss=1.13] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 7 training loss = 0.9868, training accuracy = 67.51%, val_loss = 1.1340, val_accuracy = 63.56% & LR = 0.0700\n",
      "\n",
      "Saving model with lowest val_loss = 1.1340\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.38batch/s, accuracy=70.2, loss=0.912] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.17batch/s, val_acc=51.3, val_loss=1.48] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 8 training loss = 0.9121, training accuracy = 70.20%, val_loss = 1.4816, val_accuracy = 51.32% & LR = 0.0800\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.32batch/s, accuracy=72.6, loss=0.855] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.29batch/s, val_acc=66.5, val_loss=1.04] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 9 training loss = 0.8554, training accuracy = 72.59%, val_loss = 1.0441, val_accuracy = 66.54% & LR = 0.0900\n",
      "\n",
      "Saving model with lowest val_loss = 1.0441\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.36batch/s, accuracy=73.6, loss=0.835] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.61batch/s, val_acc=67.5, val_loss=1.04] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 10 training loss = 0.8350, training accuracy = 73.58%, val_loss = 1.0413, val_accuracy = 67.50% & LR = 0.1000\n",
      "\n",
      "Saving model with lowest val_loss = 1.0413\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.38batch/s, accuracy=74.8, loss=0.792] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.35batch/s, val_acc=70, val_loss=0.973]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 11 training loss = 0.7921, training accuracy = 74.79%, val_loss = 0.9731, val_accuracy = 70.01% & LR = 0.1000\n",
      "\n",
      "Saving model with lowest val_loss = 0.9731\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.41batch/s, accuracy=75.7, loss=0.766] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.82batch/s, val_acc=63.2, val_loss=1.23] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 12 training loss = 0.7663, training accuracy = 75.68%, val_loss = 1.2288, val_accuracy = 63.23% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.48batch/s, accuracy=76.3, loss=0.754] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.86batch/s, val_acc=60, val_loss=1.26]   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 13 training loss = 0.7542, training accuracy = 76.26%, val_loss = 1.2585, val_accuracy = 60.04% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=77, loss=0.737]   \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.64batch/s, val_acc=70.5, val_loss=1.03] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 14 training loss = 0.7370, training accuracy = 76.95%, val_loss = 1.0251, val_accuracy = 70.45% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=77.1, loss=0.736] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.63batch/s, val_acc=74.6, val_loss=0.847]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 15 training loss = 0.7363, training accuracy = 77.15%, val_loss = 0.8468, val_accuracy = 74.56% & LR = 0.1000\n",
      "\n",
      "Saving model with lowest val_loss = 0.8468\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=77.5, loss=0.72]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.50batch/s, val_acc=77.7, val_loss=0.815]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 16 training loss = 0.7202, training accuracy = 77.47%, val_loss = 0.8145, val_accuracy = 77.66% & LR = 0.1000\n",
      "\n",
      "Saving model with lowest val_loss = 0.8145\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.42batch/s, accuracy=78.1, loss=0.705] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.56batch/s, val_acc=63.3, val_loss=1.12] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 17 training loss = 0.7049, training accuracy = 78.10%, val_loss = 1.1165, val_accuracy = 63.30% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.44batch/s, accuracy=78.1, loss=0.704] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.72batch/s, val_acc=73.4, val_loss=0.851]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 18 training loss = 0.7040, training accuracy = 78.10%, val_loss = 0.8506, val_accuracy = 73.42% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.49batch/s, accuracy=78.2, loss=0.693] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.90batch/s, val_acc=72.7, val_loss=0.897]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 19 training loss = 0.6932, training accuracy = 78.24%, val_loss = 0.8967, val_accuracy = 72.71% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=78.6, loss=0.691] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.85batch/s, val_acc=77.7, val_loss=0.773] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 20 training loss = 0.6908, training accuracy = 78.59%, val_loss = 0.7733, val_accuracy = 77.73% & LR = 0.1000\n",
      "\n",
      "Saving model with lowest val_loss = 0.7733\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=79.1, loss=0.676] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.97batch/s, val_acc=68.2, val_loss=1.05] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 21 training loss = 0.6759, training accuracy = 79.06%, val_loss = 1.0453, val_accuracy = 68.22% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=78.8, loss=0.679] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.01batch/s, val_acc=68.3, val_loss=1.01] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 22 training loss = 0.6788, training accuracy = 78.81%, val_loss = 1.0072, val_accuracy = 68.30% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.52batch/s, accuracy=79.1, loss=0.672] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.21batch/s, val_acc=66.9, val_loss=1]    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 23 training loss = 0.6720, training accuracy = 79.13%, val_loss = 1.0000, val_accuracy = 66.86% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=79.5, loss=0.663] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.76batch/s, val_acc=72.5, val_loss=0.907]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 24 training loss = 0.6632, training accuracy = 79.48%, val_loss = 0.9073, val_accuracy = 72.48% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=79.7, loss=0.656] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.44batch/s, val_acc=72.7, val_loss=0.852]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 25 training loss = 0.6560, training accuracy = 79.69%, val_loss = 0.8516, val_accuracy = 72.65% & LR = 0.1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=85.8, loss=0.447] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.78batch/s, val_acc=86.8, val_loss=0.48]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 26 training loss = 0.4468, training accuracy = 85.83%, val_loss = 0.4801, val_accuracy = 86.84% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.4801\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=87.6, loss=0.391] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.64batch/s, val_acc=87.7, val_loss=0.437] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 27 training loss = 0.3907, training accuracy = 87.62%, val_loss = 0.4374, val_accuracy = 87.71% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.4374\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=88.3, loss=0.359] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.49batch/s, val_acc=88.3, val_loss=0.425] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 28 training loss = 0.3589, training accuracy = 88.33%, val_loss = 0.4252, val_accuracy = 88.28% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.4252\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.45batch/s, accuracy=88.9, loss=0.343]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.24batch/s, val_acc=87.6, val_loss=0.426] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 29 training loss = 0.3427, training accuracy = 88.87%, val_loss = 0.4260, val_accuracy = 87.58% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.41batch/s, accuracy=89.3, loss=0.329]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.74batch/s, val_acc=88.5, val_loss=0.399] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 30 training loss = 0.3292, training accuracy = 89.30%, val_loss = 0.3992, val_accuracy = 88.52% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3992\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=89.8, loss=0.314]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.49batch/s, val_acc=89.5, val_loss=0.377] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 31 training loss = 0.3137, training accuracy = 89.81%, val_loss = 0.3766, val_accuracy = 89.55% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3766\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=90, loss=0.306]    \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.86batch/s, val_acc=89.1, val_loss=0.38]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 32 training loss = 0.3055, training accuracy = 90.03%, val_loss = 0.3799, val_accuracy = 89.14% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=90.5, loss=0.297]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.74batch/s, val_acc=89.8, val_loss=0.374] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 33 training loss = 0.2965, training accuracy = 90.51%, val_loss = 0.3740, val_accuracy = 89.76% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3740\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.41batch/s, accuracy=90.8, loss=0.286]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.18batch/s, val_acc=89.4, val_loss=0.356] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 34 training loss = 0.2861, training accuracy = 90.83%, val_loss = 0.3565, val_accuracy = 89.38% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3565\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.45batch/s, accuracy=91.1, loss=0.277]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.16batch/s, val_acc=89.7, val_loss=0.364] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 35 training loss = 0.2773, training accuracy = 91.13%, val_loss = 0.3644, val_accuracy = 89.72% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.44batch/s, accuracy=91.1, loss=0.275]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.74batch/s, val_acc=88.8, val_loss=0.37]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 36 training loss = 0.2752, training accuracy = 91.07%, val_loss = 0.3703, val_accuracy = 88.81% & LR = 0.0100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=91.4, loss=0.266]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.59batch/s, val_acc=89.6, val_loss=0.353] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 37 training loss = 0.2662, training accuracy = 91.39%, val_loss = 0.3527, val_accuracy = 89.62% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3527\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=91.5, loss=0.259]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.85batch/s, val_acc=90.1, val_loss=0.351] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 38 training loss = 0.2588, training accuracy = 91.47%, val_loss = 0.3511, val_accuracy = 90.07% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3511\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.44batch/s, accuracy=91.7, loss=0.255]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.18batch/s, val_acc=89.8, val_loss=0.35]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 39 training loss = 0.2552, training accuracy = 91.70%, val_loss = 0.3497, val_accuracy = 89.79% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3497\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=91.8, loss=0.253]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.71batch/s, val_acc=90.4, val_loss=0.334] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 40 training loss = 0.2525, training accuracy = 91.77%, val_loss = 0.3336, val_accuracy = 90.42% & LR = 0.0100\n",
      "\n",
      "Saving model with lowest val_loss = 0.3336\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.42batch/s, accuracy=93.1, loss=0.215] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.10batch/s, val_acc=91, val_loss=0.305]   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 41 training loss = 0.2151, training accuracy = 93.10%, val_loss = 0.3048, val_accuracy = 91.05% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.3048\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.43batch/s, accuracy=93.4, loss=0.205] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.16batch/s, val_acc=91.2, val_loss=0.294] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 42 training loss = 0.2051, training accuracy = 93.35%, val_loss = 0.2937, val_accuracy = 91.24% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2937\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.40batch/s, accuracy=93.6, loss=0.198]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.95batch/s, val_acc=91.3, val_loss=0.291] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 43 training loss = 0.1975, training accuracy = 93.55%, val_loss = 0.2907, val_accuracy = 91.30% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2907\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.45batch/s, accuracy=93.7, loss=0.195]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.80batch/s, val_acc=91.2, val_loss=0.289] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 44 training loss = 0.1950, training accuracy = 93.67%, val_loss = 0.2891, val_accuracy = 91.25% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2891\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=93.8, loss=0.191]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.21batch/s, val_acc=91.2, val_loss=0.287] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 45 training loss = 0.1908, training accuracy = 93.82%, val_loss = 0.2873, val_accuracy = 91.18% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2873\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=93.6, loss=0.195]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.12batch/s, val_acc=91.2, val_loss=0.29]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 46 training loss = 0.1945, training accuracy = 93.63%, val_loss = 0.2895, val_accuracy = 91.16% & LR = 0.0010\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=93.9, loss=0.186]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.75batch/s, val_acc=91.3, val_loss=0.287] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 47 training loss = 0.1856, training accuracy = 93.90%, val_loss = 0.2867, val_accuracy = 91.27% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2867\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.48batch/s, accuracy=94.1, loss=0.181]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.07batch/s, val_acc=91.3, val_loss=0.281] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 48 training loss = 0.1814, training accuracy = 94.09%, val_loss = 0.2807, val_accuracy = 91.26% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2807\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.51batch/s, accuracy=94.1, loss=0.182] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.33batch/s, val_acc=91.4, val_loss=0.285] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 49 training loss = 0.1818, training accuracy = 94.06%, val_loss = 0.2846, val_accuracy = 91.37% & LR = 0.0010\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.48batch/s, accuracy=94.1, loss=0.181]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.23batch/s, val_acc=91.5, val_loss=0.278] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 50 training loss = 0.1814, training accuracy = 94.07%, val_loss = 0.2778, val_accuracy = 91.49% & LR = 0.0010\n",
      "\n",
      "Saving model with lowest val_loss = 0.2778\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.44batch/s, accuracy=94.1, loss=0.179]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.05batch/s, val_acc=91.4, val_loss=0.28]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 51 training loss = 0.1793, training accuracy = 94.14%, val_loss = 0.2800, val_accuracy = 91.39% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=94.2, loss=0.178]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.56batch/s, val_acc=91.5, val_loss=0.277] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 52 training loss = 0.1777, training accuracy = 94.19%, val_loss = 0.2775, val_accuracy = 91.50% & LR = 0.0001\n",
      "\n",
      "Saving model with lowest val_loss = 0.2775\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.47batch/s, accuracy=94.3, loss=0.176]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.86batch/s, val_acc=91.6, val_loss=0.277] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 53 training loss = 0.1758, training accuracy = 94.31%, val_loss = 0.2774, val_accuracy = 91.57% & LR = 0.0001\n",
      "\n",
      "Saving model with lowest val_loss = 0.2774\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.49batch/s, accuracy=94.4, loss=0.174]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.05batch/s, val_acc=91.6, val_loss=0.276] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 54 training loss = 0.1738, training accuracy = 94.37%, val_loss = 0.2763, val_accuracy = 91.61% & LR = 0.0001\n",
      "\n",
      "Saving model with lowest val_loss = 0.2763\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.48batch/s, accuracy=94.3, loss=0.177]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.94batch/s, val_acc=91.6, val_loss=0.272] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 55 training loss = 0.1771, training accuracy = 94.28%, val_loss = 0.2720, val_accuracy = 91.64% & LR = 0.0001\n",
      "\n",
      "Saving model with lowest val_loss = 0.2720\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.52batch/s, accuracy=94.2, loss=0.179]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.11batch/s, val_acc=91.7, val_loss=0.274] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 56 training loss = 0.1791, training accuracy = 94.17%, val_loss = 0.2740, val_accuracy = 91.71% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.45batch/s, accuracy=94.2, loss=0.176]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.31batch/s, val_acc=91.8, val_loss=0.273] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 57 training loss = 0.1765, training accuracy = 94.23%, val_loss = 0.2733, val_accuracy = 91.75% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.37batch/s, accuracy=94.3, loss=0.177]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.51batch/s, val_acc=91.4, val_loss=0.277] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 58 training loss = 0.1767, training accuracy = 94.31%, val_loss = 0.2771, val_accuracy = 91.44% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.46batch/s, accuracy=94.4, loss=0.174]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.38batch/s, val_acc=91.5, val_loss=0.276] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 59 training loss = 0.1737, training accuracy = 94.37%, val_loss = 0.2755, val_accuracy = 91.51% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.56batch/s, accuracy=94.3, loss=0.174]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.17batch/s, val_acc=91.4, val_loss=0.276] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 60 training loss = 0.1741, training accuracy = 94.31%, val_loss = 0.2758, val_accuracy = 91.42% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.48batch/s, accuracy=94.5, loss=0.172]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.06batch/s, val_acc=91.5, val_loss=0.278] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 61 training loss = 0.1716, training accuracy = 94.45%, val_loss = 0.2777, val_accuracy = 91.55% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.55batch/s, accuracy=94.3, loss=0.175]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.78batch/s, val_acc=91.7, val_loss=0.272] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 62 training loss = 0.1753, training accuracy = 94.30%, val_loss = 0.2715, val_accuracy = 91.69% & LR = 0.0001\n",
      "\n",
      "Saving model with lowest val_loss = 0.2715\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:46<00:00,  8.49batch/s, accuracy=94.2, loss=0.177]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.80batch/s, val_acc=91.4, val_loss=0.277] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 63 training loss = 0.1768, training accuracy = 94.16%, val_loss = 0.2771, val_accuracy = 91.40% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.51batch/s, accuracy=94.4, loss=0.172]  \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 21.00batch/s, val_acc=91.6, val_loss=0.278] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 64 training loss = 0.1724, training accuracy = 94.40%, val_loss = 0.2781, val_accuracy = 91.60% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: : 100%|██████████| 391/391 [00:45<00:00,  8.50batch/s, accuracy=94.5, loss=0.172] \n",
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.80batch/s, val_acc=91.5, val_loss=0.275] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "epoch: 65 training loss = 0.1723, training accuracy = 94.55%, val_loss = 0.2749, val_accuracy = 91.53% & LR = 0.0001\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(num_epochs):\n",
    "\n",
    "    train_loss, train_acc = train_model_progress(model, train_loader)\n",
    "    val_loss, val_acc = test_model_progress(model, test_loader)\n",
    "    \n",
    "    print(f\"\\nepoch: {epoch + 1} training loss = {train_loss:.4f}, \"\n",
    "          f\"training accuracy = {train_acc:.2f}%, val_loss = {val_loss:.4f}\"\n",
    "          f\", val_accuracy = {val_acc:.2f}% & \"\n",
    "          f\"LR = {optimizer.param_groups[0]['lr']:.4f}\\n\")\n",
    "    \n",
    "    training_history_lr_scheduler[epoch + 1] = {\n",
    "        'loss': train_loss, 'acc': train_acc,\n",
    "        'val_loss': val_loss, 'val_acc': val_acc,\n",
    "        'lr': optimizer.param_groups[0]['lr']\n",
    "    }\n",
    "\n",
    "    \n",
    "    # Save best weights achieved until now-\n",
    "    if (val_loss < best_val_loss):    \n",
    "        # update 'best_val_loss' variable to lowest loss encountered so far-\n",
    "        best_val_loss = val_loss\n",
    "\n",
    "        print(f\"Saving model with lowest val_loss = {val_loss:.4f}\\n\")\n",
    "        \n",
    "        # Save trained model with 'best' validation accuracy-\n",
    "        torch.save(model.state_dict(), \"VGG18_best_model.pth\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save model from last training epoch-\n",
    "# torch.save(model.state_dict(), \"VGG18_last_epoch_model.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "# Sanity check- Load model from before-\n",
    "best_model = nn.Sequential(\n",
    "    VGGBlock(input_channels = 3, num_channels = 64),\n",
    "    VGGBlock(input_channels = 64, num_channels = 128),\n",
    "    VGGBlock(input_channels = 128, num_channels = 256, use_extra_conv = True),\n",
    "    VGGBlock(input_channels = 256, num_channels = 512, use_extra_conv = True),\n",
    "    VGGBlock(input_channels = 512, num_channels = 512, use_extra_conv = True),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(in_features = 512, out_features = 256),\n",
    "    nn.Linear(in_features = 256, out_features = 10)\n",
    "    )\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best_model.load_state_dict(torch.load('VGG18_best_model.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Place model on GPU-\n",
    "# best_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.76batch/s, val_acc=91.8, val_loss=0.252] \n"
     ]
    }
   ],
   "source": [
    "# val_loss, val_acc = test_model_progress(best_model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(f\"VGG-18 'best' model metrics: val_loss = {val_loss:.4f} & val_acc = {val_acc:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# del best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "id": "RMcYwZYBr8io"
   },
   "outputs": [],
   "source": [
    "'''\n",
    "# Sanity check- Load model from before-\n",
    "trained_model = nn.Sequential(\n",
    "    VGGBlock(input_channels = 3, num_channels = 64),\n",
    "    VGGBlock(input_channels = 64, num_channels = 128),\n",
    "    VGGBlock(input_channels = 128, num_channels = 256, use_extra_conv = True),\n",
    "    VGGBlock(input_channels = 256, num_channels = 512, use_extra_conv = True),\n",
    "    VGGBlock(input_channels = 512, num_channels = 512, use_extra_conv = True),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(in_features = 512, out_features = 256),\n",
    "    nn.Linear(in_features = 256, out_features = 10)\n",
    "    )\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trained_model.load_state_dict(torch.load('VGG18_last_epoch_model.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Place model on GPU-\n",
    "# trained_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Validation: : 100%|██████████| 79/79 [00:03<00:00, 20.96batch/s, val_acc=92.9, val_loss=0.278] \n"
     ]
    }
   ],
   "source": [
    "# val_loss, val_acc = test_model_progress(trained_model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VGG-18 trained model metrics: val_loss = 0.2782 & val_acc = 92.92%\n"
     ]
    }
   ],
   "source": [
    "# print(f\"VGG-18 trained model metrics: val_loss = {val_loss:.4f} & val_acc = {val_acc:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# del trained_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check-\n",
    "training_history_lr_scheduler.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['loss', 'acc', 'val_loss', 'val_acc', 'lr'])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_history_lr_scheduler[12].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save training metrics as Python3 history for later analysis-\n",
    "with open(\"VGG18_dropout_training_history_lr_scheduler.pkl\", \"wb\") as file:\n",
    "    pickle.dump(training_history_lr_scheduler, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Visualizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAG5CAYAAAC6Fv9NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABav0lEQVR4nO3dd5xddZ3/8ddneu8pkzpJCCSkEkITBAREigoCIhZEVGy467qWddUVy7rLrmVd3bWAyg8rIEVYKULA0FuAQEIKIZlJMmnTe733fn9/fO+0ZMqdmXtn5t68n4/HPM6955x77vcehtz3fKs55xARERGZCpImuwAiIiIiPRRMREREZMpQMBEREZEpQ8FEREREpgwFExEREZkyFExERERkylAwEZEJZWZfNbNfRvtcEUkMCiYiUWRmfzWzbw+y/xIzO2hmKeHna83sL2ZWb2YNZrbFzL5rZoX9XlNqZjeb2X4zazGzXWb2/8xsyTDvf6WZPWNmbWa2fpDj55jZy2bWFL7eJyL8XA+Gy9BiZt1m1tXv+c8jujlhzrl/c859PNrnjoV5u8xsS6zeQ0RGR8FEJLr+H3C1mdlh+68Gfu+cC5jZW4D1wNPAEudcAXABEABWAZhZMfAMkAW8FcgF1gCPA28f5v3rgB8BNx5+wMxSgXuAXwD5wPuAH5rZqpE+lHPuQudcjnMuB/g98J89z51zn+r3HikjXWuKOROYDiw0s5Mm8o3j8F6JTAgFE5Ho+jNQhA8TAIRrQd4J/Ca86z+BW5xz/+6cOwTgnNvjnLvBObc+fM7ngSbgaufcTuc1OOducc79ZKg3d86tc87dAewf5HARkAf8Nny9F4GtwPHj+LyYmTOz681sB7AjvO+/zWxvuGbmJTPrfz++aWa/Cz8uC7/+GjPbY2Y1Zva1MZ6baWa3hmuhtprZl82scoTiXwPcCzwQftz/cy0zs0fMrM7MDpnZV8P7k8NNTDvNrDn8+eb2K19Kv2usN7OPhx9/xMyeNrP/MrM64JtmtsjMHjOz2vDn+b2ZFfR7/Vwzu9vMqsPn/I+ZpYfLtKLfedPNrN3MpkX4n01kylIwEYki51w7cAfw4X67rwS2OedeNbNs4DTgrhEudR5wj3MuFMWyHQL+CFwb/nI9DZgPPAVgZmeYWcMYL38pcAp9IedFYDU+DP0B+JOZZQzz+jOA44BzgW+Y2dIxnHsDUAYsxNcqfWi4AptZFnAFvgbo98BVZpYWPpYLrAMeAmYBxwCPhl/6j8D7gYvwQe+jQNtw79XPKcAufC3NdwED/j38HkuBucA3w2VIBv4C7A5/rtnAbc65TuC2wz7f+4F1zrnqCMshMmUpmIhE363Ae80sM/z8w+F9AIX4/+8O9pxsZv9pvp9Jq5l9Pby75LBz3h0+p9nMHh5H2f4IfAPoBJ4Evuac2wvgnHsq3Kw0Fv/unKsLBzOcc79zztU65wLOuR8A6fgwMZRvOefanXOvAq8SbtIa5blXAv/mnKt3zlUCPx6hzJfh78PD+ACQAlwcPvZO4KBz7gfOuQ7nXLNz7vnwsY8DX3fObQ/XPL3qnKsd4b167HfO/SR8X9qdc2865x5xznWGQ8UPgbPC556MDyxfcs61hsvxVPjYrcAHzKzn3/Crgd9GWAaRKU3BRCTKwl8e1cAlZrYQOAlfawBQD4SA0n7nfzkcCO7BfzkC1B52zn3hcz4P9PxV//N+HVC/OlK5zHeavR0flNKAZcCXzeziYV8Ymb2HvdcXws0pjeFamHx82BrKwX6P24CcMZw767ByDCjTIK4B7giHhE7gbvqac+YCO4d43XDHRnL4fZpuZreZ2T4zawJ+R999mgvsds4FDr9IOCS1AmeF/7seA9w3xjKJTCkKJiKx8Rt8ALgaeLhfX5JW4Hn8X+vDeRS4tN9fxEdwzn2qXwfUf4ugTMuB7c65vzrnQs657cD9wIURvHYkvcuUh/uT/BO+BqMwHKga8c0WsXQAmNPv+dyhTjSzOcA5wIfMj5Y6iG/WucjMSvABYtEQLx/qWGt4m9Vv38zDzjl8Ofd/D+9b6ZzLwzfP9NynvcC8YTrJ3ho+/2rgTudcxxDnicQVBROR2PgNvp/IdfQ14/T4MvBRM/uKmU2H3i/KBf3O+SG+2ee34Q6SFu73sHq4Nw33HcnA17wkmVmG+dE4AK8Ai80PGTYzW4Rvsnh1XJ/0SLn4EUbVQIqZfQPfFyPW7gD+2cwKzWw28Nlhzr0aeAPfvLQ6/HMsUInvr/EXYKaZ/UO4s2mumZ0Sfu0vge+Y2eLwfVxpZsXhpph9+LCTbGYfZehw0yMXaAEawmX+Ur9jL+DD1o1mlh3+b3l6v+O/Bd6DDye/QSRBKJiIxIBzrgI/3Debw6rYw0095+CHqr4Rbup4CD+E+Cfhc2qAU4EOfOfUZmAj/ovs08O89dVAO/Az/MigduDm8DV34jtq/hg/4udxfCfcX4Gv6TCzlrF/6l5/BR7Ef/HvDn+GkZpVouHb+GBRju+4eie+D8lgrgF+6pw72P8H+DlwjXOuGd+B9l34pqMdwNvCr/0hPgQ9jL+PvwJ6+hNdhw8XtfimsmdGKPO38MPAG/G1V3f3HHDOBcPvfwywJ/zZ3tfveCXwMr7G5ckR3kckbphzh9csiojEPzP7NHCVc+6sEU+OU2b2a3yH2q+PeLJInNAEPyKSEMysFD9U+FlgMfAF4H8mtVAxZGZl+L5KJ0xyUUSiSk05IpIo0vCz2jYDj+EnTvvppJYoRszsO8Bm4HvOufLJLo9INKkpR0RERKYM1ZiIiIjIlBEXfUxKSkpcWVnZZBdDREREouCll16qcc4NurZTXASTsrIyNmzYMNnFEBERkSgws91DHVNTjoiIiEwZCiYiIiIyZSiYiIiIyJQRF31MBtPd3U1lZSUdHVq3aiJlZGQwZ84cUlNTRz5ZRERklOI2mFRWVpKbm0tZWRlmsV60VACcc9TW1lJZWcmCBQtGfoGIiMgoxW1TTkdHB8XFxQolE8jMKC4uVi2ViIjETNwGE0ChZBLonouISCzFdTARERGRxKJgMkYNDQ389KdjWx/soosuoqGhYdhzvvGNb7Bu3boxXV9ERCReKZiM0XDBJBgMDvvaBx54gIKCgmHP+fa3v81555031uKJiIjEJQWTMfrKV77Czp07Wb16NV/60pdYv349b3vb2/jABz7AihUrALj00ks58cQTWbZsGTfddFPva8vKyqipqaGiooKlS5dy3XXXsWzZMs4//3za29sB+MhHPsKdd97Ze/4NN9zAmjVrWLFiBdu2bQOgurqat7/97axZs4ZPfvKTzJ8/n5qamiPK+ulPf5q1a9eybNkybrjhht79L774Im95y1tYtWoVJ598Ms3NzQSDQb74xS+yYsUKVq5cyU9+8pOY3UMREZHDxe1w4f6+9X+vs2V/U1SvefysPG5417Ihj994441s3ryZjRs3ArB+/XpeeOEFNm/e3DuU9te//jVFRUW0t7dz0kkncfnll1NcXDzgOjt27OCPf/wjN998M1deeSV33XUXH/rQh454v5KSEl5++WV++tOf8v3vf59f/vKXfOtb3+Kcc87hn//5n3nooYcGhJ/+vvvd71JUVEQwGOTcc8/ltddeY8mSJbzvfe/j9ttv56STTqKpqYnMzExuuukmysvLeeWVV0hJSaGurm6Md1BERGT0EiKYTBUnn3zygPk9fvzjH3PPPfcAsHfvXnbs2HFEMFmwYAGrV68G4MQTT6SiomLQa1922WW959x9990APPXUU73Xv+CCCygsLBz0tXfccQc33XQTgUCAAwcOsGXLFsyM0tJSTjrpJADy8vIAWLduHZ/61KdISfG/GkVFRaO9DSIiImOWEMFkuJqNiZSdnd37eP369axbt45nn32WrKwszj777EHn/0hPT+99nJyc3NuUM9R5ycnJBAIBwE94NpLy8nK+//3v8+KLL1JYWMhHPvIROjo6cM4NOvR3qP0iIiITQX1Mxig3N5fm5uYhjzc2NlJYWEhWVhbbtm3jueeei3oZzjjjDO644w4AHn74Yerr6484p6mpiezsbPLz8zl06BAPPvggAEuWLGH//v28+OKLADQ3NxMIBDj//PP5+c9/3ht+1JQjInJ06QwEOdDYzuZ9jYRCI/8BHG0JUWMyGYqLizn99NNZvnw5F154IRdffPGA4xdccAE///nPWblyJccddxynnnpq1Mtwww038P73v5/bb7+ds846i9LSUnJzcwecs2rVKk444QSWLVvGwoULOf300wFIS0vj9ttv5+/+7u9ob28nMzOTdevW8fGPf5w33niDlStXkpqaynXXXcdnP/vZqJddRGQ8QiFHc2eAxrZuGtq7aOsK0hkI0dE9cNvZHSQlycjJSCUnPZns9BSy01PICW9zM1LISUshKWlsNcXOObqCIdo6g7R2BWjrCtLaGfDvHX7/jvC2MxCiKxAiNSWJ9N6fZNJTk0hPTiI1JYmWjgC1rV3Ut3YN2Da0dZGRmkxJTholOekU56T7x7npTMtJpzgnjcKsNDJSk4cta31bN7trW9lT18ae2jb2N3ZQ29JJbWtX77a5I9D7mo3feDsFWWljujdjZZE0B0y2tWvXug0bNgzYt3XrVpYuXTpJJZoaOjs7SU5OJiUlhWeffZZPf/rTvZ1xY0n3XuTo1NEd5GBjBwcaO2jrCpCSnERqkpGSnERKspGalERqipFsRshByDlCzuEcuPDzQMjR0R2krStIe3eQ9q4A7V1B2rqDdITDRWcgRFcwRGe333YF/P7G9u5wEOmmoa2LaP0xbwa56SnkZaaSl5FKboZ/nJaSRGd3iM5AcEDQ6Ak+rZ0+iARiVKuQmmwUZqVRlO1DR0cgSE1LJzXNXbR3Dz4tRWZqMkXZaRRkpfa+rjsYYndtG3vr2mjuDAw4vy/opFGUnU5xdpr/Ce8769hpw4adsTKzl5xzawc7phqTOLZnzx6uvPJKQqEQaWlp3HzzzZNdJBGZIroCIerbuqhv6yIvI5XpuemkJA/fet8ZCLK7to1d1S3srG6lsr6dg43tHGjs4GBTBw1t3TEvd3pKEmk9NQm9j/1PXmYqcwqzKMhMpSArlfzMVAqy0ijITCUrPbn3NRmpfdu0lCQCwRCtnUFaOgO0dgVo6QzQ0hGgtdM/bmrvpqmjZ9tNU3uAvXVtdAVDZPTUaKQkUZCZSkZeeu/7+NqXZLLSUshOSyYrPYXstBQy05LCr+spR1/NSFpyEt1B1xd2+gWfrkCInIwUirPTKMxOIzc9Zcg+f62dAR9SWjqpbvb/nevCNSt1rd29/+331rWRlGTML8ri5AVFzC3KYn5RFvOLs5hblBWT0DFeCiZxbPHixbzyyiuTXQwRibGuQIjqlk4a2rpobOumsd3XGjS2d9PQ1k1jexe1Lb7Kvy5cJd/UMfAv4ySDabnpzMzLYGZ+BjPzMpiel0F9axe7alrZWd3C3rq2AbUQxdlplBZkMKcwk7VlhZTmZzIjL4PS/Axy0lMIhEJ0Bx2BoKM7FPLbYIhgyJGcZPjWEb9NMiMpyW8zU/2XeWZaEplpKeHn/ks8Vp3vi3NictlJ09MkNb84e+ST44yCiYjIJGvrClBR08aeujb2N7RzoLGd/Q0d7GtoZ39DO9UtnQzV6p6WnER+VirF2b7Kf/ns/N7HxTlpFGSm0djePaDmY1d1K8/srKW5I0B6ShILSrJZPiufS1bNYtH0HBaW5LBgWjY56fqKkImn3zoRkQnS2NbNMztrqKhto6KmlfLaVnbXtnKoqXPAeekpScwuyGRWQSZnHTuNWQWZzMzPoDArrV8TRioFmWlkpI69lqGtK0B6SjLJY+z4KRILCiYiIhPgxYo6PvuHl3tDSElOOmXFWbx18TTKirMoK8lmflE2swszKcxKnZD5hLLS9BUgU49+K0VEYsg5x81P7uI/HtrO3MJM/njdqSyfnUduRupkF01kSlIwmUA5OTm0tLRMdjFEZII0tnfzpT+9ysNbDnHh8pn8xxUryVMgERmWgomISAxs3tfIZ37/Mvsb2vnGO4/n2tPLtNyDSAQ0Jf0Y/dM//RM//elPe59/85vf5Ac/+AEtLS2ce+65rFmzhhUrVnDvvfeOeK1LL72UE088kWXLlg1YIfihhx5izZo1rFq1inPPPReAlpYWrr32WlasWMHKlSu56667ov/hRGTMnHP84fk9XPazZ+gOhrj9k6fx0TMWKJSIRCgxakwe/Aoc3BTda85cARfeOOThq666in/4h3/gM5/5DOBX8H3ooYfIyMjgnnvuIS8vj5qaGk499VTe/e53D/uP0q9//WuKiopob2/npJNO4vLLLycUCnHdddfxxBNPsGDBgt41a77zne+Qn5/Ppk3+8w62Po6IRFco5Khu6WRvnR/SW1nfTmN7N+3h2Urbu3tmMQ3S2N7NtoPNvHVxCf991QkUZU/sdN5HrVAIgl3hn+6Bj0PdEAr4n2Cg73GoGwJd0N0K3e3+p6vncRsEOg67Xnff4+Q0mL4Eph8PM5ZD8TGQHOOvVOegqwXaaqG11m/bavw2PRemLYFpx0Hm4CvNH6GrDTqbIDUTUrMgeWo0MyZGMJkEJ5xwAlVVVezfv5/q6moKCwuZN28e3d3dfPWrX+WJJ54gKSmJffv2cejQIWbOnDnktX784x9zzz33ALB371527NhBdXU1Z555JgsWLACgqKgIgHXr1nHbbbf1vrawMMJfQBEZoLG9mzermmls76alM0hbeBbQ1s4gbV0BmjsDHGho7w0inYHQgNfnpKeQkZrsJwlLTSYzNZmM1GRm5mfwnhNm8/G3Ljy6huGGgv5LrqMR2hugvR46wtv2+r593e2QngPpef4nI7xNz/VfkF0t/hr9f9ob/LW7Wn1g6AkOPWGiu80HjWhKyYSUdB9AktP8l3bv4xT/pf7GQ+DCU8Mnp8O0Y31IKVroy9PVOrCM3e0+7KRm+XuQltN3L9JyIC3Lv6atzoeN9jr/uL2+b1+wc/hyA+TM9KFpWvgnLRsaK6FpHzTuCz+u9NftLynFly01qy+sfOQvkFUU3Xs7gsQIJsPUbMTSFVdcwZ133snBgwe56qqrAPj9739PdXU1L730EqmpqZSVldHR0THkNdavX8+6det49tlnycrK4uyzz6ajowPn3KC1LEPtFzlaNbZ1c6CpnbRkPwV5T0BIT0kiKckIhRx769vYeqCJLfub2HKgma0HmtjX0D7kNdNTkshJT2FGXgaLp+dyzpLpzCvKYk5RFvOKsphdkDm+qbwDXdB8AJr2+y+Lpv3+p7W678sr0NH3uLvd/6WenAopGZCS5rfJaf7LMyUj/EWSCanZfV8qPfuSUiApGSw5vE3qe97d7gNEbxDo9zjYDZg/38z/EN4GuweGh86m4T9zSob/Sz41Ezpb/PmBof9t9Awy8sM/4S/vjALILe37fGnZ4XuScWR4SE6DpNTw/tTwfUj1n73neXKav0b/e5aSCUkR9HQIdEL1dqjaAoc2w6EtsGs9vPpHf7z/F3zvtdN94Ohs8SGss/nI+2DJ/l5lFUFmERTMg9LVkF0MWcWQVeK32eFtVpEPb9XboXorVG2D6m3w8m99bVCPjALInwN5s2HuyZA/2+8LdAwMev1rjVLSR74PUZYYwWSSXHXVVVx33XXU1NTw+OOPA9DY2Mj06dNJTU3lb3/7G7t37x72Go2NjRQWFpKVlcW2bdt47rnnADjttNO4/vrrKS8v723KKSoq4vzzz+d//ud/+NGPfgT4phzVmsjRoKM7yJtVLWw72Mwbh5rZftD/HGwa+sstPcV/ufTUdiQZLJyWw5r5hXzw1HksmZlLUXY62Wl9q85mpSWTOsKaMqPW3eH/un7tDqh8EVqrjjwnLQdypvsvsJR0/+WYVdTvSzct3JzQ6b8QA53+eVuLf9z7l3m7/zIaSw1CavbAIJAcboZyId9U4kKA89ukVP+FmVHQ7zX5kBl+nlnoj2UW+n2pmUe+X6DLfzF3NvptV5uvOem5VlpOZAFhsqSkQ+lK/9NfoNPfu0j/iAx2+5DS1RquRckb/efOyIfC+XDs+X37QiFo3OuDR95sXzsTBxRMxmHZsmU0Nzcze/ZsSktLAfjgBz/Iu971LtauXcvq1atZsmTJsNe44IIL+PnPf87KlSs57rjjOPXUUwGYNm0aN910E5dddhmhUIjp06fzyCOP8PWvf53rr7+e5cuXk5yczA033MBll10W888qMlGccxxo7GDrgSb/c9DXcFTUtPau45KWksQx03I4bVExx83MZW5hFt1Bv+Jre3eQjm7/uKM7SMg5Fk3LYWlpHsfNzJ24RctCIdjzLLx2G7x+r//yzZnpvzjy50HerPDPbL/NyIvu+we7+/4KDnb7MOGC4YAR9E0vLuiDUEZBOIhMcB+DlDRIKfY1AYlktLUMyanhABflPzKTknxYiTPmhlqAYQpZu3at27Bhw4B9W7duZenSpZNUoqOb7r1EU2cgyEu763lyRw0v765n20Hf76PHvKIslszMZUlpHktm5nLsjFzKirNGXCl30lS/4cPIa3+Cxj2+FuL4d8PKK2HBWb4ZQeQoZ2YvOefWDnZMNSYiMqGcc+ysbuXJHdU88UY1z+2qo707SEqSsXx2PhevLGVpaR5LZ+Zy3Mzc+JghtaUKNt0Jr90OBzb6PhmLzoFzvwFLLvJ9GEQkIgomIhITnYEg+xs6qKz3o1oq69vYW9fOS7vrezueLijJ5sq1c3jr4mmcuqh48lezbW+A3c9A+RO+Gab/EMySY/02Z4bvO9DVCtse8LUjO//mm0VKV8M7/h2WXw65Myb3s4jEqbgOJhqhMvHioelPJl5PLchj2w7x+BvV7Kxq5VBzB/1/XZKTjFkFGSyfncdn3raIMxdPY25Rlj/YfAhe+aUfvjhnLcw+MbIhiq01voaiowmWXDz6tv2uVtjznA8i5U/4a7mQ73g6Z63vo/HqbdDV3PeajHwoWgQ1b/gOi/lz4fTPwcr3+SGaIjIucRtMMjIyqK2tpbi4WOFkgjjnqK2tJSMjY7KLIlNAZyDIC+V1PLatise2VbG7to05VsVXcx8kPyeLpgVrCc09jZJZZcwpymJGbvrAfiEdTbDxD36kSvnj4REfBoTTTNEimHOSDwhzTvJDIw9uggOv9v007eu7Xv5cOPsrsPKqkSe6qtsFz/wPbPy9H7GQlOLf48wvw4Iz/Xv2hBzn/NDe6u0+jFRvh9odsPwyH0bmvWVqjxwRiTNx2/m1u7ubysrKYecIkejLyMhgzpw5pKbGQbu/RFUo5Nh2sJnny2t5dmctT79ZQ2tXkPSUJM5ekM1n0v7Cyopb/R8KScm+NgH8kNJ5p8G8U2HuqVBfAZvugO0P+lBQMN93DF3xXj9CZf8rfkjtvpdg7wuDDK0136xSuio8VHOVH467/t/8a4sXwzlfg6WXHBkY9m+Ep38EW+71YWTl+2DZpb586gciMmGG6/wat8FERGIrGHJsPdDEc7tqeb68jhcr6mho86NlZhdkctZx0zj3uGmcEXiO9Ef/xY9AWX45vP07vh/GoU2+mWTPs7D72YEBI6sYll3mA8mck4ae78E5Pw9D5QY/6+XMlTBj2eDzMTgH2/4Cj33XTzI1cwWc8w1Y/HY/6dXTP/Lb9DxYey2c8mnIK432bRORCCiYiEhEGtu7Wb+9inVbq3h8exVNHX6SrvnFWZyyoIhTFhRzysIi5hRm+SaNB7/sv+ynHw8X/icseOvgF3bON5/sfcGHkkVvi92cGaGgHyGz/t987Uz2dB+KcmbAqZ+GtR/1/UREZNJouLCIDGlvXRvrth5i3dZDPL+rjkDIUZKTxjuWzeSMxSWcvKCI0vx+s3a218NfvwvP/9zP0XHhf8Lajw3fr8MMihf5n1hLSoZV7/N9QF75LWy7H5a+G1ZdNSnTa4vI6CiYiByF6ptaeO7Ru9m3/WV+17icClfK4uk5fOLMhZx3/AxWzykg6fAF6Lrb4YWb4Mkf+I6rJ3wIzr0BcqZNzocYSXKqrx1Z+9HJLomIjIKCichRwnW3s/O5v1C/4U8c1/AkF1obAB9Ph/Y5Z5B56sdgyWl+mvD+QkG/KNnf/s2Pgjnm7XDeDb4Ph4hIlCmYiCSyQCftWx7g4LO3M+PAeo6hnUaXzZtFZzHt1CuZe+yJsOkOMl/6Ddx5rV+19IQPwppr/NLtbzwE677lO5POWgPv+bkfTisiEiPq/CqSiAKd8PJvaHvse2R1HKLO5fBC+ltIX3kZJ59zCdlZWQPPDwX97KUv3eKH8bogFJb5zqNFi/zU6sdfEvlqqSIiw1DnV5GjRXeH7/D55A+heT+vh47l0ZK/4/x3X8U75pcMPRlhUjIsPs//NB2AV37nR9u85e9hzYcnftVZETlqKZiIJILDAklt0Ro+1/URbOFZ3HzNSWSkjmJF27xSOOtL/kdEZIIpmIjEu4qn4K7roHk/zDuNF1Z/l/c/ms5JZUXc8uFRhhIRkUmmYCIS7579Xwh1w4fv5ZH2JXz69y9zwtwCfnXNSWSmKZSISHzRylMi8a5hL8xaw9+6j+f6P7zCstn53HLtSWSn6+8OEYk/CiYi8a5hD/ttOp/87UssnpHDb649mdwMdVYVkfikYCISz9oboLOR320LsbAkm9997BTysxRKRCR+KZiIxLPGvX6TVsrvPn4KhdlpI7xARGRqUzARiWMdNRUALD52KSU5WqBOROKfgolIHKvYuQ2AZccvn+SSiIhEh4KJSByrqdxBu0tj1XGLJ7soIiJRoWAiEqecc3TX7qEhbSZpmkRNRBKEgolInNpZ3Upx4CAuf85kF0VEJGoUTETi1PrtVcy2GvJmLprsooiIRI2CiUicem7bHoqtmZwZCya7KCIiUaNgIhKH2roC7KvY4Z/kz5vcwoiIRJGCiUgcem5XLTPcIf+kQMFERBKHgolIHFq/vZoFKXX+ScHcyS2MiEgUKZiIxBnnHOu3V3NSQQskpULOzMkukohI1CiYiMSZito29tS1cXxWA+TPgST9bywiiUP/oonEmfXbqwAopUb9S0Qk4SiYiMSZ9durWViSTXpLpfqXiEjCUTARiSMd3UGe21XLOYvzoOUQFMyf7CKJiESVgolIHHluVy2dgRDnzw74HfmqMRGRxKJgIhJH1m+vJj0lidW5TX6H+piISIJRMBGJI4+/Uc1pi4pJa6n0O9THREQSjIKJSJzYXdtKeU0rZx87DRr2giVD7qzJLpaISFQpmIjEicffqAbg7OOmQ8MeyJsNySmTXCoRkeiKaTAxs8+b2etmttnM/mhmGWZWZGaPmNmO8LYwlmUQSRTrt1czvziLspJsaNyrZhwRSUgxCyZmNhv4e2Ctc245kAxcBXwFeNQ5txh4NPxcRIbR0R3kmZ01vhkHfFOOOr6KSAKKdVNOCpBpZilAFrAfuAS4NXz8VuDSGJdBJO69WFFHR3fIN+MEu6F5v4YKi0hCilkwcc7tA74P7AEOAI3OuYeBGc65A+FzDgDTB3u9mX3CzDaY2Ybq6upYFVNkygqFHIeaOnh5Tz23vbiXtJQkTl1YDE37wIVUYyIiCSlmPefCfUcuARYADcCfzOxDkb7eOXcTcBPA2rVrXSzKKDIV1LR0smlfI5srG6mobWNfQxv7Gzo40NhOd7DvV//tx88gMy0Z9u3xO9THREQSUCy79J8HlDvnqgHM7G7gLcAhMyt1zh0ws1KgKoZlEJlS6lq7fAjZ18hrlQ1sqmxkf2NH7/GZeRnMLsxk9dwCLlpRyuzCTGYXZDC7IIsFJdn+pIa9fqsaExFJQLEMJnuAU80sC2gHzgU2AK3ANcCN4e29MSyDSNQ0dXTz3M5annqzhtf3NzEzL8OPkinOpqwkm7LiLKblpmNmBEOO8poWthxoZuuBpt6fQ02dvddbUJLNiWVFXDs7nxVz8lk2K4/cjNSRC9KwBzDImxO7DysiMkliFkycc8+b2Z3Ay0AAeAXfNJMD3GFmH8OHl/fGqgwi49EdDLFxbwNP7qjh6Tdr2Li3gWDIkZmazPLZeWw50MRfXz9IINTX3JKVlszM/Az21bfTGQgBkJJkHDM9h7csKmFpaS7LZ+ezbFY++ZkRhJDBNO6F3FJISYvGxxQRmVJiOjuTc+4G4IbDdnfia09EpgznHJX17Wza18hrlY1s2tfAq3sbaekMkGSwYk4Bnz5rEWcsLuGEeQWkpyQDPrzsb2inoraNippWKmpb2d/QztuOm87S0jyWluZyzPSc3vOjomGP+peISMLStJFyVAgEQzS0d9PQ1kV9Wzd1rV00tHWxt66d1/Y1sqmygfq2bgBSk40lM/O49IRZnL6ohLcsKiE/a/DajdTkJOYXZzO/OJuzeuYYibWGPTD35Il5LxGRCaZgInGrrrWL+1/bz/+9eoB9De2Ar/lwgAu3rjgc7V1BmjoCg14jOck4dkYu5x8/kxVz8lk5J5/jZuZGt4YjmkJBP1xYc5iISIJSMJH4EApCUjLtXUHWbT3En1/Zx+NvVBMIOY6dkcMpC4rAwDDMwCC8NTJSkyjMTqMwK42CrFQKs9IoyvaPS3LSyUgdJIR0t8OB12DeKRP+UYfVfABCAY3IEZGEpWAiU15N9SFyfn0mD+a/j385cDotnQFm5mXw0TMWcOnq2SwtzcXMovumT3wfnvwBfGE75M6I7rXHo3eosGpMRCQxKZjIlBEKOSpqW3l9fxNbDjSxJbz9RPuvuC7lIMltr3DRisu5dPVsTllYTHJSlMNID+dg058ABwc3TbFgEp5cLV81JiKSmBRMZMI453izqoW99W0caOzgQEMH+xvbOdjYwYHGDvY3DBxiu3hGLu+Z38lHdz4MDt45r5t3X7Eq9gWt3AANu/3jQ5tg8Xmxf89INWrWVxFJbAomEnP1rV3c/co+bnthDzuqWnr3JycZM3LTmZmfwfGz8jhv6XQWz8jl+NI8Fs8ID7G9/UOQkg5zTiepbtfEFHjznZCcDhl5cOj1iXnPSDXsgexpkJo52SUREYkJBROJCeccz+6q5bYX9vLQ5oN0BUOsnlvAd9+znCUz85hVkMG0nHRSkodZR3L3M7D1/+BtX4NgF1Q8BcEAJMfw1zYUhNfvgcVv951MD26O3XuNRcNedXwVkYSmYCJRtb+hnXs37uf2F/dQUdtGXkYK7z95LledPI+lpXmRXygUgr9+FXJnwWmf9X0+XBCa98f2i7niSWg5BCuu8KFkxyPQ3QGpGbF7z9Fo2AMzV0x2KUREYkbBRMZtX0M7D246wP2bDvDKngYATi4r4u/PXcxFK0oHH447kk1/gv2vwHt+AWlZfX0qYl1jsPkuSMuBYy/wz10QqrfBrNWxe89IhULQWAlLLp7skoiIxIyCiYxJZX0bD2w6wP2bDvLq3gYAls3K40vvOI6LVpT2rYQ7Fl1t8Oi3oHQ1rLjS7+sZhdK4d1zlHlagC7bc57/4UzNhRrhm4tDmqRFMWqsh2KmmHBFJaAomMiqbKhv5r3Vv8Ni2KgCWz87jyxccx0XLSykbTxjp79n/9bObXnYzJIX7oOSHV9JtiGEw2fkodDTA8iv886IFkJo1dfqZ9AwVVjARkQSmYCIR2bK/if9a9waPbDlEQVYqnz/vWC49YRbzi6MURno0H4Sn/guWvgvKTu/bn5oB2dP7hsvGwqY7IbMQFp7tnyclw/TjfY3JVNDz2TUdvYgkMAUTGdaOQ83817o3eGDTQXIzUvjHtx/LtaeXkZsx+KJ24/bYv/oROOd968hjBXNjV2PS1QrbH4CVV0JKWt/+mcvh9T/7SdeiPbvsaDVoDhMRSXwKJjKoPbVt/OCR7dz36n6yUpP5u3OO4eNnLBxyld2oOLgJXvkdnHY9FC868nj+3NjVXmx/ELrb+ppxesxYDi/9v/DCeXNi896Ratjra3TScye3HCIiMaRgIgM457jtxb185y9bcA4+eeYiPnHmQoqy00Z+8fjeGP76NcgsgDO/OPg5BXN9gAiF+vqeRMvmuyC3FOa/ZeD+nqG5BzdPgWCyR/1LRCThKZhIr+rmTr5y12s8uq2K048p5ntXrGJWwQTNMLr3eSh/HC640dcKDCZ/nh+V0lod3fVr2uv9fCUnf8L3K+lvxjK/PbQJjrsgeu85Fo17ofiYyS2DiEiMKZgIAA+/fpB/vnsTzZ0BvvHO4/nIW8pIitUieYN5/heQkQ9rPjz0OT19Kxr3RjeYbP0/CHXDisuPPJaeC4Vlkz8yxzlfY7LonMkth4hIjCmYHOVaOgN85/+2cPuGvSyblcdt71vN4hkT3Ieh6QBsvQ9O+RSkDTPKp2c0SsMemLM2eu+/6U4oXACz1gx+fMbyyR+Z01bn+8CoKUdEEpyCyVHs+V21fPHOV9lX3871b1vE5849lrSUKPfdiMRLt/g1ak762PDn9a8xiZbmQ34a+rd+YehRNzNXwLb7/cid4YJTLGmosIgcJRRMjkJ769q48aFt3P/aAeYWZXLHJ09jbVnR5BQm0AUbboHF50PRwuHPzciH9PzRDRnedCfseQ5O+NDgs7e+fg+40JGjcfqbsRxwULU1ujU1o6HJ1UTkKKFgchRp6Qzws/VvcvOT5SQZ/P25i/nUWQvJSpvEX4Mt90JrFZzyicjOL5g3uhqTZ34CBzbCizf7ppq1H4Xll/v1d8CPxpmxHKYvGfoaM5f77aHNkxhMwp9Zc5iISIJTMDkKhEKOO1+u5Ht/3U51cyeXrp7Fly9YMnEjbobzwi+gaBEsjLBTZ8FcqN8d2bmhIFRv9x1qZ6yADb+G+z7rhyWvusrX0lS+AOd+Y/jr5M+DtNzJ7QDbsAfS8yCjYPLKICIyARRMEtxLu+u54b7NbN7XxOq5Bfzi6hNZM2+I4bgTbd/LUPkiXPAfkc9Lkj8XKp6K7Nz6Cgi0w9xTfFPOydf5Zp0Nv/L9Wl74hT9v+SCjcfpLSvLDhieqA2xns+/70nLQT9HfcsgPpc6fO/mzz4qIxJiCSQLbfrCZD9z8HEXZafz3Vat518pZEzsEeCQv3AxpObD6A5G/pmAudDZBe4OfjG04VVv8dvpSvzWD+af5nwtuhI2/9/1LCstGft+Zy+G1O2I7Nf3tV8Obj0J365HHktP8jLgiIglOwSRBtXcF+bs/vkxuRir3ffYMpuWmT3aRBmqt8f071lwNGXmRvy6/38icEYPJVr+dNkj/kewSOP1zkb/vjOXQ+Uto2B1ZkBmtul1+yPTid/jZZ3NnQs6Mvm1moWpLROSooGCSoL79ly28caiF337s5KkXSgBevtXP4npyhJ1eexT0m8ukZ7r4oVRt8SEiGkN8+09NH4tgsu0Bv73oP2NzfRGRODEJk1ZIrN3/2gH++MIePnXWIt66eNpkF+dIwQC8+GtYcBZMO250r80PD5eNZMhw1VaYfvzoyzeY6UsBi10/k233+1oZhRIROcopmCSYvXVtfOXu11g9t4AvnH/sZBdncNsfgKZKOOWTo39tdgmkZI48ZDjQCbVv9vUvGa+0bL/i8cFN0blef601sPc5OO6i6F9bRCTOKJgkkO5giL+/7RVw8JP3n0Bq8hT9z/vCTb7m49gxLIpn5lf57ZlwbCi1b0IoEL0aE4jd1PRvPOQ74S65OPrXFhGJM1P0m0vG4oePvMErexq48fKVzC3KmuziDO7QFj8F/EkfO3Il30gVzB25xqSn42u0akzAj8ypr4COpuhdE3z/krw5ULoqutcVEYlDCiYJ4skd1fz88Z28/+S5XLyydLKLM7QXboKUjOFXER5J/tyR+5hUbYGkFChePPb3OdyMFX3XjpauNtj5GCy5SKNuRERQMEkI1c2dfP72VzlmWg7feOeyyS7O0Nob4LXbYcUVkDWOtXkK5kJbjf9SH0rVVh9KUtLG/j6H65maPpr9THb9zU8Cp/4lIiKAgkncc87xhT+9SnNHN//zgTVkpo2xeWQi7H4Guttg9QfHd52ekTmNlUOfc+j16DbjAOTN9lPCR7OfybYH/MKEZWdE75oiInFMwSTO3f7iXp54o5qvX7yU42bmRu/CL/8GHv1O9K4HUB3u9zFj+fiu0zOXSeMQHWA7W/xEaNHs+Aq+qWXmiuitmRMKwhsPwrHnQ3JqdK4pIhLnFEzi2MHGDr57/1ZOW1jMh06dH70LH3gV/vJ5ePpH/ks+Wqq2+U6eo5npdTAFI8xlUr3db6NdYwI+VFVtgVBo/Nfa+zy01Wo0johIPwomcco5x9f/vJmuYIh/v2wFFq2Ok93tcPcnwJL8cNu9z0XnuuBrTKYPMj38aOWW+o6tQ43MOXyNnGiaudw3R9WXj/9a2+73a+Acc974ryUikiAUTOLU/ZsOsG7rIb5w/rGUlURhyvUej34bqrfBFbdAUiqUPxGd64aCULNj8HVrRispGfJmDT2XSdVWPwlbLGZRnRHuXDzeDrDO+WCy4ExIj2ITnIhInFMwiUP1rV3ccO/rrJyTz0dPXxC9C+9aD8/91K9fs/SdMOek6AWT+goIdEQnmIDvADtUU07VFj/V/VjnSRnOtKVgyePvAFu11de6qBlHRGQABZM49J37t9DY3s1/XL6SlGjN7treAH/+jB9ie963/L4FZ/r+Ju31479+tCc8G26StWiukXO41AwoWTz+DrDb7/fbYy8cf5lERBKIgkmcWb+9irtf3sdnzl7E0tJxdiLt74EvQcshuOwmSAvPGrvgTD9V+u5nxn/96m1+O9pF+4aSPxeaD0Cwe+D+tjpoOQgzYhRMIDpT0297AGavhbwpPBmeiMgkSJnsAkjkWjoDfO2ezRwzPYfrzzkmehfefDdsugPe9jWYvaZv/5y1fpbW8ifG3+RQvc2HiWj1pyiY60NT076BfUliMRX94WYuh813+uHUgQ7oau330+LLdeYXYeHZg7++aT/sfxnO/UbsyigiEqcUTOLI9x7axv7Gdu781Gmkp0Sp/0TTfj80ePZaOOMfBx5LSYd5p0L5k+N/n6pt0etfAj7kgO9nMiCY9IzIiWGNSdlbfT+TJ78Pqdl+5eG0bEjL8dvm/fDby+DC/4CTPn7kVPPbH/DbJe+MXRlFROKUgkmc2FBRx2+e2801p5Vx4vxxTOfen3Nw7/UQ7IL3/AKSB/l1WHCmH6nTUg0508b2PqEg1LwBi84eV3EH6JnL5PB+JlVbICPfDymOlTlr4WsH/ZDlpEFaQzub4a7r4IEv+hloL/rewAnUtt0PRYug5NjYlVFEJE6pj0kc6AwE+ae7XmNWfiZfekeU+miEQvDUD/0Ccuf/K5QM0TS04Cy/rRhHrUldOQQ7/YiWaMmb7beHj8zp6fga6wXxUtIGDyXgm6uu+oOvgXrpFvjNpdBa6491NPoaqCUXa9E+EZFBKJjEgb+8eoCd1a18+5JlZKdHoZKr/Em4+W2+JmTJO2HtR4c+t3Q1pOWOb9hwz1T00WzKSc2AnBkDp6V3zteYxLJ/SaSSkuC8G+Cym6HyRX+/q7bCjkcg1K1hwiIiQ1BTzhTnnOPWZys4ZnoO5yyZPr6LVW2DdTfAGw/5PhrvuQlWvHf4v9yTU6Ds9PEFk6ooj8jpkT93YI1J8wFfIxHL/iWjtfJKKFoIt30AfnkeFC2A7Gl+jhgRETmCakymuI17G3itspFrTps/9mnnmw/C/30OfnYa7H7Wz1Py2Q2w6n1DN0f0V/ZWqNsJjfvG9v7VW/2EaOk5Y3v9UA6fyySWU9GPx5y1cN3foPgYP2PssRfEZvI3EZEEoBqTKe43z+4mJz2F96yZM7YLvPgrePhffAfXkz8JZ34JsotHd40FZ/ptxZOw6qrRl6FqW3TWyDlc/lzfkTQU8gGrd6jwFKox6ZE/G6590M+su+KKyS6NiMiUpRqTKay6uZP7XzvAFSfOIWcsfUucg0du8Ou7fPYFuPDG0YcS8BOKZRaOrTknGIDaKK2Rc7iCeT5wtRzyz6u2Qs5MyIrSqKVoS8vy85vEYg0fEZEEoWAyhd3+4h66giGuPm3+2C7Q0QhdzXD8u30/h7FKSvLNOeVP+LAzGvXlPjzEonnl8CHDU6Xjq4iIjJmCyRTVHQzxu+f28NbFJSyaNsa+GY2Vfps/xmag/hac6QNAffnoXlcVgxE5PXonWdvjm3Oqtk3NZhwREYmYgskU9ciWQxxs6uCa08rGfpGmcGfVvGgEk/B8JqNtzulZIycWk4kVhINJT2AKtKvGREQkzimYTFG3PlPBnMJM3jaeIcK9NSazx1+gksV+3pDRTk9ftdU3uUR7RA74icwyCvyQ4anc8VVERCKmYDIFbTvYxPPldVx96nySk8YxO2hjpZ82PWfG+Atl5ptzRtvPpHpbdGd8PVzPkOHeJqMoz5UiIiITSsFkCrr1md2kpyRx5dq547tQ0z7InRW9OTMWnAmtVVC9PbLzg91QsyM2Q4V75M8L15hsgYL5samZERGRCaNgMsU0tnXz51f2cenq2RRmp43zYvui04zTo2c+k0j7mdTt8tOvT0iNyRY144iIJAAFkynmTy/tpb07OPYhwv017o3OiJwehWW+v0j545Gd39vvI5Y1JnOhq8XX4qjjq4hI3FMwmUJCIcdvn9vN2vmFLJ+dP96LQdP+vlV4o2XBmVDxlL/+SKq3AwYlMez30TMyB+cnkhMRkbimYDKFPP5GNbtr2/jwW8rGf7HWat+MEs0aE4CyM6GjAQ5tGvnc6q1QON/PeBor+f364ajGREQk7imYTCG3PlvBtNx0Llg2c+AB52DnY74zaaSiOblafwve6reR9DOp2habidX665n9NSkFihfH9r1ERCTmFEymiN21razfXs0HTp5HWsph/1l2Pga/fQ9svS/yCzaFg0m0m3LyZvkAMFIwCXZD7ZuxDyZZxZCS6VfuTRlnZ2EREZl0CiZTxMOv+4XorjxpkCHCG//gtzU7Ir9gY3jW12jXmEC4n8nT0N4w9Dm1O31TUqybV8xg5gqYe3Js30dERCaEgskU8dSbNSyals3sgsyBBzoaYdtf/OPanZFfsLESUrP8qsDRduJHoLsNnv7R0OdUx3CNnMN9+M9w0fdj/z4iIhJzCiZTQGcgyAvldZxxTMmRB1+/BwIdkD3dzwsSqaZK34xj45g5diilK2Hl++C5n/X1ZTlc1Tb8iJwYrJFzuLRsSEmP/fuIiEjMKZhMAa/saaC9O8gZi6cdeXDjH3ytw5KLRhdMoj252uHO+ZrvlPu3fxv8ePVWP+9JLEfkiIhIwlEwmQKe2lFDcpJxysKigQdq3oS9z8PqD0DRImivg/b6yC7atC82/Ut6FMyDUz7pg9PBzUcer9qm4bsiIjJqCiZTwFNv1rBqTj55GakDD7z6R7Ak32xSvMjvi6TWJNAFzQchL4bBBOCt/wgZ+bDuhiPfv26nFtQTEZFRUzCZZI3t3bxW2XBk/5JQCF69DRadC7kzoWih318bQTBpPgC42DblgO9Ye+aX4M11sGt93/66nRAKxHaNHBERSUgKJpPsuV21hBycfngwqXjCd2Bd/X7/vHABYJHVmDTFcKjw4U6+zjfrPPKNvmnqJ2KNHBERSUgKJpPs6TdryExN5oR5hw3r3fgHSM+H4y72z1Mz/CibugiGDPeMlIl1Uw740TDn/AsceBU23+X3VW/zTVATMSJHREQSioLJJHvqzRpOWVg0cLbXjibYch8sv8wHkh7FCyOrMemdjj7GTTk9ll8BM1fCo9+GQKevMSksg9TMEV8qIiLSn4LJJNrf0M6u6tYj+5dsuRcC7bD6gwP3Fy2MbJK1pn2+/0dadvQKO5ykJDj/O9C4B1642deYqH+JiIiMQUyDiZkVmNmdZrbNzLaa2WlmVmRmj5jZjvA2BlOTxoen36wBBulfsvEPfu2XOWsH7o90yHBj5cQ04/S38Gw45jx44ns+PKl/iYiIjEGsa0z+G3jIObcEWAVsBb4CPOqcWww8Gn5+VHr6zRpKctJYMjO3b2fdLtjzjJ+75PBZW3tG5ozUnBPrydWGct63/BT6LjgxU9GLiEjCiVkwMbM84EzgVwDOuS7nXANwCXBr+LRbgUtjVYapzDnHU2/WcvoxJVj/APLqbYDByquOfFHvXCblw1+8qXJiRuQcbuZyH6hAk6uJiMiYpMTw2guBauAWM1sFvAR8DpjhnDsA4Jw7YGbTB3uxmX0C+ATAvHnzYljMybH9UDM1LZ0Dm3FCIT+p2sKzB6/xKCzz2+H6mXS1+qaevEmoMQE4/1/9Sr8zlk/O+4uISFwbscbEzNaa2efN7Htm9m0zu9LMikZ6HT70rAF+5pw7AWhlFM02zrmbnHNrnXNrp00bZA2ZOPfUjkH6l+x+Ghr2HNnptUdqpu87MlxTTuMEzmEymKwiv/pwLBYPFBGRhDdkMDGzj5jZy8A/A5nAdqAKOAN4xMxuNbPhqjIqgUrn3PPh53fig8ohMysNv0dp+JpHnaffrGFhSTazC/oNqX31j5CWC0suHvqFRQuGn8ukqWeo8CQFExERkXEYriknGzjdOdc+2EEzWw0sBvYMdtw5d9DM9prZcc657cC5wJbwzzXAjeHtvWMvfnzqCoR4vryOy9f0Cw+dLfD6n2HF5cOvyFu8CLb+39DHeydXm6SmHBERkXEYMpg45/53uBc65zZGcP2/A35vZmnALuBafC3NHWb2MXyoeW/EpU0QG/c20NYVHNiMs+XP0N0Kqz4w/IuLFkJbLbQ3QGbBkccb9wEGebOiV2AREZEJEnHnVzN7F/B1IB24yTn305FeEw4vawc5dG6k75uInnqzhiSD0xYV9+3ccAuUHAfzTh3+xf2HDM9ec+Txpkq/6F9y6pHHREREprjh+pisOmzX1cCp+H4in45loRLd02/WsHJOAfmZ4fBwcBPs2xBZp9GiniHDQ3SAbaxUM46IiMSt4UblfMbMbjKzmeHne4HvAt8G9se8ZAmquaObjXsbBk5Dv+EWSE6HVYPMXXK4niHDQwaTSZpcTUREJAqG62PyyXCtyS/MbAPwL8BbgCzgOxNUvoTz3K46giHX17+kswVeuwOWvccPtR1JWpavERlsLhPn/Do5x74juoUWERGZIMPOY+Kce9U5dwmwEbgPKHXO3eec65yIwiWip9+sISM1iTXzC/yOzXdBVzOsvTbyixQNscpwez10t6kpR0RE4tZwfUw+ZWavhOcyyQYuAArN7K9m9tYJK2GCeerNGk5eUEx6SrLf8dItfiXeuadEfpGihYPPZdIzVFhNOSIiEqeG7WMSnrH1VOBLzrmAc+7HwFXAeyakdAnmYGMHb1a1cMYx4dE4+zfC/ldg7UdHN1Nq/yHD/TWFZ32d6JWFRUREomS44cL7zOw7+Flft/XsdM7VA/8Y64Ilomd2HjYN/Uu3QEomrLxydBfqWcyvvhwyT+jb36hZX0VEJL4NV2NyCfACsA748MQUJ7FtPdBEekoSS2bmQUcTvPYnWH754BOlDadnLpPDO8A2VkJSKmQn3tpCIiJydBiuxmSWc27Iuc/NzIDZzrnK6BcrMe2sbmVBSTbJSQab/uRneh1Np9cehQv8tq584P6mfX7G16QR12YUERGZkoYLJt8zsyT8WjYvAdVABnAM8Db87K034BfrkwjsrG5h+ex8P6z3pVtgxgqYfeLoL5SWBbmzjuwA21gJ+XOjU1gREZFJMOSf1s659+LnLjkO+F/gSXxI+Th+peFznHOPTEQhE0FHd5C9dW0smpYD+172s72u/cjoOr32V7zoyCHDmlxNRETi3LBr5TjntgBfm6CyJLTdtW2EHCyalg0v/TekZsOKUXZ67a9oAWx7oO95KAjN+zWHiYiIxDV1RpggO6tbAFicH4LNd8OKyyEjb+wXLFoEbTXQ0eiftxyCUEAjckREJK4pmEyQnVU+mBxz4H4/O+uJY+j02l//VYbBN+OAgomIiMQ1BZMJsrO6hdn5GaRt/A2UroLZa8Z3wcODSVO4D7KackREJI6NGEzM7C4zuzg8QkfGaGd1K+fn74Gq18dfWwK+jwlAbU+NiSZXExGR+BdJ2PgZ8AFgh5ndaGZLYlymhOOcY2d1C6el7vA7jr9k/BdNy4bc0oFNOWk5kJE//muLiIhMkhGDiXNunXPug8AaoAJ4xMyeMbNrzSw11gVMBAebOmjrClKa3gmWDJmF0blw0aK+uUyaKn0zzliHH4uIiEwBETXPmFkx8BH8HCavAP+NDyqaxyQCO6taAShJ7YT03OiFh6IF/WpMKtWMIyIicW/YeUwAzOxuYAnwW+BdzrkD4UO3m9mGWBYuUfQMFS5I6hjfEOHDFS+C1mq/7k7jPpi5InrXFhERmQQjBhPgf5xzjw12wDm3NsrlSUg7q1vITU8hI9QC6VEMJj0jc6q3QWsV5KnGRERE4lskTTlLzayg54mZFZrZZ2JXpMSzq7qVhdNzsI7mKAeTRX5b8aTfqilHRETiXCTB5DrnXEPPE+dcPXBdzEqUgHZWt/ip6Dsbo9uU0zNkuPwJv9U6OSIiEuciCSZJZn29Nc0sGUiLXZESS0tngAONHX7xvs4o15j0DBne87x/rqYcERGJc5EEk78Cd5jZuWZ2DvBH4KHYFitxlFf7ETmLpuX4TqrpudF9g6KFEGj3j1VjIiIicS6SYPJPwGPAp4HrgUeBL8eyUImkZ0TOMdOyoLMpuk050NcBNqsYUjOje20REZEJNuKoHOdcCD/7689iX5zEs7O6heQkY15ekl/9N5pNOdAXTLRGjoiIJIBI5jFZDPw7cDyQ0bPfObcwhuVKGDurW5hflEVawDfpxKQpByB/bnSvKyIiMgkiacq5BV9bEgDeBvwGP9maRGBnVSsLp+X4ZhyI/lo2xeEhw+pfIiIiCSCSYJLpnHsUMOfcbufcN4FzYlusxBAMOcprWlk0Pdt3fIXYNOWk58H046N7XRERkUkQycyvHWaWhF9d+LPAPmB6bIuVGCrr2+gKhsJDhcMz+Ue782taNnzuVa0qLCIiCSGSGpN/ALKAvwdOBD4EXBPDMiWMnhE5vXOYQPT7mABkFUFScvSvKyIiMsGGrTEJT6Z2pXPuS0ALcO2ElCpB9KwqvGhaNtTGqClHREQkgQxbY+KcCwIn9p/5VSK3s7qFkpw0CrLS+nV+VTAREREZSiR9TF4B7jWzPwGtPTudc3fHrFQJYmd1CwtLcvyT3qYcBRMREZGhRBJMioBaBo7EcYCCyQh2VrfyjmUz/JOOJkjNVl8QERGRYUQy86v6lYxBXWsXda1dvuMrRH9lYRERkQQUycyvt+BrSAZwzn00JiVKELv6j8iB8AJ+CiYiIiLDiaQp5y/9HmcA7wH2x6Y4iWPn4cGks1k1JiIiIiOIpCnnrv7PzeyPwLqYlShB7KxuJS0lidmF4RV/O5s0CZqIiMgIIplg7XCLgXnRLkii2VnVwsKSbJKTwiOt1ZQjIiIyokj6mDQzsI/JQeCfYlaiBLGrppXjS/sFkc4mNeWIiIiMIJKmnBjMoZ7YOgNB9tS18a6Vpf12NqvGREREZAQjNuWY2XvMLL/f8wIzuzSmpYpze2rbCIYci6aHO74Gu6G7TcFERERkBJH0MbnBOdfY88Q51wDcELMSJYBBR+SAmnJERERGEEkwGeycSIYZH7V2VvuZ+xeUZPsdHeFcpxoTERGRYUUSTDaY2Q/NbJGZLTSz/wJeinXB4tnOqhZm5WeQnR7Ob6oxERERiUgkweTvgC7gduAOoB24PpaFinc7q1v6+pdA38rC6epHLCIiMpxIRuW0Al+ZgLIkBOccO6tbueLEOX07O3qCiWpMREREhhPJqJxHzKyg3/NCM/trTEsVx6qaO2npDLBoWnbfzp4aE838KiIiMqxImnJKwiNxAHDO1QPTY1aiOLez6rAROdDXx0Q1JiIiIsOKJJiEzKx3Cnozm88gqw2L1ztUuH8fk95ROepjIiIiMpxIhv1+DXjKzB4PPz8T+ETsihTfdla3kpOewvTc9L6dnU2QnAapGZNXMBERkTgQSefXh8xsDXAqYMDnnXM1MS9ZnNpV08qCkmzMrG+nFvATERGJSKSrCweBKqARON7MzoxdkeJbRU0rZSXZA3d2NmsOExERkQhEsrrwx4HPAXOAjfiak2eBc2JasjjUFQhRWd/GpSfMHnigs0n9S0RERCIQSY3J54CTgN3OubcBJwDVMS1VnNpT10bIwYKSrIEH1JQjIiISkUiCSYdzrgPAzNKdc9uA42JbrPhUXtOzRk7OwAOdTZrDREREJAKRjMqpDE+w9mfgETOrB/bHslDxqqInmBQP0sdETTkiIiIjimRUznvCD79pZn8D8oGHYlqqOLWrppWi7DTys1IHHlBTjoiISEQiqTHp5Zx7fOSzjl4VNa2UFR/WvyQUCjflKJiIiIiMJNLhwhKB8prWI/uXdLUATjUmIiIiEVAwiZK2rgAHmzqOHJHTu06O+piIiIiMRMEkSipq2oAhRuSAmnJEREQioGASJRW1fkRO2WBzmACka7iwiIjISBRMoqRnDpOywYYKg2pMREREIqBgEiXlNa3MyEsnO/2wgU6djX6rPiYiIiIjUjCJkvLwqsJH6G3KUY2JiIjISBRMoqRiqGCizq8iIiIRUzCJgsb2bmpbu4YIJs2AQVrOkcdERERkAAWTKKgYquMr9E1HbzbBpRIREYk/CiZR0DMiZ+G0IZpy1IwjIiISEQWTKCivaSXJYG5R1pEHtYCfiIhIxGIeTMws2cxeMbO/hJ8XmdkjZrYjvC2MdRlirbymldmFmaSnJB95UDUmIiIiEZuIGpPPAVv7Pf8K8KhzbjHwaPh5XKuobR28fwn4YKI5TERERCIS02BiZnOAi4Ff9tt9CXBr+PGtwKWxLEOsOecor25l4WAjckBNOSIiIqMQ6xqTHwFfBkL99s1wzh0ACG+nD/ZCM/uEmW0wsw3V1dUxLubY1bZ20dwZoGyoYKKmHBERkYjFLJiY2TuBKufcS2N5vXPuJufcWufc2mnTpkW5dNHTMyJn0DlMwM9johoTERGRiKSMfMqYnQ6828wuAjKAPDP7HXDIzEqdcwfMrBSoimEZYm7YYNLdAcEu9TERERGJUMxqTJxz/+ycm+OcKwOuAh5zzn0IuA+4JnzaNcC9sSrDRCivaSU12ZhdkHnkwd7p6PMntlAiIiJxajLmMbkReLuZ7QDeHn4etypqWplblEVK8iC3Ugv4iYiIjEosm3J6OefWA+vDj2uBcyfifSdCeU0rC4YbKgzq/CoiIhIhzfw6DqGQo6J2iFWFoS+YqI+JiIhIRBRMxuFgUwcd3aGhhwqrKUdERGRUFEzGoWdV4SEnV1NTjoiIyKgomIzDrnAwGXpytWa/VY2JiIhIRBRMxqGippWM1CRm5mUMfkKH+piIiIiMhoLJOJTX+MX7kpJs8BM6myA1C5JTJ7ZgIiIicUrBZBzKhxuRA9DRqGYcERGRUVAwGaNAMMSe2rah+5dAeJ0cNeOIiIhESsFkjPY1tBMIueFrTLSysIiIyKgomIzRrpFWFQbf+VVNOSIiIhFTMBmjikiCiWpMRERERkXBZIzKa1rJTU+hODtt6JPUx0RERGRUFEzGqLymlQXTsjEbYqgwhJty8ieuUCIiInFOwWSMeuYwGVIwAN2tasoREREZBQWTMejoDrKvoX34/iVdmo5eRERktBRMxmBvXRvORTAiB9THREREZBQUTMYgoqHCWllYRERk1BRMxqBipFWFoV+NiYKJiIhIpBRMxqC8ppXi7DTyM4dZnK8z3MdENSYiIiIRUzAZg/Ka1uFrS6CvKUc1JiIiIhFTMBmDXTWtLBwpmHQ0+q2CiYiISMQUTEapuaOb6uZOFkyLsMZETTkiIiIRUzAZpfJwx9eFJTnDn9jZDEmpkJIxAaUSERFJDAomo9QbTEaqMelo8nOYDDdlvYiIiAygYDJKO6tbMYP5xVnDn6iVhUVEREZNwWSUymtamVOYSXpK8vAndjSp46uIiMgoKZiM0q7qlpH7l4DvY5KhlYVFRERGQ8FkFJxzlNe0jty/BHxTjtbJERERGRUFk1E41NRJW1dw5DlMQE05IiIiY6BgMgq7aloAWDgtkqacRnV+FRERGSUFk1HYVR3BqsIAzvk+JqoxERERGRUFk1Eor2klMzWZmXkjTJrW1QoupD4mIiIio6RgMgq7qlsoK8kmKWmESdM0Hb2IiMiYKJiMQsQjcjq0srCIiMhYKJhEqCsQYm99e2Qjcjqb/VbzmIiIiIyKgkmE9tS1EQy5COcwafRb9TEREREZFQWTCO2q9kOFF0Qy66uackRERMZEwSRCPasKjzhUGNT5VUREZIwUTCK0q7qVkpw08jNTRz65p4+JmnJERERGRcEkQuU1rZEt3gfhphyDNAUTERGR0VAwidCumpbImnGgbwG/JN1eERGR0dA3ZwQa27upaemKbEQOaAE/ERGRMVIwicCoOr5CX42JiIiIjIqCSQTKR7OqMPhgohE5IiIio6ZgEoFd1a0kJxnzirIie4GackRERMZEwSQCu2pamVuYSVpKhLers1k1JiIiImOgYBKBXdWtkfcvAfUxERERGSMFkxGEQo6KmtbI+5eAmnJERETGSMFkBAebOmjvDkZeYxLohGCnmnJERETGQMFkBD1DhSOew6R3Ovr8GJVIREQkcSmYjKBnVeHIp6Nv9Fv1MRERERk1BZMR7KppJSstmRl56ZG9QCsLi4iIjJmCyQh6RuSYWWQv6AgHE3V+FRERGTUFkxGUj3ZETk8fE9WYiIiIjJqCyTA6A0Eq69tGP4cJqI+JiIjIGCiYDGNPbRshB4siHZED/ZpyNCpHRERktBRMhrGzepSrCoM6v4qIiIyDgskweuYwGXUwScmE5NQYlUpERCRxKZgMY1d1C9Ny08nNGEXIaKmC7JLYFUpERCSBKZgMo7ymlYWjqS0BqK+AwrJYFEdERCThKZgMY1dNa+RT0feoK1cwERERGSMFkyE0tHVR19oV+VT0AF2t0FqlYCIiIjJGCiZD2DWWjq/1u/1WwURERGRMFEyGUF49ylWFwfcvAShcEP0CiYiIHAUUTIawq6aFlCRjblFW5C/qDSZlsSiSiIhIwlMwGUJ5TSvzirJITR7FLaqvgLRcyCqKWblEREQSmYLJEN441DK6/iXQN1Q40pWIRUREZAAFk0Hsrm3lzaoWTltUPLoX1ldA4fyYlElERORooGAyiAc3HwTgguUzI39RKAQNu9W/REREZBwUTAbx4OaDrJyTz5zCUXR8bTkEgQ4FExERkXFQMDnM/oZ2Xt3bMLraEoD6cr8t0lBhERGRsVIwOcxD4WacC5eXju6FmsNERERk3BRMDvPQ5oMsmZk7thE5GOTPjUWxREREjgoxCyZmNtfM/mZmW83sdTP7XHh/kZk9YmY7wtvCWJVhtKqaO3hxd93om3HAB5P8OZCSFvVyiYiIHC1iWWMSAL7gnFsKnApcb2bHA18BHnXOLQYeDT+fEh5+/RDOjaEZB/rmMBEREZExi1kwcc4dcM69HH7cDGwFZgOXALeGT7sVuDRWZRithzYfZGFJNsfOGMWKwj00h4mIiMi4TUgfEzMrA04AngdmOOcOgA8vwPQhXvMJM9tgZhuqq6tjXsb61i6e3VXLBctnYqOdubWrzQ8XVo2JiIjIuMQ8mJhZDnAX8A/OuaZIX+ecu8k5t9Y5t3batGmxK2DYI1sPEQy5sTXjNOz2W43IERERGZeYBhMzS8WHkt875+4O7z5kZqXh46VAVSzLEKmHNh9kTmEmy2fnjf7FWlVYREQkKmI5KseAXwFbnXM/7HfoPuCa8ONrgHtjVYZINXd089SOGi5YNoZmHFAwERERiZKUGF77dOBqYJOZbQzv+ypwI3CHmX0M2AO8N4ZliMhj26roCoa4cMUYhgkD1JVDWi5kjXLRPxERERkgZsHEOfcUMFT1w7mxet+xeHDTQWbkpXPC3DFOqdIzVHgstS0iIiLS66if+bWtK8D6N6p4x7KZJCWNMVhoqLCIiEhUHPXB5PHt1XR0h8Y22ytAKORH5ah/iYiIyLgd9cHkwc0HKcpO4+SyorFdoOUQBDoUTERERKLgqA4mnYEgj22r4vzjZ5CSPMZboVWFRUREouaoDiZP7aihpTMw9mYc0FBhERGRKDqqg8mDmw+Sm5HCWxaVjP0i9RWAQcHcaBVLRETkqHXUBpPuYIhHthzi7UtnkJYyjttQXwF5syElPWplExEROVrFcoK1KS0Ycnzx/GNZWjqGKej765nDRERERMbtqA0mGanJXH1a2fgvVF8Ox7x9/NcRERGRo7cpJyq62vxw4aKyyS6JiIhIQlAwGY+G3X6rocIiIiJRoWAyHhoqLCIiElUKJuOhYCIiIhJVCibjUV8BaTmQVTzZJREREUkICibj0TNU2Ma4KrGIiIgMoGAyHprDREREJKoUTMbKOQUTERGRKFMwGauWQxDoUDARERGJIgWTsaor91sFExERkahRMBmr3qHCmlxNREQkWhRMxqq+AjAomDvZJREREUkYCiZjVV8BebMhJX2ySyIiIpIwFEzGSiNyREREok7BZKwUTERERKJOwWQsutqg5aCCiYiISJQpmIxFwx6/VTARERGJKgWTsdCqwiIiIjGhYDIWCiYiIiIxoWAyFvXlkJoN2SWTXRIREZGEomAyWrufhS33QfEiMJvs0oiIiCQUBZNIdXfAw1+HWy6E5FR4539NdolEREQSTspkFyAu7N8I93wKqrfCiR+B8/8V0nMnu1QiIiIJR8FkOMFuePKH8MR/QlYJfPBOWPz2yS6ViIhIwjq6g0nF04Ab/Fh3O/ztu7D/FVhxJVz4H5BVNKHFExEROdoc3cHkt5dCsGvo45lF8N5bYdmlE1UiERGRo9rRHUw+dDdD1pgAzFiuWhIREZEJdHQHkwVvnewSiIiISD8aLiwiIiJThoKJiIiITBkKJiIiIjJlKJiIiIjIlKFgIiIiIlOGgomIiIhMGQomIiIiMmUomIiIiMiUoWAiIiIiU4aCiYiIiEwZCiYiIiIyZSiYiIiIyJShYCIiIiJThoKJiIiITBkKJiIiIjJlmHNussswIjOrBnaP4xIlQE2UipNodG+GpnszPN2foeneDE33ZnhHy/2Z75ybNtiBuAgm42VmG5xzaye7HFOR7s3QdG+Gp/szNN2boeneDE/3R005IiIiMoUomIiIiMiUcbQEk5smuwBTmO7N0HRvhqf7MzTdm6Hp3gzvqL8/R0UfExEREYkPR0uNiYiIiMQBBRMRERGZMhI6mJjZBWa23czeNLOvTHZ5JpuZ/drMqsxsc799RWb2iJntCG8LJ7OMk8XM5prZ38xsq5m9bmafC+8/6u+PmWWY2Qtm9mr43nwrvP+ovzc9zCzZzF4xs7+En+vehJlZhZltMrONZrYhvE/3BzCzAjO708y2hf/tOU33JoGDiZklA/8LXAgcD7zfzI6f3FJNuv8HXHDYvq8AjzrnFgOPhp8fjQLAF5xzS4FTgevDvy+6P9AJnOOcWwWsBi4ws1PRvenvc8DWfs91bwZ6m3Nudb/5OXR/vP8GHnLOLQFW4X+Hjvp7k7DBBDgZeNM5t8s51wXcBlwyyWWaVM65J4C6w3ZfAtwafnwrcOlElmmqcM4dcM69HH7cjP8HYja6PzivJfw0Nfzj0L0BwMzmABcDv+y3W/dmeEf9/TGzPOBM4FcAzrku51wDujcJHUxmA3v7Pa8M75OBZjjnDoD/cgamT3J5Jp2ZlQEnAM+j+wP0NlVsBKqAR5xzujd9fgR8GQj126d708cBD5vZS2b2ifA+3R9YCFQDt4SbAX9pZtno3iR0MLFB9mlstAzLzHKAu4B/cM41TXZ5pgrnXNA5txqYA5xsZssnuUhTgpm9E6hyzr002WWZwk53zq3BN6tfb2ZnTnaBpogUYA3wM+fcCUArR2GzzWASOZhUAnP7PZ8D7J+kskxlh8ysFCC8rZrk8kwaM0vFh5LfO+fuDu/W/eknXNW8Ht9XSfcGTgfebWYV+Obic8zsd+je9HLO7Q9vq4B78M3suj/+O6oyXPsIcCc+qBz19yaRg8mLwGIzW2BmacBVwH2TXKap6D7gmvDja4B7J7Esk8bMDN/Wu9U598N+h476+2Nm08ysIPw4EzgP2IbuDc65f3bOzXHOleH/jXnMOfchdG8AMLNsM8vteQycD2xG9wfn3EFgr5kdF951LrAF3ZvEnvnVzC7Ct/8mA792zn13cks0uczsj8DZ+GW1DwE3AH8G7gDmAXuA9zrnDu8gm/DM7AzgSWATfX0FvorvZ3JU3x8zW4nvhJeM/2PmDufct82smKP83vRnZmcDX3TOvVP3xjOzhfhaEvBNF39wzn1X98czs9X4TtNpwC7gWsL/j3EU35uEDiYiIiISXxK5KUdERETijIKJiIiITBkKJiIiIjJlKJiIiIjIlKFgIiIiIlOGgomITHlmdnbPyr0iktgUTERERGTKUDARkagxsw+Z2QtmttHMfhFe/K/FzH5gZi+b2aNmNi187moze87MXjOze8ysMLz/GDNbZ2avhl+zKHz5HDO708y2mdnvw7P1YmY3mtmW8HW+P0kfXUSiRMFERKLCzJYC78Mv2rYaCAIfBLKBl8MLuT2On3EY4DfAPznnVuJn3O3Z/3vgf51zq4C3AAfC+08A/gE4Hr8y6+lmVgS8B1gWvs6/xvIzikjsKZiISLScC5wIvGhmG8PPF+Kn+L89fM7vgDPMLB8ocM49Ht5/K3BmeF2V2c65ewCccx3OubbwOS845yqdcyFgI1AGNAEdwC/N7DKg51wRiVMKJiISLQbc6pxbHf45zjn3zUHOG24dDBvmWGe/x0EgxTkXwK9WexdwKfDQ6IosIlONgomIRMujwBVmNh3AzIrMbD7+35krwud8AHjKOdcI1JvZW8P7rwYed841AZVmdmn4GulmljXUG5pZDpDvnHsA38yzOuqfSkQmVMpkF0BEEoNzbouZfR142MySgG7geqAVWGZmLwGN+H4o4Jd0/3k4ePSsrAo+pPzCzL4dvsZ7h3nbXOBeM8vA17Z8PsofS0QmmFYXFpGYMrMW51zOZJdDROKDmnJERERkylCNiYiIiEwZqjERERGRKUPBRERERKYMBRMRERGZMhRMREREZMpQMBEREZEp4/8DHUZAcEBAXTYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (9, 7))\n",
    "plt.plot(list(training_history_lr_scheduler.keys()), [training_history_lr_scheduler[k]['acc'] for k in training_history_lr_scheduler.keys()], label = 'training acc')\n",
    "plt.plot(list(training_history_lr_scheduler.keys()), [training_history_lr_scheduler[k]['val_acc'] for k in training_history_lr_scheduler.keys()], label = 'val acc')\n",
    "plt.title(\"VGG-18: Training Accuracy\")\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"accuracy (%)\")\n",
    "plt.legend(loc = 'best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAG5CAYAAAAu8Fa/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABM7UlEQVR4nO3dd5xU5d3//9dnZmd7YZcOS1ORLoggxoKJLUaNLfaSXu6Y5GfuO5r+NfVOcqeanmgSo7HHbmLsvSIgqBQLSG8LC9vrzPX745pZFtgyuztld+f9fDwmZ+bMmXOuORDm7VXNOYeIiIhIqgTSXQARERHJLAofIiIiklIKHyIiIpJSCh8iIiKSUgofIiIiklIKHyIiIpJSCh8i0q+Y2TfN7C+JPlZE+g+FD5EUMrNHzOz7Hew/y8y2mVlW9PU8M/uXme02sz1mttLM/tfMStt9ZrSZXW9mW8ys1szWmtnfzWxqF9e/wMxeNLN6M3u6g/dPMLOlZlYdPd9n4/xe/4mWodbMWsysud3rP8V1c6Kccz9yzn060cf2lJk5MzskGecWyXQKHyKp9XfgcjOz/fZfDtzinGs1s6OBp4EXgKnOuSHAqUArMBvAzIYCLwL5wHFAETAXeAY4uYvrVwLXAj/Z/w0zCwH3An8GSoALgV+a2ezuvpRz7kPOuULnXCFwC/DT2Gvn3H+1u0ZWd+cSkcFP4UMkte4DyvCBAYBobcYZwE3RXT8FbnDO/dg5tx3AObfBOfcd59zT0WP+G6gGLnfOrXHeHufcDc6533Z2cefc4865O4EtHbxdBhQD/4ie71VgFTC9D983VoPwBTN7B3gnuu/XZrYxWsOyxMza34/vmtnN0ecTo5//mJltMLOdZvatXh6bZ2Y3RmuTVpnZV81sUy++T4mZ3WRmFWa23sy+bWaB6HuHmNkzZlYVvf4d0f1mZr8ysx3R9143s5m9vKUiA57Ch0gKOecagDuBj7bbfQGw2jm33MwKgPcBd3dzqpOAe51zkQSWbTtwG/AJMwua2fuACcDzAGZ2rJnt6eXpzwYWsDfIvArMwQeeW4F/mlluF58/FpgCnAhcY2bTenHsd4CJwEH42qHLevE9AH6Lrxk6CDge/2f5ieh7PwAeBUqB8uixAKcAC4FDgSH4WqVdvby+yICn8CGSejcC55tZXvT1R6P7wP9oBYBtsYPN7KfRfh91Zvbt6O5h+x1zZvSYGjN7tA9luw24BmgCngO+5ZzbCOCcez7aBNQbP3bOVUbDF865m51zu5xzrc65XwA5+MDQme855xqcc8uB5USbn3p47AXAj5xzu51zm4Df9PRLmFkQHxy+4Zyrcc6tA36BbzYDaMEHtjHOuUbn3PPt9hcBUwFzzq1yzm3t6fVFBguFD5EUi/4gVQBnmdlBwHz8f/0D7AYiwOh2x381+qN/LxDrM7Frv2MeiB7z30A2gJn9qV2nz292V65oR9U78GEoG5gBfNXMTu/9t22zcb9rfSXa9FEVrU0pwQeqzmxr97weKOzFsWP2K8c+ZYrTMPy9Wd9u33pgbPT5VwEDFpnZCjP7JIBz7kngd8Dvge1mdp2ZFffi+iKDgsKHSHrchP+Rvxx4tF3fjjrgFeDcbj7/BHB2rK9BR5xz/9Wu0+eP4ijTTOAt59wjzrmIc+4t4N/Ah+L4bHfals+O9u/4Gr4mojQamqrwP9rJtBXfFBIzrhfn2Mne2o2Y8cBmAOfcNufcZ5xzY4DPAX+IjZhxzv3GOXcEPtQdClzdi+uLDAoKHyLpcRO+38Zn2NvkEvNV4JNm9nUzGwFgZuXApHbH/BLfRPMPMzs42qGxCN+PolPRvhy5+BqUgJnlRke5ALwGTDY/3NbM7GB8R9jlffqmByrCj9ypALLM7Bp8R9dkuxP4hpmVmtlY4ItxfCY7eo9y2/VJuRP4XzMrMrMJwP8AsU6v50f/rMDXYjkgbGbzzWxB9F7XAY1AOIHfTWRAUfgQSYNoX4EXgQLggf3eex44Ad9B8e1os8TD+OG3v40esxM4Cv8j9jxQAyzD/7B/votLXw40AH/Ej7hpAK6PnnMN8El8X4hq/LDdu4G/gq+xMLPa3n/rNo8A/wHexjdZNNK7JpCe+j6wCXgPeBy4C9+3pSsr8Pco9vgE8CV8gFiLv/e3An+LHj8feCV6nx4ArnTOvYcPV9fjA8l6fLPZzxP1xUQGGnPOdX+UiMggY2afBy5yzh2f7rKIZBrVfIhIRjA/I+wxZhYwsynAV/CdeEUkxTTboIhkimz87K2TgD3A7cAf0lkgkUylZhcRERFJKTW7iIiISEr1q2aXYcOGuYkTJ6a7GCIiIpIAS5Ys2emcG77//n4VPiZOnMjixYvTXQwRERFJADNb39F+NbuIiIhISil8iIiISEopfIiIiEhK9as+HyIiIj3V0tLCpk2baGxsTHdRMlZubi7l5eWEQqHuD0bhQ0REBrhNmzZRVFTExIkTMUv24siyP+ccu3btYtOmTUyaNKn7D6BmFxERGeAaGxsZOnSogkeamBlDhw7tUc2TwoeIiAx4Ch7p1dP7r/AhIiIiKaXwISIi0gd79uzhD3/o3RqFp512Gnv27OnymGuuuYbHH3+8V+ff38SJE9m5c2dCztUXCh8iIiJ90FX4CIfDXX72oYceYsiQIV0e8/3vf5+TTjqpt8XrlxQ+RERE+uDrX/86a9asYc6cOVx99dU8/fTTfOADH+CSSy5h1qxZAJx99tkcccQRzJgxg+uuu67ts7GaiHXr1jFt2jQ+85nPMGPGDE455RQaGhoA+PjHP85dd93Vdvx3vvMd5s6dy6xZs1i9ejUAFRUVnHzyycydO5fPfe5zTJgwodsajl/+8pfMnDmTmTNncu211wJQV1fH6aefzuzZs5k5cyZ33HFH23ecPn06hx12GFdddVWf75mG2oqIyKDxvQdXsHJLdULPOX1MMd/58IxO3//JT37Cm2++ybJlywB4+umnWbRoEW+++Wbb0NO//e1vlJWV0dDQwPz58/nIRz7C0KFD9znPO++8w2233cb111/PBRdcwN13381ll112wPWGDRvG0qVL+cMf/sDPf/5z/vKXv/C9732PE044gW984xs8/PDD+wScjixZsoQbbriBV155BeccCxYs4Pjjj2ft2rWMGTOGf//73wBUVVVRWVnJvffey+rVqzGzbpuJ4qGaDxERkQQ78sgj95nz4je/+Q2zZ8/mqKOOYuPGjbzzzjsHfGbSpEnMmTMHgCOOOIJ169Z1eO5zzz33gGOef/55LrroIgBOPfVUSktLuyzf888/zznnnENBQQGFhYWce+65PPfcc8yaNYvHH3+cr33tazz33HOUlJRQXFxMbm4un/70p7nnnnvIz8/v4d04kGo+RERk0OiqhiKVCgoK2p4//fTTPP7447z00kvk5+fz/ve/v8M5MXJyctqeB4PBtmaXzo4LBoO0trYCfqKvnujs+EMPPZQlS5bw0EMP8Y1vfINTTjmFa665hkWLFvHEE09w++2387vf/Y4nn3yyR9fbn2o+GnanuwQiIjKAFRUVUVNT0+n7VVVVlJaWkp+fz+rVq3n55ZcTXoZjjz2WO++8E4BHH32U3bu7/m1buHAh9913H/X19dTV1XHvvfdy3HHHsWXLFvLz87nsssu46qqrWLp0KbW1tVRVVXHaaadx7bXXtjUv9UVm13zsfBd+Px8+/TiMPSLdpRERkQFo6NChHHPMMcycOZMPfehDnH766fu8f+qpp/KnP/2Jww47jClTpnDUUUclvAzf+c53uPjii7njjjs4/vjjGT16NEVFRZ0eP3fuXD7+8Y9z5JFHAvDpT3+aww8/nEceeYSrr76aQCBAKBTij3/8IzU1NZx11lk0NjbinONXv/pVn8trPa2qSaZ58+a5xYsXp+6Cbz8Kt54P590AM89N3XVFRCRhVq1axbRp09JdjLRqamoiGAySlZXFSy+9xOc///mE1FD0REd/Dma2xDk3b/9jM7vmo6HSb5s6ry4TERHp7zZs2MAFF1xAJBIhOzub66+/Pt1F6lJmh496hQ8RERn4Jk+ezGuvvZbuYsQtszucquZDREQk5TI7fMRqPppr01sOERGRDJLZ4aOt5iOxs+GJiIhI5zI7fKjPh4iISMopfIDCh4iIpFRhYWGP9g82mR0+1OFUREQk5TI7fKjmQ0RE+uhrX/saf/jDH9pef/e73+UXv/gFtbW1nHjiicydO5dZs2Zx//33x31O5xxXX301M2fOZNasWW1L22/dupWFCxcyZ84cZs6cyXPPPUc4HObjH/9427GJmIE02TJ3no+WBmiNLtrTpNEuIiKDwn++DtveSOw5R82CD/2k07cvuugivvzlL3PFFVcAcOedd/Lwww+Tm5vLvffeS3FxMTt37uSoo47izDPPxMy6veQ999zDsmXLWL58OTt37mT+/PksXLiQW2+9lQ9+8IN861vfIhwOU19fz7Jly9i8eTNvvvkmQEKWvE+2zA0fsVqPrFyNdhERkV47/PDD2bFjB1u2bKGiooLS0lLGjx9PS0sL3/zmN3n22WcJBAJs3ryZ7du3M2rUqG7P+fzzz3PxxRcTDAYZOXIkxx9/PK+++irz58/nk5/8JC0tLZx99tnMmTOHgw46iLVr1/KlL32J008/nVNOOSUF37pvMjd8xPp7DBkPu9aAcxBHGhURkX6sixqKZDrvvPO466672LZtGxdddBEAt9xyCxUVFSxZsoRQKMTEiRNpbGyM63ydrbu2cOFCnn32Wf79739z+eWXc/XVV/PRj36U5cuX88gjj/D73/+eO++8k7/97W8J+27JkLl9PmI1H0MmgAv7ZhgREZFeuOiii7j99tu56667OO+88wCoqqpixIgRhEIhnnrqKdavXx/3+RYuXMgdd9xBOBymoqKCZ599liOPPJL169czYsQIPvOZz/CpT32KpUuXsnPnTiKRCB/5yEf4wQ9+wNKlS5P1NRNGNR+lE/y2qQay89NXHhERGbBmzJhBTU0NY8eOZfTo0QBceumlfPjDH2bevHnMmTOHqVOnxn2+c845h5deeonZs2djZvz0pz9l1KhR3HjjjfzsZz8jFApRWFjITTfdxObNm/nEJz5BJBIB4Mc//nFSvmMiWWdVO+kwb948t3jx4tRc7NW/wr//B07+Pjx2DXxxCQw7JDXXFhGRhOloKXdJvY7+HMxsiXNu3v7HqtllSKzmQ51ORUREUiFzw0dDJWQXQsEw/1qLy4mIiKRE5oaP+krIK4OcIv9aE42JiAxY/akLQSbq6f3P3PDRUAn5pQofIiIDXG5uLrt27VIASRPnHLt27SI3Nzfuz2TuaJe2mo9i/1rhQ0RkQCovL2fTpk1UVFSkuygZKzc3l/Ly8riPz9zw0VDph9lmR1cQVIdTEZEBKRQKMWnSpHQXQ3ogc5tdYjUfWTkQCKnmQ0REJEUyM3yEW6GxCvLL/JTqOUUKHyIiIimSmeGjcQ/gfM0HRMOHhtqKiIikQmaGj9gEY/lD/TanWDUfIiIiKZKZ4SO2rkt+qd/mFKnDqYiISIokPXyYWdDMXjOzfyX7WnGL1Xy0NbsUquZDREQkRVJR83ElsCoF14lfW81H+z4fCh8iIiKpkNTwYWblwOnAX5J5nR47oOZD4UNERCRVkl3zcS3wVSDS2QFm9lkzW2xmi1M2O11DJQSy9k6trvAhIiKSMkkLH2Z2BrDDObekq+Occ9c55+Y55+YNHz48WcXZV2yCMTP/OqcYWhv8/B8iIiKSVMms+TgGONPM1gG3AyeY2c1JvF78Gir39veAvTUgzar9EBERSbakhQ/n3Decc+XOuYnARcCTzrnLknW9HonVfMRoZVsREZGUycx5Pur3q/loW1xO4UNERCTZUhI+nHNPO+fOSMW14tJZs4vCh4iISNJlXs2Hcx00uxT7rcKHiIhI0mVe+GiuhUhLJzUfmmJdREQk2TIvfOw/wRi0Cx9a2VZERCTZMi987D+1OqjPh4iISAplXvjoqOZDo11ERERSJnPDR/uaj0DABxCFDxERkaTLvPDR1uwydN/9OUXqcCoiIpICmRc+YjUfuUP23a/F5URERFIi88JHQyXklkAwa9/9Ch8iIiIpkXnhY/8JxmJyivwcICIiIpJUmRc+9p9aPUY1HyIiIimReeGj05qPYoUPERGRFMi88NFZzUd2oUa7iIiIpEDmhY+u+nw01fiF50RERCRpMit8tDb7TqX7z/EBPny4CLTUp75cIiIiGSSzwkfbBGOlB76n9V1ERERSIrPCR0frusTkFPutVrYVERFJqswKHx2taBvTVvOhTqciIiLJlFnho8uaD61sKyIikgqZFT7iqvlQ+BAREUmmzAofXdZ8KHyIiIikQmaFj4ZKyMqF7PwD32vrcKrwISIikkyZFT46m2AM1OFUREQkRTIvfHQ0wRhAVg4Es7WyrYiISJJlVvhoqOx4grEYrWwrIiKSdJkVPrpqdoHo4nIKHyIiIsmUWeGjsxVtY3KKFT5ERESSLHPCRyQCDbu7rvlQs4uIiEjSZU74aKryq9Z2WfNRpNEuIiIiSZY54aOrCcZiVPMhIiKSdJkXPrqt+dBQWxERkWTKnPDRtq5LJ/N8gGo+REREUiBzwkdbs0s383y0NkC4JTVlEhERyUCZEz66WtE2RovLiYiIJF3mhI/6SrAA5JR0fozCh4iISNJlTvhoqPRNLoEuvrLCh4iISNJlTvjobmp1UPgQERFJgcwJH91NrQ5+enXQyrYiIiJJlDnho76bqdXBLywHmuVUREQkiTIofOzqeo4PULOLiIhICmRO+GiohPwu5vgAhQ8REZEUyIzw0VwPrY09aHZR+BAREUmWzAgf8UwwBn4YbramWBcREUmmzAgf8axoG5NTpA6nIiIiSZQZ4SPemg/QyrYiIiJJlhnho0c1H4VqdhEREUmizAgfPa75UPgQERFJlswIHz3u86HwISIikiyZEz6yiyAru/tjc4oVPkRERJIoM8JHPBOMxajmQ0REJKkyI3zEs6JtTE4RNNeAc8ktk4iISIbKjPARz4q2MTlF4CLQUp/cMomIiGSozAgfPan50BTrIiIiSZUZ4aNHNR/FfqvwISIikhSDP3yEW6Gxqmd9PkBTrIuIiCTJ4A8fjXv8tid9PkA1HyIiIkky+MNH/S6/zR8a3/EKHyIiIkmVAeEjNrtpD+b5AIUPERGRJBn84aMn67pAu/ChlW1FRESSYfCHD+egZBzkD4vveHU4FRERSaqsdBcg6aad4R/xysqBYLaaXURERJJk8Nd89IbWdxEREUkahY+OKHyIiIgkjcJHRxQ+REREkkbhoyM5xQofIiIiSaLw0ZHsQmhW+BAREUkGhY+OqNlFREQkaRQ+OqLwISIikjQKHx1R+BAREUkahY+O5BRDayO0Nqe7JCIiIoOOwkdHYlOsN2t9FxERkURT+OiI1ncRERFJGoWPjuQU+q1WthUREUm4pIUPM8s1s0VmttzMVpjZ95J1rYRrq/lQp1MREZFES+aqtk3ACc65WjMLAc+b2X+ccy8n8ZqJkVPstwofIiIiCZe08OGcc0Cs3SIUfbhkXS+h1OdDREQkaZLa58PMgma2DNgBPOaceyWZ10sYNbuIiIgkTVLDh3Mu7JybA5QDR5rZzP2PMbPPmtliM1tcUVGRzOLET+FDREQkaVIy2sU5twd4Gji1g/euc87Nc87NGz58eCqK071QAWAKHyIiIkmQzNEuw81sSPR5HnASsDpZ10uoQCC6sq2G2oqIiCRaMke7jAZuNLMgPuTc6Zz7VxKvl1g5RepwKiIikgTJHO3yOnB4ss6fdFpcTkREJCk0w2lnFD5ERESSQuGjMwofIiIiSaHw0RmFDxERkaRQ+OiMwoeIiEhSKHx0JqdIq9qKiIgkgcJHZ2JDbd3AWI5GRERkoFD46ExOEeCguS7dJRERERlUFD46o/VdREREkkLhozM5xX6r8CEiIpJQCh+dUc2HiIhIUih8dCa70G+bFT5EREQSSeGjM6r5EBERSQqFj84ofIiIiCSFwkdn8ob4bf2utBZDRERksFH46EzuED/iZff6dJdERERkUFH46IwZlE6E3e+luyQiIiKDisJHV8omQaXCh4iISCIpfHSldBLs2QCRcLpLIiIiMmgofHSlbBJEWqBqU7pLIiIiMmgofHSldJLfqt+HiIhIwih8dKUsGj7U70NERCRhFD66UjwWAiHVfIiIiCSQwkdXAkEonaCaDxERkQRS+OhO6STVfIiIiCSQwkd3yiZB5TpwLt0lERERGRQUPrpTdhA012iNFxERkQRR+OhOqUa8iIiIJJLCR3fKNNeHiIhIIil8dGfIBMBU8yEiIpIgCh/dCeVC8RjVfIiIiCSIwkc8SrW6rYiISKIofMSjbKJqPkRERBJE4SMepZOgdjs016W7JCIiIgOewkc82ka8rEtrMURERAYDhY94aK4PERGRhFH4iIfm+hAREUkYhY945JVC7hDVfIiIiCSAwke8yrS6rYiISCIofMRLc32IiIgkhMJHvMomQdVGCLemuyQiIiIDmsJHvEonQaTVBxARERHpNYWPeGnEi4iISEIofMRLc32IiIgkhMJHvIpGQzBHNR8iIiJ9pPARr0AASieq5kNERKSPFD56omyS1ncRERHpI4WPnojN9eFcuksiIiIyYCl89ETZJGipg7qKdJdERERkwIorfJjZlWZWbN5fzWypmZ2S7ML1OxrxIiIi0mfx1nx80jlXDZwCDAc+AfwkaaXqrzTXh4iISJ/FGz4suj0NuME5t7zdvswxZDxgqvkQERHpg3jDxxIzexQfPh4xsyIgkrxi9VNZOVBSrpoPERGRPsiK87hPAXOAtc65ejMrwze9ZB7N9SEiItIn8dZ8vA94yzm3x8wuA74NVCWvWP1Y2STVfIiIiPRBvOHjj0C9mc0GvgqsB25KWqn6s9JJfqhtU026SyIiIjIgxRs+Wp1zDjgL+LVz7tdAUfKK1Y+1jXhZl9ZiiIiIDFTxho8aM/sGcDnwbzMLAqHkFasf01wfIiIifRJv+LgQaMLP97ENGAv8LGml6s8014eIiEifxBU+ooHjFqDEzM4AGp1zmdnnI7cE8spU8yEiItJL8U6vfgGwCDgfuAB4xczOS2bB+jWNeBEREem1eOf5+BYw3zm3A8DMhgOPA3clq2D9Wukk2PRqukshIiIyIMXb5yMQCx5Ru3rw2cGnbBJUbYJwS7pLIiIiMuDEW/PxsJk9AtwWfX0h8FByijQAlE4CF4Y9G2DowekujYiIyIASV/hwzl1tZh8BjsEvKHedc+7epJasP2s/4kXhQ0REpEfirfnAOXc3cHcSyzJwaK4PERGRXusyfJhZDeA6egtwzrnipJSqvysaBVl5muVURESkF7oMH865zJxCvTtmUDLWdzoVERGRHsmIESuRSEeVN31UUq7wISIi0guDPnw8tnI7x/7fk+yua07siYvLoXpzYs8pIiKSAQZ9+Bhfls+WqkbuWpLgWoqSsVCzTXN9iIiI9NCgDx9TRhUxf2Ipty7akNjml+KxgIPqLYk7p4iISAYY9OED4NIFE3hvZx0vrtmVuJOWlPutml5ERER6JCPCx6kzR1GaH+KWV9Yn7qSx8FGl8CEiItITGRE+ckNBLpg3jkdXbmd7dWNiTlo81m+rNeJFRESkJ5IWPsxsnJk9ZWarzGyFmV2ZrGvF4+IjxxOOOO54dWNiTphTCLlDNNxWRESkh5JZ89EKfMU5Nw04CviCmU1P4vW6NHFYAcdNHsZtizbQGo4k5qQl5Wp2ERER6aGkhQ/n3Fbn3NLo8xpgFTA2WdeLx6ULJrC1qpGn3qpIzAmLx6rZRUREpIdS0ufDzCYChwOvdPDeZ81ssZktrqhIUCjoxInTRjCyOCdxHU81y6mIiEiPJT18mFkhfjXcLzvnqvd/3zl3nXNunnNu3vDhw5NallAwwIXzx/PM2xVsrKzv+wlLxkLDbmhOwLlEREQyRFLDh5mF8MHjFufcPcm8VrwuPnIcBty6aEPfT1asuT5ERER6KpmjXQz4K7DKOffLZF2np0aX5HHitJHc+epGmlv72PG0JNqFpSpBI2hEREQyQDJrPo4BLgdOMLNl0cdpSbxe3C5dMJ5ddc08smJb306kicZERER6LCtZJ3bOPQ9Yss7fFwsnD2dcWR43v7yeD88e0/sTFY0BTM0uIiIiPZARM5zuLxAwLjlyAq+8V8m7O2p6f6KsbCgcoREvIiIiPZCR4QPg/HnlhILGzS/3seOphtuKiIj0SMaGj2GFOXxo5mjuXrqJhuZw709UPFbNLiIiIj2QseEDfMfTmsZWHly+pfcniU2x7lziCiYiIjKIZXT4OHJSGZNHFHJLX+b8KCmHljo/2ZiIiIh0K6PDh5lx6YLxLN+4hzc3V/XuJMXRuT7U9CIiIhKXjA4fAOfMLSc3FOj9ei+a60NERKRHMj58lOSFOHP2GO5ftoXqxpaen6BYs5yKiIj0RMaHD4BLF0ygvjnM/a/1ovaicCQEQmp2ERERiZPCBzB73BBmjS3h5pc34Ho6aiUQgOLRanYRERGJk8JH1KULxvPW9hqWrO/FqJXictV8iIiIxEnhI+rDs8dQlJPFLa/0YthtSbn6fIiIiMRJ4SOqICeLc+aO5d9vbKWyrrlnHy4ZC9VbIRJJTuFEREQGEYWPdi5dMIHm1gh3L+nhWi3FYyHSAnU7klMwERGRQUTho50po4qYP7GUW15ZTyTSg46nJeP8VgvMiYiIdEvhYz+XLpjAul31vLhmV/wfKonN9aHwISIi0h2Fj/2cOnMUpfmhns14qinWRURE4qbwsZ/cUJAL5o3j0ZXb2V7dGN+H8kohlK+aDxERkTgofHTg4iPHE4447nw1zuGzZtHhtgofIiIi3VH46MDEYQUcN3kYty3aQDjejqfFY9XsIiIiEgeFj05cumA8W6oaeWp1nMNnS8ZqinUREZE4KHx04sRpIxlelMPt8Ta9lIyD2u3Q2sMJykRERDKMwkcnQsEAH5lbzlNv7WBHPB1Pi8cCDmq2JL1sIiIiA5nCRxcunD+OcMRx19I4OpK2zfWhphcREZGuKHx0YdKwAo6cVMadr27EuW46nmqWUxERkbgofHTjwnnjWLernlfeq+z6wLaJxhQ+REREuqLw0Y3TZo2mKCer+zk/svP9ZGNqdhEREemSwkc38rKDnDlnDA+9uZWqhpauDy7WRGMiIiLdUfiIw0Xzx9PYEuGB5d2MZCkp10RjIiIi3VD4iMPMscVMG13cfdNLyVjVfIiIiHRD4SMOZsaF88p5Y3MVK7ZUdX5g8Vho3ANNtSkrm4iIyECj8BGnsw8fS3ZWoOvaj9hwWzW9iIiIdErhI05D8rM5dcYo7lu2hcaWcMcHtU00pqYXERGRzih89MBF88dR1dDCIyu2dXxA21wfqvkQERHpjMJHDxx10FDGleVxR2dNL8VjAFPNh4iISBcUPnogEDAuOGIcL67ZxYZd9QceEAxB0ajkTDTmHDR20dlVRERkgFD46KHz5pUTMPjnks5qP8YmZ4r15bfDL6ZCfTfTvIuIiPRzCh89NLokj+MPHc4/F28iHOlgsblkzfXx9sPQUg+blyb+3CIiIimk8NELF84fx7bqRp59u+LAN0vG+WaX7lbB7QnnYP0L/vnW1xJ3XhERkTRQ+OiFE6aOZFhhDre8sv7AN4vHQmsDNOxO3AV3vg110aCzdXnizisiIpIGCh+9kJ0V4JIjx/HE6h1srNyv42ky5vpY95zfjjkctih8iIjIwKbw0UuXLJhAwIybX96v9qOk3G8TGj6e9zUq08+Gqg3qdCoiIgOawkcvjSrJ5dQZo7j91Y00NLeb8bQ4Gj4SNdGYcz58TDwWxszx+7YuS8y5RURE0kDhow8+dvREqhpaeGB5u6BRMBwCocTVfMT6e0w4BkbP9vu2LEvMuUVERNJA4aMP5k8sZeqoIv7+4npcbHRLIOD7fWx5DSKRvl9k3fN+O/FYyCuFIRNU8yEiIgOawkcfmBkfO3oiq7ZWs3h9u9Etcz8K7z0DD3+970Nu1z0PRWOg7CD/eswcjXgREZEBTeGjj86eM5bi3Cz+/uK6vTuP/R943xdh0Z/h8e/0PoC07+9h5veNngO71yV2KK+IiEgKKXz0UV52kAvnj+ORN7exrarR7zSDU34I8z4JL/wanvlp706+8x2o2+HDR0ys34dqP0REZIBS+EiAy4+aSNg5bl20Ye9OMzjtFzD7Enj6Rz6E9NT6dv09YsYc7rcKHyIiMkApfCTA+KH5nDBlBLe+soHm1nadTAMBOOt3MONceOwaWHR9z0687nkoGr23vwdAfhmUjNeIFxERGbAUPhLko0dPZGdtE/95c+u+bwSCcO51MOV0eOgqeO3m+E7YUX+PmNGHacSLiIgMWAofCXLcIcM4aFjBvh1PY4IhOP8GOPgEuP+L8MZd3Z9w17tQu33fJpeYMXOgci00VvW12CIiIimn8JEggYBx+fsm8NqGPby+ac+BB2TlwIW3wPij4MEroXZH1yeMrecy8bgD3xsd6/fxep/KLCIikg4KHwl03hHlFGQHufHFDla7BcjOhzN/B62N8PRPuj7ZuhegcNS+/T1iNOJFREQGMIWPBCrKDXHu3HIefH0Lu2qbOj5o2CFwxCdgyd+h4u2Oj+mqvwdA4XC/0Fw6+n20NsHqh+Cez8JLf0j99UVEZMBT+Eiwj75vAs2tEW5/dWPnB73/6xDK9xOQdWTXGqjd1nF/j5jRc1I34iXcAu8+DvddAT+bDLdfDK/fCU/+EJpqU1MGEREZNBQ+EmzyyCIWHjqc659bS1V9S8cHFQyD4/4b3nrIN6/sr6v+HjGjZ/tOqU01fS90Z7avgH/9N/xiCtz8EVj1IEw9HS69Gz72ALTUwaoHknd9EREZlBQ+kuDrp06lqqGF3z75TucHHXWFbzp59NsHLkC3PtrfY+jBnX9+zBzAwbY3ElHkA7U2w40fhuW3w0Hv951lr3oHzvkjTD7JB6PSSbDs1uRcX0REBi2FjySYPqaYC44Yx40vrWPdzrqODwrlwQnfhi1LYcU9e/d3198jZvQcv01W08vap6F+F5z3N/+YdgaEcve+bwazL/a1NHs2dHoaERGR/Sl8JMlXTjmUUDDAT/6zuvODDrsQRs2CJ77nO3KCn7+jZitMPKbrCxSN9LUjyRrxsuIeyCnxc5N0ZvZFfrv8juSUQUREBiWFjyQZUZzL548/mIdXbOOVtbs6PigQhJN/4GsOFl3n98XT3yNmzJzkjHhpaYTV//a1HVk5nR9XOgEmHAvLb+v9yr0iIpJxFD6S6NPHHcToklx++O9VRCKd/Dgf/AE45CR49mdQXxmd32MkDD2k+wuMngM734bmTpp2emvNk9BU7dek6c6ci6FyDWxclNgyiIjIoKXwkUR52UG+euoU3thcxX3LNnd+4Mnf96NWnv15fP09YkbPBheBbW8mrtDgm1zySuGg47s/dvpZftjwcnU8FRGR+Ch8JNlZs8dyWHkJP334LRqawx0fNHIGzLkUXvkj1Gzpen6P9sbM8dtENr20NMBb/4FpH/Zr0nQnp8gf++a9/rMiIiLdUPhIskDA+Pbp09lW3chfnlvb+YEf+BZkRUeTTIgzfBSNhoLhiR3x8s5j0FwbX5NLzOyLoanKhxYREZFuKHykwJGTyvjQzFH88Zk17Khu7Pig4tHwgW9C+XwYNjm+E5v5fh+JHPGy4h7IHxZfh9eYSQv9nCXLb0tcOTqy/3woIiIyICl8pMjXPzSVlnCEXzzayXouAEd/CT79eHz9PWLGzIGK1Ylp8miug7cfgelnQjAr/s8Fgn7Y8LtPQM32vpejIxVvwY9Gw4ZXknN+ERFJGYWPFJkwtICPHz2RO5dsZOWW6sSdePRscOHEdDp9+xFoqe9Zk0vMnEt8Od64s+/l6MjKB/xqwMmuXRERkaRT+EihL54wmSF5Ia65/01awglqQojNdJqITqcr7vHDfCcc3fPPDpsMY+fBsiTN+fHuY367+l8Q6aTjroiIDAgKHylUkhfiu2fOYPH63Vxz/wpcIn6kS8ohf2jfw0dTje9sOv0s34zSG3Muhh0rYNvrfSvL/uorYdOrMGI61FXAhpcSe34REUkphY8UO2vOWK54/8HctmgDN7ywru8nNPNNL1v62On0rYd9s0ZvmlxiZpwLwWxf+5FIa57085l88EeQlQcr70/s+UVEJKUUPtLgqlOm8MEZI/nhv1fy1Fs7+n7C0XOgYhVUb/E1GOGWnp9jxT1QNAbGLeh9OfLL4NBT4Y1/9q4MnXn3cT/p2aSFcMiJsOrB1Ix8aaqF2orkX0dEJMMofKRBIGD86sI5TB1VzJdufY23t9f07YRj50KkFX45DX5cDj8YBt8rg/8dA/83Ca49DF67ufPPN+zxP/AzzoZAH/9KzLkE6nf6JpxEiET8uQ4+0TcHTT/LL7y3eXFizt+V+/4LfjEF/vlx2Phq8q8nIpIhejCesmfM7G/AGcAO59zMZF1noMrPzuKvH5/Hmb97gU/d+Cr3XXEMQwu7WMStK4eeCudeD41Vfshta5NvQok9tr4O938BNi+BU39y4GJxb/0Hws19a3KJOeQkP0/Isltg6ml9P9/WZT7MTD7Fvz70g75pZ+X9MO7Ivp+/M43VfvTPiOnw7pOw4l4/B8tRV8C0Hg5FFhGRfSTzX9C/A78DbkriNQa00SV5XP/ReVz455f4r5uXcPOnF5CT1YvOnsEQHHZB5++HW+HJH8AL18K2N+CCm6B4zN73V9wDJeOgfF7Pr91RWeZcAi/9Hirfg7JJfTvfO48B5ptbAHJL4KAP+KG3p/ywZ3Oi9MS7j/lAdtrPYNRMWHYrvPxHuOsTUFwOCz7rp8TPH5q8MoiIDFJJa3Zxzj0LVCbr/IPFnHFD+Pn5s3l13W6+ec+biRkBs79gFpz8PR86dqyCPy/0C9iBH0my5knf5JKoH9GjroBAFjz/q76f693HYMzhUDBs777pZ0LVhsSuabO/VQ9CwQhfu5JTBAs+B19aAhfdCqUT4bFr4GcHww9Hwi+nw5+Og3+cA3d/Bh7+Biy+QTOyioh0Iu11x2b2WeCzAOPHj09zadLjw7PHsKailmsff4eDRxRwxfsPSc6Fpp8Fw6fC7ZfCjWfCKT/wP6yR1sQ0ucQUj4a5l8OSG+H4r/rhwL1Rtws2LYbjv7bv/imn+XCz8n4fTBKtpdHXuMw6b99hx4EgTD3dP7Yuh7VPQ91OqN8V3e6EXWv885Y62L7C15yoZkREZB9pDx/OueuA6wDmzZuXhP/sHxiuPHEyayrq+OnDb1HfFOYrpxyKJeNHa/gU+MyTcN/n4ZFvQijf/5d8on/Ej7kSlvwdXvi1/wHujTVPAg4mn7zv/vwyv/bMyvvhxO8k/sd97dN+cb1pH+78mNGz/aMjzsFj/w9e/C3kDYETvp3Y8omIDHAa7dJPmBm/vGA2F80fx++eepcrb19GY0uSZvLMLYYLb/Y/3K2NcNhFif8BHzLer3a75Mber/fy7mO+T0VHwWj6WVC51tcuJNrqByGnBCYu7N3nzeDkH8Dcj8KzP/MhRERE2ih89COhYIAfnzuLr546hQeWb+Hyv75CZV1zci5mBsf9D3z5DVh4dXKucex/Q6QFXurFj28k4of/xobY7m/qGWABWPVA38vZXrgVVj/kR9VkZff+PGZwxrUw/Wx49NuwVP2uRURikhY+zOw24CVgipltMrNPJetag4mZccX7D+F3lxzO8k1VnPuHF3hvZ13yLlhSnrxho0MPhpnnwat/8/03emLLa74vRWyI7f4Kh8P4o/2ol0Ta8BI0VMK0M/p+rkDQD4E+5CR48EpYcV/fzykiMggkc7TLxc650c65kHOu3Dn312RdazA647Ax3PaZBVQ3tnLuH17g1XUDdODQcV/xK+W+/Ieefe6dRwGDg0/o/JjpZ/mZXSve7lMR97HqQcjK9YEhEbKy4YJ/QPmRcPenfW2OiEiGU7NLP3bEhDLuveJoSvOzufT6V7h/2eZ0F6nnRkz1Q2MXXednUo3Xu4/5eUcKhnZ+TKx2YlWC1npxzq+ae/CJkF2QmHMCZOfDJXf4e3HH5bDhlcSdW0RkAFL46OcmDC3gniuOZs64IVx5+zK+cudy9tQnqR9Ishx3FTRV+wASj7qdsHkpHHJy18cVj/E1ColaaG7LUqje3PUol97KGwKX3QNFo+GW87U4nohkNIWPAWBIfjb/+PSRfPEDh3Dfss2c9MtnefjNbekuVvxGHwaHfsg3vTTFsY7Nu0/gh9jG0fQx/Sw/a2vlewe+FwnDa7fAb4/wo066s+pfYEHf2TQZCkfAR++H0glw50f9fCvVW5NzLRGRfkzhY4DIyQpy1QencP8XjmFEUQ7/dfMSvnDLUnbWNqW7aPFZeBU07IbFf+v+2Hcf8+vDjI5j7pFYLUX7US/O+SDxx6Ph/iv8LK5P/i+sf7Hrc616ECYd5+cRSZYh4+AzT8FJ3/P9P36/wM+HotlQRSSDKHwMMDPHlnD/F4/hqlMO5bGV2zn5l89w/7LNyZmWPZHK5/k1WV78LTTXd35cJOx/lA85Kb4VdksnwOg5e5sx1j0Pfz0Z7rjUn+uCm+DLr/uJ1O75nF8wriMVb8Gud/wQ3mQLZsGxX4bPv+hrhR68Em4608+OKiKSARQ+BqBQMMAXT5jMv/+/Y5kwtIArb1/GJ//+Kvcv28yWPQ3pLl7njv8q1FV0PefF5qW+hmT/WU27Mv0sv2LvjWfC30+Hqs3w4d/AFS/793KK/JDX6s3wn691fI5YzUkqwkfM0IPhYw/6sm593dfUPP8rH5pERAaxtE+vLr03eWQRd3/+aG544T1+/cQ7PPVWBQBjh+Qxb2Ip8yaWMX9iKYeOKCIQ6Afri0w4GiYcA8/8n18x9rALoGjUvse8+5ifPKyrIbb7m34WPPF9v97KyT+AIz8Dobx9jxk330+m9sxP4NBTYMY5+76/6l9QPt+vS5NKZnDEx3w/k4eugse/61fNPez81JZDRCSFrD9V18+bN88tXrw43cUYkFrDEVZvq2HxukpeXb+bV9+rZEeN7w9Skhfiovnj+MzCgxhWmJPegm5f4ZsZNr0aDRknwpxL/GJxoVy47v0QzIZPPdqz825Z5ptW8oZ0fky4Bf72Qd+8ccVLfrQMwJ4NcO0sOPn7fk2adIlE4GcH+dqXs36XvnKIiCSImS1xzs07YL/Cx+DknGPT7gYWvVfJk2/t4D9vbCUnK8hlR43nswsPZnhRmkPIzndg+W2w/HbfHJJbAtPOhNf+AR/4NhyfpCnfd62BPx0L446Ey+71/Upe/iM8/HX40lLfFJJOt17oy/gl/f9ARAa+zsKH+nwMUmbGuLJ8PnJEOb+/ZC6P/c/xnDpzFH99/j2O++mT/PBfK9lR05i+Ag6bDCde49eWufw+OPRUeOMu/16yhrqCDxcf/JFfuXbRn/2+VQ/CiBnpDx4A44/yHV/rdqa7JCIiSaOajwyztqKW3z31Lve9tplQMMClCyZwyYLxHDy8AEv0yrY91VjtV6odMye513EObrsY1jwJl90FN53l+4N84JvJvW48Nrzsm4YuuhWmnp7u0oiI9ImaXWQf63bW8bun3uXe1zYTjjgmDM3nhKkjOHHqSI6cVEZ21iCvFKutgD++z0961toI//U8jJqV7lJBSyP8ZBws+Byc8sN0l0ZEpE8UPqRDW6saeGLVDp5cvYMX3t1JU2uEwpwsjps8jBOmjmDuhFLKS/PIyepgWfuB7u1H4NYLYMgEuHK5H3nSH/z1g+Ai8OnH0l0SEZE+6Sx8aKhthhtdksdlR03gsqMm0NAc5oV3d/LE6h08uXo7/4lO4R4wf9zEYflMGFrAhDK/PWREIQcNK+gfw3h749APwgd/7Ee99JfgAb7fx0u/h5aGA4cMi4gMAqr5kA4551i5tZrVW2tYX1nP+l11rN/lt7vrW9qOK8gOMmNMCTPGFjNrbAmzxpZw0PBCggM1kPQHb/0HbrsIPv4QTDwm3aUREek11XxIj5iZDxVjSg54r7qxhfU761m9rZo3N1fx5pZqbl+0kRta1gGQnx1kXGk+gYARDEDALPrwz/Nzspg6qohpo4uYPrqEg4YXEAoO8j4mPTFugd9ufFnhQ0QGJYUP6bHi3BCzykuYVV7C+fPGARCOONZU1PLGpire2FzFlj0NRJyvQYk4R7jd8121Tfz9xV00t/rF1LKDAQ4dVci0UcVMG13M1NFFTB1VTFlBdjq/Zvrkl8HwqX7ki4jIIKTwIQkRDBiHjizi0JFFfOSI8m6Pbw1HWLuzjpVbqlm1tZqVW6t5cvUO/rlkU9sxw4tymDqqiCkji5gyqojJI4vIzw621aAEA9EalYCRFTDKCrIHTw3KuAWw8j4/62k8C+yJiAwgCh+SFlnBQFtYOfvwsYCvGamobeKtbTW8ta2GVVtreGt7Nf94eT1Nrd0vOR/rGDu2NI/y0jzKS/P9dkgexXkhcrICZMcewX2fp32Ok/2Nfx8svREqVsPI6ekujYhIQil8SL9hZowoymVEUS7HTR7etj8ccazbVcfaijqaWyOEnSMSiTbnRLctYceO6kY27W5g0+4GXl6zi23Vm4nE0Z96SH6IQ0cUceioQqZEA9GUUUUMye+82ScScThIXsfa8Uf57YaXFD5EZNBR+JB+LxgwDh5eyMHDC3v0uebWCNuqGtm0u57aplaawxGaWyM0tfqtfx5m855G3tlew/3LtlDT2Nr2+RFFOQwvyqGpNUJjS3ifbXNrBDO/aF9ZfjalBdmU5mdTVhCKbrMZWZzLiOIcRhXnMrI4l4KcHvzfrXQiFI7y/T7mf6pH31tEpL9T+JBBKzsrwPih+Ywfmh/X8c45tlU38ta2Gt7eXsPqbTXsqW8hNxQgNytITihATrstzrG7voXK+mb21DezeU8Db26uorKumebwgc1EhTlZjCjOYURRDsMKcxhakE1ZQQ5DC7Ojz31gmTA03zcDjV+gTqciMigpfIhEmRmjS/IYXZLH+6eM6PV5nHPUNLWyo7qJHdWNbK9pZHt1E9uqGtkRfb5iSzW7apuoblfTEvOrC2dzzuHlvt/HyvuhajOUjO3LVxMR6VcUPkQSzMwozg1RnBvikBFdNxU1t0bYXd/MrtpmKuuaueaBN/n7C+ui4SPa72Pjy1DykRSUXEQkNTSGTySNsrMCjCzOZfqYYo6dPIyPvW8iyzdVsXzjHhg5C0IFanoRkUFH4UOkHzln7ljys4Pc/PJ6CGZB+TyFDxEZdBQ+RPqR4twQZx8+lgeWb6GqvsX3+9j+JjRWp7toIiIJo/Ah0s9ctmACTa0R/rlkox/x4iKw6dV0F0tEJGEUPkT6meljipk3oZRbXtlAZMw8sABsfCXdxRIRSRiFD5F+6PL3TeC9nXW8sKkJRs3yM52KiAwSCh8i/dCpM0cxtCCbf7y0HsYdBZsWQ7gl3cUSEUkIhQ+RfignK8iF88fx+KrtVA6bCy31sO31dBdLRCQhFD5E+qlLFozHAf/cHp3ddIP6fYjI4KDwIdJPlZfmc+LUEVy/vAk3ZIL6fYjIoKHwIdKPXXbUBHbWNrG56DA/2Zhz6S6SiEifKXyI9GMLJw9nfFk+D1VNhLodsPu9dBdJRKTPFD5E+rFAwLjsqPH8s2Kc3/HazRA+cCVcEZGBROFDpJ87/4hxbAiWs7bwcHjuF/C7I2DpPzT0VkQGLIUPkX6utCCbMw4r58PVX6PhvJshdwg88EX47VxYfAO0Nqe7iCIiPaLwITIAXP6+CdQ1R/h/q8ZT97HH4ZJ/QsFw+NeX4TeHw6LroWFPuospIhIXc/2o9/y8efPc4sWL010MkX7HOceP/7Oa659by5iSPH587iwWTh4Ga56EZ/5v79oveWVQdhAMPdhvY4+8Uj9SxkWij/De5/lDoaQ8vV9QRAYlM1vinJt3wH6FD5GBY/G6Sr569+usrajjvCPK+fbp0xiSF/JzgGxaDJVr9z6qNgFx/v975EyYejpMOQ1GzwazpH4PEckMCh8ig0RjS5jfPvkOf3pmLaX52fzgrBl8aNboAw9saYQ962HXGmiqBgv6UGEBCAT91gI+qKx+CDa+7GtCisthyod8GJl4LARDqf+SIjIoKHyIDDIrtlTx1bteZ8WWaj44YyTfPn0648rye3/Cup3w9sM+iKx5ElobILsIJh4Dk46Hg46HEdNVKyIicVP4EBmEWsMRrn/uPX71+Ns0t0Z430FDOXfuWD40azSFOVm9P3FzPax9Gt55FN57FirX+P35w2DSQh9Eyo8EnK9haamHloa923ATZBdC3hDf3yQ3ti3xtS4ikhEUPkQGsc17GrhnySbueW0z7+2sIzcU4NQZozhnbjnHHjKMYKCPtRV7NvoQ8t4zsPYZqN3W+3PllsDwqTDmcP8YPQeGTe46lETCvmamfhc07vEje2Lbht3+eSjPn2vM4VA6UTU0Iv2AwodIBnDO8drGPdyzdBMPLt9KVUMLI4pyOHHaCGaMKWH6mGKmjioiP7sPtSLOwc53YNvrvj9IVp7/4Q/lR7e5EMyGptoDA0LDHqjfCdtXwNblvqYEIFTgO7qOmeNDSG0F1G6Huui2fpfvj9Ihg9ziaI1LdM6T3CF7w82Yw2HYoZCVDYEsCIT8Npi17+tAUIFFJMEUPkQyTFNrmKdW7+DupZtZ9F4lVQ1+RlQzmDSsgOmji9vCyMHDCykvze97DUlPRMKw823Ysgy2vOYf21737xWMgMJ2j9jr/KG+KSd3yN4mnZxiHxxam2HHyr3n2vKafx3pwXT0gax2j6APVsMPhZGzYFT0EQsyItIthQ+RDOacY0tVIyu3VLNySzUrtlSxcms1m3Y3tB2TnRVg0tACDh5RwMHDC9sek0cWkhtKUT+NSCQ6IidBIail0dey7H7Ph5BIq5+Wfp/nLT4IxfZFWv36OZFWaK7zAWbHSmht9OcMhGDEVBh1GEw+GSafAtkFiSmvyCCj8CEiB6hqaOHt7TWsrahlbUUdaypqWVNRx/pddUSi/zQEojUlU0cXM21UEVNG+dqS8tI8LFOaKcKtvtPttjf2Pra8Bg2Vvnbk0FNg+tk+iOQUpru0Iv2GwoeIxK2pNcyGXfW8s6OW1dtqWL21mtXbathQWd92TH52kJHFuQwvyvGPwpy25yOKchhZnMvI4lxK80ODM6REwrD+RVh5H6x8AOp2+CAy+SQfRMrn+TlTgn3oXyMywCl8iEif1Ta18ta2GlZvq+bdHbVU1DSxo6aJnTVNVNQ0UdN0YP+K7GCAEcU5jIqGkRHFOYwdkucfpX5bVpA9sANKJOxnmV1xH6x6wHeSBT+xW8lYGDIBSifAkIl+m1fmO+ZmRTvoZkUfoTzfWTcYinaE1fJbMrApfIhI0jU0h9lZ28SOmka2VzexraqR7TWNbK/yr7dXN7KtupH65vA+n8sNBRgTDSSjS2IhJZeR7WpQhhVmkxUcAD/GkTBsXgIVq2H3ej/LbGwbCyVxs3ZBJDpCJ5jtXwfbPQIhP1ttuNk/Wpt8f5Zwk++IG2n1nWSz8iArJxp2cvYGntySvR14Y4/cIX5/pNWPSmqu2/uIvQ7l+U7A7R8Fw/w2EIqWJ1qW1qa95XMuev1oGYLZ/nkgSyOOEi0SXcspTfe2s/Ch+kARSZi87CDjyvK7nGnVOUdVQwub9zSweXdD23ZLld++vb2Gipqmtj4nMWZQmJ1FKCtAKGiEggGyswJkBwOEggGKcrPagsrIWE1LLMgU5RBKVXAJBGHckf6xv5YG2LMBGqv889bGA7fh5mhH2LDvDNtRB9lw7BENFuFmPxQ5mL33hzyYEx0KHf1RDzdHr9HkrxPbNtdB9RY/HLphtz9/PLJy93bCTRQLRINUZz+S5u9vIOhrlWKjkmLb7ELf+Te7IPo89jrf359w9F5F2t2/cMvea3f0CDf5Sfeaa6OhKxq8Wur8vQ+0C4D7DN8ORJdWiv5Fdq7d88jea7dtW/b+OeaWQH5ZuzAYfZ5b7MvRWBWd66aq3bw3VdG/J2EfNmLbff7M2g+Lz937fMLRcNJ3E/tn2Q2FDxFJKTNjSH42Q/KzmTGmpMNjwhHHrtqmttqSWO1JbVOYlnCE5taI34b9tiXsA82r6yrZUd1Ec/jAOUFK8kIMLcxmWEEOQwuz/aMgh7KCbPKyg+SFguRnB9s9zyIvFCQnFCAnK0BOVpDsrEDfhiOH8mD4lN5/Ptmc8z+ssSDSWOV/WLML/I9UdqH/IQ/l+x/7cKv/8avftfdRt9PP5RIJR2tmojUcsefBaLhobd4btlob977uKvy4yN7/ko+NUIo9D7dEw0EtNFZD9Vb/vLnWBwYL7G3Sal9jFAwB1m7F5/YPF/3+0e+eOwSKx+69H4GsdgFx/7AYjoao6N8Xa7+16D2Jzj0TC42xe9NYBfWV/s+gerMfsdWw23+XrNy9tVJ5Q6BwFAyb4l8Hs33osfYBLbqNRO9PS+Pe2Yhbo7MTW+prFBU+RKTfCQaMEdGml1l0HFA645yjsq55n2aeHdVNVNY1sbOumV21Tby7o5ZX3mtmd30zPW15DgWNnCwfUk6ePpJPHD2RySOLenaS/srMj9bJKYQh47o/Ppjlm1kKhiW/bOIDzSBZnkDhQ0QGFTNjaGEOQwtzmD6muMtjW8MRqhpaqG8O09gSpr45TENLmIbmvc+bWsM0tURoavU1Lk2tYZpaI+yoaeLuJZu49ZUNHHvIMD5xzEQ+MGUEgVRO1CaZZZAED1D4EJEMlhUM+KDSy89X1jVz26IN/OOl9XzqxsVMGJrPx943kfPnlVOUG0poWUUGE412ERHpo5ZwhEdWbOOGF9axZP1uCnOymDm2mDFD8hhTkue3Q3Kj27y+rTgsMoBotIuISJKEggHOOGwMZxw2htc37eG2RRt4Z3str6ytZFt1I+H9hu4UZAejTUO+0+vwIr8dWphNcW6I7CzfyTU72tE1J+RH9eSGghTmZJGfE6QgO6vLzq/hiKOhJUx9cyutYcfQwmxysgZPtb0MbAofIiIJdFj5EA4rH9L2ujXs+4dsrWpg855GtuxpoKKmiZ21TeyqbWbT7nqWbdxDZd2Bw4u7kxsKUJCdRUFOFqGg0dgSob65lfpm3y9lf2UF2W2zz46KDkkeXpxLcW4W+dlZFERDTUGOH+2Tnx30k785cDic84NF29eYmxlGdEkezI+GNcgK+CHRwYAN7AnkJCkUPkREkigrGGhrbjliQufHRSKO3fXN1Da1Rju2xh7hts6ujS1h6pp8bUZdU5i65lbqmvyjJezIjQ4Xjg0Z9tsssgLGzpomtlXvnext1dZqdtb2PPD0lJmf5TY7GCAUnZclKzpPSzBgZAWMrKCRFQi0PQ9F524JBY2s6GezAtb2+VC7Y2JzvQQCRlNrmMYWf58aop2IG1r8tjnsaGmNDc2O+NfR58GAH8GUE61xygkF22qegmb7BK72U3cEAkZulq+Ryg3Ftns/G4m46Gf3hjbnovckWm5fqxVsq+3KCli0xmq/zs/NrTS0hGkJO1ojEcIRR2vY0Rrxj3AkQm5WkIIcH0YLc9o/938HOjO8KIfDx5cm9y/CfhQ+RET6gUBg7yidVGkNR9hV5wNPfTTMxIJNbBtxbr/ajWhth0V/VF3HP7Bh5/adj6XV0RwO09Lqf/T9j2Zknx/Q1rB/Xdva6o8JO5qj2/bzu7SE956jI7mhAHkhP19Lbii4N0xEf+wLc7N8cIkGodaI2xv4WsJUN7S0Bb9YJY+1/c/eexCOOBqj4aaxJUJju+OTITcUaAtd+we3gEFjS6QtkLaE4y/ISdNG8JePzU9ewTug8CEikqGyggE/I2y6C9JLkYijJeLDSDjs2iaES1czj3M+LDW2RGgNR/YNbdHAZviQ1hytzYqFnubWiA9nYdc24d3emqwsckM9+17NrRHqmlqpbWqlLtrvpzNFuamPAgofIiIyIAUCRk4gSH8ZPGQWa75Jf8fe7KwA2VnZlBZkp7soHRoAqzSJiIjIYKLwISIiIiml8CEiIiIppfAhIiIiKaXwISIiIiml8CEiIiIppfAhIiIiKaXwISIiIiml8CEiIiIpldTwYWanmtlbZvaumX09mdcSERGRgSFp4cPMgsDvgQ8B04GLzWx6sq4nIiIiA0Myaz6OBN51zq11zjUDtwNnJfF6IiIiMgAkM3yMBTa2e70pum8fZvZZM1tsZosrKiqSWBwRERHpD5IZPjpa+/eANX2dc9c55+Y55+YNHz48icURERGR/iCZ4WMTMK7d63JgSxKvJyIiIgOAOXdAZURiTmyWBbwNnAhsBl4FLnHOrejiMxXA+l5echiws5efzQS6P53Tvemc7k3XdH86p3vTuUy6NxOccwc0a2Ql62rOuVYz+yLwCBAE/tZV8Ih+ptftLma22Dk3r7efH+x0fzqne9M53Zuu6f50Tvemc7o3SQwfAM65h4CHknkNERERGVg0w6mIiIik1GAKH9eluwD9nO5P53RvOqd70zXdn87p3nQu4+9N0jqcioiIiHRkMNV8iIiIyACg8CEiIiIpNSjCh1bP3cvM/mZmO8zszXb7yszsMTN7J7otTWcZ08XMxpnZU2a2ysxWmNmV0f26P4CZ5ZrZIjNbHr0/34vu1/2JMrOgmb1mZv+Kvta9AcxsnZm9YWbLzGxxdJ/uTZSZDTGzu8xsdfTfn/dl+v0Z8OFDq+ce4O/Aqfvt+zrwhHNuMvBE9HUmagW+4pybBhwFfCH6d0X3x2sCTnDOzQbmAKea2VHo/rR3JbCq3Wvdm70+4Jyb027+Ct2bvX4NPOycmwrMxv8dyuj7M+DDB1o9dx/OuWeByv12nwXcGH1+I3B2KsvUXzjntjrnlkaf1+D/ARiL7g8AzquNvgxFHw7dHwDMrBw4HfhLu926N53TvQHMrBhYCPwVwDnX7JzbQ4bfn8EQPuJaPTfDjXTObQX/AwyMSHN50s7MJgKHA6+g+9Mm2qywDNgBPOac0/3Z61rgq0Ck3T7dG88Bj5rZEjP7bHSf7o13EFAB3BBtsvuLmRWQ4fdnMISPuFbPFYkxs0LgbuDLzrnqdJenP3HOhZ1zc/ALQR5pZjPTXKR+wczOAHY455akuyz91DHOubn45u8vmNnCdBeoH8kC5gJ/dM4dDtSRYU0sHRkM4UOr53Zvu5mNBohud6S5PGljZiF88LjFOXdPdLfuz36i1cJP4/sP6f7AMcCZZrYO37R7gpndjO4NAM65LdHtDuBefHO47o23CdgUrUUEuAsfRjL6/gyG8PEqMNnMJplZNnAR8ECay9TfPAB8LPr8Y8D9aSxL2piZ4dtdVznnftnuLd0fwMyGm9mQ6PM84CRgNbo/OOe+4Zwrd85NxP8b86Rz7jJ0bzCzAjMrij0HTgHeRPcGAOfcNmCjmU2J7joRWEmG359BMcOpmZ2Gb4+NrZ77v+ktUfqY2W3A+/FLNm8HvgPcB9wJjAc2AOc75/bvlDromdmxwHPAG+xtt/8mvt+H7o/ZYfiOb0H8f5jc6Zz7vpkNRfenjZm9H7jKOXeG7g2Y2UH42g7wTQy3Ouf+V/dmLzObg++onA2sBT5B9P9jZOj9GRThQ0RERAaOwdDsIiIiIgOIwoeIiIiklMKHiIiIpJTCh4iIiKSUwoeIiIiklMKHiPQLZvb+2GqxIjK4KXyIiIhISil8iEiPmNllZrbIzJaZ2Z+ji9HVmtkvzGypmT1hZsOjx84xs5fN7HUzu9fMSqP7DzGzx81sefQzB0dPX2hmd5nZajO7JTorLWb2EzNbGT3Pz9P01UUkQRQ+RCRuZjYNuBC/kNgcIAxcChQAS6OLiz2Dn1kX4Cbga865w/Azy8b23wL83jk3Gzga2BrdfzjwZWA6fjXQY8ysDDgHmBE9zw+T+R1FJPkUPkSkJ04EjgBeNbNl0dcH4aervyN6zM3AsWZWAgxxzj0T3X8jsDC6DshY59y9AM65RudcffSYRc65Tc65CLAMmAhUA43AX8zsXCB2rIgMUAofItITBtzonJsTfUxxzn23g+O6WrfBunivqd3zMJDlnGvFr5J6N3A28HDPiiwi/Y3Ch4j0xBPAeWY2AsDMysxsAv7fkvOix1wCPO+cqwJ2m9lx0f2XA88456qBTWZ2dvQcOWaW39kFzawQKHHOPYRvkpmT8G8lIimVle4CiMjA4ZxbaWbfBh41swDQAnwBqANmmNkSoArfLwT8UuF/ioaL2Gqe4IPIn83s+9FznN/FZYuA+80sF19r8t8J/loikmJa1VZE+szMap1zhekuh4gMDGp2ERERkZRSzYeIiIiklGo+REREJKUUPkRERCSlFD5EREQkpRQ+REREJKUUPkRERCSl/n+zzsfsg7GCPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (9, 7))\n",
    "plt.plot(list(training_history_lr_scheduler.keys()), [training_history_lr_scheduler[k]['loss'] for k in training_history_lr_scheduler.keys()], label = 'training loss')\n",
    "plt.plot(list(training_history_lr_scheduler.keys()), [training_history_lr_scheduler[k]['val_loss'] for k in training_history_lr_scheduler.keys()], label = 'val loss')\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend(loc = 'best')\n",
    "plt.title(\"VGG-18: Training Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAG5CAYAAABGA9SHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzvElEQVR4nO3de5hcd33f8c93blpddrWSJeuyK0s2GBtZ0hpbMQaCk0CS2obG9EKANhCcp3FpTRt40lKHXnJ5Qpu2CQ2kFOOAAQcKIQRSP4mLaSDQkBSwbLyzkmWBELb3rO62Z3ZXt93Z+faPc0Yar3elXWlnz5zfvF/Po0c7Z87MfufY0n70+31/v2PuLgAAgKzIpV0AAADAfBBeAABAphBeAABAphBeAABAphBeAABAphBeAABAphBeAHQ0Mxs3s6vSrgPA3BFegDZmZg+b2W/NcPwOMztsZoXk8U4z+3Mze97MKmb2hJl9wMxWNb1mg5n9oZkdTH5gHzCzT5nZtef5/j9vZn9rZifN7BszPP86M3vMzEaT97trHp/tU2b223M9v1XcfYW7H2jFe5uZm9mJ5HqPmNkHzSw/x9f+hpl9phV1AVlHeAHa26ckvd3MbNrxt0v6rLvXzOzVkr4h6W8kXevuvZJulVSTNCBJZnaZpL+VtEzSayV1S7pB0jcl/cx5vv9zkn5f0u9Mf8LMipK+LOljklZKeoukD5rZwPw/Zms0wl3KBtx9haSfUHyNfinleoDMI7wA7e3PJK1WHDgkScloyhslPZAc+i+SPunu/8ndj0iSuz/j7r/u7t9IznmvpFFJb3f3H3qs4u6fdPc/mO2bu/tfuvsXJB2c4enVknok/VHyfo9I2itp6yV83sZnfKOZPZ6MIv2tme1oeu4eM/uhmY0lI0x/r+m5d5rZ35jZfzOz5yT9RjLC8xEz+4vkNd8xs5c0vcbN7KXJ1xc692fNbJ+ZVc3sf5jZN83sn8zlM7n7fsUB8/qm9/uQmQ0nI1ePmtlrk+O3Snq/pLckozaDyfGVZvYJMzuUjOT89lxHcoCQEF6ANubupyR9QdI7mg7/vKQn3X3QzJZLepWkP73AW/20pC+7e30Bazsi6XOS7jSzvJm9StJmSd+SJDP7cTOrzPd9zewGSfdL+qeSLlM8svOgmS1JTvmh4jC3UtJvSvqMmW1oeotXSjog6XJJH0iOvS05d5Wk/U3HZzLjuWa2RtIXJf1aUtc+Sa+ex+e6Nql7f9PhRxSHmdWS/qekPzGzLnf/iqT/KOmPk2mtxmjWpxWPqL1U0isk/aykOYUnICSEF6D9fVrSm81safL4HckxKf4Bm5N0uHGymf2XZMTihJn9u+Twmmnn/FxyzpiZffUSavucpP8g6Yykv5b0b919WJLc/VvJFNZ8/bKkj7n7d9x9yt0/nbz/zcn7/om7H3T3urv/saQfSLqp6fUH3f0P3L2WhD9J+pK7f9fda5I+q6bRjxnMdu7tkva4+5eS5z6spmt6Ho+Z2QnFo1LfkPQ/Gk+4+2fc/dmk1t+TtETSNTO9iZmtk3SbpPe4+wl3Pyrpv0l66xxqAIJCeAHanLt/S9IxSXckq2J+TPG/0iXpeUl1SRuazn9fEhq+LKnR8/HstHMeTM55r6SSJJnZvckUxbiZvf9CdSUjCX+sOEyVJF0n6X1m9oaL/7SS4tGbX03CVSUZvdkkaWPyfd/RNKVUkbRNcThrGJ7hPZtDxklJK87z/Wc7d2Pze3t8V9uo8djM9jRdv9c2vccNyXu8RfGo0PKm1/yqme1NpqEqikeTmj9Ls82SipIONX32jykeYQI6CuEFyIYHFIeEt0v6alNvywlJ35H09y/w+q9JepOZzfpn3t3flUxRrHD3/ziHmrZJ2ufuDyejIPsk/YXi0YFLMSzpA+7e2/Rrmbt/zsw2S/pDSe+WdFkSwHZLam5o9kv8/rM5JKm/8SBpoj772N2va7p+f938wqQn6AuS/p/ikSolAeffKJ4GXJV8lmrTZ5n+OYYVj0CtabouPe5+3UJ+SCALCC9ANjyguG/ll3VuyqjhfZJ+KWlkvVySzKxf0pVN53xQ8RTTH5nZSyzWrfNPnyjpZelSPIKTM7OuZJWRJH1P0tUWL5e2pLH1jZIG5/G58sl7Nn6VFIeTd5nZK5P3XW5mb0jqXa74h/qxpL47FYeoxfAXkrab2ZssXsV0t6T183yP35F0l5mtV7ziq6b4sxTM7D8oboBuOCJpSyNwuvshSV+V9Htm1mNmueS/5U9c2scCsofwAmSAuz+leKnzckkPTnvuW5JeJ+kWSd9PphO+ori/4g+Sc44r7hk5rbihdkzS44p/gP6z83zrt0s6JemjiptNTykOF3L3Hype9vthxSuZvqm4cfgTUjyyYGbjF/ho9yTv2fj1dXffpTik/XfF02L7Jb0z+Z5PSPo9xSMYRyRtV7yCp+WSa/hmxau7nlW8qmqX4tGQub7HkOLr9K8lPSzpf0v6vqSnFf+3aZ7y+pPk92fN7LHk68YU3ROKr80X1TQdCHQKi6dtAQDzkYyIRJL+sbv/Vdr1AJ2EkRcAmCMz+ztm1pss236/4v6Ub6dcFtBxCC8AMHevUrzPzHFJf1fSm5qWYwNYJEwbAQCATGHkBQAAZEo73LRswaxZs8a3bNmSdhkAAGABPProo8fdfe3040GFly1btmjXrl1plwEAABaAmT0903GmjQAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKYQXgAAQKa0NLyY2a1mts/M9pvZPTM8f62Z/T8zO2Nm/2o+rwUAAJ2pZeHFzPKSPiLpNklbJb3NzLZOO+05Sf9S0u9exGsBAEAHKrTwvW+StN/dD0iSmX1e0h2Snmic4O5HJR01szfM97VYPCfO1DR+pqapumuq7qp78+9pV4dmpUJOWy5bJjNLuxQAaJlWhpc+ScNNjyNJr1yE12IBHa6e1i3/9a80USOlZMUn3/lj+qlrL0+7DABomVaGl5n+6ecL/Vozu0vSXZJ0xRVXzPHtMVe7nn5OE7W63vvTL9O6niXK5Ux5M+VzdvZr/pHfHp49MaF//2e7dXz8TNqlAEBLtTK8RJI2NT3ul3RwoV/r7vdJuk+Sdu7cOddwhDkqR1WVCjn98596iYp5Fqe1syOjp/Xv/2y3Jqf4YwAgbK38afSIpKvN7EozK0l6q6QHF+G1WECDwxVt3dBDcMmAxn+jSRqRAASuZSMv7l4zs3dLelhSXtL97r7HzN6VPH+vma2XtEtSj6S6mb1H0lZ3H53pta2qFTOr1127R6r6Bzf2p10K5qCYj+fvCC8AQtfKaSO5+0OSHpp27N6mrw8rnhKa02uxuA4cH9eJiSnt6O9NuxTMQWPkZYLwAiBwzAVgVoPDVUnSQP/KlCvBXDTCS42eFwCBI7xgVuWoomWlvK5auyLtUjAH+Vy8CoxpIwChI7xgVuWRqrb1rVQ+x1rorCjkjGkjAMEjvGBGk1N1PXFwlCmjjCnlc5qsMW0EIGyEF8xo3+ExnanVadbNmGIhx7QRgOARXjCjchQ36+5g5CVTinl6XgCEj/CCGQ2NVNS7rKgrVi9LuxTMQzGfo+cFQPAIL5jR4HBV2/tWcnfijCnlcyyVBhA8wgte5PTklPYdGdMA/S6ZU8zT8wIgfIQXvMieg6Oaqru20++SOQV6XgB0AMILXmQoqkgSIy8ZFPe8MG0EIGyEF7xIOarq8u4lWr+yK+1SME/xPi+MvAAIG+EFLzIYVdjfJaOKBaaNAISP8IIXGDs9qQPHT7C/S0bRsAugExBe8AK7R0blzuZ0WRWHF3peAISN8IIXKCfNukwbZVOJkRcAHYDwghcoR1X1r1qq1ctLaZeCi8DtAQB0AsILXmAwqrBEOsMKTBsB6ACEF5z13IkJRc+fot8lw7i3EYBOQHjBWfS7ZF+JaSMAHYDwgrPKUVVm0ra+nrRLwUUqskkdgA5AeMFZ5aiiq9YsV3dXMe1ScJGKhZwm6/S8AAgb4QWSJHfXYFSlWTfjGpvUuRNgAISL8AJJ0pHRMzo2doZm3Ywr5U3u0hSjLwACRniBpHiJtCRtZ+Ql0wr5+I80y6UBhIzwAklxv0shZ7puI826WVZMwgvLpQGEjPACSfFKo5et61ZXMZ92KbgEpbxJEsulAQSN8AK5u8pRVQOb6HfJuuLZaSPCC4BwEV6gZ547qeqpSW3v6027FFyiRnip0fMCIGCEF2gwqkoSK40CUCzQ8wIgfIQXqDxcUamQ0zXru9MuBZeInhcAnYDwApVHqtq6oefslAOyq5BLel5qTBsBCBc/rTrcVN21e6SqAaaMgsC0EYBOQHjpcD88Nq6TE1PcSToQRaaNAHQAwkuHKyfNuiyTDkOJpdIAOgDhpcOVo4qWl/K6cs2KtEvBAmCpNIBOQHjpcINRVdv6Viqfs7RLwQLg9gAAOgHhpYNN1Orae3BUA5t60y4FC6RUoOcFQPgILx3s+0fGNDFVZ3O6gJxdKk14ARAwwksHG4wqkqQd3BYgGI2l0uzzAiBkhJcOVh6uatWyojatXpp2KVggjaXS9LwACBnhpYMNRhVt7++VGc26oWCpNIBOQHjpUKcmpvSDo+PsrBuYIuEFQAcgvHSoJw5VNVV3be8jvITkXHih5wVAuAgvHWpwuLGzbm+6hWBBcXsAAJ2A8NKhylFF63qWaF1PV9qlYAGZmQo5I7wACBrhpUOVR6rcjDFQxXyOaSMAQSO8dKDR05M6cOyEdtDvEqRi3jRRY+QFQLgILx1od3In6R30uwSpVMgxbQQgaISXDlQeScILIy9BiqeNCC8AwkV46UDlqKJNq5dq1fJS2qWgBYr5nGr0vAAIGOGlAw0O06wbsmLeuD0AgKARXjrMs+NnNFI5xc66AWPaCEDoCC8d5my/CyMvwWKpNIDQEV46THm4KjNpG826wSrm2aQOQNgILx2mHFX0krUrtGJJIe1S0CLFfI59XgAEjfDSQdxdg1FVO+h3CRr7vAAIHeGlgxwePa3j42c0QL9L0Ir5nGp1el4AhIvw0kEad5LezshL0Lg9AIDQEV46SDmqqJAzbd3Qk3YpaKECS6UBBI7w0kGGRqq6Zn23uor5tEtBC5VYKg0gcC0NL2Z2q5ntM7P9ZnbPDM+bmX04eb5sZjc0PfdeM9tjZrvN7HNm1tXKWkPn7ipH7KzbCVgqDSB0LQsvZpaX9BFJt0naKultZrZ12mm3Sbo6+XWXpI8mr+2T9C8l7XT3bZLykt7aqlo7wdPPnlT11CQrjToAO+wCCF0rR15ukrTf3Q+4+4Skz0u6Y9o5d0h6wGPfltRrZhuS5wqSlppZQdIySQdbWGvwBqOKJBFeOgD7vAAIXSvDS5+k4abHUXLsgue4+4ik35X0jKRDkqru/tWZvomZ3WVmu8xs17Fjxxas+NAMRVUtKeT0snXdaZeCFisVWCoNIGytDC82w7Hpf6POeI6ZrVI8KnOlpI2SlpvZL8z0Tdz9Pnff6e47165de0kFh6wcVXXdxh4V8/Roh46eFwCha+VPskjSpqbH/Xrx1M9s5/y0pB+5+zF3n5T0JUmvbmGtQZuqu3YfpFm3UzRuzOjO6AuAMLUyvDwi6Wozu9LMSoobbh+cds6Dkt6RrDq6WfH00CHF00U3m9kyMzNJr5e0t4W1Bm3/0XGdnJii36VDNEbXWC4NIFQtuzufu9fM7N2SHla8Wuh+d99jZu9Knr9X0kOSbpe0X9JJSXcmz33HzL4o6TFJNUnfk3Rfq2oNXflss25vqnVgcRTz8Wzs5FRdpQLThADC09JbC7v7Q4oDSvOxe5u+dkl3z/LaX5f0662sr1OUo6pWLCnoqjXL0y4Fi+DcyAt9LwDCxD/LOkA5qmhbX49yuZn6oxGaRniZILwACBThJXATtbr2HhrjTtIdpJSElxo9LwACRXgJ3L7DY5qYqtPv0kGKhXM9LwAQIsJL4NhZt/PQ8wIgdISXwJWjilYtK6p/1dK0S8EiKeSSnpca00YAwkR4CVzjTtLxdjnoBCWmjQAEjvASsFMTU/rB0XENMGXUUZg2AhA6wkvA9hysaqru2k6zbkdhqTSA0BFeAjYYVSWJkZcOU2SpNIDAEV4CNhRVtL6nS5f3dKVdChZRiWkjAIEjvASsHFW1nVGXjsM+LwBCR3gJVPXUpA4cP8GUUQc6u1SaaSMAgSK8BGr3SNzvws66nefstFGNkRcAYSK8BKocNcILIy+dhmkjAKEjvASqHFV0xepl6l1WSrsULDL2eQEQOsJLoOKddRl16UTnwgs9LwDCRHgJ0PHxMxqpnNIA/S4diaXSAEJHeAnQEP0uHa2Yp+cFQNgILwEajCoyk67rI7x0onwuDi8slQYQKsJLgMpRVS9du0IrlhTSLgUpMDOV8jlGXgAEi/ASGHdPmnV70y4FKSrmjX1eAASL8BKYQ9XTOj5+RgObmDLqZMUCIy8AwkV4CUw5qkiSttPv0tGK+Rw9LwCCRXgJzGBUVSFnevmGnrRLQYpK+ZxqjLwACBThJTBDUVXXbuhWVzGfdilIUTFvTBsBCBbhJSBxs25F2/t60y4FKSvkc+ywCyBYhJeAPPXsSY2ermmAzek6XtzzwsgLgDARXgLSaNZlmTRKTBsBCBjhJSDlqKolhZxetm5F2qUgZUU2qQMQMMJLQMpRRddt7FEhz3/WTlfM5zRZo+cFQJj4KReI2lRdu0dGmTKCpGSTujojLwDCRHgJxP5j4zo1OcXOupBEzwuAsBFeAlGOqpJo1kWskGPaCEC4CC+BKEcVdS8p6MrLlqddCtoA9zYCEDLCSyDKUVXb+lYql7O0S0EbKOaNfV4ABIvwEoAztSntPTSqHfS7IFFiqTSAgBFeArDv8Jgmp1wD9LsgUeT2AAACRngJwGDSrLu9j5EXxNikDkDICC8BKA9XtHp5Sf2rlqZdCtpEscBSaQDhIrwEoBxVtaN/pcxo1kWsxLQRgIARXjLu5ERNPzg6xv4ueIFCLqepumuqToABEB7CS8btOTiquks76HdBk2IhHoVj6ghAiAgvGTc4XJEklknjBUrJzTkJLwBCRHjJuHJU1YaVXbq8uyvtUtBGimfDC9NGAMJDeMm4oZG4WRdo1ggvNUZeAASI8JJh1VOT+tHxEzTr4kWK+bjnhVsEAAgR4SXDhs7eSZqRF7xQqcC0EYBwEV4ybDCqSJJ29PWmWgfaTyFHwy6AcBFeMmwoqmrzZcu0clkx7VLQZs5OG9UILwDCQ3jJsHJUod8FMyoWGHkBEC7CS0YdGzujg9XTGqDfBTMosVQaQMAILxlVbvS7MPKCGbBUGkDICC8ZVY6qypl03caetEtBG2KpNICQEV4yqhxV9NLLV2j5kkLapaANscMugJARXjLI3VWOqkwZYVZF7m0EIGCElwwaqZzSsycmaNbFrBrTRoQXACEivGRQY2fd7Yy8YBaNkRf2eQEQIsJLBg1GVRXzppdv6E67FLQpbg8AIGSElwwqRxVdu75HSwr5tEtBmzq7VLrOyAuA8BBeMqZedw1FVW7GiPPi9gAAQtbS8GJmt5rZPjPbb2b3zPC8mdmHk+fLZnZD03O9ZvZFM3vSzPaa2ataWWtWPPXsCY2dqRFecF4slQYQspaFFzPLS/qIpNskbZX0NjPbOu202yRdnfy6S9JHm577kKSvuPu1kgYk7W1VrVlSTpp1WSaN82GpNICQtXLk5SZJ+939gLtPSPq8pDumnXOHpAc89m1JvWa2wcx6JN0i6ROS5O4T7l5pYa2ZMRhV1FXM6erLV6RdCtpYPmfKGeEFQJhaGV76JA03PY6SY3M55ypJxyR90sy+Z2YfN7PlM30TM7vLzHaZ2a5jx44tXPVtaiiq6rqNK1XI066E8yvmc9weAECQWvkT0GY4Nn0CfrZzCpJukPRRd3+FpBOSXtQzI0nufp+773T3nWvXrr2Uettebaqu3Qdp1sXclPI5TdboeQEQnlaGl0jSpqbH/ZIOzvGcSFLk7t9Jjn9RcZjpaD84Oq7Tk3UN0O+COSgWckwbAQhSK8PLI5KuNrMrzawk6a2SHpx2zoOS3pGsOrpZUtXdD7n7YUnDZnZNct7rJT3RwlozoRxVJImRF8xJMW/s8wIgSC27JbG718zs3ZIelpSXdL+77zGzdyXP3yvpIUm3S9ov6aSkO5ve4l9I+mwSfA5Me64jlaOqursK2nLZjO0/wAsU8zlNMG0EIEAtCy+S5O4PKQ4ozcfubfraJd09y2sfl7SzlfVlTTmqanvfSuVyM7UKAS9UzDNtBCBMLFnJiDO1KT15eJT9XTBnxbwRXgAEifCSEXsPjWlyyjVAvwvmiJEXAKEivGTEUNKsu53wgjmK93mh5wVAeAgvGTEYVXXZ8pL6epemXQoyIt7nhZEXAOEhvGREOapoR/9KmdGsi7kpFlgqDSBMhJcMOHGmpv1Hx2nWxbwwbQQgVISXDNhzcFR1Z3M6zE8hx7QRgDARXjLg3M66vanWgWwpFVgqDSBMhJcMGIyq2riyS2u7l6RdCjKEpdIAQkV4yYChqMKoC+YtDi/0vAAID+GlzVVPTuqpZ0+yvwvmLW7YZeQFQHgIL22uPFKRJA0w8oJ5KuVNNcILgAARXtpcOapKkrb3MfKC+WHaCECoCC9trhxVtOWyZVq5rJh2KciYAtNGAAJFeGlz5ahKsy4uSim5q7Q7oy8AwkJ4aWNHx07rUPU0m9PhohTzOblLU3XCC4CwEF7a2FDS7zKwqTfdQpBJxUL8x5u+FwChIby0scGoqpxJ123sSbsUZFAxH//xpu8FQGgIL22sHFV09eXdWlYqpF0KMqiUj+9AznJpAKEhvLQpd0+adel3wcVpjLwwbQQgNISXNjVSOaXnTkwQXnDRzoUXRl4AhIXw0qYam9OxTBoXq5BMG9HzAiA0hJc2NRhVVMybrt3QnXYpyKgSIy8AAkV4aVNDUVUv39CjJYV82qUgo85OG9XoeQEQFsJLG6rXXUNRlfsZ4ZI09nlh2ghAaAgvbehHz57Q2Jkad5LGJSmyVBpAoAgvbagcVSRJOzYx8oKLV2KpNIBAEV7aUDmqamkxr5euXZF2KcgwlkoDCNUFw4uZ5cxs92IUg1g5quq6jT0q5MmWuHgslQYQqgv+dHT3uqRBM7tiEerpeLWpuvYcrLK/Cy4ZS6UBhGquN83ZIGmPmX1X0onGQXf/uZZU1cG+f2RcpyfrGqDfBZeIaSMAoZprePnNllaBs4ZGKpLEMmlcssZSafZ5ARCaOYUXd/9mqwtBbDCqqruroC2XLU+7FGRcY6n0ZJ2RFwBhOW94MbMxSTP9s80kubv3tKSqDlaOKtrRv1K5nKVdCjLubM9LjfACICznDS/uzo11FtHpySk9eWhMv3zLVWmXggAU2ecFQKBYi9tGnjw8plrdtYN+FywAlkoDCBXhpY2c21m3N9U6EIZijtVGAMJEeGkjg8NVrVlR0saVXWmXggDkcqZCzggvAIJDeGkjcbNur8xo1sXCKOZz9LwACA7hpU2cOFPT/mPj7O+CBVXMM/ICIDyElzaxe6Qqd7GzLhZUqZAjvAAIDuGlTZSjqiRpe19vuoUgKMV8jh12AQSH8NImBqOKNq7s0truJWmXgoAUmDYCECDCS5sYGuFO0lh4xXyOfV4ABIfw0gYqJyf09LMntYN+FyywUp6eFwDhIby0gUa/ywAjL1hgLJUGECLCSxto7Ky7jWXSWGAslQYQIsJLGyhHVV25ZrlWLi2mXQoCU2TaCECACC9toBxVtaOfURcsvHifF6aNAISF8JKyo6OndXj0NCuN0BLc2whAiAgvKRtMmnUZeUErFPM5TdQILwDCQnhJWTmqKGfSdRt70i4FASpyewAAASK8pKwcVfWydd1aViqkXQoCVGKpNIAAEV5S5O4qRxWmjNAyLJUGECLCS4qi50/p+ZOT2k6zLlqETeoAhIjwkqLBZHO6AUZe0CLs8wIgRISXFA1FVZXyOV2zvjvtUhAopo0AhIjwkqLBqKJrN3RrSSGfdikIFCMvAEJEeElJve7aPTJKsy5aqtHz4k7fC4BwEF5ScuD4uMbP1NhZFy1VKsR/xGnaBRCSloYXM7vVzPaZ2X4zu2eG583MPpw8XzazG6Y9nzez75nZn7eyzjSUk511BwgvaKFi3iSJqSMAQWlZeDGzvKSPSLpN0lZJbzOzrdNOu03S1cmvuyR9dNrzvyJpb6tqTFM5qmppMa+XrF2edikIWDEf/xGvMfICICCtHHm5SdJ+dz/g7hOSPi/pjmnn3CHpAY99W1KvmW2QJDPrl/QGSR9vYY2pGYwq2tbXo0KemTu0TiO8TDDyAiAgrfzJ2SdpuOlxlByb6zm/L+l9ks77t66Z3WVmu8xs17Fjxy6p4MUyOVXXEwdH6XdBy5XyjZ4XwguAcLQyvNgMx6aPXc94jpm9UdJRd3/0Qt/E3e9z953uvnPt2rUXU+ei+/6RMZ2p1VlphJYr0PMCIECtDC+RpE1Nj/slHZzjOa+R9HNm9pTi6abXmdlnWlfq4mo06zLyglYrMvICIECtDC+PSLrazK40s5Kkt0p6cNo5D0p6R7Lq6GZJVXc/5O6/5u797r4led3X3f0XWljroipHFfV0FbTlsmVpl4LAne15qdGwCyAchVa9sbvXzOzdkh6WlJd0v7vvMbN3Jc/fK+khSbdL2i/ppKQ7W1VPOylHVe3o75XZTLNmwMIpFZg2AhCeloUXSXL3hxQHlOZj9zZ97ZLuvsB7fEPSN1pQXipOT05p3+Ex3XXLVWmXgg5wdql0nfACIBys011kTxwaVa3uNOtiUTBtBCBEhJdFVh6uSKJZF4uDhl0AISK8LLLySFVrVizRhpVdaZeCDsDtAQCEiPCyyMpRVQP9K2nWxaJg5AVAiAgvi2j8TE0/PDau7fS7YJGcuz0APS8AwkF4WUS7R6py507SWDxnbw9QY+QFQDgIL4uoHFUkiZVGWDTFZJ8XlkoDCAnhZRENRlX19S7VZSuWpF0KOgTTRgBCRHhZROWowqgLFlWRaSMAASK8LJLnT0xo+LlT7O+CRcVSaQAhIrwskvJIfCfpAUZesIhYKg0gRISXRdLYWXcb4QWLqJCLR17oeQEQEsLLIimPVHXVmuXq6SqmXQo6iJmplM8x8gIgKISXRUKzLtJSzJtqhBcAASG8LIIjo6d1ZPQMzbpIRbGQ0yTTRgACQnhZBINJv8vAJkZesPiK+ZwmGHkBEBDCyyIYGqkqnzNt3UB4weIr5ox9XgAEhfCyCAajqq6+fIWWlvJpl4IOFE8bEV4AhIPw0mLurnJU4WaMSE0xT88LgLAQXlosev6UKicntYN+F6SEnhcAoSG8tNhg407Sfb2p1oHOVWKpNIDAEF5arBxVVcrndM367rRLQYdi2ghAaAgvLTY4XNHLN/aoVOBSIx1MGwEIDT9RW6hed+0eqXIzRqSqkDdWGwEICuGlhQ4cH9eJiSlt7yO8ID3c2whAaAgvLTQ4XJUkDWzqTbcQdLRiPqfJGj0vAMJBeGmhoZGqlpXyesnaFWmXgg7GJnUAQkN4aaHBqKJtfSuVz1napaCDFfNGwy6AoBBeWmRyqq4nDo5qB/0uSFkpn1ONpdIAAkJ4aZF9h8d0plbXDvpdkLIiDbsAAkN4aZGhkaRZl2XSSFmBaSMAgSG8tEg5qmjl0qKuWL0s7VLQ4VgqDSA0hJcWGRyuakf/SpnRrIt0cXsAAKEhvLTA6ckp7Tsyph1MGaENFPM5TdVdU3UCDIAwEF5a4IlDo5qqu3b096ZdCqBiIR79Y+oIQCgILy1QHq5IkgYIL2gDpXz8x7zGyAuAQBBeWqAcVbW2e4nW9SxJuxRAxSS8TNYYeQEQBsJLCwxGFQ3QrIs2UcgzbQQgLISXBTZ2elIHjp+g3wVtozHywl4vAEJBeFlgu0dG5S5WGqFtNHpeWC4NIBSElwVWjiqSxMgL2sbZnhdGXgAEgvCywMpRVf2rlmr18lLapQCS4rtKS9IEDbsAAkF4WWDlkQpLpNFWigWWSgMIC+FlAT13YkLDz52i3wVtpcS0EYDAEF4WUKPfZTvhBW2EfV4AhIbwsoDKUVVm0vY+wgvaR2OfF5ZKAwgF4WUBlaOqrlqzXN1dxbRLAc5iqTSA0BBeFlA5qrBEGm2HpdIAQkN4WSCHq6d1dOwMzbpoO0VuDwAgMISXBTLI5nRoU0WmjQAEhvCyQIaiqvI503Ube9IuBXiBUoFpIwBhIbwskMGoopet61ZXMZ92KcAL0PMCIDSElwXg7hoaqWqAfhe0oQK3BwAQGMLLAnjmuZOqnJyk3wVtiaXSAEJDeFkA5agqSaw0Qlti2ghAaAgvC6AcVVQq5HTN+u60SwFeJJ8z5YzwAiAchJcFMBhVtXVDz9l/4QLtppjPMW0EIBj8tL1EU3XXbpp10eZK+RwjLwCCQXi5RAeOjevkxBTNumhrxQLhBUA4CC+XaJBmXWRAIWeEFwDBILxconJU0fJSXletXZF2KcCsivmcJmr0vAAIQ0vDi5ndamb7zGy/md0zw/NmZh9Oni+b2Q3J8U1m9ldmttfM9pjZr7SyzksxGFW1rW+l8jlLuxRgViWmjQAEpGXhxczykj4i6TZJWyW9zcy2TjvtNklXJ7/ukvTR5HhN0q+6+8sl3Szp7hlem7qJWl17D40yZYS2V8wzbQQgHK0ceblJ0n53P+DuE5I+L+mOaefcIekBj31bUq+ZbXD3Q+7+mCS5+5ikvZL6WljrRfn+kTFN1Oo066LtsVQaQEhaGV76JA03PY704gBywXPMbIukV0j6zkzfxMzuMrNdZrbr2LFjl1rzvAxGFUnSAOEFba7IUmkAAWlleJmpCWT6P/3Oe46ZrZD0p5Le4+6jM30Td7/P3Xe6+861a9dedLEXozxc1aplRW1avXRRvy8wX+zzAiAkrQwvkaRNTY/7JR2c6zlmVlQcXD7r7l9qYZ0XrTxS1fb+XpnRrIv2VqDnBUBAWhleHpF0tZldaWYlSW+V9OC0cx6U9I5k1dHNkqrufsjiNPAJSXvd/YMtrPGinZqY0vePjGlHH826aH/FfE4T9LwACEShVW/s7jUze7ekhyXlJd3v7nvM7F3J8/dKekjS7ZL2Szop6c7k5a+R9HZJQ2b2eHLs/e7+UKvqna8nDlU1VXdWGiETivmcJmuMvAAIQ8vCiyQlYeOhacfubfraJd09w+u+pZn7YdrG4HC8s+7Apt50CwHmoFRg2ghAONhh9yINjVS1rmeJ1vV0pV0KcEGsNgIQEsLLRRqMKtre15t2GcCcsM8LgJAQXi7C6OlJHTh2QgP0uyAjGHkBEBLCy0XY3biTNP0uyAhuDwAgJISXi1AeScILy6SREUwbAQgJ4eUilKOKNq1eqlXLS2mXAsxJvM8LIy8AwkB4uQiDw1VuxohMKSXTRvHuBACQbYSXeXp2/IxGKqdo1kWmFPM5uUtTdcILgOwjvMzT2X4XRl6QIcVC/Ee9RngBEADCyzyVh6syk7bRrIsMKebjP+r0vQAIAeFlnspRRS9Zu0IrlrT0zgrAgirm47ttcH8jACEgvMyDu2swqnIzRmROY+SF5dIAQkB4mYfDo6d1fPyMBuh3QcacCy+MvADIPsLLPDTuJL2dkRdkTGPaiJ4XACEgvMxDOaqokDNt3dCTdinAvJQYeQEQEMLLPAyNVHXN+m51FfNplwLMS2PaqEbPC4AAEF7myN1VjthZF9nU2OeFaSMAISC8zNHTz55U9dQkK42QSSyVBhASwsscDUYVSSK8IJNYKg0gJISXORqKqlpSyOll67rTLgWYN5ZKAwgJ4WWOylFVWzf2nP0hAGQJS6UBhISfxHMwVXftPlhlczpkFkulAYSE8DIH+4+O6+TEFP0uyCyWSgMICeFlDspnm3V7U60DuFgslQYQEsLLHJSjqlYsKeiqNcvTLgW4KGeXShNeAASA8DIH5aiibX09yuUs7VKAi1LMJT0v7PMCIACElwuYqNW199AYzbrItMa0Efu8AAhBIe0C2t2+w2OamKrT74JMa0wb/eXeI3ru5ETK1UCKm6jvfPUWrVpeSrsUIHMILxfw5OFRSeysi2wr5XO6dn23vvdMRd97ppJ2OVDcPN3TVdA/ee1VaZcCZA7h5QLevHOTfuJla7W2e0napQAXzcz0lffcknYZaPLj//nreuyZ59MuA8gkel7m4PKeLpnRrAtg4ezcvEq7nnpe7vQhAfNFeAGAFNy4eZWOjp1R9PyptEsBMofwAgApuHHzaknSo08zdQTMF+EFAFJwzfpurVhS0K6nn0u7FCBzCC8AkIJ8zvSKK3r16NOVtEsBMofwAgApuXHzKu07PKqx05NplwJkCuEFAFJy4+ZVqrvYeweYJ8ILAKTk+k29yhlNu8B8EV4AICXdXUVds76H8ALME+EFAFK0c/Mqfe+Z5zVVZ7M6YK4ILwCQop1bVunExNTZ+6gBuDDCCwCk6IYrVkmi7wWYD8ILAKSof9VSretZQngB5oHwAgApMjPdmNykEcDcEF4AIGU3bl6tkcopHa6eTrsUIBMILwCQsp2b6XsB5oPwAgAp27qxR13FHDdpBOaI8AIAKSvmcxro79VjjLwAc0J4AYA2cOPmVdpzcFSnJqbSLgVoe4QXAGgDO7esUq3uGowqaZcCtD3CCwC0ATarA+aO8AIAbaB3WUkvvXyFdj1F0y5wIYQXAGgTOzev0mPPVFTnJo3AeRFeAKBN3LB5laqnJvXDY+NplwK0NcILALQJNqsD5obwAgBt4so1y7V6eUm7CC/AeRFeAKBNmJluuGIVIy/ABRBeAKCN7NyySj86fkLPjp9JuxSgbRXSLgAAcM6NSd/LfX99QNes6065mvRcs75b121cmXYZaFOEFwBoI9v7Vqqnq6CPffNA2qWk7qYrV+uXXnOlfmbrOuVzlnY5aCPm3rr9BMzsVkkfkpSX9HF3/51pz1vy/O2STkp6p7s/NpfXzmTnzp2+a9euhf0QALDIqqcmVTk5kXYZqZmqu77+5FF98m+e0kjllPpXLdU7X71FP/9jm9TTVUy7PCwiM3vU3Xe+6HirwouZ5SV9X9LPSIokPSLpbe7+RNM5t0v6F4rDyyslfcjdXzmX186E8AIA4ahN1fWXe4/o/m89pe8+9ZyWl/J6885NjMS0md5lRV27vqcl7z1beGnltNFNkva7+4GkgM9LukNScwC5Q9IDHieob5tZr5ltkLRlDq8FAASskM/p1m0bdOu2DRqKqvrk3/xIn/3O0/rU3z6Vdmlo8vprL9cn3vlji/o9Wxle+iQNNz2OFI+uXOicvjm+VpJkZndJukuSrrjiikurGADQlrb3r9QH33K9fu32l+sHR8fSLgdNVi0rLfr3bGV4mWlMb/oc1WznzOW18UH3+yTdJ8XTRvMpEACQLWu7l2ht95K0y0DKWhleIkmbmh73Szo4x3NKc3gtAADoQK3cpO4RSVeb2ZVmVpL0VkkPTjvnQUnvsNjNkqrufmiOrwUAAB2oZSMv7l4zs3dLeljxcuf73X2Pmb0ref5eSQ8pXmm0X/FS6TvP99pW1QoAALKjpfu8LDaWSgMAEI7ZlkpzbyMAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJApQd3byMyOSXr6Il++RtLxBSwnNFyf2XFtZse1OT+uz+y4NrPrpGuz2d3XTj8YVHi5FGa2a6abPyHG9Zkd12Z2XJvz4/rMjmszO64N00YAACBjCC8AACBTCC/n3Jd2AW2O6zM7rs3suDbnx/WZHddmdh1/beh5AQAAmcLICwAAyBTCCwAAyBTCiyQzu9XM9pnZfjO7J+160mRm95vZUTPb3XRstZn9HzP7QfL7qjRrTIuZbTKzvzKzvWa2x8x+JTnO9ZFkZl1m9l0zG0yuz28mx7k+CTPLm9n3zOzPk8dcG0lm9pSZDZnZ42a2KznGtUmYWa+ZfdHMnkz+/nlVp1+fjg8vZpaX9BFJt0naKultZrY13apS9SlJt047do+kr7n71ZK+ljzuRDVJv+ruL5d0s6S7k/9XuD6xM5Je5+4Dkq6XdKuZ3SyuT7NfkbS36THX5pyfcvfrm/Yv4dqc8yFJX3H3ayUNKP5/qKOvT8eHF0k3Sdrv7gfcfULS5yXdkXJNqXH3/yvpuWmH75D06eTrT0t602LW1C7c/ZC7P5Z8Pab4L5A+cX0kSR4bTx4Wk18uro8kycz6Jb1B0sebDnNtZse1kWRmPZJukfQJSXL3CXevqMOvD+El/uEz3PQ4So7hnHXufkiKf4BLujzlelJnZlskvULSd8T1OSuZFnlc0lFJ/8fduT7n/L6k90mqNx3j2sRc0lfN7FEzuys5xrWJXSXpmKRPJlOOHzez5erw60N4kWyGY6wfx6zMbIWkP5X0HncfTbueduLuU+5+vaR+STeZ2baUS2oLZvZGSUfd/dG0a2lTr3H3GxRP399tZrekXVAbKUi6QdJH3f0Vkk6ow6aIZkJ4iUdaNjU97pd0MKVa2tURM9sgScnvR1OuJzVmVlQcXD7r7l9KDnN9pkmGtb+huH+K6yO9RtLPmdlTiqemX2dmnxHXRpLk7geT349K+rLi6XyuTSySFCWjmJL0RcVhpqOvD+FFekTS1WZ2pZmVJL1V0oMp19RuHpT0i8nXvyjpf6VYS2rMzBTPO+919w82PcX1kWRma82sN/l6qaSflvSkuD5y919z935336L475ivu/sviGsjM1tuZt2NryX9rKTd4tpIktz9sKRhM7smOfR6SU+ow68PO+xKMrPbFc9H5yXd7+4fSLei9JjZ5yT9pOJbrh+R9OuS/kzSFyRdIekZSW929+lNvcEzsx+X9NeShnSub+H9ivteuD5mOxQ3DuYV/8PoC+7+W2Z2mbg+Z5nZT0r6V+7+Rq6NZGZXKR5tkeIpkv/p7h/g2pxjZtcrbvQuSTog6U4lf8bUodeH8AIAADKFaSMAAJAphBcAAJAphBcAAJAphBcAAJAphBcAAJAphBcAQTCzn2zcrRlA2AgvAAAgUwgvABaVmf2CmX3XzB43s48lN3McN7PfM7PHzOxrZrY2Ofd6M/u2mZXN7Mtmtio5/lIz+0szG0xe85Lk7VeY2RfN7Ekz+2yyK7LM7HfM7InkfX43pY8OYIEQXgAsGjN7uaS3KL4R3/WSpiT9Y0nLJT2W3Jzvm4p3dpakByT9G3ffoXhn48bxz0r6iLsPSHq1pEPJ8VdIeo+krYrvxvsaM1st6e9Jui55n99u5WcE0HqEFwCL6fWSbpT0iJk9njy+SvHtFv44Oeczkn7czFZK6nX3bybHPy3pluQ+OH3u/mVJcvfT7n4yOee77h65e13S45K2SBqVdFrSx83s70tqnAsgowgvABaTSfq0u1+f/LrG3X9jhvPOd98SO89zZ5q+npJUcPea4rsU/6mkN0n6yvxKBtBuCC8AFtPXJP1DM7tcksxstZltVvx30T9MzvlHkr7l7lVJz5vZa5Pjb5f0TXcflRSZ2ZuS91hiZstm+4ZmtkLSSnd/SPGU0vUL/qkALKpC2gUA6Bzu/oSZ/TtJXzWznKRJSXdLOiHpOjN7VFJVcV+MJP2ipHuTcNK4m64UB5mPmdlvJe/x5vN8225J/8vMuhSP2rx3gT8WgEXGXaUBpM7Mxt19Rdp1AMgGpo0AAECmMPICAAAyhZEXAACQKYQXAACQKYQXAACQKYQXAACQKYQXAACQKf8fGI6EXQoIYXgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (9, 7))\n",
    "plt.plot(list(training_history_lr_scheduler.keys()), [training_history_lr_scheduler[k]['lr'] for k in training_history_lr_scheduler.keys()])\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"lr\")\n",
    "plt.title(\"VGG-18: Learning-Rate\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "VGG18_PyTorch.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}