03_advanced_neural_networks/02_representation_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ca0442b2-193e-4cf0-a5c8-1e192b04f130",
   "metadata": {},
   "source": [
    "# Foundation Models\n",
    "\n",
    "# This notebook is not officially part of the course.\n",
    "\n",
    "But you are welcome to look through it anyways, you can send questions on slack, and we are happy to talk about it.  \n",
    "\n",
    "Author: Corey Adams\n",
    "\n",
    "The previous notebook trained a classifier network which did ok.  But what if we didn't have a lot of data?  In this notebook, we'll apply that model in a new way with _representation learning_."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ba69d87a-de6c-4fba-a6e2-6de36b165b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy, random\n",
    "import torch\n",
    "import torchvision\n",
    "from torchvision.transforms import v2\n",
    "\n",
    "\n",
    "import torch.distributed as dist\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.multiprocessing as mp\n",
    "\n",
    "from torch.nn.parallel import DistributedDataParallel as DDP\n",
    "\n",
    "batch_size = 128\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a551b0f7-675c-4452-97f8-36f3a269c6de",
   "metadata": {},
   "source": [
    "Here's the Convolutional Neural Network Again:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f8068c4-c85b-46d0-af2c-c665dbb6a82c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "\n",
    "class Downsampler(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride=2):\n",
    "        super(Downsampler, self).__init__()\n",
    "\n",
    "        self.norm = nn.InstanceNorm2d(in_channels)\n",
    "\n",
    "        self.downsample = nn.Conv2d(\n",
    "            in_channels=in_channels, \n",
    "            out_channels=out_channels,\n",
    "            kernel_size = stride,\n",
    "            stride = stride,\n",
    "        )\n",
    "    \n",
    "    def forward(self, inputs):\n",
    "\n",
    "        return self.downsample(self.norm(inputs))\n",
    "        \n",
    "        \n",
    "\n",
    "class ConvNextBlock(nn.Module):\n",
    "    \"\"\"This block of operations is loosely based on this paper:\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    def __init__(self, in_channels):\n",
    "        super(ConvNextBlock, self).__init__()\n",
    "\n",
    "        # Depthwise, seperable convolution with a large number of output filters:\n",
    "        self.conv1 = nn.Conv2d(in_channels=in_channels, \n",
    "                                     out_channels=in_channels, \n",
    "                                     groups=in_channels,\n",
    "                                     kernel_size=[7,7],\n",
    "                                     padding='same' )\n",
    "\n",
    "        self.norm = nn.InstanceNorm2d(in_channels)\n",
    "\n",
    "        # Two more convolutions:\n",
    "        self.conv2 = nn.Conv2d(in_channels=in_channels, \n",
    "                                     out_channels=4*in_channels,\n",
    "                                     kernel_size=1)\n",
    "\n",
    "        self.conv3 = nn.Conv2d(in_channels=4*in_channels, \n",
    "                                     out_channels=in_channels,\n",
    "                                     kernel_size=1\n",
    "                                     )\n",
    "\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        x = self.conv1(inputs)\n",
    "\n",
    "        # The normalization layer:\n",
    "        x = self.norm(x)\n",
    "\n",
    "        x = self.conv2(x)\n",
    "\n",
    "        # The non-linear activation layer:\n",
    "        x = torch.nn.functional.gelu(x)\n",
    "\n",
    "        x = self.conv3(x)\n",
    "\n",
    "        # This makes it a residual network:\n",
    "        return x + inputs\n",
    "    \n",
    "\n",
    "class Classifier(nn.Module):\n",
    "\n",
    "\n",
    "    def __init__(self, n_initial_filters, n_stages, blocks_per_stage, n_outputs):\n",
    "        super(Classifier, self).__init__()\n",
    "\n",
    "        # This is a downsampling convolution that will produce patches of output.\n",
    "\n",
    "        # This is similar to what vision transformers do to tokenize the images.\n",
    "        self.stem = nn.Conv2d(in_channels=3,\n",
    "                                    out_channels=n_initial_filters,\n",
    "                                    kernel_size=1,\n",
    "                                    stride=1)\n",
    "        \n",
    "        self.norm1 = nn.InstanceNorm2d(n_initial_filters)\n",
    "\n",
    "        current_n_filters = n_initial_filters\n",
    "        \n",
    "        self.layers = nn.Sequential()\n",
    "        for n_blocks in range(n_stages):\n",
    "            # Add a convnext block series:\n",
    "            for _ in range(blocks_per_stage):\n",
    "                self.layers.append(ConvNextBlock(in_channels=current_n_filters))\n",
    "            # Add a downsampling layer:\n",
    "            self.layers.append(Downsampler(in_channels=current_n_filters, out_channels=2*current_n_filters))\n",
    "            # Double the number of filters:\n",
    "            current_n_filters = 2*current_n_filters\n",
    "\n",
    "\n",
    "        self.head = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            nn.LayerNorm(current_n_filters),\n",
    "            nn.Linear(current_n_filters, n_outputs)\n",
    "        )\n",
    "\n",
    "    def forward(self, inputs):\n",
    "\n",
    "        x = self.stem(inputs)\n",
    "        # Apply a normalization after the initial patching:\n",
    "        x = self.norm1(x)\n",
    "\n",
    "        # Apply the main chunk of the network:\n",
    "        x = self.layers(x)\n",
    "\n",
    "        # Normalize and readout:\n",
    "        x = nn.functional.avg_pool2d(x, x.shape[2:])\n",
    "        x = self.head(x)\n",
    "\n",
    "        return x\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "55fbbcfa-843e-4bcc-ba12-00d18b9e55a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================================================================\n",
      "Layer (type:depth-idx)                   Output Shape              Param #\n",
      "==========================================================================================\n",
      "Classifier                               [128, 256]                --\n",
      "├─Conv2d: 1-1                            [128, 32, 32, 32]         128\n",
      "├─InstanceNorm2d: 1-2                    [128, 32, 32, 32]         --\n",
      "├─Sequential: 1-3                        [128, 128, 8, 8]          --\n",
      "│    └─ConvNextBlock: 2-1                [128, 32, 32, 32]         --\n",
      "│    │    └─Conv2d: 3-1                  [128, 32, 32, 32]         1,600\n",
      "│    │    └─InstanceNorm2d: 3-2          [128, 32, 32, 32]         --\n",
      "│    │    └─Conv2d: 3-3                  [128, 128, 32, 32]        4,224\n",
      "│    │    └─Conv2d: 3-4                  [128, 32, 32, 32]         4,128\n",
      "│    └─ConvNextBlock: 2-2                [128, 32, 32, 32]         --\n",
      "│    │    └─Conv2d: 3-5                  [128, 32, 32, 32]         1,600\n",
      "│    │    └─InstanceNorm2d: 3-6          [128, 32, 32, 32]         --\n",
      "│    │    └─Conv2d: 3-7                  [128, 128, 32, 32]        4,224\n",
      "│    │    └─Conv2d: 3-8                  [128, 32, 32, 32]         4,128\n",
      "│    └─Downsampler: 2-3                  [128, 64, 16, 16]         --\n",
      "│    │    └─InstanceNorm2d: 3-9          [128, 32, 32, 32]         --\n",
      "│    │    └─Conv2d: 3-10                 [128, 64, 16, 16]         8,256\n",
      "│    └─ConvNextBlock: 2-4                [128, 64, 16, 16]         --\n",
      "│    │    └─Conv2d: 3-11                 [128, 64, 16, 16]         3,200\n",
      "│    │    └─InstanceNorm2d: 3-12         [128, 64, 16, 16]         --\n",
      "│    │    └─Conv2d: 3-13                 [128, 256, 16, 16]        16,640\n",
      "│    │    └─Conv2d: 3-14                 [128, 64, 16, 16]         16,448\n",
      "│    └─ConvNextBlock: 2-5                [128, 64, 16, 16]         --\n",
      "│    │    └─Conv2d: 3-15                 [128, 64, 16, 16]         3,200\n",
      "│    │    └─InstanceNorm2d: 3-16         [128, 64, 16, 16]         --\n",
      "│    │    └─Conv2d: 3-17                 [128, 256, 16, 16]        16,640\n",
      "│    │    └─Conv2d: 3-18                 [128, 64, 16, 16]         16,448\n",
      "│    └─Downsampler: 2-6                  [128, 128, 8, 8]          --\n",
      "│    │    └─InstanceNorm2d: 3-19         [128, 64, 16, 16]         --\n",
      "│    │    └─Conv2d: 3-20                 [128, 128, 8, 8]          32,896\n",
      "├─Sequential: 1-4                        [128, 256]                --\n",
      "│    └─Flatten: 2-7                      [128, 128]                --\n",
      "│    └─LayerNorm: 2-8                    [128, 128]                256\n",
      "│    └─Linear: 2-9                       [128, 256]                33,024\n",
      "==========================================================================================\n",
      "Total params: 167,040\n",
      "Trainable params: 167,040\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (G): 5.55\n",
      "==========================================================================================\n",
      "Input size (MB): 1.57\n",
      "Forward/backward pass size (MB): 663.09\n",
      "Params size (MB): 0.67\n",
      "Estimated Total Size (MB): 665.33\n",
      "==========================================================================================\n",
      "==========================================================================================\n",
      "Layer (type:depth-idx)                   Output Shape              Param #\n",
      "==========================================================================================\n",
      "Sequential                               [128, 128]                --\n",
      "├─Linear: 1-1                            [128, 128]                32,896\n",
      "==========================================================================================\n",
      "Total params: 32,896\n",
      "Trainable params: 32,896\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (M): 4.21\n",
      "==========================================================================================\n",
      "Input size (MB): 0.13\n",
      "Forward/backward pass size (MB): 0.13\n",
      "Params size (MB): 0.13\n",
      "Estimated Total Size (MB): 0.39\n",
      "==========================================================================================\n"
     ]
    }
   ],
   "source": [
    "def create_representation_model(n_features, rank, size):\n",
    "\n",
    "    model = Classifier(32, 2, 2, n_features)\n",
    "\n",
    "\n",
    "    model.to(f\"cuda:{rank}\")\n",
    "\n",
    "    return model\n",
    "\n",
    "model = create_representation_model(256, 0, 1)\n",
    "\n",
    "head = torch.nn.Sequential(\n",
    "    nn.Linear(256,128),\n",
    ")\n",
    "\n",
    "head.to(f\"cuda:0\")\n",
    "\n",
    "from torchinfo import summary\n",
    "    \n",
    "print(summary(model, input_size=(batch_size, 3, 32, 32)))\n",
    "print(summary(head, input_size=(batch_size, 256)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "691caee3-bb58-451b-b3d7-fc394393b95f",
   "metadata": {},
   "source": [
    "This will download the data if needed:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "422c5c32-2d59-47b4-9e69-249451f3f410",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "6eeb11a6-e248-4223-bc06-ef995f658ec8",
   "metadata": {},
   "source": [
    "\n",
    "We're going to train this on Polaris nodes which have 4 A100s (But only using one node at a time).  So, the following helper functions will automatically distribute the code and model to use all 4 GPUs at once:\n",
    "\n",
    "(They are all from the [DDP Tutorial](https://pytorch.org/tutorials/intermediate/ddp_tutorial.html) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "562b9ac3-52cf-4ee4-8e36-cd7c4b29c51a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def setup(rank, world_size):\n",
    "    os.environ['MASTER_ADDR'] = 'localhost'\n",
    "    os.environ['MASTER_PORT'] = '12355'\n",
    "\n",
    "    # initialize the process group\n",
    "    dist.init_process_group(\"gloo\", rank=rank, world_size=world_size)\n",
    "\n",
    "def cleanup():\n",
    "    dist.destroy_process_group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "24e2ebd9-47f7-4534-b67a-7ca43b9bc98b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data_loaders(transforms, batch_size, rank, seed):\n",
    "    # Start up the data loader:\n",
    "    dev = torch.device(\n",
    "        f\"cuda:{rank}\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "\n",
    "    training_data = torchvision.datasets.CIFAR10(\n",
    "        root=\"data\",\n",
    "        train=True,\n",
    "        download=True,\n",
    "        transform=transforms\n",
    "    )\n",
    "    \n",
    "    training_data, validation_data = torch.utils.data.random_split(training_data, [0.8, 0.2], generator=torch.Generator().manual_seed(55))\n",
    "    \n",
    "    # The dataloader makes our dataset iterable \n",
    "    train_dataloader = torch.utils.data.DataLoader(training_data, \n",
    "                                                   batch_size=batch_size, \n",
    "                                                   shuffle=True, \n",
    "                                                   num_workers=8)\n",
    "    \n",
    "    val_dataloader = torch.utils.data.DataLoader(validation_data, \n",
    "                                                 batch_size=batch_size, \n",
    "                                                 shuffle=True, \n",
    "                                                 num_workers=8)\n",
    "    \n",
    "\n",
    "    def preprocess(x, y):\n",
    "        # CIFAR-10 is *color* images so 3 layers!\n",
    "        return x.view(-1, 3, 32, 32).to(dev), y.to(dev)\n",
    "    \n",
    "    \n",
    "    class WrappedDataLoader:\n",
    "        def __init__(self, dl, func):\n",
    "            self.dl = dl\n",
    "            self.func = func\n",
    "    \n",
    "        def __len__(self):\n",
    "            return len(self.dl)\n",
    "    \n",
    "        def __iter__(self):\n",
    "            for b in self.dl:\n",
    "                yield (self.func(*b))\n",
    "\n",
    "\n",
    "    train_dataloader = WrappedDataLoader(train_dataloader, preprocess)\n",
    "    val_dataloader = WrappedDataLoader(val_dataloader, preprocess)\n",
    "\n",
    "    return train_dataloader, val_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4c5fcdae-fbe6-4e0b-b3d7-e8b771e9ee47",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# def demo_basic(rank, world_size, n_epochs):\n",
    "#     print(f\"Running basic DDP example on rank {rank}.\")\n",
    "#     setup(rank, world_size)\n",
    "\n",
    "    \n",
    "#     # create model and move it to GPU with id rank\n",
    "#     model = ToyModel().to(rank)\n",
    "#     ddp_model = DDP(model, device_ids=[rank])\n",
    "\n",
    "#     loss_fn = nn.MSELoss()\n",
    "#     optimizer = optim.SGD(ddp_model.parameters(), lr=0.001)\n",
    "\n",
    "#     optimizer.zero_grad()\n",
    "#     outputs = ddp_model(torch.randn(20, 10))\n",
    "#     labels = torch.randn(20, 5).to(rank)\n",
    "#     loss_fn(outputs, labels).backward()\n",
    "#     optimizer.step()\n",
    "\n",
    "#     cleanup()\n",
    "\n",
    "\n",
    "# def run_demo(demo_fn, world_size):\n",
    "#     mp.spawn(demo_fn,\n",
    "#              args=(world_size,5),\n",
    "#              nprocs=world_size,\n",
    "#              join=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "544004df-e1f3-4ee3-b37e-0e3dce527da1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "11354136-05f8-4297-908c-fc35ff0a828a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import sys, os\n",
    "# from multiprocessing import Pool\n",
    "# from multiprocessing.reduction import ForkingPickler\n",
    "# from types import FunctionType\n",
    "# import cloudpickle\n",
    "\n",
    "# assert sys.version_info >= (3, 8), 'python3.8 or greater required to use reducer_override'\n",
    "\n",
    "# def reducer_override(obj):\n",
    "#     if type(obj) is FunctionType:\n",
    "#         return (cloudpickle.loads, (cloudpickle.dumps(obj),))\n",
    "#     else:\n",
    "#         return NotImplemented\n",
    "\n",
    "# # Monkeypatch our function reducer into the pickler for multiprocessing.\n",
    "# # Without this line, the main block will not work on windows or macOS.\n",
    "# # Alterntively, moving the defintionn of foo outside of the if statement\n",
    "# # would make the main block work on windows or macOS (when run from\n",
    "# # the command line).\n",
    "# ForkingPickler.reducer_override = staticmethod(reducer_override)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9716224-0de3-4e52-85e8-7f6e26763c2a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "066d1b3a-f159-430d-81c8-05957ca3824f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dda19fe1-3e37-45b6-8bb6-574bee4b5595",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# This method is from the pytorch implementation of SimCLR:\n",
    "# https://github.com/sthalles/SimCLR/blob/master/data_aug/contrastive_learning_dataset.py\n",
    "\n",
    "def get_simclr_pipeline_transform(size, s=1):\n",
    "    \"\"\"Return a set of data augmentation transformations as described in the SimCLR paper.\"\"\"\n",
    "    color_jitter = v2.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s)\n",
    "    data_transforms = v2.Compose([v2.RandomResizedCrop(size=size, scale=[0.85,1.0]),\n",
    "                                          v2.RandomHorizontalFlip(),\n",
    "                                          v2.RandomApply([color_jitter], p=0.8),\n",
    "                                          v2.RandomGrayscale(p=0.2),\n",
    "                                          v2.ToDtype(torch.float32, scale=True),  # Normalize expects float input\n",
    "                                          # v2.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "                                          # v2.ToTensor()\n",
    "                                        ])\n",
    "    return data_transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67638fcb-43c7-4c45-a8e6-284998ea6ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms1 = get_simclr_pipeline_transform((32,32))\n",
    "transforms2 = get_simclr_pipeline_transform((32,32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a279ed33-e4e3-4f33-96f9-05971cc75551",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cadams/pythonbase/lib/python3.10/site-packages/torchvision/transforms/v2/_deprecated.py:41: UserWarning: The transform `ToTensor()` is deprecated and will be removed in a future release. Instead, please use `v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)])`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train, val = create_data_loaders(v2.ToTensor(), batch_size, 0, seed = 1234)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0b3d388c-2226-49f4-b185-825c9048f773",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch, (X, Y) = next(enumerate(train))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "582bc89f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n"
     ]
    }
   ],
   "source": [
    "X1 = transforms1(X); X2 = transforms2(X)\n",
    "\n",
    "print(type(X1))\n",
    "print(type(X2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a59b8dd8-10d2-4a7a-a321-1456eb27c0d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1085f655-7e32-449c-801c-3a73a74815b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxYklEQVR4nO3de3Dc9Xnv8c9qtbu6ryzLumH5DjZg7DYOOCqEAHaxnSkHgqcDSc7UpAwMVGYKbprEnQQCbUeUzCQkGcfMnKa4mYkhoRPDgdNAwcTySWJD7OAYQ1BsI+ObbpYtrbSSVqvd3/mDWjkCG56vLfGVxPs1szOW9vGj729/v91nr58NBUEQCACAj1iO7wUAAD6eGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC9yfS/gvbLZrI4fP67i4mKFQiHfywEAOAqCQD09PaqpqVFOztkf54y7AXT8+HHV1tb6XgYA4DwdOXJE06dPP+v5YzaANmzYoG9961tqbW3V4sWL9f3vf19XXHHFh/6/4uJiSdLXH7pFeXlR098KRwrM65o55yJzrSQVl0w116YG3J7RzKSHzLXx0phT72zGnrC0b987Tr2XX/MXTvUzZ8wy1+YEWafeLvVZx9CpIMd+9Th5ap9T71df+6lT/cmut821+XnFTr072jrNte3trU69B9L2/VM7c6ZT75On7OvODKScehfk2m57TuvrSphrK2vc7mBPnzXPXJubY78tlKTsgL129qz55tpksl/LVt0zfHt+NmMygH7yk59o3bp1euyxx7R06VI9+uijWrFihZqamlRRUfGB//f00255eVHl5VsHkP1gKSjMN9dKUmGRfYfmht0G0JDDACosGrsBlJ+f59T7ww6q9yopKTHXTtQBlM4UOfUuKHTbn/2DEXtv4x230/Ly7b1jeWGn3kGO/Wl0l3VIUqzfvn8ysl/XJCkv1+2mMROzXy75eW7bWVBgP1ZycxxvJxxusoocbgtP+7CXUcbkTQjf/va3dccdd+hLX/qSLrnkEj322GMqKCjQv/3bv43FnwMATECjPoAGBwe1e/duLV++/I9/JCdHy5cv144dO95Xn0qllEgkRpwAAJPfqA+gEydOKJPJqLKycsTvKysr1dr6/uePGxoaFI/Hh0+8AQEAPh68fw5o/fr16u7uHj4dOXLE95IAAB+BUX8TQnl5ucLhsNra2kb8vq2tTVVVVe+rj8ViisXcXjgDAEx8o/4IKBqNasmSJdq6devw77LZrLZu3aq6urrR/nMAgAlqTN6GvW7dOq1Zs0af/OQndcUVV+jRRx9VMpnUl770pbH4cwCACWhMBtAtt9yijo4O3X///WptbdWf/Mmf6Pnnn3/fGxMAAB9fY5aEsHbtWq1du/ac//+UeJ7yjR9EPXGyy9z3ZNshp3UEQ0lzbX/S7UOUeTH7B7s6Uxmn3rFYobm2xPHDheWlbh/mzZX9MlTW7RPrcrjIc0JuH7jNDNmbv/76b516J/u6neoTvfb6/n63YyWUa/9gZOUF738d94OcdLhuJvt6nHq3HD1krl157RKn3rOr7R+elqQ3dtvTRHr7HOIHJJ1qP2GuveCCi516F5SVm2sHB+3jYjBtq/X+LjgAwMcTAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAODFmEXxnK8gk1KQCUy1ebn26JFgyO0bV/u60+bak529Tr3Lpkwz1+bmusXlpHrtsSZFeW6xI9Fwn1N9R8ub5trO1han3uGQ/as88goqnHpXTL/QXNt54phT7/6UW+xMENhjgU51u8X8RKPF5trC4iKn3kUltjgtSepJ9Dv1TvfZL5PCSMip92Cv2/eSVcRtt1WSVFU+xan38S779S0ScbudKHWI4jnRYT+ukknbvuQREADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMCLcZsF193VpdRAxFSbStkzpC6otuevSdJgesBcm5N1y7LKpu0ZT0FguyxOSyROmWuHBk849f7da79wqk8m9ppr+xJuOWbZIfvlEsotc+o9u3eJuTY96Jbtlpvrdt8vJ2y/quaE3XLPuhL2Y/zQkQ6n3t2nhsy1ocCepyZJS5f8mbk2m7bnRUrSsbbjTvXxvFJzbTjXfnlLUjZrz6PML3Q7rnr6Tppr+1L2dfcPkgUHABjHGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvxm0Uz9Hj7YrFbMurdojXGUzbYy0kKRoJm2unTCl16j2Ysq8lPTjo1NslvuNkV5dT79f22OM7JGl6pT2iKKSsU+/+fnvUSzLV5dS7M2mP14kV5jv1jkXd6jMJe7zOyVNukVAnu+3HSmeHW+RQdrDQXFta6nZzFITs95/TGbd4ogGHdUtSvNBenxONOvUuKs0z1xaW2Gslqb3NHn118lTCXNvfb4vt4REQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwItxmwUXKyxVLBYx1eYVlJr7pgbdssYKCuyZXb29jhlcp+y5WsXFRU69T5w8Ya4dzNhym05L9NgzoSQpO9W+9niB23YeP3bUXBstqnTqfeKUPfMurjKn3uGs22Wel2e/XJJ9bnltvcmMuTY95JZjFsrab2KSA27HVTJt3z/9WbfjqiA+w6m+dFqVuTYTjjn1LswJzLWdp/qcencn7MdhKmXPo0wZcy55BAQA8GLUB9A3v/lNhUKhEacFCxaM9p8BAExwY/IU3KWXXqqXXnrpj38kd9w+0wcA8GRMJkNubq6qquzPiQIAPn7G5DWg/fv3q6amRnPmzNEXv/hFHT58+Ky1qVRKiURixAkAMPmN+gBaunSpNm3apOeff14bN25Uc3OzPv3pT6un58zvzGloaFA8Hh8+1dbWjvaSAADj0KgPoFWrVukv//IvtWjRIq1YsUL/+Z//qa6uLv30pz89Y/369evV3d09fDpy5MhoLwkAMA6N+bsDSktLddFFF+nAgQNnPD8WiykWc3tfPABg4hvzzwH19vbq4MGDqq6uHus/BQCYQEZ9AH35y19WY2OjDh06pF//+tf63Oc+p3A4rM9//vOj/acAABPYqD8Fd/ToUX3+859XZ2enpk2bpquuuko7d+7UtGnTnPoMZWMKZ21RPINDYXPfAbcEFHW022NN0umQU+++lD1iYyDtFq/S1tFprs3Pd7sfUlJS4lSfTNr3T1lpuVPvyxbPMtf2OcbI9L7daq6NROyRTZI0dWrcqb4/kzTXVlW6bWdekf24jUXd9n1uxnYdlqRQ9LhT76kOyUonuuyxPZJUEBQ71V80b6a5trvPFlNzWldPl7k20W8/TiRpKGXf98lkylw7MGCrHfUB9OSTT452SwDAJEQWHADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADAizH/OoZz9YlPLFdBgS1fK0gNmvteUOH2VeEXVNrrhyL2zDNJGgw5ZMH1djn1/t9P/8Rc29PV4dQ7Fqlxqi8om26unbfoKqfeU6ddYK5959gJp97x6fbMrrIStyy4YMgts+vkKXuW2eJL3I7xbEGBubav335dk6Sg334ZNu//jVvv7Nm/afm9OjP265okDQ4VOdUnM/avlDnW4bbvDx/Nmmt7km6ZdwsunGOuLXS4SHLCtgxAHgEBALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALwYt1E8gz2FCg/ZIkI6u7rMfQeybpEc0WlTzbWFEbf4jlSqz1ybdew966LPmGtbW4449Y7ku60llmOPhjl8MOPUu+34KXNtRfVMp9558ZC5NnC8KzeQ6Xf7D7lTzKWtHW5xOfmFttgUSUql7LEwkpRM2LfzVIc9zkaSaqrmmWvzqyuceoeDuFN9RJXm2qkl1U69O/LtUVmFeUNOvfOjUXNtJJtnb2y8GvMICADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAODFuM2Ce/mlNxSN2LKhckoLzX2HjvU4rWP7kU5zbUmOPVNLkgY67TlmR48edes9mDLXDg65ZYeVlRQ71ef2NJlrk332fDxJqqyy52pdctlCp96ZwJ4b2JNwO676HS/zI20t5tpj77hl+5VHbJmLkpRKJp16h4ryzbVzZtc69Z4z92pz7QUVpU69X3j21071+3a9bS/OteevSVLrSfvtREmZPb9Qki5bON1cW1psz9Pr7bVdj3kEBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPBi3GbBZWOlykTzTLV9sucf5UbdcswOHrJncIV6+p16nzh42FzbM+DWu7p2lrk2x/Ey6U+5HTaxfNt+lKRMzG0t3YMZc21uasCpdyQaNtcW5rrlAAYZ+7olKV421Vx7wdRyp96F3Wlz7VC/PWNQkloj9utm35DbZXgyaa+PJtzy16ZWXeZUn8zvNde2dJxw6j0YzpprE2m3jMFf//agufZ/3HS9uTY313Z58AgIAOCF8wDavn27brjhBtXU1CgUCunpp58ecX4QBLr//vtVXV2t/Px8LV++XPv37x+t9QIAJgnnAZRMJrV48WJt2LDhjOc/8sgj+t73vqfHHntMr7zyigoLC7VixQoNDLg9/QEAmNycXwNatWqVVq1adcbzgiDQo48+qq9//eu68cYbJUk/+tGPVFlZqaefflq33nrr+a0WADBpjOprQM3NzWptbdXy5cuHfxePx7V06VLt2LHjjP8nlUopkUiMOAEAJr9RHUCtra2SpMrKyhG/r6ysHD7vvRoaGhSPx4dPtbVu34oIAJiYvL8Lbv369eru7h4+HTni9nXCAICJaVQHUFVVlSSpra1txO/b2tqGz3uvWCymkpKSEScAwOQ3qgNo9uzZqqqq0tatW4d/l0gk9Morr6iurm40/xQAYIJzfhdcb2+vDhw4MPxzc3Oz9uzZo7KyMs2YMUP33nuv/umf/kkXXnihZs+erW984xuqqanRTTfdNJrrBgBMcM4DaNeuXbr22muHf163bp0kac2aNdq0aZO+8pWvKJlM6s4771RXV5euuuoqPf/888rLs8exSFKqKFfZmG15A2n7Z4za//B7p3VkQ/bIlLICtxiZ9tCQuba41O2pyYHAHq8yrbzaqffgkFuMTOkUh8vFYd2SFI3ZH8QXTIk79b54wVxz7bUXX+TUu/eEWxxLJD9mrr2gpMip94HGV821ie4ep967e5Lm2oNH2516v7Rjp7m2p7PTqXck7XZdnjptirn2zbft8TeS1JW0xx9VVLnFMLX3268/Q41N5tqB/j5TnfMAuuaaaxQEwVnPD4VCeuihh/TQQw+5tgYAfIx4fxccAODjiQEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwwjmK56OSiWWUE7Nljk0tKTT3LYm5ZSX1d50y13Z0HHPqPXvWmb+i4kx6U4NOvfuH7PXZtNu30OYGbvdbQqey5topxW45ZovnXWKuLZ/qlgVXlImaa4f67XmEkhTNd7sMI7Lvz3S/W15bQZl9O9887HaMv9Vsr4+UVn540f8n9QGRYO/V63D5SVKi/bBTfXfWnjWXzXPLUuzp6TfXhh2Pw+lzLzXXTq21ZyP2J3tNdTwCAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4MW6jeAqDHEWMkS/ZXlvsgyRlk0mndfSePGGuPfL2AafehSVTzLW5eXlOvcMxe302cdKpd2rQLUqkqLTGXDutqtqp94nObnPtsSNuMTKZoSFz7X81ukWg5EdDTvWV+fnm2pnVFU69TyXt+//F3+5y6v32cXtETSp8yKl3blHEXBsacts/cyrmONVfOt9en5MXc+r95v5D5tq8Arcoq7BDRFGm1x4JlO2zXd48AgIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4MW6z4EKn+hSKZk21vX2nzH2T/fbsMEl658gRc+3ggD1XSZIiOfZspYULLnHqPWvOLHPtCy+97NR72szZTvXR+TPNtSenFDj1bj/eaq49/o59X0pS2OH+WSZrO1ZPy55KONXPKiw21566yC1nbtu+V8216Ry3+6yptH0tZZX2zEBJumjxfHPtoQNvOvXeu/ctp/qIw1V/3oVznXrPrKg01+ZGA6fe4ZA9GzPbddhe22+7beMREADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADAi3EbxXNRbVx5efmm2jm1C8x9Ez1dTuv49x83m2uTKbfemaDXXBsdcovvqCyPm2unTHeL1pk6/1Kn+twae7xOdW25U++BtD3SJtHlFvMzkLBHJWU05NR7MGTvLUm1F19kri2tLHXqffKX9uMwr9geCSRJgWLm2lTSHgsjSanulLm2MG+aU+8g1uFU33TUfjsx65ILnXpHcu2XYdrxNuj13/3KXPunl9hvg1Ip277hERAAwAsGEADAC+cBtH37dt1www2qqalRKBTS008/PeL82267TaFQaMRp5cqVo7VeAMAk4TyAksmkFi9erA0bNpy1ZuXKlWppaRk+PfHEE+e1SADA5OP8JoRVq1Zp1apVH1gTi8VUVVV1zosCAEx+Y/Ia0LZt21RRUaH58+fr7rvvVmdn51lrU6mUEonEiBMAYPIb9QG0cuVK/ehHP9LWrVv1L//yL2psbNSqVauUyWTOWN/Q0KB4PD58qq2tHe0lAQDGoVH/HNCtt946/O/LLrtMixYt0ty5c7Vt2zYtW7bsffXr16/XunXrhn9OJBIMIQD4GBjzt2HPmTNH5eXlOnDgwBnPj8ViKikpGXECAEx+Yz6Ajh49qs7OTlVXV4/1nwIATCDOT8H19vaOeDTT3NysPXv2qKysTGVlZXrwwQe1evVqVVVV6eDBg/rKV76iefPmacWKFaO6cADAxOY8gHbt2qVrr712+OfTr9+sWbNGGzdu1N69e/Xv//7v6urqUk1Nja6//nr94z/+o2Ixe56RJC24uFoFBbbsrp72VnPf3/5mp9M6ujvtvRUMOvXOz4uaa/e/tc+pd/Use97UrLluOXMdxpyn4bVE7Ll0g31ueWBd7fbMriHHrLG+U1322n57rSQVhN0uw5Zjb5prTxzb79S7OCcw1yY6u5x6B2d589GZHDn0jlPvzpPd9nWE3G7q+gbcrss5Uftl2NnrlgPYfeqEuTaiAafekdwic21hnv3lkbBxHc4D6JprrlEQnP3CfuGFF1xbAgA+hsiCAwB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4MerfBzRa/rD/deXl2fLjTnaeMvc90HLMaR3pgnxz7fQZM516H9xvz+yKl9vz1CTpk1fUmWuP9rjlXiXCEaf6Hoej7Fjzcafe4ax9LQP9bjlZYYe7Z5fOmeXUuzDklgdWFLJnql1V92dOvdd/+Spz7cOP/qtT71d/9ztzbbHjV7HEy8rMtSWlU5x6l5S4Xd/yc+wH+aEDDvmSkpJ99tzAwT77baEkXTKvxlx7zbX2QOlkslfSQx9axyMgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAX4zaK5+DbzYpGbTEr7SeT5r6tPUNO6+gLFZprL75oiVPv+Z+8zlx74azZTr1butPm2lQQcuo9mHa7DJvfOWqu7U/2OvXOydi3c0phkVPvSJF932dT9mNQko62HHGqX7LwYnNtQb5bjMzOV/aYa0sdI23+7M/ssUCFRW77pzheaq7Ny7NHaklSJOIWN9XV2mmu7etyO1ayQdZcG8lx3M5o1FybydrjoKy1PAICAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeDFus+D6erNKR2wZSNGCKnPfP116hdM64hUzzLUzLr7UqXcoYs9hSvW45UedGrRnpLWfPOHUu29w0Km+OBQz18Zy3XLmJHtOVsIhy0qSjh4+aK5tPXrcqXcq2eNUX1Fxgbl238EWp94DQ/YswKrpM516h3Ls+7Ovz+0Yr62pMNfm5rrd1LmlI0rVRfbrckfr2069jx5rNtcW5Ltl2PUk7LcTh5qbzLX9ff2mOh4BAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8GLdRPNddu0L5+QWm2tf226NHiirmuC0kWmgubWqyR7dIUs/AgLk2lUw59U6n7PWpdJ9b70G3+tiQPQInGHLbzo62I+balmOHnHoP9CfMteFc27F6WjTqVt9xqtdcWzil3Kl3aZ79GM8Nu91nDWXs6y4rtMfZSFJB2OG4yrjFR4WCwKk+J2vv39nyB6ferUf2mWtnzpju1Lv7lH0E/OGtvebalPH2h0dAAAAvnAZQQ0ODLr/8chUXF6uiokI33XSTmppGBtQNDAyovr5eU6dOVVFRkVavXq22trZRXTQAYOJzGkCNjY2qr6/Xzp079eKLLyqdTuv6669XMvnHFNv77rtPzz77rJ566ik1Njbq+PHjuvnmm0d94QCAic3pNaDnn39+xM+bNm1SRUWFdu/erauvvlrd3d364Q9/qM2bN+u6666TJD3++OO6+OKLtXPnTn3qU58avZUDACa083oNqLu7W5JUVlYmSdq9e7fS6bSWL18+XLNgwQLNmDFDO3bsOGOPVCqlRCIx4gQAmPzOeQBls1nde++9uvLKK7Vw4UJJUmtrq6LRqEpLS0fUVlZWqrW19Yx9GhoaFI/Hh0+1tbXnuiQAwARyzgOovr5e+/bt05NPPnleC1i/fr26u7uHT0eO2N9WCwCYuM7pc0Br167Vc889p+3bt2v69D++77yqqkqDg4Pq6uoa8Siora1NVVVn/trsWCymWMz+lc0AgMnB6RFQEARau3attmzZopdfflmzZ88ecf6SJUsUiUS0devW4d81NTXp8OHDqqurG50VAwAmBadHQPX19dq8ebOeeeYZFRcXD7+uE4/HlZ+fr3g8rttvv13r1q1TWVmZSkpKdM8996iuro53wAEARnAaQBs3bpQkXXPNNSN+//jjj+u2226TJH3nO99RTk6OVq9erVQqpRUrVugHP/jBqCwWADB5OA2gwJCPlJeXpw0bNmjDhg3nvChJ+sSSP1FRUZGptiv7prnvvoPHnNZxKpH88KL/Fs51y48Kh8Pm2kzSLcuqP2nPa4tE3NY9t/bMr+edTcc775hr9/xut1Pv3sQpc21uOOvUu1D2rLFQyJ7rJ0nFxmP7tAKHmLTMoD1/TZK6T3WYa3NCbsfKYPKkuba0pNipdybf/tpxb6LbqXc61e9Un5OxXy6Hfv+GU+++k/b9cyLstn+mvOdllA+S7rPnNFqzKMmCAwB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4cU5fx/BRSKW7FUkPmWpnz7VHw7zd4vZ9QxUVFebamimFTr0XzJllrv31jj1OvefOm2euveAC+zZKkoK0U/n+39v7d7ccdOr9Vpc9WimW63a4p4fs8UfZtFv8TSpjj0qSpFDNFHPtqZYDTr07Ok6Ya7NZezyRJGUG7ZdhZzTi1Lu4MM9cm5drj72SpHix23X5ZLv9m5z7u+zxXpJUFCmwFw+6xU1lB+zRPbmyRx9ZjxIeAQEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8GLdZcC+9vE15ebasp8HcYnPfTKrHbSGxkLm0KOKWHxVO95tr83Pd8tdqp0+1ryPmdj/kUPNRp/pjxw/Z1xJ2yxrLybFnX2WybpdhJGK/eoRy7MeJJKXTA071iVPt5toOh8tbknIdMvJcLhNJmlZtz2ksLnTIPJMUi0XNtZGw2zEectudenOfPcMwCOz5a5IUCdtz7MJy693V3WGuray60lw7MGA7vnkEBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwYtxG8XS0dyoai5lqmw7tNfdd9KefcFrH1BJ7vE5qwB6tI0nPPPusuTa/uMSpd/tJe8RGd1+fU+/f7t7tVP+7nb8y1/Z0dzv1Li2bYq7NZu2xPZLU29trrs2k3XqHQm5XvWjUdl2QpAvnXejUO+mwnQOOx3jMIbonm3WLYeo3xr1IUl/WLaImnXaLbTrSdthcWzQl36n30NCgubag0H6cSNJnb7jeXPvpa68y1/b2Jk11PAICAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeDFus+B6TnUoGo2Yajvbjpj7nmiZ5rSOC6baM9hOtp9w6v2HprfMtamMW05Wbp49E6o76ZYF94e3fu9Un3DId8s6bmckYjtGJPd8r2g0aq7NZt2uSun0kFP90JA9y6y/L+XUOxy2b2cs5naf9WS3PTsumbTlh502NGS/DPsc8w77+tzWcszhNijH8W5/Oms/bv/nX37Bqfetn7fXhx3i9CL5PaY6HgEBALxwGkANDQ26/PLLVVxcrIqKCt10001qamoaUXPNNdcoFAqNON11112jumgAwMTnNIAaGxtVX1+vnTt36sUXX1Q6ndb111//vofOd9xxh1paWoZPjzzyyKguGgAw8Tk9cf3888+P+HnTpk2qqKjQ7t27dfXVVw//vqCgQFVVVaOzQgDApHRerwF1//eLy2VlZSN+/+Mf/1jl5eVauHCh1q9f/4EvAKZSKSUSiREnAMDkd87vgstms7r33nt15ZVXauHChcO//8IXvqCZM2eqpqZGe/fu1Ve/+lU1NTXpZz/72Rn7NDQ06MEHHzzXZQAAJqhzHkD19fXat2+ffvnLX474/Z133jn878suu0zV1dVatmyZDh48qLlz576vz/r167Vu3brhnxOJhGpra891WQCACeKcBtDatWv13HPPafv27Zo+ffoH1i5dulSSdODAgTMOoFgspljM7XvMAQATn9MACoJA99xzj7Zs2aJt27Zp9uzZH/p/9uzZI0mqrq4+pwUCACYnpwFUX1+vzZs365lnnlFxcbFaW1slSfF4XPn5+Tp48KA2b96sz372s5o6dar27t2r++67T1dffbUWLVo0JhsAAJiYnAbQxo0bJb37YdP/3+OPP67bbrtN0WhUL730kh599FElk0nV1tZq9erV+vrXvz5qCwYATA7OT8F9kNraWjU2Np7Xgk6rnFKoWMyWUfXGkD1v6k8vvdBpHatv+qy59vXf7XPq/X+3bzPXvtP8tlPviEMWXG4sz6n3qY52p/ohhwy2cNjtkwGZjD0PLD046NQ7Xho31+aG7Zl0knT8WItTfWYoa649euSYU++BAfv1J9HjlpHWN2C/zF2z+mJR+zFeOqXUqXdXt1t2XFfCln0mSbmRkFPvnJBD3uFQ2Kn3nj32PMpYxH559xlz/ciCAwB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4cc7fBzTWLlkwV/n5toiY/YfeMfd9fc+rTus48vbvzbVdp3qdeqdS9riPzNCAU++BXns0SGTILQIlnM041cshLicI3O4TBQ6pJtmM23YOGONEJCkktwiUSNjtqtfXY9+fr+3e7dR7oN8exZN1vM+acagPh90uQ5evceno6HDq3dfvFsXT12uPHCooLHTqHcm1b+f/efZlp94vv2i/PYxG7cfskPE2hUdAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC/GbRbcYKpP4ZAtc6yqYqq57/F2t0yoV3f+0lybzbrN82isyFw7JV7s1HvIIWfuZEe7U2/JIYBNUjgUmGsH+u35a5IUBPbeymadeqcc1uLYWq73/drbjptr02m3zDsXoRy3vLaMS1ifo1COvXeuY/aeS29Jyo8WmGsjsme7SVJFWYW5trQ47tS7NF5mX0eFvXZwMKXtWz+8jkdAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvxm0UT2VlpQoK8k2111bWmvtG8wud1rHh+981177++htOvaPRfnNtJDfq1DsvZq8f6O1x6t3XP+BUHzjczXGNkck6ZOCEw24xMnl5eWPWu7e326neZTsjEberdTRqj4aJORxXkpSfb7sOS1JRkdt1s7jYHk9VXl7u1HvaNHv8jSRNKbXHgeVGIk69q6qqzLWzZ8926l1aOsVcG4vaj/He3l798H9950PreAQEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8GLcZsFdUVenkuIiU22k2J7D9M7RY07rqLvySnPt1PJpTr2HBjPm2p7uXqfePQl7fUlhgVPv0hJ7BpckpRU41Y+VcK7b4Z7vkAVXWGjPPJOksjJ7Bte79WXm2tIpbr1LS0vNtUWOx0pJsb1+6lT7NkpSPB431xYWuOXMRR0z7yIO+W5BYL/ev8ueA5gTdruuZbIOeZQ59iy4bMaWF8kjIACAF04DaOPGjVq0aJFKSkpUUlKiuro6/fznPx8+f2BgQPX19Zo6daqKioq0evVqtbW1jfqiAQATn9MAmj59uh5++GHt3r1bu3bt0nXXXacbb7xRb7zx7tcQ3HfffXr22Wf11FNPqbGxUcePH9fNN988JgsHAExsTk+K33DDDSN+/ud//mdt3LhRO3fu1PTp0/XDH/5Qmzdv1nXXXSdJevzxx3XxxRdr586d+tSnPjV6qwYATHjn/BpQJpPRk08+qWQyqbq6Ou3evVvpdFrLly8frlmwYIFmzJihHTt2nLVPKpVSIpEYcQIATH7OA+j1119XUVGRYrGY7rrrLm3ZskWXXHKJWltbFY1G3/eOmsrKSrW2tp61X0NDg+Lx+PCpttb+7aYAgInLeQDNnz9fe/bs0SuvvKK7775ba9as0ZtvvnnOC1i/fr26u7uHT0eOHDnnXgCAicP5c0DRaFTz5s2TJC1ZskS/+c1v9N3vfle33HKLBgcH1dXVNeJRUFtb2wd+p3ksFlMsZv9OegDA5HDenwPKZrNKpVJasmSJIpGItm7dOnxeU1OTDh8+rLq6uvP9MwCAScbpEdD69eu1atUqzZgxQz09Pdq8ebO2bdumF154QfF4XLfffrvWrVunsrIylZSU6J577lFdXR3vgAMAvI/TAGpvb9df/dVfqaWlRfF4XIsWLdILL7ygP//zP5ckfec731FOTo5Wr16tVCqlFStW6Ac/+ME5LSzV16+BsC36YUjd5r6lRfZ4FUm6bc0t5tqBfnushSRlBu216UG3iI10yh73keztcertGlOScYjiGRpyiykJhULm2kjE7RnnSNS+neEctycTIlF7dIsk5RqvC5JbLIwkhR16O1zc79Y7xMgEgdsx7lY+lr0lZe1X5pDzWuzXiSCwX96SlOOwlmzG3jubTpvqQoHrXh9jiURC8XhcTa/9QsXGLLjcAns22WBmyGk9g+k+cy0D6MwYQGfqzQB6rwk9gBy2030tDteJkNsAcllLKLDv/J7epD6xdKW6u7tVUlJy1jqy4AAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF44p2GPtdOfhu7tTZr/T27G/gndsUxCSA0MOPUe0ySEQfunp/v67NsoSekht8tw/CQh2D/xL0kRY5yIJOW4JiEMjmUSgtvVOockhPPsLbklITgaJ0kIckhCOH37/WH7dNwNoJ6ed2Nhlnz6LzyvBABwPnp6ehSPx896/rjLgstmszp+/LiKi4tH3LtNJBKqra3VkSNHPjBbaKJjOyePj8M2SmznZDMa2xkEgXp6elRTU/OBzw6Mu0dAOTk5mj59+lnPLykpmdQ7/zS2c/L4OGyjxHZONue7nR/0yOc03oQAAPCCAQQA8GLCDKBYLKYHHnhAsVjM91LGFNs5eXwctlFiOyebj3I7x92bEAAAHw8T5hEQAGByYQABALxgAAEAvGAAAQC8mDADaMOGDZo1a5by8vK0dOlSvfrqq76XNKq++c1vKhQKjTgtWLDA97LOy/bt23XDDTeopqZGoVBITz/99IjzgyDQ/fffr+rqauXn52v58uXav3+/n8Wehw/bzttuu+19+3blypV+FnuOGhoadPnll6u4uFgVFRW66aab1NTUNKJmYGBA9fX1mjp1qoqKirR69Wq1tbV5WvG5sWznNddc8779edddd3la8bnZuHGjFi1aNPxh07q6Ov385z8fPv+j2pcTYgD95Cc/0bp16/TAAw/ot7/9rRYvXqwVK1aovb3d99JG1aWXXqqWlpbh0y9/+UvfSzovyWRSixcv1oYNG854/iOPPKLvfe97euyxx/TKK6+osLBQK1as0IBjqKtvH7adkrRy5coR+/aJJ574CFd4/hobG1VfX6+dO3fqxRdfVDqd1vXXX69k8o+hwffdd5+effZZPfXUU2psbNTx48d18803e1y1O8t2StIdd9wxYn8+8sgjnlZ8bqZPn66HH35Yu3fv1q5du3Tdddfpxhtv1BtvvCHpI9yXwQRwxRVXBPX19cM/ZzKZoKamJmhoaPC4qtH1wAMPBIsXL/a9jDEjKdiyZcvwz9lsNqiqqgq+9a1vDf+uq6sriMViwRNPPOFhhaPjvdsZBEGwZs2a4MYbb/SynrHS3t4eSAoaGxuDIHh330UikeCpp54arvn9738fSAp27Njha5nn7b3bGQRB8JnPfCb427/9W3+LGiNTpkwJ/vVf//Uj3Zfj/hHQ4OCgdu/ereXLlw//LicnR8uXL9eOHTs8rmz07d+/XzU1NZozZ46++MUv6vDhw76XNGaam5vV2to6Yr/G43EtXbp00u1XSdq2bZsqKio0f/583X333ers7PS9pPPS3d0tSSorK5Mk7d69W+l0esT+XLBggWbMmDGh9+d7t/O0H//4xyovL9fChQu1fv165680GU8ymYyefPJJJZNJ1dXVfaT7ctyFkb7XiRMnlMlkVFlZOeL3lZWVeuuttzytavQtXbpUmzZt0vz589XS0qIHH3xQn/70p7Vv3z4VFxf7Xt6oa21tlaQz7tfT500WK1eu1M0336zZs2fr4MGD+od/+AetWrVKO3bsUNjhu3jGi2w2q3vvvVdXXnmlFi5cKOnd/RmNRlVaWjqidiLvzzNtpyR94Qtf0MyZM1VTU6O9e/fqq1/9qpqamvSzn/3M42rdvf7666qrq9PAwICKioq0ZcsWXXLJJdqzZ89Hti/H/QD6uFi1atXwvxctWqSlS5dq5syZ+ulPf6rbb7/d48pwvm699dbhf1922WVatGiR5s6dq23btmnZsmUeV3Zu6uvrtW/fvgn/GuWHOdt23nnnncP/vuyyy1RdXa1ly5bp4MGDmjt37ke9zHM2f/587dmzR93d3fqP//gPrVmzRo2NjR/pGsb9U3Dl5eUKh8PvewdGW1ubqqqqPK1q7JWWluqiiy7SgQMHfC9lTJzedx+3/SpJc+bMUXl5+YTct2vXrtVzzz2nX/ziFyO+NqWqqkqDg4Pq6uoaUT9R9+fZtvNMli5dKkkTbn9Go1HNmzdPS5YsUUNDgxYvXqzvfve7H+m+HPcDKBqNasmSJdq6devw77LZrLZu3aq6ujqPKxtbvb29OnjwoKqrq30vZUzMnj1bVVVVI/ZrIpHQK6+8Mqn3qyQdPXpUnZ2dE2rfBkGgtWvXasuWLXr55Zc1e/bsEecvWbJEkUhkxP5samrS4cOHJ9T+/LDtPJM9e/ZI0oTan2eSzWaVSqU+2n05qm9pGCNPPvlkEIvFgk2bNgVvvvlmcOeddwalpaVBa2ur76WNmr/7u78Ltm3bFjQ3Nwe/+tWvguXLlwfl5eVBe3u776Wds56enuC1114LXnvttUBS8O1vfzt47bXXgnfeeScIgiB4+OGHg9LS0uCZZ54J9u7dG9x4443B7Nmzg/7+fs8rd/NB29nT0xN8+ctfDnbs2BE0NzcHL730UvCJT3wiuPDCC4OBgQHfSze7++67g3g8Hmzbti1oaWkZPvX19Q3X3HXXXcGMGTOCl19+Odi1a1dQV1cX1NXVeVy1uw/bzgMHDgQPPfRQsGvXrqC5uTl45plngjlz5gRXX32155W7+drXvhY0NjYGzc3Nwd69e4Ovfe1rQSgUCv7rv/4rCIKPbl9OiAEUBEHw/e9/P5gxY0YQjUaDK664Iti5c6fvJY2qW265Jaiurg6i0WhwwQUXBLfccktw4MAB38s6L7/4xS8CSe87rVmzJgiCd9+K/Y1vfCOorKwMYrFYsGzZsqCpqcnvos/BB21nX19fcP311wfTpk0LIpFIMHPmzOCOO+6YcHeezrR9koLHH398uKa/vz/4m7/5m2DKlClBQUFB8LnPfS5oaWnxt+hz8GHbefjw4eDqq68OysrKglgsFsybNy/4+7//+6C7u9vvwh399V//dTBz5swgGo0G06ZNC5YtWzY8fILgo9uXfB0DAMCLcf8aEABgcmIAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALz4fwP4xuPRJgjCAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwm0lEQVR4nO3de2zU95n3/c/M2DM+2xhjbAdDIAfSHKBP2Yb6aZtNAxugUpQ03Kv0IC3pVomShWgTttuWVc+7K7Kp1KatKJHuzSZbqUnarJpE6bNNNqGFqF1ICw2lSVsaWBqONmCwx56zZ37PH1V8304guS6w+WLn/ZJGwvbF5e/vMHPNeGY+E4uiKBIAAOdYPPQCAADvTAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQVaEX8EaVSkWHDx9WY2OjYrFY6OUAAJyiKNLQ0JC6uroUj5/+cc55N4AOHz6s7u7u0MsAAJylAwcOaNasWaf9+YQNoA0bNuhrX/uaent7tXDhQn3729/W1Vdf/bb/r7GxUZL0r4/+s+rqaky/q75hmnldI6WEufZP9WVzbV1dtat3ImF/hHfoUL+r9/SW0x/0N1p4xXtcvVta7PtbkuJRxVwbcyZD2TtL5WjE1fvYiZfNtS/tetLV++Tga676mlSdufZEf9rVu/94n7k2kyu6erdMbzPXRs4/eOSHh8y1VZ4TRVKs7PsPMcfiOzpnu3q3ddivywnZbjNfFy/bbw9nd19srs1kclqy4q7R2/PTmZAB9P3vf19r167VAw88oMWLF+v+++/XsmXLtHv3brW3t7/l/339z251dTWqq681/b76BvuVc6ToG0AlxwCqr5+4AVRXl3X1rq+vN9c2NjW5ejc56yfrAMqX7Puwrj7l7O07V2pTSXNtLuvrnaqx3wyMVOzXB0mqcfT2DqBoxN7bPYBGfNsZj+xPp9fW+o5PXZ393ErIdx56BlCD43b2dW/3NMqEvAjh61//um677TZ98pOf1OWXX64HHnhAdXV1+rd/+7eJ+HUAgElo3AdQsVjUjh07tHTp0v/zS+JxLV26VFu3bn1TfaFQUDqdHnMBAEx94z6Ajh8/rnK5rJkzZ475/syZM9Xb2/um+vXr16u5uXn0wgsQAOCdIfj7gNatW6fBwcHRy4EDB0IvCQBwDoz7ixDa2tqUSCTU1zf2lTV9fX3q6Oh4U30qlVIq5XviDAAw+Y37I6BkMqlFixZp06ZNo9+rVCratGmTenp6xvvXAQAmqQl5GfbatWu1atUq/dmf/Zmuvvpq3X///cpkMvrkJz85Eb8OADAJTcgAuuWWW3Ts2DF98YtfVG9vr9797nfrmWeeedMLEwAA71wTloSwZs0arVmz5oz//8DxQypkbc8NVUYy5r7FvO/dblUJ+/NT5YKrtaqr7W8urHK+ibLZ8abYet+bp5WIhn3/oeJYu+MNfZIUl30fZnO+N/P+4Q+/dfS2vytfkoYyvrcb5PP2fVhxZii2TG811yZSvmNfcZy3AydOuHrPmGZ7o7okXXXpBa7eKpRc5YdfGzTXlp3nYXbQvs/b2uzJE5JUW29PNSk63sRfNCbOBH8VHADgnYkBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACGLConjOVqWUVqVoi1nJpe2xGUPpvGsddbWN5tpU0h4LI0nFmD0uJx758nJqq+2faV/KH3L1Hjx21FWfH7ZHJSUS9ngVSaqtbzfXxlO+3v3HD5trc3lnRE3FfnwkaTBrj/rx7sOaWnt9XYP9nJWkbKZori1kffukuiUy19YmfNf7eFXOVd853X7dL0a+j5/Jl+1rT6V8t0GNjfYonnTavk8yGVsuGY+AAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEGct1lw9TUJ1dUmTLXlij2jKCo5M7sSMXNtWb68tlLRnn1VLPruKxw+YM/sKmb3uHpn0vtd9YWMJ1fLmWPW0GGubZkxy9W7WLDnr8Vj9lwySYrHfcfT0304a89GlKRjJ+xZfcOO3EVJKhXs53hbywxX79ZpDebaTNp+LCUpXhx01aeqm+3FkT0fT5IKJXsWXLUvZk75UtZcm83br8fZgq2WR0AAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCDO2yieKKooiiqm2mpbYo8kqb7OF/VSqdjWIElFY/zE60ole0zJ8LCv974/ps21Q2lfjExNlS/OaKQwYq7N5+3RR5JUPnHcXNt3ss/V2xOZkkz6MlAqke++XyZjX8vAkH1/S9JJRxRPPutbd7LKfhPTMcN33Uwk7NFXhYI9ckaS4iXfTWOqwR59VZXw9a5xnFspY3zZ6wZO2iOHBtP26302a4sP4hEQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIIjzNguuVCqraMxKa6i3ZyXFfVFJymRsmUaSFI/7dmcuZ8+nyuR8+Wu5vD0PLFnl2ym10+y5V5IUkz1PL5vzZd5VEvb7UCdzJVfv+sZ6c208Zt9GSaqqsueYSVImaz9X0kP23DhJGs7YswDLRed91sh+HhZGfHlthYr9PKxJ+nLmapP2Yy9JtY0t5tpizJcbWC7b157N+87x4UzB3ttx3czlbX15BAQACGLcB9CXv/xlxWKxMZfLLrtsvH8NAGCSm5A/wV1xxRV6/vnn/88vcUSyAwDeGSZkMlRVVamjo2MiWgMApogJeQ7o1VdfVVdXl+bNm6dPfOIT2r9//2lrC4WC0un0mAsAYOob9wG0ePFiPfzww3rmmWe0ceNG7du3Tx/84Ac1NDR0yvr169erubl59NLd3T3eSwIAnIfGfQCtWLFCf/mXf6kFCxZo2bJl+s///E8NDAzoBz/4wSnr161bp8HBwdHLgQMHxntJAIDz0IS/OqClpUWXXnqp9uzZc8qfp1IppVK+18UDACa/CX8f0PDwsPbu3avOzs6J/lUAgElk3AfQpz/9aW3ZskV//OMf9d///d/6yEc+okQioY997GPj/asAAJPYuP8J7uDBg/rYxz6m/v5+zZgxQx/4wAe0bds2zZgxw9WnULTH5sRkj58o5H2RKZmsI9oiZo8dkeR6xV+xmHH1rqu1x+uMjPhiYSoVX31DgyPSpjrm6p0bSZprsyd9US+SPeqltqbR1TmW9MUf1dfb43LyI7YIq9fV1Ni3M16x729JSlTZI6Qap9m3UZJKskcOZQq+86q+qc1Vn2roMtcWcr7jk3PUF/t9UVbZjP02K5ezx/ZYa8d9AD322GPj3RIAMAWRBQcACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACGLCP47hTLW2XKS6OlvmWHNDnblvOebL4GrrtGcllUr2rCRJ2vPqb821mWOHXL0l+0dcJKqnuzrXNs1y1XfPnmeujVfZc+Mk6ehJe/ZV3aAvJ6uxwb4PUzWOzEBJQ1nfJ/82N9jP25wjw06SKhXHdaLky1TLDNnP23i8z9W7Ujxurs07cswkqVjvOw+LlQZz7ckhez6eJB05Zs+8iyJfZmRzY5O5Nha3ryMWt+XX8QgIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABDEeRvFk0jOViJpi9iZ3jHb3Le6vta1jpGoYq4tFrKu3tm8PeqlWLLHDUlSstoemZKqaff1rpnjqk9UX2hfS509GkSSOlL2+1CtMyJX71TSHlEzUvId+0Ri0FXfOt1+3iZqfed4ccQWmyJJxWze1bvvkC1OS5JGir51K95o7y1fVFJ1te86MVKZZq4tlXxRSfmsvd53hkvTp9lvV+pq7edJzPjYhkdAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCDO2yy437w2pJraEVPt0cieq9VYZ+v5upF8zlybGU67evf12Wv7c82u3omcPRWqVLLnxklS//Hjrvrf77bvw2nTZ7h6z+joNNeW7bF+kqTiSNFcmy8VXL2Pnzzpqh8YtJ9bdUl7xqAklUv27SyWfNefUtGekTerq9XVu7P9AnNtPuPLsMum7blnklTMJu29h33Xt5G8/bpcW+97TNHU0GCurWpwbGPWdp3nERAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgiPM2C25376CqU7Z8rV8fs+dkpZx5YPnj9swub0baSMW+mCjmu69QnbQf2tp4xtU7KvhytRSzZ1+1z2x3tZ59of34RLJnaknSyLA9w86bBXd04ISrfuiEvb5xxHeuVPKOLLgqX++GVnuGYVPLla7eF9bPM9fmc0Ou3r29h1z1+az9+A8XfOfKUL5krm2JEq7epYr9dqL7wi5z7fCwLQOQR0AAgCDcA+iFF17QDTfcoK6uLsViMT355JNjfh5Fkb74xS+qs7NTtbW1Wrp0qV599dXxWi8AYIpwD6BMJqOFCxdqw4YNp/z5fffdp29961t64IEH9OKLL6q+vl7Lli1TPu/8sw0AYEpzPwe0YsUKrVix4pQ/i6JI999/vz7/+c/rxhtvlCR997vf1cyZM/Xkk0/qox/96NmtFgAwZYzrc0D79u1Tb2+vli5dOvq95uZmLV68WFu3bj3l/ykUCkqn02MuAICpb1wHUG9vryRp5syZY74/c+bM0Z+90fr169Xc3Dx66e7uHs8lAQDOU8FfBbdu3ToNDg6OXg4cOBB6SQCAc2BcB1BHR4ckqa+vb8z3+/r6Rn/2RqlUSk1NTWMuAICpb1wH0Ny5c9XR0aFNmzaNfi+dTuvFF19UT0/PeP4qAMAk534V3PDwsPbs2TP69b59+7Rz5061trZq9uzZuvvuu/VP//RPuuSSSzR37lx94QtfUFdXl2666abxXDcAYJJzD6Dt27frQx/60OjXa9eulSStWrVKDz/8sD7zmc8ok8no9ttv18DAgD7wgQ/omWeeUU1Njev3DBcqqpYtquZYvz2mJFEYca3jxIHD5tpMxh7dIkl1zS3m2vrmaa7exXjKXlvtjPmpaXTVS2VzZX9VtatzQ5U95qelscHVu2N6q7m2mLVFj7xuemebq76pyhGx0jvo6l00xqZIUm/ZfiwladBRfzztu26+ut9+vR/J+mKYSnHfUwG5mD3O6ljGd65k7ElJyg/6ssYqfzhqrm3sskcfZYq266V7AF177bWKotMfzFgspq9+9av66le/6m0NAHgHCf4qOADAOxMDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEIQ7iudcGTjWq6qkLc8sVrGHJSXjvqyxZLU9a0yNta7ecvSO1/gOVbLWngWX9Oav1SZd9TVJe47ZtFZfzlx7e7u59pILZ7l6/z+O+lx6wNW7Ounb5+31debaI7/+nat33+FTf1jkqSRO+nLmSieHzbUH+o+5er968H/MtcW0I1BNUk3Cvr8lKXJE9R085tvOXMmeY9c24suMzFfZr2+/2G1fd96YjcgjIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEOdtFM8FrY2qTtWYanMDJ819jx3vc62jqd4er1NydZYK5bK5NlbO+JoXK/beJV+0TirmiCeS1NVijweZ097l6t2Ysh+fhDOO5fixfntxVHD1rvEtRUOlvLl2pNp+XknSoSH7deIP+w+4eqdlj4SqVDvybCTlRuz7JJOzRwJJUnXku77VOmK4qup921kasp8sBedjivqWTnNt0wz7dbM6Y9t/PAICAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABHHeZsGVM1nFS7ZMq6GT9syuvoO+LKtEtS2PTpJqGhpcvZN1debaWM6Xv1bK23Oy4qlGV+/q1jZXfaLGnpN18MgxV+9CLmuuLZdHXL2Lsue71TivSTMd+0SSujtmmGsHcwOu3r/87e/MtXteO+TqPVy275i44/ogSfGEPfOuIWG/HktSR3uHq37WBfb6fMWe0yhJh4/ab99qa337MCF7zlx52H5dq2RttTwCAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEcd5G8QwNnFRVddJUe/jIEXPfXDbnWkey2l47d/aFrt4XXXKxufa1gwddvU9k7NtZ32mPeZGkcrevvjdhv59z9NAJV+/+o/bonnjkizMql+1RL9HgsKv3nHpf/NFgxr4Pd+x9xdU7XbJHrORLvn1Y29Biru2+5CJX78gRI7P/D//j6r17t68+4dgv7Z0zXb0vmN5qrq2q8sX8JJQ211YG7DFmlZzt9odHQACAIBhAAIAg3APohRde0A033KCuri7FYjE9+eSTY35+6623KhaLjbksX758vNYLAJgi3AMok8lo4cKF2rBhw2lrli9friNHjoxeHn300bNaJABg6nG/CGHFihVasWLFW9akUil1dPg+TwMA8M4yIc8Bbd68We3t7Zo/f77uvPNO9fef/gOVCoWC0un0mAsAYOob9wG0fPlyffe739WmTZv0L//yL9qyZYtWrFhx2pe0rl+/Xs3NzaOX7u7u8V4SAOA8NO7vA/roRz86+u+rrrpKCxYs0EUXXaTNmzdryZIlb6pft26d1q5dO/p1Op1mCAHAO8CEvwx73rx5amtr0549e07581QqpaampjEXAMDUN+ED6ODBg+rv71dnZ+dE/yoAwCTi/hPc8PDwmEcz+/bt086dO9Xa2qrW1lZ95Stf0cqVK9XR0aG9e/fqM5/5jC6++GItW7ZsXBcOAJjc3ANo+/bt+tCHPjT69evP36xatUobN27Url279O///u8aGBhQV1eXrr/+ev3jP/6jUqmU6/d8+P99j2prak21//HkYXPf7DFfZldlxP6qvESx3dW7pd6++4dm+l7WHlUS5trGOb5Hp81dvhyzVJUt00+SsrlBV+9sxn5eFTIFV++yRuy18byr9wWX+nLPprW3mGuHX/ZtZ6FsPw8rke96PFIo2dcx5NuHNfV15trqVLOr98mTfa76IwPHzbUzZl/g6p2I249PZvioq/fBvbvMtbFL55hrC3nbOegeQNdee62iKDrtz5999llvSwDAOxBZcACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIMb984DGy+9f3q5k0pYhNnD8iLlvZSTnWkdtnT376mjvAVfvI4fs9bUz7DlMklQre/5a5fTJSqeUkO8/FIbt+XuZgQFX75FMxlybG/B92m6hYK+vjRddvY/1nfrjSU4nfdx+riSKWVfvkaw9r22k5NvO4UH7sR9M24+lJFWlbFmRkpQv2nP9JCka8dWni/Z9eGzAl0eZy9hvs2Jl3z5MxO37sDZZb1+HMYuSR0AAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCDO2yiefb2HVVVlW16uyj5Hp3f7Im36jx8319aq2tX7gu655trpF17q6t1fjplrs9W+0yCTL7jqB48PmGujki/mp1S0R8OkHOeJJM2a3mmurYvnXb3rE/bjI0nvvmK+ufaWm1e6ev9/z/3cXLvtV7929a7IHjtTU1fn6t3Q1GKuratvcPWuS9kjuCQpXrbXHth/zNU7l7VH8cQrvhimSy/sMte+b/EHzbUZY0QWj4AAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQZy3WXB96bwSxiy4rGrMfTvnvMu1ju7L6s21F3TYs8MkqaHdnktXjiVdvROOuxa5QV9+1LETJ1z1+Yw9Dyw+4mqtxtpac21VjS/fK1GxZ96lT5x09e66sNtV39rSZq4dSvuOZ0N9o7l2/nxfJmEUt5+I9Y1Nrt71jny3VI39PJEkjVRc5Sf6+s21h3J9rt4JR25gPO67SU+m7Psl5biujVRs4Xg8AgIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABHHeRvHMuujdqk7aolMubphu7jvnsitc66h1xINUxxKu3kXZIzZODuVdvYfy9vpsvujqXeUrV03Ffj+n7LxPNDJiz+45edwXgZLuP2auLQynXb1ntLa76g8ctfcfkS+Kp6bBfo7Pa7DH30jSSLlkrm1qtEcCSVKDo74q4btuRmVblMzr6uL2+hPHD7p6DwzYY35qU77rTzZrj9Xq6z3g6Gs7B3kEBAAIwjWA1q9fr/e+971qbGxUe3u7brrpJu3evXtMTT6f1+rVqzV9+nQ1NDRo5cqV6uvz3fMEAEx9rgG0ZcsWrV69Wtu2bdNzzz2nUqmk66+/XplMZrTmnnvu0dNPP63HH39cW7Zs0eHDh3XzzTeP+8IBAJOb6zmgZ555ZszXDz/8sNrb27Vjxw5dc801Ghwc1IMPPqhHHnlE1113nSTpoYce0rve9S5t27ZN73vf+8Zv5QCASe2sngMaHByUJLW2tkqSduzYoVKppKVLl47WXHbZZZo9e7a2bt16yh6FQkHpdHrMBQAw9Z3xAKpUKrr77rv1/ve/X1deeaUkqbe3V8lkUi0tLWNqZ86cqd7e3lP2Wb9+vZqbm0cv3d2+D+oCAExOZzyAVq9erZdfflmPPfbYWS1g3bp1GhwcHL0cOGB/qR8AYPI6o/cBrVmzRj/60Y/0wgsvaNasWaPf7+joULFY1MDAwJhHQX19fero6Dhlr1QqpVTK91HJAIDJz/UIKIoirVmzRk888YR+8pOfaO7cuWN+vmjRIlVXV2vTpk2j39u9e7f279+vnp6e8VkxAGBKcD0CWr16tR555BE99dRTamxsHH1ep7m5WbW1tWpubtanPvUprV27Vq2trWpqatJdd92lnp4eXgEHABjDNYA2btwoSbr22mvHfP+hhx7SrbfeKkn6xje+oXg8rpUrV6pQKGjZsmX6zne+My6LBQBMHa4BFEXR29bU1NRow4YN2rBhwxkvSpLmzL1SyZpaU211XYu5b1m+55v6B3Pm2kKu4OpddGSw5fO+fK9cwV4fjdjzuiQpVvLVF/ND5trBE77UjP7jR8y1w+mTrt7FYsVcG3fmAB49Oexbi+Oqmqz1ZarVGTMXJSke8x37qGw/x2trfdfNVMqepRiL7MdSkmIxX306Zr+dGB70vdBq4Li9vrptmqv34An77cT/7PmduTZvzKIkCw4AEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMQZfRzDuTBSLis+UjbVDvUfN/ctVOzRIJI0UrFHcuTzvpiSvCO6p+yINJGk2hr7oa2u+NZ9su/UHy54On29h8y1w4MnXL2LBXukTbns287q6hpzbW1Ds6t3VBlx1cfsqTMqj/gioQple32llHH1TjoSirwxWbms7fZBkkoF3z5R2Xd8+g/90Vx7dP9eV+/0iWPm2rqE7xxvSc4w1w54bmeN+5tHQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgzuMsuIziZVvWUyJun6Mz66td62ibNs1ce+zYgKt3FNWaa5uaGly9m5vsvQuZIVfvV4oDrvq+/7Hnu40M97t6V8cjc22iYs8OkySV7LlaVSVHWJuk6nLWVZ9L99lrHRmDklRx5J6NFHOu3qkqexhcfZ0vC67ascvjsmc6SlI04qvvP3TYXDvc78s7VNF+HhaGfFl9sdY2e/GIY4cba3kEBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAI4ryN4knE8koYkx+aGxrNfWdNb3KtY+4FM821f4zZI00kqaml1VzbcUGnq3cU2eM7eg8ddPXuba5x1TfU2O/nDKno6p1w3IeKx5xxLJH9eJYyvniVojNyaKDvNXPtyIgvciiSPc4oVeW7yUgk7PE6sbJ9HZJUVW2P1UpV2yOBJClT8MUZpQfS5tqo5DsPUwnHPnce+0Iub65tqLfHgVVV2Y4Nj4AAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQZy/WXDRiBLGLK6ahD3/qEq+vLa+QwfMtdnhQVfvaW0t5tpS2Z7ZJEl9x46aa1/9w+9dvQ8esueSSVK5Ys+lS6bs+V6S5IgxU9kXkyXP/bNK5GteKuZc9QnZ88OSKV/uWRTZd2JzY72rd319nbk26ch2k6REwn58Yp4TRdLxfl+2Xzpjz4KrTvqOj+L27ayt9e3Djs7p5tpL5s8z12aztvObR0AAgCBcA2j9+vV673vfq8bGRrW3t+umm27S7t27x9Rce+21isViYy533HHHuC4aADD5uQbQli1btHr1am3btk3PPfecSqWSrr/+emUymTF1t912m44cOTJ6ue+++8Z10QCAyc/1HNAzzzwz5uuHH35Y7e3t2rFjh6655prR79fV1amjo2N8VggAmJLO6jmgwcE/Pene2jr2g9W+973vqa2tTVdeeaXWrVunbDZ72h6FQkHpdHrMBQAw9Z3xq+AqlYruvvtuvf/979eVV145+v2Pf/zjmjNnjrq6urRr1y599rOf1e7du/XDH/7wlH3Wr1+vr3zlK2e6DADAJHXGA2j16tV6+eWX9bOf/WzM92+//fbRf1911VXq7OzUkiVLtHfvXl100UVv6rNu3TqtXbt29Ot0Oq3u7u4zXRYAYJI4owG0Zs0a/ehHP9ILL7ygWbNmvWXt4sWLJUl79uw55QBKpVJKpeyfGw8AmBpcAyiKIt1111164okntHnzZs2dO/dt/8/OnTslSZ2dnWe0QADA1OQaQKtXr9Yjjzyip556So2Njert7ZUkNTc3q7a2Vnv37tUjjzyiD3/4w5o+fbp27dqle+65R9dcc40WLFgwIRsAAJicXANo48aNkv70ZtP/20MPPaRbb71VyWRSzz//vO6//35lMhl1d3dr5cqV+vznPz9uCwYATA3uP8G9le7ubm3ZsuWsFjRqpCglYqbS+qR9M4q5078k/FRe+b09Jy1ftGeeSVKlyv4q+OPpAVfv1w7YM+x+//JvXL179/uy4Eqlorm2ts6eHSZJIyP2bL9y3penF1Uc71KI+d7REI/7nn6dNq317YteX4qr85/eCmFv7tvOfN7eO5vzHZ9K2Z6P5zkHJeng4SOu+oGsPTuuqtZ3hJLJpLn2XVde5uq9/MPLzbVXXHG5uXZ4eNhURxYcACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACCIM/48oAk3kpMStqiNjulN5raZQd8nrvYfP2quPXjEF9+RK9ljShpaprl69x21r/t4X5+rd6Hgi0yJucNh7KKKPY6lpqbG1/ttoqf+b7msb5+UR+y9JWkonTHXVlUlXL1LJXuE1KBjHZIviscTqyRJUWQ/9q64IUm9vQdd9SfT9iiemPPq0NDcYK79s6sXu3pfftW7zbWRI4YpStiOJY+AAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEGct1lwHdObzNldF8/pMvc9dNAXxJSI2zO7Tp447uo9Ui6ba5taW129szl7Nlk+68v3ihzrlqRY3H4/p+zIdpOkSsW+lvraWlfvmOP+WXY45+pdKvlyzw7uP2Cu9Waqec6VXMGeGyf5zvFE3JdhV1tnP56e3DhJOjk47KofGs6aa6uqfTe79Xn7Pjx8uN/V+79//itzbbI6aa7N5Wz7g0dAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgztsonj1/eEXJZLWp9rW9vzP3zQzbY0ckqe+oPdqiUi66epcK9viW/PCQq3fZEfUSK/uiW7xRPGVHvSdaR5LKjtgZb+SQIvv9sypH3JAkZYfSrvrf/+635tpS0XceliN7PFVFviirWMwer+ONqEkk7L0956AkDQ0PuupLBXttImaPtJGk7LD9HP/ZC7909d7xC/t5VVNjX/dIyRbZxCMgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBDnbRZcsZhXFNnym37z61+Z++Zyvpysqqpac22yyjfPE7LnUw2dOO7qXSjaspgkKYoiV++KMzuuZMyF+tNaKq7enqUXneuuOJYSOfLUJKn/+FFX/Ygj884rHrdnqkUx731W+36JJXz7sCphv/mqqvLd1KUSvu2saWwx19bVNrh6t81ot9e22WslqaVpmrm2dXqzubZQyOv5Z9++jkdAAIAgXANo48aNWrBggZqamtTU1KSenh79+Mc/Hv15Pp/X6tWrNX36dDU0NGjlypXq6+sb90UDACY/1wCaNWuW7r33Xu3YsUPbt2/XddddpxtvvFGvvPKKJOmee+7R008/rccff1xbtmzR4cOHdfPNN0/IwgEAk5vrD6M33HDDmK//+Z//WRs3btS2bds0a9YsPfjgg3rkkUd03XXXSZIeeughvetd79K2bdv0vve9b/xWDQCY9M74OaByuazHHntMmUxGPT092rFjh0qlkpYuXTpac9lll2n27NnaunXrafsUCgWl0+kxFwDA1OceQL/5zW/U0NCgVCqlO+64Q0888YQuv/xy9fb2KplMqqWlZUz9zJkz1dvbe9p+69evV3Nz8+ilu7vbvREAgMnHPYDmz5+vnTt36sUXX9Sdd96pVatW6be/tX+s6xutW7dOg4ODo5cDBw6ccS8AwOThfh9QMpnUxRdfLElatGiRfvnLX+qb3/ymbrnlFhWLRQ0MDIx5FNTX16eOjo7T9kulUkqlUv6VAwAmtbN+H1ClUlGhUNCiRYtUXV2tTZs2jf5s9+7d2r9/v3p6es721wAAphjXI6B169ZpxYoVmj17toaGhvTII49o8+bNevbZZ9Xc3KxPfepTWrt2rVpbW9XU1KS77rpLPT09vAIOAPAmrgF09OhR/dVf/ZWOHDmi5uZmLViwQM8++6z+4i/+QpL0jW98Q/F4XCtXrlShUNCyZcv0ne9854wWtvJ//S/V1dWZah968H+b++7c+WvXOsrlrLm2usr3p8R4zB49Uir4oliGh4btvUfskUCSVDZGJI3Wl+313lggT8SK90+9McfxyWQyrt7+7bTH5SSTSVdvT31NTY2rt/U6LEkNDfWu3m98wdNbmT69zdW7tXW6q76hvtFcW+vYJ5LU2dlprp01a5ard1OTPV4nlbSfg8PDw9rw7Xvfts41gB588MG3/HlNTY02bNigDRs2eNoCAN6ByIIDAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAE4U7DnmivR5Rksznz/ymV7DE1lUrFtZ5KxR7H4u3tiajx1HrX4t4nka/eEzvjjaiZyO30RPF4130+baen3nseeupHRnxxU6VSyVxbLBZdvQuFgqu+uqraXBuL++7353L220JvJFQ8bo/XKTmieF5fx9ud57HIe02YYAcPHuRD6QBgCjhw4MBb5tOddwOoUqno8OHDamxsHHMPNJ1Oq7u7WwcOHFBTU1PAFU4stnPqeCdso8R2TjXjsZ1RFGloaEhdXV2Kv8UjvvPuT3DxePwtJ2ZTU9OUPvivYzunjnfCNkps51RzttvZ3Pz2Sdu8CAEAEAQDCAAQxKQZQKlUSl/60pfcHyo22bCdU8c7YRsltnOqOZfbed69CAEA8M4waR4BAQCmFgYQACAIBhAAIAgGEAAgiEkzgDZs2KALL7xQNTU1Wrx4sX7xi1+EXtK4+vKXv6xYLDbmctlll4Ve1ll54YUXdMMNN6irq0uxWExPPvnkmJ9HUaQvfvGL6uzsVG1trZYuXapXX301zGLPwttt56233vqmY7t8+fIwiz1D69ev13vf+141Njaqvb1dN910k3bv3j2mJp/Pa/Xq1Zo+fboaGhq0cuVK9fX1BVrxmbFs57XXXvum43nHHXcEWvGZ2bhxoxYsWDD6ZtOenh79+Mc/Hv35uTqWk2IAff/739fatWv1pS99Sb/61a+0cOFCLVu2TEePHg29tHF1xRVX6MiRI6OXn/3sZ6GXdFYymYwWLlyoDRs2nPLn9913n771rW/pgQce0Isvvqj6+notW7ZM+Xz+HK/07LzddkrS8uXLxxzbRx999Byu8Oxt2bJFq1ev1rZt2/Tcc8+pVCrp+uuvHxN+ec899+jpp5/W448/ri1btujw4cO6+eabA67az7KdknTbbbeNOZ733XdfoBWfmVmzZunee+/Vjh07tH37dl133XW68cYb9corr0g6h8cymgSuvvrqaPXq1aNfl8vlqKurK1q/fn3AVY2vL33pS9HChQtDL2PCSIqeeOKJ0a8rlUrU0dERfe1rXxv93sDAQJRKpaJHH300wArHxxu3M4qiaNWqVdGNN94YZD0T5ejRo5GkaMuWLVEU/enYVVdXR48//vhoze9+97tIUrR169ZQyzxrb9zOKIqiP//zP4/+9m//NtyiJsi0adOif/3Xfz2nx/K8fwRULBa1Y8cOLV26dPR78XhcS5cu1datWwOubPy9+uqr6urq0rx58/SJT3xC+/fvD72kCbNv3z719vaOOa7Nzc1avHjxlDuukrR582a1t7dr/vz5uvPOO9Xf3x96SWdlcHBQktTa2ipJ2rFjh0ql0pjjedlll2n27NmT+ni+cTtf973vfU9tbW268sortW7dOmWz2RDLGxflclmPPfaYMpmMenp6zumxPO/CSN/o+PHjKpfLmjlz5pjvz5w5U7///e8DrWr8LV68WA8//LDmz5+vI0eO6Ctf+Yo++MEP6uWXX1ZjY2Po5Y273t5eSTrlcX39Z1PF8uXLdfPNN2vu3Lnau3ev/uEf/kErVqzQ1q1blUjYP2PlfFGpVHT33Xfr/e9/v6688kpJfzqeyWRSLS0tY2on8/E81XZK0sc//nHNmTNHXV1d2rVrlz772c9q9+7d+uEPfxhwtX6/+c1v1NPTo3w+r4aGBj3xxBO6/PLLtXPnznN2LM/7AfROsWLFitF/L1iwQIsXL9acOXP0gx/8QJ/61KcCrgxn66Mf/ejov6+66iotWLBAF110kTZv3qwlS5YEXNmZWb16tV5++eVJ/xzl2znddt5+++2j/77qqqvU2dmpJUuWaO/evbrooovO9TLP2Pz587Vz504NDg7qP/7jP7Rq1Spt2bLlnK7hvP8TXFtbmxKJxJtegdHX16eOjo5Aq5p4LS0tuvTSS7Vnz57QS5kQrx+7d9pxlaR58+apra1tUh7bNWvW6Ec/+pF++tOfjvnYlI6ODhWLRQ0MDIypn6zH83TbeSqLFy+WpEl3PJPJpC6++GItWrRI69ev18KFC/XNb37znB7L834AJZNJLVq0SJs2bRr9XqVS0aZNm9TT0xNwZRNreHhYe/fuVWdnZ+ilTIi5c+eqo6NjzHFNp9N68cUXp/Rxlf70qb/9/f2T6thGUaQ1a9boiSee0E9+8hPNnTt3zM8XLVqk6urqMcdz9+7d2r9//6Q6nm+3naeyc+dOSZpUx/NUKpWKCoXCuT2W4/qShgny2GOPRalUKnr44Yej3/72t9Htt98etbS0RL29vaGXNm7+7u/+Ltq8eXO0b9++6Oc//3m0dOnSqK2tLTp69GjopZ2xoaGh6KWXXopeeumlSFL09a9/PXrppZei1157LYqiKLr33nujlpaW6Kmnnop27doV3XjjjdHcuXOjXC4XeOU+b7WdQ0ND0ac//elo69at0b59+6Lnn38+es973hNdcsklUT6fD710szvvvDNqbm6ONm/eHB05cmT0ks1mR2vuuOOOaPbs2dFPfvKTaPv27VFPT0/U09MTcNV+b7ede/bsib761a9G27dvj/bt2xc99dRT0bx586Jrrrkm8Mp9Pve5z0VbtmyJ9u3bF+3atSv63Oc+F8Visei//uu/oig6d8dyUgygKIqib3/729Hs2bOjZDIZXX311dG2bdtCL2lc3XLLLVFnZ2eUTCajCy64ILrllluiPXv2hF7WWfnpT38aSXrTZdWqVVEU/eml2F/4wheimTNnRqlUKlqyZEm0e/fusIs+A2+1ndlsNrr++uujGTNmRNXV1dGcOXOi2267bdLdeTrV9kmKHnroodGaXC4X/c3f/E00bdq0qK6uLvrIRz4SHTlyJNyiz8Dbbef+/fuja665JmptbY1SqVR08cUXR3//938fDQ4Ohl2401//9V9Hc+bMiZLJZDRjxoxoyZIlo8Mnis7dseTjGAAAQZz3zwEBAKYmBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgiP8fHkLuy/+P5K0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAArtklEQVR4nO3de3DV9Z3/8ddJSE5CLieEkBs3uQkiBLcsYsbKUqFcdpYBZbrYOrPYujq6wVllu23Zqde9xLWzrW2Xwh/rQjtTpNUpWN0WK2jCdgUUBBFdImAgYG5ySU7uCcn394djfhsB+bxDDp8kPB8zZ4bkvHjn8z3f7znvfHPOeZ9QEASBAAC4yuJ8LwAAcG2iAQEAvKABAQC8oAEBALygAQEAvKABAQC8oAEBALygAQEAvBjiewGf19XVpcrKSqWlpSkUCvleDgDAKAgCNTQ0KD8/X3Fxlz7P6XcNqLKyUqNHj/a9DADAFTp58qRGjRp1yetj1oDWrl2rH/zgB6qurtaMGTP005/+VDfffPNl/19aWpokqaKiQunp6U4/KxqNOq/r2LFjzllJOn/+vHO2sbHRVNtiyBDbrjpx4oRz9ty5c6bas2bNMuWzs7NNeQvL2pubm021I5GIc/aLfsu7mJycHFN+xIgRpjx6sh7j7e3tpnxnZ2fMaluO26SkJFNtyyS2yspK52xzc7NWrFjR/Xh+KTFpQL/61a+0evVqrV+/XrNnz9azzz6rhQsXqqys7LIPRp/92S09Pd25AVmkpqaa8h0dHc7ZWI7Vszag5ORk52xLS4updkpKiil/uYPwSlj2j/VPupZjxdqArLdJLO4L1xJLg5Cktra2mNW3NiDLsRXLBmS930uXv8/F5EUIP/zhD3Xffffpm9/8pqZOnar169dr6NCh+s///M9Y/DgAwADU5w2ovb1d+/bt0/z58///D4mL0/z587Vr164L8m1tbYpGoz0uAIDBr88b0OnTp9XZ2XnB37hzcnJUXV19Qb64uFiRSKT7wgsQAODa4P19QGvWrFF9fX335eTJk76XBAC4Cvr8RQhZWVmKj49XTU1Nj+/X1NQoNzf3gnw4HFY4HO7rZQAA+rk+PwNKTEzUzJkztWPHju7vdXV1aceOHSosLOzrHwcAGKBi8jLs1atXa+XKlfrTP/1T3XzzzXr22WfV1NSkb37zm7H4cQCAASgmDWjFihX65JNP9Nhjj6m6ulo33XSTtm3bZn7zHQBg8IrZJIRVq1Zp1apVvf7/QRDE5I2d1jeBWd7oGEuWiQySdP311ztny8rKTLVLSkpMectEgby8PFPtYcOGOWeHDh1qqh0fH++ctR4nra2tprzF2bNnTXnLlADrdlqe37X+gmq5L586dcpU23qsWN4san2Ta0NDg3PW+oZ1i48//tg56/rmdu+vggMAXJtoQAAAL2hAAAAvaEAAAC9oQAAAL2hAAAAvaEAAAC9oQAAAL2hAAAAvaEAAAC9iN7fhCjU3NzuPlbCMTElMTDStwzIOyDrmx1LbMupDso3kSE5ONtW2jG6RbCM8Tp8+bao9YcIE52xWVpapdnp6uilvYR2tVFFR4Zytqqoy1baMhrHefyz3zaamJlPt2tpa56z1vmkdC2S9D1lkZGTErHZXV5dzNikpqc/rcgYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8KLfzoJrbW1VQkKCU9Y1J0mpqammdYTDYeesdR5UY2OjKW9huU0ss8Ak+8wuy3aGQiFT7aFDhzpnLbOsrDo7O015ywwuSSorK3POWucGRqNR52xeXp6ptmU7T506ZapdWVnpnLXMpOtN3jKvzXqsDBs2zDlrXbdlRp5lPp7rYwRnQAAAL2hAAAAvaEAAAC9oQAAAL2hAAAAvaEAAAC9oQAAAL2hAAAAvaEAAAC9oQAAAL/rtKJ7z58/r/PnzTlnLuA/LiBrJNl7HOkYmCALnbENDg6m2ZS2TJk0y1baOErGM+rGOErGM4rGOkXE9/qzrkOzjciwjofLz8021LeNYrFpaWpyzHR0dptqW48o69sp6jFvy1scJy/gwy2OKleWx0PX24AwIAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4EW/nQWXkJDgPLfNMm/KOofJMpssKSnJVNsy3+vs2bOm2pZZYyNHjjTVvummm0x56/w9C8v8MMtsN8l2rMR6Fpxlvpv1OLSwzkirqqpyzlpn0tXU1JjyFhMmTDDlZ8yY4Zy13h8sj0GWuZiS1Nra6py17HvX2ZWcAQEAvOjzBvTEE08oFAr1uEyZMqWvfwwAYICLyZ/gbrzxRm3fvv3//5Ah/fYvfQAAT2LSGYYMGaLc3NxYlAYADBIxeQ7oyJEjys/P1/jx43X33XeroqLiktm2tjZFo9EeFwDA4NfnDWj27NnauHGjtm3bpnXr1qm8vFy33XbbJV8VUVxcrEgk0n0ZPXp0Xy8JANAP9XkDWrx4sb72ta+poKBACxcu1O9+9zvV1dXp17/+9UXza9asUX19fffl5MmTfb0kAEA/FPNXB2RkZOj666/X0aNHL3p9OBw2vR8GADA4xPx9QI2NjTp27Jjy8vJi/aMAAANInzegb3/72yotLdXx48f15ptv6o477lB8fLy+/vWv9/WPAgAMYH3+J7hTp07p61//us6cOaMRI0boy1/+snbv3q0RI0aY6oTDYeeRIidOnHCum5iYaFpHRkaGczY5OdlUe/jw4c5Z63upLNuZnp5uqp2SkmLKx5J1fw5UsRyvY2EZCyNJo0aNcs5aj/Gmpibn7Pjx4021b7nlFlN+oD6NYBkhZRll5Xqc9HkD2rx5c1+XBAAMQsyCAwB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4EfOPY+gty8c0WGaZNTY2mtaRlpbmnLXOsrLMgsvMzDTVtqylq6vLVLuystKUt6zdOtstLs79d6i2tjZT7YE636s/OXPmTEyykm02mfW+ea0cK2fPnnXOWmZuuj7OcgYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCi347iGTJkiPP4DMtomJMnT5rWYcmnpqaaag8bNsw5e9NNN5lqW9TX15vyP//5z035srIy56xlNIgk5efnm/IWBQUFztkpU6aYaqekpJjyWVlZzlnLeCJJeu2115yz+/fvN9W2yMjIMOXb29uds83NzabaW7duNeUta58+fbqp9tixY52zHR0dptqW+9vp06edsy0tLU45zoAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXvTbWXBvvfWW87ysaDTqXDcUCpnWkZiY6JwdMWKEqXZOTo4pHyt1dXWmfHZ2tilvmWPW1dVlqp2ZmemcHT9+vKn2+fPnnbPW2W7WuYFtbW3OWes8sNbWVues9Rh3neco2fe9hfU2OXHihCm/a9cu5+zBgwdNtWfOnOmctdzXJOm6665zzlpmDLpmOQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeNFvZ8EdPXpUycnJTtlPPvnEue4HH3xgWsfQoUOds4WFhabalvle1nltlplQlnlqkjRt2jRTfvLkyc7ZjIwMU+1rhWUu3UcffWSqPWvWLOdsbW2tqXYQBM7Zs2fPmmo3Nzc7Z2tqaky1XR97PmO5D1nnUVpuw7KyMlPt8vJy56zlfux6vHIGBADwwtyAdu7cqSVLlig/P1+hUEhbt27tcX0QBHrssceUl5en5ORkzZ8/X0eOHOmr9QIABglzA2pqatKMGTO0du3ai17/zDPP6Cc/+YnWr1+vPXv2KCUlRQsXLjSNfAcADH7m54AWL16sxYsXX/S6IAj07LPP6vvf/76WLl0qSfrFL36hnJwcbd26VXfdddeVrRYAMGj06XNA5eXlqq6u1vz587u/F4lENHv27Et+YFNbW5ui0WiPCwBg8OvTBlRdXS3pwk/6zMnJ6b7u84qLixWJRLovo0eP7sslAQD6Ke+vgluzZo3q6+u7LydPnvS9JADAVdCnDSg3N1fSha+5r6mp6b7u88LhsNLT03tcAACDX582oHHjxik3N1c7duzo/l40GtWePXvMb9IEAAxu5lfBNTY26ujRo91fl5eX68CBA8rMzNSYMWP08MMP65/+6Z80adIkjRs3To8++qjy8/O1bNmyvlw3AGCAMzegvXv36itf+Ur316tXr5YkrVy5Uhs3btR3vvMdNTU16f7771ddXZ2+/OUva9u2bUpKSjL9nMmTJys1NdUp+1//9V/OdS3jOyQ5r0GS+Q23p0+fds6mpKSYaltG2iQmJppqW8eanDt3zjn7f19B6WLIkH47TapPWbZz4sSJptqWsU2dnZ2m2h9++KFz1jpuqqOjwzlrfR+idRTPhAkTnLPx8fGm2qdOnXLOjh8/3lR71KhRztlLPY1yMU1NTU4587137ty5XzibKBQK6amnntJTTz1lLQ0AuIZ4fxUcAODaRAMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB40W8HacXHxzvPTJo6dapz3WHDhpnW8aUvfck5Gw6HTbXfeecd5+zBgwdNtf/vRPLLsc69amhoMOUt9a3zwGL5Me+u86wk+21inU1myYdCIVNtyyxA6/3HMgOyqqrKVNsy19E6A/KTTz6J2Vqsn/rc1tbmnLXOdbR8/I3l8c319uAMCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgRb8dxRMKhZxHiowdO9a57ogRI0zrSElJcc5++OGHptpvvfWWc9Y6XmXWrFkxq/3mm2+a8rW1tc7ZPXv2mGpbRtRYxipJtu2sqakx1c7MzDTlLSNWrPtz0qRJztlIJGKqPX78eOfsmTNnTLXffvtt52xubq6p9syZM035nTt3OmcPHz5sqt3S0uKc3b9/v6l2QkKCc9YyJsv1fskZEADACxoQAMALGhAAwAsaEADACxoQAMALGhAAwAsaEADACxoQAMALGhAAwAsaEADACxoQAMCLfjsL7ne/+53C4bBT9qtf/apz3eHDh5vW0dXV5Zx99913TbVPnTrlnL3hhhtMtUePHu2c/eSTT0y1gyAw5S3OnTtnyh86dMg5e+utt5pqZ2RkOGf37t1rqm2d12aZBZeWlmaqHRfn/nuodd2WY8U6q6+pqck5a50Fl5+fb8q/8cYbzlnrzDuLIUNsD+l/8Rd/4Zy13IaNjY36l3/5l8vmOAMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHjRb0fxnDp1ynn8yIkTJ5zrDhs2zLSO1NRU52xNTY2pdnNzc8xq//d//7dz1jqKp6qqypRPSEhwzlpH8cTHxztnjx8/bqpt2T+WsTCSVFlZacqnp6c7Z5OTk021Ozs7nbOW21uS8zgtyXZ7S9LUqVNjsg7J9pgiSUePHnXOdnR0mGpbWB8nWlpanLOTJk1yzkajUaccZ0AAAC9oQAAAL8wNaOfOnVqyZIny8/MVCoW0devWHtffc889CoVCPS6LFi3qq/UCAAYJcwNqamrSjBkztHbt2ktmFi1apKqqqu7L888/f0WLBAAMPuYXISxevFiLFy/+wkw4HDZ//gYA4NoSk+eASkpKlJ2drcmTJ+vBBx/8wg9gamtrUzQa7XEBAAx+fd6AFi1apF/84hfasWOH/vVf/1WlpaVavHjxJV/qWVxcrEgk0n2xfJInAGDg6vP3Ad11113d/54+fboKCgo0YcIElZSUaN68eRfk16xZo9WrV3d/HY1GaUIAcA2I+cuwx48fr6ysrEu+USscDis9Pb3HBQAw+MW8AZ06dUpnzpxRXl5erH8UAGAAMf8JrrGxscfZTHl5uQ4cOKDMzExlZmbqySef1PLly5Wbm6tjx47pO9/5jiZOnKiFCxf26cIBAAObuQHt3btXX/nKV7q//uz5m5UrV2rdunU6ePCgfv7zn6uurk75+flasGCB/vEf/9E8iyknJ8f5/+zfv9+5bkpKinkdschK0pAh7jf/+fPnTbWrq6uds7GcYSdJ2dnZztnCwkJT7SVLljhnrdtpyWdmZppqW2eqNTQ0OGfPnj1rqm155enQoUNNtS33+6SkJFPtiRMnOmcjkYiptnXeoWWeYigUMtW23ObWff/b3/7WOWu5H7vORjQ3oLlz5yoIgkte/+qrr1pLAgCuQcyCAwB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB40eefB9RXcnJylJyc7JR98803neumpaWZ1tHR0eGc/ZM/+RNTbcvsuNbWVlNtyyy4ioqKmNWWpGHDhjlnly1bZqo9ZcoU56xldpikS36EyMW4zr76TFZWlilfW1vrnI2Ls/1eaZkzaJ1JaJl7Zv0oFst8N2vtOXPmmPKLFi1yzlqOK8n2mPXOO++Yah86dMg5u337dues6+MVZ0AAAC9oQAAAL2hAAAAvaEAAAC9oQAAAL2hAAAAvaEAAAC9oQAAAL2hAAAAvaEAAAC/67SieIAjU1dXllE1JSXGuaxkNItlGeFhHveTl5TlnreueNm2ac7a9vd1U+8yZM6Z8Z2enc9a6nQ0NDc7Z+Ph4U23X40+SSkpKTLVvvPFGUz43N9c5ax07Y9k/dXV1ptqW+2ZSUpKpdktLiylvYVm3JD3xxBPO2aqqKlPtw4cPO2ctx6z06eOsq7KyMues6wgzzoAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXvTbWXDl5eUKh8NO2Wg06ly3pqbGtI6RI0c6Z6+77jpT7dTUVOesZWaTNd/Y2Giq7bpfPhMX5/57jnU7a2trnbPW2WGW2VcHDhyIWW1Jmjx5snN25syZptpDhw51zlr3vetMMMk+Z86yPyORiKl2YmKiKW+ZM2h5TJFscwCnTp1qqv3RRx85Zw8dOuScbWtrc8pxBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8KLfjuL58MMPNWSI2/IsIzws4yQk25iSu+++21Q7PT3dOWsdURMKhZyzlpFAktTe3m7Kd3V1OWet25mQkOCcdT2ePmMZ22Qd3WIdC7Rnzx7n7LvvvmuqPWXKFOdsQUGBqXZ+fr5z1nocJicnO2ct9+Pe5C1rt44FSktLc85mZGSYat94443OWcvjVXNzs1OOMyAAgBemBlRcXKxZs2YpLS1N2dnZWrZs2QVDFVtbW1VUVKThw4crNTVVy5cvNw8ABQAMfqYGVFpaqqKiIu3evVuvvfaaOjo6tGDBAjU1NXVnHnnkEb388st64YUXVFpaqsrKSt155519vnAAwMBm+qP4tm3beny9ceNGZWdna9++fZozZ47q6+v13HPPadOmTbr99tslSRs2bNANN9yg3bt365Zbbum7lQMABrQreg6ovr5ekpSZmSlJ2rdvnzo6OjR//vzuzJQpUzRmzBjt2rXrojXa2toUjUZ7XAAAg1+vG1BXV5cefvhh3XrrrZo2bZokqbq6WomJiRe8EiMnJ0fV1dUXrVNcXKxIJNJ9GT16dG+XBAAYQHrdgIqKinTo0CFt3rz5ihawZs0a1dfXd19Onjx5RfUAAANDr94HtGrVKr3yyivauXOnRo0a1f393Nxctbe3q66ursdZUE1NzSU/VjYcDps/5hcAMPCZzoCCINCqVau0ZcsWvf766xo3blyP62fOnKmEhATt2LGj+3tlZWWqqKhQYWFh36wYADAomM6AioqKtGnTJr300ktKS0vrfl4nEokoOTlZkUhE9957r1avXq3MzEylp6froYceUmFhIa+AAwD0YGpA69atkyTNnTu3x/c3bNige+65R5L0ox/9SHFxcVq+fLna2tq0cOFC/exnP+uTxQIABg9TA3KZ05WUlKS1a9dq7dq1vV6UJMXHxys+Pt4p+/777zvXtc7g2r59u3P25ZdfNtWeNWuWczYlJcVUOzs72zlrnZF2+vRpU761tdU5a5l7Jcn5GJGkzs5OU23LPDDrfK/KykpT3rJ26zFumR334YcfmmovWrTIObt48WJT7Y6ODufsiRMnTLWts/2ysrKcs5bZlZJtOy0zICWZXnVs+SuW69tpmAUHAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCiVx/HcDW0tbU5jx9pb2+P2TrOnz/vnD18+LCp9oQJE5yz1vEdlpEpjY2Nptrnzp0z5S23oWX8jSSlpqY6Zz//QYmXU1NT45y1jnqJpa6urpjlLSOeJOmuu+5yzt5xxx2m2rFk2fdWOTk5pnxVVZVz1rpuy2PnsGHDnLOuI7I4AwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB40W9nwX388ceKi3Prj64z42KtoqLClA+Hw87Z8ePHm2q3trY6Z5uamky1rbPgamtrY5KVbGs/duyYqfbx48dN+WtBQkKCKX/mzBnnrOWYlaSkpCRT3sI6r806T9EiLS3NOXvy5ElTbcv+qaysdM42NDQ45TgDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB4QQMCAHhBAwIAeEEDAgB40W9H8XzrW99yHrXxb//2b851q6qqTOtwHQckSTU1NabaltE9I0aMMNWOj493ziYmJppqRyIRU769vd05e/78eVPtlJQU52xLS4uptmXf9yfWdVvydXV1ptovvviiczYUCplqf+1rX3POpqenm2q3tbWZ8m+//bZz1npfHjNmjHM2IyPDVLu6uto5++///u/OWdfbb2DewwAAAx4NCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgRb+dBbdq1Srn+U3hcNi57qOPPmpaR1NTk3P2yJEjptolJSXO2Wg0aqptmdeWnJxsqm2dHZeQkOCcHTZsmKm2ZRbcQJ3tZmWZAyjFdp7ewYMHnbNDhw411bbkb775ZlPtw4cPm/K//e1vnbOjR4821f7Lv/xL52wQBKbax48fd85u377dOdvZ2emUuzbukQCAfsfUgIqLizVr1iylpaUpOztby5YtU1lZWY/M3LlzFQqFelweeOCBPl00AGDgMzWg0tJSFRUVaffu3XrttdfU0dGhBQsWXPBnqvvuu09VVVXdl2eeeaZPFw0AGPhMzwFt27atx9cbN25Udna29u3bpzlz5nR/f+jQocrNze2bFQIABqUreg6ovr5ekpSZmdnj+7/85S+VlZWladOmac2aNWpubr5kjba2NkWj0R4XAMDg1+tXwXV1denhhx/WrbfeqmnTpnV//xvf+IbGjh2r/Px8HTx4UN/97ndVVlam3/zmNxetU1xcrCeffLK3ywAADFC9bkBFRUU6dOiQ/vjHP/b4/v3339/97+nTpysvL0/z5s3TsWPHNGHChAvqrFmzRqtXr+7+OhqNml+mCAAYeHrVgFatWqVXXnlFO3fu1KhRo74wO3v2bEnS0aNHL9qAwuGw6X08AIDBwdSAgiDQQw89pC1btqikpETjxo277P85cOCAJCkvL69XCwQADE6mBlRUVKRNmzbppZdeUlpamqqrqyV9+q775ORkHTt2TJs2bdKf//mfa/jw4Tp48KAeeeQRzZkzRwUFBTHZAADAwGRqQOvWrZP06ZtN/68NGzbonnvuUWJiorZv365nn31WTU1NGj16tJYvX67vf//7fbZgAMDgYP4T3BcZPXq0SktLr2hBvXHbbbc5Z0OhkKl2R0dHTLKSbU6WZZ6aJCUlJTlnreu2rsUyl851/l9v8l/0doCLsd4uFmlpaaa8ZcaXdTtd53b1RmNjo3PWOn/tD3/4g3PWOqfxgw8+MOXfeecd5+zSpUtNta+//npT3uKtt95yzn788cfO2a6uLqccs+AAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF70+vOAYm3Xrl1KSUlxyr755pvOda1jSmLp+PHjztnU1FRTbcvIobq6OlPt9vZ2U94yumfo0KGm2pZRPNaP/Yjlp/MmJiaa8pZxOa5jUD7T1tbmnI2PjzfVtqz73LlzptqW8TqfDU52ZR0LVFNT45y1Hle1tbXO2ezsbFNty31z+PDhztnOzk6n24QzIACAFzQgAIAXNCAAgBc0IACAFzQgAIAXNCAAgBc0IACAFzQgAIAXNCAAgBc0IACAFzQgAIAXoSAIAt+L+L+i0agikYgWLFjgPKfIMivp3XffNa3HOvcsViwzmyTbLDjr7DDLfC9J6i+HWFxc7H7fst6Glv0j9Z/b0MqynZa5fpKUlZXlnE1OTjbVts6Os8xTvOGGG0y1x40b55zNzMw01R4yxH0caFNTk3O2o6NDL774ourr679wv3IGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwggYEAPCCBgQA8IIGBADwwn0Ow1V2xx13OI/PqKiocK47ZswY0zoOHjzonD1z5oypdkdHh3O2tbXVVLu/jBDqT6zjcmJpoI7WsbJsZ319vam2JZ+WlmaqbR19ZdnOQ4cOmWq/9957ztm8vDxT7b/+6792zo4cOdI529LSohdffPGyOc6AAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF7QgAAAXtCAAABe0IAAAF6Egn42lCoajSoSiaiurk7p6em+l6MTJ044Z62zrKqrq52zR48eNdU+cuSIc9ayjZJUWVlpytfU1Dhnz507Z6odjUZNeVyb4uPjTflwOGzKNzc3m/IWcXHu5wmW2W6StH79eutynESjUWVkZKi+vv4LH8c5AwIAeGFqQOvWrVNBQYHS09OVnp6uwsJC/f73v+++vrW1VUVFRRo+fLhSU1O1fPly02+/AIBrh6kBjRo1Sk8//bT27dunvXv36vbbb9fSpUv1/vvvS5IeeeQRvfzyy3rhhRdUWlqqyspK3XnnnTFZOABgYDN9HtCSJUt6fP3P//zPWrdunXbv3q1Ro0bpueee06ZNm3T77bdLkjZs2KAbbrhBu3fv1i233NJ3qwYADHi9fg6os7NTmzdvVlNTkwoLC7Vv3z51dHRo/vz53ZkpU6ZozJgx2rVr1yXrtLW1KRqN9rgAAAY/cwN67733lJqaqnA4rAceeEBbtmzR1KlTVV1drcTERGVkZPTI5+TkfOGrvYqLixWJRLovo0ePNm8EAGDgMTegyZMn68CBA9qzZ48efPBBrVy5Uh988EGvF7BmzRrV19d3X06ePNnrWgCAgcP0HJAkJSYmauLEiZKkmTNn6u2339aPf/xjrVixQu3t7aqrq+txFlRTU6Pc3NxL1guHw+bX3AMABr4rfh9QV1eX2traNHPmTCUkJGjHjh3d15WVlamiokKFhYVX+mMAAIOM6QxozZo1Wrx4scaMGaOGhgZt2rRJJSUlevXVVxWJRHTvvfdq9erVyszMVHp6uh566CEVFhbyCjgAwAVMDai2tlZ/9Vd/paqqKkUiERUUFOjVV1/VV7/6VUnSj370I8XFxWn58uVqa2vTwoUL9bOf/SwmC79axo4dG7PaBQUFztkFCxbEbB2xZhmvY33jsmWckXXk0EcffeSctY5KstSWpE8++cQ5ax1n1NLS4pzt6uoy1bawjsuxTBFLTEw01c7OzjblR44cGbPaeXl5ztmVK1eaavtmakDPPffcF16flJSktWvXau3atVe0KADA4McsOACAFzQgAIAXNCAAgBc0IACAFzQgAIAXNCAAgBc0IACAFzQgAIAXNCAAgBfmadix9tl4DT6YbvCw7MvGxkZT7aamJuesZeSM9OmHJbrq6Ogw1T5//rwpbxmBYxlRY81ba8dqHda8tbZ15JBlf1qPFctxaL3/xOpx9rO6l7vdQ0Esj6heOHXqFB9KBwCDwMmTJzVq1KhLXt/vGlBXV5cqKyuVlpamUCjU/f1oNKrRo0fr5MmTSk9P97jC2GI7B49rYRsltnOw6YvtDIJADQ0Nys/PV1zcpZ/p6Xd/gouLi/vCjpmenj6od/5n2M7B41rYRontHGyudDsjkchlM7wIAQDgBQ0IAODFgGlA4XBYjz/+uMLhsO+lxBTbOXhcC9sosZ2DzdXczn73IgQAwLVhwJwBAQAGFxoQAMALGhAAwAsaEADAiwHTgNauXavrrrtOSUlJmj17tt566y3fS+pTTzzxhEKhUI/LlClTfC/riuzcuVNLlixRfn6+QqGQtm7d2uP6IAj02GOPKS8vT8nJyZo/f76OHDniZ7FX4HLbec8991ywbxctWuRnsb1UXFysWbNmKS0tTdnZ2Vq2bJnKysp6ZFpbW1VUVKThw4crNTVVy5cvV01NjacV947Lds6dO/eC/fnAAw94WnHvrFu3TgUFBd1vNi0sLNTvf//77uuv1r4cEA3oV7/6lVavXq3HH39c77zzjmbMmKGFCxeqtrbW99L61I033qiqqqruyx//+EffS7oiTU1NmjFjhtauXXvR65955hn95Cc/0fr167Vnzx6lpKRo4cKFam1tvcorvTKX205JWrRoUY99+/zzz1/FFV650tJSFRUVaffu3XrttdfU0dGhBQsW9BgG+8gjj+jll1/WCy+8oNLSUlVWVurOO+/0uGo7l+2UpPvuu6/H/nzmmWc8rbh3Ro0apaefflr79u3T3r17dfvtt2vp0qV6//33JV3FfRkMADfffHNQVFTU/XVnZ2eQn58fFBcXe1xV33r88ceDGTNm+F5GzEgKtmzZ0v11V1dXkJubG/zgBz/o/l5dXV0QDoeD559/3sMK+8bntzMIgmDlypXB0qVLvawnVmprawNJQWlpaRAEn+67hISE4IUXXujO/O///m8gKdi1a5evZV6xz29nEATBn/3ZnwV/+7d/629RMTJs2LDgP/7jP67qvuz3Z0Dt7e3at2+f5s+f3/29uLg4zZ8/X7t27fK4sr535MgR5efna/z48br77rtVUVHhe0kxU15erurq6h77NRKJaPbs2YNuv0pSSUmJsrOzNXnyZD344IM6c+aM7yVdkfr6eklSZmamJGnfvn3q6OjosT+nTJmiMWPGDOj9+fnt/Mwvf/lLZWVladq0aVqzZo2am5t9LK9PdHZ2avPmzWpqalJhYeFV3Zf9bhjp550+fVqdnZ3Kycnp8f2cnBwdPnzY06r63uzZs7Vx40ZNnjxZVVVVevLJJ3Xbbbfp0KFDSktL8728PlddXS1JF92vn103WCxatEh33nmnxo0bp2PHjukf/uEftHjxYu3atUvx8fG+l2fW1dWlhx9+WLfeequmTZsm6dP9mZiYqIyMjB7Zgbw/L7adkvSNb3xDY8eOVX5+vg4ePKjvfve7Kisr029+8xuPq7V77733VFhYqNbWVqWmpmrLli2aOnWqDhw4cNX2Zb9vQNeKxYsXd/+7oKBAs2fP1tixY/XrX/9a9957r8eV4Urddddd3f+ePn26CgoKNGHCBJWUlGjevHkeV9Y7RUVFOnTo0IB/jvJyLrWd999/f/e/p0+frry8PM2bN0/Hjh3ThAkTrvYye23y5Mk6cOCA6uvr9eKLL2rlypUqLS29qmvo93+Cy8rKUnx8/AWvwKipqVFubq6nVcVeRkaGrr/+eh09etT3UmLis313re1XSRo/fryysrIG5L5dtWqVXnnlFb3xxhs9PjYlNzdX7e3tqqur65EfqPvzUtt5MbNnz5akAbc/ExMTNXHiRM2cOVPFxcWaMWOGfvzjH1/VfdnvG1BiYqJmzpypHTt2dH+vq6tLO3bsUGFhoceVxVZjY6OOHTumvLw830uJiXHjxik3N7fHfo1Go9qzZ8+g3q/Sp5/6e+bMmQG1b4Mg0KpVq7Rlyxa9/vrrGjduXI/rZ86cqYSEhB77s6ysTBUVFQNqf15uOy/mwIEDkjSg9ufFdHV1qa2t7eruyz59SUOMbN68OQiHw8HGjRuDDz74ILj//vuDjIyMoLq62vfS+szf/d3fBSUlJUF5eXnwP//zP8H8+fODrKysoLa21vfSeq2hoSHYv39/sH///kBS8MMf/jDYv39/cOLEiSAIguDpp58OMjIygpdeeik4ePBgsHTp0mDcuHFBS0uL55XbfNF2NjQ0BN/+9reDXbt2BeXl5cH27duDL33pS8GkSZOC1tZW30t39uCDDwaRSCQoKSkJqqqqui/Nzc3dmQceeCAYM2ZM8Prrrwd79+4NCgsLg8LCQo+rtrvcdh49ejR46qmngr179wbl5eXBSy+9FIwfPz6YM2eO55XbfO973wtKS0uD8vLy4ODBg8H3vve9IBQKBX/4wx+CILh6+3JANKAgCIKf/vSnwZgxY4LExMTg5ptvDnbv3u17SX1qxYoVQV5eXpCYmBiMHDkyWLFiRXD06FHfy7oib7zxRiDpgsvKlSuDIPj0pdiPPvpokJOTE4TD4WDevHlBWVmZ30X3whdtZ3Nzc7BgwYJgxIgRQUJCQjB27NjgvvvuG3C/PF1s+yQFGzZs6M60tLQEf/M3fxMMGzYsGDp0aHDHHXcEVVVV/hbdC5fbzoqKimDOnDlBZmZmEA6Hg4kTJwZ///d/H9TX1/tduNG3vvWtYOzYsUFiYmIwYsSIYN68ed3NJwiu3r7k4xgAAF70++eAAACDEw0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4AUNCADgBQ0IAOAFDQgA4MX/A9N/tvW8KlHyAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X[0].cpu().permute((1,2,0))); plt.show()\n",
    "plt.imshow(X1[0].cpu().permute((1,2,0))); plt.show()\n",
    "plt.imshow(X2[0].cpu().permute((1,2,0))); plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10ec2a9e-08d0-4569-8175-a4f3062a1228",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7f813d20-e3f7-47c3-b4c5-7831f3c4ce4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def contrastive_loss(first_images, second_images, rank, world_size = 1, temperature=0.1):\n",
    "        # Each image is represented with k parameters,\n",
    "        # Assume the batch size is N, so the\n",
    "        # inputs have shape (N, k)\n",
    "\n",
    "        # These are pre-distributed shapes:\n",
    "        N = first_images.shape[0]\n",
    "        k = first_images.shape[1]\n",
    "\n",
    "\n",
    "        first_images = first_images / torch.norm(first_images,dim=1).reshape((-1,1))\n",
    "        second_images = second_images / torch.norm(second_images,dim=1).reshape((-1,1))\n",
    "\n",
    "        # Take the two tuples, and concatenate them.\n",
    "        # Then, reshape into Y = (1, 2N, k) and Z = (2N, 1, k)\n",
    "\n",
    "        c = torch.concat([first_images, second_images], dim=0)\n",
    "\n",
    "        # Gather all the c up if the world size > 1:\n",
    "        if world_size > 1:\n",
    "            gathered_c = torch.distributed.all_gather(tensor=c)\n",
    "            gathered_c = gathered_c.reshape((-1, first_images.shape[-1]))\n",
    "        else:\n",
    "            gathered_c = c\n",
    "\n",
    "        # Each rank computes only a slice of the global loss matrix, or\n",
    "        # the memory usage gets out of control.\n",
    "\n",
    "        # We calculate the dot product between the local and global tensors:\n",
    "        local_reps = c.reshape((c.shape[0], 1, c.shape[1]))\n",
    "        all_reps   = gathered_c.reshape((1, gathered_c.shape[0], gathered_c.shape[1]))\n",
    "\n",
    "\n",
    "        # Assume we have n images per rank, for N global images with N = n * world_size\n",
    "        # Compute the product of these tensors, which gives shape\n",
    "        # (2n, 2N, k)\n",
    "        mat =  local_reps*all_reps\n",
    "\n",
    "        # We need to compute the function (sim(x,y)) for each element in the 2N sequent.\n",
    "        # Since the are normalized, we're computing x^T . Y / (||x||*||y||),\n",
    "        # but the norms are equal to 1.\n",
    "        # So, summing the matrix over the dim = 0 and dim = 1 computes this for each pair.\n",
    "\n",
    "        sim = torch.sum(mat, dim=-1) / temperature\n",
    "\n",
    "\n",
    "\n",
    "        # Now, sim is of shape [2*n, 2*N]\n",
    "\n",
    "        # This yields a symmetric matrix, diagonal entries equal 1.  Off diagonal are symmetrics and < 1.\n",
    "\n",
    "        # sim = torch.exp(sim / temperature)\n",
    "        # Now, for every entry i in C (concat of both batches), the sum of sim[i] - sim[i][i] is the denominator\n",
    "\n",
    "        device = sim.device\n",
    "\n",
    "        # Since we have a non-symmetric matrix, need to build a non-symmetric index:\n",
    "        positive = torch.zeros(sim.shape, device=device)\n",
    "\n",
    "        # We concatenated all the local examples, and compute symmetric positive pairs\n",
    "        # So for the first N entries, the index of the positive pair is i + N  (locally)\n",
    "        # For the second N entries, the index of the positive pair is i - N (locally)\n",
    "        # with a distributed run, we've squashed all the similarity scores together.\n",
    "        # to a shape of [2*N, 2*N*Size]\n",
    "        # Each 2*N by 2*N block is the local positive indexes, all others are negative.\n",
    "        # That means that the index is shifted by global_rank*2*N\n",
    "\n",
    "        access_index_x = torch.arange(2*N)\n",
    "        # For the first N, the y-index is equal to x + 2*N\n",
    "        # For the second N\n",
    "        access_index_y = torch.arange(2*N)\n",
    "        # Shift by +/- N:\n",
    "        access_index_y[0:N] = access_index_y[0:N] + N\n",
    "        access_index_y[N:]  = access_index_y[N:] - N\n",
    "\n",
    "        access_index_y +=  rank * 2*N\n",
    "\n",
    "        # print(\"access_index_y: \", access_index_y, flush=True)\n",
    "\n",
    "        positive[access_index_x, access_index_y] = 1\n",
    "\n",
    "        # For the negative, we invert the positive and have to 0 out the self-index entries\n",
    "        negative = 1 - positive\n",
    "\n",
    "        # THESE WORK IF IT'S NOT DISTRIBUTED\n",
    "        # positive = torch.tile(torch.eye(N, device=device), (2,2))\n",
    "        # # Unsure if this line is needed?\n",
    "        # positive = positive - torch.eye(2*N, device=device)\n",
    "        #\n",
    "        # negative = - (torch.eye(2*N, device=device) - 1)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            # Here, we can compute the top-k metrics for this batch, since we have the global state:\n",
    "            # We want the top 5 entries but the self-sim is obviously perfect.\n",
    "            # So take the top 6 and reject the first.\n",
    "            topk = torch.topk(sim, k=6, dim=-1, sorted=True)\n",
    "\n",
    "            # Top 1 is just an equality check:\n",
    "            top1_acc = topk.indices[:,1] == access_index_y.to(topk.indices.device)\n",
    "            top1_acc = torch.mean(top1_acc.to(torch.float))\n",
    "          \n",
    "            # Top 5 is a little more complicated:\n",
    "            # Compute the index distance to the correct index, abs value:\n",
    "            top5_acc_dist = torch.abs(topk.indices[:,1:] - access_index_y.to(topk.indices.device).reshape(-1,1))\n",
    "            # Get the minumum value, and see if it is less than 5:\n",
    "            min_values, _ = torch.min(top5_acc_dist, dim=-1)\n",
    "            top5_acc =  min_values < 5.\n",
    "            # Average over the batch dimension:\n",
    "            top5_acc = torch.mean(top5_acc.to(torch.float))\n",
    "\n",
    "\n",
    "        negative_examples = sim * negative\n",
    "        positive_examples = sim * positive\n",
    "\n",
    "        # Now, positive/negative examples is the temperature normalized similarity.\n",
    "        # we need to sum across the whole batch dimension to compute it per-example:\n",
    "\n",
    "\n",
    "        # Compute the alignment, summed over the entire global batch:\n",
    "        alignment = torch.sum(positive_examples, dim=-1)\n",
    "\n",
    "        # Compute the exp, which we'll eventually sum and log:\n",
    "        exp = torch.sum(torch.exp(negative_examples), dim=-1)\n",
    "\n",
    "        # print(\"Alignment: \", alignment, flush=True)\n",
    "        # print(\"exp: \",       exp, flush=True)\n",
    "\n",
    "\n",
    "        # And compute the logsumexp of the negative examples:\n",
    "        log_sum_exp = torch.log(exp )\n",
    "\n",
    "\n",
    "        # Additionally, we can compute the \"floor\" of the loss at this batch size:\n",
    "        # floor = torch.log(1.*N) - 1.\n",
    "\n",
    "        loss_metrics = {\n",
    "            \"alignment\"   : torch.mean(alignment),\n",
    "            \"log_sum_exp\" : torch.mean(log_sum_exp),\n",
    "            \"top1\"        : top1_acc,\n",
    "            \"top5\"        : top5_acc,\n",
    "            # \"floor\"       : floor,\n",
    "        }\n",
    "\n",
    "        loss = torch.mean( - alignment + log_sum_exp)\n",
    "        return loss, loss_metrics\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4135ea73-85b0-431d-b942-61e9bfbd340d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_one_epoch(dataloader, t1, t2, model, head, loss_fn, optimizer, rank, size, progress_bar):\n",
    "    model.train()\n",
    "    head.train()\n",
    "    for (batch, (X, _)) in enumerate(dataloader):\n",
    "        # forward pass\n",
    "        X1 = t1(X); X2 = t2(X)\n",
    "        pred1 = head(model(X1))\n",
    "        pred2 = head(model(X2))\n",
    "        loss, metrics = loss_fn(pred1, pred2, rank, size)\n",
    "\n",
    "        # print(metrics)\n",
    "        \n",
    "        # backward pass calculates gradients\n",
    "        loss.backward()\n",
    "        \n",
    "        # take one step with these gradients\n",
    "        optimizer.step()\n",
    "        \n",
    "        # resets the gradients \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # progress_bar.refresh()\n",
    "        cpu_metrics = { key : f\"{metrics[key].detach().cpu().numpy():.2f}\" for key in metrics.keys()}\n",
    "        cpu_metrics[\"loss\"] = f\"{loss.detach().cpu().numpy():.2f}\"\n",
    "        progress_bar.update()\n",
    "        progress_bar.set_postfix(cpu_metrics)\n",
    "        # progress_bar.description = f\"Train loss: {loss.cpu():.2f} top5: {metrics['top5'].cpu():.2f}\"\n",
    "        # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e29465e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_one_epoch(dataloader, t1, t2, model, head, loss_fn, rank, size, progress_bar):\n",
    "    model.train()\n",
    "    head.train()\n",
    "    n = 0.\n",
    "    sum_metrics = None\n",
    "    for (batch, (X, _)) in enumerate(dataloader):\n",
    "        # forward pass\n",
    "        X1 = t1(X); X2 = t2(X)\n",
    "        pred1 = head(model(X1))\n",
    "        pred2 = head(model(X2))\n",
    "        loss, metrics = loss_fn(pred1, pred2, rank, size)\n",
    "\n",
    "        # print(metrics)\n",
    "        \n",
    "        # backward pass calculates gradients\n",
    "        loss.backward()\n",
    "        \n",
    "        # take one step with these gradients\n",
    "        optimizer.step()\n",
    "        \n",
    "        # resets the gradients \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # progress_bar.refresh()\n",
    "        cpu_metrics = { key : metrics[key].detach().cpu().numpy() for key in metrics.keys()}\n",
    "        if sum_metrics is None: \n",
    "            sum_metrics = cpu_metrics\n",
    "        else:\n",
    "            for key in sum_metrics.keys():\n",
    "                sum_metrics[key] += cpu_metrics[key]\n",
    "        progress_bar.update()\n",
    "        n += 1.\n",
    "        # progress_bar.description = f\"Train loss: {loss.cpu():.2f} top5: {sum_metrics['top5'].cpu():.2f}\"\n",
    "        # break\n",
    "    \n",
    "    for key in sum_metrics:\n",
    "        sum_metrics[key] = sum_metrics[key] / n\n",
    "    return sum_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "06e321f3-7cb3-43cd-bd0e-416234443a74",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.AdamW(list(model.parameters()) + list(head.parameters()), lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb095e0b-e965-4d66-90bc-23b279f32034",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "72478d32-f057-4808-92b7-3a6da64fa9e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54cc843965f4464aa62ac3bb21ed2db9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 0:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validate epoch {j}:  alignment=9.42; log_sum_exp=10.46; top1=0.97; top5=0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "480250775f914ba6865d19dde54ac2c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 1:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validate epoch {j}:  alignment=9.37; log_sum_exp=10.46; top1=0.96; top5=0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79272aea20bf4dd28317d88abc5ccef4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 2:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validate epoch {j}:  alignment=9.37; log_sum_exp=10.46; top1=0.96; top5=0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7c8d3949ada4dd2959b9be6802a8629",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 3:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validate epoch {j}:  alignment=9.38; log_sum_exp=10.45; top1=0.96; top5=0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46f0c703ec304a5e89aa4d9b37dc5efa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 4:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validate epoch {j}:  alignment=9.38; log_sum_exp=10.47; top1=0.96; top5=0.99\n"
     ]
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "for j in range(5):\n",
    "    # with tqdm(total=len(train), position=0, leave=True, desc=f\"Train Epoch {j}\") as train_bar1:\n",
    "    \n",
    "    #     train_one_epoch(train, transforms1, transforms2, model, head, contrastive_loss, optimizer, 0, 1, train_bar1)\n",
    "\n",
    "    with tqdm(total=len(val), position=0, leave=True, desc=f\"Validate Epoch {j}\") as val_bar:\n",
    "        metrics = validate_one_epoch(val, transforms1, transforms2, model, head, contrastive_loss, 0, 1, val_bar)\n",
    "        print_metrics = {\n",
    "            key : f\"{key}={metrics[key]:.2f}\" for key in metrics.keys()\n",
    "        }\n",
    "        print_metrics = \"; \".join(print_metrics.values())\n",
    "        print(f\"Validate epoch {j}: \", print_metrics)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5f602adb-f56a-48af-ba08-fad3b3020236",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, we retrain the classification head without touching the representation. This is called fine tuning.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "444e03fe",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9afda987",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2227bc84",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(dataloader, model, head, loss_fn, val_bar):\n",
    "    # Set the model to evaluation mode - some NN pieces behave differently during training\n",
    "    # Unnecessary in this situation but added for best practices\n",
    "    model.eval()\n",
    "    size = len(dataloader)\n",
    "    num_batches = len(dataloader)\n",
    "    loss, correct = 0, 0\n",
    "\n",
    "    # We can save computation and memory by not calculating gradients here - we aren't optimizing \n",
    "    with torch.no_grad():\n",
    "        # loop over all of the batches\n",
    "        for X, y in dataloader:\n",
    "\n",
    "            pred = head(model(X))\n",
    "            loss += loss_fn(pred, y).item()\n",
    "            # how many are correct in this batch? Tracking for accuracy \n",
    "            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "            val_bar.update()\n",
    "            \n",
    "    loss /= num_batches\n",
    "    correct /= (size*batch_size)\n",
    "    \n",
    "    accuracy = 100*correct\n",
    "    return accuracy, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a3a6e4ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fine_tune(dataloader, rep_model, head, loss_fn, optimizer, progress_bar):\n",
    "    head.train()\n",
    "    model.eval()\n",
    "    for batch1, (X, Y) in enumerate(dataloader):\n",
    "        # forward pass\n",
    "        # Calling detach blocks all gradients into the representation model!\n",
    "        rep = rep_model(X).detach()\n",
    "        pred = head(rep)\n",
    "        loss = loss_fn(pred, Y)\n",
    "        \n",
    "        \n",
    "        # backward pass calculates gradients\n",
    "        loss.backward()\n",
    "        \n",
    "        # take one step with these gradients\n",
    "        optimizer.step()\n",
    "        \n",
    "        # resets the gradients \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        correct = (pred.argmax(1) == Y).type(torch.float).mean().item()\n",
    "                    \n",
    "        # progress_bar.refresh()\n",
    "        cpu_metrics = {}\n",
    "        cpu_metrics[\"acc\"] = f\"{correct:.2f}\"\n",
    "        cpu_metrics[\"loss\"] = f\"{loss.detach().cpu().numpy():.2f}\"\n",
    "        progress_bar.update()\n",
    "        progress_bar.set_postfix(cpu_metrics)\n",
    "        # progress_bar.description = f\"Train loss: {loss.cpu():.2f} top5: {metrics['top5'].cpu():.2f}\"\n",
    "        # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4f4b467e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdamW (\n",
      "Parameter Group 0\n",
      "    amsgrad: False\n",
      "    betas: (0.9, 0.999)\n",
      "    capturable: False\n",
      "    differentiable: False\n",
      "    eps: 1e-08\n",
      "    foreach: None\n",
      "    fused: None\n",
      "    lr: 0.01\n",
      "    maximize: False\n",
      "    weight_decay: 0.01\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "classification_head = nn.Linear(256, 10).cuda()\n",
    "classification_loss = loss_fn = nn.CrossEntropyLoss()\n",
    "fine_tune_optimizer = torch.optim.AdamW(classification_head.parameters(), lr=0.01)\n",
    "print(fine_tune_optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2b6e2937",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a996d26fedff4a108a10ec12a45a79c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fine Tune Epoch 0:   0%|          | 0/313 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f8ab85be7ac4c0ab3260238869f4513",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 0:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: validation loss: 1.899, accuracy: 31.458\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c85b2638317b41a6ac28763c7a1dbdf9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fine Tune Epoch 1:   0%|          | 0/313 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "baa0645eaf5947519be1272cb5ad0fcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 1:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: validation loss: 1.868, accuracy: 33.752\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "409c819b2ed04f69924c5b21ca4eac15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fine Tune Epoch 2:   0%|          | 0/313 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8d202c57f6e45db9cf77a5bffe4c91b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 2:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: validation loss: 1.873, accuracy: 32.862\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5171871c949e4824bd5729d963716016",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fine Tune Epoch 3:   0%|          | 0/313 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7f72ad483054c1c838e14ee49e830fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 3:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: validation loss: 1.917, accuracy: 32.961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e02324f8f7b491e9fdb38795ba596c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fine Tune Epoch 4:   0%|          | 0/313 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00b9cd7f192c41e2958e9dec6e1984b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validate Epoch 4:   0%|          | 0/79 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: validation loss: 1.859, accuracy: 34.444\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for j in range(5):\n",
    "    with tqdm(total=len(train), position=0, leave=True, desc=f\"Fine Tune Epoch {j}\") as train_bar1:\n",
    "    \n",
    "        fine_tune(train, model, classification_head, classification_loss, fine_tune_optimizer, train_bar1)\n",
    "    with tqdm(total=len(val), position=0, leave=True, desc=f\"Validate Epoch {j}\") as val_bar:\n",
    "        acc, loss = evaluate(val, model, classification_head, classification_loss, val_bar)\n",
    "        print(f\"Epoch {j}: validation loss: {loss:.3f}, accuracy: {acc:.3f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1398afdf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}