diff --git a/siamese_network_prototype.ipynb b/siamese_network_prototype.ipynb new file mode 100644 index 0000000..43a9234 --- /dev/null +++ b/siamese_network_prototype.ipynb @@ -0,0 +1,1216 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf\n", + "# https://www.cs.cmu.edu/~rsalakhu/papers/oneshot1.pdf" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "from fastai.vision import *\n", + "from fastai.metrics import accuracy_thresh\n", + "from fastai.basic_data import *\n", + "from torch.utils.data import DataLoader, Dataset\n", + "from torch import nn\n", + "from fastai.callbacks.hooks import num_features_model, model_sizes\n", + "from fastai.layers import BCEWithLogitsFlat\n", + "from fastai.basic_train import Learner\n", + "from skimage.util import montage\n", + "import pandas as pd\n", + "from torch import optim\n", + "import re\n", + "\n", + "from utils import *" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# import fastai\n", + "# from fastprogress import force_console_behavior\n", + "# import fastprogress\n", + "# fastprogress.fastprogress.NO_BAR = True\n", + "# master_bar, progress_bar = force_console_behavior()\n", + "# fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Posing the problem as a classification task is probably not ideal. We are asking our NN to learn to recognize a whale out of 5004 possible candidates based on what it has learned about the whales. That is a tall order.\n", + "\n", + "Instead, here we will try to pose the problem as a verification task. When presented with two images of whale flukes, we will ask the network - are the images of the same whale or of different whales? In particular, we will try to teach our network to learn features that can be useful in determining the similarity between whale images (hence the name of this approach - feature learning).\n", + "\n", + "This seems like a much easier task, at least in theory. Either way, no need to start with a relatively big CNN like resnet50. Let's see what mileage we can get out of resnet18." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# new architecture calls for a new validation set, this time our validation set will consist of all whales that have exactly two images\n", + "df = pd.read_csv('data/train.csv')\n", + "im_count = df[df.Id != 'new_whale'].Id.value_counts()\n", + "im_count.name = 'sighting_count'\n", + "df = df.join(im_count, on='Id')\n", + "val_fns = set(df[df.sighting_count == 2].Image)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2570" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(val_fns)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "fn2label = {row[1].Image: row[1].Id for row in df.iterrows()}\n", + "path2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)\n", + "\n", + "name = f'res18-siamese'" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "SZ = 224\n", + "BS = 64\n", + "NUM_WORKERS = 12\n", + "SEED=0" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# data_block api creates categories based on classes it sees in the train set and\n", + "# our val set contains whales whose ids do not appear in the train set\n", + "classes = df.Id.unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "data = (\n", + " ImageItemList\n", + " .from_df(df[df.Id != 'new_whale'], f'data/train-{SZ}', cols=['Image'])\n", + " .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n", + " .label_from_func(lambda path: fn2label[path2fn(path)], classes=classes)\n", + " .add_test(ImageItemList.from_folder(f'data/test-{SZ}'))\n", + " .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n", + "# .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n", + "# .normalize(imagenet_stats)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "I am still using the ImageItemList even though I will create my own datasets. Why? Because I want to reuse the functionality that is already there (creating datasets from files, augmentations, resizing, etc).\n", + "\n", + "I realize the code is neither clean nor elegant but for the time being I am happy with this approach." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def is_even(num): return num % 2 == 0\n", + "\n", + "class TwoImDataset(Dataset):\n", + " def __init__(self, ds):\n", + " self.ds = ds\n", + " self.whale_ids = ds.y.items\n", + " def __len__(self):\n", + " return 2 * len(self.ds)\n", + " def __getitem__(self, idx):\n", + " if is_even(idx):\n", + " return self.sample_same(idx // 2)\n", + " else: return self.sample_different((idx-1) // 2)\n", + " def sample_same(self, idx):\n", + " whale_id = self.whale_ids[idx] \n", + " candidates = list(np.where(self.whale_ids == whale_id)[0])\n", + " candidates.remove(idx) # dropping our current whale - we don't want to compare against an identical image!\n", + " \n", + " if len(candidates) == 0: # oops, there is only a single whale with this id in the dataset\n", + " return self.sample_different(idx)\n", + " \n", + " np.random.shuffle(candidates)\n", + " return self.construct_example(self.ds[idx][0], self.ds[candidates[0]][0], 1)\n", + " def sample_different(self, idx):\n", + " whale_id = self.whale_ids[idx]\n", + " candidates = list(np.where(self.whale_ids != whale_id)[0])\n", + " np.random.shuffle(candidates)\n", + " return self.construct_example(self.ds[idx][0], self.ds[candidates[0]][0], 0)\n", + " \n", + " def construct_example(self, im_A, im_B, class_idx):\n", + " return [im_A, im_B], class_idx" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "train_dl = DataLoader(\n", + " TwoImDataset(data.train),\n", + " batch_size=BS,\n", + " shuffle=True,\n", + " num_workers=NUM_WORKERS\n", + ")\n", + "valid_dl = DataLoader(\n", + " TwoImDataset(data.valid),\n", + " batch_size=BS,\n", + " shuffle=False,\n", + " num_workers=NUM_WORKERS\n", + ")\n", + "\n", + "data_bunch = ImageDataBunch(train_dl, valid_dl)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_batch(batch):\n", + " stat_tensors = [torch.tensor(l).cuda() for l in imagenet_stats]\n", + " return [normalize(batch[0][0], *stat_tensors), normalize(batch[0][1], *stat_tensors)], batch[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "data_bunch.add_tfm(normalize_batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "from functional import seq\n", + "\n", + "class SiameseNetwork(nn.Module):\n", + " def __init__(self, arch=models.resnet18):\n", + " super().__init__() \n", + " self.cnn = create_body(arch)\n", + " self.head = nn.Linear(num_features_model(self.cnn), 1)\n", + " \n", + " def forward(self, im_A, im_B):\n", + " # dl - distance layer\n", + " x1, x2 = seq(im_A, im_B).map(self.cnn).map(self.process_features)\n", + " dl = self.calculate_distance(x1, x2)\n", + " out = self.head(dl)\n", + " return out\n", + " \n", + " def process_features(self, x): return x.reshape(*x.shape[:2], -1).max(-1)[0]\n", + " def calculate_distance(self, x1, x2): return (x1 - x2).abs_()\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below I include two slightly different siamese networks. I leave the code commented out and choose to use the one above." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# from functional import seq\n", + "\n", + "# def cnn_activations_count(model):\n", + "# _, ch, h, w = model_sizes(create_body(models.resnet18), (SZ, SZ))[-1]\n", + "# return ch * h * w\n", + "\n", + "# class SiameseNetwork(nn.Module):\n", + "# def __init__(self, lin_ftrs=2048, arch=models.resnet18):\n", + "# super().__init__() \n", + "# self.cnn = create_body(arch)\n", + "# self.fc1 = nn.Linear(cnn_activations_count(self.cnn), lin_ftrs)\n", + "# self.fc2 = nn.Linear(lin_ftrs, 1)\n", + " \n", + "# def forward(self, im_A, im_B):\n", + "# x1, x2 = seq(im_A, im_B).map(self.cnn).map(self.process_features).map(self.fc1)\n", + "# dl = self.calculate_distance(x1.sigmoid(), x2.sigmoid())\n", + "# out = self.fc2(dl)\n", + "# return out\n", + " \n", + "# def calculate_distance(self, x1, x2): return (x1 - x2).abs_()\n", + "# def process_features(self, x): return x.reshape(x.shape[0], -1)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# from functional import seq\n", + "\n", + "# def cnn_activations_count(model):\n", + "# _, ch, h, w = model_sizes(create_body(models.resnet18), (SZ, SZ))[-1]\n", + "# return ch * h * w\n", + "\n", + "# class SiameseNetwork(nn.Module):\n", + "# def __init__(self, lin_ftrs=2048, pool_to=3, arch=models.resnet18, pooling_layer=nn.AdaptiveMaxPool2d):\n", + "# super().__init__() \n", + "# self.cnn = create_body(arch)\n", + "# self.pool = pooling_layer(pool_to)\n", + "# self.fc1 = nn.Linear(num_features_model(self.cnn) * pool_to**2, lin_ftrs)\n", + "# self.fc2 = nn.Linear(lin_ftrs, 1)\n", + " \n", + "# def forward(self, im_A, im_B):\n", + "# x1, x2 = seq(im_A, im_B).map(self.cnn).map(self.pool).map(self.process_features).map(self.fc1)\n", + "# dl = self.calculate_distance(x1.sigmoid(), x2.sigmoid())\n", + "# out = self.fc2(dl)\n", + "# return out\n", + " \n", + "# def calculate_distance(self, x1, x2): return (x1 - x2).abs_()\n", + "# def process_features(self, x): return x.reshape(x.shape[0], -1)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "learn = Learner(data_bunch, SiameseNetwork(), loss_func=BCEWithLogitsFlat(), metrics=[lambda preds, targs: accuracy_thresh(preds.squeeze(), targs, sigmoid=False)])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "learn.split([learn.model.cnn[:6], learn.model.cnn[6:], learn.model.head])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "learn.freeze_to(-1)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n" + ] + } + ], + "source": [ + "learn.lr_find()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl8XXWd//HXJ2uzNUuTtGm6pBstpUChAUQRWhBFmJ8siorLgMzIjOuoD2ec+fkbdfTnuDsuqPxQAR1HGBFQYUSQtazStHSlLV3Tplv2Pc12P78/7m1IQ5KmTU7OTe77+XjcB/fe8733vM+l937yPd9zvsfcHREREYCksAOIiEj8UFEQEZE+KgoiItJHRUFERPqoKIiISB8VBRER6aOiICIifVQURESkj4qCiIj0SQk7wMkqLCz0srKysGOIiEwoa9eurXX3ohO1m3BFoaysjIqKirBjiIhMKGZWOZJ22n0kIiJ9VBRERKSPioKIiPRRURARkT4qCiIi0kdFQURE+qgoiIhIHxUFEZE49NzOWtbvbxz39aooiIjEof/9wCY++LO/sKe2bVzXq6IgIhJnunsjVDV00NLZw0d+tZb2rp5xW7eKgohInDnQ0EFvxLl6+Uy2H2nhX+7fhLuPy7oDKwpmdoeZVZvZ5mHarDSz9Wa2xcyeDiqLiMhEUlnfDsD7L5jLZ95yGr9ff5BfvjCiqYtGLcgJ8e4CbgV+OdhCM8sDfgxc4e77zKw4wCwiIhNGZV10HKFsWiblc/NZv7+Rrzz0CstKp7JibkGg6w6sp+Duq4H6YZq8D7jf3ffF2lcHlUVEZCKprGsnIzWZopx0kpKM775nOaX5GTyzozbwdYc5dfZpQKqZPQXkAN9396F6FbcAtwDMmTNn3AKKiIShsq6dOQWZmBkAuRmpPPiJi5g6JTXwdYc50JwCrACuAt4G/KuZnTZYQ3e/3d3L3b28qOiE14gQEZnQKuvamDMt87jnxqMgQLhFoQr4k7u3uXstsBo4O8Q8IiKhi0ScffXtlA0oCuMlzKLwe+DNZpZiZpnABcDWEPOIiISuuqWTzp4Ic6ZlhbL+wMYUzOxuYCVQaGZVwBeBVAB3v83dt5rZn4CNQAT4mbsPefiqiEgiOHbk0dyCcHoKgRUFd79hBG2+BXwrqAwiIhNNZV30HIWykHoKOqNZRCSOVNa3kZJkzMybEsr6VRREROJIZV07pfkZpCSH8/OsoiAiEkeOnaMQFhUFEZE4UlnXFtp4AqgoiIjEjcb2LpqP9jA3pHMUQEVBRCRu7I0deaTdRyIi8to5Ctp9JCIi+9RTEBGRYyrr25k+NZ2MtOTQMqgoiIjEicq6NuYWhLfrCFQURETiRmVde6hHHoGKgohIXGjv6qG6pVNFQUQkEbV29vDKwea+x/vqY4PMIR55BCoKIiKh+N6fX+XKHzzD5x/YREdXb9/sqGFNmX1MmNdoFhFJWM/vqiMvM5X/+ss+XtpTz7lz8oHwpsw+Rj0FEZFx1ny0m22Hm7nxwjJ+efP5NLR3898V+8nNSCU3c3yuxTwU9RRERMbZusoGIg7nlRVw0aJC/vSpN/OF328mLzMt7GgqCiIi461ibwPJScY5c/IAKMxO58fvXxFyqqjAdh+Z2R1mVm1mg1532cxWmlmTma2P3b4QVBYRkXjy0t56zpg5laz0+Pu7PMgxhbuAK07Q5hl3Xx67fTnALCIicaGzp5cN+xs5r6wg7CiDCqwouPtqoD6o9xcRmYg2H2imsyfCeWX5YUcZVNhHH11oZhvM7GEzO2OoRmZ2i5lVmFlFTU3NeOYTERlTa/ZG/1YuT7SewgisA+a6+9nAD4HfDdXQ3W9393J3Ly8qKhq3gCIiY61ibz3zC7MozE4PO8qgQisK7t7s7q2x+38EUs2sMKw8IiJBi0ScNXsb4nY8AUIsCmY2w8wsdv/8WJa6sPKIiARtZ00rTR3dlMfpeAIEeJ6Cmd0NrAQKzawK+CKQCuDutwHvAj5iZj1AB/Bed/eg8oiIhO2lPdHxhPPnxW9PIbCi4O43nGD5rcCtQa1fRCTeVOytpygnPdTLbZ5I2EcfiYgkjDV7Gzi/rIDYnvO4pKIgIjIODjR2cKCxI67HE0BzH4mIBKKrJ8Ltq3ex5WAz+xva+66XEM9HHoGKgohIIO5bV8W3H32V+UVZzCnI5JzZ+ZxeMpUzZk4NO9qwVBRERALwm4r9LCrO5tFPXxzXYwgDaUxBRGSM7axu4eV9jby7fPaEKgigoiAiMubuXVtFcpJxzTmlYUc5aSoKIiJjqKc3wv3rDrBqcTFFOfE5v9FwVBRERMbQ6h011LR0cn35rLCjnBIVBRGRMXRvRRXTstK4dElx2FFOiYqCiMgYqW/r4rGtR7jmnFJSkyfmz+vETC0iEod+9/IBunt9wu46AhUFEZExc+/aKs4szWXJjPg+QW04KgoiImOgsq6NrYeaue7ciXcYan8qCiIiY2BtZQMAFy6YFnKS0VFREBEZA2srG8hJT2FRcU7YUUZFRUFEZAysrWxg+Zw8kpMm1rQWAwVWFMzsDjOrNrPNJ2h3npn1mtm7gsoiIhKklqPdbD/Swoq58X2thJEIsqdwF3DFcA3MLBn4BvBIgDlERAK1fn8j7qgoDMfdVwP1J2j2CeA+oDqoHCIiQVtb2YAZLJ+dF3aUUQttTMHMSoFrgdvCyiAiMhbWVjaweHoOOVNSw44yamEONH8P+Jy7956ooZndYmYVZlZRU1MzDtFEREYmEnHW72ucFLuOINwrr5UD98QuQFEIXGlmPe7+u4EN3f124HaA8vJyH9eUIiLD2FHdSktnj4rCaLn7vGP3zewu4KHBCoKISDw7dtKaisIJmNndwEqg0MyqgC8CqQDurnEEEZkU1lY2UJidxpyCzLCjjInAioK733ASbW8KKoeISJDW7WvgnDn5E+5azEPRGc0iIqeorrWTPbVtk2bXEagoiIicsnX7GoHJM54AKgoiIqdsbWUDqcnGmaW5YUcZMyoKIiKnaF1lA2fMzGVKanLYUcaMioKIyCmob+ti/f5Gzp9XEHaUMaWiICJyCu6t2E9Xb4R3njtxr8c8GBUFEZGTFIk4v35pH+eV5bN4xsS+qM5AKgoiIifp2Z21VNa184E3zA07yphTURAROUm/erGSgqw0rlg2I+woY05FQUTkJBxq6uDxbdVcXz6L9JTJc9TRMSoKIiIn4Z6X9hNx5/3nT75dR6CiICIyYt29Ee5Zs4+LFxUxZ9rkmABvIBUFEZERenxrNUeaO3n/BXPCjhIYFQURkRH6TcV+SnKncOmS4rCjBEZFQURkhI7NiJqSPHl/OifvlomIjLH6ti4KstLCjhEoFQURkRHo6Y3Q1NFNfqaKgohIwmto7wZgWraKwikxszvMrNrMNg+x/Goz22hm682swswuCiqLiMhoNbR3AainMAp3AVcMs/xx4Gx3Xw7cDPwswCwiIqNS3xYtChpTOEXuvhqoH2Z5q7t77GEW4EO1FREJm4rCODCza81sG/A/RHsLIiJxSUVhHLj7A+6+BLgG+MpQ7czslti4Q0VNTc34BRQRiWmIFYW8zNSQkwQrLo4+iu1qWmBmhUMsv93dy929vKioaJzTiYhAfXsXOekpk3Jm1P5CKwpmttDMLHb/XCANqAsrj4jIcOrbusif5LuOAFJG0sjMFgBV7t5pZiuBs4BfunvjMK+5G1gJFJpZFfBFIBXA3W8D3gn8tZl1Ax3Ae/oNPIuIxBUVhePdB5Sb2ULg58AfgF8DVw71Ane/Ybg3dPdvAN8Y4fpFRELV0N5Fcc6UsGMEbqS7jyLu3gNcC3zP3T8NlAQXS0QkvtS3dk36E9dg5EWh28xuAG4EHoo9N7mH4EVE+qlv76Iga/L/7I20KHwIuBD4qrvvMbN5wK+CiyUiEj86uno52h3RmMIx7v4K8EkAM8sHctz960EGExGJF3VtnQBMS4CiMKKegpk9ZWZTzawA2ADcaWbfDTaaiEh8aGiLzpCqMYXX5Lp7M3AdcKe7rwDeElwsEZH4Ud+eGFNcwMiLQoqZlQDv5rWBZhGRhNCQIPMewciLwpeBR4Bd7r7GzOYDO4KLJSISP+oSqCiMdKD5XuDefo93Ez0jWURk0mto6yI5yZg6RYekAmBms8zsgdiV1I6Y2X1mNivocCIi8aC+vYv8zFSSkizsKIEb6e6jO4lObTETKAUejD0nIjLpJcrZzDDyolDk7ne6e0/sdhegOaxFJCHUtyfGZHgw8qJQa2YfMLPk2O0DaJprEUkQDW1dCXHiGoy8KNxM9HDUw8Ah4F1Ep74QEZn0EmXabBhhUXD3fe7+Dncvcvdid7+G6IlsIiKTWiTiNLR3UaAxhRP6zJilEBGJU81Hu4l4YpyjAKMrCpP/2CwRSXj1CXTiGoyuKOjSmSIy6R0rChpTAMysxcyaB7m1ED1nYbjX3hE72W3zEMvfb2YbY7fnzezsUWyHiEggjhWFRDn6aNhpLtw9ZxTvfRdwK/DLIZbvAS5x9wYzeztwO3DBKNYnIjLmGtoTq6cwormPToW7rzazsmGWP9/v4YuAps0QkbjTNxmejj4aV38DPBx2CBGRgRraupiSmkRGWnLYUcZFYD2FkTKzVUSLwkXDtLkFuAVgzpw545RMRATq27qZlpUedoxxE2pPwczOAn4GXO3uQ06b4e63u3u5u5cXFWnKJREZP/VtneRnTf4ps48JrSiY2RzgfuCD7v5qWDlERIZT396dMDOkQoC7j8zsbmAlUGhmVcAXgVQAd78N+AIwDfixmQH0uHt5UHlERE5FQ1sX86Zlhh1j3AR59NENJ1j+t8DfBrV+EZGx0JBAk+FB/Bx9JCISdzp7emnp7EmYw1FBRUFEZEiN7d0AFGSrKIiIJLz6BDtxDVQURESGlGiT4YGKgojIkBJtMjxQURARGVKiTYYHKgoiIkOqa40WhbwMndEsIpLwGtq7yMtMJSU5cX4qE2dLRURO0p7aNqbnTAk7xrhSURARGUR7Vw9/2VPPRYsKw44yrlQUREQG8dzOOrp6Ily2pDjsKONKRUFEZBBPbDtCdnoK5WUFYUcZVyoKIiIDuDtPbKvmzYsKSUtJrJ/JxNpaEZER2HKwmSPNnVyaYLuOQEVBROR1ntxWDcDKxSoKIiIJ7/Ft1Zw9O4+inMS5NvMxKgoiIv3UtnayoaqRSxOwlwAqCiIix3lqew3ucNnpKgpjyszuMLNqM9s8xPIlZvaCmXWa2WeDyiEicjKe3FZNcU46Z8ycGnaUUATZU7gLuGKY5fXAJ4FvB5hBRGTEunsjrH61hkuXFGNmYccJRWBFwd1XE/3hH2p5tbuvAbqDyiAicjLW7K2npbOHVQl4KOoxE2JMwcxuMbMKM6uoqakJO46ITFLP7KglJcm4aGFizXfU34QoCu5+u7uXu3t5UVFR2HFEZJLasL+RJSU5ZKWnhB0lNBOiKIiIBC0ScTZVNXHWrLywo4RKRUFEBNhT10ZLZw/LE7woBNZHMrO7gZVAoZlVAV8EUgHc/TYzmwFUAFOBiJl9Cljq7s1BZRIRGcrGqkYAzpqdG3KScAVWFNz9hhMsPwzMCmr9IiInY8P+JjJSk1lYlB12lFBp95GICNGewrLSqQl1PebBJPbWi4gQPWlty8HmhB9kBhUFERG2H26hsyfCWbMSezwBVBRERNhY1QTA8tnqKagoiEjC21jVSF5mKnMKMsOOEjoVBRFJeBuqmjizNDdhJ8HrT0VBRBJaR1cvrx5p4WwNMgMqCiKS4F451ERvxDXIHKOiICIJbcP+6CDz2RpkBlQURCTBbaxqZMbUKUyfOiXsKHFBRUFEEtqGqibtOupHRUFEElZTRzd7atu066gfFQURSVibYietqafwGhUFEUlYGw9Ep8s+s1RF4RgVBRFJWJuqmpg7LZO8zLSwo8QNFQURSVgbdfnN11FREJGEVNvayYHGDs7SrqPjqCiISELadCA6yHymBpmPE1hRMLM7zKzazDYPsdzM7AdmttPMNprZuUFlEREZaOP+JsxgmXoKxwmyp3AXcMUwy98OLIrdbgF+EmAWEZHjbDrQyIKibLLTA7tU/YQUWFFw99VA/TBNrgZ+6VEvAnlmVhJUHhGR/jZWNWk8YRBhjimUAvv7Pa6KPfc6ZnaLmVWYWUVNTc24hBORyetw01GqWzo1njCIMIvCYFez8MEauvvt7l7u7uVFRUUBxxKRyW5jVfSkNR2O+nphFoUqYHa/x7OAgyFlEZEEsulAE8lJxtKSqWFHiTthFoU/AH8dOwrpDUCTux8KMY+IJIgNVU0sKs4mIy057ChxJ7BhdzO7G1gJFJpZFfBFIBXA3W8D/ghcCewE2oEPBZVlvFW3HKW+rYvF03N0zVeROOPubKpq5K1LZ4QdJS4FVhTc/YYTLHfgY0GtfzzVtnbyqxcr2bC/kS0Hm6lu6QTgDfML+PyVSzWYJRJHqho6aGjv1vdyCAl3gK678+T2an705C52HGnhQ2+ax4cvnj/sscq7a1r51Yv7yM1I5ZpzZjJ3WhYAnT29/OL5vfzw8Z20dfWwqDiHixYVcsbMXCIR5ydP7+J/3fosVy+fyU1vLKPlaA+Hmjo41HSU7PQU3nH2TIrH4WpPkYjT2NFNXWsndW1dlE3LYkaurjIlienYmcyaLntwFv2DfeIoLy/3ioqKk35db8R5ePMhfvTkLrYeaqY0L4PFM3J4Yls107LS+PilC3nfBXNIT3ltH+OWg038+Kld/HHTIVKTkuiORHCHFXPzWbW4iHvXVlFZ186qxUV8/qqlLCzOPm6dzUe7ue2pXfz82T109kRelynJ4OLTinjXilm8cUEheRmpJCWdeHdTQ1sXeZmpw+6acnee3VnLD5/YScXeeiL9/jcnJxlvO2M6N71xHueV5WsXlySUrz28lTuf3cumf3vrcd/3yc7M1rp7+QnbJUpRuOelffzz/ZuYX5TFRy5ZwDXnlJKanMT6/Y184+FtvLC7juQkIyM1mSmpyaSnJHGgsYOc9BQ+eOFcbr5oHl09EX6//iAPvFzFq0daWViczf+56nRWLi4edt2HmjpYV9lI8dT0vmvB7m9o5761VTzw8gEONR0Foj/WBVlpTMtK47yyAq47t5Tls/MwMyIR5/Ft1dy+ehdr9jYwpyCTK5bN4IplM1g+K4+kJMPd6e51nt9Vyw8e38G6fY2U5E7hmnNKmZ6TTkF2OnkZqTy3q5Z7XtpPU0c3p5dM5YyZU0lJMlKSjZSkJDp7euno6qWju5feiFM2LYvFM3JYMmMqi6ZnMyX15L5IDW1dPLOzlhd21ZGabJTkZjAzbwqz8jNZPjuP5BEUQpFT0Rtxvvvn7SwoyuavzppJWkoS7/vpi7R29vCHj18UdrxxpaIwQEdXL09ur+ZtZ8x43Y/Qsb+qX9xdx9HuCB3dvRzt6mVBcTYfeMNccjNSX9f+YNNRinPSSU0e3QFcvRHnxd11bD/cQl1bJ3WtXRxuPsoLu+ro7IkwvzCLtyydzuNbj7Crpo3SvAyuO7eUjVVNPL+rlu5eJzs9BXfnaE+E3liXoDQvg4+uWsC7Vswa9K+hjq5efr/+AHev2U9tSyfdvRF6Ik5Pb4T01GQy05LJiP3476lt6+vpmMGs/AzmF2azoCibOQUZFGSnk5+ZSn5mGr0R79tFdrCxgzV7G9hQ1Yg7TJ0S3UXXfLSnL8ei4mw+9ZbTePuyGSPqJYmcjP9es4/P3bcJgBlTp/ChN5Vx65M7ecfZM/nqtWeGnG58qShMcM1Hu3l40yHuX3eAv+ypZ2nJVP7ukvlcdWYJKbFC1NTRzRPbjrCuspG0lKRYLyeJWfmZXHlmCWkpY3PEcW/E2VvXxvbDLWw/3MLu2jZ217Syu6aNju7eIV+XnpLE0plTWXlaMRefVshZs6K9gtbOHg41drD5YBM/enIXO6tbWTIjh09ffhqXnz5dxUHGRFtnD6u+/RSz8jP4xKWL+Okzu3l+Vx0A33jnmbznvDkhJxxfKgqTSGtnD1lpyXG37z8Scerbu2hs76KhvZu61i6SDGbmZVCSO4WCrLQTZu6NOA9tPMj3HtvBnto2lszI4aOrFnLVmSXH9ejaOnto6+qhKDs97j4HiU/fe+xVvvfYDu77yBtZMTcfgM0HmvjzK0f42zfPI2dK6gneYXJRUZAJpac3wh82HOTHT0V7DvMLs/irs2dSWdfG5gNN7K5twx1y0lNYUJzNwuJsyufmc9VZJQn35ZYTO9J8lJXfeopLlxTzo/drVn5QUZAJKhJxHtlymB8+sZNXDjVTkjuFM2bmsqx0KnkZqeyqaWNndSs7qlupbe1kSmoSV55ZwrvLZ3PBvAL1IgSAz/12I/e/XMVjn7mk7xDyRDfSopBw5ylIfEtKMt5+ZglXLJtBa2fPkL0Ad2f9/kZ+U1HFgxsOcv+6AyyZkcPHVi3kygG7niSxbD3UzG/W7ufmN81TQTgF6inIhNfR1cv/bDrEbU9Hdz0tKMriY6sW8o6zZ/YNysvk19bZw9Ov1vDjp3ayr66d1f+0irzMtLBjxQ3tPpKE0xtx/rT5MD98YgfbDrcwKz+DD795Pu8un62JzyaxF3bV8fNnd7N6Ry1dPRHyM1P50jvO4Orlg16eJWGpKEjCikScx7Ye4band7FuXyP5manc+MYyrjtnFnOmZYYdT8ZQdfNRLvvO02SkJXPlmSW87YwZnFeWrx7iIFQURIA1e+u57aldPL6tGoAlM3J46xkzWLW4iJLcDPIyU487Q7s34rR39ZCekjxm53kMxd052h0hNdn0I3aKPnXPy/xx02Ee+fTFzCvU+MFwNNAsApxXVsB5NxWwv76dR7Yc5tEtR7j1iR384PEdfW3SU5LISEumvauXrtiZ2/mZqXx05UI+eOHcEU/rcezM+AdePkBHV2/sOeiJOG2dPbT2u7V39tDe3Ys7ZKYlc8G8At60sJCLFhUOOeW6u1PV0MGGqkbW72tk/f5G9tS2saw0lzcvGvq1zUe7eeyVI/xx0yGqGjrITk8he0oK2ekpnDUrl2vOKaU4Z+JNkPji7jp+t/4gn7h0oQrCGFJPQRJObWsnFXsbaGjvoqG9i6b2bjq6e8lISyYzNYXMtGSe2VnL6ldrmDF1Cp+8bBHXl88ackqT7t4ID208yO2r97D1UDP5makUZqdz7Lc5yYyc2I9w9pRUstKSyUxLISs9mYy0ZA43HeXZnbXsrmkDokViQVE284uymFeYRUNbF1sPtbD1cDMtsSlC0lKSWDZzKmWFWazf39j32rzMVEpyMyjKSac4Jz0679SOWrp6I8zMncLSmbm0d0ULU1NHN5V17SQnGasWF3F9+WwuXVI86qlbxkN3b4SrfvAMbZ29PPaZSzRmNALafSQySi/uruObf9rGun2NpKckcdr0nNjEgDl09kSorGtjb107O6tbqW/rYlFxNh++eD5XL595SrNvHmjs4LmdtbxysJndtW3sqm7lQGMHWWnJLCmZyukl0UkJz5qVy5IZU4/bvXWwsYNnd9Ty8v5GalqOUtPSSU1LJ8nJxluXzuCqs0r6Jk7sb1dNK/dWVHH/uiqqWzqZXZDBx1Yu5LpzZwW++2w0frp6N1/941Z++tflXL50ethxJgQVBZEx4O48/WoNz+6oZfuRFrYeaqG2NXoRpaKcdMqmZTJ3WhZXnjmDlacVj/m8TUe7e0lLTgp8Pqie3ghPbKvmR0/uZENVE6V5GfzdJfNZPjuPgqw0CrLSyEiNj6lWDjcd5bLvPMUF86fx8xvL4yLTRBAXRcHMrgC+DyQDP3P3rw9YPhe4AygC6oEPuHvVcO+poiBhq2/rIj0liaxhLsw0Ubk7q3fU8v3HXmXdvsbjlmWnp7BqSTHvOHsmF59WGMq1CKoa2vnIr9ax/UgLj336Eh1NdhJCLwpmlgy8ClwOVAFrgBvc/ZV+be4FHnL3X5jZpcCH3P2Dw72vioJI8NydLQebOdR0lIa2Lurbu9hT08ajrxymob2bqVNSeMvS6ZxfVsA5c/JZWJwd+FnkT79awz/c8zK9vc5337Ncu41OUjwcfXQ+sNPdd8cC3QNcDbzSr81S4NOx+08Cvwswj4iMkJmxrDSXZaXHX7Ly//Yu47mdtTy44RBPbDvC/esOANFexIq5+bxl6XQuP336mF7uNRJxfvjETr73+Kssnp7DTz6wQkcbBSjIolAK7O/3uAq4YECbDcA7ie5iuhbIMbNp7l4XYC4ROUWpyUmsXFzMysXFuDuVde2s29fAun0NPLezjn/93Wb+9XebOXt2Hlcum8G1547ucNfu3gif+c0GHtxwkOvOKeWr156pI40CFmRRGKwvOXBf1WeBW83sJmA1cADoGfgiM7sFuAVgzpzEujCGSLwyM8oKsygrzOK6c2fh7uyqaeWRLUd4dMthvvbwNr75yHZWLS7m3eWzWHWSh7se7e7l479ex2Nbq/ncFUv4+0vma1B5HAQ5pnAh8CV3f1vs8b8AuPvXhmifDWxz91nDva/GFEQmhp3Vrfx2bRX3rauipqWTktwp3PjGMm44bw65mcNfA6Ots4db/rOC53fV8ZWrl/GBN8wdp9STVzwMNKcQHWi+jGgPYA3wPnff0q9NIVDv7hEz+yrQ6+5fGO59VRREJpae3ghPbq/hzuf28PyuOjLTkrl+xSwuPq2IeYVZzC7IJDU5ia7YuR87qlv56TO72VjVxLevP4trzxn270QZodAHmt29x8w+DjxC9JDUO9x9i5l9Gahw9z8AK4GvmZkT3X30saDyiEg4UpKTuHzpdC5fOp0tB5v4+bN7+PVL+/jFC5UAJCcZxTnp1LR00hOJ/pE6JTWJH73vXK5YNiPM6AlJJ6+JyLhr6uhmZ3Ure2vb2FPbxoHGDkpyp7BoejaLinNYUJStAeUxFnpPQURkKLkZqayYm8+KuflhR5EB4ndyExERGXcqCiIi0kdFQURE+qgoiIhIHxUFERHpo6IgIiJ9VBRERKSPioKIiPSZcGc0m1kTsGOQRblA0zDPDVx+7PFgbQqB2lOMOFiOkSw/Uf6Bjwe7r/zxkR9OfRtOlH+4NsPlHfh4Mubvfz8e8g+Xs/+FCVNtAAAH0klEQVTj8foNmuvuRSd8tbtPqBtw+0if7//cwOXHHg/WhujcTGOab7T5h9uegdui/OHmH802nCj/yWxDouUfi39DY5l/uJzDfO6BfwdOdJuIu48ePInnHxxm+YMjaHMqTvQep5p/4OPB7iv/5M8/XJvh8g58PBnzj3T9wxnL/AOfi5ffoGFNuN1H48HMKnwEE0fFK+UP30TfBuUPV5j5J2JPYTzcHnaAUVL+8E30bVD+cIWWXz0FERHpo56CiIj0mfRFwczuMLNqM9t8Cq9dYWabzGynmf3A+l013Mw+YWbbzWyLmX1zbFMfl2HM85vZl8zsgJmtj92uHPvkfRkC+fxjyz9rZh67rGsgAvr8v2JmG2Of/aNmNnPsk/dlCCL/t8xsW2wbHjCzvLFPflyOILbh+th3N2JmY77vfjSZh3i/G81sR+x2Y7/nh/2OnJLRHLo0EW7AxcC5wOZTeO1LwIWAAQ8Db489vwp4DEiPPS6eYPm/BHx2on7+sWWziV7qtRIonEj5gan92nwSuG2C5X8rkBK7/w3gGxPt3xBwOrAYeAooj5fMsTxlA54rAHbH/psfu58/3PaN5jbpewruvhqo7/+cmS0wsz+Z2Voze8bMlgx8nZmVEP3yvuDRT/+XwDWxxR8Bvu7unbF1VE+w/OMmwPz/AfwTEOigWBD53b25X9MsAtyGgPI/6u49saYvArOCyh/gNmx19+3xlnkIbwP+7O717t4A/Bm4Iqjv+KQvCkO4HfiEu68APgv8eJA2pUBVv8dVsecATgPebGZ/MbOnzey8QNO+3mjzA3w81v2/w8zG+5qIo8pvZu8ADrj7hqCDDmHUn7+ZfdXM9gPvB74QYNbBjMW/n2NuJvoX6ngby20YLyPJPJhSYH+/x8e2I5DtS7hrNJtZNvBG4N5+u9/SB2s6yHPH/qJLIdqNewNwHvAbM5sfq9aBGqP8PwG+Env8FeA7RL/cgRttfjPLBD5PdBfGuBujzx93/zzweTP7F+DjwBfHOOqgxip/7L0+D/QA/zWWGU9kLLdhvAyX2cw+BPxD7LmFwB/NrAvY4+7XMvR2BLJ9CVcUiPaOGt19ef8nzSwZWBt7+AeiP5z9u8WzgIOx+1XA/bEi8JKZRYjOVVITZPCYUed39yP9XvdT4KEgAw8w2vwLgHnAhtiXaxawzszOd/fDAWeHsfn309+vgf9hnIoCY5Q/Ntj5V8Bl4/HH0ABj/f9gPAyaGcDd7wTuBDCzp4Cb3H1vvyZVwMp+j2cRHXuoIojtG+sBlni8AWX0G/ABngeuj9034OwhXreGaG/g2CDOlbHn/x74cuz+aUS7djaB8pf0a/Np4J6J9PkPaLOXAAeaA/r8F/Vr8wngtxMs/xXAK0BRkLnH498QAQ00n2pmhh5o3kN070R+7H7BSLbvlHKP1//UsG7A3cAhoJtoZf0bon9p/gnYEPvH/YUhXlsObAZ2Abfy2sl+acCvYsvWAZdOsPz/CWwCNhL9i6pkIuUf0GYvwR59FMTnf1/s+Y1E56kpnWD5dxL9Q2h97BbY0VMBbsO1sffqBI4Aj8RDZgYpCrHnb4597juBD53Md+RkbzqjWURE+iTq0UciIjIIFQUREemjoiAiIn1UFEREpI+KgoiI9FFRkEnBzFrHeX0/M7OlY/RevRadMXWzmT14ollHzSzPzD46FusWGUiHpMqkYGat7p49hu+X4q9N+hao/tnN7BfAq+7+1WHalwEPufuy8cgniUU9BZm0zKzIzO4zszWx25tiz59vZs+b2cux/y6OPX+Tmd1rZg8Cj5rZSjN7ysx+a9HrB/zXsfnqY8+Xx+63xia422BmL5rZ9NjzC2KP15jZl0fYm3mB1yb+yzazx81snUXnzL861ubrwIJY7+Jbsbb/GFvPRjP7tzH8GCXBqCjIZPZ94D/c/TzgncDPYs9vAy5293OIzlD67/1ecyFwo7tfGnt8DvApYCkwH3jTIOvJAl5097OB1cCH+63/+7H1n3BOmtjcPZcRPcsc4ChwrbufS/QaHt+JFaV/Bna5+3J3/0czeyuwCDgfWA6sMLOLT7Q+kcEk4oR4kjjeAiztNyvlVDPLAXKBX5jZIqKzSqb2e82f3b3/PPgvuXsVgJmtJzqfzbMD1tPFa5MKrgUuj92/kNfmt/818O0hcmb0e++1ROfLh+h8Nv8e+4GPEO1BTB/k9W+N3V6OPc4mWiRWD7E+kSGpKMhklgRc6O4d/Z80sx8CT7r7tbH980/1W9w24D06+93vZfDvTLe/Njg3VJvhdLj7cjPLJVpcPgb8gOi1FoqAFe7ebWZ7gSmDvN6Ar7n7/zvJ9Yq8jnYfyWT2KNFrFQBgZsemLc4FDsTu3xTg+l8kutsK4L0nauzuTUQvz/lZM0slmrM6VhBWAXNjTVuAnH4vfQS4OTZnP2ZWambFY7QNkmBUFGSyyDSzqn63zxD9gS2PDb6+QnTKc4BvAl8zs+eA5AAzfQr4jJm9BJQATSd6gbu/THQWzfcSvXhNuZlVEO01bIu1qQOeix3C+i13f5To7qkXzGwT8FuOLxoiI6ZDUkUCErtKXIe7u5m9F7jB3a8+0etEwqQxBZHgrABujR0x1Mg4XfJUZDTUUxARkT4aUxARkT4qCiIi0kdFQURE+qgoiIhIHxUFERHpo6IgIiJ9/j+RAs6Sj5jeEQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learn.recorder.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Total time: 05:11

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
epochtrain_lossvalid_loss
10.5251710.7363620.532685
20.4090360.4255130.759533
30.3694580.3146080.868093
40.3285930.2960970.857588
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learn.fit_one_cycle(4, 1e-2)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(f'{name}-stage-1')" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "learn.unfreeze()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "max_lr = 5e-4\n", + "lrs = [max_lr/100, max_lr/10, max_lr]" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Total time: 15:46

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
epochtrain_lossvalid_loss
10.2999700.2851360.863424
20.2867530.2601440.887743
30.2776950.2694930.872763
40.2594900.2344930.895720
50.2291940.2249730.912257
60.2170030.2327600.897082
70.2021610.2152720.907977
80.2039440.2284680.894163
90.2014180.2221400.896498
100.1985990.2179330.899416
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learn.fit_one_cycle(10, lrs)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(f'{name}-stage-2')" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzsnXd4VcXWh99J75BAqAFClRoghCDSiwiiYEEpchUb115QP7EjNsSGXlGuegE7ioqigCAKUpQSeu8BAkgJNaQn8/2xT83ZJ/2ksd7nOc/de/bM3nNyZa8za9b6LaW1RhAEQRDyw6u8JyAIgiBUfMRYCIIgCAUixkIQBEEoEDEWgiAIQoGIsRAEQRAKRIyFIAiCUCBiLARBEIQCEWMhCIIgFIgYC0EQBKFAfMp7AqVFzZo1dXR0dHlPQxAEoVKxbt26U1rryIL6VRljER0dTUJCQnlPQxAEoVKhlDpYmH7ihhIEQRAKRIyFIAiCUCBiLARBEIQCqTJ7FoIgVB2ysrJISkoiPT29vKdSZQgICCAqKgpfX99ijRdjIQhChSMpKYnQ0FCio6NRSpX3dCo9WmuSk5NJSkqicePGxbqHuKEEQahwpKenU6NGDTEUpYRSiho1apRopSbGQhCECokYitKlpH/PS95YXMzI5u1Fu9h4+Gx5T0UQBKHCcskbi7SsHN77Yy+bk8RYCIJgkJycTIcOHejQoQN16tShfv36tvPMzMxC3eP2229n165dHp5p2XHJb3BbF2Zal+s0BEGoQNSoUYONGzcCMGHCBEJCQnj88ced+mit0Vrj5WX+m3vGjBken2dZcsmvLKx+PC3WQhCEAti7dy9t27blnnvuITY2lmPHjjF27Fji4uJo06YNEydOtPXt3r07GzduJDs7m+rVqzN+/Hjat29P165dOXHiRDl+i+IhK4vynoAgCPny4s/b2H70fKnes3W9MF64tk2xxm7fvp0ZM2Ywbdo0ACZNmkRERATZ2dn06dOHYcOG0bp1a6cx586do1evXkyaNIlx48Yxffp0xo8fX+LvUZZc8isLK7KuEAShMDRt2pTOnTvbzr/++mtiY2OJjY1lx44dbN++3WVMYGAggwYNAqBTp04kJiaW1XRLDVlZWJYW4oUShIpJcVcAniI4ONh2vGfPHt59913WrFlD9erVGT16tGkug5+fn+3Y29ub7OzsMplraXLJryyUxRE1eeHOUrnfr1uPkXQmtVTuJQhCxeb8+fOEhoYSFhbGsWPHWLhwYXlPyWNc8isL66ZFelZuqdzuni/WA9C2fhi/PNijVO4pCELFJDY2ltatW9O2bVuaNGlCt27dyntKHuOSNxalmSSak2v3ZW09cp6cXI23l2yhC0JlZsKECbbjZs2a2UJqwYim/Pzzz03HrVixwnZ89qw9j2vEiBGMGDGi9CfqYcQN5XDs+LIvDlk5zquTFXtPleh+giAIFQUxFg5LizEz1pToXt+sPex0LrkbgiBUFTxqLJRSA5VSu5RSe5VSLkHFSql7lFJblFIblVIrlFKtLe1XKqXWWa6tU0r19eQ8rSzfU/yVwF97T/HC3G1ObRfSK1/EgyAIghkeMxZKKW9gKjAIaA2MtBoDB77SWrfTWncAJgNvW9pPAddqrdsBtwHmTsHSmGcJxq7an0z0+Hms2HOKUZ+sdrn+4NcbSEg8XYInCIIgVAw8ubKIB/ZqrfdrrTOBWcBQxw5aa8e0zGAsuXFa6w1a66OW9m1AgFLK3xOTLMkG94iPVgEw+n+uhsLKsGl/iztKEIRKjyeNRX3A0YmfZGlzQil1v1JqH8bK4iGT+9wIbNBaZ3hikqoEa4sbOrp8HQBqhjjbteSLmew7mcKZi4VTqxQEQahoeNJYmL2FXX5ia62naq2bAk8CzzrdQKk2wOvAv00foNRYpVSCUirh5MmTxZtkCVYWgX7epu2f3tHZ6Tzu5cX0e+tPBkxZxs+bjspKQxAqOL1793ZJsJsyZQr33Xef2zEhISEAHD16lGHDhrm9b0JCQr7PnjJlCqmp9sTeq6++2in0trzwpLFIAho4nEcBR930BcNNdZ31RCkVBcwBbtVa7zMboLX+SGsdp7WOi4yMLIUpF431h8z/D2xSM4T7+zR1aT95IYMHv97AD+uPeHpqgiCUgJEjRzJr1iyntlmzZjFy5MgCx9arV4/vvvuu2M/Oayzmz59P9erVi32/0sKTxmIt0Fwp1Vgp5QeMAOY6dlBKNXc4HQzssbRXB+YBT2mtV3pwji4ri4k/byc9K6dQY3ccc1XCTJw0mEA/bx7q15yxPZuYjjt0WuRABKEiM2zYMH755RcyMgzvd2JiIkePHqVDhw7069eP2NhY2rVrx08//eQyNjExkbZt2wKQlpbGiBEjiImJYfjw4aSlpdn63XvvvTZp8xdeeAGA9957j6NHj9KnTx/69OkDQHR0NKdOGZGab7/9Nm3btqVt27ZMmTLF9rxWrVpx991306ZNGwYMGOD0nNLCYxncWutspdQDwELAG5iutd6mlJoIJGit5wIPKKX6A1nAGYzIJ4AHgGbAc0qp5yxtA7TWHheBn77yAHWq+TO2p+vKoCC6NathO/b38eaamLp8tGy/S793f9/Do1e2KNE8BeGSYcF4+GdL6d6zTjsYNMnt5Ro1ahAfH8+vv/7K0KFDmTVrFsOHDycwMJA5c+YQFhbGqVOnuPzyyxkyZIjb+tYffvghQUFBbN68mc2bNxMbG2u79sorrxAREUFOTg79+vVj8+bNPPTQQ7z99tssWbKEmjVrOt1r3bp1zJgxg9WrV6O1pkuXLvTq1Yvw8HD27NnD119/zccff8zNN9/M999/z+jRo0vnb2XBo3kWWuv5WusWWuumWutXLG3PWwwFWuuHtdZttNYdtNZ9tNbbLO0va62DLe3Wj0cMhdkGd0YhdaKa1QpxOp95e7zTuZ/PJZ/zKAiVFkdXlNUFpbXm6aefJiYmhv79+3PkyBGOHz/u9h7Lli2zvbRjYmKIiYmxXfv222+JjY2lY8eObNu2zVTa3JEVK1Zw/fXXExwcTEhICDfccAPLly8HoHHjxnTo0AHwnAS6aEOZ/CDIKqTsR7aDvMdH/+qEr7ezcWhZJ6xEcxMEgXxXAJ7kuuuuY9y4caxfv560tDRiY2OZOXMmJ0+eZN26dfj6+hIdHW0qSe6I2arjwIEDvPnmm6xdu5bw8HDGjBlT4H3yC4zx97dHYHp7e3vEDXXJ//Q1Wzzm1Xhyh6NSbUSwn2mfcW7cTSJjLggVm5CQEHr37s0dd9xh29g+d+4ctWrVwtfXlyVLlnDw4MF879GzZ0++/PJLALZu3crmzZsBQ9o8ODiYatWqcfz4cRYsWGAbExoayoULF0zv9eOPP5KamsrFixeZM2cOPXqUnbK1GAsTq5+Wad/gfmL2Jv7z+x4XA7I56Sz/nLf/EqgeZG4sujatYdq+ogTSIoIglA0jR45k06ZNNpXYW265hYSEBOLi4vjyyy9p2bJlvuPvvfdeUlJSiImJYfLkycTHG67q9u3b07FjR9q0acMdd9zhJG0+duxYBg0aZNvgthIbG8uYMWOIj4+nS5cu3HXXXXTs2LGUv7F7VFWJ+Y+Li9MFxS+bkZurafL0fKe2EZ0bMOlGw7cYPX6erX3p472Jrhns1N6yTiiTboyhQwP3oW0bDp2hVlgA3Sb9YWtrH1WNnx7oXuT5CsKlwI4dO2jVqlV5T6PKYfZ3VUqt01rHFTRWVhYmfqhj58x9h2ahslqTr6EA6NgwnPrVA53a4htHFH6SgiAI5cwlbyzM+HO3eTb4ruMX+HmTc17hXT0aF+sZHy8/wK9bjxVrrCAIQllzyRsLd/HRZkxZvIcHv94AQKu6YVzZujY3xTUoYJSdxEmDeaBPM9v5vV+uL/xEBeESo6q4yCsKJf17XvLGIj/c/XHjXv6N82lZBPiaa0Plx2MD7NFR8m9BEMwJCAggOTlZDEYpobUmOTmZgICAYt/jks+zcMfPm47SrVlN02unUgz12KZ5kvIKg+NKxl+S9gTBlKioKJKSkiiuQKjgSkBAAFFRUcUeL8bCgYFt6vDrtn8Ao3DRU4PyD4vbeuRciZ7nTrVWEC51fH19ady4ePuBgmeQn7YOvHJ9W6fz1xbszLf/jbHm9SwKy9nUrBKNFwRBKCvEWDjg41W0P8eTA/NfeQiCIFQVxFg44OOtuKu769L3wb7NTHqDj3fJ/3yF2cDbcew8P25wroGRlplD9Ph53P1Z0RMRBUEQiooYCwd8vJUtQ9uR266IdmmbNjrWpa04pFlqZ6Rn5TBpwU5SM7NJy8zh8dmb2HrkHKv2JzPo3eU88s1Gp3G/7TCULn/bfpy/9op0iCAInkU2uB3w9fLC28s17yLA15s7uzfmfysO2NoGtq1bKs9MSc8myM+H//yxh2l/7mP1gWTaR1Xnu3VJfLcuyalvZnauTfbccUUy6pPVJE4aXCrzEQRBMENWFg54mRgKAG+lePrqVsy4vbPp9aLy8wPdubVrIwBSMrIBmLrEqBy74dBZZv6VaDruXJp9Q7ywNTcEQRBKAzEWebiuQ3388uxFeHspvL0UvVtE8nC/5qx5pl+JntEuqho9mxs1w6/9zwp2/nOe/q1qFzjOsdzr/32/2elaRnbhSsEKgiAUB48aC6XUQKXULqXUXqXUeJPr9yiltiilNiqlViilWjtce8oybpdS6ipPztORQD9vdr080KnN6ppSSvHolS2oFVr8LEgrIQGGB/BiZg4Dpyxn8Q731bas5GcQUtKzSzwnQRAEd3jMWCilvIGpwCCgNTDS0RhY+Epr3U5r3QGYDLxtGdsaGAG0AQYCH1juVybk1Ysy28coKSH+Rd8uSst073pydFEJgiCUNp5cWcQDe7XW+7XWmcAsYKhjB621o+Z3MGDdtR0KzNJaZ2itDwB7Lfcrc5646jKP3Lc4xiLdZGUxoLXhvlq6S2QRBEHwHJ40FvWBww7nSZY2J5RS9yul9mGsLB4qytiy4Ao3le5KitUNVRQcK/j1b1WLID9vxlmECSf+kn+xd0EQhJLgSWNh5rtxyUDTWk/VWjcFngSeLcpYpdRYpVSCUirBU4Jj/j6e8X45riwKu8rYePis7TgzR9O8dijVA83LuZYVogoqCJcGnjQWSYBjsYco4KibvmC4qa4rylit9Uda6zitdVxkZGQJp2uOv69n/kQBvt58eEssbeqF2cJnrTx/Td6tHYO3f9ttO07PyiHAx4vAYsiklyaNn5pP0zxlaQVBqHp40lisBZorpRorpfwwNqznOnZQSjV3OB0M7LEczwVGKKX8lVKNgebAGg/O1S2elBEf1K4uTSNdZc7XJp52Or/boRpfbq7xSz4jK4cAX2+PGbPCkJVjbLjn5MrqQhCqOh5702its4EHgIXADuBbrfU2pdREpdQQS7cHlFLblFIbgXHAbZax24Bvge3Ar8D9WutySSTwlBvKSligqwuqee1QFo/rCcBLQ9vwzGD7SuNiZjanL2ayKekcK/eecirAtPfEhSI9+/DpVKLHz2NLUvGk1tMc8j4kz0MQqjYelfvQWs8H5udpe97h+OF8xr4CvOK52dmZNjqW5rVDTa/5ebhAUV5JD4Cr2tSmWa1QDrx2tUsY7+erDhLXKAKAbMsv+mcHt+LleTvo//ayIsl+LNl1AoBZaw/RLqpdkeee7rDhvi7xDFe4KRYlCELlR7ShMNd5+vupvizddZJqgb4efXb3ZpG2hLwnrrqMHs1r0qZeNcC8PvjkX3cxqktDp7aM7MJJf6Rn5ZCZk0tYgPGdlu02ggKK60ayVgwE2HX8ghgLQajCiNyHG+pWC2RkfMOCO5aQN4bF2I5v6hRFTFR1035NIu1quF+tPuR07WxqZt7uptzwwV/ETFhkO1+8w1hZZBfTWLy1aJftuLw32gVB8CxiLMqZ8GA/WtYxXGBB+YTQXt/BNc3k3REdABjVpVGBz7mYkc32Y/YcyBV77LLmZq6wwhDgUBY2Sza5BaFKI8aiAvDJbXG8O6JDvvkWw+MbuLSFWhL7GtcMZniccT3X5KV9IT2LNi8stJ3n5GoOnU4t6bSZt/mY7fi5H7dyKiWjxPcUBKFiIsaiAhAVHsRQk5WDI2bihd4OZWAbW9xUZpIgJy84v8RT0rNpGBFU6PldSM8qVPLdqI9XFfqegiBULsRYVCLyypj7OAgcZlk2uR/6egMbDp1x6nfg1EWn8y9WHyQzx25UHPdD8rLvZArtJizi0zw1NvIaIIDdx1Py/wKCIFRaxFhUIl4c2sbp3FEN9w9LGOziHSe4/oO/+GPncVv9izs/da7T/cbCXZy5aFepzcwnmuq1+TsA+NnB5QQwY+UBs+6CIFRRxFhUIupXD3Q6d1xZXBtTz+naHTMTeCkfccHHZm8CYHC7ui5yI47ENzZyOtrVr+bUfuRsWuEmLQhClUCMRSVj6eO9bceOCYNjroh26fvl6kNEj5+X7/3Cg305m+p+T+LV+TsBWH/oDNHj5xE9fh5pmTnENQoHYOX4vux79WoA4qMjivJVBEGoRIixqGRE1wwmuoaxOR3sED3lrn54QXyxysjZ+Gbt4Xz7bXaQBDmdmslzP20DoEawH95einb1qxHsL7kWglBVEWNRCYkM9QdwqRVeEsb/sKXQfZfvtsvBW7WpAn29nbSiBEGoWoixqIRMHRXLS9e1pUERwl+/GXs5U0fFurRf39EI2fX2UmRk53Axn/0LK+kmRiHAz9upOJMgCFULMRblSU42pJ0puF8eaoUF8K/LXbO2h3aoZ9LbIMDXm8ExrhpYEy0RVqPiGzLo3eVOyXsAwX6urqVUE2Oxan8ym5LO8di3mwqcvyAIlQ8xFuXJT/fDp9dC2tmC+xaCl65ry4Rr7XLmjpvh1lXItNHOq4vQAF/Cgwxhwf0nnfMxHMc5YpUHcTRO1vDb79cnmWaRC4JQuRFjUZ7E3AQndsKXN0FGyRPawgJ8GdPNXijJugEe4u9DRLBRftVMYTckwMdt+GxmjmsOhtWo3Nm9scs1gHd/32PaLghC5UWMRXnSrD8Mmw5HEuCbWyArvVRvH+Lvw7In+vDrIz2c2h/t38Lp/PDpNOZsOGI7dwyjdUzYa1MvzGmcY+ElRzX1d3/fk2/uhiAIlQ8xFuVN6yEw9APYvxS+uwNysgocUlgCfL1oWCOIqHBnV9Id3aPzHZeYbBcZdKyV8d7IjjSvZS8D6yhLvuBhZ4P0wZK9xZmyIAgVFI8aC6XUQKXULqXUXqXUeJPr45RS25VSm5VSvyulGjlcm2wpubpDKfWeMqsEVFXoMBKufhN2zYMf74PcwhUzKgh3f7LQAF/WPduf7ROvMr2e7eB6SknPxtfbuE+dsACn1YRj/e+WdcJ4qJ+9pPoHS/eVaO6CIFQsPGYslFLewFRgENAaGKmUap2n2wYgTmsdA3wHTLaMvQLoBsQAbYHOQC9PzbVCEH839HsBtnwL8x+DQqi8loQaIf4E+ZlLon/290EAUjOzScvK4bEBl5E4aTDB/j5OVfUC8hQ8smZ1uyM3V/PduiSyTPZBBEGo2HhyZREP7NVa79daZwKzgKGOHbTWS7TWVp/HKiDKegkIAPwAf8AXOO7BuVYMeoyD7uMgYTr89nyxDcbQDvVcdKSKgvVlvuPYBcDI0rbiWEApwMfZWPRsEel0Hj1+HnuOX7Cd/7bjOI/P3sR7sgEuCJUOT9bgrg84akgkAV3y6X8nsABAa/23UmoJcAxQwPta6x15ByilxgJjARo29HwJ1DKh3/OQcQH+eg8CwqDnE0W+xbsjOhapv5+3l1PUU3zjCM6mZnLjh38B7mt8W91T+fHq/B3MuD0eMFxaAElnRIRQECobnlxZmL1JTH8qK6VGA3HAG5bzZkArjJVGfaCvUqqny820/khrHae1jouMjMx7uXKiFAyaDDEj4I+XYdU0jz/yx/u7OZ2nZ+Uy4iN7IaP64earlMJsI6U6ZHX7WIxLcWt+C4JQfnjSWCQBjrVAo4CjeTsppfoDzwBDtNbWijrXA6u01ila6xSMFcflHpxrxcLLC4ZOhZbXwK9PwoYvPPq41nlCYp+es4Wd/9jdR71bFN4Q77co0FpZfeC07XjRdsOTeORMyUu6CoJQtnjSWKwFmiulGiul/IARwFzHDkqpjsB/MQzFCYdLh4BeSikfpZQvxua2ixuqSuPtY+RgNO0Lcx+EbXPKbSqOK4gWtY3Q2XrVXMu8Qv7qt9aa3esPlU7GuiAIZYfHjIXWOht4AFiI8aL/Vmu9TSk1USk1xNLtDSAEmK2U2qiUshqT74B9wBZgE7BJa/2zp+ZaYfHxh+FfQIMu8P3dsHuRxx5lVg8DoJlDXgXA9DGd+XevJqx4sq/be704xLmiX5KsJASh0uPJDW601vOB+Xnannc47u9mXA7wb0/OrdLgFwyjvjE0pL79F4z+HqK7l9njFz3ivFUUFR7EU4Na5TvmYqZz9nb315dw4LWr3fQWBKEyIBnclYGAajB6DoRHw1fDIWldqT/ihtj61AlzdS0Vp6jSqv2nXdpGfrzK6TxHNrkFoVIhxqKyEFwD/vUjBNeEL26A49tK9fYxUdVZ9XQ/Zt/T1db2y4PFW8GYCQyu2n/aST+q6dPzuZBeetImgiB4FuWu9nJlIy4uTickJJT3NDzPmUSYPhByc+COX6FG0/KekSk5uZpnf9zK12uMsq1Xtq7Nb9uP06ZeGNuOGol9oQE+DO1Qj5eva1eeUxWESxql1DqtdVxB/WRlUdkIj4ZbfwKdA58NhXNJ5T0jU7y9FBOG2NVdfrOEzTZwEDW8kJ5tqwEuCELFRoxFZSTyMvjXHEg/bxiMlBMFjykH/H1cq+y5S/ATBKFiI8aislK3PdzyLZw/Cp9fX6zyrGXBiif7OJ2bbaILglDxEWNRmWl4OYz4Ek7thi+GGZpSFYy8tTR8TPSkRIVWECo+YiwqO037wrAZcHQDfD2y1KvtlQbD4xrgRS69vDZxxfrH+az5Mqfrn/6VWD4TEwSh0IixqAq0ugau+xASl8PsMaVaba/EnDvCqPRZLPN/hE/9XqfJ6WX0PDyNK7y22rq8PK/4Si7/nEvn8GnJEBcETyPGoqrQfjgMfgt2L4A59xihteVFTjbsnG8kEE5pS/u9U9mfW5f7Mh/i1TZzoUZzpoVOZ0Azu4vqzMXMIj9Ga83lr/1Oj8lLSnP2giCY4FG5D6GM6XwXZKTA4hcMmZBr34WyrEZ75iBs+NxQyb1wDELqQPdxLA8dyK0/GBFbI1QwXPchYdMH8HrILBZhyISdTs0k3KHIUl601vy5+yS9WkTahA2ldKsglB2ysqhqdH8EejwO6z+FRc96vDwrOVmw/Sf4/AZ4tz0sfwvqxMCIr+DRbdDvObwj7Bnd2bkaGnSGKx4ifOcsenttBMDLYgDeWLiTtYmuciHfJhxmzIy1/G/FAVvbrLXuczTOpWZx5KxRZCk9K4f0LPOVVnpWDsM+/IstSeeK/t0F4RJCjEVVpO+zEP9v+Pt9+HOyZ56RvA9+ewHebg3f3gond0Hv8fDIFiOkt+VgQ2YdCPa3L2BrhFhWD32ehshWTPL9mDBSyMzOZeuRc0xdso+bpv3t9KifNx3lye+3AM77G75e9v98z6U579O0n7iIbpP+YOPhs7R87lfaTVhITq7mj53HcVQt2Jx0joSDZ5jwc+nKpwhCVUPcUFURpWDgJMhMgaWvgn8IdL2/5PfNzoAdP8O6mcZmuvKGywZB7G3QrB94uSbhAQT729sf7d/COPDxh+s/pOZ/+/KC7+d8vaYtM02ioo6dS2Pqkr228yHt69mOrSsHgMe+3cgnt3V2GX/d1JUAZOVopv25jzcW7qJ9g+r8ZKkOuO9kCiAy6oJQEGIsqipeXnDte4bBWPg0+IVAp9uKd6+Tu2Ddp7Dpa0g7DdUbQd/noMMtEFa3wOGOK4sAXweDUq8jM71v5C41mw3bfwHa2i7l5GpSM7Pp+tofTvey5mTMTjjsVBt893HjpZ+WmcOCrcdM5/HGwl0AbDpsL7701A/GiuX4+QzTMYIgGIixqMp4+8ANn0DmRfj5YWOF0fbGwo3NSoNtPxp7H4f+Bi9fw7XU6TZo3NswRoXE0Vjkxaf3/7Ft8RqeyPqQX5jEWUIB2Hb0HL9u/celf5pl7+GJ7zY7tR86nUpOrqbV878Wel5VRURTEMoCMRZVHR8/uPlz+HIY/DAWfIPhsoHu+/+z1TAQm76BjHMQ0RSunAjtR0FI4WtxOxLka+6eArg2thG3LLiXX7ye5SXfGTyY9RAAQ95fadq/fnX32lJnUwsXftukZjAA//689OuCCEJVxaMb3EqpgUqpXUqpvUqp8SbXxymltiulNiulfldKNXK41lAptUgptcPSJ9qTc63S+AXByFlQp52xGX3AOYOajBRY/xl83BemdTNcTi0GwG2/wIProNvDxTYUAD7e7v8z8/f1ZqduyDtZN3Ct9yqu9lrlti8YbqaVe0+ZXruQnm3a7o5FFiVcgGLUeBKES4pCGQulVFOllL/luLdS6iGlVPUCxngDU4FBQGtgpFKqdZ5uG4A4rXUMRt1tx9Cdz4A3tNatgHigYkqrVhYCwmD0DxDRBL4aAYfXGhIhPz8Cb7WEuQ8a7qqrXoPHdsKNn0DjHqWWpxEe5Ms9vVxrb4RYXFTTcq5lU24TXvKdQQ3ch7FuPHyWWz5ZbTtvWz/Mdtz7zaWFmsv+Uxc5ecF5jyJXwwdL97oZIQhCoYofKaU2AnFANLAQmAtcprV2W1hZKdUVmKC1vspy/hSA1vo1N/07Au9rrbtZjMpHWutCl2q7ZIoflZQL/xjFk84eMmpi+ARCm+uh0xhoEF+2SXwWosfPA6CZSmKe3zMsye3APVmPAAXP5a2b2vPY7E2lNpfESYOZt/kYGs01MfUKHiAIlZzSLn6Uq7XOBq4HpmitHwUKCoOpDxx2OE+ytLnjTmCB5bgFcFYp9YNSaoNS6g3LSsUJpdRYpVSCUirh5MmThfwqlzihdYziSa2uhavfNFYR138IDbuUi6FwZK+O4q3sYQz0XstQL/M9i7wMaFPbtP22ro1M2wvD/V/hDkaMAAAgAElEQVSt54GvNhR7vCBURQprLLKUUiOB24BfLG2+BYwxe/OYLmOUUqMxVi5vWJp8gB7A40BnoAkwxuVmWn+ktY7TWsdFRhbfp37JEd4Ibv4U4u+GwHy9iWXCXQ41u9fWHUVSSDte9P2UWjjX6HjtBufyqyvH9yU0wJfIUH+n9rduas+EIW1Mn7VyfN9Cz8sappuelcO51CyycnJZd9A1u1wQLgUKayxuB7oCr2itDyilGgNfFDAmCWjgcB4FHM3bSSnVH3gGGKK1znAYu0Frvd+yovkRiC3kXIVKhpfD7nK/1nU5O2AK/mTxmu8nOP6+CHEIwR0e18AWGXVla+fVxQ2x9VFKseDhHjSNDLa1d44Op371QJrXCnHq7+dj/2fwl8Pm+TSL9tTw//5N+4mLeGbOFm788G+nZEBBuFQoVOis1no78BCAUiocCNVaTypg2FqgucWwHAFGAKMcO1j2Kf4LDNRan8gzNlwpFam1Pgn0BWRDoorTv1Vt7uvdjHWHzjA5ezgv+H7Om4228vheY0URGmD/zzXXYa/txSFt+Gq1oRO1feJVNqHBVnXDWPRoLy5mZuPv44WPJTckyGJ0agT78fdT/TiXlkXnVxYDMMph8/zoOaM2yCaLbtS3CUa989SMokVdCUJVoLDRUEuVUmFKqQhgEzBDKfV2fmMsK4IHMDbEdwDfaq23KaUmKqWGWLq9AYQAs5VSG5VScy1jczBcUL8rpbZguLQ+Lsb3EyoBfpbQ2u7NauDlpYhrFM4vAUM4HBbLsJNTqUsy4KyJmOtw7OvtxRd3dqF/q9oE5snp8PZShAX44u/jjbdlBWPN4E6+mImfj5eLG8tKZIi5Cu4mER0ULkEKm5RXTWt9Xil1FzBDa/2CUmpzQYO01vOB+Xnannc47p/P2N+AmELOT6jEjO3VhKycXEbENwRAKcXa5wbA6ebwYTde9/2Ip4JeJMjPbgjSspx/3XdvXpPuzWsWew4Trm3NhJ+3O7V9teYQD/Zr7tL38dmbGNYpqtjPEoTKSGH3LHyUUnWBm7FvcAtCqRAW4MtTV7dy1o0CiGgMAybS03sLS/smEt84wpY8N3+LqxRISTCTJDmVkuk2AVAQLjUKaywmYriT9mmt1yqlmgB7PDctQbAQdyc06Y3v4udQZw+y6ql+AHxwS/HjHW6IdY3gDnGjXzV7XVKxnyMIVYlCGQut9WytdYzW+l7L+X6tdSEV6QShBCgFQ9435M9/vJ9aIX4kThrM1e0KVrt1x5vD2gM4ubUuZpoXR5q32VzB1pHsnNwC+whCZaewG9xRSqk5SqkTSqnjSqnvlVLitBXKhuoN4KpX4eAKWFvyOAcvL8X+V69m24tX2dryyn/kxTESy5GfNh6h2TMLOJh8scTzEoSKTGHdUDMwJD7qYWRh/2xpE4SyoeNoaD7AqM6XXPLa215eyhZiC3B9R7tr6n+3xXFvb2cdqx8txZLy8p3FTbXj2IUSz0kQKjKFNRaRWusZWutsy2cmICnTQtmhlFHMyccPfrwXcs3dRsWldpg9fLZfq9rc09PZWDSpGczicT3p36oW4UF28YLle4wNcKmNIVR1CmssTimlRiulvC2f0WAJfheEsiKsLgx6Aw6vhr+nluqtVR5dLMdSsNbrzWqF0iQyhLSsHJbtPmkTQATnvA9BqIoU1ljcgRE2+w9wDBiGIQEiCGVLzM3Q8hr442Wj3Gspcn+fpkwbbURZ+Xh7mda4CPD1Jj0rl1unr3Fq1+ayZ4JQZShsNNQhrfUQrXWk1rqW1vo64AYPz00QXFEKrnkH/IJhzj2QU3rSG09c1ZKBbe1RVvtfGwxA3WoBtra8GeJWcnI1W4+cI3r8PCYt2FlqcxKEikJJKuWNK7VZCEJRCKkFg9+Co+th5RSPPipx0mD+tuR2AAT6Gv9kQvPkZWRk5zLzr0QApv1Z8g14QaholMRYSCFKofxoe4NRtGnpJKNueBkRaMnNuJBHTDA1I5vle6SmilB1KYmxECetUL5c/ZZRj+PHeyAnq0we6SJJYmHCz9s5ft7I1WjiIIsuCFWFfI2FUuqCUuq8yecCRs6FIJQfwTXgminwzxZY9maZPNLdnoUj+09Kgp5Q9cjXWGitQ7XWYSafUK11YRVrBcFztLoGYobD8jfh6EaPPy7Qz9VY+Pt4EeBbkkW6IFR85L9wofIz6HUIjjSio7Lzl+0oKVZXkyMt64SSniX6UELVRoyFUPkJDDeyu0/uMDa8PYhjpneTyGAe6NOMzBzZvhOqPh41FkqpgUqpXUqpvUqp8SbXxymltiulNiulfldKNcpzPUwpdUQp9b4n5ylUAVoMMPSjVk6BJM9V4O3R3K5y8+KQNjx+1WXsOHbepV9mtqw0hKqFx4yFUsobmAoMAloDI5VSrfN02wDEaa1jgO+AyXmuvwT86ak5ClWMq16F0HqGOyorzeOPC/Iztu3ioyNsbSMt1f5EhVaoanhyZREP7LXUvsgEZgFDHTtorZdorVMtp6sAm+y5UqoTUBtY5ME5ClWJgGow9H1I3mPIgXgIq4yUtR7Gte2NrG8fL8XWI0Z97ie/N686/MP6JL5cfdBjcxMET+FJY1EfOOxwnmRpc8edwAIApZQX8BbwhMdmJ1RNmvYxquv9PRUO/u3RR1mNRYil1kWO1gyOMQzH+kNnTceM+3YTz8zZyh87j3t0boJQ2njSWJhleJvuBFpUbOOANyxN9wHztdaHzfo7jBurlEpQSiWcPCnZs4KFKydC9YaGlHmm59xBVjdUsOV/tYaxPZoA0NQkMc9RxnxBKdcQFwRP40ljkQQ0cDiPAo7m7aSU6g88AwzRWlvjErsCDyilEoE3gVuVUi5hLlrrj7TWcVrruMhIKa8hWPAPges+gDMHYPGEUr9992Y1AXv1vBwHfXIvi1TtPpPEPKt2FEBEsF+pz0sQPIknE+vWAs2VUo2BI8AIYJRjB6VUR+C/wECt9Qlru9b6Foc+YzA2wV2iqQTBLdHdocu9sPpDQ9K8Sa9Su/VbN7dn74kUm/RHppsa3Nk5ufh423+PvbVot+04ItiPv/cl07VpjVKblyB4Eo+tLLTW2cADwEJgB/Ct1nqbUmqiUmqIpdsbQAgwWym1USk111PzES5B+j0PEU3hpwcgo/TKntYKDeCKpjVt592a1TTtdzHDuZpfioP44GsLdjLy41WsO3i61OYlCJ7Eo3kWWuv5WusWWuumWutXLG3Pa63nWo77a61ra607WD5DTO4xU2v9gCfnKVRR/ILgug/hfBIsetZjj6kZ4m/avsUSGQXuy65+seqQR+YkCKWNZHALVZuGXaDrA7BuJuxd7LHHfPvvrvzxmLOra/T/VtuOs93UXZ2z4YjH5iQIpYkYC6Hq0+cZiGxpuKO2/egROfP4xhE0iQwBIDzI1+X66v3ibhIqN2IshKqPbwDc8BEob5h9G7zTBn6fCGc8kxw38/Z4l7ZlDoWRrm0v6v5C5UOMhXBpULc9PLIZRn4D9TrCinfg3fbwxY2w45dSreVtDakF2H38AulZOcxcmQjAsif68NZN7UvtWW7JzYEDy2Hr98axIJQQqUkhXDp4ecNlA43P2cOw/jPY8Dl8c4uhKRX7L4i9FapFFXyvfKgWaHdDDXhnGVe1qW0Lr60W5Iufj/NvtE2Hz9K+QXW390tOySDIz8e0loYTOdlwcIXhatv5C1y0rGaiZ8ANH0NY3eJ9IUFAVhbCpUr1BtD3GXhkKwz/Emq3hj8nw5R28NVw2L2w2L/Ia4T4ExNVzXa+cJtd2sPfx/WfXEG1uzu9vJhh0/5yaoseP4+7Pl1r7L/s/R3mPgRvtYDPhsLmb4w8k5tmwpD/wJF1MK077LFv8K87eJr0LFlxCIVHVhbCpY23j1Ftr9U1cCbRWG2s/xx2/wrVGhgrjY7/KvKv8g4NqrM56ZxLu5mx+Od8utv7LNll5KpuO2rIoO/65wJ1gr3o7bWRK/esgTc3QdoZ8AuBFgOh9VBo1t8IG7bSoAv629tQX94I3R4hMeZRbvzwb0Zf3pCXr2tXpO8lXLqIsRAEK+HRRiJf76dg5zxYNwOWvGIUVLpsEMTdDk36glfBC/IAk1rdcY3CUcpVMq16oHvpj9tnrAXAjyzYvZAtn79PHa8EZvqlcl4HQvMhhoFo2hd8A03vcSaoMZcffpKvo+YQu3IK5/9aQH3uY/c/Eab9BcEMMRaCkBdvX2hznfFJ3gfrP4UNXxj7ANUbQafbjNVGSC23tzBT0XxneAeXNj8fL7LcyIWQlU5/r3Vc7b2a/l7r4Ks0BngF8VtuJ+bldGFFbjt233BdgV/nVEoGGfhxe/Johvo244nMD5jv/xRf6CcxZNgEoWDEWAhCftRoaqjY9nkGdvxsJPf9PhGWvAotB0PcHRDd02W14SgaaKVBhN01dF/vpmxKOsu+Exc5k5pp75SVZiQPbv8Jdv3KJ34XOKuD+TUnnvm58azMbUdWEf/ZTrdEYp1Pz+Iz3Yml6lXe932P+4+/APOPwoCXwMc8C71SkbwPfn8Rjm+DRldAkz7QuBcEi/5WaSDGQhAKg48/tBtmfE7tMYzGxi+Nl3pEE+g0BjrcAsGGTlRGnrKqbeqFOZ3/38CWAFz59p9kpKUYEUzbfzI21rMuQmAEtL2eW1fV5a/cNmS7+aeamZ2Ln48XGw6dISo8iMhQ15f+12sMSRGr4sghXZthmRN40mcWd675LxxeBcNmGIaxMpJ62ghOWPsx+ARAw66w7Sdj/wkFdWOgSW/DeDTsauTdCEVGudOsqWzExcXphATP1V4WBBey0mHHXEiYDof+Bm8/aHUtxN3BjKR6vPjLDmKiqrE56RzfjL2cLk0cfuFmpMCehaycO524rAT8dToE1TTGtx4K0d15c/F+3l+yN98pbHz+Sl78eTtzNhyhbf0wfnmwh0ufYR/+RcLBM6bjd43W+P9yP+RmwzVTIOamEv1JypTsDFjzMSybbAhFxt4KvZ+G0NpGGPHRDbB/KexfAofXQG6WxZhcbhiOJr2hTkyh9qCqMkqpdVrruIL6ycpCEIqLbwDE3Gx8TuwwVhubvoat3zOmZgu6drmRZlfezUWvMKoF+RovtN0LYdscw9WUnU4br3Dm0oubbnsAGl5hRGdZKMhQgKFka9WX2nrkvGmfK5rWcGss0ptchf89K+D7u+CHu+DAnzBosnM0VUVDa2MVtvgFI4KtWX+48iUj/NmKtw806Gx8ej1hGOeDfxmGY/9SYywYK7gmvezGI7xR2X+fSoIYC0EoDWq1gkGvQ78XYPuPqIQZtNz0Gmx9m2qth0JmipEPkZMBIXWMX8GthxI77Qy5eNGrZjy1HAxFYXMgDp9OK7BPWp57XdehHj9uNOqQZeTkGDknY+bB0ldh+duQtNbI0ajVqvDfv6xISoCFzxius1qtYfQP0KxfweP8Q6DFAOMDcOEf2P+n3Xhsm2O0RzSxG47GPSHQfbLkpYYYC0EoTfyCoMMo4/PPViP8dvO3Rh5E3B2Gi6lBF5vrI5d5AJy5mMXpi5nUrx5IaIAvX612li5/7YZ2PPXDFpfHjfx4ldN5SkY2z87ZwiP9WxBd0yjtmtdYNK4Zwg0d6/PDhiP833ebaR9VnUevbGGEDUd3hx/Gwkd94OrJRtSXSbhvmXPmoLF5vfV7CK4F175rzM2rgKx2d4TWgfbDjY/WcHKX3WW1+RtI+B8oL0Mapkkfo7Z7VOeqEQhQTMRYCIKnqNMWBr9luHVQ+frGk1MyGPWJIWm+5PHeTi/4W7o0pF39au6GOtH2hYUA/LjxKImTBgOQmulsLJIvZtDrskh+2HCEpbtOsnTXScNYgJGvcc9K+OFumPug8ev7mncgwHmDvsxIPwfL34JV04yXd88noNvD4B9aes9QCmq1ND6X32NkxScl2FcdK96B5W+CbxA06masOpr2MVY2FcGQlhFiLATB0+Tz63dkfEO+XnOIWWsP29r6vLkUP4dyrFe3q0vbQhoLM/7am0z96oEcOWu4rC5m5Djd34XQ2vCvObDibSNE+Oh6I1qqnmueiMfIyTL2gJa+BqnJ0H4k9H0OqtX3/LO9faFRV+PT52nDYCWuNIzHviWw6BmjX3AtS5RVb8N4hFVtNWGPGgul1EDgXcAb+ERrPSnP9XHAXUA2cBK4Q2t9UCnVAfgQCANygFe01t94cq6CUB7c1aMxX685xNxNR53arcKDoy9vSJfGRqb15BtjiAzzp05YAM1rhdDsmQUF3n/nP+dtciLbXryKP3efZFDbOnywdJ9TP2u98IPJF6kW6Ev1ID/jV3yjbvDdnfC/K2HAyxA/1rO/prU2ggB+ew5O7YboHsZzy9JQ5SWgGrS82vgAnEuyuKyWGgZky7dGe80WhtuqdhvjU6uN4e6qIqsPj4XOKqW8gd3AlUASsBYYqbXe7tCnD7Baa52qlLoX6K21Hq6UagForfUepVQ9YB3QSmt91t3zJHRWqIwkp2TQ6WX3Ffy2T7yKID/z33TR4+fle+/ESYOd+ljdUmAo3Q6dutKp/7ODW/HyvB0ufbmYDD/dZ+hltbwGhr4PgeH5PrtYHNtklL89sAxqNDMinC4bVLFftrm5cGK7YTQOLIfjW+G8Q/XDwAi78bAakFotwS+4/Oach4oQOhsP7NVa77dMaBYwFLAZC631Eof+q4DRlvbdDn2OKqVOAJGAW2MhCJWRsEDXqnqOuDMUjoyMb8h9vZvSY/ISp3ZHQxEV7qwbVd2kmp/VUACcuJBOrVBL8lpwDRg5C1Z9AL+9ANN6wLDp0CCeH9YncUXTmtSpVoJEt/NH4feXjLDjwHAY9Iahw+Wd/9+mQuDlZexN1WkLVzxotKWeNgzI8e2G8Ti+zRCnzLpoGaQgorHFgLQ19j5qt4HwxhU658OTxqI+cNjhPAnokk//OwGXdbVSKh7wA/a5jBCESo6vtxf1qgVw9Jx75Vl31AkL4J/z6bx2Q8HKsYPa1nE6NxM6dCT+ld/ZMXGgvYaGUtD1fmhwOXx3O0wfSFbvZ3lsQXM0Xs4rkcKSkQIr34W//gM6x3jZ9nis8oerBkUYUWXR3e1tublwNtEwHFYjcmK7UXgLi3fHN8gIV85rRIIqhuCjJ42F2drR1OellBoNxAG98rTXBT4HbtNau6itKaXGAmMBGjZsWNL5CkK58MZN7bnFEglVFJY+0ZvsXPs/qRljOnP7zLWmffNqFQb4FBxyumTXCa5ul0eaPaoT3LOcnJ8exHfJi8z0jWFc1r1Fm3hujiGV8sfLkHIc2twA/V8wVH+rKl5eRg5HRBMjS99KZiqc3GExINsMI7LjF4tUiYXQus5urNptjP0RH/dqxZ7Ak8YiCWjgcB4FHM3bSSnVH3gG6KW1znBoDwPmAc9qrVflHQegtf4I+AiMPYvSm7ogVHzyrg76tHSvghsR7OzSCfAr2N3x0bL9LsYiMzuXFhNWADczyrsmL/h8xnz/p+BAlJHEVhD7/oBFzxkvxah4o/BUg84Fj6uq+AVB/U7Gx4rWhhE9vs3+ObHN2MvJsYhOevkYBqN2G2MFUq+jEZHlQTxpLNYCzZVSjYEjwAhglGMHpVRH4L/AQK31CYd2P2AO8JnWerYH5ygI5c4VTc1VUX95sLtpe34s/78+/Lr1H16Zb99/eLhfc8b2dBYJzDd01sLGw65bhD9ttG7eKr7K6cf63OZM9X2X2p8OgV5PQq//Mw8VPrHDMBJ7fzNk3m+aCa2vq9ib1+WFUkYUVWgd5+z0nCxI3utsRA6tgi2zDcNbWY2F1jpbKfUAsBAjdHa61nqbUmoikKC1ngu8AYQAsy1FYQ5prYcANwM9gRpKqTGWW47RWm/01HwFobxwLIjk46VsrqX61c2LGeVHg4ggDp9JdWobGd/Qpe63WRGmvFxWO5S1iaeJbRiOt5fRP+/yfaduyLWZr7C9y2L4cxIkroAbP7bnHKScMHI11n8KfqFGhFOXf1/SmdDFxtvX2NOo1cpQP7aSdhbSTnv88R7Ns9Bazwfm52l73uG4v5txXwBfeHJuglARWfJ4b1tUk4938X5139q1EZ/9fdB2ntdQFMS7Izrw7uI97Dp+gZum/c0j/ZvzSH8jw7uaSfRWKgHo6z5ANe4J8x4z6n1f+x6c3GlkP2enQ+e7jZWH1JYofQKrl0lQQMWN0xKESxDHAkmBBUQsuaNZrVASJw2mQYSxMnFndLo1M17cy//P7r6YMrwDQzvUJy7ankdhVbO9bfoa/v35OtN7xb70G+dbDoOxS40N2W9ugT9eMooP3bfa0JkSQ1GpEbkPQaig+BRiXyE/vr/nCv7en0xYgHm+wqe3x5Odq502yq9qY4TYhjqM8bG4oP7cfdLlHk8ObMnrv+7kTGoWs9YcYmzPFnDXYlj9X2PTtrFrfQ2hciIrC0GoAHRoYHcjWOU9SkqtsACGdnCvpeTj7eUSUWV1WTlKpOfmo/Jw5Kx9f+RcWpZx4BsI3R8RQ1HFkJWFIFQAvrunK9aUiS/u6kJ2TtlGgn9yaxxfrj5o28h2XFks2n4cM1mgTo3CGdyuHl+sMuTU61Yr+oa8UHkQYyEIFQBHl5OvtxfF3K4oNv1b16Z/69q282A/5wnkFToEeOHa1rSrXw0/by8yc3JNDYpQdRA3lCAILkSEOGcHPzxrI0EOBmTRoz2JiaqOUopNLxjV51IyClfdT6iciLEQBMGFm+Ma8KglXNaKtYhSsJ83LWrbiw8F+Hrh7aVIycgq0zkKZYsYC0EQXPD19uLh/s1d2ttHVePXR5xlPZRSeCk45FAP/O1Fuxjx0d8kp2TYN76FSo0YC0EQ3PLydW2dzjs1inDKBbGSlaP52WFf470/9rJq/2k6vbyYzq+4r9chVB5kg1sQBLf0bB7pdO5VQFJ5elYOX6w66NSWme0iGC1UQsRYCILgloY1nFcRjpLoZlw1ZRkHk1Nd2rXWhdKjEiou4oYSBCFfoh0Mxq1dG5n2aRpplAk1MxQA//ljb+lPTChTxFgIgpAv3/y7K9fE1GXnSwNpEhli2ie+gKzzD5aWjrFIz8rhvi/XceDUxYI7C6WKGAtBEPKldlgA74+KzbcUa/uo/FVP07NKJ2lv/aEzzN/yD33eXMrnfye67bfn+AWy85YHFEqEGAtBEErM8M4NCuxzMbPkSXv+DuVgn/tpm8v1mSsP0PipeVz5zjJe/3VniZ8n2BFjIQhCiXG3eb1lwgDbcWpmdomfk1aAwZnw83asC5iPlx8o8fNKm8ocGSbGQhCEUiHA1/l1Uq9agJMgYUEv+sJwsRQMTnmx49h5Wjy7gIXb/invqRQLjxoLpdRApdQupdRepdR4k+vjlFLblVKblVK/K6UaOVy7TSm1x/K5zZPzFASh5Izt0cTpPCfPHsXFUtCOclydhAeZ1+koDy6kZxE9fh5zNiS57WOtB/Lvz9exbPdJNie51jivyHgsz0Ip5Q1MBa4EkoC1Sqm5WuvtDt02AHFa61Sl1L3AZGC4UioCeAGIwyj7u84y9oyn5isIQsl4qF9z0rNz6duyFr/vOM51HY1aGp/eEc9t09eQllXyVYGjwTmTmkVGdo5tH2Px9uMlvn9xWbb7FACPfrOJ6ztGmfaZtMC+h3Lr9DUAJE4a7PnJlRKeTMqLB/ZqrfcDKKVmAUMBm7HQWi9x6L8KGG05vgr4TWt92jL2N2Ag8LUH5ysIQgnw8fbi6atbAXB5E3sJVata7YX0ohuLzOxcLqRnUSPEn2Pn0nh9gfOmdUp6Nv4hxv3v+iyhuFMvMZk5+a+aTl7IKPEzzqdnua16WBZ40g1VHzjscJ5kaXPHncCCYo4VBKGCEmD55T87wb2Lxh2Pzd5Ep5cXo7Xm4VkbuZBhGByrZlVmAeGxP208UuRnFobcXE30+Hm8tWgXAOFBdkn3C+nOwok9Jy8pkT5WelYO0ePnETNhEd+sPVTs+5QUTxoLs/AI00BrpdRoDJfTG0UZq5Qaq5RKUEolnDzpWh9YEITyp2VdQ848xL/ojox5mw1xwpMpGU77Fdbgq53HLgDQ6rlfTcf/svlYkZ9ZGKxKutbMdGs5WoB2Exaxan+y7fzQafOsdihcdNTpi5m243cX73G5/tPGI2Wyae5JY5EEOAZfRwEu5baUUv2BZ4AhWuuMoozVWn+ktY7TWsdFRkbmvSwIQgXA19sLfx8vqhdjQ9oqRRX/yu9kZdt/L7apVw2A22eu5WxqJmlZ5m6g3zy0j3EmNdPpPO9Lf8RHqwC4/8v1+d7H0RCY8d8/93HFpD9s50fPpbv0+Xj5fr5Ze9ilvbTxpLFYCzRXSjVWSvkBI4C5jh2UUh2B/2IYihMOlxYCA5RS4UqpcGCApU0QhEpIgK836W5e6Pnh7SBzu+v4Bduxv8Mv+Q4Tf3MZd1f3xkV+VlE4k2p3NX295hAZJiuE9Kwc5m1xXtkse6KP0/naxNP5PqegXJE9xy+w9ch5U1dMaeMxY6G1zgYewHjJ7wC+1VpvU0pNVEoNsXR7AwgBZiulNiql5lrGngZewjA4a4GJ1s1uQRAqHwG+XqRnFT0hLceNym2NYD/TdoCEZ/vz7DWt3V5/Zd52osfPY91B51fK0bNpHEwunObUWYeVxVM/bOHfn69z6ZOS4bqh37BGELc5iDE++PWGfJ/jU4Am/JXvLAPg950n8u1XGnhUolxrPR+Yn6fteYfj/vmMnQ5M99zsBEEoKwJ8vUnPLr0a3bXCAkzbF4/rRc0QfwDu692Uj5btJydX21YoP244Yvu1/vQPW1n4qL3qn9XdU5hw1rOpBVf/c7eS2nDYnl/RvJa5MKOVIH/3elxljWRwC4LgcQ4mp/LTxqNMXbK30L/e3fHLg93dXmtSM9h2XD3Il+xcbdvPOH4+nUe+2Wi7Xj88sNhzcLdH0sVBfTdvOdmJQ9sAOAkytq1fLd/n7D9p/1sN62Tkb1gFElJJbMkAABE0SURBVFfuPVWEGZccMRaCIJQZbyzcRa83lhZ7/C8Pds/3Bevl4LYJtLyUrTIji/Jsdl80cRMBDH1/RYHzcLdq+PKuLky6oR0ASWfsNcm7NI5gdBfD/fT5nfF0ahROkJ83R8+mmd7HjIaWcrbWxMRbPlltu/bdPV0LfZ/iIsZCEASP8+EtsU7neX91F5bIUH+31x4f0MLp3FrVzxpxVDeP6ypvRJOVTUnnmL/lWL5yHGYb2r882B0fby+bG8y6j/Fg32bMGnu5zZD5+3jz/b1XkJqZw+oD+W/FdmtmT2603vdChvPf7vlrWhMXnX89kdJAjIUgCB7HGupqxSxfIC/Wze1H+je3tTm6cK6JqevUv3ueeuG7j6cAMO5bw/WU9wV/+qL9pZuVJ7nvvi/XM+T9leS62WA3MxbWFU/NPAatX6va+ZaUTU5xn929cq+RrzHj9s5UCzRCjy9m5JBRivs/hUWMhSAIHic82DnHYvrKA/m+JAGWWYT3HA2E9YUJ8M7wDkwbbV+x1K3mvHK4s3s0ANuOngfg/q+ccx46NbIXbPpgyT7TOaw6kGzanpHHDdXMYaO6SWSw22tm7D2Rku91gD6X1SLYstmdkpHNf363Vx48WcDfsbTwaDSUIAgC4CRVbmX38RS6hpi7lSb+vJ3pK42opUBfb366vxvJF51fir7eXgxsa19d1M7jZqruIMFx2CGLun2D6qRmZLNw23Eys3Px8/HincW7i/R9MrJzCQvwYWzPJnRoEE7HhnbDk1e/yV3m+tRRsdz/1Xq3Lrm8iX7W+6RkZDNng13GZHA75xWWpxBjIQhCuZCfCq3VUICRo9G+gfuyrZueH2Ba58IxF6PHZLtm6ae3d7Yl8rV4dgH//Vcnt/dOdSOrnp6VQ4CvNw/0bW56vTDERYcDcMKNyOA3Cc5Z2cEWY7Hn+AWOWDbGH7uyRYERVaWFuKEEQSgXUgtZDCm/2t8A1YJ8qVfdNQzW3T6BY2lWwCmhbnhcA4Z2qGc7/2OXebJbRnYu/r4le31ajdmzP24F4FByKl+vsQsFrrVsfls3tq0ri/lbjtmO7+3dtERzKApiLARBKBce+GoDT3632fRa/1a1bceO0h5FJTTA2XlyWe1QAny98PM2v+frw2IY1LaO7fyr1eYqr451NMyYOsrYS5n/UA+3fXwc5pCRncNdn63lqR+2sO6gUbZn7iZDDu+Nm2IAqG8xiN2bR3J5kwha1w1zuoenEWMhCEK5kdfVYmW3gw6UNg9IKhS/PtLT6fzpwa1QShHo5/qiHxnfEICODcOd2s1qh8/f8k++G9ODY+qSOGkwreuFFWqe51KzbNFbN374l9O1CMveizX09r3f97B4xwknpduyQIyFIAjlyvpDrgUwsx1CWfOWZy0K9fO4p8IsK43broh26WtN4qsdFsCB1662tZdGOVh3jLHM41SKe/VZd99+4+GyLcsqxkIQhDJh5fi+zLy9My9c6yzyZxYNdOx8us1VFFPf/eZ2YXj1+na245go416P9m/OZ3fEO/XbcNhutBz3O8xWFoBbV1ZRsK5wrn5vuVO7Y35HceqAeAIxFoIglAn1qwfS+7Ja3N7NWT7cUXZjz/ELRI+fh9aGdlPipME0rBFUoueO6tKQ2fd05c7ujW2CgkopWtYJdeq34ZD5L/Vebyzljplrndoigv24ubN5re2i4C7s9YOl9jyKgvI0ygoxFoIglDmO0tsJifZf9FbJbYB2pRgS2jk6gufyyJbn9fkH5hN19cfOE8zfcoycXM3WI+c4fTHTVi62JDSIMDeEby7abbnuXuywrI2IGAtBEMqcni3s0hwz/0okN1fzn9+dJUAey6P1VNpUC/Slc7R9M/t/Y+Kcrv9w3xVO5/d9uZ6mT8/nmv8YQoOzSqE6XbDJRrsjeV1dz1/Tmus71gfg9m7RJX5+UagYzjBBEC4p4htH8IdDwZ41iad56zd7FvWVrWvTqEaw2dBSQynF7HuuQGtNZk6uSyhsQfvqUSWQOLeSN/S1f6vaLN5hV8d99Epng3mHpQLg2ze3z1dvyhPIykIQhDLn9m7RfDP2ctv5B0vt2kz/urwRH98aZzbMIyilTHMm6lQzL7BkZdKNMaU6j+X/14e3h7d3ahvU1nxPo6wNBXjYWCilBiqldiml9iqlxptc76mUWq+UylZKDctzbbJSaptSaodS6j1VHn8dQRA8gr+PN12a2OW3raKB4JpIV17Urx7IeyM7ur3eIR8JkqJwTUxd+rasRYOIIBddKe8CyqqWJR4zFkopb2AqMAhoDYxUSuUtjHsIGAN8lWfsFUA3IAZoC3QGenlqroIglA9392js0mYmOlhe1HOzunjl+ral9oz3R8UyfUxn27l143rK8A6l9ozSwJMmPB7Yq7XeD6CUmgUMBbZbO2itEy3X8orDa/6/vbsPsqqu4zj+/swKLEaxC67KLKQgmIIa0iqWSroaKDr5nJQzUtqTTk5mo2mWZZOp0VhT2jhGqDSGT/lApKEjGDMOIKsCkqSuDxVKAj4hPqHrtz9+35Xj7r2c6+Dde3fv9zWzc8/53d85+/t9d/Z+z/mdc38H6oH+gIB+wAuEEPqUCZ9sBJ75QFmxJ9hVwrACc04BTBm3c8Hyj8K8sw5ic8d73c4yKq2cw1DNQPZ2gTVelsvMFgMLgbX+M9/MVnetJ+mbktokta1fv77r2yGEKjeowJDT1m4X7WnNDQP5/he635XVMLB8H+T1/eqqLlFAeZNFocG2kr63L2k0sCcwnJRgWiVN6lrPzK4xsxYza2lqaur6dgihynUdkz9+QjPHjC/pmLLHjGvuPr9TT07gVy3K2eM1wIjM+nDg+RK3PQ5YYmabzGwTcDdwQM42IYTeJnP4OHHkEK740vjcKcl72uZ3P3iMW0XXnHtUOa9ZLAPGSBoJPAdMA75S4rb/Ab4h6VLSGcrngd+UpZUhhIqZOGooZ7WOZsq4ndmtqTqmteiqdY8dObllBF8/eCQD+9dVXTLrKbJtmf83b+fSVNKHfB0wy8wukfQzoM3M5kraD7gdaATeAv5nZuP8TqrfA5NIxx5/N7Nztva7WlparK2trWx9CSGEvkjSQ2aW+8WWsiaLnhTJIoQQPrxSk0XtXaUJIYTwoUWyCCGEkCuSRQghhFyRLEIIIeSKZBFCCCFXJIsQQgi5IlmEEELI1We+ZyFpPfDvbdjFDsCGj6g5fVnEqTQRp9JEnEpTzjjtYma5k+v1mWSxrSS1lfLFlFoXcSpNxKk0EafSVEOcYhgqhBBCrkgWIYQQckWy2OKaSjegl4g4lSbiVJqIU2kqHqe4ZhFCCCFXnFmEEELIVfPJQtIRkh6X1C7p/Eq3p6dJmiVpnaRVmbIhku6V9KS/Nnq5JP3WY7VS0oTMNtO9/pOSpleiL+UkaYSkhZJWS/qnpO96ecQqQ1K9pAclrfA4XezlIyUt9T7fJKm/lw/w9XZ/f9fMvi7w8sclTalMj8pLUp2kRyTN8/XqjZOZ1ewP6aFMTwGjgP7ACmBspdvVwzGYBEwAVmXKfgmc78vnA5f78lTSI25FesztUi8fAjztr42+3Fjpvn3EcRoGTPDljwNPAGMjVt3iJGCQL/cDlnr/bwamefnVwBm+fCZwtS9PA27y5bH+/zgAGOn/p3WV7l8Z4nUO8Gdgnq9XbZxq/cxif6DdzJ42s83AjcAxFW5TjzKzRcBLXYqPAa735euBYzPlsy1ZAjRIGgZMAe41s5fM7GXgXuCI8re+55jZWjN72JdfA1YDzUSsPsD7u8lX+/mPAa3ArV7eNU6d8bsVOEySvPxGM3vbzJ4B2kn/r32GpOHAUcBMXxdVHKdaTxbNwH8z62u8rNbtZGZrIX1IAjt6ebF41VQcfQhgX9JRc8SqCx9aWQ6sIyXDp4BXzOxdr5Lt8/vx8PdfBYZSA3EiPXL6POA9Xx9KFcep1pOFCpTF7WHFFYtXzcRR0iDgL8DZZrZxa1ULlNVErMysw8zGA8NJR7l7FqrmrzUZJ0lHA+vM7KFscYGqVROnWk8Wa4ARmfXhwPMVaks1ecGHTPDXdV5eLF41EUdJ/UiJ4gYzu82LI1ZFmNkrwP2kaxYNkrbzt7J9fj8e/v5g0rBoX4/TgcAXJT1LGv5uJZ1pVG2caj1ZLAPG+B0I/UkXjuZWuE3VYC7QeZfOdODOTPmpfqfPAcCrPvQyH5gsqdHvBprsZX2Gjw//EVhtZldk3opYZUhqktTgywOBw0nXdxYCJ3q1rnHqjN+JwAJLV27nAtP8LqCRwBjgwZ7pRfmZ2QVmNtzMdiV97iwws1Oo5jhV+m6ASv+Q7lp5gjSuemGl21OB/s8B1gLvkI5STieNhd4HPOmvQ7yugKs8Vo8CLZn9nEa6uNYOfK3S/SpDnA4ind6vBJb7z9SIVbc47QM84nFaBVzk5aNIH2LtwC3AAC+v9/V2f39UZl8XevweB46sdN/KGLND2HI3VNXGKb7BHUIIIVetD0OFEEIoQSSLEEIIuSJZhBBCyBXJIoQQQq5IFiGEEHJFsgi9iqQOSct9VtOHJX0up36DpDNL2O/9kuJZ0BmSrpN0Yn7NUAsiWYTe5k0zG29mnwYuAC7Nqd9AmrGzKmW+rRtCVYtkEXqzTwAvQ5qzSdJ9frbxqKTO2YMvA3bzs5EZXvc8r7NC0mWZ/Z3kz2J4QtLBXrdO0gxJy/y5FN/y8mGSFvl+V3XWz5L0rKTLfZ8PShrt5ddJukLSQuBypWdi3OH7XyJpn0yfrvW2rpR0gpdPlrTY+3qLz1eFpMskPeZ1f+VlJ3n7VkhalNMnSbrS9/E3tkyKGAJxVBN6m4E+o2k96RkTrV7+FnCcmW2UtAOwRNJc0jMm9rI0sR2SjiRN+zzRzN6QNCSz7+3MbH9JU4GfkKaqOJ00Vcd+kgYAD0i6BzgemG9ml0iqA7Yv0t6Nvs9TSXP/HO3luwOHm1mHpN8Bj5jZsZJagdnAeODH/rv39rY3et9+5Nu+LukHwDmSrgSOA/YwM+uccgO4CJhiZs9lyor1aV/gU8DewE7AY8Cskv4qoc+LZBF6mzczH/yfBWZL2os0vcYvJE0iTfncTPrA6+pw4FozewPAzLLP8uicHPAhYFdfngzskxm7H0yaf2cZMEtpcsE7zGx5kfbOybz+OlN+i5l1+PJBwAnengWShkoa7G2d1rmBmb2sNFvpWNIHPKSHdi0GNpIS5kw/K5jnmz0AXCfp5kz/ivVpEjDH2/W8pAVF+hRqUCSL0GuZ2WI/0m4izdPUBHzGzN5Rms2zvsBmovgUzm/7awdb/jcEnGVm3Sb788R0FPAnSTPMbHahZhZZfr1LmwptV6itIj086csF2rM/cBgpwXwHaDWzb0ua6O1cLml8sT75GVXM/xMKimsWodeStAfp0bgvko6O13miOBTYxau9RnoMaqd7gNMkbe/7yA5DFTIfOMPPIJC0u6SPSdrFf98fSLPRTiiy/cmZ18VF6iwCTvH9HwJssPSsjHtIH/qd/W0ElgAHZq5/bO9tGgQMNrO7gLNJw1hI2s3MlprZRcAG0nTWBfvk7Zjm1zSGAYfmxCbUkDizCL1N5zULSEfI033c/wbgr5LaSDPC/gvAzF6U9ICkVcDdZnauH123SdoM3AX8cCu/byZpSOphpXGf9aRrHocA50p6B9gEnFpk+wGSlpIOzLqdDbifAtdKWgm8wZapqH8OXOVt7wAuNrPbJH0VmOPXGyBdw3gNuFNSvcfle/7eDEljvOw+0vOaVxbp0+2ka0CPkmZi/sdW4hJqTMw6G0KZ+FBYi5ltqHRbQthWMQwVQgghV5xZhBBCyBVnFiGEEHJFsgghhJArkkUIIYRckSxCCCHkimQRQgghVySLEEIIuf4PdUEKgTJEw50AAAAASUVORK5CYII=\n", + "text/plain": [ + "

" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learn.recorder.plot_losses()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The model is not doing that well - out of presented pairs it gets roughly 10% of examples wrong. I also did a cursory error analysis (not shown here for the sake of brevity) and the model is not doing that great at all.\n", + "\n", + "How can this be? Maybe the nearly absolute positional invariance through the use of global max pooling is not working that well. Maybe there is a bug somewhere? Maybe the model has not been trained for long enough or lacks capacity?\n", + "\n", + "If I do continue to work on this I will definitely take a closer look at each of the angles I list above. For the time being, let's try to predict on the validation set and finish off with making a submission.\n", + "\n", + "The predicting part is where the code gets really messy. That is good enough for now though." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load(f'{name}-stage-2');" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "new_whale_fns = set(df[df.Id == 'new_whale'].sample(frac=1).Image.iloc[:1000])" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "data = (\n", + " ImageItemList\n", + " .from_df(df, f'data/train-{SZ}', cols=['Image'])\n", + " .split_by_valid_func(lambda path: path2fn(path) in val_fns.union(new_whale_fns))\n", + " .label_from_func(lambda path: fn2label[path2fn(path)], classes=classes)\n", + " .add_test(ImageItemList.from_folder(f'data/test-{SZ}'))\n", + " .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n", + " .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n", + " .normalize(imagenet_stats)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3570" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(data.valid_ds)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1.48 s, sys: 924 ms, total: 2.41 s\n", + "Wall time: 3.29 s\n" + ] + } + ], + "source": [ + "%%time\n", + "targs = []\n", + "feats = []\n", + "learn.model.eval()\n", + "for ims, ts in data.valid_dl:\n", + " feats.append(learn.model.process_features(learn.model.cnn(ims)).detach().cpu())\n", + " targs.append(ts)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "feats = torch.cat(feats)" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([3570, 512])" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "feats.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 21.2 s, sys: 108 ms, total: 21.3 s\n", + "Wall time: 21.3 s\n" + ] + } + ], + "source": [ + "%%time\n", + "sims = []\n", + "for feat in feats:\n", + " dists = learn.model.calculate_distance(feats, feat.unsqueeze(0).repeat(3570, 1))\n", + " predicted_similarity = learn.model.head(dists.cuda()).sigmoid_()\n", + " sims.append(predicted_similarity.squeeze().detach().cpu())" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3570" + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(sims)" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "metadata": {}, + "outputs": [], + "source": [ + "new_whale_idx = np.where(classes == 'new_whale')[0][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 135, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1.13 s, sys: 4 ms, total: 1.13 s\n", + "Wall time: 1.13 s\n" + ] + } + ], + "source": [ + "%%time\n", + "top_5s = []\n", + "for sim in sims:\n", + " idxs = sim.argsort(descending=True)\n", + " probs = sim[idxs]\n", + " top_5 = []\n", + " for i, p in zip(idxs, probs):\n", + " if len(top_5) == 5: break\n", + " if i == new_whale_idx: continue\n", + " predicted_class = data.valid_ds.y.items[i]\n", + " if predicted_class not in top_5: top_5.append(predicted_class)\n", + " top_5s.append(top_5)" + ] + }, + { + "cell_type": "code", + "execution_count": 147, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.24428104575163398" + ] + }, + "execution_count": 147, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# without predicting new_whale\n", + "mapk(data.valid_ds.y.items.reshape(-1,1), np.stack(top_5s), 5)" + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.98 0.27504668534080295\n", + "0.9822222222222222 0.2793790849673203\n", + "0.9844444444444445 0.2841456582633053\n", + "0.9866666666666667 0.2927777777777778\n", + "0.9888888888888889 0.3001960784313726\n", + "0.991111111111111 0.31275443510737627\n", + "0.9933333333333333 0.3257049486461251\n", + "0.9955555555555555 0.33599439775910367\n", + "0.9977777777777778 0.3447152194211017\n", + "1.0 0.34714285714285714\n", + "CPU times: user 12.3 s, sys: 4 ms, total: 12.3 s\n", + "Wall time: 12.3 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "for thresh in np.linspace(0.98, 1, 10):\n", + " top_5s = []\n", + " for sim in sims:\n", + " idxs = sim.argsort(descending=True)\n", + " probs = sim[idxs]\n", + " top_5 = []\n", + " for i, p in zip(idxs, probs):\n", + " if new_whale_idx not in top_5 and p < thresh and len(top_5) < 5: top_5.append(new_whale_idx)\n", + " if len(top_5) == 5: break\n", + " if i == new_whale_idx: continue\n", + " predicted_class = data.valid_ds.y.items[i]\n", + " if predicted_class not in top_5: top_5.append(predicted_class)\n", + " top_5s.append(top_5)\n", + " print(thresh, mapk(data.valid_ds.y.items.reshape(-1,1), np.stack(top_5s), 5))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are many reasons why the best threshold here might not carry over to what would make sense on the test set. It is some indication though of how our model is doing and a useful data point." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Predict" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "7960" + ] + }, + "execution_count": 163, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(data.test_ds)" + ] + }, + { + "cell_type": "code", + "execution_count": 189, + "metadata": {}, + "outputs": [], + "source": [ + "data = (\n", + " ImageItemList\n", + " .from_df(df, f'data/train-{SZ}', cols=['Image'])\n", + " .split_by_valid_func(lambda path: path2fn(path) in {'69823499d.jpg'}) # in newer version of the fastai library there is .no_split that could be used here\n", + " .label_from_func(lambda path: fn2label[path2fn(path)], classes=classes)\n", + " .add_test(ImageItemList.from_folder(f'data/test-{SZ}'))\n", + " .transform(None, size=SZ, resize_method=ResizeMethod.SQUISH)\n", + " .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n", + " .normalize(imagenet_stats)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 190, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 2.9 s, sys: 1.79 s, total: 4.69 s\n", + "Wall time: 5.03 s\n" + ] + } + ], + "source": [ + "%%time\n", + "test_feats = []\n", + "learn.model.eval()\n", + "for ims, _ in data.test_dl:\n", + " test_feats.append(learn.model.process_features(learn.model.cnn(ims)).detach().cpu())" + ] + }, + { + "cell_type": "code", + "execution_count": 195, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 9.02 s, sys: 5.04 s, total: 14.1 s\n", + "Wall time: 14.4 s\n" + ] + } + ], + "source": [ + "%%time\n", + "train_feats = []\n", + "train_class_idxs = []\n", + "learn.model.eval()\n", + "for ims, t in data.train_dl:\n", + " train_feats.append(learn.model.process_features(learn.model.cnn(ims)).detach().cpu())\n", + " train_class_idxs.append(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 196, + "metadata": {}, + "outputs": [], + "source": [ + "train_class_idxs = torch.cat(train_class_idxs)\n", + "train_feats = torch.cat(train_feats)" + ] + }, + { + "cell_type": "code", + "execution_count": 206, + "metadata": {}, + "outputs": [], + "source": [ + "test_feats = torch.cat(test_feats)" + ] + }, + { + "cell_type": "code", + "execution_count": 209, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 5min 7s, sys: 2min 58s, total: 8min 6s\n", + "Wall time: 8min 6s\n" + ] + } + ], + "source": [ + "%%time\n", + "sims = []\n", + "for feat in test_feats:\n", + " dists = learn.model.calculate_distance(train_feats, feat.unsqueeze(0).repeat(25344, 1))\n", + " predicted_similarity = learn.model.head(dists.cuda()).sigmoid_()\n", + " sims.append(predicted_similarity.squeeze().detach().cpu())" + ] + }, + { + "cell_type": "code", + "execution_count": 211, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 19.6 s, sys: 128 ms, total: 19.7 s\n", + "Wall time: 19.7 s\n" + ] + } + ], + "source": [ + "%%time\n", + "thresh = 1\n", + "\n", + "top_5s = []\n", + "for sim in sims:\n", + " idxs = sim.argsort(descending=True)\n", + " probs = sim[idxs]\n", + " top_5 = []\n", + " for i, p in zip(idxs, probs):\n", + " if new_whale_idx not in top_5 and p < thresh and len(top_5) < 5: top_5.append(new_whale_idx)\n", + " if len(top_5) == 5: break\n", + " if i == new_whale_idx: continue\n", + " predicted_class = train_class_idxs[i]\n", + " if predicted_class not in top_5: top_5.append(predicted_class)\n", + " top_5s.append(top_5)" + ] + }, + { + "cell_type": "code", + "execution_count": 221, + "metadata": {}, + "outputs": [], + "source": [ + "top_5_classes = []\n", + "for top_5 in top_5s:\n", + " top_5_classes.append(' '.join([classes[t] for t in top_5]))" + ] + }, + { + "cell_type": "code", + "execution_count": 222, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['new_whale w_9bedea6 w_448e190 w_ab629bb w_67e9aa8',\n", + " 'new_whale w_edce644 w_dd79a10 w_99af1a9 w_ae393cd',\n", + " 'new_whale w_4516ff1 w_d1207d9 w_02c7e9d w_8003858',\n", + " 'new_whale w_0369a5c w_f66ec54 w_ae8982d w_d0475b2',\n", + " 'new_whale w_8cd5c91 w_0cc0430 w_06460d7 w_e8b82f6']" + ] + }, + "execution_count": 222, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "top_5_classes[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 223, + "metadata": {}, + "outputs": [], + "source": [ + "sub = pd.DataFrame({'Image': [path.name for path in data.test_ds.x.items]})\n", + "sub['Id'] = top_5_classes\n", + "sub.to_csv(f'subs/{name}.csv.gz', index=False, compression='gzip')" + ] + }, + { + "cell_type": "code", + "execution_count": 224, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ImageId
047380533f.jpgnew_whale w_9bedea6 w_448e190 w_ab629bb w_67e9aa8
11d9de38ba.jpgnew_whale w_edce644 w_dd79a10 w_99af1a9 w_ae393cd
2b3d4ee916.jpgnew_whale w_4516ff1 w_d1207d9 w_02c7e9d w_8003858
3460fd63ae.jpgnew_whale w_0369a5c w_f66ec54 w_ae8982d w_d0475b2
479738ffc1.jpgnew_whale w_8cd5c91 w_0cc0430 w_06460d7 w_e8b82f6
\n", + "
" + ], + "text/plain": [ + " Image Id\n", + "0 47380533f.jpg new_whale w_9bedea6 w_448e190 w_ab629bb w_67e9aa8\n", + "1 1d9de38ba.jpg new_whale w_edce644 w_dd79a10 w_99af1a9 w_ae393cd\n", + "2 b3d4ee916.jpg new_whale w_4516ff1 w_d1207d9 w_02c7e9d w_8003858\n", + "3 460fd63ae.jpg new_whale w_0369a5c w_f66ec54 w_ae8982d w_d0475b2\n", + "4 79738ffc1.jpg new_whale w_8cd5c91 w_0cc0430 w_06460d7 w_e8b82f6" + ] + }, + "execution_count": 224, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.read_csv(f'subs/{name}.csv.gz').head()" + ] + }, + { + "cell_type": "code", + "execution_count": 225, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1.0" + ] + }, + "execution_count": 225, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.read_csv(f'subs/{name}.csv.gz').Id.str.split().apply(lambda x: x[0] == 'new_whale').mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 226, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████| 164k/164k [00:03<00:00, 46.1kB/s]\n", + "Successfully submitted to Humpback Whale Identification" + ] + } + ], + "source": [ + "!kaggle competitions submit -c humpback-whale-identification -f subs/{name}.csv.gz -m \"{name}\"" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}