03.Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp classification\n",
    "# default_cls_lvl 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 3. Classification\n",
    ">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST\n",
    "\n",
    "- In this chapter we will be using MNIST\n",
    "    - A dataset comprised of 70,000 small images\n",
    "    - Images represent handwritten digits by students & employees from the US census Bureau.\n",
    "- Each image is labeled with the digit it represents\n",
    "    - Ranging from 0 to 9\n",
    "        - 10 Digits\n",
    "- Scikit-learn provides many helper functions to download popular datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'feature_names', 'DESCR', 'details', 'categories', 'url'])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist = fetch_openml(name='mnist_784', version=1)\n",
    "mnist.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((70000, 784), (70000,))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = mnist['data'], mnist['target']\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((35000, 784), (35000,))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = X[:35000], y[:35000]\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_digit = X[0]\n",
    "some_digit_image = some_digit.reshape(28, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAGaElEQVR4nO3dPUiWfR/G8dveSyprs2gOXHqhcAh6hZqsNRqiJoPKRYnAoTGorWyLpqhFcmgpEmqIIByKXiAHIaKhFrGghiJ81ucBr991Z/Z4XPr5jB6cXSfVtxP6c2rb9PT0P0CeJfN9A8DMxAmhxAmhxAmhxAmhljXZ/Vcu/H1tM33RkxNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCiRNCLZvvG+B//fr1q9y/fPnyVz9/aGio4fb9+/fy2vHx8XK/ceNGuQ8MDDTc7t69W167atWqcr948WK5X7p0qdzngycnhBInhBInhBInhBInhBInhBInhHLOOYMPHz6U+48fP8r92bNn5f706dOG29TUVHnt8PBwuc+nLVu2lPv58+fLfWRkpOG2du3a8tpt27aV+759+8o9kScnhBInhBInhBInhBInhBInhGqbnp6u9nJsVS9evCj3gwcPlvvffm0r1dKlS8v91q1b5d7e3j7rz960aVO5b9iwody3bt0668/+P2ib6YuenBBKnBBKnBBKnBBKnBBKnBBKnBBqUZ5zTk5Olnt3d3e5T0xMzOXtzKlm997sPPDx48cNtxUrVpTXLtbz3zngnBNaiTghlDghlDghlDghlDghlDgh1KL81pgbN24s96tXr5b7/fv3y33Hjh3l3tfXV+6V7du3l/vo6Gi5N3un8s2bNw23a9euldcytzw5IZQ4IZQ4IZQ4IZQ4IZQ4IZQ4IdSifJ/zT339+rXcm/24ut7e3obbzZs3y2tv375d7idOnCh3InmfE1qJOCGUOCGUOCGUOCGUOCGUOCHUonyf80+tW7fuj65fv379rK9tdg56/Pjxcl+yxL/HrcKfFIQSJ4QSJ4QSJ4QSJ4QSJ4Tyytg8+PbtW8Otp6envPbJkyfl/uDBg3I/fPhwuTMvvDIGrUScEEqcEEqcEEqcEEqcEEqcEMo5Z5iJiYly37lzZ7l3dHSU+4EDB8p9165dDbezZ8+W17a1zXhcR3POOaGViBNCiRNCiRNCiRNCiRNCiRNCOedsMSMjI+V++vTpcm/24wsrly9fLveTJ0+We2dn56w/e4FzzgmtRJwQSpwQSpwQSpwQSpwQSpwQyjnnAvP69ety7+/vL/fR0dFZf/aZM2fKfXBwsNw3b948689ucc45oZWIE0KJE0KJE0KJE0KJE0KJE0I551xkpqamyv3+/fsNt1OnTpXXNvm79M+hQ4fK/dGjR+W+gDnnhFYiTgglTgglTgglTgglTgjlKIV/beXKleX+8+fPcl++fHm5P3z4sOG2f//+8toW5ygFWok4IZQ4IZQ4IZQ4IZQ4IZQ4IdSy+b4B5tarV6/KfXh4uNzHxsYabs3OMZvp6uoq97179/7Rr7/QeHJCKHFCKHFCKHFCKHFCKHFCKHFCKOecYcbHx8v9+vXr5X7v3r1y//Tp02/f07+1bFn916mzs7PclyzxrPhvfjcglDghlDghlDghlDghlDghlDghlHPOv6DZWeKdO3cabkNDQ+W179+/n80tzYndu3eX++DgYLkfPXp0Lm9nwfPkhFDihFDihFDihFDihFDihFCOUmbw+fPncn/79m25nzt3rtzfvXv32/c0V7q7u8v9woULDbdjx46V13rla2753YRQ4oRQ4oRQ4oRQ4oRQ4oRQ4oRQC/acc3JysuHW29tbXvvy5ctyn5iYmNU9zYU9e/aUe39/f7kfOXKk3FevXv3b98Tf4ckJocQJocQJocQJocQJocQJocQJoWLPOZ8/f17uV65cKfexsbGG28ePH2d1T3NlzZo1Dbe+vr7y2mbffrK9vX1W90QeT04IJU4IJU4IJU4IJU4IJU4IJU4IFXvOOTIy8kf7n+jq6ir3np6ecl+6dGm5DwwMNNw6OjrKa1k8PDkhlDghlDghlDghlDghlDghlDghVNv09HS1lyMwJ9pm+qInJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4QSJ4Rq9iMAZ/yWfcDf58kJocQJocQJocQJocQJocQJof4DO14Dhyk10VwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(some_digit_image, cmap='binary')\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Note that the label is a string, most algorithms expect numerical inputs so let's transform it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = y.astype(np.int8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- But we should always start with splitting the data into train vs. test, right?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((30000, 784), (30000,), (5000, 784), (5000,))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test = X[:30000], y[:30000], X[30000:], y[30000:]\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Some ML algorithms are sensitive to ordered rows and can perform badly if instances from the same class are grouped together, the solution to this is to shuffle.\n",
    "    - MNIST's training data is already shuffled for us."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training a Binary Classifier\n",
    "\n",
    "- Let's simplify the problem for now and try to identify one digit (number **5**).\n",
    "- This \"5-detector\" is an example of a binary classifier, it will predict, for any row, 5 or not-5.\n",
    "- Let's create the target vector for the classification task:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a fraction of the training data for the notebook to run fast locally:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_5 = (y_train == 5)\n",
    "y_test_5 = (y_test == 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- A good place to start is with a Stochastic Gradient Descent Classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf = SGDClassifier(n_jobs=-1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
       "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
       "              l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
       "              max_iter=1000, n_iter_no_change=5, n_jobs=-1, penalty='l2',\n",
       "              power_t=0.5, random_state=42, shuffle=True, tol=0.001,\n",
       "              validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X=X_train, y=y_train_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Now we will use it to predict the digit of image at index 0:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict(X=[some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- .. and it's indeed a 5.\n",
    "- Now let's evaluate this model's performance:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance Measures\n",
    "\n",
    "- Evaluating a classifier is often significantly trickier than evaluating a regressor.\n",
    "- There are many performance measures available."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Measuring Accuracy using Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.base import clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "skfolds = StratifiedKFold(n_splits=3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9458\n",
      "0.9634\n",
      "0.9603\n"
     ]
    }
   ],
   "source": [
    "for train_index, test_index in skfolds.split(X=X_train, y=y_train_5):\n",
    "    clone_clf = clone(sgd_clf)\n",
    "    X_train_folds = X_train[train_index]\n",
    "    y_train_folds = y_train_5[train_index]\n",
    "    X_test_folds = X_train[test_index]\n",
    "    y_test_folds = y_train_5[test_index]\n",
    "    clone_clf.fit(X_train_folds, y_train_folds)\n",
    "    y_pred = clone_clf.predict(X_test_folds)\n",
    "    num_correct = sum(y_pred == y_test_folds)\n",
    "    print(num_correct / len(y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Alternatively, let's use `cross_val_score()` function to do Kfold cross validation where $K=3$:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.95784036, 0.973     , 0.9585    , 0.97266667, 0.94649108])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(estimator=clone(sgd_clf), X=X_train, y=y_train_5, cv=5, scoring=\"accuracy\", n_jobs=-1)\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Above $95\\%$ accuracy, isn't that great?\n",
    "- Let's look at the performance of a very dumb classifier that classifies all training rows as \"Not-5\":"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self, X, y=None):\n",
    "        pass\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return np.zeros(shape=(len(X), 1), dtype=bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "never_5_classifier = Never5Classifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.91433333, 0.911     , 0.91016667, 0.90716667, 0.90583333])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(estimator=never_5_classifier, X=X_train, y=y_train_5, cv=5, scoring=\"accuracy\", n_jobs=-1)\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The dumb classifier has over 90% acccuracy for any validation.\n",
    "- This hapenned because of the class imabalance in the binary task we are aiming for:\n",
    "    - We originally had balanced counts for each digit.\n",
    "    - But when we turned the problem into 5 vs. non-5, we end up with the first class taking ~10% of the rows and non-5 taking >=90%.\n",
    "    - Hence, if you created a dumb model that just predicts \"non-5\" for any input, you'll get at least 90% accuracy.\n",
    "- This demonstrates why accuracy is generally not the preferred method to evaluate classifiers, especially if the data have imbalanced classes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion Matrix\n",
    "\n",
    "- A much better way to evaluate the performance of a classifier is to look at its produced confusion matrix.\n",
    "- Let's use `cross_val_predict()` to get some predictions to finally visualize the confusion matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(estimator=sgd_clf, X=X_train, y=y_train_5, cv=3, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000,)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- `corss_val_predict` returns the predictions made of the test fold.\n",
    "    - And because we are looping over the 3 folds, we endup with predictions for all data.\n",
    "- Let's take a look at the confusion matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[26515,   776],\n",
       "       [  529,  2180]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_train_5, y_pred=y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Each row in the confusion matrix represents an actual class. While each column represents a predicted class.\n",
    "    - first row is \"not 5\"\n",
    "        - Model correctly predicted \"not 5\" for 54K \"not 5\"s\n",
    "            - True Negatives\n",
    "        - Model incorrectly predicted \"5\" for 687 \"not 5\"s\n",
    "            - False Positive\n",
    "    - second row is \"5\"\n",
    "        - Model incorrectly predicted \"not 5\" for 1891 \"5\"s\n",
    "            - False Negative\n",
    "        - Model correctly predicted \"5\" for 3530 \"5\"s\n",
    "            - True Positives\n",
    "- A perfect classifier would have only True Positives and True negatives, meaning the confusion matrix will have zeros on all elements except its diagonal.\n",
    "- Let's pretend we reached perfection:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_perfect_predictions = y_train_5.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[27291,     0],\n",
       "       [    0,  2709]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_train_5, y_pred=y_train_perfect_predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The confusion matrix gives you a lot of information, but sometimes you may prefer a single-number metric.\n",
    "- An interesting one to look at is the model's accuracy over its positive predictions, named *precision*:\n",
    "\n",
    "$$precision = \\frac{TP}{TP+FP}$$\n",
    "\n",
    "- A trivial way to have a perfect precision of $1$ is to make one prediction that you're sure it's correct, by consequence $FP=0$ and $precision=1$.\n",
    "    - This would not be useful since the evaluation is ignore all but the true positives.\n",
    "- Precision is typically used along with another metric named recall.\n",
    "- recall, also called sensitivity or the True Positive rate is as follows:\n",
    "\n",
    "$$recall = \\frac{TP}{TP+FN}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision & Recal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7374830852503383"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_true=y_train_5, y_pred=y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8047249907715024"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_true=y_train_5, y_pred=y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The recall score is not good at all.\n",
    "    - its dominator consists of TPs & **False Negatives**, we can conclude that our model is predicting a lot of digits as not \"5\" but they're \"5\"s.\n",
    "    - The model is not good at predicting False Negatives.\n",
    "- Now our model doesn't look as shiny as it was when we looked at its accuracy.\n",
    "- When it claims that a digit represents a \"5\" it's only correct for $83\\%$ of the times.\n",
    "- It's often convinient to combine the precision and accuracy scores into a single metric called the $F_{1}$ score.\n",
    "    - In particular, if you want to compare two classifiers.\n",
    "    - The $F_{1}$ score represents the harmonic mean of the precision & recall metrics.\n",
    "        - Harmonic because it gives more importance to low values in its formula.\n",
    "        - Meaning, the F1 score will only give you high scores if both precision and recall are high.\n",
    "\n",
    "$$F_{1}=\\frac{2}{\\frac{1}{precision}+\\frac{1}{recall}}=2 \\times \\frac{precision \\times recall}{precision + recall}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7696381288614299"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_train_5, y_pred=y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The $F_{1}$ favors classifiers that have similar precision and recall.\n",
    "- This is not always what you want\n",
    "    - Sometimes you only care about precision\n",
    "        - Or the relation between True Positives and **False Positives**\n",
    "            - Ex. You don't want to incorrectly predict that a transaction is fraudulous.\n",
    "            - Ex. You don't want to incorrectly predict that a video is safe for kids to watch.\n",
    "                - But you don't care that much if you incorrectly predict that a video is bad.\n",
    "    - And other times you only care about recall\n",
    "        - Or the relation between True Positives and **False Negatives**\n",
    "            - Ex. You don't want to incorrectly predict that a person doesn't have cancer.\n",
    "- Unfortunately, you can't have it both ways, increasing precision will decrease recall, and increasing recall decreases precision.\n",
    "    - This is called the precision-recall trade-off."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision-Recall Trade-off"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- To understand this trade-off, we need to know how the algorithm is making its decisions.\n",
    "- For each instance, the classifier outputs a score (we can call it a probability)\n",
    "- If the score is higher than a certain threshold, we label the instance as positive (a \"5\")\n",
    "- If the score is less than the threshold, we the instance as negative (a \"not 5\")\n",
    "\n",
    "<div style=\"text-align:center\"><img style=\"width: 50%\" src=\"static/imgs/precision-recall-trade.png\" /></div>\n",
    "\n",
    "- As you can notice, if we raise the threshold, precision converges to $100%$, but as a result, we lower the recall.\n",
    "    - Same happens to recall if we lower the threshold, we get a better recall (False Negatives are fewer) but we decrease precision. \n",
    "- Scikit-learn doesn't allow you to set the threshold directly, but it can give you the scores for each of its prediction.\n",
    "    - You can set the threshold you want after getting the scores:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11968.30313346])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Let's raise the threshold:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 8000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- How do you decide which threshold to use?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores = cross_val_predict(estimator=sgd_clf, X=X_train, y=y_train_5, \n",
    "                             cv=3, method='decision_function', n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- With these decision scores, use another function to compute the recalls/precisions for all possible threshold:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(y_true=y_train_5, probas_pred=y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Finally, use `matplotlib` to plot precision and recall in function of the threshold value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label='Precision')\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"Recall\")\n",
    "    plt.xlabel(\"Threshold\")\n",
    "    plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEGCAYAAABlxeIAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5dnw8d81kx0CSUhIQgAT9n0XpGqNCwjWCi5UiytPKa+toq19n9blba2ValsfLfq4UKs8tU+tS5UqFqwrcUMQBESQLQECERDCnhASMrnfP84JmYQskzAzZ87k+n4+5zPnnPs+51z3TDLXnO0+YoxBKaWU8jgdgFJKqcigCUEppRSgCUEppZRNE4JSSilAE4JSSilbjFMbTk9PN7m5uU5tvp7y8nI6dOjgdBhh0Z7aCu2rvdrW6OXf3s8//7zUGJMRiu04lhByc3NZuXKlU5uvp6CggPz8fKfDCIv21FZoX+3VtkYv//aKSHGotqOHjJRSSgGaEJRSStk0ISillAI0ISillLJpQlBKKQUEkBBEZL6I7BWRdU2Ui4g8JiKFIrJWREYFP0yllFKhFsgewl+ASc2UTwb62sMs4KnTD0sppVS4tXgfgjHmQxHJbabKFOCvxupHe5mIpIhItjFmd5BirKf0WCl3vXsXQ7oOAUBErFek3njDstrpxso27d5E4arCgOo2nE5JSMHr8Ta5XGPLtBRzIOOd4juREJOAIHjEg0c8iPiN+81PTUzFI3p0UEWm0lKYNw+qqurm9esH111njf/Xf8GRI/WXEcmg9jaE3/4WKivrl48aBVOnWuP33gsNe/kfPx4mT7a2OWfOqTGddx5ceCEcPQoPPXRq+cSJcM45sH8/PProqeXf/S6ceSbs3g1PNfiJ3L07zJp16jKRIBg3puUAO/2mS+x5pyQEEZmFtRdBZmYmBQUFrd7Y+3vfZ/6G+dRQ06Zgm7Q5uKuLNIJfQvlQTk7XJpqGice/zCteOsZ0PJloGi7X2HisxJIcm4xXvHjwWK/iISkmiQRPwsn1iggerMQVIzGkxqWSGptKUkwSMRKDV7wkeZNIjk0mVmJJiknCK96A211WVtamvzM3cmtbFy7M5o9/7A+AiPXNPW7cAbp3/xKAP/zhLEpL4+stc+65KSfb+uCD53DsWP2/iUsu2U1KivVPPWfOeackhCuvLCExsYiKCg9z5px7SkzFxcV4vds5cCCOOXPGn1K+d+9Wqqt38vXXicyZM/aU8rKyLZSX72LLlo7MmTO6XtnAgUfo1291U29Ho8L12UogD8ix9xD+ZYwZ0kjZIuBBY8zH9vR7wM+NMZ83t84xY8aYtt6pfOzEMSqrKzFYsRtj6o0DjU43Vbb006WMHz8+oLr+09U11RyuPNzkco0t01LMgYz7anwcPH6QGlNDjanBGFM3jqk3v8pXxcHjB09ub3vxdnr27HlyunbZ2nH/19p1HK06yvHq4/Xm+S/jv93aZQ8fP0xFdQW+Gh/VNdX4jI/K6koOHj+Ir8aHz/ioMTX4anwnlw+EIMTHxJ9MMB7x4PVY4zGeGNIS04j3xhPrjSXWEwvHoHf33mR3zCarYxbpSekkxiTStUNXEmMTSY5LJqdTDh3jOga0/Ujmtrt3r7sOfD74xS/gvffgllsgISGwZd3W1tPV4E7lz40xY0KxnWDsIZQAPfymuwO7grDeJiXFJpEUmxS09WXEZ9C9U/egrS+SReo/kjGGiuoK9pTtYdfRXVT5qjjhO0GVr4r9FfupOFFBRXUFpcdKqa6pPpn0ahOKz/ioqK7g0PFDVNdUc8J3gorqCooritm+dTu7ju5qNul0iO1Ah7gOdIrvRN+0vvTs3JMRWSPondqbtMQ0RmaP1MNuTaiqAhGIjW26zokT1pd/QgJs2gTPPQfPPw+JifDCCzBiRPjiVU0LRkJYCNwqIi8C44DDoTp/oKKXiJAUm0Sv1F70Su0VtPXWJkBjDHvK9lB+opyDFQc5ePwgldWV7C3fy56yPew7to/j1cf5pvwbtuzfwnvb3qPKV3dQOz0pnTO7nUm35G6kJ6XzrR7fYkjXIWR2yKRDXPvpZA2gvBx27oSKCujTxzoGP3eu9aU+aRLccAP07l1X3+eDsWNhzRrYt886XzB3rlX28MPOtEE1rsWEICIvAPlAuoiUAPcCsQDGmHnAYuASoBA4BswIVbBKtZWIkJ2cHXD9E74TFB0sovRYKRtLN/LG5jcoOlDEF998wa6jdTvAgjCm2xh6pfYiJzmHywdezuCMwaQmpoaiGWHn84HXC598Ah07Ql4edO5cV963L7z0Eng88PHH8JvfWMOECfDyy1YSuOkmKLa7Y0tPh2uvhYICa7lewcv9KggCucro+y2UG+CWoEWkVASI9cYyIH0AAOf0PIeZo2aeLDt0/BDLS5az4/AOig4W8UHxB6zYtYKXDr7EI8seAaBjXEfG5oxlZNZIzu5xNpf1vwyvJ/CT4ZFg9mx4/PG66TvvhGuuqZs+91wYORKysuDBB615O3ZYX/T/+pd1XmDHjrpksGaN9TpyJPztb9aVRCqyONb9tVJulZKQwsV9Lj5l/t7yvXxU/BGFBwrZenArn+z8hMeWP8bDnz5Mt+RuzL14LtMGT3Mg4sAdPw7XXw+Fhdalk/6GD7f2CJYtsw4B2VdM19OzJ/znf1pDreuvh127YNgwa9rrhcGDQ9cG1XaaEJQKkq4dunLloCvrzavyVfHCly9w13t38b1XvsekNZN4ZOIjDMwY6FCUTXvgAbjnnrrpe++FY8esE7/+xo1r3XrT061BRT69bEKpEIrzxnHjiBvZPHszM0fO5N+F/2bIU0P4+Ts/p6yqzOnwTqquhtV+l8avXGnd2NUwGajopglBqTDoGNeRP1/2Z9b/eD3f7fddHlr6EMOeGsYLX76Ar8bnWFw+n3VXbmysdWXQtm3WXb+jR7e8rIo+mhCUCqNBGYN47ZrX+OfV/+R49XGmL5jOd1/4LoeOH3IknthY+PBDa9zng9xciItzJBQVATQhKOWAqQOmsuOnO7jz7Dt5s/BNBj4xkI+KPwp7HD3sW0rffLPxPntU+6IJQSmHxHhiePCiB1k0fRFVvirOf+58/veL/w3Lto8csa78+fnPrctEJzXXn7FqNzQhKOWwS/pewqpZqxiaOZQbXruB+avnh3R7//iHdXNZTg506ADf+15IN6dcRBOCUhHgjJQz+OCmDxicMZiZC2fy5IonQ7at66+vG7/00pBtRrmQJgSlIkSn+E58NOMjhmUO45bFt3BfwX1B38bu3dZVREOGWM8I0PsDlD9NCEpFkNTEVJbNXMal/S7l1x/8mle+eiVo6zbG6mbi9tth0aKgrVZFEU0ISkWYhJgEXpn2Cn3T+jLj9RkUHyoOyno9Hmu47jqriwmlGtKEoFQEio+JZ/G1iwG48uUrqayubGGJ5t18c934wMjrNUNFCE0ISkWoPml9ePayZ/l89+fMfGMmNaZtj40tLIQ//cka/+wz68oipRqjCUGpCPa9wd/j9nG387e1f2POh408DT4AmZnW6+WXWw9+V6opmhCUinB/vPiPTOw9kfs/vJ8dh3e0evmXX7auLlqwIATBqaiiCUGpCCciPDzxYaprqnl02aOtWnbDBpg5E269NUTBqaiiCUEpFxjSdQjXDbuOR5Y9wttFbwe8XO1Dbn7wgxAFpqKKJgSlXOKxSY+Rk5zDLYtvocpXFdAyRUXW6+TJIQxMRQ1NCEq5RGpiKk9c8gSFBwr55fu/bLH+unXWa1paiANTUUMTglIucln/y7hq0FU8tPQhNpZubLZuaqp1I9rixWEKTrmeJgSlXEREeGzSY8R543jk00earZuTYz30prXPQFbtlyYEpVwmOzmbKQOm8OK6FymvKm+0Tnm5dTPazp1hDk65miYEpVzo5tE3c7TqKM+ufrbR8jVrrO4qVq8Oc2DK1TQhKOVC5+edz8D0gTz+2eP4anynlP/2t9br6NFhDky5miYEpVzqJ2f9hC0HtvBW0VunlNUeKsrJCXNQytU0ISjlUtcNu46EmAReWv9SvfnGQEkJzJrlUGDKtTQhKOVSSbFJ3Dj8Rl5c9yIlR0pOzj940HoQjh4uUq2lCUEpF7t93O1U+ap44KMHTs5LS7P6MNI9BNVamhCUcrGBGQO5evDVPLv62ZPPS/jgA1i71uHAlCtpQlDK5Sb3mUyVr4pNRzcBcNttcOedDgelXCmghCAik0Rkk4gUisgpf2oi0llE3hCRL0RkvYjMCH6oSqnGTOg9AYD1R9ZTVWXtHQwf7nBQypVaTAgi4gWeACYDg4Dvi8igBtVuAb4yxgwH8oGHRSQuyLEqpRrRLbkbuSm5rD28lr//3ZrXo4ezMSl3CmQPYSxQaIzZaoypAl4EpjSoY4BkERGgI3AAqA5qpEqpJo3KHkVRWRFLl1rT117rbDzKnWICqJMD+PeIUgI07C7rcWAhsAtIBq425tQngovILGAWQGZmJgUFBW0IOfjKysoiJpZQa09thfbT3ozKDHYd38WfXy0E+rB6dYHTIYVUe/lca4WrvYEkBGlknmkwfTGwBrgA6A28IyIfGWOO1FvImKeBpwHGjBlj8vPzWx1wKBQUFBApsYRae2ortJ/2ZpVm8acn/sR1/+8DvtenT9S3ub18rrXC1d5ADhmVAP5HJLtj7Qn4mwEsMJZCYBswIDghKqVa0q9LP5JjOlGWvejkYzOVaq1AEsIKoK+I5Nkniq/BOjzkbwdwIYCIZAL9ga3BDFQp1TSPeMgru4zXNiyksOSQ0+Eol2oxIRhjqoFbgbeADcDLxpj1InKziNxsV7sf+JaIfAm8B/zCGFMaqqCVUqdK+XoKeHysOfKe06EolwrkHALGmMXA4gbz5vmN7wImBjc0pVRrfPj8VPh5J94pepurBl3pdDjKhfROZaWiRU0MnpJzKCgucDoS5VKaEJSKAsePW6+dSy9m8/7NFB4odDYg5UqaEJSKEp07V3HvtZMAeG+rnkdQracJQakokJAAr722lNuu7UvXDl35tORTp0NSLqQJQakosHQprF6dgogwtOtQNpZudDok5UKaEJSKAg89BHPn9gWgT1ofNu/f7HBEyo0CuuxUKRXZXnsNoAMAeSl5HDx+kKOVR0mOT3Y0LuUuuoeglMuVlVmvAwZYXYcNzRwKwDtb33EqJOVSmhCUcrnN9tGhq64qAWBi74kkxyXzVuFbDkal3EgTglIuV1xsvfbseQyAGE8M43uMZ9nXyxyMSrmRJgSlXG7qVNi/H/Lyyk/OG999POv2ruPw8cMORqbcRhOCUi4nAmlpEBNT95iSc3ueS42pYVmJ7iWowGlCUMrlHnoI/vSn+vPG5oxFEJZ/vdyZoJQraUJQyuWefRbeaXBBUXJ8Mrkpuazbu86ZoJQraUJQysWqq2HrVujT59SyMd3G8NGOjzCm4RNvlWqcJgSlXKy4GE6cgL59Ty27IO8C9pTtoehgUfgDU66kCUEpFyu0e7nu1+/UsrN7nA3Apzu1ozsVGE0ISrnY/v2QlNT4IaNBGYNIiElg9Z7V4Q9MuZImBKVcbPp0q+uKrKxTy7weL4MzBvPl3i/DH5hyJU0ISrmciDU0ZmjmUNZ+sza8ASnX0oSglItNmQK//W3T5YMzBrO3fC8HKg6ELyjlWpoQlHKpmhpYuNC67LQpgzIGAbBh34YwRaXcTBOCUi61fbv1Gh/fdJ2+adb1qIUHCkMfkHI9TQhKudSePdbrZZc1Xadn554AbD+0PfQBKdfThKCUS339tfWak9N0nfiYeLI7ZrP98PawxKTcTROCUi6Vng5XXAHduzdfr1+XfvqMZRUQTQhKudT558Orr0JqavP1+nfpz6bSTeEJSrmaJgSlXCrQPuv6p/dnf8V+9h/bH9qAlOtpQlDKpS69FCZMaLlevy5WR0eb9utegmqeJgSlXGrbNkhObrle/y79AfSwkWpRQAlBRCaJyCYRKRSRO5uoky8ia0RkvYh8ENwwlVL+jIENG6Bnz5br5qXmEeuJ1RPLqkUxLVUQES/wBDABKAFWiMhCY8xXfnVSgCeBScaYHSLSNVQBK6WgtNR6PXSo5boxnhj6dunLV6VftVxZtWuB7CGMBQqNMVuNMVXAi8CUBnWmAwuMMTsAjDF7gxumUsrfrl3W60UXBVZ/VPYoVny9InQBqajQ4h4CkAPs9JsuAcY1qNMPiBWRAiAZeNQY89eGKxKRWcAsgMzMTAoKCtoQcvCVlZVFTCyh1p7aCtHb3m++iWf69G7AHgoKKoDm2xp3JI7dZbt5+/23ifPEhS/QEInWz7Up4WpvIAmhsY51G17wFgOMBi4EEoFPRWSZMabeQUtjzNPA0wBjxowx+fn5rQ44FAoKCoiUWEKtPbUVoru9V18NcMbJ6ebaunX1VuZvn0+/Uf3ITckNR3ghFc2fa2PC1d5ADhmVAD38prsDuxqp829jTLkxphT4EBgenBCVUg1t2VJ3HiEQ3ZK7AfD1ka9DFJGKBoEkhBVAXxHJE5E44BpgYYM6rwPnikiMiCRhHVLS/naVCpF+/SAjoxX17XsRvtqnJ5ZV01o8ZGSMqRaRW4G3AC8w3xizXkRutsvnGWM2iMi/gbVADfCMMWZdKANXSgUuNyWXhJgEvTlNNSuQcwgYYxYDixvMm9dg+iHgoeCFppRqSqdOcNNNgdf3iIeenXtSfLg4ZDEp99M7lZVymfJyOHIEsrNbt1xOcg4lR0pCE5SKCpoQlHKZoiLrNS+vdcv1TeurT05TzdKEoJTL9OgBr7wC3/5265br16UfpcdKOVBxIDSBKdfThKCUy6SmwpVXtv6QUe2VRlv2bwlBVCoaaEJQymU++cQaWqs2IWgnd6opAV1lpJSKHL/5DRw4ACta2TVRXmoeXvFqQlBN0j0EpVymqAh69279cnHeOPJS89h8QBOCapwmBKVcpLoaiouhV6+2Ld+vSz/dQ1BN0oSglIvs3GklhbbsIQDkpeSx7eA2TKAPZFbtiiYEpVyk9h6EtiaEEVkjOFx5mA2l2tWYOpUmBKVcZPx4WL4cRo9u2/IX9bKeqPPu1neDGJWKFpoQlHKRDh1g7FhITm7b8rkpufTs3JOlO5cGNzAVFTQhKOUiL7wAb7xxeusY2nWoHjJSjdKEoJSL/P73MG9ey/WaMyB9AJtKN+Gr8QUnKBU1NCEo5SLFxZCbe3rrGNJ1CJW+Su3oTp1CE4JSLnHokDWccUbLdZvTN60vANsObQtCVCqaaEJQyiVqLznt0+f01pObkgvA9kPbT29FKupoQlDKJYrth5219jkIDWUnZxPridWEoE6hndsp5RJTp8LXX0N6+umtxyMeclNy2Xpwa3ACU1FDE4JSLuHxQLduwVlXvy792LR/U3BWpqKGHjJSyiXmzoX584Ozrj5pfSg6UKR9Gql6NCEo5RJPPQVvvhmcdfVJ60P5iXL2lO0JzgpVVNCEoJQL+HywbdvpX2FUq/bS042lG4OzQhUVNCEo5QJFRXDiRNufg9DQ8KzhACwrWRacFaqooAlBKRfYssV6Pd2b0mpldcyid2pvVu1ZFZwVqqigCUEpF9i7F+LjYcyY4K1zVPYoVu3WhKDqaEJQygVmzIDyckhLC946R2WPYuvBrRw6fih4K1WupglBKZfweoO7vlHZowBYs2dNcFesXEsTglIRzhi49FJ4/vngrndk1kgAPWykTtKEoFSEKy2FRYtg377grjejQwbdO3XXhKBO0oSgVITbYD/cbMCA4K97VPYoPt/9efBXrFwpoIQgIpNEZJOIFIrInc3UO1NEfCJyVfBCVKp922jfOzZwYPDXPSZ7DJtKN3G08mjwV65cp8WEICJe4AlgMjAI+L6IDGqi3u+Bt4IdpFLt2caNkJQEPXoEf90DMwZiMBQdLAr+ypXrBLKHMBYoNMZsNcZUAS8CUxqpNxt4FdgbxPiUaveSk+Gii6zeToOtT5rVF4Y+TlNBYN1f5wA7/aZLgHH+FUQkB7gcuAA4s6kVicgsYBZAZmYmBQUFrQw3NMrKyiImllBrT22F6Gjv+edbQ0vNaEtbj1UfA+Cdle+Qvvc0H7QQRtHwubZGuNobSEKQRuY17DN3LvALY4xPpLHq9kLGPA08DTBmzBiTn58fYJihVVBQQKTEEmrtqa3Qvtrb1rZmf5FNRXKFq96n9vS5QvjaG8hOaAngf/SyO7CrQZ0xwIsish24CnhSRKYGJUKl2rEPP7Q6tFu9OnTbGJ41nLXfrA3dBpRrBJIQVgB9RSRPROKAa4CF/hWMMXnGmFxjTC7wCvBjY8xrQY9WqXbms8+sbq9zckK3jdHZo/ly75d6pZFqOSEYY6qBW7GuHtoAvGyMWS8iN4vIzaEOUKn2bMUKq4fTrl1Dt42ze5xNjanRrrBVYM9UNsYsBhY3mDevibo3nX5YSimw9hDGjg3tNkZkjQBg8/7NTOg9IbQbUxFN71RWKkLt2wfbt8OZTV63FxyZHTPpENtBn56mNCEoFamqquD//B/rktNQ8oiHkdkj9WE5ShOCUpEqJwfmzQvuQ3GaMrTrUL785ktO+E6EfmMqYmlCUCpC7dwJPl94tjWh1wSOVh1l6c6l4dmgikiaEJSKQMbA6NHwox+FZ3v5ufl4xMN7294LzwZVRNKEoFQEKi62TiqPHBme7aUmpjI2ZyxvF70dng2qiKQJQakItGKF9RrqK4z8ndvzXFbvWU2Vryp8G1URRROCUhFoxQqIi4Nhw8K3zbE5Y6nyVfHFni/Ct1EVUTQhKBWBPvsMRoywkkK4jMuxOjHWO5bbr4DuVFZKhdddd0F1dXi32b1Td7old2P518uZzezwblxFBE0ISkWgiy8O/zZFhLO6n6V7CO2YHjJSKsJ8/DEsXGhdehpuZ+WcRdHBIvaW64MP2yNNCEpFmAcfhDvugGaeNRUy43uMB/Q8QnulCUGpCHLkCLz7Lkx16PFSI7JGEOOJ0TuW2ylNCEpFkEWLrE7trrjCme13jOvI+O7jeXfru84EoBylCUGpCLJgAWRlwVlnORfDRb0uYtXuVew/tt+5IJQjNCEoFSFqauCLL+Dyy8Hj4H/mhF4TMBje3/a+c0EoR+hlp0pFCI8HNm6E8nJn4zgz50w6xXdi0ZZFTBs8zdlgVFjpHoJSEcTjgeRkZ2OI8cRwce+LWbJ9ibOBqLDThKBUBKiqgiFD4H//1+lILOf0PIcdh3ew7eA2p0NRYaQJQakIsGQJrF8PnTs7HYnlol4XAfDO1nccjkSFkyYEpSLAX/4CqakwcaLTkVgGpg+kT1ofXt3wqtOhqDDShKCUww4csC43vf56SEhwOhqLiDC++3i+2veV06GoMNKEoJTDnnvOOofwgx84HUl9o7NHU3KkhK0HtzodigoTTQhKOeycc+BXvwrvw3ACcWm/SwF4Y9MbDkeiwkUTglIOO/NMuO8+p6M4Ve+03vTv0p9/F/3b6VBUmGhCUMohxlgPwlm71ulImnZx74v5YPsHHK8+7nQoKgw0ISjlkMWL4Xe/g+XLnY6kaRN7T6SiuoKPd3zsdCgqDDQhKOUAY6zzBr16wU03OR1N0/Jz8+kQ24EXvnzB6VBUGGhfRko54H/+B1atsu4/iI11OpqmdYjrwNQBU3lj8xvUmBo8or8ho1lAn66ITBKRTSJSKCJ3NlJ+rYistYelIjI8+KEqFR2OHoWf/hR69IBrr3U6mpZ9p+932HdsH28Xve10KCrEWkwIIuIFngAmA4OA74vIoAbVtgHnGWOGAfcDTwc7UKWiRVwcXH01/Pd/Q4wL9tEvH3g5XRK78MyqZ5wORYVYIH+OY4FCY8xWABF5EZgCnLyF0Rjj/7y9ZUD3YAapVLSorob4eHjaRT+ZEmISuGrQVTz/5fNU11QT43FBFlNtEsgnmwPs9JsuAcY1U/8HwJuNFYjILGAWQGZmJgUFBYFFGWJlZWURE0uotae2QmS1d+PGZO6/fxC/+c06evcO/kMPQtnWjGMZlFWV8ejrjzI6dXRIttEakfS5hkPY2muMaXYApgHP+E1fD/x3E3XPBzYAXVpa7+jRo02kWLJkidMhhE17aqsxkdPerVuN6drVmLw8Y/bsCc02QtnW0vJSk/K7FHPVy1eFbButESmfa7j4txdYaVr4fm3rEMhJ5RKgh990d2BXw0oiMgx4BphijNGHsSplO3gQvvMdq7+ixYshM9PpiFqvS1IXpg+ZzsJNC9l5eGfLCyhXCiQhrAD6ikieiMQB1wAL/SuISE9gAXC9MWZz8MNUyp2qquCKK6CwEF57DQYMcDqitps5aiZVvioWbVnkdCgqRFpMCMaYauBW4C2sw0EvG2PWi8jNInKzXe1XQBfgSRFZIyIrQxaxUi7i81lXFc2fD+ed53Q0p2d41nCGZw7nyRVPOh2KCpGALhcwxiwGFjeYN89vfCYwM7ihKeVeJSXWDWeZmfDmm9azkt3OIx5+OOqH3Prmrfy78N9M6jPJ6ZBUkEXBn6lSkaWgAEaOhBtusKajIRnU+uHoH5KXksfd791NjalxOhwVZFH0p6qUs44cgR/9CC64ANLT4bHHnI4o+OK8cdyXfx+r96zmla9ecTocFWSaEJQKglWroH9/64az226zejDt39/pqEJj+tDpDEwfyIMfP1h7ubmKEpoQlGqjsjJYt84a798fvvUtWLoU5s6FTp2cjS2UvB4vd4y/gzV71vBh8YdOh6OCSBOCUq105Ag88ADk5lqXlNbUQIcO8OqrMK65e/ijyLVDryU5LpnHPovC42LtmCYEpQL0xRfwH/8BOTlwzz1w1lnw179G10njQCXGJjJ77Gxe2/gaJUdKnA5HBUk7/FNWKjAnTsCHH8LXX1vThYXwyiswbRqsXAn/+peVFNqrmaNmYozhseW6lxAtNCEoZfP5YMUK67GW3/kOpKVZN5P95S9W+WWXwZ491k1mo53v381xeal5TB86nSdWPEHpsVKnw1FBoAlBtUvGwM6dsGABLLJ7YjhxAs45x3rw/bZtcN11Vvns2VZ5bCwkJTkXcyS6+9y7OXbiGD97+2d6xVEU0I7NVbsydy68/761J7BnjzUvP9/aI0hIsAl9eBgAAA8fSURBVJLD0KHu7IDOCYMyBvHzb/2cPyz9A1P6T+GKgVc4HZI6DZoQVFTZsgXWrrV+4W/ZAsuXjyApybocFKxksGULTJgAY8fCmWfCcL8Hvl50kTNxu9kDFz7Ago0LuPPdO7kg7wJSElKcDkm1kSYE5Srbt8OaNbBjBxQXW68lJfDxx+D1wkMPwZ//bNXt0gWysoShQ61LQz0eq8fR9nhVUCh5PV6evORJLv7bxVz/z+t54/tvOB2SaiNNCMoR1dVw4ACUllrDyJGQnAyffgovvGDN27sXvvnGGlatgu7d4W9/g1/+0lpHQgKccQb07GndJNa5M/zsZ1b3EXl5kJICBQWryc/PP7ldTQahMaH3BGaNnsUzq56h+FAxZ6Sc4XRIqg00IahWqamxvnzLyuDo0brxAQOs4+47dlg3aB05YpUfOmQ9IOaXv4QRI2DhQqvTt8OH66932TLrpq5Nm+C55yAjA7p2hT59rDuAvV6r3g03wOTJ0KOHVUek/nqitbsIN/jpWT9l/ur5/Oc7/8nL0152OhzVBpoQHGKMNfh81iBiPXwdrF/O1dV1ZT6fdSdserpV/tVXdeXV1dbVMVlZ0KuXNb14cd38Eyes8WHDYNQoOHbMywMPwPHjUFlZ93rFFTBxonX4ZcYMa35FRd0wZw5Mnw6ff24de2/o73+H738ftm6FO+6w5iUlWb/SO3euSwC5udaXepcudUNGRt2DY268EW66qen3rWdPa1CRp396f35x9i+Y89EcPiz+kG+f8W2nQ1Kt5MqEMHcu/POfdV+qxlgPIXn/fav8vvusL0X/8pQUePddq/yOO+C99+rKysrOpF8/eOstq3zGDPjkk7rymhrrC2ux/USIqVOtQxi1ZcZYJydff90qP/dc2Lix7gu/pgYuvNC6hBGsL8Xi4vptmjYNXrZ/VPXqdeov6BkzrOvfgZPHxP3dfrv1vpw4AVOmnPqe3X23lRAqKz3cc481LyHBGuLjrXVOnGgdUikvt+Z36gSJidaQlVUX+8MPW4d3OnasGwYPtsq/9S0roSUnQ0wjf13DhjXfC2jDX/zKXe759j08vuJxfvb2z1hy4xI6xnV0OiTVCq5MCGB9cXg81quIlRBqJSVBampdmcdj/UqtlZFhHWOuLd+//xh9+nQ4Wd6nj/UL2X8b/r9KR4+21u+//T596sonTrS+YEWsQx0eDwwaVFd+663W4ZTaMq8XBg6sK//DH6wvdq+3bujXr678pZfq1h0ba33x5uZaZfHx1l20MTFWWWysVS8tzSpPSTlBZaU1v7Ev327d6q7IaUxGRt0eQGPi4up/Fqp9SYhJ4Lmpz3H5S5dz+5u38+yUZ50OSbWGMcaRYfTo0SZSLFmyxOkQwqY9tdWY9tXeSGrrXe/eZfg1ZtHmRSFZfyS1NRz82wusNCH6XtZrLpRSQXfvefcypOsQpr86nSXbljgdjgqQJgSlVNDFx8SzaPoiMjtmcsFfL2DBhgVOh6QCoAlBKRUSPTv35PNZnzM8czg/WPgDPt7xsdMhqRZoQlBKhUzHuI4suHoBCTEJnPs/5/LiuhedDkk1QxOCUiqkeqX24oubv+Cs7mfxwzd+yFf7vnI6JNUETQhKqZDr2qErT33nKXw1PgY/OZgZr89g/7H9ToelGtCEoJQKixFZI1j/4/XMHjubv6z5C8PmDePTnZ86HZbyowlBKRU2eal5PDb5MV6+6mU84uHCv17IK1+94nRYyqYJQSkVdtMGT2PVrFXkpuQy7R/TmPz8ZJaVLHM6rHZPE4JSyhEZHTL4fNbn3Jd/H8tLljP+2fGcM/8cXt/4OserjzsdXrukCUEp5ZjE2ER+dd6vKP5JMX+8+I9sO7SNqS9Npe9/9+XZVc/iq/E5HWK7oglBKeW45PhkfnLWTyi6rYiF1ywkNSGVmW/MZMhTQ3jwowdZtXsVNaam5RWp0+La3k6VUtEnISaB7/b/Lpf2u5THP3ucZ1Y/w93v383d799NrCeWgRkDyT8jn85HO9P7cG96dO7hdMhRJaCEICKTgEcBL/CMMeZ3DcrFLr8EOAbcZIxZFeRYlVLthIgwe9xsZo+bzfq961myfQnbD21n5a6V/HnVn6moruD+DfeT1TGLrI5ZZCRlMDp7NNOHTqdrh64kxyeTGJOI6AM2WqXFhCAiXuAJYAJQAqwQkYXGGP/bDScDfe1hHPCU/aqUUqdlcNfBDO46+OT0Cd8J5i+eT1XXKlbtWcX+Y/tZtXsV72x9h999Uvdb1SteOsZ1JDk+meS45JOvibGJxHvjiY+Jt1698STGJpKakEp6UjpZHbNIjE0kMSax0dc4bxxe8RLjicHr8eKR6DnyHsgewlig0BizFUBEXgSmAP4JYQrwV7uv7mUikiIi2caY3UGPWCnVrsV6Y+mf3J/8cfn15m/Yt4GVu1ZytOooRyqPcLTyKEer7KGy7nV/xX6qfFVUVldS6auksrqSYyeOUX6ivM0xxXhi8IoXr8dbL1k0HPd6rOkfjvohd4xv5klTDgkkIeQAO/2mSzj1139jdXKAeglBRGYBswAyMzMpKChoZbihUVZWFjGxhFp7aiu0r/ZqW6EHfucUvECiPQSguqaa0qpSyqrLrERRYw1VNVXWuD3PZ3zUmBp8xmeNU3Ny3sn51NVpbP7+HfspqDw1/ta2N9gCSQiNHYQzbaiDMeZp4GmAMWPGmPz8/AA2H3oFBQVESiyh1p7aCu2rvdrW6BWu9gZy8KsE/NMu3YFdbaijlFIqggWSEFYAfUUkT0TigGuAhQ3qLARuEMtZwGE9f6CUUu7S4iEjY0y1iNwKvIV1VG6+MWa9iNxsl88DFmNdclqIddnpjNCFrJRSKhQCug/BGLMY60vff948v3ED3BLc0JRSSoVT9FxAq5RS6rRoQlBKKQVoQlBKKWXThKCUUgoAsc4HO7BhkX1AsSMbP1U6UOp0EGHSntoK7au92tbo5d/eM4wxGaHYiGMJIZKIyEpjzBin4wiH9tRWaF/t1bZGr3C1Vw8ZKaWUAjQhKKWUsmlCsDztdABh1J7aCu2rvdrW6BWW9uo5BKWUUoDuISillLJpQlBKKQVEUUIQkftFZK2IrBGRt0Wkm1/ZXSJSKCKbRORiv/mjReRLu+wxsZ/ILSLxIvKSPX+5iOT6LXOjiGyxhxv95ufZdbfYy8aFsK0PichGu73/FJGUKG7rNBFZLyI1IjKmQVlUtbWtRGSS/R4UisidTsfTHBGZLyJ7RWSd37w0EXnHfo/fEZFUv7KQf8YhbGsPEVkiIhvsv+HbI769xpioGIBOfuO3AfPs8UHAF0A8kAcUAV677DNgPNYT394EJtvzf+y3/DXAS/Z4GrDVfk21x1PtspeBa+zxecCPQtjWiUCMPf574PdR3NaBQH+gABjjNz/q2trG98drt70XEGe/J4OcjquZeL8NjALW+c37A3CnPX5nuP+eQ9jWbGCUPZ4MbLbbFLHtdfwPJEQfxF3AU37jd/mVvWW/sdnARr/53wf+5F/HHo/BukNQ/OvYZX+y54ldp/ZLejzwVpjaejnwfLS3lVMTQtS2tZXvS72YGr4vkTgAudRPCJuAbHs8G9gUrs84zO1+HZgQye2NmkNGACLyWxHZCVwL/MqenQPs9KtWYs/Lsccbzq+3jDGmGjgMdGlmXV2AQ3bdhusKtf/A+sVAM/FFS1v9tae2Nqep2N0k09hPWLRfu9rzw/EZh4V9KGcksJwIbm9AD8iJFCLyLpDVSNE9xpjXjTH3APeIyF3ArcC9WNmyIdPMfNqwTHPrapOW2mrXuQeoBp6vXawN8bmirY0t1kQcEd3WEHBDjG0Vjs845ESkI/Aq8BNjzBH78H+jVRuZF9b2uiohGGMuCrDq34FFWAmhBOjhV9Yd2GXP797IfPyWKRGRGKAzcMCen99gmQKs3bQUEYmxs7T/utqkpbbaJ4kuBS409j4hUdrWJriyrSHQ1PvgJt+ISLYxZreIZAN77fnh+IxDSkRisZLB88aYBfbsyG2v08cTg3h8rq/f+GzgFXt8MPVP1Gyl7kTNCuAs6k7UXGLPv4X6J2petsfTgG1YJ2lS7fE0u+wf1D/5+OMQtnUS8BWQ0WB+1LXVr20F1D+HELVtbeX7EmO3PY+6k8qDnY6rhZhzqX8O4SHqn2T9Qzg/4xC2U4C/AnMbzI/Y9jr+xxHEN/9VYB2wFngDyPEruwfrjP0m7LPz9vwx9jJFwOPU3bmdYH8RFGKd3e/lt8x/2PMLgRl+83vZdQvtZeND2NZCrOODa+xhXhS39XKsXzuVwDfUP4EaVW09jffoEqwrWIqwDrM5HlMzsb4A7AZO2J/rD7COeb8HbLFf0/zqh/wzDmFbz8E6TLPW73/1kkhur3ZdoZRSCoiiG9OUUkqdHk0ISimlAE0ISimlbJoQlFJKAZoQlFJK2TQhKNcRkS5i9Wq7RkT2iMjX9vghEfkqBNvLF5F/tXKZgoa9s9rzbxKRx4MXnVLBowlBuY4xZr8xZoQxZgTWzWJ/tMdHADUtLW/f0amUakATgoo2XhH5s93//Nsikggnf7E/ICIfALeLSIaIvCoiK+zhbLveeX57H6tFJNleb0cReUWs51A879cf/YV2vS/tvv7jGwYkIjNEZLO97bPD9D4o1WqaEFS06Qs8YYwZDBwCrvQrSzHGnGeMeRh4FGvP4ky7zjN2nf8L3GLvcZwLVNjzRwI/weqzvhdwtogkAH8BrjbGDMXqRuJH/sHYfdXch5UIJtjLKxWRNCGoaLPNGLPGHv8cq9+cWi/5jV8EPC4ia4CFQCd7b+AT4BERuQ0rgdR2ff2ZMabEGFOD1QVBLtaDe7YZYzbbdZ7DegCMv3FAgTFmnzGmqkEMSkUUPZaqok2l37gPSPSbLvcb92A9WKSC+n4nIouw+pxZJiK1PbE2XG8MjXcx3BjtH0a5gu4hqPbqbaxnZgAgIiPs197GmC+NMb8HVgIDmlnHRiBXRPrY09cDHzSosxzIt6+MigWmBasBSgWbJgTVXt0GjBGRtfalqjfb838iIutE5Aus8wdvNrUCY8xxYAbwDxH5EusKp3kN6uwGfg18CrwLrAp2Q5QKFu3tVCmlFKB7CEoppWyaEJRSSgGaEJRSStk0ISillAI0ISillLJpQlBKKQVoQlBKKWX7/8jnaS0i8m+NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_precision_recall_vs_threshold(precision, recall, thresholds)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- You may wonder why the precision curve is bumpier than the recall curve.\n",
    "    - The reason is that the precision may sometimes go dows when you raise the threshold.\n",
    "    - You may lose precision by upping the threshold if you lose TPs while the FPs are still there\n",
    "    - But generally, precision should increase.\n",
    "- On the other hand recall is very smooth because..\n",
    "    - Because we we add another TN while moving the threshold, nothing changes in the score.\n",
    "- Another way to select a good precision and recall trade-off is to plot precision directly against recall:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5dn/8c+VHRL2QET2HVEBJbK4BrUWxT7aSlu01dbaUp9Wq7WbXdzqY2sXn1+1tVpcWu0itdWnblSraNwA2WQXJCyyy76EhIQk1++PGUKAQIYwJ2cm832/Xnl5zpn7zFw34Hxztvs2d0dERFJXWtgFiIhIuBQEIiIpTkEgIpLiFAQiIilOQSAikuIywi7gWOXn53vPnj0bte+ePXvIzc2Nb0EJTn1ODepzajiePs+ePXuLu3es77WkC4KePXsya9asRu1bXFxMUVFRfAtKcOpzalCfU8Px9NnMPjrSazo1JCKS4hQEIiIpTkEgIpLiFAQiIilOQSAikuICCwIze9zMNpnZwiO8bmb2gJmVmNl8Mzs9qFpEROTIgjwi+BMw5iivXwz0i/5MAB4KsBYRETmCwJ4jcPe3zKznUZpcBjzpkXGwp5tZWzPr7O4bgqhn6cbdPLuskjmVS2NqX9izPef2r/fZCxGRZiXMB8q6AGvqrK+NbjssCMxsApGjBgoKCiguLj7mD5uxsYoXllfC8pIG2zrQJc+45+yWx/w5iaa0tLRRf17JTH1ODepz/IQZBFbPtnpnyXH3icBEgMLCQm/Mk3VFwPATYnsq75t/m8OMlduYX92FZZtKObdfPp8t7HbMn5kI9PRlalCfU0NQfQ4zCNYCdb9duwLrQ6rlIK2yM9i8u4L/ffVD0tOM5ZtK6d0xj9O7t8WsvvwSEUleYQbB88ANZjYJGAHsDOr6wLH60diT+OLIHvTumMuEJ2fzTskWrnhoKk99bSRllVXMWb2dBet2cdXw7ow55YSY37e8spp1O8pYv2MvG3aWs2HnXjbs2Mv6neXsq67h6+f1YVTvDuRkpgfYOxGRgwUWBGb2FJEzMvlmtha4A8gEcPeHgcnAJUAJUAZcG1Qtx6p1TiandGkDwLc/0Z8ubVvw91lruPKR6QCkpxnVNc5bH27mqa+NpGu7FsxZvZ3F63excP1OZq7azgmtc5hwbm9KNpWyfHMpKzbvYd2O8sM+q2OrbDbvrgBg+opttdtbZqUz5pQT+HjXXtZtLycjPY0LBnZi0+4KPt61l1Vb9jCkW1v+a8iJXHxqZwBqapztZZU4kJ+XHfCfkog0F0HeNXRlA6878M2gPj9ehvVoR6/8XHbt3cfAE1ozsnd7Bndty0/+tZBn5qytDQeArPQ0BpzQisqqGlZvK+Mn/1pIy6x0+nTM44ye7fh8x2706NCSzm1a0LlNDgWtc8jKSGNn+T7mfLSdB98o4YQ2Obw4fwNlldVMX76VgjY5rNpaBsDqrWV0bJVNXnYG63fuZf3Ojfx74Ub6F+Sxe28Vm3dXUFUTucySnZFGRVUNp3dK58ElU9m4ay9rtpXz9wkjOblLGzbvrmBLaQV9OuZhwNY9lWwtrTjovzU1zhXDutKjQ2oN9SuSapJuGOowtM/N4qEvDjto232fG0JVTQ1791VzZp98Cnu2o39BKzLT01i3o5xlH++mf0ErOrfJafC6QpsWmYwe2InRAzsB8MB4x4kceQBUVtVQVllFmxaZte9VWlHFKws38pspH9KpVQ6ndsmhU+tsKvbVMPujbWzaXcHm3RWsL62hZ56xZlvkaOTzE6fXW8ORPPB6SW2odMjN4qoR3VmxeQ8jerdnZ9k+dldUcdGgAoZ2a0tGuh5UF0lGCoLjcP/40+rd3qVtC7q0bdHo901LOzg4sjLSyMrIOmhbXnYGVwzryhXDuh71vSJ3GYyipsb5z+KNvFOyhW7tWtKxVTYfflxKehp0yM2mQ14W+XnZtM/NokNeFobx1/c+4oV56+mQm82MVdvYuqeS374euf32pQUHLudMfGtF7fLVI3uwbU8lW0orWLxhF8N7tufkE1uTnZnON0f3bfSfiYgER0GQItLSjDGndGbMKZ1j3ufmC/tz84X9a9d3791Hy6wM1mwrIzszjRaZ6Uz5YBOvLv6YlxdtBOCF+evpkJvF8s17AJiyZBNTlmwC4FevRB7m69a+Bfl52fTrlMfW0koWrd/FqV3bsHdfdeQoZkc5Azu3ZsbKbbx76/nHFaoi0jAFgcSsVU4mAD3zD1wzqHtUUl3jtaezANyd7WX72F5Wya3PzGfV1jI2765gzbZy1mwrZ/2OcsoqqtldUUXaOujUOoclG3fTu2MuM1ZGLpyfde/rdMjN4upRPRg9oBMnn9hap6BE4kxBIHGTfsgpLTOjfW4W7XOz+Mf1Z9Zuj9wnQIPXTsY+8DaL1u9i655KfvPaMn7z2jIA+nXKo2u7FuTlZHLNqB5s2V3Bpt0VFLTOZvfeKvburmHF5sizH+WV1dS4k5utf+oiR6L/O6TJxfpQ3kvfOgd3Z+32ct5Yuonbn1sEwLJNpSzbVArAC/OO8Aziu28etun+8UO5bGiXxhUt0owpCCShmRnd2rfkmlE9uWZUz9rt2/dU8vKijXRp24JWORlsL6skPS2NRet3snH1SubszGFErw5sL6vk2TnrALhp0ly+/fe5XHxKZ3Kz0zm3f0fO7JNPuhmtW2ToqXFJWQoCSUrtcrO4cnj3w7af178jxcVr+WnRObXb7vvsEIo/3My1f5xJjR+44+npWWsP2vfsvvl8vGsvXxjRnS+f1SvYDogkEAWBNHtmxugBnSi552LMjD2VVazeWsaT01bROieTR99ZSX5eFu+UbAHgzhcW89MXF9MiM53zBnTk410VzFm9HXe4fOiJ7Czfx+INu8hIS6NlVjrV7lTsq+GSU0/ge58cSFaGLmZLclEQSMrYf7fR/iFEfjluCAA/uXQQEBmi44X567lp0lxqHPZUVrNk42627K4gen2bN5ZupmVWOtv2VLKv+uDBch95eyWPvL0SiFzQvuTUzlx+Whd65evJbElsCgKRqLQ047KhXY75gvL+C9q3PD2Xmau2A5EL2vdPWcb9UyJ3Oj1yTSGDu7ahoHVO3OsWOV4KApHjtP+Cdt1bZJd9vJu/TP+IJ6Z9BMDXnpx12H552RmUVlRx+6WD+MrZuiYh4VEQiASgX0Er7rrsFO741Mm8XbKF1xZ/zLy1O9i7r5o128rZW1VNaUUVAD99MXJNIj8vm5su6MtZffPplZ+ru5ikySgIRAKUlmac178j5x1h/us3lmzi2j/NBGBLaQW3RZ+V2K9FZjrl+6oZ1bsDnzm9CwvX7aR1i0yKBnSkR4dcDTcucaEgEAnR6IGdWHXvWNydmau289aHm/lgwy6mLNnESZ1b88GGXQBMW7GVaSu21u63f/A/gN75ubRP38sSW86AE1rROieDod3aHfakt8iRKAhEEoCZMbxXe4b3an/YaxVV1cz5aAed2+Swe28VL8xfT05GGn+bsYayyipWbNnDCmDWv5cc9TO+clYvbrv0JJ1yksMoCEQSXHZGOqP6dKhdP7VrZPa8Wy4aULvt+f+8QeseJ1NV7bz2wcfkZmewbFMpPdq35N8LN7KltILH313J4++upG3LTGb9+EIN3ie1FAQizUDrLKNoQGRiowsHFRz02t2Xn8LO8n3c/txCnpu7nh1l++j7439zRs92nNa9HVec3pX+BXk6UkhhCgKRFNCmRSb3jz+NX1wxmBE/m8LO8n3MXLWdmau2104sdN3ZvWiZlc5NF/TT0UKKURCIpJCczHTm3XERAGWVVTw1Yw13v7gYgMfeiTwV/dvXS2idk4EDt40dxOWnddGwGc2cgkAkRbXMyuC6s3tx3dm9cHf2VFZz7R9nkJOZztvLIuMuff+Z+Xz/mfn89srT+NSQE0OuWIKiIBARzIy87IyDno5esbmU8++LzOtw41Pv8+2/z2VE7/bcdEH/eu9ukuSl4z0RqVfvjnmsuncsv7vqNACqapx3S7byuT9M46TbXqaquibkCiVeFAQiclSXDj6RVfeOZdW9Y7n3M6cCUL6vmr4//jffeXoee/dVh1yhHC8FgYjEbPzw7iy5ewz77zR9Zs5aBt72Mtf/eXa4hclxURCIyDHJyUxn5c/HsvCuT3JudAyllxdtpOetL4VcmTSWgkBEGiUvO4MnvzKcebdfVLut560vMXPVNtz9KHtKolEQiMhxadMyk9duObd2/bMPT6PXDyfzUPHyEKuSY6EgEJHj1rdTK1bdO5b7PjuEE6KzsP3i5SVc9P/eDLkyiYWCQETi5ophXZn+owv4y3UjAPjw41J63vqSThUlOAWBiMTd2f3yef+2T9SuvxV9UlkSk4JARALRLjeLp78+CoAvPT6DPRVVeggtQWmICREJzBk929Uun3zHK7XLXxzZnf+5/NQwSpJ66IhARAJjZqz8+SVce1ZPLht6Ivtnz/zL9NX0vPUlHokOgS3h0hGBiATKzLjjUycDcP/405i7ZgeXP/guAPdM/oD/LN540GB30vQCPSIwszFmttTMSszs1npeb2NmL5jZPDNbZGbXBlmPiIRvaLe2rLp3LDdf2A+Amau28+P/W0Blla4fhCWwIDCzdOBB4GJgEHClmQ06pNk3gcXuPgQoAu4zs6ygahKRxHHzhf355bjBAPz1vdX0/8m/2bRrb8hVpaYgjwiGAyXuvsLdK4FJwGWHtHGglUUmS80DtgFVAdYkIgnkc4XdePYbB04LDf/ZFO54bmGIFaUmC+pBDzMbB4xx969G168GRrj7DXXatAKeBwYCrYDPu/thI1eZ2QRgAkBBQcGwSZMmNaqm0tJS8vLyGrVvslKfU0Nz6PNd08pZufPA6aE7RuXQq036Eds3hz4fq+Pp8+jRo2e7e2F9rwV5sdjq2XZo6nwSmAucD/QBXjWzt91910E7uU8EJgIUFhZ6UVFRowoqLi6msfsmK/U5NTSHPhcVwdSSLVz16HsA3DVtL299bzTdO7Sst31z6POxCqrPQZ4aWgt0q7PeFVh/SJtrgWc9ogRYSeToQERS0Jl981l179ja9XN/9QYVVZr4JmhBBsFMoJ+Z9YpeAB5P5DRQXauBCwDMrAAYAOjGYpEUVzcMxk+cHmIlqSGwIHD3KuAG4BXgA+Bpd19kZteb2fXRZncDZ5rZAmAK8AN316AkIsKCOyPzHLy/egd7KnQPSZACfaDM3ScDkw/Z9nCd5fXARYfuJyLSKieT07q35f3VOzj5jlcOOkqQ+NIQEyKSsP5Z54njneX7QqykeVMQiEjCSk8zxg7uDMCQu/5DqU4RBUJBICIJ7XdXnla7fModrzB9xdYQq2meFAQiktD2j2C63/iJ09m8uyLEipofBYGIJDwzY9W9Y+ncJjIf8hn3vBZyRc2LgkBEksa0H15Qu/zoggqqazQXcjwoCEQkqTzxleEAvLOuij4/mkxQ46WlEgWBiCSV8/p35D/fPrd2vdcPJ7OjrDLEipKfgkBEkk7/glY8eMGBweiG/vRVpi3X3USNpSAQkaSUmxm5gNy3U2RY5isfmU6Nrhk0ioJARJLaa7ecV7s88LaXQ6wkeSkIRCTpzb39EwBUVtcwtUTjVh4rBYGIJL22LbO4/dLIlOhXPfoekxdsCLmi5KIgEJFm4Stn9+KSU08A4Bt/nRNyNclFQSAizcbvvzCsdnnbHt1SGisFgYg0K18/tzcAM1bqdtJYKQhEpFm5cFABAE9O+yjkSpKHgkBEmpXTurUFYOryrby+5OOQq0kOCgIRaVYy0tO48fy+AHzlT7NCriY5KAhEpNn5zkUDyMqIfL1pHKKGKQhEpFm64vSuANz1wuKQK0l8CgIRaZZuvrAfAP/3/joNVd0ABYGINEsFrXNql7/5Nz1gdjQKAhFptkruuRiAyQs2hlxJYlMQiEizlZF+4Cvu1DtfCbGSxKYgEJFmbdKEkQDs3lvFG0s3hVxNYlIQiEizNrJ3B56MznN87R9nhlxNYlIQiEizd06//LBLSGgKAhFp9syMSwd3DruMhJURa0Mz6wL0qLuPu78VRFEiIvG2fPMeAF6cv55LB58YcjWJJaYjAjP7BfAu8BPge9Gf7wZYl4hIXN029iQA5q7eEXIliSfWI4LLgQHuXhFkMSIiQTmjV3sAHn1nJT+JTmspEbFeI1gBZAZZiIhIkDLrPFPw8a69IVaSeGINgjJgrpn9wcwe2P8TZGEiIvF2yyf6AzD618XhFpJgYg2C54G7ganA7Do/R2VmY8xsqZmVmNmtR2hTZGZzzWyRmb0Za+EiIsfqa+dEprEsq6xm4bqdIVeTOGK6RuDuT5hZFtA/ummpu+872j5mlg48CHwCWAvMNLPn3X1xnTZtgd8DY9x9tZl1akwnRERi0SIrne+PGcAvX17Kq4s/5pQubcIuKSHEetdQEbCMyBf774EPzezcBnYbDpS4+wp3rwQmAZcd0uYq4Fl3Xw3g7nr+W0QCtf+o4P4py0KuJHHEetfQfcBF7r4UwMz6A08Bw46yTxdgTZ31tcCIQ9r0BzLNrBhoBdzv7k8e+kZmNgGYAFBQUEBxcXGMZR+stLS00fsmK/U5NajPjfPCf96gVZbFp6AmENTfc6xBkLk/BADc/UMza+guovr+dA+dHSKDSJhcALQAppnZdHf/8KCd3CcCEwEKCwu9qKgoxrIPVlxcTGP3TVbqc2pQn4/Nl3ct4k9TV3Hj62WsundsfAsLUFB/z7FeLJ5lZo9FL+wWmdkjNHyxeC3Qrc56V2B9PW1edvc97r4FeAsYEmNNIiKNclud5wjmrN4eYiWJIdYg+G9gEfAt4CZgMXB9A/vMBPqZWa/ohebxRO4+qus54BwzyzCzlkROHX0Qa/EiIo2RnmZ88uQCAD7z+6khVxO+mILA3Svc/X/d/TPu/ml3/38NPWXs7lXADcArRL7cn3b3RWZ2vZldH23zAfAyMB+YATzq7guPp0MiIrH4w9WFtcs7y456E2Szd9QgMLOno/9dYGbzD/1p6M3dfbK793f3Pu5+T3Tbw+7+cJ02v3L3Qe5+irv/5ng7JCISq/8u6gPAj/61IORKwtXQxeKbov+9NOhCRESa2o3n9+Wh4uVs31MZdimhOuoRgbtviC5uAda4+0dANpELuode+BURSSotsyK/C09dvpU128pCriY8sV4sfgvIic5JMAW4FvhTUEWJiDSVkb0jo5Je/uC7IVcSnliDwNy9DPgM8Ft3/zSgcVxFJOlNmjAKgK0pfHoo5iAws1HAF4CXottint1MRCQZPPr2irBLCEWsQXAz8EPg/6K3gPYG3giuLBGRpvPaLecB8LPJqfkYU6yjj74JvFlnfQWRh8tERJJe3055ANQcOghOijhqEJjZb9z9ZjN7gcPHCcLd/yuwykREmtDp3dsyZ/UOyiqrau8mShUN9fbP0f/+OuhCRETCNLBza+as3sGSjbs5vXu7sMtpUkcNAnffP7DcLKDc3WugdtKZ7IBrExFpMp8YVMDf3lvNX6Z9lHJBEOvF4ilAyzrrLYDX4l+OiEg4RvXuAMCz768LuZKmF2sQ5Lh76f6V6HLLo7QXEUkqOZnptcuzP0qtoaljDYI9Znb6/hUzGwaUB1OSiEg4fv3ZyHQoVzyUWkNTH8tzBP8ws7fN7G3g70SGmBYRaTbGDetau1yTQveSxjofwUxgIJEJar4BnFTnQrKISLNxwcBOAExfuTXkSppOTEEQnT3sB8BN7r4A6GlmGppaRJqda87sCUDx0s3hFtKEYj019EegEhgVXV8L/E8gFYmIhOi07m0BmPhW6ow7FGsQ9HH3XwL7ANy9HLDAqhIRCUnrnMywS2hysQZBpZm1IDrMhJn1AY46Z7GISLLq0SFyd/z9ry0LuZKmEWsQ3EFkkvluZvZXIg+YfT+wqkREQvT4l88A4Pl5qfFwWYMjK5mZAUuITEozksgpoZvcfUvAtYmIhKJPx8hopG1apMZpogaDwN3dzP7l7sM4MCmNiEiz1iE3izmrd1BVXUNGeqwnT5JTrL2bbmZnBFqJiEgC6dKuBQBvftj8byONNQhGEwmD5WY238wWmNn8IAsTEQnTXf91MgDz1uwIuZLgxTr7wsWBViEikmB65ecC8MDrJdxy0YCQqwlWQzOU5QDXA32BBcBj7l7VFIWJiISpbcus2uX3VmxlRHSY6uaooVNDTwCFRELgYuC+wCsSEUkQvxo3GIDPT5zerAehaygIBrn7F939D8A44JwmqElEJCF8trBb7fKX/jgjxEqC1VAQ7Nu/oFNCIpKKFt31SQDeXtZ8H51qKAiGmNmu6M9uYPD+ZTPb1RQFioiEKTf7wKXUZ2avDbGS4Bw1CNw93d1bR39auXtGneXWTVWkiEiYJn8rclZ81kfbQq4kGM37cTkRkTgYdGLk996nZqwJuZJgKAhERI7BltLmN/CygkBEJAbfOr8vAD96dkHIlcSfgkBEJAY3nN8PgK17KkOuJP4CDQIzG2NmS82sxMxuPUq7M8ys2szGBVmPiEhjZWVEvi7XbS8PuZL4CywIzCwdeJDIE8mDgCvNbNAR2v0CeCWoWkRE4qFTq2w27tobdhlxF+QRwXCgxN1XuHslMAm4rJ52NwLPAJsCrEVE5Li1z42MP/TGkub1dWXuwYyfET3NM8bdvxpdvxoY4e431GnTBfgbcD7wGPCiu/+znveaAEwAKCgoGDZp0qRG1VRaWkpeXl6j9k1W6nNqUJ+bxnsbqnhoXgXdWqVx91ktmvSz4fj6PHr06NnuXljfa7EOQ90YVs+2Q1PnN8AP3L06MiNm/dx9IjARoLCw0IuKihpVUHFxMY3dN1mpz6lBfW4aRcBD815ize6aUP68g+pzkEGwFuhWZ70rsP6QNoXApGgI5AOXmFmVu/8rwLpERI7bltIK8vOywy4jLoK8RjAT6GdmvcwsCxgPPF+3gbv3cvee7t4T+CfwDYWAiCSyH19yEgBXP9Z8RiMNLAiio5XeQORuoA+Ap919kZldb2bXB/W5IiJBuu7sXgB8sKH5jLsZ5Kkh3H0yMPmQbQ8foe2Xg6xFRCQe0tIOXM8sragiLzvQr9EmoSeLRUSO0adP6wLAr15eEnIl8aEgEBE5RrdePBCAJ6Z9FHIl8aEgEBE5RgWtc2qXJ81YHWIl8aEgEBFphBdvPBuAW5vBaKQKAhGRRjilS5va5V179x2lZeJTEIiINNLFp5wAwAOvLQu5kuOjIBARaaRfjhsMwKPvrGRTEo9KqiAQEWmkVjmZ9OsUGQTusXdXhlxN4ykIRESOw9+/PgqAP7y5IuRKGk9BICJyHPbPUQCwLUmnsVQQiIgcp6tH9gDgS48n50B0CgIRkeP008tOBnREICKSsvZPrLVuRzkrt+wJuZpjpyAQEYmDmy7oB8AqBYGISGo6q28+AH+ZnnwD0SkIRETiYEBBKwCmLNmE+6HTsyc2BYGISBy0aZlJy6x0AH42+YOQqzk2CgIRkTiZeuv5ADzy9koqqqpDriZ2CgIRkThp2/LAw2V/TqJJaxQEIiJxNOsnFwLwPy8lz+khBYGISBzl52XXLj8wJTmGp1YQiIjE2Wu3nAfA/776YciVxEZBICISZ32jQ1MD1NQk/q2kCgIRkQBceFIBAL1/NDnkShqmIBARCcDvrjot7BJipiAQEQlATmY6owd0BODXrywNuZqjUxCIiATkF1dE5jT+3RslIVdydAoCEZGAdGqdU7u8cWfiTm6vIBARCdAvo0cFI38+JeRKjkxBICISoM+d0a12+U/vrgyxkiNTEIiIBOz+8UMBuPOFxQk5RLWCQEQkYJcN7UJudIjqiqqakKs5nIJARKQJXDGsKwA/eGZ+yJUcTkEgItIEvhWd0/i5uevZU1EVcjUHCzQIzGyMmS01sxIzu7We179gZvOjP1PNbEiQ9YiIhCU/L5vTu7cF4OQ7Xgm5moMFFgRmlg48CFwMDAKuNLNBhzRbCZzn7oOBu4GJQdUjIhK2Z/77zNrlxet3hVjJwYI8IhgOlLj7CnevBCYBl9Vt4O5T3X17dHU60DXAekREQmVmfGlUDwAueeDtkKs5wIK6lcnMxgFj3P2r0fWrgRHufsMR2n8XGLi//SGvTQAmABQUFAybNGlSo2oqLS0lLy+v4YbNiPqcGtTn5PLll/cAcGnvTMb1z2qg9QHH0+fRo0fPdvfC+l7LaNQ7xsbq2VZv6pjZaOA64Oz6Xnf3iURPGxUWFnpRUVGjCiouLqax+yYr9Tk1qM/J5b5Wa/nOP+bx4op9/Oya0bTOyYxpv6D6HOSpobVAtzrrXYH1hzYys8HAo8Bl7r41wHpERBLCFcO6cmqXNgAMvvM/7CzfF2o9QQbBTKCfmfUysyxgPPB83QZm1h14Frja3ZNjTjcRkTh44cYDJ0A+9dt3QqwkwCBw9yrgBuAV4APgaXdfZGbXm9n10Wa3Ax2A35vZXDObFVQ9IiKJpuSeiwFYva2M9TvKQ6sj0OcI3H2yu/d39z7ufk9028Pu/nB0+avu3s7dh0Z/6r2QISLSHGWkp/HFkd0BeGLaqtDq0JPFIiIhuvnC/gD84c0VodWgIBARCVF+Xjb5edkAlFWGM/SEgkBEJGTjogPS3fC390P5fAWBiEjIvjm6DwCvL9lETU3Tz1egIBARCVmrnExG9m4PwG3PLWzyz1cQiIgkgF+Niwy+/Nf3Vjf5LGYKAhGRBNCtfUvSogPzNPW1AgWBiEiCmHfHRQC8tGADC9ftbLLPVRCIiCSIVjmZfK4wcgfRpb99h+omunCsIBARSSC/HHdgosbP/P7dJvlMBYGISIJZ/rNLAJi3dif7qmsC/zwFgYhIgklPM87q2wGAr/95duCfpyAQEUlAf7g6Mgbn60s2BX47qYJARCQB5WVncHr3tkAkDIKkIBARSVDf/kRkZNLrngh2qhYFgYhIgjqnX8fa5SemrgrscxQEIiIJ7I3vFgFwx/OLAvsMBYGISALrlZ9bu/z00spAPkNBICKS4N75wWjO6ZdPrzbBfGUrCEREElzXdi3583UjOOOEjEDeX0EgIpLiFAQiIilOQSAikuIUBCIiKaVpWq4AAAVeSURBVE5BICKS4hQEIiIpTkEgIpLiFAQiIinOgh7nOt7MbDPwUSN3zwe2xLGcZKA+pwb1OTUcT597uHvH+l5IuiA4HmY2y90Lw66jKanPqUF9Tg1B9VmnhkREUpyCQEQkxaVaEEwMu4AQqM+pQX1ODYH0OaWuEYiIyOFS7YhAREQOoSAQEUlxzTIIzGyMmS01sxIzu7We183MHoi+Pt/MTg+jzniKoc9fiPZ1vplNNbMhYdQZTw31uU67M8ys2szGNWV9QYilz2ZWZGZzzWyRmb3Z1DXGWwz/ttuY2QtmNi/a52vDqDNezOxxM9tkZguP8Hr8v7/cvVn9AOnAcqA3kAXMAwYd0uYS4N+AASOB98Kuuwn6fCbQLrp8cSr0uU6714HJwLiw626Cv+e2wGKge3S9U9h1N0GffwT8IrrcEdgGZIVd+3H0+VzgdGDhEV6P+/dXczwiGA6UuPsKd68EJgGXHdLmMuBJj5gOtDWzzk1daBw12Gd3n+ru26Or04GuTVxjvMXy9wxwI/AMsKkpiwtILH2+CnjW3VcDuHuy9zuWPjvQyswMyCMSBFVNW2b8uPtbRPpwJHH//mqOQdAFWFNnfW1027G2SSbH2p/riPxGkcwa7LOZdQE+DTzchHUFKZa/5/5AOzMrNrPZZnZNk1UXjFj6/DvgJGA9sAC4yd1rmqa8UMT9+yuYmZDDZfVsO/Qe2VjaJJOY+2Nmo4kEwdmBVhS8WPr8G+AH7l4d+WUx6cXS5wxgGHAB0AKYZmbT3f3DoIsLSCx9/iQwFzgf6AO8amZvu/uuoIsLSdy/v5pjEKwFutVZ70rkN4VjbZNMYuqPmQ0GHgUudvetTVRbUGLpcyEwKRoC+cAlZlbl7v9qmhLjLtZ/21vcfQ+wx8zeAoYAyRoEsfT5WuBej5xALzGzlcBAYEbTlNjk4v791RxPDc0E+plZLzPLAsYDzx/S5nngmujV95HATnff0NSFxlGDfTaz7sCzwNVJ/NthXQ322d17uXtPd+8J/BP4RhKHAMT2b/s54BwzyzCzlsAI4IMmrjOeYunzaiJHQJhZATAAWNGkVTatuH9/NbsjAnevMrMbgFeI3HHwuLsvMrPro68/TOQOkkuAEqCMyG8USSvGPt8OdAB+H/0NucqTeOTGGPvcrMTSZ3f/wMxeBuYDNcCj7l7vbYjJIMa/57uBP5nZAiKnTX7g7kk7PLWZPQUUAflmtha4A8iE4L6/NMSEiEiKa46nhkRE5BgoCEREUpyCQEQkxSkIRERSnIJARCTFKQhEDhEdqXSumS2MjmrZNs7v/2Uz+110+U4z+24831/kWCkIRA5X7u5D3f0UIoN/fTPsgkSCpCAQObpp1BnQy8y+Z2Yzo+PA31Vn+zXRbfPM7M/RbZ8ys/fM7H0zey361KtIwml2TxaLxIuZpRMZuuCx6PpFQD8iQyMb8LyZnQtsBX4MnOXuW8ysffQt3gFGurub2VeB7wPfaeJuiDRIQSByuBZmNhfoCcwGXo1uvyj68350PY9IMAwB/rl/WAN33z+WfFfg79Gx4rOAlU1Svcgx0qkhkcOVu/tQoAeRL/D91wgM+Hn0+sFQd+/r7o9Ft9c3Vstvgd+5+6nA14GcJqhd5JgpCESOwN13At8CvmtmmUQGPvuKmeVBZOIbM+sETAE+Z2Ydotv3nxpqA6yLLn+pSYsXOQY6NSRyFO7+vpnNA8a7+5/N7CQik70AlAJfjI6GeQ/wpplVEzl19GXgTuAfZraOyPSgvcLog0hDNPqoiEiK06khEZEUpyAQEUlxCgIRkRSnIBARSXEKAhGRFKcgEBFJcQoCEZEU9/8BmLtR2ir0BdQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(recall, precision)\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- You can see the precision really starts to fall sharply around 80% recall.\n",
    "- You would probably want to select a precision/recall trade-off just before **that drop**.\n",
    "    - For example, at around $60\\%$ recall.\n",
    "        - But ofcoures, the choice depends on your project.\n",
    "- Suppose you want to aim for $90\\%$ precision, you would look up the first graph to get the threshold\n",
    "    - and lookup the second graph to find the best Recall you can get with a precision of $90\\%$.\n",
    "- Implementation: Search for the lowest threshold that give at least $90\\%$ precision score:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12605.39256572511"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold_90_precision = thresholds[np.argmax(precision > .9)]\n",
    "threshold_90_precision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- To make predictions for the training set, you first get the score of the desired point and compare the score to the chosen threshold:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred_90 = (y_scores > threshold_90_precision)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Let's check these predictions' precision and recall:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9003181336161188"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = precision_score(y_true=y_train_5, y_pred=y_train_pred_90)\n",
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6267995570321152"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall = recall_score(y_train_5, y_train_pred_90)\n",
    "recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- But remember that a high precision classifier is not good at all with a low recall.\n",
    "- If someone shouts: \"Let's reach a 99% precision score\", you should ask: \"But at what Recall?\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The ROC Curve\n",
    "\n",
    "- The receiver operating characteristic curve is another tool used with binary classifiers.\n",
    "- ROC plots TP rate versus FP rate\n",
    "    - ROC plots recall versus 1 - specificity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_true=y_train_5, y_score=y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0,1], [0,1], 'k--', label='Random')\n",
    "    plt.grid()\n",
    "    plt.xlabel(\"False Positive Rate\")\n",
    "    plt.ylabel(\"True Positive Rate\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgUVdbA4d/JnkBIgEAEwr4HEJBFcQSDuOCK4+5EVIQBRBy3+RRhRBj3lZFV0VEGN2ZUUEQEFYmggGwiSxBkE4IsSSAh+9J9vz+6E2NMQiekutPd532ePKSqq6vPTUKdureqzhVjDEoppfxXgKcDUEop5VmaCJRSys9pIlBKKT+niUAppfycJgKllPJzQZ4OoLpiYmJMmzZtavTenJwc6tWrV7sB1XHaZv+gbfYPZ9LmTZs2pRljmlT0mtclgjZt2rBx48YavTcpKYmEhITaDaiO0zb7B22zfziTNovIL5W9pkNDSinl5zQRKKWUn9NEoJRSfk4TgVJK+TlNBEop5ecsSwQi8qaIHBeR7ZW8LiIyXUT2iMhWETnHqliUUkpVzsoewTxgaBWvXw50dH6NBuZYGItSSqlKWPYcgTFmlYi0qWKTYcB846iDvU5EokWkmTHmiFUxKaXUmTLGYDdgN4bcAhu5RcXYjWO9MTi+MKXr7AZwLmfkFnHsVD4ncwsJEMEYg83+2/7sxrDj11OEBQXy09FTNK4fSrHdUFhQSGbqYWIbR2PFoxOefKCsBXCozHKKc90fEoGIjMbRayA2NpakpKQafWB2dnaN3+uttM3+wVvbnFdsKLQ5D6LgPCCCwXFAzSs2FNl/Wy77b25eHts//Kp02W4gNdcQEgR2O9gM/JptJzRIKLIZ9mTYiQkXbM7PyCkypGQ51tmd7zcGCmyQUWAICQB7BZ/rboXH9pK29BXsuRmc98BrlvyePZkIpIJ1Ff6cjTFzgbkAffv2NTV9sk6fRPQP2uaaMcZQZDOczC0kt9DmPFN1nLHa7IZfM/IQEVJO5mJznOZiN4Ziu8FuN2xNyeSsqLDS7W12w09HswgPDiQwQNh2OJNCm52gAKHYZii02c+w1QIUVOsdB079cV1KdsWH98IqwgsQCBCh2PlzaBYVhgAigjhfK/0XEHG+BqSczKN903q0blyPhhHBBIiUfgUGON6TkVtE+0YhfP72TBa//SrRDRvx9+depnlcQ0v+tj2ZCFKAlmWW44BfPRSLUj6hZKjhlxO57EvNYV9qNqFBARzLKiA9u4CIkCDyi2ysP3CCY5n5NG0QRrHdzqETeW6LsbCCdU0iQ0sPrgEiBAT89v3+tBz6tm5YejANEBCEzMyTNGrY6HcHXruB1KwCerWMIigggMAA4dipfPq0blj63ubR4QQFCIEBgjHQsF6IM1nhPBg7Prdx/ZAyB+nfPkOkonPY2jd06FCWL1/OiBEjeOmll2jYsKFlvT5PJoLFwHgRWQCcC2Tq9QHlb2x2Q2pWAalZBeQWFlNsd5xhHz6ZR4CArcwZ+ZKtR4gICWTLwQzHmXeZ11JO5hH85VKK7Y5xalftT8up9LV2MfUICBACnQfH/GIbxTZDt+YNOJyRR6+W0YQGBfxumxM5hXRp1oBAEYIChIAAIa/IRtvG9QgNDiAiJJA2jesRGCAEBwY4Duo1PLA6ekHn1ui9dVVWVhbBwcGEhYUxYcIEHnroIS655BLLP9eyRCAi7wMJQIyIpACPA8EAxphXgaXAFcAeIBcYYVUsSp2pwmI7p/KLKCi2czQzH1NmSORUfjEncgoJCnQMeRw8keu86JdJg7Bgvv05jbhGEaRm5ZOWXUh4cCA2u6HYbndeSKy+rOPZf1hXZHPsTATCggLJK7IxsGMMAJ1jI8kptNE0MpSYyFCCAoTQoABaNYqgaWQYwUFC/dAgIsOCa/wzUmdm+fLljB49mttuu42nnnrKrcObVt41dOtpXjfAPVZ9vlJlFdns5BQUk5FbRHpOIXtTszmamc+B9BxyC2wEBgpr96bTqlEExXY7xTbD8awCTuRUNJBRfTuP/DY4nVdk+8PrrRpFcPBELue3b1x6trw/LYfz2jUqHeIIDBByC21cEt+UJvXDCA8JIDAggEARNqxfx6WDBxIREkRggHuGLlTtOHHiBA8++CD/+c9/6NKlC1deeaXbY/C6MtRKVeZETiGn8orYkWbjwHf7+W5vOl8mH0MEl4dLTnfgbxYVRkZuEZFhQaVDHEGBQlp2IWc1CCW2QRhBgY4hkm7No7DbDV2bNSA8JJAmkaGEBAbQJDKUoEAhKODMhkbK2hceoGfzXmjFihUkJiaSnp7OpEmT+Mc//kFYWJjb49BEoLxCsc3OgfQcthzKxGa3sz8tlyKbnaRdx9mbWtE4d3Lpd2WTQNPIUI5nFdCzZTQYQ9MGYbRpHEHHppE0CA8mMECIbRBaZgzbsRwWHEhwoFZkUbWradOmtG3blmXLltGrVy+PxaGJQNUJx07l82tGHtsOZ7L9cCZZ+cXsT8vhp6NZ1A8NIrug2OV9RYcKjRpE0CgihG7NG3Bj35Z0aFqfsOBAC1ug1OkZY/jPf/7D5s2bmT59Oj169GDNmjVuuxOpMpoIlGWMMWTmFfFF8jHyi2xsP5yJze4Yr9/0y0liIkOd66oetymbBAIDhPZN6tG/bSMycovo1jyKhhHBDOrUxHEvt4hfPkeg6r79+/czZswYvvzySwYOHEheXh7h4eEeTwKgiUCdgWKbnSOZ+aRmF3AkIx+D4eufjvPdnjSOnTr9gz6HM/5473r/to1Iyyrgws5NODsuivqhwXRv0YCY+qE6NKO8ks1mY9asWTz66KMEBAQwe/ZsxowZQ0BA3fl71kSgqmSzGwqL7XyRfJRFPxymXkgQu45lsTc126ULsCUXaiNCArm1fyuMgZ4towgODMBuDK0b1aN+WBAtG4YTpAd65YPS0tKYPHkyF154Ia+++iqtWrXydEh/oIlAAY4D/q6jWSzYcJAdv55i0y8nq/X+uIbhNIsKo0W044B+SXwsAzvGEBGif2LK/xQVFfHuu+9y++23Exsby+bNm2nbtm2dGAaqiP4v9TM2u2Ffajbf7E4l5WQeH21OIa/QVlozpSKhQQEUFDsKr/xtSEe6N29AgAhdmzegRXS4u0JXyits2rSJu+66i61bt9KsWTMuu+wy2rVr5+mwqqSJwAflFdrYfSyLA+k5fLyjgOUntpKeXcgXyceqfF9woKNK4/ntG3Ntrxac0zqa9k3q19mzGKXqkry8PKZOncqLL75I06ZNWbRoEZdddpmnw3KJJgIvl11QzKrdqXy4KYVdR7MqvADLoUN/XIej7MDAjjGc07ohQ7udRYA+kapUjV177bV88cUXjBo1ihdeeIHo6GhPh+QyTQReJj27gHX7TrDoh8PsPpbFwRO5lW7bMCKYFhF2Enq0pXXjCAIDhFaNIkorMSqlzsypU6cICQkhLCyMiRMn8vDDDzNkyBBPh1VtmgjquLTsAtbsTef97w9y8ERuxWf8wJ86NCa2QRjX9GxOz7hoGtYLAUoqNHZ2Z8hK+YWlS5cyduxYbrvtNp5++mkuvPBCT4dUY5oI6pgDaTm8s+4XFmw4VOXTtJfEx9IiOpwrz25Gz7hoQoL01kul3CEtLY0HHniAd955h/j4eK655hpPh3TGNBF40N7UbOYk7WVfajYBImys4pbNdjH16NkymnsGd6BD0/pujFIpVeLLL78kMTGRkydPMnnyZCZOnEhoaKinwzpjmgjcbNXuVO5b8AMnc4sq3aZ14whaRIdz3TlxXN2zGaFBWiNHqbqgWbNmdOrUiTlz5tCjRw9Ph1NrNBG4yZ7j2Tz0wY/8eCjjD68N7BjD9efEEdcwnO4torQ4mlJ1hDGGf//73/zwww/MmjWL7t27s3r1ap+72UITgUWKbXbeX3+QJVuP8P3+E394/ak/d+fWfq30lk2l6qh9+/bx17/+la+//pqEhIQ6VSSutmkiqEXGGN5Z9wuvrNhDWnbFRdceHtqZuy9s75N/TEr5ApvNxvTp05k0aRJBQUG89tprjBo1qk4ViattmghqQU5BMa+v3se/vvr5D6+1iA7n/os7ctXZzQkP0SEfpeq6tLQ0pk6dypAhQ5gzZw5xcXGeDslymgjOwM/Hskh843uOZ/3+7L99k3o8PLQLQ7o01YqaSnmBwsJC3nnnHe68805iY2PZsmULrVu39pueuyaCGsgtLGb0/E18uyftd+u7NW/Aa8P7ENcwwkORKaWqa8OGDdx1111s376duLg4Lr30Utq0aePpsNxKE0E17TxyistfWf27de+NOpfzO8R4KCKlVE3k5uYyefJkpk2bRrNmzVi8eDGXXnqpp8PyCE0E1fD++oM8unBb6fJVZzfjXzf30uEfpbzQsGHD+Oqrrxg9ejTPP/88UVFRng7JYzQRuKDIZifhhaTf1fl5e2R/BnZs4sGolFLVlZmZSWhoKGFhYTz22GNMnDiRwYMHezosj9NT2dOYtXIPHSd9XpoEujZrwNYpl2oSUMrLLFmyhG7dujF16lQABg0apEnASRNBFRZuTuGF5btKl+ObNeCzey+gQViwB6NSSlVHamoqf/nLX7j66qtp1KgR1113nadDqnN0aKgSm345wYP/+7F0+acnhmrpB6W8zBdffEFiYiKZmZlMnTqVCRMmEBIS4umw6hxNBBU4kJbD9XPWli4vu3+gJgGlvFCLFi3o2rUrc+bMoVu3bp4Op87SRFDO9sOZXDXj29LldY8O4ayoMA9GpJRyld1u54033uCHH34oPfivWrXK02HVeXqNoJxnPt9Z+v07I8/VJKCUl9izZw9DhgxhzJgx7Nq1i7y8imfzU3+kiaCM7Ycz+W5POgCzE8/hgo76kJhSdZ3NZuOll17i7LPPZvPmzbz++uusWLGC8PBwT4fmNSxNBCIyVER2icgeEZlQwetRIvKpiPwoIjtEZISV8ZzOBxsPlX5/RY9mHoxEKeWqtLQ0nnzySS655BKSk5MZNWqU39QIqi2WXSMQkUBgFnAJkAJsEJHFxpjkMpvdAyQbY64WkSbALhF51xhTaFVclbHZDf9Z+wsAb93Zz90fr5SqhoKCApYsWcKgQYNKi8S1atVKE0ANWdkj6A/sMcbscx7YFwDDym1jgEhx/PbqAyeAymdst9D6MpPHDOqkD4spVVd9//339OnTh5deeomvvvoKwK8qhVpBjDHW7FjkBmCoMWaUc3k4cK4xZnyZbSKBxUAXIBK42RjzWQX7Gg2MBoiNje2zYMGCGsWUnZ1N/foVT/z+z7V57Mu00zE6gEnn+c7YYlVt9lXaZt+Ul5fHm2++yUcffURMTAzjxo0jISHB02G51Zn8ngcPHrzJGNO3wheNMZZ8ATcCb5RZHg7MKLfNDcA0QIAOwH6gQVX77dOnj6mplStXVrg+r7DYtH5kiWn9yBKz++ipGu+/Lqqszb5M2+ybLr74YgOYu+++22RmZvpFm8s7kzYDG00lx1Urh4ZSgJZlluOAX8ttMwJY6IxzjzMRdLEwpgpNLFNRtGNspLs/XilViYyMjNLbQCdPnsw333zD7NmzadCggYcj8y1WJoINQEcRaSsiIcAtOIaByjoIDAEQkVigM7DPwpj+ILewmIU/HAbg5r4tT7O1UspdFi9e/LsicQMHDmTQoEEejso3WZYIjDHFwHhgObAT+J8xZoeIjBWRsc7NngDOF5FtwArgEWNMWsV7tMaKncdLv3/6uh7u/GilVAWOHz/OLbfcwrBhw4iJieGGG27wdEg+z9ISE8aYpcDScuteLfP9r4BHpwR6f/1BAG7t34rAAL3rQClPWrZsGYmJiWRnZ/PEE0/wyCOPEBys1X6t5ve1htbsdTxJ3Kul/85OpFRd0bJlS3r06MHs2bOJj4/3dDh+w69LTBTb7KXf67MDSrmf3W5nzpw5jBkzBoBu3bqRlJSkScDN/DoRrD/w20NkzaJ859kBpbzB7t27SUhIYNy4cezfv5/8/HxPh+S3/DoR/HAwA4CBWlxOKbcpLi7mueee4+yzz2bbtm289dZbLF++nLAwrfTrKX59jWD74UwAhnRp6uFIlPIf6enpPPfcc1xxxRXMmjWLZs20wKOn+XWPYMshR4+gdUw9D0eilG8rKCjgtddew263Exsby48//sjChQs1CdQRfp0IApxFqmLqhXo4EqV819q1a+nduzdjx47l66+/Bhx3B6m6w28TwcH0XA5n5BEg0CYmwtPhKOVzsrOzuf/++/nTn/5ETk4Oy5Yt4+KLL/Z0WKoCfnuNYN0+x/MDfVs3IjJMH1hRqrZde+21rFixgvHjx/P0008TGal1vOoql3oEIhIiIh2sDsadko+cAuBPHfSOIaVqy8mTJ0uLxE2ZMoXVq1czY8YMTQJ13GkTgYhcCWwDvnQu9xKRRVYHZrWDJ3IBaNtELxQrVRsWLlxIfHw8U6ZMAeCCCy7gggsu8GxQyiWu9Aj+CZwLZAAYY7bgmDvAq339k6PYXJvGen1AqTNx9OhRbrjhBq6//nrOOussbrnlFk+HpKrJlURQZIzJKLfOmmnN3CSv0Fb6feeztMuqVE19/vnnxMfHs2TJEp5++mnWr19P7969PR2WqiZXLhbvFJGbgAARaQvcB6yzNixr7U3NBiC2QSihQYEejkYp79W6dWt69+7NrFmz6NLF7XNKqVriSo9gPNAHsAMLgXwcycBr7TnuSAQ2u1d3bJRyO7vdzsyZM/nrX/8KQHx8PCtWrNAk4OVcSQSXGWMeMcb0dn5NAC63OjAr7TqWBUC7GN+e7Fup2rRr1y4GDRrEvffey6FDh7RInA9xJRH8o4J1k2o7EHc6dsrxB9xaLxQrdVpFRUU888wz9OzZk+TkZObNm8fnn3+uReJ8SKXXCETkMmAo0EJEXi7zUgMcw0ReK/lXxzMEbbTGkFKndfLkSV544QWuvvpqZsyYwVlnneXpkFQtq+pi8XFgO45rAjvKrM8CJlgZlNUa1QsBoHm0ntEoVZH8/HzefPNNxo4dS9OmTdm6dStxcXGeDktZpNJEYIz5AfhBRN41xvjUYOBR59BQ12YNPByJUnXPt99+y8iRI9m9ezedOnXi4osv1iTg41y5RtBCRBaIyFYR2V3yZXlkFsrOLwYgKlxrDClVIisri/HjxzNw4EAKCwv54osvtEicn3AlEcwD3gIEx91C/wMWWBiT5TJyiwBNBEqVde211zJ79mzuu+8+tm3bxiWXXOLpkJSbuPJAWYQxZrmIvGiM2Qv8Q0RWWx2YVfKLbBTa7AQHCuHB+jCZ8m8nTpwgLCyMiIgInnjiCUSEAQMGeDos5Wau9AgKRESAvSIyVkSuBrx2bsf0nEIAGkaEIM6JaZTyRx9++CFdu3YtLRJ3/vnnaxLwU64kggeA+sDfgD8BfwXusjIoK2U6h4Uiw/x2Kgbl544cOcJ1113HjTfeSMuWLUlMTPR0SMrDTns0NMZ87/w2CxgOICJeewvBoZOO8tOZecUejkQp9/vss8+47bbbyM/P57nnnuPBBx8kKEhPivxdlX8BItIPaAF8a4xJE5FuwCPARYBXJoO07AIA6ofq9QHlf9q1a0e/fv2YOXMmnTp18nQ4qo6odGhIRJ4B3gUSgWUiMglYCfwIeO1fUFqW4xpBj7hoD0eilPVsNhuvvPIKI0eOBKBr16588cUXmgTU71TVIxgG9DTG5IlII+BX5/Iu94RmjaBAxwXiCL1jSPm45ORkRo0axdq1a7niiivIz8/X+kCqQlVdLM43xuQBGGNOAD95exIAyMxzXCxuHh3u4UiUskZhYSFPPvkkvXv3Zvfu3bzzzjssWbJEk4CqVFU9gnYistD5vQBtyixjjLnudDsXkaHAK0Ag8IYx5tkKtkkA/gUEA2nGmAtdD7/6fs1wTKxtvHuSNaUqlZGRwbRp0/jzn//M9OnTadrUa+/2Vm5SVSK4vtzyzOrsWEQCgVnAJUAKsEFEFhtjkstsEw3MBoYaYw6KiOV/sQ0jHAXndE4a5UsKCgqYOXMm48aNo2nTpmzbto3mzZt7OizlJaoqOrfiDPfdH9hjjNkHICILcFx3SC6zzV+AhcaYg87PPH6Gn3laJUNDLbTyqPIRq1atYtSoUaSkpNC1a1eGDBmiSUBVi5U3ELcADpVZTgHOLbdNJyBYRJKASOAVY8z88jsSkdHAaIDY2FiSkpJqFFB2djbb9juGhnbv2kVSzr4a7cebZGdn1/jn5a38pc05OTnMnTuXxYsXExsby4svvkhgYKBftB385/dcllVttjIRVFS/ofyATBCO+ZCHAOHAWhFZZ4z5XXVTY8xcYC5A3759TUJCQo0CSkpKol3zeuw/dZzu3bqS0NsrH4WolqSkJGr68/JW/tLmiy66iKSkJB544AEuueQSLr/cq2eQrTZ/+T2XZVWbXU4EIhJqjCmoxr5TgJZlluNw3IJafps0Y0wOkCMiq4CegGVlrn84lAFA00gdGlLeJy0tjYiICCIiInjqqacQEc477zy/OzNWteu0tYZEpL+IbAN+di73FJEZLux7A9BRRNqKSAhwC7C43DafAANFJEhEInAMHe2sVguqqWVDvW1UeR9jDAsWLKBr1648/vjjAAwYMIDzzjvPw5EpX+BK0bnpwFVAOoAx5kdg8OneZIwpBsYDy3Ec3P9njNnhrGA61rnNTmAZsBVYj+MW0+01aYirip23C+lcBMpbHD58mGuvvZZbb72Vtm3bcvvtt3s6JOVjXBkaCjDG/FKuZLPNlZ0bY5YCS8ute7Xc8gvAC67srzYU2ewABAe6kgOV8qwlS5aQmJhIUVERL774Ivfffz+BgfpUvKpdriSCQyLSHzDOZwPuxcIxfKsV2Rw9guBAnYtA1X0dOnTg/PPPZ8aMGXTo0MHT4Sgf5cpp8d3Ag0Ar4BhwnnOdV8otdJSf1h6BqotsNhvTpk3jzjvvBKBLly58/vnnmgSUpVzpERQbY26xPBI3yS10jGo1CNNrBKpu2bFjByNHjuT777/nyiuv1CJxym1cOS3eICJLReQOEYm0PCKLZeU7egShwdojUHVDYWEh//znP+nduzd79+7lvffe49NPP9UkoNzmtEdDY0x74EkcD35tE5GPRcQrewjFZQoMhQZpIlB1Q0ZGBtOnT+fGG28kOTmZW2+9VefTVm7l0tHQGLPGGPM34BzgFI4Ja7xOduFviUD/oylPys3N5ZVXXsFms5UWiXv33Xdp0qSJp0NTfsiVB8rqi0iiiHyK417/VOB8yyOzQKHd0xEoBStXrqRHjx7cf//9pU8EN2vWzLNBKb/mSo9gO447hZ43xnQwxjxUZkJ7r1LsTATtm9TzbCDKL2VmZjJmzBguuugiRISVK1cyZMgQT4ellEt3DbUzxvjEuXTJNQK9dVR5wrXXXsuqVav4v//7P6ZMmUJERISnQ1IKqCIRiMhLxpiHgI9E5A/TuLgyQ1ldk+e4YYgQvVCs3CQ1NZV69eoRERHBM888Q2BgIP369fN0WEr9TlU9gv86/63WzGR1Wb7zqeJfM/I9HInydcYY3n//ff72t78xYsQIXnjhBS0Qp+qsSk+NjTHrnd92NcasKPsFdHVPeLXrZL4jEXRt5vWPQ6g6LCUlhWuuuYbExEQ6dOhQ+pSwUnWVK2Mkd1WwbmRtB+IOJeWFTjmnq1Sqti1evJj4+Hi+/vprpk2bxnfffUe3bt08HZZSVarqGsHNOOYQaCsiC8u8FAlkWB2YFU44ewTdW0R5OBLlqzp16sQFF1zAzJkzadeunafDUcolVV0jWI9jDoI4YFaZ9VnAD1YGZZWSK96Z2iNQtaS4uJh//etfbN26lfnz59OlSxeWLl16+jcqVYdUmgiMMfuB/cBX7gvHWkHOoaFG9UI8G4jyCVu3bmXkyJFs3LiRYcOGaZE45bUqvUYgIt84/z0pIifKfJ0UkRPuC7H2OG8a0sqj6owUFBTw+OOP06dPHw4ePMj//vc/Fi1apElAea2qhoZKpqOMcUcg7lBScy4gQOsMqZo7deoUs2fP5tZbb2XatGk0btzY0yEpdUaqun205GnilkCgMcYGDADGAF5Zo6GkQYFacE5VU05ODtOmTcNms9GkSRO2b9/O/PnzNQkon+DK7aMf45imsj0wH8czBO9ZGpVFjLNHEKTTVKpqWLFiBT169ODBBx/km2++ASA2NtbDUSlVe1xJBHZjTBFwHfAvY8y9QAtrw7JGyTWCAO0RKBdkZGQwatQoLr74YoKCgvjmm2+46KKLPB2WUrXOpakqReRGYDhwrXOdV15t/TXbMTikNeeUK/785z+zevVqHnnkER5//HHCw8M9HZJSlnAlEdwFjMNRhnqfiLQF3rc2LGs0DnP0BNKzCz0ciaqrjh07Rv369alXrx7PPvssQUFB9OnTx9NhKWUpV6aq3A78DdgoIl2AQ8aYpyyPzAJFzqvFbWO88lq3spAxhrfffpv4+Hgef/xxAM4991xNAsovuDJD2UBgD/Bv4E1gt4j8yerArHAkx5EJdOJ6VdbBgwe58soruf322+ncuTMjR3plKS2lasyVoaFpwBXGmGQAEekKvA30tTIwK5Qc/wuKfGKeHVULPvnkE2677TaMMUyfPp1x48YRGBjo6bCUcitXEkFISRIAMMbsFBGvrNEQ5HyQrGmDUA9HojzNGIOI0KVLFxISEpgxYwZt2rTxdFhKeYQriWCziLyGoxcAkIiXFp0rmaoyLEjP+PxVcXExL730Etu2beOdd96hc+fOfPrpp54OSymPcmWwfCywF3gYeATYh+PpYq9jK32gTK8R+KMff/yRc889lwkTJpCbm0t+vs5UpxScpkcgIj2A9sAiY8zz7gnJOoU2x786Z7F/yc/P58knn+S5556jcePGfPjhh1x//fWeDkupOqOq6qMTcZSXSAS+FJGKZirzKoXOoaGIEB0a8idZWVm89tprJCYmkpycrElAqXKqOjVOBM42xtwI9APuru7ORWSoiOwSkT0iMqGK7fqJiE1EbqjuZ1SHzXmzULAODfm87OxsXnzxxdIiccnJycybN49GjRp5OjSl6pyqjogFxpgcAGNM6mm2/QMRCXfb8KcAAByVSURBVMQxs9nlQDxwq4jEV7Ldc8Dy6uy/JopLE4HWGvJlGzZsoHv37jz88MOsWrUKgCZNmng4KqXqrqquEbQrM1exAO3Lzl1sjLnuNPvuD+wxxuwDEJEFwDAgudx29wIf4eh1WCrdOWdxiPYIfNKJEyd46KGHmDdvHp07d2b16tX86U9e+eyjUm5VVSIoP5A6s5r7bgEcKrOcApxbdgMRaQH8GbiIKhKBiIwGRoOj/G9SUlI1Q3GICDLkFgvr162lfoh/9Aqys7Nr/PPyNvfddx/bt2/nxhtvZNSoURQVFflN2/3p91xC21x7qpqzeMUZ7ruiI60pt/wv4BFjjE2qKA1tjJkLzAXo27evSUhIqFFA9i8/AyDhwoHUD3XlEQrvl5SURE1/Xt7g6NGjREZGUq9ePV5//XVCQkLIyMjw6TZXxNd/zxXRNtceK8dIUnDMblYiDvi13DZ9gQUicgC4AZgtItdikZKpKoN0qkqvZ4xh3rx5xMfHM3nyZAD69+9Pr169PByZUt7HykSwAegoIm2dJSluARaX3cAY09YY08YY0wb4EBhnjPnYqoBKHigL1ETg1Q4cOMDQoUMZMWIE3bp1Y/To0Z4OSSmv5vL4iIiEGmMKXN3eGFMsIuNx3A0UCLxpjNkhImOdr79a7WjPUEmPQOcs9l6LFi1i+PDhiAgzZ87k7rvvJiBAL/4rdSZOmwhEpD+OEtRRQCsR6QmMck5ZWSVjzFJgabl1FSYAY8ydrgRcU3a7wQAiEKA9Aq9TUiSuW7duXHzxxbzyyiu0bt3a02Ep5RNcOZWaDlwFpAMYY34EBlsZlBUKnU+TmfKXq1WdVlRUxNNPP01iYiIAnTp14uOPP9YkoFQtciURBBhjfim3zmZFMFbSOQi8z+bNm+nfvz+TJk3CZrNRUODyyKRSqhpcSQSHnMNDRkQCReR+YLfFcdW6kh5B43peOZWCX8nLy+PRRx+lf//+HD16lEWLFvHf//6X0FCdR0IpK7iSCO4GHgRaAceA86hB3SFPK0kEWnm07svJyeHf//43d9xxB8nJyVx7rWV3FCulcOFisTHmOI5bP71aXmExAOk5hR6ORFUkKyuLOXPm8NBDDxETE0NycjIxMTGeDkspv+DKXUOv88cngjHGeOXN26FaZ6jOWbZsGWPGjOHQoUP079+fhIQETQJKuZErR8WvgBXOr++ApoDXXbUreYbgrKgwzwaiSqWnp3PHHXdw+eWXU69ePb777ju/KxmgVF3gytDQf8sui8jbwJeWRWQRu/O+0QB9mKzOuO6661izZg2PPfYYkyZN0ovBSnlITSqvtQW87iZuu/PuUc0DnnXkyBEiIyOpX78+L774IiEhIfTs2dPTYSnl1047NCQiJ0XkhPMrA0dvYKL1odUu7RF4ljGGN998k65du5YWievXr58mAaXqgNNNXi9AT+Cwc5XdGO98NrckEWjBOffbt28fY8aM4auvvmLQoEGMHTvW0yEppcqoskfgPOgvMsbYnF9emQTgt4vFmgfca+HChfTo0YPvv/+eOXPmsHLlSjp16uTpsJRSZbhy19B6ETnH8kgsVtIjqGoCHFV7Ss4ZevTowdChQ9mxYwdjx47VSqFK1UGVDg2JSJAxphi4APiriOwFcnDMPGaMMV6VHEzpNQIPB+LjCgsLef7559mxYwfvvfceHTt25KOPPvJ0WEqpKlR1jWA9cA7gE8/3OytM6MViC23cuJGRI0eydetWbrnlFgoLC/WWUKW8QFWJQACMMXvdFIulsvKLAL191Ap5eXk8/vjjvPTSS5x11ll88sknXHPNNZ4OSynloqoSQRMRebCyF40xL1sQj2VKis39fDzbw5H4npycHObNm8fIkSN5/vnniY6O9nRISqlqqCoRBAL1cfYMvJ3NedtQjxZRHo7EN5w6dYrZs2fzf//3f8TExLBz504aN27s6bCUUjVQVSI4Yoz5p9sisVjJXUNBerX4jH322WeMHTuWX3/9lfPOO4+EhARNAkp5saru5fOpI2axTR8oO1OpqakkJiZy1VVXERUVxZo1a7RInFI+oKoewRC3ReEGWmLizF1//fWsW7eOKVOm8OijjxISorO9KeULKk0ExpgT7gzEaiW3j2qPoHoOHz5MVFQU9evXZ9q0aYSGhtK9e3dPh6WUqkV+85hnsbP8qCYC1xhjeP3114mPjy8tEtenTx9NAkr5IL9JBMdO5Xs6BK+xd+9ehgwZwujRo+nTpw/33HOPp0NSSlnIbxJBdIRjPPvQiVwPR1K3ffjhh/To0YNNmzYxd+5cVqxYQfv27T0dllLKQjWZmMY7OauPdoyN9GwcdZQxBhGhZ8+eXHnllUybNo24uDhPh6WUcgO/6REYtOhcRQoLC5k6dSq33HILxhg6duzIBx98oElAKT/iN4mgZD4C8a3HI87I+vXr6dOnD1OmTCEoKIjCwkJPh6SU8gC/SQQlU+roYwSQm5vL3//+dwYMGMDJkyf59NNPeffdd7VSqFJ+yn8SAToxTYm8vDzeeecdRo8eTXJyMldddZWnQ1JKeZCliUBEhorILhHZIyITKng9UUS2Or/WiIhlM5nb/bxHkJmZyVNPPUVxcTGNGzdm586dzJkzhwYNGng6NKWUh1mWCEQkEJgFXA7EA7eKSHy5zfYDFxpjzgaeAOZaFU/J2JA/5oFPP/209MGwb7/9FoCGDRt6OCqlVF1hZY+gP7DHGLPPGFMILACGld3AGLPGGHPSubgOsOxWFWeHwK9qDaWmpvLEE09wzTXX0LhxY77//nstEqeU+gMrnyNoARwqs5wCnFvF9iOBzyt6QURGA6MBYmNjSUpKqnYwu35xzFD266+HSUpKq/b7vdF9991HcnIyI0aM4NZbbyU7O7tGPztv4y/tLEvb7B+sarOViaCiU29TwTpEZDCORHBBRa8bY+biHDbq27evqclZ7f7v9sPOZOJatCAhwXfr5aSkpBAdHU39+vV56623+PHHHxkxYoSnw3KrpKQkv+v5aJv9g1VttnJoKAVoWWY5Dvi1/EYicjbwBjDMGJNuVTC/3T7qm0NDdrud1157jfj4eB577DEAzjnnHNq2bevhyJRSdZ2ViWAD0FFE2opICHALsLjsBiLSClgIDDfG7LYwltL5CHwxD/z8889cdNFFjB07lv79+3Pvvfd6OiSllBexbGjIGFMsIuOB5TjmP37TGLNDRMY6X38VmAw0BmY7z9SLjTF9rYoJfO/J4g8++IDbb7+d0NBQ/v3vfzNixAif7fUopaxhadE5Y8xSYGm5da+W+X4UMMrKGH77LMe/vlJrqKRIXO/evRk2bBgvv/wyzZs393RYSikv5DdPFvvK0FBBQQGTJ0/mpptuwhhDhw4dWLBggSYBpVSN+U0iKLldyZuHTdatW8c555zDE088QXh4uBaJU0rVCr9JBDa79/YIcnJyeOCBBzj//PPJyspi6dKlzJ8/X4vEKaVqhd8kgpSTjpnJioorfJShTsvPz2fBggWMGzeOHTt2cPnll3s6JKWUD/GbGcqaRYUDcDLXO4ZTMjIymDFjBo8++mhpkbjo6GhPh6WU8kF+0yMoEdcw3NMhnNbHH39MfHw8U6dOZc2aNQCaBJRSlvGbHoHxghGhY8eOce+99/LBBx/Qs2dPPv30U/r06ePpsJTyqKKiIlJSUsjPz//d+qioKHbu3OmhqDzDlTaHhYURFxdHcHCwy/v1n0RA3S9DfcMNN7B+/XqefPJJHn744Wr9IpXyVSkpKURGRtKmTZvf3fWXlZVFZGSkByNzv9O12RhDeno6KSkp1Sov4z+J4Lf7Rz0aR3kHDx6kYcOGREZGMn36dEJDQ4mPLz9tg1L+Kz8//w9JQFVMRGjcuDGpqanVep/fXCMozQMejeI3drudWbNm0a1bNyZPngxA7969NQkoVQFNAq6ryc/KbxIBdejJ4l27dnHhhRcyfvx4BgwYwH333efpkJRSfsxvEsFvPQLPZoL//e9/9OzZk+3bt/PWW2+xfPly2rRp49GYlFJVe+qpp+jWrRtnn302vXr14vvvv6e4uJiJEyfSsWNHevXqRa9evXjqqadK3xMYGEivXr3o1q0bPXv25OWXX8Zut3uwFZXzu2sEnuoRlBSJ69OnD9dddx0vv/wyZ511lmeCUUq5bO3atSxZsoTNmzcTGhpKWloahYWF/OMf/+Do0aNs27aNsLAwsrKyeOmll0rfFx4ezpYtWwA4fvw4f/nLX8jMzGTq1Kmeakql/CYRlHB3HsjPz+eJJ57gp59+4sMPP6R9+/a89957bo5CKd/QZsJnluz3wLNXVvrakSNHiImJKS3pEhMTQ25uLq+//joHDhwgLCwMgMjISKZMmVLhPpo2bcrcuXPp168fU6ZMqXPXPPxoaMj9DxKsWbOG3r178/TTTxMZGalF4pTyQpdeeimHDh2iU6dOjBs3jm+++YY9e/bQqlWrat2+2q5dO+x2O8ePH7cw2prxmx6BO4eGsrOzmThxIjNnzqRly5YsW7aMyy67zPoPVsrHlT1zd9dzBPXr12fTpk2sXr2alStXcvPNNzNx4sTfbfPWW2/xyiuvkJ6ezpo1a2jZsmWF+zJ19MlWP+oROLijS1ZYWMiHH37IPffcw/bt2zUJKOXlAgMDSUhIYOrUqcycOZNPP/2UgwcPkpWVBcCIESPYsmULUVFR2Gy2Cvexb98+AgMDadq0qTtDd4n/JAKLE/GJEyeYMmUKxcXFNGrUiJ07dzJjxgy/e/JRKV+za9cufv7559LlLVu20LlzZ0aOHMn48eNLS1/YbLZKh39TU1MZO3Ys48ePr3PXB8Cfhoaw7jmCjz76iHvuuYe0tDQuuugiBg0aRFRUVO1/kFLK7bKzs7n33nvJyMggKCiIDh06MHfuXKKionjsscfo3r07kZGRhIeHc8cdd5TOFpiXl0evXr0oKioiKCiI4cOH8+CDD3q4NRXzm0RQMjZUm88RHDlyhPHjx7Nw4UJ69+7NsmXL6NWrV63tXynleX369CmtAlzes88+y7PPPlvha5UNEdVF/pMInGqzR3DTTTexYcMGnn32WR566CGCgvzux6mU8gF+c+SqrUsEv/zyC40aNSIyMpIZM2YQHh5O586da2nvSinlfn50sfjMylDb7XZmzJhBt27deOyxxwDo1auXJgGllNfznx7BGTxH8NNPPzFq1Ci+++47hg4dygMPPFC7wSmllAf5T4/A+W91LxYvWLCAnj17snPnTubPn8/SpUtp3bp17QeolFIe4j+JoJo9gpIqgf369ePGG28kOTmZ4cOH18l7gJVS6kz4TyJw8XJxXl4eEyZM4Prrr8cYQ/v27XnnnXeIjY21OEKlVF1VUlK6e/fuXH311WRkZNTKfg8cOED37t1rZV9nwn8SQWmPoPIz+tWrV9OrVy+ee+45GjduTFFRkZuiU0rVZSUlpbdv306jRo2YNWuWp0OqVX5zsbhERWkgKyuLCRMmMHv2bNq2bcuXX37JxRdf7PbYlFKnl5CQADge2AoMDAQcz/SMGzeO3Nxcrrjiij+858477+TOO+8kLS2NG2644XevJSUlVevzBwwYwNatWwHHU8fDhg3j5MmTFBUV8eSTTzJs2DAOHDjA5ZdfzgUXXMCaNWto0aIFn3zyCeHh4WzatIm77rqLiIgILrjggtL95ufnc/fdd7Nx40aCgoJ4+eWXGTx4MPPmzePjjz/GZrOxbds2/v73v1NYWMjbb79NaGgoS5cupVGjRtVqQ3l+0yOoSlFRER9//DH3338/27Zt0ySglKqQzWZjxYoVXHPNNQCEhYWxaNEiNm/ezMqVK3nooYdKb1X/+eefueeee9ixYwfR0dF89NFHgKNA3fTp01m7du3v9l3Sy9i2bRvvv/8+d9xxR2kdo+3bt/Pee++xcuVKJk2aREREBD/88AMDBgxg/vz5Z9wuv+kRmHJzFqenp/PKK68wefJkGjVqxE8//aQF4pTyAiVn8BWVoY6IiKjyDD8mJqbaPQD4rW7QgQMH6NOnD5dccgngOK5MnDiRVatWERAQwOHDhzl27BgAbdu2LS0506dPHw4cOEBmZiYZGRlceOGFAAwfPpzPP/8cgG+//ZZ7770XgC5dutC6dWt2794NwODBg0vbGhUVxdVXXw1Ajx49SnsnZ8LSHoGIDBWRXSKyR0QmVPC6iMh05+tbReQcq2IpvVRsDB988AHx8fE888wzpVlZk4BSqjIl1wh++eUXCgsLS8/e3333XVJTU9m0aRNbtmwhNja29Cy+ZEYzcFxsLi4uLp2ytiJVzVVQdl8BAQGlywEBARQXF59x+yxLBCISCMwCLgfigVtFJL7cZpcDHZ1fo4E5VsVjDBRnpTNz0t3cdNNNtGzZko0bNzJw4ECrPlIp5WOioqKYPn06L774IkVFRWRmZtK0aVOCg4NZuXIlv/zyS5Xvj46OJioqim+//RZwJJISgwYNKl3evXs3Bw8edFvlAit7BP2BPcaYfcaYQmABMKzcNsOA+cZhHRAtIs2sCMZgSPvkObat+4bnn3+edevW0bNnTys+Sinlw3r37k3Pnj1ZsGABiYmJbNy4kb59+/Luu+/SpUuX077/rbfe4p577mHAgAGEh4eXrh83bhw2m40ePXpw8803M2/evN/1BKwkVk2dJiI3AEONMaOcy8OBc40x48tsswR41hjzrXN5BfCIMWZjuX2NxtFjIDY2ts+CBQuqHc9/dxXy5aZdXNGxPtf1a1vTZnmd7Oxs6tev7+kw3Erb7FuioqLo0KHDH9aXvWvIX7ja5j179pCZmfm7dYMHD95kjOlb0fZWXiyuaCCsfNZxZRuMMXOBuQB9+/Y1JbePVUdCAiQlhVCT93qzpKQkbbMf8OU279y5s8JreO6as7gucbXNYWFh9O7d2+X9Wjk0lAKUncE5Dvi1BtsopZSykJWJYAPQUUTaikgIcAuwuNw2i4HbnXcPnQdkGmOOWBiTUsoLWTWE7Ytq8rOybGjIGFMsIuOB5UAg8KYxZoeIjHW+/iqwFLgC2APkAiOsikcp5Z3CwsJIT0+ncePGWvTxNIwxpKenExYWVq33WfpAmTFmKY6Dfdl1r5b53gD3WBmDUsq7xcXFkZKSQmpq6u/W5+fnV/uA5+1caXNYWBhxcXHV2q/fPFmslPJOwcHBtG37xzv9kpKSqnVB1BdY1WatNaSUUn5OE4FSSvk5TQRKKeXnLHuy2CoikgpUXdCjcjFAWi2G4w20zf5B2+wfzqTNrY0xTSp6wesSwZkQkY2VPWLtq7TN/kHb7B+sarMODSmllJ/TRKCUUn7O3xLBXE8H4AHaZv+gbfYPlrTZr64RKKWU+iN/6xEopZQqRxOBUkr5OZ9MBCIyVER2icgeEZlQwesiItOdr28VkXM8EWdtcqHNic62bhWRNSLi9fN0nq7NZbbrJyI256x5Xs2VNotIgohsEZEdIvKNu2OsbS78bUeJyKci8qOzzV5dxVhE3hSR4yKyvZLXa//4ZYzxqS8cJa/3Au2AEOBHIL7cNlcAn+OYIe084HtPx+2GNp8PNHR+f7k/tLnMdl/jqIJ7g6fjdsPvORpIBlo5l5t6Om43tHki8Jzz+ybACSDE07GfQZsHAecA2yt5vdaPX77YI+gP7DHG7DPGFAILgGHlthkGzDcO64BoEWnm7kBr0WnbbIxZY4w56Vxch2M2OG/myu8Z4F7gI+C4O4OziCtt/guw0BhzEMAY4+3tdqXNBogUx2QF9XEkgmL3hll7jDGrcLShMrV+/PLFRNACOFRmOcW5rrrbeJPqtmckjjMKb3baNotIC+DPwKv4Bld+z52AhiKSJCKbROR2t0VnDVfaPBPoimOa223AfcYYu3vC84haP3754nwEFU1hVP4eWVe28SYut0dEBuNIBBdYGpH1XGnzv4BHjDE2H5nZypU2BwF9gCFAOLBWRNYZY3ZbHZxFXGnzZcAW4CKgPfCliKw2xpyyOjgPqfXjly8mghSgZZnlOBxnCtXdxpu41B4RORt4A7jcGJPuptis4kqb+wILnEkgBrhCRIqNMR+7J8Ra5+rfdpoxJgfIEZFVQE/AWxOBK20eATxrHAPoe0RkP9AFWO+eEN2u1o9fvjg0tAHoKCJtRSQEuAVYXG6bxcDtzqvv5wGZxpgj7g60Fp22zSLSClgIDPfis8OyTttmY0xbY0wbY0wb4ENgnBcnAXDtb/sTYKCIBIlIBHAusNPNcdYmV9p8EEcPCBGJBToD+9wapXvV+vHL53oExphiERkPLMdxx8GbxpgdIjLW+fqrOO4guQLYA+TiOKPwWi62eTLQGJjtPEMuNl5cudHFNvsUV9psjNkpIsuArYAdeMMYU+FtiN7Axd/zE8A8EdmGY9jkEWOM15anFpH3gQQgRkRSgMeBYLDu+KUlJpRSys/54tCQUkqpatBEoJRSfk4TgVJK+TlNBEop5ec0ESillJ/TRKDqHGel0C1lvtpUsW2byqo0VvMzk5wVLn8Uke9EpHMN9jG2pKSDiNwpIs3LvPaGiMTXcpwbRKSXC++53/lMgVIV0kSg6qI8Y0yvMl8H3PS5icaYnsB/gBeq+2bnffzznYt3As3LvDbKGJNcK1H+FudsXIvzfkATgaqUJgLlFZxn/qtFZLPz6/wKtukmIuudvYitItLRuf62MutfE5HA03zcKqCD871DROQHEdnmrBMf6lz/rIgkOz/nRee6KSLyd3HMe9AXeNf5meHOM/m+InK3iDxfJuY7RWRGDeNcS5liYyIyR0Q2iqMm/1Tnur/hSEgrRWSlc92lIrLW+XP8QETqn+ZzlI/TRKDqovAyw0KLnOuOA5cYY84BbgamV/C+scArxpheOA7EKSLS1bn9n5zrbUDiaT7/amCbiIQB84CbjTE9cDyJf7eINMJR1bSbMeZs4MmybzbGfAhsxHHm3ssYk1fm5Q+B68os3wz8t4ZxDgXKlsyY5Hxa/GzgQhE52xgzHUcdmsHGmMEiEgP8A7jY+bPcCDx4ms9RPs7nSkwon5DnPBiWFQzMdI6J23CUWy5vLTBJROJw1OT/WUSG4KjGucFZWiOcyucmeFdE8oADOOYx6AzsL1Ob6T/APTjKHucDb4jIZ8ASVxtmjEkVkX3OGjE/Oz/jO+d+qxNnPRwlF8rOTnWTiIzG8f+6GRCPo9REWec513/n/JwQHD835cc0EShv8QBwDEclzQAcB+LfMca8JyLfA1cCy0VkFI7aM/8xxjzqwmckGmM2liyISOOKNnLWv+mPo9DZLcB4HCWQXfVf4CbgJ2CRMcaI46jscpw4Zup6FpgFXCcibYG/A/2MMSdFZB4QVsF7BfjSGHNrNeJVPk6HhpS3iAKOOCccGY7jbPh3RKQdsM85HLIYxxDJCuAGEWnq3KaRiLR28TN/AtqISAfn8nDgG+eYepQxZimOC7EV3bmTBURWst+FwLXArTiSAtWN0xhThGOI5zznsFIDIAfIFEcFzssriWUd8KeSNolIhIhU1LtSfkQTgfIWs4E7RGQdjmGhnAq2uRnYLiJbcNSjn++8U+cfwBcishX4EsewyWkZY/JxVHb8wFnZ0o5jtrNIYIlzf9/g6K2UNw94teRicbn9nsQxr3BrY8x657pqx+m89vAS8HdjzI/AD8AO4E0cw00l5gKfi8hKY0wqjjua3nd+zjocPyvlx7T6qFJK+TntESillJ/TRKCUUn5OE4FSSvk5TQRKKeXnNBEopZSf00SglFJ+ThOBUkr5uf8HSQAhYpXCrHoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_roc_curve(fpr, tpr, label='SGD')\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The Higher the recall, the more false positives the model produces.\n",
    "- The dotted line represent the ROC curve of a completely random classifier.\n",
    "- A good classifier stays as far a way as possible from the dotted line.\n",
    "- One way to compare classifiers is to measure the area under the curve (AUC).\n",
    "    - A perfect classifier will have an ROC AUC of 1.\n",
    "    - A purely random classifier will have an ROC AUC of 0.5.\n",
    "- Example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9509867800410808"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Let's now train a `RandomForestClassifier` to compare its ROC Curve and ROC AUC score to the `SGDClassifier`'s:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_clf = RandomForestClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_probas_forest = cross_val_predict(estimator=forest_clf, X=X_train, \n",
    "                                    y=y_train_5, cv=5, \n",
    "                                    n_jobs=-1, method='predict_proba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores_forest = y_probas_forest[:, 1]  # P(X=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_forest, tps_forest, threshs_forest = roc_curve(y_true=y_train_5, y_score=y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3gU1dfA8e9JKKHEUCJFioBICV26IgQBxQqiFEUUBJGmAvr+RFQERbGASFEUVBBFUVEUEEVFIiig9BYUASkRlQ4JSUi77x93NwkxCRvIZrLZ83mefTazOztzZpPMmblz51wxxqCUUsp/BTgdgFJKKWdpIlBKKT+niUAppfycJgKllPJzmgiUUsrPFXI6gJwKDQ011apVu6DPnjlzhhIlSuRuQPmcbrN/0G32DxezzRs2bDhqjLk0s/d8LhFUq1aN9evXX9BnIyIiCA8Pz92A8jndZv+g2+wfLmabRWR/Vu9p05BSSvk5TQRKKeXnNBEopZSf00SglFJ+ThOBUkr5Oa8lAhF5V0QOi8j2LN4XEZkqIrtFZKuIXOWtWJRSSmXNm2cEc4DO2bx/I3Cl6zEQmOHFWJRSSmXBa/cRGGNWiki1bGbpAsw1tg72WhEpJSIVjTF/eysmpQoaYwzGQHKKITE5hRTXtDHYnyHda4aU9M+cO2/6zxoyzJthWZnNm349BkhJybj+c2Midb1p6yF1vWkxcc5607Z558FE/v71QKbrtduXNm/W30Xaeybdd3Du95Nh3v98F664UzLfPjud9lpyiuHMGShc1BAYCImJhpgzUKyYISAQYmPhdLShdBkICDCcOm04cjiJUkGHuKxcKbxx64STN5RVAg6mm45yvfafRCAiA7FnDZQvX56IiIgLWmFMTMwFfza/SU4xnEmC2ERDbKIhOfUPMu3ZALFxcWxf8H2m77n+LtNeM+e+nvqPlNl758yT1Xsm0/fccaT/fPrY3P+EGEghQ9zu19K9l37ZKRiSEpN4c8s3HsbIOTu0bOdN95mMsbnH9Tgntgzfb9bfvfEs1kzmOce3X2d8peDbsc3pCLznlH1K+HcPR5dOISX2JK1GvOWVfZiTiUAyee0/f9sAxpiZwEyAZs2amQu9sy6/3YmYmJzC6bhETmV4ZPaafSRxOi6Rk7EJnElI9nAtApz15mbkQwJ4+v34PhHAQGCAIAIiQoCA4HoW1+tAQIAQIIKQ9npm8wakf3atQ0TAQKFA+6+blCgYA8WL2c+ePi0EBsIlwfZzx49DoUJCmdJ2+bt3Q0iIULECJCcJmzZB9cuFihUhNlZYuxYaNYQK5YXoaFi1UmjRAi6rKPx9CCIihBtvhEtDhYMH4YflcHtXITQU9u4Rvv8e7uollC0DmzcLq1bBkMFCUFFYuQo2bRT63w+lSwtLFsPBg8LDw+zGLfgEkpKEPvfY72jeBwDCvffa7+eD94UKFaDzDfZ7ee89oWoV6NTJfj9z3xMaNoTmzey2zp4NTa8SrroKUlKE9+dC82ZC48YQHy8sWQSdOkFoqHAmRtixA8LqQpnSQkwM/LlXqHlFPB/NmshH779OSOmyPPbSq1xWubRX9mFOJoIooEq66crAIYdiuWAZd+Yn0+/IYzPfobvf93xn/l8icElQYUKK2UfhQPsPHuD6j3f/c586dYIypctk8c+dtiNwvyeu9wIy7BDSdhTnzpv5DsdOBwRkvcPB9Vl3LAEBducSkCEmMovlP+s9N+7ff/+NsLp1s40xbb32vey/n8zX85/vJ+C/MQVkmD99TJmtJ3XeAP6znozfhXt7IOuDnH37oGRJCA2FM2dg2TJo2hQuvxxWroQPPoCnnoKqVeHLL+Gll2DRYihbFl55BV5+CQ4fhrg46N8fvv0Wjh2zy771VvjrL1i+0U43aQLJyfD9VjvdsCFccQXMX2inK1aEq26D10ZCUhLc9AH06wDdu0NUFDz+MzzUBlq1ggMH4NmNMLS9Xe6uXfD+SXjwNqhcGX7/HYIj/+S526pToQLs3AlNDfS/zcb+ex3YUhe63gpFisChq+DECahb1/7OH3F9VUFB9nlkp3O/t6Htz50e2Pbc6X7XnDt9T6tzp3s0P3f6zmYZplucO92t5bnTNIPOnTuzbNky+vXrx6RJkyhdurTXWjScTASLgGEiMh9oCZzKr9cH4hKSWbTlL1b8doQTsQm5tjMPELikWNrOPKRY4f9MZ/a4pFhhgosWSt15ZsfuIDL+lRVsEdG7CW9SyekwLkhKCiQk2J1XQACcPGl3trVqQeHC8P338PXXMGmSnX/GDDh0CDp0sNPDhtnPvvoqnD0L1avD44/Diy/aHfodd8Ds2dC3r01Ks2bBkCE2ERQqBCVK2J05QFiY3UkbA8WLQ7du0KhRWqxPPQXx8WnTs2e7zk5cli61sbj9neG/e9WqtJ8rV4Z589Kmq1aFt99Om65VC557Lm26dm3o02c/FSpUB+wOvm7dc9+vXTtt+rLL7MPNnQDym+joaAoXLkxQUBCjRo3i0UcfpVOnTuf/4EXyWiIQkY+AcCBURKKAZ4DCAMaYN4GlwE3AbiAW6OetWC7UgWOxvL92H5+sj+JUXGKm82S2Mz/fTjykWGFCihemZBHPduYqf0tMhOhoKFPGTv/xB8TE2CNZgOHD4aab4Prr4fhx6NMHhg61ry1fDgMHwsyZdmf+9ddwyy3w66/QvDksWAAPPAAHD9qd5ZIl9ih+6FCoUQO2bLFHy+5EkF7RojYhXHutna5UCTZvtjtZgDZt0q7PANx8s31kNd29+7nLb5nh+KJx43OnK1f27PtT1rJlyxg4cCD33HMPzz//fJ42Y3uz19Bd53nfAEO9tf4LlZJiWPnHEeau2c+K3w+n/qM0rlKKXs2rULVMcd2ZFwDR0fbotWRJe8S9eDHcdps9atywAd5/H0aPhnLl7M54+HC7M/7zT7j3Xnt0vWuXXdYNN9imjB077M735pvt5378EQID7ZHyv//aRAD2s7Gx9ueGDSE4GEJC7HSdOvDCC2lHr9ddBx9/DKVL2+kXX7RNNoUL2+k337TP7haD6dPP3c4RI9J+LlLk3CN60T/bfOH48eOMHDmS9957jzp16nBz+uybR3yuDLW3nIpLZMGGKN5fs499x+x/aZFCAdza8DLubX05jaqUcjZAlSMnTtgdfP369qi3VSu7o3/ySdtWfskltj38f/+D3bth8GDb/HDZZXZnP2eObTIpV84e8VepAnv32jbvWbNse7rb8OHw229pzSCzZtkmk8BAO33okG1aAXvWsG5d2mcvvdQepbtdcQU88UTadI0a9uGWX5s01IVZvnw5vXv35tixYzz55JM89dRTBDnwS/b7RPDHv9HMXr2PhRv/Ii7RNo5WKlWM3q2q0rNZFcqWLOpwhCq9lBTbdp6YCCtW2B103bq2yeTBB9PanUeNss0tKa4+ntWq2eQAth181Ci4xnXBr1Ur237tbtq58077cLvrLvtwy9gefdtt9uHWrt25MfvZ2CkqB8qVK0f16tX55ptvaJyxbS0P+XUi2PVvNLdO+4mzSXZvcU3Nstzbuhod6pSjUKCWYXJCcrKrJ4/Ynfqnn8LUqfa9li1tE8k338DRo9C1q20/nzLFtn9feqntjQL2CL9hQ3s2EBBgm1fSmzAh7eeiRaFChbzZPuXfjDG89957bNy4kalTp9KgQQNWr16d2vvLKX69t5uy/A/OJqUQXvtSvh/ZlnkDWnFDvQqaBHKZ+6gc4PPP4Ycf0qYHDoSPPrI/b9xoe66sWWOnV62CadPsTh/sRVb3QVPFirbt/tFH7XSTJraJpVYtO924sb2gGqC/SpVP/Pnnn9xwww3069ePzZs3ExcXB+B4EgA/TgS7/o1m6ba/KRIYwIRuDahZLtjpkHya+6L6X3/Byy/b3jFJSXDPPbaXSkKCff+pp9IucILtYrhqlb14WqkSjBmTNu/Ikbb/emionR42zF4sdbvpprQeMErlV8nJyUydOpX69euzZs0a3njjDSIiIihWrJjToaXy26ahqcv/wBjo2bwKFUPyzy/EF8ydC6dOwUMP2b7qQUF2Bz1iBOzfb/ut161rbzi66SbbQycpyV5M/e67cy94RkWl/Vy8OIwblzatF0ZVQXD06FHGjBlDu3btePPNN6maD49e/PKM4I9/o/nKdTYwOPwKp8PJdw4ftkfyf/xhp2fNsjv1TZvs9LvvwjvvwOnTduc+fLjtM164sG2S+eEHmwAA7r7b9rBx95qpVMne+alUQZaYmMicOXNISUmhfPnybNy4ka+++ipfJgHw00Qw7YfdGAM9mlfmslL+eTZw5gysXp3WDHPXXTBggG3PP3XK3uX53Xf2vXr17GuHXAVAvvrK3vB0ySX2ou7kydC7t/25eHFo3z6t66RS/mbDhg00a9aMfv368Z3rn6hGjRr54lpAVvwuEew9EsPirYcoHCgMDq/pdDhelZwMMTFpe+Snn4YjR+zP775ru0+6m2YqV7YXVmNjbV/2xETbjx7g6qttXRr3fS4lSpxbOkApBXFxcYwaNYqWLVty5MgRFi5cyA033OB0WB7xu2sEa/cexxi4oV4FKhWws4HPPrM77EmTbO+bDh2gSJErqVrVHtVPm2bb9zdsgNtvtzvz8uXtZ1955dxlaW8bpXKma9eufPvttwwYMIBXXnmFUqV85yZUv/t3P3jC3jV8pQ/3EoqJSfv5lltseYGkJNt8M3Vq2vtDhkDz5ieoV8821ezZYy/mhobaM4AHH9SbnZS6GKdPnybeVXlv9OjRfP/998yaNcunkgD4YyI4bhNBlTK+czZw9qxt009JgfBwWwbB3V3zyBHb5TIw0NbASUkB999gjx7QqdO/qe31epFWqdyzdOlS6tevz7PPPgtAu3bt6JBZ9T8f4H+J4IS9iaNKmeIOR5K1f/+1/evBttUHBdk7YQMCYOFC223TXSr4l19sc5CIbQ7Kx9ejlCoQjh49Sp8+fbj55psJDg7mtvT1RXyU3yWCKPcZQen8kwgOH7Y7+p077fSIEfbC7Jkztkvm00/bejhgSyw8/7zd6Sul8tZ3331HWFgY8+fPZ8yYMWzcuJFWrVqd/4P5nF/tTuKTDMfOJFCkUADlgp0tJjdggK2V0769HfFp9Gg7iEjdurZC5s03p7Xfu848lVIOq1ixIrVq1WLGjBk0aNDA6XByjV8lgqNxtmG9cqlijowh8OOP0KyZ3cEXKwaPPGJvtgoLs2WL3V0y69WzD6WUs4wxvPPOO2zatInXX3+d+vXrs2rVqnx9T8CF8KumoSNxtvpZ5Ty6PnDsmL0pKyXF9tcfMsSOFXvqlO3KuWePrZcfEGArYBawvy2lfNrevXvp2LEjDzzwAJGRkfmqSFxu86tEcDTWnhFUKe29HkNRUWkDj0yYYMeG3bTJdtecONEOuu0ejUoplf8kJyczefJk6tevz7p163jrrbdYvnx5vioSl9v8qmnIfUbgrR5D8fG2C+fBg7ZOz4QJ9hqAe4SpG2/0ymqVUrno6NGjjBs3jg4dOjBjxgwq+8Hgy/51RuC+RpCLZwT//GObdyZMsN08x461vXrA9vi5+ea08WaVUvlTQkIC7777bmqRuM2bN7No0SK/SALgZ2cEcUk2EZQufvGFcrZtgyuvtCUarrzS1s0HaNv2ohetlMpD69at4/7772f79u1UrlyZ66+/nmrVqjkdVp7yqzOCs66bsIIKX1xpzI8/tuPSdutmSzssXKhdPJXyNbGxsTz22GO0atWKEydOsGjRIq6//nqnw3KEX50RJLqGTCx2gYng0CF7MbhnTzsyVokStvlHKeV7unTpwvfff8/AgQN5+eWXCfHjXhx+lQgSkm3TUFDhnJ8IxcZC8+a2zs+aNdC6dW5Hp5TytlOnTlG0aFGCgoJ4+umnGT16NO3bt3c6LMf5VdNQgqtpqFgRz88I/vgDXnvNDriye3daiWellG9ZsmQJ9erVY5xrPNS2bdtqEnDxr0SQYs8IctI0NHWqrf2TmGjvBtaLwUr5liNHjnD33Xdz6623UqZMGbp16+Z0SPmOXyWCnFwsTk62zUGTJsGyZXotQClf9O233xIWFsaCBQsYN24c69evp3nz5k6Hle/4TSJISTEkuS4WFy10/s1u0gSmT7cJwE87Eijl8ypVqkTdunXZtGkTY8aMoYiOsZopv0kE8Un2dKBY4cDz1gpJSIDOne2gLwWwrIhSBVZKSgozZ85k8ODBANSrV4+VK1dST6s4ZstvLnvGu/qOnq/HUHy8vUP45ZfzIiqlVG7ZvXs3DzzwABEREbRv3564uLgCXR8oN/nNGUFcYtoZQVaSk2330N698yoqpdTFSk5OZtKkSTRs2JCNGzcya9asAl8kLrd5NRGISGcR+V1EdovIqEzeDxGRxSKyRUR2iEg/b8US5+o7GpRN19G4OJg8Ga67zltRKKVy29GjRxk/fjydOnUiMjKSAQMGFMhS0d7ktaYhEQkEXgc6AVHAOhFZZIyJTDfbUCDSGHOriFwK/C4i84wxCbkdT7zrjCCoUNaJoGhRuO++3F6zUiq3nT17liVLltC2bdvUInFVq1bVBHCBvHlG0ALYbYzZ69qxzwe6ZJjHAMFif3slgeNAkjeCSU0EWVwjSEyEhx+2o4gppfKvX375haZNmzJp0iS+//57AC6//HJNAhfBmxeLKwEH001HAS0zzDMdWAQcAoKBnsaYlIwLEpGBwECA8uXLExERkeNgdh6zieBM9OlMP799+yV8+GEDihX7E2MO5Xj5+VVMTMwFfV++TLe5YIqLi+Pdd9/ls88+IzQ0lGeeeYYiRYoU+O1Oz2u/Z2OMVx5Ad+DtdNN9gGkZ5rkTmAwIUBP4E7gku+U2bdrUXIif/jhiLn98iblr5pos5zl71piUlAtafL61YsUKp0PIc7rNBVPHjh0NYAYPHmxOnTrlF9uc0cVsM7DeZLFf9WbTUBRQJd10ZeyRf3r9gM9dce52JYI63ggm2VVeIjCTQev/+QeOH7eDx+vZpVL5x8mTJ1PHCh4zZgw//vgjb7zxBpdcconDkRUs3kwE64ArRaS6iBQBemGbgdI7AHQAEJHyQG1grzeCyS4R9OgBZcvCiRPeWLNS6kIsWrTonCJx1157LW212JdXeC0RGGOSgGHAMmAn8IkxZoeIDBKRQa7ZngOuFpFtwHLgcWPMUW/Ek5oIMjnk//hj+OwzHVJSqfzg8OHD9OrViy5duhAaGsqdd97pdEgFnlfvLDbGLAWWZnjtzXQ/HwLypJJPUjZnBBUr2tHGlFLO+uabb+jduzcxMTE899xzPP744xTWio9e5zclJlJM5ongqafg1ClbblqvDyjlrCpVqtCgQQPeeOMNwsLCnA7Hb/hNiQl301BAhkQQEwMffqhJQCknpKSkMGPGDB588EHAFomLiIjQJJDH/C4RFMqQCF57zY5DrJTKW7t27SI8PJwhQ4bw559/Eh8f73RIfsvvEkFmF4u1NpVSeScpKYmXXnqJhg0bsm3bNmbPns2yZcsICgpyOjS/5XeJIH3T0Lx5UKsWHDyY1aeUUrnt2LFjvPTSS9x0001ERkbSt29fLQ/hMP9JBOa/TUPlykHz5lChglNRKeUfzp49y1tvvUVKSgrly5dny5YtfP7551SsWNHp0BR+lAiSMjkj6NTJnhVo7zSlvGfNmjU0adKEQYMG8cMPPwC2d5DKP/wmEaRkuFhsjG0Scp0oKKVyWUxMDMOHD+eaa67hzJkzfPPNN3Ts2NHpsFQm/CYRpF4jcLVF7t8PVavCjBlORqVUwdW1a1emTJnC0KFD2b59OzfccIPTIakseJQIRKSIiNT0djDelLHWUMmSMHMm6N+mUrnnxIkTqUXixo4dy6pVq5g2bRrBwcEOR6ayc95EICI3A9uA71zTjUVkobcDy23uO4vdlwhCQ+GBB+CKKxwMSqkC5PPPPycsLIyxY8cC0KZNG9q0aeNsUMojnpwRPIsdUOYkgDFmM3bsAJ/k7qb23Xd6I5lSueGff/7hzjvv5I477qBChQr06tXL6ZBUDnlSayjRGHMyQz9fn7vEmjHgp5+GI0dgzx5HwlGqQPj666/p3bs3sbGxvPDCCzz22GNaJM4HeZIIdopIDyBARKoDjwBrvRtW7nP3DnKns2nTIMBvLpUr5R2XX345TZo04fXXX6dOHa+MKaXygCe7wmFAUyAF+ByIxyYDn2I4NxM0bw5NmzoXj1K+KCUlhenTp/PAAw8AEBYWxvLlyzUJ+DhPEsENxpjHjTFNXI9RwI3eDiy3pZ0RCAcPwptvwr//OhuTUr7k999/p23btjz00EMcPHhQi8QVIJ4kgqcyee3J3A4kr4jAb7/B4MGwebPT0SiV/yUmJjJhwgQaNWpEZGQkc+bM4euvv9YicQVIltcIROQGoDNQSUReTffWJdhmIp/Vpg3s3GlvKFNKZe/EiRO88sor3HrrrUybNo0KWpyrwMnuYvFhYDv2msCOdK9HA6O8GZQ3GFfbkGDLTmuTplJZi4+P591332XQoEGUK1eOrVu3UrlyZafDUl6SZSIwxmwCNonIPGOMzzcGpl4jEPj2Wzh0CPr2dTQkpfKln376if79+7Nr1y5q1apFx44dNQkUcJ5cI6gkIvNFZKuI7HI/vB5ZLnPfRyAIc+fC+PGOhqNUvhMdHc2wYcO49tprSUhI4Ntvv9UicX7Ck/sI5gDjgYnY3kL98MFrBOnPCN5+G2JjnY1Hqfyma9eurFixgkceeYTx48dTsmRJp0NSecSTRFDcGLNMRCYaY/YAT4nIKm8H5i0CBAXZh1L+7vjx4wQFBVG8eHGee+45RITWrVs7HZbKY540DZ0VW19ij4gMEpFbgXJejivXmXRFJl57DZYudTAYpfKBBQsWULdu3dQicVdffbUmAT/lSSIYAZQEHgauAR4A7vdmUN6QOgCNCBMmwJIljoajlGP+/vtvunXrRvfu3alSpQq9e/d2OiTlsPM2DRljfnH9GA30ARARn+tCkHaxGP7+G5KSnIxGKWd89dVX3HPPPcTHx/PSSy8xcuRIChXypIVYFWTZnhGISHMR6Soioa7peiIyFx8sOuc+JRCxF4qLFHE4HqUcUKNGDZo3b86WLVv43//+p0lAAdkkAhGZAMwDegPfiMiTwApgC1Arb8LLPe4zghPHhYcfhl9+yXZ2pQqE5ORkpkyZQv/+/QGoW7cu3377LbVq+dy/sPKi7A4HugCNjDFxIlIGOOSa/j1vQvOO+Hj44APQsTNUQRcZGcmAAQNYs2YNN910E/Hx8VofSGUqu6aheGNMHIAx5jjwmy8nAffF4kqVICEBrr/e2XiU8paEhATGjx9PkyZN2LVrFx988AFLlizRJKCylN0ZQQ0R+dz1swDV0k1jjOl2voWLSGdgChAIvG2MeTGTecKB14DCwFFjTDvPw/ecu/uonGc+pXzdyZMnmTx5MrfffjtTp06lXDmf6+2t8lh2ieCODNPTc7JgEQkEXgc6AVHAOhFZZIyJTDdPKeANoLMx5oCIeO0v1n1G8NdfMHAgjBsHFSt6a21K5a2zZ88yffp0hgwZQrly5di2bRuXXXaZ02EpH5Fd0bnlF7nsFsBuY8xeABGZj73uEJlunruBz40xB1zrPHyR68yS+2Lx6dPC4sXwxBPeWpNSeWvlypUMGDCAqKgo6tatS4cOHTQJqBzxZt+xSsDBdNNRQMsM89QCCotIBBAMTDHGzM24IBEZCAwEKF++PBERETkOZv/+BACKF9vLRx9FsX8/7N+f48X4nJiYmAv6vnyZv2zzmTNnmDlzJosWLaJ8+fJMnDiRwMBAv9h28J/fc3re2mZvJoLMmuNNhulC2PGQOwDFgDUistYYc051U2PMTGAmQLNmzUx4eHiOg1kb9xv8uYfqNWoQHl4zx5/3VREREVzI9+XL/GWbr7vuOiIiIhgxYgSdOnXixht9bgTZi+Ivv+f0vLXNnpSYAEBEiuZw2VFAlXTTlbFdUDPO840x5owx5iiwEmiUw/XkyC9r4e67IcXn6qcqBUePHiXWVTr3+eefZ/Xq1bz66qsUK1bM4ciULztvIhCRFiKyDfjDNd1IRKZ5sOx1wJUiUl1EigC9gEUZ5vkSuFZEColIcWzT0c4cbUEOJSbCl19CgMcpUCnnGWOYP38+devW5ZlnngGgdevWtGrVyuHIVEHgye5wKnALcAzAGLMFaH++DxljkoBhwDLszv0TY8wOVwXTQa55dgLfAFuBX7FdTLdfyIZ46tq2cOaMN9egVO7666+/6Nq1K3fddRfVq1fn3nvvdTokVcB4co0gwBiz31aiTpXsycKNMUuBpRleezPD9CvAK54sTyl/s2TJEnr37k1iYiITJ05k+PDhBAYGOh2WKmA8OSM4KCItACMigSIyHPC5oSrd1q6x9xEo5Qtq1qzJ1VdfzdatW3n00Uc1CSiv8CQRDAZGAlWBf4FWrtd8ivvO4uho+PNPh4NRKgvJyclMnjyZvn37AlCnTh2+/vpratb0n55uKu95kgiSjDG9jDGhrkcvVw8fn3T99cK33zodhVL/tWPHDq655hpGjhzJ0aNHiY+Pdzok5Sc8SQTrRGSpiNwnIsFej8jL4uLsmARK5RcJCQk8++yzNGnShD179vDhhx+yePFiLRKn8sx5E4Ex5gpgPPbGr20i8oWI+GwR56++ggcfdDoKpdKcPHmSqVOn0r17dyIjI7nrrrsQPVpRecij3vTGmNXGmIeBq4DT2AFrfJIxoPfeKKfFxsYyZcoUkpOTU4vEzZs3j0svvdTp0JQf8uSGspIi0ltEFmP7+h8BrvZ6ZF7SvTu89prTUSh/tmLFCho0aMDw4cNT68ZU1FK4ykGenBFsx/YUetkYU9MY82i6Ae19R8YqR0rlsVOnTvHggw9y3XXXISKsWLGCDh06OB2WUh7dUFbDGFNgKvO88w4kbYOHHnI6EuVvunbtysqVK/m///s/xo4dS/HixZ0OSSkgm0QgIpOMMY8Cn4nIf46nPRmhLD8qFAiFvFlzVal0jhw5QokSJShevDgTJkwgMDCQ5s2bOx2WUufIbpf4ses5RyOT5Xd9+8EgrwyGqVQaY7I48Q0AACAASURBVAwfffQRDz/8MP369eOVV17RAnEq38ryGoEx5lfXj3WNMcvTP4C6eROeUr4nKiqK2267jd69e1OzZs3Uu4SVyq88uVh8fyav9c/tQPLKxImwKGMxbKVyyaJFiwgLC+OHH35g8uTJ/Pzzz9SrV8/psJTKVnbXCHpixxCoLiKfp3srGDjp7cBym/siR8WKULq0o6GoAqxWrVq0adOG6dOnU6NGDafDUcoj2V0j+BU7BkFl4PV0r0cDm7wZlDfd0xuuvdbpKFRBkZSUxGuvvcbWrVuZO3cuderUYenSpef/oFL5SJaJwBjzJ/An8H3ehaOU79i6dSv9+/dn/fr1dOnShfj4eK0PpHxSltcIRORH1/MJETme7nFCRI7nXYi5a8IE+Oknp6NQvuzs2bM888wzNG3alAMHDvDJJ5+wcOFCTQLKZ2XXNOQejjI0LwLJKzVqQJkyTkehfNnp06d54403uOuuu5g8eTJly5Z1OiSlLkp23UfddxNXAQKNMclAa+BBoEQexOYVPXtCWJjTUShfc+bMGSZPnkxycjKXXnop27dvZ+7cuZoEVIHgSffRL7DDVF4BzMXeQ/ChV6PyAmO02JC6MMuXL6dBgwaMHDmSH3/8EYDy5cs7HJVSuceTRJBijEkEugGvGWMeAip5Nyzvee452OSzfZ5UXjp58iQDBgygY8eOFCpUiB9//JHrrrvO6bCUynWeVN1JEpHuQB+gq+u1wt4LybvCwvQ+AuWZ22+/nVWrVvH444/zzDPPUEwHslAFlCeJ4H5gCLYM9V4RqQ585N2wvOfOO6FaNaejUPnVv//+S8mSJSlRogQvvvgihQoVomnTpk6HpZRXeTJU5XbgYWC9iNQBDhpjnvd6ZErlIWMM77//PmFhYTzzzDMAtGzZUpOA8guejFB2LbAbeAd4F9glItd4OzBveeIJOHTI6ShUfnLgwAFuvvlm7r33XmrXrk3//j5bSkupC+JJ09Bk4CZjTCSAiNQF3geaeTMwb2nQAEqWdDoKlV98+eWX3HPPPRhjmDp1KkOGDCEwMNDpsJTKU54kgiLuJABgjNkpIkW8GJNXuHuP9ugBl1zibCzKecYYRIQ6deoQHh7OtGnTqKYXj5Sf8iQRbBSRt7BnAQC98eGic4ULidMhKAclJSUxadIktm3bxgcffEDt2rVZvHix02Ep5ShP7iMYBOwB/gc8DuzF3l3skx4f5XQEyilbtmyhZcuWjBo1itjYWOLj450OSal8IdszAhFpAFwBLDTGvJw3IXlX+3CnI1B5LT4+nvHjx/PSSy9RtmxZFixYwB133OF0WErlG9lVHx2NLS/RG/hORDIbqczndO7sdAQqr0VHR/PWW2/Ru3dvIiMjNQkolUF2TUO9gYbGmO5Ac2BwThcuIp1F5HcR2S0iWTbKiEhzEUkWkTtzuo6c0pJD/iEmJoaJEyemFomLjIxkzpw5lNHSs0r9R3aJ4Kwx5gyAMebIeeb9DxEJxI5sdiMQBtwlIv+p++ma7yVgWU6Wf6FeeCEv1qKctG7dOurXr8///vc/Vq5cCcCll17qcFRK5V/ZXSOokW6sYgGuSD92sTGm23mW3QLYbYzZCyAi84EuQGSG+R4CPsOedXhdu3Z5sRblhOPHj/Poo48yZ84cateuzapVq7jmGp+991GpPJNdIsjYkDo9h8uuBBxMNx0FtEw/g4hUAm4HriObRCAiA4GBYMv/RkRE5DAUOBh1FoAKFfcQEXEgx5/3VTExMRf0ffmiRx55hO3bt9O9e3cGDBhAYmKi32y7P/2e3XSbc092YxYvv8hlZ9ZhP2ML/WvA48aYZJGs+/cbY2YCMwGaNWtmwsPDcxzMTzGRsO9Pqla9gvDwGjn+vK+KiIjgQr4vX/HPP/8QHBxMiRIlmDVrFkWKFOHkyZMFepszU9B/z5nRbc49OWr3z6Eo7OhmbpWBjFV+mgHzRWQfcCfwhoh0xYtmzPDm0lVeMcYwZ84cwsLCGDNmDAAtWrSgcePGDkemlO/xZiJYB1wpItVdJSl6AYvSz2CMqW6MqWaMqQYsAIYYY77wYkxce603l67ywr59++jcuTP9+vWjXr16DBw40OmQlPJpnpSYAEBEihpjzno6vzEmSUSGYXsDBQLvGmN2iMgg1/tv5jjaXHBVEyfWqnLLwoUL6dOnDyLC9OnTGTx4MAEB3jyeUargO28iEJEW2BLUIUBVEWkEDHANWZktY8xSYGmG1zJNAMaYvp4EfLHOepzKVH7iLhJXr149OnbsyJQpU7j88sudDkupAsGTQ6mpwC3AMQBjzBagvTeD8gb3Veo333I0DJVDiYmJvPDCC/Tu3RuAWrVq8cUXX2gSUCoXeZIIAowx+zO8luyNYPJCW71G4DM2btxIixYtePLJJ0lOTuasns4p5RWeJIKDruYhIyKBIjIc2OXluLzmqqucjkCdT1xcHE888QQtWrTgn3/+YeHChXz88ccULVrU6dCUKpA8SQSDgZFAVeBfoBUXUHcov4iNdToCdT5nzpzhnXfe4b777iMyMpKuXb3ao1gpv+fJ4PWHjTG9jDGhrkcvY8zRvAjOGz74wOkIVGaio6N5+eWXSU5OJjQ0lMjISN555x1Kly7tdGhKFXie9BqaxX/vCMYY45Odtzt2dDoCldE333zDgw8+yMGDB2nRogXh4eGEhoY6HZZSfsOTpqHvgeWux89AOcBnr9rV8J/qEvnesWPHuO+++7jxxhspUaIEP//8s9+VDFAqPzjvGYEx5uP00yLyPvCd1yLysugYpyNQbt26dWP16tU8/fTTPPnkk3oxWCmHeHxncTrVAZ/txP399zDiRqej8F9///03wcHBlCxZkokTJ1KkSBEaNWrkdFhK+bXzNg2JyAkROe56nMSeDYz2fmje0ayZ0xH4J2MM7777LnXr1k0tEte8eXNNAkrlA+cbvF6ARsBfrpdSjPHtwR6rVHY6Av+zd+9eHnzwQb7//nvatm3LoEGDnA5JKZVOtmcErp3+QmNMsuvh00kA4HS00xH4l88//5wGDRrwyy+/MGPGDFasWEGtWrWcDksplY4nvYZ+FZECcz+uawhb5WXuY4YGDRrQuXNnduzYwaBBg7RSqFL5UJZNQyJSyBiTBLQBHhCRPcAZ7Mhjxhjjk8mhqU9G7TsSEhJ4+eWX2bFjBx9++CFXXnkln332mdNhKaWykd01gl+Bq4ACcX+/u1GrYkVn4yjI1q9fT//+/dm6dSu9evUiISFBu4Qq5QOySwQCYIzZk0ex5InTp52OoOCJi4vjmWeeYdKkSVSoUIEvv/yS2267zemwlFIeyi4RXCoiI7N60xjzqhfi8ZqUFPv83Xcw8mZnYylozpw5w5w5c+jfvz8vv/wypUqVcjokpVQOZJcIAoGSuM4MfJ24rlG2a+dsHAXF6dOneeONN/i///s/QkND2blzJ2XLlnU6LKXUBcguEfxtjHk2zyLxMnc201pmF++rr75i0KBBHDp0iFatWhEeHq5JQCkfll1fvgJxJuCW4rpYrNcILtyRI0fo3bs3t9xyCyEhIaxevVqLxClVAGSXCDrkWRR5IMFVL3XLFmfj8GV33HEHn376KWPHjmXjxo20bNnS6ZCUUrkgy6YhY8zxvAzE24oUsc8NGjgbh6/566+/CAkJoWTJkkyePJmiRYtSv359p8NSSuUiv7nN031Dq3Zo8YwxhlmzZhEWFpZaJK5p06aaBJQqgPwmESQm2eeTJ52Nwxfs2bOHDh06MHDgQJo2bcrQoUOdDkkp5UV+kwjiXIPW6zWC7C1YsIAGDRqwYcMGZs6cyfLly7niiiucDksp5UUXMjCNTypRwj43bepsHPmVMQYRoVGjRtx8881MnjyZypW1ZrdS/sBvzggCA+1zyZLOxpHfJCQkMG7cOHr16oUxhiuvvJJPP/1Uk4BSfsRvEoH7GsGpU87GkZ/8+uuvNG3alLFjx1KoUCESEhKcDkkp5QC/SQQxrkHr9xSoEnoXJjY2lscee4zWrVtz4sQJFi9ezLx587RSqFJ+ym8SQXBJe2txnToOB5IPxMXF8cEHHzBw4EAiIyO55ZZbnA5JKeUgr14sFpHOwBRsAbu3jTEvZni/N/C4azIGGGyM8Uq/nkDXlpYoUaAqZ3js1KlTTJ8+nccff5yyZcuyc+dOSpcu7XRYKp9KTEwkKiqK+Ph4p0PJUkhICDt37nQ6jDzlyTYHBQVRuXJlChcu7PFyvZYIRCQQeB3oBEQB60RkkTEmMt1sfwLtjDEnRORGYCbglboF7ubvU354H8HixYsZNGgQ//zzD9dccw3h4eGaBFS2oqKiCA4Oplq1aojkz4On6OhogoODnQ4jT51vm40xHDt2jKioKKpXr+7xcr3ZNNQC2G2M2WuMSQDmA13Sz2CMWW2MOeGaXAt4ratKjGvQ+j/3eWsN+c+RI0d47rnnuO222yhbtiy//PKLFolTHomPj6ds2bL5NgmozIkIZcuWzfGZnDebhioBB9NNR5H90X5/4OvM3hCRgcBAgPLlyxMREZHjYGJi7ClBkcJ7iYjYn+PP+6JHHnmEyMhI+vXrx1133UVMTMwFfXe+xl+2M73c3uaQkBBi3D0s8qnk5GSio6OdDiNPebrN8fHxOfp78GYiyOxQwmQ6o0h7bCJok9n7xpiZ2GYjmjVrZi7kqPbH6B0QtY+wejUIb+P5KZOviYqKolSpUpQsWZLZs2ezZcsW+vXr53RYeSoiIsLvznxye5t37tyZ75tdtGkoa0FBQTRp0sTj5XqzaSgKqJJuujJwKONMItIQeBvoYow55q1gzrrOlArqNYKUlBTeeustwsLCePrppwG46qqrctROqFR+8/zzz1OvXj0aNmxI48aN+eWXX0hKSmL06NE0btw49fH888+nfiYwMJDGjRtTr149GjVqxKuvvkqKe6xalSlvnhGsA64UkerAX0Av4O70M4hIVeBzoI8xZpcXY+GUa0CaAwezn88X/fHHHzzwwAP8+OOPdOjQgYceesjpkJS6aGvWrGHJkiVs3LiRokWLcvToURISEnjqqaf4559/WLt2LZdeeinR0dFMmjQp9XPFihVj8+bNABw+fJi7776bU6dOMW7cOKc2Jd/z2hmBMSYJGAYsA3YCnxhjdojIIBEZ5JptDFAWeENENovIem/FU6aMfa5dy1trcMann35Kw4YN2bx5M++88w7fffcdNWrUcDosVcCEh8OcOfbnxEQ7/cEHdjo21k5//LGdPnXKTn/+uZ0+etROL15sp//5x7N1/v3334SGhqbe6BgaGkqpUqWYNWsW06ZNIygoCIDg4GDGjh2b6TLKlSvHzJkzmT59OsZk2jKt8PJ9BMaYpcDSDK+9me7nAcAAb8bg5q41VFBunnUXiWvSpAldunTh1Vdf5bLLLnM6LKVyzfXXX8+zzz5LrVq16NixIz179qR06dJUrVqV4OBgjy8U16hRg5SUFA4fPkz58uW9HLVv8pvqo+7eVCd9vNbQ2bNnef7559m5cyeffPIJNWvWZP78+U6HpQq49B1QChc+d7p48XOnQ0LOnQ4NPXe6QgXP1lmyZEk2bNjAqlWrWLFiBT179mT06NHnzDN79mymTJnCsWPHWL16NVWqVMl0WXo2kD2/KTHhLjZ36C9n47gYa9eu5aqrruK5556jWLFiWiROFXiBgYGEh4czbtw4pk+fzuLFizlw4EDq2UC/fv3YvHkzISEhJCcnZ7qMvXv3EhgYSLly5fIydJ/iN4kgNNQ+X3mls3FciDNnzjBixAiuvvpqoqOjWbp0KXPnztUicapA+/333/njjz9Spzdv3kzt2rXp378/w4YNS71pKjk5OcuDoiNHjjBo0CCGDRumN8dlw2+ahtzXCHJQfiPfiI+PZ/78+QwZMoQJEyb4Xd9p5Z9iYmJ46KGHOHnyJIUKFaJmzZrMnDmTkJAQnn76aVq2bElISAjFihXjvvvuS71GFhcXR+PGjUlMTKRQoUL06dOHkSNHOrw1+ZvfJIJY11CVvnKN4OTJk0ybNo0nnngitUhcqVKlnA5LqTzTtGlTVq9enel7L774Ik8++WSmB0VZNRGprPlN09AJ141khw87G4cnvvjiC8LCwhg3blzqP4ImAaWUt/hNIqjg6jV2RT7uYv/vv//So0cPbr/9dsqVK8cvv/xC27ZtnQ5LKVXA+U3TkPsaQaF8vMV33nknv/76K+PHj+d///tfjuqJK6XUhcrHu8Xc5S6keDKf1Ro6cOAApUuXJjg4mKlTp1K0aFHCwsKcDksp5Uf8pmnInQCOHnE2DreUlBRef/116tWrx5gxYwBo0qSJJgGlVJ7zm0RQ0VV9oXo+uEbw+++/065dO4YNG0br1q155JFHnA5JKeXH/CYRBLq21H2twCmffPIJjRo1Yvv27cyePZtly5ZRrVo1Z4NSKh9yl5OuX78+t956KydzqV1337591K9fP1eWld7YsWOpVKlSamnsUaNG5fo63DZv3szSpUvPP6OH/CYRnDpta404dY3AXeukadOmdOvWjZ07d9K3b1+921GpLLjLSW/fvp0yZcrw+uuvOx3SeY0YMYLNmzezefNmXnzxRY8/l9N7H3I7EfjNxWL3gDQnjufteuPj43nuuef47bffWLBgAVdccQUffvhh3gah1EWoNuorryx334s3ezxv69at2bp1K2DvOO7SpQvHjh0jOTmZ8ePH06VLF/bt28eNN95ImzZtWL16NZUqVeLLL7+kWLFibNiwgfvvv5/ixYvTpk3aQIjx8fEMHjyY9evXU6hQIV599VXat2/PnDlz+OKLL0hOTmb79u08+uijJCQk8P7771O0aFGWLl1KGXdt+/NYvnw5jz32GElJSTRv3pwZM2ZQtGhRqlWrxv3338+3337LsGHDaN68OUOHDuXIkSMUL16cWbNmUadOHT799FPGjRtHYGAgJUuWZMWKFYwZM4a4uDh++uknnnjiCXr27JmzLz8DvzkjqFTZPlfLwwG7Vq9eTZMmTXjhhRcIDg7WInFKXYDk5GSWL1/ObbfdBthhGBcuXJhalfTRRx9NPeP+448/GDp0KDt27KBUqVJ89tlngC1ON3XqVNasWXPOst1nGdu2beOjjz7ivvvuS61htH37dj788EN+/fVXnnzySYoXL86mTZto3bo1c+fOzTTWyZMnpzYNLVu2jPj4ePr27cvHH3/Mtm3bSEpKYsaMGanzBwUF8dNPP9GrVy8GDhzItGnT2LBhAxMnTmTIkCEAPPvssyxbtowtW7Ywf/58ihQpwrPPPkvPnj3ZvHnzRScB8KMzggA599mbYmJiGD16NNOnT6dKlSp888033HDDDd5fsVJekJMj99zkrhm0b98+mjZtSqdOnQDbzDp69GgiIiIoVKgQf/31F//++y8A1atXp3HjxoBtht23bx+nTp3i5MmTtGvXDoA+ffrw9ddfA/DTTz+ljuhXp04dLr/8cnbtsoMltm/fnuDgYIKDgwkJCeHWW28FoEGDBqlnJxmNGDGCxx57LHV6y5YtVK9enVq17IhY9913H6+//jrDhw8HSN2Jx8TEsHr1arp375762bNnzwJwzTXX0LdvX3r06JH6HeQ2vzkjcJehzotrBAkJCSxYsIChQ4eyfft2TQJKXQD3NYL9+/eTkJCQevQ+b948jhw5wsqVK9m8eTPly5dPPYpPX5E3MDCQpKSk1EGcMpPdOAXplxUQEJA6HRAQQFJSkkfbcL5xEEqUKAHY7uSlSpVKvb6wefNmdu7cCcCbb77J+PHjOXjwIG3atOHYsdwf2t1vEoG72NwJLyWC48ePM3bsWJKSkihTpgw7d+5k2rRpWilUqYsUEhLC1KlTmThxIomJiZw6dYpy5cpRuHBhVqxYwf79+7P9fKlSpQgJCeGnn34CbCJxa9u2ber0rl27OHDgALVr18612OvUqcO+ffvYvXs3AO+//37qmUl6l1xyCdWrV+fTTz8FbALZsmULAHv27KFly5Y8++yzlC1bloMHD+ZohDZP+E0iqOoauKja5bm/7M8++4ywsDDGjx+fWiQuJCQk91eklJ9q0qQJjRo1Yv78+fTu3Zv169fTrl075s2bR506dc77+dmzZzN06FBat25NsWLFUl8fMmQIycnJNGjQgJ49ezJnzpxcHecjKCiI2bNn0717dxo0aEBAQACDBg3KdN558+bxzjvv0KhRI+rVq8eXX34JwP/93//RoEED6tevz9VXX02jRo1o3749kZGRNG7cmI/dg0VfDGOMTz2aNm1qLsSYL7aZyx9fYmb/tPeCPp+ZQ4cOmW7duhnANGnSxGzatCnXlp1bVqxY4XQIeU63+eJFRkbm6vK84fTp006HkOc83ebMfn/AepPFftVvLhYfP2Gfc/MaQY8ePVi3bh0vvvgijz76KIXyc0U7pZTKgt/sudwXi0+fvrhuQ/v376dMmTIEBwczbdo0ihUrlqttikopldf85hqBu4rD5Rd4jSAlJYVp06ZRr149nn76aQAaN26sSUAp5fP85ozgYvz2228MGDCAn3/+mc6dOzNixAinQ1JKqVzjN2cEx12lJXJ6jWD+/Pk0atSInTt3MnfuXJYuXcrlF3paoZRS+ZDfJILTp+2ze4Ca80lJSQGgefPmdO/encjISPr06aNF4pRSBY7fJAL3NYLKlbOfLy4ujlGjRnHHHXdgjOGKK67ggw8+oHz58l6PUSmVxtfKUPsyv0kE57nTG4BVq1bRuHFjXnrpJcqWLUtiYqL3A1NKZcoXy1D7Kr+5WOwuz+HuRppedHQ0o0aN4o033qB69ep89913dOzYMW8DVCofCw8P/89rPXr0YMiQIcTGxnLTTTf95/2+ffvSt29fjh49yp133nnOexERETlavy+XofYFfnNG4C7LERv73/cSExP54osvGD58ONu2bdMkoFQ+4ktlqH2V35wRVKsGq/6By1xjFx87dowpU6YwZswYypQpw2+//aYF4pTKQnZH8MWLF8/2/dDQ0ByfAYBvlqH2VV49IxCRziLyu4jsFpH/DOAp1lTX+1tF5CpvxgP2j+jTTz8lLCyMCRMmpB4haBJQKn8pCGWofYXXEoGIBAKvAzcCYcBdIhKWYbYbgStdj4HADLzkyBFIij7Gq48NokePHlSpUoX169dz7bXXemuVSqlc4MtlqH2FN88IWgC7jTF7jTEJwHygS4Z5ugBzXcXx1gKlRKSiN4KJOQNHv3yJyA0/8vLLL7N27VoaNWrkjVUppXKZr5ah9hWS3anRRS1Y5E6gszFmgGu6D9DSGDMs3TxLgBeNMT+5ppcDjxtj1mdY1kDsGQPly5dvOn/+/BzH88nvCXy74XduvLIkdzTPw4GLHRYTE0PJkiWdDiNP6TZfvJCQEGrWrJlry/OG5ORkAgMDnQ4jT3m6zbt37+ZUhi6S7du332CMaZbZ/N68WJxZo1zGrOPJPBhjZgIzAZo1a2Yy68p2PuHhEBFRJNNucAVZRESEbrMfyO1t3rlzZ76/bhYdHZ3vY8xtnm5zUFAQTZo08Xi53mwaigKqpJuuDBy6gHmUUkp5kTcTwTrgShGpLiJFgF7AogzzLALudfUeagWcMsb87cWYlFIe8lazsfKuC/m9ea1pyBiTJCLDgGVAIPCuMWaHiAxyvf8msBS4CdgNxAL9vBWPUspzQUFBHDt2jLJly2qhRR9ijOHYsWMEBQXl6HNevaHMGLMUu7NP/9qb6X42wFBvxqCUyrnKlSsTFRXFkSNHnA4lS/Hx8Tne4fk6T7Y5KCiIyuerrpmB39xZrJTyXOHChalePX/3rouIiMjRBdGCwFvb7De1hpRSSmVOE4FSSvk5TQRKKeXnvHZnsbeIyBEg++IiWQsFjuZiOL5At9k/6Db7h4vZ5suNMZdm9obPJYKLISLrs7rFuqDSbfYPus3+wVvbrE1DSinl5zQRKKWUn/O3RDDT6QAcoNvsH3Sb/YNXttmvrhEopZT6L387I1BKKZWBJgKllPJzBTIRiEhnEfldRHaLyKhM3hcRmep6f6uIXOVEnLnJg23u7drWrSKyWkR8fpzO821zuvmai0iya9Q8n+bJNotIuIhsFpEdIvJjXseY2zz42w4RkcUissW1zT5dxVhE3hWRwyKyPYv3c3//ZYwpUA9syes9QA2gCLAFCMswz03A19gR0loBvzgddx5s89VAadfPN/rDNqeb7wdsFdw7nY47D37PpYBIoKprupzTcefBNo8GXnL9fClwHCjidOwXsc1tgauA7Vm8n+v7r4J4RtAC2G2M2WuMSQDmA10yzNMFmGustUApEamY14HmovNuszFmtTHmhGtyLXY0OF/mye8Z4CHgM+BwXgbnJZ5s893A58aYAwDGGF/fbk+22QDBYgdOKIlNBEl5G2buMcasxG5DVnJ9/1UQE0El4GC66SjXazmdx5fkdHv6Y48ofNl5t1lEKgG3A29SMHjye64FlBaRCBHZICL35ll03uHJNk8H6mKHud0GPGKMScmb8ByR6/uvgjgeQWbDKWXsI+vJPL7E4+0RkfbYRNDGqxF5nyfb/BrwuDEmuYCMsuXJNhcCmgIdgGLAGhFZa4zZ5e3gvMSTbb4B2AxcB1wBfCciq4wxp70dnENyff9VEBNBFFAl3XRl7JFCTufxJR5tj4g0BN4GbjTGHMuj2LzFk21uBsx3JYFQ4CYRSTLGfJE3IeY6T/+2jxpjzgBnRGQl0Ajw1UTgyTb3A140tgF9t4j8CdQBfs2bEPNcru+/CmLT0DrgShGpLiJFgF7AogzzLALudV19bwWcMsb8ndeB5qLzbrOIVAU+B/r48NFheufdZmNMdWNMNWNMNWABMMSHkwB49rf9JXCtiBQSkeJAS2BnHseZmzzZ5gPYMyBEpDxQG9ibp1HmrVzffxW4MwJjTJKIDAOWYXscvGuM2SEig1zvv4ntrt9WDQAABDxJREFUQXITsBuIxR5R+CwPt3kMUBZ4w3WEnGR8uHKjh9tcoHiyzcaYnSLyDbAVSAHeNsZk2g3RF3j4e34OmCMi27DNJo8bY3y2PLWIfASEA6EiEgU8AxQG7+2/tMSEUkr5uYLYNKSUUioHNBEopZSf00SglFJ+ThOBUkr5OU0ESinl5zQRqHzHVSl0c7pHtWzmrZZVlcYcrjPCVeFyi4j8LCK1L2AZg9wlHUSkr4hclu69t0UkLJfjXCcijT34zHDXPQVKZUoTgcqP4owxjdM99uXRensbYxoB7wGv5PTDrn78c12TfYHL0r03wBgTmStRpsX5Bp7FORzQRKCypIlA+QTXkf8qEdnoelydyTz1RORX11nEVhG50vX6Pelef0tEAs+zupVATddnO4jIJhHZ5qoTX9T1+osiEulaz0TXa2NF5DGx4x40A+a51lnMdSTfTEQGi8jL6WLuKyLTLjDONaQrNiYiM0Rkvdia/ONcrz2MTUgrRGSF67XrRWSN63v8VERKnmc9qoDTRKDyo2LpmoUWul47DHQyxlwF9ASmZvK5QcAUY0xj7I44SkTquua/xvV6MtD7POu/FdgmIkHAHKCnMaYB9k78wSJSBlvVtJ4xpiEwPv2HjTELgPXYI/fGxpi4dG8vALqlm+4JfHyBcXYG0pfMeNJ1t3hDoJ2INDTGTMXWoWlvjGkvIqHAU0BH13e5Hhh5nvWoAq7AlZhQBUKca2eYXmFguqtNPBlbbjmjNcCTIlIZW5P/DxHpgK3Guc5VWqMYWY9NME9E4oB92HEMagN/pqvN9B4wFFv2OB54W0S+ApZ4umHGmCMistdVI+YP1zp+di03J3GWwJZcSD86VQ8RGYj9v64IhGFLTaTXyvX6z671FMF+b8qPaSJQvmIE8C+2kmYAdkd8DmPMhyLyC3AzsExEBmBrz7xnjHnCg3X0Nsasd0+ISNnMZnLVv2mBLXTWCxiGLYHsqY+BHsBvwEJjjBG7V/Y4TuxIXS8CrwPdRKQ68BjQ3BhzQkTmAEGZfFaA74wxd+UgXlXAadOQ8hUhwN+uAUf6YI+GzyEiNYC9ruaQRdgmkuXAnSJSzjVPGRG53MN1/gZUE5Garuk+wI+uNvUQY8xS7IXYzHruRAPBWSz3c6ArcBc2KZDTOI0xidgmnlauZqVLgDPAKbEVOG/MIpa1wDXubRKR4iKS2dmV8iOaCJSveAO4T0TWYpuFzmQyT09gu4hsxtajn+vqqfMU8K2IbAW+wzabnJcxJh5b2fFTV2XLFOxoZ8HAEtfyfsSerWQ0B3jTfbE4w3JPYMcVvtwY86vrtRzH6br2MAl4zBizBdgE7ADexTY3uc0EvhaRFcaYI9geTR+51rMW+10pP6bVR5VSys/pGYFSSvk5TQRKKeXnNBEopZSf00SglFJ+ThOBUkr5OU0ESinl5zQRKKWUn/t/Zp7Uk+si3tcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fpr, tpr, 'b:', label='SGD')\n",
    "plot_roc_curve(fpr_forest, tps_forest, label=\"Random Forest\")\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We can see that `Random Forest classifier` is superior to `SGD` because its curve is much closer to the top left corner of the graph.\n",
    "- Let's check its area under the curve (AUC):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9921782134037131"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Let's measure the precision & recall for `RF`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_preds_forest = cross_val_predict(estimator=forest_clf, X=X_train, \n",
    "                                   y=y_train_5, cv=5, \n",
    "                                   n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9859026830377444"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_true=y_train_5, y_pred=y_preds_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8002953119232189"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_true=y_train_5, y_pred=y_preds_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- You now know how to ...\n",
    "    - train classifiers\n",
    "    - choose the appropriate metric for your task\n",
    "    - evaluate your classifier using cross-validation\n",
    "    - select the precision-recall ratio that fits your needs\n",
    "    - use ROC and ROC AUC to compare various models.\n",
    "- Time to move to multiclass classification (beyond binary classification):"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiclass Classification\n",
    "\n",
    "- There are techniques you can use to perform multiclass classification with binary classifiers.\n",
    "- One way to create a system that can classify an image into 10 labels (from 0 to 9) is to train 10 binary classifiers.\n",
    "    - One for each digit.\n",
    "    - Then when you want to classify an image, you get each classifier's associated positive score and just pick the model class w/ the highest score.\n",
    "    - This is called one-versus-the-rest startegy, or 1-vs-all.\n",
    "- Another startegy is to train a binary classifier for each pair of digits.\n",
    "    - This is called 1-vs-1.\n",
    "    - If there are $N$ classes, you need to train $\\frac{N \\times (N-1)}{2}$.\n",
    "    - For the MNIST Problem, this means training $45$ binary classifiers.\n",
    "    - **and see which class wins the most duals**.\n",
    "    - Even though we need to create many classifiers, they are needed to be trained on only the data points where one of the classes appeared.\n",
    "- Scikit-learn detects when you try to use a binary classification algorithm for a multiclass classification problem\n",
    "    - It then automatically use OvO or OvR depending on the algorithm to train."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_clf = LinearSVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((30000, 784), (30000,))"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Note — Support Vector Machines trained with non-linear kernels has a compexity of O(n_samples^2 * m_features).\n",
    "- that is why we are training `LinearSVC` instead of `SVC`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "          intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "          multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "          verbose=0)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf.fit(X=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3], dtype=int8)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- In this case, the Support Vector Classifier used `OvO` to predict the class of `some_digit`\n",
    "    - Where it tries $N*(N-1)/2$ classifier for all pairs of classes.\n",
    "    - After input a new sample, it runs all 45 trained models and picks the one who won the most duals.\n",
    "- If we inspect the score for a specific instance input, we will notice that the model outputs 10 scores, for each digit.\n",
    "    - Meaning a performance of each digit versus all others. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.07249823, -9.07709945, -1.73230222,  0.72046274, -5.12401579,\n",
       "         0.19933341, -5.44531551, -5.64787969, -3.6019937 , -3.92171732]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = svm_clf.decision_function(X=[some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=int8)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf.classes_[np.argmax(some_digit_scores)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- If you want to force your algorithms to follow a particular startegy, `OvO` or `OvR` for example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.multiclass import OneVsRestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "ovr_clf = OneVsRestClassifier(estimator=LinearSVC(), n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=LinearSVC(C=1.0, class_weight=None, dual=True,\n",
       "                                        fit_intercept=True, intercept_scaling=1,\n",
       "                                        loss='squared_hinge', max_iter=1000,\n",
       "                                        multi_class='ovr', penalty='l2',\n",
       "                                        random_state=None, tol=0.0001,\n",
       "                                        verbose=0),\n",
       "                    n_jobs=-1)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ovr_clf.fit(X=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3], dtype=int8)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ovr_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovr_clf.estimators_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- SGD or Random Forest Classifiers can directly classify an instance into multiple classes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf = SGDClassifier(n_jobs=-1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
       "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
       "              l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
       "              max_iter=1000, n_iter_no_change=5, n_jobs=-1, penalty='l2',\n",
       "              power_t=0.5, random_state=42, shuffle=True, tol=0.001,\n",
       "              validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5], dtype=int8)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -50458.5075631 , -136486.63842062,  -14135.49961456,\n",
       "         -10399.0572803 ,  -63068.57630296,   -3043.28005489,\n",
       "         -64103.92249307,  -68757.6494235 ,  -44670.30230201,\n",
       "         -27252.81246434]])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.decision_function([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Let's evaluate the performance of our classifiers using cross-validation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.87824783, 0.82422526, 0.85585736, 0.88610972, 0.86140761])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(estimator=sgd_clf, X=X_train, y=y_train, cv=5, scoring='accuracy', n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- It gets over 84% over all test folds.\n",
    "    - If you use a random classifier, you would get 10% accuracy, so 80% isn't that bad.\n",
    "- But we can do much better, by simply scaling the input, we can push accuracy to over 86%:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_scaled = scaler.fit_transform(X=X_train.astype(np.float64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90739507, 0.90503166, 0.89218464, 0.90628648, 0.90226818])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(estimator=sgd_clf, X=X_train_scaled, y=y_train, scoring='accuracy', cv=5, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error Analysis\n",
    "\n",
    "- If this were a real machine learning project, you would need to follow the steps in your checklist:\n",
    "    - You'd explore data preparation options.\n",
    "    - Try out, shortlisting, & hyper-parameter finetuning multiple models.\n",
    "    - & don't forget to automate as much as possible.\n",
    "- Here, we will assume that you have found a promissing model and that you want to improve it.\n",
    "    - One of the few ways is to analyze the types of errors it makes.\n",
    "- First, let's look at the confusion matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_predict = cross_val_predict(estimator=sgd_clf, X=X_train_scaled, y=y_train, cv=5, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2806,    0,   15,    4,    4,   29,   17,    2,   84,    0],\n",
       "       [   1, 3249,   26,   14,    3,   24,    3,    3,   93,    7],\n",
       "       [  14,   17, 2587,   57,   38,   14,   38,   21,  156,    6],\n",
       "       [  13,   15,   72, 2651,    2,   95,   11,   27,  150,   37],\n",
       "       [   9,    8,   27,    7, 2623,    4,   20,   10,  120,   98],\n",
       "       [  16,   11,   19,   82,   39, 2253,   37,    7,  204,   41],\n",
       "       [  16,    6,   22,    0,   22,   46, 2801,    5,   57,    0],\n",
       "       [  11,    3,   36,   16,   28,    2,    3, 2821,   53,  134],\n",
       "       [   6,   35,   23,   58,    3,   55,   23,    3, 2648,   21],\n",
       "       [  16,   13,   17,   31,   61,   15,    1,   86,  123, 2640]])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mtrx = confusion_matrix(y_true=y_train, y_pred=y_train_predict)\n",
    "conf_mtrx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- It's often better to look at an image representation of the confusion matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAALLklEQVR4nO3d32vd9R3H8dcrv2pbJ7VsN6ay+mO6iTBTwrAWvLBebFMUYRdOFOZNbzatIojuxn9ARC+GEOq8sehFDThkOAfqxUBqY1qwMRakaq1W7CjTImpMznsXObI0yTzfY76ffM/x/XyA0MbTT9+kefb86Pe844gQgB+2gaYHAFAeoQMJEDqQAKEDCRA6kAChAwk0FrrtX9s+Zvtd2w81NUdVti+2/artWdsztvc2PVMVtgdtH7b9YtOzVGF7i+0Dtt9pf653Nj1TJ7bvb39NHLX9rO3zmp5puUZCtz0o6S+SfiPpKkm/t31VE7N0YV7SAxHxC0nXSvpjH8wsSXslzTY9RBeekPRSRPxc0i/V47PbHpV0r6TxiLha0qCk25udaqWm7tF/JendiDgeEXOSnpN0a0OzVBIRpyJiuv3js1r8AhxtdqrvZnubpJsk7Wt6lipsXyDpeklPSVJEzEXEf5qdqpIhSRttD0naJOnjhudZoanQRyV9uOTnJ9Xj0Sxle7ukMUkHm52ko8clPSip1fQgFV0q6bSkp9tPN/bZ3tz0UN8lIj6S9KikE5JOSfosIl5udqqVmgrdq3ysL67FtX2+pOcl3RcRnzc9z/9j+2ZJn0bEm03P0oUhSTskPRkRY5K+kNTTr9/YvlCLj0YvkXSRpM2272x2qpWaCv2kpIuX/HybevDhznK2h7UY+f6ImGx6ng52SbrF9vtafGp0g+1nmh2po5OSTkbEt4+UDmgx/F52o6T3IuJ0RHwjaVLSdQ3PtEJToR+S9DPbl9ge0eKLF39raJZKbFuLzx1nI+KxpufpJCIejohtEbFdi5/fVyKi5+5ploqITyR9aPvK9od2S3q7wZGqOCHpWtub2l8ju9WDLyAONfGbRsS87T9J+ocWX6X8a0TMNDFLF3ZJukvSW7aPtD/254j4e4Mz/RDdI2l/+w7guKS7G57nO0XEQdsHJE1r8V9mDkuaaHaqlczbVIEfPq6MAxIgdCABQgcSIHQgAUIHEmg8dNt7mp6hG/02r8TM66HX5208dEk9/QlaRb/NKzHzeujpeXshdACFFblgZuvWrTE6Wu3NaGfOnNHWrVsr3fbo0aNrGQvo2uJVrZ1FROXbLv01JUTEikGKXAI7OjqqF154ofZzL7vsstrP/Fa3f0hV9eOVhwMDZR7olfxclDp7eHi4yLmSNDc3V+zs5XjoDiRA6EAChA4kQOhAAoQOJFAp9H7bwQ7gXB1D79Md7ACWqHKP3nc72AGcq0rofb2DHUC10CvtYLe9x/aU7akzZ86sfTIAtakSeqUd7BExERHjETFe9dp1AOujSuh9t4MdwLk6vqmlT3ewA1ii0rvX2t+kgG9UAPQprowDEiB0IAFCBxIgdCABQgcSKLIc0naRBV4ld44NDg4WObcf96SxP+9/NmzYUOzsr7/+usi5qy2H5B4dSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEEKn2Txe+jxMrggYFyfy+98cYbRc7duXNnkXMlqdVqFTm31Frmkn9+pWYeHh4ucq5Ubt3zarhHBxIgdCABQgcSIHQgAUIHEiB0IAFCBxLoGLrti22/anvW9oztvesxGID6VLlgZl7SAxExbftHkt60/c+IeLvwbABq0vEePSJORcR0+8dnJc1KGi09GID6dPUc3fZ2SWOSDpYYBkAZla91t32+pOcl3RcRn6/y//dI2lPjbABqUil028NajHx/REyudpuImJA00b59mXcYAPheqrzqbklPSZqNiMfKjwSgblWeo++SdJekG2wfaf/328JzAahRx4fuEfEvSfW/uRzAuuHKOCABQgcSIHQgAUIHEiB0IAGX2J5pO0psgS1paKjMQtxDhw4VOVeSrrnmmiLnbtiwoci5Jbeeltowu3HjxiLnStKXX35Z+5mtVksRsSI+7tGBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUig2Lrn2g8trNS64FarVeRcSZqeni5y7o4dO4qcW3IFeImvY6nsuucSXxtzc3NqtVqsewYyInQgAUIHEiB0IAFCBxIgdCABQgcSqBy67UHbh22/WHIgAPXr5h59r6TZUoMAKKdS6La3SbpJ0r6y4wAooeo9+uOSHpRU7npOAMV0DN32zZI+jYg3O9xuj+0p21O1TQegFlXu0XdJusX2+5Kek3SD7WeW3ygiJiJiPCLGa54RwBp1DD0iHo6IbRGxXdLtkl6JiDuLTwagNvw7OpDAUDc3jojXJL1WZBIAxXCPDiRA6EAChA4kQOhAAoQOJFBsC2yJjZ+lNn1K5TaUDg8PFzlXkhYWFoqcOzk5WeTc2267rci5Urltu1u2bClyriSdPXu29jMXFhYUEWyBBTIidCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSYAtsW6ktsP0488BAmb//jx07VuRcSbr88suLnDs01NW3J+zK/Px8kXPZAgskRehAAoQOJEDoQAKEDiRA6EAChA4kUCl021tsH7D9ju1Z2ztLDwagPlWvBnhC0ksR8TvbI5I2FZwJQM06hm77AknXS/qDJEXEnKS5smMBqFOVh+6XSjot6Wnbh23vs7258FwAalQl9CFJOyQ9GRFjkr6Q9NDyG9neY3vK9lTNMwJYoyqhn5R0MiIOtn9+QIvhnyMiJiJiPCLG6xwQwNp1DD0iPpH0oe0r2x/aLentolMBqFXVV93vkbS//Yr7cUl3lxsJQN0qhR4RRyTxkBzoU1wZByRA6EAChA4kQOhAAoQOJEDoQALF1j3XfmhhpVYc9+O651arVeTcko4fP17k3CuuuKLIuZI0MjJS+5lfffWVFhYWWPcMZEToQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiTQV1tgS21qlcptax0cHCxyriTNz88XOXdoqOo32e3OwsJCkXOlcn9+R44cKXKuJI2NjdV+ZkQoItgCC2RE6EAChA4kQOhAAoQOJEDoQAKEDiRQKXTb99uesX3U9rO2zys9GID6dAzd9qikeyWNR8TVkgYl3V56MAD1qfrQfUjSRttDkjZJ+rjcSADq1jH0iPhI0qOSTkg6JemziHi59GAA6lPlofuFkm6VdImkiyRttn3nKrfbY3vK9lT9YwJYiyoP3W+U9F5EnI6IbyRNSrpu+Y0iYiIixiNivO4hAaxNldBPSLrW9ibblrRb0mzZsQDUqcpz9IOSDkialvRW+9dMFJ4LQI0qvfE4Ih6R9EjhWQAUwpVxQAKEDiRA6EAChA4kQOhAAoQOJFBs3fPitTUo+Xkotf661FrmUiuZJWlkZKTIuSXXdb/++uu1n3nHHXdoZmaGdc9ARoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKltsCelvRBxZv/WNK/ax+inH6bV2Lm9dAr8/40In6y/INFQu+G7amIGG90iC7027wSM6+HXp+Xh+5AAoQOJNALoU80PUCX+m1eiZnXQ0/P2/hzdADl9cI9OoDCCB1IgNCBBAgdSIDQgQT+C+YJxU1zS2pdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(A=conf_mtrx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- First, let's focus the plot on the errors.\n",
    "- Instead of plotting the absolute numbers, we will plot counts over the number of images of the class:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_sums = conf_mtrx.sum(axis=1, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "normed_conf_mtrx = conf_mtrx / row_sums"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Then we fill the diagonal with zeros to highlight only the errors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.fill_diagonal(normed_conf_mtrx, val=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAL+UlEQVR4nO3dW4ic9RnH8d8vMxti1BzYVjSJ1ASKTRCLshRPBDxctFWbm4IWFPQmXrSeEEV7o/eKqFCERaugQZEoWKRYC8aL3kTXGPCwNoimmmRzMNiYhF2zmzy92AnE3dR5V99/3hmf7weE7Dg+Pqz5+s5s3v2vI0IAftzmNb0AgPIIHUiA0IEECB1IgNCBBAgdSKCx0G3/2va/bX9i+/6m9qjK9rm2N9ketf2h7Tub3qkK2y3b79l+reldqrC9xPZG2x93PteXNr1TN7bv7vye+MD2C7YXNL3TTI2Ebrsl6S+SfiNpjaQ/2F7TxC5zMCXpnohYLekSSX/sg50l6U5Jo00vMQePS3o9In4h6Zfq8d1tL5d0h6ShiLhAUkvSjc1uNVtTV/RfSfokIj6NiCOSXpS0rqFdKomIsYjY0vn1QU3/Blze7FbfzfYKSddKeqrpXaqwvUjSWklPS1JEHImI/za7VSVtSafZbktaKGlXw/vM0lToyyV9ccLHO9Tj0ZzI9nmSLpK0udlNunpM0n2SjjW9SEWrJO2T9Ezn7cZTtk9veqnvEhE7JT0i6XNJY5IORMQbzW41W1Oh+ySP9cW9uLbPkPSypLsi4uum9/l/bF8naW9EvNv0LnPQlnSxpCcj4iJJhyX19NdvbC/V9KvRlZKWSTrd9k3NbjVbU6HvkHTuCR+vUA++3JnJ9oCmI98QEa80vU8Xl0v6ne3tmn5rdJXt55tdqasdknZExPFXShs1HX4vu0bSZxGxLyImJb0i6bKGd5qlqdDfkfRz2yttz9f0Fy/+1tAuldi2pt87jkbEo03v001EPBARKyLiPE1/ft+MiJ670pwoInZL+sL2+Z2Hrpb0UYMrVfG5pEtsL+z8HrlaPfgFxHYT/9KImLL9J0n/0PRXKf8aER82scscXC7pZknv297aeezPEfH3Bnf6Mbpd0obOBeBTSbc2vM93iojNtjdK2qLpP5l5T9Jws1vNZr5NFfjx4844IAFCBxIgdCABQgcSIHQggcZDt72+6R3mot/2ldj5VOj1fRsPXVJPf4JOot/2ldj5VOjpfXshdACFFblhxnbf3YXTarUqPS8iNH2nYzVHjx79vis1Zv78+UXmzuVz0Suf58HBwUrPm5iY0IIFcztvYv/+/d9npa4iYtYnrpFbYHvRokWLisw9ePBgkbklLVu2rMjckp+Lr776qsjc66+/vshcSXr22WeLzZ6Jl+5AAoQOJEDoQAKEDiRA6EAClULvtzPYAXxb19D79Ax2ACeockXvuzPYAXxbldD7+gx2ANXujKt0Bnvnu3d6+sZ+IKsqoVc6gz0ihtU5/bIf73UHfsyqvHTvuzPYAXxb1yt6n57BDuAElb57rfNDCvhBBUCf4s44IAFCBxIgdCABQgcSIHQgAc6M65jrwX5VTU1NFZkrSYcPHy4yd3x8vMjcUue6lVTyv1+JQzgnJydP+jhXdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEihy3HOr1dKZZ55ZZG4pY2NjReZeeeWVReZK0t69e4vM3blzZ5G5q1evLjJXkvbv319k7g033FBkriS99NJLxWbPxBUdSIDQgQQIHUiA0IEECB1IgNCBBAgdSKBr6LbPtb3J9qjtD23feSoWA1CfKjfMTEm6JyK22D5T0ru2/xkRHxXeDUBNul7RI2IsIrZ0fn1Q0qik5aUXA1CfOb1Ht32epIskbS6xDIAyKt/rbvsMSS9Luisivj7J318vaX3n17UtCOCHqxS67QFNR74hIl452XMiYljSsCS12+2obUMAP1iVr7pb0tOSRiPi0fIrAahblffol0u6WdJVtrd2/vpt4b0A1KjrS/eI+Jck3nQDfYw744AECB1IgNCBBAgdSIDQgQSKnAIbETp27FiJ0cWsW7euyNxXX321yFxJmjevzP+nb7vttiJzn3vuuSJzJWnp0qVF5n799aybQGuzatWq2mdu3779pI9zRQcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFH1P+jzFutVixYsKD2uSV2Pe6ss84qMnfPnj1F5krSxMREkbntdpFTwDU4OFhkriQdOnSoyNyHHnqoyFxJ2rVrV+0zN2zYoD179sz6oahc0YEECB1IgNCBBAgdSIDQgQQIHUiA0IEEKoduu2X7PduvlVwIQP3mckW/U9JoqUUAlFMpdNsrJF0r6amy6wAooeoV/TFJ90k6VnAXAIV0Dd32dZL2RsS7XZ633vaI7ZGS96QDmLsqV/TLJf3O9nZJL0q6yvbzM58UEcMRMRQRQ/ase+oBNKhr6BHxQESsiIjzJN0o6c2IuKn4ZgBqw5+jAwnM6RuPI+ItSW8V2QRAMVzRgQQIHUiA0IEECB1IgNCBBIqcAjswMBBLliypfe4333xT+8zjFi5cWGTuunXrisyVpLfffrvI3K1btxaZe+GFFxaZK0nbtm0rMnd8fLzIXEkaGhqqfebo6KgOHz7MKbBARoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAJFToFtt9uxePHi2uceOnSo9pnHDQ4OFpk7NjZWZK4knXPOOUXmLl++vMjckZGRInMlaWBgoMjcK664oshcSdq0aVORuRHBKbBARoQOJEDoQAKEDiRA6EAChA4kQOhAApVCt73E9kbbH9setX1p6cUA1Kdd8XmPS3o9In5ve76kMj9jGEARXUO3vUjSWkm3SFJEHJF0pOxaAOpU5aX7Kkn7JD1j+z3bT9k+vfBeAGpUJfS2pIslPRkRF0k6LOn+mU+yvd72iO2REvfPA/j+qoS+Q9KOiNjc+XijpsP/logYjoihiBiyZ91TD6BBXUOPiN2SvrB9fuehqyV9VHQrALWq+lX32yVt6HzF/VNJt5ZbCUDdKoUeEVslDRXeBUAh3BkHJEDoQAKEDiRA6EAChA4kQOhAAlX/HH1OIkJHjtT/fS9Hjx6tfeZxixYtKjJ3YmKiyFyp3BHHW7ZsKTK31WoVmStJk5OTReaOj48XmStJTzzxRO0zH3744ZM+zhUdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUjAEVH70Hnz5sX8+fNrn7ty5craZx538ODBInNXr15dZK4kbdq0qcjctWvXFpm7bdu2InMlaffu3UXmljx5+Oyzz6595pdffqnJyUnPfJwrOpAAoQMJEDqQAKEDCRA6kAChAwkQOpBApdBt3237Q9sf2H7B9oLSiwGoT9fQbS+XdIekoYi4QFJL0o2lFwNQn6ov3duSTrPdlrRQ0q5yKwGoW9fQI2KnpEckfS5pTNKBiHij9GIA6lPlpftSSeskrZS0TNLptm86yfPW2x6xPVLi/nkA31+Vl+7XSPosIvZFxKSkVyRdNvNJETEcEUMRMWTPuqceQIOqhP65pEtsL/R0wVdLGi27FoA6VXmPvlnSRklbJL3f+WeGC+8FoEbtKk+KiAclPVh4FwCFcGcckAChAwkQOpAAoQMJEDqQAKEDCRQ57rndbsfixYtrnzs1NVX7zOMGBgaKzF2yZEmRuZK0atWqInPfeeedInMPHDhQZK4k3XLLLUXmrlmzpshcSbr33nuLzI0IjnsGMiJ0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIocgqs7X2S/lPx6T+R9GXtS5TTb/tK7Hwq9Mq+P4uIn858sEjoc2F7JCKGGl1iDvptX4mdT4Ve35eX7kAChA4k0AuhDze9wBz1274SO58KPb1v4+/RAZTXC1d0AIUROpAAoQMJEDqQAKEDCfwPbQ/RI7nsAs0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(A=normed_conf_mtrx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We can clearly see the kind of errors the classifier makes.\n",
    "- Remember that the rows represent acutal labels and the columns represent the predicted class.\n",
    "- The column of class 8 is quite bright, which tells us that many input images get misclassified as 8s.\n",
    "- However, the row for 8s is not that bad, telling us that when given input images of 8s, the model does a good job of classifying them as 8s.\n",
    "- As you can see, the confusion matrix is not necessarly symmetrical.\n",
    "- In our specific example, 3s and 5s are often confused.\n",
    "- Analyzing your confusion matrix often gives you insights into how to improve the classifier itself.\n",
    "- Looking at this plot, it seems that our efforts should be focused on reducing the false 8s.\n",
    "- For example, you could try collecting images of digits that look like 8s but are not, then training the classifier to better distinguish.\n",
    "- Analyzing your errors could be also a good way into knowing what the model is learning and why is it failling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_a, cl_b = 3, 5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_aa = X_train[(y_train == cl_a) & (y_train_predict == cl_a)]\n",
    "X_ab = X_train[(y_train == cl_a) & (y_train_predict == cl_b)]\n",
    "X_ba = X_train[(y_train == cl_b) & (y_train_predict == cl_a)]\n",
    "X_bb = X_train[(y_train == cl_b) & (y_train_predict == cl_b)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import ceil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_digits(instances, images_per_row=10, **options):\n",
    "    \"\"\"Plots digits on a grid of rows and columns\n",
    "    \n",
    "    # Arguments\n",
    "        instances: np.ndarray, the digits, where each is a flat array\n",
    "        images_per_row: int, how many digits to be displayed per row\n",
    "        options: other arguments for `plt.imshow()`\n",
    "    \"\"\"\n",
    "    size = 28\n",
    "    n_images = instances.shape[0]\n",
    "    images_per_row = min(images_per_row, n_images)\n",
    "    images = [instance.reshape(size, size) for instance in instances]\n",
    "    n_rows = ceil(n_images / images_per_row)\n",
    "    row_images = list()\n",
    "    n_empty = (n_rows * images_per_row) - n_images\n",
    "    images.append(np.zeros((size, size * n_empty)))\n",
    "    for row in range(n_rows):\n",
    "        rimages = images[row*images_per_row : (row+1)*images_per_row]\n",
    "        row_images.append(np.concatenate(rimages, axis=1))\n",
    "    image = np.concatenate(row_images, axis=0)\n",
    "    plt.imshow(image, cmap='binary', **options)\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAccAAAHBCAYAAAAcpXCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd1gU19fHv0tssYENFUWJQcXe0BgFW+z+BDHYIir23lBj7AW7wRI1sfeGSlBRo1jAaMAS7CJgbCAqikovUu77x7z3yC67sCw7u8bcz/PsE9mZnTnZnbl37info2CMQSAQCAQCwUdMjG2AQCAQCASfGmJyFAgEAoFABTE5CgQCgUCggpgcBQKBQCBQQUyOAoFAIBCoICZHgUAgEAhUKJDLdlHnIRBkR2FsA3RE3M8CQXbU3s9i5SgQCAQCgQpichQIBAKBQAUxOQoEAoFAoIKYHAUCgUAgUEFMjgKBQCAQqJBbtupnS3BwMADgxIkT2LJlC5o2bQoAaNiwIe0zadIkFCpUyCj2CQQC3Xj58iV++uknAEB8fDxSUlIAAObm5ihVqhQWLlwIAChRooTRbBR8+ihy6cqRr9TvsLAwfPjwAZcuXQIAjBkzBgqF5iz4Hj16AAAOHjwo66S0adMmTJs2DYB082ji/PnzaNeunV7OmZCQAADw9PRE4cKFAQA3btyg8+/duxdt27ZFpUqV1H6+QoUKcHR0BADY2trqxSZdSE5OxpkzZwAACxYswK1bt9Tut337dpQqVYr+tra2Rt26dQ1iowH47Eo5MjIysHLlSgDA77//juvXr2fbZ9CgQXByckLz5s0BAOXLl5fJzLxz7949AMCsWbNw+fJlGj9MTU1pn5cvXyIuLg5z5swBAJokBbnz9OlTxMbGAgAsLCxQrlw52c6Vnp6Oly9fApDmglOnTsHf3x8AoFAoUKNGDQDAuXPnULlyZX2cUu39rPfJ8d69e9i1axcA4PDhw8jMzERkZKR0MMZynBw5gwYNwpo1awAAJUuWzKsJufLu3TvUrl0bABAVFaVxPzMzM3h6eqJjx475PuePP/4IADQA6YKJieQFr1OnDvr27QsA6NevH7766qt826dKaGgoAGD58uVK7yclJeHQoUN5Pl7dunVx5MgRAKCL+1/MZzc5JiYm0kqqYMGCdK198cUXdM8mJiYCAOrXrw8AOHXqFCwsLGQ1WBvOnz8PFxcXAEBKSgp27NiBli1bAoDSIH727Fk4ODhgzJgxAAAPDw/ZbQsMDKT/XrlyRWnb4cOH1X6mefPmmDx5MgCgd+/e8hqogdTUVAQHB+PgwYMAAG9vbzx8+BCANIZNnToVJ06cACCNCVWqVAEAenDKDxs3bsTYsWOV3uPzVNb5w83NLV/jaRZEnaNAIBAIBNqg95Wjg4MDTp48qf5gWq4cAeDixYsAADs7u7yaoBUbN24EAEydOhWJiYmoWrUqAODZs2dK+02ePBmrVq3K9/m+/vprAMDjx4+V3i9btiwAoF69etk+Y2NjAwAICQlBTEwMbt68mW0fHx8f/O9//8u3farwJ8Hnz5/r7Zh8ZTJo0CD88ssvOh2Dr15SUlLoyVWTa1cdEyZM0MdK+7NbOaanp9NKpnbt2nRdli5dGl9++SUAaaXj7e0NT09PAMDo0aOxfv16uW3OlXHjxmH//v0ApByCFi1aqN3vxx9/xMqVK+l+5qszuTh06BD69OkDQFpRWVpaqt2vcuXKtOI6cuQIDh8+jF69etExDMXTp09x7do1ANKq+u+//0ZmZiZt5/dvuXLlkJ6eTmPmpUuXMHHiRAAgj19+sLCwUPLo1ahRA9WrVwcApbmlYsWK+hqfDONWXbduHSZNmkR/m5ubY+jQoQCAzMxMctcAQEBAAE2Cqsg9OXIaNWqEW7du0eR09+5dpe2PHj1CtWrV8n2eR48eAZDclTVr1qT3ixYtCkD6oXMiPj6ebMw6gY8YMQKbNm3Kt32q8JisquvJ1NQUc+fOBQBs3ryZ3K95oW7dujSYaRuH3L9/Py5fvoyAgAAAwJ07d/J8XgCoXr06xcDNzc11OgY+w8lRW1JSUijU0axZM1y+fDnfRuWXu3fvUqy+dOnS2bZzN+r27dvxww8/YPPmzQCAAgU+nXzEiIgIAEDLli0RERFBDyByu1UzMzNp/Fi+fHm2xQGftPv27QtXV1cA0oNUQEAAFi9eDEDKFXFzcwMAveSKjBw5Elu3bgUgTYxnz54lt6qVlRXt16RJE5rM84lwqwoEAoFAoA16f3QaPXo0ZZ0CUnC/QoUKaveNi4ujlQNP2gGkJxFeWiE3s2bNwuLFizW65lJTU/VyHu5W5f/NKz4+PkpPdUWKFAEADBs2LP/GqYEfd/To0UrvFyhQgFyuPXv2pIC4m5sbevXqhfDwcADA27dvNR47MjIS0dHRebKnf//+UCgU+OKLLwCAXDoA0KpVK3ID8kSrrPBMxrVr1+Lhw4fYt28fAPndap8bwcHBmDp1KtLT0wHknMxmSNSFJNLS0gAAx48fx549e+j9fv36fVIrRgBYtWqVkjvS09PTYIk4w4YNw44dO+jvb7/9FoD0PXXq1IlCEAULFsTt27cBSKGJlJQUzJgxAwDQpUsXvVYXFCtWjFaKDRo0QOXKlTFixAgAHxNzAMDe3l5v51SH3q+SAgUKaPStq3LmzBm8f/8+2/uWlpZU7iA3zs7OsLOzQ6dOnQBkd9fNnTtXY1aZ3Hz48AGAdDHyDGAOdy82atRIlnPzVH6eBaiOqlWrKsWcDh06RPurmxy5C3nr1q1o06ZNnuypUaMGChcujNmzZwPQ3t0UERGBP//8M5vdAu1IT0/Htm3bAAAzZsxATEwMXXPGui+0YfDgwQCAffv2wdraGgCwdOlSdOjQwZhmEVknxIiICIoxenh4aD1+6oOBAwcqlat9//33AEBx5oyMDABSLHHZsmUAgAEDBmD06NF6CTdpguemdOvWDQDIzZo1Z0WOXIusCLeqQCAQCASqMMZyesnGgQMHWNu2bZmJiUm2V2xsrJynVmLPnj3sxx9/ZAqFgikUCgYpaYFeq1evNpgtWTl//jwbPHgwGzx4MNlSqFAhVqhQIbZx40aWnJzMkpOTjWJbQkICS0hIYI8fP2ZNmzalV82aNel7VH2VLFmS7d+/n+3fv98gNj5+/Jg9fvyY2draKtnh5OTE4uLiWFxcXH4On9t986m+8kyPHj3ou2vYsCHbvXs3y8jIYBkZGbocThaio6PZsWPH2LFjx9ioUaPYV199pXQPX7p0iV26dMnYZjJPT0/m6enJmjdvzgAwS0tLZmlpyQICAoxtWjaePHnCfH19mbOzM3N2dmZmZmZszZo1bM2aNSwhIUHWc1+7do3mgsDAQLZ//366Bk1MTFiHDh1Yhw4dWGpqqr5OqfZ+Majzfe/evbQ0f/ToEbkNOVy6rWDBgrLaERISAicnJ7KDxyfU4eDgIKstqvDsq06dOlFsh8NdCpaWlhR7MzSJiYkUj+QZdblhamqKbdu2oWfPnnKaBkAqSD537hzFKF6/fq20feHChUI2TAu4CtLx48fpvS1bthhVnQn46K7/888/4eXlBQC4cOECKaqoo3///gCkeP+cOXPQtm1b+Q39f3gW6pQpU8gNbWlpCQ8PD8rwNDYhISEAJEUzX19fAMCGDRuU9qlUqRKGDBkCQIoJyolqKaCq0ATPSJZb2lPvk+PTp08pAH7u3DmlbZcuXcpW58jTwpcvX46uXbsC+OjvlosHDx7g6dOnAJDjxAhIvnZda/J0gU84qhMj8DE5qFu3bpSw1L17d/To0UNtUoIcxMTEaD0pclavXm2QiREA5s+fj59//lnj9nHjxqF48eL0Nx/sXV1dldLE/+vw66t06dI0IbVp0wa9e/fG9OnTAUCpJMkQ3Lt3D+3btwegORlowIABVHIASHWDWSUag4KC6AFdNdlM3wQGBlLNpaWlJSWATZ482aBxxZx49uwZLUqyJh/WqFED06ZNo2S2devW4ZtvvgEgJQfqmlioDSNGjMDp06cBSPkBFStWpNjjyZMnkZSUBEAai8zMzGSzQ8QcBQKBQCBQRZO/leUxRnHnzh12584dZmVlpTaOaGJiQj7jrC8HBwfm4OCgL9+x1qxdu5atXbuWFSlSJFucMevL2dnZoHb99ddf7K+//mJdunRh5cqVY+XKlcvRPgDMxMSEubm5MTc3NxYVFSWrfW/fvmXNmjVjzZo10xhfVH1VqFCB3bhxQ1a7OE5OTlrblfXVu3dvlp6eztLT07U5jbFjhwaLOb5//545OjoyR0dHVqFCBfo9K1SowMaOHctCQkJYSEiILofOM97e3nTNV69enXXp0oV16dKF7dy5k6WlpbG0tLQcP+/n58esra3pGDNmzJDVXh5bBMA8PT1lPZeuDBgwgGy0tLRk69atY+vWrcu236FDhyjnoUePHgaPOYeHh7Pw8HClucPGxoYNHjyYBQQEsICAgPzkEai9X/SmkMOVZRwcHLKpLGSdiDXJx504cYLcqobkjz/+QExMDP2dnp6O8ePHAwBiY2Ph7OxstJR1XjMYHR2NqKgo/P777wCAbdu2QdPv1qZNG5w/fx4AlNSI9Mk///wDAAgKClJ6f9CgQdniyJypU6dixYoVstiTleDgYI01lq9fv8aePXsozf/Zs2fkIkxNTSWxaj8/v9xq4f6TCjnPnz/HjBkzSNIsLS2NXNQzZsxA7969ZXW3ZWZm4sWLFwCkOLYuseMnT57Q72xvb5/nEEFecHNzw+rVqwFIblVnZ2cAQK9evaie0NgcOXKEwmDLly8nyUp1tG7dGoAU742JiVHqeCI33OV76tQpUu0KDAxUmlPc3d1JnY2XjWmJUMgRCAQCgUAb9K6t+uzZM3oS6dy5Mym5qLJt2zalRBcfHx+jrBxVYYxhwYIFAKR+hdbW1pRY9KkUj+/du5eK769evZptO28zxdtkGYrU1FQSIOb6lZwCBQrgwoULAOTXy1WF63/Onj0bu3fvJoUfQOqpCUh6jnwlfO/ePbVKO1n4T64cOXxl/vPPP+OPP/4AIIlntG7dmrwuhkrA0gUuYvD8+XMSrpZLdERdtiogrR55gs6nsorMibS0NHTu3BmA9FtfvXpVVhGAnOAJOefOncPJkydJIAAA9eqcP39+Xg6p/n7W5G9lMtc5xsTEKPmPT548KefptCYlJUUpnmdjY8MiIiJYRESEsU1TgsdYWrdunS0GOXz4cDZ8+HCj2HXx4kV28eJFZmZmli2uZ4x6s8DAQGZnZ8fs7OxyrCdbvXo12VmxYsXcDmvs2KHBYo65kZiYyBITE1nv3r2ZQqGgOLm6uJVcBAQEsFmzZrFZs2ZptX/Dhg1Zw4YNGQCWlJTEkpKSZLZQgsfNJk+erFTnGB4ebpDz55fJkyeT7efOnTO2OYwx6TvNOsbY2toyW1vbvB7G+HWOWeF1VJ8a/MmDM3ToUH11m9YrPCbWuHHjbJ1NjNlMmK/KVGuQRo0ahWbNmhncnpUrV9KTZk7xlGbNmpHNr169MohtnwM8trN3714wxqih9fLly/Hdd98BAGrVqiXLuXm50+7du3Va+TVs2FD2muqs8PKNVatWoXnz5pg6dSoASQqRN0XWJ7wT0M2bN2m1bG5urlOsNiMjA+/evQMg1Tny1nHG5uXLl0p5LPq8d0XMUSAQCAQCFfK1cuQF9GfOnKGnxNwK+Ldv3w4ASj0f5eLt27ek6tCnTx/88MMPGvflChuqsTJDxU5evnyJLVu20OomN2FtLgjMlfI5BQsWpGJdQ/Pw4UP6jt+8eaO0rXjx4rIrWqjj7du31CS6X79+mDlzJlq1akXbeRzo2LFjGjNtBblTsGBB7N69G8nJyQCkYm0euz948KAs5+S5DX5+ftnE5TXh7e1NPUi7d+9utA4dvXv3pnjnlClTqAGzPlVz+L34+PFjxMXFAZAaFZcvXx4AUKdOHXTo0IHGvKz37NixY1G+fHkEBwcDkPqpPnz4EIAUzzO0cpg6wsLClDpAAVLfSX2h85Vx6dIlLFmyBADg6+tLijPqlB/4cvzUqVOYMmUKgI8d3blbRg5VnIkTJ5L8VVhYGDVErVSpEin1BwUFISwsjMoMYmNj6fNTp07NJl2kb7gboHPnzrhz545SWYkmoqKi6GbiSS6cWrVq6dzKhR+LJ9VwfvvtNwCg1mM8hTstLY1clh4eHjh9+rTGMh5j0bBhQxo4fX198ddff1F7K+Bjq7SsikT8AU6QN4oUKULdJU6ePJmt1EffcPm4Bg0aaNW4+sWLF9i+fTtN4Fm7URgDPhFOmTIFV65c0fvxuarMokWLaCETGRlJ1/yNGzeU2nllZcKECQA+loOZmJhQJ52ffvpJ77bqgru7O169eqWxPDC/CLeqQCAQCAQq6FzK0bBhQyr8B4AxY8YAgNpg79mzZwFIq7Sss3ybNm3oc7yPmD4JDAyklWrWgLeVlRXq1KkDQFoBc5cDIIl7c9fm9evXZRfZ5W4AXozMXYA1a9ZUWk0nJyfT6nbVqlVKNgMfv3cfHx8q1s0rR48eBZC7K5kXUUdHR5OLKidcXV3h5OSE7t2762RXfsipvEQVLla+YcOG3ITd/9OlHHQwxpCQkADg4/XH/27atCnpn16/fl0WcQCeZHLv3j0aP6ZNm4YmTZrQPsnJyXRvLVu2DKGhodQH8OjRo0YT8Ac+jkktWrSgsg7uEdInt27dIm/I33//TSGkFy9eoGDBgkrJNdzz16hRI9y5c4dE2p2dncnbZsyEv7dv36Jjx44ApLGSZREBKFq0KLl+K1asmJfDqr2f9TY5agNjjPzdDg4OWLt2rcY6SH3BJ0dra2uaiHMiq9CyIdiyZQuAjwMzp1GjRkqiujExMTRxqlKiRAl4e3sDAMV+dUHbyVEbzMzMaOC5dOlSjpmicsNjiQkJCdi4cSOio6Oz7fPNN99QnFcLN42YHCG51Rs0aABAcrm3a9eOxNs9PDxw69YtANKEJYc6Eq+xnDlzJp2rUKFCsLKyIgWptLQ0CvmULVsW9vb2JDxuzEE+IiKCHjIB4K+//gKgPiwlF6GhoShcuLDSgzbPLDZkFm9uvHjxArt37wYghXh4rBaQfm/+sDNlyhQ0b95cl1MIhRyBQCAQCLRB55XjzZs3sW7dOgDArl27NB7A2tqakm7s7e0xfPhwADBYiyVOamqqkiIPX4UdOHAAAGiVdv78eTRu3Nhgdj158gQAMGvWLLJFG/iT3aRJk/D999/rJUOVfyfr16/H/v37ASi3scmNQYMGUQDfw8ND1nYyRkasHCF5gvi9P3PmTLx69YpWbAqFgv49fvx4Wdu+vX79Gn5+fgCA06dP49GjR7h06RIAKXmMu1yXLl2qVeKO3AQGBqJPnz6knvMp9Xb81Dh//jwmT55MrbOyenVq1KiBKVOmUH/ZfKBftyrwceDcuXMnZs+eDUDKTOXptR07doSjoyNlOQo0k5qaCm9vb8oYrVGjhlKj2axuyXbt2lEvPR530Te8J2JOEnTz58+nXnCAlB1nzBiOARGTowovX77Ezp07SRw/KCiIXPxeXl7Ut/Vzh7vmIyIi8O2335Kb7/nz51Q2dOXKlX+dfJyh4QLzLVu2RHh4OAncjxgxAn369AEAVK9eXV/i5/qfHAWC/yhichSoRZOWKgCaKFetWiUmxE8LEXMUCAQCgUAbxMpRIMg7YuUoEHw+iJWjQCAQCATaICZHgUAgEAhUEJOjQCAQCAQqiMlRIBAIBAIVxOQoEAgEAoEKYnIUCAQCgUAFMTkKBILPDl9fX/j6+kKhUMDMzAxmZmaydLwQfL6IOsdPDN7y5969e6SwYWpqips3b1KrmVGjRmHgwIEAPjYjFRiU/2ydY3p6OnWHDwgIwOLFiwFIeryOjo75Pbze4G3yOnXqpPR+2bJl4erqCgCydAoR/CsRdY4CgUAgEGhDAWMbYGwuXryIoKAguLu7AwBiY2OVtjPGqH9YpUqVZLPj0aNHmDNnDvWoi4mJoV6XBQsWREJCAgoXLgwAGDx4MPV9y0//xrzi6emJ06dPUyeGQYMG4ciRIwAAW1tbLFy4EPb29gazR2AYuHfp8OHDGDlyJGJiYgBIXovMzEwAQFhYmNHsywvR0dHknTEkSUlJ1Cf24sWLuHHjBv755x8AUmNe3gic9yYUKHPmzBkAkog9//0ePnyI6Oho6tcJAE5OTgCk5gzjxo3L1znz5VYdMGAAAKBq1apkVG5UqVIFAFCuXDmt9peD7du3Y8GCBQCkyTAuLk5jg1vGGCwsLAAAX3zxBQYNGgQAcHFx0Wuz1M6dO8PExIS6bZcpU4bEiW1sbBAbG4svv/wSANC9e3fq0sGbHMvJqVOnAEg3bk7fU6lSpWiy5B3E9QXvAOPr6wsA8PPzw+rVqzXu7+LiQnbw61SPDVz/M27VuLg4ajMXFhaGvn37ok6dOgCkZrm8a8tff/2la6NZWXj//j0A6f5QbWk0atQoAMCvv/4qqw2pqak4efIkAKnLzdWrVwFAqXs9h9/b586dM+j3+ObNG4SHhwMAHjx4QG7y0NBQ2Nvb07g+adIkg9kEABkZGQCAJUuWwMPDgxoyf/PNN2STqakpqlWrpvR9zZgxAwBw4cIFcv9rgeZBLYdXjtja2jJbW1umUCiYiYkJMzExUftvhUJB/7aysmJWVlbM1taWBQcH53aKPBEZGclSU1NZamqq2u3nzp1j586dY6ampmSjqs2qL03bVq9erVfbnz17pvW+DRs2ZEWKFGFFihRhsbGxerVDlcjISNaoUSPWqFEjrb6npk2bsqZNm7K4uDi92gGAriN+rpIlS9KrePHirHjx4kr78Je1tTWztrZmjx8/1ps5/9JXntm0aRNr06YNa9OmDYuJiWGMMebr68t8fX1ZtWrV2LRp09i0adN0ObSsJCYmssTERDZq1Khs18Po0aPZ6NGj9X7O6OhoNmPGDDZjxgx26NAh1rhxY7X3SrFixVifPn1Yy5YtWcuWLZXGyMOHD+vdLk1cvHiRxm/VsbpOnTpK97uLi4vB7Prw4QNbunQpW7p0KQPAxo4dy548ecKePHnC0tLSNH7u6tWrNCe1bNkyL6dUe7+ImKNAIBAIBCroJVt106ZNCA0NBQDqwK0J7i8ODQ1Fo0aNEBQUpJWh2rBlyxby3atrsMxjh3v27MH8+fMBAEWLFkVsbCxKlSoFALCysiI33PHjx+Hv76/kAuFuzxMnTqB69ep6s10buFvGzs4OPXv2BAAcPHhQo6tTH1y+fBlbt24FANy/f59cahxbW1sAwIQJE5TsuHXrFurVq6c3O8qUKYPChQtj+vTpAIBixYopuct4PGfv3r0AgGfPngEANmzYgLS0NABSPMLLy0sf5vxn3Kqq+Pr64vvvvwcAWFpa4u+//wYg3UefAh8+fABjDBMmTAAgjQlZKVy4MHx8fAAA7du31+u5AwMDYWdnR38zFffpiBEjAEhNwRs3boxZs2YBAHbv3o1u3boBkOL63MUqF4sWLQIAzJkzB+bm5ujfvz8AYObMmShbtizt9+bNGzRr1gyAlOV7/fp1We3iPHr0CLVr1wYgub6HDBmS4xj36tUrAICjoyPu3LkDQIqP5yF+q/bgeknIGTlypNb78smwWbNm2Qba/MJjI5qoXLkyAMDV1RXm5uYApMnu5s2baNWqFQBg3bp1mDJlitrPW1lZ4cSJEwBg8IkxMTGRUtDLlCmDDRs2AICsEyMgTcT8hk9MTESxYsWUthvqhuGTnybKlCkDAJg4cSI+fPiAGzduAAAOHTpEXcVLlCghr5GfKTdv3gQgxep37dpF3ddXrVr1yUyK586dAyDF9jIzM+nvrBQoUACnT59G69atZbGhUKFCKFSoEAAp3li5cmUak+bMmUP77d27l8YiALCwsMC8efMAQPaJ8cGDB1i+fDkAoHXr1li1ahUaN26sdt9Lly4hOjoaAJQmTbnZt28f5RMMHTo0x31v3LhB+zx58gTbt28HoJ/EJuFWFQgEAoFABYOXcnA3nEKhQI8ePQx9egBS6jR/2rh48SIyMzOpJEJdKQd3h9SuXdvgK0ZASj/v3bs3Hj16BAA4f/68QZ/kOKqrRgBwcHAA8DHdn/+m+nSp5oXU1FTMmDEDa9asoffq168PAPDw8DCKTf92/Pz8AEiF/gAoi7p48eKIjIwEIG+ZkzqOHDlC5/b29kZAQAAASaRAlb59+wIAdu7cSSs7OWjSpAllp7579w7Ozs5K27mL18PDAwqFgu4RX19f8mTJzZo1a5CYmAgA+O2331CrVi2l7Xzb0qVLsXjxYgrf6CkcoRVz587NcTsvH/Ly8sKYMWNo5Xv8+HEqcdMHBp0cFy1aRG7A2bNn0xdvaN6+fYvff/8dADBt2jSlUo6sbsqGDRti6NCh5Db+4osvDGbjq1evsHv3bgDAgQMHcOvWLapz9Pb2xu3btwEAP/zwA0qXLm0wu5KTk6mmbffu3YiPjwcgfW8KhQKdO3em/eR2EaljyZIlShMjIA1UAODm5oYpU6bQZCnQDv7AY2pqiqioKOzfvx8AYG9vT4NRv379MHPmTHK5ysXKlSsBSAMoL+/JiQEDBuC3334DAFknRk67du3Uvh8REUExxujoaDg7O2Pbtm0ADOvur1WrFo1xfGJ88+YNAGlc4fdOaGgo9uzZY7QxWhOxsbE0Hnt6emL+/PnkktY3wq0qEAgEAoEqmmo8mI51UeqYNWsWmzVrFlMoFFSH8ubNG30dPs/069dPYy2jo6Mju3LlCrty5Qp78eKFwW3z9/dn/v7+zMrKikHKLszxVb9+fYPaN2fOHK3qQY8fP25QuzgTJ05kFSpUYPXq1WP16tXLVvtYtWpV5u7uztzd3TXWw2qBsesVDVbnqI6kpCSWlJTEli5dyjp16sQ6derEALAGDRqwFy9eyHrf1KpVi9WqVUttPau617Jly6gu88mTJ+zhw4ey2aaOzZs3s82bN7Ny5crRvTFv3jyWlJRkUDs4r1+/plpz/reNjQ2zsbFhCoWCde7cmXXu3Nmo47Mqr1+/pnvW1NSUVatWjVWrVo35+PiwzMxMfZ6cZvwAACAASURBVJxC7f0i+83k7u6uVGT65s0bo3/xXbp00TiwDxs2jEVGRrLIyEij2Hb9+nV2/fp15uTkxObOncvmzp3Ldu7cyd6/f6/02rBhA9uwYQMrVKgQW7hwoUFsu3jxIitdurTSd+fn58f8/PxY3bp1ld63t7enQdTQPHjwgP79559/shUrVrAVK1Ywa2trpYGzc+fO7N27d+zdu3d5PYWxJzmjTo5Z4b/x9OnTWcGCBZmrqytzdXXNsVg7P7Rv3561b99e68kx66tmzZqsWrVqdF/duHGDJSQksISEBFlsHT58OJ27WLFizMfHh/n4+MhyrrywePFitnjxYsaYJPKQdXz+FEhNTWUXL15kFy9eZOPGjWOmpqb0Wr58uRwPYGrvF1m7cqxevRpTpkwhybg9e/Z8EtqbL1++hKenJ/39559/4tixY/R3yZIlAQBHjx6VLe1bX3Tv3h2XL18muSw5OXHiBGbMmEFB++PHj1M90pEjR9CvXz+l/Xn6Oq8pNTavXr3Cnj17KEaRkpJCdZMjR46ElZWVtof6z9Y55sTQoUMplX7Pnj0k4adPeBJO+/btqbY6P3ApOQ8PD73HyE1MTCi+t3TpUpLa+9TgccWjR49SYt3p06ezdTSRk+TkZIp3Xr16VWk8BkA60yNHjqRkREdHR1hZWekjZiu6cggEAoFAoA2yrhxHjRqFLVu20MqRi8ICUqZU2bJls6USG4uFCxcCADZu3EiKC4BUVMxLOYoXL24U23Li6NGjGDx4sEFWjrmxdetW+q4AYMyYMQA+lgB8KmzcuBEAMHnyZMp47NWrF5YuXQoAqFatWm6HECtHNXh6epK61Nq1azF69GjZzuXl5YW7d+8CkO7dAgWkxPsvvvgCHz58oD6nY8aMwbVr1wBIqk2aMlyHDx+OTZs26dXGrCvHMmXKoE+fPgCAVq1a4dtvv1USAjAmSUlJAKTMXt7IwNzcHOPGjcPs2bMNYoO3tzeGDBkCQFI309TI4s6dO6SCExERASsrK3Tp0gUA4OzsTJ6+PFYWqL2fZZ0c//zzTxw9epQk5UJCQkg+TqFQKMkrHTly5JNIG75+/TrJK504cQKMMURERAAwfC2XNhw9ehQuLi7UtsUY9Y+c27dvq1Xb8PHxQdeuXY1gUc5cu3aNZLvevn1Lg9e+fftyayItJkcNcNf67du3SQ1LrpIeXvc7adIkTJ06FYCk+rJy5UqqG+RddACpBtLd3Z3s4t1mAKmG+d69e3q1b+/evdTlImtLL8YYypYtq/S98EkoN5UvueFj3/Lly5GQkEBKNXJ35UhISKCSK76Y0kRKSgoASR3H39+fSovu379PbvIFCxbkpXZUuFUFAoFAINAGWVeOqoSEhJAgdEhICK0sAaBx48YG0+nMjdevXwOQFC8iIyMpkL5s2TJjmqWWrVu3Ytq0aZ+EW/Xx48fUWy2rFmqzZs0QGBhoLLNyZPPmzQAk4fQPHz4AkFR2cun9KFaOGuBKKs7OzuT2rFu3rizn4kopGRkZeerVyX/nQ4cOYeDAgQDkWTlmJSwsjNRzGGM4efIkqfqkpqZSIoxCoUCNGjVoJeno6GgUTeAlS5Zg9uzZtPry8/P7ZEJgqkRFRQGQGmCsWLECANC7d28SWdBCf9rwbtXcOH36NLnbmjRp8slMjnxgt7OzQ2hoKPbt2wcA2bIxPwXMzc2Rlpb2SUyOAMjFwWNPgNSF4Pjx48YySSvs7OxosFq/fj3FSzXw2U+OXIGpaNGiWksmxsbGomPHjvRvfgyu7JRfjhw5QiGaEydO0MQ2YMAAbeLExMyZMwEAO3bsoIHV0tISFy9eBIC8ZC3niwsXLgCQFJxWrVoFQIr/8YcKAGjZsiU1ZZbrIUMTs2fPxpIlSwB8WuNzTkyePBmAJJPHmw80atQot4/J15VDV44ePUqzOr9gPwUOHDgA4GOcgA/sckyOXAty+vTpWLZsmVZPwBkZGRg/fjwASfopNy1CQ8Jjn1nherqfMs7OzjQ58gHzv0pYWBi1KpozZ45WSRmxsbFwcHCg5JcVK1bobVLkxMfHY926dfT3ggULAAC7du1C+fLl6f2ZM2dSORa3DQAlXPGYY3p6Or7++msA0j2lTjtYTrJKzXEd1vj4eHh7e2Ps2LEApJZxvXr1on/z7jOGYPjw4dSuLiQkBCEhIQAAGxsbg9lgTETMUSAQCAQCVTSpAzCZFDU4mzZtYgBIDkoO9uzZQ+oPANjz58/Z8+fPlfbx9/dnHh4ebNWqVWzVqlXMyspKSVWjcePGLCYmhsXExMhiI1eYAcC6dOnCQkNDWWhoqNp9Hz16xB49esTatWunJB9nKMWhgwcPsuHDh2vc/vDhQ5KmUigUrHjx4qx48eIGsS2/rF69mn7zHj165La7sZVuZFXIGTZsGCtbtiwrW7Zsrko3XGqxcePGDADr2bMn69mzJ8vIyND2dFqzfft2nZRxNL0aNGjA7t69y+7evcv8/f31bm9+4CowWdW7AgMDDW4Hl/sEwEaNGsVGjRol+znj4uJYs2bNWLNmzfL0uZiYGNa8eXPWvHlzVqFCBRYVFcWioqK0+aja+8XgblWuAL9lyxYoFIpsbV30RVBQEEaPHq0UjOWNikuXLo09e/YAkMpNsnblAJQDuBMnTpS10wB3OVpaWuKPP/7AgwcPAEguIF6WcfnyZYSHh+PIkSMAJNdLw4YNAUhxW0OVbwQFBcHLy4saiXbo0IG6cjx69AgDBw5EeHg4AKmui9dM/duoWrWqsU0wKi9fvkRaWhoAySWpyZV37do1qjF7//49Fi9ejGnTpgFAbqUwOmFnZ0cKPMuXL8+zQk7x4sWRkJBAaiuvXr2i5BxDKGHxmGZ8fHyuzXi565cxRu5pQ3QVycqbN2+o2bFCoTCYO9XT0zNPMWQ+Bk2aNIni3Dt37sx3GzDhVhUIBAKBQAWDrxx5L66goCD079+flGn0TY0aNdC3b1960gSQWwYiAElXtWXLlgCkZIRvvvlGFvs4XHXn5MmT6N+/P2Wq5ZT80759e0pZzpqIYAhiYmJIveLbb7+lzN7Q0FClFffYsWNhZ2dnUNuywpOobG1tYWFhkev+/Ekd0NyT77/CyJEjqeygQ4cOlPBlYWGB0NBQ8mCcOnWKVF78/f1Rr149bdLmdaZ69eqUORsVFUXNqydMmKBVUtqCBQuQkZFB+qy8wN1Q8OzJOXPmYNiwYSQ6kRWu+8wFChQKBa0y1QlsyMnatWup9K5Ro0aYOHGiQc6bkJBA542MjISFhYXa6yozMxMBAQGk4XzlyhWaT3r37p1vOwxayjFgwAAqi/j+++9x+PBhfR4+Gy9evECdOnUASO6hrF8wl5uysLBAZmYm3VzVqlVD27ZtZbVLEw8ePMDBgwcBAL/88gspaDRp0gR9+vShm6REiRIGbbzM2bFjB4YNG6Z2G/t/tSMuzebi4mKUZscA8Pz5c/oNixcvrqSSogoXO46IiKBas/96neOHDx9I+i3rwyVXteKZoL1798bPP/8MALI3Of4cePnyJQCgbdu2ePjwodI2fu3xMYrf6506daJJ1BCZqrzuNyQkBGvWrFFSMNMk6aZv7t27R9nSycnJ6NOnD93Px44dI7m79PR0/PXXXzTO7NmzB99//70upxQKOQKBQCAQaIPsLasuX75MygpLliwhjb6ZM2caJJGEB8HXrl1LbVAmTpxIK8qhQ4fKbsPnQnx8PFq0aIHg4GB6r2nTpgCkhIYuXbqgRYsWAAyfPKBKcnIyAKBjx47466+/ct2/adOmmDVrFgCpDVgu7sHPeuUokJeYmBh4enrin3/+AaDs0udeIl64bkgP0aJFi8hFqVAoULRoUao/N3QdOk/sO3HiBAICAiiRMywsjMZuMzMztG7dmlbZFStW1PV0hlfICQoKQrdu3Wg57uTkRAoaAoEhSE9Ph+o1zlX9jx8/Tg9rJUuWzMtAJCZHwWeHt7c3uSW///57/PDDDwZzpRqZT08+TiD4lyImR4Hg80HEHAUCgUAg0AYxOQoEAoFAoIKYHAUCgUAgUEFMjgKBQCAQqCAmR4FAIBAIVBCTo0AgEAgEKojJUSAQCAQCFcTkKBAIBAKBCgbvyvEpcuzYMRKg9vf3zyYd5ufnB8AwPd8E+sXDw4N69q1ZswampqYYPHhwtv06duyIJk2aGNo8gR7YsmULdWN4/vw5AGDIkCEAgLp162Ly5MlGs00Tr1+/pnElKCgIT58+BQDqeJJViJz3cwwKCkLt2rXzdd74+HjMnj0bgNTcwNzcnDolubi4oEaNGvk6vr45duwYpk+fTo0ifv75Z3Tu3Nkg55ZFISc2NhaApJLu7u5OuniMMXTr1g0A4OjoiOHDh+ty+HwTHBxMLVGCg4OxYMECJCYmko2qkyPvQtCrVy9Srf8vce/ePQDA8OHDMXbsWPTs2RMAULRoUWOapZHAwEC6gZKSkpCZmZnrZ0qWLIlz584BgDaTpFDIMTLx8fHo378/AODMmTM0mdjb26Nw4cI4f/48ACAtLQ1Hjx4FADg4OBjcTt40Ojg4GF5eXgCAZ8+ewcvLi7pLZB1vatSogSJFilAHjzdv3tD/5+7du/Ntz8uXL1GpUiW125ydnak7Ud26dfN9Ll1JTU2lrjAHDx4knWQA6NmzJ32PekQo5AgEAoFAoA16d6v+888/6NKlCwDg8ePHAJSfjP744w8AwO3bt9GkSRODNvDkTU7/97//kRtDm+asfCX8zz//0AqzWLFiOtsRGhoKALh+/Trq1asHQFKhj42NpY4XS5cu1enYmzdvRocOHQAAVlZWOtuYFd6t4sqVK7hy5Qq5Jbt160bNmjt16gRbW1vqwGJIYmNjqeOKj48Pzp49i/j4eADa/b4AEBcXl5eVo8DI9OrVC76+vgCAr776inp28pXPoUOHAAB9+/ale8qQK8fAwED4+fnh7NmzAIA///wz2z68CUPXrl3pvmnevDk2bdqEDRs2AJB6t3JxfH1QsmRJtG/fHgBw8+ZNAFJzYUBy6V65cgUAMHXqVIwePTq3vqaysGTJEuzYsQOANIa5u7tj/PjxAKS+q3w1LrdtenOr8sbFP/74I7UbKVu2LPr370+TZVJSEsaNGwdAakQ8YMAA7Ny5UyfDdYE3GLWxsSE3TN26dTFixAi1+0dHR8Pd3V3J/887dHfq1EknGzIzM6mD+fTp02FtbQ0AePXqFV2k+YW7fjU1Js4rU6ZMAQBs2LABqampGvcrUKAAuWMaNGhAk6jcsdoHDx5kcwPx38za2poeQACpmzy3a8KECTQhAtJNCUi/Sy78J92qycnJ2TqccAoVKkRxIUNQv359+s1XrlyZzVV448YNAICtrS1NBnwylYvLly9T8+dz584pfV/29vb46quvAAC1atVC165dla7LFy9eAAAGDRqES5cu0bbZs2fD0dFRVrtPnz4NQBq7eQgFkCb4b775RtZzq6N27dp49OgRACmePHDgQPqt79+/T4sVHu7SA8KtKhAIBAKBNujtUY8nuDx79gxlypQBAHh6eqJNmzZK+/3+++8AgL179+rr1FpTvXp1AB9dLgCof5k6Hj58CHd3d73aEBsbq7Qy4Q1PtYG7MBs0aJBjA1++EtbXypGvdB0dHbFz5060bNmStvGVtK+vLxISEnDr1i0AwK1bt3Dy5EkAkhu5dOnSerFFHcWKFaPVAWfRokUAAHNzc1StWlVpG3/yzBroFyjDm4Rz9zQAjB49mkITqhw6dAjOzs4GsQ2QenFqChsEBAQoNdt2cXGR1RYeapk8eTKtWOvUqYMuXbrAzc0NAGBqaooiRYqo/Zy7uzu2bNkCAKhUqRL27NmDrl27Ashf+EZbePLa+vXrlVaOhobfsw8ePMDGjRsBAAMHDsSLFy8QHR0NAChSpIjWoZL8orfJkWcXJSQkYMKECQCQbUB8/fq1UqaRra2tvk6fJ3KaEAFQdu3EiROV3m/dujXs7e3zde4SJUpg/vz5AIBdu3YhJiYGAPD+/XsAH7+TX375hT7z66+/YsyYMeRjr1ixIrmuASnLjLtl5KRVq1Zo1aqV0ntDhw4FIMXsMjIy0K5dOwDS5Mgv6Nu3b6Nt27ay2VWlShWcOXNGq31v3LiBMWPGAJBivpwCBQrkp5P4v54PHz7gp59+or/37dsH4OO98KlhZWVFtm3duhUpKSkAgJiYGGzatAklSpQAAIwdOxa9evWS1RY+gWW9nnIiOTkZ58+fR/fu3QFID+2bNm0CAIM+YHD4d8cna2PBS1u6deum9EDz/PlzREVFAZCyVflvKzuMsZxeemX79u1MoVAwhULBOnXqpO/D6w1HR0fm6OjITExMmImJCStVqhQrVaoUO336tN7P5efnx/z8/NjcuXNZ8+bN2bp169i6devydAwrKyv6XrO+DMmzZ8/YpEmTWMGCBVnBggUZAFaoUCFWqFAhduPGDYPaoolHjx6xfv360e9qYmLCzM3Nmbm5OfPx8cnLoXK7bz7VlxJv3rxhb968YdHR0axPnz5qryFtX4cPH87L96cXnJycmJOTk1p7+D37+PFjg9uljqSkJHbs2DF27NgxZm9vz0xMTNjEiRPZxIkTjWZjREQEu3v3LpswYQKbMGFCtu/wypUrBrUnJSWFpaSksLS0NHovOjqaNW7cmMYVPz8/OU6t9n4RMUeBQCAQCFSQPb0sLCwMly5dAiAVkfPC8d69e8t9ap2YOHFitrTrgQMHAtA9QzUneEy2TZs2cHR01KoUIioqimI/R44coYJhjr5KOHIjMTGRyjy2b9+uFJ+qUKECDhw4AABo1KiRQexRR0JCArlrBg4ciLi4ONpWq1YtLFiwAIBU3vNfo1y5cgByLnepUqUKmjdvDkAqw8r6G2clMDAQJUuWpPIEQ8AzVG1sbBAREQHgo2uQ29m4cWMKBfTs2ROVKlXC27dvAQB9+vSR3cawsDAAUkbt9u3bAQAWFhb4/fffZc9C5WRkZOD27dsApBwBHhe9ceOGUnhGlZ9++gnffvstAMndy9V5VGOn+oIrAaWlpeHChQsAgB07duDGjRsoVaoUAKm6gIt6mJjIu7aTRSGHX5hTp06Fp6cn/c0Yo/TblStXom/fvobzH+dATEwMnJycAKiXj+M1NmvXrpXVjosXL1LMUTUQf/XqVZKW+vvvvylhQh08XtSvXz+ZLJXo1asX2QRINw2PmcybN4/KVAxJSkoKPnz4QJP2vXv3lB52ChYsSBJZx44do/T6PPJZlHLw61z1eu/duzfFiL/++mtKdqpZsyaVQ6mjbNmy+PXXXwEYPnbGE0liYmLw8OFDeiD38fGhyZDDk3V69epFiTByDPj37t3Dd999B0Aq4Ro7diwAKU5vaWmp9/OpwicRd3d3egjMD/xBvmXLlhgyZIiu906uLFy4EPPmzdO4nSsGOTk5oXnz5hoVf/KAKOUQCAQCgUAbZFk5btu2DQCyFdczFd3SGjVq4PLlywBA5R+Ggiv1BAcHY/369eReULUR+Lhy5OLkcnDlyhUkJSXR0yXPYuWkpKRQCYI6atasCUBym1SoUAEA8OWXX8pkrUS1atXw5MkT+vvMmTMGdaupY8KECdiwYYOScENW2rdvr3Vmaw58FitHXtL0448/Ku00Z84cykLOip2dHQICAnI8gZmZGQBJiIK7M83NzXW3OJ+kp6fTv+/cuYNr167h2rVrAICdO3eSu27Xrl1UPqEPd929e/fQoUMHEvb4448/YGdnp9Vn+eobAGVW6wJXkrG2tia3syrFihXLsWqAu2BV3emWlpa0Wte39+/AgQNYsWIFAEn9qHXr1rh//z4AKSOYh5TevHmDmjVrkpB669atdV2Rq72fZZkceVmBnZ0dihcvTgo5bm5uNCktXLgQz549w5w5cwCAyhsMwW+//UaTkOrgaW1tTTfHixcvkJCQIOvk6OPjA0CKh+U0+eVE9erVSVzZkPJtI0aMIDHk1NRU1KlTh8oBevbsaRRh8vHjx+c4OVpbW5PLlUuO6cBnMTnmlXfv3mHq1Kn0d2xsLLy9vTXuz+uKDx48aNS4syp80jhx4gRdAwkJCXQt56cukqtTzZkzB2/evMHVq1cBSAuBkJAQAICXlxd+++03mjjVxXz59Vu+fHlSsGnYsKFONoWGhlJ5XVa6d++Ob7/9Nke5RD45JiYm0qR97NgxpKSkkKzdihUrDKqOxCfHbdu2wcvLC3fv3gUgfVfr1q0DII0/eXjIMdzkqA2+vr5KrUcuXLiQTTBALiZNmkR1hAqFAubm5jRgcnk7QNI8PHPmjKyTIy985ZqQuhAWFmaU+B4gtdEBgC5duijVxHXu3Jlqt6pUqWIwew4fPozLly8rTY5copAPRvxG9vLy0jUR5z85OaoSFxdHmraAVFOoTgKxfv36lJxlDO3dnOADbd26dWmSOnHiBFq0aJHnY4WHh9PneJIcj48FBgaS1jQgraa5h8fc3BwDBgygbUFBQSRrGBwcDAsLCwCSlBxvL2VMFixYoBTDXLt2LY2RxoAvrNauXUset2XLlmkjA8kRMUeBQCAQCLTBaM2O69Wrh6pVq1Ks7++//zbYytHOzo5WFrVq1cKoUaM07ssLQuWCZ/XVqVMH+/fvJ8m1kSNHUswmLS0Nrq6u2ewCJHfmhg0bsHr1atlszAnuklmxYgUWL15McninT5+mmNbYsWNlj39yevXqlU0RhXso+vfvj9jYWIpDZWRkGMSmz5WSJUsqrXgsLCyoI0xW7ty5g1evXgH49FaOPNPR09OTvAi7du3SaeV49uzZbGVVXCZToVDQvTJ58mTY2tqS21kVFxcXEvj38vIi9+WiRYvIVWisXrgAsmWHGkKdKyf4ynH8+PE0ns6cOZO6H/ESmrxitMmxYsWKMDMzy7HORi6cnZ1zTDXn7sG3b99CoVDIquVnY2ND/+XlJOpQrQvlbXjq1q2LkydP0s2S307huuLq6opmzZpRDOfvv//GtGnTAABVq1aVXcIrJ3jMu0KFCjrHdQXZ2b17NzXsBaBRd/VThtu/ceNGpeQdXejQoQO5QL/88ku0aNGCtFUVCgXVH2uTwMJr/n744QeKhZ49ezZPWsxywcMlHGONOaqUKVOGci/c3NxoUuzZs6dO4RPhVhUIBAKBQAWjrRw/Zfgq7O+//zayJZrhnSZcXFywd+9eElO/fv06de8wNLVr16YEjfXr11PD5hEjRsDBwYGehg0Nd3WJLhz5x9/fn7IWT506pbRyzImFCxcCQL4E6F++fEkF/ar9O3UhJSWFMlO9vb2pDKVp06Y6Ha9KlSq4c+cOAOCLL77QS7/B169fU8ejOnXqUG9VY8B/Q/7/yMnJ42VoTE1NAUjlSXzluHbtWp1WjkabHP39/fHgwQNjnV4jFy9ezCYf17NnTyNZoxmupM9jOdy/buw4mjqlkZSUFFy5ciXfTY8jIiLyXMcUFRVFDw7GcOF/TkRGRsLBwUGnptw5KTppw5kzZzBgwABqiaYrUVFROH/+PABg9erVlG2tUCiwa9cuAICDg4POx+d1k/mFlyc0aNCA3hs1apTRakYXLlyIxYsXA/hYCsNb4hmjZCs3uPwcoHtTZINPjvyLPXXqFD58+EDv67tb/LFjxxAeHq4xxThrKQfwsc9kVvk4MzMzHD16NFubJl3gsa6scZlChQrpXIJx8+ZNAFDqZG8M+P/XnTt3cPXqVXq6jI+Pp+9x6dKlOv++vO3V+PHjcevWLYoVDx8+XGOJyOvXr2mFMXjwYKVWQsWLFye9yGbNmulk03+VYsWKwc7OjurucoMnbvTt2zff5168eDESEhJ0Kjjn7Y4CAgJw6NAhHDx4EIByHHDjxo1qk4m0gSekfPnll/meHLkWNfdeFS1alGqHeW22Pnn79i3dU8WLF6e6a0BKBuKlZsHBwTR2A1JCJdealkPjlJ/rzJkzqFatmlZxzQ8fPlDiopeXF8kE8t6QeUXEHAUCgUAgUEHvK8e0tDTKpCxRogSqVaumtI0r4nh4eEChUFB2o65+flX409/Vq1eRmJiI9evXq92PCyirZqIqFAqUL18egORe0ceqEVBWwuGYmZnR0+CECROoS4ImuCzf06dPsXXrVqVt3BUkZ1wvPT0dSUlJOHHiBACpI4ivry+A7I1SCxUqRNmqPBVdF7gwNC8LWbJkCQBJ+ouLOqty69Ytiotk/X3Lly+PJUuWZCuLEWiHmZkZNm/eTN+tq6srrezVwePiK1euzPe5u3btisuXL9Oqf9y4cRRL5tKJnMzMTJK5CwkJofBNVFQUKlSoQKuyH374AS1btgQAaiSuC9zrtGXLFupgMn78+GxxUe7N4K5cHr7Jmttw5coVlC5dGj169AAgxc74MfXN+/fv0b17d1y5cgWA9B1klfFMSEhQWi1y6tWrh7Nnz8rm4o2JiUG9evUAAJUrVyZ3tzpu376NU6dOAZDixtxL1LhxY4qN5zauakLvCjmvXr0id0qZMmWwZs0alC1bFoDkXssaz7OwsCDfOg+G5xd+o+SW8sz/v7/66itafgPSjcVdrvpsUcXrpviFqErlypXxxRdfKL3HLwrGGFxdXck9xOONnL59+1J6tRxdTrgykKenp0b7Aek35KoU2rbfyg0+kPzyyy84cOCA1jFV/vuampqSdJmXl5e+YkJCIQdSjW1O4wd3t2W9v3Tl1atX6NChA2lsqtNA5qhu43Fqc3NzzJ49W++tonhoYeDAgfTgaGZmhipVqigpNfF2aU+fPiU7+TbO119/jYULF+rFFZ0bYWFhVEqWGzY2NqRh+v3338v6EB4UFESLnJSUFJo/1PHmzRsaD01NTTFx4kQAwJQpU/ISaxQKOQKBQCAQaIOsK8dsB2OMlu1jx46Vpa/Znj17AEiKLcHBwZQIoinjdMCAAZT+KydcKSOrWzU/cF3G2bNnY8CAdizzCQAAIABJREFUAbL1xTx69Cg9xZYqVQrx8fGk3mFnZ0dahr169YKTk5OsCijr168ntwn/PjXBM+vc3Nz0snJRQawcjcCrV6/Ic8EbGABS/9KsJSXdu3en+8PGxoZUfHJageiDtLQ0SpTLTVx8zJgxlOVZtmxZCi8VKVJELyUg2sAYw/r162m1pUr16tXp3h81ahQqVqxoELuAj5nlfn5+SpnOT58+pfmlYMGCaNu2LbnvK1asqFF1KBcMIzweHx9PLkQee+RMnToV3bt3BwCt27d8LvDM3OjoaCXxYO7+5aUY6rC1taU4KIfXEOqj3isnhg4dim+++QaAVK94584dyrD9FFO4DYSYHAWCzwfhVhUIBAKBQBuM1rJKIMEbhvLEJHXY2dnp3f0syBdi5SgQfD58Wv0cBYJ/MWJyFAg+H4RbVSAQCAQCbRCTo0AgEAgEKojJUSAQCAQCFcTkKBAIBAKBCmJyFAgEAoFABTE5CgQCgUCggtGaHQsEgn83M2fOpI4ITZo0Qf/+/WFrawtA9wazAsGngqhz/Jfg4+NDrXcuXLiAM2fO0Lbx48eTPuLXX38tmw1btmzBkiVL0KZNGwBA/fr1qbPAiBEjULx4cdnO/Ykh6hwBJCcnU+u3AwcO4PTp00hOTgYA2NvbY/ny5QCA0qVL6/O0nw1clzhrp5iTJ0+ia9euxjIJT548AQB899139G9AahnXq1cvY5klN6LOUSAQCAQCbZBl5cg7J6xduxbXr19HWFgYAKBOnTqk8s57gxkS3jj3/v371LMRyL033Lx58wAAI0eONKgyfWRkJACp48XNmzepG4Y6eGeRChUq0AozPwQGBgIAfv31V1LFj4qKQnp6utJ+/PqxsLCAu7s7Bg8enO9z/wsQK8f/h1+Tmzdvxo8//ogdO3YAkLwZt27dAgCNXXr0xfPnz3Hp0iUAwLlz53Dq1Cm8evWKtltZWQGQBPyPHDlC74eEhGRrlGxIeMeO+vXrU4/H5s2bU6NmQ/P48WPqo5h11QhIK0dnZ2e9n5M3gV+1ahU1ss7K0KFDYW5uTp11ZOojKa983Pnz5wEAo0ePph9aXRdpPgn16NGDOmjLRWBgIHWDvnjxIjULVjfAa9M49ddff1XqqCEnkZGRdDFevXoVAKgZcteuXfHs2TMAoK7snAIFCmDjxo0AgCFDhuh07hMnTtBn3759m60pKx9snj59qrStZMmS8PLyAgC0a9dOp3Prmx07duD58+d5+sx3331HnWU0ICbH/2fhwoUAgPnz5wP4+JA2ePBgrFq1St+nU2Lnzp0AgJ9++gmvX7+m983MzFCggJROoVAoqCNObGwsihQpQqGHCxcu6NwlXp8sW7YMM2fOBAAUK1YM+/btg4ODg8HtmDdvHtzd3dVuk2ty5P/fy5Yty3G/Bg0aAABatWqFGTNmAPjYtk8PCLeqQCAQCATaoJeVo7e3N6004uLi4OjoCACYMWMGgoODaal++fJl9OnTBwBQqFChHN2EuvL27VsAklvn3Llz9Lfq6rBGjRoApEajjDHMnTsXAODv708NkocNG4bIyEj6XIsWLch9IzcuLi7Yv38//V29enX6jqdPn46XL18CkPpBjh49mnpnmpmZ0WqycuXKOp27RYsWtFrNepy5c+fC1NSUnuJu376NP//8EwCwadMmpKeno0uXLgCkBA25GjDnREhICDp27EhegrS0NORyjWejcOHCuHDhQk6rx//0ypGHSZYtW4ZDhw4BAJKSkmBmZoaffvoJADBo0KBsPUj1TceOHQFIrlTec7Rbt24YM2aMUhIQd7Feu3YNVapUQcOGDWW1S1tSUlIAAE5OTpRgZ2pqiqNHj9IYZEjGjRtHnjaOq6srAOC3336TxaXJm1QfOnQIBw8eBAD4+vqiWLFiAKRwzbNnz2j1D4C8AqdOnUL79u31YYZmt2EOL61o164dq1ChAqtQoQK7ePGixv2ePHnCIN2grFChQtoePk/4+PgwHx8fVqRIEWZiYkKvKlWqsK1bt7KtW7eyw4cPs4iICBYREZHjsdzc3JhCoaBj2NnZyWKzOsLDw5mtrS2ztbVllSpVYsePH1e7n4+PDytdujRTKBRMoVCw8uXL63zOmzdvsps3b7LKlSvT/7OTk5NWn61SpYrS97169Wqd7cgrd+/eZa6urszV1ZUVKVKErrH8vAoXLpzTKXO7bz7VV565dOkSmzx5Mps8eTKrWrUqc3FxYfb29sze3l7p+6pVqxa7du2aLqfQiQ0bNtC5y5Urx1JTU1lqaqrBzq8NDx48oPHIx8eHubi4MBcXF2Zpack2b97MPD09maenJ927CoWCWVlZGc1ed3d3JVuGDRvGkpOTWXJyskHO/+7dO/bu3Tt2+PBhtnz5crZ8+XLGGGOnTp1iQ4YMYUOGDGEKhYJ+97Zt2+rr1GrvF+FWFQgEAoFABb2IAPz666+UIdmqVSt9HFJn/ve//wGQEjGyZqxNmjRJ62Ps2rULAMhlaAwsLS3h7e1Nf6u6SLltLi4uiIuLo2Qd7rbWBe4uY4zhxo0bAD4GwnOjVatW2Ldvn87nzg+///47JXdxVxXHyclJKXBfvnx5jBs3DgDw/v17ShpJSUmhTEtAqtP7r/LixQtKsNmyZYtSOCI4OBj//PMPAOka5a7+qVOnGrTO9cCBA/TvuLg4fPfdd0rbuTuwfv36aNq0qcHs4rx+/Rrdu3fHo0ePsm1jjGVL7CtatCgAwM3NzSD2ZeXs2bMAgMWLF9N7w4YNw9q1a1GkSBGD2cHrPVUTf8qWLatWVCIkJERWe/QyOdasWVOrlGj+IwBAtWrV9HFqjfCSkbySNXMMkC5k/v+2d+9evdimLTnFDNesWQNAGhgAkI1r167V+Xzdu3cHALRt2zbPWXwODg5KMVJDMnfuXIwaNQqAlBJetmxZ9O/fHwBQpkwZSgNX5e7du2jSpAkA4NixY/R+s2bNdM70/Rzo3r07bt68SX/zAbJ8+fLYunUr7t+/D0CK9T1+/BgADC4A4eTklC2LmnP37l0MGzYMgGT70aNHAQCdOnUymH0pKSlqJ0ZN8Ozu8ePHy2USwcs0tm3bBi8vL4rXZs0B6devH7788kvZbdHEmzdvAAB79uzB7Nmzsz30AsDAgQNltcEg8nG87GDatGkU1F25cqUhTq0VYWFh9LTy8OFDpZutZs2aJJFVtWpVo9jH4eUy3bp1UwpQA4CdnV2+j89vBn3cFMePH8/Taj2/mJubA8g9JdzPz4/q7+bOnUu1ZllxdnZGv3799G/kv4R69eqR0o1qzWzHjh3pCd/Kygq3b98GADRq1Aht2rSh+9vV1RVlypQBAFnKJdzc3DSussLDw+lBfP78+ejduzcA4MiRI5QcKDeVK1fGzp07sWTJEgCSGk7WchNVeELOokWLZK8B55P2mTNnEBoaqraMLSQkBG3btpXVDk2MGzcOf/zxB4Ds9ZY2Njb0QDtlyhRZ7RAxR4FAIBAIVJBdW/Xhw4ekyXf79m2KCXJlBGPz9OlTtG3bFuHh4dm2OTo6YvXq1UZbMfJyDT8/Pxw4cICUarKudqpUqYKDBw9SerohYwQAEB8fDwDYt28fxo4dS+97eHgYdOWYE9z9cu3aNURGRqpdLRYuXJjEExwcHHLTA/3PlHIEBARQLJqXP9WtW5e2v3//HgBQokQJdO3aFdeuXZNOxBicnJwAQHaxj5w4ePAgfvjhBwDSau748eMGLeXgq7SkpCTExsYqbeOuwoEDB1J+ROHChbFlyxa4uLjIbtvbt2/x9OlTWFhYAADGjBlD4YUSJUpg0aJFlMPAPTOGoGvXrjh9+rTabaVKlVLyOtrb26N69er5PaW8Cjnq8PDwwLJlyxAdHQ1Aikts3rwZgOSizEWFRO+8ePECgCSTtG7dOgCSwowm1wIgudiWLl0KQP446T///EPuvPT0dBrE1cUumjdvDkCKG9SqVUtWuzRx7Ngxin2qJi8Zc3K8cOECgoKCAEgxC+6a4d8nT9CwsLCgAbxp06aoXbu2tqf4z0yOutKqVSuSkTx69KhB431ZyczMpAfxXr16oXbt2uRW/1S4efMm1eu9f/8eS5cuxfTp0w1ux/v37ynWzidJ3mTg4MGDBpsg379/j61btwKQFgaqE2XWWLOpqSmNmdOnT9d1ISMUcgQCgUAg0ApNBZBMx6Jhxj4W4mct2FR98YJXDw8P5uHhwdLS0nQ9nVb4+/szU1NTZmpqqlSsbmJiolTor26bjY0Ns7GxYU+fPpXVxjlz5igV4WrzGj16NEtISJDVLlXCw8NZeHi40u+rzjZ/f3/m7+9vUNsYY8zJyUnjdefh4UHFxvnA2MX8BhMB0BVesK1QKFi3bt1YRkYGy8jIMKQJ2XB1daV7ZvTo0Ua1RRU+ZgJgPXr0MJqoAb83Bg4cyEqVKkW/Ydu2bdnr16/Z69evDWpPWloaS0hIYGfOnGFnzpxh8+bN0zjmfPnll2zlypVs5cqVeT2N2vtFFrfq3bt3AUgxO8YYzMzMAEjZbVldGlygHAAmTpxIvuSCBQvqctocUSgUal2nbdq0QcOGDSkOkTU92M3NDatXr6bPBQQEkDtTDlauXEnu3qwMHz4c1tbWJJg+aNAgpe1XrlxBs2bNZLNLlYyMDADA2LFjyU2u7rvl6f2hoaH6FAnOlQULFlB89v3790rXXJs2beDp6QkgX3EU4VbNhaFDhyrVjfLUfJ7BagyePXuGBg0aUOzP2F05snLixAkAIMFx3uvRmE2jjx8/TvWiMTExJNenKR5oaBISErBw4ULKFUhOTqYx0sbGBpcvXwag1TUnn3ycLmRkZDBPT09WrFgxVqxYMQaA3b9/n92/f1+W8+3bt4/17duXXidPnmQnT55k8fHxGj+za9cupVVlt27dZLFNW9LT01l6ejr7P/bOPD7G6/vjn7EksWQR+5qovZaEKvGlhNpiCw1CS+2UWEqVUsS+xhZLqSUqpSixFi0aEcRSmtjFUntELYlIkJD7++P53WNmMonJ5JmZ0PN+veYlM8/Mc4+Z5z7nnuWe06ZNG50V0/Hjx60iz40bN0RERISIiIgQx44dE8eOHRN9+/YVffv2Jatbo9GI8PBwq8gnhLISXrNmjVizZo1wcnISAES1atVEtWrVxMqVK009rbUtQLNajk+ePMnk16H72SdPnogaNWrQ7+/o6EivW5uBAweS5REVFWVVWVJTU0VqaqrYsGGD8PT0FJ6enkKj0YiuXbuK5ORkkZycbFX5hBBi+/btYvv27cLJyUmULl1alC5dWjx69MjaYhmkU6dOOvfFyMhIERkZacxHuXwcwzAMwxhFelpTWChGERwcLIKDgwUAMXnyZDF58mRLDGs01io8boiUlBSRkpIimjZtmi0sR0MkJiaKxMRE0a5dO5LPxcXFokWp02Pbtm1kPQIQJUuWNDWOYm0L0CyW46FDh8ShQ4dE06ZNM/t9CCEUq7FBgwaiQYMGOjEhY4vXW4J9+/bR7//dd99ZVZa4uDgRFxeXJnY2c+ZMq8pliM6dO5N8q1evtrY4Brl//77w8PAQHh4eQqPRULFyIzA4XyxSIScjZMUKGZdk0kfG+mSlHEsTGRmJ4OBgamgr23xpo10jUqbQ3759GyNGjLBYu6/08Pb2RsOGDbFjxw4AShzsyJEjAJTm2/9lUlJSqHlwZppDy4bmmzdvRu/evXXKfMl4lazTqgaTJk0CoMTA5s+fn+nP165dm/6WezKtQVhYWJZKPVoKWYkrMTHRypK8naJFi9I9qXXr1ggNDQWgXCum6Bd2qzIMwzCMHla3HM2RmaoW2sWo1eLs2bOoXr16pj4TFxeHkJAQTJ06Nc2xli1bokaNGmqJlyHt2rXDnTt3qMFtRjRq1Ig6tBw6dAgi46xoi/D8+XOq6AMoVm7p0qWtKFH2YfXq1ZRdWqRIEdy9excAULJkSZ33vXr1iiyumTNn4vLlywCUSljAm6zlgIAA6n6SXuF3U/j1118BAPXq1TPp89rVetSqfJWamgpAsbhlhaqMMqFv3ryJr776ir47bYoUKWLVDFV95G+9e/duajIsvUPZEZmJDrzZDWGoaLkxWF05tm7dGoCiAGRB2exCWFiYzk09Kzf44OBgAEoVh5YtWwJQKvBXrlwZgGEXj3S7XL16FefOndM5NnLkSADKNg9LlYyT22EGDRoEQFF6srVTqVKldG5YkZGRKFu2LAAgPDw83QpEmeHJkydUdef169fUdb5y5co0cQ0hb0Jff/01QkND6WY9fPjwbHfNWQvt0mYPHjwgV2ibNm1oa9apU6fw7NkzcusLIXR+VwcHB2rFZK7WS7GxsfSv/Lto0aJGfTYmJgbff/89Pdcug5cV5ILL1dUV5cuXB6BU1mrevDlVmFm2bBmVqJwxY0aa+SCv5T59+qBcuXKqyJUV4uPjcePGDXTv3p1ek/eqrLTFMychISE6Bo1c+Mpi+JkmvWCkUCEh588//xR//PGHwWOpqanC39+fkl3y5Mkjbt68KW7evJnVYXWQ3e03bdokkpKSRFJSUobv379/v2jbtq1o27atcHR0pGIFrq6u4uTJkybLcfXqVXH16lXh5OREge0CBQqIxo0bi8aNGxu14V+mUu/cuZOScyzJ6tWrhZubW5oiCRqNRhQpUkTUq1ePHqVKlaJjaiUzffvttwY39t+7d4/eEx8fL4QQIjY2VsTGxgp/f3/RunVr0bp1awFA5M6dWzRs2FA0bNgwK6JYO7FG9YScsLAwur60rzmkU+BBHitfvrwoX768GDJkiLh//34WvlLj0J4v7u7uwt3dXezZsyfDTfMvXrwQL168EOPGjdMpXHHp0iVVZJJbMvz9/XW+Hzs7O+Hk5CScnJxE7ty503yntra2wtbWVnTr1k08fPhQPHz4UBV53obcfhUcHKzz+rNnz0RAQIAICAgQZcqU0fm/uLi4iIsXL4qLFy9aRMYDBw6IAwcOiI0bN6a73S4pKUls3LhRBAYGisDAQLpfy8eRI0fEkSNHjBmOt3IwDMMwjDGYtfC4o6MjUlNTqdCzl5cXNdTcsmULIiMjySUYFhZmliovsufX/PnzyYVWoUIFaiq6Y8cOHDx4kNwcFy5c0IlLFS9enJqlame6mcrgwYOxdOlSo94r3X9OTk7o2rUr+vfvDwBWKzQOKJX85fcxdepUcsfpdxwA3rihS5UqhcmTJ1P2oqkULFgQjx8/TvP6l19+SXGQEydOoE6dOtiyZQuAN5VZACW+PW7cOINZtpnkvayQIysJLVmyROd1OWdjY2NRpkwZer1Xr16oWbMmAHV6gBpDREQEAKU6k2zIrNFoqGFvpUqVqJi8RDbhlvFKOY8CAgJUbdJ8584deHt7A4BOs2h9hBAIDg4mF6w5q24ZQmYlz5s3D15eXvT62rVryVUNKPcf2TR81KhRFq0mJO/PS5YsgYuLC4VsZIF2QKnUs3nzZp3PSVfqkCFDMGzYMABG5bVYvitHaGgofHx8qK2NPjY2NnTh+vj4ZGWodJHNWBs3bmzwBg6kjZ3I+NVXX32F3r17w83NTTV5zp49S50stMtr6TNu3DhK3JEtv7Ijp0+fBqBctPrfb6NGjQAoCxC1bkIyWUF7AWMI7eQB2Wmgbt266NKlixpivJfK8V3ixYsXVGrx3LlzlLYvt6HI+5r2vLa1tcWQIUMwe/Zss8klG7uvW7cuTdNi2Tqrfv368PX1fVtbNLOzePFiHD16FBs2bKDX5MKhVKlS6Nq1q9Xin3JBNnDgQFLmEkO/LaAkj8mY8ldffZWZ4bgrB8MwDMMYg9mbHT9+/JjciHv37oWrq6sysEaDYcOGqeKqNIbTp0+TSa5v4Whbjq6urhg7diwAJXOMyV4cP34cgG7/yNOnT2Pr1q30vEqVKrSC7NixoznEYMsxmyHd7cHBwQgJCaHrQ6PRUKGRSZMmWdyFyWSNV69e4eXLl5S5P3nyZLIqW7RogWbNmpGVmCNHDlPd+5Z3qzLMeworR4Z5f2C3KsMwDMMYAytHhmEYhtGDlSPDMAzD6MHKkWEYhmH0YOXIMAzDMHqwcmQYhmEYPVg5MgzDMIwerBwZhmEYRg9WjgzDMMw7S6tWrdCqVSvcvn1b1fOapdmxLOV1//79NEW9f//9dwDA9u3bUaBAAUybNg2AUqpNzY7hmZXx2rVrAJRK9emxc+dOtGnTxqxyyW4Sz58/p9eWLFkCPz8/6pbg5+cHR0dHAEDbtm3NKs+7iiwAHR4ejm7dugEAxo8fj6lTpxp8f+PGjanCv7ULQjPpIzuHDB8+HBUqVAAArFy5ko5/9NFH+OSTT2ie1q9f3/Rmt1kgOTkZycnJCAwMBKCUrPzzzz8BANevX4e/vz8GDhwIwKiuEUwG/PPPPwCADRs2UCk5e3v7rJ84vUaPwsRmxzt37hT58+cX+fPnf2uzVO1Hz549RXx8PDWrNSceHh4myejk5CROnz4tTp8+rbpMKSkpYtGiRdQAFQaa+mo/pPxeXl7i2rVrqsvzrvLy5UsxceJEUalSJVGpUiVha2tLzasdHBwy/H3btWsn2rVrZ0zTWWs3LVa92bExJCYmilu3bol+/fqJfv366Xx3I0aMyOrpjWLTpk1i06ZNOo22tZtvy9fl3+PHj7eIXPrs2bNHlC9fnuapoTk8fvx4q8n3PnH+/Hlx/vx5UbRoUbFs2TKxbNmyzJ6Cmx0zDMMwjDGo7lZ9+vQpEhMTDR7TaDTo3LkzAKBcuXKYM2cOUlJSAAA//fQTcuRQdPWqVavUFkuHS5cupZFRujYGDx5MnULOnz+PxYsX4+nTpwAU18jixYvNImNAQADGjBlj9PufPXsGANizZw+6deuGo0ePqirPu8rcuXMxadIknddkQ14fHx+dPpDXr1+n3pqA4jYHgG3btnFHFi3kd9azZ09s27aNwh81atTA2bNnAcDs4QZJqVKlAGTc43TTpk3097JlyzB48GAAQJEiRcwrnBaOjo64e/euTnhEnxUrVgAA6tWrh6pVqwKATjNpc/DgwQPMmTMHAPDw4UMkJSXBxcUFgNID8YMPPjDr+OagePHiAIAvvvgCBQoUUO28Zok56iO7M/fp00enC3tUVBR+++03en7q1ClLiAPgzUSRjXBlKy3Z7FPSr18/lC1blp6bS8Y9e/akeS0sLAzAG8U9ffp0AMDRo0epRQ+jy8GDB9M91qNHD53nMTExOHbsGADg2LFjKFy4MABQh3YGSEpKQs+ePQEoi4Y2bdrA398fgNI5Xs6j+vXrW0Qe2RFe/muIxo0bY9CgQQCAR48eITk52SKyaVOvXj2EhYXRouHBgwdp3iNb6NnZ2ZldRtlwvlq1arS4sbe3R3R0NL1n7ty59L3JRtLvArKx+datW1GxYkXVzstuVYZhGIbRQ3XL0dvbG8OGDQMAXLt2DUIITJkyBQDg7u5O74uNjTW4mrIE586dQ86cOQEAxYoVy/C948aNs4RI+OWXX+Dj40Mr4vz58+N///sfgDcrI+n2O3z4MLmV7t+/r8r4KSkpiIuLAwCEhoZS9i6gNBMGlO+qRIkStDpr2LAhNRfNnz+/KnJklbFjx+LWrVv03M/PL92M3vDwcLIcAZBLqVGjRuYV8h0iOTkZFy5cAABMmDAB48aNo7kza9Yssn4smWluiKSkJADAvn378NVXX1GGvJubG2V2W5rChQvj1atXBo998cUXWLZsGQAgX758ZpflxYsXAJQs365duwIAChYsiI0bN1KG/MGDBynzt3379vj000/NLpcpSL0RExODI0eOYObMmQCAnDlzomXLluoNlF6mjlAhu80QcXFxIi4uTrRo0SJNtuDy5cvF8uXLzTGsSTJ269ZNFC1alOSzt7cXQUFBIigoyCzjxsbGipSUFJGSkmLw+IMHD8SDBw9E5cqVhZ2dnbCzsxMARL169bI8dr9+/d6aIWvo4erqKlxdXUX79u1F+/btRWBgoAgMDBSXLl3KskzmICkpSYwcOVKMHDlSODs761x/AQEBIiAgwJjTWDvr1CrZqkII8fTpU1GrVi1Rq1YtUaNGDXHv3j1x7949NU6dJRYuXCgWLlxI2aqOjo7C0dFRhISEWE2mdevWGZwzXbp0EUlJSVaTKz2+++47mgvNmze3tjhEUlKSiImJETExMWLRokWiatWqomrVqrTTQN6Dzp8/b+oQnK3KMAzDMMZgkYQcyYsXL9C+fXsAb5JNJO7u7mjVqpUlxTHIhg0bsHz5cgBpZfT29qbkBHPwtmy6NWvWAFCybdVm/fr1cHJyAgCMGDEi3fedOHGCMhQfP36MGzduAAD9u23bNgDK/0UmEHXs2NFqri1tdu3ahXnz5ukk7UiX9aJFi9C3b18rSfZuEBERgUGDBtH1t3v3bsoUtCbjxo2jLHIAyJs3L4KCggAAHTp0sJZY6Ny5M3bt2gVACZtIwsPDMWfOHHz55ZcA3iQDWpuCBQvS39lFplmzZmH9+vU4c+YMAOgUlJHIgi4ffvihqmNbRDkuXLgQAODv70/bIvTZu3evRVOttYmOjsaGDRsAKBmh+pljMlYqq11YmtevX2PSpEmYNWuW2cYICAigSiK9evUy6jMXL16kSjR79uzB06dPSYE/ePCAlE1wcHCGWaRq8+TJE7rOFi9eTBMrPDycYi+AkiUoY8oDBgywmHzvGhEREQCUeO6LFy9w5MgRAECtWrWsKRZtMfntt990tuhs2LDBYltLMiJXrlxYv349AKBu3bq0WLx79y78/f3x008/AVCqhVWpUgUAKJ5rDRITEyGEAAB88sknVpMDePPbLl++nCrgACD5tLl+/ToA3ZwWVUjP3ypUilEsWrRIpxqN9qNYsWIiPDxchIeHpxtnswQffPCBwYopjo6OYvny5eL27dvi9u3bFpfr5MmT4uTJk6JHjx4Zxv2KFi0qDhw4IA7LWZO4AAAgAElEQVQcOGBxGbVJTU2lahVOTk4kX5kyZcSzZ88sIsPjx4/FJ598km4VnDx58ogpU6aIKVOmiLVr15o6jLVjhxaLOR48eFC4u7sLd3d3kSNHDuHo6CgmTZokJk2aJMqUKaPzKF26NP3dtGlT8fTpU/H06VNThjWKCxcuiAsXLuj8viZUR7EYSUlJIikpSaxZs0Y0bdpUZw537txZdO7cWdy/f99q8tWoUYO+xw4dOojIyEiS2dJERkaKyMhIksfBwUE4ODgId3d3cf/+fXq4u7uTDomNjTX1muOYI8MwDMMYg0YYMFO1yPCgUQNoNAb9xICyCVVWs2jRokVWhzKZ9GR0cXHBzp07Ua1aNStI9WZ7RHoVh7SRsbOHDx+aPb539epVAMqWErlN4tWrV+jTpw/i4+MBACdPnsTLly8BAA4ODjh27Bi5jszJ5MmTMXHixHSPOzk5qVFAwfAFnf3J9HweMWKEThUh4I1rq3DhwrSV5+nTp4iLi9PZQnHo0CEAKhWBTkc24E3YRuLj40NuVQ8PD1U3hqvF48eP6XuVW90AoHr16vj999+tEsvdvHkzvv32WwCg7VBya1njxo115DQ3srLQ/Pnz4ezsTEXmy5Ytm6aKj4+PDwDg+PHj6N69OwBgxowZmRnO4Hw2u3IcNGgQTZLk5GRcuXLF4Pvi4+Ph4OCQ1eFMIiwsDJ999hkApLlxuri4ULKJpbGzswMAUjIyYWbVqlW4c+cOANCeUklcXJxZleOqVauoilBqaiopZQDp7unq0aMHxSLNze3bt1GnTh08fPgQgBKv1cbR0ZEmfhZu2v8Z5Xj48GF89913AICKFSvqJM15eHhQObfo6GiEhoZShZVOnTpRHN9cnDt3DoBSuk67XZEQbzoBFSxYkCoeyde7dOkCQNnP6ubmZlYZM0Lea3799Veaxy9fvkS9evWsVg4yJiYGgLLI3rx5MyU6xcXF0f5f2V0kuyDjk+7u7tRRZ+PGjZkphWdwPrNblWEYhmH0MLvlCAD37t0DoKzipRt1zZo1OH/+PL3nr7/+smr2m7RoHz16RJVpAKBo0aI4ceIEgDc1Yi2F7Pe2d+9e2NraonXr1gCUGoiyaoksWCyxhOUoe6bJQvGAYkVWqVIFly9fBgCdjN/KlStj7969VFQ5PTe7msjMXlk5BVCyjePj49GgQQMASgq4dvp6JvjPWI6Z4c8//0SzZs0AWMZylJw7d44yP1esWIGnT58avMa0LUpA8RwcOHAAgNIH0prISl2xsbHInTs3QkJCAFiuoHt6hIeHA1B2Gsi/f/nlF3Ts2NGaYhlk0KBB+OGHHwAo1dmyajlaRDka4u7du2jbti01L+3cuTN+/vlnANBx1Vma5ORk2ssoJ7f0X48ePTrL509NTUVwcDAA0IQGlMVCehX5z549i9y5c6Ny5cr0mrWUIwAqVqwdy4mOjtZ5PnbsWCqPJYsey2LGPXr0MFscKiNcXV11SsuFhITQvttMwsrRAI0bN6YQyowZMzBq1ChzDpcuCQkJVAbtypUr1NxAKke5YHr06BEppatXryJv3rxWkRfQVY7Am0Luhw8ftppM+siFRXBwMDUQz058/vnnOHnyJABlP3YmOnQYnM9W00IlS5ZEyZIlSTleuHABd+/eBQBqoWINUlNT6WZuDtavX69TSMDDwwPAm/iiIapXr57mNe1uJpbGUIKD/mvTp0+n+NTYsWMRHh6OIUOGAFAsThmbMpb9+/djxYoVFC9q3759li3QhQsXmqocGT3CwsJw8OBBartmLcUIKBbh8OHD0z3+66+/AgC6dOlCtYl3795tNWsoJSUlzf49Y5LwLI0lPD6mIGutHjp0iLyU169fz7I3gGOODMMwDKOH1SzHhQsXkr8fUFK/rWkxSv766y/8/vvv9LxUqVIU+1MDWTEDUDoZyAr5xlQHko2hr1y5gtWrV+sck24Z7TigOZg+fTplFcsmsukhY3u7d+9Gjx49KI7i5+eXactx+fLl2LJlC636nz9/ThV9TKV58+ZZ+jyjuCYBYPz48bCzs6OYY3bl0aNHWLp0KT2XrlSZdas20hs2b948zJ07V+eYdO/26tXLah2K3nUOHz5MPYLj4uIoK16NGHKmlKMsvTVz5kycOnWK9v9NmDCB9jtl9LkHDx5g7969AIAxY8bolPLK7M3SELt27aK6qIBy82vXrh0ARflkJKNMYda/gAsVKmS2LSZ58+bNVD1AuT1BP84IKDcnwHx7yiSzZ8+m5s85cuQgt2RsbCzs7Owo3nn69GlqdZWQkIDdu3fTOTL6HdLDzc2NWusASrk3ub8toxjr0aNHqaG1/I1lYlV2jJu8C8ib+uPHj+lmdOTIEQwYMACNGze2omQZM2/ePGzbto3K3wFv8ghkeENt5D7Bu3fv0vf2ww8/4NChQzRnZZxWG2uV0kwPuW3GnMhG8sWLF0eJEiXSfZ/cWrd3714dw6VmzZpUr1YN2K3KMAzDMHpkKltVVk/Qt646deqEcuXKAVCsSO3kEu2Vmv7ntImLi8uyhXb69Gna7iCzviT9+vXTsSq1OXbsGLy9vQEA//77L71euXJl7Ny5k/5vatCqVSvs2bOHnsuN/StWrEizWpJbSKSFJDe0a294BpQC0KGhoQBg9kIKo0aNwpw5c0z6rLTwtmzZkulGqleuXEGrVq10mjDLVaLMijVEx44ddaxWjUZD1oIsBG0C2TMz4e1kOVv11KlTNI+qVatG32VycnKagguW4qeffqJkEXlNSHdvVFQUVXY5ePAgNBoNuVItUaBcFvA+fPgwhQE8PT3x999/p+tKLVKkCCUqWrvribQYmzVrhmfPngFQkq/Mse1u9uzZAJRMZ5mte/fuXTRr1owSljZv3kzFExISEiCEQI8ePQAoFZNq1KhhytBZ38oRFRUFQMkUlN0Y9HFwcNDJakpKSkJqaiqAtNVKPD09yZ3q4+OjSjZUejLmzJmTyrHp73d68eIFVaEBQO7iPXv2oGTJklmWSZtNmzbB19c3y+eRMcbx48ejWLFiVOHH3Lx69YrSpWfMmIGLFy/SserVq9ME0q6knz9/fvj6+qJw4cIAgD59+pg0dkBAANatWwfgze+cWTp06KDjnjWR/6RyTEpKgq+vL3VbGT16NLnIDx06ZJXtOQDg6+uL48ePA1C6XwDKghcAVZIClHlfqVIlWhRZop2VXKS3bNmSFF5G5MqVCytWrDBra7zM0K9fPwDK/mZZLUeNEFhGVK9eXWcPPPCmZKH2fTtv3rzo3bs3uafl/cUEuEIOwzAMwxiDSUUAoqKi0KNHD+qZl1HtUX0rTQaaV65cifr162dmo2amuHz5MtavX08b7o2VsVKlSpQ0ZI7sWSEENm/eDADo378/4uLijPpcmTJldPY7zpw5EwCsVhTdWsh9TB988EGavptvw93dHQEBAWjSpElWxfhPWo6BgYEYPnw4FYG+f/9+tmgq7OvrS3MKSHvPkXtwhwwZAl9fX1OrImWJXbt2YezYsQBAzcINsXbtWiqebW3+/PNPdOrUCYAS9rKU23zfvn3kIXr+/Dl+/fVXNG3aFABQrlw58pIVL15crfuf+hVytBvdbt++HQB0tkEAuhdqnz59yCUj3R/mRsrYtm3bdDOulixZQjJ++umnNPnNzR9//EEZqCNHjkRMTAz92HIztaRcuXIW+87eBYKCgihNXqZyayPj29oxWF9fX3KtZ5H/lHKU8bsGDRogOjqaXKnBwcFWVYqSR48ekeJZuXIlvv76a4olVqhQgdy95q4c9TZkPsO8efOwefNmuuf4+PjAz88PgFIcJTtstk9ISECTJk0o4/zbb7+lBfl7SPYqH8cw7zDWv3uZhknzWXpdGjRogLZt29JixNrJIoz6yO0U06ZNw/bt2ynhavLkyVYt62lmOObIMAzDMMbAliPDZJ7/lOXI/HeQRRzs7e0xbtw42rLxHluNALtVGUY1WDkyzPsDu1UZhmEYxhhYOTIMwzCMHm9zJL+r7iOGYdLC85lhjIQtR4ZhGIbRg5UjwzAMw+jBypFhGIZh9GDlyDAMwzB6sHJkGIZhGD1YOTIMwzCMHqwcGYZhGEYPVo4MwzAMowcrR4ZhGIbRg5UjwzAMw+jBypFhGIZh9GDlyDAMwzB6sHJkGIZhGD1YOTIMwzCMHqwcGYZhGEYPVo4MwzAMowcrR4ZhGIbRg5UjwzAMw+jBypFhGIZh9GDlyDAMwzB6sHJkGIZhGD1YOTIMwzCMHqwcGYZhGEYPVo4MwzAMowcrR4ZhGIbRg5UjwzAMw+jBypFhGIZh9GDlyDAMwzB6sHJkGIZhGD1YOTIMwzCMHqwcGYZhGEYPVo4MwzAMowcrR4ZhGIbRg5UjwzAMw+jBypFhGIZh9GDlyDAMwzB6sHJkGIZhGD1YOTIMwzCMHrneclxYRAqGebfQWFsAE+H5zDBpMTif2XJkGIZhGD1YOTIMwzCMHqwcGYZhGEYPVo4MwzAMowcrR4ZhGIbRg5UjgKVLl6JEiRIoUaIENBoNxo4di7Fjx1pbLIZhGMZKvG0rR5Z5/fo14uPj07y+ePFiJCUl4fLlywCAJUuWYOTIkQCAX375BXZ2dgCA7777Dv7+/qrLlZiYCAAYPXo0lixZAo1Gyea1sbFBrlzK1/Ly5UvY2tqqPjZjOgkJCQAABwcHeq1bt25YuXKlzvs2bNgAALh27VqG5xs2bBgAoGDBgmqKyTDMO45qyvHWrVsAgOTkZBw9ehQAcPjwYcTFxWHz5s3pfq506dIAgCFDhmDr1q0AAHt7e7i5uQEAGjVqpJaIOly/fh2AYjVqo9FokCOHYlC/fv3aLGObg+fPnwMA4uLidF4vXry4NcQxG3IRY2tri+TkZADAunXrsG7dOpPOt2fPHgBAaGgo8ufPr46QzDvB7du3MX/+fABAREQEjh07BgDw8PBARESEVWSaOHEiAGDSpEkZvi80NBSenp7mFyibkZqaSve4O3fuYP369XRs8eLFZPQ4ODhg9uzZAIABAwaYNBa7VRmGYRhGD1Usx7///htNmjQBAIMu1PTImTMnpk6dCgDIly8fvvjiCwBAiRIlUKBAAQBApUqV1BDRaLy8vFCvXj0AQN68eS06dkZcuXIFv//+OwDg9OnT9Lqvry+OHj2K8PBwAMDBgwd1PpeammoR+eLi4siSAxS35uPHjwEo1t6LFy8QFBQEAKhatSoOHDhg0jjSuhsxYgRmzpwJQLmOnJycAAA+Pj44d+4cuesB4MsvvwQAlClTBpcvX8avv/5Kx8aMGaNzXkY9bt68CUDxavzyyy8AgB9++AEA0Lp1awCga8ISbNq0CceOHSOrUFqK+khvlqWYOHHiWy1FfRo3bgwhsk/BIzn35W8OABUqVFDl3PHx8di+fTsAYN++fel6iRwdHWlMe3t7NG3aNEvjat7yBRv17T9+/Bh169YFkHGMp27duqT0QkNDYWNjkyllqiY3btwAAAwdOhQ7d+4kd926devQtWtXq8iUHj/88AMGDRpEMr6NIkWKAFAWGdqK1BwcP34cAODt7Y1///1X55i8tgzJnVWX9aNHj1C4cGEAys3s5MmTAN78380Ml49Lh/379wMAQkJCSCHGxcWluQbkovfixYtmkeP27dsAgPnz51NYR76mjYeHBwBg+PDh6Ny5s1lkSQ+5kG3cuLFR7/f09NQJM0kXrKV58eIFzp8/jyNHjgBQFutRUVEAQP8C6i3Mx48fj2nTphk85uTkhIoVKwJQfmtp2GQSLh/HMAzDMMagilvV2dkZc+bMAQDs3LkTNWvWBKBYZQDg7u4OQFlV5suXDwBw7tw5BAYGqjG8Sbi6ugIAduzYYbRFZmkWL14MQHEhZoSbmxtlb7q5uWHgwIEAgCpVqphXQCjZvgDSWI3aeHt7o2TJkmjevDkAoFy5clkeN1++fJSQcPDgQbJE/vjjD3z88cdZPj+TOfr06YNz587hxIkTaY45ODhQyKR27dr4/PPPKRvdXHzzzTcAoONC79SpEzp27EjPLW0p6qNtMcprWVqG8nl2Sbo5efIktmzZAgDYtWsXLly4kK5nyFhL2Bj69euHn3/+mZ7b2tqSrqlatSoKFSqE6tWrqzaeNqq4VbV5+vQp7O3tAShZQitXrqT/3Oeff26KjGZHo9HQD9y5c2faBmBNunbtStm7ycnJqFixIsqXLw8AKFCggE4GVo0aNXS2Npgb6Upt1aqVTlzRzc0N/fr1AwBUq1YNDRs2NKscMm7ZsWNHcs83bNgQY8eOJUVsJrLnaurtqOpWffToEcVsV6xYAWdnZ3zwwQcAlC1Y1apVAwDkyZMHZcqUUXPoDBkxYgS5Uj08PDB8+HAAMNXlpgoHDx4kN6p0h8rn2UUBAkBYWBgA4MmTJ5g7dy4OHz6c5j2urq5pFt5yd4GPjw9q166tmjw1a9bUcdUWLVoUMTExqp3//2G3KsMwDMMYg+qWozbffvst5s6dSyuj/fv30x7C7IS25ZgvXz7K/JTuYEsikwbq1KmD2NhYej0mJgZFixa1uDz6/PXXX2jTpg0AxZUqLdalS5eiY8eOyJ07t8Vl2r9/P7nI4uLiUKBAgQwtR7nq7d27N0qVKmXKkGw5Avj6668pNDJkyBBMmzbN6lm/mzZtgq+vLyXabNq0yeLZp9pIF6N2Fnl2yjIF3uyNrlmzJu7evQsAePXqFYA3snp5eZEH0M7OzmKZ/L1798aaNWvo+aRJkzB+/Hi1hzE4n82qHBMTE9G2bVu6MPbu3Wtud5dJSCUu6du3LwAl5mdjY2NRWWSMxNfXV+f1lStX0gXZpUsXi8qkjaurq07Wn8wYlXFO+ft+/PHHVGnIEvzxxx8AFHf0kydPjPpMmTJl0Lt3bwDKlo5MKPb/pHJMSkrCrFmzsHbtWgDAwoUL6ebZokULs8cRjaFevXo4duwYuVI9PDxIOVrarXrw4EGd+Jus9GWtLFN9Xr9+jfHjx9Pvee/ePaoI5uLigsaNG+Ozzz4DoMxnuV3Kkvzyyy/44osv6F4SFhZmjt/R8soRULZ21KpVC4CSdisvltq1a8PPzy9bJMOcO3cOXl5eAEArJ0DZs2PpVefff/8NAGjSpInONhchBF0gFSpUoNWTpbedfPrppwZXwfq/Y4sWLbBixQoAQMmSJS0m3x9//IGuXbvio48+AqBYN9pcunSJ4lGRkZF48eIFACWxaMqUKQBgjFK3/kVrGlmaz2PHjsXMmTNp4RYUFJQtFCLwJrnm119/hYeHBzp16gQAWLBgAVmRc+fOteh81p4Tnp6eCA0NtdjYxnD06FE0aNBAZw7LqjJff/21RRe36SGVo1TashKYynDMkWEYhmGMweyWIwDKuuzVqxeePn1Kr8+YMYOql1i7BqiMM2pnWAYHB6Nbt25WkScpKQkLFiygv6dNm6azEs2ZMycAoGfPnpgwYYLFVsRjxozBs2fPDB47efIkbcYHgPr16wMADh06ZBHZJFFRUWStFipUKN33LVu2jNxvL1++pIIJRsSa/5OWo7z+ZLWSdu3aZV2iLCAr3YwYMYKq3cydOxedOnWi+TBixAiqn3rr1i2rWY6A7nYNT09Pq2epRkdH49NPP8WdO3cAKPLKeRMQEJAmtGMNrGk5QgiR0UNVzpw5I5o1ayaaNWsmNBqN0Gg0YuDAgWLgwIHizp07ag+XKcLDw0V4eDjJpdFoRNu2ba0qU3pER0cLKDc6AUB4eXll+ZwbNmwQ8+bNE/PmzRMpKSkmn2fv3r1i79694pNPPiH51qxZk6VzmhNXV1fh6uoqNBqNKF26tChdurRISkp628feNm+y6yNL1KlTR+d7+uOPP7J6yiwxd+5cMXfuXAGA/tZn+PDhdB3eunXLovL5+/vrzFP9h7+/v/D397eoTPpcv35deHl5CS8vL2FjY0Oy1a5d26pySdavXy80Go2ws7MTdnZ25hrG4HxhtyrDMAzD6JOe1hRmsByFEOLJkyfiyZMnYu3atSJHjhxkpTVt2tQcwxmNpSzHESNGiGfPnolnz56ZfI4nT56ImjVripo1awqNRiNat26dZbkaNWokKlasKCpWrCj+/vvvLJ/vzJkz9D1WrFhRJCQkZPmc5qBp06aiadOmOr+7EbJa2wI0u+V47NgxcezYMfHy5Ut67dGjR8Lf35++JwcHB3HhwgVx4cKFzJxadTKyCEuXLm01y1EbaSUasiZDQ0OtJpc2+/fvp9+2cuXK1hZHCCHEgwcPRLFixYSNjY2wsbER165dM8cwBueLxdORZDpw9+7d0bdvX6SkpABQ4lLZsWKE2sybN4/6XY4ZM8akuE3evHlRrFgxei6LuWeFPHny4OrVqwCUTFOZWffhhx9m6jyyOn///v3ptY8++ki1fVHnzp0DoHQpkVU5ZFUWU5DFsT/88EM8fPgQgLJtRj/L9b+ArDzSunVrncLdMu7u7OyMwYMHY/LkyQCUxtPGbpsxJ4biiPPmzQOg7BuWcWVr7nfU3r7h6emJSZMm6RQeFxnnflgEbRmsWU1Im8KFC8PGxob0RP369XXud59//jkGDx4MAKpvNbGocjxz5gyl0Z88eZL+w4ByczJ3ubGM+P777y0yjp+fH+1l7N69O3r06AFA2URtbIuXEydOYO/evfTc1Gae2mzcuJEaP0+aNImUW/PmzTFhwgSjzyNvAidOnKCb0cKFC1Ur/iA3Is+ePRt+fn4AgEWLFpl8PpmwIxOcAOgUX/gvIbdcxcfHU0q/fkKaTBIDgGbNmlGJOHOzadMmo2uhzps3j+QsXbo0Kcfsgqenp045OUsYA7LMX9WqVdNNMgwICKC/y5Yta3aZjKV27dq0WIuNjdWZnxMmTMCff/4JQNlHqt21JKtwzJFhGIZh9EnP3ypUijleunRJ+Pn5CT8/P1G8eHGd2I5GoxG5c+cWuXPnViXj0lRmzpwpJk6cKCZOnCg0Go1wcnISTk5OZsvGW7BggViwYIEoVqwYfQ/Ozs5i3bp1IiEhId2YV2RkpIiMjBSlS5emz3l5eYnHjx+rKl9ERITIkSMHPapWrSqqVq0qxo0bl2Em56pVq0StWrVErVq1RI4cOUTz5s1F8+bNVc1EDg4OFsHBwTrXjqenp8nni46OFtHR0cLJyYm+UyNiU9aOHZol5jh9+nQxffp0kSdPHoPZlRUrVhQAKMP31KlTbzulagAQHh4eYuPGjWLjxo1pjg8fPlwnM7VTp06iU6dOVo0zpkdoaKjw9PQkWbNy/RqLHKtChQoiKipKREVF0bE7d+6IO3fuCHt7e5oDx48fN7tMxvL69WsxZ84cMWfOHLFnzx4xZcoUMWXKFFGjRg0dXTJ8+HBThzA4X8yyz/H+/fsAgPXr12Px4sXUWFifjz/+mNyZ1twzpV1bFVA6CgDA9OnTzTru8ePH8e233wIAVb+Xe+z0a1QKIfDPP/8AUMo8SXbv3o2WLVuqLptsk7VhwwaKRWk0GtjZ2VHJNW2uXbuG33//nZ63b98eISEhqsslG6guXbqUWqK5uLjQd5NZZNzy7Nmz1PrmxIkTtK8qHd7rfY4BAQG051N2PgGUa7BOnTpUarF8+fI67mhzUqZMGdy+fZtc9R4eHrS3UbucoYeHh1XrqcqwQlhYGPz9/XVcpuk1N/b39zd7STntCkKSypUrIyEhQacqmHS/mvvepwYxMTFo2LAhrl27BkCZy3/99RcAZPa65Ao5DMMwDGMMqlmOMkh6/vx5yh66dOlSmvfVrVsXADBq1Ch4e3tbtEtHZGQkAOCnn34iK6d8+fLYuXMnWY42Njb46aefAKQt/m0OkpKSAABTp07FqlWr0m0aLIQgGe3t7bF8+XIASk9Fc/ZyvHv3LlavXg1AsdYePHiQRi5AsSqrVq1KWZ6+vr7U2NocpKamok+fPgCAdevWUXPbGTNmvPWzV65cAQB06NAB0dHRAJROHfv27QMAFClS5G2neK8tx+zIpk2bMH/+fLIWAVD91FKlStHf1syynDhxIiZNmmT0+6VVaYmaqzdv3gSgWKmyP2JUVBQqVaqEVq1aAVCScGSyjjWKjJuCfpWrly9fAkBmuwOpX3hcNrodMGAAKR5p4mojy4h98803aNGiBQBl64AluXHjBlauXAkAmDVrFl6/fk3HhBDUSX7w4MGk3C3N3bt3sWTJEgBpMyaluwNQfngXFxeLygYorW0SEhIQFBREr2krRz8/PxQsWNBi8iQmJgJQCmDLCdKrVy+MGjWKGkPrM2HCBHK9aGf8Dho0CIsXLzZ2aFaOTBoy00TBEq7U9JDbrW7duoXixYubdRFrCeR2s0uXLqmqHNmtyjAMwzB6mGQ5Hj9+HLNnz6Yi07JwrT558+bF0KFDKenGGiuUs2fPAlCsBGlpzJo1i1YYn3/+OerXr08uVGdnZ4vLyGSNJ0+e0L687du3459//qGVY4MGDSjZCVAKF0uvQf/+/WkPZ8GCBd+WhKMNW45MGvTdqv7+/ggLCwOgFBvPLn0c3yfu3btHLcnu3Lljfbfqd999RzcjiTRt27ZtS5lCI0eOfGd818z7watXr7BhwwaK42i7gAGgX79+aNKkCQAlg8/EfqKsHBkmGzBu3DjKrK1evTplWauRrWqRllUM857BypFhsgE7duxA+/btASjbzoytoqQHxxwZhmEYxhjYcmSYzMOWI8O8P7DlyDAMwzDGwMqRYRiGYfRg5cgwDMMwerByZBiGYRg9WDkyDMMwjB6sHBmGYRhGD1aODMMwDKNHLrVOJNueaHc6cHNzQ+vWren56NGjdT6TM2dO5M2bN825EhMTkSuXIlom6l1myIsXLzB//nwAwJQpU6hTSJs2bdC2bVvqJuHo6KjKeGrz6NEj/PzzzwCAyZMnU0cUQGmeumvXLgAw+H0yDGNd4uLi0KFDBwBKJxsfHx9q8G6NDjv/JV6/foroxSkAACAASURBVI1Xr15lWpew5cgwDMMweqhSIefOnTv45JNPACg9wtI9mVbDXkBpKmuoFl5wcDCtpo4dO6aK9Xjs2DH873//S/d41apVASjV8+UKz8bGhv5f1kAW0Z0/fz6OHDmCGzdu0DHZ6aROnTpo0KABChQooOrY0gOg7QmQPHr0CIDSqLl06dLpnkP2+Ozduze+/PJLVeXLiH///Zeuw4sXLyIkJAQAsG3bNp1rsH379uRNyOTqnSvkMEYhm5f36NGDGqzLa7BEiRIAFG9PgwYNAAADBw5E7dq1rSPsO4rstpSamoqFCxcCABISEuj48ePHERYWhgx0nfkKj//5559o1qwZAKUtlbxxb9u2DTdu3KDu8frKMSPk+Xbu3JnZ9iMGGTJkCDUSBoBq1aoBAM6dO0eyAboNS3PlyoXKlSuTAi9WrBgpIR8fnyzLlBFXrlzBRx99BAB49uwZhBAoVaoUAKXbyZAhQwAAOXKob/w/fvyYJqi2Qs6IjH7bLl26YP369WqJlyEhISH45ptvqPO5RqPR+W215RRCoEiRIgCUbuxVqlQxdhhWjsxbiYuLw+effw4ApBgBw3NFXqNFixbFrl27aO4z6ZOcnIwTJ07A29sbAHRCTYbIrHJUJea4fPly+rthw4YUWxw9ejSuXr1KFgTwRssHBgYCAK5fvw4AePr0KbW38vb2xtChQwFkui9XukhrB1AUuOwGHx0djZ07d9KKIz4+nnqCvXr1CufOnSMFCrxRnlOnTsWYMWNUkU0b2aX7iy++wLNnz+j1Nm3aUJuwTNzETSI4OJiUop2dHVq0aAFA6ZXo7u6O58+fAwAuX76c7jl69uxJceP+/fubVV59Hj58qDMRMvpbLtwaNWpE10SZMmUsJOl/g6ioKLpxXb16FWFhYahYsSIAUD9Na3Djxg26d8nFFKD09vTz80PlypVNPrcQAh06dMChQ4fSHAsMDMTUqVPJqtTmwYMHWL58OX788UeTx7Y0169fpx6p2lSoUEG1MeT1Ex0djfDwcADKtZTe95QvXz44ODjQ87Fjx2Z6TI45MgzDMIweWXKr3r17FwDw8ccfIzY2FgDg5eVFmZPGIFfrDx8+RLFixQAA7u7uRn/eWNauXYuePXsCUKwhaRlJt5rk0KFDuHfvHgAgIiICJ06coGM2Nja04nV2dsasWbNUl1Nai9qrHkCJnVWqVEn18Qzh7e2NnTt3AlAs2DVr1gAAXr58iVy5cpH1pb1a1HcV5cmTx9RGwlnm448/xqlTpwAolr7s96Yfc9SPPwYHBwMwKuOX3aoGSEhIwMmTJwEAW7dupXh1TEwMkpKSdN4r8wikF0JN7t69i7Vr1wJQPAIy1yAhIQGrV6/G4sWLASjWyJMnTwCkdbk5OjoiLi4u02PLz+hbjU5OTti6dSsAxbuWHhcuXIBGozG7d8gUbt++DQDYuHEjtmzZQt/Z33//jZSUFHpf2bJlAQDXrl1TZdyXL1/Cy8sLgBL+sLOzAwCMGTMGz549g7OzMwBg06ZN6NatGwDAw8MjwxwTPdR3q8obS1biXpYKPkulBijbOuTNv0+fPjrv075wu3TpYhHZtMmTJw8AoF27dtixY4dFx5YuxtOnT6Nx48YAgKCgIOqq/S5sE/n333913KodOnTAli1bACiu8O3bt+Phw4cAdOOp4eHh9Dq7VY3j6NGjCAgIoOenT59ONyGvefPmAJSQgKenp1m3L7Ru3RpRUVEAlHCEnNNLlizRkU97cdSiRQtSSJ6eniaPLV2l+u7UpKQkWiwkJiaiUaNGBufThx9+aPLY5kAaQIMHD8b+/fsBvAmNGcrTqFSpkqkNhw0SFRWF77//HqGhoQCU+6Nc3PTu3Vvnvd99951q4wLsVmUYhmGYNKiSrerr64vNmzcDUNyBU6ZMAaAkZeTPnz+rMqrC7du3aQV58+ZNuLq6AlCyyNQMHKvFiRMn4OHhQc9LlChBqydzyStXtq1ataLkn5EjR5plLHPy5ZdfUsEEjUaDYcOGAVCC8oUKFSIL8dSpU+jRowcAZRUsXYJGWI7/Sbfqq1evsGrVKtoaExYWRglk9vb2cHZ2pgS6L7/8kjwv+fPnp/CFObKr9enRowe5VbWtGiEEXF1dyVuUJ08efPHFFwCU8Ioaskmr6vDhw/j666/JOyG/JymHp6cnRo0aBQBo2bJllsdVgwsXLpBV1rVrV/zyyy/44Ycf6Lj0vg0fPhwtWrTAkSNHAACdOnVSrViLRFr+7dq1w61bt8ij5uvri6CgIFXHgjm3ckRFRVG1hzt37tDr06dPx9ChQ+k/Zm2kD7xZs2Z00bq4uOCbb77B4MGDrShZWh4/fox58+YBAAICAvDy5Uv07dsXADBx4kSULFlS9THld7B06VIcP34cgPGZsXJyqJVdnBUuXryos51IXuNVqlTB1KlT8dlnn2V1iPdaOb569Yqyxe3t7env169fw9XVleLhLVu2RK9evQAoCwprxsmkUhoyZAh+/vlnioHVrl0b5cuXB6C44f73v/8hX758FpNLxrHj4+NpUXHw4EFoNBrKzu/evTvtc+zYsaPFZJNIF/CwYcMoA33q1KnIlSsX3WcaN25M93FL3M/lvmj5/fn5+QEAhg4dqhMiUwmD85ndqgzDMAyjhyqWIwD4+/sDUFYc2hQpUgTffvstAGXFMXDgwEwLqTZXrlyhBAG5v0kG4du1a0d/myNr1hQGDBiAH3/8kVxEJUqUwNWrVwGAMrfUoGvXrgCUbLTMUrduXQBviisMGjSI9jk6OTllWEnHHMjsyJCQEFqFajQa5M2blyogSdebCbzXlmNiYiLs7e3puczyXrBgAVauXEnPZZZgdkBaP3LuyiIi27dvV3WOqEFYWBjmz5+vk3AnQz4jRowgL5y5ef36NVatWoWlS5cCUPYQjh8/HgDMsofbWGJjY6kIgkwIkjg7O9O9pnfv3qhevboaWfyG57MQIqOH0bx69Uq8evVKREVFiWLFiolixYqJHDly6Dw0Go1wcHAQDg4OomvXrmLVqlWZGUJVLl68KC5evCi8vb2FRqMRUG4cOjJWrVpVLFiwQMTExIiYmBirySqEEL169RIajYYe4eHhIjw8XNUx5P/T29tbuLu7C3d3d50xNRqNyJ07t3B1dRWurq46r8vvztCjWLFi4tdff1VVVlOYMmWKjpzdunUz9VRvmzfZ9WEUz5490/n9hg0bJoYNG5YtfkNDrFmzRtja2gpbW1sBQLi6uor4+HgRHx9vbdHSJTExUURERIiIiAjRpk0b+q7t7e3F3LlzxY0bN8SNGzfMKsPq1at17n3r168363jGkpSUJJo1ayaaNWtGsqX3KFKkCF2fSUlJpg5pcL6oZjka4ocffsD+/fspBfjp06fp7n3r3r07VcsoW7asRQL32kyePBkAMHv2bIpf6NO/f3+KV8mqMZYiPj6e9vpERESQ3//w4cOoVauW6uPJ2oQyeUWSI0cOitlo1y8MCwvDzp07aS+XPra2tpTwIq1La+Dj44Nt27YBAGrVqkUyZZL32nJMSUlB9+7dASiWl6wYBShJGdKq8PHxyRYJdx06dMD27dt1XpNei4IFC1Icb9GiRRaXzVgGDRoE4E21Mbml4+zZs2Ybs169ejh+/DglBk2ZMiVb5AwAwKVLlwAA58+fx/Lly6nGtayOAyjbZrSrrwUFBcHX1xdApuOiHHNkGIZhGKNIz6QUmXSrGkNgYKDw9vYW3t7eBt2u8tGzZ09x7949ce/ePbVFeCvHjx8X06dPF9OnTxcNGzZM4yJ0dHQUjo6OIigoSCQlJWXFlM80p06dEqdOnRL29vYkT+3atcXly5ctJoOxLFq0SLRs2VK0bNlS2NnZ6bhCDhw4YDW5BgwYQN+di4uL+Pfff8W///6b2dNY2z1qVreqNqtWrSKXpY2Njc5cqFSpkoiMjBSRkZGmnFo1QkNDRd26dUXdunVJVu1Qjo2NjbCxsRGvX7+2qpwZIb9HPz8/nXuhOZAu5y+//FJnXhYuXFh069ZNdOvWTcyYMcMsY2vz+vVrcfXqVXH16lWTPv/48WMxc+ZMnf/D/v37xf79+zN7KoPzxaLKUZv4+HgRFBQkvLy8hJeXlyhdunQaBdmzZ0+rXtApKSli3759ol+/fqJfv34iT548OjeH69evi+vXr1tcrq1bt9KE12g0YsiQIRaXITOsXLlS2Nra6sT6LL2wkAwYMICusdq1a5t6GmsrOYspR20SEhKEv7+/KFWqlChVqpTQaDSiUKFColChQuLYsWNZPb1RfP/99+L7778XW7duzfB9u3fvFrt37xalSpWiG+e4ceNEYmKiReQ0lfPnz+vE882BXBBGRkaKRYsWiTFjxogxY8aI6tWr07i5cuUStWvXFkuXLhVLly41ixyLFi0SFSpUEBUqVDD5HFu2bNFRjnPmzBFz5szJ7GkMzhd2qzIMwzCMPulpTWFmy1GfoKAgHctRZrxaw7rQJi4uTvj5+Qk/P780LlZrWY5BQUE6cnTt2tXiMmSWESNG6Mg8efJkMXnyZIvLoe1WZctRF5kh+Ta3d2xsrIiNjRW1atWi79LPz+9tp88y0dHRonjx4qJ48eKiZcuWRn1mwYIFOpnW0dHRZpYyazx48EDUqVOH7oNqz5G1a9eK9evXG8xMff78ubhy5Yq4cuWKWLRokahVq5aoWLGiqFixolnuwx06dKD/59q1a006h77lePPmTXHz5s3MnsbgfMlS4fHo6GgAwJkzZ7B69WoASkeESZMmGX2OPXv2AADthZTI0lPWqK4jC3CfO3cOffr00en1JosFt2/fHoULF7aoXPHx8QCU6vMSe3t7dOrUyWxjJiUlqVJwfPz48Zg/fz49f1tj0qxw8eJFXLx4EQAMVsORGdNyvyOjIIvNe3t7o0mTJum+759//gHwpkuD9mfNScWKFem3mzt3rlGf0e45Wq9ePYuVihRCYPXq1fj4448BADVq1Mjw/bKbR48ePahTEaDbhzYryJJrW7Zs0em/q42dnR1VE/L19UVgYCB1uTBHFmuxYsWQmpoKQCnjWadOHQDI1L7FM2fOqC6XxGTleOHCBYwYMQIAsG/fProIjG0TcubMGZw+fZqUovbNsnTp0llqNGoKslP9rl27qHSavAlIcuXKRSXWZs6caVH5Tp06RYUWZA1UAJg0aZJZb/IBAQFUh/azzz4zOXVfptZLZHsytdBWhpcvX6atOfrK8cGDB0qwHUChQoVUleFdR17vp06dorZtwJtCEykpKfjtt9+omfmjR4+owELbtm3NLp+3tzdt2Rg6dCj95l999VWa9165cgWA8n+Rv3dkZCRu3rxp1o4g2vTv3x+tWrUCAOoClB5yDhtqjqwG0ngpXLhwuqUnT548SYuJFStW4OrVq7RI0p+/ajBp0iSq3bpu3TrapjFr1iyjOiIFBARgxowZ9HzKlCkoUaKEavJxzJFhGIZh9DCpCEBKSgq8vb1x9OhRAMCyZctohaTfpBcAbQxPSUnBgQMH6DV9l4EsMbZjx463uiGMISkpicx22ZNQyitXchcuXAAAatasXaRAo9GgXLlyGDduHAClkbParlRZcEBu8Je9B48ePYrz588DANasWYM7d+7gxYsXABQXh3SttmjRwqzlsc6cOUNl9KpWrYqRI0dSJ4vMcPr0aerdWapUKWpGrMb3OXXqVGo8nTdvXowdO5Y6cWgTEhKCjh070m8cGxtrqvX4XhYBkBbYjz/+qPO67KgRHx+vUxCgYcOG1AhbehfMyZgxY8jSePr0qdGfk793UFAQWbrmRgihc8/ZvXt3mu4b0gpesGABDh48qCNro0aNAIA68WQVWdj8yJEj8Pb2BqAU5tAO0Whja2uLKVOmpAl3qYkQgu67EyZMwIoVKwAohUb69+8PQLnnaHPnzh389NNPABRvo3a3k+joaFPd5uqVj/v9999Fjhw5hJ2dnbCzs6OgbXqPXLlyiVy5cgmNRpNmT6MMsE+bNk2VMm2yLFzfvn1FgQIFKBhftGhRehgqcYb/D+ja29uLGjVqiBo1aog1a9ZkSRZjWLdunVi3bl26pdfko3jx4qJ169aidevWqpeNy4jY2FiRJ08e2saSI0cOUaVKFVGlShWxfv16kZCQIBISEtJ87tmzZxTc79Gjh87ezClTpqgiW1hYmAgLC9MpCbdlyxaDsjx79kxUqVKFUvrHjRuXlaGtnVhjloSc5ORkkZycLJYsWUIlFLWvQWdnZ1GhQgURGBgoAgMDRXJysolfn+nIbQjDhg2j7UxA2rJicj+yh4eH2L59u9i+fbtF5UxNTRXVq1en+1ypUqXofrh7925Rr149YW9vL+zt7Wk/ppxfbdq0EXFxcSIuLk41eeQ87d+/v8581mg0wsnJSTg5OQkPDw/a471v3z7VxjaGkJAQ4ebmJtzc3N5aMk77Ub58ebFixQqxYsUK8erVK1OH560cDMMwDGMMJrlVX758CQ8Pj0xnCgkhdNyWrVq1oiQT6XLLKl9//TUAIDAw0Kj3Fy1aFHXr1iV38PDhw1GzZk1VZDGGlStXAgC5EWTgW7qpAaUrwkcffWTxzhYSmb3btm3bdGuRygaykp9++gmvXr2i5zlz5qT6iD///HOWA+chISH45ptvACidVWRz488//zzNe2WdysuXL6N9+/bUIy4LWbjvpVtVGxlukL89oLhOLeE+NZZjx44BAIUbtClatCgA4/uRmoObN2+iffv2AHSzKvXvgwAoS3T8+PFo2bKlWZPFZPa97Hkp54GaySymIF32W7duxbRp0wAoOwb0kQ2qx48fDxcXFzXCSuo2O3758iUVcD5w4ABWrVpl8H0ff/wxtY/JlSuXTmd5W1tb1bOgZAykd+/eaY7JDKiiRYvi008/JfnkRLIGsnh3//79ERkZSfFZS2frGkNsbCzOnz9Pv7vsGg4YnvCShg0bokWLFqq2wRk3bhymT59OY8us1EKFCkGj0VC6ukajoWzFKlWqYMuWLWrcMN975ciog1REbdq0oQWH/lzp1KkTNmzYYBX5sityYS1zRrSROkPF5hRceJxhGIZhjMGsLauY9xOZIabdiDQwMBC3bt0iy9fPzw9t2rQBoDSgtbW1VVWGpKQkhISEAFAy8OS4Dx480LEWfXx8yAofM2aMKgUNwJYjw7xPqOtWZZj/MKwcGeb9gd2qDMMwDGMMrBwZhmEYRg9WjgzDMAyjBytHhmEYhtGDlSPDMAzD6MHKkWEYhmH0YOXIMAzDMHqo38HSALKG3/Lly6lFCaDUQS1YsKAlRGAYhmEYo2HLkWEYhmH0MHuFnOjoaHh6egIAYmJidI65ublR82NLWpCzZ8/G6NGjAQAeHh5U3R9QKr7Lhqr58uXD69evdT4ri96mV2RbTWQz6E2bNmHQoEHpjjl69GjMmDHD7PK8q8iC5zNnzgTwpsFyo0aNqOj88OHDUa5cOWNPyRVymEzx8OFDauYrqVevHgDQ/VGyZs0auleuX7+emp4HBgZi8ODB5hf2v4d1ysft27cPzZs3BwDUqFEDvXr1omMBAQHUmmX//v1mbdOiTWRkJHXlePz4cZrjzs7OAJQu1OHh4TrHpkyZAkDpCmFuZL3Qjh07Ztj1wsbGBqNGjQKgtHFRo9OJXCAMGjSIXsufPz+6du0KR0dHAErd0nz58gEAqlevnuH5Xrx4QTVZZXswS5Camkq1Va9cuQIbGxuquyrd/QBQrFgxzJo1CwCM6RbPyvEtPHz4kObzqVOnqGtPSkoKVq5ciUWLFgGA1W/2z549A6B0yrC3t1f13DNmzKBrKjU1FYmJiTrHbWxsACjdieRiffXq1bh16xZ1pXB2dsZXX30FQJnb8jOMqnD5OIZhGIYxBotajh06dKBOCoDSUDUgIAAAUKZMGUycODGrwxnNnj17AAD9+vXT6S7xNrp27QpAcXeYk8OHD6NTp04AlE4TGVmO2jx58kQVy+zKlSsAlF6M2klU+shGoxUqVEhzTF5bGo0GL1++xLJlywCkdSOZkzNnzsDNzQ2A4hKPiIggmW/cuEHNpkNDQzFp0iQAbxpmZwBbjgY4ceIEfvvtNwDAjh070KFDBwDA9OnTyWsAAEWKFMGuXbsAqNfkPCNkE11pJUoCAgIwd+5cAIpF+5Z7YaaYPn06pkyZovP/ls2+P/jgAzRr1oya9mrP6zJlyqBx48bUj7ZixYooUqSIanK9a7x+/Rp//fUX6tatm+bY48eP8ffff+u8tmnTJgDA/fv3KVQXEBBA3sB0MDifzZ6tamdnR+2Knj9/TheLjY0NPvzwQ6xevdrcIhjEy8sLALBs2TK0bduWXvfw8ECNGjUAAP/++y8cHBx02i1pN/g1JytXrtTpwp4Rjo6O5P5Uy+0ild0///xD5/7ll1/oRiORXdjPnj2b4fmGDx9ula7s//zzD/1du3ZtnZtxtWrVqK0Wkz7yt5WNxKtWrUrHZDwsJCQE9+7d03FVR0VFpTmXk5MTBg0aZDalmJCQAHt7e7ouIyIiKB6/b98+s4ypjWwELhXjt99+CwDo1asXihUrBgAUlvjrr78AADt37iRXfq9evVCyZEmzy/k2kpKScOvWLQQFBQFQ2sJJ6tevr6PQo6OjUbFiRQBv4vpqMXz4cKxYsUJn8S0XMYmJibhx48Zbz9GtWzc0adIk02OzW5VhGIZh9DC75fjJJ5+Q1t6zZw+OHz9Or2cHTp8+rfN8wYIFBk14S7N9+3ad5y4uLuSmGj58uM6xXLlyoXjx4maRw87OjpJtfv75ZwBvMos9PT1Rvnx5AMqKuVq1amQt5MuXD5999hkAJQPYxcUFOXPmNIuMhpAJDdJtBgBly5a12PjvEzJ57eHDhyafw8PDA4CScWkOq1G6/mfPng0HBwccPXoUAPDHH39k+LkyZcoAUDKX1eDp06cA3jQEDwsLA2DYoho5ciQAYOjQoXBxcVFlfLWYOnVqulbgkSNH0g3xqGE5pqSkoHHjxgBAv+O5c+fouHQzazQayjaXVKpUCQDQokULum4/+ugjk+SwSBEAmYkVGhpKLo46deqo3h3eFOTFK8mu/n0vLy/MmzfPKmNL14VUOHLit2zZkrKPs9tWEqmkw8PDaQJNmDDBmiK9k1y7do1clMaQN29eAEDp0qUppvbRRx/RAtkcc/7FixdYsmQJANAckeOtXbsWy5cvB6CESYA3MeX8+fOjS5cuAIDcuXOrIkuzZs0AAP/73/9w9OhRXLp0CQCwceNG+Pr66rzXXAvarHDmzBkAb+azDIV88MEHtNiV90gZkvj555/p/50VpEt+xIgRpBQBRVf06NEDgHJ9yb/NjdkTcrRp1aoVJcKMGDECM2bMsFpq8pMnTwAo8ZOYmBjUrFkTgLJSkQkb1qRAgQK0CgUAd3d39O/fHwAwYMAAi8oik3M+/fRTPHz4UOdmKRVPzpw50adPH/j4+AAAypcvTzdKS5OSkkIxnA0bNpClcvLkSbWG+E8l5Mib4dssx/r162PBggUATF+tZwZ5HY4aNYq2huTLlw+ffvop5TJYqwLXvXv34OvrSzd5BwcH/PjjjwBAiXbZBZlHcP78eTRs2BCAEs8bOnQopk2bBkBZSJgbaf3rLxrKli1Li5c8efJQgqe7uzuAN99nFraw8VYOhmEYhjEGi1qO9+/fx8KFCwEovumtW7eiVatWANTLsjSWnTt3AgDatWsHQEm9Bt5UU7E2+pajEIK+I2dnZ+zevRvAm9WTJbhz5w5iY2OpEMKOHTvSfa+bmxsVJpDbXyzF33//jVq1atHzvn37AgAmTpwIR0dHNVbB/1nLUaPRkKutXbt2VG3os88+g7Ozs+ob6TPC398fADB58mR6rVChQujevTu6desGQMkMzUTlI1WJiYkhqyYiIoIKZqxZs4ZclNmByMhIAIorWFrjQgiMHj1a9ezTjEjPctRHe4sY8H/tnXl8TNf//18zJJLKhixSsnyoiiWI8qHVStSW0th3gvZTtdUS0spXLW2p2Gl0C2KLqi2CFGlTglgjQZFEipJYQhJEGiKSeP/+uL9zzEwmyWRyZybV83w87sNk5t573u69557zXs77De6vjYiIQNOmTQGgotY/02TI0eTu3bsApM508uRJLFmyBMAL57SxYM7aQ4cOwcLCAidPngRg3MFGGywUfOjQoWprpDTXObq4uACQfCpeXl5GfSmxlHqqGT8iIyMRGRmJX3/9FYBkqmHyenp64vjx4/zlYGji4+O1BlVZWVnBwcGBh50PGDAAvr6+AID69etXpIl/1eDI+sSFCxdgZWWFq1evAjC9f575C7dt21bqPnXr1oW/vz8AaRA1tsuEvfD79OmD+Ph4AJIisH37drUlZKbi/v37fPKgORmvXr06H9yXLl2KV1991aCy5OfnA5DWkLP1itnZ2WpZuphsAPDLL7/g+vXrPFiHiLiytXDhQjRv3lzXpoVZVSAQCAQCXTC65si4d+8eWrZsCaVSGp9HjBjBTZty5AYtDxYwcPbsWVhaWqotMu3ZsycASTX38/ODu7s7AMnUaSwmTJjAo+yAkpqjKiNGjMDGjRuNJVqZsOTKY8eOVTN/BAUF8ftraLKzs7m5/NatW/z76tWrqyUFAF4kOwgLC6vI8qJ/lebIZuYtW7YEEWHmzJkApHB/U8KKFsybN4/fO5YLmWVOUdWGxo0bhxUrVpgk4O7+/fvcSsGWjzHXSPfu3Y0uD+Ps2bM8iC4tLY1rs46Ojti5cycePXoEQAp6CQkJAYASyydMDYus/f7773m2s2rVqnFz+5gxY8rL2629PxNRWZtBOXv2LLVv357at29PAGjKlCk0ZcoUKi4uNnTT1Lp1a2rdujVBemGUuXl5eZGXlxfdu3fP4HLpwvDhw0mhUPDN3d3d1CJxnj9/Ts+fP6f9+/eryahQKOjw4cN0trfOIQAAIABJREFU+PBho8rz6NEjSktLo7S0NMrKyqKjR4/Shx9+SB9++KHaPfbz86vIacvrN1V1qxTLly8nhUJBzs7O5OzsTOfPn6/sKQ3G2bNn6ezZs+Tv708tWrSgFi1aEAAKDw+np0+f0tOnT40uU1hYGIWFhfH+wJ69qKgog7Z79+5dvY7buXMn+fr6kq+vL9nZ2dH69etp/fr18gonI/fu3aPg4GAKDg6mmjVr8uv8ySefUE5OTlmHau0vJh0ciYj2799P+/fv5w8vAFq1apXB242IiKCIiAiytLSkevXqUZs2bahNmzY0ceJE/tnc3FztBdqqVSsqLCykwsJCWWS4cuWKXselp6fTsmXLaNmyZWRpaUkWFhY0Z84cmjNnjmyyVZZHjx5R3bp1qW7duvwhjY2NpdjYWFOLxgdpMThWjJycHJo0aRK/n926davsKcskMDCQNm3aRJs2barUeY4cOUJHjhyhmjVrEgDKyMigjIwMmaTUneLiYiouLqb169eTUqnk19HCwoLu3LlDd+7ckbW9q1ev0tWrV2nUqFEUExNDMTExFT7HuXPn6Ny5c2RpaUkdO3akjh07yiqjoYiLiyNra2uytrYmAOUN6lr7i/A5CgQCgUCggVEy5JQFSwD+3//+l0dgxsbGGrzOGwulvnLlCmrWrAk7O7sS+yQmJmLNmjXc93f+/Hnuy2jbtq3ebaempgIAevfuzf03I0aM4P7X8nBxccG0adMAAKtWrUJ6ejr3/9SuXRtTpkzRWza50EzYTmX7to2KthqegvKxtbXFokWLeNaXQ4cOcb8ZixKUE09PT760at26dYiNjdXrPKzPFhUVwd3d3WSZuVj/HjlyJIiIFzE4f/48xowZA0CK2peras348eMBSFHaXbp00escaWlp/LPcKe527drFYyVCQkJkPf/bb7/N01326dMHUVFRGD16dMVOUppKSUYyq6piZWVFVlZWJIlVNejSpYua+e3ChQt04cKFSp0zLi6O4uLiSKlU8m3t2rV6ncvd3V3tPP7+/pWSTS7i4uLI3NyczM3NSaFQkJ2dHV25ckVvU7JcnDlzhhwcHMjBwUHtvm7btq0ipzG1edQkZlUGM0srFAqysbEhGxsb+uabb2SPFbhz5w55eHiQh4cHVa9endq1a0d79+6lvXv36nyOmJgYsrCwIAsLCwJAY8aMkVXGyrBz507auXMn1ahRg/ffN998k+Li4mQ5f+PGjalx48Y0cuRIOnHiBJ04caJCx589e5ZfOwsLC+7HlQsA3LTct29fWV1WRMTf1ayNskTRtgmzqkAgEAgEmpQ2apKRNceNGzdSjRo1qEaNGuTl5WXMpktl7NixpFQq1TSM/Px8ys/Pr9R5tWmOVlZWdOjQIZ2Onzx5Mrm7u5O7u7varFOpVNLChQsrJRvj/v37dP/+ffrzzz/1Ol41Kk+hUJCPj48sculDbm4u5ebm0pdffsktE2wbNGgQDRo0qKJaj6k1QJNqjmyGP2rUKLV7/ODBA7ma4Ny+fZtu375NHh4eBIDMzMzIzMyMAgMD6fr163T9+nWtx+3bt4/27dtHrVq1Urvf586dk11GbZw4cYK2b9/Og2LKYtWqVWRpaUmWlpakVCrJzs6Ozpw5Q2fOnKmUDExzVCgUZG9vT/b29tSzZ08eNauN1NRUSk1NpQULFqhFfPr4+NDdu3f1jnzVRlBQENdKFQoF9e7dm3r37k3379+v9LmfP39OK1asoBUrVuitOercmZycnMjV1ZVcXV1lC+HeuHEjffDBB/TBBx+ohTbv2bNHr/MVFRVRy5YtqWXLluTs7Eyenp7chObg4EC7du2iXbt2lXmOU6dO8Rem5sC4bNkyvlShMsTHx1N8fDzZ2NioDWwbNmzgyw40t/DwcPLz8yM/Pz9ycXFRO06pVFLt2rWpdu3a9Pfff1dKNgYbwJ2dnXkId25ubrnHsYg8Pz8/ft0UCgXduHFDFrkqSmJiotpyIdVt4sSJ+k52TD3IGWRw/Omnn+inn36in3/+WScT19WrV9UGx5CQkPKa0Jtbt27R8OHD1e4fe+EHBgbSmjVraM2aNTR79mxyc3PjE23V/efNm2fwaG428FhbW1comjckJIRCQkKoZs2apFQqqU+fPtSnT59KTTjYYPbdd9/x94NmHyhrc3FxoaCgIAoKCtJbhvIIDw+n8PBwatCgAX+OHBwcKDIykiIjI+nZs2cVOl9WVhZlZWXxZUdse++998o6TGt/0TkJQLt27Xj6I0tLS/Tv318tfyXDwcGB5zUEJIfurl27+N/JyckApDx4jx49wvPnzwEAXl5e3Lnv5ORU6oL3sigqKuJpwFjaJsbYsWO5A1wzycCxY8e4jBs3biwRsMFKbs2bN0+20jYAsGPHDp4CqzyISk8CYGdnh3379gF4UTdPLjp37swDIf73v//xRf6lwdI+qf6/Xn31VSQnJ8PGxkZW2bRx/fp1JCYm8hy+CQkJJUousUoDQUFBOgdBafBSJgFgaboePnzIF3wDL/JUmpubo7i4mFdpiYyMVMtr2qZNG/6OMATFxcW82saRI0fw008/6XQcywUcFBRk0AQj+/fvx6BBgwBIqdD8/f1ha2sLQEpAoIqjo6PWiiELFizA7Nmz+d/9+/fnfaoysMXxsbGxSEpKAiC9I9l7Q5N+/frhq6++4rlzDU1sbCwmT54MAFw+QBoXevTowXOm+vn5lciNzK7Pvn37eBpQluIQADp16oSVK1fyurRaEOnjBAKBQCDQBZ01x2fPniEvLw+AlMCXzcw1USqVanX8ioqKtBZL/fjjj1GrVi2exq1Xr16yhFjHxMQAAPz9/dW0R3Nz81LP//jxY67BqmJvb4/9+/fzeoD6aLNl8eTJE572imXGL43SNMf69etj27ZtsmuMjIMHD/JCptWqVcPnn3/Oi8VqW/7CNMbt27dzTXHdunWyVCGIiIjgRW2rVavG76ejoyOePHkCANizZ0+J543J2atXL0ydOhUtW7YEAH21RuAl1xx//PFHte//85//AJCqJeTm5qpVZVeladOmpf4mN8XFxVw7CA0NRWRkJIAXhblZJZhZs2bx6vDVqlUzqEypqak8RVx6enqZ+7Zu3ZprQ6r9Oj09Xa0Ae5MmTYx2TU0NK0bt6+vLLYys1iSjTp063GLJCppnZmaWOJetrS0vbj5x4sTyqj7JV5WDiFBYWMjz7rFCo6Xx4YcfAoBaVnczMzPZBxtVYmJi8MMPP+D48eMAtF9AbXTv3p3nGhw8eLDBTYFhYWEAgE8++UStCocmRMQ7+bhx4/ikok6dOryEkCF49uwZXyN17NgxAOCThcmTJ2PYsGF83wkTJvC1RaoDv2pnrwwLFizA559/rtO+7L7169cPAQEBAIAWLVrIIgde0sExIiICADB9+vRyX+6amJmZISIigleHNzZsQlRUVARAcv0Ahh8QNfn5558BSH3h6dOnanlcmYysn7N3r7b3IDP/zpgxQ810/W/h2rVrAKRqICdOnOA5ctPS0kpcN3atmjRpwt+RGzdu5M+ADgizqkAgEAgEumCyqhzGgpnbAgMD1Zy1LDNPly5dYGtry01v3bp1M6hGWxqbN2/G0qVLcfHixRK/DRs2DB06dODO/tq1axtVNma+6tq1q1rGDCJSm52pmjO7dOnCnf1yBTEVFBTw2nzPnj3Dnj17AEhaArs2rEYju4cG0hxeSs2RceDAAV7VhNXuLI3XXnsNgDTDZ8cIJI4cOQJvb2/+9y+//ALgRcAJy/6j+b6ZOHEit86MHDnSGKJWebKzswFINUVZxiMvLy8A4CbTt99+W9/TV41ix4J/Lrdu3cKxY8d4iRhtAzkzXQYHB/MJyEvISz04Ai9M6MOGDcNbb70F4EVUYLNmzQBIJaxYVKuxJ2wCgYwIs6pAIBAIBLogNEeBoOK89JqjQPAvQmiOAoFAIBDoghgcBQKBQCDQQAyOAoFAIBBoIAZHgUAgEAg0EIOjQCAQCAQaiMFRIBAIBAINxOAoEAgEAoEGYnAUvNTcvHkTtra2sLW1hb+/Px4/fmxqkQQCwT8Aw1X+1IGsrCye/T8lJYUXHN69ezfPvJ6YmKi1qPLLTo8ePXDgwAEAwKhRo+Dq6qp1v1atWqFv374Gywe7a9cuDBgwgH/u06dPqfs+evSIF8LdsGEDGjZsCAC8KoYp+PXXX3nJpfDwcJPJIRDoysWLF9G9e3cAwN27d3kB999//x2vv/66KUUzOYcPH8aXX37JP8+dOxdffPGFQdoSmqNAIBAIBBoYVXPMysriRUnj4uJw7NgxXuVBoVCUWd/s34ZSqeTXYdOmTWq/1axZk1fAqFatGtLS0uDk5GQQORISErgc/v7+GDJkCLp16wYAuH37Nvbu3QtAqtCRnZ3Ni5QOGDAAixYtMohMFeHmzZtwc3MztRgvFdOmTQMA7Ny5k3+3bds2vPnmm6YSqQTMInX06FEcPXoUALBmzRoAwNdffw0AataY//73vybVyhITEwEAixcvxqlTp3g1lHXr1vHE7/9mrZFph0xrZKj+LbcGaZTcqvPnzwcAzJ49m79oWWV71QGRFfBt0qQJLz/Sr18/2NvbyyEGf5Hv3r1b7ftTp04hJSWF/92oUSMA4CZCbTg7OyM2NhYAeIFNOXn//fexf/9+AMCkSZPQoEED/tuIESN4+Zt69erxYsSG4MaNG2jfvj0AoG3btrh9+zYyMjL470OGDAEgDZTNmzfnxV2nTJmCGjVqGEwuXWnZsiU3C8+ePbvMfVkR2nKqhgMvaW5VVtJtxYoV/LtTp07x+79jx44Sx7i4uAAAjh8/zj+bksLCQuTl5fGKMPHx8Tod16xZM5w6dQqANPmUC1bd5LPPPuN9xdnZGQB4P9q6dSsvw1RQUIAWLVrwa83eRVWNoqIiPH/+HADw3Xff8c/+/v5wdHSUta1OnTrh8OHD5e5XzlhWFiK3qkAgEAgEumAUs+qff/4JQNIOVU2mCoUCY8eOBQD07duXm+vkpKCgAAAQExODTz/9VE2e0ihLY2RkZGRws+G6desqKWXZDBgwAO+8847ad6NGjTJomwx3d3e0bNkSgKSFRUVFGaXdypKTkwMAuH//Ps6cOQMA+OGHHwCA/39UZ+V//PEHn+UbysFf1WHaijYNEQAGDhyI+vXr8wCRgQMHVgltUZU5c+ZUyJxft25dAJJ2olTKryv07NkTAJCXl4fTp0/z75nlTJM5c+ZgwoQJsmtfusLMucyS1qRJEwCS+4b1j82bN+PXX3/FjRs3Shy/b98+HDp0SDZ5vvjiixJaI7PY+fj4GNQFZ5TBcdmyZQCAM2fOIDU1FQDwyiuvIDw8HH379jVo26xAq2ZFbVawtbxIWG9vb9y/fx8AcOnSJbXfWFX6l53evXsDAKZPn46GDRvigw8+MLFE5cN82bdv3+b3r7yBPSgoyOByVWWWL1/OPzPT6rZt2zBo0CBTiaQTBQUFWLJkCQB1kzAAODk5oUePHgCAMWPGlDjWysoKANC8eXNZZXry5AkGDx6M3NxcAJIi8MorrwAA6tSpgylTpuD9998HULV8iSwa/ZdffkG3bt2wb98+AMCdO3cQGBgIAGqDPOOjjz4CIJmPDUlsbCx8fHz43z4+PvD29jZIW8KsKhAIBAKBBkbRHFmEampqKleDjaE1AlAzsbCZ2o4dO2BmZgZAMhcIyoY5258+fYrt27dzjbl6dZMuky0TNuMFXgRgWVtb48KFC1rN6r169Sphuv63snz5ch6JeurUqSqvOSYlJWHOnDlq33l4eACQImqbNm1qdJn27duH/fv38/fd9OnT0b9/fwDgAU5Vhbt37wIAZs6cyYMAnZ2d8c033/A+/uqrr8LLywsA4OvrW0ILZ4F4derUkVU2Fo2qakpVhX1vCAz+dktMTORRgkQEBwcHADDKwLh161a+tAAAJk6cCODFjVSFZU7Jzc3l/qrdu3fDx8cHnp6eAF6YYIxB3759+YO6adMmvPHGG9wsY2zYvTp06BD27NmDhIQEACU7+eHDh7F48WIA4EtNAMDOzg6ffvqpUUP9nzx5AgDw9PREx44dAQCWlpY8LF5QNizCd8WKFTyJQ1XyLxYUFCApKQmAFBmtipubG0+gUVWW8bRr167KDYqAFN07c+ZMAMD69et5pG5oaCifYDCYz96Y+Pj46BSpaggMPjhOmzYN2dnZAABHR0f+0BqDsLAwtfBett7p0aNHvGOxFz1zLqsu6WCwpRrLly/n/gtD89///hc2NjYApP/H1atXeUBRp06dYGlpaRQ5AGm5CCAFCxw4cID7ICdMmMD3OXLkCE6ePMk1ctVw+GfPnuHAgQPcpxcUFGTQZR4PHz7kQVJjxowx6rV6WVB9kbMAHba+kS35uHnzJt/HxcUF9evXN9oAeufOHbRp06bE940aNUJERESVGRQZZ8+e5ROOqsKhQ4ewePFinD9/HgDQpUsXfP755wBKamimwtvbWy0rjjHlEj5HgUAgEAg0IaKytkoxYsQIAkAKhYIUCgU5ODjQ2LFjaezYsZSQkECPHz+ubBNlMmTIEIK08FmWrW/fvgaVV5PTp0/T6dOnaciQIWRjY8Ovo7e3N//N2Kxdu5YsLCzIwsKClEolubi4kIuLC3Xq1Ik+/vhjSkpKoqSkJLVjHj16RN27dyelUklKpZJWrlxpUBl//PFHfs+cnJwoMDCQAgMD6ZtvvqGsrCw5miiv31TVTWfS09MpPT2dAPB7vGzZMnJxcSmzjwwcOJAGDhxI6enpFWmuwkRFRfH+oLq1bt2aDh48SBcuXKALFy4YVIbyGDduHL8uCoWCGjduTI0bN6aAgACTynXv3j26d+8eubu7q907Ozs7cnBwIAcHB2revDlt27aNHjx4QA8ePKCCggKTyasqY2xsrEGa0LYZNENO27ZtkZiYWCIrDvvctGlTfPXVVwCkTDhyk5OTg+DgYACSY/7vv/8GIDmN/fz8+H61atVS+1uV2bNn84AOc3NzHppt7OwvW7Zs4f+XpKQkvj5rz549aNu2rVFlmTp1KgBpvejSpUsBoNygh9zcXJ4A3NHREcePHwcA1K5dW1bZnj17Bh8fH276c3JywqNHjwBIftAuXbogLCwMAEpN5q4DL2WGHG0MGjRIbd1j+/btuel0wIAB3I+8Y8cO3Lp1Sy2lHLvHcppa2Rq64cOH4969e6Xu5+7uDkByQajSvXt3owYYMR/31atX+ZIiQLp2LFPOhAkTjLqc49q1awCkZWx5eXla3x95eXnc9QQA8+bN475nOTMI6YJqhhwfHx/MnTuXf5YJkSFHIBAIBAJdMKjmmJKSAm9vbx6Qo6k5KhQKnjc1ISGhMjN5g3HgwAG1IBxW9mjEiBFGl4Vdx6lTp3Jt1sbGBpcuXZJdA3vw4AEAeTU7lhzd3Nyc57xks2e5WLx4MebMmYPvv/8egJTAIC8vDwAQHByMDRs2YPjw4QDANUg9+Ndojtu3b+faYEBAQLkRxyzpxuDBg3lQz/bt22XRHh8+fMgtJoWFhXqdo3r16lyWffv2lYjINBR//vknj/bMyMjA+fPneSauOnXq8HJ9LKe0Mbh8+TLy8vK0Bjbl5eUhMjKSW4YuXLiAn3/+GcCLfMrGRFsmnHLGrgqdXuu3pdlbSQafIxFRWloaJSYmlthmzZpFSqWS+wrCw8PlaE529u/fr2bzDg8PrxKyLliwgBYsWEAKhYKOHj0qyzlzcnIoJyeHhg8fTu3ataN27drJcl6Gk5MTOTk5kVKp1OqbrAwxMTEUExNDFhYWNHHixFL36927N/eZ/v777/o2Z2rfocF9jpVl2bJlvM9s27ZNlnM+ePBAq59R3+3rr7+WRS59ePjwIc2dO5fmzp1LSqWSbG1tydbWlhISEmRvKz09na5cuUJXrlyp8LEHDx6kgwcPUqNGjWjlypUGjxcoDXatVN/FPj4+cvkgtfYXgy/lcHV11aoR7tq1iwtRldF3hmpo2PIKQNKI2rVrB0CnihKlwkLzf/75Z8yYMaNyAmrw9OlTnkzAzc2Nr3eVC+YHeeedd/iSA23MmjUL0dHRAIDk5GR07txZVjkEEtOmTcP06dON0pa9vX2ZRbhZysq4uDijyKMLdnZ2PIevUqnklYs6deqEuLg4nv9XDqZOncr9nTt27KhQ31P1R27duhWA5CNlS7aMhWq+Y9Vix4cPHy41QUBlET5HgUAgEAg0MHr+L7bIfsGCBWp2ZFOkeNKFLVu2mFqEcomOjuZ+tcr4CJmG37RpU/z+++8AJD+THFpeVFQUn71Onz5dds2R+cJ+++23Mvdr06YNr/6yZcsWjBs3DgCMPhP+N7Fz505ZIkStrKxw8ODBEt9bW1tr9ZsxmG+5KmmOqsyZM4dnw4qPj8e+fft4/VZra+tKn/+TTz7hke6dO3fmWpguKwTy8/MBSJWKmL/XlMXov/jiC64hskhk9q/cVki9B8f58+fzcOl+/frplNosMTGRB7ew/whbFlBedQxT8Pz5c750AwAcHBx4QeaXEZaRZ9KkSRg/fjwAqZCpHCWcVMPCGzZsWOnzVYbXXnsNgDRgs1SBcg/WghfIlTbNzMysxNKM8tizZ0+JCRMLAmSuiKoAyz9dr149zJ49G1lZWQBKVhnRh1atWvESWAcPHuTVhby9vUvNhfrs2TOsXr2aZ+WqXbs2l8XU+ajZ4BgbG6v2PHTq1EnWXKvCrCoQCAQCgSalRepQGdFtycnJZGVlxaO+Zs2aVWYoUHJyMiUnJ1PHjh15lhSFQkFKpZKOHj0qW7SlXBQXF1NxcXGJ6Kjx48ebWjROQEAABQQEkEKhoFGjRlFRUREVFRXJdn6WzUOhUFBcXBzFxcXpfa6LFy+SQqEgGxsbsrGxoatXr1Zavt27d1NWVlaFM95cu3aN7OzsyM7OjqZMmcLvdQUxddRplY9WZdl1IGO0qiaZmZmUmZlJ165dK/Hbxo0baePGjWRnZ6cWoerg4EDR0dEUHR1tEJkYqamptHz58gof9/HHH5NSqaTBgwfT4MGDZZOH9WEvLy9+X5ydnalv3760atUqWrVqFR08eJAWLlxICxcuJB8fH7V339atW2WTpSKUF42q+Y5mUa0VRL5oVXt7e9jb23M/19dff42uXbsCAK+AAABZWVnYvHkzFi5cCADIzMzk9mpW7LgqlgmaNWsWAHA7PavGMWnSJFnOzypqr1+/vkJr7dgasq+//pr7bps2bYoFCxbIbupgZp733nsPw4YNAyD560JDQ3U2QV69ehWAlLBdoVDw6yqHWTUrK4sncu7cuTOvHFJW0drLly/jo48+4s9g586dDVL9XQC1SFVDVGN58OABBg8eDAD4+++/cebMGf7brl27ePHdoqIiteNGjx6N7t27yy6PJn/88QemT5+ORo0aAXhRLq807ty5AwBYvXo1FAqF2ntUDtj6yejoaJ5h5vr164iMjOR9XRss01STJk1klUdXvvzyS3Tq1EktK45qVKq2KFa50DsJwOXLl3kQjUKh4MEcY8aM4SH7YWFhSEtL05o+bufOnQYpW8VKFc2fPx/r169Hly5dALxYvK+NzMxMrF69GoDkY8vMzATwoo7hnj17AEg1/+SApWEKCQnBxo0bAQA9e/bknzW5fv06Nm/ezNPfFRUV8YW4a9asMWg6p5SUFO4nTktLQ5s2bfgD2blz5xJp9Fjpr8jISF6qLD8/H0uXLuWDrFwDUlRUFADpvrC6c9bW1ujZs6fafix93IEDB1BUVMSDrIYOHapv0/+aJADAiyU+06dP5xO00mAv07feeov7Gtl3cnL9+nU+yapRowavUu/q6orAwEDeV1R5++23sXv3btkTZmhjx44dGDJkCPf17dq1i/v3Xn/9dbVr8ueff+KTTz4BIPWfxo0b8wm03PURVXn+/DnWrFnD+6wq3bp1Q+PGjXm/MlUQzhdffFHqoKetnBUbRCsYJyHSxwkEAoFAoAuVSh/n7e0NQAqRZudRKBQlPrPZ06RJk7hpzVAcOXIEwIuIJlan8dVXX+X7nDx5kqdrio+Px40bN5CRkVHiXBYWFvi///s/XoewMgvsVWGz7yFDhnDNq1q1alzrLY3Ro0cDkEK/WVi1tsLNcsMSFYeEhCA2NpZHnr755ps8GQGzCrDi0snJyfy3KVOmGGRBOHvOrly5wq/pzp078ccff/B9unbtCltbWwBSgvmpU6dyE1ElZsP/Ks2R4erqivbt23PLh6a59OTJk9zUefPmTWzbtg0ADJLoOzc3F/7+/gBeWBBKo0OHDgCkReyqyTMMycmTJ9GtWzeulSkUCq4Fvvbaazh16pTW58/S0hLh4eFGKQb/T6Es7VETOTXHSg2O6enpACTTHnsp7t69W21wnDlzJsaMGQOgUlUQdIb58JjPQR9Yto21a9caxKzBrs/evXtx9uxZAJLp9sKFC/xF0rhxY369Ro4cCQAmN3EA0kuJZfPYs2cP9yuywfHDDz8EIA3abDlIVV3DWgn+lYPj9u3bERgYyM2sLi4uass0Tp06xT8vXbrU4NUvWMaWH374odQ1jO3bt+drdnVZbiYnGRkZPDfpuXPn+Pdnz55Fbm6uWj9mS8T27Nkje77hl4nDhw+Xupxn7ty5+i47E2ZVgUAgEAh0waBVOUzB6dOnAQA9evTglSXKom7duujUqRM3ndauXZvP3Ey92FVQZflXao6AZC5l9R137Nihpi22b9+em7flrOFYHg8ePMCoUaMASJU2AGDDhg0ApIXhxpRF8I9EfrNqVSY5ORlLly7lEaCenp48C0/79u15qZpmzZoZNCJM8FLyrx0cBYKXEGFWFQgEAoFAF15azVEgMCBCcxQIXh6E5igQCAQCgS6IwVEgEAgEAg3E4CgQCAQCgQZicBQIBAKBQAMxOAoEAoFAoIEYHAUCgUAg0EAMjgKYa1huAAAU/0lEQVSBQCAQaKBXsWN9SUxM5Blr8vLykJmZyeuavfLKK/xz/fr1jSmWGvn5+ViyZAkAqSZkYWEhr/vH6gAK/jlkZGRg3rx5AKTE2CEhIbzwq6BypKWlYcWKFQCAhIQEbN261aR9VyCQE4MPjnFxcTwzfXx8PO7du6d1v4cPH/L0bufOnTN4aZm8vDwAL3KxHj16FAAQExOjli/SlBUwdCUlJYWXu0pJSeFlugxRHiw/P5+XhKpZsyYvR6ZUKlGrVi0kJCQAALy8vHD9+nUAQE5Ojto5mjdvDisrK9llU4VVB4mKikJ2djYA6V4GBATA19eX78fKV/Xq1cvgMr1sfP3117h48SIAYNmyZVW2mkRRUREAICsrq8z9nJycAMhXjFsfsrKyeHFuhpubGwDAzMzMFCL9axFmVYFAIBAINDCI5shS0q1btw6ffPIJCgoKSuyjrXCwnZ1dqb/JSWFhIS+UumfPHgBQq0HJGD16NN5//320bdvWoPJUlNWrV3ONJyUlBcHBwUhLSwMgyc/q1jVr1kzWoqkpKSkYOHAgL3ZsYWGBWrVqAZAqmFhbWyMlJQUA4OHhgdu3bwMA/v77b7Xz9OzZE7/88otscmnj/v37av8yEhMTkZiYWGL/nj17IjIyEsCLupkC7Vy6dAkAsH79el7c+MaNG3j27Bm8vLz4fqxPVa9e3ehaOSvQ7ebmhgULFgAovwAueycw14+xuHv3Ln766ScAwPfff4+//vpL7Xdm6XjttdewatUqo8pWVXj48CEAyaoYHR0NAFiyZEkJyx57V/fo0YMXPtYXg+RWZf8RzWoXLi4uvHr45s2bTfYSKigo4Bfx0qVLeOWVV9CzZ08AgJ+fH/r16wdAqspdVcyq48aNw+rVqwG8KCys+ll1cGefZ82axf1tlYGZJT09PXH37l2t+9jZ2aGwsBD/+c9/AAAODg4l9mHPQ9++fTFs2LBKy1UWx44dAwB07NhR68SH4e3tDQCYOHEiBgwYoOvpq8ZDUXFkya3K3CTz58/nJr8rV67g6dOn6o39/+tujMmQKn/88Qfv376+vmW23axZMwDAo0ePcOvWLQDA8+fPDSofez+yieXkyZN1GvQ++OADrFu3zqCyVUUSExPRu3dvAFIMAcPZ2Rn5+fkl3DYA4OjoqLZvOYjcqgKBQCAQ6IJBVDdm0mB88MEHAIDg4GAewGFKCgsLuWkIACIiItC9e3cTSlQ+kZGRapoP++zh4YFmzZrB3t6e/8Y+y2VSffz4MQDJ/FOvXj1e0JYFMACSifX58+d8Nmzq4Bam0QCAjY0NACkgrFatWlxDcHd3578JU6pu5ObmYvbs2QCArl27Yu/evQCkGX1hYSGPOH/48CFcXV0BwKjRwd9++y2mTZvGC5UrlUpYWFgAkKxC7F3E6NixIwApEIa5BAwFC7QZNmwYunTpgunTpwOA2rsIADp06IDAwEAA4JYY4IWWayyePHmC9PR0AEB6ejpyc3MBSAF1mZmZ/Lc2bdrwY9zc3GBpaSmrHCEhIWpa4KeffgpAsqbl5eVp1RDlcM3J/kYoKirC/Pnz1b5jvqcFCxYgIiKCf9+jRw/uBzBlpNvrr79usrZ14ejRo6hduzaPtiMi3qkPHz5s8PbXr1/PPwcEBOCtt94yeJuVJSoqin9mnbdx48YAUCUmaP9Ujhw5ws2lX331Ff+e9V82IBqbkJAQAMBnn32GoqIiPpD079+fD+ZvvPFGqce7ubmpTagMAYsej46ORo8ePfj3c+fOVfNz1qlTh8cNmIrly5dj48aNPBpZE1XXjiotWrTA+fPnZZMjPz+fR70DwIwZM/hzxya0zZs3l609VYRZVSAQCAQCDWTXHCMiIriphfHbb7+p/ctYs2YNX1/Yq1cvLFq0SG5xtGJmZsZnG5cuXUJ+fj7XdlXXOE2cOBG2trZGkaksLl++jNTUVLWZmpxRqGWRmpqKZcuW8b9DQkLU1mGxCM/s7GwMGjSIX69WrVrxmbqdnZ3Rr+ODBw/4586dOxu17ZeZL774An369AEg3eOqwG+//YYZM2YAAJ49ewYAPKJR1fRvCoqLiwEAQUFB+PHHHwFILpEWLVrwfd58802DR+jrCruOixcvho2NDe/7mv23NM1Rbitc//79cfz4cR4499VXXxnPBUJEZW0VZuDAgaRQKCq8WVhY0IYNG2jDhg36NFshnj59Sp6enuTp6cnbhxTJpyaTpaUlnT592uDylMaNGzfoxo0b5OjoqCYbAJo1axbNmjXL4DJERUXxa6PvNnz4cLp3757BZWVcu3aN7O3tyd7enhQKBV2+fJkuX74sZxPl9ZuqulWKX3/9laysrOjQoUN06NChyp6u0ty8eZNu3rxJrVq1Uuu348ePN7VoRESUl5dHn332GX322Wdq/aF79+6mFk2N4uJiKi4upsTERKpXrx7Vq1ePatWqRd9//72pRePvPXYdMzIy1Lbs7GxZmtG2ybaUIzk5GYDk32Eh3dWrV8cbb7yBbt26AZD8Ev379wcgZaiZP38+Nm3aBECaYbHZU3JyMho0aKBr0xUmLy+PB2IwmB+tb9++3OELAF26dMG2bdsAvAi9NgYpKSn8WqWmpmpdvgFImi9b82gI/vrrL77EwcfHp0TmIubHy8nJwdOnT7lWuXXrVly+fBmAdL1btmyJM2fOADB8po+goCAsXrwYgHQ/g4KCAEiz4YsXL/JZ6MqVK/UNHvhXLeVgAUxt27aFubk5X1Nrag4cOAAAfBkWw8rKiluhxo8fb3S5GFu2bMGIESMAvFjWAgAjR47EjBkzuDZm6Gxg5cGyhXl6evJ7e+zYsSoRW6BUKktdTkdEqFWrFo+J6Nq1q6z9WfgcBQKBQCDQpDSVkvQ0w/Tr148aNGhADRo0oAMHDpS7f0BAAAUEBKiZRY4ePapP0zpTXFxMwcHBFBwcTA0aNKANGzZQTk4O5eTk0JMnTygqKoqioqLI3NycFAoF/fjjj/Tjjz8aVCZGQkICJSQkUMeOHdVMvdAwq7LPAQEBRpFLH5KSkigpKYlsbW0JAM2bN4/mzZtn8HbHjx/Pr0+9evW0mswVCgX16dOH7t+/T/fv369oE6Y2jxrVrDpx4kSaOHEiKRSKEm4GZvqPioqiW7du6duEXhQWFlJhYSGdOXOG/Pz8yM/Pr4SbxNramlxdXcnV1ZXWrl1rVPmIiLs/3n77bfLy8iIvLy8um4uLC7m4uNCXX37JTZumwN/fn/z9/dX6hp2dHX322We0ZcsW2rJli9Flio+Pp/j4eFIoFKRUKsnR0ZEcHR3J19eXPDw8yMPDg9zd3UmpVPJt7969+jantb/I3pmys7O5L0AX7t27R/fu3aPmzZvzGzN58mR9mpadmjVrEgAaOnQoDR061ODtJScn84dA9YFYuXIlJSYmUnh4OIWHh/MHRqlUUmhoqMHlqizR0dFkbW1NLVq0oBYtWhi8va5du2r1fdatW5fCwsLIycmJnJycCABFR0dTdHR0RZsw9SBntMHxyJEj/Pp5eHiQmZlZmf5lX19f8vX1pbS0NH2ak42CggIqKCggHx8fMjMz43J/+umnVFRUREVFRUaX6fHjx/T48WOKi4uj8ePHU69evahXr15Uo0YN8vb2Jm9vb/rtt9+MLldgYCAFBgZSt27daODAgTxuBADVqFGDatSoQWFhYUaXi4ho5MiRtHXrVvrrr7/or7/+IiKizMxMyszMpDt37lBQUBB/F44aNUrfZrT2F2FWFQgEAoFAk9JGTaqEGUYfVE2rFhYWxmy6VIYOHUoKhYLeffddevfdd6mgoMCg7Y0YMULNBNimTRtq06YNZWVlEdGLGZOHhwfXKiMiIgwqk1y8++67PELY0OYjVbNq69atKSwsjMLCwigjI4OIiJvNzczMaMyYMTRmzJiKNmFqDdDgmmNeXh7l5eVRhw4d1J5Jb29v+u677+i7776jCxcuqG3BwcFkZ2dHdnZ21KhRo4o0VyrM3FiZaGdmGmTPxIEDB3Ry+RiLn376ictmZmZGUVFRphaJiIiOHj1KdevWpbp16xIAys7Olis6VDbWrl3LNce2bdvy57aCaO0vVTJnlrYqHpUlMzOTZ1ZYsmSJTlFNs2bNwtatWxEbGwsAOH78ODp16iS7bIwmTZrwyCyFQoHly5cDeJEOjq0pTE1N5VleWJL0qk6PHj3wzTffAJAqZWhLTC4X5ubmqFmzJgBg0KBBvLYj4/333wcgrd1ikciff/65wTOk/JOIj48HACQlJeGjjz4CIEV+enl5lRo96OnpCQ8PDwDyPZcsUrZp06aIjo7WKxWdanozALwChmpdT32ZN28e9u/fDwA8dV5FGTZsGEaPHg1ASm156NAh/oyaknfeeYdXSmrcuHGJCH9TU1xcjF9//VXyD0JKJ8f6vRxUmcGRhf3LTWZmJgApPdTChQsBgOdaLI8mTZpg4MCB2LFjBwDg0KFDBh0cPTw8MHXqVADSEoR33nmH/5aWlsbTYBEROnToIGvbeXl5Bs2Hum/fPr4URq6BkZWe8vT0VFtEvXLlSqxcubLc4/Ly8vhE7OLFi2JwVIE956yChK6wgUIuvvzySwBSSL8+Yfq3bt1CWFgY/9vKygrTpk2TTb79+/fzd8yTJ0/0Svt27NgxniwAeFGcWS5WrVrFU09OmDABdevW1em4lJQU3LlzB4A0IWe5VTWrLZmKrKwsRERE8Mma3KUFhc9RIBAIBAINdNYcL126xM0RCoUCEyZM4L85OzvzLPgVhZnaWLonAGqL8CsLWyhcUFCA9u3bA9Be108bCoVC7/+XPvTr10+rOSolJQWff/45r6vo6OjIE4/LhaurK77//nsAUo1DORPBb968GUlJSTrPWHUhPz+fm9jq16/P69ypatvauH37Nk8nV1BQwLVlY1c8eBlRrcohV9o2dl8VCgUCAwN5OrPWrVuXeRzTxAICAnixAysrK4SGhqoVZK4sSUlJvJh3gwYNeDWN0aNHq1XK0QbTyubNm8drSNaqVQuTJk2STT4AiImJ4TUtk5KSMHToUABStRTNPsksKd9++y3mz5+P/Px8AJJLx5hJUHRh586dAF64neS2fOk8OHp6eqoNKsxECUi2e2aq7NevHwYNGgSg/LIhixYtQnBwMP+bqetyPhzsobC3t6+wOeDBgweIiYmRTZaKwrJV9O/fHykpKTyzy+TJk2X3Nbq5ufFOU7t2bT5p6dGjB2rXrq3XOc+ePQsAWL16NTIzM/Hee+/JI6wG165d41lSzp8/XyK7UmFhIQCpUkdAQAA3D5mZmfEJn2ppIIF+TJ48mZdfYtmQKsvBgwcBAP/73/9w5MgR+Pj4AJDyurLsMy4uLiWOY/EFp0+f5n6o0NBQ/ozLxW+//cbLYF2+fJlP7L/99lu89dZbCAgIAKA9KxTLj3zjxg3+/tywYQMaNWokq4ybN2/Gzz//DEB6b7PYBScnJzRs2FDtvf7kyRMAwLlz50BEfALKig1XBdgzxgq5Dx8+HID8fViYVQUCgUAg0EDn3KrHjh3Dli1bAEiz81OnTpV6EMu3+eqrr2LgwIFa9wkNDcX169f5LB4AL/65ZMkSXeUvF2trawCS2ez48eMAoLOptHfv3oiKiuJZ4K9evWq0enXz58/HqlWrAEgVLxo3bsxNz4aQ4ffff+e5XFXviaOjI8aNGwdAmq03b95cp5ltcHAwD4rJzMxEp06duInM3d1dFplZ9ZfBgwdzc9DAgQOxYMECNGzYEIA0K2eBTExLZEyZMgUrVqzQp+mXOrdqUVER1q5dC0AqVF6jRo1S92V5lGfPno2VK1dybS40NFTWShOpqamYPn26zgE/7L1maWmJjz/+GADKDNKqDEzbiouLw9atWwFIJj+Ws7Q8XF1d4enpCeCFpcuQMG02ISEBx44d07pP3bp18emnn8oavCQXrG9fv34dzZs3R0JCAoBKFTjW2p/1Sjz+/Plzteiq0NBQvuwgIyOjQksxWHiwv78/N+Wplo2qLOxFnJ6ejs2bNwOQwvs1y54wf56FhQUvX7V8+XIUFRVxMy+TrzL069cPu3fvBqCeQJx9ZvdD9bO3tzd++OEHgyYYB6Tk4QCwadMmHiWoWvoJAGrWrMmjBjt06MBlOnz4MK5evcr3e/DgAfejvPfee1i5cqXBikpPmjSJF4vNy8uDubk5v7/FxcX8Bc58yMz0NWfOHJ0jlzV4qQfHR48ecf/SiRMnuK8ekJ5TtrwiLi4O3377LQBpwvzVV1/xaGtDlBV69uwZX2KiWoBbG+xFOWPGDNkmYxWBlcJjk6+EhARcuXKF/+7s7Mwjznv27MmXchib1NRUrd/b2trKGiNQWdiYMnbsWF6somfPnli0aBGaNm1a2dOLxOMCgUAgEOiCbCWrGBs3buQaGnOma6NLly5wc3PD5MmTAYCbFeSGmYeYaQWQIho1Z7ZsNlyzZk2cP38eAFCjRg2MGjWKlz+SYxGsagmW8jRHFoV14MCBcqPz5IYFA4WGhnJTz8WLF3U+3sLCgptvxo4da/A1hKw0UWhoaInf2DX18/PDzJkz1TQhPXmpNcenT5/yhfN2dnZ8QfrRo0eRk5Oj5lJhvy1atMjglo1/MtnZ2WpFwl955RVZI8JfZtLT03nwzbp163gClIMHD8qhNQJymlXLg5lc//rrLwDAmjVrAEhRkMyv1bBhQ1nNp6XBwqw//vhjng1FG+w6uLm58SiySZMmyV5XskmTJtyUQUT8haKZ2aFv376YOXOmrG3rC4v2LCwsxKZNm3i19RMnTvB9WO23+vXrA5BMqXrWVquUjMeOHcPkyZORlJTEf/v9998BSGbgsvxnFeClHhyBF5HGw4cP5wk6FAoFfH19edaUAQMG/GMyNAn+mZw7dw5+fn7IyMgAIMWxREVFAZBiIGTCeIOjoHSePHmilg2IpdvSJ7OGwGS89IOjQGBK2HKNzp07Izs7m0++t2zZonXpTiURPkeBQCAQCHRBaI4CQcURmqNAYEDYco0BAwbA3t4eY8aMAQBu0pcZYVYVCGRCDI4CwcuDMKsKBAKBQKALYnAUCAQCgUCD8tJY/FPNRwKBoCSiPwsEOiI0R4FAIBAINBCDo0AgEAgEGojBUSAQCAQCDcTgKBAIBAKBBmJwFAgEAoFAAzE4CgQCgUCgwf8DmhStq4Dbd+IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,8))\n",
    "plt.subplot(221); plot_digits(X_aa[:25], images_per_row=5)\n",
    "plt.subplot(222); plot_digits(X_ab[:25], images_per_row=5)\n",
    "plt.subplot(223); plot_digits(X_ba[:25], images_per_row=5)\n",
    "plt.subplot(224); plot_digits(X_bb[:25], images_per_row=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The two 5x5 blocks on the left show instances classified as 3s, and the two 5x5 blocks on the right show instances classified as 5s.\n",
    "- Most misclassifed images seem like obvious mistakes to us.\n",
    "    - And it's hard for us to understand why the classifier made the mistakes it did.\n",
    "- The true reason is that when we used `SGDClassifier`, which is a linear model, it just assigns weights to each pixel, and when it gets a new image, it sums up the pixel intensities times the trained weights, and since 5s and 3s share most of the pixel intensity locations, the model get sometimes confused.\n",
    "- The main difference between a 5 and a 3 is the position of the small line that joins the top line to the bottom arc.\n",
    "- Meaning if you draw a 3 with a slightly shifted line to the left, the classifier might assign it a 5, and same to 5s classified as 3s.\n",
    "- In other words, this classifier is quite sensitive to image rotation and shifting.\n",
    "- So one way to reduce errors is to preprocess the images to make sure the digits are well centered and not too rotated."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multilabel Classification\n",
    "\n",
    "- In some cases, you want your classifier to output multiple classes per instance.\n",
    "    - e.g. Face recognition in images.\n",
    "        - What should it do when it recognizes several people in a picture?\n",
    "- Such system that outputs multiple binary scores that don't necessarly sum up to 1 is called a multilabel classifier.\n",
    "- Let's go through a simple example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_multilabel = np.c_[y_train_large, y_train_odd]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_clf = KNeighborsClassifier(n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=-1, n_neighbors=5, p=2,\n",
       "                     weights='uniform')"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.fit(X=X_train, y=y_multilabel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Now you can make a prediction, and notice it outputs multiple labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False,  True]])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.predict(X=[some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fa6d52866d8>"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOUElEQVR4nO3dX4xUdZrG8ecFwT8MKiyt2zJEZtGYIRqBlLAJG0Qni38SBS5mAzGIxogXIDMJxEW5gAsvjO7MZBQzplEDbEYmhJEIiRkHCcYQE0OhTAuLLGpapkeEIkTH0QsU373ow6bFrl81VafqlP1+P0mnquup0+dNhYdTXae6fubuAjD0DSt6AACtQdmBICg7EARlB4Kg7EAQF7RyZ+PGjfOJEye2cpdAKD09PTp58qQNlDVUdjO7XdJvJQ2X9Ly7P5G6/8SJE1UulxvZJYCEUqlUNav7abyZDZf0rKQ7JE2WtNDMJtf78wA0VyO/s0+X9IG7f+TupyX9QdLcfMYCkLdGyj5e0l/7fd+b3fYdZrbEzMpmVq5UKg3sDkAjGin7QC8CfO+9t+7e5e4ldy91dHQ0sDsAjWik7L2SJvT7/seSPmlsHADN0kjZ90q61sx+YmYjJS2QtD2fsQDkre5Tb+7+jZktk/Sa+k69vejuB3ObDECuGjrP7u6vSno1p1kANBFvlwWCoOxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiCIhlZxRfs7c+ZMMv/888+buv9169ZVzb766qvktocPH07mzz77bDJfuXJl1Wzz5s3JbS+66KJkvmrVqmS+Zs2aZF6EhspuZj2SvpB0RtI37l7KYygA+cvjyH6Lu5/M4ecAaCJ+ZweCaLTsLunPZrbPzJYMdAczW2JmZTMrVyqVBncHoF6Nln2mu0+TdIekpWY269w7uHuXu5fcvdTR0dHg7gDUq6Gyu/sn2eUJSdskTc9jKAD5q7vsZjbKzEafvS5pjqQDeQ0GIF+NvBp/paRtZnb257zk7n/KZaoh5ujRo8n89OnTyfytt95K5nv27KmaffbZZ8ltt27dmsyLNGHChGT+8MMPJ/Nt27ZVzUaPHp3c9sYbb0zmN998czJvR3WX3d0/kpR+RAC0DU69AUFQdiAIyg4EQdmBICg7EAR/4pqDd999N5nfeuutybzZf2baroYPH57MH3/88WQ+atSoZH7PPfdUza666qrktmPGjEnm1113XTJvRxzZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIzrPn4Oqrr07m48aNS+btfJ59xowZybzW+ejdu3dXzUaOHJncdtGiRckc54cjOxAEZQeCoOxAEJQdCIKyA0FQdiAIyg4EwXn2HIwdOzaZP/XUU8l8x44dyXzq1KnJfPny5ck8ZcqUKcn89ddfT+a1/qb8wIHqSwk8/fTTyW2RL47sQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxAE59lbYN68ecm81ufK11peuLu7u2r2/PPPJ7dduXJlMq91Hr2W66+/vmrW1dXV0M/G+al5ZDezF83shJkd6HfbWDPbaWZHssv0JxgAKNxgnsZvkHT7ObetkrTL3a+VtCv7HkAbq1l2d39T0qlzbp4raWN2faOk9PNUAIWr9wW6K939mCRll1dUu6OZLTGzspmVK5VKnbsD0Kimvxrv7l3uXnL3UkdHR7N3B6CKest+3Mw6JSm7PJHfSACaod6yb5e0OLu+WNIr+YwDoFlqnmc3s82SZksaZ2a9ktZIekLSFjN7QNJRST9v5pBD3aWXXtrQ9pdddlnd29Y6D79gwYJkPmwY78v6oahZdndfWCX6Wc6zAGgi/lsGgqDsQBCUHQiCsgNBUHYgCP7EdQhYu3Zt1Wzfvn3Jbd94441kXuujpOfMmZPM0T44sgNBUHYgCMoOBEHZgSAoOxAEZQeCoOxAEJxnHwJSH/e8fv365LbTpk1L5g8++GAyv+WWW5J5qVSqmi1dujS5rZklc5wfjuxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EATn2Ye4SZMmJfMNGzYk8/vvvz+Zb9q0qe78yy+/TG577733JvPOzs5kju/iyA4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQXCePbj58+cn82uuuSaZr1ixIpmnPnf+0UcfTW778ccfJ/PVq1cn8/HjxyfzaGoe2c3sRTM7YWYH+t221sz+Zmb7s687mzsmgEYN5mn8Bkm3D3D7b9x9Svb1ar5jAchbzbK7+5uSTrVgFgBN1MgLdMvMrDt7mj+m2p3MbImZlc2sXKlUGtgdgEbUW/bfSZokaYqkY5J+Ve2O7t7l7iV3L3V0dNS5OwCNqqvs7n7c3c+4+7eS1kuanu9YAPJWV9nNrP/fFs6XdKDafQG0h5rn2c1ss6TZksaZWa+kNZJmm9kUSS6pR9JDTZwRBbrhhhuS+ZYtW5L5jh07qmb33XdfctvnnnsumR85ciSZ79y5M5lHU7Ps7r5wgJtfaMIsAJqIt8sCQVB2IAjKDgRB2YEgKDsQhLl7y3ZWKpW8XC63bH9obxdeeGEy//rrr5P5iBEjkvlrr71WNZs9e3Zy2x+qUqmkcrk84FrXHNmBICg7EARlB4Kg7EAQlB0IgrIDQVB2IAg+ShpJ3d3dyXzr1q3JfO/evVWzWufRa5k8eXIynzVrVkM/f6jhyA4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQXCefYg7fPhwMn/mmWeS+csvv5zMP/300/OeabAuuCD9z7OzszOZDxvGsaw/Hg0gCMoOBEHZgSAoOxAEZQeCoOxAEJQdCILz7D8Atc5lv/TSS1WzdevWJbft6empZ6Rc3HTTTcl89erVyfzuu+/Oc5whr+aR3cwmmNluMztkZgfN7BfZ7WPNbKeZHckuxzR/XAD1GszT+G8krXD3n0r6V0lLzWyypFWSdrn7tZJ2Zd8DaFM1y+7ux9z9nez6F5IOSRovaa6kjdndNkqa16whATTuvF6gM7OJkqZKelvSle5+TOr7D0HSFVW2WWJmZTMrVyqVxqYFULdBl93MfiTpj5J+6e5/H+x27t7l7iV3L3V0dNQzI4AcDKrsZjZCfUX/vbuf/TOo42bWmeWdkk40Z0QAeah56s3MTNILkg65+6/7RdslLZb0RHb5SlMmHAKOHz+ezA8ePJjMly1blszff//9854pLzNmzEjmjzzySNVs7ty5yW35E9V8DeY8+0xJiyS9Z2b7s9seU1/Jt5jZA5KOSvp5c0YEkIeaZXf3PZIGXNxd0s/yHQdAs/A8CQiCsgNBUHYgCMoOBEHZgSD4E9dBOnXqVNXsoYceSm67f//+ZP7hhx/WNVMeZs6cmcxXrFiRzG+77bZkfvHFF5/3TGgOjuxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EESY8+xvv/12Mn/yySeT+d69e6tmvb29dc2Ul0suuaRqtnz58uS2tT6uedSoUXXNhPbDkR0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgghznn3btm0N5Y2YPHlyMr/rrruS+fDhw5P5ypUrq2aXX355clvEwZEdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Iwd0/fwWyCpE2S/lnSt5K63P23ZrZW0oOSKtldH3P3V1M/q1QqeblcbnhoAAMrlUoql8sDrro8mDfVfCNphbu/Y2ajJe0zs51Z9ht3/6+8BgXQPINZn/2YpGPZ9S/M7JCk8c0eDEC+zut3djObKGmqpLOf8bTMzLrN7EUzG1NlmyVmVjazcqVSGeguAFpg0GU3sx9J+qOkX7r73yX9TtIkSVPUd+T/1UDbuXuXu5fcvdTR0ZHDyADqMaiym9kI9RX99+7+siS5+3F3P+Pu30paL2l688YE0KiaZTczk/SCpEPu/ut+t3f2u9t8SQfyHw9AXgbzavxMSYskvWdmZ9cefkzSQjObIskl9UhKr1sMoFCDeTV+j6SBztslz6kDaC+8gw4IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxBEzY+SznVnZhVJH/e7aZykky0b4Py062ztOpfEbPXKc7ar3X3Az39radm/t3OzsruXChsgoV1na9e5JGarV6tm42k8EARlB4IouuxdBe8/pV1na9e5JGarV0tmK/R3dgCtU/SRHUCLUHYgiELKbma3m9lhM/vAzFYVMUM1ZtZjZu+Z2X4zK3R96WwNvRNmdqDfbWPNbKeZHckuB1xjr6DZ1prZ37LHbr+Z3VnQbBPMbLeZHTKzg2b2i+z2Qh+7xFwtedxa/ju7mQ2X9L+S/l1Sr6S9kha6+/+0dJAqzKxHUsndC38DhpnNkvQPSZvc/frsticlnXL3J7L/KMe4+3+2yWxrJf2j6GW8s9WKOvsvMy5pnqT7VOBjl5jrP9SCx62II/t0SR+4+0fuflrSHyTNLWCOtufub0o6dc7NcyVtzK5vVN8/lparMltbcPdj7v5Odv0LSWeXGS/0sUvM1RJFlH28pL/2+75X7bXeu0v6s5ntM7MlRQ8zgCvd/ZjU949H0hUFz3Oumst4t9I5y4y3zWNXz/LnjSqi7AMtJdVO5/9muvs0SXdIWpo9XcXgDGoZ71YZYJnxtlDv8ueNKqLsvZIm9Pv+x5I+KWCOAbn7J9nlCUnb1H5LUR8/u4Judnmi4Hn+Xzst4z3QMuNqg8euyOXPiyj7XknXmtlPzGykpAWSthcwx/eY2ajshROZ2ShJc9R+S1Fvl7Q4u75Y0isFzvId7bKMd7VlxlXwY1f48ufu3vIvSXeq7xX5DyWtLmKGKnP9i6S/ZF8Hi55N0mb1Pa37Wn3PiB6Q9E+Sdkk6kl2ObaPZ/lvSe5K61VeszoJm+zf1/WrYLWl/9nVn0Y9dYq6WPG68XRYIgnfQAUFQdiAIyg4EQdmBICg7EARlB4Kg7EAQ/weypTV95ccHFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(some_digit.reshape(28, 28), cmap='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Got it right! 5 is indeed less than 7 and odd.\n",
    "- There are many ways to evaluate a multilabel classifier, selecting the best metrics will depend on your project.\n",
    "- One approach is to simply calculate $F_1$ score for each label category, then average all scores into one number:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_knn_pred = cross_val_predict(estimator=knn_clf, X=X_train, y=y_multilabel, cv=5, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9720267778020544"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = f1_score(y_multilabel, y_train_knn_pred, average=\"macro\")\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- This assumes that all label categories are equally important, which may not be true.\n",
    "- In particular, if you have more pictures of alice than bob you may want to give more weight to bob pics in evaluation.\n",
    "    - This is called unbalanced classes within a label category.\n",
    "    - One simple way to do it is to give each class a score corresponding to it support.\n",
    "        - The less samples we have of bob, the more important one score of him is."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multioutput Classification\n",
    "\n",
    "- The last type of classification we will discuss is called multioutput-multiclass classification.\n",
    "    - Example. Identifying animals in a picture & for each animal we have N classes.\n",
    "    - Meaning, each label can be multiclass.\n",
    "- To demonstrate this, let's build a system that removes noise from images.\n",
    "- It will take as input, the noisy digit images, and will hopefully output an image that look like the original one from the dataset.\n",
    "- Notice that the system is a multioutput classification one\n",
    "    - Output: 728 pixels, each pixel intensity ranges from 0 to 255.\n",
    "- Let's start with training data by adding a mask of noise, the original image will serve as the output (target):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise = np.random.randint(low=0, high=100, size=(X_train.shape[0], 784))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 784)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_mod = X_train + noise\n",
    "X_train_mod.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise = np.random.randint(low=0, high=100, size=(X_test.shape[0], 784))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 784)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_mod = X_test + noise\n",
    "X_test_mod.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_mod = X_train.copy()\n",
    "y_test_mod = X_test.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a look at an X -> Y from the training set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAACmCAYAAAB5qlzZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAARpUlEQVR4nO3dSWwV9BbH8QNlLLTMQ7FlLpQOCJQhqSUiTWCjcaFBjBOKrkh0oRunhcaYaGJM1JjoQkg0UUI0ujIMKoNAsEwByozQIlOBMrVQRt/iLXzvnd+57/b5/FfK97P89Vx6b4c/Nz3/8/93+v333w0AkEbn9n4CAHAnYdEFgIRYdAEgIRZdAEiIRRcAEmLRBYCEumT64MGDB91+stbWVll7/fp1l926dUvWlpaWuuzKlSuy9tixYy6rqKjI+jlEz7exsTGrx5uZ9erVy2X19fUuu3r1qnx8dXW1y+rq6mTtwIEDXXbu3DlZO2LECJf99ttvsjY/P99lffr0kbWbN292WVlZmazNzc11WfR8r1275rLi4uJOsvivx15J/NXkzzbvdAEgIRZdAEiIRRcAEuqUaQx4xYoV7oMzZsyQtervdbt375a13bt3d9mFCxdk7d133+2yy5cvy9qRI0e67OzZs7L2yJEjLisoKJC16u/NRUVFLlN/5zUzGz9+vMtaWlpkbc+ePV2mvrZmZup716mT/hPp6dOnXRb9zV19f/bv3y9r+/fv77Lotal89uzZ/E0XHRV/0wWA9saiCwAJsegCQEIsugCQEIsuACSUcSJNdcK7dNEPOXXqlMui3QBq58Ds2bNl7fnz510Wdd137NjhsuLiYlk7ZswYl23ZskXWjhs3zmU///yzy6LXUFtb67Joek11+NWUmpnemdGtWzdZO2TIkKweb2ZWWFjosqqqKlm7ceNGl1VWVsraGzduyBy4k/BOFwASYtEFgIRYdAEgIRZdAEgoYyNNjdWqppCZbvZERwcOHjzYZdGo68mTJ102evRoWavGjqPxYjVuq0Z7zcyGDh3qsmHDhrns4MGD8vGqaRY10tQIrmrkmenX0Lmz/n9UfX2jcWr1Oi5duiRrJ02a5LLomM6oAQrcSXinCwAJsegCQEIsugCQEIsuACTEogsACWU8xHzXrl3ugz169JC1zc3NLou62Gr3weHDh2Wt+nzqYHMzPdZ64sQJWat2BDQ0NMhatSMgJyfHZdGlkGpXRPS8ysvLXbZ9+3ZZq0Zzo4PjBw0a5DL1Gsz01zEa/1Zj2mpHg5nZ6tWrXVZTU8Mh5uioOMQcANobiy4AJMSiCwAJsegCQEIZG2mbNm1yH5w2bZqsVWOma9askbW5ubkuKysrk7X5+fku+/HHH2WtGjNVI7xmesR506ZNslbdeKvGdfPy8uTj1QjtqFGjZG3Xrl1ddv369axrVUPTTJ9lq16Xmf76Tp06Vdaqf+PQoUOyVjVQc3JyaKSho6KRBgDtjUUXABJi0QWAhFh0ASAhFl0ASCjj7oXLly+7D6pbcM3Mpk+f7rJofLWkpCTrWrXToS0HdUejrurG2sbGRlnbt29fl6kdCdHtuuqG3miXwYABA1wWHTauRntVZqZ3Oqxbt07WTp482WUbNmyQtepA+oqKClmrXkdeXh67F9BRsXsBANobiy4AJMSiCwAJsegCQEIZG2lNTU3ug62trbJ2586dLotGXdU4aFNTk6w9fvy4y3r37i1re/XqldXzMtMNK9VcMzPbu3evy15++WWX9evXTz7+wIEDLotuA54zZ47LxowZI2tXrlzpsqefflrWqu9zQUGBrFVfG/UazMxmzpzpsugG5paWFpcVFhbSSGtHavT9888/d9natWvl43ft2pX153rvvfdcpm7VNtNN3ieeeELWzpgxI+vnkBiNNABobyy6AJAQiy4AJMSiCwAJsegCQEIZdy/s3Lkz69uAVRcyGsFV46Bq3NdM7xyYOHGirFXjwT/99JOsVa9D3VZrZrZq1SqXbd261WVR1/7ZZ5+VuaJ2CUQ3/KrXsGDBAln7xhtvuCwavVbUoe9mbdsxosa/c3Nz2b2QwNKlS2X+wgsvuOz06dMui9aJWbNmuezMmTOytq6uLsMz/O+fb968ebL2q6++yvrfTYzdCwDQ3lh0ASAhFl0ASIhFFwAS6pLpg0VFRS47deqUrFU31l68eFHWqj+0Dx8+XNaq5lTUAFLn1kYNIDWSGp0vq27Hfe6551ymzhQ2M3vyySddFjX4VqxY4bIdO3bIWnV+r2psmekbeqNzetWYduTcuXMui77mJ06ccFk04oz/Tt3wbGZWW1vrMvXzaqZ/D+69916Xvf766/Lx1dXVLotG3FUjbPny5bJWiW6kvt3wThcAEmLRBYCEWHQBICEWXQBIiEUXABLKuHth//79LosOG1cd6+iA4549e7pMdVzNdCc9Okhd3YQbdUc//fRTl125ckXWPv/88y578cUXXdanTx/5+F9++cVlakeEmdmyZctcpm4eNtOj11GHNz8/32XR81XjmqWlpbJWfc3UTgkzs7KyMpnjf/PFF1/IfOHChVn/G+rQfDUyrH5+ItHIcVt2KqidU0899VTWj/87450uACTEogsACbHoAkBCLLoAkFDG83QbGhrcB6Mx0/79+7ssagDt27fPZWqc1Ew33aLR0e+//95ln3zyiaxVo8TvvvuurH3sscdcpkac1Zirmdn48eNdFjW8VOMw+h49+uijLps/f76sVaPX0bjukSNHssrM9M3B0c/IzZs3XVZSUsJ5ull47bXXXPb222/L2k6d/Jd00aJFsvatt95yWVuaZsqECRNkrhrzkW+++cZlDz744P/8nNoJ5+kCQHtj0QWAhFh0ASAhFl0ASIhFFwASyjgG3NjY6LLoNmA1mnv06FFZqw4mr6mpkbVqlPjLL7+UtWqnQnRQt/p8U6ZMkbVqp4PaVaEOhDbTndhoB4Yaq62srJS1VVVVWT0+em63bt2Steq5qddrZnbt2jWXRePF0eHW+MObb74pc7VToXv37rJ27ty5LnvnnXdkbfR9/U/R6L06dL++vl7Wql040eHot+FOhazxThcAEmLRBYCEWHQBICEWXQBIKOMY8IEDB9wHi4uLZa36g3rnznpNHzhwoMuiBlC/fv1c9sgjj8ja3Nxcl0UjjS+99JLL1Hm8ZmYTJ0502apVq1y2ePFi+Xg14qy+Xma6kaIaZmZ69HrIkCGyVp1xG90mq8aWu3TRPdeuXbu6LGrQHT9+3GXl5eV37Bjw+fPnXVZSUiJrT58+7bIHHnhA1n777bd/6nkdPHjQZWoU3sxs8+bNWf+7Dz/8sMs+++wzWRuNkt9mGAMGgPbGogsACbHoAkBCLLoAkFDGibRTp0657Pr167J28ODBLjt79qyszcvLc9nw4cNl7bp161wWTcWpRtqHH34oa9W0XXTep7qEsnfv3i6LLtcsLCx0mboQ0Mxs1qxZLisvL5e1alpv0qRJslY10tpSG31/VNNFnd1rxsWU/0lN86mGWeSDDz6QufrZjpq83333ncvUxaTR2djq7N6ogf7444+7rIM0zNqEd7oAkBCLLgAkxKILAAmx6AJAQiy6AJBQm28Djjr86ixaNfZppm8LXb9+vaxVN8g+88wzsvauu+5yWTRerJ5bc3OzrB0xYoTLKioqXBaNRKoOf1FRkaxVt+6ePHlS1qrdIaojbmY2dOhQl0U3MKsx4I0bN8rabt26uSza4aI63dXV1YwB/4voJl21IyH63VVf57ZQv0fR51K/R2onk1l8W3YHxhgwALQ3Fl0ASIhFFwASYtEFgIQyjgGrc2/VSKuZbupEZ7Bu2LDBZVHDa/To0S6Lxh+XLFniMjVGbKYbYaqBYGY2f/58l5WWlrps3rx58vFq3HbhwoWydu/evS6LxiqbmppcNmrUKFmrGqBRM081NaPavn37ukydgWxm9uuvv8r8TqW+dtFZuPfff7/LojH7sWPHuiy66HHBggUuU+c0q98BM91Ii2rxT7zTBYCEWHQBICEWXQBIiEUXABJi0QWAhDLuXlBje1EHWh2+rbqzZnpHQjRmqg45jkYl1a6Gixcvylr13NTNtmb6cPQffvjBZepQcTPd+c/JyZG1Ko/GaqdNm+ay3bt3y1p1ELo6TN5MjwdHuyK2bNnissrKyqz/Xfy7GTNmyLwth5v/WWvXrnXZmjVrZK0aOVa/3/gD73QBICEWXQBIiEUXABJi0QWAhDI20g4fPuyymTNnytp9+/a5TN0mbGbWp08fl6nRwyiPxh8vXLjgsuiP+uo5ROeQqnNyb9y44bJolLmlpcVlVVVVWT+vqImimprRCK5qKEZnpLa2trpMnf1qppt8qrlmpseW8fejfo6j3w2VMwacGe90ASAhFl0ASIhFFwASYtEFgIRYdAEgoYy7F3r06OGy6LbZkpISl0U7B1Tnf+vWrbJWHSxeUFAga9UYb/R8lUOHDslc3Y6rOvHRCOeePXtcpm7RNdO7AaJxavXaosPG1Wjn1KlTZa0ae66vr5e10SixMnfu3Kxr0X74Pv21eKcLAAmx6AJAQiy6AJAQiy4AJJSxkTZ58mSX9ezZU9ZeunTJZVevXpW16tzZqDmmGjXR+bKqkXbz5k1Zq0ZrL1++LGvVmb4q27Rpk3y8Omt48ODBWT+vqFaN60a3H6txzSNHjsja7t27uyxq/KlG57Zt22StGhVXTUq0r+XLl7f3U+jQeKcLAAmx6AJAQiy6AJAQiy4AJMSiCwAJZdy9cPToUZepG4LN9E4Hdai4mVl+fr7Loi62unVXjRyb6cOXo667eg7RzozVq1e7TB1MHj2vLl38l/nYsWOydsiQIS6LxpPV7pAxY8bI2ubmZpdFI7xqJ4n6WTAzW79+vcvmzJkja3fu3Clz/L1EP2/4/+CdLgAkxKILAAmx6AJAQiy6AJBQxkZaTk6Oy9RZq2a6iaVuoDUzGzFihMt2794ta4cOHeoyNXJsphth0Y236obf6LWVl5e7TI3QRmPEquG1d+9eWXvmzBmXRef0qhHnhoaGrJ/D8OHDZa26xTl6beprps7uNTMbO3aszPH3om78jn6P0Ha80wWAhFh0ASAhFl0ASIhFFwASYtEFgITafBvwuXPnZO306dNdFnW81S6DW7duyVp1+Pbx48dlrdptUV1dLWtV51517aPnpg4xV11fM7Ovv/7aZf3795e1hYWFLovGZ9WuigEDBshadXNwNCJ9/vz5rB5vpseWo10Ra9ascdmwYcNkLdpPRUWFy4qLi2WtGhmOxogHDRr0555YB8E7XQBIiEUXABJi0QWAhFh0ASChjI001VBRDTMzPSZ49uxZWXvjxg2XlZWVyVrV3Bo3bpysVWe+RiOp6sbb6DzdyspKl6km45QpU+Tj1RnEy5Ytk7Wq8ReNAatRZNXgM9PjxUVFRbJWvbaoVt1e3Lkz/5d3NK+88orMFy5cmHXtRx995LLS0tI/98RuQ/x2AEBCLLoAkBCLLgAkxKILAAmx6AJAQp0yHU7c1NTkPhgdNq4OMVc37prpTni0c0A9v2jMsKmpyWUjR46UteqmYjVWG30+tTNj1KhR8vGLFi1yWV1dnaxVY8Dvv/++rJ04caLL1Ni0mT5QPhqnVrna7WFmds8997hM7XoxMztw4IDLqqqq9BP+63EqdxtEFxLMmzfPZStXrpS1Dz30kMsWL14sa6NdOLcZ+bPNO10ASIhFFwASYtEFgIRYdAEgoYyNtH379rkPRmewtrS0uKykpETWqsZUQUGBrK2trXXZ1KlTZa0aGe7Xr5+s3bNnj8smTJgga/v27euy7du3uyw6y7axsdFlS5YskbVLly512bZt22Tt/v37XRa9hry8PJepMWIzfQOzeg1munkYnbmsGmk1NTU00m5jqsH26quvytqPP/7YZdFZ0R1kPJhGGgC0NxZdAEiIRRcAEmLRBYCEWHQBIKGMuxdqa2vdB9X4rJkeB1W385rpg69VZqZHXVtbW2XtunXrXFZTUyNrVTe+d+/eslZ149XYszoo3Mysa9euLotuV1W3BEcjmA0NDS4bO3asrG1ubnZZ9Hp37NjhsuiG3/r6epdFnefNmze77L777mP3Ajoqdi8AQHtj0QWAhFh0ASAhFl0ASChjIw0A8P/FO10ASIhFFwASYtEFgIRYdAEgIRZdAEiIRRcAEvoHxlbaLqj12roAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(nrows=1, ncols=2)\n",
    "ax[0].imshow(X_train_mod[4:5].reshape(28, 28), cmap='binary'); ax[0].axis('off')\n",
    "ax[1].imshow(y_train_mod[4:5].reshape(28, 28), cmap='binary'); ax[1].axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Now let's train the classifier to clean this image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=-1, n_neighbors=5, p=2,\n",
       "                     weights='uniform')"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.fit(X=X_train_mod, y=y_train_mod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_digit = knn_clf.predict(X=[X_test_mod[234]]).reshape(28, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAACmCAYAAAB5qlzZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQyElEQVR4nO3dSWzV5ffH8VNkHgqFthRKGW2hlTAUFKSgSBQTVgYXRldqjJqYuHPjmBhMNCYuXMjCkJhAdKeBqNE4hUGQMMgMZWoppVBaOgAWW6H+Nv+F/s/n6EXg+Sp9v5bH822/9/bL8eae5zxP3u+//24AgDT6ZX0DANCXUHQBICGKLgAkRNEFgIQougCQEEUXABLq/zf/nfVkuNXysviltbW17tm+dOmSzO3p6XGxcePGyVz1M3755ReZW1lZ6WK9vb0yt6WlRcaVQYMGudjo0aNlbltbW07X19fXy+svXrzoYosWLZK5p06dcjH1HpiZtba2utjIkSNl7pUrV1wser+Ki4tdbNiwYTK3trY2p+vNzI4ePepiixcvls82n3QBICGKLgAkRNEFgITy/mYMmO90catl8p1uV1eXe7a3bNkic+fOnetix44dk7n9+/s2SUVFhcwdOnSoi23atEnmqu9kS0tLZa76DvrMmTMyd968eS52/vx5F/v111/l9VevXpVxpbCw0MXUd8pmZoMHD3axw4cPy1z1HXK/fvrz5IABA1ysq6tL5ubn57tYdL/qu/iioiK+0wWArFF0ASAhii4AJETRBYCEKLoAkBCrF5C1TFYvtLW1uWc7mkxqaGhwsajjrabPJk+eLHMbGxtdLFrp0NzcnPM9qM79jBkzZK7quqvXO3z4cHl9XV2di6lVGWZ6BUT0etVE2Z133ilzd+/e7WJRXRsyZIiLqZUSZnq1RUdHh8wtKChwsdGjR7N6AQCyRtEFgIQougCQEEUXABKikYasZdJIa25uds/2hQsXZK5qNqlRWTOzZcuWudiBAwdkblFRkYtF48WzZ892sb1798rc8ePHu1i0bWV7e7uL1dTUuNjWrVvl9WoUOWo2qfd34sSJMleNMquxXDM9mqu2ezQzu3btmotFI86q+ajGiM30NpAPPPAAjTQAyBpFFwASougCQEIUXQBIiKILAAmxegFZy2T1Qmdnp3u2T548KXPVWOuECRNkrupu79mzR+aqVRHRCK3aHL2srEzmqo3Fo3/nTU1NMv7/lZeXy7habRH9TPXaos3Gf/vtNxeLVhlMnTrVxaLXq1YvqMM1zcwGDhyYc64aUS4tLWX1AgBkjaILAAlRdAEgIYouACTkv50H+oARI0a4WNQkUfuqtra2ylx1am/U1KmurnaxQ4cOydxRo0a52B133CFz1Z6+R44ckbnq9OE33njDxdauXSuvX7lypYtNmzZN5qpR5KhxqJqa0b7E586dc7HoPVe/L9preN++fS4WnX5cXFws4wqfdAEgIYouACRE0QWAhCi6AJAQRRcAEmL1AvokNVJaUlIic9VqgLNnz8rczs5OF4tOGVYbbUeddNWNj1ZQqDHcy5cvy9z169e72Pz5812svr5eXq9er+r6m+mx2vvuu0/mqjHraAPxkSNHutjRo0dlbl6en8yNVq2o50E9C2b6ZOcpU6bIXD7pAkBCFF0ASIiiCwAJUXQBICH20/2DdevWybgat3z88cdv9e30FZnsp1tbW+ue7Wh/WjVCG+UeP37cxebOnStzt23b5mLq1F8zs/3797tYNEKrGk7Lly+XuaqhuGLFChd766235PWqGRg1ptQoszq52Mxsw4YNLqZGrM3Mli5d6mJRw0u93paWFpmrmoT33nuvzFWNtIqKCvbTBYCsUXQBICGKLgAkRNEFgIQougCQUJ9dvVBXV+di6lTRyN+8b0msWrXKxV599dUM7uSGZLJ64aeffnJ/QDX+ama2ZcsWF1u4cKHMbWtrc7FolYE68Xbv3r0yt6qqysXGjBkjc9esWeNiL7/8cs4/98svv3Sx5uZmeb1aqRDdl3pvohOY3333XReLNhDfvHmzi6n31szsxIkTLqZWVZjpE5ijlRlqNUtBQQGrFwAgaxRdAEiIogsACVF0ASCh234/3Y6ODhm/nqaZOsU0pY8++kjGX3vtNRf7DzbSMtHb2+ti0R6sqqHS09Mjc9Upwzt37sz5HmbNmiVzVROqoaFB5r7//vsuFo2vvvTSSy52PfvTqoby6dOnZa5qsA0aNEjmHjx40MWefPJJmXvmzBkXmz59usxVe/qWlpbK3Pb29pxiZmbl5eUyrvBJFwASougCQEIUXQBIiKILAAlRdAEgodt+9cI777yTc+6/YbRXbb781FNPyVx1cmw0rjl27Ngbu7HbTGFhoYupVQpmZsOHD885V52EG214rlbQqA3TzfTm29Gpu9euXXOx2tpamas26FfPUDQirVZmzJs3T+aqFRDRahv13kQrMNRG6F1dXTJX/d3USgkzs+LiYheLTiQ+f/68i02aNEnm8kkXABKi6AJAQhRdAEiIogsACd32jbS3335bxg8fPpz4TnKjmjaRDz74wMVomOVGjetGjdQpU6a42K5du2SuOh03Gh1VJ/xGDSDV8Prwww9l7uTJk13smWeekbmKajZduHBB5qqmnYqZma1evdrF1P62kZqaGhlX9xbt06vGgKurq2WuGmeeM2eOzFWNShppAPAvQNEFgIQougCQEEUXABKi6AJAQrf96oWI6vCm9Pnnn9/wz7j//vtvwp30TWpFgVrRYKY3G1cj2GZmTU1NLlZfXy9zKysr/+IO/0yd8Bvdr1rpsGzZMpmrRn7VibfR6bpqPPnjjz+WuQcOHHCx6Bnu7Ox0sWhz9CVLlrhYfn5+zj9Xjd6b6fFttTrFzOyuu+6ScYVPugCQEEUXABKi6AJAQhRdAEiozzbSNm3a5GLLly+/Jb/r008/dbFHH3005+s3bNgg4/368f/Mf6q7u9vFVMPMzKyxsdHFohNkz54962LRaLY64Xf27NkyV53QGzV11DhzNBarnqEdO3a42NChQ+X1qun2/fffy1zVdIv2f37ooYdcbNSoUTJX/VtW475mumlWVVUlcw8dOuRiEydOlLmXL192McaAAeBfgKILAAlRdAEgIYouACTUZxtpTz/9tIudOXMmgzv5M3X43j333JPBndzeVGNINasiEyZMkHHVjIsOhVTNnh9++EHmqvuNJt1UkzA6AHLPnj0upppN0cSVmjJbvHixzM3Ly3MxNU1mZvbII4+42Llz52Ru1HxU1MGSqglmZlZeXu5i0d9SNflopAHAvwBFFwASougCQEIUXQBIiKILAAnlRSeg/p+//I//ZStXrnSxzz77LIM7+bMvvvjCxVasWJHBnSTjW9oJdHd3u2d78+bNMledABvtL6u642o1gZlZXV3dX97jH5WUlLjYvn37ZO4TTzzhYtHqA7ViZ9iwYS5WVFQkr1cn8UarKj755BMXi1YvqD2BOzo6ZG5hYaGLDR48WOaqsWU1jm2mVx9EuWrVSllZmXy2+aQLAAlRdAEgIYouACRE0QWAhPpsI+1WUGOOkVdeeUXGV61adbNu578ik0bali1b3LMdNV9UEysaA1YuXbok42q0d+fOnTJX7Xur7svM7Ntvv3WxaP/fqVOnutiRI0dcbP369fL6Xbt2udjDDz8sc1944QUXU404M31w7Pnz52WuqmHR/r/qb6z2+TUzGz9+vItdu3ZN5qpR4oKCAhppAJA1ii4AJETRBYCEKLoAkBBFFwAS6rObmN8o1cVUHVczPRb57LPP3uQ7wvXo6elxMTXCa6bHTHfv3i1zVSe8q6tL5qrNt9X4q5lZcXGxi0Ubas+fP9/FZs6cKXMbGhpcTG3IHY1Ijx492sVqampkrhrjjVaBqFOC1Qhv9HPVBuRm+u8erXRQudH498GDB10s2sydT7oAkBBFFwASougCQEIUXQBIiEbaP7R69WoXi/YRVc2GiRMn3uxbwnVQI7RXrlyRuWqMV+05a6b3VY1GR9VzEf1c1eyJRmgrKipc7OzZszJXNdK2b9+e832pBl3UxFKn+UZ75I4ZM8bF1L7GZrq5pU40NjMbMmSIi23btk3mVlVVybiixrTD3JwzAQA3jKILAAlRdAEgIYouACRE0QWAhFi98A8dOHAg51x1wi+ylZ+f72Kq629m9t1337nY3XffLXMHDRrkYmqE18zs1KlTLhaNIre2trrY2LFjZe6ePXtcrH9//U9d/dx169a5WHTy8HPPPedi0abgCxYscLHGxkaZq04pjlZgqBUJ0cqDgQMHulh0UrJa2RGtcCkoKJBxhU+6AJAQRRcAEqLoAkBCFF0ASIjTgHOgGh5q79yioiJ5fXSKKcwso9OAN27c6J7tq1evylwVHzFihMydNWuWi504cULmRg0nRY26qsaUmT6Vuq2tTeY+//zzLrZ//34XGzdunLz+66+/drGo6ab2FY5Ge1VDMnq/mpqaXEyNWJuZnT592sU6Oztlrhq9VnsNm+lnZM6cOZwGDABZo+gCQEIUXQBIiKILAAlRdAEgIcaAc7BmzZqc8t58881bfCe4WdSYqDrR1UyvSlHjpGa6cx+dEq1WJESn46rx02i1xa5du1wsGnX96quvcsp98cUX5fU//viji0UrO9RqgGjsWZ10HK0CUicoDx8+XOaq9zwaka6urs7pd5nFJxUrfNIFgIQougCQEEUXABKi6AJAQowB50CNVSonT56U8SlTptzM27ndZDIG3NLS4p5tNSJqpk9ujppuqkF37NgxmavGWtUYsZneI3fJkiUyV50+rBpIZmaPPfaYi6km1tq1a+X1I0eOdLHBgwfLXHVCb9TwUiPD0X66Dz74oIvV1dXJXHUicTS+39PT42LTpk2TuWpP37ygcPBJFwASougCQEIUXQBIiKILAAlRdAEgIcaA/0CNT14PVin8d6iNxaMTfjdt2uRi0Um8qnMfjY6qU4Lr6+tlbmVlpYv9/PPPMletSNq2bZvM3bhxo4u9/vrrLlZeXi6vV5uNX7hwQeaqeHRScklJSU6/y0xvQq5WcJiZTZ061cXUIQVmZmVlZS4WrVqZOXOmi6kVDWZ80gWApCi6AJAQRRcAEqLoAkBCNNL+oc2bN2d9C7gBam/XqEmiTpYdNmyYzFUNq2gv29LSUhfbunWrzG1vb3exaE9fNbb83nvvyVw1ilxTU+Ni0YnG+fn5Lhbtkav2rY2aY6qhOH36dJmrGmFR42/37t0uFp0crE7+jcaWo72NFT7pAkBCFF0ASIiiCwAJUXQBICGKLgAkxCbmyFomm5h/88037tleunSpzL18+bKLqU22zXR3O9rEXHXjo/FVFY9WL+zfv9/FZsyYIXPVxuKtra0539elS5dcbM6cOTJXbQre0NAgc+fNm+di0Ubs6v2NRvLVRujRKcPq715QUCBz1WseMGAAm5gDQNYougCQEEUXABKi6AJAQjTSkLVMGmnd3d3u2d6xY4fMVU0S1RQy0+OratTWTI8Sjxs3TuZevHjRxdQos5lZW1ubizU1NclctS+w2rt3+/bt8nq1761qzpmZtbS0uNjChQtlrnptjY2NMledwh2NIi9YsMDFOjo6ZK5qpEX7KKt7q6iooJEGAFmj6AJAQhRdAEiIogsACVF0ASAhVi8ga5msXti+fbt7tqPRXrXRtjop1sysu7vbxfr1059tmpubXWzSpEkyV610OH78eM6506ZNk7lqFYYa+VWjwWZ69UK0ykBtYh6NSF/Pqcq9vb0uFo0ML1q0yMXUacJmZmPGjHGxQ4cOyVx1v9OnT2f1AgBkjaILAAlRdAEgIYouACT0d400AMBNxCddAEiIogsACVF0ASAhii4AJETRBYCEKLoAkND/AG6qht7WYk81AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(nrows=1, ncols=2)\n",
    "ax[0].imshow(clean_digit, cmap='binary'); ax[0].axis('off')\n",
    "ax[1].imshow(X_test_mod[234].reshape(28, 28), cmap='binary'); ax[1].axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Looks close enough to the target\n",
    "- And with this, we finish our classification chapter.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1. Try to build a classifier for the MNIST dataset that achieves over 97% accuracy over the test set.**\n",
    "\n",
    "- KNeighbors classifier works well for this task.\n",
    "- You just need to find good hyperparameter values\n",
    "    - Try a grid search on `weights` and `n_neighbors` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "sns.set_style()\n",
    "from sklearn.datasets import openml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = openml.fetch_openml(name='mnist_784', version=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((70000, 784), (70000,))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = mnist.data, mnist.target\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 784), (60000,), (10000, 784), (10000,))"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test = X[:60000], y[:60000], X[60000:], y[60000:]\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_clf = KNeighborsClassifier(n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = cross_val_score(estimator=knn_clf, X=X_train, y=y_train, scoring='accuracy', cv=5, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9697626 , 0.97017164, 0.96941667, 0.96815871, 0.96898966])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = {\n",
    "    'weights': ['uniform', 'distance'],\n",
    "    'n_neighbors': [3, 4, 5]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_clf = GridSearchCV(estimator=knn_clf, param_grid=grid, scoring='accuracy', cv=5, verbose=3, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 6 candidates, totalling 30 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  26 out of  30 | elapsed: 63.4min remaining:  9.7min\n",
      "[Parallel(n_jobs=-1)]: Done  30 out of  30 | elapsed: 63.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30,\n",
       "                                            metric='minkowski',\n",
       "                                            metric_params=None, n_jobs=-1,\n",
       "                                            n_neighbors=5, p=2,\n",
       "                                            weights='uniform'),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'n_neighbors': [3, 4, 5],\n",
       "                         'weights': ['uniform', 'distance']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='accuracy', verbose=3)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.fit(X=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9716166666666667"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 4, 'weights': 'distance'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train the model with the best parameters over the whole training set (without cross validation):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_classifier = KNeighborsClassifier(n_neighbors=4, weights='distance', n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=-1, n_neighbors=4, p=2,\n",
       "                     weights='distance')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_classifier.fit(X=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we'll evaluated our model over the test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9714"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_classifier.score(X=X_test, y=y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We got $97.14\\%$ accuracy! Finally, Let's save the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/02/knn_MNIST_97.joblib']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(knn_classifier, 'models/02/knn_MNIST_97.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2. Write a function that can shift an MNIST image in any direction**\n",
    "\n",
    "- Left/Right/Up/Down by 1 pixel.\n",
    "- Then for each image in the training set, create 4 shifted copies\n",
    "- Add the copies to the training set\n",
    "- Finally, train your model on this training set and evaluate it on the test set\n",
    "- This technique of expanding the training set is commonly referred to as *Data Augmentation* or *Training set expansion*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.ndimage.interpolation import shift\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shift_digit(imgs, direction):\n",
    "    \"\"\"Shifts image by 1 pixel in any direction\n",
    "    \n",
    "    # Arguments\n",
    "        imgs: np.ndarray, contains flattened digit images\n",
    "        direction: string, direction you want to shift the pixels in. \\in {'left', 'right', 'up', 'down'}\n",
    "    \n",
    "    # Returns\n",
    "        shifted: np.ndarray, the resulting shifted image\n",
    "    \"\"\"\n",
    "    dir_to_shift = {\n",
    "        'up': [0, -1, 0],\n",
    "        'down': [0, 1, 0],\n",
    "        'left': [0, 0, -1],\n",
    "        'right': [0, 0, 1]\n",
    "    }\n",
    "    return shift(imgs.reshape(-1, 28, 28), shift=dir_to_shift[direction], cval=0).reshape(-1, 28*28)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's create 4 images for each image present in the training dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "up_shifted_X_train = shift_digit(X_train, 'up')\n",
    "down_shifted_X_train = shift_digit(X_train, 'down')\n",
    "left_shifted_X_train = shift_digit(X_train, 'left')\n",
    "right_shifted_X_train = shift_digit(X_train, 'right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(240000, 784)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aug_X_train = np.concatenate((up_shifted_X_train, down_shifted_X_train, left_shifted_X_train, right_shifted_X_train), axis=0)\n",
    "aug_X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(240000,)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aug_y_train = np.concatenate((y_train, y_train, y_train, y_train), axis=0)\n",
    "aug_y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add in shuffling by index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_X_y(X, y):\n",
    "    \"\"\"Shuffles two arrays using indexes\n",
    "    \n",
    "    # Arguments\n",
    "        X: np.ndarray, input\n",
    "        y: np.ndarray, output\n",
    "    \n",
    "    # Returns\n",
    "        shuffled_X: np.ndarray\n",
    "        shuffled_y: np.ndarray\n",
    "    \"\"\"\n",
    "    indices = list(range(X.shape[0]))\n",
    "    random.shuffle(indices)\n",
    "    return X[indices], y[indices]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we add the augmented imagery to `X_train` & `y_train`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 784), (240000, 784), (60000,), (240000,))"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, aug_X_train.shape, y_train.shape, aug_y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.concatenate((X_train, aug_X_train), axis=0)\n",
    "y_train = np.concatenate((y_train, aug_y_train), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((300000, 784), (300000,))"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = shuffle_X_y(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAGL0lEQVR4nO3dz4vNexzH8TNSJkRKJKUmU8TGisUUi1n6tRGShZWN2dgpYeNPQFLGFsnCwsZCjbIZNSdJKUM2k4U6KL9Wc3e3bvd836d75o55HfN4LL36nnNKT9/y6XvO0Pz8fAvIs2KpPwDQnTghlDghlDghlDgh1Moeu//KhcU31O0P3TkhlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDghlDgh1Mql/gDL0dWrVxu3S5culdeOjIyU++XLl8t9//795b558+bGbc2aNeW1/L/cOSGUOCGUOCGUOCGUOCGUOCHU0Pz8fLWXI919/Pix3A8ePNi4zczM/N8f5x96/H23jh071rjduXOnvHbt2rV9fSZaQ93+0J0TQokTQokTQokTQokTQokTQokTQjnnXASHDh0q98ePH/+mT/Jvvc45h4a6Hrm1Wq1Wa3R0tLx2eHi43Pfs2VPu7Xa7cbt27Vp5ba9H4cI554RBIk4IJU4IJU4IJU4IJU4IJU4I5ZyzD2/evCn38fHxcp+bm+v7vatnQVutVuvevXvl/ujRo3I/f/584/bjx4/y2q9fv5b7Qqxfv77cO53Oor33b+CcEwaJOCGUOCGUOCGUOCGUOCGUOCGUc84uvnz5Uu69nkv88OFD3++9evXqcr9161a5nzp1qu/37mV2drbcp6eny73XM5nPnz9v3Hp9J+5inrH+Bs45YZCIE0KJE0KJE0KJE0KJE0KJE0KtXOoPkOjly5flvpBzzFar1dq9e3fjdvHixfLakydPLui9F2L79u0L2ns9S1qdc544caK89k/kzgmhxAmhxAmhxAmhxAmhxAmhPDLWxaZNm8r906dP5d7rp/Du3r3buB05cqS8Ntm3b9/KfWxsrNxfv37duL19+7a8dtu2beUeziNjMEjECaHECaHECaHECaHECaHECaGW5SNjU1NT5f758+cFvf7Tp0/Lfd++fQt6/VS3b98u916P4lWPww34OWZf3DkhlDghlDghlDghlDghlDghlDgh1LJ8nnNubq7cJycny33jxo3l3utn+NatW1fuqXp9JejOnTvLvdfPG96/f79xGx8fL68dcJ7nhEEiTgglTgglTgglTgglTgglTgi1LM856c/ExES537hxo9xv3rxZ7mfPnv3Pn+kP4ZwTBok4IZQ4IZQ4IZQ4IZQ4IZQ4IdSy/N5amj18+LBxu379enntihX1v/Vbtmzp6zMtV+6cEEqcEEqcEEqcEEqcEEqcEMpRyjIzPT1d7ufOnWvchoa6Ptn0tytXrpT74cOHy51/cueEUOKEUOKEUOKEUOKEUOKEUOKEUL4a8w/z6tWrcj9w4EC5dzqdxm3Hjh3lte12u9xXrVpV7suYr8aEQSJOCCVOCCVOCCVOCCVOCCVOCOWcc8D8+vWr3Hft2lXu7969K/fR0dHGbWpqqrzWV1/2zTknDBJxQihxQihxQihxQihxQihxQijfWxvm+/fv5d7rZ/jev39f7sPDw+V+4cKFxs055u/lzgmhxAmhxAmhxAmhxAmhxAmhPDIW5sGDB+V+/PjxBb3+5ORkuZ85c2ZBr09fPDIGg0ScEEqcEEqcEEqcEEqcEEqcEMo55xJ48uRJ43b06NHy2p8/f5b72NhYuT979qzcWRLOOWGQiBNCiRNCiRNCiRNCiRNCiRNCOedcBLOzs+W+d+/exq3T6ZTXjoyMlHuvn+nbunVrubMknHPCIBEnhBInhBInhBInhBInhBInhPITgIvg9OnT5V6dZfY6x3zx4kW5b9iwodwZHO6cEEqcEEqcEEqcEEqcEEqcEEqcEMo5Zx9mZmbKvd1u9/3aExMT5e4cc/lw54RQ4oRQ4oRQ4oRQ4oRQ4oRQvhoTlp6vxoRBIk4IJU4IJU4IJU4IJU4IJU4IJU4IJU4IJU4IJU4IJU4IJU4IJU4IJU4I1eurMbs+ZwYsPndOCCVOCCVOCCVOCCVOCCVOCPUXsaUJDyT3O7sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_train[99].reshape(28,28), cmap='binary')\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'7'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[99]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we train our classifier with the best hyper-parameters we found:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_classifier = KNeighborsClassifier(n_neighbors=4, weights='distance', n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=-1, n_neighbors=4, p=2,\n",
       "                     weights='distance')"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_classifier.fit(X=X_train, y=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we evaluate our model over the test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9763"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_hat = knn_classifier.predict(X=X_test)\n",
    "accuracy_score(y_true=y_test, y_pred=y_hat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We got $97.63\\%$ test accuracy, $+0.5\\%$ more than $97.14\\%$ by using data augmentation!\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}