4_baseline_modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries and dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix, recall_score\n",
    "from sklearn.metrics import plot_confusion_matrix, f1_score\n",
    "from scoring_functions import scoring_functions as f\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import make_pipeline, Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "\n",
    "# imp allows to reload the above function file when tinkering with it without kernel restart\n",
    "import imp\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('complaints_processed_1year_0719.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "df['processed narrative'].isnull().sum()\n",
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change product names to integers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0])"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['product'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    49399\n",
       "1     1236\n",
       "Name: product, dtype: int64"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['product'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "product_dict ={'Non Money Transfer': 0, 'Money Transfer': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.8/site-packages/pandas/core/generic.py:6619: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return self._update_inplace(result)\n"
     ]
    }
   ],
   "source": [
    "df['product'].replace(product_dict, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0])"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['product'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    49399\n",
       "1     1236\n",
       "Name: product, dtype: int64"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['product'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>narrative</th>\n",
       "      <th>processed narrative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>My wife received settlement from XXXX XXXX com...</td>\n",
       "      <td>wife received settlement company amount opted ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>My girlfriend sent me {$5000.00} on XX/XX/XXXX...</td>\n",
       "      <td>girlfriend sent using paypal help pay work don...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>On XX/XX/2021, I initiated a wire transfer of ...</td>\n",
       "      <td>initiated wire transfer gemini account externa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Can not link my PNC Bank account to numerous X...</td>\n",
       "      <td>link pnc bank account numerous platform</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>I receive two payment for the amount of {$1500...</td>\n",
       "      <td>receive two payment amount cash app try cash m...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   product                                          narrative  \\\n",
       "0        1  My wife received settlement from XXXX XXXX com...   \n",
       "1        1  My girlfriend sent me {$5000.00} on XX/XX/XXXX...   \n",
       "2        1  On XX/XX/2021, I initiated a wire transfer of ...   \n",
       "3        1  Can not link my PNC Bank account to numerous X...   \n",
       "4        1  I receive two payment for the amount of {$1500...   \n",
       "\n",
       "                                 processed narrative  \n",
       "0  wife received settlement company amount opted ...  \n",
       "1  girlfriend sent using paypal help pay work don...  \n",
       "2  initiated wire transfer gemini account externa...  \n",
       "3            link pnc bank account numerous platform  \n",
       "4  receive two payment amount cash app try cash m...  "
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train-test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(['product'], axis=1)\n",
    "y = df['product']\n",
    "X = X['processed narrative']\n",
    "vect_y_train = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    1\n",
       "3    1\n",
       "4    1\n",
       "Name: product, dtype: int64"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vect_y_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CountVector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000\n"
     ]
    }
   ],
   "source": [
    "# CountVector\n",
    "vectorizer = CountVectorizer(ngram_range=(2,3),stop_words = 'english', min_df = 2, max_features = 5000)\n",
    "vect_X_train = vectorizer.fit_transform(X)\n",
    "feature_names =  vectorizer.get_feature_names()\n",
    "print(len(feature_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "start = time.time()\n",
    "\n",
    "pipe = Pipeline([('CountVect', CountVectorizer(stop_words='english',\n",
    "                                              ngram_range=(2, 3), max_features = 5000)),\n",
    "                   ('clf', LogisticRegression(random_state=0, \n",
    "                                              fit_intercept=False, max_iter = 5000))\n",
    "                    ])\n",
    "                 \n",
    "param_grid = {'CountVect__min_df': [2, 5, 10],\n",
    "              'clf__C': np.logspace(-3,3,7),\n",
    "              'clf__penalty': ['l1', 'l2'],\n",
    "              'clf__solver': ['liblinear', 'newton-cg', 'lbfgs'],\n",
    "             } \n",
    "# param_grid = {'CountVect__max_features': [1000,2000, 3000, 4000, 5000],\n",
    "#               'CountVect__min_df': [2, 5, 10],\n",
    "#               'clf__C': np.logspace(-3,3,7),\n",
    "#               'clf__penalty': ['l1', 'l2'],\n",
    "#               'clf__solver': ['liblinear', 'newton-cg', 'lbfgs'],\n",
    "#               'clf__max_iter': [1000, 5000, 10000]\n",
    "#              } \n",
    "grid = GridSearchCV(pipe, param_grid, cv=5, scoring='f1', n_jobs = -1)\n",
    "\n",
    "grid_countvect = grid.fit(X, y)\n",
    "\n",
    "end=time.time()\n",
    "hours, rem = divmod(end-start, 3600)\n",
    "minutes, seconds = divmod(rem, 60)\n",
    "print(\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CountVect__min_df': 2,\n",
       " 'clf__C': 0.01,\n",
       " 'clf__penalty': 'l2',\n",
       " 'clf__solver': 'liblinear'}"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_countvect.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=100)\n",
    "pipe = Pipeline([('CountVect', CountVectorizer(stop_words='english',\n",
    "                                              ngram_range=(2, 3), min_df = 2, max_features=5000)),\n",
    "                   ('clf', LogisticRegression(penalty = 'l2', C=0.01, solver = 'liblinear', random_state=0, \n",
    "                                              fit_intercept=False, max_iter=5000))\n",
    "                    ])\n",
    "model_lr = pipe.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "def proba_to_classes(y_scores, t):\n",
    "    \"\"\"\n",
    "    This function adjusts class predictions based on the prediction threshold (t).\n",
    "    Will only work for binary classification problems.\n",
    "    \"\"\"\n",
    "    return [1 if y >= t else 0 for y in y_scores]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv('complaints_processed_1year_0719_test.csv')\n",
    "df_test.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "df_test.head()\n",
    "if df_test['processed narrative'].isnull().sum() != 0:\n",
    "    df_test = df_test.dropna()\n",
    "df_test['processed narrative'].isnull().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    181687\n",
      "1      4944\n",
      "Name: product, dtype: int64\n",
      "Best Threshold=0.500867, F-Score=0.926\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "from numpy import argmax\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "df_test = pd.read_csv('complaints_processed_1year_0719_test.csv')\n",
    "df_test.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "if df_test['processed narrative'].isnull().sum() != 0:\n",
    "    df_test = df_test.dropna()\n",
    "df_test['processed narrative'].isnull().sum()\n",
    "product_dict ={'Non Money Transfer': 0, 'Money Transfer': 1}\n",
    "df_test['product'].replace(product_dict, inplace=True)\n",
    "print(df_test['product'].value_counts())\n",
    "X_test = df_test['processed narrative']\n",
    "y_test = df_test['product']\n",
    "\n",
    "pred_proba = model_lr.predict_proba(X_val)\n",
    "\n",
    "pred_proba_class_1 = pred_proba[:,1]\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_val, pred_proba_class_1)\n",
    "\n",
    "# convert to f score\n",
    "fscore = (2 * precision * recall) / (precision + recall)\n",
    "# locate the index of the largest f score\n",
    "ix = argmax(fscore)\n",
    "print('Best Threshold=%f, F-Score=%.3f' % (thresholds[ix], fscore[ix]))\n",
    "# plot the roc curve for the model\n",
    "no_skill = len(y_val[y_val==1]) / len(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkP0lEQVR4nO3deXxU5d338c8vCRBIQgirQEBQsIoQgsaFxRsVNxDx0adtrFWUopTeuNWlgLeP6/1qafVWitpaqqi3VqW1VFERqYqAFZWgEVlUAghGKIQ9CTu5nj/OJGQlkzBnJsn5vl+vec3Muc6c+V0Yz3fOdh1zziEiIsEVF+sCREQkthQEIiIBpyAQEQk4BYGISMApCEREAi4h1gXUVfv27V2PHj1iXYaISKOydOnSrc65DtW1Nbog6NGjBzk5ObEuQ0SkUTGz9TW1adeQiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnG9BYGYzzGyLmS2vod3MbJqZ5ZnZMjM7za9aRESkZn5uETwHXHKU9uFA79BjHPBHH2sREZEa+HYdgXNuoZn1OMoslwP/67xxsD82szZm1tk5t8mXgjavhBX/8GXRTVpyRzjjBjCLdSUi4pNYXlDWFfiu3Pv80LQqQWBm4/C2GujevXv9vm3r17Dw4fp9NrBC96r4wQhI7RrbUkTEN7EMgup+YlZ7lxzn3HRgOkBWVlb97qRz6hXeQ8L3+Yvw+gRwh2NdiYj4KJZnDeUD3cq9Twc2xqgWEZHAimUQzAZGh84eOhvY5dvxARERqZFvu4bM7GXgXKC9meUD9wHNAJxzTwFzgBFAHrAHGONXLSIiUjM/zxr6SS3tDpjg1/dLBC15GpyDfTvh9DHQVZd8iDQljW4YaomipI7e879+DwmJcGgfNGulIBBpYhQEUrOTLoK71kKzltC8FUyp56m7ItKgKQjk6JLaRX6ZLnQGsC5SE2kQFAQSWQf3QeGmI4/dpc8bofDfUBh67nAy/HxBrKsVERQEUleH9sPWPNj5LexYDzvXh543eK/3bKv6mYSW0LozpHSGrllQ8DVsXxv10kWkegoCqQODpc96j1LxzSG1G7TpDieP9F6XrvRTOnuvE9tU3A00dzJ8XuPtU0UkyhQEEr5Lpni//NOOhzbHeyv/lM4QF4HrEg8fhF35R7Ysdm7wjiWc91+RWb6I1EhBIOHLPOqlIXVzcC/8Y3xoxb8Bdn8PrqTqfKddC2k9Ive9IlKFgkCiL62nN5DdukXeVsXxg73nNt1DWxvdYd1CmH3zkTOMRMQ3CgKJvrPGwRljIS6+5nnWL45ePdVxDvbu8M522v29t9uq/GtXAj95BRJbx7bOxqKkBPbv9v5Nqzx2Vnx/oAguuB/Ss2JddWAoCCQ2jhYC5b11BxRt8Y4bDLsXzrwxMt9/+BDszocd33qP0hX9rnxvZb97IxzcU/EzFu8dE4lvBjvWebu0jusbmXoak0MHYM9WKN4aet7mnS1W7Uo+9Ni3s/pdf6WaJ0PLNEhMhc3LYd0CBUEUKQikYUrrAc1TvJV0u17ejYUKvg7/86W/6HesO7Ky37G+4oq//H0WLA6Sj/NuwNOpL/S+2Hvdugu0TveekztBfAKsnA1/vRZwUFQAu74LbSl874XLrtBWw76dcOWfoUtm/f8dSkpg7/bQNRj/9q7JKPp3ufehR59RcMlv6vcdB/dVXbEXF5Sbtq1c21bvl31NElO9FXrpI+34iu+reyS2gYTm3ucP7Yf/7gjzf+MNazJQw5FFg4JAGqbjB8Ld+Ufe/7Zn9fMd2g/b18G21bB1NWzL8563fuOtiMtL6uAFTPoZ0O9H3uu0Ht7KKqWLt5Kviz8NrXrTnvgWXoC0bOvVsGVlzUFwcF9o6+P7I1silVfwRZuh5GDVzya2CZ2i2wl2HoDvPqnYXhqERZuPLKdoMxRu9oKkaMuR6TWt2OMSoFV7SGoPrdpBlwGh9x28K87L2kLtLduEv6VXk4QWcOGD8MFvYfOKY1uWhE1BII1H4SbIfQm2rPK2DrZ+4+0yKr/LIfk4aN/buxtd+97egem0Ht4B6BbJkamj+0A4/Xpo0RpS06F1V+85Nd1bIZp5Wx2/7w/b1sDK170V/a58b+uhdIuheEvVZZdfwbc/x9sKSekMKccdeSR38sZ/KvXi/4XvP4OXrz6ywi/aDIcPVF1+syRv2cmdoNOp0GtYaMXevuKKPam99+s+FsOADL4VPv1z9L83wBQE0jjEN4ev3vQe8c2hXW/o3B/6/dB73b6X9xyNg7fJHeCy39deL8CiR45Ma5YUCoyu3rGF1G4Vg6R1l4or+HB17u8FwY51kNzR25WW0skLxdKVfunrFil1X36sbF8Ln0z3dkklpmo3kY/MNbLT87KyslxOTk6sy5BoW/+Rt9+6wynQ9oS678aJha/nAu7Iyr5lmgbaC9cfBsGWSruGJq73AmH/bu9YRWq3I8cWpFZmttQ5V+0ReAWBiDQ8RaGD1a3aw7KZMO+/vOM4e7bB4f3ePFljYeSjsa2zETlaEDSCn1UiEjjJHbwHwInnww8u9Xb7JXXwHh89Dt9+CGvmw4nn1W3ZB/Z4W5fFBd5B8+IC73hNUcGR6aVtnTNg9OuR718DoyAQkYatUx/4yUsVp615H9bOh3fv84LgwJ7QabWhA+VlK/kt3m6kshV+gXfBWnVahIImuaN3ooFzsOETeP+/Q5/f6p2Jde4kOGGo//2OIu0aEpHGp+QwvJTthUFCSzhQWM1M5p39VLoVkdzxyOvq3jdLrPjxRf8D7z3oXWNSetrslhXeKLsnnn8kbJq1hAsfavCDI+oYgYg0PStnw4pZobOiQo/Ss6WSO3qn8h7LdQ3OedeitEg9spL/bQ9vq6BU6b28b1tOYXwbZs6cyerVq+nduzfZ2dmkpDScs7QUBCIikbDjW9i3C5I6elsbX7wCs29iyZBnGXbl9ZSUlFBcXExSUhJxcXHMmTOHIUOGxLpq4OhB0LC3ZUREGpK0Ht51G61DY06FxL96PR/+tIS+bfYBUFxcTGFhISNGjKCoqIZjEg2IgkBEpL4692drs3QOHIaMTvFMHtKcE9KOXCtSUlLCzJkzY1hgeBQEIiL11TmDh3ePZNjzhRTud1x+cjPeuSaJpNDGQnFxMXl5ebGtMQwKAhGRY9C7d2+seRLH/U8hF75QzAlpxiMXeWcgJSUl0atXrxhXWDsFgYjIMcjOziYuLo49B+HdtYd5dPEBxmc1Z3ivBOLi4sjOzo51ibVSEIiIHIOUlBTmzJlDSkoKSUlJ3PP+flYUOGZc3pJ5r71McnKERr31ka4sFhE5RkOGDGHjxo3MnDmTvLw8vunekv9TMI3jts4EN6LBDzaoIBARiYDk5GTGjh17ZMKiJO/K5C//Bhk/jl1hYdCuIRERPwy+DbqdBW/d6V2E1oApCERE/BAXD2fcCPt3eYPhNWAKAhERvzTwYwOlfA0CM7vEzL42szwzm1RNe5qZ/cPMlpnZp2bW1896RESkKt+CwMzigSeB4UAf4Cdm1qfSbHcDuc65DGA0UMuNYEVEJNL83CI4E8hzzq11zh0AXgEurzRPH+A9AOfcV0APM+vkY00iItG3+HF47T+h4JtYV1ItP4OgK/Bduff5oWnlfQFcCWBmZwLHA+mVF2Rm48wsx8xyCgoKfCpXRCTCkkK32/z8L5D7F/hmbmzrqYGfQVDdUZLKNz+YAqSZWS5wM/A5cKjKh5yb7pzLcs5ldejQIeKFioj44oSh8Kt18Ku1sa7kqPy8oCwf6FbufTqwsfwMzrndwBgAMzNgXeghItI0tGoL+xv2PQn83CJYAvQ2s55m1hy4CphdfgYzaxNqA7gBWBgKBxERiRLftgicc4fM7CbgHSAemOGcW2Fm40PtTwGnAP9rZoeBlcDYGhcoIiK+8HWsIefcHGBOpWlPlXu9GOjtZw0iInJ0urJYRCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiLRUrQZvp4L2xvWaKQKAhERv1loVbv4CXg5G966M7b1VOLrWEMiIgI0bwWXPwklh+CTP8Gh/bGuqAIFgYhINAy4xnte9rfY1lEN7RoSEQk4BYGISMApCEREAk5BICISTe4wrP8QvlsS60rKKAhERKKpRYr3/NfRsa2jHAWBiEg0jXocOvWFQ3tjXUkZBYGISDSlHAddT4f4FrGupIyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREos5B0b9h/q9jXQigIBARib423b3nBb+F/UWxrQUFgYhI9P3HXTDs3lhXUaZJDEN98OBB8vPz2bdvX6xLabASExNJT0+nWbNmsS5FRADiQv8vfvAbuOABiI/d6rhJBEF+fj4pKSn06NEDM4t1OQ2Oc45t27aRn59Pz549Y12OiAAkdfCeFz8BGT+Gzv1jVkqT2DW0b98+2rVrpxCogZnRrl07bTGJNCT9r4JRT3iv354E007z7mccA00iCACFQC307yPSwJjBcf0gLgEKN8L2NbDpi5iU0iR2DYmINEpdMuG/Nns3t38wLWZlhLVFYGaDzeyfZvaNma01s3VmtjaMz11iZl+bWZ6ZTaqmPdXM3jCzL8xshZmNqU8nGgIz44477ih7/8gjj3D//feH/fnNmzczcuRI+vfvT58+fRgxYgQAH3zwASNHjqwy/+zZs5kyZQoA999/P4888ggA119/Pa+++uox9EREoiqGB4lLhVvBM8AvgaXA4XA+YGbxwJPAhUA+sMTMZjvnVpabbQKw0jl3mZl1AL42s7845w6E3YMGokWLFsyaNYvJkyfTvn37On/+3nvv5cILL+TWW28FYNmyZUedf9SoUYwaNapetYqIlBfuMYJdzrm3nXNbnHPbSh+1fOZMIM85tza0Yn8FuLzSPA5IMW8HdjKwHThUlw40FAkJCYwbN47HHnusStv69esZNmwYGRkZDBs2jA0bNlSZZ9OmTaSnp5e9z8jIqDLPkiVLGDBgAGvXruW5557jpptuimwnRCSQwt0imG9mDwOzgP2lE51znx3lM12B78q9zwfOqjTPE8BsYCOQAmQ750rCrKlG2X9aXGXayIzOXDuwB3sPHOb6Zz+t0v7D09P5UVY3thcf4BcvLq3QNvPnA8P63gkTJpCRkcGvfvWrCtNvuukmRo8ezXXXXceMGTO45ZZbeO2116p8Njs7myeeeIILLriAMWPG0KVLl7L2jz76iJtvvpnXX3+d7t27s3DhwrBqEhGpTbhBULoCzyo3zQHnH+Uz1Z2m4iq9vxjIDS3nROCfZrbIObe7woLMxgHjALp37x5mydHXunVrRo8ezbRp02jZsmXZ9MWLFzNr1iwArr322ipBAXDxxRezdu1a5s6dy9tvv82AAQNYvnw5AKtWrWLcuHHMmzevQjiIiERCWEHgnDuvHsvOB7qVe5+O98u/vDHAFOecA/LMbB1wMlDhJ7tzbjowHSArK6tymFRxtF/wLZvHH7W9bVLzsLcAqnPbbbdx2mmnMWZMzce9azqVs23btlx99dVcffXVjBw5koULF9KuXTs6d+7Mvn37+PzzzxUEIhJx4Z41lGpmj5pZTujxP2aWWsvHlgC9zaynmTUHrsLbDVTeBmBY6Ds6AT8Aaj0bqSFr27YtP/7xj3nmmWfKpg0aNIhXXnkFgL/85S8MGTKkyufef/999uzZA0BhYSFr1qwp2/pp06YNb731FnfffTcffPCB/50QkUAJ92DxDKAQ+HHosRt49mgfcM4dAm4C3gFWAX91zq0ws/FmNj4020PAIDP7EngPmOic21r3bjQsd9xxB1u3HunGtGnTePbZZ8nIyOCFF17g97//fZXPLF26lKysLDIyMhg4cCA33HADZ5xxRll7p06deOONN5gwYQKffPJJVPohIsFg3l6ZWmYyy3XOZdY2LRqysrJcTk5OhWmrVq3ilFNOiXYpjY7+nUQaqJIS74Kyc++Gcyf68hVmttQ5l1VdW7hbBHvNrGx/hpkNBvZGojgREYmtcM8a+gXwfOi4gOGd73+9X0WJiEj0hHvWUC7Q38xah97vPvonRESksThqEJjZNc65F83s9krTAXDOPepjbSIiEgW1bREkhZ5T/C5ERERi46hB4Jz7U+j5geiUIyIi0RbuBWW/M7PWZtbMzN4zs61mdo3fxTUmycnJx7yMnJwcbrnllhrbv/32W1566aWw5xeRxqb20/n9EO7poxeFDhCPxBs64iTgLt+qCqisrCymTZtWY3vlIKhtfhFpZD74DSx4OOpfG24QNAs9jwBeds5t96meJiU3N5ezzz6bjIwMrrjiCnbs2AF4w0mXXkF811130bdvX6DiTWgWLFhAZmYmmZmZDBgwgMLCQiZNmsSiRYvIzMzkscceqzB/UVERY8aMoV+/fmRkZPD3v/89Np0WkbozgzNu9F5/+Bj8e3lUvz7c6wjeMLOv8C4i+8/QTWQa5p3Q354E//4ysss8rh8Mn1Lnj40ePZrHH3+coUOHcu+99/LAAw8wdepUxowZw/Tp0xk0aBCTJlW5cRvg3eHsySefZPDgwRQVFZGYmMiUKVN45JFHePPNNwEqjDv00EMPkZqaypdfen0vDR0RaQTM4NJH4NtFUPAVvPlLuOGfUfv6sLYInHOTgIFAlnPuIFBM1ZvMSDm7du1i586dDB06FIDrrruOhQsXsnPnTgoLCxk0aBAAV199dbWfHzx4MLfffjvTpk1j586dJCQcPbPfffddJkyYUPY+LS129z8VkXr60fPQPBkORXfghtquIzjfOfe+mV1Zblr5WWb5VVi91eOXezSFM7YTwKRJk7j00kuZM2cOZ599Nu+++26ty61peGsRaSQ6ngzdB8Ke2m4AGVm1bREMDT1fVs2j6h3VpUxqaippaWksWrQIgBdeeIGhQ4eSlpZGSkoKH3/8MUDZ8NSVrVmzhn79+jFx4kSysrL46quvSElJobCwsNr5L7roIp544omy99o1JCLhqu06gvtCzzXfZUUA2LNnT4V7Dt9+++08//zzjB8/nj179nDCCSfw7LPeyN3PPPMMN954I0lJSZx77rmkpla9tcPUqVOZP38+8fHx9OnTh+HDhxMXF0dCQgL9+/fn+uuvZ8CAAWXz33PPPUyYMIG+ffsSHx/Pfffdx5VXXllluSIilYU7DPWvgd8553aG3qcBdzjn7vG3vKqawjDURUVFZdcdTJkyhU2bNlV7j4JIa2z/TiKB9OIPvV1D4+ZHdLGRGIZ6eGkIADjnduCdSir18NZbb5GZmUnfvn1ZtGgR99wT9TwVESkT7umj8WbWwjm3H8DMWgIt/CuracvOziY7OzvWZYiIAOEHwYvAe2b2LN410D8DnvetKhERiZpw70fwOzNbBlyAd2Oah5xz7/hamYiIREW4WwTg3YD+kHPuXTNrZWYpzrnqz2UUEZFGI9zRR28EXgX+FJrUFXjNp5pERIKr5BBs/Ayei96lWuGeNTQBGAzsBnDOrQY6+lWU3woLC3n66aeZOHEiTz/9dI0XadVFfHw8mZmZ9O/fn9NOO42PPvqoXsuZOnUqe/bsOeZ6RKSR6jPKe/52UdS+MtxdQ/udcwdKhzAwswRiNXD2Mfrwww8ZMWIEJSUlFBcXk5SUxO23386cOXMYMmRIvZfbsmVLcnNzAXjnnXeYPHkyCxYsqPNypk6dyjXXXEOrVq3qXYuINGJZP4OCbyD3pdrnjZBwtwgWmNndQEszuxD4G/CGf2X5o7CwkBEjRlBYWEhxcTEAxcXFZdOLiooi8j27d++uMOjbww8/zBlnnEFGRgb33Xdf2fdeeuml9O/fn759+zJz5kymTZvGxo0bOe+88zjvvPMiUouISG3C3SKYCNwAfAn8HJgDPO1XUX6ZOXMmJSUl1baVlJQwc+ZMxo4dW69l7927l8zMTPbt28emTZt4//33AZg3bx6rV6/m008/xTnHqFGjWLhwIQUFBXTp0oW33noL8EYrTU1N5dFHH2X+/Pm0b9++fp0UEamjWoPAzOKAZc65vsCf/S/JP6tXry7bEqisuLiYvLy8ei+7/K6hxYsXM3r0aJYvX868efOYN29e2bhARUVFrF69mnPOOYc777yTiRMnMnLkSM4555x6f7eINDUO9u+CDZ9A97N8/7Zadw0550qAL8ysu+/V+Kx3794kJSVV25aUlESvXr0i8j0DBw5k69atFBQU4Jxj8uTJ5ObmkpubS15eHmPHjuWkk05i6dKl9OvXj8mTJ/Pggw9G5LtFpAmID90UcsZFsHen718X7jGCzsCK0I3rZ5c+/CzMD9nZ2cTFVd/luLi4iA378NVXX3H48GHatWvHxRdfzIwZM8qOP3z//fds2bKFjRs30qpVK6655hruvPNOPvvsM4CjDjUtIgFx5s+h6+ne68MHff+6cI8RPOBrFVGSkpLCnDlzqpw1FBcXx5w5c8pGBK2P0mME4N0k5vnnnyc+Pp6LLrqIVatWMXDgQACSk5N58cUXycvL46677iIuLo5mzZrxxz/+EYBx48YxfPhwOnfuzPz5kR19UEQaiTbdoP9P4PulUfm6ow5DbWaJwHigF96B4mecc4eiUlkNIjEMdVFRETNnziQvL49evXqRnZ19TCHQWGgYapFG5NM/w5w74c48SO5wzIs72jDUtW0RPA8cBBYBw4E+wK3HXFGMJScn1/vsIBGRpqa2IOjjnOsHYGbPAJ/6X5KIiERTbQeLy45SxHqXUG3CvSl8UOnfR0RqUlsQ9Dez3aFHIZBR+trMdte2cDO7xMy+NrM8M5tUTftdZpYbeiw3s8Nm1raunUhMTGTbtm1a2dXAOce2bdtITEyMdSki0gDVdvP6+Pou2MzigSeBC4F8YImZzXbOrSy3/IeBh0PzXwb80jm3va7flZ6eTn5+PgUFBfUtt8lLTEwkPT091mWISANUl/sR1NWZQJ5zbi2Amb0CXA6srGH+nwAv1+eLmjVrRs+ePetVpIhI0IV7QVl9dAW+K/c+PzStCjNrBVwC/L2G9nFmlmNmOfrVLyISWX4GgVUzraad+JcB/6ppt5BzbrpzLss5l9Whw7GfTysi0mi8eRscqH6MtEjxMwjygW7l3qcDG2uY9yrquVtIRKRJSg7d++urN2H2LbArH/bU+RBqWPwMgiVAbzPraWbN8Vb2VcYnMrNUYCjwuo+1iIg0LqeMgp++6r1e/io8dip8/Adfvsq3g8XOuUNmdhPwDhAPzHDOrTCz8aH2p0KzXgHMc875u+0jItKYmEGvC+Da12DxE9DpVDj5Un++qrGde1/dWEMiInJ0RxtryM9dQyIi0ggoCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjA+RoEZnaJmX1tZnlmNqmGec41s1wzW2FmC/ysR0REqkrwa8FmFg88CVwI5ANLzGy2c25luXnaAH8ALnHObTCzjn7VIyIi1fNzi+BMIM85t9Y5dwB4Bbi80jxXA7OccxsAnHNbfKxHRESq4WcQdAW+K/c+PzStvJOANDP7wMyWmtno6hZkZuPMLMfMcgoKCnwqV0QkmPwMAqtmmqv0PgE4HbgUuBj4f2Z2UpUPOTfdOZflnMvq0KFD5CsVEQkw344R4G0BdCv3Ph3YWM08W51zxUCxmS0E+gPf+FiXiIiU4+cWwRKgt5n1NLPmwFXA7ErzvA6cY2YJZtYKOAtY5WNNIiJSiW9bBM65Q2Z2E/AOEA/McM6tMLPxofannHOrzGwusAwoAZ52zi33qyYREanKnKu8275hy8rKcjk5ObEuQ0SkUTGzpc65rOradGWxiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJuIRYFxBN2X9aXGXayIzOXDuwB3sPHOb6Zz+t0v7D09P5UVY3thcf4BcvLq3Sfs3Zx3NZ/y5s3LmXX87MrdJ+4zkncEGfTqwpKOLuWV9Wab/5/N4M6d2eFRt38eAbK6u0/+qSH3D68W1Zun47v5v7dZX2ey/rw6ldUvlw9VYef391lfZfX9mPEzsk8+7Kzfx50doq7Y9lZ9KlTUve+GIjL368vkr7H685nbZJzflbzne8ujS/SvtzY86kZfN4Xlj8LW8u21SlfebPBwIwfeEa3lu1pUJbYrN4nv/ZmQBMe281/8rbWqE9rVVznrr2dAB+O/crPlu/o0J759REpl41AIAH3ljByo27K7Sf0CGJ31yZAcDkWctYW1Bcob1Pl9bcd9mpANz2yuds2rWvQvtpx6cx8ZKTARj/wlJ27DlQoX1wr/bcMqw3ANfN+JR9Bw9XaB92SkfG/ceJgP729LcXmb+90j5FmrYIREQCzpxzsa6hTrKyslxOTk6syxARaVTMbKlzLqu6Nm0RiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBrdBeUmVkBUPV69PC0B7bWOlfToj4Hg/ocDMfS5+Odcx2qa2h0QXAszCynpivrmir1ORjU52Dwq8/aNSQiEnAKAhGRgAtaEEyPdQExoD4Hg/ocDL70OVDHCEREpKqgbRGIiEglCgIRkYBrkkFgZpeY2ddmlmdmk6ppNzObFmpfZmanxaLOSAqjzz8N9XWZmX1kZv1jUWck1dbncvOdYWaHzeyH0azPD+H02czONbNcM1thZguiXWOkhfG3nWpmb5jZF6E+j4lFnZFiZjPMbIuZLa+hPfLrL+dck3oA8cAa4ASgOfAF0KfSPCOAtwEDzgY+iXXdUejzICAt9Hp4EPpcbr73gTnAD2NddxT+O7cBVgLdQ+87xrruKPT5buC3odcdgO1A81jXfgx9/g/gNGB5De0RX381xS2CM4E859xa59wB4BXg8krzXA78r/N8DLQxs87RLjSCau2zc+4j51zpHbg/BtKjXGOkhfPfGeBm4O/AlmraGptw+nw1MMs5twHAOdfY+x1Onx2QYmYGJOMFwaHolhk5zrmFeH2oScTXX00xCLoC35V7nx+aVtd5GpO69mcs3i+KxqzWPptZV+AK4Kko1uWncP47nwSkmdkHZrbUzEZHrTp/hNPnJ4BTgI3Al8CtzrmS6JQXExFffyUcUzkNk1UzrfI5suHM05iE3R8zOw8vCIb4WpH/wunzVGCic+6w92Ox0QunzwnA6cAwoCWw2Mw+ds5943dxPgmnzxcDucD5wInAP81skXNut8+1xUrE119NMQjygW7l3qfj/VKo6zyNSVj9MbMM4GlguHNuW5Rq80s4fc4CXgmFQHtghJkdcs69FpUKIy/cv+2tzrlioNjMFgL9gcYaBOH0eQwwxXk70PPMbB1wMvBpdEqMuoivv5rirqElQG8z62lmzYGrgNmV5pkNjA4dfT8b2OWc2xTtQiOo1j6bWXdgFnBtI/51WF6tfXbO9XTO9XDO9QBeBf6zEYcAhPe3/TpwjpklmFkr4CxgVZTrjKRw+rwBbwsIM+sE/ABYG9Uqoyvi668mt0XgnDtkZjcB7+CdcTDDObfCzMaH2p/CO4NkBJAH7MH7RdFohdnne4F2wB9Cv5APuUY8cmOYfW5Swumzc26Vmc0FlgElwNPOuWpPQ2wMwvzv/BDwnJl9ibfbZKJzrtEOT21mLwPnAu3NLB+4D2gG/q2/NMSEiEjANcVdQyIiUgcKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBCpRmi00lwzWx4a2bJNhJf/rZm1D70uiuSyRepKQSBSvb3OuUznXF+8AcAmxLogEb8oCERqt5jQoF5mdqKZzQ0N6LbIzE4OTe9kZv8IjYn/hZkNCk1/LTTvCjMbF8M+iNSoyV1ZLBJJZhaPN3zBM6FJ04HxzrnVZnYW8Ae8wc6mAQucc1eEPpMcmv9nzrntZtYSWGJmf28C4zxJE6MgEKleSzPLBXoAS/FGtEzGu8HP38qNZtoi9Hw+MBrAOXcY2BWafouZXRF63Q3oDSgIpEFREIhUb69zLtPMUoE38Y4RPAfsdM5lhrMAMzsXuAAY6JzbY2YfAIl+FCtyLHSMQOQonHO7gFuAO4G9wDoz+xGU3Tu29N7P7wG/CE2PN7PWQCqwIxQCJ+PdVlCkwVEQiNTCOfc53r1yrwJ+Cow1sy+AFRy5beKtwHmhETCXAqcCc4EEM1uGN0Lmx9GuXSQcGn1URCTgtEUgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMD9f/L5GLCBF9QfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([0,1], [no_skill,no_skill], linestyle='--', label='No Skill')\n",
    "plt.plot(recall, precision, label='Logistic')\n",
    "plt.scatter(recall[ix], precision[ix], marker='o', color='black', label='Best', s = 50)\n",
    "# axis labels\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.legend()\n",
    "# show the plot\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(thresholds[ix])\n",
    "pred_proba_test = model_lr.predict_proba(X_test)\n",
    "pred_proba_test_class_1 = pred_proba_test[:, 1]\n",
    "y_hat = proba_to_classes(pred_proba_test_class_1, thresholds[ix])\n",
    "print(f1_score(y_test, y_hat))\n",
    "# predict_proba = model_lr.predict_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "FNs_pred_proba = []\n",
    "pred_neg = []\n",
    "for index, (label, label_hat) in enumerate(zip(y_test, y_hat)):\n",
    "    if label == 1 and label_hat == 0:\n",
    "        FNs_pred_proba.append(pred_proba_test_class_1[index])\n",
    "    if label_hat == 0:\n",
    "        pred_neg.append(pred_proba_test_class_1[index])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  3.   2.  13.  41. 351.]\n",
      "[20869. 20996. 30767. 24818. 61656.]\n",
      "[1.43753893e-04 9.52562393e-05 4.22530633e-04 1.65202675e-03\n",
      " 5.69287661e-03]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAFTCAYAAAB4XsS+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAArO0lEQVR4nO3deXxc5X3v8c9Pm+V9lfcVMBjb4E2sTkgCIdiQWM7WQil745CEpr1ZbkmT3rZZ2t6mTW5IKJQESmhICVkqmaUBkgCBAIllLNt4A4NXSbZlbEvetP/uH+eY1yC0jGwdnZkz3/frpZc0M+fR/B4t3znznOc8x9wdERGJTl7cBYiIJJ2CVkQkYgpaEZGIKWhFRCKmoBURiZiCVkQkYgVxF9AfxowZ49OnT4+7DBFJmNWrV+9395KetsuJoJ0+fTqVlZVxlyEiCWNmO9LZTkMHIiIRU9CKiERMQSsiEjEFrYhIxBS0IiIRU9CKiERMQSsiEjEFrYhIxBS0IiIRU9CKSM77zea9fP7htRw61hzJ91fQikjOe3jVbp59tY4hA6JZlUBBKyI5rf54C7/ZvI8PzZtAQX40kaigFZGc9stXamlua2f5/EmRPYeCVkRyWvmaGmaMGcy5k4dH9hwKWhHJWXvqG3lp25uUzZ+ImUX2PApaEclZK9dW406kwwagoBWRHFa+poZ5U0YwfczgSJ9HQSsiOem1vYfZWNvA8vkTI38uBa2I5KTyqmry84wPnqugFRHpc+5ORVUNi88YQ8nQAZE/n4JWRHLO6h0H2X3weL8MG4CCVkRyUHlVNcWFeXxgzvh+eT4FrYjklJa2dh5bV8vls8dHtrZBRwpaEckpv321joPHWvpt2AAUtCKSY8qrahg5qJBLzizpt+dU0IpIzjjS1MpTG/dw1bkTKIxopa7OKGhFJGc8uWEPjS3RrtTVGQWtiOSM8qoaJo8cyKJpI/v1eRW0IpIT6g438fxrdZGv1NUZBa2I5IRH19XQ3g8rdXVGQSsiOaG8qobZE4Yxc9zQfn9uBa2IJN62/UdZu+sQyxf039zZVApaEUm8iqpqzGDZvP4fNgAFrYgk3ImVui6cMZrxw4tjqUFBKyKJtm53Pdv2H41t2AAUtCKScOVV1RTl57Fk7oTYalDQikhitba188jaWi6dNZbhAwtjqyPSoDWzJWa2xcy2mtntnTxuZnZH+Pg6M1vYU1sz+zszqzazqvDjyij7ICLZ64XX32T/kaZYhw0gwqA1s3zgTmApMBu4xsxmd9hsKTAz/FgB3JVm22+7+/zw4/Go+iAi2a28qpqhxQW896yxsdYR5R7t+cBWd3/D3ZuBh4CyDtuUAQ944CVghJlNSLOtiEiXjje38cQre7hy7gSKC/NjrSXKoJ0E7Eq5vTu8L51temp7WzjUcJ+Z9e/qECKSFX61aS9Hm9soi3nYAKIN2s5WbfA0t+mu7V3A6cB8oBb4106f3GyFmVWaWWVdXV1aBYtIclRUVTN+WDEXzhgddymRBu1uYErK7clATZrbdNnW3fe6e5u7twPfJxhmeAd3v8fdS929tKSk/1ZSF5H4HTzazDNb6lg2fyJ5ef27UldnogzaVcBMM5thZkXA1cDKDtusBK4PZx9cCNS7e213bcMx3BM+DLwSYR9EJAs9tr6W1nanrB+vC9adyC4B6e6tZnYb8ASQD9zn7hvM7Nbw8buBx4Erga3AMeCm7tqG3/qfzWw+wVDCduCTUfVBRLJTRVU1M8cOYfaEYXGXAkQYtADh1KvHO9x3d8rXDnwm3bbh/df1cZkikiC7Dx5j1faDfPGKs/p9ge+u6MwwEUmUiqrgUNCyeZkxbAAKWhFJkGClrmpKp41kyqhBcZfzFgWtiCTGptrDvLr3CGUL4ll3tisKWhFJjIqqagryjKvOiW+lrs4oaEUkEdrbnZVra3jPmSWMGlwUdzlvo6AVkUT4/bYD1NY3ZtywAShoRSQhKqqqGVyUz+Vnj4u7lHdQ0IpI1mtqbePx9bVcMWc8A4viXamrMwpaEcl6T2+uo6GxNSOHDUBBKyIJUFFVzZghRSw+Pf6VujqjoBWRrNbQ2MKvN+/jg+dOpCA/MyMtM6sSEUnTL9fvobm1neUZOmwACloRyXLlVdVMHz2IeZOHx11KlxS0IpK19tQ38uIbb1I2f1LGrNTVGQWtiGStR9bW4E5GDxuAglZEslh5VTXzJg9nxpjBcZfSLQWtiGSlrfsOs6GmgbL5mb03CwpaEclS5WtqyDP44LzMWqmrMwpaEck67k7F2moWnzGGsUOL4y6nRwpaEck6L+88yK4Dx1meBcMGoKAVkSxUvqaG4sI8rpg7Pu5S0qKgFZGs0tLWzmPra3n/2eMYMiDSC3n3GQWtiGSV516r48DR5qwZNgAFrYhkmfI1NYwYVMglZ5bEXUraFLQikjWONLXy5MY9XHXOBIoKsie+sqdSEcl5T27YQ2NLZq/U1RkFrYhkjfKqGiaNGMiiqSPjLqVXFLQikhXqDjfx/Gt1lM2fSF5e5q7U1RkFrYhkhUfX1dCeBSt1dUZBKyJZobyqhrMnDOPMcUPjLqXXFLQikvG27T/K2l2HWD5/YtylnBQFrYhkvIqqasxgmYJWRKTvuTsVVTVcMGMUE4YPjLuck6KgFZGMtm53Pdv2H82qU247UtCKSEYrr6qmKD+Ppedk/gLfXYk0aM1siZltMbOtZnZ7J4+bmd0RPr7OzBb2ou0XzMzNbEyUfRCR+LS2tfPI2lreN6uE4QML4y7npEUWtGaWD9wJLAVmA9eY2ewOmy0FZoYfK4C70mlrZlOAy4GdUdUvIvF74fU32X+kKauHDSDaPdrzga3u/oa7NwMPAWUdtikDHvDAS8AIM5uQRttvA/8b8AjrF5GYlVdVM7S4gPfNGht3KackyqCdBOxKub07vC+dbbpsa2bLgGp3X9vXBYtI5jje3MYTr+xh6dzxFBfmx13OKYlyefLOTkbuuAfa1Tad3m9mg4AvAx/o8cnNVhAMRzB16tSeNheRDPOrTXs52tyW9cMGEO0e7W5gSsrtyUBNmtt0df/pwAxgrZltD+9/2czeceEgd7/H3UvdvbSkJHsWCBaRQEVVNeOGDeCC00bHXcopizJoVwEzzWyGmRUBVwMrO2yzErg+nH1wIVDv7rVdtXX39e4+1t2nu/t0gkBe6O57IuyHiPSzg0ebeWZLHcvmTSQ/y1bq6kxkQwfu3mpmtwFPAPnAfe6+wcxuDR+/G3gcuBLYChwDbuqubVS1ikhmeWx9La3tTlkChg0g2jFa3P1xgjBNve/ulK8d+Ey6bTvZZvqpVykimaaiqpozxg5hzsRhcZfSJ3RmmIhklN0Hj7Fq+0GWz5+IWfYPG4CCVkQyTEVVcMw8KcMGoKAVkQwSrNRVzaJpI5kyalDc5fQZBa2IZIxNtYd5de+RrF3guysKWhHJGBVV1RTkGVedq6AVEelz7e3OyrU1XHJmCaMGF8VdTp9S0IpIRvj9tgPU1jdSlrBhA1DQikiGqKiqZlBRPpfPHhd3KX1OQSsisWtqbePx9bVcMWc8g4oiPY8qFgpaEYnd05vraGhsTeSwAShoRSQDVFRVM2ZIEe86I5lXplLQikisGhpb+PXmfXzw3IkU5CczkpLZKxHJGr9cv4fm1vbEDhuAglZEYlZeVc200YOYP2VE3KVERkErIrHZU9/Ii2+8Sdn8SYlZqaszCloRic0ja2twJ3FrG3SkoBWR2JRXVXPu5OGcVjIk7lIipaAVkVhs3XeYDTUNiVp3tisKWhGJRfmaGvIMPjRvQtylRE5BKyL9zt2pWFvN4jPGMHZocdzlRE5BKyL97uWdB9l14HhODBuAglZEYlC+poYBBXlcMSd5K3V1RkErIv2qpa2dx9bX8v7Z4xhaXBh3Of1CQSsi/eq51+o4cLSZ5TkybAAKWhHpZ+VrahgxqJD3nFkSdyn9RkErIv3maFMrT23cy5XnTKCoIHfiJ+2emtngKAsRkeR7cuMejre05dSwAaQRtGZ2sZltBDaFt+eZ2b9FXpmIJE75mhomjRhI6bSRcZfSr9LZo/02cAXwJoC7rwUuibIoEUme/UeaeH7rfpbNn0heXnJX6upMWkMH7r6rw11tEdQiIgn26Noa2to954YNANK53OQuM7sYcDMrAj5LOIwgIpKu8qoaZo0fylnjh8ZdSr9LZ4/2VuAzwCRgNzAf+HSENYlIwmzff5SqXYdYviD39mYhvT3as9z92tQ7zGwx8LtoShKRpKmoqsEMls1L9gLfXUlnj/a7ad4nIvIO7k5FVTXnTx/FxBED4y4nFl3u0ZrZRcDFQImZfS7loWFAftSFiUgyrK+u5439R/nEJafFXUpsutujLQKGEITx0JSPBuBj6XxzM1tiZlvMbKuZ3d7J42Zmd4SPrzOzhT21NbOvhdtWmdmTZpab70VEskT5mhqK8vO4cm7yF/juSpd7tO7+LPCsmd3v7jt6+43NLB+4E7ic4CDaKjNb6e4bUzZbCswMPy4A7gIu6KHtN939b8Ln+CzwfwgO2IlIhmlrdx5ZV8N7zyph+KDcWKmrM+kcDDtmZt8E5gBvLYXu7pf20O58YKu7vwFgZg8BZUBq0JYBD7i7Ay+Z2QgzmwBM76qtuzektB8MeBp9EJEYvPD6fuoON+XsbIMT0jkY9iCwGZgB/D2wHViVRrtJQOqJDrvD+9LZptu2ZvYNM9sFXEuwRysiGah8TQ1DBxRw6ayxcZcSq3SCdrS73wu0uPuz7n4zcGEa7To7x67j3mdX23Tb1t2/7O5TCF4Ebuv0yc1WmFmlmVXW1dWlUa6I9KXGljae2LCHJXPHU1yY28fP0wnalvBzrZldZWYLgMlptNsNTEm5PRmoSXObdNoC/Bj4aGdP7u73uHupu5eWlOTOupcimeJXm/ZypKk154cNIL2g/bqZDQc+D3wB+AHwv9JotwqYaWYzwlN3rwZWdthmJXB9OPvgQqDe3Wu7a2tmM1PaLyMY1hCRDFO+poaxQwdw4Wmj4y4ldt0eDAuP/s9090eBeuB96X5jd281s9uAJwjm3d7n7hvM7Nbw8buBx4Erga3AMeCm7tqG3/qfzOwsoB3YgWYciGScQ8eaefbVfdxw0XTyc2ylrs50G7Tu3mZmywiWSuw1d3+cIExT77s75WsnWEchrbbh/Z0OFYhI5nhsfS0tba5hg1A607teMLPvAT8Bjp64091fjqwqEclqFWtqOL1kMHMmDou7lIyQTtBeHH7+asp9DvQ0j1ZEctDug8f4w/YDfP7yMzHTsAGkEbTunva4rIjIyrXBBKGyHFzguyu5cxlKEekXFWtqWDh1BFNHD4q7lIyhoBWRPrOptoEtew/rIFgHCloR6TPlVdXk5xlXnZO7K3V1prv1aLu90q27/7bvyxGRbNXe7jxSVcMlM8cwesiAuMvJKN0dDPtiJ/c5MI/glNjcPnlZRN7mD9sPUFPfyF8tnRV3KRmnu/VoP5R628zeBXwZqKWLhVxEJHdVVFUzqCify2ePi7uUjNPj9C4zuwz4G4K92X9w96cir0pEskpTaxuPravlA7PHMagonen5uaW7MdqrCPZg64Evu7uueisinXpmSx0Nja2UabZBp7p76XmEYLnCN4G/6niGh7svi7AuEckiFVXVjB5cxLvPGBN3KRmpu6DVGWEi0qOGxhZ+tWkf15w3hYJ8zRjtTHdBu83dd/ZbJSKSlX75yh6aW9s1bNCN7l5+yk98YWY/j74UEclGFVXVTBs9iAVTRsRdSsbqLmhTB2VPi7oQEck+exsaeeH1NymbN1ErdXWju6D1Lr4WEQHgkbU1uKNhgx50N0Y7z8waCPZsB4ZfE952d9eKviI5rryqmnMmDef0kiFxl5LRujszTKfYikiXtu47wivVDXzlqrPjLiXjaS6GiJyUiqpq8gyWzZsYdykZT0ErIr3m7lRU1XDx6WMYO6w47nIynoJWRHrt5Z2H2HngGGXztTebDgWtiPRaRVU1AwryWDJ3fNylZAUFrYj0SktbO4+uq+X9Z49jaHFh3OVkBQWtiPTK86/t58DRZg0b9IKCVkR6pbyqmuEDC3nvWWPjLiVrKGhFJG1Hm1p5csNerjxnAkUFio906SclIml7auNejre0sVzDBr2ioBWRtJVXVTNxeDHnTR8VdylZRUErImnZf6SJ517bz7L5k8jL00pdvaGgFZG0PLaulrZ2Z/kCDRv0loJWRNJSXlXNrPFDmTVeC/f1loJWRHq0482jrNl5iLL5Wnf2ZChoRaRHFVU1ACzTbIOToqAVkW65O+VV1Zw/YxSTRgyMu5yspKAVkW69Ut3AG3VHWa5hg5MWadCa2RIz22JmW83s9k4eNzO7I3x8nZkt7KmtmX3TzDaH2/+3mY2Isg8iua68qprCfOPKc7RS18mKLGjNLB+4E1gKzAauMbPZHTZbCswMP1YAd6XR9ilgrrufC7wKfCmqPojkurZ255G1Nbz3rLGMGFQUdzlZK8o92vOBre7+hrs3Aw8BZR22KQMe8MBLwAgzm9BdW3d/0t1bw/YvAZMj7INITnvx9TfZd7hJwwanKMqgnQTsSrm9O7wvnW3SaQtwM/A/nT25ma0ws0ozq6yrq+tl6SICwbDBkAEFXHa2Vuo6FVEGbWfn6Hma2/TY1sy+DLQCD3b25O5+j7uXuntpSUlJGuWKSKrGljZ++coelswdT3GhLop9Krq83Hgf2A1MSbk9GahJc5ui7tqa2Q3AB4HL3L1jeItIH/j1pn0caWrVsEEfiHKPdhUw08xmmFkRcDWwssM2K4Hrw9kHFwL17l7bXVszWwL8FbDM3Y9FWL9ITiuvqmbs0AFcdProuEvJepHt0bp7q5ndBjwB5AP3ufsGM7s1fPxu4HHgSmArcAy4qbu24bf+HjAAeMrMAF5y91uj6odILjp0rJlntuzj+oumk6+Vuk5ZlEMHuPvjBGGaet/dKV878Jl024b3n9HHZYpIB4+v30NLm2vYoI/ozDAReYfyqmpOKxnM3ElaqasvKGhF5G2qDx3nD9sOsHz+JMLhOTlFCloReZuV4Updupx431HQisjbVFRVs2DqCKaNHhx3KYmhoBWRt2ze08DmPYd1EKyPKWhF5C3la2rIzzOuOndC3KUkioJWRABob3dWVlXz7pljGDNkQNzlJIqCVkQAWLX9ADX1jRo2iICCVkQAKK+qYWBhPpfPHhd3KYmjoBURfr56Nz9bvYulc8czeECkJ4zmJP1ERXKYu/Otp17lu7/ZykWnjeZvPzQn7pISSUErkqMaW9r44s/W8cjaGv6odDJfX34ORQV6kxsFBa1IDnrzSBMr/nM1q3cc5ItXnMWn33u6TreNkIJWJMds3XeEm+9fxZ6GRr73Jwv44Lk61TZqClqRHPLC6/u59T9XU5ifx0MrLmTh1JFxl5QTFLQiOeKnlbv40i/WM33MYP7jxvOYMmpQ3CXlDAWtSMK1tzv/+tQW7nz6dRafMZp/u3YRwwcWxl1WTlHQiiRYY0sbX/jpWh5dV8vV503ha8vnUpivmQX9TUErklBvHmniEw9U8vLOQ9y+dBafvOQ0zSyIiYJWJIG27jvMTfevYl9DE3ddu5Cl52g1rjgpaEUS5ndb93Prj1YzoCCYWbBAMwtip6AVSZCfrNrJl//7FWaMGcx9mlmQMRS0IgnQ3u5888kt3PXM67x75hjuvHYhw4o1syBTKGhFslxjSxuff3gtj62v5Zrzp/LVsjmaWZBhFLQiWazucDCzYO3uQ/z1lbP4xLs1syATKWhFstRre4OZBfuPBDMLlszVzIJMpaAVyULPv7afT/1oNcVF+fxkxUXMmzIi7pKkGwpakSzz0B928pXyVzi9ZAj33XQek0YMjLsk6YGCViRLtLc7//eJzfz7s29wyZkl3PknCxiqmQVZQUErkgWON7fxuYer+J9X9nDtBVP5+2VzKNDMgqyhoBXJcPsON/KJB1azbvchvnLV2dzyrhmaWZBlFLQiGWzLnsPcfP8qDhxt5u4/XcQVc8bHXZKcBAWtSIb67at1fObBlykuyufhT17EOZOHx12SnCQFrUgGevD3O/g/FRuYOXYI996omQXZTkErkkHa251//J9NfP+5bbznzBK+p5kFiRDpYUszW2JmW8xsq5nd3snjZmZ3hI+vM7OFPbU1s4+b2QYzazez0ijrF+lPx5vb+NSDq/n+c9u47sJp3HtDqUI2ISLbozWzfOBO4HJgN7DKzFa6+8aUzZYCM8OPC4C7gAt6aPsK8BHg36OqXaS/7Wto5M8eqGR9dT1/88HZ3Lx4umYWJEiUQwfnA1vd/Q0AM3sIKANSg7YMeMDdHXjJzEaY2QRgeldt3X1TeF+EpYv0n817Grj5P1Zx8FgL91xXyuWzx8VdkvSxKIcOJgG7Um7vDu9LZ5t02nbLzFaYWaWZVdbV1fWmqUi/eWbLPj5214u0ufPTWy9SyCZUlEHb2S6np7lNOm275e73uHupu5eWlJT0pqlIv/jPl3Zwyw8rmTJqEOWfWczcSZq+lVRRDh3sBqak3J4M1KS5TVEabUWyUlu784+Pb+IHz2/j0lljueOaBQwZoAlASRblHu0qYKaZzTCzIuBqYGWHbVYC14ezDy4E6t29Ns22IlnnWHMrt/5oNT94fhs3Xjyde65bpJDNAZH9ht291cxuA54A8oH73H2Dmd0aPn438DhwJbAVOAbc1F1bADP7MPBdoAR4zMyq3P2KqPoh0lf2NjRyyw9XsbGmgb/90GxuWjwj7pKkn1hwwD/ZSktLvbKyMu4yJIdtrGnglh+uov54C9+9ZgGXna2DXklgZqvdvcf5/HrPIhKxpzfv47Yfv8yQ4gIe/uRFOuiVgxS0IhF64MXt/N3KDcwaP4z7bjyP8cOL4y5JYqCgFYlAW7vz9cc28h+/2877zx7Ld65ewGAd9MpZ+s2L9LGjTa38xUNr+NWmfdy0eDpfuWo2+Xk6kzGXKWhF+tCe+mBmwabaBr5aNofrL5oed0mSARS0In1kQ009t9xfyeHGFu694TzeN2ts3CVJhlDQivSB32zey20/XsPwgYX89NaLmT1xWNwlSQZR0Iqcovt/t42vPrqR2ROHce8N5zFumGYWyNspaEVOUlu787VHN3L/C9t5/9njuOOa+Qwq0r+UvJP+KkROwpGmVj77X2v4zeZ93PKuGfz1lWdrZoF0SUEr0ku19ce5+f5Ktuxp4GvL53LdhdPiLkkynIJWpBdeqa7nlh+u4khjK/feeB7vO0szC6RnClqRNP1q417+/L/WMHJQIT/71MWcPUEzCyQ9ClqRHrg7//G77XztsY2cM2k4P7i+lLGaWSC9oKAV6UZrWztffXQjD7y4gyvmjOPbf6yZBdJ7+osR6cKRplZu+/HLPLOljhWXnMbtS2aRp5kFchIUtCKdqDl0nJvvX8Vr+47wjQ/P5doLNLNATp6CVqSD9buDmQXHmtu478bzeM+ZuoqynBoFrUiKJzfs4S8eqmLU4CJ+/qkLOGv80LhLkgRQ0ErOcnd2HjjG6h0H3/rYvOcw8yYP5/s3lDJ2qGYWSN9Q0ErOaGpt45XqBlbvOBAG6yH2H2kCYOiAAuZPHcGH5k3k5sUzGFiUH3O1kiQKWkmsusNNvLzzIC/vOEjljoOs311Pc1s7ANNGD+KSmWNYOG0kpdNHMnPsUK1VIJFR0EoitLc7r+47/LZhgB1vHgOgKD+PuZOGcePi6SycOpKF00ZoWED6lYJWstKRplbW7jpE5faDrN55kDU7D3K4sRWAMUOKWDh1JNdeMJVF00YyZ+Jwigs1FCDxUdBKxnN3dh88zss7gz3Vyu0H2byngXYHMzhr3FA+NG8ii6YGwwBTRw3CTMMAkjkUtJJxmlvb2VjbQOX2A2+F696G4KDVoKJ8FkwdwW2XzmTRtJHMnzKC4QMLY65YpHsKWondgaPNvLwjGAJYvf0ga3cfoqk1OGg1eeRALjxtNIumjWTRtJGcNW4oBfl5MVcs0jsKWulX7e3OG/uPBGOrYbi+UXcUgII8Y86k4fzphdPeClZdf0uSQEErkTrW3MraXfVvzV19eech6o+3ADByUCGLpo3k44umsGjaSM6drINWkkwKWulTNYeOv22K1cbaBtraHYAzxg5h6dzxwdzVaSOZMWawDlpJTlDQyklraWtnc+1hKk/sre44SE19IwADC/OZN2U4n3rP6SyaNpIFU0cwYlBRzBWLxENB28GbR5r43MNrKS7Mo7gwnwEFb/984usBhfkUp3x+27aFeRQX5L+jfbavZXroWDNrdh4KpljtOMDaXfUcb2kDYOLwYhZOG8knpo2kdNooZk0YSqEOWokACtp3aGlzDh1voamhjcaWNppa22lsaaOxpZ3G1jbcT/57F+XnBSHcaYDnMaAg+FxcEIT1gILOQ/tt23byvQakPF6Un3dSb8/dnW37j1IZ7qmu3nGQ1/YdASA/z5gzcRh/fN6Utw5aTRwx8OR/MCIJp6DtYPzwYio+s7jTx9ydljansbWNppb2MIiDEO7qc8ewfuvx1PvD71d/vKXTts3hVKeTYcY7Qz0lwAd0Etp7GxpZveMgB48FB62GFRewaNpIyuZPZNG0UcybMlyXcxHpBf239IKZUVRgFBXkQT/OOmpvd5rb3hnWbw/x1OB+e5B3ts2Jz4cbW9nf2kxTyuPDBxXy/rPHvbW3enrJkKwf9hCJU6RBa2ZLgO8A+cAP3P2fOjxu4eNXAseAG9395e7amtko4CfAdGA78EfufjDKfsQtL88ozsvX1CeRLBXZ0QozywfuBJYCs4FrzGx2h82WAjPDjxXAXWm0vR34tbvPBH4d3hYRyVhRHhY+H9jq7m+4ezPwEFDWYZsy4AEPvASMMLMJPbQtA34Yfv1DYHmEfRAROWVRBu0kYFfK7d3hfels013bce5eCxB+HtvZk5vZCjOrNLPKurq6k+6EiMipijJoOzt60nFyVFfbpNO2W+5+j7uXuntpSYmuYioi8YkyaHcDU1JuTwZq0tymu7Z7w+EFws/7+rBmEZE+F2XQrgJmmtkMMysCrgZWdthmJXC9BS4E6sPhgO7argRuCL++AaiIsA8iIqcssuld7t5qZrcBTxBM0brP3TeY2a3h43cDjxNM7dpKML3rpu7aht/6n4CHzewWYCfw8aj6ICLSF8xP5ZzSLFFaWuqVlZVxlyEiCWNmq929tKfttOqHiEjEFLQiIhFT0IqIRCwnxmjNrA7Y0ctmY4D9EZSTDdT33JXL/T+Zvk9z9x4n6udE0J4MM6tMZ5A7idT33Ow75Hb/o+y7hg5ERCKmoBURiZiCtmv3xF1AjNT33JXL/Y+s7xqjFRGJmPZoRUQipqAVEYmYgjZNdjLX7BZJGDPLycw41X5rjLYH4Zq3h9z9uJmZ5/APzMxmAe3AcXff1dP2SWNm88Mvm919Y5y19DczuwxodPffmVmeu7fHXVN/MbMPAKOAn5zs/39Ovjqly8yWA48BN5jZEHf3XN2zNbMPAT8CvgV82cwuj7mkfmVmVwL3A58FvmRmC+KtqP+EV57+FvCsmb3b3dvNLC8X/hfM7Arg/wG1p7KTpaDtQrgnezuwkeBqD1fnatia2WLgnwkWWr+JYB3ghbEW1Y/M7F3A14EV7n4z0AwUmllhyjaJ/Ztw9wPAfQQvNI+Z2RJ3b0/6uzszKyW4MvdX3P1ZMxtuZmPNbHxvv1dkC38nwEHgVuBV4GNAKYCZ/dTd63Ps7dNI4PsnFl83s98CXzezu4AjOfBzGAx8zt3/EO7dfYDgoqBvmNnr7n5HUkMnZbgsD/hP4BfAT8zsHwB3939O8JDaGGADsN/MFgFfI3iRbTSzF939O+l+I+3RdsHdG4FN7n7M3R8guLzOucAfhZucFVtx/czdHyW8xLuZFRBcobjR3RvCt5ETYy0wYu7+hLs/E/b9r4B/BK4juIzSZWaW2L+FlAB9Hlju7o8D3yP4GQwPH0vk3ry7/5JgL/424EGgnGDn6wfAJWZ2RrrfS3u03XD3ppRX6wcJ/qBmmtnPCP/B3D0nLg7p7m+Gn1vN7ADhi7SZXQ8sNrPPu/uROGuMWtj3r53op5m9AHyK4ABh0h0CisMxyw8D/0IwVv0rd3861soicOL/3t1/bmbNwGh3vz987BDwaXpxZW4FbajjUEDKD9rDr9uBB8zsYWARcEkuhGwXbwsNOGZmXwWWAX+S1JDt2P8O/bwCmAoc7ffC+kFq3939tXAc+hfA9WEA/R6ojbXICJ3IBHd/JHU8HlgCTCS4zmF63yuZQyu9Y2aXAu8FXgeed/fXw/vfCl8zywcmAc8SvIVaG1O5kQoP/Awj+Nt4rJPHDSgC1hDsyX3U3bf0b5XR6an/4TYFwBeAPwauc/dX+rHEyHTV9xOBa2bnho+tNbN8d2+Lrdg+1lPfU25/luCAcK9+7zk/RmtmlwA/BaqBMuA2M/sEQDj++Nb4k7vvBBYkOGTfD/yMYEbBv5jZN8xsdOo24U5+E8FUr+sSFrI99h+CIQSCv5drExSyXfY9Za92XRiyRoKGS9Lpe4o6gndwvfq95/werZldA0wJj56OB94NXApUuvu94TbnE6yk/tMYS41M+I+TB9wBVLn798Ofxd0ER12/6e6Hwm0vA15097TfNmW6Xvb/UuD37p6I4YJe9v19wGp3b4ir3r7Un7/3nN+jJRhnucnMprn7HuBJ4GlgjplNDn8ZEwmOuiZSuJfaRjAcMMfMRoU/i08SzLT4QsrmSwimvSRGL/u/FHjHXm626mXfrwRG9H+V0ejP33vOB627VwAPAV8wswnuXg/8FpgNLA5/GeXunthB/xSbCcapZpnZAHffC6wAPhLuyeLuXwyHUJIol/uvvkfY95wO2pTx13LgMPAVM5sRvqq9CEy1UFw19id3f57gj+4vgVIzGxO+wPwSaI2ztv6Qy/1X36Pte06N0YYTy0cBlUB76lFTC063u5TglewXBEcW35Wkgz0dpR5RTT2KbGZfAM4mOOCxi+Ct1HvcfWtsxUboxFhdLvY/1/reySyCfvm7z5mgNbOPAP9AcLS4GlgNPODuB81sIXCeu/+7BROyC4BX3f21+CqOjpmVAG+emFURTt058Xk2wbzQ0cAsgrGq+919c5w196Xw911E8GL7h9SwSXr/zewCgrfJze7+bHhfXvi3kPS+LwHGAys9WL+h//ru7on/AAqBnxCMuQJ8lGCRlL8DLgO2AZfHXWc//SyWA88A1wL54X0nXnBLgX3Au1O2t7hr7uP+LyVYKOhbQA1wdcpjie4/wYHMTcBXwlD5QMpj5ye572F/HgWeIziNfkyH33tdlH3PpTHaYcDM8Ov/JvihFxIE7Qp3fyrpY7FmNg34BrCX4GDfx8O3Tm5mA4F3EcwRfC6JP4twwv2/Ap9w988BtxCuymZmg0hw/8O+fwv4lLt/Hfh7oCD8mwBYDPxpEvueYi3BLKPLgaVmVmBmQwj6fnWUfc+loYPLgT8nmBv3nAVnel0LvNeDpe+6Ot00McI/qtnAVuAa4EyCg36/cPdmMxvq7octoSuTmdl5wAx3f9iCFfNnECz/d5W7HzGzwe5+NIn9D98aF7l7lQWLAG0Bfg7MB77t7icWDUrs/0B4HOZsgpX5ygj24JuB77r7gSj7nkt7tM8RzJG9zswucfc2D1blmm7hyvlJ/QM7wYPz9Nd5MD51L8ESkBcRLAMJ4TzBpIXMCe6+iuAUajw4h/11grfQ+eEmie2/u28MQzaf4HTzv3T3G4FPAHeY2cXhdkn+H8gDbvRgNbpa4IsE84KPQLR9z5mg9WDZwwcJ3j58ycxWmNkNwDgSvDBGR+7eGL5yNxIsAfcqwfzBB4EXzGxkrAVG5MRbQg/mSBLO2htAsH5FoZndCJSb2dAEv3XGgyPs5e5+b7jnvopgCczETuFK+d3/AXjagquFXA18BygBlocvQNHVkOwXsHcysyKCMZlPAo3Ad9x9TbxVRaOz6WwdZxmE21UQHGVd7glax6GL/ndcpe0nwHaCv4lbPTlrF/TY93C7awjW2F3mCTkZoYu+54eff0iwGNDH3P1RM/sYwSnl1ZHWlGtBe0L4CuZJfJsInU5nqySYrtKQMqUlHziDYFjl/e6+Lr6K+1Y6/Q+3e5Rg3Poqd98UW8F9KM3f/TCCGShfJDgQtCG2gvtQd31P2WZef+9Q5GzQJpkFa2f+CLjDg6uWfhS4EGgiOBhY32H7Enevi6HUSPSm/xYsXP6CJ2dCfm/6/j5gu7tvi6favnUSf/f9duAvZ8Zoc1Bn09mKCGYbYGYXWHBlV5IUsil66v9FZvY+d38gKSGbIp2+X+buTyclZFP01PfzzWwp9O+BPwVtArl7C8GcyY9YeHlogtXHqoB3hweBphGsWpQ4afZ/CsEUp0TpRd83xldlNNLs+/Twdr/S0EFCmVkx8GcEB7l+5O6/De9/Gviku78aZ31Ry+X+q++Z13ddMyyhwmlcDxJcQO5LZjaLYKxqLFDfbeMEyOX+q++Z13ft0SZcLk1n60wu9199z5y+K2hzRNKns/Ukl/uvvsffdwWtiEjENOtARCRiCloRkYgpaEVEIqagFRGJmIJWcpKZTTezd6zUZWY/CBfJFukzOmFBJIW7/1ncNUjyaI9WclmBmf3QzNaZ2c/MbJCZPRNe8gQzO2Jm3zCztWb2kpmNi7tgyU4KWsllZwH3uPu5QAPw6Q6PDwZecvd5wG8JLvsi0msKWsllu9z9d+HXPyK4Cm6qZoJl9gBWE6z8JNJrClrJZR1Pi+x4uyVlzdI2dExDTpKCVnLZVDO7KPz6GoK1S0X6nIJWctkm4AYzW0dwMb+7Yq5HEkqLyoiIREx7tCIiEVPQiohETEErIhIxBa2ISMQUtCIiEVPQiohETEErIhIxBa2ISMT+P+DEL0UdadWqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "FNn, bins, _ = plt.hist(FNs_pred_proba, bins=5, range = [0,0.5])\n",
    "print(FNn)\n",
    "pred_Nn, _, _ = plt.hist(pred_neg, bins=5, range = [0,0.5])\n",
    "print(pred_Nn)\n",
    "FN_rate = np.divide(np.array(FNn), np.array(pred_Nn))\n",
    "print(FN_rate)\n",
    "plt.close()\n",
    "plt.figure(figsize = (5, 5))\n",
    "plt.plot(FN_rate)\n",
    "plt.xticks(np.arange(5), [\"0-0.1\", \n",
    "                          \"0.1-0.2\", \n",
    "                          \"0.2-0.3\", \n",
    "                          \"0.3-0.4\", \n",
    "                          \"0.4-0.5\"],\n",
    "                        rotation=45)\n",
    "#                           \"0.5-0.6\", \n",
    "#                           \"0.6-0.7\", \n",
    "#                           \"0.7-0.8\"],\n",
    "\n",
    "plt.ylabel('FN rate')\n",
    "plt.xlabel('bin')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.43753893e-04, 9.52562393e-05, 4.22530633e-04, 1.65202675e-03,\n",
       "       5.69287661e-03])"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FN_rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found input variables with inconsistent numbers of samples: [2013, 2517]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/qc/ys_y69ln715f54sflsztxbzw0000gn/T/ipykernel_19575/923293693.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mgrid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_grid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'f1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mgrid_tfidf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;31m# extra_args > 0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    757\u001b[0m             \u001b[0mrefit_metric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 759\u001b[0;31m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroups\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindexable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    760\u001b[0m         \u001b[0mfit_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_check_fit_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    761\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mindexable\u001b[0;34m(*iterables)\u001b[0m\n\u001b[1;32m    354\u001b[0m     \"\"\"\n\u001b[1;32m    355\u001b[0m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_make_indexable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mX\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterables\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[0;34m(*arrays)\u001b[0m\n\u001b[1;32m    317\u001b[0m     \u001b[0muniques\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muniques\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m         raise ValueError(\"Found input variables with inconsistent numbers of\"\n\u001b[0m\u001b[1;32m    320\u001b[0m                          \" samples: %r\" % [int(l) for l in lengths])\n\u001b[1;32m    321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Found input variables with inconsistent numbers of samples: [2013, 2517]"
     ]
    }
   ],
   "source": [
    "pipe = Pipeline([('TfidfVect', TfidfVectorizer(stop_words='english',\n",
    "                                              ngram_range=(2, 3), max_features = 5000)),\n",
    "                   ('clf', LogisticRegression(random_state=0, \n",
    "                                              fit_intercept=False))\n",
    "                    ])\n",
    "\n",
    "param_grid = {\n",
    "              'TfidfVect__min_df': [2, 5, 10],\n",
    "              'clf__C': np.logspace(-3,3,7),\n",
    "              'clf__penalty': ['l1', 'l2'],\n",
    "              'clf__solver': ['liblinear', 'newton-cg', 'lbfgs'],\n",
    "              'clf__max_iter': [1000, 5000, 10000]\n",
    "             } \n",
    "grid = GridSearchCV(pipe, param_grid, cv=5, scoring='f1', n_jobs = -1)\n",
    "\n",
    "grid_tfidf = grid.fit(X_train, y)\n",
    "\n",
    "\n",
    "%%time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse._csr.csr_matrix"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train logistic regression model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "# clf = LogisticRegression(penalty='l1', solver='liblinear', random_state=0, fit_intercept=False, max_iter = 10000).fit(vect_X_train, y_train)\n",
    "clf = LogisticRegression(random_state=0, fit_intercept=False, max_iter = 10000).fit(vect_X_train, y_train)\n",
    "\n",
    "# clf.predict(vect_X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y_pred_proba = clf.predict_proba(vect_X_train)\n",
    "train_y_predict_prob_class_1 = train_y_pred_proba[:,1]\n",
    "train_y_predict_class = [1 if train_y_predict_prob_class_1[i] > 0.96 else 0 for i in range(len(train_y_predict_prob_class_1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The accuracy score is: 94.28%'"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f'The accuracy score is: {round(accuracy_score(y_train, train_y_predict_class)*100, 2)}%'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 94.3%\n",
      "Precision: 96.8%\n",
      "Recall: 81.8%\n",
      "F1: 87.2%\n",
      "\n",
      "\n",
      "Classification Report\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97     77516\n",
      "           1       1.00      0.64      0.78     14423\n",
      "\n",
      "    accuracy                           0.94     91939\n",
      "   macro avg       0.97      0.82      0.87     91939\n",
      "weighted avg       0.95      0.94      0.94     91939\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/keqiaoli/miniconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEGCAYAAADyuIefAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhrklEQVR4nO3deZgV1Z3/8feHzRZl32QzECX6UyciOorR8cFlBB1/Yubn7kSSGIlRE2eyzGg2JiZmmcxMRmMkEiWiMVGSmEgSFAnG0ZhgwH2X1qiAAkIjCCjQ3d/fH3UaCuy+fa/07eX25/U89dyqb52qey48fDmnTlUdRQRmZpbp0tYVMDNrT5wUzcxynBTNzHKcFM3McpwUzcxyurV1BfIG9u8ao0Z2b+tqWAleeKJnW1fBSvAOG9kSm7Ur55h47B6xpqauqLIPP7F5XkRM2pXva23tKimOGtmdv8wb2dbVsBJMHDa2ratgJXgoFuzyOVbX1PHQvBFFle0+9MWBu/yFraxdJUUz6wiCuqhv60qUjZOimZUkgHoq96EPJ0UzK1k9bimamQEQBFvdfTYzywRQ5+6zmdl2vqZoZpYEUFfBb9dyUjSzklXuFUUnRTMrURC+pmhm1iACtlZuTnRSNLNSiTp26fHpds1J0cxKEkC9W4pmZtu5pWhmlmQ3bzspmpkBWVLcGpX7fmonRTMrSSDqKvil/ZX7y8ysbOpDRS2FSNpP0mO5Zb2kf5bUX9J8SUvSZ79UXpKukVQt6QlJ43LnmpLKL5E0JRc/VNKT6ZhrJDXb73dSNLOSNFxTLGYpeJ6I5yNibESMBQ4FNgG/Ai4HFkTEGGBB2gY4CRiTlqnAdABJ/YFpwBHA4cC0hkSaylyYO67ZqRGcFM2sRKIuuhS1lOB44MWIeAWYDMxK8VnAaWl9MnBzZBYCfSUNBSYC8yOiJiLWAvOBSWlf74hYGBEB3Jw7V5N8TdHMSpK9ebvohDdQ0uLc9oyImNFIubOBn6X1IRHxelpfAQxJ68OBpbljlqVYofiyRuIFOSmaWUkixJboWmzx1RFxWKECknoApwJXvPu7IiS16q3i7j6bWcnqUVFLkU4CHomIlWl7Zer6kj5XpfhyID/d54gUKxQf0Ui8ICdFMytJNtDSpailSOewvesMMAdoGEGeAtyZi5+fRqHHA+tSN3secKKkfmmA5URgXtq3XtL4NOp8fu5cTXL32cxKpFIHUZo+k7QH8PfAJ3PhbwOzJV0AvAKcmeJzgZOBarKR6o8BRESNpK8Di1K5KyOiJq1fDNwE7A7clZaCnBTNrCQlDrQUPlfERmDATrE1ZKPRO5cN4JImzjMTmNlIfDFwUCl1clI0s5LVNXNjdkfmpGhmJQnE1qjc1FG5v8zMyqJhoKVSOSmaWUkCuftsZpbXUgMt7ZGTopmVJIIWuyWnPXJSNLOSZAMtRT/m1+E4KZpZyTzQYmaWBM2/QLYjc1I0s5K5pWhmlmTzPjspmpklzU810JE5KZpZSbIpTj36bGYGZG/edvfZzCzHN2+bmSXZ+xR9TdHMLGm5N2+3R06KZlaS7JYctxTNzIDKf/a5ctvAZlY29XQpammOpL6SfiHpOUnPSjpSUn9J8yUtSZ/9UllJukZStaQnJI3LnWdKKr9E0pRc/FBJT6Zjrkmz+hXkpGhmJcleHaailiJcDdwdEfsDBwPPApcDCyJiDLAgbUM2P/SYtEwFpgNI6g9MA44ADgemNSTSVObC3HGTmquQk6KZlaw+VNRSiKQ+wDHAjQARsSUi3gQmA7NSsVnAaWl9MnBzZBYCfSUNBSYC8yOiJiLWAvOBSWlf74hYmGYCvDl3rib5mqKZlSR7S07R7amBkhbntmdExIy0Php4A/ixpIOBh4HLgCFpInuAFcCQtD4cWJo717IUKxRf1ki8ICdFMytJ9phf0UlxdUQc1sS+bsA44NMR8ZCkq9neVc6+KyIkxXuu7HvgpFikpdW78c2LRm3bXvFqDz7yhRU8u7gny16sAmDj+q7s0buO6b9/flu5Vcu6c+GE/fmnz63gjE+9AcCvbhjIXbcOIAJOOq+Gf7wwi69f25VvXjSKlct6MGTEFr50/cv06lvXej+yExo0bAtfuPpV+g6qhYC5PxnAr28cBMCpH3+DUz+6hvo6eGhBb278xrA2rm170WKP+S0DlkXEQ2n7F2RJcaWkoRHxeuoCr0r7lwMjc8ePSLHlwISd4vel+IhGyhdU1qQoaRLZhdSuwA0R8e1yfl85jdx387ZkV1cH5407kKNOenNbQgO4/mvD2KPXjkns+q8N52+Pe2vb9svPVXHXrQO45ncv0L1H8MVz9+GIE9YxfPQWZl87mEOOfouzPr2K278/mNuvHcwnvvw6Vj51tWLGlcOofrInu+9Rx7V3v8Aj9/ei36BaPjRxPZ864QNs3dKFPgO2tnVV25WWeKIlIlZIWippv4h4HjgeeCYtU4Bvp8870yFzgEsl3UY2qLIuJc55wDdzgysnAldERI2k9ZLGAw8B5wPfb65eZRtokdQV+AHZiNEBwDmSDijX97Wmxx7oxdD3bWbIiO3/UCLg/jl9Ofa0tdtif7qrD3uN3ML7PvDOttirS3Zj/0M2UdUz6NoNPnjkBh6c2xeAP8/rwwln1gBwwpk1/PnuPq3zgzqxmlXdqX6yJwBvb+zK0uoqBg7dyinnr+b2awezdUv2T2Tdmu5tWc12pYVHnz8N3CrpCWAs8E2yZPj3kpYAJ6RtgLnAS0A18CPg4qw+UQN8HViUlitTjFTmhnTMi8BdzVWonC3Fw4HqiHgJIGX3yWT/C3Ro993ZlwmnvblD7KmH9qDfoFqGv38LAG9v7MLs6wbzrdte5BfTB28rN2r/d7jpO0NZX9OVHlX1LLq3N2M+uAmAtau7M2BILQD9B9eydrX/IbamISO2sM9Bb/PcIz35xFde46AjNvLRf1vBls3iR1cO44XHe7Z1FduNlnpLTkQ8BjR2zfH4RsoGcEkT55kJzGwkvhg4qJQ6lTMpNjYidMTOhSRNJbvniL2Ht/9LnFu3iIX39OHjX9yxW/uHX/djQq6VeMt/7sWHL3yD3feo36Hc3mM2c+bFq7jinH2o6lnP+w98my6NPBwgQStfX+7UqnrW8ZUbXuaHXx3Gpg1d6doVevWt5bJT9mW/sW/zpetfYcr4/aGCX4RQLM/RUmZpeH4GwGEHV7X7LLDo3l7s+zeb6DeodlusrhYenNuHa+9+YVvsuUd78sff9eXGbwxjw/quqEvQY7dg8sdXM+ncGiadm7XuZ35rKIOGZq3LfgO3smZlNwYMqWXNym70HVCLlV/XbsFXbniZe+/ox4N39QVg9evd02UN8fxjPamvhz7961hX0+b/ZNpcALV+IcR70tRIUYd236/7vavr/MgDvRi572YGDdt+jfG/f129bf2W/9yLqj3qmPzx1QC8ubobfQfWsmpZdx6c24erf7sEgPEnruf3s/tz1qdX8fvZ/Tly4rry/6BOL/jsfy1l6ZIq7pgxaFv0T3f35uCjNvD4n/Zk+Ps3071HsK6mcp/3LZVfMvveLALGSBpNlgzPBs4t4/eV3TubuvDIA7247D+W7hD/3zt37Do358pPjOKttd3o2j249JvL2LNPNmJ91qUrueqiUdx92wAGD89uybHyOvDwjZxwxlpeeqaK6+Zndxf8+FtDmXdbfz7730u5/t7n2bpVfPeykbjrnBTxtEpHpuzaZZlOLp0M/A/ZLTkzI+KqQuUPO7gq/jJvZKEi1s5MHDa2ratgJXgoFrA+anYpo/Xbf3AcN/P0osrecdT0hwvcvN0ulfUCSUTMJRtGN7MKUsktRV81NrOS+CWzZmY5gait90CLmdk2nrjKzKxBuPtsZraNrymame3ESdHMLAlEnQdazMy280CLmVkSHmgxM9tROCmamTWo7BdCOCmaWcncUjQzSyKgrr5yk2LljqubWdnUo6KW5kh6WdKTkh6TtDjF+kuaL2lJ+uyX4pJ0jaRqSU9IGpc7z5RUfomkKbn4oen81enYZivlpGhmJQmy7nMxS5GOjYixufcuXg4siIgxwIK0DdnMoGPSMhWYDlkSBaaRzQF1ODAtN93pdODC3HGTmquMk6KZlSgbaClmeY8mA7PS+izgtFz85sgsBPpKGgpMBOZHRE1ErAXmA5PSvt4RsTDNBHhz7lxNclI0s5JFFLcUcyrgHkkPp5k9AYZERMN0mSuAIWm9sRlChzcTX9ZIvCAPtJhZyUroGg9suFaYzEgzeDY4OiKWSxoMzJf03I7fE6FWnuvXSdHMSpKNPhfdyVxdaI6WiFiePldJ+hXZNcGVkoZGxOupC7wqFW9qhtDlwISd4vel+IhGyhfk7rOZlawlus+S9pDUq2EdOBF4CpgDNIwgTwHuTOtzgPPTKPR4YF3qZs8DTpTULw2wnAjMS/vWSxqfRp3Pz52rSW4pmlnJWujm7SHAr9JdMt2An0bE3ZIWAbMlXQC8ApyZys8FTgaqgU3Ax7K6RI2kr5NNqwxwZUTUpPWLgZuA3YG70lKQk6KZlSQo6Xabps8T8RJwcCPxNcDxjcQDuKSJc80EZjYSXwwcVEq9nBTNrGStOvLRypwUzaw0AVHBj/k5KZpZyfxCCDOznCJvzO6QmkyKkr5PgUsHEfGZstTIzNq1hmefK1WhluLiAvvMrLMKoDMmxYiYld+W1DMiNpW/SmbW3lVy97nZJ1okHSnpGeC5tH2wpOvKXjMza6dE1Be3dETFPOb3P2Sv5lkDEBGPA8eUsU5m1t5FkUsHVNToc0Qs3emFtXXlqY6ZtXvReQdaGiyV9CEgJHUHLgOeLW+1zKxd66CtwGIU032+iOx5w+HAa8BYmnj+0Mw6CxW5dDzNthQjYjVwXivUxcw6ivq2rkD5FDP6/H5Jv5H0hqRVku6U9P7WqJyZtUMN9ykWs3RAxXSffwrMBoYCw4CfAz8rZ6XMrH1rwTla2p1ikmLPiLglImrT8hOgqtwVM7N2rDPekpPmUgW4S9LlwG1kP/Mssjfgmlln1UG7xsUoNNDyMFkSbPj1n8ztC+CKclXKzNq31p1fr3UVevZ5dGtWxMw6iBB00Ef4ilHUbH6SDpJ0pqTzG5ZyV8zM2rEWvKYoqaukRyX9Nm2PlvSQpGpJt0vqkeK7pe3qtH9U7hxXpPjzkibm4pNSrDpdBmxWMbfkTAO+n5Zjgf8ATi3u55pZRWrZgZadn5L7DvC9iNgXWAtckOIXAGtT/HupHJIOAM4GDgQmAdelRNsV+AFwEnAAcE4qW1AxLcXTyWbWWhERHyObfatPEceZWaVqoaQoaQTwD8ANaVvAccAvUpFZwGlpfXLaJu0/PpWfDNwWEZsj4q9kU6AenpbqiHgpIraQDRZPbq5OxSTFtyOiHqiV1BtYBYws4jgzq0Qte/P2/wD/yvZnZAYAb0ZEbdpeRvaIMelzKUDavy6V3xbf6Zim4gUVkxQXS+oL/IhsRPoR4M9FHGdmFUpR3AIMlLQ4t0zddg7pFGBVRDzcVr+jMcU8+3xxWv2hpLuB3hHxRHmrZWbtWvHXC1dHxGFN7DsKOFXSyWQPhPQGrgb6SuqWWoMjgOWp/HKyXuoySd3ILuOtycUb5I9pKt6kJluKksbtvAD9gW5p3cw6qRJaik2KiCsiYkREjCIbKLk3Is4D/kA2lgEwBbgzrc9J26T990ZEpPjZaXR6NDAG+AuwCBiTRrN7pO+Y09xvK9RS/K9Cv4fsYmiLeuGJnkwcNralT2tlVH/02LaugpXi0T+1zHnK+0TLvwG3SfoG8ChwY4rfCNwiqRqoIUtyRMTTkmYDzwC1wCURUQcg6VJgHtAVmBkRTzf35YVu3j72Pf8kM6tcZXiuOSLuA+5L6y+RjRzvXOYd4Iwmjr8KuKqR+FxKfCy5qOkIzMx20Bkf8zMza4oq+CWzTopmVroKbikW85ifJP2TpK+m7b0lvau/b2adQ7Ejzx31TTrF3Lx9HXAkcE7afovseUIz66wqeDqCYrrPR0TEOEmPAkTE2oa3VphZJ9VBW4HFKCYpbk1vmwgASYOo6Lm8zKw5HbVrXIxikuI1wK+AwZKuIruT/MtlrZWZtV/RyUefI+JWSQ+TvT5MwGkR8Wwzh5lZJevMLUVJewObgN/kYxHxajkrZmbtWGdOisDv2D6BVRUwGnie7C23ZtYJdeprihHxN/nt9Iaci5sobmbWoZX8REtEPCLpiHJUxsw6iM7cUpT02dxmF2Ac8FrZamRm7VtnH30GeuXWa8muMf6yPNUxsw6hs7YU003bvSLi861UHzNr50QnHWhpmCNB0lGtWSEz6wA6Y1Ikm+NgHPCYpDnAz4GNDTsj4o4y183M2qMO/AacYhRzTbGKbMas49h+v2IATopmnVUnHWgZnEaen2J7MmxQwf9PmFlzKrmlWOh9il2BPdPSK7fesJhZZxVFLgVIqpL0F0mPS3pa0tdSfLSkhyRVS7q94VWFaQrT21P8IUmjcue6IsWflzQxF5+UYtWSLi/mpxVqKb4eEVcWcxIz60Rabja/zcBxEbFBUnfgj5LuAj4LfC8ibpP0Q+ACYHr6XBsR+0o6G/gOcJakA8imOz0QGAb8XtIH0nf8APh7YBmwSNKciHimUKUKtRQ75mtzzazsWmI6gshsSJvd09Iwp/wvUnwWcFpan5y2SfuPl6QUvy0iNkfEX4FqsilSDweqI+KliNgC3JbKFlQoKR7f3MFm1kkV330eKGlxbpmaP42krpIeA1YB84EXgTcjojYVWQYMT+vDgaUAaf86YEA+vtMxTcULarL7HBE1zR1sZp1TCY/5rY6Iw5raGRF1wFhJfcleZr3/LlduFxUzcZWZ2XbFthJLuO4YEW8CfyCbJK+vpIYG2whgeVpfDoyE7OESoA/Z7YLb4jsd01S8ICdFMyuJSlgKnkcalFqISNqdbEDkWbLkeHoqNgW4M63PSduk/fdGRKT42Wl0ejQwhuzhk0XAmDSa3YNsMGZOc7+v5FeHmZm10OjzUGBWesdCF2B2RPxW0jPAbZK+ATwK3JjK3wjcIqkaqCFLckTE05JmA8+QvbTmktQtR9KlwDyyWwxnRsTTzVXKSdHMStYSN29HxBPAIY3EXyIbOd45/g5wRhPnugq4qpH4XGBuKfVyUjSz0lXwEy1OimZWGr9k1sxsJ24pmpltV8kvhHBSNLPSOSmamW3nlqKZWYOg075k1szsXTrtxFVmZk1yUjQz205RuVnRSdHMStNyb95ul5wUzaxkvqZoZpbjx/zMzPLcUjQzS4qYlKojc1I0s9I5KZqZZXzztpnZTlRfuVnRSdHMSuP7FK2QQcO28IWrX6XvoFoImPuTAfz6xkH83Slv8pHPrWDkmM185uQxLHmi57Zjzrp0JZPOqaGuXkz/8jAe/t/ebfgLKtfnPvUgR4xbxpvrqpj6+ck77Dv9lKf55PmL+X8XnMX6t6o47uiXOGvyU0jBpre7c80N43nplf4FzzPlrEf50GFLiYA311Xx3euOZs3annQGLXFLjqSRwM3AELI0OyMirpbUH7gdGAW8DJwZEWslCbgaOBnYBHw0Ih5J55oCfDmd+hsRMSvFDwVuAnYnm6vlsjQDYJPKNsWppJmSVkl6qlzf0R7U1YoZVw5j6oT9ueyUMfzfj65m7zHv8PJzVVz5iVE8uXCPHcrvPeYdJkx+k6nH7seXzh3Npd9aTpcuFfzfbhu65759+OI3T3hXfNCAjRz6wddY+cb2v5sVq/bkc/8+kamfn8ytv/wg/zz1z82e5+dzDuSTXziVi/71VBY+MpJ/Ov3x8vyQ9qhl5n2uBT4XEQcA44FLJB0AXA4siIgxwIK0DXAS2fSlY4CpwHSAlESnAUeQTXg1TVK/dMx04MLccZOaq1Q5532+qZgKdHQ1q7pT/WTWOnh7Y1eWVlcxcOhWllZXsezFqneVP3LiOu67sy9bt3Rh5dLdeO3lHux3yKbWrnan8OSze/HWht3eFb9oyiJ+dOuh5NsLz7wwmA0bs7LPLhnEoAEbmz3Pprd7bFuv2q2WCn4c+F0UxS2FRMTrDS29iHiLbM7n4cBkYFYqNgs4La1PBm6OzEKgr6ShwERgfkTURMRaYD4wKe3rHRELU+vw5ty5mlS27nNE3C9pVLnO3x4NGbGFfQ56m+ceaboLNXDoVp59eHsLZfXrPRiw19bWqJ4BRx72Kmtqem7rGjdm0nFLWPToiKLO97GzH+GEY15k46YefOFrE1uqmu1bQAn/AwyUtDi3PSMiZuxcKOWKQ4CHgCER8XratYKsew1ZwlyaO2xZihWKL2skXlA5W4pFkTRV0mJJi7eyua2r855V9azjKze8zA+/OoxNG7q2dXWsEbv1qOWcDz/JTbePbbLMwQe+zknHVvOjW8cVdc4f3zaO8y4+g3v/+H4mT3quhWra/qm+uAVYHRGH5ZbGEuKewC+Bf46I9fl9qYXXqm3wNk+KETGj4Q+sO+/uonQEXbsFX7nhZe69ox8P3tW3YNnVr3dn0LAt27YHDt3CmhXdy1xDAxg65C32GryB6787h1uu/QWDBmxi+nd+S78+bwMweu8aPvvJP/HV7x7LWxvefemjkAUPjOboI14pR7XbnYb7FHe1+wwgqTtZQrw1Iu5I4ZWp60v6XJXiy4GRucNHpFih+IhG4gW1eVLs+ILP/tdSli6p4o4Zg5otvfCePkyY/Cbde9QzZORmho/ewvOPdo4Ry7b28tJ+nHnhWXzk0tP5yKWn88aannzq305h7brdGTRgA9M+fx/fufbvWP56n6LON3yv7Y2aD/3tUpa+VtxxHV5E8UsBaTT5RuDZiPjv3K45wJS0PgW4Mxc/X5nxwLrUzZ4HnCipXxpgORGYl/atlzQ+fdf5uXM1ybfk7KIDD9/ICWes5aVnqrhu/vMA/PhbQ+neI7j4G8vpM6CWr9/yV158uoovnbsPr7xQxf2/6cuM+56nrk5c+8Xh1NerjX9FZfriZf/LBw9YSZ9e7/DT6T/n5tljufsPYxot+5HTn6D3npv5zCcWAlBX14VLrjil4HkuOO9hRgxdT4RYuXoPrp4xvtV+W1troSdajgI+Ajwp6bEU+yLwbWC2pAuAV4Az0765ZLfjVJPdkvMxgIiokfR1YFEqd2VE1KT1i9l+S85daSlIzdyy855J+hkwARgIrASmRcSNhY7prf5xhI4vS32sPOqPHtvWVbASLHr0Ota/tXyX/hfu1XdEHHLMZUWVfeA3//pwRBy2K9/X2so5+nxOuc5tZm3Lzz6bmTUIoK5ys6KTopmVzC1FM7O8Cn58x0nRzErmlqKZWQO/OszMbDsB8kCLmdl28jVFM7PE3Wczs7zmn2vuyJwUzaxkHn02M8tzS9HMLAmPPpuZ7ahyc6KTopmVzrfkmJnlOSmamSUB1Ld1JcrHSdHMSiLC3Wczsx3UV25T0bP5mVlpGrrPxSzNkDRT0ipJT+Vi/SXNl7QkffZLcUm6RlK1pCckjcsdMyWVXyJpSi5+qKQn0zHXpFn9CnJSNLOSKaKopQg3AZN2il0OLIiIMcCCtA1wEjAmLVOB6ZAlUWAacARwODCtIZGmMhfmjtv5u97FSdHMStcC8z5np4n7gZqdwpOBWWl9FnBaLn5zZBYCfSUNBSYC8yOiJiLWAvOBSWlf74hYGNm0pTfnztUkX1M0sxKV9EKIgZIW57ZnRMSMZo4ZkiayB1gBDEnrw4GluXLLUqxQfFkj8YKcFM2sNKXN5rd6V+Z9joiQWvf1E+4+m1nJWvCaYmNWpq4v6XNVii8HRubKjUixQvERjcQLclI0s9K10DXFJswBGkaQpwB35uLnp1Ho8cC61M2eB5woqV8aYDkRmJf2rZc0Po06n587V5PcfTaz0gRQ3zI9Wkk/AyaQXXtcRjaK/G1gtqQLgFeAM1PxucDJQDWwCfgYQETUSPo6sCiVuzIiGgZvLiYb4d4duCstBTkpmlmJWu7N2xFxThO7jm+kbACXNHGemcDMRuKLgYNKqZOTopmVzo/5mZklAdRV7mN+TopmVqKAcFI0M9vO3Wczs6QFR5/bIydFMyudW4pmZjlOimZmSQTU1bV1LcrGSdHMSueWoplZjpOimVmD8Oizmdk2AeGbt83McvyYn5lZElHRU5w6KZpZ6TzQYma2XbilaGbWoOVeMtseOSmaWWn8Qggzs+0CCD/mZ2aWhF8ya2a2g3D32cwsp4Jbiop2NIok6Q2yeV4rzUBgdVtXwkpSqX9n74uIQbtyAkl3k/35FGN1REzale9rbe0qKVYqSYsj4rC2rocVz39nnVeXtq6AmVl74qRoZpbjpNg6ZrR1Baxk/jvrpHxN0cwsxy1FM7McJ0UzsxwnxTKSNEnS85KqJV3e1vWx5kmaKWmVpKfaui7WNpwUy0RSV+AHwEnAAcA5kg5o21pZEW4COtTNxtaynBTL53CgOiJeiogtwG3A5DaukzUjIu4Hatq6HtZ2nBTLZziwNLe9LMXMrB1zUjQzy3FSLJ/lwMjc9ogUM7N2zEmxfBYBYySNltQDOBuY08Z1MrNmOCmWSUTUApcC84BngdkR8XTb1sqaI+lnwJ+B/SQtk3RBW9fJWpcf8zMzy3FL0cwsx0nRzCzHSdHMLMdJ0cwsx0nRzCzHSbEDkVQn6TFJT0n6uaSeu3CumySdntZvKPSyCkkTJH3oPXzHy5LeNetbU/Gdymwo8bv+XdLnS62j2c6cFDuWtyNibEQcBGwBLsrvlPSe5vGOiE9ExDMFikwASk6KZh2Rk2LH9QCwb2rFPSBpDvCMpK6SvitpkaQnJH0SQJlr0/sdfw8MbjiRpPskHZbWJ0l6RNLjkhZIGkWWfP8ltVL/TtIgSb9M37FI0lHp2AGS7pH0tKQbADX3IyT9WtLD6ZipO+37XoovkDQoxfaRdHc65gFJ+7fIn6ZZ8p5aFta2UovwJODuFBoHHBQRf02JZV1E/K2k3YAHJd0DHALsR/ZuxyHAM8DMnc47CPgRcEw6V/+IqJH0Q2BDRPxnKvdT4HsR8UdJe5M9tfN/gGnAHyPiSkn/ABTzNMjH03fsDiyS9MuIWAPsASyOiH+R9NV07kvJJpS6KCKWSDoCuA447j38MZo1ykmxY9ld0mNp/QHgRrJu7V8i4q8pfiLwwYbrhUAfYAxwDPCziKgDXpN0byPnHw/c33CuiGjqvYInAAdI2xqCvSXtmb7jH9Oxv5O0tojf9BlJH07rI1Nd1wD1wO0p/hPgjvQdHwJ+nvvu3Yr4DrOiOSl2LG9HxNh8ICWHjfkQ8OmImLdTuZNbsB5dgPER8U4jdSmapAlkCfbIiNgk6T6gqonikb73zZ3/DMxakq8pVp55wKckdQeQ9AFJewD3A2ela45DgWMbOXYhcIyk0enY/in+FtArV+4e4NMNG5LGptX7gXNT7CSgXzN17QOsTQlxf7KWaoMuQENr91yybvl64K+SzkjfIUkHN/MdZiVxUqw8N5BdL3wkTb50PVmP4FfAkrTvZrI3wewgIt4AppJ1VR9ne/f1N8CHGwZagM8Ah6WBnGfYPgr+NbKk+jRZN/rVZup6N9BN0rPAt8mScoONwOHpNxwHXJni5wEXpPo9jad4sBbmt+SYmeW4pWhmluOkaGaW46RoZpbjpGhmluOkaGaW46RoZpbjpGhmlvP/AVcUd0FJZ5n9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "imp.reload(f)\n",
    "f.scoring(y_train, train_y_predict_class, clf, vect_X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run model on Validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_pred_proba = clf.predict_proba(vect_X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_y_predict_prob_class_1 = val_pred_proba[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_y_predict_class = [1 if val_y_predict_prob_class_1[i] > 0.96 else 0 for i in range(len(val_y_predict_prob_class_1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22985"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_predict_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 92.4%\n",
      "Precision: 94.4%\n",
      "Recall: 76.5%\n",
      "F1: 82.2%\n",
      "\n",
      "\n",
      "Classification Report\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96     19393\n",
      "           1       0.97      0.53      0.69      3592\n",
      "\n",
      "    accuracy                           0.92     22985\n",
      "   macro avg       0.94      0.77      0.82     22985\n",
      "weighted avg       0.93      0.92      0.91     22985\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/keqiaoli/miniconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEGCAYAAADyuIefAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAibUlEQVR4nO3deZgV1Z3/8fenmx1ZBRFZBBU1riiMS4yORqPoOFETJ2Iy0Ulco7jM6Biz/GLGxIkxUSNJNHHhUTOJS1wiGgQNUTHGBVAjKiIgIiDIvmNLd39/f1RdKLCXe5u+3F4+r+eph7rf2s7ttr+eU6fqHEUEZmaWKCt1AczMmhInRTOzDCdFM7MMJ0UzswwnRTOzjDalLkBWr57lMWhA21IXwwrw7hudSl0EK8DHrOOTqNC2nOOEYzrHsuVVee079Y2KCRExYluut701qaQ4aEBbXpkwoNTFsAKcsMvQUhfBCvByTNzmcyxdXsXLE/rntW/bvrN7bfMFt7MmlRTNrDkIqqK61IUoGidFMytIANW03Jc+nBTNrGDVuKZoZgZAEGx089nMLBFAlZvPZmab+Z6imVkqgKoWPLqWk6KZFazl3lF0UjSzAgXhe4pmZjkRsLHl5kQnRTMrlKhim16fbtKcFM2sIAFUu6ZoZrZZS64pejxFMytI8vC28lrqI2mMpMWS3szEHpD0erq8L+n1ND5I0obMtt9kjhkmaZqkWZJGS1Ia7ynpaUkz03971FcmJ0UzK0gAG6MsryUPdwNbjLcYEWdExNCIGAo8DDyS2Tw7ty0iLszEbwPOA4akS+6cVwMTI2IIMDH9XCcnRTMrSCCqKMtrqfdcEZOA5TVtS2t7XwHuq+sckvoCXSPipUjmbL4XODXdfApwT7p+TyZeKydFMytYdSivBeglaUpmOb+AyxwJfBQRMzOxwZJek/ScpCPTWD9gfmaf+WkMoE9ELEzXFwF96ruoO1rMrCC5e4p5WhoRwxt4qTPZspa4EBgYEcskDQP+JGnffE8WESGp3n5zJ0UzK5Coyu9+YcOvILUBvgQMy8UiogKoSNenSpoN7AksALLzI/RPYwAfSeobEQvTZvbi+q7t5rOZFSQZebssr2UbHAe8ExGbmsWSeksqT9d3I+lQeS9tHq+WdFh6H/Is4LH0sLHA2en62Zl4rVxTNLOCRIhPorxRziXpPuBoknuP84FrIuIuYCSf7mA5CrhW0kaSMSkujIhcJ81FJD3ZHYEn0wXgeuBBSecAc0k6burkpGhmBatupIe3I+LMWuL/UUPsYZJHdGrafwqwXw3xZcCxhZTJSdHMCpJ0tLTcO29OimZWoOJ3tJSSk6KZFSTX0dJSOSmaWcGqouUOCOGkaGYFCcTGaLmpo+V+MzMrCne0mJllBHLz2cwsyx0tZmapCPxIjplZTtLR0jiv+TVFTopmVjB3tJiZpYJNA8i2SE6KZlYw1xTNzFLJvM9OimZmqfymL22unBTNrCDJFKfufTYzA5KRt918NjPL8MPbZmapZDxF31M0M0u17JG3W+43M7OiSB7JUV5LfSSNkbRY0puZ2A8lLZD0erqclNn2HUmzJM2QdEImPiKNzZJ0dSY+WNLLafwBSe3qK5OTopkVJPfucz5LHu4GRtQQvzkihqbLOABJ+5BMfbpvesytksrTuaB/DZwI7AOcme4L8NP0XHsAK4Bz6iuQk6KZFaymie9rWuoTEZOA5fXumDgFuD8iKiJiDjALOCRdZkXEexHxCXA/cIokAZ8HHkqPvwc4tb6LOCmaWUGSocOU10Iyyf2UzHJ+npcZJemNtHndI431A+Zl9pmfxmqL7wisjIjKreJ1ckeLmRWsgAEhlkbE8AJPfxvwI5Lblz8CbgS+WeA5GsxJ0cwKkoySU7xGZkR8lFuXdAfwRPpxATAgs2v/NEYt8WVAd0lt0tpidv9auflsZgVJXvMry2tpCEl9Mx9PA3I902OBkZLaSxoMDAFeASYDQ9Ke5nYknTFjIyKAZ4DT0+PPBh6r7/quKebpxv8cwMt/6Ur3XpXc/swMAGa/1YFfXj2ADevK6NP/E77967l07lINwP2/3Inx9+1IeVnwrR8vYPjRa1i8oC0/u2wgK5e0BQUn/fsyTjt3KQDXXbAr82d3AGDd6nI6d63itr/MKM2XbeHatq/mxkdm0bZdUN4meP7P3fndz3cGgv/49iKOPHkl1dXiiXt35LG7egNwwOFrufDaBbRpE6xa3ob//vIepf0SJdV4NUVJ9wFHk9x7nA9cAxwtaShJ/n0fuAAgIt6S9CDwNlAJXBwRVel5RgETgHJgTES8lV7i28D9kn4MvAbcVV+ZipoUJY0AbiEp6J0RcX0xr1dMx5+xnC9+Yyk/u2zgptgvrhzIeT9YwAGHr2PCfT156LadOPuqRcx9tz3PPtaD2595h+UfteXqM3bnrr9Np7xNcP4PPmTIARtYv7aMUSP25OCj1rDrnhV877dzN533t/+zC527VJXia7YKGyvEVf+2Ox+vL6e8TXDTn2Yx+a9dGDikgt67bOTco/YmQnTbcSMAnbtWMeon8/ne13ZjyYJ2m+KtWWO90RIRZ9YQrjVxRcR1wHU1xMcB42qIv0fSO523ojWf63l2qNnZ/7B1dOmxZaKa/1579j9sHQAHHbWGv/25OwAvTujG0aesoF37YOeBn7DLoApmvNaJHftUMuSADQB02qGaAXtUsHRh2y3OGQGTxnbnmFNXFP9LtVri4/XJM3Rt2gblbYMIOPmspfz+5j5E2omwalnyuznmtBW8MK4bSxa02yLeWhXY+9zsFPOeYo3PDhXxetvdrnt+zIvjuwHw/BPdWfJh8seydGFbeu+yuTbRq+9Gli3a8g9p0bx2zH6zI3sfvH6L+Jsvd6ZH70r67fZJkUvfupWVBbc+PYMH3niL1ybtwIzXOtN310/45y+u5JdPvsuP/+89dhlcAUD/3SrYoXsVNzw0i1+Nf5fjTs/3sbqWqzrK8lqao2KWurZnh7Yg6fzcM0xLljWvJuN/3fQBj9+zIxefsCcb1pbRpl3kddyGdWX86NxBXHjtgk33IHOe+VMPjnYtseiqq8VFX9iLrw3bh72GrmfXvTbQtn3wSYW45MQ9efL3PbnipuQ/3/I2wZD9N/D/vj6Y7351N756+Uf0262ixN+gdHJztDTGa35NUck7WiLiduB2gOEHdsgvqzQRA4dU8JP73wNg/uz2vDyxK5DUDHO1RkhqjjvunNQcKzfCj84dxOe/tILPnbRqi/NVVcIL47rxq/HvbqdvYOtWl/OPv+/APx2zhqUL2/K3cUnN/4Unu3HFzUlSXLKwLatXtKFiQzkVG2Dayzuw2z4bWPBe+1IWvWQCqGymtcB8FPOb1fVMUYuwcmny/5TqavjDLX04+evLADjs+NU8+1gPPqkQiz5ox4I57dnroPVEwE1XDGTAkAq+fMGST53v1ee7MGCPii2a3tb4uvWspHPXpFXSrkM1Bx+1lnmzOvD38V058Ii1ABxw+Drmp0nvxfHd2Pef1lFWHrTvWM3eB63ng5mtMyHmtOTmczFripueHSJJhiOBrxbxekX1k2/tyhsv7sCq5W342rB9+PoVi9iwvozH7+4FwBEnruL4kcm9pkF7fcxR/7qS84/em/LyYNT/zqe8PLlfOPGhngz+zAa+ddxeAHzjOx9yyLFrAHjuMTedt4eefTZy5S0fUFYGZWUw6fFuvPyXrrz5Sme+/au5fOm8pWxYV8Yvrkz+nz5vVgemPNuF30ycQVSL8X/oydwZHUv8LUqoGTeN86Hk+cYinTwZ8ucXbH526FNd6VnDD+wQr0wYUNcu1sScsMvQUhfBCvByTGR1LN+mjNZj753i82NOr39H4JEjbpvagNf8Sqqo9xRre3bIzJq3llxTLHlHi5k1L7lBZlsqJ0UzK0ggKqubZydKPpwUzaxgnrjKzCwn3Hw2M9vE9xTNzLbipGhmlgpElTtazMw2c0eLmVkq3NFiZralcFI0M8tp2QNCOCmaWcFack2x5XYhmVlRREBVtfJa6iNpjKTFkt7MxH4m6R1Jb0h6VFL3ND5I0gZJr6fLbzLHDJM0TdIsSaMlKY33lPS0pJnpvz3qK5OTopkVrBrlteThbmDEVrGngf0i4gDgXeA7mW2zI2JoulyYid8GnEcyF/SQzDmvBiZGxBBgYvq5Tk6KZlaQIGk+57PUe66IScDyrWJPRURl+vElklH7ayWpL9A1Il6KZIDYe4FT082nAPek6/dk4rVyUjSzAhU0cVWv3MR06XJ+gRf7JvBk5vNgSa9Jek7SkWmsH8nEeDnZSfL6RMTCdH0R0Ke+C7qjxcwKVsCA/UsbOvK2pO8BlcDv09BCYGBELJM0DPiTpH3zPV9EhKR6S+6kaGYFK3bvs6T/AE4Gjk2bxEREBVCRrk+VNBvYk2QOqGwTOztJ3keS+kbEwrSZvbi+a7v5bGYFSXqfy/JaGkLSCOAq4IsRsT4T7y2pPF3fjaRD5b20ebxa0mFpr/NZwGPpYWOBs9P1szPxWrmmaGYFa6z57iTdBxxNcu9xPnANSW9ze+Dp9Mmal9Ke5qOAayVtBKqBCyMi10lzEUlPdkeSe5C5+5DXAw9KOgeYC3ylvjI5KZpZwRqr+RwRZ9YQvquWfR8GHq5l2xRgvxriy4BjCymTk6KZFSTI73Gb5spJ0cwKVrzZ4kvPSdHMChMQebzC11w5KZpZwdx8NjPLaKze56ao1qQo6ZfUcesgIi4tSonMrEnLvfvcUtVVU5yy3UphZs1HAK0xKUbEPdnPkjplny43s9arJTef630PR9Lhkt4G3kk/Hyjp1qKXzMyaKBHV+S3NUT4vJ/4COAFYBhAR/yB53cbMWqvIc2mG8up9joh56TuIOVXFKY6ZNXnRejtacuZJ+iwQktoClwHTi1ssM2vSmmktMB/5NJ8vBC4mGcn2Q2Bo+tnMWi3luTQ/9dYUI2Ip8LXtUBYzay6qS12A4smn93k3SY9LWpJORfhYOsCjmbVGuecU81maoXyaz38AHgT6ArsAfwTuK2ahzKxpi8hvaY7ySYqdIuJ3EVGZLv8HdCh2wcysCWuNj+RI6pmuPinpauB+kq95BjBuO5TNzJqqZto0zkddHS1TSZJg7ttfkNkWJPMomFkrVP9Eoc1XXe8+D96eBTGzZiIEzfQVvnzkNQehpP0kfUXSWbml2AUzsyaske4pShqTPtXyZibWU9LTkmam//ZI45I0WtIsSW9IOjhzzNnp/jMlnZ2JD5M0LT1mtLZ6Na8m+TyScw3wy3Q5BrgB+GL9X9fMWqzG62i5GxixVexqYGJEDAEmpp8BTiSZ63kIcD5wG2zq/7gGOBQ4BLgml0jTfc7LHLf1tT4ln5ri6SRTBC6KiG8ABwLd8jjOzFqqRkqKETEJWL5V+BQgN3ThPcCpmfi9kXgJ6C6pL8mANU9HxPKIWAE8DYxIt3WNiJciIoB7M+eqVT7vPm+IiGpJlZK6AouBAXkcZ2YtUfEHme0TEQvT9UVAn3S9HzAvs9/8NFZXfH4N8TrlkxSnSOoO3EHSI70WeDGP48yshSqg97mXpOwo/rdHxO35HhwRIW3fvu583n2+KF39jaTxJNXRN4pbLDNr0vJPU0sjYniBZ/9IUt+IWJg2gRen8QVs2Urtn8YWAEdvFX82jfevYf861XpPUdLBWy9AT6BNttfHzFofRX5LA40Fcj3IZwOPZeJnpb3QhwGr0mb2BOB4ST3SDpbjgQnpttWSDkt7nc/KnKtWddUUb6xjWwCfr+/khXp3WidGDCz0fypWSmUHDil1EawAmvFC45yoke4pSrqPpJbXS9J8kl7k64EHJZ0DzAW+ku4+DjgJmAWsB74BEBHLJf0ImJzud21E5DpvLiLp4e4IPJkudarr4e1jCvhuZtZaNOJ7zRFxZi2bjq1h36CWsVwjYgwwpob4FGC/QsqU13QEZmZbaI2v+ZmZ1UYteJBZJ0UzK1wLrinm85qfJP27pB+knwdKOqT4RTOzpijfnufmOpJOPq/53QocDuRuiK4Bfl20EplZ09eCpyPIp/l8aEQcLOk1gIhYIaldkctlZk1ZM60F5iOfpLhRUjnpj0FSb1r0XF5mVp/m2jTORz5JcTTwKLCTpOtIRs35flFLZWZNV7Ty3ueI+L2kqSQPUwo4NSKmF71kZtZ0teaaoqSBJK/UPJ6NRcQHxSyYmTVhrTkpAn9m8wRWHYDBwAxg3yKWy8yasFZ9TzEi9s9+TkfIuaiW3c3MmrWC32iJiFclHVqMwphZM9Gaa4qS/ivzsQw4GPiwaCUys6attfc+A10y65Uk9xgfLk5xzKxZaK01xfSh7S4RceV2Ko+ZNXGilXa0SGoTEZWSjtieBTKzZqA1JkXgFZL7h69LGgv8EViX2xgRjxS5bGbWFDXjEXDykc89xQ7AMpI5WXLPKwbgpGjWWrXSjpad0p7nN9mcDHNa8P8nzKw+rbWmWA7swJbJMKcF/0jMrF4tOAPUlRQXRsS1260kZtY8NNJsfpL2Ah7IhHYDfgB0B84DlqTx70bEuPSY7wDnAFXApRExIY2PAG4hqczdGRHXN7RcdSXF5jlsrpkVXWM0nyNiBjAUNj3+t4BkmMJvADdHxM+3uKa0DzCSZNyFXYC/SNoz3fxr4AvAfGCypLER8XZDylVXUvzUvKtmZkAxms/HArMjYq5Ua33sFOD+iKgA5kiaBeTmi5oVEe8BSLo/3bdBSbHWOVoiYnlDTmhmLZ+q81uAXpKmZJbzaznlSOC+zOdRkt6QNEZSjzTWD5iX2Wd+Gqst3iD5TFxlZrZZFLDA0ogYnllu3/p06ZxPXyR5FhrgNmB3kqb1QuDGIn6bT/G8z2ZWENHoHQ4nAq9GxEcAuX8BJN0BPJF+XAAMyBzXP41RR7xgrimaWeHyrynm40wyTWdJfTPbTiN5VhpgLDBSUntJg4EhJG/eTQaGSBqc1jpHpvs2iGuKZlawxnp4W1Jnkl7jCzLhGyQNJUmr7+e2RcRbkh4k6UCpBC6OiKr0PKOACSSP5IyJiLcaWiYnRTMrXCMlxYhYB+y4Vezrdex/HXBdDfFxwLjGKJOTopkVxoPMmpltpZW+5mdmVqPWOiCEmVnNnBTNzDZzTdHMLCdotYPMmpl9SquduMrMrFZOimZmmylablZ0UjSzwjTSyNtNlZOimRXM9xTNzDL8mp+ZWZZrimZmqXDz2cxsS06KZmYJP7xtZrYVVbfcrOikaGaF8XOKVp/OXSu5/Ia5DNpzAxHi5v/elSNGrOTQ41ZSubGMD+e256Yrd2Xd6uTHPXjv9Vz6kw/o1KWK6mq49F8/w8YKzyFWTL16rePKK1+mR4+PiYAnn9ydxx7bi8GDV3DJJVPo0KGSxYs7c8MNh7N+fVsABg1ayaWXTqZTp41UV4vLLjuejRvLadOmiosumsr++y8hAu655wBeeGFAPSVoWfxITgNIGgOcDCyOiP2KdZ2m4MIfzmPqs9247sLdadO2mvYdq+n4fFfG/LQf1VXim9+ZzxkXL2LMT/pTVh5cdcv73HD5IOZM70SX7pVUbWzkCSPtU6qqyrjjjqHMnt2Tjh03Mnr0U7z22s5cfvlk7rxzKNOm7cTxx7/Hl788nd/97gDKyqq56qoX+dnPDmPOnB506VJBVVXyexo58m1WruzAeef9C1LQpcsnJf52JdCCa4rFrJ7cDYwo4vmbhE5dqtj/kLWMvz+Ze6dyYxnrVrfh1ee7Up3+Eb3zamd67bwRgGFHrWbO9I7Mmd4JgDUr21Bd7aRYbCtWdGT27J4AbNjQlnnzurLjjhvo128N06b1BuDVV/vwuc/NB2DYsEXMmdOdOXN6ALBmTXuqq5M/l+OPf48HHtgHgAixenX77f11Sk6R31LveaT3JU2T9LqkKWmsp6SnJc1M/+2RxiVptKRZkt6QdHDmPGen+8+UdPa2fLei1RQjYpKkQcU6f1Ox84AKVi1vwxU3zmXwZ9Yza1onbvvhACo2lG/a5/gzljHp8eSPq99uHxPAdb+bSbeeG3n28Z489JudS1T61mmnnday++4rmDFjR+bO7crhhy/gxRf7c+SR8+jVaz0A/fqtIQJ+/ONn6datgueeG8hDD32Gzp2TWuFZZ03jgAMWs3DhDtx66zBWruxQyq+0fQXQuANCHBMRSzOfrwYmRsT1kq5OP38bOJFkruchwKHAbcChknoC1wDD09JNlTQ2IlY0pDAlv5El6XxJUyRN2RgVpS5OwcrbBHvst54nftebUSftw8cbyjnjokWbto8ctZCqSvHXR5NaSnl5sO/wtfz00sFc8eW9OeKElQw9YnWpit/qdOiwke9//wV++9uDWL++LTfffCgnnzyT0aMn0LFjJZWVyZ9EeXmw775LueGGw7nyymP57GfnM3ToIsrLg969NzB9ei8uueQEpk/fkXPPfa3E32r7U3V+SwOdAtyTrt8DnJqJ3xuJl4DukvoCJwBPR8TyNBE+zTa0UkueFCPi9ogYHhHD26r5NUOWLmzH0oXtmPF6ZwCeH9edPfZLahtfOH0phx67ihsuHUzydFey/7RXdmD1ijZUfFzG5Ge6bdrfiqu8vJrvf/8FnnlmV/7+96RjZP78rnzve8dw6aUn8NxzA1m4cAcAli7tyJtv9mb16vZUVLRh8uS+7L77ClavbsfHH5fzwgv9AXj++QHssUeDKiTNVu45xTybz71ylZ50OX+r0wXwlKSpmW19ImJhur4I6JOu9wPmZY6dn8ZqizdIyZNic7diSVuWLGxH/90+BuCgI9bwwcyODPvnVZz+rY/44Tm7U/Hx5h/z1EldGbzXBtp3qKasPNj/sGR/K7bg8stfYd68rjz66N6bot26Jb83KRg58m3GjdsDgKlT+zJo0Crat6+krKya/fdfwgcfdAPEyy/344ADFgMwdOhHabwVich/gaW5Sk+63L7V2T4XEQeTNI0vlnTUlpeK7f4AkB/JaQS3/mAAV42eQ9u2wcIP2nHTlYMY/fg7tG1Xzf/+fiYA77zWmV9+d1fWrmrDI3f2YfQT04mAyc9045W/trI/qhLYd9+lHHfc+8yZ041f/Wo8kDxKs8suazj55FkA/P3v/XnqqcEArF3bjkce2YtbbnmKCDF5cl8mT94FgDFjDuTKK1/iggteZdWq9tx006Gl+VIl1FhvtETEgvTfxZIeBQ4BPpLUNyIWps3jxenuC4Dss0/909gC4Oit4s82tEyKIo2gK+k+koL2Aj4CromIu+o6pmtZzziszQlFKY8Vh/YdUuoiWAFemnEnq9Z/uE2PO3Tp3j8OOuqyvPZ9/vGrpkbE8Jq2SeoMlEXEmnT9aeBa4FhgWaajpWdEXCXpX4BRwEkkHS2jI+KQtKNlKpDrjX4VGBYRyxvy/YrZ+3xmsc5tZqXVSDXFPsCjkiDJRX+IiPGSJgMPSjoHmAt8Jd1/HElCnAWsB74BEBHLJf0ImJzud21DE2KuIGZm+QugatuzYkS8BxxYQ3wZSW1x63gAF9dyrjHAmG0uFE6KZtYAHiXHzCzLs/mZmW3mmqKZWY6HDjMz20yAGqGjpalyUjSzgsn3FM3MUm4+m5llhXufzcyy3PtsZpblmqKZWSrc+2xmtqWWmxOdFM2scH4kx8wsy0nRzCwVQMMnpWrynBTNrCAi3Hw2M9tCdcutKjopmllh3Hw2M9uSm89mZlktOCmW1b+LmVlWLRPf17TUQdIASc9IelvSW5IuS+M/lLRA0uvpclLmmO9ImiVphqQTMvERaWxWOi1qg7mmaGaFaaTZ/IBK4IqIeFVSF2CqpKfTbTdHxM+zO0vaBxgJ7AvsAvxF0p7p5l8DXwDmA5MljY2ItxtSKCdFMytYY9xTjIiFwMJ0fY2k6UC/Og45Bbg/IiqAOZJmAYek22alU6Yi6f503wYlRTefzaxw+Tefe0maklnOr+l0kgYBBwEvp6FRkt6QNEZSjzTWD5iXOWx+Gqst3iBOimZWmACqI78FlkbE8Mxy+9ank7QD8DBweUSsBm4DdgeGktQkb9xu3w03n82sYI038raktiQJ8fcR8QhARHyU2X4H8ET6cQEwIHN4/zRGHfGCuaZoZoVrnN5nAXcB0yPipky8b2a304A30/WxwEhJ7SUNBoYArwCTgSGSBktqR9IZM7ahX801RTMrTABVjfJKyxHA14Fpkl5PY98FzpQ0NL3S+8AFABHxlqQHSTpQKoGLI6IKQNIoYAJQDoyJiLcaWignRTMrUEBse1KMiL+RTCO9tXF1HHMdcF0N8XF1HVcIJ0UzK1wLfqPFSdHMCpPrfW6hnBTNrHCuKZqZZTgpmpmlIqCqqtSlKBonRTMrnGuKZmYZTopmZjnh3mczs00CohEe3m6qnBTNrHCN85pfk+SkaGaFifAUp2ZmW3BHi5nZZuGaoplZTuMNMtsUOSmaWWE8IISZ2WYBhF/zMzNLReMMMttUOSmaWcHCzWczs4wWXFNUNKFeJElLgLmlLkcR9AKWlroQVpCW+jvbNSJ6b8sJJI0n+fnkY2lEjNiW621vTSoptlSSpkTE8FKXw/Ln31nr5XmfzcwynBTNzDKcFLeP20tdACuYf2etlO8pmplluKZoZpbhpGhmluGkWESSRkiaIWmWpKtLXR6rn6QxkhZLerPUZbHScFIsEknlwK+BE4F9gDMl7VPaUlke7gaa1cPG1ricFIvnEGBWRLwXEZ8A9wOnlLhMVo+ImAQsL3U5rHScFIunHzAv83l+GjOzJsxJ0cwsw0mxeBYAAzKf+6cxM2vCnBSLZzIwRNJgSe2AkcDYEpfJzOrhpFgkEVEJjAImANOBByPirdKWyuoj6T7gRWAvSfMlnVPqMtn25df8zMwyXFM0M8twUjQzy3BSNDPLcFI0M8twUjQzy3BSbEYkVUl6XdKbkv4oqdM2nOtuSaen63fWNViFpKMlfbYB13hf0qdmfastvtU+awu81g8lXVloGc225qTYvGyIiKERsR/wCXBhdqOkBs3jHRHnRsTbdexyNFBwUjRrjpwUm6/ngT3SWtzzksYCb0sql/QzSZMlvSHpAgAlfpWO7/gXYKfciSQ9K2l4uj5C0quS/iFpoqRBJMn3P9Na6pGSekt6OL3GZElHpMfuKOkpSW9JuhNQfV9C0p8kTU2POX+rbTen8YmSeqex3SWNT495XtLejfLTNEs1qGZhpZXWCE8Exqehg4H9ImJOmlhWRcQ/SWoPvCDpKeAgYC+SsR37AG8DY7Y6b2/gDuCo9Fw9I2K5pN8AayPi5+l+fwBujoi/SRpI8tbOZ4BrgL9FxLWS/gXI522Qb6bX6AhMlvRwRCwDOgNTIuI/Jf0gPfcokgmlLoyImZIOBW4FPt+AH6NZjZwUm5eOkl5P158H7iJp1r4SEXPS+PHAAbn7hUA3YAhwFHBfRFQBH0r6aw3nPwyYlDtXRNQ2ruBxwD7SpopgV0k7pNf4UnrsnyWtyOM7XSrptHR9QFrWZUA18EAa/z/gkfQanwX+mLl2+zyuYZY3J8XmZUNEDM0G0uSwLhsCLomICVvtd1IjlqMMOCwiPq6hLHmTdDRJgj08ItZLehboUMvukV535dY/A7PG5HuKLc8E4FuS2gJI2lNSZ2AScEZ6z7EvcEwNx74EHCVpcHpszzS+BuiS2e8p4JLcB0lD09VJwFfT2IlAj3rK2g1YkSbEvUlqqjllQK62+1WSZvlqYI6kf0uvIUkH1nMNs4I4KbY8d5LcL3w1nXzptyQtgkeBmem2e0lGgtlCRCwBzidpqv6Dzc3Xx4HTch0twKXA8LQj520294L/D0lSfYukGf1BPWUdD7SRNB24niQp56wDDkm/w+eBa9P414Bz0vK9had4sEbmUXLMzDJcUzQzy3BSNDPLcFI0M8twUjQzy3BSNDPLcFI0M8twUjQzy/j/Dh3ejhMTZFEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "f.scoring(y_val, val_y_predict_class, clf, vect_X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "FN_pred_prob = [y_predict_prob_class_1[i] if y_predict_prob_class_1[i] < 0.96 else 0 for i in range(len(y_predict_prob_class_1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAE/CAYAAABB8mpwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc9klEQVR4nO3de7xnVV3/8dc7EK8oKEhcRrEcTbSfSvMDupiUioDlUJmBF9AfiZp2+UkX8lc/SNLoovajFMXkJxgK5I0pMSSkKAtiUERBjREHGa4Dw60oFP30x15HvnM4l++cOTNnzjqv5+PxfZzvWXvvtdd3nX2+7+9ee519UlVIkqR+fc9CN0CSJG1Zhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16LVpJXJfmnOW57YJJ1Myx/T5LfnWrdJFclOXAu+x2jXSck+cv2/AlJ/j3JdvNU97SvaR7qfk6Sr85XfVtCkr9P8ovt+cuTfHqcdeewn3n9uc2yrzm3U0uLYS8AkqxN8p/tTWrisUeSvZNUkvMmrf+XSU5YoOZucVX1uqo6cZplT6+qv4eNw3kLtOEbVfWoqvr2TOuN+6Fnpte0qdox8eSRuv+xqp46H3VvDVV1ZlUdNB91td+d54/UPdbPbWub3M7Fvh9tGsNeo366vUlNPG4cWbZ/kh/Zmo1Jsv3W3F/PtsZZpqRtl2Gvcf0R8NapFiTZJcnfJLkzyYYk/5hkymOrnRH+SpJrk9yW5I8n1m1nqJ9N8s4ktwMnJHlMkjOSrE9yXZLfmVR3kvx5kruSfCXJ80YWvDrJl5Pc0/b32ina8+bWjrVJXj5S/oEkvz/Na1ib5PlJDgbeDPxCGwn5QpKfT3L5pPXflOTcaep6UpJ/aG28ANhlZNnEqMr2I/1zbVv3620o+mnAe4Afbm24c6T9pyQ5L8l/AD8x1Wua4fVvNDw8OnqQ5OJW/IW2z1/Igy91PK3VcWeGyx4vntS370ryyfZaLk3y/VP1z6S2PrTV94yRsl0zjEg9PsnO7Thcn+SO9nyvaeraaDQkyQva8XNXkj8HMrLs+5N8Jsntra/OTLJTW/ZB4AnAX7e++M0pfm57JFmV4XdjTZLXjNR9QpJz2jF+T+urFTP0wby1s5X/VZKbW30XJ3n6SH2HJrm6teuGJL8+suynklzRfh7/nOR/zLQfLTzDXuN6N/CUTD08dyywDtgV2I0hAGe6D/PPACuAfYGVwP8aWbY/cG2r563AnwGPAb4PeC5wJPDqSet/jSEkjwc+luSxbdmtwE8Bj27bvDPJviPbfm/bbk/gKODUJGMPRVfV3wJvA85uIyHPBFYBT2ohPOGVwBnTVPMh4PLWjhNbOx4kySOBk4FDqmpH4EeAK6rqy8DrgH9pbdhpZLOXMfThjsBUw/xzev1V9ePt6TPbPs+e1NaHAH8NfBp4PPDLwJmT6j4c+D1gZ2AN03yQnLTf+4CPAUeMFL8U+IequpXh/ez/A09kCJz/BP58tnqT7NLq/R2G/vga8KOjqwB/AOwBPA1YBpzQ2vRK4Bs8MCr2R1Ps4iyG3489gJcAb0vykyPLX9zW2Ynh+JmyzVuonZ8CljP8nD4HnDlS3/uB17bj7RnAZ1o7ng2cBrwWeBzwXmBVkoeO2R9aAIa9Rn2ifVK/M8knJi37T4Y35KnOdr8F7A48saq+1a7fzhT2f1hVG6rqG8CfsvGb941V9WdVdT/wTYZQ+O2quqeq1gJvZwjPCbcCf9r2ezbwVeBFAFX1yar6Wg3+gSF8njOpLb9bVfe15Z9kCI85a4F0NvAKgHamtDfwN5PXTfIE4H+OtOFihpCczneAZyR5eFXdVFVXzdKcc6vqs1X1nar6r2nWmdfX3xwAPAo4qaq+WVWfYXj9oz/nj1fVv7af85nAs8as+0MMx8SEl7Uyqur2qvpoVd1bVfcwHK/PHaPOQ4GrquojVfUthmPy5omFVbWmqi5o/bQeeMeY9ZJkGUMg/1ZV/VdVXQH8BcOH1gn/VFXntWv8HwSeubXaWVWntd+t+xg+GDwzyWPa4m8B+yR5dFXdUVWfa+XHAO+tqkur6ttVdTpwH8PPXdsow16jDquqndrjsCmW/wWwW5KfnlT+xwxnZ5/OMMx83Cz7uX7k+XUMZyJTLdsFeEhbZ3T9PUe+v2HSB4vv1pfkkCSXtOHTOxneLHcZWfeOqvqPGdoyV6cDL0sShg8m57Q308n2mKYND9LW+QWGs/ib2hD4D8zSjutnWb6lXv8ewPVV9Z1JdY/+3G4eeX4vw4eDcVwEPCLJ/kn2ZviQ8HGAJI9I8t4Ml3vuBi4Gdsrs8xX2YKSv2vH03e+T7JbkrDaUfTfwl2x8HM1W94b24WPCbH3xsEw9X2Ve25lkuyQnJflaW39tWzSxzc8x/M5cl+FS0w+38icCx46cGNzJMIowH8eOthDDXmOrqm8yDL2eyMi1wnZmcGxVfR/DkOSbMnLtfArLRp4/ARidCDga3LcxnF08cdL6N4x8v2cL1Y3qS/JQ4KPAnwC7teHt80bbDezchsena8s4HjSCUVWXMIxKPIfhzPOD02x70zRtmHpHVedX1QsYRlG+ArxvujbMUj5hptf/H8AjRpZ97yx1jboRWJaN51ZM/rnNSTv7PYdhlOAI4G9GgvRY4KnA/lX1aGDickMeVNHGbmLkmGzH0+gx+jaGvvzBVu8rJtU5Uz/fCDw2yY4jZXPti/lu58sYLqM9n+FS2d4TVQNU1WVVtZJhiP8TDP0OwweMt46cGOxUVY+oqg9Psx9tAwx7baoPAg8DDp4oaJN1ntzefO4Cvs0w5Dyd38gwmWoZ8KsMw94PMvLG/tYkOyZ5IvAmhjOWCY8HfiXJQ5L8PMO1yvOAHYCHAuuB+5McAkz1p1a/l2SHJM9huL7/V7N3wUZuAfbOgycknsFw7fVbVTXln8VV1XXA6pE2/BgwedQE+O5Z28oWzvcB/84DfXwLsFeSHTax7TD9678C+Nl2tvxk4OhJ293CMI9iKpcynKH+Zvu5HNhe11njNCjDxL4TZljlQwyjHC9vzyfsyHC56c42b+P4cfbHcPni6Ul+tp1R/wobf7jZkaG/70qyJ/Abk7afti+q6nrgn4E/SPKwNpHtaDY+hsc13+3ckeFYup3hg93bJha0Y+LlSR7TLhnczQPH2/uA17XRlSR5ZJIXjXygmenY0AIx7LVJWgD/X+CxI8XLgb9jeKP5F+DdVXXRDNWcyzAp7QqGN7D3z7DuLzOcZV7LMMnsQwyTgyZc2vZ/G8M12pe0a7f3MLwZngPcwXAWs2pS3Te3ZTcyXDd+XVV9ZYa2TGUiHG9P8rmR8g8yTGqa7U39ZQyTDDcwhNN0E/m+h+GDzo1t3ecCr2/LPgNcBdyc5LZNaPtMr/+dDKMTtzBcljhz0rYnAKe3YdyNrvO3EaCfBg5h+Lm8GzhyE/p2GfDZ6RZW1aUMx8QeDBPMJvwp8PC2z0uAvx1nZ1V1G/DzwEkMwbd80v5/j2Ey6V0Mx+vHJlXxB8DvtL74dR7sCIaz5hsZLjkcX1V/N07btnA7z2C4pHADcDVDn416JbC2DfG/juHDFVW1GngNw4fZOxgu4b1qhv1oG5CZ51FJ8ytJAcuras1Ct2VLSvJwhsmD+1bVNQvdnsUiw5/KnVNVW/WeDlLvvGmJtGW8HrjMoN80VbWO4c8KJc2jWYfxkyxLclGGmytcleRXW/kJbdbnFe1x6Mg2v53h5hFfTfLCkfKDW9majMzYznBjkUtb+dlzvPYobROSrGWYi3DsAjdFkoAxhvGT7A7sXlWfaxMwLgcOY/h73H+vqj+ZtP4+wIeB/Riuqf0d8JS2+N+AFzDcYOIy4IiqujrJOcDHquqsJO8BvlBVp8zTa5QkaUmb9cy+3bzjc+35PcCX2fhvRCdbCZzVbuzwdYbJG/u1x5qqurZN4DkLWNlmcP8k8JG2/ekMHyYkSdI82KTZ+BluYvFshhnQAG9McmWS05Ls3Mr2ZOObeaxrZdOVPw64s91Ja7RckiTNg7En6CV5FMNNSn6tqu5OcgrDzVWqfX07G9/jfN4lOYbhVo088pGP/KEf+IHZbiAmSVIfLr/88tuqate5bDtW2Gf4xxYfBc6sqo8BVNUtI8vfxwP3/r6Bje/qtBcP3C1qqvLbGW5puX07ux9dfyNVdSpwKsCKFStq9erV4zRfkqRFL8mUt9Mexziz8cNw05MvV9U7Rsp3H1ntZ4AvteergMMz/DvKJzHc+OFfGSbkLW8z73dg+GcWq9r9nS9i+G9QMPz3rSn/HagkSdp045zZ/yjDnZS+mOSKVvZm4Igkz2IYxl/L8O8Oqaqr2uz6q4H7gTe0u66R5I3A+cB2wGn1wH/t+i3grAz/a/vzzHxHNUmStAkW7R30HMaXJC0lSS6vqhVz2dZ740uS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnq3Kxhn2RZkouSXJ3kqiS/2sofm+SCJNe0rzu38iQ5OcmaJFcm2XekrqPa+tckOWqk/IeSfLFtc3KSbIkXK0nSUjTOmf39wLFVtQ9wAPCGJPsAxwEXVtVy4ML2PcAhwPL2OAY4BYYPB8DxwP7AfsDxEx8Q2jqvGdnu4M1/aZIkCcYI+6q6qao+157fA3wZ2BNYCZzeVjsdOKw9XwmcUYNLgJ2S7A68ELigqjZU1R3ABcDBbdmjq+qSqirgjJG6JEnSZtqka/ZJ9gaeDVwK7FZVN7VFNwO7ted7AtePbLaulc1Uvm6K8qn2f0yS1UlWr1+/flOaLknSkjV22Cd5FPBR4Neq6u7RZe2MvOa5bQ9SVadW1YqqWrHrrrtu6d1JktSFscI+yUMYgv7MqvpYK76lDcHTvt7aym8Alo1svlcrm6l8rynKJUnSPBhnNn6A9wNfrqp3jCxaBUzMqD8KOHek/Mg2K/8A4K423H8+cFCSndvEvIOA89uyu5Mc0PZ15EhdkiRpM20/xjo/CrwS+GKSK1rZm4GTgHOSHA1cB7y0LTsPOBRYA9wLvBqgqjYkORG4rK33lqra0J7/EvAB4OHAp9pDkiTNgwyX2xefFStW1OrVqxe6GZIkbRVJLq+qFXPZ1jvoSZLUOcNekqTOGfaSJHVunAl6i8rex31yQfa79qQXLch+JUmajWf2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHVu1rBPclqSW5N8aaTshCQ3JLmiPQ4dWfbbSdYk+WqSF46UH9zK1iQ5bqT8SUkubeVnJ9lhPl+gJElL3Thn9h8ADp6i/J1V9az2OA8gyT7A4cDT2zbvTrJdku2AdwGHAPsAR7R1Af6w1fVk4A7g6M15QZIkaWOzhn1VXQxsGLO+lcBZVXVfVX0dWAPs1x5rquraqvomcBawMkmAnwQ+0rY/HThs016CJEmayeZcs39jkivbMP/OrWxP4PqRdda1sunKHwfcWVX3TyqXJEnzZK5hfwrw/cCzgJuAt89Xg2aS5Jgkq5OsXr9+/dbYpSRJi96cwr6qbqmqb1fVd4D3MQzTA9wALBtZda9WNl357cBOSbafVD7dfk+tqhVVtWLXXXedS9MlSVpy5hT2SXYf+fZngImZ+quAw5M8NMmTgOXAvwKXAcvbzPsdGCbxraqqAi4CXtK2Pwo4dy5tkiRJU9t+thWSfBg4ENglyTrgeODAJM8CClgLvBagqq5Kcg5wNXA/8Iaq+nar543A+cB2wGlVdVXbxW8BZyX5feDzwPvn68VJkqQxwr6qjpiieNpArqq3Am+dovw84Lwpyq/lgcsAkiRpnnkHPUmSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc7OGfZLTktya5EsjZY9NckGSa9rXnVt5kpycZE2SK5PsO7LNUW39a5IcNVL+Q0m+2LY5OUnm+0VKkrSUjXNm/wHg4EllxwEXVtVy4ML2PcAhwPL2OAY4BYYPB8DxwP7AfsDxEx8Q2jqvGdlu8r4kSdJmmDXsq+piYMOk4pXA6e356cBhI+Vn1OASYKckuwMvBC6oqg1VdQdwAXBwW/boqrqkqgo4Y6QuSZI0D+Z6zX63qrqpPb8Z2K093xO4fmS9da1spvJ1U5RLkqR5stkT9NoZec1DW2aV5Jgkq5OsXr9+/dbYpSRJi95cw/6WNgRP+3prK78BWDay3l6tbKbyvaYon1JVnVpVK6pqxa677jrHpkuStLTMNexXARMz6o8Czh0pP7LNyj8AuKsN958PHJRk5zYx7yDg/Lbs7iQHtFn4R47UJUmS5sH2s62Q5MPAgcAuSdYxzKo/CTgnydHAdcBL2+rnAYcCa4B7gVcDVNWGJCcCl7X13lJVE5P+folhxv/DgU+1hyRJmiezhn1VHTHNoudNsW4Bb5imntOA06YoXw08Y7Z2SJKkufEOepIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6t1lhn2Rtki8muSLJ6lb22CQXJLmmfd25lSfJyUnWJLkyyb4j9RzV1r8myVGb95IkSdKo+Tiz/4mqelZVrWjfHwdcWFXLgQvb9wCHAMvb4xjgFBg+HADHA/sD+wHHT3xAkCRJm29LDOOvBE5vz08HDhspP6MGlwA7JdkdeCFwQVVtqKo7gAuAg7dAuyRJWpI2N+wL+HSSy5Mc08p2q6qb2vObgd3a8z2B60e2XdfKpit/kCTHJFmdZPX69es3s+mSJC0N22/m9j9WVTckeTxwQZKvjC6sqkpSm7mP0fpOBU4FWLFixbzVK0lSzzbrzL6qbmhfbwU+znDN/ZY2PE/7emtb/QZg2cjme7Wy6colSdI8mHPYJ3lkkh0nngMHAV8CVgETM+qPAs5tz1cBR7ZZ+QcAd7Xh/vOBg5Ls3CbmHdTKJEnSPNicYfzdgI8nmajnQ1X1t0kuA85JcjRwHfDStv55wKHAGuBe4NUAVbUhyYnAZW29t1TVhs1olyRJGjHnsK+qa4FnTlF+O/C8KcoLeMM0dZ0GnDbXtkiSpOl5Bz1Jkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSerc9gvdgKVs7+M+udX3ufakF231fUqSFpZn9pIkdc6wlySpcw7ja1YLcbkBvOQgSfPFM3tJkjrnmb264qRHSXqwbSbskxwM/D9gO+AvquqkBW6SJPwAtZQtpkt4Hqcz2ybCPsl2wLuAFwDrgMuSrKqqqxe2ZdKWsZjeRBeLxdSnBpO2tm3lmv1+wJqquraqvgmcBaxc4DZJktSFbSXs9wSuH/l+XSuTJEmbKVW10G0gyUuAg6vqF9v3rwT2r6o3TlrvGOCY9u1Tga/Oc1N2AW6b5zqXOvt0/tmn888+nX/26fx7alXtOJcNt4lr9sANwLKR7/dqZRupqlOBU7dUI5KsrqoVW6r+pcg+nX/26fyzT+effTr/kqye67bbyjD+ZcDyJE9KsgNwOLBqgdskSVIXtokz+6q6P8kbgfMZ/vTutKq6aoGbJUlSF7aJsAeoqvOA8xa4GVvsEsESZp/OP/t0/tmn888+nX9z7tNtYoKeJEnacraVa/aSJGkLWZJhn+TgJF9NsibJcVMsf2iSs9vyS5PsvQDNXFTG6NM3Jbk6yZVJLkzyxIVo52IyW5+OrPdzSSqJM59nMU6fJnlpO1avSvKhrd3GxWaM3/0nJLkoyefb7/+hC9HOxSLJaUluTfKlaZYnycmtv69Msu9YFVfVknowTAD8GvB9wA7AF4B9Jq3zS8B72vPDgbMXut3b8mPMPv0J4BHt+evt083v07bejsDFwCXAioVu97b8GPM4XQ58Hti5ff/4hW73tvwYs09PBV7fnu8DrF3odm/LD+DHgX2BL02z/FDgU0CAA4BLx6l3KZ7Zj3Nr3pXA6e35R4DnJclWbONiM2ufVtVFVXVv+/YShnspaHrj3kL6ROAPgf/amo1bpMbp09cA76qqOwCq6tat3MbFZpw+LeDR7fljgBu3YvsWnaq6GNgwwyorgTNqcAmwU5LdZ6t3KYb9OLfm/e46VXU/cBfwuK3SusVpU293fDTDJ1NNb9Y+bcN3y6pqYf4DzOIzznH6FOApST6b5JL23zg1vXH69ATgFUnWMfzF1S9vnaZ1a063l99m/vROS0OSVwArgOcudFsWsyTfA7wDeNUCN6U32zMM5R/IMPp0cZIfrKo7F7JRi9wRwAeq6u1Jfhj4YJJnVNV3FrphS8lSPLMf59a8310nyfYMQ0+3b5XWLU5j3e44yfOB/wO8uKru20ptW6xm69MdgWcAf59kLcO1u1VO0pvROMfpOmBVVX2rqr4O/BtD+Gtq4/Tp0cA5AFX1L8DDGO6br7kZ6/12sqUY9uPcmncVcFR7/hLgM9VmRmhKs/ZpkmcD72UIeq+Dzm7GPq2qu6pql6rau6r2ZpgH8eKqmvO9s5eAcX73P8FwVk+SXRiG9a/dim1cbMbp028AzwNI8jSGsF+/VVvZl1XAkW1W/gHAXVV102wbLblh/Jrm1rxJ3gKsrqpVwPsZhprWMEyUOHzhWrztG7NP/xh4FPBXba7jN6rqxQvW6G3cmH2qTTBmn54PHJTkauDbwG9UlaN60xizT48F3pfkfzNM1nuVJ0/TS/Jhhg+cu7R5DscDDwGoqvcwzHs4FFgD3Au8eqx67XNJkvq2FIfxJUlaUgx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSercfwNr76AfpsHMOgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (8, 5))\n",
    "plt.hist(FN_pred_prob, bins=10, range = [0,1], rwidth=0.7, align='left')\n",
    "plt.xlim(-0.05,1)\n",
    "plt.ylim(0, 25000)\n",
    "plt.title('FNs probability distribution, validation dataset')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 92.2%\n",
      "Precision: 94.2%\n",
      "Recall: 76.1%\n",
      "F1: 81.8%\n",
      "\n",
      "\n",
      "Classification Report\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96     24191\n",
      "           1       0.97      0.53      0.68      4540\n",
      "\n",
      "    accuracy                           0.92     28731\n",
      "   macro avg       0.94      0.76      0.82     28731\n",
      "weighted avg       0.93      0.92      0.91     28731\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/keqiaoli/miniconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEGCAYAAADyuIefAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdpElEQVR4nO3deZgV1bnv8e/bTTM0QzMTBBQVFIkRRCISjfORluReh5Nj1CRyPRqciGbQHExuJGpITExigonmEkPEOB0TNRLDICF6FCOjQSZFBkVABqGRqWno4b1/1OrugvSwN/RmD/37PE89XXvttave3f3wslatqrXM3RERkUheugMQEckkSooiIjFKiiIiMUqKIiIxSooiIjEt0h1AXNfO+d63T0G6w5AkvLu4MN0hSBLK2MN+32eHc4wR57X1bSWVCdVduHjfDHcvPpzzHWkZlRT79ilg3ow+6Q5DkjDiqMHpDkGSMNdnHfYxtpZUMndG74TqFvRc3fWwT3iEZVRSFJFs4FR6VbqDSBklRRFJigNV5O5DH0qKIpK0KtRSFBEBwHHK1X0WEYk4UKnus4hILV1TFBEJHKjM4dm1lBRFJGm5e0VRSVFEkuS4rimKiFRzh/LczYlKiiKSLKOSw3p8OqMpKYpIUhyoUktRRKSWWooiIkF087aSoogIECXFcs/d+amVFEUkKY5RmcOT9ispikjSqlzdZxERQNcURUQOYlTqmqKISCSaeVtJUUQEAHdjv+enO4yUUVIUkaRV6ZqiiEgkGmhR91lEJNBAi4hIDQ20iIgcpFI3b4uIRByj3HM3deTuNxORlNBAi4hIjGPqPouIxOXyQEvufjMRSQl3qPS8hLaGmFkfM3vZzJab2TIzuy2UdzazmWa2MvzsFMrNzCaY2SozW2xmQ2LHGhXqrzSzUbHy08xsSfjMBDNrtImrpCgiSYkGWvIT2hpRAXzL3QcCZwC3mNlAYCwwy937A7PCa4CLgf5hGw08DFESBcYBw4DTgXHViTTU+Wrsc8WNBaWkKCJJqyQvoa0h7r7R3d8M+7uAt4FewCXA5FBtMnBp2L8EeMwjc4COZtYTGAHMdPcSd98OzASKw3sd3H2OuzvwWOxY9dI1RRFJimPJTDLb1cwWxF5PdPeJB1cys77AqcBcoIe7bwxvbQJ6hP1ewLrYx9aHsobK19dR3iAlRRFJWhK35Gx196ENVTCzdsCzwNfdfWf8sp+7u5kd0QVV1X0WkaRE6z7nJbQ1xswKiBLiE+7+XCjeHLq+hJ9bQvkGoE/s471DWUPlvesob5CSoogkyahMcGvwKFGT8HfA2+7+89hbU4DqEeRRwAux8mvCKPQZwI7QzZ4BXGRmncIAy0XAjPDeTjM7I5zrmtix6qXus4gkJVritEkmmT0T+AqwxMwWhbLvAPcBz5jZdcBa4Irw3lRgJLAKKAWuBXD3EjO7F5gf6t3j7iVh/2bgUaANMC1sDVJSFJGkuFtCXePGj+Ozod7m5AV11HfglnqONQmYVEf5AuDkZOJSUhSRpGk+RRGRIJpPUc8+i4gEmnlbRKRGdEuOWooiIkDts8+5SklRRJKWy1OHKSmKSFKiqcPUfRYRqaFriiIiQTRLjrrPIiJA9WN+SorN3pYNBdx/29F8/FEBmDPyy9u47PqtTP7JJ3hjRhFm0LFrObf/4gO6fKICgLf+0Y7f3NWLigoo6lzJT59bBcDzj3Rl2hNdcIeLv1TC5V/9qOY8L/yuK1Me7UpevjPsgp1c/72NdcYjhy8vz3lw+rts21jAXaOOY9CZu/jqXRspKHBWLm7Dz7/Vh6pK45Thu/n+799j07qWALw+tYgnHvhEmqNPJ7UUD5mZFQO/BPKBR9z9vlSeL5XyWzij7/qQ/qfspXR3HmOKT2DI2bv4wk1bGPXtTQD8+ZGuPP7AJ7jtx+vZvSOfX93Zm/FPrKZ773I+3hr9qt9/pzXTnujChL++S0FL5ztXH8+wC3fQ69j9LHq9Hf+YUcTDf1tBy1Ze8xlJjUuv38q6la0pbFeJmXPHL9fxX1ccz4Y1rbjmjk382xUlzHiqCwBL57blrlHHpTnizJHLT7SkLN2bWT7wa6J1FQYCV4X1F7JSlx4V9D9lLwCF7aro028fWzcW0LZ9VU2dsr15VM+P+fLzHTlz5Md0710OQMeuUevxg5WtGHBqKa0LnfwWcMrw3bw+tSMALz7WhS+O2UzLVn7AZ6Tpde25n9Mv2Mm0JzsD0KFTJeX7jQ1rWgHw5v+046yRO9IZYsaqHn1OZMtGqWwDnw6scvc17r4feJpojYWst2ldS1YvbcOAIaUA/P6+T/Cl0wby9+c6cc0dUXd3/ZrW7P44nzv+vR+3jDiBmX+M1tHpO6CMpfPasrMkn7JSY/7fO/DRhwUAbFjdmqVz23Hr5/pz++X9WLGoTXq+YDNw490f8sgPeuJV0T/cHSX55Ldw+p8S/U3P+vwOuh1VXlP/pNNKeXjmCn7w+BqOOaEsLTFnkqaaZDYTpbJ/Vte6CcMOrmRmo4lW5uLoXpnfXdy7J497r+/LjfdsqGklXjt2E9eO3cTTD3ZnyqRuXHPHJiorYOWSQn78zGr27TW+/r9P4KQhpRzdfx9X3LyFO686ntaFVRz3yb3khYcDKith18f5/PLFlaxYVMj4G/oyec7bNL4ooyRj2IU7+XhrC1YtKeSU4btDqfGjm47hxrs/pKBlFQv/pz1VoROwakkbvnL6SZSV5vPp83cybtJ7/OdZJ6Ut/nRLco2WrJP2VO7uE919qLsP7dYlsx8dqiiHe6/vy/mXb6+za3X+ZduZPbUIgG49yzntnF20LqyiqEslnxq2mzXLWwNQfHUJv57xLj97fhXtiirpfVzU8ujas5wzR+7ADAacWkpeXtSCkaY18NN7OOOinUyeu5w7H17LoLN28+0H1/L2wrZ867J+3Pq5E1gytx0bVkdd6dLd+ZSVRn+H+X/vQH6B06Fz87204UCF5yW0ZaNURl3fuglZyR1+/q2j6dN/H/9+Q+1o8YY1LWv235hRRJ9++wAYXryDZfPbUlkBZaXGO/8s5Oj+0XvVAyhb1hfw+tQizrvsYwA+U7yDt15vB8D61a0o328Uda48El+vWfn9j3ry5aEDGTVsID+66Rjemt2On3ztGIq6RN3lgpZVXHHzFl78QzTI0qlbOVEqgBMHR/9Z7Wzm/1mp+3xo5gP9zexYomR4JXB1Cs+XUsvmtWXWnzpz7El7uenCEwG49s4Pmf5UF9avbkVeHnTvtZ9bfxytqHh0/30MPXcnN14wAMtziq8uoe+AqEV4z/V92bW9BfkFzpgfrqddUZT4RlxZws+/2YfR551IQYFzxy8/UNf5CPqPmz9i2IU7sTz46+QuvPV6ewA++/kdfP6arVRWGPvK8vjRTcdQ/4TRzYDndvfZohm+U3Rws5HAL4huyZnk7uMbqj90UGufN6NPQ1Ukw4w4anC6Q5AkzPVZ7PSSw8ponQZ09/MnfSGhus+d+fDCxpY4zTQpHdlw96lEi82ISA7J5ZZi5g/3ikhG0SSzIiIxjlFRlZ2DKIlQUhSRpOXyY35KiiKSHFf3WUSkhq4piogcRElRRCRwjEoNtIiI1NJAi4hI4BpoERE5kCspiohUy+0JIZQURSRpaimKiATuUFmlpCgiUkOjzyIigaPus4hIjAZaREQOkMIJ+9NOSVFEkpbL3efcfYBRRFIiGn3OS2hrjJlNMrMtZrY0VvZ9M9tgZovCNjL23p1mtsrMVpjZiFh5cShbZWZjY+XHmtncUP7fZla7/GY9lBRFJGnuiW0JeBQorqP8AXcfHLapAGY2kGhV0E+GzzxkZvlmlg/8GrgYGAhcFeoC/Dgcqx+wHbiusYCUFEUkae6W0Nb4cfxVoCTB014CPO3u+9z9PWAVcHrYVrn7GnffDzwNXGJmBpwP/Cl8fjJwaWMnUVIUkaQ4iSXEkBS7mtmC2DY6wdOMMbPFoXvdKZT1AtbF6qwPZfWVdwE+dveKg8obpKQoIknzBDdgq7sPjW0TEzj8w8DxwGBgI/CzJg6/QRp9FpHkOHgKH/Nz983V+2b2W+DF8HID0CdWtXcoo57ybUBHM2sRWovx+vVSS1FEktZU1xTrYmY9Yy8vA6pHpqcAV5pZKzM7FugPzAPmA/3DSHNLosGYKe7uwMvAF8LnRwEvNHZ+tRRFJGlNdfO2mT0FnEt07XE9MA4418wGE/XA3wduiM7py8zsGWA5UAHc4u6V4ThjgBlAPjDJ3ZeFU/wX8LSZ/QD4J/C7xmKqNyma2YPUXBb4V+5+a2MHF5Hc05TPPrv7VXUU15u43H08ML6O8qnA1DrK1xCNTiesoZbigmQOJCLNhAM5/ERLvUnR3SfHX5tZobuXpj4kEcl0ufzsc6MDLWY23MyWA++E14PM7KGURyYiGcrwqsS2bJTI6PMvgBFEw9u4+1vA2SmMSUQyXRI3KmabhEaf3X1d9MRMjcrUhCMiGc9ze5acRJLiOjP7DOBmVgDcBryd2rBEJKNlaSswEYl0n28EbiF6ZvBDokdvbklhTCKS8SzBLfs02lJ0963Al45ALCKSLarSHUDqJDL6fJyZ/cXMPgqTQb5gZscdieBEJANV36eYyJaFEuk+Pwk8A/QEjgL+CDyVyqBEJLM14SSzGSeRpFjo7n9w94qwPQ60TnVgIpLBmuMtOWbWOexOC2sePE30Nb9IHc8YikgzkqVd40Q0NNCykCgJVn/7G2LvOXBnqoISkcxmWdoKTERDzz4feyQDEZEs4QZZ+ghfIhJ6osXMTiZaJavmWqK7P5aqoEQkwzXHlmI1MxtHNAnkQKJriRcDswElRZHmKoeTYiKjz18ALgA2ufu1wCCgKKVRiUhma46jzzF73b3KzCrMrAOwhQMXiRGR5qS5TjIbs8DMOgK/JRqR3g28kcqgRCSzNcvR52rufnPY/Y2ZTQc6uPvi1IYlIhmtOSZFMxvS0Hvu/mZqQhKRTNdcW4o/a+A9B85v4lh4d0khxUcPberDSgrlDeqf7hAkCbbi9aY5UHO8puju5x3JQEQkS2TxyHIiErp5W0TkAEqKIiK1LIcnmVVSFJHk5XBLMZGZt83Mvmxmd4XXR5vZ6akPTUQykXniWzZK5DG/h4DhwFXh9S7g1ymLSEQyXw4vR5BI93mYuw8xs38CuPt2M2uZ4rhEJJNlaSswEYkkxXIzyyf8GsysGzm9lpeINCZbu8aJSCQpTgCeB7qb2XiiWXP+b0qjEpHM5c189NndnzCzhUTThxlwqbu/nfLIRCRzNeeWopkdDZQCf4mXufsHqQxMRDJYc06KwF+pXcCqNXAssAL4ZArjEpEM1qyvKbr7p+Kvw+w5N9dTXUQkqyX9RIu7v2lmw1IRjIhkiebcUjSzb8Ze5gFDgA9TFpGIZLbmPvoMtI/tVxBdY3w2NeGISFbI4ZZig4/5hZu227v73WEb7+5PuHvZEYpPRDKM0XTPPpvZJDPbYmZLY2WdzWymma0MPzuFcjOzCWa2yswWx1cHMLNRof5KMxsVKz/NzJaEz0wws0afPaw3KZpZC3evBM5s/KuJSLPSdEucPgoUH1Q2Fpjl7v2BWeE1RGvO9w/baOBhiJIoMA4YBpwOjKtOpKHOV2OfO/hc/6KhluK88HORmU0xs6+Y2eXVW2MHFpEc1YSz5Lj7q0DJQcWXAJPD/mTg0lj5Yx6ZA3Q0s57ACGCmu5e4+3ZgJlAc3uvg7nPc3YHHYseqVyLXFFsD24jWZKm+X9GB5xL4rIjkosQHWrqa2YLY64nuPrGRz/Rw941hfxPQI+z3AtbF6q0PZQ2Vr6+jvEENJcXuYeR5KbXJsFoOX2YVkcYkcfP2Vnc/5NXo3N3Njuyt4g11n/OBdmFrH9uv3kSkuWq6a4p12Ry6voSfW0L5BqBPrF7vUNZQee86yhvUUEtxo7vf09gBRKSZSf1qflOAUcB94ecLsfIxZvY00aDKDnffaGYzgB/GBlcuAu509xIz22lmZwBzgWuABxs7eUNJMTunzRWRlGuqDq2ZPQWcS3TtcT3RKPJ9wDNmdh2wFrgiVJ8KjARWEU1Scy1ASH73AvNDvXvcvXrw5maiEe42wLSwNaihpHhBol9MRJqZJkqK7n5VPW/9S/4JI8i31HOcScCkOsoXACcnE1O9STGWaUVEDtDcH/MTEamV+muKaaWkKCJJMXJ7wEFJUUSSp5aiiEitZj3ztojIv1BSFBEJNMmsiMhB1FIUEamla4oiInFKiiIitdRSFBGp5iQzyWzWUVIUkaRUL1yVq5QURSR5SooiIrXMczcrKimKSHI0S46IyIF0TVFEJEaP+YmIxKmlKCISuLrPIiIHUlIUEYno5m0RkYNYVe5mRSVFEUmO7lOUxlx23WaKr9qKO7z/Tht+dntfvvGTtZxwyh4qKowVi9oy4c5jqKwwTjljF+MeWcWmda0AeH16R5785VFp/ga5r6Cgkvvvn0VBQRX5+VXMnt2Hxx//FPff/zfatKkAoGPHMlas6MK9936W3r138s1vzqVfv+1MnnwKzz47oOZYjz46hdLSAqqqjMpK47bbRqTra6WNbsk5BGY2Cfg8sMXdT07VedKtS4/9XHLtFkZf8En278vjOw+t4dz/VcLLf+7MT27rC8DYB9+j+Mqt/PXxbgAsnd+ecdf2S2PUzU95eR5jx55HWVkB+flV/PSnf2PBgp7ccceFNXW++93ZzJnTC4Bdu1rym98MYfjwDXUeb+zY89m5s9URiT0j5XBLMS+Fx34UKE7h8TNGfgunZesq8vKdVm2q2La5JfNfLqJ6hdwVi9rStef+dIfZzBllZQUAtGhRRYsWjnvt6sWFheUMGrSZN97oDcCOHa15990uVFTk8grHh848sS0bpayl6O6vmlnfVB0/U2zb3JI/TezBH+YsYV9ZHm++2oE3X+tQ835+C+eCy7fx8N19aspOGrKbh6YvZ9vmAh4Z35u177ZJR+jNTl5eFRMmvMRRR+3mxRf7sWJFl5r3hg9fz1tv9aC0tKDR47gb48e/gjtMm3Y806Y1s1a/A5oQInXMbDQwGqA1hWmOJnntiioY/m87+D9nnszunS347sOrOf+ybfz9+egf3JjxH7BkXnuWzWsPwKqlhVwz/FOUlebz6fN2cNdvV3PdOTl7dSGjVFXlMWZMMW3b7ud735vNMcd8zNq1HQE455wPmDHjuISOc/vtF7BtWyFFRWX88IevsG5dB5Yu7Z7CyDNPLl9TTGX3OSHuPtHdh7r70ALLvms0p561i83rWrKjpIDKCuP16Z046bQ9AHzp6x9S1Lmciff0rqlfujufstJ8AOa/XESLFk6HThVpib252rOnJYsXd2fo0E0AdOiwjxNP3Ma8eYkNeG3bFv3nvWNHa/7xj16ceGJJymLNRNX3KeZq9zntSTHbbdnQkgFD9tCqdRXgDD5zJ+tWtab4yq2cdvZO7htz3AHXrjp1K6f6KvUJg/Zgec7O7fnpCb4ZKSoqo23b6Lpuy5YVnHrqJtati1rvZ521jnnzjqK8vPG/Q6tWFbRpU16zP2TIJt5/vyh1gWci98S3LJT27nO2W7GoLa9N7cSvpi6nstJYvayQaU925c/v/JPNG1rywJ/fAWpvvTlr5HY+/5WPqKww9pXl8aMxxxH93yup1KlTGbffPoe8PMcMXnutD/PmRSPN55yzlmeeGXhQ/b1MmPAShYXlVFUZl166ghtuGEmHDvv43vdmA5CfX8UrrxzDwoU9j/j3SbdsbQUmwjxF2dzMngLOBboCm4Fx7v67hj7TIa+zn9Gi+d3zlc3sk/3THYIkYc6KR9hR+uFh/S/cvmNvP/Xs2xKq+9pfvr3Q3YcezvmOtFSOPl+VqmOLSHrlcktR3WcRSY4DlbmbFZUURSRpaimKiMRl6chyInRLjogkranuUzSz981siZktMrMFoayzmc00s5XhZ6dQbmY2wcxWmdliMxsSO86oUH+lmY06nO+mpCgiyfEktsSc5+6DY6PUY4FZ7t4fmBVeA1wM9A/baOBhiJIoMA4YBpwOjKtOpIdCSVFEkmKAVXpC2yG6BJgc9icDl8bKH/PIHKCjmfUERgAz3b3E3bcDMzmMyWiUFEUkaeae0JYAB14ys4VhHgSAHu6+MexvAnqE/V7Authn14ey+soPiQZaRCQ5yXWNu1ZfKwwmuvvE2Ouz3H2DmXUHZprZOwecyt3NjuxYt5KiiCQpqeeatzb0RIu7bwg/t5jZ80TXBDebWU933xi6x1tC9Q1An9jHe4eyDURPz8XLX0k0wIOp+ywiSWuK0Wcza2tm7av3gYuApcAUoHoEeRTwQtifAlwTRqHPAHaEbvYM4CIz6xQGWC4KZYdELUURSV7T3KfYA3jezCDKRU+6+3Qzmw88Y2bXAWuBK0L9qcBIYBVQClwbheIlZnYvMD/Uu8fdD3k+NyVFEUmOczgjy7WHcV8DDKqjfBtwQR3lDtxSz7EmAZMOOyiUFEXkUOTuAy1KiiKSvARvt8lKSooikjwlRRGRwIEcXrhKSVFEkmIk/LRKVlJSFJHkVeVuU1FJUUSSo+6ziMiB1H0WEYlTUhQRqZa9C90nQklRRJKj1fxERA6ka4oiInFKiiIigQNVSooiIoEGWkREDqSkKCISOFCZu4+0KCmKSJIcXElRRKSWus8iIoFGn0VEDqKWoohIjJKiiEjgDpWV6Y4iZZQURSR5aimKiMQoKYqIVHONPouI1HBw3bwtIhKjx/xERAJ3LXEqInIADbSIiNRytRRFRKppklkRkVqaEEJEpJYDrsf8REQC1ySzIiIHcHWfRURicrilaJ5Bo0hm9hGwNt1xpEBXYGu6g5Ck5Orf7Bh373Y4BzCz6US/n0RsdffiwznfkZZRSTFXmdkCdx+a7jgkcfqbNV956Q5ARCSTKCmKiMQoKR4ZE9MdgCRNf7NmStcURURi1FIUEYlRUhQRiVFSTCEzKzazFWa2yszGpjseaZyZTTKzLWa2NN2xSHooKaaImeUDvwYuBgYCV5nZwPRGJQl4FMiqm42laSkpps7pwCp3X+Pu+4GngUvSHJM0wt1fBUrSHYekj5Ji6vQC1sVerw9lIpLBlBRFRGKUFFNnA9An9rp3KBORDKakmDrzgf5mdqyZtQSuBKakOSYRaYSSYoq4ewUwBpgBvA084+7L0huVNMbMngLeAE40s/Vmdl26Y5IjS4/5iYjEqKUoIhKjpCgiEqOkKCISo6QoIhKjpCgiEqOkmEXMrNLMFpnZUjP7o5kVHsaxHjWzL4T9RxqarMLMzjWzzxzCOd43s39Z9a2+8oPq7E7yXN83s9uTjVHkYEqK2WWvuw9295OB/cCN8TfN7JDW8Xb36919eQNVzgWSTooi2UhJMXu9BvQLrbjXzGwKsNzM8s3sfjObb2aLzewGAIv8Kszv+Dege/WBzOwVMxsa9ovN7E0ze8vMZplZX6Lk+43QSv2smXUzs2fDOeab2Znhs13M7CUzW2ZmjwDW2Jcwsz+b2cLwmdEHvfdAKJ9lZt1C2fFmNj185jUzG9Akv02R4JBaFpJeoUV4MTA9FA0BTnb390Ji2eHunzazVsDrZvYScCpwItHcjj2A5cCkg47bDfgtcHY4Vmd3LzGz3wC73f2nod6TwAPuPtvMjiZ6auckYBww293vMbPPAYk8DfKf4RxtgPlm9qy7bwPaAgvc/Rtmdlc49hiiBaVudPeVZjYMeAg4/xB+jSJ1UlLMLm3MbFHYfw34HVG3dp67vxfKLwJOqb5eCBQB/YGzgafcvRL40Mz+XsfxzwBerT6Wu9c3r+CFwECzmoZgBzNrF85xefjsX81sewLf6VYzuyzs9wmxbgOqgP8O5Y8Dz4VzfAb4Y+zcrRI4h0jClBSzy153HxwvCMlhT7wI+Jq7zzio3sgmjCMPOMPdy+qIJWFmdi5Rgh3u7qVm9grQup7qHs778cG/A5GmpGuKuWcGcJOZFQCY2Qlm1hZ4FfhiuObYEzivjs/OAc42s2PDZzuH8l1A+1i9l4CvVb8ws8Fh91Xg6lB2MdCpkViLgO0hIQ4gaqlWywOqW7tXE3XLdwLvmdl/hHOYmQ1q5BwiSVFSzD2PEF0vfDMsvvT/iHoEzwMrw3uPEc0EcwB3/wgYTdRVfYva7utfgMuqB1qAW4GhYSBnObWj4HcTJdVlRN3oDxqJdTrQwszeBu4jSsrV9gCnh+9wPnBPKP8ScF2Ibxla4kGamGbJERGJUUtRRCRGSVFEJEZJUUQkRklRRCRGSVFEJEZJUUQkRklRRCTm/wOfPofjOyh/DgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_pred_proba = clf.predict_proba(vect_X_test)\n",
    "test_y_predict_prob_class_1 = test_pred_proba[:,1]\n",
    "test_y_predict_class = [1 if test_y_predict_prob_class_1[i] > 0.96 else 0 for i in range(len(test_y_predict_prob_class_1))]\n",
    "f.scoring(y_test, test_y_predict_class, clf, vect_X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_FN_pred_prob = [test_y_predict_prob_class_1[i] if test_y_predict_prob_class_1[i] < 0.96 else 0 for i in range(len(test_y_predict_prob_class_1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAE/CAYAAABB8mpwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcP0lEQVR4nO3deZxlZX3n8c83IMQFBaVlEFpx6ahoZpT0AJnEiEuwwcQmE6OQRNBgSFyy6RjRJANxSXASNXFiMBh7ADcgRqUTMEgQQ/SVJjQuCKihxUa62RqaTYkL+ps/zlN4u6itq6u7up76vF+v+6p7n/Occ57z1Kn7Pefc555KVSFJkvr1I/PdAEmStH0Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMey16SV6a5DOznPewJBummP6eJH80Ud0kVyU5bDbrnUG7Tk7ygfb80Um+mWSXOVr2pNs0B8t+RpKvztXydlZJKskT5rsdWjwMe82pJOuT/GcLl7HHo5Ic0N7gzh9X/wNJTp6n5m53VfWbVfXmSaY9pao+DVuG83Zowzeq6iFV9f2p6s30oGeqbdpa40Ovqv61qp44F8ueZr2zPsCbYFnrkzx3LpY1wbLH/m523R7L39Hr0fwx7LU9/HwLl7HHDSPTDknyP3ZkY3wDmztzdXVA0o5l2GtH+z/AWyeakGTvJP+Y5I4km5P8a5IJ99F2FvLbSa5NcmuSPxur287aPpvknUluA05O8rAkZybZlOS6JH84btlJ8ldJ7kzylSTPGZnwsiRfTnJ3W99vTNCeN7Z2rE/yKyPlpyd5yyTbsD7Jc5OsAN4IvLhdCflikl9Kcvm4+q9Jcu4ky3pskn9pbbwQ2Htk2hZnba1/rm11v57kV5I8GXgP8JOtDXeMtP/UJOcn+RbwrIm2aYrt/3SSl4+8vu+MOsklrfiLbZ0vHv+xQJInt2XckeFjjxeM69t3JzmvbculSR4/Uf+Ma+tk27p7kj9P8o0kN2f4uOKBbdqE+2aS9wOPBv6hLev3J1nn65LcmOSGJL82btrzk3w+yV1Jrs+WV7rG+uiOtvyfTPL4JJ9Kclvr8w8m2XNkea9PsrH1yVfH9uXW3hOTfK3Ne06Sh0+2nun6UQtMVfnwMWcPYD3w3AnKDwAK2APYOFYH+ABwcnv+pwxvwg9oj2cAmWQ9BVwMPJzhzfY/gJe3aS8F7gV+C9gVeCBwJnBuW/8Brf7x4+r/Xlvvi4E7gYe36c8HHg8EeCZwD3BQm3ZYm/cdwO5t+reAJ7bppwNvGam7YaK+Ak4GPjAybXdgM/DkkbLPA784SX/820gbfga4e2x5I32/K/Bg4K6R9u0LPGWkHz4zbrmnt774KYaTgx+dYJum2v5Pj/1eJlpHa9cTRl7f10ftd7GO4UBoN+DZbbtG+/Y24OC2bR8EzprhfjrRtr4TWM2wT+0B/APwp9Ptm0yyz48sdwVwM/DU1v8fGt3uts0/3vr3v7a6R43/3Y0s7wnAz7b+XsIQ1H/Rpj0RuB541Mj8j2/PfwdYA+zf5v0b4MOTrcdHXw/P7LU9fLydAd2R5OPjpv0nw5n9RGe732MIn8dU1fdq+Px2qn/e8Laq2lxV3wD+AjhmZNoNVfV/q+pe4LvA0cAbquruqloPvB14yUj9WxjeML9XVWcDX2UIearqvKr6Wg3+Bfgkw5v9qD+qqu+06ecBL5qi3dOqqu8AZwO/CpDkKQxvyP84vm6SRwP/faQNlzAE1WR+ADw1yQOr6saqumqa5pxbVZ+tqh9U1bcnqTOn298cCjwEOKWqvltVn2LY/tHf88eq6t/b7/mDwNNms6IkAU4Afq/tU3cDf8Kw38DW75ujXgT8v6q6sqq+xXBgd5+q+nRVfan17xXAhxkOmiZUVeuq6sLW35sYDrTG6n+fIcgPTPKAqlpfVV9r034T+IOq2tD2r5OBF8aPuRYFw17bw1FVtWd7HDXB9L8F9kny8+PK/4zhTO6T7TLzidOs5/qR59cBj5pk2t4MZ2PXjau/38jrjePevO9bXpIjkqxpl2/vAI5k5DI5cHt7E5+sLbN1BvDLLYheApzT3qTHe9QkbbifVufFDG/8N7ZL4E+aph3XTzN9e23/o4Drq+oH45Y9+nu7aeT5PQwHB7OxBHgQcPnYgSrwT60ctn7fHPUo7r+v3ifJIUkuzvAR050Mv5vR/Ytx9fdJcla7VH8Xw9WxvWE4EAB+lyHIb2n1xn4XjwE+NrJ9X2Y4ONhnK7ZFC5Rhrx2uqr4L/DHwZoZL42Pld1fVa6vqccALgNdk5LPzCSwdef5oYHQg4Ghw38pwZvaYcfU3jrzer4XqFstLsjvw98CfA/tU1Z7A+aPtBvZK8uAp2jIT9ztLrKo1DFclngH8MvD+Sea9cZI2TLyiqguq6mcZzlS/Arx3sjZMUz5mqu3/FkOIjvkv0yxr1A3A0mw5tmL87222xm/TrQxXnZ4ycqD6sKp6CEy7b07XPzdy/3111IcYPj5YWlUPY/i4YGz/mmjZf9LKf7yqHspw9Wf07+hDVfXTDPt7AW9rk64HjhjZvj2r6kerauMMtkELnGGv+fJ+hs9/V4wVJPm5JE9ooXsnw1nHDyaZH+B1SfZKspTh88izJ6pUw1fOzgHemmSPJI8BXsNwRjTmkcBvJ3lAkl8CnswQ6rsxXBbdBNyb5Ajg8AlW88dJdkvyDODngL+bvgu2cDNwQO4/IPFM4K+A71XVhF8Vq6rrgLUjbfhpYPxVE+C+s8KVLZy/A3yTH/bxzcD+SXbbyrbD5Nv/BeB/JnlQhq/YHT9uvpuBx02yzEsZztZ/v/1eDmvbddZMGpRhYN/Jk0zeYlvb1YP3Au9M8sg2/35JnteeT7VvTrUNMOx7L01yYJIHASeNm74HsLmqvp3kYIYDuzGb2noeN67+N4E7k+wHvG5km5+Y5NntIPXbDAcwY+18D8PfwGNa3SVJVk6xHnXEsNe8aAH8vxkGQ41ZBvwzwxvZvwF/XVUXT7GYc4HLGQLlPOB9U9T9LYazzGuBzzCcTa0amX5pW/+tDGMKXlhVt7XPbn+b4Q37doY34tXjln1Tm3YDw+fGv1lVX5miLRMZC8fbknxupPz9DAO7pvsO/i8DhzAM6juJ4SBhIj/CcKBzQ6v7TOAVbdqngKuAm5LcuhVtn2r738lwdeJmho8lPjhu3pOBM9ql5S0+529XgH4eOILh9/LXwLFb0bdLgc9OMm2ibX09w6X6Ne3y+D8zDHiDqffNPwX+sG3D/xq/oqr6BMOYkk+15X9qXJVXAm9KcjfD38Q5I/Pew7A/frYt/1CGq2IHMRx0nAd8dGRZuwOnMPTXTQwHsW9o0/6SYd/9ZFvXGoZ9ZrL1qCNjo0mlBSVJAcvaZ5TdyvDVr1sYRv9fM9/tWSiS7M8wxmGH3tNB2lk5ClPaub0CuMyg3zpVtQEw6KVm2sv4SZa2kaJXZ7ipxe+08pPbaNAvtMeRI/O8Icm6DDd0eN5I+YpWtm50NGuGG4Jc2srPnuVnhlJXkqxnGIvw2nluiqQFbtrL+En2Bfatqs8l2YPhM9KjGL47+s2q+vNx9Q9k+J7owQxfOfln4Mfa5P9guBnEBuAy4JiqujrJOcBHq+qsJO8BvlhVp87RNkqStKhNe2bfbrrxufb8bobvZu43xSwrGe5i9Z2q+jrDgJSD22NdVV3bBt6cBaxso1ufDXykzX8Gw8GEJEmaA1s1Gj/JAcDTGUYuA7w6yRVJViXZq5Xtx5Y3kNjQyiYrfwRwR7sD1mi5JEmaAzMeoJfkIQw3F/ndqroryakMN0Wp9vPtwK9NsYhtluQEhlta8uAHP/gnnvSk6W78JUlSHy6//PJbq2rJ9DXvb0Zhn+QBDEH/war6KEBV3Twy/b388J7dG9nyblH788M7Xk1UfhuwZ5Jd29n9aP0tVNVpwGkAy5cvr7Vr186k+ZIkLXhJJrwN9kzMZDR+GG5W8uWqesdI+b4j1X4BuLI9Xw0cneHfRT6W4WYU/84wIG9ZG3m/G8M/mFjd7kd+MfDCNv9xDDdLkSRJc2AmZ/Y/xfBPOL6U5Aut7I3AMUmexnAZfz3wGwBVdVUbXX81w7++fFW7WxpJXg1cAOwCrBr5b1uvB87K8D+yP8/Ud0KTJElbYcHeQc/L+JKkxSTJ5VW1fDbzem98SZI6193tcg848bx5We/6U54/L+uVJGk6ntlLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlz04Z9kqVJLk5ydZKrkvxOK394kguTXNN+7tXKk+RdSdYluSLJQSPLOq7VvybJcSPlP5HkS22edyXJ9thYSZIWo5mc2d8LvLaqDgQOBV6V5EDgROCiqloGXNReAxwBLGuPE4BTYTg4AE4CDgEOBk4aO0BodX59ZL4V275pkiQJZhD2VXVjVX2uPb8b+DKwH7ASOKNVOwM4qj1fCZxZgzXAnkn2BZ4HXFhVm6vqduBCYEWb9tCqWlNVBZw5sixJkrSNtuoz+yQHAE8HLgX2qaob26SbgH3a8/2A60dm29DKpirfMEH5ROs/IcnaJGs3bdq0NU2XJGnRmnHYJ3kI8PfA71bVXaPT2hl5zXHb7qeqTquq5VW1fMmSJdt7dZIkdWFGYZ/kAQxB/8Gq+mgrvrldgqf9vKWVbwSWjsy+fyubqnz/CcolSdIcmMlo/ADvA75cVe8YmbQaGBtRfxxw7kj5sW1U/qHAne1y/wXA4Un2agPzDgcuaNPuSnJoW9exI8uSJEnbaNcZ1Pkp4CXAl5J8oZW9ETgFOCfJ8cB1wIvatPOBI4F1wD3AywCqanOSNwOXtXpvqqrN7fkrgdOBBwKfaA9JkjQHpg37qvoMMNn33p8zQf0CXjXJslYBqyYoXws8dbq2SJKkrecd9CRJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzk0b9klWJbklyZUjZScn2ZjkC+1x5Mi0NyRZl+SrSZ43Ur6ila1LcuJI+WOTXNrKz06y21xuoCRJi91MzuxPB1ZMUP7Oqnpae5wPkORA4GjgKW2ev06yS5JdgHcDRwAHAse0ugBva8t6AnA7cPy2bJAkSdrStGFfVZcAm2e4vJXAWVX1nar6OrAOOLg91lXVtVX1XeAsYGWSAM8GPtLmPwM4aus2QZIkTWVbPrN/dZIr2mX+vVrZfsD1I3U2tLLJyh8B3FFV944rlyRJc2S2YX8q8HjgacCNwNvnqkFTSXJCkrVJ1m7atGlHrFKSpAVvVmFfVTdX1fer6gfAexku0wNsBJaOVN2/lU1WfhuwZ5Jdx5VPtt7Tqmp5VS1fsmTJbJouSdKiM6uwT7LvyMtfAMZG6q8Gjk6ye5LHAsuAfwcuA5a1kfe7MQziW11VBVwMvLDNfxxw7mzaJEmSJrbrdBWSfBg4DNg7yQbgJOCwJE8DClgP/AZAVV2V5BzgauBe4FVV9f22nFcDFwC7AKuq6qq2itcDZyV5C/B54H1ztXGSJGkGYV9Vx0xQPGkgV9VbgbdOUH4+cP4E5dfyw48BJEnSHPMOepIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS56YN+ySrktyS5MqRsocnuTDJNe3nXq08Sd6VZF2SK5IcNDLPca3+NUmOGyn/iSRfavO8K0nmeiMlSVrMZnJmfzqwYlzZicBFVbUMuKi9BjgCWNYeJwCnwnBwAJwEHAIcDJw0doDQ6vz6yHzj1yVJkrbBtGFfVZcAm8cVrwTOaM/PAI4aKT+zBmuAPZPsCzwPuLCqNlfV7cCFwIo27aFVtaaqCjhzZFmSJGkOzPYz+32q6sb2/CZgn/Z8P+D6kXobWtlU5RsmKJckSXNkmwfotTPymoO2TCvJCUnWJlm7adOmHbFKSZIWvNmG/c3tEjzt5y2tfCOwdKTe/q1sqvL9JyifUFWdVlXLq2r5kiVLZtl0SZIWl9mG/WpgbET9ccC5I+XHtlH5hwJ3tsv9FwCHJ9mrDcw7HLigTbsryaFtFP6xI8uSJElzYNfpKiT5MHAYsHeSDQyj6k8BzklyPHAd8KJW/XzgSGAdcA/wMoCq2pzkzcBlrd6bqmps0N8rGUb8PxD4RHtIkqQ5Mm3YV9Uxk0x6zgR1C3jVJMtZBayaoHwt8NTp2iFJkmbHO+hJktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6tw2hX2S9Um+lOQLSda2socnuTDJNe3nXq08Sd6VZF2SK5IcNLKc41r9a5Ict22bJEmSRs3Fmf2zquppVbW8vT4RuKiqlgEXtdcARwDL2uME4FQYDg6Ak4BDgIOBk8YOECRJ0rbbHpfxVwJntOdnAEeNlJ9ZgzXAnkn2BZ4HXFhVm6vqduBCYMV2aJckSYvStoZ9AZ9McnmSE1rZPlV1Y3t+E7BPe74fcP3IvBta2WTl95PkhCRrk6zdtGnTNjZdkqTFYddtnP+nq2pjkkcCFyb5yujEqqoktY3rGF3eacBpAMuXL5+z5UqS1LNtOrOvqo3t5y3Axxg+c7+5XZ6n/bylVd8ILB2Zff9WNlm5JEmaA7MO+yQPTrLH2HPgcOBKYDUwNqL+OODc9nw1cGwblX8ocGe73H8BcHiSvdrAvMNbmSRJmgPbchl/H+BjScaW86Gq+qcklwHnJDkeuA54Uat/PnAksA64B3gZQFVtTvJm4LJW701VtXkb2iVJkkbMOuyr6lrgv01QfhvwnAnKC3jVJMtaBayabVskSdLkvIOeJEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1DnDXpKkzhn2kiR1zrCXJKlzhr0kSZ0z7CVJ6pxhL0lS5wx7SZI6Z9hLktQ5w16SpM4Z9pIkdc6wlySpc4a9JEmdM+wlSeqcYS9JUucMe0mSOmfYS5LUOcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjpn2EuS1Lld57sBi9kBJ563w9e5/pTn7/B1SpLml2f2kiR1zrCXJKlzhr0kSZ0z7CVJ6pwD9DSt+RhICA4mlKS5YthLkhY8v900NcNeXfEPXpo7XtXrx04T9klWAH8J7AL8bVWdMs9NkrabhfQm6gGUtPDtFGGfZBfg3cDPAhuAy5Ksrqqr57dlkhYKD6Cm5gHU4razjMY/GFhXVddW1XeBs4CV89wmSZK6sLOE/X7A9SOvN7QySZK0jVJV890GkrwQWFFVL2+vXwIcUlWvHlfvBOCE9vKJwFfnuCl7A7fO8TIXO/t07tmnc88+nXv26dx7YlXtMZsZd4rP7IGNwNKR1/u3si1U1WnAadurEUnWVtXy7bX8xcg+nXv26dyzT+eefTr3kqyd7bw7y2X8y4BlSR6bZDfgaGD1PLdJkqQu7BRn9lV1b5JXAxcwfPVuVVVdNc/NkiSpCztF2ANU1fnA+fPcjO32EcEiZp/OPft07tmnc88+nXuz7tOdYoCeJEnafnaWz+wlSdJ2sijDPsmKJF9Nsi7JiRNM3z3J2W36pUkOmIdmLigz6NPXJLk6yRVJLkrymPlo50IyXZ+O1PvFJJXEkc/TmEmfJnlR21evSvKhHd3GhWYGf/uPTnJxks+3v/8j56OdC0WSVUluSXLlJNOT5F2tv69IctCMFlxVi+rBMADwa8DjgN2ALwIHjqvzSuA97fnRwNnz3e6d+THDPn0W8KD2/BX26bb3aau3B3AJsAZYPt/t3pkfM9xPlwGfB/Zqrx853+3emR8z7NPTgFe05wcC6+e73TvzA/gZ4CDgykmmHwl8AghwKHDpTJa7GM/sZ3Jr3pXAGe35R4DnJMkObONCM22fVtXFVXVPe7mG4V4KmtxMbyH9ZuBtwLd3ZOMWqJn06a8D766q2wGq6pYd3MaFZiZ9WsBD2/OHATfswPYtOFV1CbB5iiorgTNrsAbYM8m+0y13MYb9TG7Ne1+dqroXuBN4xA5p3cK0tbc7Pp7hyFSTm7ZP2+W7pVU1P/8BZuGZyX76Y8CPJflskjXtv3FqcjPp05OBX02ygeEbV7+1Y5rWrVndXn6n+eqdFockvwosB545321ZyJL8CPAO4KXz3JTe7MpwKf8whqtPlyT58aq6Yz4btcAdA5xeVW9P8pPA+5M8tap+MN8NW0wW45n9TG7Ne1+dJLsyXHq6bYe0bmGa0e2OkzwX+APgBVX1nR3UtoVquj7dA3gq8Okk6xk+u1vtIL0pzWQ/3QCsrqrvVdXXgf9gCH9NbCZ9ejxwDkBV/Rvwowz3zdfszOj9drzFGPYzuTXvauC49vyFwKeqjYzQhKbt0yRPB/6GIej9HHR6U/ZpVd1ZVXtX1QFVdQDDOIgXVNWs7529CMzkb//jDGf1JNmb4bL+tTuwjQvNTPr0G8BzAJI8mSHsN+3QVvZlNXBsG5V/KHBnVd043UyL7jJ+TXJr3iRvAtZW1WrgfQyXmtYxDJQ4ev5avPObYZ/+GfAQ4O/aWMdvVNUL5q3RO7kZ9qm2wgz79ALg8CRXA98HXldVXtWbxAz79LXAe5P8HsNgvZd68jS5JB9mOODcu41zOAl4AEBVvYdh3MORwDrgHuBlM1qufS5JUt8W42V8SZIWFcNekqTOGfaSJHXOsJckqXOGvSRJnTPsJUnqnGEvSVLnDHtJkjr3/wHhoALyVyTYwQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (8, 5))\n",
    "plt.hist(test_FN_pred_prob, bins=10, range = [0,1], rwidth=0.7, align='left')\n",
    "plt.xlim(-0.05,1)\n",
    "plt.ylim(0, 25000)\n",
    "plt.title('FNs probability distribution, test dataset')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}