VanTineEtAl_Fig5_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# To analyze and classify the harmonized dataset\n",
    "* note that some classifiers may require using a computer cluster due to memory requirements\n",
    "* These package versions were used for the manuscript: scikit-learn v1.3.2, catboost v1.2.3, xgboost v2.0.3, lightgbm v4.3.0, joblib v1.3.2, optuna v3.5.0, numpy v1.23.4, pandas v2.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import copy\n",
    "#import chart_studio.plotly as py\n",
    "#import plotly.express as px\n",
    "#import plotly.graph_objects as go\n",
    "#from matplotlib import colors\n",
    "import os\n",
    "import json\n",
    "import sys\n",
    "from numpy import ma\n",
    "import seaborn as sns\n",
    "import logging\n",
    "#pd.set_option('display.max_columns',200)\n",
    "#import plotnine as p9\n",
    "#from plotnine import *\n",
    "#from plotnine.data import *\n",
    "\n",
    "from scipy import stats\n",
    "from scipy.stats import zscore\n",
    "#from natsort import natsorted\n",
    "from glob import glob\n",
    "import glob\n",
    "from statistics import mean\n",
    "#import shutil\n",
    "import random\n",
    "import copy\n",
    "#from scipy.stats import shapiro, skew\n",
    "\n",
    "#import lime\n",
    "#import lime.lime_tabular\n",
    "\n",
    "#logger = logging.getLogger(__name__)\n",
    "#logger.setLevel(logging.INFO)\n",
    "#logging.basicConfig()\n",
    "\n",
    "import joblib\n",
    "\n",
    "#import sklearn.datasets\n",
    "\n",
    "#from sklearn.preprocessing import FunctionTransformer\n",
    "#from sklearn.metrics import plot_roc_curve\n",
    "#from sklearn.inspection import plot_partial_dependence, permutation_importance\n",
    "#from sklearn.metrics import RocCurveDisplay, plot_confusion_matrix, plot_precision_recall_curve, roc_curve\n",
    "\n",
    "#import PipelineProfiler\n",
    "from joblib import dump, load\n",
    "import scikitplot as skplt\n",
    "#from sklearn.metrics import classification_report\n",
    "import optuna\n",
    "import scipy.stats as stats\n",
    "import math\n",
    "import random\n",
    "from catboost import CatBoostClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "#from scipy.stats import shapiro, skew\n",
    "from sklearn import tree\n",
    "from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "#from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier, AdaBoostClassifier, RandomForestClassifier, StackingClassifier, VotingClassifier, BaggingClassifier, ExtraTreesClassifier, GradientBoostingClassifier\n",
    "from sklearn.feature_selection import RFECV, RFE, SelectKBest, chi2\n",
    "from sklearn.gaussian_process import GaussianProcessClassifier\n",
    "from sklearn.linear_model import RidgeClassifier, SGDClassifier, ElasticNetCV, LassoLarsCV, ElasticNet, LogisticRegression\n",
    "import sklearn.metrics\n",
    "from sklearn.metrics import balanced_accuracy_score, accuracy_score, f1_score, matthews_corrcoef, mean_squared_error, r2_score, mean_absolute_error, cohen_kappa_score, make_scorer\n",
    "from sklearn.metrics import classification_report, roc_auc_score, auc, RocCurveDisplay, roc_curve\n",
    "import sklearn.model_selection\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score, StratifiedKFold, KFold, train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import RobustScaler, StandardScaler, QuantileTransformer, FunctionTransformer, MinMaxScaler, PowerTransformer, OrdinalEncoder, OneHotEncoder, LabelEncoder\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "\n",
    "# Hide warnings \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "#plotly_template = 'simple_white'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set working directory\n",
    "\n",
    "os.chdir('/Users/jasonheld/Manuscripts/2022_Sarcoma-Exosomes/vanTine_001_FinalDataWithControls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make some lists to fill later\n",
    "\n",
    "dfTrainModelEvalAllSeeds = pd.DataFrame()\n",
    "seedListStats = []\n",
    "trainFracListStats = []\n",
    "finalClassifierList = []\n",
    "tunedClassifierListForValidation = []\n",
    "finalImportancesMeanList = []\n",
    "finalFeaturesList = []\n",
    "\n",
    "y_testList = []\n",
    "y_predList = []\n",
    "y_probList = []\n",
    "seedList = []\n",
    "seedListStats = []\n",
    "trainFracListStats = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# key parameters\n",
    "\n",
    "# adjust based on whether classifying cancer or drug sensitivity\n",
    "target = 'sensitivity' # 'sensitivity', 'cancer',\n",
    "\n",
    "seed = 7 # use seeds 1-7 for each classification to randomize 7 times\n",
    "np.random.seed(7) # match the seed value\n",
    "\n",
    "quantType = 'Intensity_Raw' # 'Intensity_Raw', 'areaHarmonized', 'NormalizedIntensity_Log2'\n",
    "trainFrac = .75\n",
    "nJobs = 6\n",
    "\n",
    "if target ==  'sensitivity':\n",
    "    nSamplesVal = 30\n",
    "if target ==  'cancer':\n",
    "    nSamplesVal = 35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function takes a dataframe and a target variable to create an X (predictors) dataframe and a y Series\n",
    "\n",
    "def X_y_split(df, target):\n",
    "    \n",
    "    categorical_features = []\n",
    "    continuous_features = []\n",
    "    binary_features = []\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if df[col].dtype == object:\n",
    "            categorical_features.append(col)\n",
    "        \n",
    "        else:\n",
    "            if df[col].nunique() <= 2:\n",
    "                binary_features.append(col)\n",
    "            else:\n",
    "                continuous_features.append(col)\n",
    "                \n",
    "    if categorical_features: # if this list isn't empty\n",
    "        if target in categorical_features:\n",
    "            categorical_features.remove(target)\n",
    "        df.drop(categorical_features, axis = 1, inplace = True)\n",
    "\n",
    "    X, y = df.drop([target], axis = 1), df[target]\n",
    "    \n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function parses columns by type.\n",
    "\n",
    "def columns_catNumOrBin(df):\n",
    "\n",
    "    categorical_features = []\n",
    "    continuous_features = []\n",
    "    binary_features = []\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if df[col].dtype == object:\n",
    "            categorical_features.append(col)\n",
    "        else:\n",
    "            if df[col].nunique() <= 2:\n",
    "                binary_features.append(col)\n",
    "            else:\n",
    "                continuous_features.append(col)\n",
    "    \n",
    "    return categorical_features,continuous_features, binary_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# impute 0s\n",
    "\n",
    "def imputeWideDFMinOr0(df):\n",
    "    \n",
    "    for col in df.columns:\n",
    "\n",
    "        if df[col].dtype == object:\n",
    "            continue\n",
    "        \n",
    "        else:\n",
    "        \n",
    "            if quantType == 'NormalizedIntensity_Log2':\n",
    "                df[col].fillna(value = df[col].min(), inplace = True)\n",
    "            \n",
    "            if quantType == 'Intensity_Raw':\n",
    "                df[col].fillna(value = df[col].min(), inplace = True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# make pipeline\n",
    "\n",
    "pipeline = Pipeline(steps=[\n",
    "    ('Transforming Distribution',  RobustScaler(with_centering = True, with_scaling = True, quantile_range=(25, 75), unit_variance = False)),\n",
    "    ('Standard Scaler', StandardScaler()),\n",
    "#    ('RFE', RFECV(estimator=RandomForestClassifier(random_state=seed), cv=cv, scoring=make_scorer(quadratic_weighted_kappa), n_jobs = nJobs)),\n",
    "    ('Model', None),\n",
    "]#, verbose=True,\n",
    ")\n",
    "\n",
    "pipeline"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# temp. for testing\n",
    "\n",
    "dfTrain = pd.read_excel('dfTrain_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "dfTest = pd.read_excel('dfTest_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "\n",
    "X, y = X_y_split(dfTrain, target)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# load key variables if on disk\n",
    "\n",
    "dfTrain = pd.read_excel('dfTrain_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "#dfTest = pd.read_excel('dfTest_target-' + target + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "\n",
    "# RFECV\n",
    "fileName = 'selectedFeaturesRFECV_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "with open(fileName + '.json', 'r') as f:\n",
    "    selected_featuresFromDisk = json.load(f)\n",
    "\n",
    "print (\"There are \" + str(len(selected_featuresFromDisk)) + \" selected features\")\n",
    "\n",
    "# filter dfTrain with RFE list\n",
    "colsToKeep = []\n",
    "colsToKeep = copy.deepcopy(selected_featuresFromDisk)\n",
    "colsToKeep.append(target)\n",
    "#print(colsToKeep) # QC\n",
    "dfTrain = dfTrain[colsToKeep] # selected features from RFE\n",
    "\n",
    "X, y = X_y_split(dfTrain, target)\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "y = pd.Series(le.fit_transform(y))\n",
    "categorical_features,continuous_features, binary_features = columns_catNumOrBin(X)\n",
    "\n",
    "imputeWideDFMinOr0(X)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# can move to cell above once all are saved\n",
    "dfML = pd.read_excel('dfML_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_seed-' + str(seed) + '.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load sarcoma cohort results\n",
    "# use ProteinQuantitationWithSampleMetadata.xlsx from the Gitub repo\n",
    "\n",
    "dfSarcomaRaw = pd.read_excel('/Users/jasonheld/Manuscripts/2022_Sarcoma-Exosomes/vanTine_001_FinalDataWithControls/MQ/vanTine_001_90_samples/Protein/Model/Protein_pVals_Final_WithMetaData.xlsx', sheet_name='Sheet1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tidyDFSarcomaRawAndChooseQuantType (df, quantType_):\n",
    "\n",
    "    global quantType\n",
    "    quantType = quantType_\n",
    "    \n",
    "    df = df.loc[:,~df.columns.str.contains('CON_')] # remove contaminants\n",
    "    df = df.query(\"dataType == @quantType\")\n",
    "    \n",
    "    totalCols = df.columns\n",
    "    numCols = df._get_numeric_data().columns\n",
    "    catCols = list(set(totalCols)-set(numCols))\n",
    "    \n",
    "    df = pd.melt(df,\n",
    "        id_vars = catCols,\n",
    "        value_vars = numCols,\n",
    "        var_name = 'protAcc',\n",
    "        value_name=\"area\",\n",
    "        ignore_index=False, \n",
    "        )\n",
    "    \n",
    "    #remove rows without areas & duplicates\n",
    "    df.dropna(subset=['area'], inplace=True)\n",
    "    df.drop_duplicates(inplace = True)\n",
    "    \n",
    "    # add a column\n",
    "    dataset = 'sarcoma'\n",
    "    df.insert(loc=1, column='dataset', value=dataset)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tidy the dataframe\n",
    "\n",
    "dfSarcoma = tidyDFSarcomaRawAndChooseQuantType(dfSarcomaRaw, quantType)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# works! But, can be slow so only download when needed\n",
    "\n",
    "import re\n",
    "import requests\n",
    "from requests.adapters import HTTPAdapter, Retry\n",
    "\n",
    "re_next_link = re.compile(r'<(.+)>; rel=\"next\"')\n",
    "retries = Retry(total=5, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504])\n",
    "session = requests.Session()\n",
    "session.mount(\"https://\", HTTPAdapter(max_retries=retries))\n",
    "\n",
    "def get_next_link(headers):\n",
    "    if \"Link\" in headers:\n",
    "        match = re_next_link.match(headers[\"Link\"])\n",
    "        if match:\n",
    "            return match.group(1)\n",
    "\n",
    "def get_batch(batch_url):\n",
    "    while batch_url:\n",
    "        response = session.get(batch_url)\n",
    "        response.raise_for_status()\n",
    "        total = response.headers[\"x-total-results\"]\n",
    "        yield response, total\n",
    "        batch_url = get_next_link(response.headers)\n",
    "\n",
    "\n",
    "url = 'https://rest.uniprot.org/uniprotkb/search?fields=accession%2Creviewed%2Cid%2Cprotein_name%2Clength%2Cgene_primary%2Cgene_names%2Corganism_name&format=tsv&query=%28%28organism_id%3A9606%29+AND+%28reviewed%3Atrue%29%29&size=500'\n",
    "progress = 0\n",
    "with open('uniprotDownload.tsv', 'w') as f:\n",
    "    for batch, total in get_batch(url):\n",
    "        lines = batch.text.splitlines()\n",
    "        if not progress:\n",
    "            print(lines[0], file=f)\n",
    "        for line in lines[1:]:\n",
    "            print(line, file=f)\n",
    "        progress += len(lines[1:])\n",
    "        print(f'{progress} / {total}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>protAcc</th>\n",
       "      <th>Reviewed</th>\n",
       "      <th>Protein names</th>\n",
       "      <th>geneNamePrimary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A0A0C5B5G6</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Mitochondrial-derived peptide MOTS-c (Mitochon...</td>\n",
       "      <td>MT-RNR1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A0A1B0GTW7</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Ciliated left-right organizer metallopeptidase...</td>\n",
       "      <td>CIROP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A0JNW5</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Bridge-like lipid transfer protein family memb...</td>\n",
       "      <td>BLTP3B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A0JP26</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>POTE ankyrin domain family member B3</td>\n",
       "      <td>POTEB3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A0PK11</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Clarin-2</td>\n",
       "      <td>CLRN2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20428</th>\n",
       "      <td>Q9H2U6</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Putative uncharacterized protein encoded by LI...</td>\n",
       "      <td>LINC00597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20429</th>\n",
       "      <td>Q9H379</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Putative uncharacterized protein PRO3102</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20430</th>\n",
       "      <td>Q9P1C3</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Putative uncharacterized protein PRO2829</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20431</th>\n",
       "      <td>Q9UFV3</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Putative uncharacterized protein DKFZp434L187</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20432</th>\n",
       "      <td>Q9Y6C7</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Putative uncharacterized protein encoded by LI...</td>\n",
       "      <td>LINC00312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20433 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          protAcc  Reviewed  \\\n",
       "0      A0A0C5B5G6  reviewed   \n",
       "1      A0A1B0GTW7  reviewed   \n",
       "2          A0JNW5  reviewed   \n",
       "3          A0JP26  reviewed   \n",
       "4          A0PK11  reviewed   \n",
       "...           ...       ...   \n",
       "20428      Q9H2U6  reviewed   \n",
       "20429      Q9H379  reviewed   \n",
       "20430      Q9P1C3  reviewed   \n",
       "20431      Q9UFV3  reviewed   \n",
       "20432      Q9Y6C7  reviewed   \n",
       "\n",
       "                                           Protein names geneNamePrimary  \n",
       "0      Mitochondrial-derived peptide MOTS-c (Mitochon...         MT-RNR1  \n",
       "1      Ciliated left-right organizer metallopeptidase...           CIROP  \n",
       "2      Bridge-like lipid transfer protein family memb...          BLTP3B  \n",
       "3                   POTE ankyrin domain family member B3          POTEB3  \n",
       "4                                               Clarin-2           CLRN2  \n",
       "...                                                  ...             ...  \n",
       "20428  Putative uncharacterized protein encoded by LI...       LINC00597  \n",
       "20429           Putative uncharacterized protein PRO3102             NaN  \n",
       "20430           Putative uncharacterized protein PRO2829             NaN  \n",
       "20431      Putative uncharacterized protein DKFZp434L187             NaN  \n",
       "20432  Putative uncharacterized protein encoded by LI...       LINC00312  \n",
       "\n",
       "[20433 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make a uniprot df for annotation\n",
    "# use uniprotAnnotationsForFigure2.xlxs on the Github repor\n",
    "\n",
    "dfUniprot = pd.read_csv('uniprotDownload.tsv', delimiter = '\\t')\n",
    "\n",
    "dfUniprot = dfUniprot.rename(columns={'Entry' : 'protAcc'})\n",
    "dfUniprot = dfUniprot.rename(columns={'Gene Names (primary)' : 'geneNamePrimary'})\n",
    "dfUniprot = dfUniprot.drop(columns=['Length','Entry Name','Organism','Gene Names'])\n",
    "dfUniprot['protAcc'] = dfUniprot['protAcc'].astype(str)\n",
    "dfUniprot.drop_duplicates(subset=['protAcc'], inplace = True)\n",
    "dfUniprot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annotate sarcoma df info by merging with dfUniprot\n",
    "\n",
    "dfSarcoma = dfSarcoma.merge(dfUniprot, on='protAcc', how='left')\n",
    "\n",
    "totalCols = dfSarcoma.columns\n",
    "numCols = dfSarcoma._get_numeric_data().columns\n",
    "catCols = list(set(totalCols)-set(numCols))\n",
    "\n",
    "dfSarcoma['area'] = dfSarcoma['area'].astype(float) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter dfSarcoma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tidy timing\n",
    "\n",
    "def filterTiming(df):\n",
    "    if target == 'sensitivity':\n",
    "        df.query('timing == \"Pre\"', inplace = True)\n",
    "    elif target == 'cancer':\n",
    "        df.query('timing != \"Post\"', inplace = True)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfFiltered = filterTiming(dfSarcoma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeCrapome(df):\n",
    "    crapome =['KRT1', 'KRT2', 'KRT3', 'KRT4', 'KRT5', 'KRT6', 'KRT7', 'KRT8', 'KRT9', 'KRT10', 'KRT11', 'KRT12', 'KRT13', 'KRT14', 'KRT15', 'KRT16', 'KRT17', 'KRT18', 'KRT19', 'KRT20', 'KRT21', 'KRT22', 'KRT23', 'KRT24']\n",
    "#    df.loc[~df['geneNamePrimary'].str.contains('|'.join(crapome))]\n",
    "    df.query('geneNamePrimary not in @crapome', inplace = True)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfFiltered = removeCrapome(dfSarcoma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count protein IDs\n",
    "\n",
    "def filterNDetectionsPerProtAcc(df, nSamplesVal_):\n",
    "\n",
    "    global nSamplesVal\n",
    "    nSamplesVal = nSamplesVal_\n",
    "    print(nSamplesVal)\n",
    "    \n",
    "    #drop 0 values\n",
    "    df['area'].replace(0, np.nan, inplace=True)\n",
    "    df.dropna(subset = ['area'], inplace = True)\n",
    "    \n",
    "    # count\n",
    "    df['nSamples'] = df.groupby('protAcc')['protAcc'].transform('count')\n",
    "    \n",
    "    # filter by N detections\n",
    "    df = df.query('nSamples >= @nSamplesVal')  # variable set above\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    }
   ],
   "source": [
    "dfFiltered = filterNDetectionsPerProtAcc(dfFiltered, nSamplesVal)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### impute NANs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imputeDfMinOr0 (df, colToImpute):\n",
    "    \n",
    "    if quantType == 'NormalizedIntensity_Log2':\n",
    "        df[colToImpute].fillna(value = df[colToImpute].min(), inplace = True)\n",
    "    \n",
    "    if quantType == 'Intensity_Raw':\n",
    "        df[colToImpute].fillna(value = df[colToImpute].min(), inplace = True)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfFiltered = imputeDfMinOr0(dfFiltered, 'area')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pivot to wide data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features,continuous_features, binary_features = columns_catNumOrBin(dfFiltered)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save dfFiltered"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# to save\n",
    "\n",
    "df = dfFiltered\n",
    "\n",
    "df = df.reset_index(drop = True)\n",
    "df.to_excel('dfFiltered_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_seed-' + str(seed) + '.xlsx')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# To rescue from HD (only if needed)\n",
    "\n",
    "dfFiltered = pd.read_excel('dfFilteredForAutoSKLearn_nSampleFilter.xlsx', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pivot df by geneNamePrimary\n",
    "\n",
    "indexListTemp = categorical_features.copy()\n",
    "indexListTemp.remove('geneNamePrimary')\n",
    "indexListTemp.remove('protAcc')\n",
    "indexListTemp.remove('Reviewed')\n",
    "indexListTemp.remove('Protein names')\n",
    "\n",
    "dfML = pd.pivot(\n",
    "    dfFiltered,\n",
    "    values = 'area',\n",
    "    columns = 'geneNamePrimary',\n",
    "    index = ['timing','dataset', 'gender', 'sample', 'cancerType', 'dataType', 'patient', 'group', 'age', 'cancer', 'sensitivity', 'race']\n",
    ").reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## remove categorical columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features,continuous_features, binary_features = columns_catNumOrBin(dfML)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dfRemoveAllCatColsButTarget(df):\n",
    "    \n",
    "    categorical_features = []\n",
    "    continuous_features = []\n",
    "    binary_features = []\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if df[col].dtype == object:\n",
    "            categorical_features.append(col)\n",
    "        \n",
    "        else:\n",
    "            if df[col].nunique() <= 2:\n",
    "                binary_features.append(col)\n",
    "            else:\n",
    "                continuous_features.append(col)\n",
    "    \n",
    "    categorical_features.remove(target)\n",
    "        \n",
    "    df.drop(categorical_features, axis = 1, inplace = True)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfML = dfRemoveAllCatColsButTarget(dfML)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# save\n",
    "df = dfML\n",
    "\n",
    "df = df.reset_index(drop = True)\n",
    "df.to_excel('dfML_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_seed-' + str(seed) + '.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split dfML into training, validation, and testing datasets prior to any transformation or feature selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dfML.copy() \n",
    "\n",
    "# random splitting into each set\n",
    "dfTrain = df.groupby(target, group_keys=False).sample(frac=trainFrac, random_state=seed)\n",
    "dfTest = df[~df.index.isin(dfTrain.index)]\n",
    "\n",
    "# validation fraction, taken from dfTrain.\n",
    "valFrac = .20\n",
    "\n",
    "dfValidation = dfTrain.groupby(target, group_keys=False).sample(frac=valFrac, random_state=seed)\n",
    "dfTrain= dfTrain[~dfTrain.index.isin(dfValidation.index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save\n",
    "df = dfML \n",
    "df = df.reset_index(drop = True)\n",
    "df.to_excel('dfTest_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "\n",
    "df = dfTest\n",
    "df = df.reset_index(drop = True)\n",
    "df.to_excel('dfTest_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "\n",
    "df = dfValidation\n",
    "df = df.reset_index(drop = True)\n",
    "df.to_excel('dfValidation_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "\n",
    "df = dfTrain\n",
    "df = df.reset_index(drop = True)\n",
    "df.to_excel('dfTrain_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into X and y\n",
    "\n",
    "X, y = X_y_split(dfTrain, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>geneNamePrimary</th>\n",
       "      <th>A1BG</th>\n",
       "      <th>ABCF1</th>\n",
       "      <th>ABI1</th>\n",
       "      <th>ACOT9</th>\n",
       "      <th>ACTA1</th>\n",
       "      <th>ACTB</th>\n",
       "      <th>ACTG1</th>\n",
       "      <th>ACTN1</th>\n",
       "      <th>ACTN4</th>\n",
       "      <th>ACTR2</th>\n",
       "      <th>...</th>\n",
       "      <th>YWHAB</th>\n",
       "      <th>YWHAE</th>\n",
       "      <th>YWHAG</th>\n",
       "      <th>YWHAH</th>\n",
       "      <th>YWHAQ</th>\n",
       "      <th>YWHAZ</th>\n",
       "      <th>ZC3HAV1</th>\n",
       "      <th>ZDHHC2</th>\n",
       "      <th>ZMPSTE24</th>\n",
       "      <th>ZYX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>7632000.0</td>\n",
       "      <td>496590.0</td>\n",
       "      <td>2438242.0</td>\n",
       "      <td>291130.0</td>\n",
       "      <td>36487000.0</td>\n",
       "      <td>32881000.0</td>\n",
       "      <td>10357000.0</td>\n",
       "      <td>7327800.0</td>\n",
       "      <td>360258.0</td>\n",
       "      <td>589420.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3628570.0</td>\n",
       "      <td>6010120.0</td>\n",
       "      <td>2205670.0</td>\n",
       "      <td>6052910.0</td>\n",
       "      <td>3996620.0</td>\n",
       "      <td>25551700.0</td>\n",
       "      <td>613469.0</td>\n",
       "      <td>287020.0</td>\n",
       "      <td>82566.0</td>\n",
       "      <td>6430000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4123600.0</td>\n",
       "      <td>368730.0</td>\n",
       "      <td>749190.0</td>\n",
       "      <td>813307.0</td>\n",
       "      <td>83363000.0</td>\n",
       "      <td>45029000.0</td>\n",
       "      <td>22867000.0</td>\n",
       "      <td>23313700.0</td>\n",
       "      <td>1850240.0</td>\n",
       "      <td>1927730.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2183930.0</td>\n",
       "      <td>6493540.0</td>\n",
       "      <td>1821520.0</td>\n",
       "      <td>3524830.0</td>\n",
       "      <td>3553360.0</td>\n",
       "      <td>19557400.0</td>\n",
       "      <td>776766.0</td>\n",
       "      <td>123410.0</td>\n",
       "      <td>137218.0</td>\n",
       "      <td>3732000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>3933800.0</td>\n",
       "      <td>107730.0</td>\n",
       "      <td>2887500.0</td>\n",
       "      <td>200200.0</td>\n",
       "      <td>71462000.0</td>\n",
       "      <td>50501000.0</td>\n",
       "      <td>20002000.0</td>\n",
       "      <td>16157000.0</td>\n",
       "      <td>1076000.0</td>\n",
       "      <td>1149880.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3104980.0</td>\n",
       "      <td>8533300.0</td>\n",
       "      <td>3162760.0</td>\n",
       "      <td>4963630.0</td>\n",
       "      <td>3661240.0</td>\n",
       "      <td>22837200.0</td>\n",
       "      <td>77191.0</td>\n",
       "      <td>349010.0</td>\n",
       "      <td>113450.0</td>\n",
       "      <td>7282800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4497800.0</td>\n",
       "      <td>437075.0</td>\n",
       "      <td>2074010.0</td>\n",
       "      <td>425770.0</td>\n",
       "      <td>110345000.0</td>\n",
       "      <td>12099000.0</td>\n",
       "      <td>31026000.0</td>\n",
       "      <td>54004000.0</td>\n",
       "      <td>8077700.0</td>\n",
       "      <td>5941300.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5485280.0</td>\n",
       "      <td>19672600.0</td>\n",
       "      <td>5545970.0</td>\n",
       "      <td>11060700.0</td>\n",
       "      <td>7227400.0</td>\n",
       "      <td>43943000.0</td>\n",
       "      <td>234738.0</td>\n",
       "      <td>22105.0</td>\n",
       "      <td>434008.0</td>\n",
       "      <td>16252500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2556590.0</td>\n",
       "      <td>207772.0</td>\n",
       "      <td>601030.0</td>\n",
       "      <td>42573.0</td>\n",
       "      <td>87707140.0</td>\n",
       "      <td>65904000.0</td>\n",
       "      <td>31700000.0</td>\n",
       "      <td>35191000.0</td>\n",
       "      <td>2366940.0</td>\n",
       "      <td>2859450.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2598800.0</td>\n",
       "      <td>9861400.0</td>\n",
       "      <td>3171220.0</td>\n",
       "      <td>6229500.0</td>\n",
       "      <td>4119620.0</td>\n",
       "      <td>27531600.0</td>\n",
       "      <td>102305.0</td>\n",
       "      <td>225020.0</td>\n",
       "      <td>498364.0</td>\n",
       "      <td>6656900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7657700.0</td>\n",
       "      <td>708890.0</td>\n",
       "      <td>340690.0</td>\n",
       "      <td>720100.0</td>\n",
       "      <td>9466000.0</td>\n",
       "      <td>9294300.0</td>\n",
       "      <td>1763900.0</td>\n",
       "      <td>1159670.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>256360.0</td>\n",
       "      <td>...</td>\n",
       "      <td>241781.0</td>\n",
       "      <td>1270330.0</td>\n",
       "      <td>92610.0</td>\n",
       "      <td>298685.0</td>\n",
       "      <td>46829.0</td>\n",
       "      <td>2931930.0</td>\n",
       "      <td>686690.0</td>\n",
       "      <td>6110100.0</td>\n",
       "      <td>133720.0</td>\n",
       "      <td>420970.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2522360.0</td>\n",
       "      <td>187310.0</td>\n",
       "      <td>439230.0</td>\n",
       "      <td>504190.0</td>\n",
       "      <td>55457000.0</td>\n",
       "      <td>41600000.0</td>\n",
       "      <td>15211000.0</td>\n",
       "      <td>9072400.0</td>\n",
       "      <td>1471740.0</td>\n",
       "      <td>1457540.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2866490.0</td>\n",
       "      <td>8142400.0</td>\n",
       "      <td>3163310.0</td>\n",
       "      <td>4933440.0</td>\n",
       "      <td>3430980.0</td>\n",
       "      <td>24067800.0</td>\n",
       "      <td>208061.0</td>\n",
       "      <td>267930.0</td>\n",
       "      <td>66941.0</td>\n",
       "      <td>5779200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>4749800.0</td>\n",
       "      <td>733689.0</td>\n",
       "      <td>1901780.0</td>\n",
       "      <td>416150.0</td>\n",
       "      <td>79739400.0</td>\n",
       "      <td>45568000.0</td>\n",
       "      <td>28951000.0</td>\n",
       "      <td>32928500.0</td>\n",
       "      <td>2849210.0</td>\n",
       "      <td>2803710.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4057260.0</td>\n",
       "      <td>10469000.0</td>\n",
       "      <td>3718840.0</td>\n",
       "      <td>8389400.0</td>\n",
       "      <td>5930700.0</td>\n",
       "      <td>34845200.0</td>\n",
       "      <td>143474.0</td>\n",
       "      <td>90475.0</td>\n",
       "      <td>204736.0</td>\n",
       "      <td>6548200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>5821700.0</td>\n",
       "      <td>790330.0</td>\n",
       "      <td>34386.0</td>\n",
       "      <td>186680.0</td>\n",
       "      <td>29609471.0</td>\n",
       "      <td>17135000.0</td>\n",
       "      <td>6221500.0</td>\n",
       "      <td>2599980.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>256360.0</td>\n",
       "      <td>...</td>\n",
       "      <td>811680.0</td>\n",
       "      <td>2587520.0</td>\n",
       "      <td>900830.0</td>\n",
       "      <td>1549680.0</td>\n",
       "      <td>828280.0</td>\n",
       "      <td>10830600.0</td>\n",
       "      <td>32445.0</td>\n",
       "      <td>190790.0</td>\n",
       "      <td>215637.0</td>\n",
       "      <td>1497080.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7486400.0</td>\n",
       "      <td>1196700.0</td>\n",
       "      <td>1043800.0</td>\n",
       "      <td>337322.0</td>\n",
       "      <td>11505000.0</td>\n",
       "      <td>29364000.0</td>\n",
       "      <td>9394400.0</td>\n",
       "      <td>6441500.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>307360.0</td>\n",
       "      <td>...</td>\n",
       "      <td>241781.0</td>\n",
       "      <td>661170.0</td>\n",
       "      <td>115300.0</td>\n",
       "      <td>172095.0</td>\n",
       "      <td>46829.0</td>\n",
       "      <td>3430480.0</td>\n",
       "      <td>920900.0</td>\n",
       "      <td>445150.0</td>\n",
       "      <td>248387.0</td>\n",
       "      <td>420970.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5963800.0</td>\n",
       "      <td>761030.0</td>\n",
       "      <td>1117200.0</td>\n",
       "      <td>821444.0</td>\n",
       "      <td>85647550.0</td>\n",
       "      <td>45643000.0</td>\n",
       "      <td>29081000.0</td>\n",
       "      <td>37752000.0</td>\n",
       "      <td>3004220.0</td>\n",
       "      <td>2549800.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3125640.0</td>\n",
       "      <td>8732300.0</td>\n",
       "      <td>2066800.0</td>\n",
       "      <td>5177900.0</td>\n",
       "      <td>4897630.0</td>\n",
       "      <td>29950100.0</td>\n",
       "      <td>783448.0</td>\n",
       "      <td>77077.0</td>\n",
       "      <td>288725.0</td>\n",
       "      <td>3914200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3307350.0</td>\n",
       "      <td>547128.0</td>\n",
       "      <td>1677330.0</td>\n",
       "      <td>489406.0</td>\n",
       "      <td>75005000.0</td>\n",
       "      <td>26004000.0</td>\n",
       "      <td>28262000.0</td>\n",
       "      <td>11684400.0</td>\n",
       "      <td>1042460.0</td>\n",
       "      <td>2211940.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3878920.0</td>\n",
       "      <td>10807100.0</td>\n",
       "      <td>3985040.0</td>\n",
       "      <td>8385000.0</td>\n",
       "      <td>7160200.0</td>\n",
       "      <td>32210400.0</td>\n",
       "      <td>666806.0</td>\n",
       "      <td>22105.0</td>\n",
       "      <td>305853.0</td>\n",
       "      <td>4020600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>7900200.0</td>\n",
       "      <td>1320600.0</td>\n",
       "      <td>1823825.0</td>\n",
       "      <td>418460.0</td>\n",
       "      <td>34257000.0</td>\n",
       "      <td>27007000.0</td>\n",
       "      <td>7265600.0</td>\n",
       "      <td>3735900.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>313276.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1411200.0</td>\n",
       "      <td>3158680.0</td>\n",
       "      <td>1305250.0</td>\n",
       "      <td>2350040.0</td>\n",
       "      <td>1487370.0</td>\n",
       "      <td>14330200.0</td>\n",
       "      <td>1431200.0</td>\n",
       "      <td>224880.0</td>\n",
       "      <td>66941.0</td>\n",
       "      <td>1740270.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>5724300.0</td>\n",
       "      <td>628750.0</td>\n",
       "      <td>447110.0</td>\n",
       "      <td>495000.0</td>\n",
       "      <td>47575000.0</td>\n",
       "      <td>36872000.0</td>\n",
       "      <td>11879000.0</td>\n",
       "      <td>7582100.0</td>\n",
       "      <td>554270.0</td>\n",
       "      <td>430760.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1974690.0</td>\n",
       "      <td>7561630.0</td>\n",
       "      <td>2493410.0</td>\n",
       "      <td>2941740.0</td>\n",
       "      <td>2256360.0</td>\n",
       "      <td>24753100.0</td>\n",
       "      <td>32445.0</td>\n",
       "      <td>127090.0</td>\n",
       "      <td>70487.0</td>\n",
       "      <td>1301220.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>40471000.0</td>\n",
       "      <td>3754400.0</td>\n",
       "      <td>183374.0</td>\n",
       "      <td>26910.0</td>\n",
       "      <td>43944000.0</td>\n",
       "      <td>34841000.0</td>\n",
       "      <td>13861000.0</td>\n",
       "      <td>6113200.0</td>\n",
       "      <td>872410.0</td>\n",
       "      <td>413300.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3252650.0</td>\n",
       "      <td>5236390.0</td>\n",
       "      <td>2524590.0</td>\n",
       "      <td>4191050.0</td>\n",
       "      <td>2182480.0</td>\n",
       "      <td>22405800.0</td>\n",
       "      <td>1130300.0</td>\n",
       "      <td>54069.0</td>\n",
       "      <td>66941.0</td>\n",
       "      <td>3122550.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>7897500.0</td>\n",
       "      <td>876180.0</td>\n",
       "      <td>244963.0</td>\n",
       "      <td>56517.0</td>\n",
       "      <td>34730000.0</td>\n",
       "      <td>32199000.0</td>\n",
       "      <td>11500000.0</td>\n",
       "      <td>6069300.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>568450.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2388980.0</td>\n",
       "      <td>3840960.0</td>\n",
       "      <td>2045710.0</td>\n",
       "      <td>3069700.0</td>\n",
       "      <td>1718780.0</td>\n",
       "      <td>20102500.0</td>\n",
       "      <td>600230.0</td>\n",
       "      <td>354960.0</td>\n",
       "      <td>68770.0</td>\n",
       "      <td>2749070.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6074000.0</td>\n",
       "      <td>434140.0</td>\n",
       "      <td>159561.0</td>\n",
       "      <td>538060.0</td>\n",
       "      <td>39485000.0</td>\n",
       "      <td>32711000.0</td>\n",
       "      <td>9034200.0</td>\n",
       "      <td>3241060.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>455300.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1215420.0</td>\n",
       "      <td>4528430.0</td>\n",
       "      <td>1542890.0</td>\n",
       "      <td>2161480.0</td>\n",
       "      <td>1891310.0</td>\n",
       "      <td>19340300.0</td>\n",
       "      <td>32445.0</td>\n",
       "      <td>267280.0</td>\n",
       "      <td>89617.0</td>\n",
       "      <td>1246550.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5570400.0</td>\n",
       "      <td>670240.0</td>\n",
       "      <td>2298605.0</td>\n",
       "      <td>380400.0</td>\n",
       "      <td>64586130.0</td>\n",
       "      <td>40935000.0</td>\n",
       "      <td>14809000.0</td>\n",
       "      <td>11001800.0</td>\n",
       "      <td>925830.0</td>\n",
       "      <td>771710.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1535260.0</td>\n",
       "      <td>4894700.0</td>\n",
       "      <td>1581800.0</td>\n",
       "      <td>2130600.0</td>\n",
       "      <td>2776350.0</td>\n",
       "      <td>19994300.0</td>\n",
       "      <td>1087000.0</td>\n",
       "      <td>2179500.0</td>\n",
       "      <td>117810.0</td>\n",
       "      <td>2945530.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>5859100.0</td>\n",
       "      <td>493600.0</td>\n",
       "      <td>452591.0</td>\n",
       "      <td>341850.0</td>\n",
       "      <td>81960000.0</td>\n",
       "      <td>74706000.0</td>\n",
       "      <td>28815000.0</td>\n",
       "      <td>21392300.0</td>\n",
       "      <td>1409100.0</td>\n",
       "      <td>2252960.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1572020.0</td>\n",
       "      <td>6711400.0</td>\n",
       "      <td>2401350.0</td>\n",
       "      <td>3103620.0</td>\n",
       "      <td>2301300.0</td>\n",
       "      <td>17463600.0</td>\n",
       "      <td>1029800.0</td>\n",
       "      <td>814310.0</td>\n",
       "      <td>192194.0</td>\n",
       "      <td>1499490.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4940330.0</td>\n",
       "      <td>609820.0</td>\n",
       "      <td>1105920.0</td>\n",
       "      <td>142990.0</td>\n",
       "      <td>63009000.0</td>\n",
       "      <td>48651000.0</td>\n",
       "      <td>11655000.0</td>\n",
       "      <td>2241600.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>1201960.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3594970.0</td>\n",
       "      <td>6801390.0</td>\n",
       "      <td>3672580.0</td>\n",
       "      <td>5622600.0</td>\n",
       "      <td>3455410.0</td>\n",
       "      <td>25705000.0</td>\n",
       "      <td>1053700.0</td>\n",
       "      <td>484580.0</td>\n",
       "      <td>366060.0</td>\n",
       "      <td>1697230.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>5185500.0</td>\n",
       "      <td>572590.0</td>\n",
       "      <td>34386.0</td>\n",
       "      <td>721420.0</td>\n",
       "      <td>42500000.0</td>\n",
       "      <td>34192000.0</td>\n",
       "      <td>10111000.0</td>\n",
       "      <td>4970400.0</td>\n",
       "      <td>336904.0</td>\n",
       "      <td>538990.0</td>\n",
       "      <td>...</td>\n",
       "      <td>241781.0</td>\n",
       "      <td>2445890.0</td>\n",
       "      <td>743460.0</td>\n",
       "      <td>1357890.0</td>\n",
       "      <td>1012660.0</td>\n",
       "      <td>9238700.0</td>\n",
       "      <td>32445.0</td>\n",
       "      <td>393260.0</td>\n",
       "      <td>159983.0</td>\n",
       "      <td>1476650.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3297100.0</td>\n",
       "      <td>304487.0</td>\n",
       "      <td>984140.0</td>\n",
       "      <td>615110.0</td>\n",
       "      <td>94534300.0</td>\n",
       "      <td>34083000.0</td>\n",
       "      <td>30532000.0</td>\n",
       "      <td>36564000.0</td>\n",
       "      <td>3164150.0</td>\n",
       "      <td>3241840.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2816030.0</td>\n",
       "      <td>10354900.0</td>\n",
       "      <td>2267970.0</td>\n",
       "      <td>7002900.0</td>\n",
       "      <td>5546200.0</td>\n",
       "      <td>25409100.0</td>\n",
       "      <td>458207.0</td>\n",
       "      <td>37211.0</td>\n",
       "      <td>1307090.0</td>\n",
       "      <td>4197320.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6673200.0</td>\n",
       "      <td>180005.0</td>\n",
       "      <td>2685300.0</td>\n",
       "      <td>479140.0</td>\n",
       "      <td>92366200.0</td>\n",
       "      <td>14751000.0</td>\n",
       "      <td>26632000.0</td>\n",
       "      <td>40377000.0</td>\n",
       "      <td>4851500.0</td>\n",
       "      <td>7352800.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7760990.0</td>\n",
       "      <td>21084700.0</td>\n",
       "      <td>4308620.0</td>\n",
       "      <td>6892410.0</td>\n",
       "      <td>8347600.0</td>\n",
       "      <td>37026700.0</td>\n",
       "      <td>526140.0</td>\n",
       "      <td>22105.0</td>\n",
       "      <td>969180.0</td>\n",
       "      <td>15917400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>4546310.0</td>\n",
       "      <td>107730.0</td>\n",
       "      <td>3201500.0</td>\n",
       "      <td>621684.0</td>\n",
       "      <td>92508300.0</td>\n",
       "      <td>38317000.0</td>\n",
       "      <td>30275000.0</td>\n",
       "      <td>45366000.0</td>\n",
       "      <td>2358470.0</td>\n",
       "      <td>3798300.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2370360.0</td>\n",
       "      <td>4994400.0</td>\n",
       "      <td>2389193.0</td>\n",
       "      <td>8123010.0</td>\n",
       "      <td>6558310.0</td>\n",
       "      <td>33648100.0</td>\n",
       "      <td>216287.0</td>\n",
       "      <td>135350.0</td>\n",
       "      <td>224550.0</td>\n",
       "      <td>16971600.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 880 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "geneNamePrimary        A1BG      ABCF1       ABI1     ACOT9        ACTA1  \\\n",
       "22                7632000.0   496590.0  2438242.0  291130.0   36487000.0   \n",
       "18                4123600.0   368730.0   749190.0  813307.0   83363000.0   \n",
       "29                3933800.0   107730.0  2887500.0  200200.0   71462000.0   \n",
       "10                4497800.0   437075.0  2074010.0  425770.0  110345000.0   \n",
       "32                2556590.0   207772.0   601030.0   42573.0   87707140.0   \n",
       "9                 7657700.0   708890.0   340690.0  720100.0    9466000.0   \n",
       "36                2522360.0   187310.0   439230.0  504190.0   55457000.0   \n",
       "21                4749800.0   733689.0  1901780.0  416150.0   79739400.0   \n",
       "30                5821700.0   790330.0    34386.0  186680.0   29609471.0   \n",
       "12                7486400.0  1196700.0  1043800.0  337322.0   11505000.0   \n",
       "3                 5963800.0   761030.0  1117200.0  821444.0   85647550.0   \n",
       "20                3307350.0   547128.0  1677330.0  489406.0   75005000.0   \n",
       "26                7900200.0  1320600.0  1823825.0  418460.0   34257000.0   \n",
       "34                5724300.0   628750.0   447110.0  495000.0   47575000.0   \n",
       "24               40471000.0  3754400.0   183374.0   26910.0   43944000.0   \n",
       "39                7897500.0   876180.0   244963.0   56517.0   34730000.0   \n",
       "6                 6074000.0   434140.0   159561.0  538060.0   39485000.0   \n",
       "4                 5570400.0   670240.0  2298605.0  380400.0   64586130.0   \n",
       "37                5859100.0   493600.0   452591.0  341850.0   81960000.0   \n",
       "13                4940330.0   609820.0  1105920.0  142990.0   63009000.0   \n",
       "31                5185500.0   572590.0    34386.0  721420.0   42500000.0   \n",
       "15                3297100.0   304487.0   984140.0  615110.0   94534300.0   \n",
       "0                 6673200.0   180005.0  2685300.0  479140.0   92366200.0   \n",
       "17                4546310.0   107730.0  3201500.0  621684.0   92508300.0   \n",
       "\n",
       "geneNamePrimary        ACTB       ACTG1       ACTN1      ACTN4      ACTR2  \\\n",
       "22               32881000.0  10357000.0   7327800.0   360258.0   589420.0   \n",
       "18               45029000.0  22867000.0  23313700.0  1850240.0  1927730.0   \n",
       "29               50501000.0  20002000.0  16157000.0  1076000.0  1149880.0   \n",
       "10               12099000.0  31026000.0  54004000.0  8077700.0  5941300.0   \n",
       "32               65904000.0  31700000.0  35191000.0  2366940.0  2859450.0   \n",
       "9                 9294300.0   1763900.0   1159670.0   336904.0   256360.0   \n",
       "36               41600000.0  15211000.0   9072400.0  1471740.0  1457540.0   \n",
       "21               45568000.0  28951000.0  32928500.0  2849210.0  2803710.0   \n",
       "30               17135000.0   6221500.0   2599980.0   336904.0   256360.0   \n",
       "12               29364000.0   9394400.0   6441500.0   336904.0   307360.0   \n",
       "3                45643000.0  29081000.0  37752000.0  3004220.0  2549800.0   \n",
       "20               26004000.0  28262000.0  11684400.0  1042460.0  2211940.0   \n",
       "26               27007000.0   7265600.0   3735900.0   336904.0   313276.0   \n",
       "34               36872000.0  11879000.0   7582100.0   554270.0   430760.0   \n",
       "24               34841000.0  13861000.0   6113200.0   872410.0   413300.0   \n",
       "39               32199000.0  11500000.0   6069300.0   336904.0   568450.0   \n",
       "6                32711000.0   9034200.0   3241060.0   336904.0   455300.0   \n",
       "4                40935000.0  14809000.0  11001800.0   925830.0   771710.0   \n",
       "37               74706000.0  28815000.0  21392300.0  1409100.0  2252960.0   \n",
       "13               48651000.0  11655000.0   2241600.0   336904.0  1201960.0   \n",
       "31               34192000.0  10111000.0   4970400.0   336904.0   538990.0   \n",
       "15               34083000.0  30532000.0  36564000.0  3164150.0  3241840.0   \n",
       "0                14751000.0  26632000.0  40377000.0  4851500.0  7352800.0   \n",
       "17               38317000.0  30275000.0  45366000.0  2358470.0  3798300.0   \n",
       "\n",
       "geneNamePrimary  ...      YWHAB       YWHAE      YWHAG       YWHAH      YWHAQ  \\\n",
       "22               ...  3628570.0   6010120.0  2205670.0   6052910.0  3996620.0   \n",
       "18               ...  2183930.0   6493540.0  1821520.0   3524830.0  3553360.0   \n",
       "29               ...  3104980.0   8533300.0  3162760.0   4963630.0  3661240.0   \n",
       "10               ...  5485280.0  19672600.0  5545970.0  11060700.0  7227400.0   \n",
       "32               ...  2598800.0   9861400.0  3171220.0   6229500.0  4119620.0   \n",
       "9                ...   241781.0   1270330.0    92610.0    298685.0    46829.0   \n",
       "36               ...  2866490.0   8142400.0  3163310.0   4933440.0  3430980.0   \n",
       "21               ...  4057260.0  10469000.0  3718840.0   8389400.0  5930700.0   \n",
       "30               ...   811680.0   2587520.0   900830.0   1549680.0   828280.0   \n",
       "12               ...   241781.0    661170.0   115300.0    172095.0    46829.0   \n",
       "3                ...  3125640.0   8732300.0  2066800.0   5177900.0  4897630.0   \n",
       "20               ...  3878920.0  10807100.0  3985040.0   8385000.0  7160200.0   \n",
       "26               ...  1411200.0   3158680.0  1305250.0   2350040.0  1487370.0   \n",
       "34               ...  1974690.0   7561630.0  2493410.0   2941740.0  2256360.0   \n",
       "24               ...  3252650.0   5236390.0  2524590.0   4191050.0  2182480.0   \n",
       "39               ...  2388980.0   3840960.0  2045710.0   3069700.0  1718780.0   \n",
       "6                ...  1215420.0   4528430.0  1542890.0   2161480.0  1891310.0   \n",
       "4                ...  1535260.0   4894700.0  1581800.0   2130600.0  2776350.0   \n",
       "37               ...  1572020.0   6711400.0  2401350.0   3103620.0  2301300.0   \n",
       "13               ...  3594970.0   6801390.0  3672580.0   5622600.0  3455410.0   \n",
       "31               ...   241781.0   2445890.0   743460.0   1357890.0  1012660.0   \n",
       "15               ...  2816030.0  10354900.0  2267970.0   7002900.0  5546200.0   \n",
       "0                ...  7760990.0  21084700.0  4308620.0   6892410.0  8347600.0   \n",
       "17               ...  2370360.0   4994400.0  2389193.0   8123010.0  6558310.0   \n",
       "\n",
       "geneNamePrimary       YWHAZ    ZC3HAV1     ZDHHC2   ZMPSTE24         ZYX  \n",
       "22               25551700.0   613469.0   287020.0    82566.0   6430000.0  \n",
       "18               19557400.0   776766.0   123410.0   137218.0   3732000.0  \n",
       "29               22837200.0    77191.0   349010.0   113450.0   7282800.0  \n",
       "10               43943000.0   234738.0    22105.0   434008.0  16252500.0  \n",
       "32               27531600.0   102305.0   225020.0   498364.0   6656900.0  \n",
       "9                 2931930.0   686690.0  6110100.0   133720.0    420970.0  \n",
       "36               24067800.0   208061.0   267930.0    66941.0   5779200.0  \n",
       "21               34845200.0   143474.0    90475.0   204736.0   6548200.0  \n",
       "30               10830600.0    32445.0   190790.0   215637.0   1497080.0  \n",
       "12                3430480.0   920900.0   445150.0   248387.0    420970.0  \n",
       "3                29950100.0   783448.0    77077.0   288725.0   3914200.0  \n",
       "20               32210400.0   666806.0    22105.0   305853.0   4020600.0  \n",
       "26               14330200.0  1431200.0   224880.0    66941.0   1740270.0  \n",
       "34               24753100.0    32445.0   127090.0    70487.0   1301220.0  \n",
       "24               22405800.0  1130300.0    54069.0    66941.0   3122550.0  \n",
       "39               20102500.0   600230.0   354960.0    68770.0   2749070.0  \n",
       "6                19340300.0    32445.0   267280.0    89617.0   1246550.0  \n",
       "4                19994300.0  1087000.0  2179500.0   117810.0   2945530.0  \n",
       "37               17463600.0  1029800.0   814310.0   192194.0   1499490.0  \n",
       "13               25705000.0  1053700.0   484580.0   366060.0   1697230.0  \n",
       "31                9238700.0    32445.0   393260.0   159983.0   1476650.0  \n",
       "15               25409100.0   458207.0    37211.0  1307090.0   4197320.0  \n",
       "0                37026700.0   526140.0    22105.0   969180.0  15917400.0  \n",
       "17               33648100.0   216287.0   135350.0   224550.0  16971600.0  \n",
       "\n",
       "[24 rows x 880 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputeWideDFMinOr0(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform and scale the data as in the pipeline below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scale each gene of a Pandas DF separately.\n",
    "\n",
    "def transformX (X, transformer):\n",
    "   \n",
    "    if transformer == RobustScaler:\n",
    "        transformer = RobustScaler(with_centering = True, with_scaling = True, quantile_range=(25, 75), unit_variance = False)\n",
    "\n",
    "    X_transformed = transformer.fit_transform(X)\n",
    "    X_transformed = pd.DataFrame(X_transformed, columns=X.columns) # turn back into df\n",
    "\n",
    "    return X_transformed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform, matching the pipeline\n",
    "\n",
    "X_transformed = transformX(X, RobustScaler)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# scale to same variance, matching the pipeline\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X_transformed)\n",
    "X_scaled = pd.DataFrame(X_scaled, columns=X_transformed.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection with recursive feature extraction (RFE)\n",
    "* tune minFeaturesToSelect and cv to retain ~60-150 proteins. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define CV strategy\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "\n",
    "if target ==  'sensitivity':\n",
    "    cv = RepeatedStratifiedKFold(n_splits=3, n_repeats=5, random_state = seed)\n",
    "    cv_splits = list(cv.split(X,y))\n",
    "\n",
    "if target ==  'cancer':\n",
    "    cv = RepeatedStratifiedKFold(n_splits=4, n_repeats=5, random_state = seed)\n",
    "    cv_splits = list(cv.split(X,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for scoring\n",
    "\n",
    "def quadratic_weighted_kappa(y_true, y_pred):\n",
    "    return cohen_kappa_score(y_true, y_pred, weights='quadratic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "def customRFECV (X_input):\n",
    "\n",
    "    global selected_features\n",
    "    global rfe\n",
    "    global min_features_to_select\n",
    "    \n",
    "    estimator = RandomForestClassifier(random_state=seed)\n",
    "\n",
    "    # for sensitivity seed 1 = use cv = 6,\n",
    "    # for sensitivity seed 2 = use cv = 6,\n",
    "    # for sensitivity seed 3 = use minFeaturesToSelect = 100 and cv = 3,\n",
    "    # for sensitivity seed 4 = use cv = 4,\n",
    "    # for sensitivity seed 5 = use minFeaturesToSelect = 100 and cv = 4,\n",
    "    # for sensitivity seed 6 = use cv = 8,\n",
    "    # for sensitivity seed 7 = use cv = 6,\n",
    "\n",
    "    # for cancer seed 1 = use cv = 3,\n",
    "    # for cancer seed 2 = use cv = 3,\n",
    "    # for cancer seed 3 = use minFeaturesToSelect = 100 and cv = 3,\n",
    "    # for cancer seed 4 = use cv = 4,\n",
    "    # for cancer seed 5 = use cv = 5,\n",
    "    # for cancer seed 6 = use cv = 4,\n",
    "    # for cancer seed 7 = use cv = 5,\n",
    "    \n",
    "    minFeaturesToSelect = 100\n",
    "    \n",
    "    rfe = RFECV(\n",
    "        estimator=estimator, \n",
    "        cv=6,\n",
    "        scoring= make_scorer(quadratic_weighted_kappa),\n",
    "        n_jobs = nJobs,\n",
    "        step = 1,\n",
    "    )\n",
    "    \n",
    "    rfe.fit(X_input, y)\n",
    "    \n",
    "    selected_features = []\n",
    "    \n",
    "    for i, feature in enumerate(X_input.columns):\n",
    "        if rfe.support_[i]:\n",
    "            selected_features.append(feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "customRFECV(X_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save features\n",
    "\n",
    "fileName = '_selectedFeaturesRFECV_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "\n",
    "with open(fileName + '.json', 'w') as f:\n",
    "    json.dump(selected_features, f, indent=2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to open\n",
    "\n",
    "fileName = '_selectedFeaturesRFECV_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "\n",
    "with open(fileName + '.json', 'r') as f:\n",
    "    selected_features = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;Transforming Distribution&#x27;,\n",
       "                 RobustScaler(quantile_range=(25, 75))),\n",
       "                (&#x27;Standard Scaler&#x27;, StandardScaler()), (&#x27;Model&#x27;, None)])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;Transforming Distribution&#x27;,\n",
       "                 RobustScaler(quantile_range=(25, 75))),\n",
       "                (&#x27;Standard Scaler&#x27;, StandardScaler()), (&#x27;Model&#x27;, None)])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RobustScaler</label><div class=\"sk-toggleable__content\"><pre>RobustScaler(quantile_range=(25, 75))</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">None</label><div class=\"sk-toggleable__content\"><pre>None</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('Transforming Distribution',\n",
       "                 RobustScaler(quantile_range=(25, 75))),\n",
       "                ('Standard Scaler', StandardScaler()), ('Model', None)])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make an empty sklearn pipeline without a classifier\n",
    "\n",
    "pipeline = Pipeline(steps=[\n",
    "    ('Transforming Distribution',  RobustScaler(with_centering = True, with_scaling = True, quantile_range=(25, 75), unit_variance = False)),\n",
    "    ('Standard Scaler', StandardScaler()),\n",
    "    ('Model', None),\n",
    "])\n",
    "\n",
    "pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start with training data\n",
    "\n",
    "df = dfTrain.copy(deep = True)\n",
    "\n",
    "categorical_features,continuous_features, binary_features = columns_catNumOrBin(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature selection\n",
    "\n",
    "df = dfTrain.copy(deep = True)\n",
    "\n",
    "# filtered proteins based on RFE above\n",
    "fileName = '_selectedFeaturesRFECV_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "\n",
    "with open(fileName + '.json', 'r') as f:\n",
    "    selected_featuresFromDisk = json.load(f)\n",
    "\n",
    "colsToFilter = []\n",
    "colsToFilter = copy.deepcopy(selected_featuresFromDisk)\n",
    "colsToFilter.append(target)\n",
    "df = df[colsToFilter] # selected features from RFE\n",
    "\n",
    "X, y = X_y_split(df, target)\n",
    "\n",
    "le = LabelEncoder()\n",
    "y = pd.Series(le.fit_transform(y))\n",
    "\n",
    "categorical_features,continuous_features, binary_features = columns_catNumOrBin(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>geneNamePrimary</th>\n",
       "      <th>HNRNPR</th>\n",
       "      <th>HPR</th>\n",
       "      <th>HPSE</th>\n",
       "      <th>HPX</th>\n",
       "      <th>HRG</th>\n",
       "      <th>HRNR</th>\n",
       "      <th>HSDL2</th>\n",
       "      <th>HSP90AA1</th>\n",
       "      <th>HSP90AB1</th>\n",
       "      <th>HSP90B1</th>\n",
       "      <th>...</th>\n",
       "      <th>ST13</th>\n",
       "      <th>STAT5B</th>\n",
       "      <th>SYNRG</th>\n",
       "      <th>TFRC</th>\n",
       "      <th>TGFB1I1</th>\n",
       "      <th>THBS4</th>\n",
       "      <th>TIMP1</th>\n",
       "      <th>TIMP2</th>\n",
       "      <th>TKT</th>\n",
       "      <th>TLN2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>198080.0</td>\n",
       "      <td>10003700.0</td>\n",
       "      <td>674920.0</td>\n",
       "      <td>76598000.0</td>\n",
       "      <td>14859700.0</td>\n",
       "      <td>1044630.0</td>\n",
       "      <td>803050.0</td>\n",
       "      <td>722800.0</td>\n",
       "      <td>835130.0</td>\n",
       "      <td>2177780.0</td>\n",
       "      <td>...</td>\n",
       "      <td>621810.0</td>\n",
       "      <td>641002.0</td>\n",
       "      <td>3934500.0</td>\n",
       "      <td>528100.0</td>\n",
       "      <td>7205700.0</td>\n",
       "      <td>602830.0</td>\n",
       "      <td>175589.0</td>\n",
       "      <td>432830.0</td>\n",
       "      <td>1310900.0</td>\n",
       "      <td>375590.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>57248.0</td>\n",
       "      <td>15866200.0</td>\n",
       "      <td>1051780.0</td>\n",
       "      <td>44572000.0</td>\n",
       "      <td>12356200.0</td>\n",
       "      <td>659010.0</td>\n",
       "      <td>896280.0</td>\n",
       "      <td>890310.0</td>\n",
       "      <td>348891.0</td>\n",
       "      <td>4679200.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1028150.0</td>\n",
       "      <td>112145.0</td>\n",
       "      <td>6886400.0</td>\n",
       "      <td>357746.0</td>\n",
       "      <td>1359120.0</td>\n",
       "      <td>3735800.0</td>\n",
       "      <td>368210.0</td>\n",
       "      <td>676790.0</td>\n",
       "      <td>313390.0</td>\n",
       "      <td>687130.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>394410.0</td>\n",
       "      <td>13968700.0</td>\n",
       "      <td>898040.0</td>\n",
       "      <td>85988000.0</td>\n",
       "      <td>7706700.0</td>\n",
       "      <td>631690.0</td>\n",
       "      <td>600350.0</td>\n",
       "      <td>1687590.0</td>\n",
       "      <td>1959670.0</td>\n",
       "      <td>3858400.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1033470.0</td>\n",
       "      <td>327190.0</td>\n",
       "      <td>3479400.0</td>\n",
       "      <td>905510.0</td>\n",
       "      <td>5311100.0</td>\n",
       "      <td>4638100.0</td>\n",
       "      <td>469568.0</td>\n",
       "      <td>639440.0</td>\n",
       "      <td>1985700.0</td>\n",
       "      <td>510200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>356770.0</td>\n",
       "      <td>1646300.0</td>\n",
       "      <td>2588120.0</td>\n",
       "      <td>24667900.0</td>\n",
       "      <td>10165600.0</td>\n",
       "      <td>640742.0</td>\n",
       "      <td>202250.0</td>\n",
       "      <td>3895180.0</td>\n",
       "      <td>1890770.0</td>\n",
       "      <td>7309600.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1325710.0</td>\n",
       "      <td>357067.0</td>\n",
       "      <td>176200.0</td>\n",
       "      <td>134719.0</td>\n",
       "      <td>2976070.0</td>\n",
       "      <td>173833.0</td>\n",
       "      <td>1905360.0</td>\n",
       "      <td>35751.0</td>\n",
       "      <td>796414.0</td>\n",
       "      <td>100020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>878460.0</td>\n",
       "      <td>17399900.0</td>\n",
       "      <td>2198060.0</td>\n",
       "      <td>45095000.0</td>\n",
       "      <td>6637200.0</td>\n",
       "      <td>727743.0</td>\n",
       "      <td>614080.0</td>\n",
       "      <td>2092530.0</td>\n",
       "      <td>949310.0</td>\n",
       "      <td>6349600.0</td>\n",
       "      <td>...</td>\n",
       "      <td>938790.0</td>\n",
       "      <td>383976.0</td>\n",
       "      <td>3932800.0</td>\n",
       "      <td>350255.0</td>\n",
       "      <td>3108900.0</td>\n",
       "      <td>873290.0</td>\n",
       "      <td>823252.0</td>\n",
       "      <td>168694.0</td>\n",
       "      <td>889510.0</td>\n",
       "      <td>883680.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>183860.0</td>\n",
       "      <td>11954000.0</td>\n",
       "      <td>149170.0</td>\n",
       "      <td>75845000.0</td>\n",
       "      <td>17204500.0</td>\n",
       "      <td>1858990.0</td>\n",
       "      <td>1546200.0</td>\n",
       "      <td>508927.0</td>\n",
       "      <td>59913.0</td>\n",
       "      <td>3804000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>786660.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>6663900.0</td>\n",
       "      <td>313169.0</td>\n",
       "      <td>878750.0</td>\n",
       "      <td>1716580.0</td>\n",
       "      <td>308347.0</td>\n",
       "      <td>837250.0</td>\n",
       "      <td>70625.0</td>\n",
       "      <td>274730.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1315960.0</td>\n",
       "      <td>4611100.0</td>\n",
       "      <td>2140580.0</td>\n",
       "      <td>39022400.0</td>\n",
       "      <td>5603500.0</td>\n",
       "      <td>985332.0</td>\n",
       "      <td>731910.0</td>\n",
       "      <td>1000040.0</td>\n",
       "      <td>1365030.0</td>\n",
       "      <td>3585700.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1163090.0</td>\n",
       "      <td>1390830.0</td>\n",
       "      <td>4727300.0</td>\n",
       "      <td>434540.0</td>\n",
       "      <td>8194600.0</td>\n",
       "      <td>3433580.0</td>\n",
       "      <td>317050.0</td>\n",
       "      <td>459970.0</td>\n",
       "      <td>1322900.0</td>\n",
       "      <td>669560.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>737800.0</td>\n",
       "      <td>16069700.0</td>\n",
       "      <td>2018930.0</td>\n",
       "      <td>46356000.0</td>\n",
       "      <td>7385400.0</td>\n",
       "      <td>592344.0</td>\n",
       "      <td>400430.0</td>\n",
       "      <td>1761560.0</td>\n",
       "      <td>1113340.0</td>\n",
       "      <td>6818700.0</td>\n",
       "      <td>...</td>\n",
       "      <td>783200.0</td>\n",
       "      <td>394000.0</td>\n",
       "      <td>2993600.0</td>\n",
       "      <td>269724.0</td>\n",
       "      <td>4479950.0</td>\n",
       "      <td>783100.0</td>\n",
       "      <td>874645.0</td>\n",
       "      <td>158259.0</td>\n",
       "      <td>522899.0</td>\n",
       "      <td>390110.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>275710.0</td>\n",
       "      <td>18816200.0</td>\n",
       "      <td>149170.0</td>\n",
       "      <td>69909000.0</td>\n",
       "      <td>17475700.0</td>\n",
       "      <td>585650.0</td>\n",
       "      <td>3149400.0</td>\n",
       "      <td>163169.0</td>\n",
       "      <td>59913.0</td>\n",
       "      <td>2432610.0</td>\n",
       "      <td>...</td>\n",
       "      <td>305555.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>6021500.0</td>\n",
       "      <td>736990.0</td>\n",
       "      <td>412943.0</td>\n",
       "      <td>7531400.0</td>\n",
       "      <td>17057.0</td>\n",
       "      <td>556380.0</td>\n",
       "      <td>2333000.0</td>\n",
       "      <td>1583700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5862100.0</td>\n",
       "      <td>488340.0</td>\n",
       "      <td>149170.0</td>\n",
       "      <td>47983400.0</td>\n",
       "      <td>6317100.0</td>\n",
       "      <td>831790.0</td>\n",
       "      <td>1297000.0</td>\n",
       "      <td>110474.0</td>\n",
       "      <td>76098.0</td>\n",
       "      <td>1519580.0</td>\n",
       "      <td>...</td>\n",
       "      <td>148633.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>5252600.0</td>\n",
       "      <td>520790.0</td>\n",
       "      <td>68827.0</td>\n",
       "      <td>2703670.0</td>\n",
       "      <td>17057.0</td>\n",
       "      <td>1399640.0</td>\n",
       "      <td>70625.0</td>\n",
       "      <td>889400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1238520.0</td>\n",
       "      <td>7730400.0</td>\n",
       "      <td>1621500.0</td>\n",
       "      <td>44495800.0</td>\n",
       "      <td>13244300.0</td>\n",
       "      <td>1100628.0</td>\n",
       "      <td>673890.0</td>\n",
       "      <td>1134020.0</td>\n",
       "      <td>584590.0</td>\n",
       "      <td>4265600.0</td>\n",
       "      <td>...</td>\n",
       "      <td>568730.0</td>\n",
       "      <td>160337.0</td>\n",
       "      <td>4328300.0</td>\n",
       "      <td>272187.0</td>\n",
       "      <td>1607540.0</td>\n",
       "      <td>7864400.0</td>\n",
       "      <td>329210.0</td>\n",
       "      <td>523240.0</td>\n",
       "      <td>197355.0</td>\n",
       "      <td>504110.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1217610.0</td>\n",
       "      <td>7429300.0</td>\n",
       "      <td>1435290.0</td>\n",
       "      <td>33928400.0</td>\n",
       "      <td>18207600.0</td>\n",
       "      <td>547683.0</td>\n",
       "      <td>813210.0</td>\n",
       "      <td>1354720.0</td>\n",
       "      <td>512590.0</td>\n",
       "      <td>3825100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>621410.0</td>\n",
       "      <td>124092.0</td>\n",
       "      <td>2473400.0</td>\n",
       "      <td>975350.0</td>\n",
       "      <td>2138280.0</td>\n",
       "      <td>2765190.0</td>\n",
       "      <td>389310.0</td>\n",
       "      <td>260635.0</td>\n",
       "      <td>448829.0</td>\n",
       "      <td>356700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2904370.0</td>\n",
       "      <td>16650000.0</td>\n",
       "      <td>161540.0</td>\n",
       "      <td>106192000.0</td>\n",
       "      <td>25127600.0</td>\n",
       "      <td>2450070.0</td>\n",
       "      <td>616210.0</td>\n",
       "      <td>145231.0</td>\n",
       "      <td>171807.0</td>\n",
       "      <td>1218830.0</td>\n",
       "      <td>...</td>\n",
       "      <td>494600.0</td>\n",
       "      <td>168898.0</td>\n",
       "      <td>1267800.0</td>\n",
       "      <td>451890.0</td>\n",
       "      <td>1903880.0</td>\n",
       "      <td>6063000.0</td>\n",
       "      <td>17057.0</td>\n",
       "      <td>1138990.0</td>\n",
       "      <td>1092100.0</td>\n",
       "      <td>548310.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2118120.0</td>\n",
       "      <td>15869600.0</td>\n",
       "      <td>550020.0</td>\n",
       "      <td>64813000.0</td>\n",
       "      <td>15433100.0</td>\n",
       "      <td>1552250.0</td>\n",
       "      <td>944220.0</td>\n",
       "      <td>717930.0</td>\n",
       "      <td>325847.0</td>\n",
       "      <td>3062480.0</td>\n",
       "      <td>...</td>\n",
       "      <td>506470.0</td>\n",
       "      <td>400614.0</td>\n",
       "      <td>9382400.0</td>\n",
       "      <td>82340.0</td>\n",
       "      <td>3375010.0</td>\n",
       "      <td>5960900.0</td>\n",
       "      <td>171603.0</td>\n",
       "      <td>1726560.0</td>\n",
       "      <td>1067300.0</td>\n",
       "      <td>1140800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>882220.0</td>\n",
       "      <td>15511500.0</td>\n",
       "      <td>203860.0</td>\n",
       "      <td>168679000.0</td>\n",
       "      <td>31779900.0</td>\n",
       "      <td>1342790.0</td>\n",
       "      <td>728370.0</td>\n",
       "      <td>644090.0</td>\n",
       "      <td>248968.0</td>\n",
       "      <td>1553770.0</td>\n",
       "      <td>...</td>\n",
       "      <td>635560.0</td>\n",
       "      <td>106188.0</td>\n",
       "      <td>176200.0</td>\n",
       "      <td>36323.0</td>\n",
       "      <td>1909020.0</td>\n",
       "      <td>2296180.0</td>\n",
       "      <td>35701.0</td>\n",
       "      <td>434673.0</td>\n",
       "      <td>400460.0</td>\n",
       "      <td>861080.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1228700.0</td>\n",
       "      <td>19840200.0</td>\n",
       "      <td>379777.0</td>\n",
       "      <td>75066000.0</td>\n",
       "      <td>10682200.0</td>\n",
       "      <td>764749.0</td>\n",
       "      <td>1018600.0</td>\n",
       "      <td>110474.0</td>\n",
       "      <td>697550.0</td>\n",
       "      <td>2152660.0</td>\n",
       "      <td>...</td>\n",
       "      <td>533849.0</td>\n",
       "      <td>160820.0</td>\n",
       "      <td>1966600.0</td>\n",
       "      <td>439450.0</td>\n",
       "      <td>3686400.0</td>\n",
       "      <td>1341140.0</td>\n",
       "      <td>50095.0</td>\n",
       "      <td>654130.0</td>\n",
       "      <td>70625.0</td>\n",
       "      <td>149410.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3517950.0</td>\n",
       "      <td>9483300.0</td>\n",
       "      <td>149170.0</td>\n",
       "      <td>65298000.0</td>\n",
       "      <td>15285000.0</td>\n",
       "      <td>771640.0</td>\n",
       "      <td>1827300.0</td>\n",
       "      <td>447750.0</td>\n",
       "      <td>393074.0</td>\n",
       "      <td>3698500.0</td>\n",
       "      <td>...</td>\n",
       "      <td>760590.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>8917700.0</td>\n",
       "      <td>348406.0</td>\n",
       "      <td>1981300.0</td>\n",
       "      <td>4406780.0</td>\n",
       "      <td>17057.0</td>\n",
       "      <td>788770.0</td>\n",
       "      <td>1341400.0</td>\n",
       "      <td>905770.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3512000.0</td>\n",
       "      <td>11013000.0</td>\n",
       "      <td>311353.0</td>\n",
       "      <td>69228000.0</td>\n",
       "      <td>16635600.0</td>\n",
       "      <td>1383490.0</td>\n",
       "      <td>1380920.0</td>\n",
       "      <td>665700.0</td>\n",
       "      <td>247849.0</td>\n",
       "      <td>1961030.0</td>\n",
       "      <td>...</td>\n",
       "      <td>763640.0</td>\n",
       "      <td>33443.0</td>\n",
       "      <td>5074100.0</td>\n",
       "      <td>347610.0</td>\n",
       "      <td>1619800.0</td>\n",
       "      <td>4568200.0</td>\n",
       "      <td>41796.0</td>\n",
       "      <td>690390.0</td>\n",
       "      <td>1515800.0</td>\n",
       "      <td>455630.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>568470.0</td>\n",
       "      <td>16763100.0</td>\n",
       "      <td>967790.0</td>\n",
       "      <td>76221000.0</td>\n",
       "      <td>11356200.0</td>\n",
       "      <td>1358280.0</td>\n",
       "      <td>1476530.0</td>\n",
       "      <td>983370.0</td>\n",
       "      <td>245333.0</td>\n",
       "      <td>7727200.0</td>\n",
       "      <td>...</td>\n",
       "      <td>284509.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>9768100.0</td>\n",
       "      <td>92853.0</td>\n",
       "      <td>2799700.0</td>\n",
       "      <td>364901.0</td>\n",
       "      <td>251788.0</td>\n",
       "      <td>369307.0</td>\n",
       "      <td>70625.0</td>\n",
       "      <td>431040.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>5310080.0</td>\n",
       "      <td>18658000.0</td>\n",
       "      <td>1491800.0</td>\n",
       "      <td>81089000.0</td>\n",
       "      <td>11858600.0</td>\n",
       "      <td>253980.0</td>\n",
       "      <td>846880.0</td>\n",
       "      <td>1468510.0</td>\n",
       "      <td>1107180.0</td>\n",
       "      <td>1769940.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1840520.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>176200.0</td>\n",
       "      <td>631380.0</td>\n",
       "      <td>270841.0</td>\n",
       "      <td>1118810.0</td>\n",
       "      <td>17057.0</td>\n",
       "      <td>362739.0</td>\n",
       "      <td>70625.0</td>\n",
       "      <td>101920.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>4010430.0</td>\n",
       "      <td>6409790.0</td>\n",
       "      <td>149170.0</td>\n",
       "      <td>65457000.0</td>\n",
       "      <td>34689700.0</td>\n",
       "      <td>3465820.0</td>\n",
       "      <td>1242000.0</td>\n",
       "      <td>278190.0</td>\n",
       "      <td>102550.0</td>\n",
       "      <td>3025270.0</td>\n",
       "      <td>...</td>\n",
       "      <td>611850.0</td>\n",
       "      <td>10049.0</td>\n",
       "      <td>6757900.0</td>\n",
       "      <td>1399510.0</td>\n",
       "      <td>441473.0</td>\n",
       "      <td>16826400.0</td>\n",
       "      <td>17057.0</td>\n",
       "      <td>1764150.0</td>\n",
       "      <td>1608100.0</td>\n",
       "      <td>625310.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>489250.0</td>\n",
       "      <td>6199600.0</td>\n",
       "      <td>2528020.0</td>\n",
       "      <td>39714400.0</td>\n",
       "      <td>8269300.0</td>\n",
       "      <td>499980.0</td>\n",
       "      <td>656680.0</td>\n",
       "      <td>1735800.0</td>\n",
       "      <td>653680.0</td>\n",
       "      <td>7425500.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1007670.0</td>\n",
       "      <td>309620.0</td>\n",
       "      <td>5483700.0</td>\n",
       "      <td>523510.0</td>\n",
       "      <td>2383040.0</td>\n",
       "      <td>574350.0</td>\n",
       "      <td>834401.0</td>\n",
       "      <td>432080.0</td>\n",
       "      <td>70625.0</td>\n",
       "      <td>416280.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>247780.0</td>\n",
       "      <td>2175700.0</td>\n",
       "      <td>5810700.0</td>\n",
       "      <td>38395200.0</td>\n",
       "      <td>7941000.0</td>\n",
       "      <td>222547.0</td>\n",
       "      <td>215810.0</td>\n",
       "      <td>3081070.0</td>\n",
       "      <td>1275940.0</td>\n",
       "      <td>6533000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1539500.0</td>\n",
       "      <td>605701.0</td>\n",
       "      <td>938390.0</td>\n",
       "      <td>125811.0</td>\n",
       "      <td>2648450.0</td>\n",
       "      <td>1417130.0</td>\n",
       "      <td>2070120.0</td>\n",
       "      <td>384321.0</td>\n",
       "      <td>785430.0</td>\n",
       "      <td>481110.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>57248.0</td>\n",
       "      <td>24737700.0</td>\n",
       "      <td>1724240.0</td>\n",
       "      <td>30500700.0</td>\n",
       "      <td>33567500.0</td>\n",
       "      <td>149534.0</td>\n",
       "      <td>421910.0</td>\n",
       "      <td>1562650.0</td>\n",
       "      <td>1251810.0</td>\n",
       "      <td>3136270.0</td>\n",
       "      <td>...</td>\n",
       "      <td>259353.0</td>\n",
       "      <td>414209.0</td>\n",
       "      <td>176200.0</td>\n",
       "      <td>712550.0</td>\n",
       "      <td>2531220.0</td>\n",
       "      <td>4764200.0</td>\n",
       "      <td>152343.0</td>\n",
       "      <td>294470.0</td>\n",
       "      <td>217444.0</td>\n",
       "      <td>491810.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 86 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "geneNamePrimary     HNRNPR         HPR       HPSE          HPX         HRG  \\\n",
       "22                198080.0  10003700.0   674920.0   76598000.0  14859700.0   \n",
       "18                 57248.0  15866200.0  1051780.0   44572000.0  12356200.0   \n",
       "29                394410.0  13968700.0   898040.0   85988000.0   7706700.0   \n",
       "10                356770.0   1646300.0  2588120.0   24667900.0  10165600.0   \n",
       "32                878460.0  17399900.0  2198060.0   45095000.0   6637200.0   \n",
       "9                 183860.0  11954000.0   149170.0   75845000.0  17204500.0   \n",
       "36               1315960.0   4611100.0  2140580.0   39022400.0   5603500.0   \n",
       "21                737800.0  16069700.0  2018930.0   46356000.0   7385400.0   \n",
       "30                275710.0  18816200.0   149170.0   69909000.0  17475700.0   \n",
       "12               5862100.0    488340.0   149170.0   47983400.0   6317100.0   \n",
       "3                1238520.0   7730400.0  1621500.0   44495800.0  13244300.0   \n",
       "20               1217610.0   7429300.0  1435290.0   33928400.0  18207600.0   \n",
       "26               2904370.0  16650000.0   161540.0  106192000.0  25127600.0   \n",
       "34               2118120.0  15869600.0   550020.0   64813000.0  15433100.0   \n",
       "24                882220.0  15511500.0   203860.0  168679000.0  31779900.0   \n",
       "39               1228700.0  19840200.0   379777.0   75066000.0  10682200.0   \n",
       "6                3517950.0   9483300.0   149170.0   65298000.0  15285000.0   \n",
       "4                3512000.0  11013000.0   311353.0   69228000.0  16635600.0   \n",
       "37                568470.0  16763100.0   967790.0   76221000.0  11356200.0   \n",
       "13               5310080.0  18658000.0  1491800.0   81089000.0  11858600.0   \n",
       "31               4010430.0   6409790.0   149170.0   65457000.0  34689700.0   \n",
       "15                489250.0   6199600.0  2528020.0   39714400.0   8269300.0   \n",
       "0                 247780.0   2175700.0  5810700.0   38395200.0   7941000.0   \n",
       "17                 57248.0  24737700.0  1724240.0   30500700.0  33567500.0   \n",
       "\n",
       "geneNamePrimary       HRNR      HSDL2   HSP90AA1   HSP90AB1    HSP90B1  ...  \\\n",
       "22               1044630.0   803050.0   722800.0   835130.0  2177780.0  ...   \n",
       "18                659010.0   896280.0   890310.0   348891.0  4679200.0  ...   \n",
       "29                631690.0   600350.0  1687590.0  1959670.0  3858400.0  ...   \n",
       "10                640742.0   202250.0  3895180.0  1890770.0  7309600.0  ...   \n",
       "32                727743.0   614080.0  2092530.0   949310.0  6349600.0  ...   \n",
       "9                1858990.0  1546200.0   508927.0    59913.0  3804000.0  ...   \n",
       "36                985332.0   731910.0  1000040.0  1365030.0  3585700.0  ...   \n",
       "21                592344.0   400430.0  1761560.0  1113340.0  6818700.0  ...   \n",
       "30                585650.0  3149400.0   163169.0    59913.0  2432610.0  ...   \n",
       "12                831790.0  1297000.0   110474.0    76098.0  1519580.0  ...   \n",
       "3                1100628.0   673890.0  1134020.0   584590.0  4265600.0  ...   \n",
       "20                547683.0   813210.0  1354720.0   512590.0  3825100.0  ...   \n",
       "26               2450070.0   616210.0   145231.0   171807.0  1218830.0  ...   \n",
       "34               1552250.0   944220.0   717930.0   325847.0  3062480.0  ...   \n",
       "24               1342790.0   728370.0   644090.0   248968.0  1553770.0  ...   \n",
       "39                764749.0  1018600.0   110474.0   697550.0  2152660.0  ...   \n",
       "6                 771640.0  1827300.0   447750.0   393074.0  3698500.0  ...   \n",
       "4                1383490.0  1380920.0   665700.0   247849.0  1961030.0  ...   \n",
       "37               1358280.0  1476530.0   983370.0   245333.0  7727200.0  ...   \n",
       "13                253980.0   846880.0  1468510.0  1107180.0  1769940.0  ...   \n",
       "31               3465820.0  1242000.0   278190.0   102550.0  3025270.0  ...   \n",
       "15                499980.0   656680.0  1735800.0   653680.0  7425500.0  ...   \n",
       "0                 222547.0   215810.0  3081070.0  1275940.0  6533000.0  ...   \n",
       "17                149534.0   421910.0  1562650.0  1251810.0  3136270.0  ...   \n",
       "\n",
       "geneNamePrimary       ST13     STAT5B      SYNRG       TFRC    TGFB1I1  \\\n",
       "22                621810.0   641002.0  3934500.0   528100.0  7205700.0   \n",
       "18               1028150.0   112145.0  6886400.0   357746.0  1359120.0   \n",
       "29               1033470.0   327190.0  3479400.0   905510.0  5311100.0   \n",
       "10               1325710.0   357067.0   176200.0   134719.0  2976070.0   \n",
       "32                938790.0   383976.0  3932800.0   350255.0  3108900.0   \n",
       "9                 786660.0    10049.0  6663900.0   313169.0   878750.0   \n",
       "36               1163090.0  1390830.0  4727300.0   434540.0  8194600.0   \n",
       "21                783200.0   394000.0  2993600.0   269724.0  4479950.0   \n",
       "30                305555.0    10049.0  6021500.0   736990.0   412943.0   \n",
       "12                148633.0    10049.0  5252600.0   520790.0    68827.0   \n",
       "3                 568730.0   160337.0  4328300.0   272187.0  1607540.0   \n",
       "20                621410.0   124092.0  2473400.0   975350.0  2138280.0   \n",
       "26                494600.0   168898.0  1267800.0   451890.0  1903880.0   \n",
       "34                506470.0   400614.0  9382400.0    82340.0  3375010.0   \n",
       "24                635560.0   106188.0   176200.0    36323.0  1909020.0   \n",
       "39                533849.0   160820.0  1966600.0   439450.0  3686400.0   \n",
       "6                 760590.0    10049.0  8917700.0   348406.0  1981300.0   \n",
       "4                 763640.0    33443.0  5074100.0   347610.0  1619800.0   \n",
       "37                284509.0    10049.0  9768100.0    92853.0  2799700.0   \n",
       "13               1840520.0    10049.0   176200.0   631380.0   270841.0   \n",
       "31                611850.0    10049.0  6757900.0  1399510.0   441473.0   \n",
       "15               1007670.0   309620.0  5483700.0   523510.0  2383040.0   \n",
       "0                1539500.0   605701.0   938390.0   125811.0  2648450.0   \n",
       "17                259353.0   414209.0   176200.0   712550.0  2531220.0   \n",
       "\n",
       "geneNamePrimary       THBS4      TIMP1      TIMP2        TKT       TLN2  \n",
       "22                 602830.0   175589.0   432830.0  1310900.0   375590.0  \n",
       "18                3735800.0   368210.0   676790.0   313390.0   687130.0  \n",
       "29                4638100.0   469568.0   639440.0  1985700.0   510200.0  \n",
       "10                 173833.0  1905360.0    35751.0   796414.0   100020.0  \n",
       "32                 873290.0   823252.0   168694.0   889510.0   883680.0  \n",
       "9                 1716580.0   308347.0   837250.0    70625.0   274730.0  \n",
       "36                3433580.0   317050.0   459970.0  1322900.0   669560.0  \n",
       "21                 783100.0   874645.0   158259.0   522899.0   390110.0  \n",
       "30                7531400.0    17057.0   556380.0  2333000.0  1583700.0  \n",
       "12                2703670.0    17057.0  1399640.0    70625.0   889400.0  \n",
       "3                 7864400.0   329210.0   523240.0   197355.0   504110.0  \n",
       "20                2765190.0   389310.0   260635.0   448829.0   356700.0  \n",
       "26                6063000.0    17057.0  1138990.0  1092100.0   548310.0  \n",
       "34                5960900.0   171603.0  1726560.0  1067300.0  1140800.0  \n",
       "24                2296180.0    35701.0   434673.0   400460.0   861080.0  \n",
       "39                1341140.0    50095.0   654130.0    70625.0   149410.0  \n",
       "6                 4406780.0    17057.0   788770.0  1341400.0   905770.0  \n",
       "4                 4568200.0    41796.0   690390.0  1515800.0   455630.0  \n",
       "37                 364901.0   251788.0   369307.0    70625.0   431040.0  \n",
       "13                1118810.0    17057.0   362739.0    70625.0   101920.0  \n",
       "31               16826400.0    17057.0  1764150.0  1608100.0   625310.0  \n",
       "15                 574350.0   834401.0   432080.0    70625.0   416280.0  \n",
       "0                 1417130.0  2070120.0   384321.0   785430.0   481110.0  \n",
       "17                4764200.0   152343.0   294470.0   217444.0   491810.0  \n",
       "\n",
       "[24 rows x 86 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# impute 0s, as above\n",
    "\n",
    "imputeWideDFMinOr0(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of classification algorithms\n",
    "\n",
    "classifiers = [\n",
    "    ('HistGradientBoostingClassifier', HistGradientBoostingClassifier(random_state = seed)),\n",
    "    ('CatBoostClassifier', CatBoostClassifier(random_state = seed, verbose = False)),\n",
    "    ('LGBMClassifier', LGBMClassifier(random_state = seed, verbosity = -1)),\n",
    "    ('XGBClassifier', XGBClassifier(random_state = seed)),\n",
    "    ('AdaBoostClassifier', AdaBoostClassifier(random_state = seed)),\n",
    "    ('RandomForestClassifier', RandomForestClassifier(random_state = seed)),\n",
    "    ('BaggingClassifier', BaggingClassifier(random_state = seed)),\n",
    "    ('ExtraTreesClassifier', ExtraTreesClassifier(random_state = seed)),\n",
    "    ('GradientBoostingClassifier', GradientBoostingClassifier(random_state = seed)),\n",
    "    ('SVC', SVC(random_state = seed)),\n",
    "    ('MLPClassifier', MLPClassifier(random_state = seed)),\n",
    "    ('KNeighborsClassifier', KNeighborsClassifier(n_neighbors=3)),\n",
    "    ('GaussianProcessClassifier', GaussianProcessClassifier(random_state = seed)),\n",
    "    ('DecisionTreeClassifier', DecisionTreeClassifier(random_state = seed)),\n",
    "    ('GaussianNB', GaussianNB()),\n",
    "    ('QuadraticDiscriminantAnalysis', QuadraticDiscriminantAnalysis()),\n",
    "    ('LinearSVC', LinearSVC(random_state=seed)),\n",
    "    ('RidgeClassifier', RidgeClassifier(random_state = seed)),\n",
    "    ('SGDClassifier', SGDClassifier(random_state = seed)),\n",
    "    ('XGBClassifier', XGBClassifier(random_state = seed)),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# perform cross validation on every model in the list. Use of a network computer cluster may be required\n",
    "\n",
    "print('\\nCross-Validation:')\n",
    "\n",
    "for j, (name, clf) in enumerate(classifiers):\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresCrossValScore = []\n",
    "    r2_scores = []\n",
    "    pipeline.set_params(Model = clf)\n",
    "    \n",
    "    print('\\n')\n",
    "    print(f'\\n{name} Classifier:\\n')\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        \n",
    "        #print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa: {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### use Optuna to individually tune each model\n",
    "* score based on Matthew Correlation Coefficient (MCC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup optuna logger\n",
    "optuna.logging.get_logger(\"optuna\").addHandler(logging.StreamHandler(sys.stdout))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# CatBoostClassifier, from https://catboost.ai/docs/concepts/python-reference_catboostclassifier\n",
    "\n",
    "def objectiveCatBoostClassifier(trial):\n",
    "    \n",
    "    params = {\n",
    "        'learning_rate': trial.suggest_float('learning_rate', 1e-3, 0.1, log=True),\n",
    "        'iterations': 500,\n",
    "        'depth': trial.suggest_int('depth', 1, 10),\n",
    "        'colsample_bylevel': trial.suggest_float('colsample_bylevel', 0.05, 1.0),\n",
    "        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 0.1, 10),\n",
    "        'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 100),\n",
    "        'random_strength': trial.suggest_float('random_strength', 0.1, 10),\n",
    "        'verbose': False,\n",
    "        'grow_policy': trial.suggest_categorical('grow_policy', ['SymmetricTree', 'Depthwise', 'Lossguide']),\n",
    "        'leaf_estimation_method': trial.suggest_categorical('leaf_estimation_method', ['Newton', 'Gradient']),\n",
    "        'eval_metric': 'MCC'\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = CatBoostClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyCatBoostClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyCatBoostClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyCatBoostClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyCatBoostClassifier, load_if_exists=True)\n",
    "studyCatBoostClassifier.optimize(objectiveCatBoostClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsCatBoostClassifier = studyCatBoostClassifier.best_params\n",
    "best_rmse_scoreCatBoostClassifier = studyCatBoostClassifier.best_value\n",
    "\n",
    "print(f'\\nCatBoostClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyCatBoostClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreCatBoostClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsCatBoostClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyCatBoostClassifier, 'CatBoostClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyCatBoostClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# Histogram-based Gradient Boosting\n",
    "\n",
    "def objectiveHistGradientBoostingClassifier(trial):\n",
    "    params = {\n",
    "        'loss': trial.suggest_categorical('loss', ['log_loss']),\n",
    "        'learning_rate': trial.suggest_loguniform('learning_rate', 0.001,0.1),\n",
    "        'max_iter': trial.suggest_categorical('max_iter', [1000]),\n",
    "        'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 10,200),\n",
    "        'max_depth': trial.suggest_int('max_depth', 2, 30),\n",
    "        'max_bins': trial.suggest_int('max_bins', 100, 255),\n",
    "        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 5,100),\n",
    "        'l2_regularization': trial.suggest_float('l2_regularization', 1e-10,10.0),\n",
    "        'class_weight': trial.suggest_categorical('class_weight', ['balanced', None])\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = HistGradientBoostingClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC \n",
    "    \n",
    "studyName = 'studyHistGradientBoostingClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyHistGradientBoostingClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyHistGradientBoostingClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyHistGradientBoostingClassifier, load_if_exists=True)\n",
    "studyHistGradientBoostingClassifier.optimize(objectiveHistGradientBoostingClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsHistGradientBoostingClassifier = studyHistGradientBoostingClassifier.best_params\n",
    "best_rmse_scoreHistGradientBoostingClassifier = studyHistGradientBoostingClassifier.best_value\n",
    "\n",
    "print(f'\\nHistGradientBoostingClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyHistGradientBoostingClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreHistGradientBoostingClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsHistGradientBoostingClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyHistGradientBoostingClassifier, 'HistGradientBoostingClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyHistGradientBoostingClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# AdaBoostClassifier optimization\n",
    "\n",
    "def objectiveAdaBoostClassifier(trial):\n",
    "    \n",
    "    params = { # this is from sklearn\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 1, 200),\n",
    "        'learning_rate': trial.suggest_loguniform('learning_rate', 0.0001, 0.5),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = AdaBoostClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "\n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC            \n",
    "\n",
    "studyName = 'studyAdaBoostClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyAdaBoostClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyAdaBoostClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyAdaBoostClassifier, load_if_exists=True)\n",
    "studyAdaBoostClassifier.optimize(objectiveAdaBoostClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsAdaBoostClassifier = studyAdaBoostClassifier.best_params\n",
    "best_rmse_scoreAdaBoostClassifier = studyAdaBoostClassifier.best_value\n",
    "\n",
    "print(f'\\nAdaBoostClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyAdaBoostClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreAdaBoostClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsAdaBoostClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyAdaBoostClassifier, 'AdaBoostClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyAdaBoostClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# LGBMClassifier\n",
    "\n",
    "def objectiveLGBMClassifier(trial):\n",
    "    \n",
    "    params = {\n",
    "        'metric': 'rmse',\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 10, 20000),\n",
    "        'boosting_type': trial.suggest_categorical('boosting_type', ['gbdt','dart']),\n",
    "        'reg_alpha': trial.suggest_float('reg_alpha', 1e-5, 10.0, log=True),\n",
    "        'reg_lambda': trial.suggest_float('reg_lambda', 1e-5, 10),\n",
    "        'num_leaves': trial.suggest_int('num_leaves', 1, 1000),\n",
    "        'learning_rate': trial.suggest_loguniform('learning_rate', 0.0001, 0.5),\n",
    "        \"max_depth\": trial.suggest_int(\"max_depth\", 2, 100),\n",
    "        #\"min_split_gain\": trial.suggest_float(\"min_split_gain\", 0, 15),\n",
    "        'subsample': trial.suggest_float('subsample', 0.2, 1.0),\n",
    "        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.3, 1.0),\n",
    "        'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),\n",
    "        'min_data_per_groups' : trial.suggest_int('min_data_per_groups', 1, 100),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = LGBMClassifier(**params, random_state = seed, n_jobs = nJobs, verbosity = -1))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC       \n",
    "\n",
    "studyName = 'studyLGBMClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyLGBMClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyLGBMClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyLGBMClassifier, load_if_exists=True)\n",
    "studyLGBMClassifier.optimize(objectiveLGBMClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsLGBMClassifier = studyLGBMClassifier.best_params\n",
    "best_rmse_scoreLGBMClassifier = studyLGBMClassifier.best_value\n",
    "\n",
    "print(f'\\nLGBMClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyLGBMClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreLGBMClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsLGBMClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyLGBMClassifier, 'LGBMClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyLGBMClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# RandomForestClassifier\n",
    "\n",
    "def objectiveRandomForestClassifier(trial):\n",
    "    params = {\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 2, 500),\n",
    "        'criterion': trial.suggest_categorical('criterion', ['gini', 'entropy', 'log_loss']),\n",
    "        'max_depth': trial.suggest_int('max_depth', 1, 110),\n",
    "        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 10),\n",
    "        'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 30, 10000),\n",
    "        'bootstrap': trial.suggest_categorical('bootstrap', [True, False]),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = RandomForestClassifier(**params, random_state = seed, n_jobs = nJobs))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC \n",
    "         \n",
    "studyName = 'studyRandomForestClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyRandomForestClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyRandomForestClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyRandomForestClassifier, load_if_exists=True)\n",
    "studyRandomForestClassifier.optimize(objectiveRandomForestClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsRandomForestClassifier = studyRandomForestClassifier.best_params\n",
    "best_rmse_scoreRandomForestClassifier = studyRandomForestClassifier.best_value\n",
    "\n",
    "print(f'\\nRandomForestClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyRandomForestClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreRandomForestClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsRandomForestClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyRandomForestClassifier, 'RandomForestClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyRandomForestClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# BaggingClassifier\n",
    "\n",
    "def objectiveBaggingClassifier(trial):\n",
    "    params = {\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 2, 200),\n",
    "        'bootstrap': trial.suggest_categorical('bootstrap', [True, False]),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = BaggingClassifier(**params, random_state = seed, n_jobs = nJobs))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyBaggingClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyBaggingClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyBaggingClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyBaggingClassifier, load_if_exists=True)\n",
    "studyBaggingClassifier.optimize(objectiveBaggingClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsBaggingClassifier = studyBaggingClassifier.best_params\n",
    "best_rmse_scoreBaggingClassifier = studyBaggingClassifier.best_value\n",
    "\n",
    "print(f'\\nBaggingClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyBaggingClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreBaggingClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsBaggingClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyBaggingClassifier, 'BaggingClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyBaggingClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# ExtraTreesClassifier\n",
    "\n",
    "def objectiveExtraTreesClassifier(trial):\n",
    "    params = { # these come from SKLearn\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 2, 400),\n",
    "        'criterion': trial.suggest_categorical('criterion', ['gini', 'entropy', 'log_loss']),\n",
    "        'max_depth': trial.suggest_int('max_depth', 1, 9),\n",
    "        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 10),\n",
    "        'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 10, 10000),\n",
    "        'bootstrap': trial.suggest_categorical('bootstrap', [True, False]),\n",
    "        'min_impurity_decrease': trial.suggest_float('min_impurity_decrease',0.0, 0.01),\n",
    "        'min_weight_fraction_leaf': trial.suggest_uniform('min_weight_fraction_leaf', 0.00001,0.4),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = ExtraTreesClassifier(**params, random_state = seed, n_jobs = nJobs))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyExtraTreesClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyExtraTreesClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyExtraTreesClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyExtraTreesClassifier, load_if_exists=True)\n",
    "studyExtraTreesClassifier.optimize(objectiveExtraTreesClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsExtraTreesClassifier = studyExtraTreesClassifier.best_params\n",
    "best_rmse_scoreExtraTreesClassifier = studyExtraTreesClassifier.best_value\n",
    "\n",
    "print(f'\\nExtraTreesClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyExtraTreesClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreExtraTreesClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsExtraTreesClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyExtraTreesClassifier, 'ExtraTreesClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyExtraTreesClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# GradientBoostingClassifier\n",
    "\n",
    "def objectiveGradientBoostingClassifier(trial):\n",
    "    params = {\n",
    "        'loss': trial.suggest_categorical('loss', ['log_loss', 'exponential']),\n",
    "        'learning_rate': trial.suggest_loguniform('learning_rate', 0.0001, 0.5),\n",
    "        'n_estimators': trial.suggest_int('n_estimators', 20, 1000),\n",
    "        'subsample': trial.suggest_float('subsample', 0.1, 1.0), # fussy\n",
    "        'criterion': trial.suggest_categorical('criterion', ['friedman_mse', 'squared_error']),\n",
    "        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 5),\n",
    "        'max_depth': trial.suggest_int('max_depth', 3, 9),\n",
    "        'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 10, 10000),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = GradientBoostingClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyGradientBoostingClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyGradientBoostingClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyGradientBoostingClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyGradientBoostingClassifier, load_if_exists=True)\n",
    "studyGradientBoostingClassifier.optimize(objectiveGradientBoostingClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsGradientBoostingClassifier = studyGradientBoostingClassifier.best_params\n",
    "best_rmse_scoreGradientBoostingClassifier = studyGradientBoostingClassifier.best_value\n",
    "\n",
    "print(f'\\nGradientBoostingClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyGradientBoostingClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreGradientBoostingClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsGradientBoostingClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyGradientBoostingClassifier, 'GradientBoostingClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyGradientBoostingClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# SVC\n",
    "\n",
    "def objectiveSVC(trial):\n",
    "    params = {\n",
    "        'C': trial.suggest_float('C', 1e-10, 1e10, log=True),\n",
    "        'kernel': trial.suggest_categorical('kernel', ['rbf', 'linear', 'poly', 'sigmoid']),\n",
    "        'degree': trial.suggest_int('degree', 1, 5),\n",
    "        'gamma': trial.suggest_categorical('gamma', ['scale', 'auto']),\n",
    "        'shrinking': trial.suggest_categorical('shrinking', [True, False]),\n",
    "        'probability': trial.suggest_categorical('probability', [True, False]),\n",
    "        'decision_function_shape': trial.suggest_categorical('decision_function_shape', ['ovo', 'ovr']),\n",
    "        'probability': True\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = SVC(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = [] # new!\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC \n",
    "\n",
    "studyName = 'studySVC_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudySVC = 'sqlite:///' + studyName +'.db'\n",
    "studySVC = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudySVC, load_if_exists=True)\n",
    "studySVC.optimize(objectiveSVC, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsSVC = studySVC.best_params\n",
    "best_rmse_scoreSVC = studySVC.best_value\n",
    "\n",
    "print(f'\\nSVC:\\n')\n",
    "print('Number of finished trials:', len(studySVC.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreSVC} \\n')\n",
    "print(f'\\n Best Params = {best_paramsSVC} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studySVC, 'SVCTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studySVC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# MLPClassifier\n",
    "\n",
    "def objectiveMLPClassifier(trial):\n",
    "    params = { # these come from SKLearn\n",
    "        'activation': trial.suggest_categorical('activation', ['identity', 'logistic', 'tanh', 'relu']),\n",
    "        'solver': trial.suggest_categorical('solver', ['lbfgs', 'sgd', 'adam']),\n",
    "        'alpha': trial.suggest_float('alpha', 0.00001, 0.01),\n",
    "        'learning_rate': trial.suggest_categorical('learning_rate', ['constant', 'invscaling', 'adaptive']),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = MLPClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyMLPClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyMLPClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyMLPClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyMLPClassifier, load_if_exists=True)\n",
    "studyMLPClassifier.optimize(objectiveMLPClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsMLPClassifier = studyMLPClassifier.best_params\n",
    "best_rmse_scoreMLPClassifier = studyMLPClassifier.best_value\n",
    "\n",
    "print(f'\\nMLPClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyMLPClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreMLPClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsMLPClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyMLPClassifier, 'MLPClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyMLPClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# KNeighborsClassifier\n",
    "\n",
    "def objectiveKNeighborsClassifier(trial):\n",
    "    params = {\n",
    "        'n_neighbors': trial.suggest_int('n_neighbors', 1,30),\n",
    "        'weights': trial.suggest_categorical(\"weights\", ['uniform', 'distance']),\n",
    "        'metric': trial.suggest_categorical(\"metric\", ['euclidean', 'manhattan', 'minkowski']),\n",
    "        'algorithm': trial.suggest_categorical('algorithm', ['auto', 'ball_tree', 'kd_tree', 'brute']),\n",
    "        'leaf_size': trial.suggest_int('leaf_size', 5,100),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = KNeighborsClassifier(**params, n_jobs = nJobs))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC  \n",
    "\n",
    "studyName = 'studyKNeighborsClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyKNeighborsClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyKNeighborsClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyKNeighborsClassifier, load_if_exists=True)\n",
    "studyKNeighborsClassifier.optimize(objectiveKNeighborsClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsKNeighborsClassifier = studyKNeighborsClassifier.best_params\n",
    "best_rmse_scoreKNeighborsClassifier = studyKNeighborsClassifier.best_value\n",
    "\n",
    "print(f'\\nKNeighborsClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyKNeighborsClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreKNeighborsClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsKNeighborsClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyKNeighborsClassifier, 'KNeighborsClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyKNeighborsClassifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### gaussianProcessClassifier has no parameters to optimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# DecisionTreeClassifier\n",
    "\n",
    "def objectiveDecisionTreeClassifier(trial):\n",
    "    params = {\n",
    "        'criterion': trial.suggest_categorical('criterion', ['gini', 'entropy', 'log_loss']),\n",
    "        'splitter': trial.suggest_categorical('splitter', ['best', 'random']),\n",
    "        'max_depth': trial.suggest_int('max_depth', 1, 6),\n",
    "        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 1000),\n",
    "        'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 2, 10000),\n",
    "        'min_weight_fraction_leaf': trial.suggest_uniform('min_weight_fraction_leaf', 0.0, 0.5), \n",
    "        'ccp_alpha': trial.suggest_float('ccp_alpha', 0.00000001, 1.0, log=True),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = DecisionTreeClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = [] # new!\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC \n",
    "\n",
    "studyName = 'studyDecisionTreeClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyDecisionTreeClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyDecisionTreeClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyDecisionTreeClassifier, load_if_exists=True)\n",
    "studyDecisionTreeClassifier.optimize(objectiveDecisionTreeClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsDecisionTreeClassifier = studyDecisionTreeClassifier.best_params\n",
    "best_rmse_scoreDecisionTreeClassifier = studyDecisionTreeClassifier.best_value\n",
    "\n",
    "print(f'\\nDecisionTreeClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyDecisionTreeClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreDecisionTreeClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsDecisionTreeClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyDecisionTreeClassifier, 'DecisionTreeClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyDecisionTreeClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# GaussianNB\n",
    "\n",
    "def objectiveGaussianNB(trial):\n",
    "    params = {\n",
    "        'var_smoothing': trial.suggest_float('var_smoothing', 1e-13, 1, log = True),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = GaussianNB(**params))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC  \n",
    "\n",
    "studyName = 'studyGaussianNB_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyGaussianNB = 'sqlite:///' + studyName +'.db'\n",
    "studyGaussianNB = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyGaussianNB, load_if_exists=True)\n",
    "studyGaussianNB.optimize(objectiveGaussianNB, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsGaussianNB = studyGaussianNB.best_params\n",
    "best_rmse_scoreGaussianNB = studyGaussianNB.best_value\n",
    "\n",
    "print(f'\\nGaussianNB:\\n')\n",
    "print('Number of finished trials:', len(studyGaussianNB.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreGaussianNB} \\n')\n",
    "print(f'\\n Best Params = {best_paramsGaussianNB} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyGaussianNB, 'GaussianNBTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyGaussianNB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# QuadraticDiscriminantAnalysis\n",
    "\n",
    "def objectiveQuadraticDiscriminantAnalysis(trial):\n",
    "    params = {\n",
    "        'reg_param': trial.suggest_float('reg_param', 0, 1),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = QuadraticDiscriminantAnalysis(**params))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = [] # new!\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC  \n",
    "\n",
    "studyName = 'studyQuadraticDiscriminantAnalysis_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyQuadraticDiscriminantAnalysis = 'sqlite:///' + studyName +'.db'\n",
    "studyQuadraticDiscriminantAnalysis = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyQuadraticDiscriminantAnalysis, load_if_exists=True)\n",
    "studyQuadraticDiscriminantAnalysis.optimize(objectiveQuadraticDiscriminantAnalysis, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsQuadraticDiscriminantAnalysis = studyQuadraticDiscriminantAnalysis.best_params\n",
    "best_rmse_scoreQuadraticDiscriminantAnalysis = studyQuadraticDiscriminantAnalysis.best_value\n",
    "\n",
    "print(f'\\nQuadraticDiscriminantAnalysis:\\n')\n",
    "print('Number of finished trials:', len(studyQuadraticDiscriminantAnalysis.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreQuadraticDiscriminantAnalysis} \\n')\n",
    "print(f'\\n Best Params = {best_paramsQuadraticDiscriminantAnalysis} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyQuadraticDiscriminantAnalysis, 'QuadraticDiscriminantAnalysisTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyQuadraticDiscriminantAnalysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# LinearSVC\n",
    "\n",
    "def objectiveLinearSVC(trial):\n",
    "    params = {\n",
    "        'penalty': trial.suggest_categorical('penalty', ['l1', 'l2']),\n",
    "        'loss': trial.suggest_categorical('loss', ['hinge', 'squared_hinge']),\n",
    "        'dual': trial.suggest_categorical('dual', ['auto', True]),\n",
    "        'C': trial.suggest_float('C', 1e-10, 1e10, log=True),\n",
    "        'intercept_scaling': trial.suggest_float('intercept_scaling', 0.1, 2),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = CalibratedClassifierCV(LinearSVC(**params, random_state = seed)))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = [] # new!\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyLinearSVC_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyLinearSVC = 'sqlite:///' + studyName +'.db'\n",
    "studyLinearSVC = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyLinearSVC, load_if_exists=True)\n",
    "studyLinearSVC.optimize(objectiveLinearSVC, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning, ValueError))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsLinearSVC = studyLinearSVC.best_params\n",
    "best_rmse_scoreLinearSVC = studyLinearSVC.best_value\n",
    "\n",
    "print(f'\\nLinearSVC:\\n')\n",
    "print('Number of finished trials:', len(studyLinearSVC.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreLinearSVC} \\n')\n",
    "print(f'\\n Best Params = {best_paramsLinearSVC} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyLinearSVC, 'LinearSVCTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyLinearSVC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# RidgeClassifier\n",
    "\n",
    "def objectiveRidgeClassifier(trial):\n",
    "    params = {\n",
    "        'alpha': trial.suggest_float('alpha', 0, 10),\n",
    "        'fit_intercept': trial.suggest_categorical('fit_intercept', [True, False]),\n",
    "        'solver': trial.suggest_categorical('solver', ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = RidgeClassifier(**params, random_state = seed))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studyRidgeClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyRidgeClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyRidgeClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyRidgeClassifier, load_if_exists=True)\n",
    "studyRidgeClassifier.optimize(objectiveRidgeClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsRidgeClassifier = studyRidgeClassifier.best_params\n",
    "best_rmse_scoreRidgeClassifier = studyRidgeClassifier.best_value\n",
    "\n",
    "print(f'\\nRidgeClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyRidgeClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreRidgeClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsRidgeClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyRidgeClassifier, 'RidgeClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyRidgeClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# SGDClassifier\n",
    "\n",
    "def objectiveSGDClassifier(trial):\n",
    "    params = {\n",
    "        'loss': trial.suggest_categorical('loss', ['log_loss', 'modified_huber']),\n",
    "        'penalty': trial.suggest_categorical('penalty', ['l2', 'l1', 'elasticnet']),\n",
    "        'alpha': trial.suggest_float('alpha', 0.00001, 0.001, log=True),\n",
    "        'l1_ratio': trial.suggest_float('l1_ratio', 0.05, 0.95),\n",
    "        'power_t': trial.suggest_float('power_t', -2, 2),\n",
    "        'fit_intercept': trial.suggest_categorical('fit_intercept', [True, False]),\n",
    "    }\n",
    "    \n",
    "    pipeline.set_params(Model = SGDClassifier(**params, random_state = seed, n_jobs = nJobs))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = [] # new!\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC\n",
    "\n",
    "studyName = 'studySGDClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudySGDClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studySGDClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudySGDClassifier, load_if_exists=True)\n",
    "studySGDClassifier.optimize(objectiveSGDClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsSGDClassifier = studySGDClassifier.best_params\n",
    "best_rmse_scoreSGDClassifier = studySGDClassifier.best_value\n",
    "\n",
    "print(f'\\nSGDClassifier:\\n')\n",
    "print('Number of finished trials:', len(studySGDClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreSGDClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsSGDClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studySGDClassifier, 'SGDClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studySGDClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# XGBClassifier\n",
    "\n",
    "def objectiveXGBClassifier(trial):\n",
    "    params = {\n",
    "        'booster': trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n",
    "        \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "        \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "        'max_depth': trial.suggest_int(\"max_depth\", 2, 10),\n",
    "        'n_estimators': trial.suggest_int(\"n_estimators\", 1, 150),\n",
    "        'learning_rate': trial.suggest_uniform('learning_rate', 0.0000001, 1),\n",
    "        'subsample': trial.suggest_uniform('subsample', 0.0001, 1.0),\n",
    "        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "    }\n",
    "\n",
    "    if params[\"booster\"] in [\"gbtree\", \"dart\"]:\n",
    "        params[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n",
    "        params[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n",
    "        params[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n",
    "        params[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n",
    "        params[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "\n",
    "    if params[\"booster\"] == \"dart\":\n",
    "        params[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n",
    "        params[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n",
    "        params[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n",
    "        params[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n",
    "        \n",
    "    pipeline.set_params(Model = XGBClassifier(**params, random_state = seed, nthread = nJobs))\n",
    "    scores = []\n",
    "    scoresMCC = []\n",
    "    scoresF1Weighted = []\n",
    "    scoresAccuracy = []\n",
    "    scoresROC = [] # new!\n",
    "    scoresCrossValScore = []\n",
    "    \n",
    "    for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "        \n",
    "        pipeline.fit(X_train, y_train)\n",
    "        y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "        kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "        mcc = matthews_corrcoef(y_val, y_pred)\n",
    "        f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "        predictions = pipeline.predict_proba(X_val)[:,1] # new!\n",
    "        auc = roc_auc_score(y_val, predictions)\n",
    "        accuracy = balanced_accuracy_score(y_val, y_pred)\n",
    "        crossValScore = cross_val_score(pipeline, X_train, y_train, cv=3)\n",
    "        \n",
    "        print(f'Fold {i + 1}:\\n')\n",
    "        \n",
    "        scores.append(kappa)\n",
    "        scoresMCC.append(mcc)\n",
    "        scoresF1Weighted.append(f1Weighted)\n",
    "        scoresAccuracy.append(accuracy)\n",
    "        scoresROC.append(auc)\n",
    "        scoresCrossValScore.append(crossValScore)\n",
    "        fold_std = np.std(scores)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        fold_scoreROC = np.std(scoresROC)\n",
    "        fold_scoreCrossValScore = np.std(scoresCrossValScore)\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        mean_scoreCrossValScore = np.mean(scoresCrossValScore)\n",
    "           \n",
    "        print('* * * * * * * * * * * * * * * * * * * * * * * * * * * *\\n')\n",
    "        print(f'  Mean Quadratic Weighted Kappa: {mean_score:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient: {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score: {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized): {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  ROC: {mean_scoreROC:.4f}')\n",
    "        print(f'\\n  CrossValScore: {mean_scoreCrossValScore:.4f} \\u00B1 {fold_scoreCrossValScore:.4f}')\n",
    "        print('\\n')\n",
    "        \n",
    "    return mean_scoreMCC   \n",
    "\n",
    "studyName = 'studyXGBClassifier_' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "storagestudyXGBClassifier = 'sqlite:///' + studyName +'.db'\n",
    "studyXGBClassifier = optuna.create_study(study_name = studyName, direction='maximize', storage = storagestudyXGBClassifier, load_if_exists=True)\n",
    "studyXGBClassifier.optimize(objectiveXGBClassifier, n_trials = 100, show_progress_bar = True, catch=(FloatingPointError, RuntimeWarning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "best_paramsXGBClassifier = studyXGBClassifier.best_params\n",
    "best_rmse_scoreXGBClassifier = studyXGBClassifier.best_value\n",
    "\n",
    "print(f'\\nXGBClassifier:\\n')\n",
    "print('Number of finished trials:', len(studyXGBClassifier.trials))\n",
    "print(f'\\n Best RMSE score = {best_rmse_scoreXGBClassifier} \\n')\n",
    "print(f'\\n Best Params = {best_paramsXGBClassifier} \\n')\n",
    "\n",
    "# save\n",
    "joblib.dump(studyXGBClassifier, 'XGBClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(studyXGBClassifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate each tuned classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 863,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load optuna tuned classifiers AS PIPELINES\n",
    "tunedClassifierList = []\n",
    "\n",
    "CatBoostClassifierTuned = joblib.load('CatBoostClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "HistGradientBoostingClassifierTuned = joblib.load('HistGradientBoostingClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "LGBMClassifierTuned = joblib.load('LGBMClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "AdaBoostClassifierTuned = joblib.load('AdaBoostClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "RandomForestClassifierTuned = joblib.load('RandomForestClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "BaggingClassifierTuned = joblib.load('BaggingClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')#ExtraTreesClassifierTuned = joblib.load('ExtraTreesClassifierTuned_target-' + target + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "ExtraTreesClassifierTuned = joblib.load('ExtraTreesClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')#ExtraTreesClassifierTuned = joblib.load('ExtraTreesClassifierTuned_target-' + target + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "GradientBoostingClassifierTuned = joblib.load('GradientBoostingClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "SVCTuned = joblib.load('SVCTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "MLPClassifierTuned = joblib.load('MLPClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "KNeighborsClassifierTuned = joblib.load('KNeighborsClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "GaussianProcessClassifierTuned = GaussianProcessClassifier(random_state = seed) #has nothing to tune\n",
    "DecisionTreeClassifierTuned = joblib.load('DecisionTreeClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "GaussianNBTuned = joblib.load('GaussianNBTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "QuadraticDiscriminantAnalysisTuned = joblib.load('QuadraticDiscriminantAnalysisTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "LinearSVCTuned = joblib.load('LinearSVCTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "RidgeClassifierTuned = joblib.load('RidgeClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "SGDClassifierTuned = joblib.load('SGDClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')\n",
    "XGBClassifierTuned = joblib.load('XGBClassifierTuned_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 864,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lists to summarize results\n",
    "\n",
    "tunedClassifierList = []\n",
    "seedList = []\n",
    "trainFracList = []\n",
    "mean_scoreKappaTunedClassifierList = []\n",
    "mean_scoreMCCTunedClassifierList= []\n",
    "mean_scoreF1WeightedTunedClassifierList = []\n",
    "mean_scoreAccuracyTunedClassifierList = []\n",
    "mean_scoreROCTunedClassifierList = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# catboost with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned CatBoostClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = CatBoostClassifier(**CatBoostClassifierTuned.best_params,random_state = seed, verbose = False))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "    feature_importance.append(pipeline[-1].feature_importances_)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('CatBoostClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# HistGradientBoostingClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned HistGradientBoostingClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = HistGradientBoostingClassifier(**HistGradientBoostingClassifierTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('HistGradientBoostingClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# LGBMClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned LGBMClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = LGBMClassifier(**LGBMClassifierTuned.best_params,random_state = seed, n_jobs=nJobs, verbosity = -1))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "    feature_importance.append(pipeline[-1].feature_importances_)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('LGBMClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# AdaBoostClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned AdaBoostClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = AdaBoostClassifier(**AdaBoostClassifierTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "    feature_importance.append(pipeline[-1].feature_importances_)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('AdaBoostClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# RandomForestClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned RandomForestClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = RandomForestClassifier(**RandomForestClassifierTuned.best_params,random_state = seed, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "    feature_importance.append(pipeline[-1].feature_importances_)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('RandomForestClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# BaggingClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned BaggingClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = BaggingClassifier(**BaggingClassifierTuned.best_params,random_state = seed, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('BaggingClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# ExtraTreesClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned ExtraTreesClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = ExtraTreesClassifier(**ExtraTreesClassifierTuned.best_params,random_state = seed, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "    feature_importance.append(pipeline[-1].feature_importances_)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('ExtraTreesClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# GradientBoostingClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned GradientBoostingClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = GradientBoostingClassifier(**GradientBoostingClassifierTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "    feature_importance.append(pipeline[-1].feature_importances_)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('GradientBoostingClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# SVC with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned SVC Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = SVC(**SVCTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('SVC')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# MLPClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned MLPClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = MLPClassifier(**MLPClassifierTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('MLPClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# KNeighborsClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned KNeighborsClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = KNeighborsClassifier(**KNeighborsClassifierTuned.best_params, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('KNeighborsClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# GaussianProcessClassifier, which has no parameters to optimize\n",
    "\n",
    "print('\\nTuned gaussianProcessClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = GaussianProcessClassifier(random_state = seed, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('GaussianProcessClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# DecisionTreeClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned DecisionTreeClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = DecisionTreeClassifier(**DecisionTreeClassifierTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('DecisionTreeClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# GaussianNB with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned GaussianNB Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = GaussianNB(**GaussianNBTuned.best_params))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('GaussianNB')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# QuadraticDiscriminantAnalysis with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned QuadraticDiscriminantAnalysis Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = QuadraticDiscriminantAnalysis(**QuadraticDiscriminantAnalysisTuned.best_params))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('QuadraticDiscriminantAnalysis')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# LinearSVC with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned LinearSVC Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = LinearSVC(**LinearSVCTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('LinearSVC')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# RidgeClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned RidgeClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = RidgeClassifier(**RidgeClassifierTuned.best_params,random_state = seed))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('RidgeClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# SGDClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned SGDClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = SGDClassifier(**SGDClassifierTuned.best_params,random_state = seed, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('SGDClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# XGBClassifier with optuna-tuned parameters\n",
    "\n",
    "print('\\nTuned XGBClassifier Cross-Validation:')\n",
    "scores = []\n",
    "feature_importance = []\n",
    "print('\\n')\n",
    "scores = []\n",
    "scoresMCC = []\n",
    "scoresF1Weighted = []\n",
    "scoresAccuracy = []\n",
    "scoresROC = []\n",
    "\n",
    "for i, (train_index, val_index) in enumerate(cv_splits):\n",
    "    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n",
    "\n",
    "    pipeline.set_params(Model = XGBClassifier(**XGBClassifierTuned.best_params,random_state = seed, n_jobs=nJobs))\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    y_pred = pipeline.predict(X_val)\n",
    "        \n",
    "    kappa = cohen_kappa_score(y_val, y_pred, weights = 'quadratic')\n",
    "    mcc = matthews_corrcoef(y_val, y_pred)\n",
    "    f1Weighted = f1_score(y_val, y_pred, average = 'weighted')\n",
    "    accuracy = accuracy_score(y_val, y_pred)\n",
    "    predictions = pipeline.predict_proba(X_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, predictions)\n",
    "        \n",
    "    scores.append(kappa)\n",
    "    scoresMCC.append(mcc)\n",
    "    scoresF1Weighted.append(f1Weighted)\n",
    "    scoresAccuracy.append(accuracy)\n",
    "    scoresROC.append(auc)\n",
    "        \n",
    " #   print('===================================================')\n",
    "    \n",
    "    if i == len(cv_splits) - 1:\n",
    "        mean_score = np.mean(scores)\n",
    "        fold_std = np.std(scores)\n",
    "        mean_scoreMCC = np.mean(scoresMCC)\n",
    "        fold_stdMCC = np.std(scoresMCC)\n",
    "        mean_scoreF1Weighted = np.mean(scoresF1Weighted)\n",
    "        fold_stdF1Weighted = np.std(scoresF1Weighted)\n",
    "        mean_scoreAccuracy = np.mean(scoresAccuracy)\n",
    "        fold_stdAccuracy = np.std(scoresAccuracy)\n",
    "        mean_scoreROC = np.mean(scoresROC)\n",
    "        fold_stdROC = np.std(scoresROC)\n",
    "        \n",
    "        print(f'\\n  Mean Quadratic Weighted Kappa = = {mean_score:.4f} \\u00B1 {fold_std:.4f}')\n",
    "        print(f'\\n  Matthews correlation coefficient = = {mean_scoreMCC:.4f} \\u00B1 {fold_stdMCC:.4f}')\n",
    "        print(f'\\n  F1 weighted score = = {mean_scoreF1Weighted:.4f} \\u00B1 {fold_stdF1Weighted:.4f}')\n",
    "        print(f'\\n  Accuracy (normalized) = = {mean_scoreAccuracy:.4f} \\u00B1 {fold_stdAccuracy:.4f}')\n",
    "        print(f'\\n  AUROC = = {mean_scoreROC:.4f} \\u00B1 {fold_stdROC:.4f}')\n",
    "\n",
    "        tunedClassifierList.append('XGBClassifier')\n",
    "        seedList.append(seed)\n",
    "        trainFracList.append(trainFrac)\n",
    "        mean_scoreKappaTunedClassifierList.append(mean_score)\n",
    "        mean_scoreMCCTunedClassifierList.append(mean_scoreMCC)\n",
    "        mean_scoreF1WeightedTunedClassifierList.append(mean_scoreF1Weighted)\n",
    "        mean_scoreAccuracyTunedClassifierList.append(mean_scoreAccuracy)\n",
    "        mean_scoreROCTunedClassifierList.append(mean_scoreROC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply the individually tuned classifiers to the validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 871,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# load from saved\n",
    "dfValidationFromDisk = pd.read_excel('dfValidation_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "df = dfValidationFromDisk\n",
    "\n",
    "y_validationList = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 872,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keep only selected features\n",
    "\n",
    "fileName = '_selectedFeaturesRFECV_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "\n",
    "with open(fileName + '.json', 'r') as f:\n",
    "    selected_featuresFromDisk = json.load(f)\n",
    "\n",
    "colsToKeep = []\n",
    "colsToKeep = copy.deepcopy(selected_featuresFromDisk)\n",
    "colsToKeep.append(target)\n",
    "\n",
    "df = df[colsToKeep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 874,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "X shape: (6, 86)\n",
      "\n",
      "\n",
      "6 Samples \n",
      "\n",
      "\n",
      "86 Attributes \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HNRNPR</th>\n",
       "      <th>HPR</th>\n",
       "      <th>HPSE</th>\n",
       "      <th>HPX</th>\n",
       "      <th>HRG</th>\n",
       "      <th>HRNR</th>\n",
       "      <th>HSDL2</th>\n",
       "      <th>HSP90AA1</th>\n",
       "      <th>HSP90AB1</th>\n",
       "      <th>HSP90B1</th>\n",
       "      <th>HSPA1A</th>\n",
       "      <th>HSPA5</th>\n",
       "      <th>HSPA6</th>\n",
       "      <th>HSPA8</th>\n",
       "      <th>HTT</th>\n",
       "      <th>HYOU1</th>\n",
       "      <th>IBSP</th>\n",
       "      <th>IDH2</th>\n",
       "      <th>IGFALS</th>\n",
       "      <th>IGHG3</th>\n",
       "      <th>IGHG4</th>\n",
       "      <th>IGHV1-18</th>\n",
       "      <th>IGHV3-13</th>\n",
       "      <th>IGHV3-15</th>\n",
       "      <th>IGHV3-20</th>\n",
       "      <th>IGHV3-35</th>\n",
       "      <th>IGHV3-49</th>\n",
       "      <th>IGHV3-72</th>\n",
       "      <th>IGHV4-28</th>\n",
       "      <th>IGHV5-51</th>\n",
       "      <th>IGKC</th>\n",
       "      <th>IGKV1-16</th>\n",
       "      <th>IGKV1-17</th>\n",
       "      <th>IGKV1-33</th>\n",
       "      <th>IGKV1-5</th>\n",
       "      <th>IGKV2-24</th>\n",
       "      <th>IGKV3-20</th>\n",
       "      <th>IGKV4-1</th>\n",
       "      <th>INHBE</th>\n",
       "      <th>IQGAP2</th>\n",
       "      <th>ISOC1</th>\n",
       "      <th>ITM2B</th>\n",
       "      <th>ITPR1</th>\n",
       "      <th>JCHAIN</th>\n",
       "      <th>KGD4</th>\n",
       "      <th>MBP</th>\n",
       "      <th>MICAL1</th>\n",
       "      <th>MPP1</th>\n",
       "      <th>NFKB1</th>\n",
       "      <th>PKN1</th>\n",
       "      <th>PLXDC2</th>\n",
       "      <th>RASA1</th>\n",
       "      <th>RFTN1</th>\n",
       "      <th>RIPOR3</th>\n",
       "      <th>RNF121</th>\n",
       "      <th>ROCK2</th>\n",
       "      <th>RPLP1</th>\n",
       "      <th>RPLP2</th>\n",
       "      <th>RSU1</th>\n",
       "      <th>S100A4</th>\n",
       "      <th>SAA1</th>\n",
       "      <th>SAA2</th>\n",
       "      <th>SEPTIN11</th>\n",
       "      <th>SEPTIN2</th>\n",
       "      <th>SEPTIN6</th>\n",
       "      <th>SEPTIN7</th>\n",
       "      <th>SERPINA4</th>\n",
       "      <th>SERPINA6</th>\n",
       "      <th>SERPINB6</th>\n",
       "      <th>SERPINE1</th>\n",
       "      <th>SH3BGRL3</th>\n",
       "      <th>SH3GL1</th>\n",
       "      <th>SLC25A5</th>\n",
       "      <th>SLC2A1</th>\n",
       "      <th>SPP2</th>\n",
       "      <th>SPTA1</th>\n",
       "      <th>ST13</th>\n",
       "      <th>STAT5B</th>\n",
       "      <th>SYNRG</th>\n",
       "      <th>TFRC</th>\n",
       "      <th>TGFB1I1</th>\n",
       "      <th>THBS4</th>\n",
       "      <th>TIMP1</th>\n",
       "      <th>TIMP2</th>\n",
       "      <th>TKT</th>\n",
       "      <th>TLN2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1232310</td>\n",
       "      <td>9168200.0</td>\n",
       "      <td>36900100</td>\n",
       "      <td>3278440</td>\n",
       "      <td>NaN</td>\n",
       "      <td>131538</td>\n",
       "      <td>8513500.0</td>\n",
       "      <td>3415300.0</td>\n",
       "      <td>10903700</td>\n",
       "      <td>2181840.0</td>\n",
       "      <td>11793500</td>\n",
       "      <td>6058100</td>\n",
       "      <td>28238300</td>\n",
       "      <td>242726</td>\n",
       "      <td>6423000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7651000</td>\n",
       "      <td>307268</td>\n",
       "      <td>32247900</td>\n",
       "      <td>41904000</td>\n",
       "      <td>783480</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32495.0</td>\n",
       "      <td>138770.0</td>\n",
       "      <td>454670</td>\n",
       "      <td>48893.0</td>\n",
       "      <td>1724209.0</td>\n",
       "      <td>968950</td>\n",
       "      <td>6861770</td>\n",
       "      <td>109894000</td>\n",
       "      <td>476760.0</td>\n",
       "      <td>136070</td>\n",
       "      <td>2530739</td>\n",
       "      <td>169550</td>\n",
       "      <td>168290.0</td>\n",
       "      <td>1891180</td>\n",
       "      <td>1091810</td>\n",
       "      <td>104339.0</td>\n",
       "      <td>5097800.0</td>\n",
       "      <td>1304830.0</td>\n",
       "      <td>27768</td>\n",
       "      <td>1533800</td>\n",
       "      <td>1376300</td>\n",
       "      <td>390490</td>\n",
       "      <td>41052</td>\n",
       "      <td>198972</td>\n",
       "      <td>2698020.0</td>\n",
       "      <td>197959.0</td>\n",
       "      <td>195335</td>\n",
       "      <td>484050</td>\n",
       "      <td>1476770</td>\n",
       "      <td>46095.0</td>\n",
       "      <td>805750.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1748140</td>\n",
       "      <td>628442</td>\n",
       "      <td>391381</td>\n",
       "      <td>8788200.0</td>\n",
       "      <td>1682200.0</td>\n",
       "      <td>36356300</td>\n",
       "      <td>2276888</td>\n",
       "      <td>2080530</td>\n",
       "      <td>3235820.0</td>\n",
       "      <td>1859160.0</td>\n",
       "      <td>3195140</td>\n",
       "      <td>184207</td>\n",
       "      <td>410222</td>\n",
       "      <td>2410290</td>\n",
       "      <td>851690</td>\n",
       "      <td>3364560</td>\n",
       "      <td>1090580.0</td>\n",
       "      <td>673450.0</td>\n",
       "      <td>26142.0</td>\n",
       "      <td>1211723</td>\n",
       "      <td>636330</td>\n",
       "      <td>1354700</td>\n",
       "      <td>204402.0</td>\n",
       "      <td>568940.0</td>\n",
       "      <td>237276.0</td>\n",
       "      <td>3629600</td>\n",
       "      <td>223152</td>\n",
       "      <td>3847440.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>997620</td>\n",
       "      <td>596740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22515.0</td>\n",
       "      <td>2178090</td>\n",
       "      <td>9495600.0</td>\n",
       "      <td>32478500</td>\n",
       "      <td>4416770</td>\n",
       "      <td>NaN</td>\n",
       "      <td>300108</td>\n",
       "      <td>7319600.0</td>\n",
       "      <td>3040670.0</td>\n",
       "      <td>12274900</td>\n",
       "      <td>2899350.0</td>\n",
       "      <td>9931600</td>\n",
       "      <td>5409900</td>\n",
       "      <td>23876700</td>\n",
       "      <td>366210</td>\n",
       "      <td>5668000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6627500</td>\n",
       "      <td>193002</td>\n",
       "      <td>17108400</td>\n",
       "      <td>2300830</td>\n",
       "      <td>588700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>196790</td>\n",
       "      <td>NaN</td>\n",
       "      <td>394627.0</td>\n",
       "      <td>401433</td>\n",
       "      <td>1292632</td>\n",
       "      <td>66016000</td>\n",
       "      <td>281599.0</td>\n",
       "      <td>343700</td>\n",
       "      <td>808950</td>\n",
       "      <td>123320</td>\n",
       "      <td>94968.0</td>\n",
       "      <td>2094740</td>\n",
       "      <td>484380</td>\n",
       "      <td>155302.0</td>\n",
       "      <td>4875300.0</td>\n",
       "      <td>1307110.0</td>\n",
       "      <td>24413</td>\n",
       "      <td>1725500</td>\n",
       "      <td>928811</td>\n",
       "      <td>593890</td>\n",
       "      <td>70315</td>\n",
       "      <td>238439</td>\n",
       "      <td>1334160.0</td>\n",
       "      <td>331001.0</td>\n",
       "      <td>94747</td>\n",
       "      <td>578823</td>\n",
       "      <td>1653879</td>\n",
       "      <td>53919.0</td>\n",
       "      <td>459980.0</td>\n",
       "      <td>2788077.0</td>\n",
       "      <td>1756610</td>\n",
       "      <td>371750</td>\n",
       "      <td>819555</td>\n",
       "      <td>16325600.0</td>\n",
       "      <td>934060.0</td>\n",
       "      <td>47781100</td>\n",
       "      <td>7323820</td>\n",
       "      <td>1617730</td>\n",
       "      <td>2853890.0</td>\n",
       "      <td>1845010.0</td>\n",
       "      <td>3579430</td>\n",
       "      <td>102535</td>\n",
       "      <td>417907</td>\n",
       "      <td>828510</td>\n",
       "      <td>926390</td>\n",
       "      <td>3555310</td>\n",
       "      <td>899870.0</td>\n",
       "      <td>729909.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3764250</td>\n",
       "      <td>854560</td>\n",
       "      <td>943430</td>\n",
       "      <td>260069.0</td>\n",
       "      <td>992464.0</td>\n",
       "      <td>208263.0</td>\n",
       "      <td>2933700</td>\n",
       "      <td>6148</td>\n",
       "      <td>2713100.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>820950</td>\n",
       "      <td>587058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3561500.0</td>\n",
       "      <td>33189900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>60702000</td>\n",
       "      <td>14182800</td>\n",
       "      <td>865020.0</td>\n",
       "      <td>1636800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2778970</td>\n",
       "      <td>NaN</td>\n",
       "      <td>778140</td>\n",
       "      <td>19740</td>\n",
       "      <td>275411</td>\n",
       "      <td>1156200</td>\n",
       "      <td>841820</td>\n",
       "      <td>4096300.0</td>\n",
       "      <td>81616</td>\n",
       "      <td>1502200</td>\n",
       "      <td>8133800</td>\n",
       "      <td>1583400</td>\n",
       "      <td>223140</td>\n",
       "      <td>77621.0</td>\n",
       "      <td>102280.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>225950</td>\n",
       "      <td>365220.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>268510</td>\n",
       "      <td>2188260</td>\n",
       "      <td>340584000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>299830</td>\n",
       "      <td>663820</td>\n",
       "      <td>977060</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1242150</td>\n",
       "      <td>3134320</td>\n",
       "      <td>3643760.0</td>\n",
       "      <td>35443.0</td>\n",
       "      <td>175103.0</td>\n",
       "      <td>179410</td>\n",
       "      <td>753410</td>\n",
       "      <td>16003900</td>\n",
       "      <td>1627900</td>\n",
       "      <td>9436000</td>\n",
       "      <td>219860</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>111540</td>\n",
       "      <td>1637490</td>\n",
       "      <td>1746100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1268600.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>366613</td>\n",
       "      <td>175700</td>\n",
       "      <td>85160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16883100</td>\n",
       "      <td>1218700</td>\n",
       "      <td>62080</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>145100</td>\n",
       "      <td>5428200</td>\n",
       "      <td>628940</td>\n",
       "      <td>762270</td>\n",
       "      <td>5605200</td>\n",
       "      <td>259860</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1448264.0</td>\n",
       "      <td>96828000</td>\n",
       "      <td>1449040</td>\n",
       "      <td>59265</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12265000.0</td>\n",
       "      <td>1186050.0</td>\n",
       "      <td>179924</td>\n",
       "      <td>6187500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>752190.0</td>\n",
       "      <td>3756700</td>\n",
       "      <td>1130700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1525910.0</td>\n",
       "      <td>16314500</td>\n",
       "      <td>192014.0</td>\n",
       "      <td>57405000</td>\n",
       "      <td>10100700</td>\n",
       "      <td>772688.0</td>\n",
       "      <td>1092100</td>\n",
       "      <td>757990.0</td>\n",
       "      <td>467421.0</td>\n",
       "      <td>3004580</td>\n",
       "      <td>352530.0</td>\n",
       "      <td>2624350</td>\n",
       "      <td>345340</td>\n",
       "      <td>4449100</td>\n",
       "      <td>642560</td>\n",
       "      <td>688300</td>\n",
       "      <td>517470.0</td>\n",
       "      <td>1782830</td>\n",
       "      <td>1219970</td>\n",
       "      <td>33539900</td>\n",
       "      <td>6833340</td>\n",
       "      <td>180370</td>\n",
       "      <td>56461.0</td>\n",
       "      <td>148980.0</td>\n",
       "      <td>1243100.0</td>\n",
       "      <td>176060</td>\n",
       "      <td>401330.0</td>\n",
       "      <td>30111.0</td>\n",
       "      <td>18472</td>\n",
       "      <td>3114300</td>\n",
       "      <td>233071000</td>\n",
       "      <td>146145.0</td>\n",
       "      <td>205720</td>\n",
       "      <td>794520</td>\n",
       "      <td>1679650</td>\n",
       "      <td>4135967.0</td>\n",
       "      <td>819420</td>\n",
       "      <td>11033070</td>\n",
       "      <td>438879.0</td>\n",
       "      <td>632110.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>187710</td>\n",
       "      <td>689402</td>\n",
       "      <td>4575710</td>\n",
       "      <td>2616400</td>\n",
       "      <td>5688400</td>\n",
       "      <td>37495</td>\n",
       "      <td>950950.0</td>\n",
       "      <td>219390.0</td>\n",
       "      <td>75238</td>\n",
       "      <td>158828</td>\n",
       "      <td>599990</td>\n",
       "      <td>348490.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2816200.0</td>\n",
       "      <td>282169</td>\n",
       "      <td>1462570</td>\n",
       "      <td>2512750</td>\n",
       "      <td>2598950.0</td>\n",
       "      <td>60538.0</td>\n",
       "      <td>17736559</td>\n",
       "      <td>2528797</td>\n",
       "      <td>43388</td>\n",
       "      <td>165510.0</td>\n",
       "      <td>362334.0</td>\n",
       "      <td>511704</td>\n",
       "      <td>1996640</td>\n",
       "      <td>815585</td>\n",
       "      <td>313590</td>\n",
       "      <td>162041</td>\n",
       "      <td>568491</td>\n",
       "      <td>329616.0</td>\n",
       "      <td>599876.0</td>\n",
       "      <td>1664660.0</td>\n",
       "      <td>41016000</td>\n",
       "      <td>2034830</td>\n",
       "      <td>969550</td>\n",
       "      <td>102922.0</td>\n",
       "      <td>2821300.0</td>\n",
       "      <td>234748.0</td>\n",
       "      <td>1492000</td>\n",
       "      <td>13833800</td>\n",
       "      <td>58936.0</td>\n",
       "      <td>1173550.0</td>\n",
       "      <td>476450</td>\n",
       "      <td>360690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1287450.0</td>\n",
       "      <td>5590200</td>\n",
       "      <td>1460620.0</td>\n",
       "      <td>25920400</td>\n",
       "      <td>14014800</td>\n",
       "      <td>715937.0</td>\n",
       "      <td>321590</td>\n",
       "      <td>1200970.0</td>\n",
       "      <td>552460.0</td>\n",
       "      <td>5117300</td>\n",
       "      <td>997670.0</td>\n",
       "      <td>5067900</td>\n",
       "      <td>328720</td>\n",
       "      <td>5452800</td>\n",
       "      <td>691380</td>\n",
       "      <td>1041300</td>\n",
       "      <td>22278.0</td>\n",
       "      <td>6222200</td>\n",
       "      <td>402730</td>\n",
       "      <td>30916900</td>\n",
       "      <td>3851530</td>\n",
       "      <td>157280</td>\n",
       "      <td>272160.0</td>\n",
       "      <td>255350.0</td>\n",
       "      <td>1542600.0</td>\n",
       "      <td>346030</td>\n",
       "      <td>256152.0</td>\n",
       "      <td>557610.0</td>\n",
       "      <td>186494</td>\n",
       "      <td>1698300</td>\n",
       "      <td>197161000</td>\n",
       "      <td>325038.0</td>\n",
       "      <td>642850</td>\n",
       "      <td>2318040</td>\n",
       "      <td>1787626</td>\n",
       "      <td>99504.0</td>\n",
       "      <td>1160543</td>\n",
       "      <td>2277260</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1733330.0</td>\n",
       "      <td>350911.0</td>\n",
       "      <td>229550</td>\n",
       "      <td>1155360</td>\n",
       "      <td>4674720</td>\n",
       "      <td>1280532</td>\n",
       "      <td>591340</td>\n",
       "      <td>46762</td>\n",
       "      <td>508030.0</td>\n",
       "      <td>20316.0</td>\n",
       "      <td>71232</td>\n",
       "      <td>1052690</td>\n",
       "      <td>21033</td>\n",
       "      <td>113384.0</td>\n",
       "      <td>200318.0</td>\n",
       "      <td>2468200.0</td>\n",
       "      <td>665150</td>\n",
       "      <td>221270</td>\n",
       "      <td>856950</td>\n",
       "      <td>5710930.0</td>\n",
       "      <td>95539.0</td>\n",
       "      <td>4384700</td>\n",
       "      <td>434379</td>\n",
       "      <td>418750</td>\n",
       "      <td>985310.0</td>\n",
       "      <td>930720.0</td>\n",
       "      <td>1189740</td>\n",
       "      <td>762050</td>\n",
       "      <td>649090</td>\n",
       "      <td>659554</td>\n",
       "      <td>252566</td>\n",
       "      <td>2017000</td>\n",
       "      <td>646020.0</td>\n",
       "      <td>1984077.0</td>\n",
       "      <td>663200.0</td>\n",
       "      <td>33166000</td>\n",
       "      <td>1357000</td>\n",
       "      <td>757100</td>\n",
       "      <td>265414.0</td>\n",
       "      <td>4696000.0</td>\n",
       "      <td>224844.0</td>\n",
       "      <td>3766480</td>\n",
       "      <td>3682950</td>\n",
       "      <td>406720.0</td>\n",
       "      <td>603930.0</td>\n",
       "      <td>66336</td>\n",
       "      <td>342130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      HNRNPR       HPR       HPSE       HPX       HRG      HRNR    HSDL2  \\\n",
       "0        NaN   1232310  9168200.0  36900100   3278440       NaN   131538   \n",
       "1    22515.0   2178090  9495600.0  32478500   4416770       NaN   300108   \n",
       "2  3561500.0  33189900        NaN  60702000  14182800  865020.0  1636800   \n",
       "3  1525910.0  16314500   192014.0  57405000  10100700  772688.0  1092100   \n",
       "4  1287450.0   5590200  1460620.0  25920400  14014800  715937.0   321590   \n",
       "\n",
       "    HSP90AA1   HSP90AB1   HSP90B1     HSPA1A     HSPA5    HSPA6     HSPA8  \\\n",
       "0  8513500.0  3415300.0  10903700  2181840.0  11793500  6058100  28238300   \n",
       "1  7319600.0  3040670.0  12274900  2899350.0   9931600  5409900  23876700   \n",
       "2        NaN        NaN   2778970        NaN    778140    19740    275411   \n",
       "3   757990.0   467421.0   3004580   352530.0   2624350   345340   4449100   \n",
       "4  1200970.0   552460.0   5117300   997670.0   5067900   328720   5452800   \n",
       "\n",
       "       HTT    HYOU1       IBSP     IDH2   IGFALS     IGHG3     IGHG4  \\\n",
       "0   242726  6423000        NaN  7651000   307268  32247900  41904000   \n",
       "1   366210  5668000        NaN  6627500   193002  17108400   2300830   \n",
       "2  1156200   841820  4096300.0    81616  1502200   8133800   1583400   \n",
       "3   642560   688300   517470.0  1782830  1219970  33539900   6833340   \n",
       "4   691380  1041300    22278.0  6222200   402730  30916900   3851530   \n",
       "\n",
       "   IGHV1-18  IGHV3-13  IGHV3-15   IGHV3-20  IGHV3-35  IGHV3-49   IGHV3-72  \\\n",
       "0    783480       NaN   32495.0   138770.0    454670   48893.0  1724209.0   \n",
       "1    588700       NaN       NaN        NaN    196790       NaN   394627.0   \n",
       "2    223140   77621.0  102280.0        NaN    225950  365220.0        NaN   \n",
       "3    180370   56461.0  148980.0  1243100.0    176060  401330.0    30111.0   \n",
       "4    157280  272160.0  255350.0  1542600.0    346030  256152.0   557610.0   \n",
       "\n",
       "   IGHV4-28  IGHV5-51       IGKC  IGKV1-16  IGKV1-17  IGKV1-33  IGKV1-5  \\\n",
       "0    968950   6861770  109894000  476760.0    136070   2530739   169550   \n",
       "1    401433   1292632   66016000  281599.0    343700    808950   123320   \n",
       "2    268510   2188260  340584000       NaN    299830    663820   977060   \n",
       "3     18472   3114300  233071000  146145.0    205720    794520  1679650   \n",
       "4    186494   1698300  197161000  325038.0    642850   2318040  1787626   \n",
       "\n",
       "    IGKV2-24  IGKV3-20   IGKV4-1      INHBE     IQGAP2      ISOC1   ITM2B  \\\n",
       "0   168290.0   1891180   1091810   104339.0  5097800.0  1304830.0   27768   \n",
       "1    94968.0   2094740    484380   155302.0  4875300.0  1307110.0   24413   \n",
       "2        NaN   1242150   3134320  3643760.0    35443.0   175103.0  179410   \n",
       "3  4135967.0    819420  11033070   438879.0   632110.0        NaN  187710   \n",
       "4    99504.0   1160543   2277260        NaN  1733330.0   350911.0  229550   \n",
       "\n",
       "     ITPR1    JCHAIN     KGD4      MBP  MICAL1       MPP1     NFKB1    PKN1  \\\n",
       "0  1533800   1376300   390490    41052  198972  2698020.0  197959.0  195335   \n",
       "1  1725500    928811   593890    70315  238439  1334160.0  331001.0   94747   \n",
       "2   753410  16003900  1627900  9436000  219860        NaN       NaN  111540   \n",
       "3   689402   4575710  2616400  5688400   37495   950950.0  219390.0   75238   \n",
       "4  1155360   4674720  1280532   591340   46762   508030.0   20316.0   71232   \n",
       "\n",
       "    PLXDC2    RASA1     RFTN1     RIPOR3     RNF121    ROCK2    RPLP1  \\\n",
       "0   484050  1476770   46095.0   805750.0        NaN  1748140   628442   \n",
       "1   578823  1653879   53919.0   459980.0  2788077.0  1756610   371750   \n",
       "2  1637490  1746100       NaN  1268600.0        NaN   366613   175700   \n",
       "3   158828   599990  348490.0        NaN  2816200.0   282169  1462570   \n",
       "4  1052690    21033  113384.0   200318.0  2468200.0   665150   221270   \n",
       "\n",
       "     RPLP2        RSU1     S100A4      SAA1     SAA2  SEPTIN11    SEPTIN2  \\\n",
       "0   391381   8788200.0  1682200.0  36356300  2276888   2080530  3235820.0   \n",
       "1   819555  16325600.0   934060.0  47781100  7323820   1617730  2853890.0   \n",
       "2    85160         NaN        NaN  16883100  1218700     62080        NaN   \n",
       "3  2512750   2598950.0    60538.0  17736559  2528797     43388   165510.0   \n",
       "4   856950   5710930.0    95539.0   4384700   434379    418750   985310.0   \n",
       "\n",
       "     SEPTIN6  SEPTIN7  SERPINA4  SERPINA6  SERPINB6  SERPINE1  SH3BGRL3  \\\n",
       "0  1859160.0  3195140    184207    410222   2410290    851690   3364560   \n",
       "1  1845010.0  3579430    102535    417907    828510    926390   3555310   \n",
       "2        NaN   145100   5428200    628940    762270   5605200    259860   \n",
       "3   362334.0   511704   1996640    815585    313590    162041    568491   \n",
       "4   930720.0  1189740    762050    649090    659554    252566   2017000   \n",
       "\n",
       "      SH3GL1    SLC25A5     SLC2A1      SPP2    SPTA1     ST13    STAT5B  \\\n",
       "0  1090580.0   673450.0    26142.0   1211723   636330  1354700  204402.0   \n",
       "1   899870.0   729909.0        NaN   3764250   854560   943430  260069.0   \n",
       "2        NaN        NaN  1448264.0  96828000  1449040    59265       NaN   \n",
       "3   329616.0   599876.0  1664660.0  41016000  2034830   969550  102922.0   \n",
       "4   646020.0  1984077.0   663200.0  33166000  1357000   757100  265414.0   \n",
       "\n",
       "        SYNRG       TFRC  TGFB1I1     THBS4      TIMP1      TIMP2      TKT  \\\n",
       "0    568940.0   237276.0  3629600    223152  3847440.0        NaN   997620   \n",
       "1    992464.0   208263.0  2933700      6148  2713100.0        NaN   820950   \n",
       "2  12265000.0  1186050.0   179924   6187500        NaN   752190.0  3756700   \n",
       "3   2821300.0   234748.0  1492000  13833800    58936.0  1173550.0   476450   \n",
       "4   4696000.0   224844.0  3766480   3682950   406720.0   603930.0    66336   \n",
       "\n",
       "      TLN2  \n",
       "0   596740  \n",
       "1   587058  \n",
       "2  1130700  \n",
       "3   360690  \n",
       "4   342130  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "y shape: (6,)\n",
      "\n",
      "\n",
      "6 Samples \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    R\n",
       "1    R\n",
       "2    S\n",
       "3    S\n",
       "4    S\n",
       "Name: sensitivity, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# split into X and Y\n",
    "\n",
    "X_validation, y_validation = X_y_split(df, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 875,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    1\n",
       "3    1\n",
       "4    1\n",
       "5    1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 875,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le = LabelEncoder()\n",
    "y_validation = pd.Series(le.fit_transform(y_validation))\n",
    "y_validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 876,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "quantType is Intensity_Raw\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 876,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# impute 0s as above\n",
    "\n",
    "imputeWideDFMinOr0(X_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 878,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 878,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# impute any missing values to the min of the dataset\n",
    "\n",
    "genesWithNANs = X_validation.columns[X_validation.isna().any()].tolist()\n",
    "\n",
    "X_validation[genesWithNANs] = min(X_validation.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 883,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some lists to track valation performance\n",
    "\n",
    "classifierListVal=[]\n",
    "y_predListVal=[]\n",
    "y_validationListVal=[]\n",
    "y_probListVal=[]\n",
    "seedListStatsVal=[]\n",
    "trainFracListStatsVal=[]\n",
    "balancedAccuracyListVal =[]\n",
    "precisionListVal=[]\n",
    "recallListVal=[]\n",
    "ROCAUCScoreListVal=[]\n",
    "F1ScoreListVal=[]\n",
    "MCCListVal=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned CatBoostClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = CatBoostClassifier(**CatBoostClassifierTuned.best_params,random_state = seed, verbose = False))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('CatBoostClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned HistGradientBoostingClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = HistGradientBoostingClassifier(**HistGradientBoostingClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('HistGradientBoostingClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred # Visualizing predictions\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned LGBMClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = LGBMClassifier(**LGBMClassifierTuned.best_params,random_state = seed, verbosity = -1))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append(LGBMClassifier)\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred # Visualizing predictions\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned AdaBoostClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = AdaBoostClassifier(**AdaBoostClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('AdaBoostClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned RandomForestClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = RandomForestClassifier(**RandomForestClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('RandomForestClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned BaggingClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = BaggingClassifier(**BaggingClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('BaggingClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned ExtraTreesClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = ExtraTreesClassifier(**ExtraTreesClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('ExtraTreesClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned GradientBoostingClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = GradientBoostingClassifier(**GradientBoostingClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('GradientBoostingClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned SVC For Validation\n",
    "\n",
    "pipeline.set_params(Model = SVC(**SVCTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('SVC')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned MLPClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = MLPClassifier(**MLPClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('MLPClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0]\n",
      "Balanced Accuracy 0.9097222222222222\n",
      "Precision 0.875\n",
      "Recall 0.875\n",
      "ROC AUC score 0.9097222222222222\n",
      "F1 score 0.875\n",
      "MCC 0.8194444444444444\n",
      "Validation score 0.9230769230769231\n",
      "Confusion matrix\n",
      " [[17  1]\n",
      " [ 1  7]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAFVCAYAAAD2eLS6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGn0lEQVR4nO3deVxUVRsH8N8MMDMgu2xCCKIJ4gIKQZhrYpS+LqWFK+OklAthjpq4gUuKqREuJG6ImAu55kKooZQmSYJkKWC4gQtbKgjKOvf9g5gcGXRmGBjn+nz93E/NmXPveS6OzxzOOfdeDsMwDAghhGgdrqYDIIQQohpK4IQQoqUogRNCiJaiBE4IIVqKEjghhGgpSuCEEKKlKIETQoiWogROCCFaSlfTARBCSHOpqKhAVVWVSvvyeDwIBAI1R6RelMAJIaxUUVEBfaPWQM1jlfa3sbHBjRs3XuokTgmcEMJKVVVVQM1j8DuLAB2ecjvXViH/8jZUVVVRAieEEI3R5YGjw1dqF4bTTLGoGSVwQgi7cbh1m7L7aAHtiJIQQkgD1AMnhLAbh1O3KbuPFqAETghhNxYPoVACJ4SwG4t74NrxNUNk9OvXD/369ZO+vnnzJjgcDmJjY1s0jgkTJsDR0bFF21TVjh074OLiAj09PZiamqr9+IsWLQJHS/7RtwRNfSbl4/7XC1d005LUqB1RKik2NhYcDgcCgQB37txp8H6/fv3QpUsXDUT2ajt48CDee+89WFhYgMfjwdbWFh999BFOnTrVrO1mZWVhwoQJaN++PTZv3oxNmzY1a3stjcPhgMPhYNKkSXLfnz9/vrROcXGx0sdPSEjAokWLmhilBtX3wJXdVBAVFQVHR0cIBAJ4e3sjNTW10brV1dVYsmQJ2rdvD4FAADc3NyQmJirVHisTeL3KykqsWLFC02E0OwcHBzx58gTjx4/XdChyMQwDkUiEDz74AAUFBRCLxYiOjsa0adNw/fp1DBgwAOfOnWu29pOTkyGRSLBmzRpMmDABH330kdrbWLBgAZ48eaL24ypKIBBg//79ci8b3717d5MuRklISMDixYuV2udl/0w2h/j4eIjFYoSFhSE9PR1ubm7w8/NDYWGh3PoLFizAxo0bsW7dOly5cgWTJ0/G+++/j4sXLyrcJqsTuLu7OzZv3oy7d+82WxsMw2j0Hy4A6W8bOjo6Go2jMV9//TViY2Px+eefIy0tDfPmzcPHH3+M+fPn48KFC4iLi4OubvNNx9T/A2qOoZN6urq6Gr1i791330VpaSl+/PFHmfJz587hxo0bGDx4cIvEUVNTg6qqqpfrM6ns8Ikqk54AIiIiEBgYCJFIBFdXV0RHR8PAwAAxMTFy6+/YsQPz5s3DoEGD4OTkhClTpmDQoEH4+uuvFW6T1Ql83rx5qK2tVagXXlNTg6VLl6J9+/bg8/lwdHTEvHnzUFlZKVPP0dER//vf/3D8+HF4enpCX18fGzduRHJyMjgcDr7//nssXrwYdnZ2MDIywsiRI1FSUoLKykp8/vnnsLKygqGhIUQiUYNjb9u2DW+//TasrKzA5/Ph6uqKDRs2vDD2Z8cb62ORtz07Zv3jjz+id+/eaNWqFYyMjDB48GBcvny5QRuHDh1Cly5dIBAI0KVLFxw8ePCFcQHAkydPEB4eDhcXF6xevVruOPH48ePh5eUlfX39+nV8+OGHMDc3h4GBAd58800cO3ZMZp+nf97Lli3Da6+9BoFAgAEDBiAnJ0daz9HREWFhYQAAS0tLcDgc6XDA0///NEdHR0yYMEH6urq6GosXL8brr78OgUCA1q1bo1evXjh58qS0jrwxcGU/U2fPnoWXlxcEAgGcnJwQFxf3/B/uU+zs7NCnTx/s2rVLpnznzp3o2rWr3CHDM2fO4MMPP0Tbtm3B5/Nhb2+PGTNmyHRIJkyYgKioKOnPq34D/vvcrV69GpGRkdLzvHLlSoPPZGFhISwtLdGvXz8wDCM9fk5ODlq1agV/f3+Fz1VpLTCEUlVVhbS0NPj6+krLuFwufH19kZKSInefysrKBl/6+vr6OHv2rMLtsnoVSrt27RAQEIDNmzcjJCQEtra2jdadNGkStm/fjpEjR2LmzJk4f/48wsPDkZmZ2SBZZWdnY/To0fj0008RGBgIZ2dn6Xvh4eHQ19dHSEgIcnJysG7dOujp6YHL5eLBgwdYtGgRfvvtN8TGxqJdu3YIDQ2V7rthwwZ07twZQ4cOha6uLo4cOYKpU6dCIpFg2rRpCp93p06dsGPHDpmyhw8fQiwWw8rKSlq2Y8cOCIVC+Pn54auvvsLjx4+xYcMG9OrVCxcvXpQm+xMnTmDEiBFwdXVFeHg4/vnnH4hEIrz22msvjOXs2bO4f/8+Pv/8c4V6YwUFBejZsyceP36M4OBgtG7dGtu3b8fQoUOxb98+vP/++zL1V6xYAS6Xi1mzZqGkpAQrV67E2LFjcf78eQBAZGQk4uLicPDgQWzYsAGGhobo1q3bC+N42qJFixAeHo5JkybBy8sLpaWluHDhAtLT0zFw4MBG91PmM5WTk4ORI0di4sSJEAqFiImJwYQJE+Dh4YHOnTsrFOeYMWMwffp0lJWVwdDQEDU1Ndi7dy/EYjEqKioa1N+7dy8eP36MKVOmoHXr1khNTcW6detw+/Zt7N27FwDw6aef4u7duzh58mSDz1S9bdu2oaKiAp988gn4fD7Mzc0hkUhk6lhZWWHDhg348MMPsW7dOgQHB0MikWDChAkwMjLCt99+q9A5qqQJywhLS0tlivl8Pvj8hpflFxcXo7a2FtbW1jLl1tbWyMrKktuEn58fIiIi0KdPH7Rv3x5JSUk4cOAAamtrFY+TYaFt27YxAJjff/+duXbtGqOrq8sEBwdL3+/bty/TuXNn6euMjAwGADNp0iSZ48yaNYsBwJw6dUpa5uDgwABgEhMTZeqePn2aAcB06dKFqaqqkpaPHj2a4XA4zHvvvSdT38fHh3FwcJApe/z4cYNz8fPzY5ycnGTK+vbty/Tt21f6+saNGwwAZtu2bXJ/HhKJhPnf//7HGBoaMpcvX2YYhmEePXrEmJqaMoGBgTJ18/PzGRMTE5lyd3d3pk2bNszDhw+lZSdOnGAANDiHZ61Zs4YBwBw8ePC59ep9/vnnDADmzJkz0rJHjx4x7dq1YxwdHZna2lqGYf77eXfq1ImprKxs0N6ff/4pLQsLC2MAMEVFRTJtAWDCwsIaxODg4MAIhULpazc3N2bw4MHPjbu+jXqqfKZ++eUXaVlhYSHD5/OZmTNnPrfd+vOYNm0ac//+fYbH4zE7duxgGIZhjh07xnA4HObmzZtyfwbyPm/h4eEMh8Nhbt26JS2bNm0aIy9V1H/ujI2NmcLCQrnvPfuZHD16NGNgYMBcvXqVWbVqFQOAOXTo0AvPURUlJSUMAIb/5heMoNdCpTb+m18wABps8j4vDMMwd+7cYQAw586dkymfPXs24+XlJXefwsJCZtiwYQyXy2V0dHSYjh07MlOnTmUEAoHC58jqIRQAcHJywvjx47Fp0ybcu3dPbp2EhAQAgFgslimfOXMmADT49b1du3bw8/OTe6yAgADo6elJX3t7e4NhGHz88ccy9by9vZGXl4eamhppmb6+vvT/S0pKUFxcjL59++L69esoKSl50ak2aunSpTh69ChiY2Ph6uoKADh58iQePnyI0aNHo7i4WLrp6OjA29sbp0+fBgDcu3cPGRkZEAqFMDExkR5z4MCB0mM9T30PxsjISKFYExIS4OXlhV69eknLDA0N8cknn+DmzZu4cuWKTH2RSAQe7787zfXu3RtA3TCMupiamuLy5cv4+++/Fd5H2c+Uq6urNHagbrjH2dlZqfMwMzPDu+++i927dwMAdu3ahZ49e8LBwUFu/ac/b+Xl5SguLkbPnj3BMIxSE2kjRoyApaWlQnXXr18PExMTjBw5EgsXLsT48eMxbNgwhdtSSRPGwPPy8lBSUiLd5s6dK7cJCwsL6OjooKCgQKa8oKAANjY2cvextLTEoUOHUF5ejlu3biErKwuGhoZwcnJS+NRYn8CButnempqaRsfCb926BS6Xiw4dOsiU29jYwNTUFLdu3ZIpb9euXaNttW3bVuZ1fdKzt7dvUC6RSGQS86+//gpfX1+0atUKpqamsLS0xLx58wBA5QSemJiIxYsXY+7cuRgxYoS0vD4Zvf3227C0tJTZTpw4IZ34qz/3119/vcGxnx46aoyxsTEA4NGjRwrFe+vWLbnH7dSpk0w89Z79eZuZmQEAHjx4oFB7iliyZAkePnyIjh07omvXrpg9ezYuXbr03H2U/Uw9ex5A3bkoex5jxozByZMnkZubi0OHDmHMmDGN1s3NzcWECRNgbm4OQ0NDWFpaom/fvgCU+7w979/Ds8zNzbF27VpcunQJJiYmWLt2rcL7aoKxsbHMJm/4BKh7+IOHhweSkpKkZRKJBElJSfDx8XluGwKBAHZ2dqipqcH+/fuV+kJj9Rh4PScnJ4wbNw6bNm1CSEhIo/UUvRDj6Z7Lsxob522snPl3QufatWsYMGAAXFxcEBERAXt7e/B4PCQkJOCbb75pMKaoiBs3bmDs2LEYOHAgvvzyS5n36o+3Y8cOuT0Eda0KcXFxAQD8+eefGD58uFqO+bQX/VxV8ewYZJ8+fXDt2jX88MMPOHHiBLZs2YJvvvkG0dHRja69rqfoZ0pd5zF06FDw+XwIhUJUVlY2umSytrYWAwcOxP379zFnzhy4uLigVatWuHPnDiZMmKDU5+15/x7kOX78OIC6L9nbt2836+ogAP9OSio7Bq78OnCxWAyhUAhPT094eXkhMjIS5eXlEIlEAOp+O7ezs0N4eDgA4Pz587hz5w7c3d1x584dLFq0CBKJBF988YXCbb4SCRyo64V/9913+Oqrrxq85+DgAIlEgr///lva0wPqfv15+PBho7+CqtORI0dQWVmJw4cPy/TG6ocylPXkyRN88MEHMDU1xe7du8Hlyn6A27dvD6BucunpmfNn1Z+7vOGD7OzsF8bRq1cvmJmZYffu3Zg3b94LJzIdHBzkHrd+IkidfxdmZmZ4+PChTFlVVZXcoTZzc3OIRCKIRCKUlZWhT58+WLRoUaMJXFOfKX19fQwfPhzfffed9KIpef78809cvXoV27dvR0BAgLT86ZU19dR5hWliYiK2bNmCL774Ajt37oRQKMT58+ebdRkpuJy6Tdl9lOTv74+ioiKEhoYiPz8f7u7uSExMlE5s5ubmyvw7rKiowIIFC3D9+nUYGhpi0KBB2LFjh1JfaK/EEApQl7DGjRuHjRs3Ij8/X+a9QYMGAahbsfC0iIgIAGiRNbT1ie3pHldJSQm2bdum0vEmT56Mq1ev4uDBg9Jhhaf5+fnB2NgYy5cvR3V1dYP3i4qKAABt2rSBu7s7tm/fLvNr9cmTJxuMR8tjYGCAOXPmIDMzE3PmzJHbo/zuu++kV6wNGjQIqampMkuvysvLsWnTJjg6Oio07q6o9u3b45dffpEp27RpU4Me+D///CPz2tDQEB06dGiwHPBpmvxMzZo1C2FhYVi4cGGjdeR93hiGwZo1axrUbdWqFQA0+LJT1sOHD6UreZYvX44tW7YgPT0dy5cvb9JxX6iF1oEDQFBQEG7duoXKykqcP38e3t7e0veSk5Nlbi3Qt29fXLlyBRUVFSguLkZcXNxzV8rJ88r0wIG6S4p37NiB7OxsmaVZbm5uEAqF2LRpEx4+fIi+ffsiNTUV27dvx/Dhw9G/f/9mj+2dd94Bj8fDkCFD8Omnn6KsrAybN2+GlZVVo5OvjTl27Bji4uIwYsQIXLp0SWa81tDQEMOHD4exsTE2bNiA8ePHo0ePHhg1ahQsLS2Rm5uLY8eO4a233sL69esB1C2NHDx4MHr16oWPP/4Y9+/fx7p169C5c2eUlZW9MJ7Zs2fj8uXL+Prrr3H69GmMHDkSNjY2yM/Px6FDh5Camiq9EjMkJAS7d+/Ge++9h+DgYJibm2P79u24ceMG9u/f3+A3iaaYNGkSJk+ejBEjRmDgwIH4448/cPz48Qa9VldXV/Tr1w8eHh4wNzfHhQsXsG/fPgQFBTV6bE1+ptzc3ODm5vbcOi4uLmjfvj1mzZqFO3fuwNjYGPv375c75u7h4QEACA4Ohp+fH3R0dDBq1Cil45o+fTr++ecf/PTTT9DR0cG7776LSZMm4csvv8SwYcNeGLPKWHwzq1cqgXfo0AHjxo3D9u3bG7y3ZcsWODk5ITY2FgcPHoSNjQ3mzp0rvQikuTk7O2Pfvn1YsGABZs2aBRsbG0yZMgWWlpYNVrC8SH3vef/+/di/f7/Mew4ODtKx6DFjxsDW1hYrVqzAqlWrUFlZCTs7O/Tu3Vs6bgfUXeW3d+9eLFiwAHPnzkX79u2xbds2/PDDD0hOTn5hPFwuF3FxcRg2bBg2bdqE1atXo7S0FJaWlujTpw9WrlwpneixtrbGuXPnMGfOHKxbtw4VFRXo1q0bjhw5ovZea2BgIG7cuIGtW7ciMTERvXv3xsmTJzFgwACZesHBwTh8+DBOnDiByspKODg44Msvv8Ts2bOfe3xNf6aeR09PD0eOHEFwcDDCw8MhEAjw/vvvIygoqEEi/eCDD/DZZ59hz549+O6778AwjNIJ/PDhw4iLi8PXX38tnRcB6n4jOXnyJIRCIX7//XeZFVxqw+LbyXKYpsz2EELIS6q0tBQmJibg9w0DR1e52xwwNRWo/HkxSkpKpCupXkavVA+cEPIKoiEUQgjRUiweQqEETghhN+qBE0KIlqIeOCGEaCkW98C142uGEEJIA1rdA5dIJLh79y6MjIzogbKEsAzDMHj06BFsbW2beAGXKldWakffVqsT+N27dxvc5Y8Qwi55eXkKPTykUSweQtHqBF5/j2meqxAcHd4LahM2yE1erekQSAt5VFqKDu3sFb6XfKNa6G6EmqDVCbx+2ISjw6ME/op4ma+KI82jycOjtAqFEEK0FIuHULTja4YQQkgD1AMnhLAbDaEQQoiWYvEQCiVwQgi7UQ+cEEK0FPXACSFEO3E4HOWXImpJAteO3xMIIYQ0QD1wQgirsbkHTgmcEMJunH83ZffRApTACSGsxuYeOI2BE0JYrT6BK7upIioqCo6OjhAIBPD29kZqaupz60dGRsLZ2Rn6+vqwt7fHjBkzUFFRoXB7lMAJIazWUgk8Pj4eYrEYYWFhSE9Ph5ubG/z8/FBYWCi3/q5duxASEoKwsDBkZmZi69atiI+Px7x58xRukxI4IYSoQUREBAIDAyESieDq6oro6GgYGBggJiZGbv1z587hrbfewpgxY+Do6Ih33nkHo0ePfmGv/WmUwAkhrNYSPfCqqiqkpaXB19dXWsblcuHr64uUlBS5+/Ts2RNpaWnShH39+nUkJCRg0KBBCrdLk5iEEHZrwiqU0tJSmWI+nw8+n9+genFxMWpra2FtbS1Tbm1tjaysLLlNjBkzBsXFxejVqxcYhkFNTQ0mT55MQyiEEFKvKT1we3t7mJiYSLfw8HC1xZWcnIzly5fj22+/RXp6Og4cOIBjx45h6dKlCh+DeuCEEFaruxWKsssI6/6Tl5cn8xQoeb1vALCwsICOjg4KCgpkygsKCmBjYyN3n4ULF2L8+PGYNGkSAKBr164oLy/HJ598gvnz5yv0IGfqgRNCWI0DFXrg/2ZwY2Njma2xBM7j8eDh4YGkpCRpmUQiQVJSEnx8fOTu8/jx4wZJWkdHBwDAMIxC50Y9cEIIUQOxWAyhUAhPT094eXkhMjIS5eXlEIlEAICAgADY2dlJh2GGDBmCiIgIdO/eHd7e3sjJycHChQsxZMgQaSJ/EUrghBBWa6krMf39/VFUVITQ0FDk5+fD3d0diYmJ0onN3NxcmR73ggULwOFwsGDBAty5cweWlpYYMmQIli1bpniYjKJ99ZdQaWkpTExMwO8aSE+lf0U8+H29pkMgLaS0tBTWrU1QUlIiMw6tzP4mJiYwG7UFHJ6BUvsyVY/xYM8kldtuKdQDJ4Swmwo9cEZL7oVCCZwQwmqqDKGoei+UlkYJnBDCamxO4LSMkBBCtBT1wAkh7EYPdCCEEO3E5iEUSuCEEFajBE4IIVqKEjghhGgpNidwWoVCCCFainrghBB2o1UohBCindg8hEIJnBDCapTACSFES7E5gdMkJiGEaCnqgRNC2I0mMQkhRDuxeQiFEjghhNUogRNCiJaqfyq9svtoA0rghBBWY3MPnFahEEKIlqIeOCGE3WgVCiGEaCc2D6FQAieEsBqbEziNgRNCWI3DUW1TRVRUFBwdHSEQCODt7Y3U1NRG6/br10/65fL0NnjwYIXbowROCGG1uoTcMFE+f1O+nfj4eIjFYoSFhSE9PR1ubm7w8/NDYWGh3PoHDhzAvXv3pNtff/0FHR0dfPjhhwq3SQmcEELUICIiAoGBgRCJRHB1dUV0dDQMDAwQExMjt765uTlsbGyk28mTJ2FgYEAJnBBCpFQZPlGyB15VVYW0tDT4+vpKy7hcLnx9fZGSkqLQMbZu3YpRo0ahVatWCrdLk5iEEFZryiRmaWmpTDmfzwefz29Qv7i4GLW1tbC2tpYpt7a2RlZW1gvbS01NxV9//YWtW7cqFSf1wAkhrNaUSUx7e3uYmJhIt/Dw8GaJcevWrejatSu8vLyU2o964IQQVuNyOeByleuBM//Wz8vLg7GxsbRcXu8bACwsLKCjo4OCggKZ8oKCAtjY2Dy3rfLycuzZswdLlixRKkaAeuAvjU8/6oOsY4vx4Ldv8EvcLHh2dmi0rq4uF3M/eReXD4fhwW/f4Hx8CAb27NRo/VmigXhycT1WzRrRHKETFUR/GwXnDo4wNRSgd09v/P6c5WYAsH/fXrh1cYGpoQCe7l2R+GNCgzpZmZkY+f5QWLc2QWuTVnjrzTeQm5vbXKegNZrSAzc2NpbZGkvgPB4PHh4eSEpKkpZJJBIkJSXBx8fnufHt3bsXlZWVGDdunNLnRgn8JTDynR74aub7WLbxR/iM+QqXrt7B4W+nwdLMUG79RVOHYNKIXhCv3IvuI77Eln1nEf91INycX2tQ18O1LSaOeAuXrt5u7tMgCtr7fTzmzBZj/oIwpKSmo1s3Nwwd3Phys5Rz5yAcNxpC0UT89vtFDBk2HB+NGI7Lf/0lrXP92jUM6NcLHZ1dcPynZPyefglz5y+EQCBoqdN65YnFYmzevBnbt29HZmYmpkyZgvLycohEIgBAQEAA5s6d22C/rVu3Yvjw4WjdurXSbb4UCVyZxe9sFDzubWw7cA47Dv+GrOv5+GzZHjypqIJwuPxv7jH/88LKrSdw/OwV3LzzDzbvPYvjv17B9PFvy9Rrpc/DtuUTMHXpbjwsfdISp0IUsDYyAqKJgQiYIEInV1es+zYa+gYG2B4rf7lZ1Po1eMfvXYhnzoZLp04IW7wU7t17IPrb9dI6YaHz4ffuICxfsRLu3bvDqX17/G/IUFhZWbXUab20lF8DrvykJwD4+/tj9erVCA0Nhbu7OzIyMpCYmCid2MzNzcW9e/dk9snOzsbZs2cxceJElc5N4wlc2cXvbKOnq4Punexx6ny2tIxhGJw6nw2vbu3k7sPT00VFVbVM2ZOKKvTs3l6mLHKuPxLP/IXTTx2baFZVVRUupqfh7QGyy83eftsXqb/JX252/rcU9H/bV6Zs4Dt+OP9vfYlEgsSEY3i9Y0cMGeSHtrZW6N3TG4d/ONRs56FNWvJKzKCgINy6dQuVlZU4f/48vL29pe8lJycjNjZWpr6zszMYhsHAgQNVak/jCVzZxe9sY2FmCF1dHRTefyRTXvhPKWxaG8vd56eUTASPexvt21qCw+HgbW8XDHvbHTYW/9X/0M8D7i72WLjucLPGT5RTv9zMykp2uZmVtTXy8/Pl7lOQnw+rZ5anWVlZo6Cgrn5hYSHKysqweuUKDHznXRxJOIGhw9/HqA8/wJlffm6eE9EiLdUD1wSNrkKpX/z+9LjQ8xa/V1ZWorKyUvr62TWar4pZq/bh24Wj8ceBhWAYBtdvFyPu8G8QDnsTAPCatSlWzR6B/01Zj8qqGg1HS5qbRCIBAPxv6DAEfz4DAODm7o7zKeeweVM0evfpq8nwNI7NN7PSaAJXdvF7eHg4Fi9e3FLhtYjiB2WoqamFlbmRTLlVa2Pk/yP/C6r4QRk+Em8Gn6eL1iatcLeoBF8GD8ONO/8AALp3agvr1sZI2TVHuo+urg569WiPyf59YOL9OSQSpvlOijSqfrlZYaHscrPC5yw3s7axQeEzy9MKCwtgbW0jPaauri46dXKVqePs0gnnfj2rxui1kypDIlqSvzU/hKKMuXPnoqSkRLrl5eVpOqQmq66pxcXMPPT3dpaWcTgc9PfqiNRLN567b2VVDe4WlUBXl4vhA9xxNPkSAOB0ajY8Ri6D96gV0i3t8i3sSbgA71ErKHlrEI/HQ/ceHjh9Sna52enTSfB6U/6ktfebPkg+nSRTlvTTSXj/W5/H48HD8w1czZad6/j776to69D4clSi/TTaA1d28Xtjl7Fqu7XfncLmJeORdiUXF/66iaAx/WGgz0fcD78BALYsHY+7hSUI/Xc8+40uDrC1MsUf2bdhZ2WK+Z8OApfLQUTsTwCAsseVuHJNdra7/EkV7peUNygnLS/4czECPxbCw8MTnm94Yf3aSDwuL0eAsG652cQJAbC1s8PSZXVX/U0Lmo53BvRF5Ddf4733BmPv93uQnnYBURs2SY85Y+ZsjB/jj169+6Bvv/44cTwRCUeP4PhPyZo4xZcKPdS4mTy9+H348OEA/lv8HhQUpMnQWtS+E+mwMDNE6JTBsG5thEvZdzBsWpR0YtPexlym18zn6yFs2v/Qzs4CZY8rcfzXy5i4MA4lZbRUUBt8+JE/iouKsGRxKAry89HNzR0/HP1vuVleXi643P9+Ofbp2ROxO3ZhcdgChC2Yhw6vv47v9x9C5y5dpHWGDX8f66KisWplOGbOCEbHjs7Y/f1+vNWrV4uf38uGzUMoHIZhNPr7dHx8PIRCITZu3AgvLy9ERkbi+++/R1ZWVoOx8WeVlpbCxMQE/K6B4OjwWihiokkPfl//4kqEFUpLS2Hd2gQlJSUyl7Mrs7+JiQnc5h2BjkDxO/wBQG1FOf5YPkTltluKxu+F4u/vj6KiIoSGhiI/Px/u7u4yi98JIaQp2NwD13gCB+oWv79KQyaEkJbD5mWEWrUKhRBCyH9eih44IYQ0FxpCIYQQLcXmIRRK4IQQdlPl5lTakb8pgRNC2I164IQQoqXYPAZOq1AIIURLUQ+cEMJqNIRCCCFais1DKJTACSGsRj1wQgjRUpTACSFES7F5CIVWoRBCiJaiHjghhNVoCIUQQrQUDaEQQoiWqu+BK7upIioqCo6OjhAIBPD29kZqaupz6z98+BDTpk1DmzZtwOfz0bFjRyQkJCjcHvXACSGsxoEKPXAV2omPj4dYLEZ0dDS8vb0RGRkJPz8/ZGdnw8rKqkH9qqoqDBw4EFZWVti3bx/s7Oxw69YtmJqaKtwmJXBCCKtxORxwlczgytYHgIiICAQGBkIkEgEAoqOjcezYMcTExCAkJKRB/ZiYGNy/fx/nzp2Dnp4eAMDR0VG5OJWOkhBCiIyqqiqkpaXB19dXWsblcuHr64uUlBS5+xw+fBg+Pj6YNm0arK2t0aVLFyxfvhy1tbUKt0s9cEIIqzVlErO0tFSmnM/ng8/nN6hfXFyM2traBg9jt7a2RlZWltw2rl+/jlOnTmHs2LFISEhATk4Opk6diurqaoSFhSkUJ/XACSGs1pRJTHt7e5iYmEi38PBwtcUlkUhgZWWFTZs2wcPDA/7+/pg/fz6io6MVPgb1wAkhrMbl1G3K7gMAeXl5MDY2lpbL630DgIWFBXR0dFBQUCBTXlBQABsbG7n7tGnTBnp6etDR0ZGWderUCfn5+aiqqgKPx3txnC+sQQgh2oyjfC+8fhmKsbGxzNZYAufxePDw8EBSUpK0TCKRICkpCT4+PnL3eeutt5CTkwOJRCItu3r1Ktq0aaNQ8gYU7IEfPnxYoYMBwNChQxWuSwghza2lLuQRi8UQCoXw9PSEl5cXIiMjUV5eLl2VEhAQADs7O+kwzJQpU7B+/XpMnz4dn332Gf7++28sX74cwcHBCrepUAIfPny4QgfjcDhKzaASQghb+Pv7o6ioCKGhocjPz4e7uzsSExOlE5u5ubngcv8b9LC3t8fx48cxY8YMdOvWDXZ2dpg+fTrmzJmjcJsKJfCnu/iEEKJNOP/+UXYfVQQFBSEoKEjue8nJyQ3KfHx88Ntvv6nUFtDEScyKigoIBIKmHIIQQppVUyYxX3ZKT2LW1tZi6dKlsLOzg6GhIa5fvw4AWLhwIbZu3ar2AAkhpCla8l4oLU3pBL5s2TLExsZi5cqVMjOlXbp0wZYtW9QaHCGENFX9JKaymzZQOoHHxcVh06ZNGDt2rMz6RTc3t0avOCKEEE2pvxeKsps2UDqB37lzBx06dGhQLpFIUF1drZagCCGEvJjSCdzV1RVnzpxpUL5v3z50795dLUERQoi6sHkIRelVKKGhoRAKhbhz5w4kEgkOHDiA7OxsxMXF4ejRo80RIyGEqIzNj1RTugc+bNgwHDlyBD/99BNatWqF0NBQZGZm4siRIxg4cGBzxEgIISqjHvgzevfujZMnT6o7FkIIUbuWeqCDJqh8Ic+FCxeQmZkJoG5c3MPDQ21BEUKIunCg/CPStCN9q5DAb9++jdGjR+PXX3+VPrvt4cOH6NmzJ/bs2YPXXntN3TESQgiRQ+kx8EmTJqG6uhqZmZm4f/8+7t+/j8zMTEgkEkyaNKk5YiSEEJWx+UpMpXvgP//8M86dOwdnZ2dpmbOzM9atW4fevXurNThCCGkqNt8LRekEbm9vL/eCndraWtja2qolKEIIURdaRviUVatW4bPPPsOFCxekZRcuXMD06dOxevVqtQZHCCHqwMYlhICCPXAzMzOZb6Ty8nJ4e3tDV7du95qaGujq6uLjjz9W+OEPhBDSEtjcA1cogUdGRjZzGIQQQpSlUAIXCoXNHQchhDQLmsRsREVFBaqqqmTKjI2NmxQQIYSoE5uHUJSexCwvL0dQUBCsrKzQqlUrmJmZyWyEEPIy4ai4aQOlE/gXX3yBU6dOYcOGDeDz+diyZQsWL14MW1tbxMXFNUeMhBCiMjY/0EHpIZQjR44gLi4O/fr1g0gkQu/evdGhQwc4ODhg586dGDt2bHPESQgh5BlK98Dv378PJycnAHXj3ffv3wcA9OrVC7/88ot6oyOEkCZi8+1klU7gTk5OuHHjBgDAxcUF33//PYC6nnn9za0IIeRlweZ7oSidwEUiEf744w8AQEhICKKioiAQCDBjxgzMnj1b7QESQkhTUA/8KTNmzEBwcDAAwNfXF1lZWdi1axcuXryI6dOnqz1AQghpipacxIyKioKjoyMEAgG8vb2RmpraaN3Y2NgGvX6BQKBUe01aBw4ADg4OcHBwaOphCCGkWajSo1Ylf8fHx0MsFiM6Ohre3t6IjIyEn58fsrOzYWVlJXcfY2NjZGdnP9Wucg0rlMDXrl2r8AHre+eEEPIqiYiIQGBgIEQiEQAgOjoax44dQ0xMDEJCQuTuw+FwYGNjo3KbCiXwb775RqGDcTgcjSTwyz+Gw4iuAH0lmL2zXNMhkBbC1FSo5ThNuRKztLRUppzP54PP5zeoX1VVhbS0NMydO1daxuVy4evri5SUlEbbKSsrg4ODAyQSCXr06IHly5ejc+fOCsepUAKvX3VCCCHahgvlJ/vq69vb28uUh4WFYdGiRQ3qFxcXo7a2FtbW1jLl1tbWyMrKktuGs7MzYmJi0K1bN5SUlGD16tXo2bMnLl++rPCjKZs8Bk4IIS+zpvTA8/LyZO7vJK/3rSofHx/4+PhIX/fs2ROdOnXCxo0bsXTpUoWOQQmcEMJqHBXuRlif742NjRW6QZ+FhQV0dHRQUFAgU15QUKDwGLeenh66d++OnJwcheNUehkhIYRok/rbySq7KYPH48HDwwNJSUnSMolEgqSkJJle9vPU1tbizz//RJs2bRRul3rghBCiBmKxGEKhEJ6envDy8kJkZCTKy8ulq1ICAgJgZ2eH8PBwAMCSJUvw5ptvokOHDnj48CFWrVqFW7duYdKkSQq3SQmcEMJqLXU/cH9/fxQVFSE0NBT5+flwd3dHYmKidGIzNzcXXO5/gx4PHjxAYGAg8vPzYWZmBg8PD5w7dw6urq6Kx8kwDKNsoGfOnMHGjRtx7do17Nu3D3Z2dtixYwfatWuHXr16KXs4lZWWlsLExAQ5t4tpGeErwmHYKk2HQFoIU1OByjNLUVJSotKDYurzw2fxF8A3MFRq38rHZVjn76ly2y1F6THw/fv3w8/PD/r6+rh48SIqKysBACUlJVi+nNboEkJeLnQvlKd8+eWXiI6OxubNm6Gnpyctf+utt5Cenq7W4AghpKnogQ5Pyc7ORp8+fRqUm5iY4OHDh+qIiRBC1KYpF/K87JSO08bGRu46xbNnz0of9EAIIaT5KZ3AAwMDMX36dJw/fx4cDgd3797Fzp07MWvWLEyZMqU5YiSEEJWxeQxc6SGUkJAQSCQSDBgwAI8fP0afPn3A5/Mxa9YsfPbZZ80RIyGEqIwL5ce0uVryXHqlEziHw8H8+fMxe/Zs5OTkoKysDK6urjA0VG6ZDiGEtISWuh+4Jqh8IQ+Px1NqwTkhhGiCKpfGK1tfU5RO4P3793/uVUqnTp1qUkCEEKJOdTezUvZKzGYKRs2UTuDu7u4yr6urq5GRkYG//voLQqFQXXERQgh5AaUTeGNP51m0aBHKysqaHBAhhKgTm8fA1bZefdy4cYiJiVHX4QghRC1a4naymqK2uxGmpKRAIBCo63CEEKIWnH//KLuPNlA6gX/wwQcyrxmGwb1793DhwgUsXLhQbYERQog60CqUp5iYmMi85nK5cHZ2xpIlS/DOO++oLTBCCFEHSuD/qq2thUgkQteuXWFmZtZcMRFCCFGAUpOYOjo6eOedd+iug4QQrVH/RB5lN22g9CqULl264Pr1680RCyGEqB2bV6Go9ECHWbNm4ejRo7h37x5KS0tlNkIIeZnQ3QhR9wTlmTNnYtCgQQCAoUOHyvyawTAMOBwOamtr1R8lIYSoSJUn7LDuiTyLFy/G5MmTcfr06eaMhxBC1IpWoaCuhw0Affv2bbZgCCGEKE6pZYTaMjNLCCFSqoxpa0mqU2oSs2PHjjA3N3/uRgghLxMuOCptqoiKioKjoyMEAgG8vb2Rmpqq0H579uwBh8PB8OHDlWpPqR744sWLG1yJSQghL7OWuhthfHw8xGIxoqOj4e3tjcjISPj5+SE7OxtWVlaN7nfz5k3MmjULvXv3VrpNpRL4qFGjnhsIIYS8bFpqEjMiIgKBgYEQiUQAgOjoaBw7dgwxMTEICQmRu09tbS3Gjh2LxYsX48yZM0pfJKnwEAqNfxNCtFH9MkJlN2VUVVUhLS0Nvr6+/7XL5cLX1xcpKSmN7rdkyRJYWVlh4sSJKp2b0qtQCCHkVfHsxYl8Ph98Pr9BveLiYtTW1sLa2lqm3NraGllZWXKPffbsWWzduhUZGRkqx6dwD1wikdDwCSFE6zTlSkx7e3uYmJhIt/DwcLXE9OjRI4wfPx6bN2+GhYWFysdR2wMdCCHkZcSFCldi/rsKJS8vD8bGxtJyeb1vALCwsICOjg4KCgpkygsKCmBjY9Og/rVr13Dz5k0MGTJEWiaRSAAAurq6yM7ORvv27RWIkxBCWKwpPXBjY2OZrbEEzuPx4OHhgaSkJGmZRCJBUlISfHx8GtR3cXHBn3/+iYyMDOk2dOhQ9O/fHxkZGbC3t1fo3KgHTghhNS6U76mq0rMVi8UQCoXw9PSEl5cXIiMjUV5eLl2VEhAQADs7O4SHh0MgEKBLly4y+5uamgJAg/LnoQROCGE1Ve7vrcqqO39/fxQVFSE0NBT5+flwd3dHYmKidGIzNzcXXK56Bz0ogRNCiJoEBQUhKChI7nvJycnP3Tc2Nlbp9iiBE0JYjQPlb22iLVe9UAInhLAa3Q+cEEK0mHakY+VRAieEsFpL3cxKEyiBE0JYraVWoWgCXchDCCFainrghBBWa6kLeTSBEjghhNXYPIRCCZwQwmq0DpwQQrQU9cAJIURLsXkMXFviJIQQ8gzqgRNCWI2GUAghREvRJCYhhGgpNl9KT2PgL4mYTRvg2eV1tLU0wrv930L6hd8brZuVeRkfj/sInl1eh7UxDxuj1jaos+brr+DX1wdOtuZwdbKDcPQI5Pyd3ZynQJTw6TAPZO2aigeJX+CXKCE8Xdo8t37QiDfwx/ZPcf/H2fh7TxBWTvUFX09H+j6Xy0GoqA8yd07F/R9n4/J3UxAy7q3mPg2twAVHpU0bUAJ/CRza/z3C5s3GzJAFOHnmPDp37YZRHwxGUVGh3PpPHj+Bg6MT5i/6ElbWDR+YCgApZ89A9MkUJCSdwd4fElBTXQP/4YNRXl7enKdCFDCyXyd8NWUAlsWdhc+nMbh0rRCHvxoFS1MDufX933bF0sD+WL79DNwnbMLk1ccwsl8nLJnUT1pn5igfBA7tgRlrj8N9wiYs2HQa4lFvYur7ni10Vi+vpjwT82Wn0QT+yy+/YMiQIbC1tQWHw8GhQ4c0GY7GRK9fg3HCiRg9TghnF1esioyCvr4Bdu+IlVu/u4cnwr5cgfdH+jf6kNU9B49i1NgAuHTqjM5d3bAmegtu5+XiUkZ6M54JUUTwh17YlpCBHYmXkHWrGJ998yOeVNZA+J6b3PpvdnkNKX/dRvypK8gtKEHShRv4/tQVeLrY/lensx2O/noVieevIbegBAd/yULShRsydQj7aDSBl5eXw83NDVFRUZoMQ6OqqqpwKSMdvfu/LS3jcrno0+9tXEj9TW3tPCopAQCYmpmp7ZhEeXq6XHTv2Aan0m5KyxgGOJV2A16udnL3+e2v2+je0UY6zOLYxhR+3u2ReP7af3Uu30H/Ho7o8Jo5AKCrkxV8utjjROo1ucd8lXBU/KMNNDqJ+d577+G9997TZAgad/+fYtTW1sLS0lqm3NLKCn9fVc+YtUQiwYKQWfB6syc6uSr+xGuifhYmBtDV4aLwgexQVuGDcji3bS13n/hTV9DaxABJawLA4QB6ujrYdDgdq3adk9ZZvfscjFvx8Efsp6iVSKDD5SJsazL2JF1u1vPRBmyexNSqVSiVlZWorKyUvi4tLdVgNNojZGYwsjMv4/Dx05oOhaigt1tbzB7bE9PXJOL3zLtob2eG1dMG4t64t7Diu18BACP7uWLUgC6YsOwHXLlZhG4drLFqqi/u/VOGnSf+1PAZaBZHhUlJ6oE3g/DwcCxevFjTYaiVeWsL6OjooKioQKa8qLAQVtbWjeyluLkzp+NkYgIO/ZgEW7vXmnw80jTFJY9RUyuBlVkrmXIrs1bIvy9/gjlM1Be7T/6F2IQ/AACXbxTBQKCHKPEgfLXzVzAMsPzTt7F6dwr2nr4irdPW2gSzx/SkBM7iHrhWrUKZO3cuSkpKpFteXp6mQ2oyHo+Hbu49cCb5v96xRCLBmZ9Pw9PrTZWPyzAM5s6cjoSjP2D/keNwcGynjnBJE1XXSHDx6j307+EoLeNwgP49HJF65Y7cffQFupBIGJmy+tf1Vwzq83UhYWTr1NZKwNWSRNSc2LwKRat64Hw+v9FVF9psctB0BE+eCPfuPdDd8w1s+nYdHj8ux6hxQgBA0Cci2NjaYsGiZQDqJj6vZl2R/n/+vbv461IGWrUyRLv2HQAAIeJgHNi3B9t374ehkREKC/IBAEbGJtDX19fAWZJ6a/emYnPIEKRl38OFrLsIGuEFA4Ee4hIvAQC2hAzB3eJHCN2SDABISMlB8Egv/JFTgNTMO2hvZ4ZQUR8kpPwtTeQJKTmYM7Yn8gpKcOVmMdxft0bwh96I+/EPTZ0maQFalcDZaviIj/BPcTFWLl+CwoJ8dO7qht37j8LKqm4I5c7tPHC5//2ylH/vLgb08pK+/nZtBL5dG4GevfrgYMJPAIDYrRsBAO8P8pVpa82GLRg1NqC5T4k8x77kTFiYGiBU1AfWZq1w6VoBhs2Jl05s2lsZy/S4V+w4C4ZhEPZxH9haGKH44WMcS8nBoq3J0jridScQ9nEfrPn8XViaGuDeP2XYevQilsedaenTe+mosqpEW8bAOQzzzO9dLaisrAw5OTkAgO7duyMiIgL9+/eHubk52rZt+8L9S0tLYWJigpzbxTAyNm7ucMlLwGHYKk2HQFoIU1OByjNLUVJSAmMV/n3X54cffr+OVoZGSu1bXvYIw95wUrrtqKgorFq1Cvn5+XBzc8O6devg5eUlt+6BAwewfPly5OTkoLq6Gq+//jpmzpyJ8ePHK9yeRsfAL1y4gO7du6N79+4AALFYjO7duyM0NFSTYRFCWKSl1oHHx8dDLBYjLCwM6enpcHNzg5+fHwoL5V9RbW5ujvnz5yMlJQWXLl2CSCSCSCTC8ePHFT83TfbAm4p64K8e6oG/OtTVAz9y4YZKPfAhnu2Uatvb2xtvvPEG1q9fD6BuMYK9vT0+++wzhISEKHSMHj16YPDgwVi6dKlC9bVqFQohhLSk0tJSme3p61CeVlVVhbS0NPj6/jfnxOVy4evri5SUlBe2wzAMkpKSkJ2djT59+igcHyVwQgir1d0PXLUBFHt7e5iYmEi38PBwuW0UF9ddUW39zLUb1tbWyM/PbzS2kpISGBoagsfjYfDgwVi3bh0GDhyo8LnRKhRCCKtxOVB6PXx9/by8PJkhFHUvYzYyMkJGRgbKysqQlJQEsVgMJycn9OvXT6H9KYETQlitKcsIjY2NFRoDt7Cou6K6oED2iuqCggLY2Mi/5TNQN8zSoUPdtRvu7u7IzMxEeHi4wgmchlAIIazWEldi8ng8eHh4ICkpSVomkUiQlJQEHx8fhY8jkUgaHWeXh3rghBBWa6lnYorFYgiFQnh6esLLywuRkZEoLy+HSCQCAAQEBMDOzk46jh4eHg5PT0+0b98elZWVSEhIwI4dO7BhwwaF26QETgghauDv74+ioiKEhoYiPz8f7u7uSExMlE5s5ubmylxRXV5ejqlTp+L27dvQ19eHi4sLvvvuO/j7+yvcJq0DJ1qF1oG/OtS1Dvxk+i20MlJu//JHpRjYw0HltlsK9cAJIazWUkMomkAJnBDCbizO4JTACSGsxua7EVICJ4SwmyoPaNCO/E3rwAkhRFtRD5wQwmosHgKnBE4IYTkWZ3BK4IQQVqNJTEII0VKq3NuEnkpPCCEvARaPoNAqFEII0VbUAyeEsBuLu+CUwAkhrEaTmIQQoqVoEpMQQrQUi0dQKIETQliOxRmcVqEQQoiWoh44IYTVaBKTEEK0FE1iEkKIlmLxEDglcEIIy7E4g1MCJ4SwGpvHwGkVCiGEaClK4IQQVqufxFR2U0VUVBQcHR0hEAjg7e2N1NTURutu3rwZvXv3hpmZGczMzODr6/vc+vJQAieEsBpHxU1Z8fHxEIvFCAsLQ3p6Otzc3ODn54fCwkK59ZOTkzF69GicPn0aKSkpsLe3xzvvvIM7d+4o3CYlcEIIu7VQBo+IiEBgYCBEIhFcXV0RHR0NAwMDxMTEyK2/c+dOTJ06Fe7u7nBxccGWLVsgkUiQlJSkcJuUwAkhrMZR8Y8yqqqqkJaWBl9fX2kZl8uFr68vUlJSFDrG48ePUV1dDXNzc4XbpVUohBBWa8qFPKWlpTLlfD4ffD6/Qf3i4mLU1tbC2tpaptza2hpZWVkKtTlnzhzY2trKfAm8CPXACSGkEfb29jAxMZFu4eHhzdLOihUrsGfPHhw8eBACgUDh/agHTghhtaZcx5OXlwdjY2NpubzeNwBYWFhAR0cHBQUFMuUFBQWwsbF5blurV6/GihUr8NNPP6Fbt25KxUk9cEIIuzVhEtPY2FhmayyB83g8eHh4yExA1k9I+vj4NBraypUrsXTpUiQmJsLT01PpU6MeOCGE1VrqSkyxWAyhUAhPT094eXkhMjIS5eXlEIlEAICAgADY2dlJh2G++uorhIaGYteuXXB0dER+fj4AwNDQEIaGhgq1SQmcEMJuqlyYo8IyQn9/fxQVFSE0NBT5+flwd3dHYmKidGIzNzcXXO5/gx4bNmxAVVUVRo4cKXOcsLAwLFq0SKE2KYETQlitJe9lFRQUhKCgILnvJScny7y+efOmiq38h8bACSFES1EPnBDCbnQ7WUII0U5svp0sJXBCCKvRI9UIIURLsXgEhRI4IYTlWJzBaRUKIYRoKeqBE0JYjSYxCSFES3GgwiRms0SifpTACSGsxuIhcErghBB2o2WEhBCitdjbB9fqBM4wDADg0aNHGo6EtBSmpkLTIZAWwtRU1v3333/npCGtTuD1ibt7p3YajoQQ0lwePXoEExMTlfenIZSXlK2tLfLy8mBkZASOtvzE1aC0tBT29vYNHvdE2OlV/ftmGAaPHj2Cra1tk47D3gEULU/gXC4Xr732mqbD0Jj6xzyRV8Or+PfdlJ53PeqBE0KIlqILeQghRFuxeAyF7oWihfh8PsLCwhp9QjZhF/r7Jo3hMLRGhxDCQqWlpTAxMcHfecUwUnLu4FFpKV63t0BJSclLPe9AQyiEEFajSUxCCNFSNIlJCCHaisWTmJTACSGsxuL8TatQtFFUVBQcHR0hEAjg7e2N1NRUTYdEmsEvv/yCIUOGwNbWFhwOB4cOHdJ0SOQlQwlcy8THx0MsFiMsLAzp6elwc3ODn58fCgsLNR0aUbPy8nK4ubkhKipK06FotfpJTGU3VSjTubp8+TJGjBgBR0dHcDgcREZGKt0eJXAtExERgcDAQIhEIri6uiI6OhoGBgaIiYnRdGhEzd577z18+eWXeP/99zUdipbjKP1HlUEUZTtXjx8/hpOTE1asWAEbGxuVzowSuBapqqpCWloafH19pWVcLhe+vr5ISUnRYGSEvLxaqgeubOfqjTfewKpVqzBq1CiVL9KiBK5FiouLUVtbC2tra5lya2tr5OfnaygqQtirtLRUZqusrJRbT1OdK0rghBBWa0oP3N7eHiYmJtItPDxcbhua6lzRMkItYmFhAR0dHRQUFMiUFxQUqDyGRghp3LP3YH/Z7kdDPXAtwuPx4OHhgaSkJGmZRCJBUlISfHx8NBgZIS8v5acw/7tys/4e7PVbYwlcU50rSuBaRiwWY/Pmzdi+fTsyMzMxZcoUlJeXQyQSaTo0omZlZWXIyMhARkYGAODGjRvIyMhAbm6uZgPTMi0xiampzhUNoWgZf39/FBUVITQ0FPn5+XB3d0diYmKDsTei/S5cuID+/ftLX4vFYgCAUChEbGyshqLSPi11JaZYLIZQKISnpye8vLwQGRkp07kKCAiAnZ2ddBy9qqoKV65ckf7/nTt3kJGRAUNDQ3To0EGxOOl2soQQNqq/neztwgdK3xK2tLQUr1mZKX072fXr12PVqlXSztXatWvh7e0NAOjXrx8cHR2lX743b95Eu3YNH8jet29fJCcnK9QeJXBCCCtpIoG3NBpCIYSwGt1OlhBCtBQ90IEQQrQUm28nSwmcEMJuLM7glMAJIazG5jFwupCHEEK0FCVwolYTJkzA8OHDpa/79euHzz//vMXjSE5OBofDwcOHDxuto+xTbhYtWgR3d/cmxXXz5k1wOBzp1ZWk+T16VKrSpg1oCOUVMGHCBGzfvh0AoKenh7Zt2yIgIADz5s2Drm7zfgQOHDgAPT09heomJyejf//+ePDgAUxNTZs1LsJ+PB4PNjY2eL2dvUr729jYgMfjqTkq9aIE/op49913sW3bNlRWViIhIQHTpk2Dnp4e5s6d26BuVVWV2j645ubmajkOIcoSCAS4ceMGqqqqVNqfx+NBIBCoOSr1oiGUVwSfz4eNjQ0cHBwwZcoU+Pr64vDhwwD+G/ZYtmwZbG1t4ezsDKDuVpofffQRTE1NYW5ujmHDhuHmzZvSY9bW1kIsFsPU1BStW7fGF198gWcv7H12CKWyshJz5syBvb09+Hw+OnTogK1bt+LmzZvS+36YmZmBw+FgwoQJAOpuChQeHo527dpBX18fbm5u2Ldvn0w7CQkJ6NixI/T19dG/f3+ZOBU1Z84cdOzYEQYGBnBycsLChQtRXV3doN7GjRthb28PAwMDfPTRRygpKZF5f8uWLejUqRMEAgFcXFzw7bffKh0LUQ+BQNDgjoKKbi978gYogb+y9PX1ZXomSUlJyM7OxsmTJ3H06FFUV1fDz88PRkZGOHPmDH799VcYGhri3Xffle739ddfIzY2FjExMTh79izu37+PgwcPPrfdgIAA7N69G2vXrkVmZiY2btwIQ0ND2NvbY//+/QCA7Oxs3Lt3D2vWrAEAhIeHIy4uDtHR0bh8+TJmzJiBcePG4eeffwZQ90XzwQcfYMiQIcjIyMCkSZMQEhKi9M/EyMgIsbGxuHLlCtasWYPNmzfjm2++kamTk5OD77//HkeOHEFiYiIuXryIqVOnSt/fuXMnQkNDsWzZMmRmZmL58uVYuHChdAiLELViCOsJhUJm2LBhDMMwjEQiYU6ePMnw+Xxm1qxZ0vetra2ZyspK6T47duxgnJ2dGYlEIi2rrKxk9PX1mePHjzMMwzBt2rRhVq5cKX2/urqaee2116RtMQzD9O3bl5k+fTrDMAyTnZ3NAGBOnjwpN87Tp08zAJgHDx5IyyoqKhgDAwPm3LlzMnUnTpzIjB49mmEYhpk7dy7j6uoq8/6cOXMaHOtZAJiDBw82+v6qVasYDw8P6euwsDBGR0eHuX37trTsxx9/ZLhcLnPv3j2GYRimffv2zK5du2SOs3TpUsbHx4dhGIa5ceMGA4C5ePFio+0SoigaA39FHD16FIaGhqiuroZEIsGYMWOwaNEi6ftdu3aVGff+448/kJOTAyMjI5njVFRU4Nq1aygpKcG9e/ekd1oDAF1dXXh6ejYYRqmXkZEBHR0d9O3bV+G4c3Jy8PjxYwwcOFCmvKqqCt27dwcAZGZmysQBQKV7MMfHx2Pt2rW4du0aysrKUFNT0+BGRm3btoWdnZ1MOxKJBNnZ2TAyMsK1a9cwceJEBAYGSuvU1NTAxMRE6XgIeRFK4K+I/v37Y8OGDeDxeLC1tW2w+qRVq1Yyr8vKyuDh4YGdO3c2OJalpaVKMejr6yu9T1lZGQDg2LFjMokTUO/jrVJSUjB27FgsXrwYfn5+MDExwZ49e/D1118rHevmzZsbfKHo6OioLVZC6lECf0W0atVK4ZvEA0CPHj0QHx8PKyurRm+n2aZNG5w/fx59+vQBUNfTTEtLQ48ePeTW79q1KyQSCX7++WeZp3fXq/8NoLa2Vlrm6uoKPp+P3NzcRnvunTp1kk7I1vvtt99efJJPOXfuHBwcHDB//nxp2a1btxrUy83Nxd27d2Fraytth8vlwtnZGdbW1rC1tcX169cxduxYpdonRBU0iUnkGjt2LCwsLDBs2DCcOXMGN27cQHJyMoKDg3H79m0AwPTp07FixQocOnQIWVlZmDp16nMvnHF0dIRQKMTHH3+MQ4cOSY/5/fffAwAcHBzA4XBw9OhRFBUVoaysDEZGRpg1axZmzJiB7du349q1a0hPT8e6deukE4OTJ0/G33//jdmzZyM7Oxu7du1S+ok1r7/+OnJzc7Fnzx5cu3YNa9eulTshKxAIIBQK8ccff+DMmTMIDg7GRx99JH3u4eLFixEeHo61a9fi6tWr+PPPP7Ft2zZEREQoFQ8hCtH0IDxpfk9PYirz/r1795iAgADGwsKC4fP5jJOTExMYGMiUlJQwDFM3aTl9+nTG2NiYMTU1ZcRiMRMQENDoJCbDMMyTJ0+YGTNmMG3atGF4PB7ToUMHJiYmRvr+kiVLGBsbG4bD4TBCoZBhmLqJ18jISMbZ2ZnR09NjLC0tGT8/P+bnn3+W7nfkyBGmQ4cODJ/PZ3r37s3ExMQoPYk5e/ZspnXr1oyhoSHj7+/PfPPNN4yJiYn0/bCwMMbNzY359ttvGVtbW0YgEDAjR45k7t+/L3PcnTt3Mu7u7gyPx2PMzMyYPn36MAcOHGAYhiYxiXrRE3kIIURL0RAKIYRoKUrghBCipSiBE0KIlqIETgghWooSOCGEaClK4IQQoqUogRNCiJaiBE4IIVqKEjghhGgpSuCEEKKlKIETQoiWogROCCFa6v/0cOc8IcMzGQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 400x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Tuned KNeighborsClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = KNeighborsClassifier(**KNeighborsClassifierTuned.best_params))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('KNeighborsClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 1 1 1 1 1 1 1]\n",
      "Balanced Accuracy 0.9444444444444444\n",
      "Precision 0.8\n",
      "Recall 1.0\n",
      "ROC AUC score 0.9444444444444444\n",
      "F1 score 0.888888888888889\n",
      "MCC 0.8432740427115678\n",
      "Validation score 0.9230769230769231\n",
      "Confusion matrix\n",
      " [[16  2]\n",
      " [ 0  8]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAFaCAYAAAAHLgZvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+aUlEQVR4nO3deVxUVf8H8M8MwgyKbLKJIqgkiAsoPhLmmhgtP5fKJVfEpTTJhTR3cUtMy0hTedQU19Q0zS3SSNKSNBfKSjEUd0EJWVVQ5vz+8GFyBHRmWIZ7/bx93dfLOXPuvd/B65cz55x7rkIIIUBERJKjNHUARERkHCZwIiKJYgInIpIoJnAiIoliAicikigmcCIiiWICJyKSKCZwIiKJYgInIpIoJnAiIoliAiciKqNDhw6ha9eucHV1hUKhwM6dO5+6T3x8PFq2bAmVSgVPT0/ExMQYfF4mcCKiMsrLy4Ovry+WLl2qV/2UlBS89tpr6NSpExITEzF27FgMGzYM3333nUHnVXAxKyKi8qNQKLBjxw706NGj1DoTJ07E3r178ccff2jL3nrrLWRmZiI2Nlbvc1UrS6BERFXZvXv3UFBQYNS+QggoFAqdMpVKBZVKVea4EhISEBQUpFMWHByMsWPHGnQcJnAikqV79+7BsmYt4MEdo/a3srJCbm6uTllERARmzpxZ5thSU1Ph7OysU+bs7Izs7GzcvXsXlpaWeh2HCZyIZKmgoAB4cAeqJqGAmYVhOxcWIPfPNbhy5Qqsra21xeXR+i5PTOBEJG/VLKAwMyzxiv/1nFhbW+sk8PLi4uKCtLQ0nbK0tDRYW1vr3foGmMCJSO4UyoeboftUoMDAQOzbt0+n7MCBAwgMDDToOJxGSERURrm5uUhMTERiYiKAh9MEExMTcfnyZQDA5MmTMWjQIG39ESNG4MKFC/jggw9w9uxZLFu2DFu3bsW4ceMMOi9b4EQkbwrFw83QfQxw/PhxdOrUSfs6PDwcABASEoKYmBjcuHFDm8wBoH79+ti7dy/GjRuHzz77DHXr1sWqVasQHBxsWJicB05EcpSdnQ0bGxuoWoYZ3gdemI/8k58jKyurQvrAywtb4EQkb5XQAjcV9oFLUMeOHdGxY0ft64sXL0KhUBi1lkJZDB48GB4eHpV6TmOtX78e3t7eMDc3h62tbbkff+bMmcVu+niWmeqaLJny34FMfTeJpEZpRGmgmJgYKBQKqNVqXLt2rdj7HTt2RNOmTU0Q2bNtx44deOWVV+Dg4AALCwu4urqid+/e+OGHHyr0vGfPnsXgwYPRsGFDrFy5EitWrKjQ81U2hUIBhUKBYcOGlfj+1KlTtXXS09MNPv6+ffvK5eYVkylqgRu6SYAsE3iR/Px8zJ8/39RhVDh3d3fcvXsXAwcONHUoJRJCIDQ0FG+88QbS0tIQHh6O6OhojBo1ChcuXEDnzp1x5MiRCjt/fHw8NBoNPvvsMwwePBi9e/cu93NMmzYNd+/eLffj6kutVmP79u0l3jb+5ZdfQq1WG33sffv2YdasWQbtU9WvSbmQdQL38/PDypUrcf369Qo7hxDCpP9xAWi/bZiZmZk0jtJ88skniImJwdixY3HixAlMmTIFQ4YMwdSpU3H8+HGsW7cO1apV3HDMzZs3AaBCuk6KVKtWrUxJsqxefvllZGdn49tvv9UpP3LkiHblu8rw4MEDFBQUVK1r0tDuE2PmjZuINKI00pQpU1BYWKhXK/zBgweYM2cOGjZsCJVKBQ8PD0yZMgX5+fk69Tw8PPB///d/+O6779CqVStYWlriv//9L+Lj46FQKLB161bMmjULderUQc2aNdGzZ09kZWUhPz8fY8eOhZOTE6ysrBAaGlrs2GvWrMGLL74IJycnqFQq+Pj4YPny5U+N/fH+xqJYStoe77P+9ttv0a5dO9SoUQM1a9bEa6+9hj///LPYOXbu3ImmTZtCrVajadOm2LFjx1PjAoC7d+8iMjIS3t7e+Pjjj0vsJx44cCBat26tfX3hwgX06tUL9vb2qF69Op5//nns3btXZ59Hf94ffvgh6tatC7Vajc6dOyM5OVlbz8PDAxEREQAAR0dHKBQKbXfAo39/lIeHBwYPHqx9ff/+fcyaNQvPPfcc1Go1atWqhbZt2+LAgQPaOiX1gRt6Tf30009o3bo11Go1GjRogHXr1j35h/uIOnXqoH379ti0aZNO+caNG9GsWbMSuwwPHz6MXr16oV69elCpVHBzc8O4ceN0GiSDBw/WLpH66HUE/Hvdffzxx4iKitJ+zr/++qvYNXnz5k04OjqiY8eOeHTiW3JyMmrUqIE+ffro/VkNJuMuFFnPQqlfvz4GDRqElStXYtKkSXB1dS217rBhw7B27Vr07NkT77//Po4ePYrIyEicOXOmWLJKSkpC37598c4772D48OHw8vLSvhcZGQlLS0tMmjQJycnJWLJkCczNzaFUKnH79m3MnDkTv/zyC2JiYlC/fn3MmDFDu+/y5cvRpEkTdOvWDdWqVcPu3bvx7rvvQqPRYNSoUXp/7saNG2P9+vU6ZZmZmQgPD4eTk5O2bP369QgJCUFwcDA++ugj3LlzB8uXL0fbtm1x6tQpbbLfv38/3nzzTfj4+CAyMhL//PMPQkNDUbdu3afG8tNPPyEjIwNjx47VqzWWlpaGNm3a4M6dOxg9ejRq1aqFtWvXolu3bti2bRtef/11nfrz58+HUqnE+PHjkZWVhQULFqB///44evQoACAqKgrr1q3Djh07sHz5clhZWaF58+ZPjeNRM2fORGRkJIYNG4bWrVsjOzsbx48fx8mTJ9GlS5dS9zPkmkpOTkbPnj0xdOhQhISEYPXq1Rg8eDD8/f3RpEkTveLs168fxowZg9zcXFhZWeHBgwf46quvEB4ejnv37hWr/9VXX+HOnTsYOXIkatWqhWPHjmHJkiW4evUqvvrqKwDAO++8g+vXr+PAgQPFrqkia9aswb179/D2229DpVLB3t4eGo1Gp46TkxOWL1+OXr16YcmSJRg9ejQ0Gg0GDx6MmjVrYtmyZXp9RqNUwTsxy42QoTVr1ggA4tdffxXnz58X1apVE6NHj9a+36FDB9GkSRPt68TERAFADBs2TOc448ePFwDEDz/8oC1zd3cXAERsbKxO3YMHDwoAomnTpqKgoEBb3rdvX6FQKMQrr7yiUz8wMFC4u7vrlN25c6fYZwkODhYNGjTQKevQoYPo0KGD9nVKSooAINasWVPiz0Oj0Yj/+7//E1ZWVuLPP/8UQgiRk5MjbG1txfDhw3XqpqamChsbG51yPz8/Ubt2bZGZmakt279/vwBQ7DM87rPPPhMAxI4dO55Yr8jYsWMFAHH48GFtWU5Ojqhfv77w8PAQhYWFQoh/f96NGzcW+fn5xc53+vRpbVlERIQAIG7duqVzLgAiIiKiWAzu7u4iJCRE+9rX11e89tprT4y76BxFjLmmDh06pC27efOmUKlU4v3333/ieYs+x6hRo0RGRoawsLAQ69evF0IIsXfvXqFQKMTFixdL/BmUdL1FRkYKhUIhLl26pC0bNWqUKClVFF131tbW4ubNmyW+9/g12bdvX1G9enVx7tw5sXDhQgFA7Ny586mf0RhZWVkCgFA9/4FQt51u0KZ6/gMBQGRlZVVIbOVFIr9mjNegQQMMHDgQK1aswI0bN0qsU7QmQdHdU0Xef/99ACj29b1+/fql3jE1aNAgmJuba18HBARACIEhQ4bo1AsICMCVK1fw4MEDbdmji9hkZWUhPT0dHTp0wIULF5CVlfW0j1qqOXPmYM+ePYiJiYGPjw+Ah+suZGZmom/fvkhPT9duZmZmCAgIwMGDBwEAN27cQGJiIkJCQmBjY6M9ZpcuXbTHepLs7GwAQM2aNfWKdd++fWjdujXatm2rLbOyssLbb7+Nixcv4q+//tKpHxoaCguLf1eaa9euHYCH3TDlxdbWFn/++Sf+/vtvvfcx9Jry8fHRxg487O7x8vIy6HPY2dnh5ZdfxpdffgkA2LRpE9q0aQN3d/cS6z96veXl5SE9PR1t2rSBEAKnTp3S+7xvvvkmHB0d9ar7+eefw8bGBj179sT06dMxcOBAdO/eXe9zGYV94NI2bdo0PHjwoNS+8EuXLkGpVMLT01On3MXFBba2trh06ZJOef369Us9V7169XReFyU9Nze3YuUajUYnMf/8888ICgpCjRo1YGtrC0dHR0yZMgUAjE7gsbGxmDVrFiZPnow333xTW16UjF588UU4OjrqbPv379cO/BV99ueee67YsR/tOipN0V1sOTk5esV76dKlEo/buHFjnXiKPP7ztrOzAwDcvn1br/PpY/bs2cjMzESjRo3QrFkzTJgwAb///vsT9zH0mnr8cwAPP4uhn6Nfv344cOAALl++jJ07d6Jfv36l1r18+TIGDx4Me3t7WFlZwdHRER06dABg2PX2pP8Pj7O3t8fixYvx+++/w8bGBosXL9Z7XypO1n3gRRo0aIABAwZgxYoVmDRpUqn19L0R40nLPZbWz1taufjfgM758+fRuXNneHt7Y9GiRXBzc4OFhQX27duHTz/9tFifoj5SUlLQv39/dOnSBXPnztV5r+h469evh4uLS7F9y2tWiLe3NwDg9OnTT3zElLGe9nM1RmFhoc7r9u3b4/z58/jmm2+wf/9+rFq1Cp9++imio6NLnXtdRN9rqrw+R7du3aBSqRASEoL8/PxSp0wWFhaiS5cuyMjIwMSJE+Ht7Y0aNWrg2rVrGDx4sEHXmyHLnwLQPvfx9u3buHr1aoXODgLwv0FJQ/vAOYhZpUybNg0bNmzARx99VOw9d3d3aDQa/P3339qWHvBwQC0zM7PUr6Dlaffu3cjPz8euXbt0WmNFXRmGunv3Lt544w3Y2triyy+/hFKpewE3bNgQwMPBpccf7fSoos9eUvdBUlLSU+No27Yt7Ozs8OWXX2LKlClPHch0d3cv8bhnz57Viac82NnZITMzU6esoKCgxK42e3t7hIaGIjQ0FLm5uWjfvj1mzpxZagI31TVlaWmJHj16YMOGDdqbpkpy+vRpnDt3DmvXrtVZJe/RmTVFyvMO09jYWKxatQoffPABNm7ciJCQEBw9erRCp5FCqXi4GbqPBDwTXSjAw4Q1YMAA/Pe//0VqaqrOe6+++iqAhzMWHrVo0SIAqJQ5tEWJ7dEWV1ZWFtasWWPU8UaMGIFz585hx44d2m6FRwUHB8Pa2hrz5s3D/fv3i71/69YtAEDt2rXh5+eHtWvX6nytPnDgQLH+6JJUr14dEydOxJkzZzBx4sQSW5QbNmzAsWPHADz8tzh27BgSEhK07+fl5WHFihXw8PDQq99dXw0bNsShQ4d0ylasWFGsBf7PP//ovLaysoKnp2ex6YCPMuU1NX78eERERGD69Oml1inpehNC4LPPPitWt0aNGgBQ7JedoTIzM7UzeebNm4dVq1bh5MmTmDdvXpmO+1Qy7gN/ZlrgwMNbitevX4+kpCSdqVm+vr4ICQnBihUrkJmZiQ4dOuDYsWNYu3YtevToobNMZEV56aWXYGFhga5du+Kdd95Bbm4uVq5cCScnp1IHX0uzd+9erFu3Dm+++SZ+//13nf5aKysr9OjRA9bW1li+fDkGDhyIli1b4q233oKjoyMuX76MvXv34oUXXsDnn38O4OHUyNdeew1t27bFkCFDkJGRgSVLlqBJkybFnhlYkgkTJuDPP//EJ598goMHD6Jnz55wcXFBamoqdu7ciWPHjmnvxJw0aRK+/PJLvPLKKxg9ejTs7e2xdu1apKSkYPv27cW+SZTFsGHDMGLECLz55pvo0qULfvvtN3z33XfFWq0+Pj7o2LEj/P39YW9vj+PHj2Pbtm0ICwsr9dimvKZ8fX3h6+v7xDre3t5o2LAhxo8fj2vXrsHa2hrbt28vsc/d398fADB69GgEBwfDzMwMb731lsFxjRkzBv/88w++//57mJmZ4eWXX8awYcMwd+5cdO/e/akxG03Gi1k9Uwnc09MTAwYMwNq1a4u9t2rVKjRo0AAxMTHYsWMHXFxcMHnyZO1NIBXNy8sL27Ztw7Rp0zB+/Hi4uLhg5MiRcHR0LDaD5WmKWs/bt2/H9u3bdd5zd3fX9kX369cPrq6umD9/PhYuXIj8/HzUqVMH7dq1Q2hoqHafl19+GV999RWmTZuGyZMno2HDhlizZg2++eYbxMfHPzUepVKJdevWoXv37lixYgU+/vhjZGdnw9HREe3bt8eCBQu0TyJxdnbGkSNHMHHiRCxZsgT37t1D8+bNsXv37nJvtQ4fPhwpKSn44osvEBsbi3bt2uHAgQPo3LmzTr3Ro0dj165d2L9/P/Lz8+Hu7o65c+diwoQJTzy+qa+pJzE3N8fu3bsxevRoREZGQq1W4/XXX0dYWFixRPrGG2/gvffew+bNm7FhwwYIIQxO4Lt27cK6devwySefaMdFgIffSA4cOICQkBD8+uuvOjO4yo2M54FzPXAikiXteuAdIqCoZtgyB+LBPeT/OIvrgRMRmRS7UIiIJErGXShM4EQkb2yBExFJFFvgREQSJeMWuDR+zRARUTGSboFrNBpcv34dNWvW5ANliWRGCIGcnBy4urqW8QYuY+6slEbbVtIJ/Pr168VW+SMiebly5YpeDw8plYy7UCSdwIvWmLbo/KHBE/VJmk5EDzB1CFRJcnNy8HxzT73Xki8VVyOsmoq6TRTV1FCYG7akJUlTzZpV9644qhhl7h7lLBQiIomScReKNH7NEBFRMWyBE5G8sQuFiEiiZNyFwgRORPLGFjgRkUSxBU5EJE0KhcLwqYgSSeDS+J5ARETFsAVORLIm5xY4EzgRyZvif5uh+0gAEzgRyRpb4EREEsUETkQkUXJO4JyFQkQkUWyBE5GsybkFzgRORPLGWShERNLEFjgRkUQ9XArF0AReMbGUNyZwIpI1BYxogUskg3MWChGRRLEFTkSyxj5wIiKp4iwUIiKJMqIFLtgCJyIyPWO6UAwf9DQNJnAikjU5J3DOQiEiKidLly6Fh4cH1Go1AgICcOzYsSfWj4qKgpeXFywtLeHm5oZx48bh3r17ep+PCZyI5E1h5GagLVu2IDw8HBERETh58iR8fX0RHByMmzdvllh/06ZNmDRpEiIiInDmzBl88cUX2LJlC6ZMmaL3OZnAiUjWirpQDN0AIDs7W2fLz88v9TyLFi3C8OHDERoaCh8fH0RHR6N69epYvXp1ifWPHDmCF154Af369YOHhwdeeukl9O3b96mt9kcxgRORrJUlgbu5ucHGxka7RUZGlniOgoICnDhxAkFBQdoypVKJoKAgJCQklLhPmzZtcOLECW3CvnDhAvbt24dXX31V78/GQUwikrWyDGJeuXIF1tbW2nKVSlVi/fT0dBQWFsLZ2Vmn3NnZGWfPni1xn379+iE9PR1t27aFEAIPHjzAiBEj2IVCRFSkLC1wa2trna20BG6M+Ph4zJs3D8uWLcPJkyfx9ddfY+/evZgzZ47ex2ALnIiojBwcHGBmZoa0tDSd8rS0NLi4uJS4z/Tp0zFw4EAMGzYMANCsWTPk5eXh7bffxtSpU6FUPr19zRY4EclbJcxCsbCwgL+/P+Li4rRlGo0GcXFxCAwMLHGfO3fuFEvSZmZmAAAhhF7nZQuciGStsm7kCQ8PR0hICFq1aoXWrVsjKioKeXl5CA0NBQAMGjQIderU0Q6Edu3aFYsWLUKLFi0QEBCA5ORkTJ8+HV27dtUm8qdhAiciWausBN6nTx/cunULM2bMQGpqKvz8/BAbG6sd2Lx8+bJOi3vatGlQKBSYNm0arl27BkdHR3Tt2hUffvih/nEKfdvqVVB2djZsbGygCv4ECnNLU4dDlSApJtTUIVAlycnJRtP6zsjKytKZCaKvovzgOmwTlBbVDdpXU3AH11f1M/rclYV94EREEsUuFCKSN64HTkQkTXJejZAJnIhkjQmciEiijHkqvUIifShM4EQka3JugXMWChGRRLEFTkTyxlkoRETSJOcuFCZwIpI1JnAiIolSKB5uhu4jBUzgRCRrDxO4oS3wCgqmnHEWChGRRLEFTkTyZkQXCmehEBFVARzEJCKSKA5iEhFJlFKpgFJpWEYWBtY3FSZwIpI1ObfAOQulinjn1SY4u6o/bm8fjkMfv4FWzzk9sX5Yt+b4bXlfZGwbjr9XD8SCYW2gMv/3QahWluZYOOwFJH0xABnbhuPggtfh/5xjRX8M0tPaL6LxQgsvNKpji+4vtUPiyV9LrXvu7F94Z/BbeKGFF9wdLPFF9JJidY4e+QlD+r2J/zSpD3cHS3y3b1dFhk9VRJVI4EuXLoWHhwfUajUCAgJw7NgxU4dUqXq2bYiPhr2AD788jsCx2/B7yj/YNfv/4GhT8nM++3R4DnNCAjBv83H4vbsZI5YcRM+2npg9KEBbZ/l7HfFii7oYsigOrd7bgu9PXcHeOV3hal+jsj4WlWL3jq8wd/pEjJkwFXt+SEDjJs0xsFc3pN+6WWL9u3fuoJ57fUycPgeOTi4l1rlzJw+NmzbDnAVRFRi5NBUNYhq6SYHJE/iWLVsQHh6OiIgInDx5Er6+vggODsbNmyVfzHI0uocv1nz3F9bHJeHsldt4b9mPuJt/HyFdvEus/7y3MxLOpGLLj3/j8s0cxJ26iq2H/karRg9b7WoLM/Ro0wBT1yTg5z9v4MKNbHz45XGcv5GN4a82qcyPRiVYtXwx3hoYit79BqGRV2PM+2QJLC0tsXXT2hLr+7ZshamzItHtjd5QqSxKrNMpKBgTpszEy691r8jQJamoC8XQTQpMnsAXLVqE4cOHIzQ0FD4+PoiOjkb16tWxevVqU4dWKcyrKdHC0xE//HZVWyYE8EPiNbT2ci5xn1/OpqFFQ0dtN4uHc00Et3JH7PHLAIBqZkpUM1PiXkGhzn73Ch6gjU/JLTiqHAUFBTj92ym07fCitkypVKJthxdx8tdn65tnZZFzC9ykg5gFBQU4ceIEJk+erC1TKpUICgpCQkJCsfr5+fnIz8/Xvs7Ozq6UOCuSg7Ua1cyUuHn7rk75zcw78KprW+I+W378G7Ws1Yj7qAcUCsC8mhlW7PsTC786CQDIvXsfv5xJxeS3/JF09TbSMu+id3tPBHg54/wN6f/MpOz2P+koLCyEg6PuGIeDoxPO/51koqjkTc7zwE3aAk9Pf3gxOzvrtjSdnZ2RmpparH5kZCRsbGy0m5ubW2WFWqW0a+qKCb1aYkz0YQSO3YY+H8bilf/Uw6Q+/to6QxbFQaFQ4MLaEGR9/TZGdW2GrYeSoRHChJETVT45d6FIahrh5MmTER4ern2dnZ0t+SSenn0PDwo1cLLTHbB0sq2O1Nt3StwnYkBrfHnwHGL2nwEA/HkpA9XV1bA0rAM+2noCQgApqdl4afI3qK6qBuvqFki9fQfrP+iClFS2wE3JrpYDzMzMig1Ypt+6WeoAJVFpTNoCd3B4eDGnpaXplKelpcHFpfjFrFKpYG1trbNJ3f0HGpxKvoVOzetqyxQKoJNvHRxLSitxH0tVNWg0ui3potePf/W7k/8AqbfvwLaGBYJauGHP0ZRy/gRkCAsLCzTzbYGfDx3Ulmk0Gvx86CBa/qe1CSOTr6KHGhu0SWQxFJO2wC0sLODv74+4uDj06NEDwMOLOS4uDmFhYaYMrVIt3vkbVo57ESeSb+H4uTSEdW+O6mpzrPv+LABg1bgXcf2fPMxYdxQAsO/YRYzu4YvfLqTj2Lk0NKxtgxn9W2PfsUvaRB7Uwg0KBXDuWiYa1rbBvNBAnLuaiXXfs5/V1IaNHI33w4ajuZ8/fFu2wuroz3Hnzh306jsIADDu3aFwqe2KidPnAHg4VvR30hnt31NvXMefp39DjRpW8GjQEACQl5uLiynntee4cuki/jz9G2zt7FCnbr1K/oRVi5xv5DF5F0p4eDhCQkLQqlUrtG7dGlFRUcjLy0NoaKipQ6s02346DwcbS8zo/x8421XH7xfS0T1iD25mPhzYdHO00um7nr/lYTdJxIDWcK1VA+nZd7H32CXMXH9UW8emhgVmDwpAHQcrZOTcwzdHLiBi/TE8KNRU+ucjXV1f74V//knHovmzcetmGnyaNse6rd/A0enhWND1q1egVP775Tgt9QZe7fS89vWKpVFYsTQKz7dphy279gMAfk88ibd6BGvrzJk+EQDQ860B+OTzlZXxsaosOQ9iKoQw/ajW559/joULFyI1NRV+fn5YvHgxAgICnrpfdnY2bGxsoAr+BArzkm96IXlJinl2frE/63JystG0vjOysrKM6i4tyg9+U3fDTG3YDWyF9/KQ+GFXo89dWUzeAgeAsLCwZ6rLhIgqj5xb4Ca/kYeIiIxTJVrgREQVhYOYREQSJecuFCZwIpI3PhOTiEia2AInIpIoOfeBcxYKEZFEsQVORLLGLhQiIomScxcKEzgRyRpb4EREEsUETkQkUXLuQuEsFCIiiWILnIhkjV0oREQSJecuFCZwIpI1tsCJiCRKASNa4BUSSfljAiciWVMqFFAamMENrW8qnIVCRCRRbIETkazJeRCTLXAikrWiQUxDN2MsXboUHh4eUKvVCAgIwLFjx55YPzMzE6NGjULt2rWhUqnQqFEj7Nu3T+/zsQVORLKmVDzcDN3HUFu2bEF4eDiio6MREBCAqKgoBAcHIykpCU5OTsXqFxQUoEuXLnBycsK2bdtQp04dXLp0Cba2tnqfkwmciORNYcS0QCMS+KJFizB8+HCEhoYCAKKjo7F3716sXr0akyZNKlZ/9erVyMjIwJEjR2Bubg4A8PDwMOiceiXwXbt26X3Abt26GRQAEVFFKksfeHZ2tk65SqWCSqUqVr+goAAnTpzA5MmTtWVKpRJBQUFISEgo8Ry7du1CYGAgRo0ahW+++QaOjo7o168fJk6cCDMzM73i1CuB9+jRQ6+DKRQKFBYW6lWXiKiqc3Nz03kdERGBmTNnFquXnp6OwsJCODs765Q7Ozvj7NmzJR77woUL+OGHH9C/f3/s27cPycnJePfdd3H//n1EREToFZ9eCVyj0eh1MCKiqkbxvz+G7gMAV65cgbW1tba8pNa3sTQaDZycnLBixQqYmZnB398f165dw8KFC8s3gZfm3r17UKvVZTkEEVGFKssgprW1tU4CL42DgwPMzMyQlpamU56WlgYXF5cS96lduzbMzc11uksaN26M1NRUFBQUwMLC4ulxPrXGYwoLCzFnzhzUqVMHVlZWuHDhAgBg+vTp+OKLLww9HBFRhaqMaYQWFhbw9/dHXFyctkyj0SAuLg6BgYEl7vPCCy8gOTlZp4fj3LlzqF27tl7JGzAigX/44YeIiYnBggULdE7StGlTrFq1ytDDERFVqKJBTEM3Q4WHh2PlypVYu3Ytzpw5g5EjRyIvL087K2XQoEE6g5wjR45ERkYGxowZg3PnzmHv3r2YN28eRo0apfc5De5CWbduHVasWIHOnTtjxIgR2nJfX99SO+uJiEylstZC6dOnD27duoUZM2YgNTUVfn5+iI2N1Q5sXr58GUrlv21mNzc3fPfddxg3bhyaN2+OOnXqYMyYMZg4caLe5zQ4gV+7dg2enp7FyjUaDe7fv2/o4YiIZCMsLAxhYWElvhcfH1+sLDAwEL/88ovR5zO4C8XHxweHDx8uVr5t2za0aNHC6ECIiCpCZXWhmILBLfAZM2YgJCQE165dg0ajwddff42kpCSsW7cOe/bsqYgYiYiMJucHOhjcAu/evTt2796N77//HjVq1MCMGTNw5swZ7N69G126dKmIGImIjMYW+GPatWuHAwcOlHcsRETlTs4PdDD6Rp7jx4/jzJkzAB72i/v7+5dbUERE5UUBw9emkkb6NiKBX716FX379sXPP/+sXfYwMzMTbdq0webNm1G3bt3yjpGIiEpgcB/4sGHDcP/+fZw5cwYZGRnIyMjAmTNnoNFoMGzYsIqIkYjIaJX5QIfKZnAL/Mcff8SRI0fg5eWlLfPy8sKSJUvQrl27cg2OiKisKuuBDqZgcAJ3c3Mr8YadwsJCuLq6lktQRETlhdMIH7Fw4UK89957OH78uLbs+PHjGDNmDD7++ONyDY6IqDzIcQohoGcL3M7OTuc3Ul5eHgICAlCt2sPdHzx4gGrVqmHIkCF6P/yBiKgyyLkFrlcCj4qKquAwiIjIUHol8JCQkIqOg4ioQnAQsxT37t1DQUGBTpk+T68gIqoscu5CMXgQMy8vD2FhYXByckKNGjVgZ2ensxERVSUKIzcpMDiBf/DBB/jhhx+wfPlyqFQqrFq1CrNmzYKrqyvWrVtXETESERmtaC0UQzcpMLgLZffu3Vi3bh06duyI0NBQtGvXDp6ennB3d8fGjRvRv3//ioiTiIgeY3ALPCMjAw0aNADwsL87IyMDANC2bVscOnSofKMjIiojOS8na3ACb9CgAVJSUgAA3t7e2Lp1K4CHLfOixa2IiKoKOa+FYnACDw0NxW+//QYAmDRpEpYuXQq1Wo1x48ZhwoQJ5R4gEVFZyLkFbnAf+Lhx47R/DwoKwtmzZ3HixAl4enqiefPm5RocEVFZ8YEOT+Du7g53d/fyiIWIqNwZ06KWSP7WL4EvXrxY7wOOHj3a6GCIiEh/eiXwTz/9VK+DKRQKkyTwy5uG8Q7QZ4Tdf8JMHQJVElFY8PRKepDznZh6JfCiWSdERFKjhOGzNQye3WEiZe4DJyKqyp75FjgRkVQpjFiNUCL5mwmciORNzsvJSqWrh4iIHsMWOBHJmpz7wI1qgR8+fBgDBgxAYGAgrl27BgBYv349fvrpp3INjoiorIq6UAzdpMDgBL59+3YEBwfD0tISp06dQn5+PgAgKysL8+bNK/cAiYjKQs5roRicwOfOnYvo6GisXLkS5ubm2vIXXngBJ0+eLNfgiIjKig90eERSUhLat29frNzGxgaZmZnlERMRUbmR8408Bsfp4uKC5OTkYuU//fST9kEPRERU8QxO4MOHD8eYMWNw9OhRKBQKXL9+HRs3bsT48eMxcuTIioiRiMhocu4DN7gLZdKkSdBoNOjcuTPu3LmD9u3bQ6VSYfz48XjvvfcqIkYiIqMpYcR64BJ5Lr3BCVyhUGDq1KmYMGECkpOTkZubCx8fH1hZWVVEfEREZfLMrwdeEgsLC/j4+JRnLERE5U7Ot9IbnMA7der0xLuUfvjhhzIFRERUnh4uZmXonZgVFEw5MziB+/n56by+f/8+EhMT8ccffyAkJKS84iIioqcwOIGX9nSemTNnIjc3t8wBERGVJzn3gZfbfPUBAwZg9erV5XU4IqJyIee1UMptNcKEhASo1eryOhwRUblQ/O+PoftIgcEJ/I033tB5LYTAjRs3cPz4cUyfPr3cAiMiKg+chfIIGxsbnddKpRJeXl6YPXs2XnrppXILjIioPDCB/09hYSFCQ0PRrFkz2NnZVVRMRESkB4MGMc3MzPDSSy9x1UEikoyiJ/IYukmBwbNQmjZtigsXLlRELERE5U7Os1CMeqDD+PHjsWfPHty4cQPZ2dk6GxFRVcLVCAHMnj0b77//Pl599VUAQLdu3XS+ZgghoFAoUFhYWP5REhEZyZgn7MjuiTyzZs3CiBEjcPDgwYqMh4ioXFXmLJSlS5di4cKFSE1Nha+vL5YsWYLWrVs/db/Nmzejb9++6N69O3bu3Kn3+fRO4EIIAECHDh30PjgR0bNiy5YtCA8PR3R0NAICAhAVFYXg4GAkJSXBycmp1P0uXryI8ePHo127dgaf06A+cKmMzBIRaRnT/21Eqlu0aBGGDx+O0NBQ+Pj4IDo6GtWrV3/iEiOFhYXo378/Zs2aZdQjKQ2aB96oUaOnJvGMjAyDgyAiqihKKAx+wk5R/ccnZqhUKqhUqmL1CwoKcOLECUyePPnfYyiVCAoKQkJCQqnnmT17NpycnDB06FAcPnzYoBgBAxP4rFmzit2JSURUlZVlNUI3Nzed8oiICMycObNY/fT0dBQWFsLZ2Vmn3NnZGWfPni3xHD/99BO++OILJCYmGhbcIwxK4G+99dYT+3KIiKqasgxiXrlyBdbW1tryklrfxsjJycHAgQOxcuVKODg4GH0cvRM4+7+JSIrKMo3Q2tpaJ4GXxsHBAWZmZkhLS9MpT0tLg4uLS7H658+fx8WLF9G1a1dtmUajAQBUq1YNSUlJaNiw4dPjfGqN/ymahUJERLosLCzg7++PuLg4bZlGo0FcXBwCAwOL1ff29sbp06eRmJio3bp164ZOnTohMTGxWNdNafRugRf9diAikpLKeiJPeHg4QkJC0KpVK7Ru3RpRUVHIy8tDaGgoAGDQoEGoU6cOIiMjoVar0bRpU539bW1tAaBY+ZOU2wMdiIiqIiWM6EIxYh5hnz59cOvWLcyYMQOpqanw8/NDbGysdmDz8uXLUCrL7SFoAJjAiUjmKvOZmGFhYQgLCyvxvfj4+CfuGxMTY/D5mMCJSNaUMHzVvvJtJ1ccJnAikjVj1veWyqw7qfyiISKix7AFTkSyZszSJtJofzOBE5HMcT1wIiIJk0Y6NhwTOBHJWmVOI6xsTOBEJGuchUJERFUOW+BEJGu8kYeISKLk3IXCBE5EssZ54EREEsUWOBGRRMm5D1wqcRIR0WPYAiciWWMXChGRRHEQk4hIongrPRGRRCmhMPgZl8Y8E9MUOIhZRUUvWwovTw/YWqnRrk0Afj127In1t2/7Cr5NvWFrpUYrv2aI/XZfJUVKZfVCy4bYFvUOLuz/EHdPfY6uHZs/dZ92/s/hyKaJyDz6Kf74JgIDugZUQqTSVNQCN3STApMm8EOHDqFr165wdXWFQqHAzp07TRlOlfHV1i2YOCEcU6dFIOHYSTRv7oturwXj5s2bJdZPOHIEIQP6IiR0KH759RS6du+B3m/2wJ9//FHJkZMxaliqcPrcNYyN3KJXfXfXWtixZAQOHT+HgLfm4/NNB7F8Rj8EBTau4EipqjFpAs/Ly4Ovry+WLl1qyjCqnMVRixA6dDgGDQ5FYx8fLFkWDcvq1bE2ZnWJ9Zd+/hleCn4Z4e9PgHfjxoiYNQd+LVoietnnlRw5GWP/z39h1rI92HXwd73qD+/ZFhev/YNJi3YgKSUN0VsOYUdcIt7r36mCI5UmhZF/pMCkCfyVV17B3Llz8frrr5syjCqloKAAp06ewIudg7RlSqUSL74YhGO/JJS4z9FfEtDpxSCdsi4vBeNoKfVJ2gJ86+Pg0SSdsgNHziCgeX0TRVS1ybkLRVKDmPn5+cjPz9e+zs7ONmE0FSM9PR2FhYVwcnLWKXdydkZS0tkS90lLTYWT82P1nZyRlpZaYXGS6TjXskZaRo5O2c2MbNjUtIRaZY57+fdNFFnVpDBiEJMt8AoQGRkJGxsb7ebm5mbqkIioipNzC1xSCXzy5MnIysrSbleuXDF1SOXOwcEBZmZmuHkzTaf8ZloaXFxcStzH2cUFN9Meq38zDc7OJdcnaUv7JxvO9jV1ypzsrZGVc5et7xIwgVcRKpUK1tbWOpvcWFhYoEVLfxz8IU5bptFocPBgHFo/H1jiPgHPByL+YJxOWdz3BxBQSn2StqO/paBjay+dss7Pe+Po7ykmiohMRVIJ/Fkxemw41nyxEhvWrcXZM2cwetRI3MnLw6CQUADA0MGDMH3qZG39UWFjsP+7WER9+gmSzp7F3NkzcfLEcYx4N8xEn4AMUcPSAs0b1UHzRnUAAB51aqF5ozpwc7EDAMx+rxtWzRmorb9y20+oX7cWPhzTHY08nPF2r3Z4s0sLLNl40CTxV3VynoVi0kHM3NxcJCcna1+npKQgMTER9vb2qFevngkjM61evfsg/dYtzJ41A2mpqWju64dv9sTC+X8DlVeuXIZS+e/v3sA2bRCzfhNmRUxDxLQp8HzuOWzdvhNNmjY11UcgA7T0ccf+VWO0rxeMfxMAsH7XL3g7YgNcHKzh5mKvff/S9X/w+nvRWDD+DYzq1xHX0jIxcvYmfJ9wptJjlwKl4uFm6D5SoBBCCFOdPD4+Hp06FZ+7GhISgpiYmKfun52dDRsbG6T9kyXL7hQqzu4//FbxrBCFBcg/vRJZWcb9/y7KD7t+TUENq5pP3+ERebk56Paf+kafu7KYtAXesWNHmPD3BxE9A+S8mBX7wImIJEpSN/IQERnq4Xrght7IIw1M4EQka3IexGQCJyJZM2ZaIKcREhFVAXIexGQCJyJZk/MzMTkLhYhIotgCJyJZU0IBpYF9IlJ5JiYTOBHJmpy7UJjAiUjeZJzBmcCJSNY4jZCISKqMeUCDNPI3Z6EQEUkVW+BEJGsy7gJnAicimZNxBmcCJyJZ4yAmEZFEcS0UIiKJknEPCmehEBFJFRM4EcmbwsjNCEuXLoWHhwfUajUCAgJw7NixUuuuXLkS7dq1g52dHezs7BAUFPTE+iVhAiciWVMY+cdQW7ZsQXh4OCIiInDy5En4+voiODgYN2/eLLF+fHw8+vbti4MHDyIhIQFubm546aWXcO3aNb3PyQRORLJWNIhp6GaoRYsWYfjw4QgNDYWPjw+io6NRvXp1rF69usT6GzduxLvvvgs/Pz94e3tj1apV0Gg0iIuL0/ucTOBEJGtl6UHJzs7W2fLz80s8R0FBAU6cOIGgoCBtmVKpRFBQEBISEvSK886dO7h//z7s7e31/mxM4EQkb2XI4G5ubrCxsdFukZGRJZ4iPT0dhYWFcHZ21il3dnZGamqqXmFOnDgRrq6uOr8EnobTCImISnHlyhVYW1trX6tUqgo5z/z587F582bEx8dDrVbrvR8TOBHJWlnuxLS2ttZJ4KVxcHCAmZkZ0tLSdMrT0tLg4uLyxH0//vhjzJ8/H99//z2aN29uUJzsQiEiWauMQUwLCwv4+/vrDEAWDUgGBgaWut+CBQswZ84cxMbGolWrVgZ/NrbAiUjWKutOzPDwcISEhKBVq1Zo3bo1oqKikJeXh9DQUADAoEGDUKdOHW0/+kcffYQZM2Zg06ZN8PDw0PaVW1lZwcrKSq9zMoETkbxVUgbv06cPbt26hRkzZiA1NRV+fn6IjY3VDmxevnwZSuW/nR7Lly9HQUEBevbsqXOciIgIzJw5U69zMoETkaxV5mqEYWFhCAsLK/G9+Ph4ndcXL1406hyPYh84EZFEsQVORLLG5WSJiCRKzsvJMoETkbzJOIMzgRORrPGRakREEiXnPnDOQiEikii2wIlI1mTcBc4ETkQyJ+MMzgRORLLGQUwiIqky5hFp0sjfTOBEJG8y7kHhLBQiIqliC5yI5E3GTXAmcCKSNQ5iEhFJlJzvxGQCJyJZk3EPChM4EcmcjDM4Z6EQEUkUW+BEJGscxCQikigFjBjErJBIyh8TOBHJmoy7wJnAiUjeOI2QiEiy5NsGl3QCF0IAAHKys00cCVUWUVhg6hCokhT9Wxf9P6fiJJ3Ac3JyAACe9d1MHAkRVZScnBzY2NgYvT+7UKooV1dXXLlyBTVr1oRCKj/xcpCdnQ03NzdcuXIF1tbWpg6HKtiz+u8thEBOTg5cXV3LdBz5dqBIPIErlUrUrVvX1GGYjLW19TP1H/pZ9yz+e5el5V2ELXAiIonijTxERFIl4z4UroUiQSqVChEREVCpVKYOhSoB/72pNArBOTpEJEPZ2dmwsbHB31fSUdPAsYOc7Gw85+aArKysKj3uwC4UIpI1DmISEUkUBzGJiKRKxoOYTOBEJGsyzt+chSJFS5cuhYeHB9RqNQICAnDs2DFTh0QV4NChQ+jatStcXV2hUCiwc+dOU4dEVQwTuMRs2bIF4eHhiIiIwMmTJ+Hr64vg4GDcvHnT1KFROcvLy4Ovry+WLl1q6lAkrWgQ09BNCjiNUGICAgLwn//8B59//jkAQKPRwM3NDe+99x4mTZpk4uiooigUCuzYsQM9evQwdSiSUTSNMOV6hsFTAbOzs1Hf1b7KTyNkC1xCCgoKcOLECQQFBWnLlEolgoKCkJCQYMLIiKouObfAmcAlJD09HYWFhXB2dtYpd3Z2RmpqqomiIiJT4SwUIpI1Od/Iwxa4hDg4OMDMzAxpaWk65WlpaXBxcTFRVERkKkzgEmJhYQF/f3/ExcVpyzQaDeLi4hAYGGjCyIiqLoWRf6SAXSgSEx4ejpCQELRq1QqtW7dGVFQU8vLyEBoaaurQqJzl5uYiOTlZ+zolJQWJiYmwt7dHvXr1TBiZtMi5C4UJXGL69OmDW7duYcaMGUhNTYWfnx9iY2OLDWyS9B0/fhydOnXSvg4PDwcAhISEICYmxkRRSY+c78TkPHAikqWieeBXb942ah54XSc7zgMnIqKKwS4UIpI1LidLRCRRHMQkIpIoOQ9isg+ciORNYeRmBEOXev7qq6/g7e0NtVqNZs2aYd++fQadjwmciGStsm7kMXSp5yNHjqBv374YOnQoTp06hR49eqBHjx74448/9P9snEZIRHJUNI0wNd3wqYDZ2dlwcbAxaBqhoUs99+nTB3l5edizZ4+27Pnnn4efnx+io6P1Oif7wIlI1nJysg0elMzJyQbwMJE/SqVSQaVSFatftNTz5MmTtWVPW+o5ISFBe3NWkeDgYIOevMQuFCpXgwcP1nnoQMeOHTF27NhKjyM+Ph4KhQKZmZml1jH0MWUzZ86En59fmeK6ePEiFAoFEhMTy3QcejoLCwu4uLjgufpucK5lY9D2XH03WFlZwc3NDTY2NtotMjKyxHMZs9RzampqmZeGZgv8GTB48GCsXbsWAGBubo569eph0KBBmDJlCqpVq9hL4Ouvv4a5ubledePj49GpUyfcvn0btra2FRoXyZ9arUZKSgoKCgqM2l8IAcVjTfeSWt+mxAT+jHj55ZexZs0a5OfnY9++fRg1ahTMzc11vvIVKSgogIWFRbmc197evlyOQ2QMtVoNtVpd4ecxZqlnFxeXMi8NzS6UZ4RKpYKLiwvc3d0xcuRIBAUFYdeuXQD+7fb48MMP4erqCi8vLwDAlStX0Lt3b9ja2sLe3h7du3fHxYsXtccsLCxEeHg4bG1tUatWLXzwwQd4fEz88S6U/Px8TJw4EW5ublCpVPD09MQXX3yBixcvahdusrOzg0KhwODBgwE8HAyKjIxE/fr1YWlpCV9fX2zbtk3nPPv27UOjRo1gaWmJTp066cSpr4kTJ6JRo0aoXr06GjRogOnTp+P+/fvF6v33v/+Fm5sbqlevjt69eyMrK0vn/VWrVqFx48ZQq9Xw9vbGsmXLDI6FpMWYpZ4DAwN16gPAgQMHDFsaWpDshYSEiO7du+uUdevWTbRs2VL7vpWVlRg4cKD4448/xB9//CEKCgpE48aNxZAhQ8Tvv/8u/vrrL9GvXz/h5eUl8vPzhRBCfPTRR8LOzk5s375d/PXXX2Lo0KGiZs2aOufq0KGDGDNmjPZ17969hZubm/j666/F+fPnxffffy82b94sHjx4ILZv3y4AiKSkJHHjxg2RmZkphBBi7ty5wtvbW8TGxorz58+LNWvWCJVKJeLj44UQQly+fFmoVCoRHh4uzp49KzZs2CCcnZ0FAHH79u1Sfy4AxI4dO7Sv58yZI37++WeRkpIidu3aJZydncVHH32kfT8iIkLUqFFDvPjii+LUqVPixx9/FJ6enqJfv37aOhs2bBC1a9cW27dvFxcuXBDbt28X9vb2IiYmRgghREpKigAgTp06pe8/H0nE5s2bhUqlEjExMeKvv/4Sb7/9trC1tRWpqalCCCEGDhwoJk2apK3/888/i2rVqomPP/5YnDlzRkRERAhzc3Nx+vRpvc/JBP4MeDSBazQaceDAAaFSqcT48eO17zs7O2sTsxBCrF+/Xnh5eQmNRqMty8/PF5aWluK7774TQghRu3ZtsWDBAu379+/fF3Xr1i01gSclJQkA4sCBAyXGefDgwWJJ9969e6J69eriyJEjOnWHDh0q+vbtK4QQYvLkycLHx0fn/YkTJxqcwB+3cOFC4e/vr30dEREhzMzMxNWrV7Vl3377rVAqleLGjRtCCCEaNmwoNm3apHOcOXPmiMDAQCEEE7jcLVmyRNSrV09YWFiI1q1bi19++UX7XocOHURISIhO/a1bt4pGjRoJCwsL0aRJE7F3716Dzsc+8GfEnj17YGVlhfv370Oj0aBfv36YOXOm9v1mzZrp9Hv/9ttvSE5ORs2aNXWOc+/ePZw/fx5ZWVm4ceMGAgICtO9Vq1YNrVq1KtaNUiQxMRFmZmbo0KGD3nEnJyfjzp076NKli055QUEBWrRoAQA4c+aMThwAjHpC0ZYtW7B48WKcP38eubm5ePDgQbE5wPXq1UOdOnV0zqPRaJCUlISaNWvi/PnzGDp0KIYPH66t8+DBA9jY2BgcD0lPWFgYwsLCSnwvPj6+WFmvXr3Qq1cvo8/HBP6M6NSpE5YvXw4LCwu4uroWm31So0YNnde5ubnw9/fHxo0bix3L0dHRqBgsLS0N3ic3NxcAsHfvXp3ECZTvjICEhAT0798fs2bNQnBwMGxsbLB582Z88sknBse6cuXKYr9QzMzMyi1WoiJM4M+IGjVqwNPTU+/6LVu2xJYtW+Dk5FTqnWi1a9fG0aNH0b59ewAPW5onTpxAy5YtS6zfrFkzaDQa/PjjjwgKCir2ftE3gMLCQm2Zj48PVCoVLl++XGrLvXHjxtoB2SK//PLL0z/kI44cOQJ3d3dMnTpVW3bp0qVi9S5fvozr16/D1dVVex6lUgkvLy84OzvD1dUVFy5cQP/+/Q06P5ExOAuFStS/f384ODige/fuOHz4MFJSUhAfH4/Ro0fj6tWrAIAxY8Zg/vz52LlzJ86ePYt33333iTfOeHh4ICQkBEOGDMHOnTu1x9y6dSsAwN3dHQqFAnv27MGtW7eQm5uLmjVrYvz48Rg3bhzWrl2L8+fP4+TJk1iyZIl2bvuIESPw999/Y8KECUhKSsKmTZsMfuTYc889h8uXL2Pz5s04f/48Fi9ejB07dhSrp1arERISgt9++w2HDx/G6NGj0bt3b+3Ur1mzZiEyMhKLFy/GuXPncPr0aaxZswaLFi0yKB4ivZRDvz1VcSXNQtHn/Rs3bohBgwYJBwcHoVKpRIMGDcTw4cNFVlaWEOLhoOWYMWOEtbW1sLW1FeHh4WLQoEFPnIVy9+5dMW7cOFG7dm1hYWEhPD09xerVq7Xvz549W7i4uAiFQqEd8NFoNCIqKkp4eXkJc3Nz4ejoKIKDg8WPP/6o3W/37t3C09NTqFQq0a5dO7F69WqDBzEnTJggatWqJaysrESfPn3Ep59+KmxsbLTvR0RECF9fX7Fs2TLh6uoq1Gq16Nmzp8jIyNA57saNG4Wfn5+wsLAQdnZ2on379uLrr78WQnAQk8oXF7MiIpIodqEQEUkUEzgRkUQxgRMRSRQTOBGRRDGBExFJFBM4EZFEMYETEUkUEzgRkUQxgRMRSRQTOBGRRDGBExFJ1P8D6qH0Ww2slHQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 400x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# GaussianProcessClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = GaussianProcessClassifier(random_state = seed, n_jobs=nJobs))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('GaussianProcessClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned DecisionTreeClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = DecisionTreeClassifier(**DecisionTreeClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('DecisionTreeClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned GaussianNB For Validation\n",
    "\n",
    "pipeline.set_params(Model = GaussianNB(**GaussianNBTuned.best_params))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('GaussianNB')\n",
    "y_pred = pipeline.predict(X_validation) # .predict(X)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned QuadraticDiscriminantAnalysis For Validation\n",
    "\n",
    "pipeline.set_params(Model = QuadraticDiscriminantAnalysis(**QuadraticDiscriminantAnalysisTuned.best_params))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('QuadraticDiscriminantAnalysis')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned LinearSVC For Validation\n",
    "\n",
    "pipeline.set_params(Model = LinearSVC(**LinearSVCTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('LinearSVC')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned RidgeClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = RidgeClassifier(**RidgeClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('RidgeClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned SGDClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = SGDClassifier(**SGDClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('SGDClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned XGBClassifier For Validation\n",
    "\n",
    "pipeline.set_params(Model = XGBClassifier(**XGBClassifierTuned.best_params,random_state = seed))\n",
    "pipeline.fit(X_train, y_train)\n",
    "classifierListVal.append('XGBClassifier')\n",
    "y_pred = pipeline.predict(X_validation)\n",
    "y_prob = pipeline.predict_proba(X_validation)\n",
    "y_pred\n",
    "print(y_pred)\n",
    "y_predListVal.append(y_pred)\n",
    "y_validationListVal.append(y_validation)\n",
    "y_probListVal.append(y_prob)\n",
    "seedListStatsVal.append(seed)\n",
    "trainFracListStatsVal.append(trainFrac)\n",
    "\n",
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "print(\"MCC\", sklearn.metrics.matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "balancedAccuracyListVal.append(sklearn.metrics.balanced_accuracy_score(y_validation, y_pred))\n",
    "precisionListVal.append(sklearn.metrics.precision_score(y_validation, y_pred))\n",
    "recallListVal.append(sklearn.metrics.recall_score(y_validation, y_pred))\n",
    "ROCAUCScoreListVal.append(sklearn.metrics.roc_auc_score(y_validation, y_pred))\n",
    "F1ScoreListVal.append(sklearn.metrics.f1_score(y_validation, y_pred))\n",
    "MCCListVal.append(matthews_corrcoef(y_validation, y_pred))\n",
    "\n",
    "s = pipeline.score(X_validation, y_validation)\n",
    "print(f\"Validation score {s}\")\n",
    "\n",
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_validation, y_pred))\n",
    "\n",
    "skplt.metrics.plot_confusion_matrix(y_validation, y_pred, normalize=True,figsize=(4,4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make final pipeline for each seed and 'test' it on the testing dataset\n",
    "* XGBClassifier for sarcoma sensitivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final pipeline\n",
    "\n",
    "pipeline.set_params(Model = XGBClassifier(**XGBClassifierTuned.best_params, n_jobs=nJobs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit the pipeline\n",
    "\n",
    "pipeline.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 902,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "# load test data from saved\n",
    "dfTestFromDisk = pd.read_excel('dfTest_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.xlsx')\n",
    "df = dfTestFromDisk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 903,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select features based on dfTrain's RFE\n",
    "\n",
    "fileName = '_selectedFeaturesRFECV_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed)\n",
    "\n",
    "with open(fileName + '.json', 'r') as f:\n",
    "    selected_featuresFromDisk = json.load(f)\n",
    "\n",
    "colsToKeep = []\n",
    "colsToKeep = copy.deepcopy(selected_featuresFromDisk)\n",
    "colsToKeep.append(target)\n",
    "\n",
    "df = df[colsToKeep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 905,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "X shape: (10, 86)\n",
      "\n",
      "\n",
      "10 Samples \n",
      "\n",
      "\n",
      "86 Attributes \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HNRNPR</th>\n",
       "      <th>HPR</th>\n",
       "      <th>HPSE</th>\n",
       "      <th>HPX</th>\n",
       "      <th>HRG</th>\n",
       "      <th>HRNR</th>\n",
       "      <th>HSDL2</th>\n",
       "      <th>HSP90AA1</th>\n",
       "      <th>HSP90AB1</th>\n",
       "      <th>HSP90B1</th>\n",
       "      <th>HSPA1A</th>\n",
       "      <th>HSPA5</th>\n",
       "      <th>HSPA6</th>\n",
       "      <th>HSPA8</th>\n",
       "      <th>HTT</th>\n",
       "      <th>HYOU1</th>\n",
       "      <th>IBSP</th>\n",
       "      <th>IDH2</th>\n",
       "      <th>IGFALS</th>\n",
       "      <th>IGHG3</th>\n",
       "      <th>IGHG4</th>\n",
       "      <th>IGHV1-18</th>\n",
       "      <th>IGHV3-13</th>\n",
       "      <th>IGHV3-15</th>\n",
       "      <th>IGHV3-20</th>\n",
       "      <th>IGHV3-35</th>\n",
       "      <th>IGHV3-49</th>\n",
       "      <th>IGHV3-72</th>\n",
       "      <th>IGHV4-28</th>\n",
       "      <th>IGHV5-51</th>\n",
       "      <th>IGKC</th>\n",
       "      <th>IGKV1-16</th>\n",
       "      <th>IGKV1-17</th>\n",
       "      <th>IGKV1-33</th>\n",
       "      <th>IGKV1-5</th>\n",
       "      <th>IGKV2-24</th>\n",
       "      <th>IGKV3-20</th>\n",
       "      <th>IGKV4-1</th>\n",
       "      <th>INHBE</th>\n",
       "      <th>IQGAP2</th>\n",
       "      <th>ISOC1</th>\n",
       "      <th>ITM2B</th>\n",
       "      <th>ITPR1</th>\n",
       "      <th>JCHAIN</th>\n",
       "      <th>KGD4</th>\n",
       "      <th>MBP</th>\n",
       "      <th>MICAL1</th>\n",
       "      <th>MPP1</th>\n",
       "      <th>NFKB1</th>\n",
       "      <th>PKN1</th>\n",
       "      <th>PLXDC2</th>\n",
       "      <th>RASA1</th>\n",
       "      <th>RFTN1</th>\n",
       "      <th>RIPOR3</th>\n",
       "      <th>RNF121</th>\n",
       "      <th>ROCK2</th>\n",
       "      <th>RPLP1</th>\n",
       "      <th>RPLP2</th>\n",
       "      <th>RSU1</th>\n",
       "      <th>S100A4</th>\n",
       "      <th>SAA1</th>\n",
       "      <th>SAA2</th>\n",
       "      <th>SEPTIN11</th>\n",
       "      <th>SEPTIN2</th>\n",
       "      <th>SEPTIN6</th>\n",
       "      <th>SEPTIN7</th>\n",
       "      <th>SERPINA4</th>\n",
       "      <th>SERPINA6</th>\n",
       "      <th>SERPINB6</th>\n",
       "      <th>SERPINE1</th>\n",
       "      <th>SH3BGRL3</th>\n",
       "      <th>SH3GL1</th>\n",
       "      <th>SLC25A5</th>\n",
       "      <th>SLC2A1</th>\n",
       "      <th>SPP2</th>\n",
       "      <th>SPTA1</th>\n",
       "      <th>ST13</th>\n",
       "      <th>STAT5B</th>\n",
       "      <th>SYNRG</th>\n",
       "      <th>TFRC</th>\n",
       "      <th>TGFB1I1</th>\n",
       "      <th>THBS4</th>\n",
       "      <th>TIMP1</th>\n",
       "      <th>TIMP2</th>\n",
       "      <th>TKT</th>\n",
       "      <th>TLN2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1102700</td>\n",
       "      <td>17927800</td>\n",
       "      <td>537610.0</td>\n",
       "      <td>54012000</td>\n",
       "      <td>7777300</td>\n",
       "      <td>1265511</td>\n",
       "      <td>431740</td>\n",
       "      <td>1132990.0</td>\n",
       "      <td>928250</td>\n",
       "      <td>1477050</td>\n",
       "      <td>485188.0</td>\n",
       "      <td>1062450</td>\n",
       "      <td>261650.0</td>\n",
       "      <td>5198000</td>\n",
       "      <td>1229340.0</td>\n",
       "      <td>376202</td>\n",
       "      <td>783640.0</td>\n",
       "      <td>812550.0</td>\n",
       "      <td>1502570</td>\n",
       "      <td>13091000</td>\n",
       "      <td>11408900</td>\n",
       "      <td>361290.0</td>\n",
       "      <td>299110.0</td>\n",
       "      <td>174780.0</td>\n",
       "      <td>444130.0</td>\n",
       "      <td>190240</td>\n",
       "      <td>178790</td>\n",
       "      <td>273948.0</td>\n",
       "      <td>147400.0</td>\n",
       "      <td>1497460</td>\n",
       "      <td>159608000</td>\n",
       "      <td>240710.0</td>\n",
       "      <td>227000</td>\n",
       "      <td>1033300</td>\n",
       "      <td>821700</td>\n",
       "      <td>76354.0</td>\n",
       "      <td>1228500</td>\n",
       "      <td>4266690</td>\n",
       "      <td>275284.0</td>\n",
       "      <td>913410.0</td>\n",
       "      <td>365850</td>\n",
       "      <td>126920.0</td>\n",
       "      <td>39243</td>\n",
       "      <td>4910150</td>\n",
       "      <td>1662100.0</td>\n",
       "      <td>4626600</td>\n",
       "      <td>234651.0</td>\n",
       "      <td>811500</td>\n",
       "      <td>60947.0</td>\n",
       "      <td>334299</td>\n",
       "      <td>268096</td>\n",
       "      <td>2254700</td>\n",
       "      <td>224780</td>\n",
       "      <td>654590.0</td>\n",
       "      <td>8461900</td>\n",
       "      <td>587076</td>\n",
       "      <td>839326</td>\n",
       "      <td>1263300</td>\n",
       "      <td>5890230</td>\n",
       "      <td>219873.0</td>\n",
       "      <td>6565400</td>\n",
       "      <td>1545080.0</td>\n",
       "      <td>179552.0</td>\n",
       "      <td>604690.0</td>\n",
       "      <td>385727.0</td>\n",
       "      <td>709050</td>\n",
       "      <td>2121130</td>\n",
       "      <td>808600</td>\n",
       "      <td>396801</td>\n",
       "      <td>178146</td>\n",
       "      <td>633422</td>\n",
       "      <td>736950.0</td>\n",
       "      <td>166882.0</td>\n",
       "      <td>379196</td>\n",
       "      <td>48859000</td>\n",
       "      <td>783610</td>\n",
       "      <td>934150.0</td>\n",
       "      <td>263813.0</td>\n",
       "      <td>1359600.0</td>\n",
       "      <td>21818</td>\n",
       "      <td>5226000</td>\n",
       "      <td>1152980</td>\n",
       "      <td>50325.0</td>\n",
       "      <td>371920</td>\n",
       "      <td>499931.0</td>\n",
       "      <td>485050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1633140</td>\n",
       "      <td>12903200</td>\n",
       "      <td>735870.0</td>\n",
       "      <td>47108000</td>\n",
       "      <td>9058400</td>\n",
       "      <td>573660</td>\n",
       "      <td>1974050</td>\n",
       "      <td>468820.0</td>\n",
       "      <td>668330</td>\n",
       "      <td>1870680</td>\n",
       "      <td>113983.0</td>\n",
       "      <td>781430</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2942000</td>\n",
       "      <td>797900.0</td>\n",
       "      <td>412870</td>\n",
       "      <td>894180.0</td>\n",
       "      <td>706680.0</td>\n",
       "      <td>1560620</td>\n",
       "      <td>21107700</td>\n",
       "      <td>2486156</td>\n",
       "      <td>152680.0</td>\n",
       "      <td>89821.0</td>\n",
       "      <td>175450.0</td>\n",
       "      <td>2031300.0</td>\n",
       "      <td>618540</td>\n",
       "      <td>459634</td>\n",
       "      <td>NaN</td>\n",
       "      <td>68498.0</td>\n",
       "      <td>1305260</td>\n",
       "      <td>161096000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>111430</td>\n",
       "      <td>585736</td>\n",
       "      <td>2073600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2552612</td>\n",
       "      <td>1877860</td>\n",
       "      <td>97405.0</td>\n",
       "      <td>216232.0</td>\n",
       "      <td>2784690</td>\n",
       "      <td>242630.0</td>\n",
       "      <td>198980</td>\n",
       "      <td>7438100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4039500</td>\n",
       "      <td>163494.0</td>\n",
       "      <td>1268740</td>\n",
       "      <td>210417.0</td>\n",
       "      <td>324360</td>\n",
       "      <td>733130</td>\n",
       "      <td>620378</td>\n",
       "      <td>500110</td>\n",
       "      <td>1062443.0</td>\n",
       "      <td>365850</td>\n",
       "      <td>6465562</td>\n",
       "      <td>1290537</td>\n",
       "      <td>3497880</td>\n",
       "      <td>3129840</td>\n",
       "      <td>142820.0</td>\n",
       "      <td>4777780</td>\n",
       "      <td>556940.0</td>\n",
       "      <td>32367.0</td>\n",
       "      <td>320155.0</td>\n",
       "      <td>269704.0</td>\n",
       "      <td>491030</td>\n",
       "      <td>1643870</td>\n",
       "      <td>462620</td>\n",
       "      <td>543800</td>\n",
       "      <td>374086</td>\n",
       "      <td>150320</td>\n",
       "      <td>1301670.0</td>\n",
       "      <td>74456.0</td>\n",
       "      <td>984440</td>\n",
       "      <td>41544000</td>\n",
       "      <td>2046030</td>\n",
       "      <td>1508180.0</td>\n",
       "      <td>1393400.0</td>\n",
       "      <td>7313800.0</td>\n",
       "      <td>155024</td>\n",
       "      <td>8602300</td>\n",
       "      <td>2365660</td>\n",
       "      <td>123506.0</td>\n",
       "      <td>474180</td>\n",
       "      <td>1645800.0</td>\n",
       "      <td>827290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1188300</td>\n",
       "      <td>18561900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52207000</td>\n",
       "      <td>16656200</td>\n",
       "      <td>1802410</td>\n",
       "      <td>1699000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>152619</td>\n",
       "      <td>1993280</td>\n",
       "      <td>NaN</td>\n",
       "      <td>487210</td>\n",
       "      <td>83980.0</td>\n",
       "      <td>1025060</td>\n",
       "      <td>NaN</td>\n",
       "      <td>105630</td>\n",
       "      <td>1560100.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1206290</td>\n",
       "      <td>23948000</td>\n",
       "      <td>16008200</td>\n",
       "      <td>404380.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>39840.0</td>\n",
       "      <td>1429000.0</td>\n",
       "      <td>321340</td>\n",
       "      <td>277760</td>\n",
       "      <td>NaN</td>\n",
       "      <td>233840.0</td>\n",
       "      <td>1643710</td>\n",
       "      <td>369524000</td>\n",
       "      <td>23982.0</td>\n",
       "      <td>87229</td>\n",
       "      <td>513670</td>\n",
       "      <td>122090</td>\n",
       "      <td>NaN</td>\n",
       "      <td>385210</td>\n",
       "      <td>2677390</td>\n",
       "      <td>158127.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1808040</td>\n",
       "      <td>NaN</td>\n",
       "      <td>411670</td>\n",
       "      <td>5055040</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3430200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>75882</td>\n",
       "      <td>NaN</td>\n",
       "      <td>81312</td>\n",
       "      <td>177240</td>\n",
       "      <td>1315631</td>\n",
       "      <td>951420</td>\n",
       "      <td>2687800.0</td>\n",
       "      <td>662900</td>\n",
       "      <td>1000580</td>\n",
       "      <td>636390</td>\n",
       "      <td>111490</td>\n",
       "      <td>1293715</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3097900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>750480.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>92038</td>\n",
       "      <td>2641210</td>\n",
       "      <td>572143</td>\n",
       "      <td>459050</td>\n",
       "      <td>944540</td>\n",
       "      <td>338730</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1905489</td>\n",
       "      <td>73584000</td>\n",
       "      <td>2464790</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12719000.0</td>\n",
       "      <td>52436</td>\n",
       "      <td>1797110</td>\n",
       "      <td>10896700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>558230</td>\n",
       "      <td>1341600.0</td>\n",
       "      <td>263630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>367960</td>\n",
       "      <td>14700200</td>\n",
       "      <td>2156090.0</td>\n",
       "      <td>40419000</td>\n",
       "      <td>9278300</td>\n",
       "      <td>1553340</td>\n",
       "      <td>1042580</td>\n",
       "      <td>944190.0</td>\n",
       "      <td>792880</td>\n",
       "      <td>5798000</td>\n",
       "      <td>445284.0</td>\n",
       "      <td>5668300</td>\n",
       "      <td>1142800.0</td>\n",
       "      <td>9961200</td>\n",
       "      <td>156190.0</td>\n",
       "      <td>1095300</td>\n",
       "      <td>395590.0</td>\n",
       "      <td>5434000.0</td>\n",
       "      <td>723610</td>\n",
       "      <td>37083000</td>\n",
       "      <td>27915800</td>\n",
       "      <td>373610.0</td>\n",
       "      <td>85671.0</td>\n",
       "      <td>212770.0</td>\n",
       "      <td>1060400.0</td>\n",
       "      <td>533420</td>\n",
       "      <td>682414</td>\n",
       "      <td>361738.0</td>\n",
       "      <td>337170.0</td>\n",
       "      <td>2406440</td>\n",
       "      <td>328546000</td>\n",
       "      <td>191042.0</td>\n",
       "      <td>199030</td>\n",
       "      <td>3253200</td>\n",
       "      <td>2535560</td>\n",
       "      <td>705860.0</td>\n",
       "      <td>1985600</td>\n",
       "      <td>4611010</td>\n",
       "      <td>135039.0</td>\n",
       "      <td>2180340.0</td>\n",
       "      <td>1235440</td>\n",
       "      <td>53129.0</td>\n",
       "      <td>446253</td>\n",
       "      <td>1724690</td>\n",
       "      <td>1447100.0</td>\n",
       "      <td>1811300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1364250</td>\n",
       "      <td>47583.0</td>\n",
       "      <td>80896</td>\n",
       "      <td>334361</td>\n",
       "      <td>845264</td>\n",
       "      <td>266310</td>\n",
       "      <td>93027.0</td>\n",
       "      <td>2032200</td>\n",
       "      <td>783380</td>\n",
       "      <td>522470</td>\n",
       "      <td>551170</td>\n",
       "      <td>4587300</td>\n",
       "      <td>63482.0</td>\n",
       "      <td>16616170</td>\n",
       "      <td>2687220.0</td>\n",
       "      <td>506140.0</td>\n",
       "      <td>1424290.0</td>\n",
       "      <td>1878090.0</td>\n",
       "      <td>1432530</td>\n",
       "      <td>692350</td>\n",
       "      <td>599390</td>\n",
       "      <td>411130</td>\n",
       "      <td>430050</td>\n",
       "      <td>1236663</td>\n",
       "      <td>768240.0</td>\n",
       "      <td>2450260.0</td>\n",
       "      <td>1156810</td>\n",
       "      <td>61286000</td>\n",
       "      <td>1653030</td>\n",
       "      <td>556780.0</td>\n",
       "      <td>97915.0</td>\n",
       "      <td>3072500.0</td>\n",
       "      <td>178004</td>\n",
       "      <td>3316470</td>\n",
       "      <td>1093660</td>\n",
       "      <td>585945.0</td>\n",
       "      <td>118918</td>\n",
       "      <td>NaN</td>\n",
       "      <td>258140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1602380</td>\n",
       "      <td>8187360</td>\n",
       "      <td>696380.0</td>\n",
       "      <td>70799000</td>\n",
       "      <td>8538700</td>\n",
       "      <td>876897</td>\n",
       "      <td>2342830</td>\n",
       "      <td>1692750.0</td>\n",
       "      <td>896370</td>\n",
       "      <td>2323850</td>\n",
       "      <td>759830.0</td>\n",
       "      <td>2547430</td>\n",
       "      <td>239070.0</td>\n",
       "      <td>7722100</td>\n",
       "      <td>146060.0</td>\n",
       "      <td>233964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2095110.0</td>\n",
       "      <td>1399030</td>\n",
       "      <td>30172900</td>\n",
       "      <td>20998300</td>\n",
       "      <td>384770.0</td>\n",
       "      <td>389760.0</td>\n",
       "      <td>628430.0</td>\n",
       "      <td>15963.0</td>\n",
       "      <td>922310</td>\n",
       "      <td>334031</td>\n",
       "      <td>540590.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2948610</td>\n",
       "      <td>241053000</td>\n",
       "      <td>55297.0</td>\n",
       "      <td>314290</td>\n",
       "      <td>1063000</td>\n",
       "      <td>871120</td>\n",
       "      <td>3658900.0</td>\n",
       "      <td>2064619</td>\n",
       "      <td>5381438</td>\n",
       "      <td>1319950.0</td>\n",
       "      <td>1629920.0</td>\n",
       "      <td>717340</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100080</td>\n",
       "      <td>4806700</td>\n",
       "      <td>479700.0</td>\n",
       "      <td>4429300</td>\n",
       "      <td>245584.0</td>\n",
       "      <td>1338250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49069</td>\n",
       "      <td>35739</td>\n",
       "      <td>2853450</td>\n",
       "      <td>33882</td>\n",
       "      <td>126425.0</td>\n",
       "      <td>4913800</td>\n",
       "      <td>994710</td>\n",
       "      <td>427800</td>\n",
       "      <td>760740</td>\n",
       "      <td>7260710</td>\n",
       "      <td>262003.0</td>\n",
       "      <td>15979890</td>\n",
       "      <td>2415130.0</td>\n",
       "      <td>342695.0</td>\n",
       "      <td>470450.0</td>\n",
       "      <td>1057120.0</td>\n",
       "      <td>890620</td>\n",
       "      <td>1222670</td>\n",
       "      <td>1914640</td>\n",
       "      <td>428710</td>\n",
       "      <td>653880</td>\n",
       "      <td>784300</td>\n",
       "      <td>1081370.0</td>\n",
       "      <td>762440.0</td>\n",
       "      <td>559656</td>\n",
       "      <td>30311600</td>\n",
       "      <td>1843740</td>\n",
       "      <td>592830.0</td>\n",
       "      <td>147636.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55840</td>\n",
       "      <td>3949800</td>\n",
       "      <td>404920</td>\n",
       "      <td>208071.0</td>\n",
       "      <td>172606</td>\n",
       "      <td>121550.0</td>\n",
       "      <td>131240</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    HNRNPR       HPR       HPSE       HPX       HRG     HRNR    HSDL2  \\\n",
       "0  1102700  17927800   537610.0  54012000   7777300  1265511   431740   \n",
       "1  1633140  12903200   735870.0  47108000   9058400   573660  1974050   \n",
       "2  1188300  18561900        NaN  52207000  16656200  1802410  1699000   \n",
       "3   367960  14700200  2156090.0  40419000   9278300  1553340  1042580   \n",
       "4  1602380   8187360   696380.0  70799000   8538700   876897  2342830   \n",
       "\n",
       "    HSP90AA1  HSP90AB1  HSP90B1    HSPA1A    HSPA5      HSPA6    HSPA8  \\\n",
       "0  1132990.0    928250  1477050  485188.0  1062450   261650.0  5198000   \n",
       "1   468820.0    668330  1870680  113983.0   781430        NaN  2942000   \n",
       "2        NaN    152619  1993280       NaN   487210    83980.0  1025060   \n",
       "3   944190.0    792880  5798000  445284.0  5668300  1142800.0  9961200   \n",
       "4  1692750.0    896370  2323850  759830.0  2547430   239070.0  7722100   \n",
       "\n",
       "         HTT    HYOU1       IBSP       IDH2   IGFALS     IGHG3     IGHG4  \\\n",
       "0  1229340.0   376202   783640.0   812550.0  1502570  13091000  11408900   \n",
       "1   797900.0   412870   894180.0   706680.0  1560620  21107700   2486156   \n",
       "2        NaN   105630  1560100.0        NaN  1206290  23948000  16008200   \n",
       "3   156190.0  1095300   395590.0  5434000.0   723610  37083000  27915800   \n",
       "4   146060.0   233964        NaN  2095110.0  1399030  30172900  20998300   \n",
       "\n",
       "   IGHV1-18  IGHV3-13  IGHV3-15   IGHV3-20  IGHV3-35  IGHV3-49  IGHV3-72  \\\n",
       "0  361290.0  299110.0  174780.0   444130.0    190240    178790  273948.0   \n",
       "1  152680.0   89821.0  175450.0  2031300.0    618540    459634       NaN   \n",
       "2  404380.0       NaN   39840.0  1429000.0    321340    277760       NaN   \n",
       "3  373610.0   85671.0  212770.0  1060400.0    533420    682414  361738.0   \n",
       "4  384770.0  389760.0  628430.0    15963.0    922310    334031  540590.0   \n",
       "\n",
       "   IGHV4-28  IGHV5-51       IGKC  IGKV1-16  IGKV1-17  IGKV1-33  IGKV1-5  \\\n",
       "0  147400.0   1497460  159608000  240710.0    227000   1033300   821700   \n",
       "1   68498.0   1305260  161096000       NaN    111430    585736  2073600   \n",
       "2  233840.0   1643710  369524000   23982.0     87229    513670   122090   \n",
       "3  337170.0   2406440  328546000  191042.0    199030   3253200  2535560   \n",
       "4       NaN   2948610  241053000   55297.0    314290   1063000   871120   \n",
       "\n",
       "    IGKV2-24  IGKV3-20  IGKV4-1      INHBE     IQGAP2    ISOC1     ITM2B  \\\n",
       "0    76354.0   1228500  4266690   275284.0   913410.0   365850  126920.0   \n",
       "1        NaN   2552612  1877860    97405.0   216232.0  2784690  242630.0   \n",
       "2        NaN    385210  2677390   158127.0        NaN  1808040       NaN   \n",
       "3   705860.0   1985600  4611010   135039.0  2180340.0  1235440   53129.0   \n",
       "4  3658900.0   2064619  5381438  1319950.0  1629920.0   717340       NaN   \n",
       "\n",
       "    ITPR1   JCHAIN       KGD4      MBP    MICAL1     MPP1     NFKB1    PKN1  \\\n",
       "0   39243  4910150  1662100.0  4626600  234651.0   811500   60947.0  334299   \n",
       "1  198980  7438100        NaN  4039500  163494.0  1268740  210417.0  324360   \n",
       "2  411670  5055040        NaN  3430200       NaN    75882       NaN   81312   \n",
       "3  446253  1724690  1447100.0  1811300       NaN  1364250   47583.0   80896   \n",
       "4  100080  4806700   479700.0  4429300  245584.0  1338250       NaN   49069   \n",
       "\n",
       "   PLXDC2    RASA1   RFTN1     RIPOR3   RNF121    ROCK2    RPLP1    RPLP2  \\\n",
       "0  268096  2254700  224780   654590.0  8461900   587076   839326  1263300   \n",
       "1  733130   620378  500110  1062443.0   365850  6465562  1290537  3497880   \n",
       "2  177240  1315631  951420  2687800.0   662900  1000580   636390   111490   \n",
       "3  334361   845264  266310    93027.0  2032200   783380   522470   551170   \n",
       "4   35739  2853450   33882   126425.0  4913800   994710   427800   760740   \n",
       "\n",
       "      RSU1    S100A4      SAA1       SAA2  SEPTIN11    SEPTIN2    SEPTIN6  \\\n",
       "0  5890230  219873.0   6565400  1545080.0  179552.0   604690.0   385727.0   \n",
       "1  3129840  142820.0   4777780   556940.0   32367.0   320155.0   269704.0   \n",
       "2  1293715       NaN   3097900        NaN  750480.0        NaN        NaN   \n",
       "3  4587300   63482.0  16616170  2687220.0  506140.0  1424290.0  1878090.0   \n",
       "4  7260710  262003.0  15979890  2415130.0  342695.0   470450.0  1057120.0   \n",
       "\n",
       "   SEPTIN7  SERPINA4  SERPINA6  SERPINB6  SERPINE1  SH3BGRL3     SH3GL1  \\\n",
       "0   709050   2121130    808600    396801    178146    633422   736950.0   \n",
       "1   491030   1643870    462620    543800    374086    150320  1301670.0   \n",
       "2    92038   2641210    572143    459050    944540    338730        NaN   \n",
       "3  1432530    692350    599390    411130    430050   1236663   768240.0   \n",
       "4   890620   1222670   1914640    428710    653880    784300  1081370.0   \n",
       "\n",
       "     SLC25A5   SLC2A1      SPP2    SPTA1       ST13     STAT5B       SYNRG  \\\n",
       "0   166882.0   379196  48859000   783610   934150.0   263813.0   1359600.0   \n",
       "1    74456.0   984440  41544000  2046030  1508180.0  1393400.0   7313800.0   \n",
       "2        NaN  1905489  73584000  2464790        NaN        NaN  12719000.0   \n",
       "3  2450260.0  1156810  61286000  1653030   556780.0    97915.0   3072500.0   \n",
       "4   762440.0   559656  30311600  1843740   592830.0   147636.0         NaN   \n",
       "\n",
       "     TFRC  TGFB1I1     THBS4     TIMP1   TIMP2        TKT    TLN2  \n",
       "0   21818  5226000   1152980   50325.0  371920   499931.0  485050  \n",
       "1  155024  8602300   2365660  123506.0  474180  1645800.0  827290  \n",
       "2   52436  1797110  10896700       NaN  558230  1341600.0  263630  \n",
       "3  178004  3316470   1093660  585945.0  118918        NaN  258140  \n",
       "4   55840  3949800    404920  208071.0  172606   121550.0  131240  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "y shape: (10,)\n",
      "\n",
      "\n",
      "10 Samples \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    S\n",
       "1    S\n",
       "2    S\n",
       "3    S\n",
       "4    R\n",
       "Name: sensitivity, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# split into X and Y\n",
    "\n",
    "X_test, y_test = X_y_split(df, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 906,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    1\n",
       "3    1\n",
       "4    0\n",
       "5    0\n",
       "6    0\n",
       "7    1\n",
       "8    0\n",
       "9    1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 906,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# label encode y, as above\n",
    "y_test = pd.Series(le.fit_transform(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 908,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "quantType is Intensity_Raw\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 908,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# impute 0s, as above\n",
    "\n",
    "imputeWideDFMinOr0(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 910,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 910,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# impute any new proteins in X_test with the min value of the dataset, as above\n",
    "genesWithNANs = X_test.columns[X_test.isna().any()].tolist()\n",
    "\n",
    "X_test[genesWithNANs] = min(X_test.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 916,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 0, 1, 1, 1, 0, 1, 1, 1])"
      ]
     },
     "execution_count": 916,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 917,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.4860709e-01, 5.5139291e-01],\n",
       "       [2.2952062e-01, 7.7047938e-01],\n",
       "       [8.3521491e-01, 1.6478509e-01],\n",
       "       [2.3738742e-03, 9.9762613e-01],\n",
       "       [7.8916550e-05, 9.9992108e-01],\n",
       "       [3.0276352e-01, 6.9723648e-01],\n",
       "       [8.6963093e-01, 1.3036905e-01],\n",
       "       [2.8118110e-01, 7.1881890e-01],\n",
       "       [4.2102933e-03, 9.9578971e-01],\n",
       "       [2.6153922e-03, 9.9738461e-01]], dtype=float32)"
      ]
     },
     "execution_count": 917,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = pipeline.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 918,
   "metadata": {},
   "outputs": [],
   "source": [
    "# re-format this output\n",
    "y_prob = [i[1] for i in y_prob]\n",
    "\n",
    "# append to lists\n",
    "y_predList.append(y_pred)\n",
    "y_testList.append(y_test)\n",
    "y_probList.append(y_prob)\n",
    "seedListStats.append(seed)\n",
    "trainFracListStats.append(trainFrac)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 921,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Balanced Accuracy 0.5416666666666667\n",
      "Precision 0.625\n",
      "Recall 0.8333333333333334\n",
      "ROC AUC score 0.5416666666666667\n",
      "F1 score 0.7142857142857143\n",
      "Test score 0.6\n"
     ]
    }
   ],
   "source": [
    "print(\"Balanced Accuracy\", sklearn.metrics.balanced_accuracy_score(y_test, y_pred))\n",
    "print(\"Precision\", sklearn.metrics.precision_score(y_test, y_pred))\n",
    "print(\"Recall\", sklearn.metrics.recall_score(y_test, y_pred))\n",
    "print(\"ROC AUC score\", sklearn.metrics.roc_auc_score(y_test, y_pred))\n",
    "print(\"F1 score\", sklearn.metrics.f1_score(y_test, y_pred))\n",
    "\n",
    "s = pipeline.score(X_test, y_test)\n",
    "print(f\"Test score {s}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 922,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix\n",
      " [[1 3]\n",
      " [1 5]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Confusion matrix\\n\", sklearn.metrics.confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot normalized confusion matrix from the current seed\n",
    "skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True,figsize=(2,2))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 923,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "from collections.abc import Iterable\n",
    "\n",
    "def flatten(lis):\n",
    "     for item in lis:\n",
    "         if isinstance(item, Iterable) and not isinstance(item, str):\n",
    "             for x in flatten(item):\n",
    "                 yield x\n",
    "         else:        \n",
    "             yield item\n",
    "             \n",
    "cm = confusion_matrix(list(flatten(y_testList)), list(flatten(y_predList))) # for many trials\n",
    "\n",
    "# for all models\n",
    "from collections.abc import Iterable\n",
    "\n",
    "cm = confusion_matrix(list(flatten(y_testList)), list(flatten(y_predList))) # for all models\n",
    "\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=le.classes_)\n",
    "disp.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 924,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAADRCAYAAADIUNdbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2KklEQVR4nO2deVhTRxeHfwmQhDWA7IiAUEFE0aJQq6JUFK11+dxXAlaqVtxQ6y4iKlo3xA1FkaW24l63ohbFXSlutYpYEBSRRYuAgLJlvj+QW0ICBAjEwrw+8zzec+fOnAk3J3POmTuXRQghoFAoLR62vBWgUCifBtQYUCgUANQYUCiUj1BjQKFQAFBjQKFQPkKNAYVCAUCNAYVC+Qg1BhQKBQA1BhQK5SPUGFSiT58+6NOnD3OckpICFouF0NDQJtXD3d0dZmZmTdpnfYmIiIC1tTWUlJSgqakp8/ZXrlwJFosl83b/qzTmPVknYxAaGgoWiwUej4e0tDSx83369IGtra3MlKNIx/HjxzFw4EDo6OiAw+HAyMgIo0ePxsWLFxu13ydPnsDd3R0WFhYIDg7Gnj17GrW/pobFYoHFYmHKlCkSzy9dupSp8+bNmzq3f/bsWaxcubKBWsoQUgf2799PABAAxMvLS+x87969SYcOHerS5CdF7969Se/evZljoVBI3r9/T0pLS5tUD4FAQExNTWutJxQKibu7OwFAunTpQtasWUP27dtHVq9eTezt7QkAcv369UbTc9euXQQA+fvvvxutj5KSEvL+/ftGa78mABAej0c0NTVJUVGR2Hlzc3PC4/EIAPL69es6tz9jxgxSx69go96T9XITOnfujODgYLx69UpWNkkMQgjev3/faO1LQ8UsSEFBQa56VMemTZsQGhqKOXPm4M6dO1iyZAkmT56MpUuXIi4uDuHh4VBUVGy0/rOysgCgUdyDChQVFcHj8Rqt/doYMGAA8vLy8Ntvv4nIb9y4geTkZAwaNKhJ9CgtLUVxcXGj3pP1MgZLlixBWVkZ1q1bV2vd0tJS+Pn5wcLCAlwuF2ZmZliyZAmKiopE6pmZmeGbb77BuXPn0LVrVygrK2P37t2IiYkBi8XCoUOH4OvrC2NjY6irq2PkyJHIzc1FUVER5syZAz09PaipqcHDw0Os7f379+Orr76Cnp4euFwubGxssGvXrlp1r+qfVegiqVT18X/77Tf06tULqqqqUFdXx6BBg/Do0SOxPk6cOAFbW1vweDzY2tri+PHjteoFAO/fv4e/vz+sra2xceNGiX71pEmT4ODgwBw/e/YMo0aNgra2NlRUVPDFF1/gzJkzItdU/rzXrFmD1q1bg8fjoW/fvkhMTGTqmZmZwcfHBwCgq6sLFovFTHkr/78yZmZmcHd3Z45LSkrg6+uLzz77DDweD61atULPnj1x4cIFpo6kmEFd76lr167BwcEBPB4Pbdu2RXh4eM0fbiWMjY3h5OSEn3/+WUR+4MABdOzYUaJbfPXqVYwaNQpt2rQBl8uFiYkJ5s6dK/Lj5u7ujh07djCfV0UB/r3vNm7ciICAAGacjx8/Frsns7KyoKuriz59+oBUegA5MTERqqqqGDNmjNRjrdfPhrm5Odzc3BAcHIxFixbByMio2rpTpkxBWFgYRo4ciXnz5uH27dvw9/dHfHy82I2fkJCAcePGYerUqfD09ISVlRVzzt/fH8rKyli0aBESExOxbds2KCkpgc1m4+3bt1i5ciVu3bqF0NBQmJubY8WKFcy1u3btQocOHTBkyBAoKiri1KlT+P777yEUCjFjxgypx92+fXtERESIyHJycuDt7Q09PT1GFhERAYFAAFdXV6xfvx6FhYXYtWsXevbsiXv37jGG4/z58xgxYgRsbGzg7++Pf/75Bx4eHmjdunWtuly7dg3Z2dmYM2eOVL8SmZmZ+PLLL1FYWIhZs2ahVatWCAsLw5AhQ3DkyBH873//E6m/bt06sNlszJ8/H7m5ufjxxx8xYcIE3L59GwAQEBCA8PBwHD9+HLt27YKamho6depUqx6VWblyJfz9/TFlyhQ4ODggLy8PcXFxuHv3Lvr161ftdXW5pxITEzFy5Eh8++23EAgECAkJgbu7O+zt7dGhQwep9Bw/fjxmz56N/Px8qKmpobS0FIcPH4a3tzc+fPggVv/w4cMoLCzE9OnT0apVK8TGxmLbtm14+fIlDh8+DACYOnUqXr16hQsXLojdUxXs378fHz58wHfffQculwttbW0IhUKROnp6eti1axdGjRqFbdu2YdasWRAKhXB3d4e6ujp27twp1RgB1C9m8Mcff5CkpCSiqKhIZs2axZyvGjO4f/8+AUCmTJki0s78+fMJAHLx4kVGZmpqSgCQqKgokbqXLl0iAIitrS0pLi5m5OPGjSMsFosMHDhQpH737t3F/O3CwkKxsbi6upK2bduKyKrGDJKTkwkAsn//fomfh1AoJN988w1RU1Mjjx49IoQQ8u7dO6KpqUk8PT1F6mZkZBA+ny8i79y5MzE0NCQ5OTmM7Pz58wRArTGDrVu3EgDk+PHjNdarYM6cOQQAuXr1KiN79+4dMTc3J2ZmZqSsrIwQ8u/n3b59exE/uaK/hw8fMjIfHx+J/jIA4uPjI6aDqakpEQgEzLGdnR0ZNGhQjXpX9FFBfe6pK1euMLKsrCzC5XLJvHnzauy3YhxTp04lKSkpRElJiezZs4fk5uaSw4cPEwDkyZMnEj8DSfebv78/YbFY5Pnz54ysuphBxX2noaFBsrKyJJ6rek+OGzeOqKiokKdPn5INGzYQAOTEiRO1jrEy9U4ttm3bFpMmTcKePXuQnp4usc7Zs2cBAN7e3iLyefPmAYDYFNXc3Byurq4S23Jzc4OSkhJz7OjoCEIIJk+eLFLP0dERqampKC0tZWTKysrM/3Nzc/HmzRv07t0bz549Q25ubm1DrRY/Pz+cPn0aoaGhsLGxAQBcuHABOTk5GDduHN68ecMUBQUFODo64tKlSwCA9PR03L9/HwKBAHw+n2mzX79+TFs1kZeXBwBQV1eXStezZ8/CwcEBPXv2ZGRqamr47rvvkJKSgsePH4vU9/DwAIfDYY579eoFoNzVkBWampp49OgR/v77b6mvqes9ZWNjw+gOlLs0VlZWUo9j974ImJmZoaSkBN999x34fD5GjRoFoDx7Vvk+q6Dy/VZQUIA3b97gyy+/BCEE9+7dk6pfABgxYgR0dXWlqrt9+3bw+XyMHDkSy5cvx6RJkzB06FCp+wIauM5g2bJlKC0trTZ28Pz5c7DZbFhaWorIDQwMoKmpiefPn4vIzc3Nq+2rTZs2IscVXyATExMxuVAoFPmSX79+HS4uLlBVVYWmpiZ0dXWxZMkSAKi3MYiKioKvry8WL16MESNGMPKKG/urr76Crq6uSDl//jwTdKsY+2effSbWdmX3qDo0NDQAAO/evZNK3+fPn0tst3379iL6VFD189bS0gIAvH37Vqr+pGHVqlXIyclBu3bt0LFjRyxYsAB//vlnjdfU9Z6qOg6gfCxSj6O0EFxbD3DtpooWWw9kZGSgrKxM7JIXL17A3d0d2traUFNTg66uLnr37g2gbvdbTd+HqmhrayMwMBB//vkn+Hw+AgMDpb62ggaFmtu2bYuJEydiz549WLRoUbX1pF00UtmiVqU6v7g6OfkYTElKSkLfvn1hbW2NzZs3w8TEBBwOB2fPnsWWLVvEfDBpSE5OxoQJE9CvXz+sXr1a5FxFexERETAwMBC7VlbRfWtrawDAw4cPMWzYMJm0WZnaPtf6UPWL4+TkhKSkJPz66684f/489u7diy1btiAoKKja3H4F0t5TMhmHkjJYClzR69mS2y0rK0O/fv2QnZ2NhQsXwtraGqqqqkhLS4O7u3ud7reavg+SOHfuHIByg/3y5cs6Z3kafGcuW7YMP/30E9avXy92ztTUFEKhEH///TfzCwSUB7NycnJgamra0O5r5dSpUygqKsLJkydFfiUqput15f379xg+fDg0NTXxyy+/gM0WnVxZWFgAKA/suLi4VNtOxdglTZETEhJq1aNnz57Q0tLCL7/8giVLltQaRDQ1NZXY7pMnT0T0kQVaWlrIyckRkRUXF0t0J7W1teHh4QEPDw/k5+fDyckJK1eurNYYyOWeYiuUl8oQyZ/3w4cP8fTpU4SFhcHNzY2RV2RIoqOjsXLlSmRkZEj1ZQ0ICMCuXbvw4sUL6OjooH///hLrRUVFYe/evfjhhx9w4MABCAQC3L59u04/Pg1ejmxhYYGJEydi9+7dyMjIEDn39ddfAygfUGU2b94MAE2So634klT+JcjNzcX+/fvr1d60adPw9OlTHD9+nJk6V8bV1RUaGhpYu3YtSkpKxM6/fv0aAGBoaIjOnTsjLCxMZOp44cIFMf9dEioqKli4cCHi4+OxcOFCib90P/30E2JjYwGU/y1iY2Nx8+ZN5nxBQQH27NkDMzMzqeIU0mJhYYErV66IyPbs2SM2M/jnn39EjtXU1GBpaSmWIqyMfO4pNsCqUqr56ki63wgh2Lp1KwDg4MGD8PHxwd27d5mZY3Uxk3v37mHRokXw8fFBfHw89u3bh9OnT4vVy8nJYTIya9euxd69e3H37l2sXbu2TqOUyZx16dKliIiIQEJCgki6xs7ODgKBAHv27EFOTg569+6N2NhYhIWFYdiwYXB2dpZF9zXSv39/cDgcDB48GFOnTkV+fj6Cg4Ohp6dXbeCzOs6cOYPw8HCMGDECf/75p4h/q6amhmHDhkFDQwO7du3CpEmT8Pnnn2Ps2LHQ1dXFixcvcObMGfTo0QPbt28HUJ4uHTRoEHr27InJkycjOzsb27ZtQ4cOHZCfn1+rPgsWLMCjR4+wadMmXLp0CSNHjoSBgQEyMjJw4sQJxMbG4saNGwCARYsW4ZdffsHAgQMxa9YsaGtrIywsDMnJyTh69KjYDKchTJkyBdOmTcOIESPQr18/PHjwAOfOnYOOjo5IPRsbG/Tp0wf29vbQ1tZGXFwcjhw5Ai8vr2rblss9JWlmUI2bYG1tDQsLC8yfPx9paWnQ0NDA0aNHmRiFk5MTPDw8AACLFy/G2LFjMWbMGMybNw8KCgoYO3Ys01ZKSgp69OiB8ePHAyhfNzF48GDs27dPpM/Zs2fjn3/+we+//w4FBQUMGDAAU6ZMwerVqzF06FDY2dlJN866pB4qpxarIhAICACx5cglJSXE19eXmJubEyUlJWJiYkIWL15MPnz4IFLP1NRUYpqpItV1+PBhqXSRlOo5efIk6dSpE+HxeMTMzIysX7+ehISEEAAkOTmZqVdbarHycuyqpWoq8NKlS8TV1ZXw+XzC4/GIhYUFcXd3J3FxcSL1jh49Stq3b0+4XC6xsbEhx44dk3o5cgVHjhwh/fv3J9ra2kRRUZEYGhqSMWPGkJiYGJF6SUlJZOTIkURTU5PweDzi4OBATp8+Laa3pM9bUkqrutRiWVkZWbhwIdHR0SEqKirE1dWVJCYmiqUWV69eTRwcHIimpiZRVlYm1tbWZM2aNSIp5KqpRUIafk9V/TtXR8Xfluu4gPB6LBMpXMcFBACZO3cuAUCePXtGcnNzyYcPH8jjx4+Ji4sLUVNTIzo6OsTT05PExcURAGTmzJlM+6WlpcTa2ppwOBzCYrGYcVZ81uPHjyd8Pp/cvn2bEFL+97OwsBD5O/z6668EANm0aZOI7nl5ecTU1JTY2dmJfJ41wfo4aAqFUoW8vDzw+Xxwv/gBLMUqAcTSIhTd+lHsGh8fH4mrL1+9egVjY2PcuHED3bt3Z+Q//PADLl++zCzmqkpgYCDmz58PQghKS0sxbdo0qVbP1ofGW7hOoTQX2AoAu8pXhV2+viA1NZVJ8wIAlytqNBpCTEwM1q5di507d8LR0RGJiYmYPXs2/Pz8sHz5cpn1UwE1BhRKbSgolJfKfMwmaGhoiBiD6tDR0YGCggIyMzNF5JmZmRJT0ACYxUMVmZWOHTuioKAA3333HZYuXSrTOA9ANzehUGqHxZJc6gCHw4G9vT2io6MZmVAoRHR0tIjbUJnCwkKxL7ykbIWsoDMDCqU26pBNqAlvb28IBAJ07doVDg4OCAgIQEFBAZNdcHNzg7GxMfz9/QEAgwcPxubNm9GlSxfGTVi+fDkGDx7cKI8wU2NAodSGjIzBmDFj8Pr1a6xYsQIZGRno3LkzoqKioK+vD6B8GXPlmcCyZcvAYrGwbNkypKWlQVdXF4MHD8aaNWsaNJzqoNkECqUamGxC3zVgKYpusEJKP6Aoeilyc3Olihn8F6Azg0oIhUK8evUK6urqdBPOFgAhBO/evYORkVHNwTiWhJkB69Pc/aohUGNQiVevXok9BUlp/qSmpta8oQybLcFNaH6xd2oMKlGxN8D1B4lQk3KfgOaCw8yD8lahySElH1B8fmHte0IwzyNUkTUzqDGoRIVroKauDnX15uEHSgtLqW6PyzYnanUJZRRA/NShxoBCqQ1qDCgUCgDJi4yaYYCZGgMKpRbYbDZYVQKGhAYQKZSWB4vNAotdZSZQ9bgZQI0BhVILlV9wUkkoH2UaEWoMKJRaoG4ChUIBQN0ECoXykfJkQlU3QT66NCbUGFAotcBmSXAT6ApECqUFIsFNINRNoFBaHpKyCc3xqVZqDCiUWpAUQBQLKDYDqDGgUGqBzWaL73dAU4sUSsuDugkUCgUAdRMoFMpHqJtAoVAAUDeBQqF8hLoJFAoFQPkKRDE3ga5ApFBaHtRNoFAoAKibQKFQPsJms8CmjzBT6kv4viAE79iC11mZaN+hI1b6b4bd590k1j0YEYJjkQfw9MljAICtXRcsWOorUn+BlyeORv4kcp2Tcz+EHjrZeIOoB1MH2mDusE7Q11TGw5RseO+9gbi/X0use85vEJxsjcTkv8W9wPA15wAAe2b2xqSv2omcP383FUP9omSvfDVIeoS5GXoJ1Bg0BqePH8baFQvht2EbOtt3w/7d2yEYPQS/33wAHV09sfq3rl/B4OGjYd/tC3B5PAQFboLbqME4d+0ODAyNmXq9v+qPHwN3M8ccLrdJxiMtI3u0xXqPLzAz6Br+eJoFr8G2OLliIOy8DuF17gex+mPX/w6O4r+BOG11HmK3DMexG89E6p27m4qp2y4zx0UlZY03CAmwJMwMmuNTi80uJLpjxw6YmZmBx+PB0dERsbGxTa7DvqBAjJnogVHj3fCZVXus3rgNysrKOPxzmMT6AUGhmDR5Kmw62sHiMyusC9gFIhTixpUYkXocLge6+gZM4WtqNcFopGfWkI7Yf+EJIi4+xZOXOZgZdA3vi0oh6Gslsf7b/CJk5rxnSl87YxQWleLYjWSResUlZSL1cgqKm2I4DAoKLImludGsjEFkZCS8vb3h4+ODu3fvws7ODq6ursjKymoyHYqLi/HXg3vo0fsrRsZms9HD6Svci5POML1/X4iS0hLwtUS/7LeuX0W39m3Q94tOWLZgFt5m/yNT3RuCkiIbXSx0cPFBGiMjBLj4ZxocrMRnQ5IQuFjh8LUkFBaVish72RrieehEPNg+Clun9oC2etPOiCpem1C1NDealTHYvHkzPD094eHhARsbGwQFBUFFRQUhISFNpsPb7DcoKysTcwd09PTwOitDqjbWr1oGfQND9HT616A49e2HTTv2IuLoWSxcsRqxN67CY+xQlJU17ZS5OnTUeVBUYCMr972IPCvnPQw0VWq9vutnurA11Ubo7wki8gv3UjFlawy+XnEGy8Jj0auDIX5dPkA8oNeIVAQQq5bmhlxiBidPSh/0GjJkiFT1iouLcefOHSxevJiRsdlsuLi44ObNmxKvKSoqQlFREXOcl5cntV6Nxa6tG3D6+GH8fOIcuDweIx/8v9HM/61tbGFt0xF9utng1vUr6OHkLA9VZYqgrxUepvwjFmw8fO3f+MGjF2/x8Hk24oPGwqmDIWIevmoS3SQ9m0B3R5YRw4YNk6oei8WS+pfvzZvyX2R9fX0Rub6+Pp48eSLxGn9/f/j6+krVvrRoaetAQUEBb16LuiZvsrKgq2dQ47XBO7YgKHATIo6eQfsOHWus28bMHNqtdPA8OemTMAZv3n1AaZkQenzRF7jqaSojI6ewxmtVuIoY1dMCfgfjau0nJfMdXue+h4WhRpMZgxbydjX5uAlCoVCq0thT4MWLFyM3N5cpqampDW6Tw+HA1q4Lbly5xMiEQiFuXL2ELl0dqr1u97ZN2LZpHUIjf0Wnzva19pP+6iXeZv8DPf2aDUxTUVIqxL2kN3Du9G/2g8UCnDsaITah5pjN8C/NwVVi45fLibX2Y9xKFa3Uech4W7OBkSVslgQ3oRlag08qtfjhwwfwKk2N64KOTvkvcmZmpog8MzMTBgaSvzBcLhfcRkjPfTttFubP9ETHzvaw+7wr9u/ejsLCQowc5wYAmDfjW+gbGOGH5X4AgKDAjQhY74ctQaFobWKK15nlsQUVVTWoqqmhID8fgRvXYMA3w6CrZ4DnKc+w3ncpTM0t0Mu5n8z1ry+BJx8ieFZv3El6jbi/X8PrG1uo8JQQHv0UALB3Vh+8yi7Aip/+ELnO3cUap24/R/a7IhG5Kk8RS8d8jhM3U5DxthBtDTSwRuCApIw8XLj3ssnGJSlGQFOLjUBZWRn8/PxgbGwMNTU1PHtW7iMuX74c+/btk7odDocDe3t7REdHMzKhUIjo6Gh0795d5nrXxDf/G4UlK/2xZf0qfOPsiMd/PUBo5K/Q1St3YV69TGW+8ABwIDQYxcXFmDF5PBxtzZkSvDMAAKCgoIAnj/7Cd5NGoe8XHbFozjTY2nVB5KnfG8WY1Zcj159hcehtrBhrj9ubh8POvBWGrvqNCSqa6KrCQEs0mPiZER89bAwQGp0g1l6ZkMDWtBUOL+mPhztGI8jLCfeS3sBlySkUlwqbZEzAv88mVC31oa6p75ycHMyYMQOGhobgcrlo164dzp49W6++a4NFCCGN0rKUrFq1CmFhYVi1ahU8PT3x119/oW3btoiMjERAQEC1wT9JREZGQiAQYPfu3XBwcEBAQAAOHTqEJ0+eiMUSJJGXlwc+n48HzzKhrq7RkGH957DxjJC3Ck0OKXmPojOzkZubCw0N8b93xf1gv+IMFHiqIufKPhTgzqpB1V4ricjISLi5uSEoKAiOjo4ICAjA4cOHkZCQAD098fRrcXExevToAT09PSxZsgTGxsZ4/vw5NDU1YWdnV79B14Dc3YTw8HDs2bMHffv2xbRp0xi5nZ1dtYG/6hgzZgxev36NFStWICMjA507d0ZUVJRUhoBCqQ5JKxCF9XATKqe+ASAoKAhnzpxBSEgIFi1aJFY/JCQE2dnZuHHjBpSUlAAAZmZmdR+AlMjdTUhLS4OlpaWYXCgUoqSkpM7teXl54fnz5ygqKsLt27fh6OgoCzUpLZiaFh3l5eWJlMqp6spUpL5dXFwYWW2p75MnT6J79+6YMWMG9PX1YWtri7Vr1zZaYF3uxsDGxgZXr14Vkx85cgRdunSRg0YUiig1LToyMTEBn89nir+/v8Q2akp9Z2RIXoz27NkzHDlyBGVlZTh79iyWL1+OTZs2YfXq1bId4Efk7iasWLECAoEAaWlpEAqFOHbsGBISEhAeHo7Tp0/LWz0KReKio4rj1NRUkZiBLAO6QqEQenp62LNnDxQUFGBvb4+0tDRs2LABPj4+MuunArnPDIYOHYpTp07h999/h6qqKlasWIH4+HicOnUK/fp9OmkzSsulJjdBQ0NDpFRnDOqT+jY0NES7du2goKDAyNq3b4+MjAwUF8v+YS25GwMA6NWrFy5cuICsrCwUFhbi2rVr6N+/v7zVolAAyObZhPqkvnv06IHExEQIhf+mUZ8+fQpDQ0NwOJz6DaYGPgljAABxcXGIiIhAREQE7ty5I291KBQGWa1A9Pb2RnBwMMLCwhAfH4/p06ejoKCAyS64ubmJPFszffp0ZGdnY/bs2Xj69CnOnDmDtWvXYsaMGTIbW2XkHjN4+fIlxo0bh+vXr0NTUxNA+UKLL7/8EgcPHkTr1q3lqyClxcNmiX/562MMakt9v3jxQiQ2YWJignPnzmHu3Lno1KkTjI2NMXv2bCxcuLBhA6oGuRuDKVOmoKSkBPHx8bCyKt8EIyEhAR4eHpgyZQqioppueysKRRKS3IL6PsLs5eUFLy8viediYmLEZN27d8etW7fq1VddkbsxuHz5Mm7cuMEYAgCwsrLCtm3b0KtXLzlqRqGUw2YDCmKLjuSkTCMid2NgYmIicXFRWVkZjIzEN8ukUJoa+ghzE7FhwwbMnDkTcXH/PsseFxeH2bNnY+PGjXLUjEIpR4HFkliaG3KZGWhpaYk89VVQUABHR0coKparU1paCkVFRUyePFnqjVAolMZCljGDTxm5GIOAgAB5dEuh1AtZZRM+deRiDAQCgTy6pVDqBZ0ZyIEPHz6ILbOU9llxCqWxUGCzJGQTmp8xkHsAsaCgAF5eXtDT04Oqqiq0tLRECoUib1jVlOaG3I3BDz/8gIsXL2LXrl3gcrnYu3cvfH19YWRkhPDwcHmrR6EwM4Oqpbkhdzfh1KlTCA8PR58+feDh4YFevXrB0tISpqamOHDgACZMmCBvFSktHEl7HtZ3D8RPGbnPDLKzs9G2bVsA5fGB7OxsAEDPnj1x5coVeapGoQAo/+JXfVCJGoNGoG3btkhOLn/RprW1NQ4dOgSgfMZQ8eAShSJPWoqbIHdj4OHhgQcPHgAAFi1ahB07doDH42Hu3LlYsGCBnLWjUFpOAFHuMYO5c+cy/3dxccGTJ09w584dWFpaolOnTnLUjEIpR9JMoDnODORuDKpiamoKU1NTeatBoTDQRUeNSGBgoNR1Z82a1YiaUCi1Q5cjNyJbtmyRqh6LxZKLMTDg86ChUb93Pv5nSXkgbw2anjLpNhWlM4NGpCJ7QKH8F5D0yDJ9hJlCaYGwWEDViUAztAXUGFAotUGzCRQKBQCgwC4vVWXNDWoMKJRaoNkECoUCAFBglZeqsubGJzHZuXr1KiZOnIju3bsjLS0NABAREYFr167JWTMK5WM2oeqzCc1wZiB3Y3D06FG4urpCWVkZ9+7dY95vn5ubi7Vr18pZOwqlPJMgqTQ35G4MVq9ejaCgIAQHB0NJSYmR9+jRA3fv3pWjZhRKORUvUalc2HL/5sgeuccMEhIS4OTkJCbn8/nIyclpeoUolCq0lEVHcrdvBgYGSExMFJNfu3aN2fSEQpEn1E1oIjw9PTF79mzcvn0bLBYLr169woEDBzB//nxMnz5d3upRKC1mcxO5uwmLFi2CUChE3759UVhYCCcnJ3C5XMyfPx8zZ86Ut3oUCl101FSwWCwsXboUCxYsQGJiIvLz82FjYwM1NTV5q0ahAKCLjpocDocDGxsbeatBoYjBljAzoNmERsDZ2bnGnWYvXrzYhNpQKOK0lGyC3I1B586dRY5LSkpw//59/PXXX/SdjJRPAknZg2YYP5S/Mahu16OVK1ciPz+/ibWhUMRpKY8wf7Kez8SJExESEiJvNSgU+myCvLl58yZ4vBa2DyHlk4RdTakPO3bsgJmZGXg8HhwdHREbGyvVdQcPHgSLxcKwYcPq2XPtyN1NGD58uMgxIQTp6emIi4vD8uXL5aQVhfIvskotRkZGwtvbG0FBQXB0dERAQABcXV2RkJAAPT29aq9LSUnB/Pnz0atXrzr3WRfkPjPg8/kiRVtbG3369MHZs2fh4+Mjb/UoFLA/ZhMql/oYg82bN8PT0xMeHh6wsbFBUFAQVFRUanSHy8rKMGHCBPj6+jb68ny5zgzKysrg4eGBjh07QktLS56qUCjVwmKJb4BacZyXlyci53K54HK5Ym0UFxfjzp07WLx4MSNjs9lwcXHBzZs3q+171apV0NPTw7fffourV6/WfxBSINeZgYKCAvr370+fTqR80lSdFVRed2BiYiIys/X395fYxps3b1BWVgZ9fX0Rub6+PjIyMiRec+3aNezbtw/BwcGyHVA1yD1mYGtri2fPnsHc3FzeqlAoEqkpZpCamgoNDQ1GLmlWUB/evXuHSZMmITg4GDo6OjJpszbkHjNYvXo15s+fj9OnTyM9PR15eXki5b9K0M4dsLI0g6YaD72+dMQfNUSNHz96hLGjR8DK0gzKSixs2xogVqfiXNUyZ+aMRhxF3Zk62glPzvji7a0tuBI+H1071PzeTK/xffDg+HJk39yMv3/zw4/zhoPL+fc3ynNUT8RGLkbm1Q3IvLoBMWHz0L9H0y5bZ7FYEgsAaGhoiJTqjIGOjg4UFBSQmZkpIs/MzISBgYFY/aSkJKSkpGDw4MFQVFSEoqIiwsPDcfLkSSgqKiIpKUnm45SbMVi1ahUKCgrw9ddf48GDBxgyZAhat24NLS0taGlpQVNT8z8bRzh8KBILF3hj6TIf3Iy9i06d7DBkkCuysrIk1i8sLIS5eVv4rVkn8cYAgGs3/0ByajpTzkRdAAAMHzmq0cZRV0b2/xzr5/0Pa3b/hu7j1+PPp2k4uXMGdLUkP3Q2ZkBX+M0airW7f0Pn4asxzfcARrraY9XMIUydtMwcLN/2K76c8CN6TNiAmNinOLzlO7RvK/lzagxqchOkhcPhwN7eHtHR0YxMKBQiOjoa3bt3F6tvbW2Nhw8f4v79+0wZMmQInJ2dcf/+fZiYmDR4XFWRm5vg6+uLadOm4dKlSzJr88qVK9iwYQPu3LmD9PR0HD9+vFHzstURGLAZHt96ws3dAwCwbWcQfvvtDMJCQ7Dgh0Vi9bt264au3boBAJYvFT8PALq6uiLHG39ch7YWFujl1FvG2tefWRO/wv5jNxBx8hYAYOaagxjYqwMEw7pj4/4LYvW/sDPHzfvPEBkVBwB4kZ6NQ1Fx6GZrxtQ5e+UvkWtW7jgFz1E94dDJHPHPJPvaskZWy5G9vb0hEAjQtWtXODg4ICAgAAUFBfDwKL9P3NzcYGxsDH9/f/B4PNja2opcr6mpCQBiclkhN2NACAEA9O4tu5u5oKAAdnZ2mDx5stj6haaiuLgY9+7ewYKFolHjr75yQeyt6qPGde3j4M8/YdYc7xof8mpKlBQV0KW9CTaEnGdkhBBcvJ0Ah06S40G3HiRj7KBu6NrBFHGPnsPMuBVce3TAz2cku1RsNgsj+n0OVWUObv/ZdO/rZIMFNlhisroyZswYvH79GitWrEBGRgY6d+6MqKgoJqj44sULsOX4OKRcA4iyvpEHDhyIgQMHyrTNulIRNdbTE40a6+nrIyHhiUz6OPnrCeTk5GCim7tM2pMFOlpqUFRUQFb2OxF51j95sDLTl3hNZFQcWmmpInr/XLDAgpKSAvYcvipiUACgg6URYsLmgcdRRP77IoyZF4wnTTQrAGS7n4GXlxe8vLwknouJianx2tDQ0Hr1KS1yNQbt2rWr1SBkZ2c3Wv9FRUXM1uyAeM74UyVs/z64DhgIIyMjeavSIHrZf4YFk10x2z8Sfzx8DgsTHWxcMBLpngOwLjiKqfc0JROOY/3BV1PG/1y6IHjVJPSfsrXJDAJbQoyAbm4iY3x9fcHn8+XWv7+/P3x9fWXaZkXUOCtLNGqcVU3UuK48f/4cF6N/x8HDxxrclix58zYfpaVl0NNWF5HrtdJAxj+SjazP94Pwy5lYhB4vd58eJb6CijIXO5aNw/q95xhXsqS0DM9S3wAA7sWnwr5DG8wY1wcz1xxsxBH9S02LjpoTcjUGY8eOrXFNdmOzePFieHt7M8d5eXkNjtJyOBx0+dwely5GY8jQYQDKo8aXLkVj2veSp4d1ISJsP/T09DDw60ENbkuWlJSW4V58KpwdrXAq5k8A5W6gs0M7BEVekXiNMo8DoZCIyIRC4cdrAUIkXVX+q1w5/djY0G3PGplPIfBV3dLRhjJrjjc8Jwtgb98VXbs5YHtgAAoLCuAmKI8af+vuBiNjY/itKV+tVlxcjPjHj5n/v3qVhgf370NNTQ0WlpZMu0KhEOFh+zFhkgCKinJfLyZG4E8XEbxqEu48foG4v1LgNd4ZKspchP9anl3Y6zcJr7JysWLbSQDlmYJZE53xIOElYh+mwMJEFyumf4OzVx4yRmLVzCE4d/0RUtPfQl2VhzEDu8Kp62cY/P3OJhsX3emokSHVmf1mwKjRY/Dm9Wus8l2BzIwMdLLrjF9P/xs1Tk0VjRqnv3qFL7p1YY4DNm9EwOaN6OXUG+ejYxj5xejfkfriBQTuk5tsLHXhyPm70NFSw4rpg6DfSh1/JqRh6IwdTFDRxEBbZCawbm8UCCHw+f4bGOnx8eZtPs5c+Qsrt59i6uhqq2GfnxsMdDSQm/8Bf/2dhsHf78TF27IJxkpDS3ETWKQZfSvz8/OZF7J06dIFmzdvhrOzM7S1tdGmTZtar8/LywOfz0fmP7kiS0xbAlrdGu7C/NcgZcUoehiM3FzJf++K++HsnWSoqomeL8jPw9f25tVe+1/k05trNoC4uDg4OzszxxXxAIFA0OhpGUrzhcYM/oP06dOnWbsfFPnA+liqypobzcoYUCiNgQIkBBCboTmgxoBCqYXKTylWljU3qDGgUGpDQjahGU4MqDGgUGqjpaQWqTGgUGqBZhMoFAoAmk2gUCgfoQFECoUCgL54lUKhVNBC/ARqDCiUWqABRAqFAoCmFikUykdYH/9VlTU3qDGgUGqBBhApFAoAmlqkUCgfoTEDCoUCgBoDCoXyEZaE1CJ1EyiUFkgLWXNEjQGFUhs0gEihUADQ1CKFQqmghfgJ1BhQKLXAhoRnE5qhNaDGgEKpBeomtEAq3rnw7j/yanZZQsqK5a1Ck1Mx5trftdEy/ARqDCrx7l35OwEtzRv2JmbKf4t3796Bz+dXe57ODFogRkZGSE1Nhbq6epOnjipeB5+amtps3t0nDfIcNyEE7969g5GRUY316H4GLRA2m43WrVvLVQcNDY0WZQwqkNe4a5oRMLQML4EaAwqlNlqKm8CWtwIUyqdOxQrEqqU+7NixA2ZmZuDxeHB0dERsbGy1dYODg9GrVy9oaWlBS0sLLi4uNdZvKNQYfCJwuVz4+PiAy+XKW5Um5b8wblY1pa5ERkbC29sbPj4+uHv3Luzs7ODq6oqsrCyJ9WNiYjBu3DhcunQJN2/ehImJCfr374+0tLR6j6UmWIS+w5xCkUheXh74fD6SX2WLxTPy8vJgbqSN3NxcqWMdjo6O6NatG7Zv3w4AEAqFMDExwcyZM7Fo0aJary8rK4OWlha2b98ONze3ug+oFujMgEKphYr9DKoWoNwoVC5FRUUS2yguLsadO3fg4uLCyNhsNlxcXHDz5k2p9CgsLERJSQm0tbUbPCZJUGNAodRCTcbAxMQEfD6fKf7+/hLbePPmDcrKyqCvry8i19fXR0ZGhlR6LFy4EEZGRiIGRZbQbAKFUgs1rTOouj6isWIf69atw8GDBxETEwMej9cofVBjQKHUQk3LDKRdH6GjowMFBQVkZmaKyDMzM2FgYFDjtRs3bsS6devw+++/o1OnTtIrXkeom/AJUJd0U3PhypUrGDx4MIyMjMBisXDixAl5q1Qtskgtcjgc2NvbIzo6mpEJhUJER0eje/fu1V73448/ws/PD1FRUejatWu9xyAN1BjImbqmm5oLBQUFsLOzw44dO+StSq1ULDqqWuqKt7c3goODERYWhvj4eEyfPh0FBQXw8PAAALi5uWHx4sVM/fXr12P58uUICQmBmZkZMjIykJGRgfz8fFkNTRRCkSsODg5kxowZzHFZWRkxMjIi/v7+ctSqaQFAjh8/Lm81xMjNzSUASPqbHFJQLBQp6W9yCACSm5tbpza3bdtG2rRpQzgcDnFwcCC3bt1izvXu3ZsIBALm2NTUlAAQKz4+PjIaoSg0ZiBHKtJNlX8N6ppuojQ++e/eiQUQ8z8+4VpXvLy84OXlJfFcTEyMyHFKSkq9+qgv1BjIkZrSTU+ePJGTVpQKOBwODAwM8Fk1j7QbGBiAw+E0sVaNBzUGFEo18Hg8JCcno7hY8sYvHA6n0dJ88oAaAznSkHQTpWng8XjN6gtfEzSbIEfqm26iUBoDOjOQM97e3hAIBOjatSscHBwQEBAgkm5qruTn5yMxMZE5Tk5Oxv3796GtrY02bdrIUbMWTKPkKCh1oqZ0U3Pl0qVLEtNmlVNrlKaFPsJMoVAA0JgBhUL5CDUGFAoFADUGFArlI9QYUCgUANQYUCiUj1BjQKFQAFBjQKFQPkKNAYVCAUCNwX8Gd3d3DBs2jDnu06cP5syZ0+R6xMTEgMViIScnp9o6dd3GbOXKlejcuXOD9EpJSQGLxcL9+/cb1E5LhhqDBuDu7s7sh8fhcGBpaYlVq1ahtLS00fs+duwY/Pz8pKorzReYQqEPKjWQAQMGYP/+/SgqKsLZs2cxY8YMKCkpiexeVEFxcbHMNsNorBdpUFoudGbQQLhcLgwMDGBqaorp06fDxcUFJ0+eBPDv1H7NmjUwMjKClZUVgPK99kePHg1NTU1oa2tj6NChIltclZWVwdvbG5qammjVqhV++OEHVH2EpKqbUFRUhIULF8LExARcLheWlpbYt28fUlJS4OzsDADQ0tICi8WCu7s7gPLHpf39/WFubg5lZWXY2dnhyJEjIv2cPXsW7dq1g7KyMpydneu1FdfChQvRrl07qKiooG3btli+fDlKSkrE6u3evRsmJiZQUVHB6NGjkZubK3J+7969aN++PXg8HqytrbFz584660KpHmoMZIyysrLIzjjR0dFISEjAhQsXcPr0aZSUlMDV1RXq6uq4evUqrl+/DjU1NQwYMIC5btOmTQgNDUVISAiuXbuG7OxsHD9+vMZ+3dzc8MsvvyAwMBDx8fHYvXs31NTUYGJigqNHjwIAEhISkJ6ejq1btwIA/P39ER4ejqCgIDx69Ahz587FxIkTcfnyZQDlRmv48OEYPHgw7t+/jylTpkj1TsCqqKurIzQ0FI8fP8bWrVsRHByMLVu2iNRJTEzEoUOHcOrUKURFReHevXv4/vvvmfMHDhzAihUrsGbNGsTHx2Pt2rVYvnw5wsLC6qwPpRrk/NTkfxqBQECGDh1KCCFEKBSSCxcuEC6XS+bPn8+c19fXJ0VFRcw1ERERxMrKigiFQkZWVFRElJWVyblz5wghhBgaGpIff/yROV9SUkJat27N9EVI+U66s2fPJoQQkpCQQACQCxcuSNSz4nHht2/fMrIPHz4QFRUVcuPGDZG63377LRk3bhwhhJDFixcTGxsbkfMLFy4Ua6sqqGW34w0bNhB7e3vm2MfHhygoKJCXL18yst9++42w2WySnp5OCCHEwsKC/PzzzyLt+Pn5ke7duxNCCElOTiYAyL1796rtl1IzNGbQQE6fPg01NTWUlJRAKBRi/PjxWLlyJXO+Y8eOInGCBw8eIDExEerq6iLtfPjwAUlJScjNzUV6ejocHR2Zc4qKiujatauYq1DB/fv3oaCggN69e0utd2JiIgoLC9GvXz8ReXFxMbp06QIAiI+PF9EDQL12YIqMjERgYCCSkpKQn5+P0tJSsbcQtWnTBsbGxiL9CIVCJCQkQF1dHUlJSfj222/h6enJ1CktLQWfz6+zPhTJUGPQQJydnbFr1y5wOBwYGRlBUVH0I1VVVRU5zs/Ph729PQ4cOCDWlq6ubr10UFZWrvM1FS/iOHPmjMiXEJDt+wJv3ryJCRMmwNfXF66uruDz+Th48CA2bdpUZ12Dg4PFjJOCgoLMdG3pUGPQQFRVVWFpaSl1/c8//xyRkZHQ09Or9h19hoaGuH37NpycnACU/wLeuXMHn3/+ucT6HTt2hFAoxOXLlyW+obdiZlJWVsbIbGxswOVy8eLFi2pnFO3bt2eCoRXcunWr9kFW4saNGzA1NcXSpUsZ2fPnz8XqvXjxAq9evYKRkRHTD5vNhpWVFfT19WFkZIRnz55hwoQJdeqfIj00gNjETJgwATo6Ohg6dCiuXr2K5ORkxMTEYNasWXj58iUAYPbs2Vi3bh1OnDiBJ0+e4Pvvv69xjYCZmRkEAgEmT56MEydOMG0eOnQIAGBqagoWi4XTp0/j9evXyM/Ph7q6OubPn4+5c+ciLCwMSUlJuHv3LrZt28YE5aZNm4a///4bCxYsQEJCAn7++WeEhobWabyfffYZXrx4gYMHDyIpKQmBgYESg6E8Hg8CgQAPHjzA1atXMWvWLIwePZrZJdrX1xf+/v4IDAzE06dP8fDhQ+zfvx+bN2+ukz6UGpB30OK/TOUAYl3Op6enEzc3N6Kjo0O4XC5p27Yt8fT0ZF7VVVJSQmbPnk00NDSIpqYm8fb2Jm5ubtUGEAkh5P3792Tu3LnE0NCQcDgcYmlpSUJCQpjzq1atIgYGBoTFYjH7DAqFQhIQEECsrKyIkpIS0dXVJa6uruTy5cvMdadOnSKWlpaEy+WSXr16kZCQkDoHEBcsWEBatWpF1NTUyJgxY8iWLVsIn89nzvv4+BA7Ozuyc+dOYmRkRHg8Hhk5ciTJzs4WaffAgQOkc+fOhMPhEC0tLeLk5ESOHTtGCKEBRFlA90CkUCgAqJtAoVA+Qo0BhUIBQI0BhUL5CDUGFAoFADUGFArlI9QYUCgUANQYUCiUj1BjQKFQAFBjQKFQPkKNAYVCAUCNAYVC+cj/AS/gWy1NMgXwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 200x200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot normalized confusion matrix for all seeds/models in the current loop\n",
    "skplt.metrics.plot_confusion_matrix(list(flatten(y_testList)), list(flatten(y_predList)), normalize=True,figsize=(2,2))\n",
    " \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test stastistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 931,
   "metadata": {},
   "outputs": [],
   "source": [
    "TN = cm[0][0]\n",
    "FN = cm[1][0] # type II error\n",
    "TP = cm[1][1]\n",
    "FP = cm[0][1] # Type I error\n",
    "\n",
    "# Sensitivity, hit rate, recall, or true positive rate\n",
    "TPR = TP/(TP+FN)\n",
    "# Specificity or true negative rate\n",
    "TNR = TN/(TN+FP) \n",
    "# Precision or positive predictive value\n",
    "precision = TP/(TP+FP)\n",
    "# Negative predictive value\n",
    "NPV = TN/(TN+FN)\n",
    "# Fall out or false positive rate\n",
    "FPR = FP/(FP+TN)\n",
    "# balanced accurcy\n",
    "bACC = (TPR + TNR) / 2\n",
    "# False negative rate\n",
    "FNR = FN/(TP+FN)\n",
    "# False discovery rate\n",
    "FDR = FP/(TP+FP)\n",
    "# F1 score\n",
    "F1 = (2 * TP) / (2 * TP + FP + FN)\n",
    "# False ommission rate\n",
    "FOR = 1 - NPV\n",
    "# LR+, positive liklihood ratio\n",
    "LRpos = TPR / FPR\n",
    "# LR-, negative liklihood ratio\n",
    "LRneg = FNR / TNR\n",
    "# diagnostic odds ratio\n",
    "oddsRatio = LRpos/LRneg\n",
    "\n",
    "# Overall accuracy\n",
    "ACC = (TP+TN)/(TP+FP+FN+TN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 932,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the sensitivity/TPR is: 0.8095238095238095\n",
      "the specificity/TNR is: 0.6071428571428571\n",
      "the accuracy is: 0.7285714285714285\n",
      "the balanced accuracy is: 0.7083333333333333\n",
      "the positive predictive value (PPV) aka precision is 0.7555555555555555\n",
      "the negative predictive value (NPV) is 0.68\n",
      "the diagnostic odds ratio is 6.568181818181818\n",
      "The positive LR is 2.0606060606060606\n",
      "The negative LR is 0.3137254901960784\n",
      "The F1 score is 0.7816091954022989\n"
     ]
    }
   ],
   "source": [
    "print(\"the sensitivity/TPR is: \" + str(TPR))\n",
    "print(\"the specificity/TNR is: \" + str(TNR))\n",
    "print(\"the accuracy is: \" + str(ACC))\n",
    "print(\"the balanced accuracy is: \" + str(bACC))\n",
    "print(\"the negative predictive value (NPV) is \" + str(NPV))\n",
    "print(\"the diagnostic odds ratio is \" + str(oddsRatio))\n",
    "print(\"The positive LR is \" + str(LRpos))\n",
    "print(\"The negative LR is \" + str(LRneg))\n",
    "print(\"The F1 score is \" + str(F1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def true_false_positive(threshold_vector, y_test):\n",
    "    true_positive = np.equal(threshold_vector, 1) & np.equal(y_test, 1)\n",
    "    true_negative = np.equal(threshold_vector, 0) & np.equal(y_test, 0)\n",
    "    false_positive = np.equal(threshold_vector, 1) & np.equal(y_test, 0)\n",
    "    false_negative = np.equal(threshold_vector, 0) & np.equal(y_test, 1)\n",
    "\n",
    "    tpr = true_positive.sum() / (true_positive.sum() + false_negative.sum())\n",
    "    fpr = false_positive.sum() / (false_positive.sum() + true_negative.sum())\n",
    "\n",
    "    return tpr, fpr\n",
    "\n",
    "def roc_from_scratch(probabilities, y_test, partitions=100):\n",
    "    roc = np.array([])\n",
    "    for i in range(partitions + 1):\n",
    "        \n",
    "        threshold_vector = np.greater_equal(probabilities, i / partitions).astype(int)\n",
    "        tpr, fpr = true_false_positive(threshold_vector, y_test)\n",
    "        roc = np.append(roc, [fpr, tpr])\n",
    "        \n",
    "    return roc.reshape(-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calc the AUROC\n",
    "partitions = 100\n",
    "\n",
    "ROC = roc_from_scratch(y_prob, y_test, partitions=partitions) # y_prob[:,1]\n",
    "fpr, tpr = ROC[:, 0], ROC[:, 1]\n",
    "rectangle_roc = 0\n",
    "for k in range(partitions):\n",
    "        rectangle_roc = rectangle_roc + (fpr[k]- fpr[k + 1]) * tpr[k]\n",
    "print('The AUROC is: ' + str(rectangle_roc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# overlaid ROC\n",
    "from sklearn.metrics import auc\n",
    "\n",
    "tprs = []\n",
    "aucs = []\n",
    "mean_fpr = np.linspace(0, 1, 100) # regularly spaced points\n",
    "\n",
    "#set up plotting area\n",
    "plt.figure(figsize=(3, 3))\n",
    "ax = plt.axes() # enables overlay\n",
    "\n",
    "plt.plot([0, 1], [0, 1],'--', color = 'black', lw = 2)\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "\n",
    "fprList = []\n",
    "tprList = []\n",
    "\n",
    "for y_prob, y_test in zip(y_probList,y_testList):\n",
    "    \n",
    "    ROC = roc_from_scratch(np.reshape(y_prob,(-1)),y_test,partitions=100) # y_prob[:,1]\n",
    "    plt.plot(ROC[:,0],ROC[:,1],color='black', alpha = 0.025)\n",
    "    plt.title('ROC Curve',fontsize=12)\n",
    "    plt.xlabel('False Positive Rate',fontsize=12)\n",
    "    plt.ylabel('True Positive Rate',fontsize=12)\n",
    "\n",
    "    sort = np.argsort(ROC[:,0])\n",
    "    interp_tpr = np.interp(mean_fpr, ROC[:,0][sort], ROC[:,1][sort]) # interp needs to be sorted\n",
    "    interp_tpr[0] = 0.0\n",
    "    tprs.append(interp_tpr)\n",
    "    rectangle_roc = 0\n",
    "    for k in range(partitions):\n",
    "        fpr, tpr = ROC[:, 0], ROC[:, 1]\n",
    "        rectangle_roc = rectangle_roc + (fpr[k]- fpr[k + 1]) * tpr[k]\n",
    "    print('AUROC is ' + str(rectangle_roc))    \n",
    "    aucs.append(rectangle_roc)\n",
    "\n",
    "mean_tpr = np.mean(tprs, axis=0)\n",
    "mean_tpr[-1] = 1.0\n",
    "mean_auc = auc(mean_fpr, mean_tpr)\n",
    "std_auc = np.std(aucs)\n",
    "ax.plot(mean_fpr, mean_tpr, color='b',\n",
    "        label=r'Mean ROC (AUC = %0.2f $\\pm$ %0.2f)' % (mean_auc, std_auc),\n",
    "        lw=2, alpha=.8)\n",
    "\n",
    "std_tpr = np.std(tprs, axis=0)\n",
    "tprs_upper = np.minimum(mean_tpr + std_tpr, 1)\n",
    "tprs_lower = np.maximum(mean_tpr - std_tpr, 0)\n",
    "ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.3,\n",
    "                label=r'$\\pm$ 1 std. dev.')\n",
    "\n",
    "ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],\n",
    "       title=\"ROC curve\")\n",
    "\n",
    "plt.plot()\n",
    "ax.legend(bbox_to_anchor=(3.3, .5), loc='center right', borderaxespad=0)\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "y_testListFlat = list(chain.from_iterable(y_testList)) \n",
    "y_predListFlat = list(chain.from_iterable(y_predList))\n",
    "y_probListFlat = list(chain.from_iterable(y_probList))\n",
    "y_1MinusProbListFlat = [1 - x for x in y_probListFlat]\n",
    "y_predProbAFormattedLikeSKLearn = np.stack((y_1MinusProbListFlat, y_probListFlat), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 934,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://towardsdatascience.com/roc-curve-and-auc-from-scratch-in-numpy-visualized-2612bb9459ab\n",
    "\n",
    "def true_false_positive(threshold_vector, y_test): # threshold_vector is input as y_prob\n",
    "    true_positive = np.equal(threshold_vector, 1) & np.equal(y_test, 1)\n",
    "    true_negative = np.equal(threshold_vector, 0) & np.equal(y_test, 0)\n",
    "    false_positive = np.equal(threshold_vector, 1) & np.equal(y_test, 0)\n",
    "    false_negative = np.equal(threshold_vector, 0) & np.equal(y_test, 1)\n",
    "\n",
    "    tpr = true_positive.sum() / (true_positive.sum() + false_negative.sum())\n",
    "    fpr = false_positive.sum() / (false_positive.sum() + true_negative.sum())\n",
    "\n",
    "    return tpr, fpr # ROC[:,0] is fpr, ROC[:,1] is tpr\n",
    "\n",
    "def roc_from_scratch(probabilities, y_test, partitions=1000):\n",
    "    roc = np.array([])\n",
    "    for i in range(partitions + 1):\n",
    "        \n",
    "        threshold_vector = np.greater_equal(probabilities, i / partitions).astype(int)\n",
    "        tpr, fpr = true_false_positive(threshold_vector, y_test)\n",
    "        roc = np.append(roc, [fpr, tpr])\n",
    "        \n",
    "    return roc.reshape(-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 935,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'True Positive Rate')"
      ]
     },
     "execution_count": 935,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAGNCAYAAAD3m81fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJd0lEQVR4nO3deXiM5/4G8HuyTSKRhawIiag9pKKJIIKmjRY9SglKglJLLJU6JShSJVpblJRSRC0Va+ugFOXYq0KKxr4L2ZBFItvM8/vDL3OMLJLJTGaSuT/XNZdrnvd53/f7JDL3vLtECCFARER6yUDbBRARkfYwBIiI9BhDgIhIjzEEiIj0GEOAiEiPMQSIiPQYQ4CISI8xBIiI9BhDgIhIjzEEiIj0GEOAqrTo6GhIJBLFy8jICHXr1sWQIUOQkJBQ7DxCCKxfvx6dOnWCtbU1atSoAXd3d3z11VfIysoqcV07d+7Ee++9B1tbW5iYmKBOnTro168f/vjjjzLVmpOTg8WLF8Pb2xtWVlYwNTVF48aNMXbsWFy7dk2l8RNVlIT3DqKqLDo6GkOHDsVXX30FV1dX5OTk4PTp04iOjoaLiwsuXboEU1NTRX+ZTIaBAwdiy5Yt8PX1Re/evVGjRg0cO3YMmzZtQvPmzXHw4EE4ODgo5hFCYNiwYYiOjsabb76Jjz76CI6Ojnj06BF27tyJ2NhYnDhxAu3bty+xztTUVHTr1g2xsbHo0aMH/P39YWFhgatXr2Lz5s1ITExEXl6eRn9WRMUSRFXY2rVrBQDx119/KbVPnjxZABAxMTFK7XPnzhUAxKRJk4osa9euXcLAwEB069ZNqX3+/PkCgPjss8+EXC4vMt9PP/0k/vzzz1Lr7N69uzAwMBDbtm0rMi0nJ0d8/vnnpc5fVvn5+SI3N1ctyyL9wBCgKq2kENi9e7cAIObOnatoy87OFjY2NqJx48YiPz+/2OUNHTpUABCnTp1SzFOrVi3RtGlTUVBQoFKNp0+fFgDEiBEjytTfz89P+Pn5FWkPDg4WDRo0ULy/ffu2ACDmz58vFi9eLBo2bCgMDAzE6dOnhaGhoZg1a1aRZVy5ckUAEEuXLlW0PX36VEyYMEHUq1dPmJiYCDc3NzFv3jwhk8nKPVaqenhMgKqlO3fuAABsbGwUbcePH8fTp08xcOBAGBkZFTtfUFAQAGD37t2KeZ48eYKBAwfC0NBQpVp27doFABg8eLBK87/O2rVrsXTpUnz66adYuHAhnJyc4Ofnhy1bthTpGxMTA0NDQ/Tt2xcAkJ2dDT8/P2zYsAFBQUH47rvv0KFDB4SFhSE0NFQj9ZJuKf4vgaiKSU9PR2pqKnJycvDnn38iPDwcUqkUPXr0UPSJj48HALRu3brE5RROu3z5stK/7u7uKtemjmWU5sGDB7hx4wbs7OwUbYGBgRg5ciQuXbqEli1bKtpjYmLg5+enOOaxaNEi3Lx5E+fPn8cbb7wBABg5ciTq1KmD+fPn4/PPP4ezs7NG6ibdwC0Bqhb8/f1hZ2cHZ2dnfPTRRzA3N8euXbtQr149RZ/MzEwAQM2aNUtcTuG0jIwMpX9Lm+d11LGM0vTp00cpAACgd+/eMDIyQkxMjKLt0qVLiI+PR2BgoKJt69at8PX1hY2NDVJTUxUvf39/yGQyHD16VCM1k+7glgBVC1FRUWjcuDHS09OxZs0aHD16FFKpVKlP4YdwYRgU59WgsLS0fO08r/PyMqytrVVeTklcXV2LtNna2uLtt9/Gli1bMHv2bAAvtgKMjIzQu3dvRb/r16/jwoULRUKkUHJystrrJd3CEKBqwcvLC23btgUA9OrVCx07dsTAgQNx9epVWFhYAACaNWsGALhw4QJ69epV7HIuXLgAAGjevDkAoGnTpgCAixcvljjP67y8DF9f39f2l0gkEMWcuS2TyYrtb2ZmVmx7//79MXToUMTFxcHDwwNbtmzB22+/DVtbW0UfuVyOd955B1988UWxy2jcuPFr66WqjbuDqNoxNDREREQEHj58iGXLlinaO3bsCGtra2zatKnED9SffvoJABTHEjp27AgbGxv8/PPPJc7zOj179gQAbNiwoUz9bWxskJaWVqT97t275Vpvr169YGJigpiYGMTFxeHatWvo37+/Uh83Nzc8e/YM/v7+xb7q169frnVS1cMQoGqpc+fO8PLyQmRkJHJycgAANWrUwKRJk3D16lVMmzatyDx79uxBdHQ0AgIC0K5dO8U8kydPxuXLlzF58uRiv6Fv2LABZ86cKbEWHx8fdOvWDT/++CN++eWXItPz8vIwadIkxXs3NzdcuXIFKSkpira///4bJ06cKPP4AcDa2hoBAQHYsmULNm/eDBMTkyJbM/369cOpU6ewf//+IvOnpaWhoKCgXOukKkjb56gSVURJ1wkIIcTWrVsFALF8+XJFW0FBgejTp48AIDp16iSWLFkiVq5cKYKCgoSBgYFo0aKFSExMVFqOTCYTgwcPFgBEmzZtxNy5c8WaNWvE3LlzhZeXlwAgTp48WWqdycnJwsPDQ0gkEvHBBx+IJUuWiB9//FFMnjxZNGjQQJiYmCj6xsfHCwMDA/Hmm2+KZcuWiRkzZgh7e3vh7u5e4nUCJdmwYYMAIGrWrCl69uxZZHpWVpZo06aNMDIyEsOHDxfLly8XCxYsEMHBwcLc3FykpKSUOi6q+hgCVKWVFgIymUy4ubkJNzc3pQu9ZDKZWLt2rejQoYOwtLQUpqamokWLFiI8PFw8e/asxHVt27ZNvPvuu6JWrVrCyMhIODk5icDAQHHkyJEy1ZqdnS0WLFgg3nrrLWFhYSFMTEzEG2+8IcaNGydu3Lih1HfDhg2iYcOGwsTERHh4eIj9+/eXerFYSTIyMoSZmZkAIDZs2FBsn8zMTBEWFiYaNWokTExMhK2trWjfvr1YsGCByMvLK9PYqOrivYOIiPQYjwkQEekxhgARkR5jCBAR6TGGABGRHmMIEBHpMYYAEZEe0/t7B8nlcjx8+BA1a9aERCLRdjlERBUmhEBmZibq1KkDA4PSv+vrfQg8fPiQ90snomrp/v37SrdTL47eh0DhLYPv37+vuOUvEVFVlpGRAWdn5zI9w0LvQ6BwF5ClpSVDgIiqlbLs4uaBYSIiPcYQICLSYwwBIiI9xhAgItJjDAEiIj3GECAi0mMMASIiPaZTIXD06FH07NkTderUgUQiKfah3K86cuQI2rRpA6lUikaNGiE6OlrjdRKReq2/ehgjjyzDxqtHtF2KTmm8cRSs1wxA042jNbYOnQqBrKwstG7dGlFRUWXqf/v2bXTv3h1dunRBXFwcPvvsMwwfPhz79+/XcKVEpA5xqbdgu3YQxp1YiZhbJxBy4gfYrh2Ei6m3tV2aVg07FAnrNQOQnJsOAEjMTYP1mgEYeXiZ2tels88Ylkgk2LlzJ3r16lVin8mTJ2PPnj24dOmSoq1///5IS0vDvn37yrSejIwMWFlZIT09nVcME1Uy27WDUCBkRdoNYIDorhO0UJFuCPpjcYnT0ob9/Nr5y/O5VqVvG3Hq1Cn4+/srtQUEBOCzzz4rcZ7c3Fzk5uYq3mdkZGiqPCIqxfqrh4sNAACQQ17qB6E+a7pxNK58vFxty6vSIZCYmAgHBwelNgcHB2RkZOD58+cwMzMrMk9ERATCw8Mrq0QiKsHxR/+UOt1WWhONrOpUUjW643Ty1VKnJ+amqXV9VToEVBEWFobQ0FDF+8K77RFR5ero1AIxt06UOD287UB83KRz5RWkIxpvHKU4FlAcR6m1WtenUweGy8vR0RFJSUlKbUlJSbC0tCx2KwAApFKp4o6hvHMokfYMbtIFRhLDYqcZSQz1MgAA4NrHK0qdrs5dQUAVDwEfHx8cOnRIqe3AgQPw8fHRUkVEVB6He86GwSsfQ0YSQxzuOVtLFemGQNcO5WqvCJ0KgWfPniEuLg5xcXEAXpwCGhcXh3v37gF4sSsnKChI0X/UqFG4desWvvjiC1y5cgXff/89tmzZgokTJ2qjfCIqJ3dbV6x/+8Xfa21pTUR1GInUoRvgbuuq5cq064cuY5E27GfFrh9HqTXShv2MH7qMVfu6dOqYwNmzZ9GlSxfF+8J998HBwYiOjsajR48UgQAArq6u2LNnDyZOnIglS5agXr16+PHHHxEQEFDptRNRxbhZOertLqCSqHvXT3F0KgQ6d+6M0i5bKO5q4M6dO+P8+fMarIqIqPrSqd1BRERUuRgCRER6jCFARKTHGAJERHqMIUBEpMcYAkREeowhQESkxxgCRER6jCFARKTHGAJERHqMIUBEpMcYAkREeowhQESkxxgCRER6jCFARKTHGAJERHqMIUBEpMcYAkREeowhQESkxxgCRER6jCFARKTHGAJERHqMIUBEpMcYAkREeowhQESkxxgCRER6jCFARKTHGAJERHqMIUBEpMcYAkREeowhQESkxxgCRER6jCFARKTHGAJERHqMIUBEpMcYAkSkE84kX4f1mgHaLkPvGGm7ACLSX45rBiEHMqU26zUDYA5jJAz7SUtV6RduCRCR1rwaAIWykF/JlegvhgARacXrdv1w11DlYAgQEekxhgARkR5jCBCRVqQN+7lC00k9GAJEpDXmMC5XO6kfTxElIq1JGPYT9tw9i48PLVS0cQugcnFLgIh0gpf9GwwALdC5EIiKioKLiwtMTU3h7e2NM2fOlNo/MjISTZo0gZmZGZydnTFx4kTk5ORUUrVERFWbToVATEwMQkNDMXPmTJw7dw6tW7dGQEAAkpOTi+2/adMmTJkyBTNnzsTly5exevVqxMTEYOrUqZVcORFR1aRTIbBo0SKMGDECQ4cORfPmzbFixQrUqFEDa9asKbb/yZMn0aFDBwwcOBAuLi549913MWDAgNduPRAR0Qs6EwJ5eXmIjY2Fv7+/os3AwAD+/v44depUsfO0b98esbGxig/9W7duYe/evXj//fdLXE9ubi4yMjKUXkRE+kpnzg5KTU2FTCaDg4ODUruDgwOuXLlS7DwDBw5EamoqOnbsCCEECgoKMGrUqFJ3B0VERCA8PFyttRMRVVU6syWgiiNHjmDu3Ln4/vvvce7cOezYsQN79uzB7NmzS5wnLCwM6enpitf9+/crsWIiIt2iM1sCtra2MDQ0RFJSklJ7UlISHB0di53nyy+/xODBgzF8+HAAgLu7O7KysvDpp59i2rRpMDAomnFSqRRSqVT9AyAiqoJ0ZkvAxMQEnp6eOHTokKJNLpfj0KFD8PHxKXae7OzsIh/0hoaGAAAhhOaKJSKqJnRmSwAAQkNDERwcjLZt28LLywuRkZHIysrC0KFDAQBBQUGoW7cuIiIiAAA9e/bEokWL8Oabb8Lb2xs3btzAl19+iZ49eyrCgIiISqZTIRAYGIiUlBTMmDEDiYmJ8PDwwL59+xQHi+/du6f0zX/69OmQSCSYPn06EhISYGdnh549e2LOnDnaGgIRUZUiEXq+3yQjIwNWVlZIT0+HpaWltssh0juF9w7ysn8Dv/f4StvlVAvl+VzTmWMCRERU+RgCRER6jCFARKTHGAJERHqsQmcH5ebm4ty5c0hOTkaHDh1ga2urrrqIiKgSqLwl8N1338HJyQkdO3ZE7969ceHCBQAv7gFka2tb4p0/iYhId6gUAmvXrsVnn32Gbt26YfXq1UpX59ra2qJr167YvHmz2ookIiLNUCkEFi5ciH/961/YtGkTevbsWWS6p6cn/vnnnwoXR0REmqVSCNy4cQPvvfdeidNr1aqFx48fq1wUERFVDpVCwNraGqmpqSVOj4+PL/HOn0REpDtUCoH3338fK1euRFpaWpFp//zzD1atWoUPPvigorUREZGGqRQCX3/9NWQyGVq2bKm4idu6deswaNAgtG3bFvb29pgxY4a6ayUiIjVTKQTq1KmD2NhYdOvWDTExMRBCYP369fjPf/6DAQMG4PTp07xmgIioClD5YjF7e3v8+OOP+PHHH5GSkgK5XA47O7tin+ZFRES6SaVP7GHDhuHPP/9UvLezs4ODg4MiAM6cOYNhw4app0IiItIYlUIgOjoaN2/eLHH67du3sW7dOpWLIiKiyqGRfTcPHz6EmZmZJhZNRERqVOZjAr/++it+/fVXxfuVK1fi4MGDRfqlpaXh4MGDeOutt9RTIRERaUyZQyA+Ph5bt24FAEgkEvz555+IjY1V6iORSGBubo5OnTph0aJF6q2UiIjUrswhEBYWhrCwMACAgYEBVq9ejYEDB2qsMCIi0jyVThGVy+XqroOIiLSAJ/UTEekxlUPgt99+wzvvvIPatWvDyMgIhoaGRV5ERKTbVAqB7du3o0ePHkhKSkL//v0hl8sxYMAA9O/fH2ZmZmjVqhXvHUREVAWoFAIRERHw8vLC+fPnER4eDuDFVcQbN27EpUuX8OjRI7i6uqq1UCIiUj+VQiA+Ph79+/eHoaEhjIxeHFvOz88HALi4uGDMmDH45ptv1FclERFphEohUKNGDZiYmAB48YAZqVSKR48eKaY7ODjg9u3b6qmQqJo49OBvfHt+Ow4/uKDtUnTSmeTraLpxtLbL0DsqnSLapEkTxMfHK957eHhg/fr1GDRoEAoKCrBp0ybUr19fbUUSVWW3M5Lw9n+m40nuM0VbLakFDvf8Gg0sHbRYmfYNOxSJHXf/dzPKxNw0WK8ZgEDXDvihy1gtVqY/VNoS+PDDD/Hrr78iNzcXADBt2jQcOXIE1tbWsLOzw7FjxzBlyhS1FkpUVb0aAADwJPcZuvxnupYq0h0vB8DLYm6fqORK9JdECCHUsaBjx45hx44dMDQ0RPfu3dGlSxd1LFbjMjIyYGVlhfT0dFhaWmq7HKpmDj34G31+n1fidAOJBBJIKrEi3SETpV906ii1xpWPl1dSNdVLeT7XVH6ozKt8fX3h6+ureJ+ZmYmaNWuqa/FEVVJsyo1Sp8uFAKCW72HVTmJumrZL0AtqC4FCycnJiIyMxPLly/H06VN1L56oSvG0a1Tq9NV+49DBqVklVaNbOuyYjMd5mSVOd5RaV14xeqxcIZCcnIyffvoJN2/ehI2NDfr06QNPT08AQEJCAubMmYPo6Gjk5OSgc+fOmqiXqEp5u15r1JJaFDkmALw4ONzHrb0WqtINNwethPWaASVO566gylHmELhy5Qo6deqEx48fo/AwwrfffosNGzZAIpFg+PDhyMnJQZ8+ffDvf/9bEQ5E+u5wz6/RpYSzg/RdoGuHYg8CB7p20EI1+qnMB4b79u2LPXv2YPHixfD19cXt27cxceJEZGRkID09HT179sS8efPQsGFDTdesVjwwTJWl3Y5/40raA3zRujemevbVdjk6penG0UjMTePBYDXRyIHho0ePYvTo0Rg5ciQAoHnz5jAyMsJ7772H4OBgrF27tmJVE1Vz1ibmAICWtRtouRLdww9+7SnzdQKPHz9Gq1atlNpat24N4MV1A0REVPWUOQTkcjmMjY2V2grfW1hYqLcqIiKqFOU6O+js2bMwNTVVvM/MzIREIsHx48eRlpZWpH/v3r0rXCAREWlOuUIgMjISkZGRRdpnzZpVpE0ikUAmk6laFxERVYIyh8Dhw4c1WQcREWlBmUPAz89Pk3UQEZEW8EHzRER6jCFARKTHGAJERHpM50IgKioKLi4uMDU1hbe3N86cOVNq/7S0NISEhMDJyQlSqRSNGzfG3r17K6laIqKqTe23kq6ImJgYhIaGYsWKFfD29kZkZCQCAgJw9epV2NvbF+mfl5eHd955B/b29ti2bRvq1q2Lu3fvwtrauvKLJyKqgnQqBBYtWoQRI0Zg6NChAIAVK1Zgz549WLNmTbGPq1yzZg2ePHmCkydPKq5ednFxqcySiYiqNJV3B927dw+jRo1CkyZNUKtWLRw9ehQAkJqaivHjx+P8+fPlWl5eXh5iY2Ph7+//v+IMDODv749Tp04VO8+uXbvg4+ODkJAQODg4oGXLlpg7d26pF6nl5uYiIyND6UVEpK9UCoH4+Hi8+eabiImJgaurK9LT01FQUAAAsLW1xfHjx7Fs2bJyLTM1NRUymQwODg5K7Q4ODkhMTCx2nlu3bmHbtm2QyWTYu3cvvvzySyxcuBBff13yfdojIiJgZWWleDk7O5erTiKi6kSlEPjiiy9gbW2Na9euYcOGDXj1kQTdu3fHsWPH1FJgaeRyOezt7bFy5Up4enoiMDAQ06ZNw4oVK0qcJywsDOnp6YrX/fv3NV4nEZGuUumYwNGjRzFjxgzY2dnh8ePHRabXr18fCQkJ5Vqmra0tDA0NkZSUpNSelJQER0fHYudxcnKCsbExDA0NFW3NmjVDYmIi8vLyYGJiUmQeqVQKqVRartqIiKorlbYE5HI5atSoUeL0lJSUcn/QmpiYwNPTE4cOHVJaz6FDh+Dj41PsPB06dMCNGzcgl8sVbdeuXYOTk1OxAUBERMpUCoE2bdpgz549xU4rKCjA5s2b0a5du3IvNzQ0FKtWrcK6detw+fJljB49GllZWYqzhYKCghAWFqboP3r0aDx58gQTJkzAtWvXsGfPHsydOxchISGqDIuISO+otDsoLCwMPXr0wOjRo9G/f38AL3bbHDx4EHPnzsXly5fLfWAYAAIDA5GSkoIZM2YgMTERHh4e2Ldvn+Jg8b1792Bg8L/ccnZ2xv79+zFx4kS0atUKdevWxYQJEzB58mRVhkVEpHfK/KD5V61fvx4TJkxAeno6hBCQSCQQQsDS0hLLly/HgAED1F2rRvBB81RZuu2ehdPJV/FT14n4wMVL2+VQNaaRB82/avDgwejduzcOHDiA69evQy6Xw83NDQEBAahZs6aqiyUiokqkUggUfvM3NzdHr1691FwSERFVFpUODBfuez9x4oS66yEiokqkUgj4+flhzZo16NSpE+rXr49Jkya99m6fRESke1QKgZ9//hnJycnYvHkzvLy8sHz5cvj4+MDNzQ1Tp05FXFycmsskIiJNUPkGcmZmZujbty+2bduG5ORkbNiwAe7u7li8eDE8PT3RtGlTddZJREQaoJaHypibm2PAgAHYsGED5s+fDwsLC1y/fl0diyYiIg2q8PMEsrOzsWvXLmzZsgX79u1Dbm4u3NzcMH78eHXUR0REGqRSCOTk5GDPnj2IiYnB3r17kZ2dDRcXF4wfPx6BgYF488031V0nERFpgEohYGdnh+zsbNSpUweffvopAgMD4e3tre7aiIhIw1QKgSFDhiAwMBAdO3ZUdz1ERFSJVAqBpUuXqrsOIiLSgjKFQOHzgzt16qT0/nUK+xMRkW4qUwh07twZEokEz58/h4mJieJ9SQrvLVTaA9+JiEj7yhQChw8fBgDF07oK3xMRUdVWphDw8/Mr9T0REVVNKl0x3LVrV6VnAb/q8OHD6Nq1q8pFERFR5VApBI4cOYKkpKQSpycnJ+O///2vykURVUc3Mx4BAH7853ctV0L0PyrfO6i0A8M3btzg08WI/l/M9aOwXjMAKTkZAICjSf/Aes0AbL/B53GQ9pX5GcPr1q3DunXrALzYEmjWrJniAfAvS0tLw4ULF/D+++9j165d6q1WA/iMYdI06zUlP287bdjPlVgJ6QuNPGM4OzsbKSkpiveZmZkwMFDekCh85OSoUaMwY8aMcpZNVP2M+e/yUqeP/e8PWOY3spKqISqqzCEwevRojB49GgDg6uqKJUuW4IMPPtBYYUTVwYmk+FKnH0u6VEmVEBVPpdtG3L59W911EFVLHRya4+6zkq+w93VoWYnVEBVVphC4d+8eAKB+/fpK71+nsD+RvvrebzQ23Sw5BLgriLStTCHg4uKidNuIwvevw9tGEAGrO43FJ0eXFdtOpG1lCoE1a9ZAIpHA2NhY6T0RvV6fRh3Qp1EHvLFpJFJyMtDJoQV2dZ+u7bKIAJQxBIYMGVLqeyJ6PTdLJ6TkZGB4i3e1XQqRgloeNF8oLy8PWVlZ6lwkERFpkEohsHnzZkycOFGpLTw8HBYWFrC2tsaHH36IZ8+eqaVAIiLSHJVCYOHChUrf+E+ePInw8HAEBARg4sSJ2LdvH+bMmaO2IomISDNUuk7g5s2bCA4OVrzftGkTHB0dsXPnThgZGUEul2P79u2IiIhQW6FERKR+Km0J5ObmwtTUVPH+999/x3vvvQcjoxeZ0rx5czx48EA9FRIRkcaoFAKurq44ePAgAODs2bO4ceMGunXrppielJQECwsL9VRIREQao9LuoJEjR2LChAmIj4/HgwcPUK9ePfTo0UMx/cSJE2jRooXaiiQiIs1QKQTGjRsHU1NT7N27F56enpg8eTLMzMwAAE+ePEFiYiJGjRql1kKJiEj9VAoBABgxYgRGjBhRpL1WrVo4e/ZshYoiIqLKoXIIFIqPj8fdu3cBAA0aNEDz5s0rXBQREVUOlUPg119/RWhoKO7cuaPU7urqikWLFvFZA0REVYBKZwft3bsXffr0AQDMnTsXO3fuxM6dOzF37lwIIdC7d2/s27dPrYUSEZH6lfkZwy/z8fFBbm4ujh07BnNzc6VpWVlZ6NixI0xNTXHq1Cm1FaopfMYwVZZuu2fhdPJV/NR1Ij5w8dJ2OVSNledzTaUtgQsXLiA4OLhIAACAubk5hgwZggsXLqiyaCIiqkQqhYCpqSmePHlS4vQnT54oXVFMRES6SaUQ6Nq1K5YsWVLs7p4///wT3333Hfz9/StcHBERaZZKZwd9++238PHxQceOHeHl5YUmTZoAAK5evYozZ87A3t4e33zzjVoLJSIi9VP53kEXLlzA+PHj8fTpU8TExCAmJgZPnz7FhAkT8Pfff8PFxUXNpRIRkbqVOwRkMhkSExNhaWmJxYsX48qVK3j+/DmeP3+OK1euYNGiRbC3t69QUVFRUXBxcYGpqSm8vb1x5syZMs23efNmSCQS9OrVq0LrJyLSF2UOASEEpk6dChsbG9StWxeWlpb48MMPSz1ArIqYmBiEhoZi5syZOHfuHFq3bo2AgAAkJyeXOt+dO3cwadIk+Pr6qrUeIqLqrMwhEB0djXnz5sHa2hp9+vSBu7s7fv31VwwdOlStBS1atAgjRozA0KFD0bx5c6xYsQI1atTAmjVrSpxHJpPh448/Rnh4OBo2bKjWeoiIqrMyh8Dy5cvx5ptv4urVq9iyZQtiY2Mxbtw47NmzB6mpqWopJi8vD7GxsUpnFhkYGMDf37/UC8+++uor2Nvb45NPPnntOnJzc5GRkaH0IiLSV2UOgZs3byIoKEhxy2gAGDNmDORyOa5fv66WYlJTUyGTyeDg4KDU7uDggMTExGLnOX78OFavXo1Vq1aVaR0RERGwsrJSvJydnStcNxFRVVXmEHj69Cns7OyU2mxtbQEAOTk56q2qjDIzMzF48GCsWrVKUcvrhIWFIT09XfG6f/++hqskItJd5bpOQCKRaKoOAC9CxdDQEElJSUrtSUlJcHR0LNL/5s2buHPnDnr27Klok8vlAAAjIyNcvXoVbm5uSvNIpVJIpVINVE9EVPWUKwSmTJmCiIgIxXuZTAYAGD58eJH7CEkkEvz999/lKsbExASenp44dOiQ4jRPuVyOQ4cOYezYsUX6N23aFBcvXlRqmz59OjIzM7FkyRLu6iEieo0yh0CnTp2K3RKo6DUBrwoNDUVwcDDatm0LLy8vREZGIisrS3EWUlBQEOrWrYuIiAiYmpqiZcuWSvNbW1sDQJF2IiIqqswhcOTIEQ2W8T+BgYFISUnBjBkzkJiYCA8PD+zbt09xsPjevXswMFDpQmciInpFhR8vqQljx44tdvcP8Powio6OVn9BRETVFL9SExHpMYYAEZEeYwgQEekxhgARkR5jCBAR6bEKnR2UkJCAo0ePIjk5GX369EG9evUgk8mQnp4OKysrGBoaqqtOIiLSAJW2BIQQCA0NhaurKz7++GOEhobi2rVrAIBnz57BxcUFS5cuVWuhRESkfiqFwPz587FkyRJMmjQJBw4cgBBCMc3Kygq9e/fG9u3b1VYkUXVwOvkqAGDIH5HaLYToJSrtDlq1ahWCgoIwd+5cPH78uMj0Vq1a4bfffqtwcUTVQYdtk/BPRoLivRwC1msGwMO6AY70nqfFyohU3BK4f/8+2rdvX+J0c3NzPqyF6P+9HAAvi0u7W8mVEBWlUgjY29uXeh/+2NhY1K9fX+WiiKoL2zUfV2g6kaapFAK9e/fGihUrcOvWLUVb4R1Gf//9d0RHR6Nv377qqZCoCiuAvELTiTRNpRAIDw+Hk5MTPDw8EBQUBIlEgm+++QYdO3bEe++9h1atWmHq1KnqrpWoShFCwAClP4jJiJfqkJap9D/QysoKp0+fxhdffIGEhASYmpriv//9L9LS0jBz5kwcO3YMNWrUUHetRFXGw6wn6H9wPuQQpfZLHbaxkioiKp5EvHx+px7KyMiAlZUV0tPTYWlpqe1yqIoTQmDj9f9i6pn1yMjLhomBEWqbWOBRTlqRvjw7iDSlPJ9rOvk8AaKq6MGzx/jsxCocTHjxWNU2tm6I8h2JZjYvHnNqu+ZjFEAOIxhwC4B0hkohMGzYsNf2kUgkWL16tSqLJ6pShBBYf+0wpp/ZgIz855AaGmPqmx8hpGV3GBn879Yp/OAnXaRSCPzxxx9Fnjcsk8nw6NEjyGQy2NnZFXnwPFF1dP9ZKsYfX4nDDy8CAN6yewPLfEeiiXVdLVdGVDYqhcCdO3eKbc/Pz8cPP/yAyMhIHDhwoCJ1Eek0IQSirx7Cl2c24llBDkwNjTGtTT+MafE+DPkMbKpCNHJgeMyYMbh79y727Nmj7kWrHQ8MU3ndyUzG+OMrcfTRPwCAdvZNsMx3JBpZOWm5MqIXtH5guHXr1li/fr0mFk2kNXIhx+rLBzHr7CZkFeTCzNAEX7YNxMhm3fjtn6osjYTAgQMHeJ0AVSu3M5Iw9vgPOJF4GQDg49AUUb4j0dDSUcuVEVWMSiHw1VdfFduelpaGo0eP4ty5c5gyZUqFCiPSBXIhx8r43/FV7GZkF+SihpEUs9oOwPBm78BAwm//VPWpdEzAoIRNXxsbG7i5uWH48OEYMWJEkTOIdBGPCVBJbqY/wtjjK3Eq6QoAoKNjcyzr+ClcLB20XBlR6TR+TEAu502vqPqSyeVYEb8PX8fG4LksD+ZGUnz11scY2vRtfvunaqfcIfD8+XNMmzYNXbp0Qc+ePTVRE5HWXE9/iJBjK3Am+ToAwM+pJb7r+Cka1LTTcmVEmlHuEDAzM8MPP/yA5s2ba6IeIq2QyeWI+mcP5p7bihxZPmoam2H2Wx8juEnXKrFbk0hVKu0O8vT0xKVLl9RdC5FWXE1LQMixFTibcgMA0LVuKyzpMALOFrZaroxI81QKgcjISLz//vto2bIlhgwZAiMj3oeOqp4CuQxLL+7GvLjtyJXlw9LYDHO8B2PQG5357Z/0RpnPDjp69CiaNWsGOzs7uLu74/Hjx0hKSoJUKkXdunVhZmamvGCJBH///bdGilYnnh2kn+Kf3kfIsRU4n/ri6Xjv1PNAZIfhqGteW8uVEVWcRs4O6tKlCzZs2IABAwagdu3asLW1RZMmTSpcLFFlypcXYMmF/+DbuB3IkxfA0qQG5nkHYUCjTvz2T3qpzCEghEDhRsORI0c0VQ+Rxlx6chchx37A349vAwACnNsgssMncKpRS8uVEWkPd+ZTtZcvL8Civ3/Fgr93Il8ug7WJOb5pF4x+bh357Z/0XrlCgH8wVNVceHwHY46twKUndwEA3eu3xcL2w+BYw0bLlRHphjIfGDYwMChXCEgkEhQUFKhcWGXhgeHqKU9WgAV/78Siv39FgZChltQC832GorerD7/MULWnsdtG+Pv7o3HjxhUqjkjT4lJvYcyxFYh/eh8A8IGLFxb4DIW9mbV2CyPSQeUKgeDgYAwcOFBTtRBVSK4sH9+e34HIi7sgE3LUNq2JhT7D0Mu1nbZLI9JZPDBM1cK5lJsIObYCl9MeAAA+dG2H+e2GwtaMu/iISsMQoCotpyAP885vx3eX/gO5ELAztcSC9sPwLxdvbZdGVCUwBKjK+iv5OkKOrcC19IcAgL4NO2BeuyDUNuW3f6KyKnMI8BkCpCueF+Rh7rmtiPpnD+RCwMHMGovaf4LuDdpquzSiKodbAlSlnE66irHHfsCNjEcAgEA3X8xrFwQbqYWWKyOqmhgCVCVkF+RidmwMVvyzDwICTjVssLj9cHSr30bbpRFVaQwB0nknEi9j3PGVuJWRCAAY+IYf5noNgjW//RNVGEOAdFZWfg7CYzdjZfx+AECdGrWwpMMIvOPsod3CiKoRnXxqdlRUFFxcXGBqagpvb2+cOXOmxL6rVq2Cr68vbGxsYGNjA39//1L7U9Vw7FE8OvwyWREAQY274FTv+QwAIjXTuRCIiYlBaGgoZs6ciXPnzqF169YICAhAcnJysf2PHDmCAQMG4PDhwzh16hScnZ3x7rvvIiEhoZIrJ3V4lp+DSSfXoOdvs3EnMxn1zGtjR0AYvuv4KaxMami7PKJqp8w3kKss3t7eeOutt7Bs2TIAL05NdXZ2xrhx4zBlypTXzi+TyWBjY4Nly5YhKCjotf15Aznd8d+HlzD2+A+4/ywVADC0ydsIf2sgLPnhT1QuGruBnKbl5eUhNjYWYWFhijYDAwP4+/vj1KlTZVpGdnY28vPzUatW8Q8Kyc3NRW5uruJ9RkZGxYqmCsvIy8bMvzZh7dVDAABnC1ss6zgSfnVaarkyoupPp0IgNTUVMpkMDg4OSu0ODg64cuVKmZYxefJk1KlTB/7+/sVOj4iIQHh4eIVrJfX4I+ECxh9fiQdZjwEAw5u+g5lvDUBNY7PXzElE6qBTIVBR8+bNw+bNm3HkyBGYmpoW2ycsLAyhoaGK9xkZGXB2dq6sEun/pedlY/qZDVh/7TAAoIGFPZb5joSvU3MtV0akX3QqBGxtbWFoaIikpCSl9qSkJDg6OpY674IFCzBv3jwcPHgQrVq1KrGfVCqFVCpVS72kmgP34/DZyVVIyHoCABjZvBtmeAbC3Lj44CYizdGps4NMTEzg6emJQ4cOKdrkcjkOHToEHx+fEuf79ttvMXv2bOzbtw9t2/L+MboqLfcZxhxbgb4HvkFC1hM0tHTE3vdn4pt2wQwAIi3RqS0BAAgNDUVwcDDatm0LLy8vREZGIisrC0OHDgUABAUFoW7duoiIiAAAfPPNN5gxYwY2bdoEFxcXJCa+uKrUwsICFha8olRX/HYvFhNP/IjE52mQQILRLd7DdM9+qGHErTIibdK5EAgMDERKSgpmzJiBxMREeHh4YN++fYqDxffu3YOBwf82YJYvX468vDx89NFHSsuZOXMmZs2aVZmlUzGe5j7DlNM/IebmMQBAI0snRPmOgrcDH1NKpAt07jqBysbrBDRnz92zCD25GknP02AgkSCkRXdMbdMXZkYm2i6NqFqrstcJUPXwOCcDk0+vw7ZbJwEAja3qIMp3FN6yf0PLlRHRqxgCpFa77pzB5ydXIyUnAwYSCSa498Rkjz4w5bd/Ip3EECC1SH2egX+fXoudt08DAJpZ10OU7yi0sXPTcmVEVBqGAFXYL7dP4/NTa/A4JxOGEgNMbPUB/u3RG1JDY22XRkSvwRAglSU/T8OkU2ux686LW3c3t3HG976j4GHbUMuVEVFZMQSo3IQQ2H7rJL44HY0nuc9gJDFEaOt/YVLrD2FiyP9SRFUJ/2KpXJKy0xB6cjX23DsLAGhZqwG+9x2FVrVdtFsYEamEIUBlIoTAlpvHMfn0OqTlZcHYwBCTWn+I0Nb/grEB/xsRVVX866XXepT9BJ+dWI39988BAFrXdkWU70i0rNVAy5URUUUxBKhEQgj8fOMowv5cj/S8LJgYGGHym30w3r0Hv/0TVRP8S6ZiJWQ9xmcnfsSBB3EAgDa2bojyHYlmNnz2AlF1whAgJUIIrL9+BNP/XI+M/OeQGhoj7M2PMLZldxgZGGq7PCJSM4YAKdx/looJJ1bhj4QLAIC2do0Q5TsKTazrarkyItIUhgBBCIF1V//Al39tRGb+c5gaGmNam34Y0+J9GBro1HOHiEjNGAJ67m5mCsYfX4n/ProEAGhn3wTLfEeikZWTlisjosrAENBTciHH2iuHMOOvjcgqyIWZoQm+bBuIkc268ds/kR5hCOihOxlJGHt8JY4nxgMAfByaIsp3JBpaOmq5MiKqbAwBPSIXcqy6/DvCz25GdkEuahhJMavtAAxv9g4MJPz2T6SPGAJ64lZGIsYe/wEnE68AADo6Nseyjp/CxdJBy5URkTYxBKo5mVyOHy7vw+yzMXguy4O5kRRfvfUxhjZ9m9/+iYghUJ1dT3+Iscd+wJ/J1wAAfk4t8V3HT9Ggpp2WKyMiXcEQqIZkcjm+/2cv5pzbghxZPmoam2H2Wx8juElXSCQSbZdHRDqEIVDNXEtLQMixH/BXynUAQNe6rbCkwwg4W9hquTIi0kUMgWqiQC7Dskt7EHF+G3Jl+bA0NsMc78EY9EZnfvsnohIxBKqBy0/vI+TYDziXehMA8E49D0R2GI665rW1XBkR6TqGQBVWIJdhycX/4Jvz25EnL4ClSQ3M8w7CgEad+O2fiMqEIVBF/fPkHkKOrUDc49sAgADnNojs8AmcatTScmVEVJUwBKqYfHkBFl/YhflxO5Avl8HaxBzftAtGP7eO/PZPROXGEKhCLjy+g5BjK3DxyV0AQPf6bbGw/TA41rDRcmVEVFUxBKqAPFkBFv79Cxb+/QsKhAy1pBaY7zMUvV19+O2fiCqEIaDj4lJvI+TYCvzz9B4A4AMXLyzwGQp7M2vtFkZE1QJDQEflyvIxP24HFl/YBZmQo7ZpTSz0GYZeru20XRoRVSMMAR10PvUmxhxdgctpDwAAH7q2w/x2Q2FrZqnlyoioumEI6JCcgjx8E7cd313cDZmQw87UEgvaD8O/XLy1XRoRVVMMAR1xNvkGQo6vwNW0BABA34YdMK9dEGqb8ts/EWkOQ0DLnhfkIeL8Viy7tAdyIeBgZo1F7T9B9wZttV0aEekBhoAW/Zl0DSHHVuBGxiMAQKCbL+a1C4KN1ELLlRGRvmAIaEF2QS6+jt2C5f/8BgEBpxo2WNx+OLrVb6Pt0ohIzzAEKtnJxCsYe/wH3MpIBAAMfMMPc70GwZrf/olICxgClSQrPwdfxcZgZfx+CAjUqVELSzqMwDvOHtoujYj0GEOgEhx/FI+xx3/AncxkAEBQ4y6Y7TUIViY1tFwZEek7hoAGPcvPQfjZn7Hq8u8AgHrmtfFdx0/RtW4rLVdGRPQCQ0BD/vvwEsYdX4l7z1IAAEObvI3wtwbCkt/+iUiHMATULCMvGzP/2oS1Vw8BAJwtbLGs40j41Wmp5cqIiIpiCKjR4YQLGHd8FR5kpQIAhjd9BzPfGoCaxmZaroyIqHgG2i6gOFFRUXBxcYGpqSm8vb1x5syZUvtv3boVTZs2hampKdzd3bF3716N13gj/REO3I/DzfRHSM/LxoTjq/Dh/gg8yEpFAwt7/Oe9L7Gg/TAGABHpNJ3bEoiJiUFoaChWrFgBb29vREZGIiAgAFevXoW9vX2R/idPnsSAAQMQERGBHj16YNOmTejVqxfOnTuHli3Vvwvmae4zDD+yFIcSLijapIbGyJXlAwBGNu+GGZ6BMDc2Vfu6iYjUTSKEENou4mXe3t546623sGzZMgCAXC6Hs7Mzxo0bhylTphTpHxgYiKysLOzevVvR1q5dO3h4eGDFihWvXV9GRgasrKyQnp4OS8vX36ytz/4IHHl4CTIhV2o3MzTB9oAwtHds+tplEBFpUnk+13Rqd1BeXh5iY2Ph7++vaDMwMIC/vz9OnTpV7DynTp1S6g8AAQEBJfbPzc1FRkaG0qusbqQ/wqGEC0UCAACey/LgYGZV5mUREekCnQqB1NRUyGQyODg4KLU7ODggMTGx2HkSExPL1T8iIgJWVlaKl7Ozc5nru52RVOr0W6+ZTkSka3QqBCpDWFgY0tPTFa/79++XeV5XS4dSpzd8zXQiIl2jUweGbW1tYWhoiKQk5W/USUlJcHR0LHYeR0fHcvWXSqWQSqUq1dfIyglv121V5JiAocQAneu0hJuVk0rLJSLSFp3aEjAxMYGnpycOHTqkaJPL5Th06BB8fHyKncfHx0epPwAcOHCgxP4VtbrzOHR+5cKvznVaYnXncRpZHxGRJunUlgAAhIaGIjg4GG3btoWXlxciIyORlZWFoUOHAgCCgoJQt25dREREAAAmTJgAPz8/LFy4EN27d8fmzZtx9uxZrFy5UiP1WUstsD0gDDfTH+FWRhIaWjpwC4CIqiydC4HAwECkpKRgxowZSExMhIeHB/bt26c4+Hvv3j0YGPxvA6Z9+/bYtGkTpk+fjqlTp+KNN97AL7/8opFrBF7mZuXED38iqvJ07jqBylbe6wSIiHRdlb1OgIiIKhdDgIhIjzEEiIj0GEOAiEiPMQSIiPQYQ4CISI/p3HUCla3wDNny3E2UiEiXFX6eleUKAL0PgczMTAAo191EiYiqgszMTFhZlX6Le72/WEwul+Phw4eoWbMmJBJJmefLyMiAs7Mz7t+/X20vMqvuY+T4qr7qPkZVxyeEQGZmJurUqaN0h4Xi6P2WgIGBAerVq6fy/JaWltXyP9/LqvsYOb6qr7qPUZXxvW4LoBAPDBMR6TGGABGRHmMIqEgqlWLmzJkqP6CmKqjuY+T4qr7qPsbKGJ/eHxgmItJn3BIgItJjDAEiIj3GECAi0mMMASIiPcYQKEVUVBRcXFxgamoKb29vnDlzptT+W7duRdOmTWFqagp3d3fs3bu3kipVXXnGuGrVKvj6+sLGxgY2Njbw9/d/7c9E28r7Oyy0efNmSCQS9OrVS7MFVlB5x5eWloaQkBA4OTlBKpWicePGOv//tLxjjIyMRJMmTWBmZgZnZ2dMnDgROTk5lVRt+Rw9ehQ9e/ZEnTp1IJFI8Msvv7x2niNHjqBNmzaQSqVo1KgRoqOjK1aEoGJt3rxZmJiYiDVr1oh//vlHjBgxQlhbW4ukpKRi+584cUIYGhqKb7/9VsTHx4vp06cLY2NjcfHixUquvOzKO8aBAweKqKgocf78eXH58mUxZMgQYWVlJR48eFDJlZdNecdX6Pbt26Ju3brC19dX/Otf/6qcYlVQ3vHl5uaKtm3bivfff18cP35c3L59Wxw5ckTExcVVcuVlV94xbty4UUilUrFx40Zx+/ZtsX//fuHk5CQmTpxYyZWXzd69e8W0adPEjh07BACxc+fOUvvfunVL1KhRQ4SGhor4+HixdOlSYWhoKPbt26dyDQyBEnh5eYmQkBDFe5lMJurUqSMiIiKK7d+vXz/RvXt3pTZvb28xcuRIjdZZEeUd46sKCgpEzZo1xbp16zRVYoWoMr6CggLRvn178eOPP4rg4GCdDoHyjm/58uWiYcOGIi8vr7JKrLDyjjEkJER07dpVqS00NFR06NBBo3WqQ1lC4IsvvhAtWrRQagsMDBQBAQEqr5e7g4qRl5eH2NhY+Pv7K9oMDAzg7++PU6dOFTvPqVOnlPoDQEBAQIn9tU2VMb4qOzsb+fn5qFWrlqbKVJmq4/vqq69gb2+PTz75pDLKVJkq49u1axd8fHwQEhICBwcHtGzZEnPnzoVMJqussstFlTG2b98esbGxil1Gt27dwt69e/H+++9XSs2aponPGb2/gVxxUlNTIZPJ4ODgoNTu4OCAK1euFDtPYmJisf0TExM1VmdFqDLGV02ePBl16tQp8p9SF6gyvuPHj2P16tWIi4urhAorRpXx3bp1C3/88Qc+/vhj7N27Fzdu3MCYMWOQn5+PmTNnVkbZ5aLKGAcOHIjU1FR07NgRQggUFBRg1KhRmDp1amWUrHElfc5kZGTg+fPnMDMzK/cyuSVAKpk3bx42b96MnTt3wtTUVNvlVFhmZiYGDx6MVatWwdbWVtvlaIRcLoe9vT1WrlwJT09PBAYGYtq0aVixYoW2S1ObI0eOYO7cufj+++9x7tw57NixA3v27MHs2bO1XZrO4pZAMWxtbWFoaIikpCSl9qSkJDg6OhY7j6OjY7n6a5sqYyy0YMECzJs3DwcPHkSrVq00WabKyju+mzdv4s6dO+jZs6eiTS6XAwCMjIxw9epVuLm5abboclDl9+fk5ARjY2MYGhoq2po1a4bExETk5eXBxMREozWXlypj/PLLLzF48GAMHz4cAODu7o6srCx8+umnmDZt2mvvra/rSvqcsbS0VGkrAOCWQLFMTEzg6emJQ4cOKdrkcjkOHToEHx+fYufx8fFR6g8ABw4cKLG/tqkyRgD49ttvMXv2bOzbtw9t27atjFJVUt7xNW3aFBcvXkRcXJzi9cEHH6BLly6Ii4vTuSfPqfL769ChA27cuKEINwC4du0anJycdC4AANXGmJ2dXeSDvjD0RDW4TZpGPmdUPqRczW3evFlIpVIRHR0t4uPjxaeffiqsra1FYmKiEEKIwYMHiylTpij6nzhxQhgZGYkFCxaIy5cvi5kzZ1aJU0TLM8Z58+YJExMTsW3bNvHo0SPFKzMzU1tDKFV5x/cqXT87qLzju3fvnqhZs6YYO3asuHr1qti9e7ewt7cXX3/9tbaG8FrlHePMmTNFzZo1xc8//yxu3bolfv/9d+Hm5ib69eunrSGUKjMzU5w/f16cP39eABCLFi0S58+fF3fv3hVCCDFlyhQxePBgRf/CU0T//e9/i8uXL4uoqCieIqpJS5cuFfXr1xcmJibCy8tLnD59WjHNz89PBAcHK/XfsmWLaNy4sTAxMREtWrQQe/bsqeSKy688Y2zQoIEAUOQ1c+bMyi+8jMr7O3yZroeAEOUf38mTJ4W3t7eQSqWiYcOGYs6cOaKgoKCSqy6f8owxPz9fzJo1S7i5uQlTU1Ph7OwsxowZI54+fVr5hZfB4cOHi/2bKhxTcHCw8PPzKzKPh4eHMDExEQ0bNhRr166tUA28lTQRkR7jMQEiIj3GECAi0mMMASIiPcYQICLSYwwBIiI9xhAgItJjDAEiIj3GECAi0mMMAdIJR44cgUQiwZEjR7RdikZJJBLMmjWrTH1dXFwwZMgQjdZDxBCgComOjoZEIin2NWXKFG2XV6pXazc1NUXjxo0xduzYIndq1JSTJ09i1qxZSEtLq5T1lYWLi4vSz8Xc3BxeXl746aefVF7m3r17yxx+VLl4K2lSi6+++gqurq5KbS1bttRSNeVTWHtOTg6OHz+O5cuXY+/evbh06RJq1Kih1nU9f/4cRkb/+7M7efIkwsPDMWTIEFhbWyv1vXr1qtZufezh4YHPP/8cAPDo0SP8+OOPCA4ORm5uLkaMGFHu5e3duxdRUVEMAh3EECC1eO+993T61tKlebn24cOHo3bt2li0aBF+/fVXDBgwQK3rKs8DeKRSqVrXXR5169bFoEGDFO+HDBmChg0bYvHixSqFAOku7g4ijbp79y7GjBmDJk2awMzMDLVr10bfvn1x586d1857/fp19OnTB46OjjA1NUW9evXQv39/pKenK/XbsGEDPD09YWZmhlq1aqF///64f/++yjV37doVAHD79m0AQEFBAWbPng03NzdIpVK4uLhg6tSpyM3NVZrv7NmzCAgIgK2tLczMzODq6ophw4Yp9Xn5mMCsWbPw73//GwDg6uqq2P1S+LN5+ZjA2bNnIZFIsG7duiL17t+/HxKJBLt371a0JSQkYNiwYXBwcIBUKkWLFi2wZs0alX8mdnZ2aNq0KW7evKnUfuzYMfTt2xf169eHVCqFs7MzJk6ciOfPnyv6DBkyBFFRUYrxF74KyeVyREZGokWLFjA1NYWDgwNGjhyJp0+fqlwvlR23BEgt0tPTkZqaqtRma2uLv/76CydPnkT//v1Rr1493LlzB8uXL0fnzp0RHx9f4u6WvLw8BAQEIDc3F+PGjYOjoyMSEhKwe/dupKWlwcrKCgAwZ84cfPnll+jXrx+GDx+OlJQULF26FJ06dcL58+eL7GIpi8IPutq1awN4sXWwbt06fPTRR/j888/x559/IiIiApcvX8bOnTsBAMnJyXj33XdhZ2eHKVOmwNraGnfu3MGOHTtKXE/v3r1x7do1/Pzzz1i8eLHisZZ2dnZF+rZt2xYNGzbEli1bEBwcrDQtJiYGNjY2CAgIAPDiSVPt2rWDRCLB2LFjYWdnh99++w2ffPIJMjIy8Nlnn5X7Z1JQUIAHDx7AxsZGqX3r1q3Izs7G6NGjUbt2bZw5cwZLly7FgwcPsHXrVgDAyJEj8fDhQxw4cADr168vsuyRI0ciOjoaQ4cOxfjx43H79m0sW7YM58+fx4kTJ2BsbFzueqkcKnQjatJ7a9euLfZ+6IX/tbKzs4vMc+rUKQFA/PTTT4q2wvuqHz58WAghFA/Z2Lp1a4nrvnPnjjA0NBRz5sxRar948aIwMjIq0l5S7QcPHhQpKSni/v37YvPmzaJ27drCzMxMPHjwQMTFxQkAYvjw4UrzTpo0SQAQf/zxhxBCiJ07dwoA4q+//ip1nXjl+Qvz588XAMTt27eL9G3QoIHSvfLDwsKEsbGxePLkiaItNzdXWFtbi2HDhinaPvnkE+Hk5CRSU1OVlte/f39hZWVV7O/k1fW+++67IiUlRaSkpIiLFy+KwYMHCwAiJCREqW9xy4qIiBASiUTxYBQhhAgJCRHFfdwcO3ZMABAbN25Uat+3b1+x7aR+3B1EahEVFYUDBw4ovQAoPfc0Pz8fjx8/RqNGjWBtbY1z586VuLzCb/r79+9HdnZ2sX127NgBuVyOfv36ITU1VfFydHTEG2+8gcOHD5epdn9/f9jZ2cHZ2Rn9+/eHhYUFdu7cibp162Lv3r0AgNDQUKV5Cg+a7tmzBwAUWxy7d+9Gfn5+mdZbXoGBgcjPz1fauvj999+RlpaGwMBAAC8eobh9+3b07NkTQgiln0tAQADS09NL/bm/vFw7OzvY2dnB3d0d69evx9ChQzF//nylfi//frOyspCamor27dtDCIHz58+/dj1bt26FlZUV3nnnHaVaPT09YWFhUebfIamOu4NILby8vIo9MPz8+XNERERg7dq1SEhIUHrO66v79l/m6uqK0NBQLFq0CBs3boSvry8++OADDBo0SBEQ169fhxACb7zxRrHLKOtuhKioKDRu3BhGRkZwcHBAkyZNFGfl3L17FwYGBmjUqJHSPI6OjrC2tsbdu3cBAH5+fujTpw/Cw8OxePFidO7cGb169cLAgQPVdoC3devWaNq0KWJiYvDJJ58AeLEryNbWVnEcIyUlBWlpaVi5ciVWrlxZ7HKSk5Nfuy5vb298/fXXkMlkuHTpEr7++ms8ffq0yLOI7927hxkzZmDXrl1F9uGX9vstdP36daSnp8Pe3l7lWqliGAKkUePGjcPatWvx2WefwcfHB1ZWVpBIJOjfv7/SA8+Ls3DhQgwZMgS//vorfv/9d4wfPx4RERE4ffo06tWrB7lcDolEgt9++03xMPGXWVhYlKnGkgLsZS8fyCxp+rZt23D69Gn85z//wf79+zFs2DAsXLgQp0+fLnMtrxMYGIg5c+YgNTUVNWvWxK5duzBgwADFaaeFP9NBgwYVOXZQqFWrVq9dj62tLfz9/QEAAQEBaNq0KXr06IElS5YotopkMhneeecdPHnyBJMnT0bTpk1hbm6OhIQEDBky5LW/38J67e3tsXHjxmKnF3d8hNSLIUAatW3bNgQHB2PhwoWKtpycnDJfHOXu7g53d3dMnz4dJ0+eRIcOHbBixQp8/fXXcHNzgxACrq6uaNy4sUbqb9CgAeRyOa5fv45mzZop2pOSkpCWloYGDRoo9W/Xrh3atWuHOXPmYNOmTfj444+xefNmDB8+vNjlvy5cXhUYGIjw8HBs374dDg4OyMjIQP/+/RXT7ezsULNmTchkMsWHuDp0794dfn5+mDt3LkaOHAlzc3NcvHgR165dw7p16xAUFKToW7gr8GUljdPNzQ0HDx5Ehw4dlHYtUeXhMQHSKENDQ6VdQACwdOlSyGSyUufLyMhAQUGBUpu7uzsMDAwUp2b27t0bhoaGCA8PL7IOIQQeP35c4frff/99AEBkZKRS+6JFiwC8+HAEgKdPnxapwcPDAwCKnEr6MnNzcwAocyg2a9YM7u7uiImJQUxMDJycnNCpUyfFdENDQ/Tp0wfbt2/HpUuXisyfkpJSpvUUZ/LkyXj8+DFWrVqlWBcApXELIbBkyZIi85Y0zn79+kEmk2H27NlF5ikoKNCpK6mrK24JkEb16NED69evh5WVFZo3b45Tp07h4MGDitMvS/LHH39g7Nix6Nu3Lxo3boyCggKsX79e8SEHvPgW+fXXXyMsLAx37txBr169ULNmTdy+fRs7d+7Ep59+ikmTJlWo/tatWyM4OBgrV65EWloa/Pz8cObMGaxbtw69evVCly5dAADr1q3D999/jw8//BBubm7IzMzEqlWrYGlpqQiS4nh6egIApk2bhv79+8PY2Bg9e/ZUfGgWJzAwEDNmzICpqSk++eSTIlcVz5s3D4cPH4a3tzdGjBiB5s2b48mTJzh37hwOHjyIJ0+eqPSzeO+999CyZUssWrQIISEhaNq0Kdzc3DBp0iQkJCTA0tIS27dvL/b8/sJxjh8/HgEBATA0NET//v3h5+eHkSNHIiIiAnFxcXj33XdhbGyM69evY+vWrViyZAk++ugjleqlMtLOSUlUXRSeZlnSqZFPnz4VQ4cOFba2tsLCwkIEBASIK1euFDn98dVTRG/duiWGDRsm3NzchKmpqahVq5bo0qWLOHjwYJF1bN++XXTs2FGYm5sLc3Nz0bRpUxESEiKuXr1aodoL5efni/DwcOHq6iqMjY2Fs7OzCAsLEzk5OYo+586dEwMGDBD169cXUqlU2Nvbix49eoizZ88qLQuvnCIqhBCzZ88WdevWFQYGBkqni776Myp0/fp1xWm4x48fL7bmpKQkERISIpydnYWxsbFwdHQUb7/9tli5cmWpYy1cb/fu3YudFh0dLQCItWvXCiGEiI+PF/7+/sLCwkLY2tqKESNGiL///lupjxBCFBQUiHHjxgk7OzshkUiKnC66cuVK4enpKczMzETNmjWFu7u7+OKLL8TDhw9fWy9VjESIV7ZhiYhIb/CYABGRHmMIEBHpMYYAEZEeYwgQEekxhgARkR5jCBAR6TGGABGRHmMIEBHpMYYAEZEeYwgQEekxhgARkR5jCBAR6bH/A+Q659Th0ItiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(4,4))\n",
    "\n",
    "ROC = roc_from_scratch(y_prob,y_test,partitions=100)\n",
    "plt.scatter(ROC[:,0],ROC[:,1],color='#0F9D58',s=20)\n",
    "plt.plot(ROC[:,0],ROC[:,1],color='#0F9D58')\n",
    "plt.title('ROC Curve',fontsize=12)\n",
    "plt.xlabel('False Positive Rate',fontsize=12)\n",
    "plt.ylabel('True Positive Rate',fontsize=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 936,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5513929,\n",
       " 0.7704794,\n",
       " 0.16478509,\n",
       " 0.9976261,\n",
       " 0.9999211,\n",
       " 0.6972365,\n",
       " 0.13036905,\n",
       " 0.7188189,\n",
       " 0.9957897,\n",
       " 0.9973846]"
      ]
     },
     "execution_count": 936,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 937,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x180979e10>"
      ]
     },
     "execution_count": 937,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAGNCAYAAAD3m81fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABh10lEQVR4nO3deXxMV/8H8M+dmcxMNklkT4RsYkuIUqp2TRuq+iitoI+dorQ0VUsVVS3dKEV5aIlSTWir9ZRGSfnZqZ0mSEKEkA3ZM5nMzPn9cZ8MI5NkZjJbMt/36zUvnTvn3Pneofd77j3nnsMxxhgIIYTYJIGlAyCEEGI5lAQIIcSGURIghBAbRkmAEEJsGCUBQgixYZQECCHEhlESIIQQG0ZJgBBCbBglAUIIsWGUBAghxIZREiANWlxcHDiOU79EIhH8/f0xduxYZGVlaa3DGMPWrVvRq1cvuLq6wsHBAREREfjoo49QWlpa43ft2rULAwYMgIeHB8RiMfz8/DBs2DD89ddfOsUqk8nw1VdfoWvXrnBxcYFUKkVYWBimT5+O69evG3T8hNQXR3MHkYYsLi4O48aNw0cffYSgoCDIZDKcPHkScXFxCAwMxJUrVyCVStXllUolRo4ciR07dqBnz54YMmQIHBwccOTIEWzfvh1t27bFgQMH4O3tra7DGMP48eMRFxeHjh074tVXX4WPjw/u3buHXbt24ezZszh27BieffbZGuPMz89H//79cfbsWbz00kuIioqCk5MTrl27hvj4eGRnZ0Mul5v0tyJEK0ZIA7Z582YGgP39998a2+fMmcMAsISEBI3tS5cuZQDYrFmzqu1r9+7dTCAQsP79+2ts/+KLLxgANnPmTKZSqarV+/7779mpU6dqjXPgwIFMIBCwn376qdpnMpmMvfvuu7XW11VlZSWrqKgwyr6IbaAkQBq0mpLA77//zgCwpUuXqreVlZUxNzc3FhYWxiorK7Xub9y4cQwAO3HihLpO06ZNWevWrZlCoTAoxpMnTzIAbNKkSTqV7927N+vdu3e17WPGjGEtWrRQv7958yYDwL744gv21VdfseDgYCYQCNjJkyeZUChkH374YbV9XL16lQFgq1evVm97+PAhmzFjBmvWrBkTi8UsJCSEffrpp0ypVOp9rKThoT4B0ihlZGQAANzc3NTbjh49iocPH2LkyJEQiURa640ePRoA8Pvvv6vrPHjwACNHjoRQKDQolt27dwMARo0aZVD9umzevBmrV6/GG2+8geXLl8PX1xe9e/fGjh07qpVNSEiAUCjEa6+9BgAoKytD7969sW3bNowePRpff/01unfvjnnz5iE2NtYk8RLrov3/BEIamMLCQuTn50Mmk+HUqVNYvHgxJBIJXnrpJXWZ5ORkAECHDh1q3E/VZykpKRp/RkREGBybMfZRmzt37iAtLQ2enp7qbTExMZg8eTKuXLmC8PBw9faEhAT07t1b3eexYsUKpKen4/z582jZsiUAYPLkyfDz88MXX3yBd999FwEBASaJm1gHuhIgjUJUVBQ8PT0REBCAV199FY6Ojti9ezeaNWumLlNcXAwAcHZ2rnE/VZ8VFRVp/FlbnboYYx+1GTp0qEYCAIAhQ4ZAJBIhISFBve3KlStITk5GTEyMetvOnTvRs2dPuLm5IT8/X/2KioqCUqnE4cOHTRIzsR50JUAahbVr1yIsLAyFhYXYtGkTDh8+DIlEolGm6iRclQy0eTJRNGnSpM46dXl8H66urgbvpyZBQUHVtnl4eOC5557Djh07sGTJEgD8VYBIJMKQIUPU5VJTU3Hp0qVqSaRKbm6u0eMl1oWSAGkUunTpgs6dOwMABg8ejB49emDkyJG4du0anJycAABt2rQBAFy6dAmDBw/Wup9Lly4BANq2bQsAaN26NQDg8uXLNdapy+P76NmzZ53lOY4D0zJyW6lUai1vb2+vdfvw4cMxbtw4XLhwAZGRkdixYweee+45eHh4qMuoVCo8//zzmD17ttZ9hIWF1RkvadjodhBpdIRCIZYtW4a7d+9izZo16u09evSAq6srtm/fXuMJ9fvvvwcAdV9Cjx494Obmhh9//LHGOnUZNGgQAGDbtm06lXdzc0NBQUG17bdu3dLrewcPHgyxWIyEhARcuHAB169fx/DhwzXKhISEoKSkBFFRUVpfzZs31+s7ScNDSYA0Sn369EGXLl2wcuVKyGQyAICDgwNmzZqFa9euYf78+dXq7NmzB3FxcYiOjsYzzzyjrjNnzhykpKRgzpw5Wlvo27Ztw+nTp2uMpVu3bujfvz++/fZb/Prrr9U+l8vlmDVrlvp9SEgIrl69iry8PPW2ixcv4tixYzofPwC4uroiOjoaO3bsQHx8PMRicbWrmWHDhuHEiRPYt29ftfoFBQVQKBR6fSdpgCw9RpWQ+qjpOQHGGNu5cycDwNatW6feplAo2NChQxkA1qtXL7Zq1Sq2YcMGNnr0aCYQCFi7du1Ydna2xn6USiUbNWoUA8CeeuoptnTpUrZp0ya2dOlS1qVLFwaAHT9+vNY4c3NzWWRkJOM4jr388sts1apV7Ntvv2Vz5sxhLVq0YGKxWF02OTmZCQQC1rFjR7ZmzRq2cOFC5uXlxSIiImp8TqAm27ZtYwCYs7MzGzRoULXPS0tL2VNPPcVEIhGbOHEiW7duHfvyyy/ZmDFjmKOjI8vLy6v1uEjDR0mANGi1JQGlUslCQkJYSEiIxoNeSqWSbd68mXXv3p01adKESaVS1q5dO7Z48WJWUlJS43f99NNP7IUXXmBNmzZlIpGI+fr6spiYGHbo0CGdYi0rK2Nffvkle/rpp5mTkxMTi8WsZcuW7K233mJpaWkaZbdt28aCg4OZWCxmkZGRbN++fbU+LFaToqIiZm9vzwCwbdu2aS1TXFzM5s2bx0JDQ5lYLGYeHh7s2WefZV9++SWTy+U6HRtpuGjuIEIIsWHUJ0AIITaMkgAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJIAIYTYMJufO0ilUuHu3btwdnYGx3GWDocQQuqNMYbi4mL4+flBIKi9rW/zSeDu3bs0XzohpFG6ffu2xnTq2th8EqiaMvj27dvqKX8JIaQhKyoqQkBAgE5rWNh8Eqi6BdSkSRNKAoSQRkWXW9zUMUwIITaMkgAhhNgwSgKEEGLDbL5PQBeMMSgUCoNXliKkoRAKhRCJRGYdLs0YUJ71APKCEohdnWDv3xQ0Wvt/v8vO3ZCnpkHcMhT2r71skt+FkkAd5HI57t27h7KyMkuHQohZODg4wNfXF2Kx2KTfI8spROZ/9kD26z5I76ZDoFJAJRBB5hcC6eBoNJ88EFJvF5PGYI1kfyRBOX4ipNkZkACQ/G+7KgaQ+QRCuOlbSAc8Z7Tvs/n1BIqKiuDi4oLCwsJqo4NUKhVSU1MhFArh6ekJsVhMD5SRRosxBrlcjry8PCiVSrRs2bLOB40MlRl/DMXzl8IpLwOME6DCwQ1MKAKnVEBS9hAcU6HEMxDOn7yP5sO7myQGa1TQawCcjyTWWa64Z3+4Hv6jxs9rO689ia4EaiGXy6FSqRAQEAAHBwdLh0OIydnb28POzg63bt2CXC6HVCo1+ndkxh9D6bsfwKk4D8VeoYBE8zsqm/oCFTI456aj9N0PkImPbSIR6JoAAMD5SCIKeg2oNRHoyqo6hg8fPoxBgwbBz88PHMdpXZT7SYcOHcJTTz0FiUSC0NBQxMXFGT0uU7WGCLFGpvz3LsspRPH8pXAozkORf5tqCUBNIkWRfxs4FOeheP5SyHIKTRaTNZD9kaRzAqjifCQRsj+S6v3dVnV2Ky0tRYcOHbB27Vqdyt+8eRMDBw5E3759ceHCBcycORMTJ07Evn37TBwpIcQQmf/ZA6e8DBR7hYCrI9lwAgFKvELglHcLtzfqd4JsaJTjJxpUTzHhjXp/t1XdDhowYAAGDBigc/n169cjKCgIy5cvBwC0adMGR48exVdffYXo6GhThWkQxoDyckAuB8RiwN4eNAKC2BTGANmv+yDmBNWuAARKOQQqLaPvBICAKVCZ8BPY+F6N8v8ZxgBpdoZBde3v3QBj9TuXWFUS0NeJEycQFRWlsS06OhozZ86ssU5FRQUqKirU74uKikwVHgBAJgMuXACOHgXS0gCFAhCJgNBQoEcPIDISMMFt13qLi4vDzJkzUVBQAAD48MMP8euvv+LChQs6lddVRkYGgoKCcP78eURGRtYrZmLdyrMeQHo3HRUObhrbBUo5HMvya6zHiYRwufMP5PsOQuJR91w4DY386Kl6nYjLd+6Gw7CXDa7foJNAdnY2vL29NbZ5e3ujqKgI5eXlsLe3r1Zn2bJlWLx4sVniS0sDvvsOSE/nM7W7O+DoCFRWAn//DZw+DYSEABMm8EnB2PLy8rBw4ULs2bMHOTk5cHNzQ4cOHbBw4UJ07157R1tMTAxefPFFnb/ryfJ1JQ1ie+QFJRCoFFCINQdZCBh/BcDAQSW0q15RJIFQUQG5SAJJ06bmCNWs5A8L63UilqemoT7DVhp0EjDEvHnzEBsbq35fNduesaWlAatWAdnZQMuW/C2gx3l58beGUlP5cjNmGD8RDB06FHK5HFu2bEFwcDBycnKQlJSE+/fv11nX3t5eaxI1Vnlzq6yshJ2dlhMMMRuxqxNUAn4YqDYqoQhlDh7VttvJKsEJHdC0b1+gWeNLAsKs+8DGNQbXF7es34nDqjqG9eXj44OcnByNbTk5OWjSpEmNJySJRKKeMdRUM4fKZPwVQHY20LZt9QRQRSzmP8/O5svLZMaLoaCgAEeOHMFnn32Gvn37okWLFujSpQvmzZuHl19+WV1m8uTJ8Pb2hlQqRXh4OH7//XcA/O0dV1fXGvefnp6O4OBgTJ8+HYwxjfJxcXFYvHgxLl68CI7jwHGcXqO2rly5ggEDBsDJyQne3t4YNWoU8vMf3S5ITExEjx494OrqCnd3d7z00ktIT09Xf56RkQGO45CQkIDevXtDKpXihx9+wNixYzF48GB8+eWX8PX1hbu7O6ZNm4bKykrdf1hiMHv/ppD5hUBS9lCvepKyh5D5hcDev/ElAACwf83wWznGqN+gk0C3bt2QlKQ5RGr//v3o1q2bhSLiXbjA3wJq2bLuDhuO468A0tOBixeNF4OTkxOcnJzw66+/avSBVFGpVBgwYACOHTuGbdu2ITk5GZ9++imEQmGd+7506RJ69OiBkSNHYs2aNdUeoIuJicG7776Ldu3a4d69e7h37x5iYmJ0irugoAD9+vVDx44dcebMGSQmJiInJwfDhg1TlyktLUVsbCzOnDmDpKQkCAQCvPLKK1CpVBr7mjt3LmbMmIGUlBT1QIGDBw8iPT0dBw8exJYtWxAXF2eSYcWkOo4DpIOjwTEVUKFbi4erkIFjDPZDXmyUncIA/7vIfAINqlvuG1zv38WqbgeVlJQgLS1N/f7mzZu4cOECmjZtiubNm2PevHnIysrC999/DwCYMmUK1qxZg9mzZ2P8+PH466+/sGPHDuzZs8dShwDG+E5gjqv5CuBJEglf/sgRoEsX44waEolEiIuLw6RJk7B+/Xo89dRT6N27N4YPH4727dvjwIEDOH36NFJSUhAWFgYACA4OrnO/x48fx0svvYT58+fj3Xff1VrG3t4eTk5OEIlE8PHx0SvuNWvWoGPHjli6dKl626ZNmxAQEIDr168jLCwMQ4cO1aizadMmeHp6Ijk5GeHh4ertM2fOxJAhQzTKurm5Yc2aNRAKhWjdujUGDhyIpKQkTJo0Sa84iWGaTx6I9C0/wDk3HUX+bWodJspUKjjnpqPEswVCJvU3Y5TmJ9z0LfBiVN0FnyD6bkO9v9uqrgTOnDmDjh07omPHjgCA2NhYdOzYEQsXLgQA3Lt3D5mZmeryQUFB2LNnD/bv348OHTpg+fLl+Pbbby06PLS8nO8PcHfXr567O1+vvNx4sQwdOhR3797F7t270b9/f/WDdXFxcbhw4QKaNWumTgC6yMzMxPPPP4+FCxfWmABqM2XKFPUVipOTk9YyFy9exMGDBzXKtW7dGgDUt3xSU1MxYsQIBAcHo0mTJggMDFTH97jOnTtX23+7du00rnZ8fX2Rm5ur97EQw0i9XeD8yfsoc/ZEk6wUcDVcEXAVMjTJSuHLLX2/0c8hJB3wHIp76pfoSnr1N8ocQlZ1JdCnTx/UNpWRtsv2Pn364Pz58yaMSj9yOT8M1NFRv3oiEVBWxtc35gwVUqkUzz//PJ5//nksWLAAEydOxKJFizBr1iy99+Xp6Qk/Pz/8+OOPGD9+vN79KR999FGd31tSUoJBgwbhs88+q/aZr68vAGDQoEFo0aIFNm7cCD8/P6hUKoSHh0Mul2uUd9Tyl/Bk5zDHcdVuIxHTaj68OzLxMT93UG46BEwBTiQERBLYyarmDmIo8WyBJkvfR0BM458yAgBcD/+h89QRJb36w+X/6j9lBGBlVwKNgVjMn9D17Wusen7AxBM3om3btigtLUX79u1x584dXL9+Xee69vb2+P333yGVShEdHY3i4uIay4rF4mpTb3t5eSE0NFT90uapp57CP//8g8DAQI2yoaGhcHR0xP3793Ht2jV88MEHeO6559CmTRs8fKhfRyOxvObDuyPk+HbIZy9AcWB7cGAQKirAMRWKQp6CfPYChBzfbjMJoIrr4T9QufcAyn2135ot9w1G5d4DRksAgJVdCTQG9vZ8R+/ff/PDQHV1/z7w9NN8fWO4f/8+XnvtNYwfPx7t27eHs7Mzzpw5g88//xz/+te/0Lt3b/Tq1QtDhw7FihUrEBoaiqtXr4LjOPTvX/NlqaOjI/bs2aN+ujsxMVHrrZ3AwEB1n06zZs3g7OwMiUSiZY+apk2bho0bN2LEiBGYPXs2mjZtirS0NMTHx+Pbb7+Fm5sb3N3dsWHDBvj6+iIzMxNz586t129FLEPq7YKWH8SATegNeeJfkAslaNqvr82vJyAd8BxwN13regJOJvhd6ErAyDiOfxKYMf7Wji4qKvjyPXsabyoJJycndO3aFV999RV69eqF8PBwLFiwAJMmTcKaNfyY5J9//hlPP/00RowYgbZt22L27Nk6LZzj5OSEP/74A4wxDBw4EKWlpdXKDB06FP3790ffvn3h6emJH3/8Uae4/fz8cOzYMSiVSrzwwguIiIjAzJkz4erqCoFAAIFAgPj4eJw9exbh4eF455138MUXX+j34xCrwnGAxMMZzsHecGhm2wngcRwHOAx7Ga7zY+EwzDQLygC0nkCt827LZDLcvHkTQUFBek2pK5MBixfzHb1t29Z+YmcMSE7mrx4WLbLOKSSIbTH0373BsrP5S2c3N74FRepNn/UE6ErABKRSfioIHx/+BK9lmD4AfntyMl9u4kRKAIQQ86M+ARMJDeWngnhy7iCRiO8Evn+fvwoIDeUTQEiIpSMmhNgiSgImVHWL5+JF/kGwtDR+GKhIxHcC9+wJdOhAVwCEEMuhJGBiUinQtSv/JDCtJ0AIsTaUBMyE4/iHwGipYkKINaGOYUIIsWGUBAghxIZREiCEEBtGfQLmVDW7nDkmCSKEEB3QlYCpVVYCN24A+/YB27YB27fzf+7bx29vZKtacRyHX3/91eTf06dPH8ycOdPk32ONzPUbE9tAScCU8vKAX3/lXykpgEDAjxkVCPj3VZ/l5Znk62/fvo3x48fDz88PYrEYLVq0wIwZM3RaY9hQ9+7dw4ABA0y2/4aoallLfX344YeIjIystp1+Y2JMlARMJS8PSEwEbt8GAgP5R4Ld3QEXF/7PkBB+++3bfDkjJ4IbN26gc+fOSE1NxY8//oi0tDSsX78eSUlJ6NatGx48eFCv/de0Lq+Pj49Os4USw9FvTIyJkoApVFYChw7xJ/bQUOCJhUzU7Oz4z/Py+PJGvDU0bdo0iMVi/Pnnn+jduzeaN2+OAQMG4MCBA8jKysL8+fPVZbXdXnB1dVUv4lPTwu3aPL6vqno7duxAz549YW9vj6effhrXr1/H33//jc6dO8PJyQkDBgxA3mNJsKrlvHjxYnh6eqJJkyaYMmVKtUVjHldRUYFZs2bB398fjo6O6Nq1Kw4dOqT+PC4uDq6urvj999/RqlUrODg44NVXX0VZWRm2bNmCwMBAuLm54e2339aYSVXX/e7btw9t2rSBk5MT+vfvj3v37gHgW/NbtmzBb7/9Bo7jwHGcuv6cOXMQFhYGBwcHBAcHY8GCBerkGhcXh8WLF+PixYvqelV/H0/+fV2+fBn9+vWDvb093N3d8cYbb6CkpKTa7/nll1/C19cX7u7umDZtWo2JnNgW6hg2hdu3H10B6LLSfIsWQGYmcOcOEBRU769/8OAB9u3bh08++QT2TyxQ4OPjg9dffx0JCQn45ptvqi0SX5u5c+di+fLl6Nixo16zSy5atAgrV65E8+bNMX78eIwcORLOzs5YtWoVHBwcMGzYMCxcuBDr1q1T10lKSoJUKsWhQ4eQkZGBcePGwd3dHZ988onW75g+fTqSk5MRHx8PPz8/7Nq1C/3798fly5fRsmVLAEBZWRm+/vprxMfHo7i4GEOGDMErr7wCV1dX7N27Fzdu3MDQoUPRvXt3xMTE6LXfL7/8Elu3boVAIMC///1vzJo1Cz/88ANmzZqFlJQUFBUVYfPmzQCApk2bAgCcnZ0RFxcHPz8/XL58GZMmTYKzszNmz56NmJgYXLlyBYmJiThw4AAAwMWl+hKLpaWliI6ORrdu3fD3338jNzcXEydOxPTp0zVW4jt48CB8fX1x8OBBpKWlISYmBpGRkbS2MqEkYBKpqfx9/5quAJ4kFvPJ4Pp1oySB1NRUMMbQpk0brZ9XrcaVl5cHLz1WvtG2cLsuZs2apV73ecaMGRgxYgSSkpLQvTu/atSECROqLR0qFouxadMmODg4oF27dvjoo4/w3nvvYcmSJRA8sTh5ZmYmNm/ejMzMTPj5+am/MzExEZs3b1YvWl9ZWYl169Yh5H+z9b366qvYunUrcnJy4OTkhLZt26Jv3744ePAgYmJi9Nrv+vXr1fudPn06PvroIwD82gv29vaoqKiAj4+PRtwffPCB+r8DAwMxa9YsxMfHY/bs2bC3t4eTkxNEIlG1eo/bvn07ZDIZvv/+e/VymmvWrFEv0ent7Q0AcHNzw5o1ayAUCtG6dWsMHDgQSUlJlAQIJQGjk8uBrCzA1VW/eq6ufL2qyYWMwNhLRWhbuF0X7du3V/931UkpIiJCY9uTi7136NABDo/NsdGtWzeUlJTg9u3baNGihUbZy5cvQ6lUIiwsTGN7RUUF3N3d1e8dHBzUJ+qq7w0MDNRYGe3xWAzdr66L1yckJODrr79Geno6SkpKoFAo9F63OSUlBR06dNBYT7l79+5QqVS4du2a+vdu164dhEKhRoyXL1/W67tI40RJwNgUCkCl0v9ELhLxq9EoFPVOAqGhoeA4DikpKXjllVeqfZ6SkgI3Nzd4enoC4O8xP5kwtN0v1rZwuy4eX9y96vbTk9vqs9h7SUkJhEIhzp49q3GiA6Bxgte2yHxtC8/XZ791JeATJ07g9ddfx+LFixEdHQ0XFxfEx8dj+fLldRytYWo7TmLbKAkYm0jE3wpSKPSrp1Dw9UT1/ytxd3fH888/j2+++QbvvPOORr9AdnY2fvjhB4wePVp9Qvb09FR3ZAL87aSysrJ6x1EfFy9eRHl5uTr2kydPwsnJCQEBAdXKduzYEUqlErm5uejZs6fRYjDWfsVicbVlO48fP44WLVpodNDfunWrznpPatOmDeLi4lBaWqpO0seOHYNAIECrVq0MjpnYDhodZGxiMeDvDxQU6FevoICvZ6RbQWvWrEFFRQWio6Nx+PBh3L59G4mJiXj++efh7++v0cHar18/rFmzBufPn8eZM2cwZcqUai1Hc5PL5ZgwYQKSk5Oxd+9eLFq0CNOnT6/WHwAAYWFheP311zF69Gj88ssvuHnzJk6fPo1ly5Zhz549BsdgrP0GBgbi0qVLuHbtGvLz81FZWYmWLVsiMzMT8fHxSE9Px9dff41du3ZVq3fz5k1cuHAB+fn5qNCyRN3rr78OqVSKMWPG4MqVKzh48CDeeustjBo1Sn0riJDaUBIwhZYt+VtCug7Bk8v5ZcaeuPdcvxBa4syZMwgODsawYcMQEhKCN954A3379sWJEyfUI1QAYPny5QgICEDPnj0xcuRIzJo1S+N+vCU899xzaNmyJXr16oWYmBi8/PLL+PDDD2ssv3nzZowePRrvvvsuWrVqhcGDB+Pvv/9G8+bN6xWHMfY7adIktGrVCp07d4anpyeOHTuGl19+Ge+88w6mT5+OyMhIHD9+HAsWLNCoN3ToUPTv3x99+/aFp6cnfvzxx2r7dnBwwL59+/DgwQM8/fTTePXVV/Hcc89hzZo19TpuYjtooXkTLDSPykr+SeDbt/nnAOpaaT4tDQgIAAYP1n1EUSM2duxYFBQU0NQIFkILzTd8tNC8pdnZAX36AJ6e/Am+poec5HL+c09PvjwlAEKImVHHsKl4egL9+/NPAmdm8lcDrq6PVpovKOCvAgICHiUMQggxM0oCpuTpyd/iuXOHfxAsK4sfBioQAG3a8H0AzZrRFcATnnxwjBBiOpQETM3Ojn8KOCiI1hMghFgdSgLmJBbTyZ8QYlWoY5gQQmwYXQkQQizviy/4vrOuXQEtz0MQ06ErAUKIZUydyo+a8/UFjhwBbt4E4uP5bRzHf05MjpIAIcT87OyA9etrL7N+PY2cMwNKAsSoaKF502vwC83b2ek+waJCQYnAxCgJNGK00Lx1oIXmHzN1qmEz7NKtIZOhJNBI0ULzjVeD/o3rugVk7HqkTpQEDKFUWualB1ponhaaf/L3pIXmiTY0RFRfSiWwd69lvvvFF4EnVrjShhaap4XmrXKh+REj6l+fho8aHSWBRogWmqeF5q1yofkrVyxbn2hFSUBfQiHfIrfUd+uBFpqnheataqH58PD6ncjDw40XC1GjJGAIPU/G5kYLzT9CC83XHKPZF5r/8Uf+YbD61CdGR0mgEaKF5o2DFpontoBGBzVStNA8LTRvlQvNT5li3nqkTpQEGilaaJ4WmrdK69bx62noQyTi6xHTYFZmzZo1rEWLFkwikbAuXbqwU6dO1Vr+q6++YmFhYUwqlbJmzZqxmTNnsvLycp2/r7CwkAFghYWF1T4rLy9nycnJeu2P1N+YMWPYv/71L0uHYbPM8u9eJGKMX2C19pdIZLoYGrHazmtPsqorgYSEBMTGxmLRokU4d+4cOnTogOjo6BpHWmzfvh1z587FokWLkJKSgu+++w4JCQl4//33zRw5IUQvlZV13+KZMoUvR0zKqpLAihUrMGnSJIwbNw5t27bF+vXr4eDggE2bNmktf/z4cXTv3h0jR45EYGAgXnjhBYwYMQKnT582c+SEEL2tW8e39+/dA3r25JdgHT780XUA3QIyC6sZHSSXy3H27FnMmzdPvU0gECAqKgonTpzQWufZZ5/Ftm3bcPr0aXTp0gU3btzA3r17MWrUqBq/p6KiQqODraioyHgHQYyCFpq3Qe+9B7i5AT16WDoSm2M1SSA/Px9KpbLaiAZvb29cvXpVa52RI0ciPz8fPXr0AGMMCoUCU6ZMqfV20LJly7B48WKjxk4IIQ2VVd0O0tehQ4ewdOlSfPPNNzh37hx++eUX7NmzB0uWLKmxzrx581BYWKh+3b59u87vYUZ+8pYQa0b/3m2L1VwJeHh4QCgUIicnR2N7Tk5OjXOnLFiwAKNGjcLEiRMB8FMRlJaW4o033sD8+fO1jimXSCQ6T8NbNVa+rKys2kRshDRWVQ8KWvpZEWIeVpMExGIxOnXqhKSkJPUCHCqVCklJSZg+fbrWOmVlZdVO9FWP9xujNSMUCuHq6qoeneTg4KDXrJuENCSMMZSVlSE3Nxeurq7VpsogjZPVJAEAiI2NxZgxY9C5c2d06dIFK1euRGlpKcaNGwcAGD16NPz9/bFs2TIAwKBBg7BixQp07NgRXbt2RVpaGhYsWIBBgwYZ7R9w1VWILhOCEdIYuLq61jpzKWlcrCoJxMTEIC8vDwsXLkR2djYiIyORmJio7izOzMzUaPl/8MEH4DgOH3zwAbKysuDp6YlBgwbVOOe8ITiOg6+vL7y8vGgRDtLo2dnZ0RWAjeGYjfcCFRUVwcXFBYWFhXpP40sIMYLsbODvv2mIqBHpc15r0KODCCGE1A8lAUIIsWGUBAghxIZREiCEEBtWr9FBFRUVOHfuHHJzc9G9e3d4eHgYKy5CCCFmYPCVwNdffw1fX1/06NEDQ4YMwaVLlwDwcwB5eHjUOPMnIYQQ62FQEti8eTNmzpyJ/v3747vvvtN4OtfDwwP9+vVDfH0WlCaEEGIWBiWB5cuX41//+he2b9+OQYMGVfu8U6dO+Oeff+odHCGEENMyKAmkpaVhwIABNX7etGlT3L9/3+CgCCGEmIdBScDV1RX5+fk1fp6cnExzjxBCSANgUBJ48cUXsWHDBhQUFFT77J9//sHGjRvx8ssv1zc2QgghJmZQEvj444+hVCoRHh6unsRty5Yt+Pe//43OnTvDy8sLCxcuNHashBBCjMygJODn54ezZ8+if//+SEhIAGMMW7duxX//+1+MGDECJ0+epGcGCCGkATDKLKJ5eXlQqVTw9PTUupqXNaNZRAmxMJpF1OhMPovo+PHjcerUKfV7T09PeHt7qxPA6dOnMX78eEN2TQghxIwMSgJxcXFIT0+v8fObN29iy5YtBgdFCCHEPExy7+bu3bu0MDshhDQAOk8g99tvv+G3335Tv9+wYQMOHDhQrVxBQQEOHDiAp59+2jgREkIIMRmdk0BycjJ27twJgF9399SpUzh79qxGGY7j4OjoiF69emHFihXGjZQQQojRGTQ6SCAQYNu2bRg5cqQpYjIrGh1EiIXR6CCj0+e8ZtB6AiqVyqDACCGEWJeGNaifEEKIURmcBP744w88//zzcHd3h0gkglAorPYihPAYA8pyS1CQno+y3BLU/xHNxoMxoOLoKRRviUfZjt3025iZQbeDfv75ZwwbNgzt2rXD8OHDsW7dOowcORKMMfz2229o2bIlBg8ebORQCWl4ZA/KkB5/Gvd/OwZBRjoEKgVUAhFUgSFw/1d3hAzvAmlTB0uHaRGyP5KgHD8R0uwMiPC/k9G3a6GKAWQ+gRBu+hbSAc9ZOMrGz6CO4c6dO8POzg5Hjx7Fw4cP4eXlhQMHDqBfv37IyMjAM888g88//xyjR482RcxGRR3DxFQyElOQuWAjpFlpYJwQ8ibugEgEKBQQF90Hx5SQ+Yei+ZJJCOzfxtLhmlVBrwFwPpJYZ7ninv3hevgPM0TUuJh82ojk5GQMHz4cQqEQIhF/MVFZWQkACAwMxJtvvonPPvvMkF0T0ihkJKYg892VkGaloyygFWQh4VB5+kLl5gmVpy9kIeEoC2gFaVY6Mt9diYzEFEuHbDa6JgAAcD6SiIJeNS9gRerPoCTg4OAAsVgMgF9gRiKR4N69e+rPvb29cfPmTeNESEgDI3tQxl8BPMxGWXA4OIlUazlOIkVZcDikD7ORuWAjZA/KzByp+cn+SNI5AVRxPpII2R9JJoqIGJQEWrVqheTkZPX7yMhIbN26FQqFAjKZDNu3b0fz5s2NFiQhDUl6/GlIs9JQFhAGro5ZdTmBAOXNwiDJuoEbO86YKULLUY6faFA9xYQ3jBwJqWJQEnjllVfw22+/oaKiAgAwf/58HDp0CK6urvD09MSRI0cwd+5cowZKSEPAGHD/t2NgnLDGK4BqpFKA45C/60ijHhnDGCDNzjCorv29G436t7Eko6wnAABHjhzBL7/8AqFQiIEDB6Jv377G2K3JUccwMaay3BKc6/k2mEAAlaevxmfS8gdwKL8PaPk/TlD0AFAxhHz2BqSujXPyRdmRU7D7/GOD61ck/AaHYbRsrS5M/sSwNj179kTPnj3V74uLi+Hs7Gys3RPSIMiLZRCoFFCKqw/7FMtLwdXU5hIIIFBWQFEmB1x0vIJoYBR378KuHvXlqWmwzcG0pmW0JFAlNzcXK1euxLp16/Dw4UNj754QqyZ2lkIl4IeB1qTE0RtysaPGNoHiHjixCi36RwNeTqYO0yIEhRXAtk0G1xe3DDViNKSKXkkgNzcX33//PdLT0+Hm5oahQ4eiU6dOAICsrCx88skniIuLg0wmQ58+fUwRLyFWzd7TCarAEEiunIHsidtBVZhAACbQ/F/PrqQQFeGdYR/gAXDmiNT87F9/FapR9aj/Gt0KMgWdk8DVq1fRq1cv3L9/H1XdCJ9//jm2bdsGjuMwceJEyGQyDB06FO+99546ORBiSzgOcP9Xd5RePgVWIdOtc1gmAxiDxys9wTXSBADwv43MJ9CgzuFy32A4NeLfxpJ0Hh20YMEClJSU4JtvvsGVK1fw3//+F8HBwZg5cybGjh2LAQMG4Nq1a4iPj6cEQGxayPAukPmHwuH2dbA6ZtxlKhXs71xHhX8wgod1NlOEliPc9K1B9UTfbTByJKSKzlcChw8fxtSpUzF58mQAQNu2bSESiTBgwACMGTMGmzdvNlmQhDQk0qYOaL5kEjLfXQmHG1dQ3iyMHwb6JJkMDneuQ+bmgxYfT7KJOYSkA55DQc/+ej0wVtKrP1xoDiGT0TkJ3L9/H+3bt9fY1qFDBwD8cwOEkEf4uYBm8k8O30kFOA4CkQIcKsHJhZCW3QIYg8w/BC0+noQW0bYzd5Dr4T90njqipFd/uPwfzR1kSjrfDlKpVLCz0xzgVfXeyalxjmYgpD4C+7dBl30fw3lhLCrCOwNMBUFlBTiVChXhneG8MBZd9n1sUwmgiuvhP1C59wDKfYO1fl7uG4zKvQcoAZiBXqODzpw5A+ljl7XFxcXgOA5Hjx5FQUFBtfJDhgypd4CENGTSpg5oO6UX2OReqNizH4q7uRB06wr78NBG3QmsC+mA54C76WAMKN+5G/LUNIhbhsL+tZepE9iMdH5iWFDHHCjVdsxxUCqVBgVlTvTEMDGbY8eABw+Azp0BX+3DRwkxBpM8MXzw4MF6B0YIIcS66JwEevfubco4CCGEWAAtNE8IITaMkgAhhNgwSgKEEGLDrC4JrF27FoGBgZBKpejatStOnz5da/mCggJMmzYNvr6+kEgkCAsLw969e80ULSGENGxGn0q6PhISEhAbG4v169eja9euWLlyJaKjo3Ht2jV4eXlVKy+Xy/H888/Dy8sLP/30E/z9/XHr1i24urqaP3hCCGmArCoJrFixApMmTcK4ceMAAOvXr8eePXuwadMmrctVbtq0CQ8ePMDx48fVTy8HBgaaM2RCCGnQDL4dlJmZiSlTpqBVq1Zo2rQpDh8+DADIz8/H22+/jfPnz+u1P7lcjrNnzyIqKupRcAIBoqKicOLECa11du/ejW7dumHatGnw9vZGeHg4li5dWutDahUVFSgqKtJ4EUKIrTIoCSQnJ6Njx45ISEhAUFAQCgsLofjfSkoeHh44evQo1qxZo9c+8/PzoVQq4e3trbHd29sb2dnZWuvcuHEDP/30E5RKJfbu3YsFCxZg+fLl+PjjmtcxXbZsGVxcXNSvgIAAveIkhJDGxKAkMHv2bLi6uuL69evYtm0bnpx5YuDAgThy5IhRAqyNSqWCl5cXNmzYgE6dOiEmJgbz58/H+vXra6wzb948FBYWql+3b982eZyEEGKtDOoTOHz4MBYuXAhPT0/cv3+/2ufNmzdHVlaWXvv08PCAUChETk6OxvacnBz4+PhorePr6ws7OzsIhUL1tjZt2iA7OxtyuRxisbhaHYlEAolEoldshBDSWBl0JaBSqeDgUPMCGHl5eXqfaMViMTp16oSkpCSN70lKSkK3bt201unevTvS0tKgemz1puvXr8PX11drAiCEEKLJoCTw1FNPYc+ePVo/UygUiI+PxzPPPKP3fmNjY7Fx40Zs2bIFKSkpmDp1KkpLS9WjhUaPHo158+apy0+dOhUPHjzAjBkzcP36dezZswdLly7FtGnTDDksQgixOQbdDpo3bx5eeuklTJ06FcOHDwfA37Y5cOAAli5dipSUFL07hgEgJiYGeXl5WLhwIbKzsxEZGYnExER1Z3FmZqbGlNYBAQHYt28f3nnnHbRv3x7+/v6YMWMG5syZY8hhEUKIzdF5PYEnbd26FTNmzEBhYSEYY+A4DowxNGnSBOvWrcOIESOMHatJ0HoCxGxoPQFiJiZZT+BJo0aNwpAhQ7B//36kpqZCpVIhJCQE0dHRcHZ2NnS3hBBCzMigJFDV8nd0dMTgwYONHBIhhBBzMahjuOre+7Fjx4wdDyGEEDMyKAn07t0bmzZtQq9evdC8eXPMmjWrztk+CSGEWB+DksCPP/6I3NxcxMfHo0uXLli3bh26deuGkJAQvP/++7hw4YKRwySEEGIKBk8gZ29vj9deew0//fQTcnNzsW3bNkREROCrr75Cp06d0Lp1a2PGSQghxASMsqiMo6MjRowYgW3btuGLL76Ak5MTUlNTjbFrQgghJlTv9QTKysqwe/du7NixA4mJiaioqEBISAjefvttY8RHCCHEhAxKAjKZDHv27EFCQgL27t2LsrIyBAYG4u2330ZMTAw6duxo7DgJIYSYgEFJwNPTE2VlZfDz88Mbb7yBmJgYdO3a1dixEUIIMTGDksDYsWMRExODHj16GDseQgghZmRQEli9erWx4yCEEGIBOiWBqvWDe/XqpfG+LlXlCSGEWCedkkCfPn3AcRzKy8shFovV72tSNbdQbQu+E2Jz7twBMjMBd3eaRZRYDZ2SwMGDBwFAvVpX1XtCSB0yM4HPPgP++APIzgaUSkAkAry9gQEDgDlzgObNLR0lsWEGryfQWNB6AsRk1q0DFi4Eior490IhwHH8q7KS39akCfDRR8DUqZaLkzQ6+pzXDHpiuF+/fhprAT/p4MGD6NevnyG7JqRxWLcOmDePTwBuboCXF+DkBNjb8yd+Ly9+e1ERX27dOktHTGyUQUng0KFDyMnJqfHz3Nxc/N///Z/BQRHSoGVm8lcA5eX8/X87O+3l7Oz4z8vL+fKZmeaNkxDUY+6g2jqG09LSaHUxYrs+++zRFYBQWHtZoRBwdeXLL19ulvAIeZzOzwls2bIFW7ZsUb//+OOPsXHjxmrlCgoKcOnSJbz44ovGiZCQhuaPP/g/a7oCeNL/Blzgv/8FVq0yTUyE1EDnJFBWVoa8vDz1++LiYggEmhcSVUtOTpkyBQsXLjRelIQ0FGlpQF4eIJHoV08i4eulpQGhoaaJjRAtdE4CU6dOxdT/jWAICgrCqlWr8PLLL5ssMEIapLw8QKXS/SqgikDAjxjKy6MkQMzKoGkjbt68aew4CGkcPD35E7pKpV89lYqv5+lpmrgIqYFOSSDzf6MWmv/voZZMHUcxNKeHYIitCQ3lT+RZWfrVq6gA/P3pKoCYnU5JIDAwUGPaiKr3daFpI4hNGjAA+PZb/vaOLreF5HL+z0GDTBsXIVrolAQ2bdoEjuNg979/0FXvCSFazJkD7NgBPHzIPwdQ2zBRpRIoKOAfIHv3XbOFSEgVmjaCpo0gplD1xHB5Of8cgFjM3/JRqfj/Fgr5K4CCAv4p4mXLaOoIYjT6nNfqvcbw4+RyOSorK+Ho6GjM3RLS8FSd0Bcu5E/0wKO5g2QymjuIWA2DnhiOj4/HO++8o7Ft8eLFcHJygqurK1555RWUlJQYJUBCGqypU4GzZ4EpU/hOXwBQKADG+PdTpvCfUwIgFmTQ7aCnn34aHTt2xIYNGwAAx48fR48ePTBw4EC0adMGq1evxsyZM7Fs2TKjB2xsdDuImE1CAj8/UPfuwLPPWjoa0oiZ/HZQeno6xowZo36/fft2+Pj4YNeuXRCJRFCpVPj5558bRBIgxGyaNQMcHICgIEtHQoiaQbeDKioqIJVK1e///PNPDBgwACIRn1Patm2LO3fuGCdCQgghJmNQEggKCsKBAwcAAGfOnEFaWhr69++v/jwnJwdOTk7GiZAQQojJGHQ7aPLkyZgxYwaSk5Nx584dNGvWDC+99JL682PHjqFdu3ZGC5IQQohpGJQE3nrrLUilUuzduxedOnXCnDlzYG9vDwB48OABsrOzMWXKFKMGSgghxPjoYTEaHUTM5dgx4MEDoHNnwNfX0tGQRsysD4slJyfj1q1bAIAWLVqgbdu29d0lIYQQMzE4Cfz222+IjY1FRkaGxvagoCCsWLGC1hoghJAGwKDRQXv37sXQoUMBAEuXLsWuXbuwa9cuLF26FIwxDBkyBImJiUYNlBBCiPEZ1CfQrVs3VFRU4MiRI9XmCSotLUWPHj0glUpx4sQJowVqKtQnQMyG+gSImehzXjPoSuDSpUsYM2aM1oniHB0dMXbsWFy6dMmQXRNCCDEjg5KAVCrFgwcPavz8wYMHGk8UE0IIsU4GJYF+/fph1apVWm/3nDp1Cl9//TWioqLqHRwhhBDTMmh00Oeff45u3bqhR48e6NKlC1q1agUAuHbtGk6fPg0vLy989tlnRg2UEEKI8Rk8d9ClS5fw9ttv4+HDh0hISEBCQgIePnyIGTNm4OLFiwgMDDRyqIQQQoxN7ySgVCqRnZ2NJk2a4KuvvsLVq1dRXl6O8vJyXL16FStWrICXl1e9glq7di0CAwMhlUrRtWtXnD59Wqd68fHx4DgOgwcPrtf3E0KIrdA5CTDG8P7778PNzQ3+/v5o0qQJXnnllVo7iA2RkJCA2NhYLFq0COfOnUOHDh0QHR2N3NzcWutlZGRg1qxZ6Nmzp1HjIYSQxkznJBAXF4dPP/0Urq6uGDp0KCIiIvDbb79h3LhxRg1oxYoVmDRpEsaNG4e2bdti/fr1cHBwwKZNm2qso1Qq8frrr2Px4sUIDg42ajyEENKY6ZwE1q1bh44dO+LatWvYsWMHzp49i7feegt79uxBfn6+UYKRy+U4e/asxsgigUCAqKioWh88++ijj+Dl5YUJEybU+R0VFRUoKirSeBFCiK3SOQmkp6dj9OjR6imjAeDNN9+ESqVCamqqUYLJz8+HUqmEt7e3xnZvb29kZ2drrXP06FF899132Lhxo07fsWzZMri4uKhfAQEB9Y6bEEIaKp2TwMOHD+Hp6amxzcPDAwAgk8mMG5WOiouLMWrUKGzcuFEdS13mzZuHwsJC9ev27dsmjpIQQqyXXs8JcBxnqjgA8ElFKBQiJydHY3tOTg58fHyqlU9PT0dGRgYGDRqk3qZSqQAAIpEI165dQ0hIiEYdiUQCiURigugJIaTh0SsJzJ07F8uWLVO/VyqVAICJEydWm0eI4zhcvHhRr2DEYjE6deqEpKQk9TBPlUqFpKQkTJ8+vVr51q1b4/LlyxrbPvjgAxQXF2PVqlV0q4cQQuqgcxLo1auX1iuB+j4T8KTY2FiMGTMGnTt3RpcuXbBy5UqUlpaqRyGNHj0a/v7+WLZsGaRSKcLDwzXqu7q6AkC17YQQQqrTOQkcOnTIhGE8EhMTg7y8PCxcuBDZ2dmIjIxEYmKiurM4MzMTAoFBDzoTQgh5Aq0xTOsJEHOh9QSImZh8PQFCCCGNAyUBQgixYZQECCHEhlESIEbFGFBWBhQU8H/ado/TI4wBsh8SULLia5St3UC/C7Ea1DFMHcNGIZMBFy4AR48CaWmAQgGIREBoKNCjBxAZCdjiiqOydd+BmzENosqKap8p7CRgq9ZCOrXuOa8I0Yc+57V6JYGsrCwcPnwYubm5GDp0KJo1awalUonCwkK4uLhAKBQaumuzoSRQf2lpwHffAenpAMcB7u6AnR1QWQncv8+3gkNCgAkT+KRgK0qaBcM+62ad5cr9g+B054YZIiK2wuSjgxhjiI2NRVBQEF5//XXExsbi+vXrAICSkhIEBgZi9erVhuyaNDBpacCqVfyfoaFAmzaAlxfg5sb/2aYNv/3xcrZA1wQAAPZZN1HSjKZAJ5ZhUBL44osvsGrVKsyaNQv79+/H4xcTLi4uGDJkCH7++WejBUmsk0zGXwFkZwNt2wJisfZyYjH/eXY2X95C8w2ajWzddzongCr2WTchW/ediSIipGYGJYGNGzdi9OjRWLp0KSIjI6t93r59e/WVAWm8LlzgbwG1bMnfBqoNx/FXBOnpgJ5TSjU43IxphtWb+ZaRIyGkbgYlgdu3b+PZZ5+t8XNHR0darKWRY4zvBOa4mq8AniSR8OWPHGm8o4YYg9ZOYF2I5OWN9nch1sugJODl5VXrPPxnz55F8+bNDQ6KWL/ycv7+vru7fvXc3fl65eWmicvSyr9YadH6hOjLoCQwZMgQrF+/HjduPBrRUDXD6J9//om4uDi89tprxomQWCW5nB8GamenXz2RiK8nl5smLkuTX7lcdyET1idEXwYlgcWLF8PX1xeRkZEYPXo0OI7DZ599hh49emDAgAFo37493n//fWPHSqyIWMyf0Csr9atX9fyArreQGhpxeIRF6xOiL4OSgIuLC06ePInZs2cjKysLUqkU//d//4eCggIsWrQIR44cgYODg7FjJVbE3p7v6L1/X7969+/z9R5bqrpRsX9vpkXrE6IvvVYWe5y9vT0++OADfPDBB8aMhzQQHMc/CXz6NH9rR5eWfUUF33Has2fdo4kaKo4DFCIxRAr973cpxPaQNNLfhVgvmjuIGCwykn8SODW17tE+jPEdwiEhQIcOZgnP/MrLgVOnwE18w6DqbCU9YEnMz6ArgfHjx9dZhuM4fPcdPfzSmEml/FQQq1YBycn8bR6JpHq5igo+Afj4ABMnNtI5hDIzgX/+ARQKiF/qj9Jdv0Kac0fn6uXNguBEcwgRCzAoCfz111/V1htWKpW4d+8elEolPD09qy08Txqn0FBgxozqcwdVjQKqmjsoNJRPACEhlo7YyMrL+aff8vL4925uQGQkHLNv6z53ULMgON2muYOIZRh1FtHKykr85z//wcqVK7F//34EBQUZa9cmQxPIGYdMxp8LjxypPotoz578LaBGdwVw6xZ/CaRQAAIB0Lo1EBys0eEhW/cduJlvQSSv/mCEQmwPtnI1zSJKjM5ss4jW5M0338StW7ewZ88eY+/a6CgJGBdjfOO4qrPY3r4RdgKXlfEZLz+ff9+0Kd9BUsvVL2P8g2DyK5chDo+A/XszG9/vQqyGPuc1g0cH1aZDhw7YunWrKXZNrBzHAQ4O/KvRYexR61+pBIRCvvUfFFRnpuM4wGH2TDTGn4U0bCZJAvv376fnBEjjUlrKt/6rHozQofVPSENgUBL46KOPtG4vKCjA4cOHce7cOcydO7degRFiFRgDMjKAlJRHrf82bYDAwEZ4n4vYIoP6BAQC7Y8XuLm5ISQkBBMnTsSkSZOqjSCyRtQnQGpUWsrPl/3gAf/e3Z1v/dNVLrFyJu8TUKlUBgVGSIPAGHDzJnD16qPWf9u2QIsW1PonjY7eTwyXl5cjNjYW//3vf00RDyGWVVICHDvGP/ilVAIeHkCfPnT7hzRael8J2Nvb4z//+Q/atm1ringIsQzGgBs3+Na/SsU/5FDV+iekETPodlCnTp1w5coVY8dCiGWUlPD3/h8+5N97evJPtzXWqU4JeYxBSWDlypV48cUXER4ejrFjx0IkMslIU0JMizF+rotr1x61/tu1A2hVPGJDdB4ddPjwYbRp0waenp6IiIjA/fv3kZOTA4lEAn9/f9g/0WriOA4XG8CK4jQ6yEYVF/Ot/4IC/r2XVyOd24LYIpOMDurbty+2bduGESNGwN3dHR4eHmjVqlW9gyXErFQqvvV//Tr/33Z2fOs/IMDSkRFiETonAcYYqi4aDh06ZKp4CDGdoiK+9V9YyL/39gbat6fWP7FpdDOfNH4qFT+1aWrqo9Z/eDjQrJmlIyPE4vRKAg3hCWBCNBQVAefP838C/Mo2ERHU+ifkf3TuGBYIBHolAY7joFAoDA7MXKhjuJFSqfiWf9Xal2Ix3/r397d0ZISYnMmmjYiKikJYWFi9giPE5AoL+Xv/Va1/X1++9a9t7UtCbJxeSWDMmDEYOXKkqWIhpH5UKn7UT1rao9Z/RATg52fpyAixWtQxTBqHggK+9V9czL/38+MTgFhsyagIsXqUBEjDplLxT/ymp/Otf4mEP/n7+lo6MkIaBEoCpOF6+JBv/ZeU8O/9/fnOX2r9E6IznZMArSFArIZSybf+b9x41Ppv354f/kkI0QtdCZCG5cEDvvVfWsq/b9aMb/3b2Vk0LEIaKkoCpGFQKvm5/m/c4N9LpXzr39vbsnER0sBREiDW7/594OLFR63/gAB+0jdq/RNSb5QEiPVSKoGUFH69X4Bv/XfowE/7TAgxCr3XGDaHtWvXIjAwEFKpFF27dsXp06drLLtx40b07NkTbm5ucHNzQ1RUVK3lSQNx/z5w6NCjBNC8Ob/WLyUAQozK6pJAQkICYmNjsWjRIpw7dw4dOnRAdHQ0cnNztZY/dOgQRowYgYMHD+LEiRMICAjACy+8gKysLDNHToxCoQAuXwaOHwfKyvglHp95hr8CoNs/hBidzhPImUvXrl3x9NNPY82aNQD4oakBAQF46623MHfu3DrrK5VKuLm5Yc2aNRg9enSd5WkCOSuSn8+P/Ckv59+3aMEv9k7LlxKiF5NNIGdqcrkcZ8+exbx589TbBAIBoqKicOLECZ32UVZWhsrKSjRt2lTr5xUVFaioqFC/L6qaZIxYjkIBJCcDt27x7+3tgchIwMPDomERYgus6nZQfn4+lEolvJ8Y9uft7Y3s7Gyd9jFnzhz4+fkhKipK6+fLli2Di4uL+hVAywpaVl4ef++/KgEEBvL3/ikBEGIWVpUE6uvTTz9FfHw8du3aBWkNi4bMmzcPhYWF6tft27fNHCUBAFRW8sM+T57kb/84OADPPsvP+0O3fwgxG6v6v83DwwNCoRA5OTka23NycuBTx5QAX375JT799FMcOHAA7du3r7GcRCKBhOaVt6zcXD4ByGT8+6AgoE0bQCi0bFyE2CCruhIQi8Xo1KkTkpKS1NtUKhWSkpLQrVu3Gut9/vnnWLJkCRITE9G5c2dzhEoMUVnJd/yeOsUnAEdHoHt3ftoHSgCEWIRVXQkAQGxsLMaMGYPOnTujS5cuWLlyJUpLSzFu3DgAwOjRo+Hv749ly5YBAD777DMsXLgQ27dvR2BgoLrvwMnJCU5OThY7DvKEnBzg0qVHrf/gYKB1azr5E2JhVpcEYmJikJeXh4ULFyI7OxuRkZFITExUdxZnZmZCIHh0AbNu3TrI5XK8+uqrGvtZtGgRPvzwQ3OGTrSprASuXAHu3OHfOzryI39qGL1FCDEvq3tOwNzoOQETys7mW/8VFQDH8a3/Vq2o9U+IiTXY5wRIIyGX863/qqe2nZz41r+bm0XDIoRUR0mAGNe9e/y0D1Wt/5AQvvUvsKoxCISQ/6EkQIxDLudP/nfv8u+dnfnWv6urJaMihNSBkgCpv7t3+QQgl/Ot/9BQICyMWv+ENACUBIjhKir4k/+9e/z7Jk341r+Li0XDIoTojpIAMUxWFt/5W9X6b9mSf1Hrn5AGhZIA0U9FBT/ss2pCvyZNgI4d+T8JIQ0OJQGiuzt3+NZ/ZSXf4m/Zkr//T61/QhosSgKkbjIZ3/qvmtjPxYW/90+tf0IaPEoCpHa3bwP//POo9R8Wxo/9p9Y/IY0CJQGinUzGT/dctbazqyvf+nd2tmRUhBAjoyRAqsvM5Fv/CgXf4m/Vim/9c5ylIyOEGBklAfJIeTnf+s/L49+7ufGtf5qSm5BGi5IA4d26xS/2XtX6b92an/WTWv+ENGqUBGxdWRnf+s/P5983bcq3/h0dLRoWIcQ8KAnYKsYetf6VSn6O/9at+fV+qfVPiM2gJGCLysr4tX7v3+ffU+ufEJtFScCWMAZkZAApKY9a/23aAIGB1PonxEZRErAVpaX8vf+q1r+7O9/6d3CwaFiEEMuiJNDYMQbcvAlcvfqo9d+2LdCiBbX+CSGUBBq1khK+9f/gAf/ewwPo0IFa/4QQNUoCjRFjwI0bfOtfpQJEoketf0IIeQwlgcampIQf+fPwIf/e05Nv/dvbWzQsQoh1oiTQWDAGpKcD1649av23awc0b27pyAghVoySQGNQXMy3/gsK+PdeXnzrXyq1ZFSEkAaAkkBDxhiQlgZcv863/u3s+NZ/QIClIyOENBCUBBqqoiK+9V9YyL/39gbat6fWPyFEL5QEGhqVim/9p6Y+av2HhwPNmlk6MkJIA0RJoCEpKgLOn+f/BAAfHyAiglr/hBCDURJoCFQqvuWfmsr3A4jFfOvf39/SkRFCGjhKAtausJC/91/V+vf15Vv/EolFwyKENA6UBKyVSsWP+klLe9T6j4gA/PwsHRkhpBGhJGCNCgr41n9xMf/ez49PAGKxJaMihDRClASsiUrFP/Gbns63/iUS/uTv62vpyAghjRQlAWvx8CHf+i8p4d/7+/Odv9T6J4SYECUBS1Mq+db/jRuPWv/t2/PDPwkhxMQoCVjSgwd867+0lH/frBnf+rezs2hYhBDbQUnAEpRKfq7/Gzf491Ip3/r39rZsXIQQm0NJwNyebP0HBPCTvlHrnxBiAZQEzEWpBFJS+PV+Ab7136EDP+0zIYRYCCUBc7h/n2/9l5Xx75s355d7pNY/IcTCKAkYgDGgvFAOeZkCYgcR7F3E4DgtBRUKvvWfkcG/t7fnW/+enuYM16wYA8rLAbmcH91qbw/tvw0hxCpQEtCDrLgSKX/eRtofqShLywKUKkAogEOoP0IHtESbFwIgdf5f6z4/H7h48VHrv0ULvvUvapw/uUzGX+wcPcrPdKFQ8IcaGgr06AFERtJkp4RYI44xxiwdhCUVFRXBxcUFhYWFaNKkSY3lbpzKw4llh6DIuA3GCcC5uYITicAUCrCHBeCYCqLAAHR7rweCnfOAW7f4ivb2/BnQw8M8B2QBaWnAd9/xDzpzHODuzt/pqqzk74QxBoSEABMm8EmBEGJaup7XAEoCOv1YN07l4cgHiWC5eRCFBEIgqX4vX1VRCeG1K3CXlKL9yAj4tHIBAgOBNm0abesf4BPAqlVAdjbQsqX2B5zlcn4WbB8fYMYMSgSEmJo+SUBgppj0snbtWgQGBkIqlaJr1644ffp0reV37tyJ1q1bQyqVIiIiAnv37jVaLLLiSpxYdggsNw92rUO1JgBOpUDTsjvwbSqH4GE+ruy6DlnE0/y8P404Achk/BVAdjZ/p6umGS7EYv7z7Gy+vExm3jgJITWzuiSQkJCA2NhYLFq0COfOnUOHDh0QHR2N3NxcreWPHz+OESNGYMKECTh//jwGDx6MwYMH48qVK0aJJ+XP21Bk3IYoJBCcoHoPp0RWAO/cy3AsywPHcSgPDkdWiQuuni83yvdbswsX+FtALVvW3fnLcfwVQHo631VCCLEOVpcEVqxYgUmTJmHcuHFo27Yt1q9fDwcHB2zatElr+VWrVqF///5477330KZNGyxZsgRPPfUU1qxZU+9YGAPS/kgF4wTVrgA4lQKuD2/A4/51CJWVUIgkyPNog2KvUDCBEKl7rqMx32hjjO8E5jjd57iTSPjyR46gUf82hDQkVpUE5HI5zp49i6ioKPU2gUCAqKgonDhxQmudEydOaJQHgOjo6BrLV1RUoKioSONVk/JCOcrSssC5uWp+wBg881LgWJYPAChx8kGuZzjkEmcAAOfmirK0LJQXyus65AarvJzvD3B316+euztfr7zxXygR0iBYVRLIz8+HUqmE9xNz6Hh7eyM7O1trnezsbL3KL1u2DC4uLupXQEBAjfHIyxSAUgXuyfv6HIdiZ19167/QpTmYQPjoY5EIUKr4+o2UXM4PA9X3eTeRiK8nb7z5kZAGxaqSgDnMmzcPhYWF6tft27drLCt2EAFCAZii+sm83MEDOV4R6tb/45hCAQgFfP1GSizmT+iVlfrVq3p+gJZJIMQ6WFUS8PDwgFAoRE5Ojsb2nJwc+NQwv76Pj49e5SUSCZo0aaLxqom9ixgOof5gDwu0F+C0/3zsYQEcQv1h79J4z3T29nxH7/37+tW7f5+vZ29vmrgIIfqxqiQgFovRqVMnJCUlqbepVCokJSWhW7duWut069ZNozwA7N+/v8by+uA4IHRAS3BMBVWFbk1epUwOMIaWA8Ma9XQJHMc/CcyY7rd2Kir48j170lQShFgLq0oCABAbG4uNGzdiy5YtSElJwdSpU1FaWopx48YBAEaPHo158+apy8+YMQOJiYlYvnw5rl69ig8//BBnzpzB9OnTjRJPmxcCIAoMgCI9A0xV+5AWpmJQ3rgFu8BmaB3VzCjfb80iI/kngVNT6x7twxjfIRwSwk+fRAixDlaXBGJiYvDll19i4cKFiIyMxIULF5CYmKju/M3MzMS9e/fU5Z999lls374dGzZsQIcOHfDTTz/h119/RXh4uFHikTrbodu8PuC8PFF5NY1v6WuhlMlReTUNnJcnnn2/z6M5hBoxqZSfCsLHB0hO5lv62lRU8J/7+AATJ9IcQoRYE5o2Qs+5gyoz7gAcV23uIDAGu8BmePb9Pgjq0nhnCdVG29xBVaOAHp87aOJE/k9CiGnR3EF60OfHkhVX4uqBO0jdc73aLKItB4ahdVQzm7gC0EYm458EPnKk+iyiPXvyt4DoCoAQ86AkoAd9fqwqOq8nYINoPQFCLE+f81rjHchuQhwHOLiK4eDaeIeAGorjAAcH/kUIsX5W1zFMCCHEfCgJEEKIDaMkQAghNszm+wSq+sVrm02UEEIakqrzmS7jfmw+CRQXFwNArbOJEkJIQ1RcXAwXF5day9j8EFGVSoW7d+/C2dkZnB5jGYuKihAQEIDbt2/rPLS0oWnsx0jH1/A19mM09PgYYyguLoafnx8Egtrv+tv8lYBAIECzZobP81PXTKSNQWM/Rjq+hq+xH6Mhx1fXFUAV6hgmhBAbRkmAEEJsGCUBA0kkEixatAgSicTSoZhMYz9GOr6Gr7EfozmOz+Y7hgkhxJbRlQAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJJALdauXYvAwEBIpVJ07doVp0+frrX8zp070bp1a0ilUkRERGDv3r1mitRw+hzjxo0b0bNnT7i5ucHNzQ1RUVF1/iaWpu/fYZX4+HhwHIfBgwebNsB60vf4CgoKMG3aNPj6+kIikSAsLMzq/53qe4wrV65Eq1atYG9vj4CAALzzzjuQyWRmilY/hw8fxqBBg+Dn5weO4/Drr7/WWefQoUN46qmnIJFIEBoairi4uPoFwYhW8fHxTCwWs02bNrF//vmHTZo0ibm6urKcnByt5Y8dO8aEQiH7/PPPWXJyMvvggw+YnZ0du3z5spkj152+xzhy5Ei2du1adv78eZaSksLGjh3LXFxc2J07d8wcuW70Pb4qN2/eZP7+/qxnz57sX//6l3mCNYC+x1dRUcE6d+7MXnzxRXb06FF28+ZNdujQIXbhwgUzR647fY/xhx9+YBKJhP3www/s5s2bbN++fczX15e98847Zo5cN3v37mXz589nv/zyCwPAdu3aVWv5GzduMAcHBxYbG8uSk5PZ6tWrmVAoZImJiQbHQEmgBl26dGHTpk1Tv1cqlczPz48tW7ZMa/lhw4axgQMHamzr2rUrmzx5sknjrA99j/FJCoWCOTs7sy1btpgqxHox5PgUCgV79tln2bfffsvGjBlj1UlA3+Nbt24dCw4OZnK53Fwh1pu+xzht2jTWr18/jW2xsbGse/fuJo3TGHRJArNnz2bt2rXT2BYTE8Oio6MN/l66HaSFXC7H2bNnERUVpd4mEAgQFRWFEydOaK1z4sQJjfIAEB0dXWN5SzPkGJ9UVlaGyspKNG3a1FRhGszQ4/voo4/g5eWFCRMmmCNMgxlyfLt370a3bt0wbdo0eHt7Izw8HEuXLoVSqTRX2Hox5BifffZZnD17Vn3L6MaNG9i7dy9efPFFs8RsaqY4z9j8BHLa5OfnQ6lUwtvbW2O7t7c3rl69qrVOdna21vLZ2dkmi7M+DDnGJ82ZMwd+fn7V/lFaA0OO7+jRo/juu+9w4cIFM0RYP4Yc340bN/DXX3/h9ddfx969e5GWloY333wTlZWVWLRokTnC1oshxzhy5Ejk5+ejR48eYIxBoVBgypQpeP/9980RssnVdJ4pKipCeXk57O3t9d4nXQkQg3z66aeIj4/Hrl27IJVKLR1OvRUXF2PUqFHYuHEjPDw8LB2OSahUKnh5eWHDhg3o1KkTYmJiMH/+fKxfv97SoRnNoUOHsHTpUnzzzTc4d+4cfvnlF+zZswdLliyxdGhWi64EtPDw8IBQKEROTo7G9pycHPj4+Git4+Pjo1d5SzPkGKt8+eWX+PTTT3HgwAG0b9/elGEaTN/jS09PR0ZGBgYNGqTeplKpAAAikQjXrl1DSEiIaYPWgyF/f76+vrCzs4NQKFRva9OmDbKzsyGXyyEWi00as74MOcYFCxZg1KhRmDhxIgAgIiICpaWleOONNzB//vw659a3djWdZ5o0aWLQVQBAVwJaicVidOrUCUlJSeptKpUKSUlJ6Natm9Y63bp10ygPAPv376+xvKUZcowA8Pnnn2PJkiVITExE586dzRGqQfQ9vtatW+Py5cu4cOGC+vXyyy+jb9++uHDhgtWtPGfI31/37t2RlpamTm4AcP36dfj6+lpdAgAMO8aysrJqJ/qqpMcawTRpJjnPGNyl3MjFx8cziUTC4uLiWHJyMnvjjTeYq6sry87OZowxNmrUKDZ37lx1+WPHjjGRSMS+/PJLlpKSwhYtWtQghojqc4yffvopE4vF7KeffmL37t1Tv4qLiy11CLXS9/ieZO2jg/Q9vszMTObs7MymT5/Orl27xn7//Xfm5eXFPv74Y0sdQp30PcZFixYxZ2dn9uOPP7IbN26wP//8k4WEhLBhw4ZZ6hBqVVxczM6fP8/Onz/PALAVK1aw8+fPs1u3bjHGGJs7dy4bNWqUunzVENH33nuPpaSksLVr19IQUVNavXo1a968OROLxaxLly7s5MmT6s969+7NxowZo1F+x44dLCwsjInFYtauXTu2Z88eM0esP32OsUWLFgxAtdeiRYvMH7iO9P07fJy1JwHG9D++48ePs65duzKJRMKCg4PZJ598whQKhZmj1o8+x1hZWck+/PBDFhISwqRSKQsICGBvvvkme/jwofkD18HBgwe1/j9VdUxjxoxhvXv3rlYnMjKSicViFhwczDZv3lyvGGgqaUIIsWHUJ0AIITaMkgAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJIAIYTYMEoChBBiwygJEEKIDaMkQKzCoUOHwHEcDh06ZOlQTIrjOHz44Yc6lQ0MDMTYsWNNGg8hlARIvcTFxYHjOK2vuXPnWjq8Wj0Zu1QqRVhYGKZPn15tpkZTOX78OD788EMUFBSY5ft0ERgYqPG7ODo6okuXLvj+++8N3ufevXt1Tn7EvGgqaWIUH330EYKCgjS2hYeHWyga/VTFLpPJcPToUaxbtw579+7FlStX4ODgYNTvKi8vh0j06H+748ePY/HixRg7dixcXV01yl67ds1iUx9HRkbi3XffBQDcu3cP3377LcaMGYOKigpMmjRJ7/3t3bsXa9eupURghSgJEKMYMGCAVU8tXZvHY584cSLc3d2xYsUK/PbbbxgxYoRRv0ufBXgkEolRv1sf/v7++Pe//61+P3bsWAQHB+Orr74yKAkQ60W3g4hJ3bp1C2+++SZatWoFe3t7uLu747XXXkNGRkaddVNTUzF06FD4+PhAKpWiWbNmGD58OAoLCzXKbdu2DZ06dYK9vT2aNm2K4cOH4/bt2wbH3K9fPwDAzZs3AQAKhQJLlixBSEgIJBIJAgMD8f7776OiokKj3pkzZxAdHQ0PDw/Y29sjKCgI48eP1yjzeJ/Ahx9+iPfeew8AEBQUpL79UvXbPN4ncObMGXAchy1btlSLd9++feA4Dr///rt6W1ZWFsaPHw9vb29IJBK0a9cOmzZtMvg38fT0ROvWrZGenq6x/ciRI3jttdfQvHlzSCQSBAQE4J133kF5ebm6zNixY7F27Vr18Ve9qqhUKqxcuRLt2rWDVCqFt7c3Jk+ejIcPHxocL9EdXQkQoygsLER+fr7GNg8PD/z99984fvw4hg8fjmbNmiEjIwPr1q1Dnz59kJycXOPtFrlcjujoaFRUVOCtt96Cj48PsrKy8Pvvv6OgoAAuLi4AgE8++QQLFizAsGHDMHHiROTl5WH16tXo1asXzp8/X+0Wiy6qTnTu7u4A+KuDLVu24NVXX8W7776LU6dOYdmyZUhJScGuXbsAALm5uXjhhRfg6emJuXPnwtXVFRkZGfjll19q/J4hQ4bg+vXr+PHHH/HVV1+pl7X09PSsVrZz584IDg7Gjh07MGbMGI3PEhIS4ObmhujoaAD8SlPPPPMMOI7D9OnT4enpiT/++AMTJkxAUVERZs6cqfdvolAocOfOHbi5uWls37lzJ8rKyjB16lS4u7vj9OnTWL16Ne7cuYOdO3cCACZPnoy7d+9i//792Lp1a7V9T548GXFxcRg3bhzefvtt3Lx5E2vWrMH58+dx7Ngx2NnZ6R0v0UO9JqImNm/z5s1a50Ov+qdVVlZWrc6JEycYAPb999+rt1XNq37w4EHGGFMvsrFz584avzsjI4MJhUL2ySefaGy/fPkyE4lE1bbXFPuBAwdYXl4eu337NouPj2fu7u7M3t6e3blzh124cIEBYBMnTtSoO2vWLAaA/fXXX4wxxnbt2sUAsL///rvW78QT6y988cUXDAC7efNmtbItWrTQmCt/3rx5zM7Ojj148EC9raKigrm6urLx48ert02YMIH5+vqy/Px8jf0NHz6cubi4aP07efJ7X3jhBZaXl8fy8vLY5cuX2ahRoxgANm3aNI2y2va1bNkyxnGcemEUxhibNm0a03a6OXLkCAPAfvjhB43tiYmJWrcT46PbQcQo1q5di/3792u8AGise1pZWYn79+8jNDQUrq6uOHfuXI37q2rp79u3D2VlZVrL/PLLL1CpVBg2bBjy8/PVLx8fH7Rs2RIHDx7UKfaoqCh4enoiICAAw4cPh5OTE3bt2gV/f3/s3bsXABAbG6tRp6rTdM+ePQCgvuL4/fffUVlZqdP36ismJgaVlZUaVxd//vknCgoKEBMTA4BfQvHnn3/GoEGDwBjT+F2io6NRWFhY6+/++H49PT3h6emJiIgIbN26FePGjcMXX3yhUe7xv9/S0lLk5+fj2WefBWMM58+fr/N7du7cCRcXFzz//PMasXbq1AlOTk46/x0Sw9HtIGIUXbp00doxXF5ejmXLlmHz5s3IysrSWOf1yXv7jwsKCkJsbCxWrFiBH374AT179sTLL7+Mf//73+oEkZqaCsYYWrZsqXUfut5GWLt2LcLCwiASieDt7Y1WrVqpR+XcunULAoEAoaGhGnV8fHzg6uqKW7duAQB69+6NoUOHYvHixfjqq6/Qp08fDB48GCNHjjRaB2+HDh3QunVrJCQkYMKECQD4W0EeHh7qfoy8vDwUFBRgw4YN2LBhg9b95Obm1vldXbt2xccffwylUokrV67g448/xsOHD6utRZyZmYmFCxdi9+7d1e7h1/b3WyU1NRWFhYXw8vIyOFZSP5QEiEm99dZb2Lx5M2bOnIlu3brBxcUFHMdh+PDhGguea7N8+XKMHTsWv/32G/7880+8/fbbWLZsGU6ePIlmzZpBpVKB4zj88ccf6sXEH+fk5KRTjDUlsMc93pFZ0+c//fQTTp48if/+97/Yt28fxo8fj+XLl+PkyZM6x1KXmJgYfPLJJ8jPz4ezszN2796NESNGqIedVv2m//73v6v1HVRp3759nd/j4eGBqKgoAEB0dDRat26Nl156CatWrVJfFSmVSjz//PN48OAB5syZg9atW8PR0RFZWVkYO3ZsnX+/VfF6eXnhhx9+0Pq5tv4RYlyUBIhJ/fTTTxgzZgyWL1+u3iaTyXR+OCoiIgIRERH44IMPcPz4cXTv3h3r16/Hxx9/jJCQEDDGEBQUhLCwMJPE36JFC6hUKqSmpqJNmzbq7Tk5OSgoKECLFi00yj/zzDN45pln8Mknn2D79u14/fXXER8fj4kTJ2rdf13J5UkxMTFYvHgxfv75Z3h7e6OoqAjDhw9Xf+7p6QlnZ2colUr1SdwYBg4ciN69e2Pp0qWYPHkyHB0dcfnyZVy/fh1btmzB6NGj1WWrbgU+rqbjDAkJwYEDB9C9e3eNW0vEfKhPgJiUUCjUuAUEAKtXr4ZSqay1XlFRERQKhca2iIgICAQC9dDMIUOGQCgUYvHixdW+gzGG+/fv1zv+F198EQCwcuVKje0rVqwAwJ8cAeDhw4fVYoiMjASAakNJH+fo6AgAOifFNm3aICIiAgkJCUhISICvry969eql/lwoFGLo0KH4+eefceXKlWr18/LydPoebebMmYP79+9j48aN6u8CoHHcjDGsWrWqWt2ajnPYsGFQKpVYsmRJtToKhcKqnqRurOhKgJjUSy+9hK1bt8LFxQVt27bFiRMncODAAfXwy5r89ddfmD59Ol577TWEhYVBoVBg69at6pMcwLciP/74Y8ybNw8ZGRkYPHgwnJ2dcfPmTezatQtvvPEGZs2aVa/4O3TogDFjxmDDhg0oKChA7969cfr0aWzZsgWDBw9G3759AQBbtmzBN998g1deeQUhISEoLi7Gxo0b0aRJE3Ui0aZTp04AgPnz52P48OGws7PDoEGD1CdNbWJiYrBw4UJIpVJMmDCh2lPFn376KQ4ePIiuXbti0qRJaNu2LR48eIBz587hwIEDePDggUG/xYABAxAeHo4VK1Zg2rRpaN26NUJCQjBr1ixkZWWhSZMm+Pnnn7WO7686zrfffhvR0dEQCoUYPnw4evfujcmTJ2PZsmW4cOECXnjhBdjZ2SE1NRU7d+7EqlWr8OqrrxoUL9GRZQYlkcaiaphlTUMjHz58yMaNG8c8PDyYk5MTi46OZlevXq02/PHJIaI3btxg48ePZyEhIUwqlbKmTZuyvn37sgMHDlT7jp9//pn16NGDOTo6MkdHR9a6dWs2bdo0du3atXrFXqWyspItXryYBQUFMTs7OxYQEMDmzZvHZDKZusy5c+fYiBEjWPPmzZlEImFeXl7spZdeYmfOnNHYF54YIsoYY0uWLGH+/v5MIBBoDBd98jeqkpqaqh6Ge/ToUa0x5+TksGnTprGAgABmZ2fHfHx82HPPPcc2bNhQ67FWfe/AgQO1fhYXF8cAsM2bNzPGGEtOTmZRUVHMycmJeXh4sEmTJrGLFy9qlGGMMYVCwd566y3m6enJOI6rNlx0w4YNrFOnTsze3p45OzuziIgINnv2bHb37t064yX1wzH2xDUsIYQQm0F9AoQQYsMoCRBCiA2jJEAIITaMkgAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJIAIYTYMEoChBBiwygJEEKIDaMkQAghNoySACGE2LD/Bxx6BBV3rJdWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# another way (1 seed)\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob) # y_prob[:,1]\n",
    "#fpr, tpr, thresholds = roc_curve(y_test, y_prob) # way that usualy works\n",
    "\n",
    "plt.figure(figsize=(4, 4))\n",
    "plt.scatter(fpr, tpr, s=100, alpha=0.5, color=\"blue\", label=\"Scikit-learn\")\n",
    "plt.scatter(ROC[:, 0], ROC[:, 1], color=\"red\", s=100, alpha=0.3, label=\"Our implementation\")\n",
    "plt.plot(ROC[:, 0], ROC[:, 1], color=\"red\", alpha=0.3, label=\"Our implementation\")\n",
    "plt.title(\"ROC Curve\", fontsize=12)\n",
    "plt.xlabel(\"False Positive Rate\", fontsize=12)\n",
    "plt.ylabel(\"True Positive Rate\", fontsize=12)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 938,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x180a4ddb0>"
      ]
     },
     "execution_count": 938,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAGNCAYAAAD3m81fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABh10lEQVR4nO3deXxMV/8H8M+dmcxMNklkT4RsYkuIUqp2TRuq+iitoI+dorQ0VUsVVS3dKEV5aIlSTWir9ZRGSfnZqZ0mSEKEkA3ZM5nMzPn9cZ8MI5NkZjJbMt/36zUvnTvn3Pneofd77j3nnsMxxhgIIYTYJIGlAyCEEGI5lAQIIcSGURIghBAbRkmAEEJsGCUBQgixYZQECCHEhlESIIQQG0ZJgBBCbBglAUIIsWGUBAghxIZREiANWlxcHDiOU79EIhH8/f0xduxYZGVlaa3DGMPWrVvRq1cvuLq6wsHBAREREfjoo49QWlpa43ft2rULAwYMgIeHB8RiMfz8/DBs2DD89ddfOsUqk8nw1VdfoWvXrnBxcYFUKkVYWBimT5+O69evG3T8hNQXR3MHkYYsLi4O48aNw0cffYSgoCDIZDKcPHkScXFxCAwMxJUrVyCVStXllUolRo4ciR07dqBnz54YMmQIHBwccOTIEWzfvh1t27bFgQMH4O3tra7DGMP48eMRFxeHjh074tVXX4WPjw/u3buHXbt24ezZszh27BieffbZGuPMz89H//79cfbsWbz00kuIioqCk5MTrl27hvj4eGRnZ0Mul5v0tyJEK0ZIA7Z582YGgP39998a2+fMmcMAsISEBI3tS5cuZQDYrFmzqu1r9+7dTCAQsP79+2ts/+KLLxgANnPmTKZSqarV+/7779mpU6dqjXPgwIFMIBCwn376qdpnMpmMvfvuu7XW11VlZSWrqKgwyr6IbaAkQBq0mpLA77//zgCwpUuXqreVlZUxNzc3FhYWxiorK7Xub9y4cQwAO3HihLpO06ZNWevWrZlCoTAoxpMnTzIAbNKkSTqV7927N+vdu3e17WPGjGEtWrRQv7958yYDwL744gv21VdfseDgYCYQCNjJkyeZUChkH374YbV9XL16lQFgq1evVm97+PAhmzFjBmvWrBkTi8UsJCSEffrpp0ypVOp9rKThoT4B0ihlZGQAANzc3NTbjh49iocPH2LkyJEQiURa640ePRoA8Pvvv6vrPHjwACNHjoRQKDQolt27dwMARo0aZVD9umzevBmrV6/GG2+8geXLl8PX1xe9e/fGjh07qpVNSEiAUCjEa6+9BgAoKytD7969sW3bNowePRpff/01unfvjnnz5iE2NtYk8RLrov3/BEIamMLCQuTn50Mmk+HUqVNYvHgxJBIJXnrpJXWZ5ORkAECHDh1q3E/VZykpKRp/RkREGBybMfZRmzt37iAtLQ2enp7qbTExMZg8eTKuXLmC8PBw9faEhAT07t1b3eexYsUKpKen4/z582jZsiUAYPLkyfDz88MXX3yBd999FwEBASaJm1gHuhIgjUJUVBQ8PT0REBCAV199FY6Ojti9ezeaNWumLlNcXAwAcHZ2rnE/VZ8VFRVp/FlbnboYYx+1GTp0qEYCAIAhQ4ZAJBIhISFBve3KlStITk5GTEyMetvOnTvRs2dPuLm5IT8/X/2KioqCUqnE4cOHTRIzsR50JUAahbVr1yIsLAyFhYXYtGkTDh8+DIlEolGm6iRclQy0eTJRNGnSpM46dXl8H66urgbvpyZBQUHVtnl4eOC5557Djh07sGTJEgD8VYBIJMKQIUPU5VJTU3Hp0qVqSaRKbm6u0eMl1oWSAGkUunTpgs6dOwMABg8ejB49emDkyJG4du0anJycAABt2rQBAFy6dAmDBw/Wup9Lly4BANq2bQsAaN26NQDg8uXLNdapy+P76NmzZ53lOY4D0zJyW6lUai1vb2+vdfvw4cMxbtw4XLhwAZGRkdixYweee+45eHh4qMuoVCo8//zzmD17ttZ9hIWF1RkvadjodhBpdIRCIZYtW4a7d+9izZo16u09evSAq6srtm/fXuMJ9fvvvwcAdV9Cjx494Obmhh9//LHGOnUZNGgQAGDbtm06lXdzc0NBQUG17bdu3dLrewcPHgyxWIyEhARcuHAB169fx/DhwzXKhISEoKSkBFFRUVpfzZs31+s7ScNDSYA0Sn369EGXLl2wcuVKyGQyAICDgwNmzZqFa9euYf78+dXq7NmzB3FxcYiOjsYzzzyjrjNnzhykpKRgzpw5Wlvo27Ztw+nTp2uMpVu3bujfvz++/fZb/Prrr9U+l8vlmDVrlvp9SEgIrl69iry8PPW2ixcv4tixYzofPwC4uroiOjoaO3bsQHx8PMRicbWrmWHDhuHEiRPYt29ftfoFBQVQKBR6fSdpgCw9RpWQ+qjpOQHGGNu5cycDwNatW6feplAo2NChQxkA1qtXL7Zq1Sq2YcMGNnr0aCYQCFi7du1Ydna2xn6USiUbNWoUA8CeeuoptnTpUrZp0ya2dOlS1qVLFwaAHT9+vNY4c3NzWWRkJOM4jr388sts1apV7Ntvv2Vz5sxhLVq0YGKxWF02OTmZCQQC1rFjR7ZmzRq2cOFC5uXlxSIiImp8TqAm27ZtYwCYs7MzGzRoULXPS0tL2VNPPcVEIhGbOHEiW7duHfvyyy/ZmDFjmKOjI8vLy6v1uEjDR0mANGi1JQGlUslCQkJYSEiIxoNeSqWSbd68mXXv3p01adKESaVS1q5dO7Z48WJWUlJS43f99NNP7IUXXmBNmzZlIpGI+fr6spiYGHbo0CGdYi0rK2Nffvkle/rpp5mTkxMTi8WsZcuW7K233mJpaWkaZbdt28aCg4OZWCxmkZGRbN++fbU+LFaToqIiZm9vzwCwbdu2aS1TXFzM5s2bx0JDQ5lYLGYeHh7s2WefZV9++SWTy+U6HRtpuGjuIEIIsWHUJ0AIITaMkgAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJIAIYTYMJufO0ilUuHu3btwdnYGx3GWDocQQuqNMYbi4mL4+flBIKi9rW/zSeDu3bs0XzohpFG6ffu2xnTq2th8EqiaMvj27dvqKX8JIaQhKyoqQkBAgE5rWNh8Eqi6BdSkSRNKAoSQRkWXW9zUMUwIITaMkgAhhNgwSgKEEGLDbL5PQBeMMSgUCoNXliKkoRAKhRCJRGYdLs0YUJ71APKCEohdnWDv3xQ0Wvt/v8vO3ZCnpkHcMhT2r71skt+FkkAd5HI57t27h7KyMkuHQohZODg4wNfXF2Kx2KTfI8spROZ/9kD26z5I76ZDoFJAJRBB5hcC6eBoNJ88EFJvF5PGYI1kfyRBOX4ipNkZkACQ/G+7KgaQ+QRCuOlbSAc8Z7Tvs/n1BIqKiuDi4oLCwsJqo4NUKhVSU1MhFArh6ekJsVhMD5SRRosxBrlcjry8PCiVSrRs2bLOB40MlRl/DMXzl8IpLwOME6DCwQ1MKAKnVEBS9hAcU6HEMxDOn7yP5sO7myQGa1TQawCcjyTWWa64Z3+4Hv6jxs9rO689ia4EaiGXy6FSqRAQEAAHBwdLh0OIydnb28POzg63bt2CXC6HVCo1+ndkxh9D6bsfwKk4D8VeoYBE8zsqm/oCFTI456aj9N0PkImPbSIR6JoAAMD5SCIKeg2oNRHoyqo6hg8fPoxBgwbBz88PHMdpXZT7SYcOHcJTTz0FiUSC0NBQxMXFGT0uU7WGCLFGpvz3LsspRPH8pXAozkORf5tqCUBNIkWRfxs4FOeheP5SyHIKTRaTNZD9kaRzAqjifCQRsj+S6v3dVnV2Ky0tRYcOHbB27Vqdyt+8eRMDBw5E3759ceHCBcycORMTJ07Evn37TBwpIcQQmf/ZA6e8DBR7hYCrI9lwAgFKvELglHcLtzfqd4JsaJTjJxpUTzHhjXp/t1XdDhowYAAGDBigc/n169cjKCgIy5cvBwC0adMGR48exVdffYXo6GhThWkQxoDyckAuB8RiwN4eNAKC2BTGANmv+yDmBNWuAARKOQQqLaPvBICAKVCZ8BPY+F6N8v8ZxgBpdoZBde3v3QBj9TuXWFUS0NeJEycQFRWlsS06OhozZ86ssU5FRQUqKirU74uKikwVHgBAJgMuXACOHgXS0gCFAhCJgNBQoEcPIDISMMFt13qLi4vDzJkzUVBQAAD48MMP8euvv+LChQs6lddVRkYGgoKCcP78eURGRtYrZmLdyrMeQHo3HRUObhrbBUo5HMvya6zHiYRwufMP5PsOQuJR91w4DY386Kl6nYjLd+6Gw7CXDa7foJNAdnY2vL29NbZ5e3ujqKgI5eXlsLe3r1Zn2bJlWLx4sVniS0sDvvsOSE/nM7W7O+DoCFRWAn//DZw+DYSEABMm8EnB2PLy8rBw4ULs2bMHOTk5cHNzQ4cOHbBw4UJ07157R1tMTAxefPFFnb/ryfJ1JQ1ie+QFJRCoFFCINQdZCBh/BcDAQSW0q15RJIFQUQG5SAJJ06bmCNWs5A8L63UilqemoT7DVhp0EjDEvHnzEBsbq35fNduesaWlAatWAdnZQMuW/C2gx3l58beGUlP5cjNmGD8RDB06FHK5HFu2bEFwcDBycnKQlJSE+/fv11nX3t5eaxI1Vnlzq6yshJ2dlhMMMRuxqxNUAn4YqDYqoQhlDh7VttvJKsEJHdC0b1+gWeNLAsKs+8DGNQbXF7es34nDqjqG9eXj44OcnByNbTk5OWjSpEmNJySJRKKeMdRUM4fKZPwVQHY20LZt9QRQRSzmP8/O5svLZMaLoaCgAEeOHMFnn32Gvn37okWLFujSpQvmzZuHl19+WV1m8uTJ8Pb2hlQqRXh4OH7//XcA/O0dV1fXGvefnp6O4OBgTJ8+HYwxjfJxcXFYvHgxLl68CI7jwHGcXqO2rly5ggEDBsDJyQne3t4YNWoU8vMf3S5ITExEjx494OrqCnd3d7z00ktIT09Xf56RkQGO45CQkIDevXtDKpXihx9+wNixYzF48GB8+eWX8PX1hbu7O6ZNm4bKykrdf1hiMHv/ppD5hUBS9lCvepKyh5D5hcDev/ElAACwf83wWznGqN+gk0C3bt2QlKQ5RGr//v3o1q2bhSLiXbjA3wJq2bLuDhuO468A0tOBixeNF4OTkxOcnJzw66+/avSBVFGpVBgwYACOHTuGbdu2ITk5GZ9++imEQmGd+7506RJ69OiBkSNHYs2aNdUeoIuJicG7776Ldu3a4d69e7h37x5iYmJ0irugoAD9+vVDx44dcebMGSQmJiInJwfDhg1TlyktLUVsbCzOnDmDpKQkCAQCvPLKK1CpVBr7mjt3LmbMmIGUlBT1QIGDBw8iPT0dBw8exJYtWxAXF2eSYcWkOo4DpIOjwTEVUKFbi4erkIFjDPZDXmyUncIA/7vIfAINqlvuG1zv38WqbgeVlJQgLS1N/f7mzZu4cOECmjZtiubNm2PevHnIysrC999/DwCYMmUK1qxZg9mzZ2P8+PH466+/sGPHDuzZs8dShwDG+E5gjqv5CuBJEglf/sgRoEsX44waEolEiIuLw6RJk7B+/Xo89dRT6N27N4YPH4727dvjwIEDOH36NFJSUhAWFgYACA4OrnO/x48fx0svvYT58+fj3Xff1VrG3t4eTk5OEIlE8PHx0SvuNWvWoGPHjli6dKl626ZNmxAQEIDr168jLCwMQ4cO1aizadMmeHp6Ijk5GeHh4ertM2fOxJAhQzTKurm5Yc2aNRAKhWjdujUGDhyIpKQkTJo0Sa84iWGaTx6I9C0/wDk3HUX+bWodJspUKjjnpqPEswVCJvU3Y5TmJ9z0LfBiVN0FnyD6bkO9v9uqrgTOnDmDjh07omPHjgCA2NhYdOzYEQsXLgQA3Lt3D5mZmeryQUFB2LNnD/bv348OHTpg+fLl+Pbbby06PLS8nO8PcHfXr567O1+vvNx4sQwdOhR3797F7t270b9/f/WDdXFxcbhw4QKaNWumTgC6yMzMxPPPP4+FCxfWmABqM2XKFPUVipOTk9YyFy9exMGDBzXKtW7dGgDUt3xSU1MxYsQIBAcHo0mTJggMDFTH97jOnTtX23+7du00rnZ8fX2Rm5ur97EQw0i9XeD8yfsoc/ZEk6wUcDVcEXAVMjTJSuHLLX2/0c8hJB3wHIp76pfoSnr1N8ocQlZ1JdCnTx/UNpWRtsv2Pn364Pz58yaMSj9yOT8M1NFRv3oiEVBWxtc35gwVUqkUzz//PJ5//nksWLAAEydOxKJFizBr1iy99+Xp6Qk/Pz/8+OOPGD9+vN79KR999FGd31tSUoJBgwbhs88+q/aZr68vAGDQoEFo0aIFNm7cCD8/P6hUKoSHh0Mul2uUd9Tyl/Bk5zDHcdVuIxHTaj68OzLxMT93UG46BEwBTiQERBLYyarmDmIo8WyBJkvfR0BM458yAgBcD/+h89QRJb36w+X/6j9lBGBlVwKNgVjMn9D17Wusen7AxBM3om3btigtLUX79u1x584dXL9+Xee69vb2+P333yGVShEdHY3i4uIay4rF4mpTb3t5eSE0NFT90uapp57CP//8g8DAQI2yoaGhcHR0xP3793Ht2jV88MEHeO6559CmTRs8fKhfRyOxvObDuyPk+HbIZy9AcWB7cGAQKirAMRWKQp6CfPYChBzfbjMJoIrr4T9QufcAyn2135ot9w1G5d4DRksAgJVdCTQG9vZ8R+/ff/PDQHV1/z7w9NN8fWO4f/8+XnvtNYwfPx7t27eHs7Mzzpw5g88//xz/+te/0Lt3b/Tq1QtDhw7FihUrEBoaiqtXr4LjOPTvX/NlqaOjI/bs2aN+ujsxMVHrrZ3AwEB1n06zZs3g7OwMiUSiZY+apk2bho0bN2LEiBGYPXs2mjZtirS0NMTHx+Pbb7+Fm5sb3N3dsWHDBvj6+iIzMxNz586t129FLEPq7YKWH8SATegNeeJfkAslaNqvr82vJyAd8BxwN13regJOJvhd6ErAyDiOfxKYMf7Wji4qKvjyPXsabyoJJycndO3aFV999RV69eqF8PBwLFiwAJMmTcKaNfyY5J9//hlPP/00RowYgbZt22L27Nk6LZzj5OSEP/74A4wxDBw4EKWlpdXKDB06FP3790ffvn3h6emJH3/8Uae4/fz8cOzYMSiVSrzwwguIiIjAzJkz4erqCoFAAIFAgPj4eJw9exbh4eF455138MUXX+j34xCrwnGAxMMZzsHecGhm2wngcRwHOAx7Ga7zY+EwzDQLygC0nkCt827LZDLcvHkTQUFBek2pK5MBixfzHb1t29Z+YmcMSE7mrx4WLbLOKSSIbTH0373BsrP5S2c3N74FRepNn/UE6ErABKRSfioIHx/+BK9lmD4AfntyMl9u4kRKAIQQ86M+ARMJDeWngnhy7iCRiO8Evn+fvwoIDeUTQEiIpSMmhNgiSgImVHWL5+JF/kGwtDR+GKhIxHcC9+wJdOhAVwCEEMuhJGBiUinQtSv/JDCtJ0AIsTaUBMyE4/iHwGipYkKINaGOYUIIsWGUBAghxIZREiCEEBtGfQLmVDW7nDkmCSKEEB3QlYCpVVYCN24A+/YB27YB27fzf+7bx29vZKtacRyHX3/91eTf06dPH8ycOdPk32ONzPUbE9tAScCU8vKAX3/lXykpgEDAjxkVCPj3VZ/l5Znk62/fvo3x48fDz88PYrEYLVq0wIwZM3RaY9hQ9+7dw4ABA0y2/4aoallLfX344YeIjIystp1+Y2JMlARMJS8PSEwEbt8GAgP5R4Ld3QEXF/7PkBB+++3bfDkjJ4IbN26gc+fOSE1NxY8//oi0tDSsX78eSUlJ6NatGx48eFCv/de0Lq+Pj49Os4USw9FvTIyJkoApVFYChw7xJ/bQUOCJhUzU7Oz4z/Py+PJGvDU0bdo0iMVi/Pnnn+jduzeaN2+OAQMG4MCBA8jKysL8+fPVZbXdXnB1dVUv4lPTwu3aPL6vqno7duxAz549YW9vj6effhrXr1/H33//jc6dO8PJyQkDBgxA3mNJsKrlvHjxYnh6eqJJkyaYMmVKtUVjHldRUYFZs2bB398fjo6O6Nq1Kw4dOqT+PC4uDq6urvj999/RqlUrODg44NVXX0VZWRm2bNmCwMBAuLm54e2339aYSVXX/e7btw9t2rSBk5MT+vfvj3v37gHgW/NbtmzBb7/9Bo7jwHGcuv6cOXMQFhYGBwcHBAcHY8GCBerkGhcXh8WLF+PixYvqelV/H0/+fV2+fBn9+vWDvb093N3d8cYbb6CkpKTa7/nll1/C19cX7u7umDZtWo2JnNgW6hg2hdu3H10B6LLSfIsWQGYmcOcOEBRU769/8OAB9u3bh08++QT2TyxQ4OPjg9dffx0JCQn45ptvqi0SX5u5c+di+fLl6Nixo16zSy5atAgrV65E8+bNMX78eIwcORLOzs5YtWoVHBwcMGzYMCxcuBDr1q1T10lKSoJUKsWhQ4eQkZGBcePGwd3dHZ988onW75g+fTqSk5MRHx8PPz8/7Nq1C/3798fly5fRsmVLAEBZWRm+/vprxMfHo7i4GEOGDMErr7wCV1dX7N27Fzdu3MDQoUPRvXt3xMTE6LXfL7/8Elu3boVAIMC///1vzJo1Cz/88ANmzZqFlJQUFBUVYfPmzQCApk2bAgCcnZ0RFxcHPz8/XL58GZMmTYKzszNmz56NmJgYXLlyBYmJiThw4AAAwMWl+hKLpaWliI6ORrdu3fD3338jNzcXEydOxPTp0zVW4jt48CB8fX1x8OBBpKWlISYmBpGRkbS2MqEkYBKpqfx9/5quAJ4kFvPJ4Pp1oySB1NRUMMbQpk0brZ9XrcaVl5cHLz1WvtG2cLsuZs2apV73ecaMGRgxYgSSkpLQvTu/atSECROqLR0qFouxadMmODg4oF27dvjoo4/w3nvvYcmSJRA8sTh5ZmYmNm/ejMzMTPj5+am/MzExEZs3b1YvWl9ZWYl169Yh5H+z9b366qvYunUrcnJy4OTkhLZt26Jv3744ePAgYmJi9Nrv+vXr1fudPn06PvroIwD82gv29vaoqKiAj4+PRtwffPCB+r8DAwMxa9YsxMfHY/bs2bC3t4eTkxNEIlG1eo/bvn07ZDIZvv/+e/VymmvWrFEv0ent7Q0AcHNzw5o1ayAUCtG6dWsMHDgQSUlJlAQIJQGjk8uBrCzA1VW/eq6ufL2qyYWMwNhLRWhbuF0X7du3V/931UkpIiJCY9uTi7136NABDo/NsdGtWzeUlJTg9u3baNGihUbZy5cvQ6lUIiwsTGN7RUUF3N3d1e8dHBzUJ+qq7w0MDNRYGe3xWAzdr66L1yckJODrr79Geno6SkpKoFAo9F63OSUlBR06dNBYT7l79+5QqVS4du2a+vdu164dhEKhRoyXL1/W67tI40RJwNgUCkCl0v9ELhLxq9EoFPVOAqGhoeA4DikpKXjllVeqfZ6SkgI3Nzd4enoC4O8xP5kwtN0v1rZwuy4eX9y96vbTk9vqs9h7SUkJhEIhzp49q3GiA6Bxgte2yHxtC8/XZ791JeATJ07g9ddfx+LFixEdHQ0XFxfEx8dj+fLldRytYWo7TmLbKAkYm0jE3wpSKPSrp1Dw9UT1/ytxd3fH888/j2+++QbvvPOORr9AdnY2fvjhB4wePVp9Qvb09FR3ZAL87aSysrJ6x1EfFy9eRHl5uTr2kydPwsnJCQEBAdXKduzYEUqlErm5uejZs6fRYjDWfsVicbVlO48fP44WLVpodNDfunWrznpPatOmDeLi4lBaWqpO0seOHYNAIECrVq0MjpnYDhodZGxiMeDvDxQU6FevoICvZ6RbQWvWrEFFRQWio6Nx+PBh3L59G4mJiXj++efh7++v0cHar18/rFmzBufPn8eZM2cwZcqUai1Hc5PL5ZgwYQKSk5Oxd+9eLFq0CNOnT6/WHwAAYWFheP311zF69Gj88ssvuHnzJk6fPo1ly5Zhz549BsdgrP0GBgbi0qVLuHbtGvLz81FZWYmWLVsiMzMT8fHxSE9Px9dff41du3ZVq3fz5k1cuHAB+fn5qNCyRN3rr78OqVSKMWPG4MqVKzh48CDeeustjBo1Sn0riJDaUBIwhZYt+VtCug7Bk8v5ZcaeuPdcvxBa4syZMwgODsawYcMQEhKCN954A3379sWJEyfUI1QAYPny5QgICEDPnj0xcuRIzJo1S+N+vCU899xzaNmyJXr16oWYmBi8/PLL+PDDD2ssv3nzZowePRrvvvsuWrVqhcGDB+Pvv/9G8+bN6xWHMfY7adIktGrVCp07d4anpyeOHTuGl19+Ge+88w6mT5+OyMhIHD9+HAsWLNCoN3ToUPTv3x99+/aFp6cnfvzxx2r7dnBwwL59+/DgwQM8/fTTePXVV/Hcc89hzZo19TpuYjtooXkTLDSPykr+SeDbt/nnAOpaaT4tDQgIAAYP1n1EUSM2duxYFBQU0NQIFkILzTd8tNC8pdnZAX36AJ6e/Am+poec5HL+c09PvjwlAEKImVHHsKl4egL9+/NPAmdm8lcDrq6PVpovKOCvAgICHiUMQggxM0oCpuTpyd/iuXOHfxAsK4sfBioQAG3a8H0AzZrRFcATnnxwjBBiOpQETM3Ojn8KOCiI1hMghFgdSgLmJBbTyZ8QYlWoY5gQQmwYXQkQQizviy/4vrOuXQEtz0MQ06ErAUKIZUydyo+a8/UFjhwBbt4E4uP5bRzHf05MjpIAIcT87OyA9etrL7N+PY2cMwNKAsSoaKF502vwC83b2ek+waJCQYnAxCgJNGK00Lx1oIXmHzN1qmEz7NKtIZOhJNBI0ULzjVeD/o3rugVk7HqkTpQEDKFUWualB1ponhaaf/L3pIXmiTY0RFRfSiWwd69lvvvFF4EnVrjShhaap4XmrXKh+REj6l+fho8aHSWBRogWmqeF5q1yofkrVyxbn2hFSUBfQiHfIrfUd+uBFpqnheataqH58PD6ncjDw40XC1GjJGAIPU/G5kYLzT9CC83XHKPZF5r/8Uf+YbD61CdGR0mgEaKF5o2DFpontoBGBzVStNA8LTRvlQvNT5li3nqkTpQEGilaaJ4WmrdK69bx62noQyTi6xHTYFZmzZo1rEWLFkwikbAuXbqwU6dO1Vr+q6++YmFhYUwqlbJmzZqxmTNnsvLycp2/r7CwkAFghYWF1T4rLy9nycnJeu2P1N+YMWPYv/71L0uHYbPM8u9eJGKMX2C19pdIZLoYGrHazmtPsqorgYSEBMTGxmLRokU4d+4cOnTogOjo6BpHWmzfvh1z587FokWLkJKSgu+++w4JCQl4//33zRw5IUQvlZV13+KZMoUvR0zKqpLAihUrMGnSJIwbNw5t27bF+vXr4eDggE2bNmktf/z4cXTv3h0jR45EYGAgXnjhBYwYMQKnT582c+SEEL2tW8e39+/dA3r25JdgHT780XUA3QIyC6sZHSSXy3H27FnMmzdPvU0gECAqKgonTpzQWufZZ5/Ftm3bcPr0aXTp0gU3btzA3r17MWrUqBq/p6KiQqODraioyHgHQYyCFpq3Qe+9B7i5AT16WDoSm2M1SSA/Px9KpbLaiAZvb29cvXpVa52RI0ciPz8fPXr0AGMMCoUCU6ZMqfV20LJly7B48WKjxk4IIQ2VVd0O0tehQ4ewdOlSfPPNNzh37hx++eUX7NmzB0uWLKmxzrx581BYWKh+3b59u87vYUZ+8pYQa0b/3m2L1VwJeHh4QCgUIicnR2N7Tk5OjXOnLFiwAKNGjcLEiRMB8FMRlJaW4o033sD8+fO1jimXSCQ6T8NbNVa+rKys2kRshDRWVQ8KWvpZEWIeVpMExGIxOnXqhKSkJPUCHCqVCklJSZg+fbrWOmVlZdVO9FWP9xujNSMUCuHq6qoeneTg4KDXrJuENCSMMZSVlSE3Nxeurq7VpsogjZPVJAEAiI2NxZgxY9C5c2d06dIFK1euRGlpKcaNGwcAGD16NPz9/bFs2TIAwKBBg7BixQp07NgRXbt2RVpaGhYsWIBBgwYZ7R9w1VWILhOCEdIYuLq61jpzKWlcrCoJxMTEIC8vDwsXLkR2djYiIyORmJio7izOzMzUaPl/8MEH4DgOH3zwAbKysuDp6YlBgwbVOOe8ITiOg6+vL7y8vGgRDtLo2dnZ0RWAjeGYjfcCFRUVwcXFBYWFhXpP40sIMYLsbODvv2mIqBHpc15r0KODCCGE1A8lAUIIsWGUBAghxIZREiCEEBtWr9FBFRUVOHfuHHJzc9G9e3d4eHgYKy5CCCFmYPCVwNdffw1fX1/06NEDQ4YMwaVLlwDwcwB5eHjUOPMnIYQQ62FQEti8eTNmzpyJ/v3747vvvtN4OtfDwwP9+vVDfH0WlCaEEGIWBiWB5cuX41//+he2b9+OQYMGVfu8U6dO+Oeff+odHCGEENMyKAmkpaVhwIABNX7etGlT3L9/3+CgCCGEmIdBScDV1RX5+fk1fp6cnExzjxBCSANgUBJ48cUXsWHDBhQUFFT77J9//sHGjRvx8ssv1zc2QgghJmZQEvj444+hVCoRHh6unsRty5Yt+Pe//43OnTvDy8sLCxcuNHashBBCjMygJODn54ezZ8+if//+SEhIAGMMW7duxX//+1+MGDECJ0+epGcGCCGkATDKLKJ5eXlQqVTw9PTUupqXNaNZRAmxMJpF1OhMPovo+PHjcerUKfV7T09PeHt7qxPA6dOnMX78eEN2TQghxIwMSgJxcXFIT0+v8fObN29iy5YtBgdFCCHEPExy7+bu3bu0MDshhDQAOk8g99tvv+G3335Tv9+wYQMOHDhQrVxBQQEOHDiAp59+2jgREkIIMRmdk0BycjJ27twJgF9399SpUzh79qxGGY7j4OjoiF69emHFihXGjZQQQojRGTQ6SCAQYNu2bRg5cqQpYjIrGh1EiIXR6CCj0+e8ZtB6AiqVyqDACCGEWJeGNaifEEKIURmcBP744w88//zzcHd3h0gkglAorPYihPAYA8pyS1CQno+y3BLU/xHNxoMxoOLoKRRviUfZjt3025iZQbeDfv75ZwwbNgzt2rXD8OHDsW7dOowcORKMMfz2229o2bIlBg8ebORQCWl4ZA/KkB5/Gvd/OwZBRjoEKgVUAhFUgSFw/1d3hAzvAmlTB0uHaRGyP5KgHD8R0uwMiPC/k9G3a6GKAWQ+gRBu+hbSAc9ZOMrGz6CO4c6dO8POzg5Hjx7Fw4cP4eXlhQMHDqBfv37IyMjAM888g88//xyjR482RcxGRR3DxFQyElOQuWAjpFlpYJwQ8ibugEgEKBQQF90Hx5SQ+Yei+ZJJCOzfxtLhmlVBrwFwPpJYZ7ninv3hevgPM0TUuJh82ojk5GQMHz4cQqEQIhF/MVFZWQkACAwMxJtvvonPPvvMkF0T0ihkJKYg892VkGaloyygFWQh4VB5+kLl5gmVpy9kIeEoC2gFaVY6Mt9diYzEFEuHbDa6JgAAcD6SiIJeNS9gRerPoCTg4OAAsVgMgF9gRiKR4N69e+rPvb29cfPmTeNESEgDI3tQxl8BPMxGWXA4OIlUazlOIkVZcDikD7ORuWAjZA/KzByp+cn+SNI5AVRxPpII2R9JJoqIGJQEWrVqheTkZPX7yMhIbN26FQqFAjKZDNu3b0fz5s2NFiQhDUl6/GlIs9JQFhAGro5ZdTmBAOXNwiDJuoEbO86YKULLUY6faFA9xYQ3jBwJqWJQEnjllVfw22+/oaKiAgAwf/58HDp0CK6urvD09MSRI0cwd+5cowZKSEPAGHD/t2NgnLDGK4BqpFKA45C/60ijHhnDGCDNzjCorv29G436t7Eko6wnAABHjhzBL7/8AqFQiIEDB6Jv377G2K3JUccwMaay3BKc6/k2mEAAlaevxmfS8gdwKL8PaPk/TlD0AFAxhHz2BqSujXPyRdmRU7D7/GOD61ck/AaHYbRsrS5M/sSwNj179kTPnj3V74uLi+Hs7Gys3RPSIMiLZRCoFFCKqw/7FMtLwdXU5hIIIFBWQFEmB1x0vIJoYBR378KuHvXlqWmwzcG0pmW0JFAlNzcXK1euxLp16/Dw4UNj754QqyZ2lkIl4IeB1qTE0RtysaPGNoHiHjixCi36RwNeTqYO0yIEhRXAtk0G1xe3DDViNKSKXkkgNzcX33//PdLT0+Hm5oahQ4eiU6dOAICsrCx88skniIuLg0wmQ58+fUwRLyFWzd7TCarAEEiunIHsidtBVZhAACbQ/F/PrqQQFeGdYR/gAXDmiNT87F9/FapR9aj/Gt0KMgWdk8DVq1fRq1cv3L9/H1XdCJ9//jm2bdsGjuMwceJEyGQyDB06FO+99546ORBiSzgOcP9Xd5RePgVWIdOtc1gmAxiDxys9wTXSBADwv43MJ9CgzuFy32A4NeLfxpJ0Hh20YMEClJSU4JtvvsGVK1fw3//+F8HBwZg5cybGjh2LAQMG4Nq1a4iPj6cEQGxayPAukPmHwuH2dbA6ZtxlKhXs71xHhX8wgod1NlOEliPc9K1B9UTfbTByJKSKzlcChw8fxtSpUzF58mQAQNu2bSESiTBgwACMGTMGmzdvNlmQhDQk0qYOaL5kEjLfXQmHG1dQ3iyMHwb6JJkMDneuQ+bmgxYfT7KJOYSkA55DQc/+ej0wVtKrP1xoDiGT0TkJ3L9/H+3bt9fY1qFDBwD8cwOEkEf4uYBm8k8O30kFOA4CkQIcKsHJhZCW3QIYg8w/BC0+noQW0bYzd5Dr4T90njqipFd/uPwfzR1kSjrfDlKpVLCz0xzgVfXeyalxjmYgpD4C+7dBl30fw3lhLCrCOwNMBUFlBTiVChXhneG8MBZd9n1sUwmgiuvhP1C59wDKfYO1fl7uG4zKvQcoAZiBXqODzpw5A+ljl7XFxcXgOA5Hjx5FQUFBtfJDhgypd4CENGTSpg5oO6UX2OReqNizH4q7uRB06wr78NBG3QmsC+mA54C76WAMKN+5G/LUNIhbhsL+tZepE9iMdH5iWFDHHCjVdsxxUCqVBgVlTvTEMDGbY8eABw+Azp0BX+3DRwkxBpM8MXzw4MF6B0YIIcS66JwEevfubco4CCGEWAAtNE8IITaMkgAhhNgwSgKEEGLDrC4JrF27FoGBgZBKpejatStOnz5da/mCggJMmzYNvr6+kEgkCAsLw969e80ULSGENGxGn0q6PhISEhAbG4v169eja9euWLlyJaKjo3Ht2jV4eXlVKy+Xy/H888/Dy8sLP/30E/z9/XHr1i24urqaP3hCCGmArCoJrFixApMmTcK4ceMAAOvXr8eePXuwadMmrctVbtq0CQ8ePMDx48fVTy8HBgaaM2RCCGnQDL4dlJmZiSlTpqBVq1Zo2rQpDh8+DADIz8/H22+/jfPnz+u1P7lcjrNnzyIqKupRcAIBoqKicOLECa11du/ejW7dumHatGnw9vZGeHg4li5dWutDahUVFSgqKtJ4EUKIrTIoCSQnJ6Njx45ISEhAUFAQCgsLofjfSkoeHh44evQo1qxZo9c+8/PzoVQq4e3trbHd29sb2dnZWuvcuHEDP/30E5RKJfbu3YsFCxZg+fLl+PjjmtcxXbZsGVxcXNSvgIAAveIkhJDGxKAkMHv2bLi6uuL69evYtm0bnpx5YuDAgThy5IhRAqyNSqWCl5cXNmzYgE6dOiEmJgbz58/H+vXra6wzb948FBYWql+3b982eZyEEGKtDOoTOHz4MBYuXAhPT0/cv3+/2ufNmzdHVlaWXvv08PCAUChETk6OxvacnBz4+PhorePr6ws7OzsIhUL1tjZt2iA7OxtyuRxisbhaHYlEAolEoldshBDSWBl0JaBSqeDgUPMCGHl5eXqfaMViMTp16oSkpCSN70lKSkK3bt201unevTvS0tKgemz1puvXr8PX11drAiCEEKLJoCTw1FNPYc+ePVo/UygUiI+PxzPPPKP3fmNjY7Fx40Zs2bIFKSkpmDp1KkpLS9WjhUaPHo158+apy0+dOhUPHjzAjBkzcP36dezZswdLly7FtGnTDDksQgixOQbdDpo3bx5eeuklTJ06FcOHDwfA37Y5cOAAli5dipSUFL07hgEgJiYGeXl5WLhwIbKzsxEZGYnExER1Z3FmZqbGlNYBAQHYt28f3nnnHbRv3x7+/v6YMWMG5syZY8hhEUKIzdF5PYEnbd26FTNmzEBhYSEYY+A4DowxNGnSBOvWrcOIESOMHatJ0HoCxGxoPQFiJiZZT+BJo0aNwpAhQ7B//36kpqZCpVIhJCQE0dHRcHZ2NnS3hBBCzMigJFDV8nd0dMTgwYONHBIhhBBzMahjuOre+7Fjx4wdDyGEEDMyKAn07t0bmzZtQq9evdC8eXPMmjWrztk+CSGEWB+DksCPP/6I3NxcxMfHo0uXLli3bh26deuGkJAQvP/++7hw4YKRwySEEGIKBk8gZ29vj9deew0//fQTcnNzsW3bNkREROCrr75Cp06d0Lp1a2PGSQghxASMsqiMo6MjRowYgW3btuGLL76Ak5MTUlNTjbFrQgghJlTv9QTKysqwe/du7NixA4mJiaioqEBISAjefvttY8RHCCHEhAxKAjKZDHv27EFCQgL27t2LsrIyBAYG4u2330ZMTAw6duxo7DgJIYSYgEFJwNPTE2VlZfDz88Mbb7yBmJgYdO3a1dixEUIIMTGDksDYsWMRExODHj16GDseQgghZmRQEli9erWx4yCEEGIBOiWBqvWDe/XqpfG+LlXlCSGEWCedkkCfPn3AcRzKy8shFovV72tSNbdQbQu+E2Jz7twBMjMBd3eaRZRYDZ2SwMGDBwFAvVpX1XtCSB0yM4HPPgP++APIzgaUSkAkAry9gQEDgDlzgObNLR0lsWEGryfQWNB6AsRk1q0DFi4Eior490IhwHH8q7KS39akCfDRR8DUqZaLkzQ6+pzXDHpiuF+/fhprAT/p4MGD6NevnyG7JqRxWLcOmDePTwBuboCXF+DkBNjb8yd+Ly9+e1ERX27dOktHTGyUQUng0KFDyMnJqfHz3Nxc/N///Z/BQRHSoGVm8lcA5eX8/X87O+3l7Oz4z8vL+fKZmeaNkxDUY+6g2jqG09LSaHUxYrs+++zRFYBQWHtZoRBwdeXLL19ulvAIeZzOzwls2bIFW7ZsUb//+OOPsXHjxmrlCgoKcOnSJbz44ovGiZCQhuaPP/g/a7oCeNL/Blzgv/8FVq0yTUyE1EDnJFBWVoa8vDz1++LiYggEmhcSVUtOTpkyBQsXLjRelIQ0FGlpQF4eIJHoV08i4eulpQGhoaaJjRAtdE4CU6dOxdT/jWAICgrCqlWr8PLLL5ssMEIapLw8QKXS/SqgikDAjxjKy6MkQMzKoGkjbt68aew4CGkcPD35E7pKpV89lYqv5+lpmrgIqYFOSSDzf6MWmv/voZZMHUcxNKeHYIitCQ3lT+RZWfrVq6gA/P3pKoCYnU5JIDAwUGPaiKr3daFpI4hNGjAA+PZb/vaOLreF5HL+z0GDTBsXIVrolAQ2bdoEjuNg979/0FXvCSFazJkD7NgBPHzIPwdQ2zBRpRIoKOAfIHv3XbOFSEgVmjaCpo0gplD1xHB5Of8cgFjM3/JRqfj/Fgr5K4CCAv4p4mXLaOoIYjT6nNfqvcbw4+RyOSorK+Ho6GjM3RLS8FSd0Bcu5E/0wKO5g2QymjuIWA2DnhiOj4/HO++8o7Ft8eLFcHJygqurK1555RWUlJQYJUBCGqypU4GzZ4EpU/hOXwBQKADG+PdTpvCfUwIgFmTQ7aCnn34aHTt2xIYNGwAAx48fR48ePTBw4EC0adMGq1evxsyZM7Fs2TKjB2xsdDuImE1CAj8/UPfuwLPPWjoa0oiZ/HZQeno6xowZo36/fft2+Pj4YNeuXRCJRFCpVPj5558bRBIgxGyaNQMcHICgIEtHQoiaQbeDKioqIJVK1e///PNPDBgwACIRn1Patm2LO3fuGCdCQgghJmNQEggKCsKBAwcAAGfOnEFaWhr69++v/jwnJwdOTk7GiZAQQojJGHQ7aPLkyZgxYwaSk5Nx584dNGvWDC+99JL682PHjqFdu3ZGC5IQQohpGJQE3nrrLUilUuzduxedOnXCnDlzYG9vDwB48OABsrOzMWXKFKMGSgghxPjoYTEaHUTM5dgx4MEDoHNnwNfX0tGQRsysD4slJyfj1q1bAIAWLVqgbdu29d0lIYQQMzE4Cfz222+IjY1FRkaGxvagoCCsWLGC1hoghJAGwKDRQXv37sXQoUMBAEuXLsWuXbuwa9cuLF26FIwxDBkyBImJiUYNlBBCiPEZ1CfQrVs3VFRU4MiRI9XmCSotLUWPHj0glUpx4sQJowVqKtQnQMyG+gSImehzXjPoSuDSpUsYM2aM1oniHB0dMXbsWFy6dMmQXRNCCDEjg5KAVCrFgwcPavz8wYMHGk8UE0IIsU4GJYF+/fph1apVWm/3nDp1Cl9//TWioqLqHRwhhBDTMmh00Oeff45u3bqhR48e6NKlC1q1agUAuHbtGk6fPg0vLy989tlnRg2UEEKI8Rk8d9ClS5fw9ttv4+HDh0hISEBCQgIePnyIGTNm4OLFiwgMDDRyqIQQQoxN7ySgVCqRnZ2NJk2a4KuvvsLVq1dRXl6O8vJyXL16FStWrICXl1e9glq7di0CAwMhlUrRtWtXnD59Wqd68fHx4DgOgwcPrtf3E0KIrdA5CTDG8P7778PNzQ3+/v5o0qQJXnnllVo7iA2RkJCA2NhYLFq0COfOnUOHDh0QHR2N3NzcWutlZGRg1qxZ6Nmzp1HjIYSQxkznJBAXF4dPP/0Urq6uGDp0KCIiIvDbb79h3LhxRg1oxYoVmDRpEsaNG4e2bdti/fr1cHBwwKZNm2qso1Qq8frrr2Px4sUIDg42ajyEENKY6ZwE1q1bh44dO+LatWvYsWMHzp49i7feegt79uxBfn6+UYKRy+U4e/asxsgigUCAqKioWh88++ijj+Dl5YUJEybU+R0VFRUoKirSeBFCiK3SOQmkp6dj9OjR6imjAeDNN9+ESqVCamqqUYLJz8+HUqmEt7e3xnZvb29kZ2drrXP06FF899132Lhxo07fsWzZMri4uKhfAQEB9Y6bEEIaKp2TwMOHD+Hp6amxzcPDAwAgk8mMG5WOiouLMWrUKGzcuFEdS13mzZuHwsJC9ev27dsmjpIQQqyXXs8JcBxnqjgA8ElFKBQiJydHY3tOTg58fHyqlU9PT0dGRgYGDRqk3qZSqQAAIpEI165dQ0hIiEYdiUQCiURigugJIaTh0SsJzJ07F8uWLVO/VyqVAICJEydWm0eI4zhcvHhRr2DEYjE6deqEpKQk9TBPlUqFpKQkTJ8+vVr51q1b4/LlyxrbPvjgAxQXF2PVqlV0q4cQQuqgcxLo1auX1iuB+j4T8KTY2FiMGTMGnTt3RpcuXbBy5UqUlpaqRyGNHj0a/v7+WLZsGaRSKcLDwzXqu7q6AkC17YQQQqrTOQkcOnTIhGE8EhMTg7y8PCxcuBDZ2dmIjIxEYmKiurM4MzMTAoFBDzoTQgh5Aq0xTOsJEHOh9QSImZh8PQFCCCGNAyUBQgixYZQECCHEhlESIEbFGFBWBhQU8H/ado/TI4wBsh8SULLia5St3UC/C7Ea1DFMHcNGIZMBFy4AR48CaWmAQgGIREBoKNCjBxAZCdjiiqOydd+BmzENosqKap8p7CRgq9ZCOrXuOa8I0Yc+57V6JYGsrCwcPnwYubm5GDp0KJo1awalUonCwkK4uLhAKBQaumuzoSRQf2lpwHffAenpAMcB7u6AnR1QWQncv8+3gkNCgAkT+KRgK0qaBcM+62ad5cr9g+B054YZIiK2wuSjgxhjiI2NRVBQEF5//XXExsbi+vXrAICSkhIEBgZi9erVhuyaNDBpacCqVfyfoaFAmzaAlxfg5sb/2aYNv/3xcrZA1wQAAPZZN1HSjKZAJ5ZhUBL44osvsGrVKsyaNQv79+/H4xcTLi4uGDJkCH7++WejBUmsk0zGXwFkZwNt2wJisfZyYjH/eXY2X95C8w2ajWzddzongCr2WTchW/ediSIipGYGJYGNGzdi9OjRWLp0KSIjI6t93r59e/WVAWm8LlzgbwG1bMnfBqoNx/FXBOnpgJ5TSjU43IxphtWb+ZaRIyGkbgYlgdu3b+PZZ5+t8XNHR0darKWRY4zvBOa4mq8AniSR8OWPHGm8o4YYg9ZOYF2I5OWN9nch1sugJODl5VXrPPxnz55F8+bNDQ6KWL/ycv7+vru7fvXc3fl65eWmicvSyr9YadH6hOjLoCQwZMgQrF+/HjduPBrRUDXD6J9//om4uDi89tprxomQWCW5nB8GamenXz2RiK8nl5smLkuTX7lcdyET1idEXwYlgcWLF8PX1xeRkZEYPXo0OI7DZ599hh49emDAgAFo37493n//fWPHSqyIWMyf0Csr9atX9fyArreQGhpxeIRF6xOiL4OSgIuLC06ePInZs2cjKysLUqkU//d//4eCggIsWrQIR44cgYODg7FjJVbE3p7v6L1/X7969+/z9R5bqrpRsX9vpkXrE6IvvVYWe5y9vT0++OADfPDBB8aMhzQQHMc/CXz6NH9rR5eWfUUF33Has2fdo4kaKo4DFCIxRAr973cpxPaQNNLfhVgvmjuIGCwykn8SODW17tE+jPEdwiEhQIcOZgnP/MrLgVOnwE18w6DqbCU9YEnMz6ArgfHjx9dZhuM4fPcdPfzSmEml/FQQq1YBycn8bR6JpHq5igo+Afj4ABMnNtI5hDIzgX/+ARQKiF/qj9Jdv0Kac0fn6uXNguBEcwgRCzAoCfz111/V1htWKpW4d+8elEolPD09qy08Txqn0FBgxozqcwdVjQKqmjsoNJRPACEhlo7YyMrL+aff8vL4925uQGQkHLNv6z53ULMgON2muYOIZRh1FtHKykr85z//wcqVK7F//34EBQUZa9cmQxPIGYdMxp8LjxypPotoz578LaBGdwVw6xZ/CaRQAAIB0Lo1EBys0eEhW/cduJlvQSSv/mCEQmwPtnI1zSJKjM5ss4jW5M0338StW7ewZ88eY+/a6CgJGBdjfOO4qrPY3r4RdgKXlfEZLz+ff9+0Kd9BUsvVL2P8g2DyK5chDo+A/XszG9/vQqyGPuc1g0cH1aZDhw7YunWrKXZNrBzHAQ4O/KvRYexR61+pBIRCvvUfFFRnpuM4wGH2TDTGn4U0bCZJAvv376fnBEjjUlrKt/6rHozQofVPSENgUBL46KOPtG4vKCjA4cOHce7cOcydO7degRFiFRgDMjKAlJRHrf82bYDAwEZ4n4vYIoP6BAQC7Y8XuLm5ISQkBBMnTsSkSZOqjSCyRtQnQGpUWsrPl/3gAf/e3Z1v/dNVLrFyJu8TUKlUBgVGSIPAGHDzJnD16qPWf9u2QIsW1PonjY7eTwyXl5cjNjYW//3vf00RDyGWVVICHDvGP/ilVAIeHkCfPnT7hzRael8J2Nvb4z//+Q/atm1ringIsQzGgBs3+Na/SsU/5FDV+iekETPodlCnTp1w5coVY8dCiGWUlPD3/h8+5N97evJPtzXWqU4JeYxBSWDlypV48cUXER4ejrFjx0IkMslIU0JMizF+rotr1x61/tu1A2hVPGJDdB4ddPjwYbRp0waenp6IiIjA/fv3kZOTA4lEAn9/f9g/0WriOA4XG8CK4jQ6yEYVF/Ot/4IC/r2XVyOd24LYIpOMDurbty+2bduGESNGwN3dHR4eHmjVqlW9gyXErFQqvvV//Tr/33Z2fOs/IMDSkRFiETonAcYYqi4aDh06ZKp4CDGdoiK+9V9YyL/39gbat6fWP7FpdDOfNH4qFT+1aWrqo9Z/eDjQrJmlIyPE4vRKAg3hCWBCNBQVAefP838C/Mo2ERHU+ifkf3TuGBYIBHolAY7joFAoDA7MXKhjuJFSqfiWf9Xal2Ix3/r397d0ZISYnMmmjYiKikJYWFi9giPE5AoL+Xv/Va1/X1++9a9t7UtCbJxeSWDMmDEYOXKkqWIhpH5UKn7UT1rao9Z/RATg52fpyAixWtQxTBqHggK+9V9czL/38+MTgFhsyagIsXqUBEjDplLxT/ymp/Otf4mEP/n7+lo6MkIaBEoCpOF6+JBv/ZeU8O/9/fnOX2r9E6IznZMArSFArIZSybf+b9x41Ppv354f/kkI0QtdCZCG5cEDvvVfWsq/b9aMb/3b2Vk0LEIaKkoCpGFQKvm5/m/c4N9LpXzr39vbsnER0sBREiDW7/594OLFR63/gAB+0jdq/RNSb5QEiPVSKoGUFH69X4Bv/XfowE/7TAgxCr3XGDaHtWvXIjAwEFKpFF27dsXp06drLLtx40b07NkTbm5ucHNzQ1RUVK3lSQNx/z5w6NCjBNC8Ob/WLyUAQozK6pJAQkICYmNjsWjRIpw7dw4dOnRAdHQ0cnNztZY/dOgQRowYgYMHD+LEiRMICAjACy+8gKysLDNHToxCoQAuXwaOHwfKyvglHp95hr8CoNs/hBidzhPImUvXrl3x9NNPY82aNQD4oakBAQF46623MHfu3DrrK5VKuLm5Yc2aNRg9enSd5WkCOSuSn8+P/Ckv59+3aMEv9k7LlxKiF5NNIGdqcrkcZ8+exbx589TbBAIBoqKicOLECZ32UVZWhsrKSjRt2lTr5xUVFaioqFC/L6qaZIxYjkIBJCcDt27x7+3tgchIwMPDomERYgus6nZQfn4+lEolvJ8Y9uft7Y3s7Gyd9jFnzhz4+fkhKipK6+fLli2Di4uL+hVAywpaVl4ef++/KgEEBvL3/ikBEGIWVpUE6uvTTz9FfHw8du3aBWkNi4bMmzcPhYWF6tft27fNHCUBAFRW8sM+T57kb/84OADPPsvP+0O3fwgxG6v6v83DwwNCoRA5OTka23NycuBTx5QAX375JT799FMcOHAA7du3r7GcRCKBhOaVt6zcXD4ByGT8+6AgoE0bQCi0bFyE2CCruhIQi8Xo1KkTkpKS1NtUKhWSkpLQrVu3Gut9/vnnWLJkCRITE9G5c2dzhEoMUVnJd/yeOsUnAEdHoHt3ftoHSgCEWIRVXQkAQGxsLMaMGYPOnTujS5cuWLlyJUpLSzFu3DgAwOjRo+Hv749ly5YBAD777DMsXLgQ27dvR2BgoLrvwMnJCU5OThY7DvKEnBzg0qVHrf/gYKB1azr5E2JhVpcEYmJikJeXh4ULFyI7OxuRkZFITExUdxZnZmZCIHh0AbNu3TrI5XK8+uqrGvtZtGgRPvzwQ3OGTrSprASuXAHu3OHfOzryI39qGL1FCDEvq3tOwNzoOQETys7mW/8VFQDH8a3/Vq2o9U+IiTXY5wRIIyGX863/qqe2nZz41r+bm0XDIoRUR0mAGNe9e/y0D1Wt/5AQvvUvsKoxCISQ/6EkQIxDLudP/nfv8u+dnfnWv6urJaMihNSBkgCpv7t3+QQgl/Ot/9BQICyMWv+ENACUBIjhKir4k/+9e/z7Jk341r+Li0XDIoTojpIAMUxWFt/5W9X6b9mSf1Hrn5AGhZIA0U9FBT/ss2pCvyZNgI4d+T8JIQ0OJQGiuzt3+NZ/ZSXf4m/Zkr//T61/QhosSgKkbjIZ3/qvmtjPxYW/90+tf0IaPEoCpHa3bwP//POo9R8Wxo/9p9Y/IY0CJQGinUzGT/dctbazqyvf+nd2tmRUhBAjoyRAqsvM5Fv/CgXf4m/Vim/9c5ylIyOEGBklAfJIeTnf+s/L49+7ufGtf5qSm5BGi5IA4d26xS/2XtX6b92an/WTWv+ENGqUBGxdWRnf+s/P5983bcq3/h0dLRoWIcQ8KAnYKsYetf6VSn6O/9at+fV+qfVPiM2gJGCLysr4tX7v3+ffU+ufEJtFScCWMAZkZAApKY9a/23aAIGB1PonxEZRErAVpaX8vf+q1r+7O9/6d3CwaFiEEMuiJNDYMQbcvAlcvfqo9d+2LdCiBbX+CSGUBBq1khK+9f/gAf/ewwPo0IFa/4QQNUoCjRFjwI0bfOtfpQJEoketf0IIeQwlgcampIQf+fPwIf/e05Nv/dvbWzQsQoh1oiTQWDAGpKcD1649av23awc0b27pyAghVoySQGNQXMy3/gsK+PdeXnzrXyq1ZFSEkAaAkkBDxhiQlgZcv863/u3s+NZ/QIClIyOENBCUBBqqoiK+9V9YyL/39gbat6fWPyFEL5QEGhqVim/9p6Y+av2HhwPNmlk6MkJIA0RJoCEpKgLOn+f/BAAfHyAiglr/hBCDURJoCFQqvuWfmsr3A4jFfOvf39/SkRFCGjhKAtausJC/91/V+vf15Vv/EolFwyKENA6UBKyVSsWP+klLe9T6j4gA/PwsHRkhpBGhJGCNCgr41n9xMf/ez49PAGKxJaMihDRClASsiUrFP/Gbns63/iUS/uTv62vpyAghjRQlAWvx8CHf+i8p4d/7+/Odv9T6J4SYECUBS1Mq+db/jRuPWv/t2/PDPwkhxMQoCVjSgwd867+0lH/frBnf+rezs2hYhBDbQUnAEpRKfq7/Gzf491Ip3/r39rZsXIQQm0NJwNyebP0HBPCTvlHrnxBiAZQEzEWpBFJS+PV+Ab7136EDP+0zIYRYCCUBc7h/n2/9l5Xx75s355d7pNY/IcTCKAkYgDGgvFAOeZkCYgcR7F3E4DgtBRUKvvWfkcG/t7fnW/+enuYM16wYA8rLAbmcH91qbw/tvw0hxCpQEtCDrLgSKX/eRtofqShLywKUKkAogEOoP0IHtESbFwIgdf5f6z4/H7h48VHrv0ULvvUvapw/uUzGX+wcPcrPdKFQ8IcaGgr06AFERtJkp4RYI44xxiwdhCUVFRXBxcUFhYWFaNKkSY3lbpzKw4llh6DIuA3GCcC5uYITicAUCrCHBeCYCqLAAHR7rweCnfOAW7f4ivb2/BnQw8M8B2QBaWnAd9/xDzpzHODuzt/pqqzk74QxBoSEABMm8EmBEGJaup7XAEoCOv1YN07l4cgHiWC5eRCFBEIgqX4vX1VRCeG1K3CXlKL9yAj4tHIBAgOBNm0abesf4BPAqlVAdjbQsqX2B5zlcn4WbB8fYMYMSgSEmJo+SUBgppj0snbtWgQGBkIqlaJr1644ffp0reV37tyJ1q1bQyqVIiIiAnv37jVaLLLiSpxYdggsNw92rUO1JgBOpUDTsjvwbSqH4GE+ruy6DlnE0/y8P404Achk/BVAdjZ/p6umGS7EYv7z7Gy+vExm3jgJITWzuiSQkJCA2NhYLFq0COfOnUOHDh0QHR2N3NxcreWPHz+OESNGYMKECTh//jwGDx6MwYMH48qVK0aJJ+XP21Bk3IYoJBCcoHoPp0RWAO/cy3AsywPHcSgPDkdWiQuuni83yvdbswsX+FtALVvW3fnLcfwVQHo631VCCLEOVpcEVqxYgUmTJmHcuHFo27Yt1q9fDwcHB2zatElr+VWrVqF///5477330KZNGyxZsgRPPfUU1qxZU+9YGAPS/kgF4wTVrgA4lQKuD2/A4/51CJWVUIgkyPNog2KvUDCBEKl7rqMx32hjjO8E5jjd57iTSPjyR46gUf82hDQkVpUE5HI5zp49i6ioKPU2gUCAqKgonDhxQmudEydOaJQHgOjo6BrLV1RUoKioSONVk/JCOcrSssC5uWp+wBg881LgWJYPAChx8kGuZzjkEmcAAOfmirK0LJQXyus65AarvJzvD3B316+euztfr7zxXygR0iBYVRLIz8+HUqmE9xNz6Hh7eyM7O1trnezsbL3KL1u2DC4uLupXQEBAjfHIyxSAUgXuyfv6HIdiZ19167/QpTmYQPjoY5EIUKr4+o2UXM4PA9X3eTeRiK8nb7z5kZAGxaqSgDnMmzcPhYWF6tft27drLCt2EAFCAZii+sm83MEDOV4R6tb/45hCAQgFfP1GSizmT+iVlfrVq3p+gJZJIMQ6WFUS8PDwgFAoRE5Ojsb2nJwc+NQwv76Pj49e5SUSCZo0aaLxqom9ixgOof5gDwu0F+C0/3zsYQEcQv1h79J4z3T29nxH7/37+tW7f5+vZ29vmrgIIfqxqiQgFovRqVMnJCUlqbepVCokJSWhW7duWut069ZNozwA7N+/v8by+uA4IHRAS3BMBVWFbk1epUwOMIaWA8Ma9XQJHMc/CcyY7rd2Kir48j170lQShFgLq0oCABAbG4uNGzdiy5YtSElJwdSpU1FaWopx48YBAEaPHo158+apy8+YMQOJiYlYvnw5rl69ig8//BBnzpzB9OnTjRJPmxcCIAoMgCI9A0xV+5AWpmJQ3rgFu8BmaB3VzCjfb80iI/kngVNT6x7twxjfIRwSwk+fRAixDlaXBGJiYvDll19i4cKFiIyMxIULF5CYmKju/M3MzMS9e/fU5Z999lls374dGzZsQIcOHfDTTz/h119/RXh4uFHikTrbodu8PuC8PFF5NY1v6WuhlMlReTUNnJcnnn2/z6M5hBoxqZSfCsLHB0hO5lv62lRU8J/7+AATJ9IcQoRYE5o2Qs+5gyoz7gAcV23uIDAGu8BmePb9Pgjq0nhnCdVG29xBVaOAHp87aOJE/k9CiGnR3EF60OfHkhVX4uqBO0jdc73aLKItB4ahdVQzm7gC0EYm458EPnKk+iyiPXvyt4DoCoAQ86AkoAd9fqwqOq8nYINoPQFCLE+f81rjHchuQhwHOLiK4eDaeIeAGorjAAcH/kUIsX5W1zFMCCHEfCgJEEKIDaMkQAghNszm+wSq+sVrm02UEEIakqrzmS7jfmw+CRQXFwNArbOJEkJIQ1RcXAwXF5day9j8EFGVSoW7d+/C2dkZnB5jGYuKihAQEIDbt2/rPLS0oWnsx0jH1/A19mM09PgYYyguLoafnx8Egtrv+tv8lYBAIECzZobP81PXTKSNQWM/Rjq+hq+xH6Mhx1fXFUAV6hgmhBAbRkmAEEJsGCUBA0kkEixatAgSicTSoZhMYz9GOr6Gr7EfozmOz+Y7hgkhxJbRlQAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJJALdauXYvAwEBIpVJ07doVp0+frrX8zp070bp1a0ilUkRERGDv3r1mitRw+hzjxo0b0bNnT7i5ucHNzQ1RUVF1/iaWpu/fYZX4+HhwHIfBgwebNsB60vf4CgoKMG3aNPj6+kIikSAsLMzq/53qe4wrV65Eq1atYG9vj4CAALzzzjuQyWRmilY/hw8fxqBBg+Dn5weO4/Drr7/WWefQoUN46qmnIJFIEBoairi4uPoFwYhW8fHxTCwWs02bNrF//vmHTZo0ibm6urKcnByt5Y8dO8aEQiH7/PPPWXJyMvvggw+YnZ0du3z5spkj152+xzhy5Ei2du1adv78eZaSksLGjh3LXFxc2J07d8wcuW70Pb4qN2/eZP7+/qxnz57sX//6l3mCNYC+x1dRUcE6d+7MXnzxRXb06FF28+ZNdujQIXbhwgUzR647fY/xhx9+YBKJhP3www/s5s2bbN++fczX15e98847Zo5cN3v37mXz589nv/zyCwPAdu3aVWv5GzduMAcHBxYbG8uSk5PZ6tWrmVAoZImJiQbHQEmgBl26dGHTpk1Tv1cqlczPz48tW7ZMa/lhw4axgQMHamzr2rUrmzx5sknjrA99j/FJCoWCOTs7sy1btpgqxHox5PgUCgV79tln2bfffsvGjBlj1UlA3+Nbt24dCw4OZnK53Fwh1pu+xzht2jTWr18/jW2xsbGse/fuJo3TGHRJArNnz2bt2rXT2BYTE8Oio6MN/l66HaSFXC7H2bNnERUVpd4mEAgQFRWFEydOaK1z4sQJjfIAEB0dXWN5SzPkGJ9UVlaGyspKNG3a1FRhGszQ4/voo4/g5eWFCRMmmCNMgxlyfLt370a3bt0wbdo0eHt7Izw8HEuXLoVSqTRX2Hox5BifffZZnD17Vn3L6MaNG9i7dy9efPFFs8RsaqY4z9j8BHLa5OfnQ6lUwtvbW2O7t7c3rl69qrVOdna21vLZ2dkmi7M+DDnGJ82ZMwd+fn7V/lFaA0OO7+jRo/juu+9w4cIFM0RYP4Yc340bN/DXX3/h9ddfx969e5GWloY333wTlZWVWLRokTnC1oshxzhy5Ejk5+ejR48eYIxBoVBgypQpeP/9980RssnVdJ4pKipCeXk57O3t9d4nXQkQg3z66aeIj4/Hrl27IJVKLR1OvRUXF2PUqFHYuHEjPDw8LB2OSahUKnh5eWHDhg3o1KkTYmJiMH/+fKxfv97SoRnNoUOHsHTpUnzzzTc4d+4cfvnlF+zZswdLliyxdGhWi64EtPDw8IBQKEROTo7G9pycHPj4+Git4+Pjo1d5SzPkGKt8+eWX+PTTT3HgwAG0b9/elGEaTN/jS09PR0ZGBgYNGqTeplKpAAAikQjXrl1DSEiIaYPWgyF/f76+vrCzs4NQKFRva9OmDbKzsyGXyyEWi00as74MOcYFCxZg1KhRmDhxIgAgIiICpaWleOONNzB//vw659a3djWdZ5o0aWLQVQBAVwJaicVidOrUCUlJSeptKpUKSUlJ6Natm9Y63bp10ygPAPv376+xvKUZcowA8Pnnn2PJkiVITExE586dzRGqQfQ9vtatW+Py5cu4cOGC+vXyyy+jb9++uHDhgtWtPGfI31/37t2RlpamTm4AcP36dfj6+lpdAgAMO8aysrJqJ/qqpMcawTRpJjnPGNyl3MjFx8cziUTC4uLiWHJyMnvjjTeYq6sry87OZowxNmrUKDZ37lx1+WPHjjGRSMS+/PJLlpKSwhYtWtQghojqc4yffvopE4vF7KeffmL37t1Tv4qLiy11CLXS9/ieZO2jg/Q9vszMTObs7MymT5/Orl27xn7//Xfm5eXFPv74Y0sdQp30PcZFixYxZ2dn9uOPP7IbN26wP//8k4WEhLBhw4ZZ6hBqVVxczM6fP8/Onz/PALAVK1aw8+fPs1u3bjHGGJs7dy4bNWqUunzVENH33nuPpaSksLVr19IQUVNavXo1a968OROLxaxLly7s5MmT6s969+7NxowZo1F+x44dLCwsjInFYtauXTu2Z88eM0esP32OsUWLFgxAtdeiRYvMH7iO9P07fJy1JwHG9D++48ePs65duzKJRMKCg4PZJ598whQKhZmj1o8+x1hZWck+/PBDFhISwqRSKQsICGBvvvkme/jwofkD18HBgwe1/j9VdUxjxoxhvXv3rlYnMjKSicViFhwczDZv3lyvGGgqaUIIsWHUJ0AIITaMkgAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJIAIYTYMEoChBBiwygJEEKIDaMkQKzCoUOHwHEcDh06ZOlQTIrjOHz44Yc6lQ0MDMTYsWNNGg8hlARIvcTFxYHjOK2vuXPnWjq8Wj0Zu1QqRVhYGKZPn15tpkZTOX78OD788EMUFBSY5ft0ERgYqPG7ODo6okuXLvj+++8N3ufevXt1Tn7EvGgqaWIUH330EYKCgjS2hYeHWyga/VTFLpPJcPToUaxbtw579+7FlStX4ODgYNTvKi8vh0j06H+748ePY/HixRg7dixcXV01yl67ds1iUx9HRkbi3XffBQDcu3cP3377LcaMGYOKigpMmjRJ7/3t3bsXa9eupURghSgJEKMYMGCAVU8tXZvHY584cSLc3d2xYsUK/PbbbxgxYoRRv0ufBXgkEolRv1sf/v7++Pe//61+P3bsWAQHB+Orr74yKAkQ60W3g4hJ3bp1C2+++SZatWoFe3t7uLu747XXXkNGRkaddVNTUzF06FD4+PhAKpWiWbNmGD58OAoLCzXKbdu2DZ06dYK9vT2aNm2K4cOH4/bt2wbH3K9fPwDAzZs3AQAKhQJLlixBSEgIJBIJAgMD8f7776OiokKj3pkzZxAdHQ0PDw/Y29sjKCgI48eP1yjzeJ/Ahx9+iPfeew8AEBQUpL79UvXbPN4ncObMGXAchy1btlSLd9++feA4Dr///rt6W1ZWFsaPHw9vb29IJBK0a9cOmzZtMvg38fT0ROvWrZGenq6x/ciRI3jttdfQvHlzSCQSBAQE4J133kF5ebm6zNixY7F27Vr18Ve9qqhUKqxcuRLt2rWDVCqFt7c3Jk+ejIcPHxocL9EdXQkQoygsLER+fr7GNg8PD/z99984fvw4hg8fjmbNmiEjIwPr1q1Dnz59kJycXOPtFrlcjujoaFRUVOCtt96Cj48PsrKy8Pvvv6OgoAAuLi4AgE8++QQLFizAsGHDMHHiROTl5WH16tXo1asXzp8/X+0Wiy6qTnTu7u4A+KuDLVu24NVXX8W7776LU6dOYdmyZUhJScGuXbsAALm5uXjhhRfg6emJuXPnwtXVFRkZGfjll19q/J4hQ4bg+vXr+PHHH/HVV1+pl7X09PSsVrZz584IDg7Gjh07MGbMGI3PEhIS4ObmhujoaAD8SlPPPPMMOI7D9OnT4enpiT/++AMTJkxAUVERZs6cqfdvolAocOfOHbi5uWls37lzJ8rKyjB16lS4u7vj9OnTWL16Ne7cuYOdO3cCACZPnoy7d+9i//792Lp1a7V9T548GXFxcRg3bhzefvtt3Lx5E2vWrMH58+dx7Ngx2NnZ6R0v0UO9JqImNm/z5s1a50Ov+qdVVlZWrc6JEycYAPb999+rt1XNq37w4EHGGFMvsrFz584avzsjI4MJhUL2ySefaGy/fPkyE4lE1bbXFPuBAwdYXl4eu337NouPj2fu7u7M3t6e3blzh124cIEBYBMnTtSoO2vWLAaA/fXXX4wxxnbt2sUAsL///rvW78QT6y988cUXDAC7efNmtbItWrTQmCt/3rx5zM7Ojj148EC9raKigrm6urLx48ert02YMIH5+vqy/Px8jf0NHz6cubi4aP07efJ7X3jhBZaXl8fy8vLY5cuX2ahRoxgANm3aNI2y2va1bNkyxnGcemEUxhibNm0a03a6OXLkCAPAfvjhB43tiYmJWrcT46PbQcQo1q5di/3792u8AGise1pZWYn79+8jNDQUrq6uOHfuXI37q2rp79u3D2VlZVrL/PLLL1CpVBg2bBjy8/PVLx8fH7Rs2RIHDx7UKfaoqCh4enoiICAAw4cPh5OTE3bt2gV/f3/s3bsXABAbG6tRp6rTdM+ePQCgvuL4/fffUVlZqdP36ismJgaVlZUaVxd//vknCgoKEBMTA4BfQvHnn3/GoEGDwBjT+F2io6NRWFhY6+/++H49PT3h6emJiIgIbN26FePGjcMXX3yhUe7xv9/S0lLk5+fj2WefBWMM58+fr/N7du7cCRcXFzz//PMasXbq1AlOTk46/x0Sw9HtIGIUXbp00doxXF5ejmXLlmHz5s3IysrSWOf1yXv7jwsKCkJsbCxWrFiBH374AT179sTLL7+Mf//73+oEkZqaCsYYWrZsqXUfut5GWLt2LcLCwiASieDt7Y1WrVqpR+XcunULAoEAoaGhGnV8fHzg6uqKW7duAQB69+6NoUOHYvHixfjqq6/Qp08fDB48GCNHjjRaB2+HDh3QunVrJCQkYMKECQD4W0EeHh7qfoy8vDwUFBRgw4YN2LBhg9b95Obm1vldXbt2xccffwylUokrV67g448/xsOHD6utRZyZmYmFCxdi9+7d1e7h1/b3WyU1NRWFhYXw8vIyOFZSP5QEiEm99dZb2Lx5M2bOnIlu3brBxcUFHMdh+PDhGguea7N8+XKMHTsWv/32G/7880+8/fbbWLZsGU6ePIlmzZpBpVKB4zj88ccf6sXEH+fk5KRTjDUlsMc93pFZ0+c//fQTTp48if/+97/Yt28fxo8fj+XLl+PkyZM6x1KXmJgYfPLJJ8jPz4ezszN2796NESNGqIedVv2m//73v6v1HVRp3759nd/j4eGBqKgoAEB0dDRat26Nl156CatWrVJfFSmVSjz//PN48OAB5syZg9atW8PR0RFZWVkYO3ZsnX+/VfF6eXnhhx9+0Pq5tv4RYlyUBIhJ/fTTTxgzZgyWL1+u3iaTyXR+OCoiIgIRERH44IMPcPz4cXTv3h3r16/Hxx9/jJCQEDDGEBQUhLCwMJPE36JFC6hUKqSmpqJNmzbq7Tk5OSgoKECLFi00yj/zzDN45pln8Mknn2D79u14/fXXER8fj4kTJ2rdf13J5UkxMTFYvHgxfv75Z3h7e6OoqAjDhw9Xf+7p6QlnZ2colUr1SdwYBg4ciN69e2Pp0qWYPHkyHB0dcfnyZVy/fh1btmzB6NGj1WWrbgU+rqbjDAkJwYEDB9C9e3eNW0vEfKhPgJiUUCjUuAUEAKtXr4ZSqay1XlFRERQKhca2iIgICAQC9dDMIUOGQCgUYvHixdW+gzGG+/fv1zv+F198EQCwcuVKje0rVqwAwJ8cAeDhw4fVYoiMjASAakNJH+fo6AgAOifFNm3aICIiAgkJCUhISICvry969eql/lwoFGLo0KH4+eefceXKlWr18/LydPoebebMmYP79+9j48aN6u8CoHHcjDGsWrWqWt2ajnPYsGFQKpVYsmRJtToKhcKqnqRurOhKgJjUSy+9hK1bt8LFxQVt27bFiRMncODAAfXwy5r89ddfmD59Ol577TWEhYVBoVBg69at6pMcwLciP/74Y8ybNw8ZGRkYPHgwnJ2dcfPmTezatQtvvPEGZs2aVa/4O3TogDFjxmDDhg0oKChA7969cfr0aWzZsgWDBw9G3759AQBbtmzBN998g1deeQUhISEoLi7Gxo0b0aRJE3Ui0aZTp04AgPnz52P48OGws7PDoEGD1CdNbWJiYrBw4UJIpVJMmDCh2lPFn376KQ4ePIiuXbti0qRJaNu2LR48eIBz587hwIEDePDggUG/xYABAxAeHo4VK1Zg2rRpaN26NUJCQjBr1ixkZWWhSZMm+Pnnn7WO7686zrfffhvR0dEQCoUYPnw4evfujcmTJ2PZsmW4cOECXnjhBdjZ2SE1NRU7d+7EqlWr8OqrrxoUL9GRZQYlkcaiaphlTUMjHz58yMaNG8c8PDyYk5MTi46OZlevXq02/PHJIaI3btxg48ePZyEhIUwqlbKmTZuyvn37sgMHDlT7jp9//pn16NGDOTo6MkdHR9a6dWs2bdo0du3atXrFXqWyspItXryYBQUFMTs7OxYQEMDmzZvHZDKZusy5c+fYiBEjWPPmzZlEImFeXl7spZdeYmfOnNHYF54YIsoYY0uWLGH+/v5MIBBoDBd98jeqkpqaqh6Ge/ToUa0x5+TksGnTprGAgABmZ2fHfHx82HPPPcc2bNhQ67FWfe/AgQO1fhYXF8cAsM2bNzPGGEtOTmZRUVHMycmJeXh4sEmTJrGLFy9qlGGMMYVCwd566y3m6enJOI6rNlx0w4YNrFOnTsze3p45OzuziIgINnv2bHb37t064yX1wzH2xDUsIYQQm0F9AoQQYsMoCRBCiA2jJEAIITaMkgAhhNgwSgKEEGLDKAkQQogNoyRACCE2jJIAIYTYMEoChBBiwygJEEKIDaMkQAghNoySACGE2LD/Bxx6BBV3rJdWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# another way (1 seed)\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob) # y_prob[:,1]\n",
    "#fpr, tpr, thresholds = roc_curve(y_test, y_prob) # way that usualy works\n",
    "\n",
    "plt.figure(figsize=(4, 4))\n",
    "plt.scatter(fpr, tpr, s=100, alpha=0.5, color=\"blue\", label=\"Scikit-learn\")\n",
    "plt.scatter(ROC[:, 0], ROC[:, 1], color=\"red\", s=100, alpha=0.3, label=\"Our implementation\")\n",
    "plt.plot(ROC[:, 0], ROC[:, 1], color=\"red\", alpha=0.3, label=\"Our implementation\")\n",
    "plt.title(\"ROC Curve\", fontsize=12)\n",
    "plt.xlabel(\"False Positive Rate\", fontsize=12)\n",
    "plt.ylabel(\"True Positive Rate\", fontsize=12)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 939,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUROC is: 0.49999999999999994\n"
     ]
    }
   ],
   "source": [
    "# calc the AUROC of most recent seed\n",
    "partitions = 1000\n",
    "\n",
    "ROC = roc_from_scratch(y_prob, y_test, partitions=partitions) # y_prob[:,1]\n",
    "#ROC = roc_from_scratch(y_prob, y_test, partitions=partitions) # way that usually works\n",
    "fpr, tpr = ROC[:, 0], ROC[:, 1]\n",
    "rectangle_roc = 0\n",
    "for k in range(partitions):\n",
    "        rectangle_roc = rectangle_roc + (fpr[k]- fpr[k + 1]) * tpr[k]\n",
    "print('The AUROC is: ' + str(rectangle_roc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 940,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUROC is: 0.6386054421768709\n"
     ]
    }
   ],
   "source": [
    "# calc the AUROC of aggregated list\n",
    "partitions = 1000\n",
    "\n",
    "ROC = roc_from_scratch(y_probList, y_testList, partitions=partitions) # y_prob[:,1]\n",
    "#ROC = roc_from_scratch(y_prob, y_test, partitions=partitions) # way that usually works\n",
    "fpr, tpr = ROC[:, 0], ROC[:, 1]\n",
    "rectangle_roc = 0\n",
    "for k in range(partitions):\n",
    "        rectangle_roc = rectangle_roc + (fpr[k]- fpr[k + 1]) * tpr[k]\n",
    "print('The AUROC is: ' + str(rectangle_roc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 941,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUROC is 1.0\n",
      "AUROC is 0.5\n",
      "AUROC is 0.5833333333333333\n",
      "AUROC is 0.8333333333333334\n",
      "AUROC is 0.75\n",
      "AUROC is 0.5833333333333334\n",
      "AUROC is 0.49999999999999994\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0oAAAFACAYAAACC3oyiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAACCVUlEQVR4nO3dd3xT1fsH8E+apunee9AWZMsQEARRhpWCwBcEmQItS0CQUUFBkCGjCrIEBEGgICCIiCgosgVkyVBBkCWzdLfpTtMk9/dHfr2SDkjTtEnbz/v1ygty77n3PjdpIU/OOc+RCIIggIiIiIiIiERW5g6AiIiIiIjI0jBRIiIiIiIiKoSJEhERERERUSFMlIiIiIiIiAphokRERERERFQIEyUiIiIiIqJCmCgREREREREVwkSJiIiIiIioECZKREREREREhTBRIiIiIiIiKoSJEhksJiYGEolEfFhbWyMgIACRkZGIjY0t9hhBEPDVV1/h5ZdfhqurK+zt7dGoUSN89NFHyM7OLvFau3fvRpcuXeDp6QkbGxv4+/ujb9++OHLkSHndHhERERGRSCIIgmDuIKhyiImJwdChQ/HRRx8hNDQUSqUSZ86cQUxMDEJCQnDlyhXY2tqK7TUaDQYOHIhvvvkGL730Enr16gV7e3ucOHEC27ZtQ4MGDXDo0CH4+PiIxwiCgGHDhiEmJgbPPfcc3njjDfj6+iIuLg67d+/GhQsX8Ntvv6FNmzbmeAmIiIiIqJqwNncAVPl06dIFLVq0AACMGDECnp6e+OSTT/DDDz+gb9++YruFCxfim2++weTJk7Fo0SJx+1tvvYW+ffuiZ8+eiIyMxM8//yzuW7x4MWJiYjBx4kQsWbIEEolE3Dd9+nR89dVXsLY2749tdnY2HBwczBoDEREREZUvDr2jMnvppZcAALdv3xa35ebmYtGiRahTpw6io6OLHNO9e3dERERg//79OHPmjHhMdHQ06tWrh08//VQvSSowePBgtGzZ8onxaLVaLF++HI0aNYKtrS28vLzQuXNnnD9/HgBw9+5dSCQSxMTEFDlWIpFg9uzZ4vPZs2dDIpHg6tWrGDhwINzc3NC2bVsxvnv37hU5x7Rp02BjY4O0tDRx29mzZ9G5c2e4uLjA3t4e7dq1w2+//fbE+yAiIiIi82GiRGV29+5dAICbm5u47eTJk0hLS8PAgQNL7AEaMmQIAGDv3r3iMampqRg4cCCkUqnR8QwfPhwTJ05EUFAQPvnkE0ydOhW2trZiQmaMPn36ICcnBwsWLMDIkSPRt29fSCQSfPPNN0XafvPNN+jUqZP4ehw5cgQvv/wyMjIyMGvWLCxYsAAKhQIdO3bEuXPnjI6JiIiIiMoPh95RqaWnpyM5ORlKpRJnz57FnDlzIJfL0a1bN7HN1atXAQBNmjQp8TwF+65du6b3Z6NGjYyO7ejRo4iJicH48eOxfPlycfu7776LskzHa9KkCbZt26a37YUXXsCOHTswZcoUcdvvv/+Of//9V+yVEgQBo0ePRocOHfDzzz+LvWSjRo1Cw4YNMWPGDBw4cMDouIiIiIiofLBHiUotLCwMXl5eCAoKwhtvvAEHBwf88MMPCAwMFNtkZmYCAJycnEo8T8G+jIwMvT+fdMzT7Nq1CxKJBLNmzSqyr7ihfIYaPXp0kW39+vXDhQsX9IYc7tixA3K5HD169AAA/PHHH7h58yYGDhyIlJQUJCcnIzk5GdnZ2XjllVdw/PhxaLVao+MiIiIiovLBRIlKbdWqVTh48CC+/fZbvPbaa0hOToZcLtdrU5DsFCRMxSmcTDk7Oz/1mKe5ffs2/P394e7ubvQ5ihMaGlpkW58+fWBlZYUdO3YA0PUe7dy5E126dBHv5ebNmwCAiIgIeHl56T2+/PJL5OXlIT093aSxEhEREVHZcegdlVrLli3Fqnc9e/ZE27ZtMXDgQFy/fh2Ojo4AgPr16wMA/vrrL/Ts2bPY8/z1118AgAYNGgAA6tWrBwC4fPlyiceYQkk9SxqNpsRj7Ozsimzz9/fHSy+9hG+++QYffPABzpw5g/v37+OTTz4R2xT0Fi1atAhNmzYt9twFrxkRERERWQ72KFGZSKVSREdH49GjR1i5cqW4vW3btnB1dcW2bdtKTEA2b94MAOLcprZt28LNzQ1ff/31E5OWJ6lVqxYePXqE1NTUEtsUFFlQKBR624urYPc0/fr1w59//onr169jx44dsLe3R/fu3fXiAXS9ZWFhYcU+ZDJZqa9LREREROWLiRKVWfv27dGyZUssW7YMSqUSAGBvb4/Jkyfj+vXrmD59epFj9u3bh5iYGISHh+OFF14Qj3n//fdx7do1vP/++8UWX9iyZcsTK8X17t0bgiBgzpw5RfYVnM/Z2Rmenp44fvy43v7PP//c8Jt+7HpSqRRff/01du7ciW7duumtsdS8eXPUqlULn376KbKysoocn5SUVOprEhEREVH549A7MokpU6agT58+iImJEQsfTJ06FZcuXcInn3yC06dPo3fv3rCzs8PJkyexZcsW1K9fH5s2bSpynr///huLFy/G0aNH8cYbb8DX1xfx8fH4/vvvce7cOZw6darEODp06IDBgwfjs88+w82bN9G5c2dotVqcOHECHTp0wLhx4wDoFsr9+OOPMWLECLRo0QLHjx/HjRs3Sn3f3t7e6NChA5YsWYLMzEz069dPb7+VlRW+/PJLdOnSBQ0bNsTQoUMREBCA2NhYHD16FM7Ozvjxxx9LfV0iIiIiKmcCkYE2btwoABB+//33Ivs0Go1Qq1YtoVatWoJardbbvnHjRuHFF18UnJ2dBVtbW6Fhw4bCnDlzhKysrBKv9e233wqdOnUS3N3dBWtra8HPz0/o16+fcOzYsafGqVarhUWLFgn16tUTbGxsBC8vL6FLly7ChQsXxDY5OTnC8OHDBRcXF8HJyUno27evkJiYKAAQZs2aJbabNWuWAEBISkoq8Xrr1q0TAAhOTk5Cbm5usW0uXbok9OrVS/Dw8BDkcrkQHBws9O3bVzh8+PBT74eIiIiIKp5EEMqwuAwREREREVEVxDlKREREREREhTBRIiIiIiIiKoSJEhERERERUSFMlIiIiIiIiAphokRERERERFQIEyUiIiIiIqJCqv2Cs1qtFo8ePYKTkxMkEom5wyGiQgRBQGZmJvz9/WFlxe92iIiIqGJU+0Tp0aNHCAoKMncYRPQUDx48QGBgoLnDICIiomqi2idKTk5OAHQfwpydnc0cDREVlpGRgaCgIPF3lYiIiKgiVPtEqWC4nbOzMxMlIgvGobFERERUkTjgn4iIiIiIqBAmSkRERERERIUwUSIiIiIiIirEohKl48ePo3v37vD394dEIsH333//1GOOHTuGZs2aQS6X45lnnkFMTEy5x0lERERERFWbRSVK2dnZaNKkCVatWmVQ+zt37qBr167o0KED/vjjD0ycOBEjRozAL7/8Us6REhERERFRVWZRVe+6dOmCLl26GNx+zZo1CA0NxeLFiwEA9evXx8mTJ7F06VKEh4eXV5hEVEaZmcCWLcDgwYCjo7mjISIiIirKohKl0jp9+jTCwsL0toWHh2PixIklHpOXl4e8vDzxeUZGhsHXy8/PhyAIpY7TklW1exIEAbm5uVXqnnJycqDVas0dhkncvSvDypXATz95Q6WyhSAIePttlv0mIiIiy1OpE6X4+Hj4+PjobfPx8UFGRgZyc3NhZ2dX5Jjo6GjMmTPHqOtVpQ/fBarCPanVaty/fx85OTnQaDR6iXBVkJOTY+4QykSrBS5e9MKPP4bi4kVvvX0rVmgxZowUXCKJiIiILE2lTpSMMW3aNERFRYnPMzIyEBQUVKpzyGSyKrf4ZWW9J41Gg7t37yIzMxO2trawsrKCVquFRCKBjY1NpbynwqRSKQCI91dZZGVJsXevJ3bu9MaDB7aF9ubC2XkvNmzoConE3izxERERET1JpU6UfH19kZCQoLctISEBzs7OxfYmAYBcLodcLi/TdavCh+/CKuM9CYKAe/fuIT4+Hi4uLpDJZNBoNLC2toaVlVWJPwOVTUGi5ODgYOZIDHP7tg22bnXF99+7ICdHP7FzckpBZuYnqF37BI4c2YnAQCZJREREZJkqdaLUunVr/PTTT3rbDh48iNatW5spIqpIaWlpYmIsk8nMHU61ptEAv/7qgC1b3HDqVNGErnXrbPTuHYc2bVJx6pQPBgzYBU9PTzNESkRERGQYi0qUsrKycOvWLfH5nTt38Mcff8Dd3R01atTAtGnTEBsbi82bNwMARo8ejZUrV+K9997DsGHDcOTIEXzzzTfYt2+fuW6BKoharcajR48glUphY2Nj7nCqLYXCCrt2ueDrr13x8KH++2Bnp0X37mkYNCgd3t5JsLa2RkhILbRu3RIqlcpMERMREREZxqISpfPnz6NDhw7i84K5RBEREYiJiUFcXBzu378v7g8NDcW+ffswadIkLF++HIGBgfjyyy9ZGrwaSExMRHp6Otzd3c0dSrV0/boNtmxxw48/OkOp1B9eV6OGCm++qUDjxhcRFTUML7wwBTVqdERISAhcXFzMFDERERFR6UiEqlD2rAwyMjLg4uKC9PR0ODs7P7FtwbfgVakHozLeU05ODq5duwaJRFJk3o5Go4FSqaxSc5Sys7MBmH+OkloNHD7siC1b3PD770XnFr30UhYGDVLgpZeycevWDURGRiI1NRU2Njb48ccf0alTJ7FtaX7uSvM7SkRERGQqFtWjRPQ0giDg0aNHyMvLY29SBUlLk+Kbb3TD6+Lj9eeCOTho8PrrGXjzzTSEhuYDAK5fv47IyEikpaUBAJ599lm0bNmywuMmIiIiKgsmSlSppKamIikpCc7OzpWyUl9l8vffcmzZ4oZ9+5ygUukPrwsNzcOgQQr06JEOR8f/OqULJ0ktWrTAwYMH4erqWpGhExEREZUZEyWqNFQqFWJjY2Ftbc0qd+UkPx84eNAJX33lhkuX9IcuSiQC2rXLxqBBaWjTJgeFl3T6559/EBkZCYVCAQBo2bIlDhw4wHlJREREVCkxUaJKQRAExMXFITMzk0PuykFSkhTffOOK7dtdkZSk/8+Cs7MGvXqlY+BABWrUyC/2+GvXriEyMhLp6ekAgBdeeAH79+9nkkRERESVFhMlqhTS09MRHx8PJycnWBXuyiCj/fmnLbZsccP+/U7Iz9cfyli7dh7efDMN//tfBuztS675cvXqVURGRiIjIwOAbn2z/fv3s/ACERERVWpMlMji5efn4+HDhwAAuVxu5mgqP5VKgp9/dsKWLa64fFl/eJ2VlYBXXsnCm2+moVWrXDxtGpggCLh165ZYma9Nmzb4+eefmSQRERFRpcdEiSxawZA7rplUdgkJ1ti+3QXffOOKlBT9X30XFw369lWgf38FAgLUBp1Pq9VCoVAgPDwcAQEBWLduHX744Qc4OTmVR/hEREREFYqJElk0hUKBuLg4ODo6csidEQQBuHjRDlu2uOLgQSeo1fpdRPXrKzFoUBq6ds2Era3hS6ppNBqkpaXB1dUVoaGhaNy4MQYMGMBKhERERFRlMFEii6VSqfDgwQMAgK2trZmjqVyUSgn27XPCli1uuHZN/7WTSgV06pSJQYMUaNbs6cPrCrt06RLOnz+PkSNHIiQkRBwOySSJiIiIqhImSmSRBEHAw4cPWeWulB49ssbXX7ti504XKBT6v94eHur/H16XDh8fw4bXFXb+/HmMHj0aWVlZCAwMxMSJE00QNREREZHlYaJEFikpKQkJCQlwdnbmkLunEATg7Fk7bN3qhsOHHaHV6vfsNGqUi0GDFOjSJRM2NoYPryvs3LlzGDNmjFi4Yc+ePXjnnXcglUrLFD8RERGRJWKiRBYnOzsbDx8+hI2NDWxsbMwdjsXKyZHgxx+dsWWLG27e1K8GKJMJ6Nw5E4MGpaFJE2WZr3Xq1Cm88847YpLUoUMH/Pjjj0ySiIiIqMpiokQWJT8/H/fv34dSqeSQuxI8eCDDtm2u2LXLBRkZ+omKl5ca/fsr0LevAl5emjJfSxAEnDx5EhMnThSTpI4dO+LHH3+Evb19mc9PREREZKmYKJHFKJiXlJqaCjc3NxYHKOT8eTt8+aU7fv3VAYKg/9o891wuBg9Ow6uvZkImM831BEHAr7/+iqioKOTk5AAAwsLCsGfPHiZJREREVOUxUSKLkZiYiPj4eDg7O3NIVyEXLzpizJggvfLeNjZadO2qG17XsGGeSa+n1Wpx7NgxTJ48WUySXn31VezZswd2dnZPOZqIiIio8mOiRBZBoVDgwYMHkMvlnJdUjG3bvMUkyc8vH/37K9CnTzrc3cs+vK4wtVqNpKQkzJs3T0ySOnXqhO+//55JEhEREVUbTJTI7HJycnDv3j1oNBo4OTmZOxyLo1BIceKEMwDdHKQDB/412fC6wlQqFTIyMuDv74/vv/8e4eHhaNWqFXbv3s21rIiIiKhaYaJEZqVSqXD37l1kZ2ezeEMJDhxwg1qtK5HevXtGuSVJSqUS2dnZ8Pf3R1BQEKytrXHq1CmEhIQwSSIiIqJqhwvUkNmo1WrcvXsXaWlpLN7wBHv3eoh/79EjvVyukZ2djatXryIwMBDBwcGwttZ9h1KvXj0mSURERFQtMVEis9BqtXjw4AGSkpLg6urKRWWLkZ0twS+/OOLvvx0AAPXrK1G3rsrk18nIyMCZM2cwbNgwzJ071+TnJyIiIqqMOPSOKlxBkhQXFwcXFxex96K6S0mR4sIFO/Fx7ZotNJr/etl69Mgw6fUEQYBCocClS5cwefJkKJVKfPnll2jevDlGjx5t0msRERERVTb8hEoVShAExMbGIjY2Fo6OjpCV14QbCycIuoVjz5+3w8WLdjh/3h5375Zc7c/VNd+kiZJWq4VCocBff/2FqKgoKJVKAMD//vc/DB061GTXISIiIqqsmChRhSlIkh4+fAgHBwfI5XJzh1RhNBrg+nX5Yz1G9khKevKvX+3aeWjWLBfPPpuGNm0y4OZmmrlCGo0GCoUCV65cwcSJE8UkqUePHvjmm29Ynp2IiIgITJSoggiCgIcPH+LBgwdwcHCo8gUClEoJ/vrLVuwtunTJFtnZJS+iK5MJaNhQiRYtctCsWS6aNcuFq6sWgK7Qgqmo1WooFApcu3YN48ePR16ebqHanj17YseOHUySiIiIiP4fEyUqdxqNBg8fPhSH21XFniSFwgqXLul6i86ft8fff9siP7/kKn729lo891yumBg1aaKEra1QrjEWrJF0/fp1jBs3TkySXn/9dezYsaPaDoMkIiIiKg4TJSpXarUa9+/fR1xcHJycnKpMkvTokbU4hO7CBTvcvPnk+/LyUqNZs1w0b56DFi1yUadOHiqyhkVubi5ycnJw//59jBkzBiqVrnpe79698fXXXzNJIiIiIiqkTB/V8vLycPHiRSQmJuLFF1+Ep6enqeKiKiAvLw93795FcnIyXFxcKu2Hca0WuH3bRuwtunDBDnFxT76X4GCV2FvUokUuatTIhzHLRAlC2XuZsrKyoFarERwcjNDQUNSsWRP//PMP3njjDWzbtq3Svi9ERERE5cnoROmzzz7D7NmzkZ6uWwDz4MGD6NixI5KTk1GvXj0sXLgQw4YNM1mgVD7y8/NN8mG8sKysLNy7dw/p6elwdXWFIAhiL0Z50mg0yM/Ph5WVFaTSkucEPUl+vgRXr+p6iy5dcsAff9gjPb3kXxWpVEDdurlo1iwHzz2XjWbNcuDhoS50TqNCKRNBEJCRkQGJRILQ0FB4eXlBIpHgyJEjWLJkCRYsWMAkiYiIiKgERiVKGzduxMSJE9G/f3906tRJLyHy9PREx44dsX37diZKlYCpkyRBEJCSkoIHDx4gLy8Pbm5uFr+YbFaWFf780x4XLzrg0iV7XL5sj7y8kmO2tdWiUaMcNGumS4oaN86Bg4O2XGOUlLI7qmCNJBsbG4SEhMDd3V3c5+fnh0WLFpk6RCIiIqIqxahEafHixejRowe2bduGlJSUIvubN2+Ozz77rMzBUcWRyWSl/jBemFqtxqNHj/Do0SNYW1vDx8fHRNEZTqPRQKvVwsrKqsQKbomJUly8+N/8on/+kUOrLfneXV3VaN48VxxGV7++EvqntqypflqtFmlpaXB0dMTVq1cxbdo0fPfdd3BwcDB3aERERESVhlGf8G7duoXx48eXuN/d3b3YBIosV1mTpKysLNy/fx9paWkWVbRBEIC7d2ViUnThgh3u339yCeyAABVatMj9/+ILuahZUwUL7xQTaTQapKWlwc3NDZcvX8aQIUOQn5+P7t27Y+/evbC3tzd3iERERESVglGJkqurK5KTk0vcf/XqVfj6+hoV0KpVq7Bo0SLEx8ejSZMmWLFiBVq2bFli+2XLlmH16tW4f/8+PD098cYbbyA6OrrKr9NjKTQaDRISEhAXFweVSgU3Nzej5waZSmamFfbt88Qff7jg4kU7pKSU/GMukQioXVtXeKF5c11i5OurLrG9JcvPz0d6ejo8PT3x559/YtCgQVCrdffi5+fHNZKIiIiISsGoROm1117D2rVr8fbbbxfZ9/fff2PdunVGzU/asWMHoqKisGbNGrRq1QrLli1DeHg4rl+/Dm9v7yLtt23bhqlTp2LDhg1o06YNbty4gcjISEgkEixZssSYW6NSyMzMxKNHj5CSkgJbW1u9eTDmcu2aLcaPr4VHj4rv0ZLJtGjcWCkOo3vuuVw4O5fv/KKKkJeXh6ysLPj5+eHChQt48803xSRp0KBBiImJMXsCS0RERFSZSAQjZvM/evQIrVq1giAI6N69O9auXYtBgwZBo9Fg165d8PPzw7lz50pdLrxVq1Z4/vnnsXLlSgC6uRZBQUF45513MHXq1CLtx40bh2vXruHw4cPitnfffRdnz57FyZMnDbpmRkYGXFxckJ6eDmdn5ye2LajaVpW+mc/KygIAODo6GnyMSqVCQkIC4uPjoVar4ezsDOuKXBSoBD/95IRp03z1CjE4OWnw3HP/zS9q1EgJubx8F3Y1taf93OXk5ECpVCIgIABnz57FgAEDoNFoAACDBw/Gxo0bLS5JKs3vUml+R4mIiIhMxahPt/7+/rhw4QI++OAD7NixA4Ig4KuvvoKTkxMGDBiAjz/+uNRJkkqlwoULFzBt2jRxm5WVFcLCwnD69Olij2nTpg22bNmCc+fOoWXLlvj333/x008/YfDgwSVeJy8vD3l5eeLzjIyMUsVZnWk0GqSkpCAuLg5ZWVlwcHCwmA+uly/b4v33/ZCfr5tr1ahRNmbPTkK9enmwsBzBpDIzM6HRaBAcHIyTJ09i4MCBYpIUERGB9evXW1ySRERERFQZGN0N4O3tjS+//BJffvklkpKSoNVq4eXlZXQp6OTkZGg0miKV0nx8fPDPP/8Ue8zAgQORnJyMtm3bQhAEqNVqjB49Gh988EGJ14mOjsacOXOMirG60mq1UCgUiI+PF0tOu7u7W0zZ7/R0K0yc+F+S1K1bCj788CFcXKruPDVBEJCeng6pVIpatWrhyJEjeknS0KFDsW7dOiZJREREREYy6pPusGHDcPbsWfG5l5cXfHx8xA/O586dq5A1lI4dO4YFCxbg888/x8WLF/Hdd99h3759mDt3bonHTJs2Denp6eLjwYMH5R6npcp/yiqoWq0WqampuHHjBq5fv47MzEy4urrCycnJYpIkQQA++MAXsbG6IVxNm+Zgxox7sLGpXMPrSqOg/LdcLsczzzwDDw8PbN68WUyShg0bhi+//JJJEhEREVEZGNWjFBMTg7CwMLRq1arY/Xfu3MGmTZuwYcMGg8/p6ekJqVSKhIQEve0JCQklVtD78MMPMXjwYIwYMQIA0KhRI2RnZ+Ott97C9OnTi/0wL5fLLaZ0tblotVokJCQgMTERrq6u8PDw0NuvVquhUCiQmJiI9PR0SCQSi5mHVFhMjBsOH3YCALi4aPDppw8hk5k5qHKk0WigUCjg5OSE0NBQcW7Zzp070aNHDwQHB+OLL76wmESWiIiIqLIql0++jx49gp2dXamOsbGxQfPmzXH48GH07NkTgO4D/eHDhzFu3Lhij8nJySnygbDgW3QjalRUC3l5eXjw4AESExMB6Io52NnZwdHREUqlUkyQsrKyYG1tbbEJEgBcumSLxYu9xOcLF8bBz08NpdKMQZWjgp4kd3d3hISE6P2O2dnZ4YcffoCNjQ2TJCIiIiITMPgT8J49e7Bnzx7x+dq1a3Ho0KEi7RQKBQ4dOoTnn3++1MFERUUhIiICLVq0QMuWLbFs2TJkZ2dj6NChAIAhQ4YgICAA0dHRAIDu3btjyZIleO6559CqVSvcunULH374Ibp3785hR4UIgoC0tDQ8fPgQmZmZcHFxgbW1NWJjY/HgwQNkZ2cjNTUVSqUScrkcbm5uFv2BOy3NClFR/lCrdfOSRo5MQbt22fj/0WdVUl5eHuzt7VGzZk3s27cPLVu2RGBgoLifa4cRERERmY7BidLVq1exc+dOAIBEIsHZs2dx4cIFvTYSiQQODg54+eWXjVrHqF+/fkhKSsLMmTMRHx+Ppk2bYv/+/WKBh/v37+t9eJ8xYwYkEglmzJiB2NhYeHl5oXv37pg/f36pr12V5efnIy4uDnFxcZBIJPDw8IBEokswnJ2dkZqaCpVKBTs7O719lkqrBaZO9UNcnG6MXYsWOZgwoeQFkKuK/Px8ODo6YufOnYiIiEBoaCh+/fVXBAQEmDs0IiIioirHqHWUrKyssGXLFgwcOLA8YqpQVX0dpYyMDDx8+BBpaWlwdHQs0uuQnZ0NAHBwcDBHeEZZt85dHHLn7q7G7t334OOjW1xVo9FAqVTCysqq1MM/LVXBz11WVhbOnDmDd999Vxxa+tFHH+HDDz80Z3hG4TpKREREZOmMmnyi1WpNHQeZmEajQXx8PB49egSNRmNR5bzL4sIFOyxbplujSyIRsHBhnJgkVXX79u3DggULxCRp7NixmDFjhpmjIiIiIqqaLHOWPpVJZmam2Itkb29fZb6FT02VYtIkP2g0uqGBY8akoG3bHDNHVTG+++47vSTpnXfewfLlyy1+mCQRERFRZWV0F8PPP/+MV199FR4eHrC2toZUKi3yoIqlVqsRGxuL69evQ6FQwNXVtcoMP9NqgSlT/JCYqJuX1KpVNsaOTTFzVBXju+++w+zZs8UkacKECUySiIiIiMqZUYnSrl270K1bNyQkJKB///7QarUYMGAA+vfvDzs7OzRu3BgzZ840daz0BBkZGbh58ybu3bsHqVQKd3f3KpWsfvGFO377TTePytNTjU8/jUMVur0S7dy5E7NnzxafT5gwAUuXLmWSRERERFTOjBp6Fx0djZYtW+LkyZNIS0vD6tWrMWzYMHTs2BF3797FCy+8gNDQUFPHSsXIz89HfHw84uPjodFo4OrqWqUSJAA4c8YOK1bo5iVZWQn49NM4eHlV4Trg/+/ixYt6hRqGDh3KJImIiIioghjVo3T16lX0798fUqlUXIw0Pz8fABASEoK3334bn3zyiemipCIK1kW6fv067t+/DxsbG7i5uVW5JCkpSYopU/yh1eqSg7FjU/DCC9VjXtJzzz2HN998EwDQt29fzJs3j0kSERERUQUxqkfJ3t5eLOvr6uoKuVyOuLg4cb+Pjw/u3LljmgipCKVSiUePHiExMRESiaTKVLQrTKMBJk/2Q1KS7se0TZtsjB5dPeYlARDXCHv++efRpEkTLihLREREVIGM+nRdt25dXL16VXzetGlTfPXVV1Cr1VAqldi2bRtq1KhhsiBJR6PRICEhAdeuXUNcXBwcHBzg6upaJZMkAFi1ygNnz+rmJXl752PRoqo/LyktLU3vuUQiQbt27WBtbQ25XG6mqIiIiIiqH6M+Yb/++uvYs2cP8vLyAADTp0/HsWPH4OrqCi8vL5w4cQJTp041aaDVmSAISE9Px40bN3D79m1otVp4eHhUqoVvS+u33+yxerUHAN28pMWL4+DhUbXnJW3duhXh4eG4fPmyuE2r1SIrKws2NjZMlIiIiIgqkEQoqDlcRidOnMB3330HqVSKrl27okOHDqY4bbnLyMiAi4sL0tPTn7rekEqlAoAKTVCUSiXi4uKQlJQErVYLZ2dnk85Dys7OBgA4ODiY7JxllZAgxeuvhyA1VTfkLioqCW+9lWrQsRqNBkqlElZWVpWqNPpXX32F+fPnAwCcnJywZ88eODo6Ij8/H7a2tvD19YW/v7+ZozSd0vwuleZ3lIiIiMhUTLbg7EsvvYSXXnpJfJ6ZmQknJydTnb7aUavVSE5ORlxcHHJzc+Ho6FgtehTUauDdd/3FJKlduyyMGGFYklRZbd68GQsWLBCf9+nTBzKZDHK5HMHBwXB0dBSLphARERFRxTD5p6/ExEQsW7YMq1evLjLfgp6uoJpdXFwcFAoFbG1t4e7uXm2qnX32mSfOn7cHAPj65uPjj+NRRadgAQA2bdqE6Oho8XlkZCTGjx8PX19fcTHngt4XIiIiIqo4pUqUEhMTsXnzZty+fRtubm7o3bs3mjdvDgCIjY3F/PnzERMTA6VSifbt25dHvFVaZmYm4uLikJqaColEUiXLfT/J8eMOWLtWNy/J2lrAkiWP4OZWdeclxcTE4OOPPxafv/XWW5g1axa8vLwgk8nMGBkRERERGZwo/fPPP3j55ZeRkpKCgmlNCxcuxJYtWyCRSDBixAgolUr07t0bU6ZMERMoerrc3FwkJiYiMTER+fn5cHZ2rnYflOPirPHee77i86ioJDRrpjRjROVrw4YNWLhwofg8KioK0dHRVbpABxEREVFlYnCi9OGHHyIrKwuff/45XnrpJdy5cweTJk3CxIkTkZ6eju7du+Pjjz9GzZo1yzPeKkWlUiEpKQkJCQniPKTqOFk9Px+IivKHQqH7cezYMRNDh1bdYZtr167FkiVLxOczZszA3LlzzRgRERERERVmcKJ0/PhxjBkzBqNGjQIANGjQANbW1ujSpQsiIiKwcePGcguyqlGr1UhJSUF8fDyysrJgZ2cHDw+PajMPqbBly7xw6ZKuQl1AgArR0fGoii+FWq1GZmamXq/RRx99hA8//NCMURERERFRcQxOlFJSUtC4cWO9bU2aNAGgW1eJnk6j0SAtLQ3x8fFIT0+HXC6Hu7t7lV0w1hBHjjhg/Xp3AIBMJmDp0ji4uGjNHJVpabVaZGZmQqPRwN3dHVOnTkVwcDBSUlIwffp0c4dHRERERMUwOFHSarVF5s0UPHd0dDRtVFWMVqtFWloaEhISoFAoYG1tXe0TJAB4+NAa06b5ic+nTElE48ZVZ16SIAjIzs6GUqmEi4sL/Pz84ObmBisrK4wePdrc4RERERHRE5Sq6t358+dha2srPs/MzIREIsHJkyehUCiKtO/Vq1eZA6zMtFot0tPTER8fD4VCASsrK7i6ularSnYlUal085LS03WvRadOmRg8WGHeoExIqVQiKysL9vb22Lt3Lxo1aoRnn33W3GERERERkYEkQkEJu6cobe+HRCKBRmP5pZ0zMjLg4uKC9PT0pxZSKFjP5mmVyQoSpISEBKSlpUEikcDJycniFg0VBAE5OTkAAAcHhwq99oIFXti8WTfkLihIhe++uwcnp7IPudNoNFAqlbCysoKdnV2Zz1da+fn5yMzMhLW1NXx8fPDll19i7ty5kEql2LZtG/r27Vuq8wmCgPz8fABP/7mrLEp7T6X5HSUiIiIyFYM/uR89erQ846gU8vPz8aS88vEEqaAHydHREVKpFFqt1iIXDlWr1RAEoUJjO3TIWUySZDItFi26B7lcCVOEoNFokJubC4lEAq224uY6FcxD0mq18PDwgK+vLz777DNxMVmNRoMbN24gKyurwmIiIiIiIuMZnCi1a9euPOOoFEpKkrRaLRQKBRITE8UEydnZuVIMsTOwQ9FkHjywwcyZgeLz996LQ4MGpp2XJAhChVUQLOiVy8vLg4uLC3x8fODm5obo6GgxSQKAefPmYcKECUZfpypWRKyK90RERERVh2WNBaskZDKZOLRQoVAgISEB6enpkEgk8PDwsLghdk9SMASq4J7Kk0olwZQpNZCVpUsgX3stA4MGZUEiMd2QMo1GA5lMBisrK9jb25frPeXl5SEzMxMODg545pln4OXlBalUitmzZ+slSUuWLMGkSZPKLQ4iIiIiMr3K84negkgkEiQnJyM+Ph4ZGRmQSqVwdnauVAlSYRXx7f7HH3vh6lVdMZDgYBU++iihXNdLKq970mg0yMjIgJWVFfz9/eHn5wdbW1sIgoCZM2di3rx5Yttly5aVqSeJiIiIiMyj8n6yNyOlUol79+5BrVazip2Bfv7ZCdu2uQEA5HItli9/BEfHyrVekiAIyMrKgkqlgru7O/z8/ODs7AyJRAJBEPDhhx9i/vz5Yvvly5dj/PjxZoyYiIiIiIzFRMkIgiBAq9XCwcGBSZIB7tyRYcYMH/H5jBmJqFcvz4wRlV5BuW8HBwfUqFEDHh4eeu/9nTt3sGTJEvH5ihUrMG7cOHOESkREREQmUL1XPDWSIAgVWjCgMlMqJZg40R/Z2bqk4n//S8cbb6SbOSrDqdVqpKamQqVSITAwEPXr14e3t3eRBLlmzZrYu3cv7O3tsXLlSiZJRERERJUce5SMUNGV4iqz+fO9cf26bl5SzZp5mDWrfOclmcrjw+w8PDzEYXZP0rFjR9y6dQt+fn4VFCURERERlReje5Tu37+P0aNHo27dunB3d8fx48cBAMnJyRg/fjwuXbpksiAtEXuUnu6HH5yxc6crAMDWVotlyx7BwcHyk8y8vDykpKRAJpPhmWeeQe3atYskSYIg4JdffimSNDNJIiIiIqoajEqUrl69iueeew47duxAaGgo0tPToVarAQCenp44efIkVq5cadJALUnB0Dsq2e3bNpg16795SbNmJaBOHctbcPdxGo0GaWlpUCqVCAwMRL169eDt7Q0rK/1fE0EQMGXKFHTu3Blz5swxU7REREREVJ6MSpTee+89uLq64saNG9iyZUuRpKFr1644ceKESQKkyicnRzcvKTdX9+PVq1c6Xn89w8xRlaxgmJ1CoYCLiwvq1q2L4OBgyOXyYtu+++67WLx4MQBgzpw5uHjxYkWHTERERETlzKg5SsePH8fMmTPh5eWFlJSUIvtr1KiB2NhYowJatWoVFi1ahPj4eDRp0gQrVqxAy5YtS2yvUCgwffp0fPfdd0hNTUVwcDCWLVuG1157zajrG6IgMeTQu+LNneuDmzd1SUbt2nn48MMEM0dUMpVKhYyMDNjb26NWrVrw9PQssZKhIAiYNGkSli9fDkD3/q9btw7NmjWryJCJiKgYgiBArVZDo9GYOxQismBSqRTW1tYGfY43KlHSarWwt7cvcX9SUlKx38Y/zY4dOxAVFYU1a9agVatWWLZsGcLDw3H9+nV4e3sXaa9SqfDqq6/C29sb3377LQICAnDv3j24urqW+tqlwWF3JfvuO2fs3u0CALC3162XZGdnea+XVqtFRkYGBEGAn58f/Pz8YGdnV2J7QRAwYcIErFixAoAuSfryyy8xbNiwigqZiIhKoFKpEBcXh5ycHHOHQkSVgL29Pfz8/GBjY/PEdkYlSs2aNcO+ffvw9ttvF9mnVquxfft2vPDCC6U+75IlSzBy5EgMHToUALBmzRrs27cPGzZswNSpU4u037BhA1JTU3Hq1CnIZDIAQEhISKmvawwWcyjqxg0bfPTRf/OS5syJR82aljcvKTc3F9nZ2XBxcUFAQABcXV2f+F4KgoDx48eL8+4kEgnWr18v/pwSEZH5aLVa3LlzB1KpFP7+/rCxseH/z0RULEEQoFKpkJSUhDt37qB27dpF5qI/zqhEadq0aejWrRvGjBmD/v37AwASEhJw6NAhLFiwANeuXSt1MQeVSoULFy5g2rRp4jYrKyuEhYXh9OnTxR7zww8/oHXr1hg7diz27NkDLy8vDBw4EO+//36Jw6fy8vKQl/ffYqcZGaWfO8NiDkVlZ+vmJSmVuh+2vn0V6N4908xR6VOr1cjIyIBMJkNISAh8fHxgbf3kXwFBEDBu3Dh8/vnnAHRJ0saNGxEREVERIRMR0VOoVCpotVoEBQU9cbQLEREA2NnZQSaT4d69e1CpVLC1tS2xrVGJUpcuXRATE4MJEyZg7dq1AIBBgwZBEAQ4Oztj8+bNePnll0t1zuTkZGg0Gvj4+Oht9/HxwT///FPsMf/++y+OHDmCN998Ez/99BNu3bqFt99+G/n5+Zg1a1axx0RHR5usUhm/sdIRBGD2bF/8+69uuGW9ekp88EGimaP6z+NrInl6esLf3x+Ojo4GHTtz5ky9JCkmJgZDhgwpz3CJiMgIT/pWmIjocYb+e2H0grODBw9Gr169cPDgQdy8eRNarRa1atVCeHg4nJycjD1tqWi1Wnh7e2Pt2rWQSqVo3rw5YmNjsWjRohITpWnTpiEqKkp8npGRgaCgoFJdl71J+nbudMGPP+rWGXJw0GDZskewtbWM1ygvLw+ZmZlwcHBAcHAwPDw8SvWf6ZAhQ7BhwwbExcVh06ZNGDx4cDlGS0RERESWwqhEqWB+joODA3r27GmSQAqqjSUk6FdIS0hIgK+vb7HH+Pn5QSaT6Q2zq1+/PuLj46FSqYqdoCWXy40qNPE4Jkr/uXZNjnnz/iu0MW9eAkJC8s0YkY4gCFAoFAAAf39/+Pv7G/W+165dG8eOHcOlS5fQt29fE0dJRERERJbKqH7qgIAATJgwAb/99pvJArGxsUHz5s1x+PBhcZtWq8Xhw4fRunXrYo958cUXcevWLWi1WnHbjRs3DKpiQWWXlWWFiRP9oVLpfowGDkxDly7mn5ekUqmQmpoKR0dH1KlTByEhIQYnSVqtFvn5+ole7dq1mSQRERERVTNGJUrt2rXDhg0b8PLLL6NGjRqYPHkyzp07V+ZgoqKisG7dOmzatAnXrl3DmDFjkJ2dLVYXGzJkiF6xhzFjxiA1NRUTJkzAjRs3sG/fPixYsABjx44tcyxPwh4l3bykmTN9cO+eLiFt2FCJqVOTzByTgIyMDOTk5MDf3x916tSBm5ubwXPJtFot3nrrLQwcOLBIskRERERE1YtRQ+++/vpr5ObmYu/evdixYwdWr16NpUuXIiQkBP369UPfvn3RtGnTUp+3X79+SEpKwsyZMxEfH4+mTZti//79YoGH+/fv680vCQoKwi+//IJJkyahcePGYk/X+++/b8xtlUp1L+Tw9deu+Okn3bwkJyfdvCQbG/MlkFqtFgqFAjY2NuLCsQUl4w09fsSIEdi4cSMAXQ/n1q1byytcIiIiMlJKSgrq16+Pc+fOVdiyMFR59O/fH88//zzefffdMp/L6BIxdnZ26NOnD7799lskJiZiy5YtaNSoEZYuXYrmzZujXr16Rp133LhxuHfvHvLy8nD27Fm0atVK3Hfs2DHExMTotW/dujXOnDkDpVKJ27dv44MPPiixNLipVPfy4FeuyBEd7SU+nz8/HkFB5uuB0Wg0SE1NhbOzM2rXrg13d/dSJbIajQbDhw8XkySpVGqyuXdERERPEhkZCYlEgtGjRxfZN3bsWEgkEkRGRlZ8YIUUxCmRSCCTyRAaGor33nsPSqVSr92DBw8wbNgwcU2r4OBgTJgwASkpKUXOGR8fj3feeQc1a9aEXC5HUFAQunfvrjcNozjz589Hjx49ik2STp8+DalUiq5duxbZ1759e0ycOLHI9piYGLi6upokNlNYtWoVQkJCYGtri1atWj1x1FZISIj4vjz+eHx01fHjx9G9e3f4+/tDIpHg+++/L/d7AEp3H4bGacj9zpgxA/Pnz0d6enqZ78EktTQdHBwwYMAAbNmyBYsWLYKjoyNu3rxpilNbpOq82GxGhm5eUn6+7kcnIiIVnTplmS2e/Px8pKWlwcvLC8888wwcHBxKdXxBklSQgEulUmzfvh19+vQph2iJiIiKCgoKwvbt25GbmytuUyqV2LZtG2rUqGHGyPR17twZcXFx+Pfff7F06VJ88cUXelWG//33X7Ro0QI3b97E119/jVu3bmHNmjXifPPU1FSx7d27d9G8eXMcOXIEixYtwuXLl7F//3506NDhiVMocnJysH79egwfPrzY/evXr8c777yD48eP49GjR0bdp7GxmcKOHTsQFRWFWbNm4eLFi2jSpAnCw8ORmFj8siu///474uLixMfBgwcBQO9zTHZ2Npo0aYJVq1YZHVf79u2LdFaY8j4MjdOQ+3322WdRq1YtbNmyxeB4S1LmRCknJwfbt29Hr1694O3tjQkTJsDHxwcffPBBmYOzZNWxR0kQgOnTffHwoW5eUuPGuXj3XfPNS1KpVEhPT4efn5/4jU9paDQaDB06FJs2bQIAWFtbY8eOHXjjjTfKI1wiIqJiNWvWDEFBQfjuu+/Ebd999x1q1KiB5557Ttym1WoRHR2N0NBQ2NnZoUmTJvj222/1zrV//360bdsWrq6u8PDwQLdu3XD79m29Nu3bt8f48ePx3nvvwd3dHb6+vpg9e/ZT45TL5fD19UVQUBB69uyJsLAw8YMqoOsBs7GxwYEDB9CuXTvUqFEDXbp0waFDhxAbG4vp06eLbd9++21IJBKcO3cOvXv3Rp06ddCwYUNERUXhzJkzJcbw008/QS6X44UXXiiyLysrCzt27MCYMWPQtWvXUn2wf5yxsZnCkiVLMHLkSAwdOhQNGjTAmjVrYG9vjw0bNhTb3svLC76+vuJj7969qFWrFtq1aye26dKlC+bNm4fXX3+9XGN/XGnvw9A4DblfAOjevTu2b99e5vswKlFSKpXYtWsX+vbtC29vbwwcOBB//PEHxo8fj/Pnz+PGjRuYO3dumYOzVNUxSQKAzZvdcPCgbo0sFxcNlix5BHMVFyxYHykwMBDBwcGwti7ddDuNRoPIyEh89dVXAHRJ0jfffIPevXuXR7hERFTBWrQAAgMr9tGihfHxDhs2TBwCDgAbNmwQi1kViI6OxubNm7FmzRr8/fffmDRpEgYNGoRff/1VbJOdnY2oqCicP38ehw8fhpWVFV5//XW9CsEAsGnTJjg4OODs2bNYuHAhPvroI72k52muXLmCU6dOiVWGU1NT8csvv+Dtt9+GnZ2dXltfX1+8+eab2LFjBwRBQGpqKvbv34+xY8cWOxKk8DC4x504cQLNmzcvdt8333yDevXqoW7duhg0aBA2bNhQ6s9sZYkNABYsWABHR8cnPu7fv1/ssSqVChcuXEBYWJi4zcrKCmFhYTh9+vRTY1epVNiyZQuGDRtm1pFPZb2P0lynpPtt2bIlzp07h7y8vDJdw6hiDl5eXmJlsbfeegv9+vXTm0tUHVS3oXd//mmLRYv+m5cUHR2HwEC1WWLJy8tDVlYWAgMDERgYWOrV2DUaDSIiIsRiDdbW1ti5cyfnJRERVSHx8UBsrLmjMNygQYMwbdo03Lt3DwDw22+/Yfv27Th27BgA3f99CxYswKFDh8RlU2rWrImTJ0/iiy++EL9RL/yF34YNG+Dl5YWrV6/i2WefFbc3btxYHDZXu3ZtrFy5EocPH8arr75aYox79+6Fo6Mj1Go18vLyYGVlhZUrVwIAbt68CUEQUL9+/WKPrV+/PtLS0pCUlIS7d+9CEASj5rPfu3cP/v7+xe5bv349Bg0aBEA3TDA9PR2//vor2rdvb/D5b926ZXRsADB69OinLilSUvzJycnQaDRiEbMCPj4++Oeff5567e+//x4KhcIkc9oWLFiABQsWiM9zc3Nx5swZjBs3Ttx29erVYoeGlvU+DPWk+/X394dKpUJ8fDyCg4ONvoZRiVJkZCT69euHtm3bGn3hyqy6FXNQKKwwaZI/1Gpdcjh8eCo6dsw2SywqlapMSRKg+7bt+vXrAACZTIadO3eiR48epg6ViIjMqIS16i32ml5eXuJwMUEQ0LVrV3h6eor7b926hZycnCKJjEql0hued/PmTcycORNnz55FcnKy2JN0//79IonS4/z8/J44fwQAOnTogNWrVyM7OxtLly6FtbV1kcTMkM9HZfkMlZubC1tb2yLbr1+/jnPnzmH37t0AdF+C9uvXD+vXry9VolTWz3fu7u5wd3cv0zmMtX79enTp0qXERKw0Cid8b775Jnr37o1evXqJ20xxnbJ40v0W9Grm5OSU6RpGJUorVqwo00Uru+pUzEEQgGnT/PDoka7U9nPP5WLiRPPMS8rPz0dGRkaZkiQAcHZ2xoEDB9CtWzdMnToV3bt3N3GkRERkbufPmzuC0hs2bJj4jX3hCe1ZWbrCSfv27UNAQIDevsfn6Hbv3h3BwcFYt24d/P39odVq8eyzz0KlUukdU3gJDYlEUmR4XmEODg545plnAOh6qpo0aSIWVnjmmWcgkUhw7dq1YueYXLt2DW5ubvDy8oK1tTUkEolRvQuenp5IS0srsn39+vVQq9V6H5oFQYBcLsfKlSvh4uICZ2fnYiuhKRQKuLi4AND1rhkbG1C0J6Y4JfXEeHp6QiqVIiEhQW97QkICfJ+Shd+7dw+HDh3Sm+dWFoUTPjs7O3h7e4vv/5OU5T4M9bT7LSgc4uXlVex+QxmUKB0/fhwA8PLLL+s9f5qC9lVRdelR2rDBDUePOgIAXF3VWLr0EUqxPJHJqNVqsXBDUFCQ0UlSATc3N5w4caLM5yEiIjKVzp07Q6VSQSKRIDw8XG9fgwYNIJfLcf/+/SIT1wukpKTg+vXrWLduHV566SUAwMmTJ8slVisrK3zwwQeIiorCwIED4eHhgVdffRWff/45Jk2apDdPKT4+Hlu3bsWQIUMgkUjg7u6O8PBwrFq1CuPHjy8yF0ihUJQ4F+i5554rUs1MrVZj8+bNWLx4MTp16qS3r2fPnvj6668xevRo1K1bFwcOHChyzosXL6JOnToAUKbYgLINvbOxsUHz5s1x+PBhcTqAVqvF4cOH9Ya8FWfjxo3w9vYutix6RSvLfRjqafd75coVBAYG6vXKGsOgRKl9+/aQSCTIzc2FjY2N+LwkBT0uGo2mTMFZquqSJF28aIslS/7LxBctioOvb8XPS9JoNFAoFPDy8kKNGjVKndzk5+fjo48+QlRUFNzc3MTtTJKIiMiSSKVSXLt2Tfz745ycnDB58mRMmjQJWq0Wbdu2RXp6On777Tc4OzsjIiICbm5u8PDwwNq1a+Hn54f79+9j6tSp5RZvnz59MGXKFKxatQqTJ0/GypUr0aZNG4SHh2PevHkIDQ3F33//jSlTpiAgIADz588Xj121ahVefPFFtGzZEh999BEaN24MtVqNgwcPYvXq1eLrUFh4eDimTZuGtLQ08f/0vXv3Ii0tDcOHDxd7hgr07t0b69evx+jRozFmzBisXLkS48ePx4gRIyCXy7Fv3z58/fXX+PHHH8scG1D2oXdRUVGIiIhAixYt0LJlSyxbtgzZ2dliYY+VK1di9+7deus5abVabNy4EREREcUWt8rKysKtW7fE53fu3MEff/wBd3f3EsvPZ2Vlib2YAMQKcvHx8eI2Ly+vEtcufdp9FHcvhsb5tPsFdEU/CifNxjAoUTp69CgAiJVNCp5T1ZWWJkVUlD80Gl1CPGpUCl56qWzjPI0hCAIUCgXc3d0REhJS6up2+fn5GDhwIL799lv8/PPPOHjwoF6yREREZEmcnZ1L3Dd37lx4eXkhOjoa//77L1xdXdGsWTNxSRYrKyts374d48ePx7PPPou6devis88+K9UcndKwtrbGuHHjsHDhQowZMwa1a9fG+fPnMWvWLPTt2xepqanw9fVFz549MWvWLL0EombNmrh48SLmz5+Pd999F3FxcfDy8kLz5s2xevXqEq/ZqFEjNGvWDN988w1GjRoFQDfsLiwsrEiSBOgSpYULF+Kvv/5C48aNcfz4cUyfPh1hYWFQqVSoV68edu7cic6dO5c5NlPo168fkpKSMHPmTMTHx6Np06bYv3+/WBghOTm5SLn3Q4cO4f79+xg2bFix5zx//jw6dOggPo+KigIARERElFhC/dNPP8WcOXOeGOudO3eKXfTXkPso7l4MjfNp96tUKvH9999j//79T4zfEBKhunSPlCAjIwMuLi5IT09/4j9OwH/jgzMzM3Hnzh14eHhURIgmp9EAW7e6Ij1disGD70Mmg17XslYLjB4dgOPHdUPunn8+Bxs3PkApcxSTUCgUsLOzQ+3atYuUGy2ORqNBeno6pFIp7O3tMWDAAOzatQuAbgz3L7/8UuKQBbJMpfkdJaLqR6lU4s6dOwgNDS12kj9VPfv27cOUKVNw5coVjg6hIlavXo3du3cXO8yygKH/bhj109WxY0e9Lr/Cjh49io4dOxpz6kqhMueWKhXw7rt+WLDAB6tWeWLnzqKT3NatcxeTJHd3NT799JFZkqSsrCxYW1sjJCTEoCTpcSqVCv369dNLkvbs2cMkiYiIqJLr2rUr3nrrLcRWpvrvVGFkMpnJCs8ZlSgdO3asSCWLxyUmJuotfkaWQamU4J13ArB//3/fyp84od9N/fvvdli+XDfxTSIR8OmncfDxqfi5Znl5ecjPz0eNGjVK3YugUqkwdOhQsUSora0tfvjhhyITY4mIiKhymjhxIoKCgswdBlmgESNGoG7duiY5l9H9lU8q5nDr1i04OTkZe2qLVxl7lLKyrDByZCB+/dVRb/ulS47IzdW9lykpUrz7rh+0Wt3zt99OQZs2FT8vSaPRIDMzE/7+/qWuVlKQJO3btw+ALkn68ccfTTKhj4iIiIiqD4MHVG3atAmbNm0Sn8+bNw/r1q0r0k6hUOCvv/7Ca6+9ZpoILVBlS5TS0qzw1luBuHxZN3zNwUGDunXzcPGiPfLzrXDpkhM6dtRgyhQ/JCbqan+/8EI23n47pcJjLSje4OnpiYCAgFKtV5WXl4e+ffuKk/fs7Ozw448/4pVXXimvcImIiIioijI4UcrJyUFS0n8LjWZmZhaZQCeRSODg4IDRo0dj5syZpovSwlSmBWeTkqQYNiwIN2/qFqNzcdHgyy8f4NEjGS5etAcAnD7thBs3pDh1SlfQwctLjU8/jUMJFR/LVWZmJhwcHFCjRo0SS06WZOXKldi7dy8AXYXGvXv3Vum5ckRERERUfgxOlMaMGYMxY8YAAEJDQ7F8+XL873//K7fALFll6VGKjbXGsGFBuHdPV9bdy0uN9esfoE4dFYKD82FlJUCrlWDfPg9kZOiSEisrAZ9++gienhU/L0mpVEKj0SAoKKjUxRsAYMKECThz5gx+/PFH7Ny5k0kSERERERnNqFpmd+7cMXUclUplSJTu3JFh2LAgxMXphtL5++dj48YHCA7OBwA4O2vRuLESf/xhh/T0/34M3nknGa1a5VZ4vBqNBllZWQgKCjJ6nSNra2t89dVX+P3339G4cWMTR0hERERE1YlBidL9+/cBQFwVt+D505S02m9lZ+lD765fl2PYsECkpOje3uBgFWJiHsDPT63X7sUXs/HHH//13LRtm41Ro1IrNNYC6enpcHNzg7+/v8GvrVKpRFxcHEJDQ8VtMpkMDRs2LK8wiYiIiKiaMChRCgkJgUQiQW5uLmxsbMTnT6PRVPzwrYpgyT1Kf/5pi5EjA8WhdHXrKrF+/cNih9K1bZuNVat0VeV8fPKxcGEczLFuW3Z2NmQyGYKCgmBt4IJNubm56NmzJy5fvoxjx46hTp065RwlEREREVUnBn0q3bBhAyQSCWQymd7z6spSe5TOnLHD228HIidHl+00aZKLtWsfwsVFW2z7pk2VGDAgEVeu2GP27GS4u1d8YqtWq6FUKhESEmJwSfnc3Fz06NEDBw8eBAD873//w5UrVwxOsoiIqHrIz8+vsC9tpVKp+DmJiKoGgz5ZRkZGPvF5daPVFp94mNOxYw6YMMEfeXm6JKlVq2ysWhULR8eSe78kEmDy5IcAAAcHhwqJs7D09HR4eHjAx8fHoPY5OTno0aMHDh06BABwdHTE+vXrmSQREZGe/Px8XL9+Hbm5FTPv1s7ODnXr1q02yVL79u3RtGlTLFu2zCLOQ1QeTDrQSqVSITs725SntFiW1KO0f78jxo0LEJOkdu2y8MUXT06SLEF2djbkcjkCAgIMKgWek5OD//3vf2KS5OTkhF9++QUvvvhieYdKRESVjEajQW5uLqytrWFra1uuD2tra+Tm5pplysHx48fRvXt3cY7v999/X6bztW/fHhMnTjRJbESVnVGJ0vbt2zFp0iS9bXPmzIGjoyNcXV3x+uuvIysryyQBWiJLmqO0a5czoqL8oVbrErfXXsvAypWxsLW1nBiLU/AfmJ+fHxwdHZ/aPicnB927d8fhw4cB/JcktWnTprxDJSKiSsza2ho2Njbl+jDlqIb27dsjJibG4PbZ2dlo0qQJVq1aZbIYiEjHqERp8eLFej1Hp06dwpw5cxAeHo5JkyZh//79mD9/vsmCtDSWkiht3uyK6dP9oNXqkqQ33lBg0aI4VIZe/4yMDLi7uxs05C47OxvdunXDkSNHAADOzs44cOAAWrduXd5hEhERWbQuXbpg3rx5eP311w0+5ttvv0WjRo1gZ2cHDw8PhIWFITs7G5GRkfj111+xfPlySCQSSCQS3L17F9nZ2RgyZAgcHR3h5+eHxYsXGxWrIefRarWIjo5GaGgo7Ozs0KRJE3z77bcAgLVr18Lf37/IFIgePXpg2LBhRsVE9CRGJUq3b9/WW6dm27Zt8PX1xe7du7Fw4UKMHTsWu3btMlmQlsbcxRwEAVi92h0LFvyXZEREpGLu3AQYMILN7JRKJaRSqUFD7vLz89GtWzccPXoUwH9J0gsvvFARoRIREVUpcXFxGDBgAIYNG4Zr167h2LFj6NWrFwRBwPLly9G6dWuMHDkScXFxiIuLQ1BQEKZMmYJff/0Ve/bswYEDB3Ds2DFcvHix1Nc25DzR0dHYvHkz1qxZg7///huTJk3CoEGD8Ouvv6JPnz5ISUkRPxMAQGpqKvbv348333yzzK8NUWFG9RXn5eXB1tZWfH7gwAF06dJF7Hpu0KABPv/8c9NEaIHMWcxBEIBPP/XC+vXu4ra3307GO++kwIKmTZVIq9WKC8s6Ozs/tb1MJkOHDh1w7NgxuLi44MCBA2jZsmUFREpERFT+FixYgAULFojPc3NzcebMGYwbN07cdvXqVZOtTRkXFwe1Wo1evXohODgYANCoUSNxv42NDezt7eHr6wsAyMrKwvr167Flyxa88sorAIBNmzYhMDCwVNc15Dx5eXlYsGABDh06JI4aqVmzJk6ePIkvvvgC27ZtQ5cuXbBt2zbxHN9++y08PT3RoUMHI18RopIZlSiFhobi0KFDGDFiBM6fP49bt27pDbVLSEgwaN5JZWaOHiWtFvjoIx9s3+4qbpsyJRHDh6dVeCzGysrKgpOTk/gPsCFmzpwJGxsbvPLKK3j++efLMToiIqKKNXr0aPTt21d8/uabb6J3797o1auXuM3f399k12vSpAleeeUVNGrUCOHh4ejUqRPeeOMNuLm5Fdv+9u3bUKlUaNWqlbjN3d0ddevWLdV1DTnPrVu3kJOTg1dffVXvWJVKheeeew6A7vUZOXIkPv/8c8jlcmzduhX9+/eHlTkWgqQqz6hEadSoUZgwYQKuXr2Khw8fIjAwEN26dRP3//bbb2jYsKHJgrQ05pijpFYDH3zgix9+cAEASCQCZs9OQL9+6RUei7HUajXUajVCQ0NhY2NTYrvihjZOnTq1vMMjIiKqcO7u7nB3/2+UiJ2dHby9vfHMM8+Uy/WkUikOHjyIU6dO4cCBA1ixYgWmT5+Os2fPIjQ0tFyuaaiCQmD79u1DQECA3j65XA4A6N69OwRBwL59+/D888/jxIkTWLp0aYXHStWDUen3O++8gy+++AK1atVCjx49cODAAdjZ2QHQjRWNj4+v0mNFKzpRUqkkmDjRX0ySpFIBCxfGVaokCdAVcPDw8ND7D6G4Np06dRKr2xEREZFpSSQSvPjii5gzZw4uXboEGxsb7N69G4Bu6N3jZc5r1aoFmUyGs2fPitvS0tJw48aNUl3TkPM0aNAAcrkc9+/fxzPPPKP3CAoKAgDY2tqiV69e2Lp1K77++mvUrVsXzZo1M+p1IHoao+tZjhw5EiNHjiyy3d3dHefPny9TUJauIos55ORIMG5cAE6d0i0IK5NpsWxZHF55pXKVX8/NzYVMJoO/v3+J3eMZGRno3LkzTp8+jd9++w179+5Fx44dKzhSIiKqStRqtUVfIysrS29Jle3btwMA4uPjxW1eXl4lFj/KysrCrVu3xOd37tzBH3/8AXd392LnNZ09exaHDx9Gp06d4O3tjbNnzyIpKQn169cHAISEhODs2bO4e/cuHB0d4e7ujuHDh2PKlCnw8PCAt7c3pk+fXuT/8pUrV2L37t0lftHp6Oj41PM4OTlh8uTJmDRpErRaLdq2bYv09HT89ttvcHZ2RkREBADd8Ltu3brh77//xqBBg4pc62mxEBmqzIX/r169inv37gEAgoOD0aBBgzIHZekqqphDRoYVRo0KxKVLut46OzstVq2KRZs2ORVyfVMRBAHZ2dmoUaNGiXPX0tPT0blzZ5w5cwaAbuhBSeOliYiInkYqlcLOzg65ubkVkizZ2dkZtHh6YZ9++inmzJnzxDZ37txBSEhIsfvOnz+vV8ggKioKABAREVHsekzOzs44fvw4li1bhoyMDAQHB2Px4sXo0qULAGDy5MmIiIhAgwYNkJubizt37mDRokXIyspC9+7d4eTkhHfffRfp6fqjWpKTk3H79u0n3och55k7dy68vLwQHR2Nf//9F66urmjWrBk++OADsU3Hjh3h7u6O69evY+DAgUWuY0gsRIaQCEaOI9uzZw+ioqJw9+5dve2hoaFYsmQJ/ve//xkd1KpVq7Bo0SLEx8ejSZMmWLFihUGVzrZv344BAwagR48eBq9MnZGRARcXF6Snpz+1ClvBNz7//vsv1Go1HBwcDLqGMVJTpRgxIhBXr+qqCzo6arB2bSyaNcs16XUK1sMqz3vJzMyETCZD/fr1i52blJ6ejvDwcLE73t3dHYcPH0bTpk1LfS2NRoP09HRIpVK4uLiUNXSyAKX5HSWi6kepVOLOnTsIDQ3Vq8gL6JaYeHwYWXmSSqWQVYaFDInoif9uPM6oHqWffvoJvXv3RnBwMBYsWCB21167dg1r165Fr169sHfvXnTu3LnU596xYweioqKwZs0atGrVCsuWLUN4eDiuX78Ob2/vEo+7e/cuJk+ejJdeesmYWyqV8p6jlJBgjWHDAnH7tm7iopubGuvXP0SDBnnlet3yoNFooFKpUKNGjWKTJIVCgfDwcJw7dw4A4OHhgcOHD6NJkyYVHSoREVUxMpmMyQsRGc2oHqXWrVsjLy8PJ06cKNITkZ2djbZt28LW1hanT58udUCtWrXC888/j5UrVwLQDXMLCgrCO++8U2LlM41Gg5dffhnDhg3DiRMnoFAoyq1HSRAE/Pvvv9BoNOXSC/PwoQyRkYF4+FCXVHh752PjxoeoVUtl8msB5d+jlJaWBmdnZ9StW7fIkASFQoFOnTrh999/BwB4enri8OHDeosZlxZ7lKoe9igR0ZMY+s0wEVEBQ//dMKrq3V9//YWIiIhiP1w7ODggMjISf/31V6nPq1KpcOHCBYSFhf0XoJUVwsLCnph0ffTRR/D29sbw4cOfeo28vDxkZGToPUqrvIo53L5tg4EDg8QkKTBQhW3bHpRbklTe8vPzAQB+fn5FkqS0tDS8+uqrYpLk5eWFo0ePlilJIiIiIiIyFaMSJVtbW6Smppa4PzU11ahvdZKTk6HRaODj46O33cfHR6/6y+NOnjyJ9evXY926dQZdIzo6Gi4uLuKjoNykoQRBKJehd1evyjFoUBASE3VDBGrVysPWrQ8QGJhv8mtVlIJy4K6urkX2nTt3DpcuXQKgS5KOHDmCZ599toIjJCIiIiIqnlGJUseOHbF8+fJie3nOnj2Lzz77TK9XqLxkZmZi8ODBWLduHTw9PQ06Ztq0aUhPTxcfDx48KNU1C5IkU/YoXbxohyFDgpCWppsy1qCBElu2PICPT/lX6SkvSqUSMpkMvr6+xb5W4eHh2LZtG/z9/XH06FEmSUREVCbmWAyeiConQ/+9MKqYw8KFC9G6dWu0bdsWLVu2RN26dQEA169fx7lz5+Dt7Y1PPvmk1Of19PSEVCpFQkKC3vaEhAT4+voWaX/79m3cvXsX3bt3F7cVlO62trbG9evXUatWLb1j5HK5uLqzJTh1yh5jxwYgN1eXszZrloMvvoiFk1PFlCAvD4IgICsrC4GBgXByciqxXd++fdGtWzfY29tXYHRERFSVFBRryMnJgZ2dnZmjIaLKICdHt9TO04q9GJUohYaG4q+//kJ0dDR+/vln7NixA4BuHaUJEyZg6tSpT6xQVxIbGxs0b94chw8fRs+ePQHoEp/Dhw9j3LhxRdrXq1cPly9f1ts2Y8YMZGZmYvny5aUeVmcIU/YoHT7siIkT/ZCfr0uSXnwxGytWxMLevnJ/K5abmwtbW1u9n4GUlBQcOnQI/fr102vLJImIiMpCKpXC1dUViYmJAHT/r1TUovBEVLkIgoCcnBwkJibC1dX1qWuflTpR0mg0SEpKgqurK5YuXYqlS5caHWxxoqKiEBERgRYtWqBly5ZYtmwZsrOzMXToUADAkCFDEBAQgOjoaNja2hYZslUwH6Y8h3KZonv/xx+dMHWqHzQa3T/mr76aicWL42BjU7mTpIIfwODgYPGbveTkZISFheHPP/9EWloaRo8ebeYoiYioKikYdVKQLBERPYmrq2uxo9UKMzhREgQB06dPx8qVK5GdnQ2pVIquXbti/fr1cHd3L1Owj+vXrx+SkpIwc+ZMxMfHo2nTpti/f79Y4OH+/fuwsjJqapVJmKKYw/btLpgzxweCoEuS/ve/dCxYEA9ro/r3LEt2djbs7e3h5eUFQJckvfLKK2IVxLlz5+LNN9984pA8IiKi0pBIJPDz84O3t7dYcZWIqDgymeypPUkFDF5HaePGjRg+fDgCAwPxwgsv4Pbt27h06RK6d++OPXv2lClgcyrtOkoqlQp37tyBTCYzaq7T+vVuWLTovyFp/fsrMHNmAsyV+5lyHSWtVovU1FSEhobC398fSUlJeOWVV8ThkQWFG+rUqVPma5WE6yhVPVxHiYiIiMzB4D6M1atX47nnnsPJkyfFIVUTJkzAqlWrkJycbHDVuepKEIAVKzzw+ef/vU4jRqTg3XeTUVWGUmdnZ8PR0RFeXl5ITEzEK6+8gitXrgAAAgICcPToUdSuXdvMURIRERERPZ3B/Ri3b9/GkCFD9CrKvP3229Bqtbh582a5BGeJCobelWaiqCAA0dFeeknSxIlJVSpJ0mq1yMvLg6+vL1JTU9GhQwcxSQoMDMSxY8eYJBERERFRpWFwj1JaWpo476RAQS+SUqk0bVRViEYDzJrlg2+/dRW3ffBBAoYMUZgtpvKQlZUFJycnqNVqdOrUCVevXgXwX5JUuEw7EREREZElK1X5AJbbLF0xB5UKeP99P/z8s25ehZWVgLlz49G7d0Z5hljhtFot8vPzERwcjNdff11MkoKCgnD06FEmSURERERU6RhczMHKygpBQUF6E+Q1Gg2uXbuG0NDQIsUAJBIJ/vzzT9NGWw5KW8whNzcXd+/ehZ2d3RMXqVIqJZgwwR+//uoIALC2FvDpp4/QuXOWSeMvK1MUc8jIyICdnR3q1auHP/74A2FhYXBxccHRo0dRs2ZNU4VqEBZzqHpYzIGIiIjMweAepZdffrnYHiVjFpat6rKyJBg7NgBnz+qSD7lci88+e4R27bLNHJnpaTQa5OfnIyQkBNbW1mjRogUOHToEDw8PhIaGVng8Wq22wq9JRERERFWPwYnSsWPHyjGMyuNpxRwUCiuMGhWIP//UFb2wt9di9eqHaNUqtyLDNFhubm6Z1oV68OABfH19YWNjg6wsXW9ZvXr1AEB8XpE0Gk2FX5OIiIiIqh7zrdxaBSUnSxERESQmSS4uGsTEPLDYJAlAmZKkuLg4jB07FitWrDDrIsDFsbR4iIiIiKhyKVUxByq5mMOjR9YYNiwId+/aAAA8PdVYv/4h6tbNq+gQjWJnZ1eq5CI+Ph5jxoxBbGwstm7dirp16+LDDz8sxwgNo9FooNFoDF5xmYiIiIioOEyUSqkgSXp86N29ezIMHRqER490xR38/PKxYcMDhIbmmyVGY5QmSYqLi8OQIUPw4MEDAEDNmjURGRlZTpEREREREVU8jk8ywuM9Sjdu2GDQoBpikhQcrMLWrfcrVZJUGo8ePdJLkmrVqoVjx44hKCjIzJEREREREZkOe5RK6fEepcuXbTFiRCDS03XDvGrXzsOGDQ/g5VU1CwoUJEkPHz4EAISGhuLYsWMIDAw0c2RERERERKbFHiUj/f67PSIj/0uSGjXKxebN96tskhQbG6uXJNWoUQNHjx5lkkREREREVVKZepRiY2Nx/PhxJCYmonfv3ggMDBQX/HRxcamSE+oFQcCpUy744IMg5OXp8sznn8/B6tUP4ehofAU5S1aQJMXGxgIAgoKCsGfPHgQHB5s5MiIiIiKi8mFUj5IgCIiKikJoaCjefPNNREVF4caNGwB0a+eEhIRgxYoVJg3UUty/L8X779cVk6SXX87CunVVN0kCAGtra1hb63LqGjVqYP369WjYsKGZoyIiIiIiKj9GJUqLFi3C8uXLMXnyZBw8eFCvuIGLiwt69eqFXbt2mSxIS3L4sBxqte5lCwvLxMqVsbC1rbpJEgD4+Phg8+bNaN++PT777DM0atQIMpnM3GEREREREZUbo4berVu3DkOGDMGCBQuQkpJSZH/jxo3x888/lzk4S5T72Nqx3btnwMbGfLFUJB8fHyxevBhWVlZwd3c3dzhEREREROXKqB6lBw8eoE2bNiXud3BwQEZGhtFBWTKl8r/1k6pqT9L9+/fx/vvvQ6lUitsEQUBubi68vLxgU12yQyIiIiKqtozqUfL29hbX0SnOhQsXUKNGDaODsmSPJ0pyudaMkZSPe/fuISIiAvHx8UhKSsLnn38OW1tb5Obmws7ODh4eHuYOkYiIiIio3BnVo9SrVy+sWbMG//77r7hNItElEAcOHEBMTAz69OljmggtjH6iVLV6lO7evYshQ4YgPj4eAJCUlIScnBwAEHuTbG1tzRkiEREREVGFMCpRmjNnDvz8/NC0aVMMGTIEEokEn3zyCdq2bYsuXbqgcePG+OCDD0wdq0V4bDRalRp6V5AkJSQkAABq166NmJgYuLu7Q6lUQi6XszeJiIiIiKoNoxIlFxcXnDlzBu+99x5iY2Nha2uLX3/9FQqFArNmzcKJEydgb29v6lgtQm5u1Rt6d/fuXQwePBiJiYkAgDp16mDTpk1iYpSdnQ13d/cq+54SERERERVm9IKzdnZ2mDFjBmbMmGHKeCxeXl7VKuZw9+5djB8/HmlpaQCAunXrIiYmBm5ubgCAvLw8yGQyeHl5mTNMIiIiIqIKZVSPUnVWlare/fvvv3pJUr169fSSJEDXm+Tm5gYHBwdzhUlEREREVOGM6lEaNmzYU9tIJBKsX7/emNNbtMcTJRubyj307vPPPxeTpPr162PDhg16SVJ+fj4kEgm8vLzEYh1ERERERNWBUYnSkSNHinxw1mg0iIuLg0ajgZeXV5XtgahKPUpz587FgwcPkJubi40bN8LV1VVvf1ZWFtzc3ODs7GyeAImIiIiIzMSoROnu3bvFbs/Pz8cXX3yBZcuW4eDBg2WJy2IVJErW1gKsjZ7hZX5arRZ5eXn4+OOPodVqiyRJGo0GgiCwN4mIiIiIqiWTzlGSyWQYN24cOnXqhHHjxpny1BajoJhDZax4d+vWLSQkJEAQBKSlpcHKygpWVlbF9hhlZWXB2dkZLi4uZoiUiIiIiMi8yqWYQ5MmTXD8+PHyOLXZFfQoVbZhd9evX8eQIUMQGRmJ27dvw9HREQ4ODlCr1UXaarVaqNVqeHt7QyqVmiFaIiIiIiLzKpdE6eDBg1V2zZ3/epQqT6J0/fp1REZGIjU1FXfu3MGqVasQEhICJyenYhOlnJwcODg46BV2ICIiIiKqToyaZfPRRx8Vu12hUOD48eO4ePEipk6dWqbALFVBj1JlGXr3zz//IDIyEgqFAgDQsGFDrFy5Es7OzsjLyyvSXhAEKJVK1KxZE9aVeRIWEREREVEZGPVJePbs2cVud3NzQ61atbBmzRqMHDnS6KBWrVqFRYsWIT4+Hk2aNMGKFSvQsmXLYtuuW7cOmzdvxpUrVwAAzZs3x4IFC0psX1aVaejdtWvXEBkZifT0dAC6JGnnzp0IDQ0FANjY2EAikUCr/S/py83NhZ2dHdzd3c0SMxERERGRJTAqUXr8g7Wp7dixA1FRUVizZg1atWqFZcuWITw8HNevX4e3t3eR9seOHcOAAQPQpk0b2Nra4pNPPkGnTp3w999/IyAgwKSxaTRAfr5utKKlD727evUqhg4dKiZJzz77LLZu3Yq6deuKVezkcjlkMhk0Go14XE5ODoKCgiCXy80SNxERERGRJSj1HKXc3FxERUXhxx9/LI94sGTJEowcORJDhw5FgwYNsGbNGtjb22PDhg3Ftt+6dSvefvttNG3aFPXq1cOXX34JrVaLw4cPmzw2pfK/v9vaWu7Qu7///lsvSWrUqBFiYmLQoEEDWFn995bb2NjA2toa+fn5AIC8vDzY2NjAw8PDLHETEREREVmKUidKdnZ2+OKLL5CQkGDyYFQqFS5cuICwsDBxm5WVFcLCwnD69GmDzpGTk4P8/PwSh47l5eUhIyND72GoxxMlS+1RevjwoV6S1LhxY6xbtw7PPvtskTlHVlZWsLOzEws6ZGdnw93dvcouFkxEREREZCijqt41b95cnBNkSsnJydBoNPDx8dHb7uPjg/j4eIPO8f7778Pf318v2XpcdHQ0XFxcxEdQUJDB8RXMTwIsN1Hy9/dHly5dAOjKtK9cuRINGjQocSidra0t1Go18vPzIZFI4OnpWZHhEhERERFZJKPmKC1btgyvvfYann32WURGRlpMdbSPP/4Y27dvx7Fjx2Bra1tsm2nTpiEqKkp8npGRYXCylJv7398tdeidlZUVZs2ahYCAAHTt2hUNGjSAk5NTie1tbGwA6HqTXF1di118loiIiIioujE4wzl+/Djq168PLy8vREREwMrKCqNGjcL48eMREBAAOzs7vfYSiQR//vlnqYLx9PSEVCotMqwvISEBvr6+Tzz2008/xccff4xDhw6hcePGJbaTy+VGFyqw1B6l/Px8yGQy8blGo0Hv3r0RHBz81PlGcrkcEokEGo0GXl5eYqEHIiIiIqLqzOChdx06dMChQ4cAAB4eHqhbty5efvlltGrVCoGBgfDw8NB7GFNe2sbGBs2bN9crxFBQmKF169YlHrdw4ULMnTsX+/fvR4sWLUp9XUPp9yhZRqL0xx9/oHPnzvj7778B6JKk9PR0+Pr6ws/P76nH29jYQCaTwdHREa6uruUcLRERERFR5WBwj5IgCBAEXXJw7Nix8ooHUVFRiIiIQIsWLdCyZUssW7YM2dnZGDp0KABgyJAhCAgIQHR0NADgk08+wcyZM7Ft2zaEhISIc5kcHR3h6Oho0tj0e5TMP/Tu0qVLGDFiBLKzszFs2DBs2bIF7u7u8PDwQFBQkF6Fu5LY2NjAzs4OPj4+kEqlFRA1EREREZHls4zJRY/p168fkpKSMHPmTMTHx6Np06bYv3+/WODh/v37egnA6tWroVKp8MYbb+idZ9asWSUujGssS6p6d/HiRYwcORLZ2dkAgPr168PJyQmOjo4IDg7WG4r3JFZWVggICGARByIiIiKix5QqUaqo+Svjxo3DuHHjit1XuDfr7t275R/Q/7OUoXcXL17EiBEjkJOTAwBo3bo1Fi1aBBsbG9SoUaPIfLGSaLVaqNVqyOVyaLXacl1IuKIU9HoSEREREZVFqcqDDxo0CFKp1KCHpVTCMyVLGHp34cIFvSSpTZs2WLZsGaysrFCjRg2D5xmp1Wqo1WpotVpxWGVVeBARERERmUKpspmwsDDUqVOnvGKxeObuUTp//jzeeustvSRpxYoVyM3NRUBAALy9vZ96Dq1WC41GIyYVVlZWsLKyqlKJrUQiYfU+IiIiIiqTUn06joiIwMCBA8srFov3eI9SRSdKv//+O0aNGiUmSW3btsWKFSuQk5MDT09PBAYGPjU5KOhBAnTJxOM9f4YUfiAiIiIiqi6qTjdCBXi8mIONTcUOvbt9+7aYJL300ktYuXIlcnNz4ejoiBo1ajyxR6i4XqSq1INERERERGRq/LRcCo8nShXdo9S/f3+o1WqcOHECn332GTQajTgv6UnFG4rrRWLvERERERHRkzFRKoXcXPMNvQN0xTQGDhwIjUaDrKwshIaGlli8oXAvUkGRDSIiIiIiejqDE6WqUDq6rPTXUSrf1+P06dPIyMhAeHh4kX3p6enw8/MT15YqrHAvkrW1NYsbEBERERGVAnuUSqGiepROnTqFMWPGQKPRQCKRoFOnTgB0awSlpaXBzc0NQUFBRYbQsReJiIiIiMg0OFmlFB4vDy6Xl0+i9Ntvv2HMmDHIy8uDWq3Gvn37xMQnMzMTdnZ2CA4Ohkwm0zuuYF0kQRAgkUggk8mYJBERERERGYk9SqWQl/ff38tj6N2JEycwduxYqFQqALp1qxYtWgSJRAKlUgmNRoOaNWvCwcFBPIa9SEREREREpsdEqRTKc+hd4STp1VdfxZIlSyCTyaBWq5GdnY2goCC4u7uLx3AuEhERERFR+WCiVArlVczh+PHjGDdunJgkderUCYsXL4ZMJoMgCEhPT4enpyf8/f0hkUjYi0REREREVM6YKJVCefQo/frrrxg3bhzy8/MBAOHh4fj000/FOUjp6eniorJSqZS9SEREREREFYCJUikU9ChJpQKsTfDKZWRkYPLkyWKS1KVLFyxcuFBMknJycsRFZW1sbMQeJ10M7EUiIiIiIiovrHpXCgWJkqmG3Tk7O2Pp0qWwsbHBa6+9hkWLFolJUn5+PnJzc+Hv7w9HR0eo1WoAYEU7IiIiIqIKwB6lUigYemfK+Ult27bF119/jbp168L6/7uptFot0tPT4eXlBQ8PD3GoHXuRiIiIiIgqBnuUSuG/HiXj5yfduXOnyLaGDRuKSRKgm5fk4OAAPz8/WFlZsReJiIiIiKiCMVEqhYIeJVtb43qUDh48iO7du2PNmjUltsnMzIQgCAgMDIRcLodUKoVMJmPBBiIiIiKiCsREqRQKepRsbErfo3TgwAFMmjQJarUay5Ytw5EjR/T2C4KA3Nxc5OTkwN/fHy4uLuxFIiIiIiIyEyZKBtJqAZXKuB6lX375BVFRUWJBhh49eqBdu3aPnVuLvLw8KBQKeHl5wd/fn71IRERERERmxGIOBjJ2sdn9+/fj3XffhUajAQC8/vrrmDdvHqRSKQRBgEajgVarRUZGBlxcXFCzZk2x8h0REREREZkHEyUD5eb+93dDF5v9+eefMXnyZDFJ6tWrF+bOnQupVAqtViv2MOXk5MDGxga1atWCXC43eexERERERFQ6HHpnoMcTJUN6lH766Se9JKl3796YN28erKysoFarxSRJrVZDo9GgRo0acHZ2LpfYiYiIiIiodJgoGejxROlpxRx++eUXvSSpT58+mDt3LgDdQrIF6yJJJBLk5OTAx8cHPj4+5RM4ERERERGVGhMlA5Vm6F3t2rXh4eEBQJckzZ49W2+oXcG6SNnZ2XBxcUFgYCCsrPhWEBERERFZCs5RMtDjxRyeVvWuZs2a2LRpE3bv3o0JEyaIPUsAIJVKIZVKkZOTAysrKwQFBXFeEhERERGRhWE3hoH05ygV7VESBP1toaGhmDBhgt4wu4J1kdRqNXJzcxEQEAAXF5dyjZuIiIiIiEqPiZKBnjT0bvfu3XjvvffEoXUajUZvLpJUKhXXRRIEQVwvydfXt8LiJyIiIiIiw3HonYFK6lH67rvvMH36dAiCAK1Wi/nz54vzjaysrCCVSvUWjs3IyICTkxOCgoI4L4mIiIiIyEIxUTJQceXBd+3ahRkzZojD7pycnMSkyNraukgipPz/iU5BQUGwtbWtgKiJiIiIiMgY7NIwkH4xBwHffvutXpI0cOBATJs2TRxmVzhJ0mg0yMrKgr+/P9zc3CoydCIiIiIiKiWLTJRWrVqFkJAQ2NraolWrVjh37twT2+/cuRP16tWDra0tGjVqhJ9++snkMT3eo3T58lm9JOnNN9/E+++/D5lMBmtra72hdgDEeUmenp7w8/MzeWxERERERGRaFpco7dixA1FRUZg1axYuXryIJk2aIDw8HImJicW2P3XqFAYMGIDhw4fj0qVL6NmzJ3r27IkrV66YNK7HE6Xvvtsq/n3QoEGYOnUqbGxsSpxzlJWVBTs7OwQFBUEqlZo0LiIiIiIiMj2LS5SWLFmCkSNHYujQoWjQoAHWrFkDe3t7bNiwodj2y5cvR+fOnTFlyhTUr18fc+fORbNmzbBy5UqTxvV4ogToxuENGTIE06ZNEyvaFScvLw/5+fkICgqCvb29SWMqq8fXdyIiIiIiov9YVDEHlUqFCxcuYNq0aeI2KysrhIWF4fTp08Uec/r0aURFReltCw8Px/fff19s+7y8POTl5YnPMzIyDIotK0sDoKA3KBf9+/fH6NGjkZmZWWx7QRCQnZ2N/Px8+Pr6QiKRIDU11aBrERERERGReVlUopScnAyNRgMfHx+97T4+Pvjnn3+KPSY+Pr7Y9vHx8cW2j46Oxpw5c0odW17ef51v7dq1wuDBbaFQKJ54TMFwOw8PjxJ7nCwBhwMSEREREemzqESpIkybNk2vByojIwNBQUFPPa5NGwmys7VIS1Nixoz3ERIiPPUYW1tblgEnIiIiIqqELCpR8vT0hFQqRUJCgt72hIQE+Pr6FnuMr69vqdrL5XLI5fJSx9a3L9C3rxUAy5pnREREREREpmdRxRxsbGzQvHlzHD58WNym1Wpx+PBhtG7duthjWrdurdceAA4ePFhieyIiIiIioqexqB4lAIiKikJERARatGiBli1bYtmyZcjOzsbQoUMB6CrNBQQEIDo6GgAwYcIEtGvXDosXL0bXrl2xfft2nD9/HmvXrjXnbRARERERUSVmcYlSv379kJSUhJkzZyI+Ph5NmzbF/v37xYIN9+/f11uvqE2bNti2bRtmzJiBDz74ALVr18b333+PZ5991ly3QERERERElZxEEISnVyWowjIyMuDi4oL09HQ4OzubOxwiKoS/o0RERGQOFjVHiYiIiIiIyBIwUSIiIiIiIiqEiRIREREREVEhFlfMoaIVTNHKyMgwcyREVJyC381qPp2SiIiIKli1T5QyMzMBAEFBQWaOhIieJDMzEy4uLuYOg4iIiKqJal/1TqvV4tGjR3BycoJEInli24yMDAQFBeHBgwdVpvoW78nyVbX7AUp3T4IgIDMzE/7+/npLAxARERGVp2rfo2RlZYXAwMBSHePs7FxlPrAW4D1Zvqp2P4Dh98SeJCIiIqpo/HqWiIiIiIioECZKREREREREhTBRKgW5XI5Zs2ZBLpebOxST4T1Zvqp2P0DVvCciIiKqWqp9MQciIiIiIqLC2KNERERERERUCBMlIiIiIiKiQpgoERERERERFcJEiYiIiIiIqJBqnSitWrUKISEhsLW1RatWrXDu3Lkntt+5cyfq1asHW1tbNGrUCD/99JPefkEQMHPmTPj5+cHOzg5hYWG4efNmed5CEaW5p3Xr1uGll16Cm5sb3NzcEBYWVqR9ZGQkJBKJ3qNz587lfRt6SnNPMTExReK1tbXVa1PZ3qf27dsXuSeJRIKuXbuKbcz5Ph0/fhzdu3eHv78/JBIJvv/++6cec+zYMTRr1gxyuRzPPPMMYmJiirQp7e8nERERkSlV20Rpx44diIqKwqxZs3Dx4kU0adIE4eHhSExMLLb9qVOnMGDAAAwfPhyXLl1Cz5490bNnT1y5ckVss3DhQnz22WdYs2YNzp49CwcHB4SHh0OpVFrkPR07dgwDBgzA0aNHcfr0aQQFBaFTp06IjY3Va9e5c2fExcWJj6+//roibgdA6e8JAJydnfXivXfvnt7+yvY+fffdd3r3c+XKFUilUvTp00evnbnep+zsbDRp0gSrVq0yqP2dO3fQtWtXdOjQAX/88QcmTpyIESNG4JdffhHbGPO+ExEREZmUUE21bNlSGDt2rPhco9EI/v7+QnR0dLHt+/btK3Tt2lVvW6tWrYRRo0YJgiAIWq1W8PX1FRYtWiTuVygUglwuF77++utyuIOiSntPhanVasHJyUnYtGmTuC0iIkLo0aOHqUM1WGnvaePGjYKLi0uJ56sK79PSpUsFJycnISsrS9xm7vepAABh9+7dT2zz3nvvCQ0bNtTb1q9fPyE8PFx8XtbXiIiIiKisqmWPkkqlwoULFxAWFiZus7KyQlhYGE6fPl3sMadPn9ZrDwDh4eFi+zt37iA+Pl6vjYuLC1q1alXiOU3JmHsqLCcnB/n5+XB3d9fbfuzYMXh7e6Nu3boYM2YMUlJSTBp7SYy9p6ysLAQHByMoKAg9evTA33//Le6rCu/T+vXr0b9/fzg4OOhtN9f7VFpP+10yxWtEREREVFbVMlFKTk6GRqOBj4+P3nYfHx/Ex8cXe0x8fPwT2xf8WZpzmpIx91TY+++/D39/f70PqJ07d8bmzZtx+PBhfPLJJ/j111/RpUsXaDQak8ZfHGPuqW7dutiwYQP27NmDLVu2QKvVok2bNnj48CGAyv8+nTt3DleuXMGIESP0tpvzfSqtkn6XMjIykJuba5KfZSIiIqKysjZ3AGQZPv74Y2zfvh3Hjh3TK37Qv39/8e+NGjVC48aNUatWLRw7dgyvvPKKOUJ9otatW6N169bi8zZt2qB+/fr44osvMHfuXDNGZhrr169Ho0aN0LJlS73tle19IiIiIrJ01bJHydPTE1KpFAkJCXrbExIS4OvrW+wxvr6+T2xf8GdpzmlKxtxTgU8//RQff/wxDhw4gMaNGz+xbc2aNeHp6Ylbt26VOeanKcs9FZDJZHjuuefEeCvz+5SdnY3t27dj+PDhT71ORb5PpVXS75KzszPs7OxM8r4TERERlVW1TJRsbGzQvHlzHD58WNym1Wpx+PBhvd6Ix7Vu3VqvPQAcPHhQbB8aGgpfX1+9NhkZGTh79myJ5zQlY+4J0FWAmzt3Lvbv348WLVo89ToPHz5ESkoK/Pz8TBL3kxh7T4/TaDS4fPmyGG9lfZ8AXXn6vLw8DBo06KnXqcj3qbSe9rtkivediIiIqMzMXU3CXLZv3y7I5XIhJiZGuHr1qvDWW28Jrq6uQnx8vCAIgjB48GBh6tSpYvvffvtNsLa2Fj799FPh2rVrwqxZswSZTCZcvnxZbPPxxx8Lrq6uwp49e4S//vpL6NGjhxAaGirk5uZa5D19/PHHgo2NjfDtt98KcXFx4iMzM1MQBEHIzMwUJk+eLJw+fVq4c+eOcOjQIaFZs2ZC7dq1BaVSaZH3NGfOHOGXX34Rbt++LVy4cEHo37+/YGtrK/z99996912Z3qcCbdu2Ffr161dku7nfp8zMTOHSpUvCpUuXBADCkiVLhEuXLgn37t0TBEEQpk6dKgwePFhs/++//wr29vbClClThGvXrgmrVq0SpFKpsH//frHN014jIiIiovJWbRMlQRCEFStWCDVq1BBsbGyEli1bCmfOnBH3tWvXToiIiNBr/8033wh16tQRbGxshIYNGwr79u3T26/VaoUPP/xQ8PHxEeRyufDKK68I169fr4hbEZXmnoKDgwUARR6zZs0SBEEQcnJyhE6dOgleXl6CTCYTgoODhZEjR1b4h9XS3NPEiRPFtj4+PsJrr70mXLx4Ue98le19EgRB+OeffwQAwoEDB4qcy9zv09GjR4v9OSq4h4iICKFdu3ZFjmnatKlgY2Mj1KxZU9i4cWOR8z7pNSIiIiIqbxJBEATz9GURERERERFZpmo5R4mIiIiIiOhJmCgREREREREVwkSJiIiIiIioECZKREREREREhTBRIiIiIiIiKoSJEhERERERUSFMlIiIiIiIiAphokRERERERFQIE6Vq5NixY5BIJDh27Ji5QylXEokEs2fPNqhtSEgIIiMjyzUeIiIiIqp8mChVAjExMZBIJMU+pk6dau7wnqhw7La2tqhTpw7GjRuHhISEConh1KlTmD17NhQKRYVczxAhISF6r4uDgwNatmyJzZs3G33On376yeAEkYiIiIiezNrcAZDhPvroI4SGhupte/bZZ80UTekUxK5UKnHy5EmsXr0aP/30E65cuQJ7e3uTXis3NxfW1v/9aJ86dQpz5sxBZGQkXF1d9dpev34dVlbm+b6gadOmePfddwEAcXFx+PLLLxEREYG8vDyMHDmy1Of76aefsGrVKiZLRERERCbARKkS6dKlC1q0aGHuMIzyeOwjRoyAh4cHlixZgj179mDAgAEmvZatra3BbeVyuUmvXRoBAQEYNGiQ+DwyMhI1a9bE0qVLjUqUiIiIiMh0OPSuCrh37x7efvtt1K1bF3Z2dvDw8ECfPn1w9+7dpx578+ZN9O7dG76+vrC1tUVgYCD69++P9PR0vXZbtmxB8+bNYWdnB3d3d/Tv3x8PHjwwOuaOHTsCAO7cuQMAUKvVmDt3LmrVqgW5XI6QkBB88MEHyMvL0zvu/PnzCA8Ph6enJ+zs7BAaGophw4bptXl8jtLs2bMxZcoUAEBoaKg41K3gtXl8jtL58+chkUiwadOmIvH+8ssvkEgk2Lt3r7gtNjYWw4YNg4+PD+RyORo2bIgNGzYY/Zp4eXmhXr16uH37tt72EydOoE+fPqhRowbkcjmCgoIwadIk5Obmim0iIyOxatUq8f4LHgW0Wi2WLVuGhg0bwtbWFj4+Phg1ahTS0tKMjpeIiIioKmOPUiWSnp6O5ORkvW2enp74/fffcerUKfTv3x+BgYG4e/cuVq9ejfbt2+Pq1aslDm1TqVQIDw9HXl4e3nnnHfj6+iI2NhZ79+6FQqGAi4sLAGD+/Pn48MMP0bdvX4wYMQJJSUlYsWIFXn75ZVy6dKnIcDZDFCQDHh4eAHS9TJs2bcIbb7yBd999F2fPnkV0dDSuXbuG3bt3AwASExPRqVMneHl5YerUqXB1dcXdu3fx3XfflXidXr164caNG/j666+xdOlSeHp6AtAlJYW1aNECNWvWxDfffIOIiAi9fTt27ICbmxvCw8MBAAkJCXjhhRcgkUgwbtw4eHl54eeff8bw4cORkZGBiRMnlvo1UavVePjwIdzc3PS279y5Ezk5ORgzZgw8PDxw7tw5rFixAg8fPsTOnTsBAKNGjcKjR49w8OBBfPXVV0XOPWrUKMTExGDo0KEYP3487ty5g5UrV+LSpUv47bffIJPJSh0vERERUZUmkMXbuHGjAKDYhyAIQk5OTpFjTp8+LQAQNm/eLG47evSoAEA4evSoIAiCcOnSJQGAsHPnzhKvfffuXUEqlQrz58/X23758mXB2tq6yPaSYj906JCQlJQkPHjwQNi+fbvg4eEh2NnZCQ8fPhT++OMPAYAwYsQIvWMnT54sABCOHDkiCIIg7N69WwAg/P7770+8JgBh1qxZ4vNFixYJAIQ7d+4UaRscHCxERESIz6dNmybIZDIhNTVV3JaXlye4uroKw4YNE7cNHz5c8PPzE5KTk/XO179/f8HFxaXY96TwdTt16iQkJSUJSUlJwuXLl4XBgwcLAISxY8fqtS3uXNHR0YJEIhHu3bsnbhs7dqxQ3K/0iRMnBADC1q1b9bbv37+/2O1EREREJAgceleJrFq1CgcPHtR7AICdnZ3YJj8/HykpKXjmmWfg6uqKixcvlni+gh6jX375BTk5OcW2+e6776DVatG3b18kJyeLD19fX9SuXRtHjx41KPawsDB4eXkhKCgI/fv3h6OjI3bv3o2AgAD89NNPAICoqCi9YwoKHezbtw8AxJ6rvXv3Ij8/36Drlla/fv2Qn5+v10t14MABKBQK9OvXDwAgCAJ27dqF7t27QxAEvdclPDwc6enpT3zdHz+vl5cXvLy80KhRI3z11VcYOnQoFi1apNfu8fc3OzsbycnJaNOmDQRBwKVLl556nZ07d8LFxQWvvvqqXqzNmzeHo6Ojwe8hERERUXXCoXeVSMuWLYst5pCbm4vo6Ghs3LgRsbGxEARB3Fd4rtHjQkNDERUVhSVLlmDr1q146aWX8L///Q+DBg0Sk6ibN29CEATUrl272HMYOmRr1apVqFOnDqytreHj44O6deuK1ebu3bsHKysrPPPMM3rH+Pr6wtXVFffu3QMAtGvXDr1798acOXOwdOlStG/fHj179sTAgQNNVpShSZMmqFevHnbs2IHhw4cD0A278/T0FOdVJSUlQaFQYO3atVi7dm2x50lMTHzqtVq1aoV58+ZBo9HgypUrmDdvHtLS0mBjY6PX7v79+5g5cyZ++OGHInOKnvT+Frh58ybS09Ph7e1tdKxERERE1Q0TpSrgnXfewcaNGzFx4kS0bt0aLi4ukEgk6N+/P7Ra7ROPXbx4MSIjI7Fnzx4cOHAA48ePR3R0NM6cOYPAwEBotVpIJBL8/PPPkEqlRY53dHQ0KMaSkrzHPV58oKT93377Lc6cOYMff/wRv/zyC4YNG4bFixfjzJkzBsfyNP369cP8+fORnJwMJycn/PDDDxgwYIBYcrzgNR00aFCRuUwFGjdu/NTreHp6IiwsDAAQHh6OevXqoVu3bli+fLnYu6bRaPDqq68iNTUV77//PurVqwcHBwfExsYiMjLyqe9vQbze3t7YunVrsfuLm69FREREVN0xUaoCvv32W0RERGDx4sXiNqVSafACq40aNUKjRo0wY8YMnDp1Ci+++CLWrFmDefPmoVatWhAEAaGhoahTp065xB8cHAytVoubN2+ifv364vaEhAQoFAoEBwfrtX/hhRfwwgsvYP78+di2bRvefPNNbN++HSNGjCj2/E9LwArr168f5syZg127dsHHxwcZGRno37+/uN/LywtOTk7QaDRiomMKXbt2Rbt27bBgwQKMGjUKDg4OuHz5Mm7cuIFNmzZhyJAhYtuCYZePK+k+a9WqhUOHDuHFF1/UG8ZHRERERCXjHKUqQCqV6g23A4AVK1ZAo9E88biMjAyo1Wq9bY0aNYKVlZVYlrtXr16QSqWYM2dOkWsIgoCUlJQyx//aa68BAJYtW6a3fcmSJQB0CQQApKWlFYmhadOmAFCkjPjjHBwcAMDgxLF+/fpo1KgRduzYgR07dsDPzw8vv/yyuF8qlaJ3797YtWsXrly5UuT4pKQkg65TnPfffx8pKSlYt26deC0AevctCAKWL19e5NiS7rNv377QaDSYO3dukWPUarXBrwsRERFRdcIepSqgW7du+Oqrr+Di4oIGDRrg9OnTOHTokFh6uyRHjhzBuHHj0KdPH9SpUwdqtRpfffWVmAgAut6IefPmYdq0abh79y569uwJJycn3LlzB7t378Zbb72FyZMnlyn+Jk2aICIiAmvXroVCoUC7du1w7tw5bNq0CT179kSHDh0AAJs2bcLnn3+O119/HbVq1UJmZibWrVsHZ2dnMdkqTvPmzQEA06dPR//+/SGTydC9e3cxsShOv379MHPmTNja2mL48OHifKoCH3/8MY4ePYpWrVph5MiRaNCgAVJTU3Hx4kUcOnQIqampRr0WXbp0wbPPPoslS5Zg7NixqFevHmrVqoXJkycjNjYWzs7O2LVrV7HrHxXc5/jx4xEeHg6pVIr+/fujXbt2GDVqFKKjo/HHH3+gU6dOkMlkuHnzJnbu3Inly5fjjTfeMCpeIiIioirLPMX2qDQKSmyXVBY7LS1NGDp0qODp6Sk4OjoK4eHhwj///FOk9HXh8uD//vuvMGzYMKFWrVqCra2t4O7uLnTo0EE4dOhQkWvs2rVLaNu2reDg4CA4ODgI9erVE8aOHStcv369TLEXyM/PF+bMmSOEhoYKMplMCAoKEqZNmyYolUqxzcWLF4UBAwYINWrUEORyueDt7S1069ZNOH/+vN65UKg8uCAIwty5c4WAgADByspKr1R44deowM2bN8US7CdPniw25oSEBGHs2LFCUFCQIJPJBF9fX+GVV14R1q5d+8R7Lbhu165di90XExMjABA2btwoCIIgXL16VQgLCxMcHR0FT09PYeTIkcKff/6p10YQBEGtVgvvvPOO4OXlJUgkkiKlwteuXSs0b95csLOzE5ycnIRGjRoJ7733nvDo0aOnxktERERU3UgEodBYJiIiIiIiomqOc5SIiIiIiIgKYaJERERERERUCBMlIiIiIiKiQpgoERERERERFcJEiYiIiIiIqBAmSkRERERERIUwUSIiIiIiIiqEiRIREREREVEhTJSIiIiIiIgKYaJERERERERUCBMlIiIiIiKiQpgoERERERERFfJ/bdhCV2UorB8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# overlaid ROC\n",
    "from sklearn.metrics import auc\n",
    "\n",
    "tprs = []\n",
    "aucs = []\n",
    "mean_fpr = np.linspace(0, 1, 100) # regularly spaced points\n",
    "\n",
    "#set up plotting area\n",
    "plt.figure(figsize=(3, 3))\n",
    "ax = plt.axes() # enables overlay\n",
    "\n",
    "plt.plot([0, 1], [0, 1],'--', color = 'black', lw = 2)\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "\n",
    "#plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "\n",
    "fprList = []\n",
    "tprList = []\n",
    "\n",
    "for y_prob, y_test in zip(y_probList,y_testList):\n",
    "    \n",
    "    ROC = roc_from_scratch(np.reshape(y_prob,(-1)),y_test,partitions=1000) # y_prob[:,1]\n",
    "    #ROC = roc_from_scratch(y_prob,y_test,partitions=100) # way that usually works\n",
    "\n",
    "    #plt.scatter(ROC[:,0],ROC[:,1],color='black',s=20, alpha = 0.8)\n",
    "    plt.plot(ROC[:,0],ROC[:,1],color='black', alpha = 0.025)\n",
    "    plt.title('ROC Curve',fontsize=12)\n",
    "    plt.xlabel('False Positive Rate',fontsize=12)\n",
    "    plt.ylabel('True Positive Rate',fontsize=12)\n",
    "    #print(ROC[:,1])\n",
    "\n",
    "    sort = np.argsort(ROC[:,0])\n",
    "    interp_tpr = np.interp(mean_fpr, ROC[:,0][sort], ROC[:,1][sort]) # interp needs to be sorted\n",
    "    interp_tpr[0] = 0.0\n",
    "    tprs.append(interp_tpr)\n",
    "    rectangle_roc = 0\n",
    "    for k in range(partitions):\n",
    "        fpr, tpr = ROC[:, 0], ROC[:, 1]\n",
    "        rectangle_roc = rectangle_roc + (fpr[k]- fpr[k + 1]) * tpr[k]\n",
    "    print('AUROC is ' + str(rectangle_roc))    \n",
    "    aucs.append(rectangle_roc)\n",
    "\n",
    "    #if auc < 0.5:\n",
    "    #    probs = [-x for x in probs]\n",
    "\n",
    "mean_tpr = np.mean(tprs, axis=0)\n",
    "#print(mean_tpr)\n",
    "mean_tpr[-1] = 1.0\n",
    "mean_auc = auc(mean_fpr, mean_tpr)\n",
    "std_auc = np.std(aucs)\n",
    "\n",
    "#print(\"mean_fpr is: \" + str(mean_fpr)) # QC\n",
    "#print(\"mean_tpr is: \" + str(mean_tpr)) # QC\n",
    "\n",
    "ax.plot(mean_fpr, mean_tpr, color='b',\n",
    "        label=r'Mean ROC (AUC = %0.2f $\\pm$ %0.2f)' % (mean_auc, std_auc),\n",
    "        lw=2, alpha=1)\n",
    "\n",
    "std_tpr = np.std(tprs, axis=0)\n",
    "tprs_upper = np.minimum(mean_tpr + std_tpr, 1)\n",
    "tprs_lower = np.maximum(mean_tpr - std_tpr, 0)\n",
    "ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.3,\n",
    "                label=r'$\\pm$ 1 std. dev.')\n",
    "\n",
    "ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],\n",
    "       title=\"ROC curve\")\n",
    "\n",
    "plt.plot()\n",
    "ax.legend(bbox_to_anchor=(3.3, .5), loc='center right', borderaxespad=0)\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.savefig('ROC_overlaid_' + target + '_quantType_' + quantType + '.pdf',format='pdf', dpi=500, bbox_inches='tight')# axis labels\n",
    "plt.savefig('ROC_overlaid_' + target + '_quantType_' + quantType + '.svg',format='svg', dpi=500, bbox_inches='tight')# axis labels. Saves to Fugure2 folder currently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 942,
   "metadata": {},
   "outputs": [],
   "source": [
    "# testing to make line smoother\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 943,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in progress\n",
    "\n",
    "from itertools import chain\n",
    "y_testListFlat = list(chain.from_iterable(y_testList)) \n",
    "y_predListFlat = list(chain.from_iterable(y_predList))\n",
    "#y_probListFlat = list(chain.from_iterable(y_probList)) # usually works\n",
    "y_probListFlat = list(chain.from_iterable(y_probList)) # y_probList[:,1]\n",
    "y_1MinusProbListFlat = [1 - x for x in y_probListFlat]\n",
    "y_predProbAFormattedLikeSKLearn = np.stack((y_1MinusProbListFlat, y_probListFlat), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 944,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAToAAAE8CAYAAABQG31BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABb60lEQVR4nO3dd3zM9x/A8ddl78QKCSE2tcWoLYQYVVuIWtVqjValahUxWpRabY3SmrWV8jOigiiqKGKLEBolCWlIJJF19/n9cXKcDLm4yyXxeT4e93Dfz32+3+/7G7l3vuMzFEIIgSRJUiFmYuwAJEmSDE0mOkmSCj2Z6CRJKvRkopMkqdCTiU6SpEJPJjpJkgo9megkSSr0ZKKTJKnQk4lOkqRCTyY6SZIKPZno9GDNmjUoFArNy8zMjNKlSzN48GDu3buX6TpCCNavX0/Lli1xcnLCxsaGWrVqMWPGDBISErLc186dO+nYsSPFixfHwsICV1dX+vTpw+HDhw11eAVWo0aNUCgULFu2LNPPp02bhkKhIDo6OtPPa9asSevWrTOUx8XFMX36dOrUqYOdnR3W1tbUrFmT8ePHc//+fX0ewiupVCrmzp1L+fLlsbKyonbt2mzatEmnbQQGBtKmTRscHR2xt7fHw8ODLVu2aNUZM2YM9evXp2jRotjY2FC9enWmTZtGfHy8Pg/HYMyMHUBhMmPGDMqXL09SUhJ//fUXa9as4fjx41y+fBkrKytNPaVSia+vL1u3bqVFixZMmzYNGxsbjh07xvTp09m2bRuBgYGULFlSs44Qgvfff581a9ZQr149/Pz8KFWqFBEREezcuZO2bdty4sQJmjZtaoxDz3dCQ0M5c+YM7u7ubNiwgeHDh+tlu2FhYXh5eREeHk7v3r0ZNmwYFhYWXLx4kZ9//pmdO3dy48YNvewrJ7788kvmzJnDhx9+SMOGDdm1axe+vr4oFAr69u37yvVXr17N0KFDadeuHbNmzcLU1JSQkBDu3r2rVe/MmTO0aNGCIUOGYGVlxfnz55kzZw6BgYH88ccfmJjk83MmIb221atXC0CcOXNGq3z8+PECEFu2bNEqnzVrlgDE2LFjM2xr9+7dwsTERHTo0EGrfN68eQIQn332mVCpVBnWW7dunTh16pQejib34uPjjbr/F02dOlU4OzuLX3/9VSgUCnH79u0Mdfz9/QUgHj58mOk2atSoIVq1aqVZTk1NFXXq1BE2Njbi2LFjGerHxsaKSZMm6esQXunff/8V5ubmYuTIkZoylUolWrRoIcqUKSPS0tKyXf/27dvC2tpafPrpp7na/7fffisAcfLkyVytn5dkotODrBLdnj17BCBmzZqlKUtMTBRFihQRVapUEampqZlub8iQIVq/QImJiaJo0aKiWrVqr/zlzY5SqRSLFi0SNWvWFJaWlqJ48eLC29tbE/ft27cFIFavXp1hXUD4+/trltOTxJUrV0S/fv2Ek5OTqFu3riYh37lzJ8M2JkyYIMzNzUVMTIym7K+//hLe3t7CwcFBWFtbi5YtW4rjx4/n+hjTVapUSYwYMUIkJycLJycn8fXXX2eoo2ui27x5swAy3ZYxLFmyRPN/8KKNGzcKINNk/KLx48cLCwsL8fjxYyGEEE+ePMn0j2hWtm/fLgCxf/9+3YPPY/n8fLNgu3PnDgBFihTRlB0/fpxHjx7h6+uLmVnmdw4GDhwIwJ49ezTrxMTE4Ovri6mpaa7jGTp0KJ999hlubm588803TJgwASsrK/76669cb7N3794kJiYya9YsPvzwQ/r06YNCoWDr1q0Z6m7dupX27dtrfh6HDx+mZcuWxMXF4e/vz6xZs3j8+DFt2rTh9OnTuY7p1KlT3Lx5k379+mFhYUGPHj3YsGFDrreXbvfu3QAMGDAg19tITU0lOjo6Ry+VSpXtts6fP4+trS3Vq1fXKm/UqJHm8+wEBgZSrVo19u3bR5kyZbC3t6dYsWJMmTIl032npaURHR3N/fv3+f3335k8eTL29vaa/eVrxs60hUH6GV1gYKB4+PChuHv3rti+fbsoUaKEsLS0FHfv3tXUXbRokQDEzp07s9xeTEyMAESPHj2EEEIsXrz4leu8yuHDhwWQ6WVK+l/x3JzR9evXL0PdJk2aCA8PD62y06dPC0CsW7dOs8/KlSsLb29vrbOIxMREUb58edGuXbvcHKYQQohRo0YJNzc3zXZ///13AYjz589r1dP1jK5evXrC0dEx13EJIcSRI0cEkKNXZpfbL+rcubOoUKFChvKEhAQBiAkTJmS7voODgyhSpIiwtLQUU6ZMEdu3bxe+vr5Zrnvy5Emt+KpWrSqOHDmiy+EbjXwYoUdeXl5ay+7u7vzyyy+UKVNGU/bkyRMA7O3ts9xO+mdxcXFa/2a3zqv8+uuvKBQK/P39M3ymUChyvd2PP/44Q5mPjw+fffYZt27domLFigBs2bIFS0tLunbtCkBwcDChoaFMnjyZ//77T2v9tm3bsn79elQqlc43udPS0tiyZQuDBg3SHFebNm1wdnZmw4YN1K1bNxdHqRYXF/da/wcAderU4eDBgzmqW6pUqWw/f/r0KZaWlhnK0x98PX36NNv14+PjUalUzJkzh/HjxwPQs2dPYmJiWLx4MZMmTdI63rfeeouDBw+SkJDAn3/+SWBgoHzq+iZasmQJVapUITY2llWrVvHHH39k+EVM/8VJT3iZeTkZOjg4vHKdV7l16xaurq4ULVo019vITPny5TOU9e7dGz8/P7Zs2cKkSZMQQrBt2zY6duyoOZbQ0FAABg0alOW2Y2NjtS77c+L333/n4cOHNGrUiJs3b2rKPT092bRpE998841OyfPFPwIODg6EhYXpFM/LihQpkuEPYm5ZW1uTnJycoTwpKUnz+avWT0hIoF+/flrl/fr1IyAggPPnz9OyZUtNuYODgyb2rl27snHjRrp27cq5c+eoU6fO6x6OQclEp0eNGjWiQYMGAHTr1o3mzZvj6+tLSEgIdnZ2AJr7KRcvXqRbt26ZbufixYuA+i8oQLVq1QC4dOlSluvoQ1ZndkqlMst1Mvsyubq60qJFC7Zu3cqkSZP466+/CA8P55tvvtHUSb8HNG/evCzPstJ/ZrpIvxfXp0+fTD8/evQonp6ewKvPfBITE7WaBVWrVo3z589z9+5d3NzcdI4NICUlhZiYmBzVLVGiRLb3ZF1cXDhy5AhCCK3/u4iICED9/5AdV1dXQkNDtZoxATg7OwPw6NGjbNfv0aMHAwYMYPPmzfk+0cmHEQZiamrK7NmzuX//Pj/88IOmvHnz5jg5ObFx48YsE8i6desAeOeddzTrFClShE2bNmWbdLJTsWJF7t+/n+2XLP3s6fHjx1rl//zzj8778/Hx4cKFC4SEhLBlyxZsbGzo0qWLVjzw/Cwhs5e5ublO+0xISGDXrl34+Piwbdu2DC8XFxethxLlypUDICQkJMO2EhMTuXv3rqYOoIn/l19+0SmuF/3555+4uLjk6PVyW7aX1a1bl8TERK5du6ZVfurUKc3n2fHw8ADI0Kg9vdFziRIlsl0/OTkZlUpFbGxstvXyBWPfJCwMsmpeIoQQjRo1EiVLlhRPnz7VlH311VcCEOPHj89Qf8+ePcLExER4e3trlc+ZM0cA4vPPP8+0CcD69euzbUeXk4cRQghRvHhx0b17d63PP//88ywfRmR1Iz8qKkqYmpoKf39/4erqKvr06aP1uVKpFBUrVhSVK1cWT548ybD+gwcPsjyWrKxfv14A4o8//sj08w8//FA4OTmJpKQkTYwWFhaiR48eQqlUatVduHChAMRvv/2mKUtJSRG1atUStra24s8//8yw/bi4uFe2o4uJiREHDx7M0evF35nM3L17N8t2dKVLl9ZqinT//n1x7do1kZKSoinbuXOnALRiViqVonnz5qJo0aKan9OjR4+01kuX3o7u559/zjbO/EAmOj3ILtFt27ZNAGLZsmWasrS0NNGzZ08BiJYtW4rFixeLFStWiIEDBwoTExNRo0YNERkZqbUdpVIpBgwYIABRv359MWvWLLFq1Soxa9Ys0ahRIwFk+uV7Ufr6HTt2FIsXLxYLFy4UPXr0EN9//72mzoQJEwQghg4dKpYtWyb69esnPDw8dE50Qgjh5eUl7O3tBSB+/fXXDJ8fOXJEWFlZibJlywp/f3+xYsUK4e/vL1q2bCneeecdrbqA1hPQzHTo0EEUK1Ysy7aG//vf/zLEkv5Hp1mzZuKbb74R33//vejXr58ARPv27TMkwNDQUFGuXDlhZmYmfH19xZIlS8SKFSvE6NGjRYkSJUSVKlWyjVHfvvjiCwGIYcOGiZUrV4rOnTsLQGzYsEGr3qBBgzI8yVWpVKJt27ZCoVCIYcOGiSVLloh27doJQPz444+aejt37hRubm5izJgxYunSpWLRokWiZ8+eQqFQiAYNGojk5OS8Otxck4lOD7JLdOlnLhUrVtT6AiqVSrF69WrRrFkz4eDgIKysrESNGjXE9OnTs+1hsH37dtG+fXtRtGhRYWZmJlxcXISPj48ICgp6ZZxpaWli3rx5olq1asLCwkKUKFFCdOzYUZw9e1ZTJzExUQwdOlQ4OjoKe3t70adPH/HgwYNcJbqVK1cKQNjb22d5dnL+/HnRo0cPUaxYMWFpaSnKlSsn+vTpIw4dOqSp8+TJEwGIvn37ZrmvqKgoYWZmJgYMGJBlncTERGFjY5PhjPWXX34Rb7/9trC1tRWWlpaiWrVqYvr06Zozmpc9evRITJ06VdSqVUvY2NgIKysrUbNmTTFx4kQRERGR5f4NQalUilmzZoly5coJCwsLUaNGDfHLL79kqJdZohNC/bMdPXq0KFWqlLCwsBC1atXKsP7NmzfFwIEDRYUKFYS1tbXmd9Xf3z9f9YbJjkIIOa+rlL/t27ePd955hwsXLlCrVi1jhyMVQPJhhJTvHTlyhL59+8okJ+WaPKOTJKnQk+3oJEnKP1RKuHcM4iPAzgVKtwCT3PfvTmfUS9c//viDLl264OrqikKh4LfffnvlOkFBQdSvXx9LS0sqVarEmjVrDB6nJEl5IHQHrHSHrZ6wz1f970p3dflrMmqiS0hIoE6dOixZsiRH9W/fvk3nzp3x9PQkODiYzz77jA8++IADBw4YOFJJkgwqdAfs7gXx/2qXx99Tl79msss39+gUCgU7d+7MtovT+PHj2bt3L5cvX9aU9e3bl8ePHxMQEJAHUUqSpHcqpfrM7eUkp6EA+zLwwe1cX8YWqHt0J0+ezNAh2tvbm88++yzLdZKTk7U6PqtUKmJiYihWrNhrjdohSZKe3P0Dop8nOSHgSTK4OoB6/AUBT+6q7925tc7VLgpUoouMjMzQAblkyZLExcXx9OnTTDuYz549m+nTp+dViJIk6cndyVDG6YWC+Ihcb6tAJbrcmDhxIn5+fprl2NhYypYty927dzVDBkmSlIeUqRCyFc4thEfq4bpmH4Y5zyaym+AJc46A/ctD7dm55HqXBSrRlSpViqioKK2yqKgoHBwcshx7y9LSMtPBCR0cHGSik6S8pEqD4GXw9zz1pSiAFUw78DzJfdMZPm6iTnTP7yw9u0dXukWud12gekY0adKEQ4cOaZUdPHiQJk2aGCkiSZJyTGEKV1ZrkpwQ4H+yLNOfDbg87x0Y5/nyffNny56LXqs9nVETXXx8PMHBwQQHBwPq5iPBwcGEh4cD6svO9IliQD1sd1hYGOPGjeP69essXbqUrVu3MmbMGGOEL0lSdp5qD5GPQgGNJqjfV+hMVNtdLAlST9Y+f/58xi74FexKa69jXwbe3Q6Ve7xeLMYcUSCriUIGDRokhFCPuPDy0DxHjhwRdevWFRYWFqJChQqZTuSSndjYWAGI2NhY/RyEJEnaHt0S4uDHQiy0FOLeS0OHKdOEeHhJsxgcHCyWLFmi9XnsFfU0obFX9qjr60G+aUeXV+Li4nB0dCQ2Nlbeo5MkfXp4CU7PgZDNIJ5Nl1ihC3TfrakihODOnTuZzjWSzhDf0QJ1j06SpHzo/knY2QXW1YbrG58nOXM7KFZdsyyEYOLEidSqVYvjx4/naYgF6qmrJEn5hBDwz+9wajb8e1T7M+viUH801B0JVkWeVReMHz+eefPmAeqJnpo3b55n4cpEJ0mS7iL+gl87aJfZu0GDsVBrKJjbaoqFEIwbN45vv/0WgB9++IHhw4fnZbQy0UmSlAsub4NLY4g4BUWqqp+mVvcFUwutakIIxo4dy4IFCwBYunRpnic5kIlOkqTspMTDpZVw7wR02fa8Fa9CAS2+gafRUKlbpm3chBD4+fmxaNEiAJYtW8bHH3+cd7G/QCY6SZIyehoD57+H899B0rO5gP89qt2p3q1VtptIS0vj1q1bAPz4448MGzbMQMG+mkx0kiQ99+QenF0AF3+E1ATtz/79Q6fRQ8zNzdm2bRuHDx+mY8eO+o1TRzLRSZIEj27CmblwdS0oU56XK0yhWj9oNB6K13zlZoQQ7Nixgx49eqBQKLC0tDR6kgPZjk6SpL/nw+qq6ntx6UnO1BLqjIChodBpfY6T3KhRo+jVq5fWiEH5gTyjk6Q3ncvbzxv5WjhA3RFQ/zOwLZntai9SqVSMGjWKZcuWoVAoqF27tmFizSWZ6CTpTSEE3N4PZtZQ1vN5eelmULknlKyvPouzctJpsyqVihEjRvDjjz+iUChYtWoVgwcP1mvor0smOkkq7FRKuLFN3Q/14QVwrgfvnX1xwDf1CCG52bRKxfDhw1mxYgUKhYI1a9ZojTiUX8hEJ0mFVVqy+uHCmbnw+Nbz8gfn4W6Q9lldLo0cOVKT5NauXcuAAQNee5uGIB9GSFJhk/JE/YDhp/Jw8CPtJFeqIby745Vt4HKqRYsWmJubs27dunyb5CAfTXeYV+QwTVKhJQScnAHnF0PSI+3PyraFRhOhbBvtS1Y9CA8Pp2zZsnrbnhymSZLeVCql+nLz2ib1vyplxjoKhfoe3ItJrlJ36H8aegdCubavneSUSiWTJ0/m3r17mjJ9JjlDkffoJCm/C90Bh0drT/BsV0bdiLfOx2Dywte48UQI+x9U7w8Nx6vHg9MTpVLJ0KFDWbt2LTt37iQ4OBhzc3O9bd+QZKKTpPwsdAfs7oV6loEXxP8Lhz+BxzfVE8ekK9UQhv2rUxu4nFAqlbz//vusW7cOU1NTpk2bVmCSHMhLV0nKv1RK9Zncy0nuRcFL1fOkvsgASW7w4MGaJLd582Z69+6t130Ymkx0kpRf3TumfbmaGVWq+p6dgSiVSgYNGsQvv/yCmZkZW7ZsoVevXgbbn6HIS1dJyq/iI3JW72m0wUL48ssv2bBhgybJ9ejxmtMOGok8o5Ok/Cqnl6B2LgYL4dNPP6VGjRps3bq1wCY5kGd0kpR/3Q16RQWFeoLn0i30ulshBIpnzVBcXV0JDg7GzKxgpwp5RidJ+VWdj8E2q7O1Z+3hPBdlOox5bqWmpuLr68vGjRs1ZQU9yYFMdJKUf9m5Qv8z6oln7Mpof2ZfRt0Rv7L+LidTU1Pp168fmzdv5oMPPiAyMlJv2za2gp+qJamwUCnVA1+aWz8vsy8NLWZDs6+ePYWNUN+TK91C72dyffv2ZceOHVhYWLBt2zZKlSqlt+0bm0x0kpQfqJRwYAjE34du/9NOdqBOajrM16CLlJQU+vbty86dO7G0tGTnzp35YvhzfZKXrpJkbColBAyGq+sh/BD8r5e6g34eSElJoU+fPpok99tvvxW6JAfyjE6SjEuVBvsHwfVnN/9NzKDWh3ofYSQrv/zyC7t27cLS0pJdu3bh7e2dJ/vNazLRSZKxqNJg/0C4vkm9bGKuniS6Utc8C2HIkCFcu3aNdu3a0b59+zzbb16T49FJkjGo0mDfAAjZrF42MYcu26HSuwbfdXJyMgCWlpYG31duyPHoJKkwUKXB3v7Pk5yphXrU3zxIcklJSfTo0YNevXppEt6bQF66SoajUhq0SUSe7EMf239xG9Yl4OKPEPpsMpr0JFehs/5izkJSUhLdu3cnICAAa2trrly5Qv369Q2+3/xAJjrJMLIaLLLNYv01cjX0PvSx/cy2YfrsktHUAt7dCRU6vX6sr5CUlES3bt04cOAA1tbW7N27941JciAvXSVDSB8s8uUhhuLvqctDd+T/fehj+1ltQ5mi/rfBF3mS5J4+fUrXrl05cOAANjY27Nu3D0/P158BrCAx+sOIJUuWMG/ePCIjI6lTpw7ff/89jRo1yrL+okWLWLZsGeHh4RQvXpxevXoxe/ZsrKyscrQ/+TDCwFRKWOme/ThqZtZQvhNU6gZvvffCummwt9+r9yEE3DsOiVGv3sfLzTQUpvDOZu2ySz/DnQDt7d/eB2lPs9j4s870H9yGoDGQkMlwSrpsQ9+X8y9IT3IHDx7UJLlWrfQzA5ihGOI7atRL1y1btuDn58fy5ctp3LgxixYtwtvbm5CQEJydnTPU37hxIxMmTGDVqlU0bdqUGzduMHjwYBQKBQsWLDDCEUgZ5GSwyLSnEPorOFXULhcCbuRuIuUs9/Eyk0x+5R8E67hfAU/uqo/1TgA8Cs1FgC9sw0A9HgBCQkL4888/sbW1Zd++fbRs2dJg+8rPjHrpumDBAj788EOGDBnCW2+9xfLly7GxsWHVqlWZ1v/zzz9p1qwZvr6+uLu70759e/r168fp06fzOHIpSzkdLLIw0MexGvjnVbduXQICAti/f/8bm+TAiGd0KSkpnD17lokTJ2rKTExM8PLy4uTJk5mu07RpU3755RdOnz5No0aNCAsLY9++fdlOnJucnKz1GD0uLk5/ByFllNNBIN/ZCu4vNVA1MYNhd1+97v2TsKdPzvbh2uTV9ZrNUM+opev27VzA5w/1JXduYzTAoJmJiYn8888/VK+ungGsefPmet9HQWO0RBcdHY1SqaRkSe1RVEuWLMn169czXcfX15fo6GiaN2+OEIK0tDQ+/vhjJk2alOV+Zs+ezfTp0/Uau5SN0i3UTybj75H5pC7P7k1V7pHx3pTi2WevUrlH7veRGasi6peu28+uqYku29CjhIQEunTpwoULFzh8+DB16tTR6/YLqgL11DUoKIhZs2axdOlSzp07x44dO9i7dy8zZ87Mcp2JEycSGxured29m4MzBin3TEzVzS8AzeCQGnoaLNLQ+9DH9vPi5/CShIQEOnfuzJEjR0hNTSUxMVFv2y7ojJboihcvjqmpKVFR2k/OoqKishwHa8qUKQwYMIAPPviAWrVq0b17d2bNmsXs2bNRqVSZrmNpaYmDg4PWSzKwyj3Ug0LaldYu1+dgkYbehz62nxc/h2fi4+Pp1KkTR48excHBgd9//50mTXJw2f6GMNqlq4WFBR4eHhw6dIhu3boBoFKpOHToEKNGjcp0ncTERExMtHOzqan6L+Ib1mU3/6vcAyp2NWyvBUPvQx/bz4Ofw5MnT+jUqRPHjx/XJLnGjRvrbfuFgjCizZs3C0tLS7FmzRpx9epVMWzYMOHk5CQiIyOFEEIMGDBATJgwQVPf399f2Nvbi02bNomwsDDx+++/i4oVK4o+ffrkeJ+xsbECELGxsXo/HknKa3FxcaJZs2YCEI6OjuLUqVPGDum1GeI7atR2dD4+Pjx8+JCpU6cSGRmpeRSe/oAiPDxc6wxu8uTJKBQKJk+ezL179yhRogRdunTh66+/NtYhSNm5su7ZzXjU8x7k0RhrbxITExNMTU1xdHTk4MGDNGzY0Ngh5UtG7xmR12TPiDy0sSlEPGsq5KcERYF69lVgxMfHc/v2bWrVqmXsUPRCDtMkSRKxsbH8/PPPmmU7O7tCk+QMRY5eIkkFSGxsLN7e3pw6dYqYmBi++OILY4dUILzWGV1SUpK+4pAk6RUeP35M+/btOXXqFEWLFsXLy8vYIRUYOic6lUrFzJkzKV26NHZ2doSFhQHqNm4vnk5LkqQ/6Unu9OnTFC1alEOHDlGvXj1jh1Vg6JzovvrqK9asWcPcuXOxsLDQlNesWZOffvpJr8FJkgSPHj2iXbt2nDlzhmLFinH48GHq1q1r7LAKFJ0T3bp161ixYgX9+/fXNNYFqFOnTpZ9VCVJyp3U1FTat2/P33//TfHixWX/1VzSOdHdu3ePSpUqZShXqVSkpqbqJShJktTMzc0ZPHgwJUqU4PDhw9SuXdvYIRVIOie6t956i2PHjmUo3759u7xnIEkGMHLkSG7cuCGbkLwGnZuXTJ06lUGDBnHv3j1UKhU7duwgJCSEdevWsWfPHkPEKBVURauAUj6Z19V///3HmDFjWLhwIcWKFQPAycnJuEEVcLnqGXHs2DFmzJjBhQsXiI+Pp379+kydOrVAzPQte0ZI+Vl0dDReXl5cuHCBjh07sm/fPmOHlOcM8R2VXcAkKZ+Ijo6mbdu2XLx4kVKlSnHkyBGqVatm7LDyXL7oAlahQgX++++/DOWPHz+mQoUKeglKkt40Dx8+pE2bNm98kjMUnRPdnTt3UCqVGcqTk5O5d++eXoKSpDfJgwcPaNOmDZcuXcLFxYWgoCCZ5PQsxw8jdu/erXl/4MABHB0dNctKpZJDhw7h7u6u1+CkAi5wBPx3Rf2+zxE5ekkWBg0axOXLl3F1deXIkSNUqVLF2CEVOjlOdOmjACsUCgYNGqT1mbm5Oe7u7syfP1+vwUkF3IPg58M0SVn6/vvvee+991i/fj2VK1c2djiFUo4TXfqcDOXLl+fMmTMUL17cYEFJUmGnVCo1PYsqVarEyZMnUciBSQ1G52uJ27dvyyQnSa8hIiICDw8PraYjMskZVq7Go0tISODo0aOEh4eTkpKi9dmnn36ql8AkqTC6f/8+np6e3Lhxg88++4x27dphbm5u7LAKPZ0T3fnz5+nUqROJiYkkJCRQtGhRoqOjsbGxwdnZWSY6ScrCvXv38PT0JDQ0lLJlyxIQECCTXB7R+dJ1zJgxdOnShUePHmFtbc1ff/3FP//8g4eHB99++60hYpQKqhfbot8NAlXGZklviheTXLly5QgKCpLtTvOQzokuODiYzz//XDP7UHJyMm5ubsydO5dJkyYZIkapIArdAQ/OPV/e1hZWuqvL3zD//vsvrVu31kpy5cuXN3ZYbxSdE525ublmCkJnZ2fCw8MBcHR05O7du/qNTiqYQnfA7l6g0r5/S/w9dfkbluyWLFnCzZs3cXd3JygoSLY3NQKd79HVq1ePM2fOULlyZVq1asXUqVOJjo5m/fr11KxZ0xAxSgWJSgmHRwOZdaEWgAKOfKaevV6Ps9XnZ1999RVKpZKRI0dSrlw5Y4fzRtL5jG7WrFm4uLgA8PXXX1OkSBGGDx/Ow4cP+fHHH/UeoFTA3DsG8f9mU0HAk7vqeoVYVFSUpqukqakpc+fOlUnOiHQ+o2vQoIHmvbOzMwEBAXoNSCrg4iP0W68A+ueff/D09KRZs2asWbNGa8oByTj01vnw3LlzvPPOO/ranFRQ2bnot14Bc+fOHVq3bs3t27c5efJkpiP9SHlPp0R34MABxo4dy6RJkzTTHF6/fp1u3brRsGFDTTcx6Q0mFGBVDMiqpb8C7N2gdIu8jCpP3L59m9atW3Pnzh0qV67M0aNHcXZ2NnZYEjpcuv788898+OGHFC1alEePHvHTTz+xYMECPvnkE3x8fLh8+TLVq1c3ZKxSfpf4APb7QlL6WYwC7YcSz5Kf56JC9yAiLCwMT09PwsPDqVy5MkeOHKF06dLGDkt6JsdndIsXL+abb74hOjqarVu3Eh0dzdKlS7l06RLLly+XSe5Np1LCvgEQf1+9XKwm2L30RbcvA+9uh8o98j4+AwoLC6N169aEh4dTpUoVgoKCZJLLZ3J8Rnfr1i169+4NQI8ePTAzM2PevHmUKVPGYMFJBcipWfDP7+r3NiWh90GwLvHsKWyE+p5c6RaF7kwO1N+NqKgoqlatypEjRzStEqT8I8eJ7unTp9jY2ADqkRYsLS3lf6ikFn4ETk5Tv1eYQOdNYFtKvezW2lhR5Zl27dqxd+9eatSoIb8T+ZROzUt++ukn7OzsAEhLS2PNmjUZhmySnfrfMAmRsLcfiGcPoppOh7Kexo0pD4SGhgJoBsr08vIyZjjSK+R4FjB3d/dXjpmlUCg0T2PzKzkLmB6plLC9Hdw9ol4u1x567i/0Q6aHhobSunVrFAoFQUFBVKpUydghFSqG+I7m+Izuzp07etmhVIicnPE8ydm5QqdfCn2SCwkJwdPTk4iICGrUqCH/WBYQhfu3UjKcpMdwYan6vcIUOm8GmxJGDcnQXkxyNWvW5PDhw7KdXAFh9ES3ZMkS3N3dsbKyonHjxpw+fTrb+o8fP2bkyJG4uLhgaWlJlSpV3sjZzI3OygneOwsub0Pzr6FM4WsA/KLr16/TunVrIiIiqFWrlkxyBUyuhlLXly1btuDn58fy5ctp3LgxixYtwtvbm5CQkEx/iVJSUmjXrh3Ozs5s376d0qVL888//+Dk5JT3wUvgUBZ8/iiUTUZeFBISQuvWrYmKiqJ27docOnRIzptSwOT4YYQhNG7cmIYNG/LDDz8A6pnG3Nzc+OSTT5gwYUKG+suXL2fevHlcv34910NQy4cRkq5iYmLw8vJCpVIRGBgok5yBGeI7arRL15SUFM6ePav1WN7ExAQvLy9Onsx8LtDdu3fTpEkTRo4cScmSJalZsyazZs3SDIeTmeTkZOLi4rReUi6F7YO9vpD8Zv0MixYtSmBgoDyTK8Bylehu3brF5MmT6devHw8ePABg//79XLlyJcfbiI6ORqlUUrJkSa3ykiVLEhkZmek6YWFhbN++HaVSyb59+5gyZQrz58/nq6++ynI/s2fPxtHRUfNyc3PLcYzSC+Luwv4BcH0TbGgACVHGjsigLl++zMqVKzXLRYsWpVixYkaMSHodOie6o0ePUqtWLU6dOsWOHTuIj48H4MKFC/j7++s9wBepVCqcnZ1ZsWIFHh4e+Pj48OWXX7J8+fIs15k4cSKxsbGalxzuPReUqbDHB5Ji1MtF3wKbwnsj/tKlS3h6ejJs2DA2b95s7HAkPdD5YcSECRP46quv8PPzw97eXlPepk0bzb22nChevDimpqZERWmfGURFRVGqVKlM13FxccHc3FxrIMPq1asTGRlJSkoKFhYWGdaxtLTE0tIyx3FJz6iUz/up3vwNIp7dTnBwhw6roZBOuHzx4kXatm1LdHQ0Hh4etG/f3tghSXqg8xndpUuX6N69e4ZyZ2dnoqOjc7wdCwsLPDw8OHTokKZMpVJx6NAhmjRpkuk6zZo14+bNm1rj3t24cQMXF5dMk5yUS6E71DN2bfWEfb5wY6u6XGEKXbaCVRGjhmcoFy5coE2bNkRHR9OgQQMOHjxI0aJFjR2WpAc6JzonJyciIjIOg33+/Hmdh6bx8/Nj5cqVrF27lmvXrjF8+HASEhIYMmQIAAMHDmTixIma+sOHDycmJobRo0dz48YN9u7dy6xZsxg5cqSuhyFlJX0Gr8zmfRBK9XwPhVBwcDBt2rThv//+o2HDhhw8eJAiRQpnQn8T6Xzp2rdvX8aPH8+2bdtQKBSoVCpOnDjB2LFjGThwoE7b8vHx4eHDh0ydOpXIyEjq1q1LQECA5gFFeHi4ZmpFADc3Nw4cOMCYMWOoXbs2pUuXZvTo0YwfP17Xw5Ayk+0MXlBYZ/CKioqibdu2xMTE0KhRI37//XccHR2NHZakRzq3o0tJSWHkyJGsWbMGpVKJmZkZSqUSX1/fAjERiGxHl427QerL1Vfpc6TQDb80c+ZM9u7dy4EDB2SSMzJDfEdz3WA4PDycy5cvEx8fT7169TTD1eR3MtFl49om9T25V+m0Ear3M3w8eSw5OVk+uMoH8kWD4ePHjwNQtmxZOnXqRJ8+fQpMkpNe4Q2awevMmTN07dpV0zwKkEmuENM50bVp04by5cszadIkrl69aoiYJGMp3QLsylDYZ/A6ffo07dq1Y/fu3UydOtXY4Uh5QOdEd//+fT7//HOOHj1KzZo1qVu3LvPmzePff7ObnV0qEExMoc3iZwsvJ7vCMYPXqVOnaNeuHbGxsTRv3pzp06cbOyQpD+ic6IoXL86oUaM4ceKEZsKctWvX4u7uTps2bQwRo5SXKvdQz9RVCGfw+uuvv2jfvj1xcXG0aNGC/fv3azV6lwqv1x69RKlUsn//fqZMmcLFixez7WCfH8iHEa8gBBybACXqgKkVKJMLxQxeJ0+exNvbmydPntCyZUv27t2rmf9Eyl+MOpT6y06cOMGGDRvYvn07SUlJdO3aldmzZ+slKMmI4u7Ambnq92XbQO9D2VYvCFJTU3nvvfd48uQJrVu3Zs+ePdja2ho7LCkP6XzpOnHiRMqXL0+bNm0IDw9n8eLFREZGsn79ejp06GCIGKW8FH7k+Xu3wjGbl7m5OTt27KBXr14yyb2hdD6j++OPP/jiiy/o06ePHJurMPo36Pn7Mq2NFYVePH36FGtrawDq1KnDtm3bjByRZCw6n9GdOHGCESNGyCRXGAnx/IzOzBpKNTRuPK/h2LFjVKhQQdPuU3qz5eiMbvfu3XTs2BFzc3N2796dbd13331XL4FJRhAb9rwzv2szMCuYDWj/+OMPOnXqREJCAgsWLKB58+bGDkkyshwlum7duhEZGYmzszPdunXLsp5Cocj3T12lbNwNev6+gPZlPXr0KJ06dSIxMZH27duzYcMGY4ck5QM5SnQvjv/24nupkLlbsB9EBAUF0blzZxITE/H29mbnzp2ae3TSm03ne3Tr1q0jOTk5Q3lKSgrr1q3TS1CSEQjx/IzOzAZKNTBmNDo7fPiw5kyuQ4cO/PbbbzLJSRo6J7ohQ4YQGxubofzJkyeaATOlAujxTYi/p35fuhmYFqwRm1esWMHTp0/p1KkTO3fuxMrKytghSfmIzs1LhBAoMpkv4N9//5XjeBVkZjbw9lT15au7t7Gj0dnatWupXbs2n3/+uRyFRMogx13A6tWrh0Kh4MKFC9SoUQMzs+c5UqlUcvv2bTp06MDWrVsNFqw+yC5ghce1a9eoVq1apn94pYLLqF3A0p+2BgcH4+3trdVP0MLCAnd3d3r27KmXoCTpVQ4cOEDXrl0ZNmwYixcvlslOylaOE136nK3u7u74+PjIeyCS0QQEBNCtWzeSk5MJDw8nLS0Nc3NzY4cl5WM636MbNGiQIeKQjCn6ivrfYm/l+/la9+/fT/fu3UlOTqZbt25s2bJFJjnplXKU6IoWLcqNGzcoXrw4RYoUyfYyISYmRm/BSXnkr5kQsgWsS0C/P6FIJWNHlKl9+/bRvXt3UlJS6N69u0xyUo7lKNEtXLhQM0DhwoUL5f2QwuTF9nNpT8HR3YjBZG3Pnj307NmTlJQUevbsyaZNm2SSk3IsR4nuxcvVwYMHGyoWyRhiQiAxSv2+TAswyfUQhQYVHx9PWloavXr1YuPGjTLJSTrR+bf63LlzmJubU6tWLQB27drF6tWreeutt5g2bRoWFgWroekbr4B0++rbty+urq40adJEJjlJZzr3jPjoo4+4ceMGAGFhYfj4+GBjY8O2bdsYN26c3gOUDOxu0PP3+awj//79+7l3755muWXLljLJSbmic6K7ceMGdevWBWDbtm20atWKjRs3smbNGn799Vd9xycZkhDPB9q0cADnekYN50U7d+7k3XffxdPTk4cPHxo7HKmA0znRCSE0I5gEBgbSqVMnANzc3IiOjtZvdJJhxVyDxAfq9/no/tyOHTvo06cPaWlpNGjQgCJFihg7JKmA0znRNWjQgK+++or169dz9OhROnfuDMDt27cpWbKk3gOUDOhu0PP3+WTY9F9//VWT5Hx9fVm3bp1Wd0NJyg2dE92iRYs4d+4co0aN4ssvv6RSJXWbq+3bt9O0aVO9BygZiEqpbjuXrkxL48XyzLZt2/Dx8UGpVPLee+/JJCfpzWvP65ouKSkJU1PTfH+zWHbqB0J3wOHRz4dNB7ArA20WG22C6v/97390794dpVLJgAEDWL16NaamBXceWSn38tW8rmfPnuXatWsAvPXWW9SvX18vAUkGFroDdvcCXvr7Fn9PXf7udqMkOw8PDypWrEiTJk34+eefZZKT9ErnRPfgwQN8fHw4evQoTk5OADx+/BhPT082b95MiRIl9B2jpC8qpfpM7uUkB8/KFHDkM6jYFUzyNtG4urpy4sQJihQpIpOcpHc636P75JNPiI+P58qVK8TExBATE8Ply5eJi4vj008/NUSMkr7cO6Z9uZqBgCd31fXywMaNG7UmrylevLhMcpJB6HxGFxAQQGBgINWrV9eUvfXWWyxZsoT27dvrNThJz+Ij9FvvNWzYsIGBAwcCULFiRd5++22D71N6c+l8RqdSqTJ94GBubi5nCMvv7Fz0Wy+X1q9fz8CBA1GpVAwdOpRGjRoZdH+SpHOia9OmDaNHj+b+/fuasnv37jFmzBjatm2bqyCWLFmCu7s7VlZWNG7cmNOnT+dovc2bN6NQKLKda1Z6QekW6qerZDX6jALs3dT1DGTt2rUMGjQIlUrFRx99xPLlyzEx0fnXUJJ0ovNv2A8//EBcXBzu7u5UrFiRihUrUr58eeLi4vj+++91DmDLli34+fnh7+/PuXPnqFOnDt7e3jx48CDb9e7cucPYsWNp0cJwX8pCx8RU3YQkU8+Sn+cigz2IWLNmDUOGDEEIwccff8zSpUtlkpPyRK7a0QkhOHTokKZ5SfXq1fHy8spVAI0bN6Zhw4b88MMPgPrS2M3NjU8++YQJEyZkuo5SqaRly5a8//77HDt2jMePH/Pbb7/laH+yHR3qJiZ7+4My6XmZvZs6yRmoacmpU6do0qQJQgiGDx/ODz/8IJOclCmjt6PbsmULu3fvJiUlhbZt2/LJJ5+81s5TUlI4e/YsEydO1JSZmJjg5eXFyZMns1xvxowZODs7M3ToUI4dy/4JYXJystaE23Fxca8Vc6FQuYd6SsNbu9TLXX6FSoZtUtKoUSM+/fRTUlNT+eGHH+TgrVKeynGiW7ZsGSNHjqRy5cpYW1uzY8cObt26xbx583K98+joaJRKZYY+siVLluT69euZrnP8+HF+/vlngoODc7SP2bNnM3369FzHWGi9mGhKNzVYkkufB1ihULBw4cJnu5ZJTspbOb52+OGHH/D39yckJITg4GDWrl3L0qVLDRlbBk+ePGHAgAGsXLmS4sWL52idiRMnEhsbq3ndvXvXwFFK6VasWEHXrl01Z9TpCU+S8lqOz+jCwsK0hlT39fVl6NChRERE4OKSu+YI6Q1Eo6KitMqjoqIoVapUhvq3bt3izp07dOnSRVOW3qTFzMyMkJAQKlasqLWOpaWlnLndCH788Uc+/vhjAH755ReGDh1q5IikN1mOz+iSk5OxtbV9vqKJCRYWFjx9+jTXO7ewsMDDw4NDhw5pylQqFYcOHaJJkyYZ6lerVo1Lly4RHByseaUPzhgcHIybm1uuY5H0Z9myZZokN2bMGN5//30jRyS96XR6GDFlyhRsbGw0yykpKXz99dc4OjpqyhYsWKBTAH5+fgwaNIgGDRrQqFEjFi1aREJCAkOGDAFg4MCBlC5dmtmzZ2NlZUXNmjW11k/vb/tyuWQcS5cuZeTIkQB8/vnnzJs3T16uSkaX40TXsmVLQkJCtMqaNm1KWFiYZjk3v9A+Pj48fPiQqVOnEhkZSd26dQkICNA8oAgPD5fNEAyhQhcwsYDUBIg8C+U7vPYDiSVLljBq1CgAvvjiC7755huZ5KR8QW/j0RUUsh0dBhmP7sGDB1SqVIknT54wbtw45syZI5OclCtGb0cnFQIGGo/O2dmZffv2ERgYiL+/v0xyUr4iz+jeJColrHTPZqgmBdiXgQ9u5/gyNjo6OsdNfSQpJwzxHZU3v94keh6PbsGCBVSrVi3HjbclyVhkonuT6HE8um+//ZbPP/+c//77j4CAgNcMTJIMSya6N4mexqObN28eX3zxBQD+/v5ZDr4gSflFrhLdsWPHeO+992jSpAn37t0D1IMpHj9+XK/BSXpWugWY22ZT4dXj0X3zzTeMGzcOgGnTpjFt2jT9xihJBqBzovv111/x9vbG2tqa8+fPa/oxxsbGMmvWLL0HKOlR2B51u7lMvXo8utmzZ2vO3qZPn46/v7/+Y5QkA9A50X311VcsX76clStXag2p3qxZM86dO6fX4CQ9SoiE3z94vmxZRPtz+zLZNi1JTU3lwIEDAMycOZOpU6caKlJJ0jud29GFhITQsmXGWd0dHR15/PixPmKS9E0ICBgCT6PVyxW7QpftcP+4+sGDnYv6cjWbJiXm5ubs2bOHnTt3MmDAgDwKXJL0Q+czulKlSnHz5s0M5cePH6dChQp6CUrSs+ClcOfZk1HbUtD+JzA1A7fWUL2f+t8sktzRo0c17+3s7GSSkwoknRPdhx9+yOjRozl16hQKhYL79++zYcMGxo4dy/Dhww0Ro/Q6Up7An1OeL3uvBpucNfCdPn06rVu3ZsaMGQYKTpLyhs6XrhMmTEClUtG2bVsSExNp2bIllpaWjB079rWHVpcMwMIefP6Aff2hdEt15/1XEEIwbdo0TYKzsrIydJSSZFC57gKWkpLCzZs3iY+P56233sLOzk7fsRnEG9sFLC0ZhArMrbOtJoTA39+fmTNnAjB37lxNmzlJygv5qlO/hYUFb731ll6CkPKA2atHWRZCMHXqVL766ivgee8HSSrodE50np6e2Y5Mcfjw4dcKSNKDpMdw/jtoOD5HCS7dlClT+PrrrwGYP38+fn5+BgpQkvKWzomubt26WsupqakEBwdz+fJlrTklJCM6NBKub1QPyfTOVihaJUerubq6ArBw4UI+++wzAwYoSXlL50SXPmXdy6ZNm0Z8fPxrByS9pmsb1UkOIO4OmOX8QcKIESNo3rw5tWvXNkxskmQkeuvU/95777Fq1Sp9bU7Kjbh/4NCI58tey8GhbJbVhRB89913/Pfff5oymeSkwkhvie7kyZOyGYIxqZSwfxAkx6qXq/eHan2zrC6EYNy4cYwePZp27dqRkpKSR4FKUt7T+dK1Rw/tvpBCCCIiIvj777+ZMmVKFmtJBvf3fPj3WS8G+7LQ5ocsqwoh+OKLL5g/fz4AH3zwARYWFnkRpSQZhc6J7sWpDUE9v2vVqlWZMWMG7du311tgkg6izsGJyc8WFNBpPVg5ZVpVCMHnn3+uudf64hysklRY6ZTolEolQ4YMoVatWhQpUuTVK0iGl5qo7vWgSlUvNxoPZTIOugDqJDdmzBgWL14MwPLly/noo4/yKlJJMhqd7tGZmprSvn17OUpJfhK8BGKuq98714em07OsOnPmTE2SW7FihUxy0htD54cRNWvW1Jq0WjKy+p/B21PA3A46bQDTrO+1+fr64ubmxsqVK/nwww/zLkZJMjKd+7oGBAQwceJEZs6ciYeHB7a22kNz5/f+o4W2r2tidI5GJUlISMjwfyZJ+YkhvqM5TnQzZszg888/x97e/vnKL3QFE0KgUChQKpV6CcxQCm2iy4RKpcLPzw8vLy/eeecdY4cjSTli1ERnampKREQE165dy7Zeq1at9BKYoRSKRBeyDZwqQsn6WVZRqVSMHDmS5cuXY2VlRVhYGC4uOZwFTJKMyKijl6Tnw/yeyAq9mBsQMFj9lLXZTGg4Dl4aZEGlUjF8+HBWrFiBQqFgxYoVMslJbzSdmpdkN2qJlAeUqbD/PUhLVC/HhmWa5D7++GNWrlyJiYkJa9eu5b333jNCsJKUf+iU6KpUqfLKZBcTE/NaAUnZ+GsmRJ5Rvy9SGVov0PpYpVIxbNgwfv75Z0xMTFi3bh39+/c3QqCSlL/olOimT5+eoWeElEfunYBT6rHiUJiqm5K8NBn12rVrNUlu/fr1+Pr6GiFQScp/dEp0ffv2xdnZ2VCxSFlJjoP9A9RDoQM0nQalGmaoNnDgQI4dO0a7du3o169f3sYoSflYjhOdvD9nREdGQ+xt9XvXZtBoouaj9OY8pqammJqayqGyJCkTOe4Zkcs5dKTXdWM7XFmjfm9hr+6w/2wO1vS+x4MHD8737RclyZhynOhUKpW8bM1rqU/h0Kjny21+AMfygDrJDRo0iPXr17Np0ybOnDljpCAlKf/T28CbkgGYW0P3PVCkClTpDW8NACAtLY2BAweyYcMGzMzM2LJlC2+//baRg5Wk/CtfJLolS5bg7u6OlZUVjRs35vTp01nWXblyJS1atKBIkSIUKVIELy+vbOsXeKUawIBz0H4lKBSkpaUxYMAANm7ciJmZGVu3bqVnz57GjlKS8jWjJ7otW7bg5+eHv78/586do06dOnh7e/PgwYNM6wcFBdGvXz+OHDnCyZMncXNzo3379ty7dy+PI89D5rZg6UhaWhrvvfcemzdvxszMjG3bttG9e3djRydJ+Z8wskaNGomRI0dqlpVKpXB1dRWzZ8/O0fppaWnC3t5erF27Nkf1Y2NjBSBiY2NzFa/BpT4V4ux3QihTM3z0999/CwsLC2Fubi5+++03IwQnSYZniO+ozkOp61NKSgpnz55l4sTnzSVMTEzw8vLi5MmTOdpGYmIiqampFC1aNNPPk5OTSU5O1izHxcW9XtCGdnwSnF0I1zeoGwU7VdR85OHhwW+//UZaWhpdunQxYpCSVLAY9dI1OjoapVJJyZIltcpLlixJZGRkjrYxfvx4XF1d8fLyyvTz2bNn4+joqHm5ubm9dtwG80+gOskBPAiGVHUSv3v3rqZKx44dZZKTJB0Z/R7d65gzZw6bN29m586dWU61OHHiRGJjYzWvF5NGvvL0PwgY9Hy5xRxSnarRt29fmjRpws2bN40XmyQVcEa9dC1evDimpqZERUVplUdFRVGqVKls1/3222+ZM2cOgYGB2U66bGlpiaWlpV7iNRgh4OBHEH9fvVzWi5SaH9PXx4edO3diYWHBrVu3qFSpknHjlKQCyqhndBYWFnh4eHDo0CFNmUql4tChQzRp0iTL9ebOncvMmTMJCAigQYMGeRGqYV1dB6G/qt9bFSWl7Ur6+PRl586dWFpasmvXLry9vY0boyQVYEY9owPw8/Nj0KBBNGjQgEaNGrFo0SISEhIYMmQIoO6oXrp0aWbPng3AN998w9SpU9m4cSPu7u6ae3l2dnbY2dkZ7Thy7XGYVu+H5FY/0Hvwp/zvf/+TSU6S9MToic7Hx4eHDx8ydepUIiMjqVu3LgEBAZoHFOHh4ZiYPD/xXLZsGSkpKfTq1UtrO/7+/kybNi0vQ399qjT1qCSp8QAkVx5Ar4kb2bNnD1ZWVuzatUtOCi5JeqDzLGAFXb6aMyJ4KRwaqX7vWIHH7wbRtmM3rl69yu7du2nXrp1x45MkIzDqnBGSAdR8Hx7fgvPfQcf1ODm7cfDgQa5evUrz5s2NHZ0kFRoFunlJgWdmRdLbX7Or5BIo3RSAokWLyiQnSXomE50RJSUl0b17d7r1/4hly5YZOxxJKrTkpWteCz8MtqV4alOebt268fvvv2NjY0P16tWNHZkkFVoy0eWlhEjY48PThCd0/dWdg6dCsLGxYd++fXK+XEkyIJno8ooQEDCExNhouq6GwNAQbG1t2bdvHy1btjR2dJJUqMlEl1eCl5J2K4B3V8OhULC1tWH//v20aNHC2JHlK0II0tLS5BwYhZipqSlmZmZ5OuGWTHR54b+r8MdYzEyhVQU4dd+a/QEH5NPVl6SkpBAREUFiYqKxQ5EMzMbGBhcXFywsLPJkf7LBsKEpU2BDY3gYrF6uO4q7Vcbl7+GijEClUhEaGoqpqSklSpTAwsJCTrFZCAkhSElJ4eHDhyiVSipXrqzV8wlkg+ECKSFwPFN/Dma6N9i5VIeWc3EztzZ2WPlOSkoKKpUKNzc3bGxsjB2OZEDW1taYm5vzzz//kJKSkuUQa/okE50BxV/fT+cRi/gjDG5EK/jf4Q3qmb2kLL38110qnPL6/1kmOgOJj31Ep3e7cywMHKxg8ucjoGQ9Y4clSW8kmegM4MmTJ3R6512OhybjaK3g98n1aDR4sbHDkqQ3lkx0evbkyRM6duzIiRMncHR05ODenTSsUw1MTI0d2ptDpYR7xyA+AuxcoHQL+fN/w8kbInr23nvvceLECZycnAgMDKRhM0/1l03KG6E7YKU7bPWEfb7qf1e6q8sNZPDgwSgUChQKBebm5pQvX55x48aRlJSUoe6ePXto1aoV9vb22NjY0LBhQ9asWZPpdn/99Vdat26No6MjdnZ21K5dmxkzZhATE2OwYymsZKLTJ5WSmYMbU7lyZQIDAwvHMO8FSegO2N0L4v/VLo+/py43YLLr0KEDERERhIWFsXDhQn788Uf8/f216nz//fd07dqVZs2acerUKS5evEjfvn35+OOPGTt2rFbdL7/8Eh8fHxo2bMj+/fu5fPky8+fP58KFC6xfv95gx/GylJSUPNuXQelthtgCwhCT46pUKvWbU3OE+BaRurGlELH/6G37b4KnT5+Kq1eviqdPn+ZuA8o0IZaXEeJbsngphPjRTV1PzwYNGiS6du2qVdajRw9Rr149zXJ4eLgwNzcXfn5+Gdb/7rvvBCD++usvIYQQp06dEoBYtGhRpvt79OhRlrHcvXtX9O3bVxQpUkTY2NgIDw8PzXYzi3P06NGiVatWmuVWrVqJkSNHitGjR4tixYqJ1q1bi379+ok+ffporZeSkiKKFSummTheqVSKWbNmCXd3d2FlZSVq164ttm3blmWc2f1/F7oJrAuDx48f07NnT6aP9qX5rSkAmN0/BnF3wKGscYMrLP5eAGcXZF8nLRmSorOpIODJXVhWCswymRXOww8a+L1WmOkuX77Mn3/+Sbly5TRl27dvJzU1NcOZG8BHH33EpEmT2LRpE40bN2bDhg3Y2dkxYsSITLfv5OSUaXl8fDytWrWidOnS7N69m1KlSnHu3DlUKpVO8a9du5bhw4dz4sQJAG7evEnv3r2Jj4/XzMty4MABEhMT6d69O6CeP/mXX35h+fLlVK5cmT/++IP33nuPEiVK5IsBK2Siew2PHz+mffv2nDlzhrCLxwgZm4qFGdBoPJSRHfX1JiVOffmpD1klw5S419rsnj17sLOzIy0tjeTkZExMTPjhhx80n9+4cQNHR0dcXDLer7WwsKBChQrcuHEDgNDQUCpUqIC5ublOMWzcuJGHDx9y5swZihYtCpCrKTIrV67M3LlzNcsVK1bE1taWnTt3MmDAAM2+3n33Xezt7UlOTmbWrFkEBgZqZu+rUKECx48f58cff5SJriB79OgR7du35++//6aYgxW/DUxSJznn+tB0urHDK1wsHMCudPZ1XnlG94xV8czP6Cxer6uRp6cny5YtIyEhgYULF2JmZkbPnj1ztS2Ry16ZwcHB1KtXT5PkcsvDw0Nr2czMjD59+rBhwwYGDBhAQkICu3btYvPmzYD6jC8xMTHDHCcpKSnUq5c/2o7KRJcLjx49ol27dpw9e5biRR04NDiO2q6AmTV02gCmedNR+Y3RIAeXlSql+ulq/D0gs0ShAPsy8MFtgzQ1sbW11Zw9rVq1ijp16vDzzz8zdOhQAKpUqUJsbCz379/H1dVVa92UlBRu3bqFp6enpu7x48dJTU3V6azO2jr7XjcmJiYZkmhqamqmx/Ky/v3706pVKx48eMDBgwextramQ4cOgPqSGWDv3r2ULq39Bym/TB4vn7rqKCYmBi8vL3WSK16Mw8PN1EkOoNW3UKyaUeN7Y5mYQpv0RtkvDwbwbNlzUZ60pzMxMWHSpElMnjyZp0+fAtCzZ0/Mzc2ZP39+hvrLly8nISGBfv36AeDr60t8fDxLly7NdPuPHz/OtLx27doEBwdn2fykRIkSREREaJUFBwfn6JiaNm2Km5sbW7ZsYcOGDfTu3VuThN966y0sLS0JDw+nUqVKWq/8MniFTHQ6+uabbzh37hwlSpTgyORa1Cry7JeqfCeoM9y4wb3pKveAd7dnvMy1L6Mur9wjz0Lp3bs3pqamLFmyBICyZcsyd+5cFi1axJdffsn169e5desWCxYsYNy4cXz++ec0btwYgMaNG2vKxo0bx8mTJ/nnn384dOgQvXv3Zu3atZnus1+/fpQqVYpu3bpx4sQJwsLC+PXXXzl58iQAbdq04e+//2bdunWEhobi7+/P5cuXc3xMvr6+LF++nIMHD9K/f39Nub29PWPHjmXMmDGsXbuWW7duce7cOb7//vssY81zent+W0C87qPr5ORkMWTIEHF515znTReWlBAiPlLPkb5ZXrt5yYuUaUKEHxHi6kb1vwZoUvKizJptCCHE7NmzRYkSJUR8fLymbNeuXaJFixbC1tZWWFlZCQ8PD7Fq1apMt7tlyxbRsmVLYW9vL2xtbUXt2rXFjBkzsm1ecufOHdGzZ0/h4OAgbGxsRIMGDcSpU6c0n0+dOlWULFlSODo6ijFjxohRo0ZlaF4yevToTLd99epVAYhy5co9b1L1jEqlEosWLRJVq1YV5ubmokSJEsLb21scPXo0023ldfMSOR5dDtext7fXHh8tNQGOjoULy6HbbqjYxUARvxmSkpK4ffs25cuXz5NheyTjyu7/2xDj0clL11d4+PAhzZs3Z/To0do3cs1twWsZDLwgk5wk5XMy0WXjwYMHtGnThkuXLrFt2zYiIyMzVipRO+8DkyRJJzLRZSE9yV2+fBkXFxeCgoJwMbkPj28ZOzRJknQkE10moqKi8PT05MqVK7i6uhIUFERVdxd1x/B1deHSKvX0hZIkFQgy0b0kMjIST09Prl69SunSpQkKCqJKlSpw5FN1/9XUeLi8CoRu/QelnHnDno29sfL6/1kmupecPn2akJAQTZKrXLkyhGyDK8/aA1nYQ6f1ciBHPUtvfCqnOnwzpP8/69qfN7dkF7CXvPvuu2zdupU6deqou/Q8uQeBHz2v0OYHcCxvvAALKVNTU5ycnHjw4AGgnvdTTndY+AghSExM5MGDBzg5OWFqmjcnDDLRAREREahUKk0/PU1nbKGCgEGQ9Ei9XKU3vDXASFEWfqVKlQLQJDup8HJyctL8f+eFNz7R3b9/H09PT1QqFUFBQdqdks8thvBD6vd2pcFrOcizDINRKBS4uLjg7OycaWdzqXAwNzfPszO5dG90ort37x6enp6EhoZStmxZ9bDR6ROrRJyCE5OfV+6wFqxfb/gbKWdMTU3z/IsgFW754mHEkiVLcHd3x8rKisaNG3P69Ols62/bto1q1aphZWVFrVq12Ldvn877vHfvHq1btyY0NJRy5cpx9OhRyqedfz6xyrEJoEpTV67QBcq1zcWRSZKUHxg90W3ZsgU/Pz/8/f05d+4cderUwdvbO8v7NH/++Sf9+vVj6NChnD9/nm7dutGtWzedRmEA6Ny5Mzdv3sTd3Z2jR4/innou84lVAML2GHRiFUmSDMvonfobN25Mw4YNNcNOq1Qq3Nzc+OSTT5gwYUKG+j4+PiQkJLBnzx5N2dtvv03dunVZvnz5K/eX3mEYoHz58hw5coRybmWeDdqYSZIDDD1ooyRJzxmiU79R79GlpKRw9uxZJk6cqCkzMTHBy8tLM4bWy06ePImfn/Zos97e3vz222+Z1k9OTiY5OVmzHBsbC6jHB/vf//5HkSJFiLseANFZJTkAAUl34XoAlGmRs4OTJClX4uLU83fo8xzMqIkuOjoapVJJyZIltcpLlizJ9evXM10nMjIy0/qZdrhHPTvR9OkZ53AIDw+nZs2augU8+R3d6kuSlGv//fef5urrdRX6p64TJ07UOgN8/Pgx5cqVIzw8/PkP8d9jsDMHSaz7HqOe0cXFxeHm5sbdu3f1dkqfFwpq3FBwYy+ocYP6qqts2bKvPcnPi4ya6IoXL46pqSlRUVFa5VFRUVk2JixVqpRO9S0tLTOdoMPR0fH5L0C1DlC8zKsnVqnWIV/co3NwcChwv7xQcOOGght7QY0b1Lex9LYtvW0pFywsLPDw8ODQoUOaMpVKxaFDhzTzQ76sSZMmWvUBDh48mGX9HMlHE6tIkqR/Rm9e4ufnx8qVK1m7di3Xrl1j+PDhJCQkMGTIEAAGDhyo9bBi9OjRBAQEMH/+fK5fv860adP4+++/GTVq1OsFko8mVpEkSb+Mfo/Ox8eHhw8fMnXqVCIjI6lbty4BAQGaBw7h4eFap7BNmzZl48aNTJ48mUmTJlG5cmV+++23HD9YsLS0xN/fP/P5Jiv3gIpd1T0j4iPAzgVKt8g3Z3LZxp6PFdS4oeDGXlDjBsPEbvR2dJIkSYZm9EtXSZIkQ5OJTpKkQk8mOkmSCj2Z6CRJKvQKZaIzxrBP+qJL7CtXrqRFixYUKVKEIkWK4OXl9cpjNRRdf+bpNm/ejEKhoFu3boYNMBu6xv748WNGjhyJi4sLlpaWVKlSxSi/M7rGvWjRIqpWrYq1tTVubm6MGTOGpKSkPIpW7Y8//qBLly64urqiUCiy7KP+oqCgIOrXr4+lpSWVKlVizZo1uu9YFDKbN28WFhYWYtWqVeLKlSviww8/FE5OTiIqKirT+idOnBCmpqZi7ty54urVq2Ly5MnC3NxcXLp0KY8j1z12X19fsWTJEnH+/Hlx7do1MXjwYOHo6Cj+/ffffB13utu3b4vSpUuLFi1aiK5du+ZNsC/RNfbk5GTRoEED0alTJ3H8+HFx+/ZtERQUJIKDg/N13Bs2bBCWlpZiw4YN4vbt2+LAgQPCxcVFjBkzJk/j3rdvn/jyyy/Fjh07BCB27tyZbf2wsDBhY2Mj/Pz8xNWrV8X3338vTE1NRUBAgE77LXSJrlGjRmLkyJGaZaVSKVxdXcXs2bMzrd+nTx/RuXNnrbLGjRuLjz76yKBxZkbX2F+WlpYm7O3txdq1aw0VYqZyE3daWppo2rSp+Omnn8SgQYOMluh0jX3ZsmWiQoUKIiUlJa9CzJSucY8cOVK0adNGq8zPz080a9bMoHFmJyeJbty4caJGjRpaZT4+PsLb21unfRWqS9f0YZ+8vLw0ZTkZ9unF+qAe9imr+oaSm9hflpiYSGpqql47Q79KbuOeMWMGzs7ODB06NC/CzFRuYt+9ezdNmjRh5MiRlCxZkpo1azJr1iyUSmVehZ2ruJs2bcrZs2c1l7dhYWHs27ePTp065UnMuaWv76fRe0boU14M+2QouYn9ZePHj8fV1TXDL4Yh5Sbu48eP8/PPPxMcHJwHEWYtN7GHhYVx+PBh+vfvz759+7h58yYjRowgNTUVf3//vAg7V3H7+voSHR1N8+bNEUKQlpbGxx9/zKRJk/Ii5FzL6vsZFxfH06dPsba2ztF2CtUZ3Ztszpw5bN68mZ07d2JlZWXscLL05MkTBgwYwMqVKylevLixw9GZSqXC2dmZFStW4OHhgY+PD19++WWORrc2pqCgIGbNmsXSpUs5d+4cO3bsYO/evcycOdPYoeWJQnVGlxfDPhlKbmJP9+233zJnzhwCAwOpXbu2IcPMQNe4b926xZ07d+jSpYumTKVSAWBmZkZISAgVK1Y0bNDP5OZn7uLikmG6vurVqxMZGUlKSgoWFhYGjRlyF/eUKVMYMGAAH3zwAQC1atUiISGBYcOG8eWXX+p1SCR9yur76eDgkOOzOShkZ3T5ZtinXMhN7ABz585l5syZBAQE0KBBg7wIVYuucVerVo1Lly4RHByseb377rt4enoSHByMm5tbvo0doFmzZty8eVOTnAFu3LiBi4tLniQ5yF3ciYmJGZJZerIW+bi7u96+n7o9J8n/Nm/eLCwtLcWaNWvE1atXxbBhw4STk5OIjIwUQggxYMAAMWHCBE39EydOCDMzM/Htt9+Ka9euCX9/f6M2L9El9jlz5ggLCwuxfft2ERERoXk9efIkX8f9MmM+ddU19vDwcGFvby9GjRolQkJCxJ49e4Szs7P46quv8nXc/v7+wt7eXmzatEmEhYWJ33//XVSsWFH06dMnT+N+8uSJOH/+vDh//rwAxIIFC8T58+fFP//8I4QQYsKECWLAgAGa+unNS7744gtx7do1sWTJEtm8JN33338vypYtKywsLESjRo3EX3/9pfmsVatWYtCgQVr1t27dKqpUqSIsLCxEjRo1xN69e/M44ud0ib1cuXIC9ZDIWi9/f/98HffLjJnohNA99j///FM0btxYWFpaigoVKoivv/5apKWl5XHUusWdmpoqpk2bJipWrCisrKyEm5ubGDFihHj06FGexnzkyJFMf2fTYx00aJBo1apVhnXq1q0rLCwsRIUKFcTq1at13q8cpkmSpEKvUN2jkyRJyoxMdJIkFXoy0UmSVOjJRCdJUqEnE50kSYWeTHSSJBV6MtFJklToyUQnSVKhJxOdxJo1a3BycjJ2GLmWkyG5Bw8ebNTh2iXjkomukBg8eDAKhSLD6+bNm8YOjTVr1mjiMTExoUyZMgwZMoQHDx7oZfsRERF07NgRgDt37qBQKDKMdbd48eLczTWgg2nTpmmO09TUFDc3N4YNG0ZMTIxO25FJWf8K1TBNb7oOHTqwevVqrbISJUoYKRptDg4OhISEoFKpuHDhAkOGDOH+/fscOHDgtbedkyG1HB0dX3s/OVGjRg0CAwNRKpVcu3aN999/n9jYWLZs2ZIn+5cyJ8/oChFLS0tKlSql9TI1NWXBggXUqlULW1tb3NzcGDFiBPHx8Vlu58KFC3h6emJvb4+DgwMeHh78/fffms+PHz9OixYtNLNJffrppyQkJGQbm0KhoFSpUri6utKxY0c+/fRTAgMDefr0KSqVihkzZlCmTBksLS2pW7cuAQEBmnVTUlIYNWoULi4uWFlZUa5cOWbPnq217fRL1/LlywNQr149FAoFrVu3BrTPklasWIGrq6vWUEsAXbt25f3339cs79q1i/r162NlZUWFChWYPn06aWlp2R6nmZkZpUqVonTp0nh5edG7d28OHjyo+VypVDJ06FDKly+PtbU1VatWZfHixZrPp02bxtq1a9m1a5fm7DAoKAiAu3fv0qdPH5ycnChatChdu3blzp072cYjqclE9wYwMTHhu+++48qVK6xdu5bDhw8zbty4LOv379+fMmXKcObMGc6ePcuECRMwNzcH1ANndujQgZ49e3Lx4kW2bNnC8ePHGTVqlE4xWVtbo1KpSEtLY/HixcyfP59vv/2Wixcv4u3tzbvvvktoaCgA3333Hbt372br1q2EhISwYcMG3N3dM91u+pwIgYGBREREsGPHjgx1evfuzX///ceRI0c0ZTExMQQEBNC/f38Ajh07xsCBAxk9ejRXr17lxx9/ZM2aNXz99dc5PsY7d+5w4MABrXHqVCoVZcqUYdu2bVy9epWpU6cyadIktm7dCsDYsWPp06cPHTp0ICIigoiICJo2bUpqaire3t7Y29tz7NgxTpw4gZ2dHR06dCAlJSXHMb2xXnfYFSl/GDRokDA1NRW2traaV69evTKtu23bNlGsWDHN8urVq4Wjo6Nm2d7eXqxZsybTdYcOHSqGDRumVXbs2DFhYmIinj59muk6L2//xo0bokqVKqJBgwZCCCFcXV3F119/rbVOw4YNxYgRI4QQQnzyySeiTZs2QqVSZbp9XphN6vbt2wIQ58+f16rz8lBQXbt2Fe+//75m+ccffxSurq5CqVQKIYRo27atmDVrltY21q9fL1xcXDKNQQj1mG8mJibC1tZWWFlZaYYgWrBgQZbrCKGeoatnz55Zxpq+76pVq2r9DJKTk4W1tbU4cOBAttuXhJD36AoRT09Pli1bplm2tbUF1Gc3s2fP5vr168TFxZGWlkZSUhKJiYnY2Nhk2I6fnx8ffPAB69ev11x+pQ9vfuHCBS5evMiGDRs09YUQqFQqbt++TfXq1TONLTY2Fjs7O1QqFUlJSTRv3pyffvqJuLg47t+/T7NmzbTqN2vWjAsXLgDqy8527dpRtWpVOnTowDvvvEP79u1f62fVv39/PvzwQ5YuXYqlpSUbNmygb9++mlF4L1y4wIkTJ7TO4JRKZbY/N4CqVauye/dukpKS+OWXXwgODuaTTz7RqrNkyRJWrVpFeHg4T58+JSUlhbp162Yb74ULF7h58yb29vZa5UlJSdy6dSsXP4E3i0x0hYitrS2VKlXSKrtz5w7vvPMOw4cP5+uvv6Zo0aIcP36coUOHkpKSkukXdtq0afj6+rJ3717279+Pv78/mzdvpnv37sTHx/PRRx/x6aefZlivbNmyWcZmb2/PuXPnMDExwcXFRTPef1xc3CuPq379+ty+fZv9+/cTGBhInz598PLyYvv27a9cNytdunRBCMHevXtp2LAhx44dY+HChZrP4+PjmT59Oj169MiwbnaTD1lYWGj+D+bMmUPnzp2ZPn26ZhKazZs3M3bsWObPn0+TJk2wt7dn3rx5nDp1Ktt44+Pj8fDw0PoDky6/PHDKz2SiK+TOnj2LSqVi/vz5mrOV9PtB2alSpQpVqlRhzJgx9OvXj9WrV9O9e3fq16/P1atXMyTUVzExMcl0HQcHB1xdXTlx4gStWrXSlJ84cYJGjRpp1fPx8cHHx4devXrRoUMHYmJiMsxhm34/7FXzrFpZWdGjRw82bNjAzZs3qVq1KvXr19d8Xr9+fUJCQnQ+zpdNnjyZNm3aMHz4cM1xNm3alBEjRmjqvHxGZmFhkSH++vXrs2XLFpydnXFwcHitmN5E8mFEIVepUiVSU1P5/vvvCQsLY/369dlOzff06VNGjRpFUFAQ//zzDydOnODMmTOaS9Lx48fz559/MmrUKIKDgwkNDWXXrl06P4x40RdffME333zDli1bCAkJYcKECQQHBzN69GgAFixYwKZNm7h+/To3btxg27ZtlCpVKtNGzs7OzlhbWxMQEEBUVBSxsbFZ7rd///7s3buXVatWaR5CpJs6dSrr1q1j+vTpXLlyhWvXrrF582YmT56s07E1adKE2rVrM2vWLAAqV67M33//zYEDB7hx4wZTpkzhzJkzWuu4u7tz8eJFQkJCiI6OJjU1lf79+1O8eHG6du3KsWPHuH37NkFBQXz66af8+++/OsX0RjL2TUJJP7Kbd2HBggXCxcVFWFtbC29vb7Fu3ToBaOYLePFhQXJysujbt69wc3MTFhYWwtXVVYwaNUrrQcPp06dFu3bthJ2dnbC1tRW1a9fO8DDhRS8/jHiZUqkU06ZNE6VLlxbm5uaiTp06Yv/+/ZrPV6xYIerWrStsbW2Fg4ODaNu2rTh37pzmc154GCGEECtXrhRubm7CxMREM/9AZj8fpVIpXFxcBCBu3bqVIa6AgADRtGlTYW1tLRwcHESjRo3EihUrsjwOf39/UadOnQzlmzZtEpaWliI8PFwkJSWJwYMHC0dHR+Hk5CSGDx8uJkyYoLXegwcPND9fQBw5ckQIIURERIQYOHCgKF68uGa+ig8//FDExsZmGZOkJueMkCSp0JOXrpIkFXoy0UmSVOjJRCdJUqEnE50kSYWeTHSSJBV6MtFJklToyUQnSVKhJxOdJEmFnkx0kiQVejLRSZJU6MlEJ0lSofd/bBTMR3T5vi8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Works\n",
    "\n",
    "#plt.style.use('ggplot')\n",
    "plt.rcParams[\"figure.figsize\"] = (3,3)\n",
    "%matplotlib inline\n",
    "\n",
    "# false positive rate\n",
    "FPR = []\n",
    "# true positive rate\n",
    "TPR = []\n",
    "# Iterate thresholds from 0.0 to 1.0\n",
    "thresholds = np.arange(0.0, 1.01, 0.1)\n",
    "# array([0. , 0.2, 0.4, 0.6, 0.8, 1. ])\n",
    "\n",
    "# get number of positive and negative examples in the dataset\n",
    "P = sum(y_testListFlat)\n",
    "N = len(y_testListFlat) - P\n",
    "\n",
    "# iterate through all thresholds and determine fraction of true positives\n",
    "# and false positives found at this threshold\n",
    "for thresh in thresholds:\n",
    "    FP=0\n",
    "    TP=0\n",
    "    thresh = round(thresh,2) #Limiting floats to two decimal points, or threshold 0.6 will be 0.6000000000000001 which gives FP=0\n",
    "    for i in range(len(y_probListFlat)):\n",
    "        if (y_probListFlat[i] >= thresh):\n",
    "            if y_testListFlat[i] == 1:\n",
    "                TP = TP + 1\n",
    "            if y_testListFlat[i] == 0:\n",
    "                FP = FP + 1\n",
    "    FPR.append(FP/N)\n",
    "    TPR.append(TP/P)\n",
    "\n",
    "# This is the AUC\n",
    "#you're integrating from right to left. This flips the sign of the result\n",
    "auc = -1 * np.trapz(TPR, FPR)\n",
    "\n",
    "plt.plot(FPR, TPR, linestyle='--', marker='o', color='darkorange', lw = 2, label='ROC curve', clip_on=False)\n",
    "plt.plot([0, 1], [0, 1], color='black', linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC curve, AUC = %.2f'%auc)\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.savefig('AUC_example.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 945,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 945,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_probList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 946,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrap #1 ROC area: 0.634\n",
      "Bootstrap #2 ROC area: 0.634\n",
      "Bootstrap #3 ROC area: 0.634\n",
      "Bootstrap #4 ROC area: 0.634\n",
      "Bootstrap #5 ROC area: 0.634\n",
      "Bootstrap #6 ROC area: 0.634\n",
      "Bootstrap #7 ROC area: 0.634\n",
      "Bootstrap #8 ROC area: 0.634\n",
      "Bootstrap #9 ROC area: 0.634\n",
      "Bootstrap #10 ROC area: 0.634\n",
      "Bootstrap #11 ROC area: 0.634\n",
      "Bootstrap #12 ROC area: 0.634\n",
      "Bootstrap #13 ROC area: 0.634\n",
      "Bootstrap #14 ROC area: 0.634\n",
      "Bootstrap #15 ROC area: 0.634\n",
      "Bootstrap #16 ROC area: 0.634\n",
      "Bootstrap #17 ROC area: 0.634\n",
      "Bootstrap #18 ROC area: 0.634\n",
      "Bootstrap #19 ROC area: 0.634\n",
      "Bootstrap #20 ROC area: 0.634\n",
      "Bootstrap #21 ROC area: 0.634\n",
      "Bootstrap #22 ROC area: 0.634\n",
      "Bootstrap #23 ROC area: 0.634\n",
      "Bootstrap #24 ROC area: 0.634\n",
      "Bootstrap #25 ROC area: 0.634\n",
      "Bootstrap #26 ROC area: 0.634\n",
      "Bootstrap #27 ROC area: 0.634\n",
      "Bootstrap #28 ROC area: 0.634\n",
      "Bootstrap #29 ROC area: 0.634\n",
      "Bootstrap #30 ROC area: 0.634\n",
      "Bootstrap #31 ROC area: 0.634\n",
      "Bootstrap #32 ROC area: 0.634\n",
      "Bootstrap #33 ROC area: 0.634\n",
      "Bootstrap #34 ROC area: 0.634\n",
      "Bootstrap #35 ROC area: 0.634\n",
      "Bootstrap #36 ROC area: 0.634\n",
      "Bootstrap #37 ROC area: 0.634\n",
      "Bootstrap #38 ROC area: 0.634\n",
      "Bootstrap #39 ROC area: 0.634\n",
      "Bootstrap #40 ROC area: 0.634\n",
      "Bootstrap #41 ROC area: 0.634\n",
      "Bootstrap #42 ROC area: 0.634\n",
      "Bootstrap #43 ROC area: 0.634\n",
      "Bootstrap #44 ROC area: 0.634\n",
      "Bootstrap #45 ROC area: 0.634\n",
      "Bootstrap #46 ROC area: 0.634\n",
      "Bootstrap #47 ROC area: 0.634\n",
      "Bootstrap #48 ROC area: 0.634\n",
      "Bootstrap #49 ROC area: 0.634\n",
      "Bootstrap #50 ROC area: 0.634\n",
      "Bootstrap #51 ROC area: 0.634\n",
      "Bootstrap #52 ROC area: 0.634\n",
      "Bootstrap #53 ROC area: 0.634\n",
      "Bootstrap #54 ROC area: 0.634\n",
      "Bootstrap #55 ROC area: 0.634\n",
      "Bootstrap #56 ROC area: 0.634\n",
      "Bootstrap #57 ROC area: 0.634\n",
      "Bootstrap #58 ROC area: 0.634\n",
      "Bootstrap #59 ROC area: 0.634\n",
      "Bootstrap #60 ROC area: 0.634\n",
      "Bootstrap #61 ROC area: 0.634\n",
      "Bootstrap #62 ROC area: 0.634\n",
      "Bootstrap #63 ROC area: 0.634\n",
      "Bootstrap #64 ROC area: 0.634\n",
      "Bootstrap #65 ROC area: 0.634\n",
      "Bootstrap #66 ROC area: 0.634\n",
      "Bootstrap #67 ROC area: 0.634\n",
      "Bootstrap #68 ROC area: 0.634\n",
      "Bootstrap #69 ROC area: 0.634\n",
      "Bootstrap #70 ROC area: 0.634\n",
      "Bootstrap #71 ROC area: 0.634\n",
      "Bootstrap #72 ROC area: 0.634\n",
      "Bootstrap #73 ROC area: 0.634\n",
      "Bootstrap #74 ROC area: 0.634\n",
      "Bootstrap #75 ROC area: 0.634\n",
      "Bootstrap #76 ROC area: 0.634\n",
      "Bootstrap #77 ROC area: 0.634\n",
      "Bootstrap #78 ROC area: 0.634\n",
      "Bootstrap #79 ROC area: 0.634\n",
      "Bootstrap #80 ROC area: 0.634\n",
      "Bootstrap #81 ROC area: 0.634\n",
      "Bootstrap #82 ROC area: 0.634\n",
      "Bootstrap #83 ROC area: 0.634\n",
      "Bootstrap #84 ROC area: 0.634\n",
      "Bootstrap #85 ROC area: 0.634\n",
      "Bootstrap #86 ROC area: 0.634\n",
      "Bootstrap #87 ROC area: 0.634\n",
      "Bootstrap #88 ROC area: 0.634\n",
      "Bootstrap #89 ROC area: 0.634\n",
      "Bootstrap #90 ROC area: 0.634\n",
      "Bootstrap #91 ROC area: 0.634\n",
      "Bootstrap #92 ROC area: 0.634\n",
      "Bootstrap #93 ROC area: 0.634\n",
      "Bootstrap #94 ROC area: 0.634\n",
      "Bootstrap #95 ROC area: 0.634\n",
      "Bootstrap #96 ROC area: 0.634\n",
      "Bootstrap #97 ROC area: 0.634\n",
      "Bootstrap #98 ROC area: 0.634\n",
      "Bootstrap #99 ROC area: 0.634\n",
      "Bootstrap #100 ROC area: 0.634\n",
      "Bootstrap #101 ROC area: 0.634\n",
      "Bootstrap #102 ROC area: 0.634\n",
      "Bootstrap #103 ROC area: 0.634\n",
      "Bootstrap #104 ROC area: 0.634\n",
      "Bootstrap #105 ROC area: 0.634\n",
      "Bootstrap #106 ROC area: 0.634\n",
      "Bootstrap #107 ROC area: 0.634\n",
      "Bootstrap #108 ROC area: 0.634\n",
      "Bootstrap #109 ROC area: 0.634\n",
      "Bootstrap #110 ROC area: 0.634\n",
      "Bootstrap #111 ROC area: 0.634\n",
      "Bootstrap #112 ROC area: 0.634\n",
      "Bootstrap #113 ROC area: 0.634\n",
      "Bootstrap #114 ROC area: 0.634\n",
      "Bootstrap #115 ROC area: 0.634\n",
      "Bootstrap #116 ROC area: 0.634\n",
      "Bootstrap #117 ROC area: 0.634\n",
      "Bootstrap #118 ROC area: 0.634\n",
      "Bootstrap #119 ROC area: 0.634\n",
      "Bootstrap #120 ROC area: 0.634\n",
      "Bootstrap #121 ROC area: 0.634\n",
      "Bootstrap #122 ROC area: 0.634\n",
      "Bootstrap #123 ROC area: 0.634\n",
      "Bootstrap #124 ROC area: 0.634\n",
      "Bootstrap #125 ROC area: 0.634\n",
      "Bootstrap #126 ROC area: 0.634\n",
      "Bootstrap #127 ROC area: 0.634\n",
      "Bootstrap #128 ROC area: 0.634\n",
      "Bootstrap #129 ROC area: 0.634\n",
      "Bootstrap #130 ROC area: 0.634\n",
      "Bootstrap #131 ROC area: 0.634\n",
      "Bootstrap #132 ROC area: 0.634\n",
      "Bootstrap #133 ROC area: 0.634\n",
      "Bootstrap #134 ROC area: 0.634\n",
      "Bootstrap #135 ROC area: 0.634\n",
      "Bootstrap #136 ROC area: 0.634\n",
      "Bootstrap #137 ROC area: 0.634\n",
      "Bootstrap #138 ROC area: 0.634\n",
      "Bootstrap #139 ROC area: 0.634\n",
      "Bootstrap #140 ROC area: 0.634\n",
      "Bootstrap #141 ROC area: 0.634\n",
      "Bootstrap #142 ROC area: 0.634\n",
      "Bootstrap #143 ROC area: 0.634\n",
      "Bootstrap #144 ROC area: 0.634\n",
      "Bootstrap #145 ROC area: 0.634\n",
      "Bootstrap #146 ROC area: 0.634\n",
      "Bootstrap #147 ROC area: 0.634\n",
      "Bootstrap #148 ROC area: 0.634\n",
      "Bootstrap #149 ROC area: 0.634\n",
      "Bootstrap #150 ROC area: 0.634\n",
      "Bootstrap #151 ROC area: 0.634\n",
      "Bootstrap #152 ROC area: 0.634\n",
      "Bootstrap #153 ROC area: 0.634\n",
      "Bootstrap #154 ROC area: 0.634\n",
      "Bootstrap #155 ROC area: 0.634\n",
      "Bootstrap #156 ROC area: 0.634\n",
      "Bootstrap #157 ROC area: 0.634\n",
      "Bootstrap #158 ROC area: 0.634\n",
      "Bootstrap #159 ROC area: 0.634\n",
      "Bootstrap #160 ROC area: 0.634\n",
      "Bootstrap #161 ROC area: 0.634\n",
      "Bootstrap #162 ROC area: 0.634\n",
      "Bootstrap #163 ROC area: 0.634\n",
      "Bootstrap #164 ROC area: 0.634\n",
      "Bootstrap #165 ROC area: 0.634\n",
      "Bootstrap #166 ROC area: 0.634\n",
      "Bootstrap #167 ROC area: 0.634\n",
      "Bootstrap #168 ROC area: 0.634\n",
      "Bootstrap #169 ROC area: 0.634\n",
      "Bootstrap #170 ROC area: 0.634\n",
      "Bootstrap #171 ROC area: 0.634\n",
      "Bootstrap #172 ROC area: 0.634\n",
      "Bootstrap #173 ROC area: 0.634\n",
      "Bootstrap #174 ROC area: 0.634\n",
      "Bootstrap #175 ROC area: 0.634\n",
      "Bootstrap #176 ROC area: 0.634\n",
      "Bootstrap #177 ROC area: 0.634\n",
      "Bootstrap #178 ROC area: 0.634\n",
      "Bootstrap #179 ROC area: 0.634\n",
      "Bootstrap #180 ROC area: 0.634\n",
      "Bootstrap #181 ROC area: 0.634\n",
      "Bootstrap #182 ROC area: 0.634\n",
      "Bootstrap #183 ROC area: 0.634\n",
      "Bootstrap #184 ROC area: 0.634\n",
      "Bootstrap #185 ROC area: 0.634\n",
      "Bootstrap #186 ROC area: 0.634\n",
      "Bootstrap #187 ROC area: 0.634\n",
      "Bootstrap #188 ROC area: 0.634\n",
      "Bootstrap #189 ROC area: 0.634\n",
      "Bootstrap #190 ROC area: 0.634\n",
      "Bootstrap #191 ROC area: 0.634\n",
      "Bootstrap #192 ROC area: 0.634\n",
      "Bootstrap #193 ROC area: 0.634\n",
      "Bootstrap #194 ROC area: 0.634\n",
      "Bootstrap #195 ROC area: 0.634\n",
      "Bootstrap #196 ROC area: 0.634\n",
      "Bootstrap #197 ROC area: 0.634\n",
      "Bootstrap #198 ROC area: 0.634\n",
      "Bootstrap #199 ROC area: 0.634\n",
      "Bootstrap #200 ROC area: 0.634\n",
      "Bootstrap #201 ROC area: 0.634\n",
      "Bootstrap #202 ROC area: 0.634\n",
      "Bootstrap #203 ROC area: 0.634\n",
      "Bootstrap #204 ROC area: 0.634\n",
      "Bootstrap #205 ROC area: 0.634\n",
      "Bootstrap #206 ROC area: 0.634\n",
      "Bootstrap #207 ROC area: 0.634\n",
      "Bootstrap #208 ROC area: 0.634\n",
      "Bootstrap #209 ROC area: 0.634\n",
      "Bootstrap #210 ROC area: 0.634\n",
      "Bootstrap #211 ROC area: 0.634\n",
      "Bootstrap #212 ROC area: 0.634\n",
      "Bootstrap #213 ROC area: 0.634\n",
      "Bootstrap #214 ROC area: 0.634\n",
      "Bootstrap #215 ROC area: 0.634\n",
      "Bootstrap #216 ROC area: 0.634\n",
      "Bootstrap #217 ROC area: 0.634\n",
      "Bootstrap #218 ROC area: 0.634\n",
      "Bootstrap #219 ROC area: 0.634\n",
      "Bootstrap #220 ROC area: 0.634\n",
      "Bootstrap #221 ROC area: 0.634\n",
      "Bootstrap #222 ROC area: 0.634\n",
      "Bootstrap #223 ROC area: 0.634\n",
      "Bootstrap #224 ROC area: 0.634\n",
      "Bootstrap #225 ROC area: 0.634\n",
      "Bootstrap #226 ROC area: 0.634\n",
      "Bootstrap #227 ROC area: 0.634\n",
      "Bootstrap #228 ROC area: 0.634\n",
      "Bootstrap #229 ROC area: 0.634\n",
      "Bootstrap #230 ROC area: 0.634\n",
      "Bootstrap #231 ROC area: 0.634\n",
      "Bootstrap #232 ROC area: 0.634\n",
      "Bootstrap #233 ROC area: 0.634\n",
      "Bootstrap #234 ROC area: 0.634\n",
      "Bootstrap #235 ROC area: 0.634\n",
      "Bootstrap #236 ROC area: 0.634\n",
      "Bootstrap #237 ROC area: 0.634\n",
      "Bootstrap #238 ROC area: 0.634\n",
      "Bootstrap #239 ROC area: 0.634\n",
      "Bootstrap #240 ROC area: 0.634\n",
      "Bootstrap #241 ROC area: 0.634\n",
      "Bootstrap #242 ROC area: 0.634\n",
      "Bootstrap #243 ROC area: 0.634\n",
      "Bootstrap #244 ROC area: 0.634\n",
      "Bootstrap #245 ROC area: 0.634\n",
      "Bootstrap #246 ROC area: 0.634\n",
      "Bootstrap #247 ROC area: 0.634\n",
      "Bootstrap #248 ROC area: 0.634\n",
      "Bootstrap #249 ROC area: 0.634\n",
      "Bootstrap #250 ROC area: 0.634\n",
      "Bootstrap #251 ROC area: 0.634\n",
      "Bootstrap #252 ROC area: 0.634\n",
      "Bootstrap #253 ROC area: 0.634\n",
      "Bootstrap #254 ROC area: 0.634\n",
      "Bootstrap #255 ROC area: 0.634\n",
      "Bootstrap #256 ROC area: 0.634\n",
      "Bootstrap #257 ROC area: 0.634\n",
      "Bootstrap #258 ROC area: 0.634\n",
      "Bootstrap #259 ROC area: 0.634\n",
      "Bootstrap #260 ROC area: 0.634\n",
      "Bootstrap #261 ROC area: 0.634\n",
      "Bootstrap #262 ROC area: 0.634\n",
      "Bootstrap #263 ROC area: 0.634\n",
      "Bootstrap #264 ROC area: 0.634\n",
      "Bootstrap #265 ROC area: 0.634\n",
      "Bootstrap #266 ROC area: 0.634\n",
      "Bootstrap #267 ROC area: 0.634\n",
      "Bootstrap #268 ROC area: 0.634\n",
      "Bootstrap #269 ROC area: 0.634\n",
      "Bootstrap #270 ROC area: 0.634\n",
      "Bootstrap #271 ROC area: 0.634\n",
      "Bootstrap #272 ROC area: 0.634\n",
      "Bootstrap #273 ROC area: 0.634\n",
      "Bootstrap #274 ROC area: 0.634\n",
      "Bootstrap #275 ROC area: 0.634\n",
      "Bootstrap #276 ROC area: 0.634\n",
      "Bootstrap #277 ROC area: 0.634\n",
      "Bootstrap #278 ROC area: 0.634\n",
      "Bootstrap #279 ROC area: 0.634\n",
      "Bootstrap #280 ROC area: 0.634\n",
      "Bootstrap #281 ROC area: 0.634\n",
      "Bootstrap #282 ROC area: 0.634\n",
      "Bootstrap #283 ROC area: 0.634\n",
      "Bootstrap #284 ROC area: 0.634\n",
      "Bootstrap #285 ROC area: 0.634\n",
      "Bootstrap #286 ROC area: 0.634\n",
      "Bootstrap #287 ROC area: 0.634\n",
      "Bootstrap #288 ROC area: 0.634\n",
      "Bootstrap #289 ROC area: 0.634\n",
      "Bootstrap #290 ROC area: 0.634\n",
      "Bootstrap #291 ROC area: 0.634\n",
      "Bootstrap #292 ROC area: 0.634\n",
      "Bootstrap #293 ROC area: 0.634\n",
      "Bootstrap #294 ROC area: 0.634\n",
      "Bootstrap #295 ROC area: 0.634\n",
      "Bootstrap #296 ROC area: 0.634\n",
      "Bootstrap #297 ROC area: 0.634\n",
      "Bootstrap #298 ROC area: 0.634\n",
      "Bootstrap #299 ROC area: 0.634\n",
      "Bootstrap #300 ROC area: 0.634\n",
      "Bootstrap #301 ROC area: 0.634\n",
      "Bootstrap #302 ROC area: 0.634\n",
      "Bootstrap #303 ROC area: 0.634\n",
      "Bootstrap #304 ROC area: 0.634\n",
      "Bootstrap #305 ROC area: 0.634\n",
      "Bootstrap #306 ROC area: 0.634\n",
      "Bootstrap #307 ROC area: 0.634\n",
      "Bootstrap #308 ROC area: 0.634\n",
      "Bootstrap #309 ROC area: 0.634\n",
      "Bootstrap #310 ROC area: 0.634\n",
      "Bootstrap #311 ROC area: 0.634\n",
      "Bootstrap #312 ROC area: 0.634\n",
      "Bootstrap #313 ROC area: 0.634\n",
      "Bootstrap #314 ROC area: 0.634\n",
      "Bootstrap #315 ROC area: 0.634\n",
      "Bootstrap #316 ROC area: 0.634\n",
      "Bootstrap #317 ROC area: 0.634\n",
      "Bootstrap #318 ROC area: 0.634\n",
      "Bootstrap #319 ROC area: 0.634\n",
      "Bootstrap #320 ROC area: 0.634\n",
      "Bootstrap #321 ROC area: 0.634\n",
      "Bootstrap #322 ROC area: 0.634\n",
      "Bootstrap #323 ROC area: 0.634\n",
      "Bootstrap #324 ROC area: 0.634\n",
      "Bootstrap #325 ROC area: 0.634\n",
      "Bootstrap #326 ROC area: 0.634\n",
      "Bootstrap #327 ROC area: 0.634\n",
      "Bootstrap #328 ROC area: 0.634\n",
      "Bootstrap #329 ROC area: 0.634\n",
      "Bootstrap #330 ROC area: 0.634\n",
      "Bootstrap #331 ROC area: 0.634\n",
      "Bootstrap #332 ROC area: 0.634\n",
      "Bootstrap #333 ROC area: 0.634\n",
      "Bootstrap #334 ROC area: 0.634\n",
      "Bootstrap #335 ROC area: 0.634\n",
      "Bootstrap #336 ROC area: 0.634\n",
      "Bootstrap #337 ROC area: 0.634\n",
      "Bootstrap #338 ROC area: 0.634\n",
      "Bootstrap #339 ROC area: 0.634\n",
      "Bootstrap #340 ROC area: 0.634\n",
      "Bootstrap #341 ROC area: 0.634\n",
      "Bootstrap #342 ROC area: 0.634\n",
      "Bootstrap #343 ROC area: 0.634\n",
      "Bootstrap #344 ROC area: 0.634\n",
      "Bootstrap #345 ROC area: 0.634\n",
      "Bootstrap #346 ROC area: 0.634\n",
      "Bootstrap #347 ROC area: 0.634\n",
      "Bootstrap #348 ROC area: 0.634\n",
      "Bootstrap #349 ROC area: 0.634\n",
      "Bootstrap #350 ROC area: 0.634\n",
      "Bootstrap #351 ROC area: 0.634\n",
      "Bootstrap #352 ROC area: 0.634\n",
      "Bootstrap #353 ROC area: 0.634\n",
      "Bootstrap #354 ROC area: 0.634\n",
      "Bootstrap #355 ROC area: 0.634\n",
      "Bootstrap #356 ROC area: 0.634\n",
      "Bootstrap #357 ROC area: 0.634\n",
      "Bootstrap #358 ROC area: 0.634\n",
      "Bootstrap #359 ROC area: 0.634\n",
      "Bootstrap #360 ROC area: 0.634\n",
      "Bootstrap #361 ROC area: 0.634\n",
      "Bootstrap #362 ROC area: 0.634\n",
      "Bootstrap #363 ROC area: 0.634\n",
      "Bootstrap #364 ROC area: 0.634\n",
      "Bootstrap #365 ROC area: 0.634\n",
      "Bootstrap #366 ROC area: 0.634\n",
      "Bootstrap #367 ROC area: 0.634\n",
      "Bootstrap #368 ROC area: 0.634\n",
      "Bootstrap #369 ROC area: 0.634\n",
      "Bootstrap #370 ROC area: 0.634\n",
      "Bootstrap #371 ROC area: 0.634\n",
      "Bootstrap #372 ROC area: 0.634\n",
      "Bootstrap #373 ROC area: 0.634\n",
      "Bootstrap #374 ROC area: 0.634\n",
      "Bootstrap #375 ROC area: 0.634\n",
      "Bootstrap #376 ROC area: 0.634\n",
      "Bootstrap #377 ROC area: 0.634\n",
      "Bootstrap #378 ROC area: 0.634\n",
      "Bootstrap #379 ROC area: 0.634\n",
      "Bootstrap #380 ROC area: 0.634\n",
      "Bootstrap #381 ROC area: 0.634\n",
      "Bootstrap #382 ROC area: 0.634\n",
      "Bootstrap #383 ROC area: 0.634\n",
      "Bootstrap #384 ROC area: 0.634\n",
      "Bootstrap #385 ROC area: 0.634\n",
      "Bootstrap #386 ROC area: 0.634\n",
      "Bootstrap #387 ROC area: 0.634\n",
      "Bootstrap #388 ROC area: 0.634\n",
      "Bootstrap #389 ROC area: 0.634\n",
      "Bootstrap #390 ROC area: 0.634\n",
      "Bootstrap #391 ROC area: 0.634\n",
      "Bootstrap #392 ROC area: 0.634\n",
      "Bootstrap #393 ROC area: 0.634\n",
      "Bootstrap #394 ROC area: 0.634\n",
      "Bootstrap #395 ROC area: 0.634\n",
      "Bootstrap #396 ROC area: 0.634\n",
      "Bootstrap #397 ROC area: 0.634\n",
      "Bootstrap #398 ROC area: 0.634\n",
      "Bootstrap #399 ROC area: 0.634\n",
      "Bootstrap #400 ROC area: 0.634\n",
      "Bootstrap #401 ROC area: 0.634\n",
      "Bootstrap #402 ROC area: 0.634\n",
      "Bootstrap #403 ROC area: 0.634\n",
      "Bootstrap #404 ROC area: 0.634\n",
      "Bootstrap #405 ROC area: 0.634\n",
      "Bootstrap #406 ROC area: 0.634\n",
      "Bootstrap #407 ROC area: 0.634\n",
      "Bootstrap #408 ROC area: 0.634\n",
      "Bootstrap #409 ROC area: 0.634\n",
      "Bootstrap #410 ROC area: 0.634\n",
      "Bootstrap #411 ROC area: 0.634\n",
      "Bootstrap #412 ROC area: 0.634\n",
      "Bootstrap #413 ROC area: 0.634\n",
      "Bootstrap #414 ROC area: 0.634\n",
      "Bootstrap #415 ROC area: 0.634\n",
      "Bootstrap #416 ROC area: 0.634\n",
      "Bootstrap #417 ROC area: 0.634\n",
      "Bootstrap #418 ROC area: 0.634\n",
      "Bootstrap #419 ROC area: 0.634\n",
      "Bootstrap #420 ROC area: 0.634\n",
      "Bootstrap #421 ROC area: 0.634\n",
      "Bootstrap #422 ROC area: 0.634\n",
      "Bootstrap #423 ROC area: 0.634\n",
      "Bootstrap #424 ROC area: 0.634\n",
      "Bootstrap #425 ROC area: 0.634\n",
      "Bootstrap #426 ROC area: 0.634\n",
      "Bootstrap #427 ROC area: 0.634\n",
      "Bootstrap #428 ROC area: 0.634\n",
      "Bootstrap #429 ROC area: 0.634\n",
      "Bootstrap #430 ROC area: 0.634\n",
      "Bootstrap #431 ROC area: 0.634\n",
      "Bootstrap #432 ROC area: 0.634\n",
      "Bootstrap #433 ROC area: 0.634\n",
      "Bootstrap #434 ROC area: 0.634\n",
      "Bootstrap #435 ROC area: 0.634\n",
      "Bootstrap #436 ROC area: 0.634\n",
      "Bootstrap #437 ROC area: 0.634\n",
      "Bootstrap #438 ROC area: 0.634\n",
      "Bootstrap #439 ROC area: 0.634\n",
      "Bootstrap #440 ROC area: 0.634\n",
      "Bootstrap #441 ROC area: 0.634\n",
      "Bootstrap #442 ROC area: 0.634\n",
      "Bootstrap #443 ROC area: 0.634\n",
      "Bootstrap #444 ROC area: 0.634\n",
      "Bootstrap #445 ROC area: 0.634\n",
      "Bootstrap #446 ROC area: 0.634\n",
      "Bootstrap #447 ROC area: 0.634\n",
      "Bootstrap #448 ROC area: 0.634\n",
      "Bootstrap #449 ROC area: 0.634\n",
      "Bootstrap #450 ROC area: 0.634\n",
      "Bootstrap #451 ROC area: 0.634\n",
      "Bootstrap #452 ROC area: 0.634\n",
      "Bootstrap #453 ROC area: 0.634\n",
      "Bootstrap #454 ROC area: 0.634\n",
      "Bootstrap #455 ROC area: 0.634\n",
      "Bootstrap #456 ROC area: 0.634\n",
      "Bootstrap #457 ROC area: 0.634\n",
      "Bootstrap #458 ROC area: 0.634\n",
      "Bootstrap #459 ROC area: 0.634\n",
      "Bootstrap #460 ROC area: 0.634\n",
      "Bootstrap #461 ROC area: 0.634\n",
      "Bootstrap #462 ROC area: 0.634\n",
      "Bootstrap #463 ROC area: 0.634\n",
      "Bootstrap #464 ROC area: 0.634\n",
      "Bootstrap #465 ROC area: 0.634\n",
      "Bootstrap #466 ROC area: 0.634\n",
      "Bootstrap #467 ROC area: 0.634\n",
      "Bootstrap #468 ROC area: 0.634\n",
      "Bootstrap #469 ROC area: 0.634\n",
      "Bootstrap #470 ROC area: 0.634\n",
      "Bootstrap #471 ROC area: 0.634\n",
      "Bootstrap #472 ROC area: 0.634\n",
      "Bootstrap #473 ROC area: 0.634\n",
      "Bootstrap #474 ROC area: 0.634\n",
      "Bootstrap #475 ROC area: 0.634\n",
      "Bootstrap #476 ROC area: 0.634\n",
      "Bootstrap #477 ROC area: 0.634\n",
      "Bootstrap #478 ROC area: 0.634\n",
      "Bootstrap #479 ROC area: 0.634\n",
      "Bootstrap #480 ROC area: 0.634\n",
      "Bootstrap #481 ROC area: 0.634\n",
      "Bootstrap #482 ROC area: 0.634\n",
      "Bootstrap #483 ROC area: 0.634\n",
      "Bootstrap #484 ROC area: 0.634\n",
      "Bootstrap #485 ROC area: 0.634\n",
      "Bootstrap #486 ROC area: 0.634\n",
      "Bootstrap #487 ROC area: 0.634\n",
      "Bootstrap #488 ROC area: 0.634\n",
      "Bootstrap #489 ROC area: 0.634\n",
      "Bootstrap #490 ROC area: 0.634\n",
      "Bootstrap #491 ROC area: 0.634\n",
      "Bootstrap #492 ROC area: 0.634\n",
      "Bootstrap #493 ROC area: 0.634\n",
      "Bootstrap #494 ROC area: 0.634\n",
      "Bootstrap #495 ROC area: 0.634\n",
      "Bootstrap #496 ROC area: 0.634\n",
      "Bootstrap #497 ROC area: 0.634\n",
      "Bootstrap #498 ROC area: 0.634\n",
      "Bootstrap #499 ROC area: 0.634\n",
      "Bootstrap #500 ROC area: 0.634\n",
      "Bootstrap #501 ROC area: 0.634\n",
      "Bootstrap #502 ROC area: 0.634\n",
      "Bootstrap #503 ROC area: 0.634\n",
      "Bootstrap #504 ROC area: 0.634\n",
      "Bootstrap #505 ROC area: 0.634\n",
      "Bootstrap #506 ROC area: 0.634\n",
      "Bootstrap #507 ROC area: 0.634\n",
      "Bootstrap #508 ROC area: 0.634\n",
      "Bootstrap #509 ROC area: 0.634\n",
      "Bootstrap #510 ROC area: 0.634\n",
      "Bootstrap #511 ROC area: 0.634\n",
      "Bootstrap #512 ROC area: 0.634\n",
      "Bootstrap #513 ROC area: 0.634\n",
      "Bootstrap #514 ROC area: 0.634\n",
      "Bootstrap #515 ROC area: 0.634\n",
      "Bootstrap #516 ROC area: 0.634\n",
      "Bootstrap #517 ROC area: 0.634\n",
      "Bootstrap #518 ROC area: 0.634\n",
      "Bootstrap #519 ROC area: 0.634\n",
      "Bootstrap #520 ROC area: 0.634\n",
      "Bootstrap #521 ROC area: 0.634\n",
      "Bootstrap #522 ROC area: 0.634\n",
      "Bootstrap #523 ROC area: 0.634\n",
      "Bootstrap #524 ROC area: 0.634\n",
      "Bootstrap #525 ROC area: 0.634\n",
      "Bootstrap #526 ROC area: 0.634\n",
      "Bootstrap #527 ROC area: 0.634\n",
      "Bootstrap #528 ROC area: 0.634\n",
      "Bootstrap #529 ROC area: 0.634\n",
      "Bootstrap #530 ROC area: 0.634\n",
      "Bootstrap #531 ROC area: 0.634\n",
      "Bootstrap #532 ROC area: 0.634\n",
      "Bootstrap #533 ROC area: 0.634\n",
      "Bootstrap #534 ROC area: 0.634\n",
      "Bootstrap #535 ROC area: 0.634\n",
      "Bootstrap #536 ROC area: 0.634\n",
      "Bootstrap #537 ROC area: 0.634\n",
      "Bootstrap #538 ROC area: 0.634\n",
      "Bootstrap #539 ROC area: 0.634\n",
      "Bootstrap #540 ROC area: 0.634\n",
      "Bootstrap #541 ROC area: 0.634\n",
      "Bootstrap #542 ROC area: 0.634\n",
      "Bootstrap #543 ROC area: 0.634\n",
      "Bootstrap #544 ROC area: 0.634\n",
      "Bootstrap #545 ROC area: 0.634\n",
      "Bootstrap #546 ROC area: 0.634\n",
      "Bootstrap #547 ROC area: 0.634\n",
      "Bootstrap #548 ROC area: 0.634\n",
      "Bootstrap #549 ROC area: 0.634\n",
      "Bootstrap #550 ROC area: 0.634\n",
      "Bootstrap #551 ROC area: 0.634\n",
      "Bootstrap #552 ROC area: 0.634\n",
      "Bootstrap #553 ROC area: 0.634\n",
      "Bootstrap #554 ROC area: 0.634\n",
      "Bootstrap #555 ROC area: 0.634\n",
      "Bootstrap #556 ROC area: 0.634\n",
      "Bootstrap #557 ROC area: 0.634\n",
      "Bootstrap #558 ROC area: 0.634\n",
      "Bootstrap #559 ROC area: 0.634\n",
      "Bootstrap #560 ROC area: 0.634\n",
      "Bootstrap #561 ROC area: 0.634\n",
      "Bootstrap #562 ROC area: 0.634\n",
      "Bootstrap #563 ROC area: 0.634\n",
      "Bootstrap #564 ROC area: 0.634\n",
      "Bootstrap #565 ROC area: 0.634\n",
      "Bootstrap #566 ROC area: 0.634\n",
      "Bootstrap #567 ROC area: 0.634\n",
      "Bootstrap #568 ROC area: 0.634\n",
      "Bootstrap #569 ROC area: 0.634\n",
      "Bootstrap #570 ROC area: 0.634\n",
      "Bootstrap #571 ROC area: 0.634\n",
      "Bootstrap #572 ROC area: 0.634\n",
      "Bootstrap #573 ROC area: 0.634\n",
      "Bootstrap #574 ROC area: 0.634\n",
      "Bootstrap #575 ROC area: 0.634\n",
      "Bootstrap #576 ROC area: 0.634\n",
      "Bootstrap #577 ROC area: 0.634\n",
      "Bootstrap #578 ROC area: 0.634\n",
      "Bootstrap #579 ROC area: 0.634\n",
      "Bootstrap #580 ROC area: 0.634\n",
      "Bootstrap #581 ROC area: 0.634\n",
      "Bootstrap #582 ROC area: 0.634\n",
      "Bootstrap #583 ROC area: 0.634\n",
      "Bootstrap #584 ROC area: 0.634\n",
      "Bootstrap #585 ROC area: 0.634\n",
      "Bootstrap #586 ROC area: 0.634\n",
      "Bootstrap #587 ROC area: 0.634\n",
      "Bootstrap #588 ROC area: 0.634\n",
      "Bootstrap #589 ROC area: 0.634\n",
      "Bootstrap #590 ROC area: 0.634\n",
      "Bootstrap #591 ROC area: 0.634\n",
      "Bootstrap #592 ROC area: 0.634\n",
      "Bootstrap #593 ROC area: 0.634\n",
      "Bootstrap #594 ROC area: 0.634\n",
      "Bootstrap #595 ROC area: 0.634\n",
      "Bootstrap #596 ROC area: 0.634\n",
      "Bootstrap #597 ROC area: 0.634\n",
      "Bootstrap #598 ROC area: 0.634\n",
      "Bootstrap #599 ROC area: 0.634\n",
      "Bootstrap #600 ROC area: 0.634\n",
      "Bootstrap #601 ROC area: 0.634\n",
      "Bootstrap #602 ROC area: 0.634\n",
      "Bootstrap #603 ROC area: 0.634\n",
      "Bootstrap #604 ROC area: 0.634\n",
      "Bootstrap #605 ROC area: 0.634\n",
      "Bootstrap #606 ROC area: 0.634\n",
      "Bootstrap #607 ROC area: 0.634\n",
      "Bootstrap #608 ROC area: 0.634\n",
      "Bootstrap #609 ROC area: 0.634\n",
      "Bootstrap #610 ROC area: 0.634\n",
      "Bootstrap #611 ROC area: 0.634\n",
      "Bootstrap #612 ROC area: 0.634\n",
      "Bootstrap #613 ROC area: 0.634\n",
      "Bootstrap #614 ROC area: 0.634\n",
      "Bootstrap #615 ROC area: 0.634\n",
      "Bootstrap #616 ROC area: 0.634\n",
      "Bootstrap #617 ROC area: 0.634\n",
      "Bootstrap #618 ROC area: 0.634\n",
      "Bootstrap #619 ROC area: 0.634\n",
      "Bootstrap #620 ROC area: 0.634\n",
      "Bootstrap #621 ROC area: 0.634\n",
      "Bootstrap #622 ROC area: 0.634\n",
      "Bootstrap #623 ROC area: 0.634\n",
      "Bootstrap #624 ROC area: 0.634\n",
      "Bootstrap #625 ROC area: 0.634\n",
      "Bootstrap #626 ROC area: 0.634\n",
      "Bootstrap #627 ROC area: 0.634\n",
      "Bootstrap #628 ROC area: 0.634\n",
      "Bootstrap #629 ROC area: 0.634\n",
      "Bootstrap #630 ROC area: 0.634\n",
      "Bootstrap #631 ROC area: 0.634\n",
      "Bootstrap #632 ROC area: 0.634\n",
      "Bootstrap #633 ROC area: 0.634\n",
      "Bootstrap #634 ROC area: 0.634\n",
      "Bootstrap #635 ROC area: 0.634\n",
      "Bootstrap #636 ROC area: 0.634\n",
      "Bootstrap #637 ROC area: 0.634\n",
      "Bootstrap #638 ROC area: 0.634\n",
      "Bootstrap #639 ROC area: 0.634\n",
      "Bootstrap #640 ROC area: 0.634\n",
      "Bootstrap #641 ROC area: 0.634\n",
      "Bootstrap #642 ROC area: 0.634\n",
      "Bootstrap #643 ROC area: 0.634\n",
      "Bootstrap #644 ROC area: 0.634\n",
      "Bootstrap #645 ROC area: 0.634\n",
      "Bootstrap #646 ROC area: 0.634\n",
      "Bootstrap #647 ROC area: 0.634\n",
      "Bootstrap #648 ROC area: 0.634\n",
      "Bootstrap #649 ROC area: 0.634\n",
      "Bootstrap #650 ROC area: 0.634\n",
      "Bootstrap #651 ROC area: 0.634\n",
      "Bootstrap #652 ROC area: 0.634\n",
      "Bootstrap #653 ROC area: 0.634\n",
      "Bootstrap #654 ROC area: 0.634\n",
      "Bootstrap #655 ROC area: 0.634\n",
      "Bootstrap #656 ROC area: 0.634\n",
      "Bootstrap #657 ROC area: 0.634\n",
      "Bootstrap #658 ROC area: 0.634\n",
      "Bootstrap #659 ROC area: 0.634\n",
      "Bootstrap #660 ROC area: 0.634\n",
      "Bootstrap #661 ROC area: 0.634\n",
      "Bootstrap #662 ROC area: 0.634\n",
      "Bootstrap #663 ROC area: 0.634\n",
      "Bootstrap #664 ROC area: 0.634\n",
      "Bootstrap #665 ROC area: 0.634\n",
      "Bootstrap #666 ROC area: 0.634\n",
      "Bootstrap #667 ROC area: 0.634\n",
      "Bootstrap #668 ROC area: 0.634\n",
      "Bootstrap #669 ROC area: 0.634\n",
      "Bootstrap #670 ROC area: 0.634\n",
      "Bootstrap #671 ROC area: 0.634\n",
      "Bootstrap #672 ROC area: 0.634\n",
      "Bootstrap #673 ROC area: 0.634\n",
      "Bootstrap #674 ROC area: 0.634\n",
      "Bootstrap #675 ROC area: 0.634\n",
      "Bootstrap #676 ROC area: 0.634\n",
      "Bootstrap #677 ROC area: 0.634\n",
      "Bootstrap #678 ROC area: 0.634\n",
      "Bootstrap #679 ROC area: 0.634\n",
      "Bootstrap #680 ROC area: 0.634\n",
      "Bootstrap #681 ROC area: 0.634\n",
      "Bootstrap #682 ROC area: 0.634\n",
      "Bootstrap #683 ROC area: 0.634\n",
      "Bootstrap #684 ROC area: 0.634\n",
      "Bootstrap #685 ROC area: 0.634\n",
      "Bootstrap #686 ROC area: 0.634\n",
      "Bootstrap #687 ROC area: 0.634\n",
      "Bootstrap #688 ROC area: 0.634\n",
      "Bootstrap #689 ROC area: 0.634\n",
      "Bootstrap #690 ROC area: 0.634\n",
      "Bootstrap #691 ROC area: 0.634\n",
      "Bootstrap #692 ROC area: 0.634\n",
      "Bootstrap #693 ROC area: 0.634\n",
      "Bootstrap #694 ROC area: 0.634\n",
      "Bootstrap #695 ROC area: 0.634\n",
      "Bootstrap #696 ROC area: 0.634\n",
      "Bootstrap #697 ROC area: 0.634\n",
      "Bootstrap #698 ROC area: 0.634\n",
      "Bootstrap #699 ROC area: 0.634\n",
      "Bootstrap #700 ROC area: 0.634\n",
      "Bootstrap #701 ROC area: 0.634\n",
      "Bootstrap #702 ROC area: 0.634\n",
      "Bootstrap #703 ROC area: 0.634\n",
      "Bootstrap #704 ROC area: 0.634\n",
      "Bootstrap #705 ROC area: 0.634\n",
      "Bootstrap #706 ROC area: 0.634\n",
      "Bootstrap #707 ROC area: 0.634\n",
      "Bootstrap #708 ROC area: 0.634\n",
      "Bootstrap #709 ROC area: 0.634\n",
      "Bootstrap #710 ROC area: 0.634\n",
      "Bootstrap #711 ROC area: 0.634\n",
      "Bootstrap #712 ROC area: 0.634\n",
      "Bootstrap #713 ROC area: 0.634\n",
      "Bootstrap #714 ROC area: 0.634\n",
      "Bootstrap #715 ROC area: 0.634\n",
      "Bootstrap #716 ROC area: 0.634\n",
      "Bootstrap #717 ROC area: 0.634\n",
      "Bootstrap #718 ROC area: 0.634\n",
      "Bootstrap #719 ROC area: 0.634\n",
      "Bootstrap #720 ROC area: 0.634\n",
      "Bootstrap #721 ROC area: 0.634\n",
      "Bootstrap #722 ROC area: 0.634\n",
      "Bootstrap #723 ROC area: 0.634\n",
      "Bootstrap #724 ROC area: 0.634\n",
      "Bootstrap #725 ROC area: 0.634\n",
      "Bootstrap #726 ROC area: 0.634\n",
      "Bootstrap #727 ROC area: 0.634\n",
      "Bootstrap #728 ROC area: 0.634\n",
      "Bootstrap #729 ROC area: 0.634\n",
      "Bootstrap #730 ROC area: 0.634\n",
      "Bootstrap #731 ROC area: 0.634\n",
      "Bootstrap #732 ROC area: 0.634\n",
      "Bootstrap #733 ROC area: 0.634\n",
      "Bootstrap #734 ROC area: 0.634\n",
      "Bootstrap #735 ROC area: 0.634\n",
      "Bootstrap #736 ROC area: 0.634\n",
      "Bootstrap #737 ROC area: 0.634\n",
      "Bootstrap #738 ROC area: 0.634\n",
      "Bootstrap #739 ROC area: 0.634\n",
      "Bootstrap #740 ROC area: 0.634\n",
      "Bootstrap #741 ROC area: 0.634\n",
      "Bootstrap #742 ROC area: 0.634\n",
      "Bootstrap #743 ROC area: 0.634\n",
      "Bootstrap #744 ROC area: 0.634\n",
      "Bootstrap #745 ROC area: 0.634\n",
      "Bootstrap #746 ROC area: 0.634\n",
      "Bootstrap #747 ROC area: 0.634\n",
      "Bootstrap #748 ROC area: 0.634\n",
      "Bootstrap #749 ROC area: 0.634\n",
      "Bootstrap #750 ROC area: 0.634\n",
      "Bootstrap #751 ROC area: 0.634\n",
      "Bootstrap #752 ROC area: 0.634\n",
      "Bootstrap #753 ROC area: 0.634\n",
      "Bootstrap #754 ROC area: 0.634\n",
      "Bootstrap #755 ROC area: 0.634\n",
      "Bootstrap #756 ROC area: 0.634\n",
      "Bootstrap #757 ROC area: 0.634\n",
      "Bootstrap #758 ROC area: 0.634\n",
      "Bootstrap #759 ROC area: 0.634\n",
      "Bootstrap #760 ROC area: 0.634\n",
      "Bootstrap #761 ROC area: 0.634\n",
      "Bootstrap #762 ROC area: 0.634\n",
      "Bootstrap #763 ROC area: 0.634\n",
      "Bootstrap #764 ROC area: 0.634\n",
      "Bootstrap #765 ROC area: 0.634\n",
      "Bootstrap #766 ROC area: 0.634\n",
      "Bootstrap #767 ROC area: 0.634\n",
      "Bootstrap #768 ROC area: 0.634\n",
      "Bootstrap #769 ROC area: 0.634\n",
      "Bootstrap #770 ROC area: 0.634\n",
      "Bootstrap #771 ROC area: 0.634\n",
      "Bootstrap #772 ROC area: 0.634\n",
      "Bootstrap #773 ROC area: 0.634\n",
      "Bootstrap #774 ROC area: 0.634\n",
      "Bootstrap #775 ROC area: 0.634\n",
      "Bootstrap #776 ROC area: 0.634\n",
      "Bootstrap #777 ROC area: 0.634\n",
      "Bootstrap #778 ROC area: 0.634\n",
      "Bootstrap #779 ROC area: 0.634\n",
      "Bootstrap #780 ROC area: 0.634\n",
      "Bootstrap #781 ROC area: 0.634\n",
      "Bootstrap #782 ROC area: 0.634\n",
      "Bootstrap #783 ROC area: 0.634\n",
      "Bootstrap #784 ROC area: 0.634\n",
      "Bootstrap #785 ROC area: 0.634\n",
      "Bootstrap #786 ROC area: 0.634\n",
      "Bootstrap #787 ROC area: 0.634\n",
      "Bootstrap #788 ROC area: 0.634\n",
      "Bootstrap #789 ROC area: 0.634\n",
      "Bootstrap #790 ROC area: 0.634\n",
      "Bootstrap #791 ROC area: 0.634\n",
      "Bootstrap #792 ROC area: 0.634\n",
      "Bootstrap #793 ROC area: 0.634\n",
      "Bootstrap #794 ROC area: 0.634\n",
      "Bootstrap #795 ROC area: 0.634\n",
      "Bootstrap #796 ROC area: 0.634\n",
      "Bootstrap #797 ROC area: 0.634\n",
      "Bootstrap #798 ROC area: 0.634\n",
      "Bootstrap #799 ROC area: 0.634\n",
      "Bootstrap #800 ROC area: 0.634\n",
      "Bootstrap #801 ROC area: 0.634\n",
      "Bootstrap #802 ROC area: 0.634\n",
      "Bootstrap #803 ROC area: 0.634\n",
      "Bootstrap #804 ROC area: 0.634\n",
      "Bootstrap #805 ROC area: 0.634\n",
      "Bootstrap #806 ROC area: 0.634\n",
      "Bootstrap #807 ROC area: 0.634\n",
      "Bootstrap #808 ROC area: 0.634\n",
      "Bootstrap #809 ROC area: 0.634\n",
      "Bootstrap #810 ROC area: 0.634\n",
      "Bootstrap #811 ROC area: 0.634\n",
      "Bootstrap #812 ROC area: 0.634\n",
      "Bootstrap #813 ROC area: 0.634\n",
      "Bootstrap #814 ROC area: 0.634\n",
      "Bootstrap #815 ROC area: 0.634\n",
      "Bootstrap #816 ROC area: 0.634\n",
      "Bootstrap #817 ROC area: 0.634\n",
      "Bootstrap #818 ROC area: 0.634\n",
      "Bootstrap #819 ROC area: 0.634\n",
      "Bootstrap #820 ROC area: 0.634\n",
      "Bootstrap #821 ROC area: 0.634\n",
      "Bootstrap #822 ROC area: 0.634\n",
      "Bootstrap #823 ROC area: 0.634\n",
      "Bootstrap #824 ROC area: 0.634\n",
      "Bootstrap #825 ROC area: 0.634\n",
      "Bootstrap #826 ROC area: 0.634\n",
      "Bootstrap #827 ROC area: 0.634\n",
      "Bootstrap #828 ROC area: 0.634\n",
      "Bootstrap #829 ROC area: 0.634\n",
      "Bootstrap #830 ROC area: 0.634\n",
      "Bootstrap #831 ROC area: 0.634\n",
      "Bootstrap #832 ROC area: 0.634\n",
      "Bootstrap #833 ROC area: 0.634\n",
      "Bootstrap #834 ROC area: 0.634\n",
      "Bootstrap #835 ROC area: 0.634\n",
      "Bootstrap #836 ROC area: 0.634\n",
      "Bootstrap #837 ROC area: 0.634\n",
      "Bootstrap #838 ROC area: 0.634\n",
      "Bootstrap #839 ROC area: 0.634\n",
      "Bootstrap #840 ROC area: 0.634\n",
      "Bootstrap #841 ROC area: 0.634\n",
      "Bootstrap #842 ROC area: 0.634\n",
      "Bootstrap #843 ROC area: 0.634\n",
      "Bootstrap #844 ROC area: 0.634\n",
      "Bootstrap #845 ROC area: 0.634\n",
      "Bootstrap #846 ROC area: 0.634\n",
      "Bootstrap #847 ROC area: 0.634\n",
      "Bootstrap #848 ROC area: 0.634\n",
      "Bootstrap #849 ROC area: 0.634\n",
      "Bootstrap #850 ROC area: 0.634\n",
      "Bootstrap #851 ROC area: 0.634\n",
      "Bootstrap #852 ROC area: 0.634\n",
      "Bootstrap #853 ROC area: 0.634\n",
      "Bootstrap #854 ROC area: 0.634\n",
      "Bootstrap #855 ROC area: 0.634\n",
      "Bootstrap #856 ROC area: 0.634\n",
      "Bootstrap #857 ROC area: 0.634\n",
      "Bootstrap #858 ROC area: 0.634\n",
      "Bootstrap #859 ROC area: 0.634\n",
      "Bootstrap #860 ROC area: 0.634\n",
      "Bootstrap #861 ROC area: 0.634\n",
      "Bootstrap #862 ROC area: 0.634\n",
      "Bootstrap #863 ROC area: 0.634\n",
      "Bootstrap #864 ROC area: 0.634\n",
      "Bootstrap #865 ROC area: 0.634\n",
      "Bootstrap #866 ROC area: 0.634\n",
      "Bootstrap #867 ROC area: 0.634\n",
      "Bootstrap #868 ROC area: 0.634\n",
      "Bootstrap #869 ROC area: 0.634\n",
      "Bootstrap #870 ROC area: 0.634\n",
      "Bootstrap #871 ROC area: 0.634\n",
      "Bootstrap #872 ROC area: 0.634\n",
      "Bootstrap #873 ROC area: 0.634\n",
      "Bootstrap #874 ROC area: 0.634\n",
      "Bootstrap #875 ROC area: 0.634\n",
      "Bootstrap #876 ROC area: 0.634\n",
      "Bootstrap #877 ROC area: 0.634\n",
      "Bootstrap #878 ROC area: 0.634\n",
      "Bootstrap #879 ROC area: 0.634\n",
      "Bootstrap #880 ROC area: 0.634\n",
      "Bootstrap #881 ROC area: 0.634\n",
      "Bootstrap #882 ROC area: 0.634\n",
      "Bootstrap #883 ROC area: 0.634\n",
      "Bootstrap #884 ROC area: 0.634\n",
      "Bootstrap #885 ROC area: 0.634\n",
      "Bootstrap #886 ROC area: 0.634\n",
      "Bootstrap #887 ROC area: 0.634\n",
      "Bootstrap #888 ROC area: 0.634\n",
      "Bootstrap #889 ROC area: 0.634\n",
      "Bootstrap #890 ROC area: 0.634\n",
      "Bootstrap #891 ROC area: 0.634\n",
      "Bootstrap #892 ROC area: 0.634\n",
      "Bootstrap #893 ROC area: 0.634\n",
      "Bootstrap #894 ROC area: 0.634\n",
      "Bootstrap #895 ROC area: 0.634\n",
      "Bootstrap #896 ROC area: 0.634\n",
      "Bootstrap #897 ROC area: 0.634\n",
      "Bootstrap #898 ROC area: 0.634\n",
      "Bootstrap #899 ROC area: 0.634\n",
      "Bootstrap #900 ROC area: 0.634\n",
      "Bootstrap #901 ROC area: 0.634\n",
      "Bootstrap #902 ROC area: 0.634\n",
      "Bootstrap #903 ROC area: 0.634\n",
      "Bootstrap #904 ROC area: 0.634\n",
      "Bootstrap #905 ROC area: 0.634\n",
      "Bootstrap #906 ROC area: 0.634\n",
      "Bootstrap #907 ROC area: 0.634\n",
      "Bootstrap #908 ROC area: 0.634\n",
      "Bootstrap #909 ROC area: 0.634\n",
      "Bootstrap #910 ROC area: 0.634\n",
      "Bootstrap #911 ROC area: 0.634\n",
      "Bootstrap #912 ROC area: 0.634\n",
      "Bootstrap #913 ROC area: 0.634\n",
      "Bootstrap #914 ROC area: 0.634\n",
      "Bootstrap #915 ROC area: 0.634\n",
      "Bootstrap #916 ROC area: 0.634\n",
      "Bootstrap #917 ROC area: 0.634\n",
      "Bootstrap #918 ROC area: 0.634\n",
      "Bootstrap #919 ROC area: 0.634\n",
      "Bootstrap #920 ROC area: 0.634\n",
      "Bootstrap #921 ROC area: 0.634\n",
      "Bootstrap #922 ROC area: 0.634\n",
      "Bootstrap #923 ROC area: 0.634\n",
      "Bootstrap #924 ROC area: 0.634\n",
      "Bootstrap #925 ROC area: 0.634\n",
      "Bootstrap #926 ROC area: 0.634\n",
      "Bootstrap #927 ROC area: 0.634\n",
      "Bootstrap #928 ROC area: 0.634\n",
      "Bootstrap #929 ROC area: 0.634\n",
      "Bootstrap #930 ROC area: 0.634\n",
      "Bootstrap #931 ROC area: 0.634\n",
      "Bootstrap #932 ROC area: 0.634\n",
      "Bootstrap #933 ROC area: 0.634\n",
      "Bootstrap #934 ROC area: 0.634\n",
      "Bootstrap #935 ROC area: 0.634\n",
      "Bootstrap #936 ROC area: 0.634\n",
      "Bootstrap #937 ROC area: 0.634\n",
      "Bootstrap #938 ROC area: 0.634\n",
      "Bootstrap #939 ROC area: 0.634\n",
      "Bootstrap #940 ROC area: 0.634\n",
      "Bootstrap #941 ROC area: 0.634\n",
      "Bootstrap #942 ROC area: 0.634\n",
      "Bootstrap #943 ROC area: 0.634\n",
      "Bootstrap #944 ROC area: 0.634\n",
      "Bootstrap #945 ROC area: 0.634\n",
      "Bootstrap #946 ROC area: 0.634\n",
      "Bootstrap #947 ROC area: 0.634\n",
      "Bootstrap #948 ROC area: 0.634\n",
      "Bootstrap #949 ROC area: 0.634\n",
      "Bootstrap #950 ROC area: 0.634\n",
      "Bootstrap #951 ROC area: 0.634\n",
      "Bootstrap #952 ROC area: 0.634\n",
      "Bootstrap #953 ROC area: 0.634\n",
      "Bootstrap #954 ROC area: 0.634\n",
      "Bootstrap #955 ROC area: 0.634\n",
      "Bootstrap #956 ROC area: 0.634\n",
      "Bootstrap #957 ROC area: 0.634\n",
      "Bootstrap #958 ROC area: 0.634\n",
      "Bootstrap #959 ROC area: 0.634\n",
      "Bootstrap #960 ROC area: 0.634\n",
      "Bootstrap #961 ROC area: 0.634\n",
      "Bootstrap #962 ROC area: 0.634\n",
      "Bootstrap #963 ROC area: 0.634\n",
      "Bootstrap #964 ROC area: 0.634\n",
      "Bootstrap #965 ROC area: 0.634\n",
      "Bootstrap #966 ROC area: 0.634\n",
      "Bootstrap #967 ROC area: 0.634\n",
      "Bootstrap #968 ROC area: 0.634\n",
      "Bootstrap #969 ROC area: 0.634\n",
      "Bootstrap #970 ROC area: 0.634\n",
      "Bootstrap #971 ROC area: 0.634\n",
      "Bootstrap #972 ROC area: 0.634\n",
      "Bootstrap #973 ROC area: 0.634\n",
      "Bootstrap #974 ROC area: 0.634\n",
      "Bootstrap #975 ROC area: 0.634\n",
      "Bootstrap #976 ROC area: 0.634\n",
      "Bootstrap #977 ROC area: 0.634\n",
      "Bootstrap #978 ROC area: 0.634\n",
      "Bootstrap #979 ROC area: 0.634\n",
      "Bootstrap #980 ROC area: 0.634\n",
      "Bootstrap #981 ROC area: 0.634\n",
      "Bootstrap #982 ROC area: 0.634\n",
      "Bootstrap #983 ROC area: 0.634\n",
      "Bootstrap #984 ROC area: 0.634\n",
      "Bootstrap #985 ROC area: 0.634\n",
      "Bootstrap #986 ROC area: 0.634\n",
      "Bootstrap #987 ROC area: 0.634\n",
      "Bootstrap #988 ROC area: 0.634\n",
      "Bootstrap #989 ROC area: 0.634\n",
      "Bootstrap #990 ROC area: 0.634\n",
      "Bootstrap #991 ROC area: 0.634\n",
      "Bootstrap #992 ROC area: 0.634\n",
      "Bootstrap #993 ROC area: 0.634\n",
      "Bootstrap #994 ROC area: 0.634\n",
      "Bootstrap #995 ROC area: 0.634\n",
      "Bootstrap #996 ROC area: 0.634\n",
      "Bootstrap #997 ROC area: 0.634\n",
      "Bootstrap #998 ROC area: 0.634\n",
      "Bootstrap #999 ROC area: 0.634\n",
      "Bootstrap #1000 ROC area: 0.634\n",
      "Confidence interval for the score: [0.634 - 0.634]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x180ad3850>"
      ]
     },
     "execution_count": 946,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAAFACAYAAAAbNX/PAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABib0lEQVR4nO3deVxN+f8H8Ndtu21atKmkyL6bTHYpkZ1hKAaR9SdrY6wz1iEzxjb0tZO9bGEsIWTfZc1kCzFK0aa9e9+/P45OXS3qunVut8/z8bgPnc/Z3rfcd58+57OIiIjAMAzDlCk1oQNgGIapiFjyZRiGEQBLvgzDMAJgyZdhGEYALPkyDMMIgCVfhmEYAbDkyzAMIwCWfBmGYQTAki/DMIwAWPJlGIYRAEu+zFf5+/tDJBLxLw0NDVhbW2PYsGF4+/ZtgecQEXbs2IH27dvDyMgIurq6aNSoERYsWICUlJRC7xUUFISuXbvC1NQUWlpasLKywoABA3D27NlixZqeno4VK1agRYsWMDQ0hLa2NmrXro3x48fjyZMncr1/hikNIja3A/M1/v7+GD58OBYsWIDq1asjPT0d165dg7+/P+zs7PDw4UNoa2vzx0skEgwaNAh79+5Fu3bt0LdvX+jq6uLixYvYvXs36tevj5CQEFhYWPDnEBG8vLzg7++PZs2a4ccff0SVKlXw7t07BAUF4fbt27h8+TJat25daJxxcXHo0qULbt++jR49esDV1RX6+vqIiIhAQEAAoqOjkZmZWarfK4YpNmKYr9i6dSsBoJs3b8qUT58+nQBQYGCgTPnixYsJAE2dOjXftY4cOUJqamrUpUsXmfKlS5cSAJo8eTJJpdJ8523fvp2uX79eZJzdu3cnNTU12r9/f7596enp9PPPPxd5fnFlZWVRRkaGQq7FVFws+TJfVVjyPXr0KAGgxYsX82WpqalkbGxMtWvXpqysrAKvN3z4cAJAV69e5c+pXLky1a1bl7Kzs+WK8dq1awSARo0aVazjnZycyMnJKV+5p6cn2dra8tuRkZEEgJYuXUorVqygGjVqkJqaGl27do3U1dVp3rx5+a7x77//EgBavXo1XxYfH0+TJk2iqlWrkpaWFtnb29OSJUtIIpGU+L0yqoG1+TJye/nyJQDA2NiYL7t06RLi4+MxaNAgaGhoFHje0KFDAQBHjx7lz/n48SMGDRoEdXV1uWI5cuQIAGDIkCFynf81W7duxerVqzF69GgsW7YMlpaWcHJywt69e/MdGxgYCHV1dfTv3x8AkJqaCicnJ+zcuRNDhw7F33//jTZt2mDmzJnw8fEplXgZ5Vfwp4NhCpCYmIi4uDikp6fj+vXrmD9/PsRiMXr06MEfEx4eDgBo0qRJodfJ2ff48WOZfxs1aiR3bIq4RlHevHmDZ8+ewczMjC9zd3fHmDFj8PDhQzRs2JAvDwwMhJOTE9+mvXz5cjx//hxhYWGoVasWAGDMmDGwsrLC0qVL8fPPP8PGxqZU4maUF6v5MsXm6uoKMzMz2NjY4Mcff4Senh6OHDmCqlWr8sckJycDACpVqlTodXL2JSUlyfxb1Dlfo4hrFKVfv34yiRcA+vbtCw0NDQQGBvJlDx8+RHh4ONzd3fmyffv2oV27djA2NkZcXBz/cnV1hUQiwYULF0olZka5sZovU2x+fn6oXbs2EhMTsWXLFly4cAFisVjmmJzkl5OEC/JlgjYwMPjqOV+T9xpGRkZyX6cw1atXz1dmamqKjh07Yu/evVi4cCEArtaroaGBvn378sc9ffoU9+/fz5e8c7x//17h8TLKjyVfptgcHR3RvHlzAECfPn3Qtm1bDBo0CBEREdDX1wcA1KtXDwBw//599OnTp8Dr3L9/HwBQv359AEDdunUBAA8ePCj0nK/Je4127dp99XiRSAQqoJelRCIp8HgdHZ0Cyz08PDB8+HDcvXsXTZs2xd69e9GxY0eYmpryx0ilUnTq1AnTpk0r8Bq1a9f+aryM6mHNDoxc1NXV4evri//++w9r1qzhy9u2bQsjIyPs3r270ES2fft2AODbitu2bQtjY2Ps2bOn0HO+pmfPngCAnTt3Fut4Y2NjJCQk5Ct/9epVie7bp08faGlpITAwEHfv3sWTJ0/g4eEhc4y9vT0+ffoEV1fXAl/VqlUr0T0Z1cCSLyO3Dh06wNHREStXrkR6ejoAQFdXF1OnTkVERARmz56d75xjx47B398fbm5uaNmyJX/O9OnT8fjxY0yfPr3AGunOnTtx48aNQmNp1aoVunTpgk2bNuHQoUP59mdmZmLq1Kn8tr29Pf7991/ExsbyZffu3cPly5eL/f4BwMjICG5ubti7dy8CAgKgpaWVr/Y+YMAAXL16FSdPnsx3fkJCArKzs0t0T0ZFCN3XjVF+hfXzJSLat28fAaC1a9fyZdnZ2dSvXz8CQO3bt6dVq1bRhg0baOjQoaSmpkYNGjSg6OhometIJBIaMmQIAaDvvvuOFi9eTFu2bKHFixeTo6MjAaArV64UGef79++padOmJBKJqFevXrRq1SratGkTTZ8+nWxtbUlLS4s/Njw8nNTU1KhZs2a0Zs0amjNnDpmbm1OjRo0K7edbmJ07dxIAqlSpEvXs2TPf/pSUFPruu+9IQ0ODRo4cSWvXrqW//vqLPD09SU9Pj2JjY4t8X4xqYsmX+aqikq9EIiF7e3uyt7eXGSAhkUho69at1KZNGzIwMCBtbW1q0KABzZ8/nz59+lTovfbv30+dO3emypUrk4aGBllaWpK7uzuFhoYWK9bU1FT666+/6Pvvvyd9fX3S0tKiWrVq0YQJE+jZs2cyx+7cuZNq1KhBWlpa1LRpUzp58mSRgywKk5SURDo6OgSAdu7cWeAxycnJNHPmTKpZsyZpaWmRqakptW7dmv766y/KzMws1ntjVAub24FhGEYArM2XYRhGACz5MgzDCIAlX4ZhGAEoVfK9cOECevbsCSsrK4hEogK7DH0pNDQU3333HcRiMWrWrAl/f/9Sj5NhGOZbKVXyTUlJQZMmTeDn51es4yMjI9G9e3c4Ozvj7t27mDx5MkaOHFlgf0qGYRhlorS9HUQiEYKCgoocbjp9+nQcO3YMDx8+5Ms8PDyQkJCA4ODgMoiSYRhGPuV6boerV6/C1dVVpszNzQ2TJ08u9JyMjAxkZGTw21KpFB8/foSJiQlEIlFphcowjJyICMnJybCysoKamlL9sf5NynXyjY6OllkHDAAsLCyQlJSEtLS0AidD8fX1xfz588sqRIZhFCQqKkpm+tLyrlwnX3l8uXpAYmIiqlWrhqioKH5aQoZhygYREBoKrF8PnDhR0BE3YGISiA8fNpXaXM1CKdfJt0qVKoiJiZEpi4mJgYGBQaFTAIrF4nxz0ALcfLAs+TJM2UhOBnbsANasAT4vQsLT0gL69EnH7dueUFe/i8OHD6NevU0q1yxYrpNvq1atcPz4cZmy06dPo1WrVgJFxDBMUZ4+Bfz8gK1bgc+Lj/CqVgX+7/+AkSMBc3NtvH+/GtnZ2fxc0apGqZLvp0+f8OzZM347MjISd+/eReXKlVGtWjXMnDkTb9++5eeDHTt2LNasWYNp06bBy8sLZ8+exd69e3Hs2DGh3gLDMF+QSoGTJ4HVqwtuWmjfHnB3j4WHhxiVK+f+9Wlubg4gd4kolSPotD5fOHfuHAHI9/L09CQiblnvL5f7PnfuHDVt2pS0tLSoRo0atHXr1hLdMzExkQBQYmKiYt4EwzBERJSQQLRyJVGtWkRc627uS1ubaMQIort3iZ4/f042NjbUunVrSkpKyncdVf2MKm0/37KSlJQEQ0NDJCYmsjZfhlGAx4+5ttzt24FPn2T32doC48YBI0YAJibA8+fP0aFDB7x58wYAMHz4cGzZskXmHFX9jCpVswPDMOWTRAIcO8Y1LYSE5N/v4gJMmAD07Amoq3Nlz549g7OzM59469evD19f3zKMWlgs+TIMI7f4eGDzZuB//wMiI2X36eoCQ4cC48cDDRrI7nv69CmcnZ3x9u1bAECDBg1w9uxZvp23ImDJl2GYEnvwgKvl7twJpKXJ7rO3B7y9geHDASOj/Oc+efIEzs7O+O+//wAADRs2xNmzZ2FmZlb6gSsRlnwZhimW7Gzg8GEu6Z4/n3+/mxvXtNC1K1DYKOCIiAg4Ozvj3bt3AIBGjRrhzJkzFS7xAiz5MgzzFXFxwMaNwNq1QFSU7L5KlYBhw7iabp06RV/n5cuXMom3cePGOHPmDExNTUsncCXHki/DMAW6c4er5e7ZA+SZiwoAl2jHjwc8PbkEXByWlpZwcHDA0aNH0aRJE5w5cwYmJiaKD7ycYMmXYRheVhZw4ACXdK9ckd0nEgHdu3NNC66uhTctFEYsFmP//v2YPXs2Zs6cWaETL6DE8/mWFVXtQ8gwJREdDWzYAKxbB3xuFeAZGQFeXlzTQo0aJbsuEX3znAyq+hlVnckxGYYpsevXgcGDgWrVgLlzZRNvgwZcMn7zBli2rOSJ99GjR2jZsiVevnyp0JhVBWt2YJgKJiMD2LuXG4V244bsPjU1oHdvrmmhQweuqUEeDx8+hIuLC2JjY+Hs7Izz58+jWrVq3xy7KmHJl2EqiLdvuZrshg3A+/ey+ypXBkaN4mYVs7X9tvs8ePAALi4uiIuLAwCYmpqqVHOBorDkyzAqjAi4fJl7gHbwINdXN6+mTbla7sCBQCFTYJfI/fv34eLigg8fPgAAvv/+e5w6dQpGBY22qOBY8mUYFZSWxnURW7MGCAuT3aeuDvTrxyXdNm3kb1r40r1799CxY0c+8bZo0QInT56EoaGhYm6gYljyZRgV8vo1N8/Cpk3A5xzIMzcHxozhXtbWir3v3bt30bFjR3z8+BEA0LJlSwQHB7PEWwSWfBmmnMtZB231am74r1Qqu//777la7oABQAEraH2zsLAwuLq68om3VatWCA4OZu28X8GSL8OUY1evAqNHAw8fypZranLJdsIEoEWL0o3hn3/+4RNv69atceLECZZ4i4ElX4Ypp1JSuBFn8fG5ZZaWwNixXEKuUqVs4vjtt9+QnJyMq1ev4sSJEyq3ynBpYcmXYcqpQ4dyE2/DhsDs2UDfvtzqv2VJJBLhzz//REZGBrS1tcv25uUYS74MU059XkcWAPeQrV27srnvzZs3kZGRgbZt2/JlIpGIJd4SYsOLGaYcevs2d7me6tW5LmNl4caNG3B1dUXXrl1x5cuZd5gSYcmXYcqh3btzezUMGVLyGcbkcf36dXTq1AlJSUn49OkTlixZUvo3VWEs+TJMOSORyDY5DBlS+ve8evUqn3gBwNnZGXv27Cn9G6sw1ubLMEouPR24eRO4dAm4eJEbLvw5B6J1a6BmzdK9/5UrV9ClSxckJycDAFxcXPDPP/9AV1e3dG+s4ljyZRglEx/PTWSek2xv3gQyMws+duTI0o3l8uXL6NKlCz59+gQA6NixI44cOcISrwKw5MswAnvzhkuyOcn24UNu1FphzM25ng3dunHL+JSWS5cuoWvXrnzi7dSpEw4fPgwdRczAw7DkyzBliQh4/Fg22b56VfQ5NWtyybZtW+7fmjUVNxlOYT58+IDu3bvzibdz5844dOgQS7wKxJIvw5SizExuIcq87bVfTniTl5oaN81jTrJt27bsRqrlZWJigr///hvDhw+Hm5sbgoKCWD9eBWPJl2EUKDmZm28hJ9lev85N71gYHR1u7oWcWm2rVsVfDbi0eXp6wsLCAh06dGCJtxSw5Msw3yAmJjfRXrwI3L2bf1axvCpXzq3RtmsHfPdd2Q8HLkxsbCzMzMxkyrp06SJQNKqPJV+GKSYi4Plz2fbap0+LPsfWVra9tm7dshkQUVJnz55F79694efnh6FDhwodToXAki/DFCI7G7h/PzfZXrrELbFeGJGIm+AmJ9G2bQvY2JRdvPI6c+YMevbsibS0NAwbNgxWVlZwdXUVOiyVx5Ivw3yWlsa10ebUaq9cAT4/7C+QpiY3UXlOom3TBjA2Lrt4FSEkJAQ9e/ZEeno6AKBnz55oV1Yz9FRwLPkyFdaHD1zvg5xke/s2kJVV+PEGBtyIspya7fffK2bRSaGcPn0avXr14hNv7969sXfvXmgpSyO0imPJl6kQiLj1zfI+HAsPL/ocS0vZ9tpGjbjFJ1XBqVOn0KtXL2RkZAAA+vTpg8DAQJZ4yxBLvoxKkkqBR49kk+2bN0WfU6eObHttjRqlP5hBCCdPnkTv3r35xNu3b18EBARAU1NT4MgqGFIya9asIVtbWxKLxeTo6EjXr18v8vgVK1ZQ7dq1SVtbm6pWrUqTJ0+mtLS0Yt8vMTGRAFBiYuK3hs4oia1biczMiLj6bsEvdXWi778nmjKF6OBBopgYoaMuGydOnCCxWEwACAD169ePMjMzhQ6rSKr6Gf2m5Juenk5XrlyhQ4cOUWxs7DcHExAQQFpaWrRlyxZ69OgRjRo1ioyMjCimkE/Grl27SCwW065duygyMpJOnjxJlpaWNGXKlGLfU1V/sBVRWhrRqFEFJ1tdXSIXF6K5c4lOnyZKThY6WmFcunSJ9PX1CQD9+OOPSp94iVT3Myp38l21ahUZGxuTmpoaqamp0ZkzZ4iIKDY2lkxMTGjz5s0lvqajoyN5e3vz2xKJhKysrMjX17fA4729vcnFxUWmzMfHh9q0aVPse6rqD7aiycggat9eNuF27Uq0bBnR9etE5SDHlJmLFy/S8OHDy0XiJVLdz6hc3b23bt2KyZMno0uXLti8eTMozxRMpqamcHFxQUBAQImumZmZidu3b8v0L1RTU4OrqyuuXr1a4DmtW7fG7du3cePGDQDAixcvcPz4cXTr1q3Q+2RkZCApKUnmxZR/M2YAFy5wX+vocJONHz8O+PgAjo5ctzCG07ZtW2zZsoW18QpMruS7bNky9O7dG7t370bPnj3z7XdwcMCjR49KdM24uDhIJBJYWFjIlFtYWCC6kJ7tgwYNwoIFC9C2bVtoamrC3t4eHTp0wKxZswq9j6+vLwwNDfmXTXnoBc8UKSgIWLGC+1pLCzh7tmxWdygPjhw5glmzZslUkBjlIFfyffbsGbp27Vro/sqVK+NDUVM3KUhoaCgWL16M//3vf7hz5w4OHjyIY8eOYeHChYWeM3PmTCQmJvKvqKioUo+TKT0vXgDDh+duL18OtGwpXDzK5NChQ/jxxx/h6+uLGTNmsASsZOTqamZkZIS4uLhC94eHh6NKCefBMzU1hbq6OmJiYmTKY2JiCr3Wb7/9hiFDhmDk5+n8GzVqhJSUFIwePRqzZ8+GWgGD6MViMcRicYliY5RTRgYwYACQmMht9+8PjBsnbEzKIigoCAMGDEB2djYA4O3bt5BKpVBXlY7KKkCumm+3bt2wYcMGJCQk5Nv36NEjbNy4Eb169SrRNbW0tODg4IAzZ87wZVKpFGfOnEGrVq0KPCc1NTVfgs35z8V+y6u+n3/mRqUBQK1awKZNqtkvt6QOHjwok3iHDBmCbdu2scSrbOR5Svf27VuqWrUqWVtb09ixY0lNTY2GDh1KP/30E2lra1P16tXl6noWEBBAYrGY/P39KTw8nEaPHk1GRkYUHR1NRERDhgyhGTNm8MfPnTuXKlWqRHv27KEXL17QqVOnyN7engYMGFDse6rqk1RVFxiY26tBLCYKCxM6IuWwb98+UldX5/vxDh06lLKzs4UO65uo6mdU7q5mMTExNGLECDI2NiaRSEQikYgMDAxo+PDhhfbLLY7Vq1dTtWrVSEtLixwdHenatWv8PicnJ/L09OS3s7KyaN68eWRvb0/a2tpkY2ND48aNo/j4+GLfT1V/sKrsyROiSpVyk+/69UJHpBz27t0rk3g9PT3LfeIlUt3PqIjo2/8+j42NhVQqhZmZWYHtrMosKSkJhoaGSExMhIGBgdDhMF+Rlsat9nDvHrf900/Ajh2suWHv3r0YNGgQJBIJAGD48OHYuHGjSjQ1qOpnVK5M6eXlhevXr/PbZmZmsLCw4BPvjRs34OXlpZgIGSaPyZNzE2/dusC6dSzxZmRkYNasWXzi9fLywqZNm1Qi8aoyuZKvv78/nj9/Xuj+yMhIbNu2Te6gGKYgu3cDGzZwX+voAPv2Afr6wsakDMRiMU6fPo1q1aph5MiR2LhxY7n7C7QiKpVZzf777z+2xDSjUP/+C4wenbvt58etGsFwqlevjhs3bpTLpr+KqtjJ9/Dhwzh8+DC/vWHDBoSEhOQ7LiEhASEhIfj+++8VEyFT4aWmcn14U1K47WHDZAdWVERnzpxB27ZtZfqsfzk6lFFuxU6+4eHh2LdvHwBAJBLh+vXruJ3TyfIzkUgEPT09tG/fHsuXL1dspEyFNWEC8PAh93WDBlyttyLbsWMHhg0bhh49emDfvn1sAvRySq7eDmpqati5cycGDRpUGjGVKVV9kqoqtm3jaroAoKcH3LwJ1KsnaEiC2rZtG4YPH84PItqwYQNGjRolcFSlS1U/o3K1+UqlUkXHwTD5PHoE/N//5W6vX1+xE6+/vz+8vLz4xOvt7c0PrWfKH9YyzyilT5+4dt60NG571CiuT29FtXXrVpnEO2HCBKxevRqiit7PrhyTO/meOHECnTp1gomJCTQ0NKCurp7vxTDyIOImyHn8mNtu3BhYtUrYmIS0efNmjBgxgk+8EydOxKpVq1jiLefkSr4HDhxAjx49EBMTAw8PD0ilUgwcOBAeHh7Q0dFB48aNMWfOHEXHylQQmzdzo9YArh/vvn3le4n2b7Fp0yaMHDmST7yTJk3CypUrWeJVBfKMSXZwcKCWLVtSdnY2xcbGkkgk4pcRioyMJAsLC9q2bZsihj+XOlUdN15e3btHpK2dO29DQIDQEQknKCiIn6cBAE2ZMoWkUqnQYZU5Vf2MylXzDQ8Ph4eHB9TV1aGhwT2zy8rKAgDY2dlh3Lhx+OOPPxTyy4GpOJKTuXbe9HRue9w4wN1d2JiE5OLiwk+n+vPPP2PZsmWsxqtC5OrtoKury/ctNDIyglgsxrt37/j9FhYWiIyMVEyETIVAxI1ge/KE2/7uO2DZMmFjEpqBgQGCg4Oxc+dO/N///R9LvCpGrppvnTp1EB4ezm83bdoUO3bsQHZ2NtLT07F7925Uq1ZNYUEyqm/9eiBnzVVDQ66dV1tb2JiEkJmZKbNtYGCAcePGscSrguRKvj/88AMOHz6MjIwMAMDs2bMRGhoKIyMjmJmZ4eLFi5gxY4ZCA2VU1507wKRJudtbtwI1aggXj1D8/Pzw/fffF7lEF6M6FDKfLwBcvHgRBw8ehLq6Orp37w5nZ2dFXLbUqeromfIiMZFrYnjxgtueNAlYuVLQkASxevVqTJw4EQDQuHFjXLlyBXp6egJHpRxU9TOqsFnN2rVrh3bt2vHbycnJqFSpkqIuz6ggIsDLKzfxOjoCf/4pbExCWLVqFSZPnsxv9+rVC7q6usIFxJQJhY9we//+PWbNmsXafJmvWr0aOHiQ+9rICAgMBCraHDErV66USby//fYbFixYwNp4K4AS1Xzfv3+P7du34/nz5zA2Nka/fv3g4OAAgFuaetGiRfD390d6ejo6dOhQGvEyKuLGDWDq1NztbdsAOzvBwhHEihUr4OPjw2/PnTsX8+bNEy4gpmwVt0Pw48ePyczMjNTU1PgFM9XV1WnPnj0UEBBA+vr6pKGhQe7u7nTr1q3S65msYKragVuZffhAZGubO5Bi6lShIyp7f/31l8wAinnz5gkdktJS1c9osZPvjz/+SDo6OrRu3Tp69OgRHT16lGrVqkUWFhakra1N/fv3p+fPn5dmrKVCVX+wykoqJerVKzfxtm5NlJkpdFRl688//5RJvPPnzxc6JKWmqp/RYjc7XLhwAf/3f/+HMWPGAADq168PDQ0NdO3aFZ6enti6dauC6+SMKlq+HDhyhPvaxITr26upKWxMZYmI8DBnZngACxYswG+//SZgRIxQip18P3z4gMaNG8uUNWnSBADX75dhvubqVSBv9+8dOwAbG+HiEYJIJMKWLVsglUpRt25dzJ49W+iQGIEUO/lKpVJoflFFydnWZ0vIMl/x4QM3T0N2Nrc9cybQtauwMQlFXV0d27dvZz0aKrgS9Xa4desWtPOM+UxOToZIJMKlS5eQkJCQ7/i+fft+c4BM+SeVAkOHAlFR3Ha7dsCCBcLGVJZWrFiBzp07o0GDBnwZS7xMsUe4lXQ5apFIBIlEIldQZUlVR88okz/+yG1uMDMD7t4FrKwEDanMLFy4EHPmzIG5uTnOnTuH+vXrCx1SuaOqn9Fi13zPnTtXmnEwKuriRSCnWVMkAnbtqjiJd8GCBZg7dy4Aro88S75MXsVOvk5OTqUZB6OC3r8HPDyAnD+AfvsN6NRJ2JjKyrx58zB//nx++6+//oK3t7eAETHKRmFzOzBMXlIpMGQI8N9/3LazM1ARVpYiIsybNw8L8jRqL1++HFOmTBEwKkYZseTLlIrFi4FTp7ivLSyA3bsBVV9TlYgwd+5cLFy4kC9bsWKFzNwNDJODJV9G4c6dAz43dUJNDdizB6hSRdiYShsR4bfffsOiRYv4slWrVvHTRDLMl1jyZRQqOhoYOJBrdgCAefO4JgdVd+nSJZnEu3r1aowfP17AiBhlp/ApJZmKSyIBBg0CYmK47c6dc3s6qLp27dphyZIlAIA1a9awxMt8lcJWsiivVLUPoRDmzAFymjutrLj+vGZmgoZU5sLCwtCsWTOhw1ApqvoZlbvm+/r1a4wdOxZ16tRB5cqVceHCBQBAXFwcJk6ciLCwMIUFySi/U6eA33/nvlZX5ybMUeXES0Qyi8jmYImXKS65km94eDiaNWuGwMBAVK9eHYmJicj+PGjf1NQUly5dwpo1a+QKyM/PD3Z2dtDW1kaLFi1w48aNIo9PSEiAt7c3LC0tIRaLUbt2bRw/flyuezPyefsWGDyYmyQS4JJwnhWlVA4RYerUqWjWrBlOnDghdDhMOSXXA7dp06bByMgI165dg0gkgrm5ucz+7t27IzAwsMTXDQwMhI+PD9atW4cWLVpg5cqVcHNzQ0RERL57ANwy2506dYK5uTn2798Pa2trvHr1CkZGRvK8LUYO2dncA7bYWG67Wzdg2jRhYypNRAQfHx+s/LzKZ9++ffH8+XNYVZRhe4ziyDMJcKVKlWjp0qVERBQXF0cikYjOnDnD79+wYQPp6uqW+LqOjo7k7e3Nb0skErKysiJfX98Cj1+7di3VqFGDMr9hNm5Vnai5rMyYkTsxuo0NUVyc0BGVHqlUSpMmTeInQReJRLRp0yahw1J5qvoZlavZQSqVFrm6amxsLMRicYmumZmZidu3b8PV1ZUvU1NTg6urK65evVrgOUeOHEGrVq3g7e0NCwsLNGzYEIsXLy5yQp+MjAwkJSXJvBj5HD8OfH7ADw0NbgFMExNhYyotRITJkydj1apVALiJozZt2oQRI0YIHBlTXsmVfL/77jscO3aswH3Z2dkICAhAy5YtS3TNuLg4SCQSWFhYyJRbWFggOjq6wHNevHiB/fv3QyKR4Pjx4/jtt9+wbNky/J7z5KcAvr6+MDQ05F82FW02bwWJiuKGD+dYsgRo1Uq4eEoTEWHixIn4+++/AeROiO7l5SVwZEy5Jk91+fjx46SmpkZjx46l0NBQEolEtHv3bjp9+jQ5OzuThoYGnT9/vkTXfPv2LQGgK1euyJT/8ssv5OjoWOA5tWrVIhsbG8rOzubLli1bRlWqVCn0Punp6ZSYmMi/oqKiVPJPmtKUmUnUqlVuc0OvXtzabKpIKpWSt7e3TFODv7+/0GFVKKra7CDXA7euXbvC398fkyZNwoYNGwAAgwcPBhHBwMAA27dvR/v27Ut0TVNTU6irqyMmp4f+ZzExMahSyNhUS0tLaGpqQj3PpAH16tVDdHQ0MjMzoaWlle8csVhc4iYRRtbMmdySQAC33Lu/PzddpCqaMmUK/Pz8AHA1Xn9/fwwdOlTgqBhVIHc/3yFDhiAqKgoHDhzAH3/8gcWLF2Pv3r2IiorCwIEDS3w9LS0tODg44MyZM3yZVCrFmTNn0KqQv2fbtGmDZ8+eQZozlhXAkydPYGlpWWDiZb7d4cPAsmXc15qawN69gLGxsDGVJhcXF2hqakJNTQ3bt29niZdRHHmqy9JS+hszICCAxGIx+fv7U3h4OI0ePZqMjIwoOjqaiIiGDBlCM2bM4I9//fo1VapUicaPH08RERF09OhRMjc3p99//73Y91TVP2lKw4sXREZGuc0Nf/8tdERl49ChQ7Rz506hw6iwVPUzKlezg7W1Nfr3748BAwagTZs2CvtF4O7ujtjYWMyZMwfR0dFo2rQpgoOD+Ydwr1+/llnOyMbGBidPnsSUKVPQuHFjWFtbY9KkSZg+fbrCYmI4mZncApg5S/X16weo4vQFRJRvfbXevXsLFA2jyuSa22HgwIE4evQoUlNTYW1tjQEDBmDAgAFwdHQsjRhLlaqOG1e0SZOAzw/7YW8P3L4NGBoKG5OiSaVSjB49Gg0bNmRz8CoRlf2MyltlTk1Npb1791K/fv1IV1eX1NTUqEaNGjRz5kwKCwtTVM281KnqnzSKtH9/blODlhbR7dtCR6R4EomEhg8fzvdqWLVqldAhMZ+p6mdU7gduOjo66N+/P/bv34/3799j586daNSoEVasWAEHBwfUrVtXUb8fGAE9fw7k7c66ahXw3XfCxVMaJBIJRowYga1btwIA1NXVYWlpKXBUjKpTyHy+enp6GDhwIHbu3ImlS5dCX18fT58+VcSlGQGlpwMDBgA5gwAHDgTGjBE2JkWTSCTw8vKCv78/AC7xBgQEoH///sIGxqi8b17JIjU1FUeOHMHevXsRHByMjIwM2Nvbs+VTVICPD3DnDvd17drA+vWq1Z9XIpFg+PDh2LFjBwBAQ0MDAQEB6Nevn8CRMRWBXMk3PT0dx44dQ2BgII4fP47U1FTY2dlh4sSJcHd3Z3OaqoDAQGDtWu5rbW1g3z6gUiVhY1IkiUSCYcOGYefOnQC4xLt371788MMPAkfGVBRyJV8zMzOkpqbCysoKo0ePhru7O1q0aKHo2BiBPHkCjByZu71mDdC4sXDxKFp2djY8PT2xe/duAFzi3bdvH/r06SNsYEyFIlfyHTZsGNzd3dG2bVtFx8MILC0N6N8f+PSJ2x4yRPaBmyqIiorCyZMnAQCamprYt28f68vLlDm2hpuq9iGU06hRwKZN3Nf16gE3bwJ6esLGVBru3buHrl27Yv369ejZs6fQ4TBFUNXPaLFqvjnrs+VMlpOz/TUlnVyHEdbOnbmJV1cX2L9fNRMvADRp0gTPnj0rcl5qhilNxar5qqmpQSQSIS0tDVpaWvx2YejzEM2iJjVXFqr6W7WkHj8GmjcHUlO57W3bAFWZQyYrKwtbtmzBqFGjZIanM+WDqn5Gi1XzPXfuHADwM4XlbDOqISWFa+fNSbxeXqqVeAcOHIgDBw7g1q1bWL9+PUvAjFJgbb4q+ls1x5o13GrCgwYBy5cXfMzw4dycvADQsCFw/TrX7FDeZWVlwcPDAwcPHgTAzeV848YNNFalrhsVgKp+RuWqAri4uMjMu/ulc+fOwcXFRe6gmG9HBMybB0yYAMTEACtWcEv/fGnr1tzEq6/PtfOqQuLNzMyEu7u7TOI9fPgwS7yM0pAr+YaGhuZbcSKv9+/f4/z583IHxXwbIuDnn4H582XLT52S3X7wAPD2zt3esAGoU6f04yttmZmZGDBgAIKCggAA2traOHLkCNzc3ASOjGFyyd34VdQDt2fPnqGSKg2HKkckEm7+hRUr8u/Lm3w/feLaedPSuO0xY7i5G8q7zMxM9O/fH4cPHwaQm3g7d+4scGQMI6vYgyy2bduGbdu28du///47Nm7cmO+4hIQE3L9/H926dVNMhEyxZWVxgyICA7ltkQhYtw6YNg1ITARCQrjkrKbGJduICO64pk2BlSuFilpxMjIy0L9/f/zzzz8AuMT7zz//wNXVVeDIGCa/Yiff1NRUxMbG8tvJycn5nhqLRCLo6elh7NixmDNnjuKiZL4qPZ2ryR49ym1raHD9dt3duRrvgQPAx4/cRDlhYcDnkbWoVImbt0FbW7jYFWX69Ol84tXR0cE///yDjh07ChwVwxRCnkmA7ezs6PDhwwqbVFhIqjBRc3IykYtL7oTnYjHR0aO5+9evz903YAC3P2d73z7h4la06OhoqlevHuno6NDZs2eFDodREFX4jBaEdTUr591Y4uOBbt2Aa9e4bT094J9/AGfn3GNevgSqV89/7vjxwOrVZRJmmYmOjsbz588VurYgI6zy/hktTLGaHV6/fg0AqFatmsz21+Qcz5SO9++Bzp2Be/e4bSMj4MQJoGVL2ePs7Lj5eJ88yS1r3hz466+yirR0pKenQyKRQC/PGOgqVaqgSpUqAkbFMMVTrORrZ2cnM7w4Z/trysPw4vIqKgro1Cn3oZm5OXD6dOFTP3bunJt8DQ2BvXsBsbhsYi0NaWlp6NOnD9LT03H8+HGZBMww5UGxku+WLVsgEomgqakps80I4/lzoGNH4NUrbtvGhuvJULt24ecMHcpNji4ScYMqCmqGKC/S0tLQu3dvnD59GgDg4eHBP2hjmPKCtfmWs/akR4+4Gu+7d9x2zZpc4rW1/fq5ERFc8i0qSSu71NRU9O7dGyEhIQAAfX19BAcHszZeFVbePqPF9c1ruOWVmZmJrKws9idgKbl1C3Bz47qMAdw8DKdOAcVdaLe8j15LTU1Fr169+KHtlSpVQnBwMFq3bi1wZAxTcnKNcAsICMCUKVNkyubPnw99fX0YGRnhhx9+wKecpRAYhbh4EXBxyU28zZsDoaHFT7zlXWpqKnr27CmTeE+ePMkSL1NuyZV8ly1bhpSUFH77ypUrmD9/Ptzc3DBlyhQEBwdj0aJFCguyogsO5mq8ycncdvv2wJkzgImJsHGVlZSUFPTo0QNnz54FABgYGODUqVNo1aqVwJExjPzkanZ4/vw5PD09+e3du3ejSpUqCAoKgoaGBqRSKQ4cOABfX1+FBVpRHTwIeHhwQ4cBoGtX1Zl5rDhyEm9oaCiA3MTLFmxlyju5ar4ZGRnQzjMe9dSpU+jatSs0NLhcXr9+fbx580YxEVZg27dzQ4ZzEu+PPwKHDlWcxAsA6urq/P81Q0NDnD59miVeRiXIlXyrV6/OP22+desWnj17hi5duvD7Y2JioK+vr5gIKyg/P8DTE5BKue1hw4A9e4DPi4lUGNra2ggKCoKHhwdOnz4NR0dHoUNiGIWQq9lhzJgxmDRpEsLDw/HmzRtUrVoVPXr04PdfvnwZDRo0UFiQFc2SJcDMmbnbEyZws45V1NVvtLW1sWfPHqHDYBiFkuvjPGHCBKxfvx729vbo3bs3Tp06BR0dHQDAx48fER0djZ9++kmhgVYERFzSzZt4Z80CVq2qOIk3OTkZgwcPLvYQdoYpr9ggCyXpwC2VApMmcWuu5ViyBJg+XbCQylxSUhK6du2KK1euoEaNGggNDYWNjY3QYTECU5bPqKJ98yCL8PBwvPo8ztXW1hb169f/5qAqmuxsYORIbrn2HH5+wLhxwsVU1hITE9GlSxdc+zw9W0JCAj5+/MiSL6Oy5E6+hw8fho+PD16+fClTXr16dSxfvhy9evX61tgqhMxMbmXhAwe4bTU1blFLVVm6vTgSExPh5uaG69evAwBMTExw5swZNGnSRODIGKb0yJV8jx8/jn79+sHW1haLFy9GvXr1AACPHz/Ghg0b0LdvXxw9elSmBwSTX2oq0K8fN4gCADQ1gYAAoG9fYeMqSwkJCXBzc8ONGzcAsMTLVCDyzMDesmVLatasGX369Cnfvk+fPlHTpk2pZcuWcs/wvmbNGrK1tSWxWEyOjo50/fr1Yp23Z88eAkC9e/cu9r2EmiU/MZGoXbvcFSV0dIiCg8s0BMHFx8fT999/TwAIAJmamtK9e/eEDotRMqq6koVcz9Dv378PT0/PAifQ0dPTw7Bhw3D//n25fhkEBgbCx8cHc+fOxZ07d9CkSRO4ubnh/fv3RZ738uVLTJ06Fe3atZPrvmXpwwduSsiLF7ntSpWAkye5IcQVRXx8PDp16oSbN28CAMzMzHDu3Dk0LmxCYoZRMXIlX21tbXzMmeGlAB8/fpQZAVcSy5cvx6hRozB8+HDUr18f69atg66uLrZs2VLoORKJBD/99BPmz5+PGjVqyHXfsvLuHeDkxM1QBgCVKwNnzwLl4HeGQm3btg23Pn8TzMzMcPbsWTRs2FDgqBim7MiVfF1cXLBq1SpcvXo1377r16/j77//lmu57szMTNy+fVvmXDU1Nbi6uhZ4rxwLFiyAubk5RowY8dV7ZGRkICkpSeZVVl694ibFefSI27a0BC5c4GYoq2gmTZqEiRMnwtzcHOfOnWOJl6lw5Hrg9ueff6JVq1Zo27YtHB0dUefzRLERERG4ceMGzM3N8ccff5T4unFxcZBIJLCwsJApt7CwwL///lvgOZcuXcLmzZtx9+7dYt3D19cX8+fPL3Fs3yoiAnB1BXKmvLCz4yZBt7cv81CUgkgkwsqVKzF9+nRYWVkJHQ7DlDm553a4f/8+Jk6ciPj4eAQGBiIwMBDx8fGYNGkS7t27Bzs7OwWHml9ycjKGDBmCjRs3wtTUtFjnzJw5E4mJifwrKiqqlKPkFrhs3z438dapw7X3VqTE++HDB9y+fVumTCQSscTLVFglrvlKJBLExsbCyMgIK1aswIoVKxQWjKmpKdTV1RETEyNTHhMTU+CKtM+fP8fLly/Rs2dPvkz6eSYaDQ0NREREwP6LDCcWiyEuw5Ujr13jpoFMSOC2mzThVp8wNy+zEAQXFxcHV1dXREZGsslxGOazYtd8iQizZs2CsbExrK2tYWBggB9++KHIB28lpaWlBQcHB361AoBLpmfOnClw4uy6deviwYMHuHv3Lv/q1asXnJ2dcffuXcFHR507xzU15CTeli25soqUeGNjY+Hi4oJ79+4hKSkJw4YNY6taMwxQ/H6+W7ZsIZFIRDY2NtS/f3/67rvvSCQSUa9evRTa9y0gIIDEYjH5+/tTeHg4jR49moyMjCg6OpqIiIYMGUIzZswo9HxPT0+l6Od79CiRWJzbj9fFhSg5WaG3UHrv37+nRo0a8f14raysKCIiQuiwmHJGVfv5FrvZYe3atWjWrBkuXbrEz2A2adIk+Pn5IS4urthtrl/j7u6O2NhYzJkzB9HR0WjatCmCg4P5h3CvX7+GmpJP8bV3L/DTT9ycDQDQsydXJmfvu3Lp/fv36NixIx4+fAgAsLa2xrlz51CrVi2BI2MYJVHcLF25cmVauXKlTNm///5LIpGIrly5ovDfCmVF0b9VN20iEolya7wDBxJlZirk0uVGdHQ01a9fn6/xWltb09OnT4UOiymnVLXmW+wqZHx8PMzMzGTKcmq76enpivttUI6tXMnNTpYzSeeoUcCOHdycDRVFTEwMXFxcEB4eDgCoWrUqQkNDUbNmTYEjYxjlUqK/30UiUWnFUa4RAQsXAlOm5Jb5+ADr1wPq6sLFVdYyMzPRsWNHPvHa2NiwxMswhSj2ZOpqamqwsbGBoaEhXyaRSPD48WNUr1493zwPIpEI9+7dU2y0peBbJ2omAqZNA/76K7ds3jxgzhygIv6u2rRpE0aNGoVq1arh3LlzSj/cm1F+FX4y9fbt2xdY8zWvSP2mviCRcBOeb9iQW7ZsGVfrrahGjhwJbW1ttGnTBtWrVxc6HIZRWmwZITl/q2ZlcSsK797NbYtEXDPDqFGlE6eyyszMhFZFW1KZKVOqWvNV7j5bSio9HejfPzfxqqsDu3ZVvMT79u1bNG3alK0szDBy+OY13CqalBSgTx9uUhwAEIu5PrwVbdWkN2/ewNnZGc+ePcPgwYOhra2NH374QeiwGKbcYMm3BBISgO7dgStXuG1dXeDIEW5i9IokKioKzs7OeP78OQDAzs4ODg4OAkfFMOULS77FFBvLrTQRFsZtGxoCx48DrVsLG1dZi4qKQocOHfDixQsAYEu8M4ycWPIthrdvgU6dgMePuW0zM25msqZNBQ2rzL1+/RrOzs584rW3t0doaCiqVq0qcGQMU/6w5PsVL15wM5NFRnLb1tZce2/dusLGVdZevXoFZ2dnRH7+RtSsWROhoaGwtrYWODKGKZ++Kfm+ffsWFy5cwPv379GvXz9UrVoVEokEiYmJMDQ0hHo5H94VHs4l3nfvuO0aNYAzZ7hVKCqSly9fwtnZGS9fvgQA1KpVC+fOnWOJl2G+gVxdzYgIPj4+qF69On766Sf4+PjgyZMnAIBPnz7Bzs4Oq1evVmigZe3OHW71iZzEW78+t/pERUu8ANezIWf16Nq1a7MaL8MogFzJd+nSpVi1ahWmTp2K06dPI+84DUNDQ/Tt2xcHDhxQWJBl7dIlwNmZW+IdABwcgPPngYq64k3btm1x7NgxfPfddwgNDWVL/zCMAsjV7LBx40YMHToUixcvxoecDJVH48aNceLEiW8OTginTwO9ewNpadx227bA0aNc74aKrEOHDrh586bSz6XMMOWFXJ+kqKgotC6ij5Wenl6ZLsmuKIcOAT165Cbezp2BkycrXuJ9/vw5li5dii9HnrPEyzCKI1fN19zcvMhVf2/fvo1q1arJHZQQQkKAAQO4yXIA4IcfgD17uBFsFcmzZ8/g7OyMN2/eICkpCQsWLGBTiTJMKZCrKtO3b1+sW7eO7+8J5M71e+rUKfj7+6N///6KibCM+PnlJt4hQ7ghwxUt8T59+hQdOnTAm89r3AcFBSElJUXgqBhGNck1q1liYiLat2+PyMhItGvXDsHBwejUqRM+ffqEq1evolmzZrhw4QJ0dXVLI2aFypkxqUmTRNy7ZwB1dSAzE6hof2HnJN7//vsPANCwYUOcPXs23+olDFPW2KxmeRgaGuLatWuYNm0a3r59C21tbZw/fx4JCQmYO3cuLl68WC4Sb1457bw6OhUv8UZERMDJyYlPvI0aNWKJl2FKGZvP9/Nv1WrVEvH6tQHMzIDPXVorhH///RcuLi5497lDc+PGjXHmzBmFrUbNMN+K1XxVXN6ab0Xx+PFjODs784m3SZMmOHv2LEu8DFMG5Ort4OXl9dVjRCIRNm/eLM/lBZGRwf1bUZIvEcHT0xPR0dEAgKZNmyIkJAQmJiYCR8YwFYNcyffs2bP5uh9JJBK8e/cOEokEZmZm+RbUVHYVreYrEomwe/dudOjQAebm5ggJCUHlypWFDothKgy5km/OBCtfysrKwvr167Fy5UqcPn36W+Iqc1lZ3L/a2sLGUZZq1qyJ8+fPw9jYmCVehiljCm3z1dTUxPjx49G5c2eMHz9ekZcuM6pc833+/Dmycn7LfGZvb88SL8MIoFQeuDVp0gQXLlwojUuXOlVNvg8ePEDLli3h4eGRLwEzDFP2SiX5nj59utz1882hisn3/v37cHZ2RlxcHA4ePIgFCxYIHRLDVHhytfkW9uFNSEjAhQsXcOfOHcyYMeObAhOKqiXfe/fuoWPHjvzscy1atMDUqVMFjophGLmS77x58wosNzY2hr29PdatW4dRo0Z9S1yCUaXke/fuXXTs2BEfP34EALRs2RLBwcEwrGjTtDGMEpIr+UqlUkXHoTRUpbdDWFgYXF1d+cTbqlUrBAcHq9QIIYYpz0rc5puWlgYfHx/8888/pRGP4FSh5nvnzh2ZGm/r1q1Z4mUYJVPi5Kujo4P169cjJiamNOIRXHlPvg8ePICrqyvi4+MBAG3atGGJl2GUkFy9HRwcHPDw4UNFx6IUynvyrVatGmrVqgWAW3vtxIkTqFSpksBRMQzzJbmS78qVKxEQEIBNmzYhOztb0TEJqrwnX0NDQ5w8eRLjx49niZdhlFixp5S8cOEC6tWrBzMzMzRq1AgfPnxATEwMxGIxrK2tofNF1hKJRLh3716pBK1IOdPVAYkADLB+PTB6tNBRlQwRsaV+GJVV4aeUdHZ2RkhICADAxMQEderUQfv27dGiRQtUrVoVJiYmMq9vGbLq5+cHOzs7aGtro0WLFrhx40ahx27cuBHt2rWDsbExjI2N4erqWuTxX1Pear7Xrl2Di4sL/3CNYZjyodhdzYiIX802NDS0tOJBYGAgfHx8sG7dOrRo0QIrV66Em5sbIiIiYG5unu/40NBQDBw4EK1bt4a2tjb++OMPdO7cGY8ePYK1tXWJ71+eku/Vq1fh5uaG5ORkdOrUCSEhITA2NhY6LIZhikHpJlNfvnw5Ro0aheHDh6N+/fpYt24ddHV1sWXLlgKP37VrF8aNG4emTZuibt262LRpE6RSKc6cOSPX/ctL8r1y5QqfeAHAyMgI4oq24ifDlGMlSr6l3a6YmZmJ27dvw9XVlS9TU1ODq6srrl69WqxrpKamIisrq9Bmj4yMDCQlJcm88ioPyffy5csyibdjx474559/yu18GgxTEZUo+Q4ePBjq6urFemlolHzwXFxcHCQSCSwsLGTKLSws+BUXvmb69OmwsrKSSeB5+fr6wtDQkH/Z2NjI7Ff25Hvp0iV06dIFnz59AgB06tSJJV6GKYdKlCFdXV1Ru3bt0orlmy1ZsgQBAQEIDQ2FdiHjhGfOnAkfHx9+OykpSSYBK/Pw4osXL6Jr165ISUkBAHTu3BmHDh3K19OEYRjlV6Lk6+npiUGDBpVWLDA1NYW6unq+0XMxMTGoUqVKkef+9ddfWLJkCUJCQtC4ceNCjxOLxUW2jSprHrtw4QK6devGJ143NzcEBQWxxMsw5ZRSPXDT0tKCg4ODzMOynIdnrVq1KvS8P//8EwsXLkRwcDCaN2/+TTEoay7bvn07n3i7dOnCarwMU87JNatZafLx8YGnpyeaN28OR0dHrFy5EikpKRg+fDgAYOjQobC2toavry8A4I8//sCcOXOwe/du2NnZ8W3D+vr60NfXL/H9lTWfrVu3Dp8+fUJycjIOHDhQaLMKwzDlg9IlX3d3d8TGxmLOnDmIjo5G06ZNERwczD+Ee/36NdTUcivsa9euRWZmJn788UeZ68ydO7fQeYeLoqzJV0NDAzt37oREImFdyhhGBRR7eLGq+nJ4cVYWIEdHDYU7d+4cLC0tUbduXaFDYRhBVfjhxRWBurpyJN6QkBB069YNzs7OiIiIEDochmFKAUu+eShDk8Pp06fRs2dPpKenIzo6GsuWLRM6JIZhSgFLvnkInXxPnjzJJ14A6NOnD9asWSNsUAzDlAqWfPMQMvkGBwejd+/eyMjIAAD88MMPCAwMhJaWlnBBMQxTapSghVN5FJZ8iQjZ2dmQSCSlct8LFy5g4sSJ/ECSzp07Y9myZZBKpXwtmGEqqszMTNja2iIzM1PpPw85UysUZx4c1tshT2+HJk0McPeu7P7MzEy8e/cOqamppXL/tLQ0vH//nt/W1dWFqakpmxydYT6TSqWIioqCjY2NTDdTZaWrqwtLS8uv/tXKar55fFnzlUqliIyMhLq6OqysrKClpaXQpJiRkYFnz57B1NQUAGBgYICqVauWi/9gDFNWJBIJ0tLSYGdnB3V1daHDKRQRITMzE7GxsYiMjEStWrWK/Cyz5JvHl8k3MzMTUqkUNjY2pTJrmLa2NiwtLfHff//B2NgYNWrUYDVehvlCTnOftra2UidfgFvdXVNTE69evUJmZmaRI1FZ8s2jsDbf0qyJWllZQUdHB0ZGRizxMowKKG6+YH/f5lEWvR0KWu3Z2NiYJV6GqWBY8s2jtJNvfHw87t+/n2/1DIZhKh6WfPMozYnC4uPj8eLFC0ilUjx9+rTUek8wDFM+sDbfPEqr5hsfH4/nz5/z25UrV2Zz8TJMBcdqvnmURj78+PGjTOI1MTGBnZ0da+NVUh8+fIC5uTlevnwpdCiMEvLw8FDYfCss+eah6OT78eNHvHjxgt82NTUt08Q7bNgwiEQijB07Nt8+b29viEQiDBs2rExiKUpOnCKRCJqamqhevTqmTZuWbzRTVFQUvLy8+D7Xtra2mDRpEj58+JDvmtHR0ZgwYQJq1KgBsVgMGxsb9OzZU2aVlIIsWrQIvXv3hp2dXb59V69ehbq6Orp3755vX4cOHTB58uR85f7+/jAyMlJIbIrg5+cHOzs7aGtro0WLFrhx40ahx+b8X/3y5e3tzR9z4cIF9OzZE1ZWVhCJRDh06FCpvwegZO8jx9u3bzF48GCYmJhAR0cHjRo1wq1bt/j9a9euRePGjWFgYAADAwO0atUKJ06ckLnGr7/+ikWLFiExMfGb3wNLvnkoMvkWlHhtbW3LvMZrY2ODgIAApKWl8WXp6enYvXs3qlWrVqaxFKVLly549+4dXrx4gRUrVmD9+vWYO3cuv//Fixdo3rw5nj59ij179uDZs2dYt24dv8TUx48f+WNfvnwJBwcHnD17FkuXLsWDBw8QHBwMZ2dnmcTxpdTUVGzevBkjRowocP/mzZsxYcIEXLhwAf/9959c71Pe2BQhMDAQPj4+mDt3Lu7cuYMmTZrAzc1NZoRlXjdv3sS7d+/41+nTpwEA/fv3549JSUlBkyZN4OfnJ3dcHTp0gL+/f6m9D4Br+mvTpg00NTVx4sQJhIeHY9myZTA2NuaPqVq1KpYsWYLbt2/j1q1bcHFxQe/evfHo0SP+mIYNG8Le3h47d+6U673KoAouMTGRABCQSKtWye5LS0uj8PBwSktLK9E1P3z4QDdv3uRfkZGRJJVKFRh18Xh6elLv3r2pYcOGtHPnTr58165d1LhxY+rduzd5enoSEZFEIqHFixeTnZ0daWtrU+PGjWnfvn0y1ztx4gS1adOGDA0NqXLlytS9e3d69uyZzDFOTk40YcIE+uWXX8jY2JgsLCxo7ty5xYozr759+1KzZs347S5dulDVqlUpNTVV5rh3796Rrq4ujR07li/r2rUrWVtb06dPn/LdKz4+vtA49u3bR2ZmZgXuS05OJn19ffr333/J3d2dFi1aJLPfycmJJk2alO+8rVu3kqGh4TfHpgiOjo7k7e3Nb0skErKysiJfX99inT9p0iSyt7cv9P8yAAoKCipxXE5OTrR169ZC92dnZ9PNmzcpOzubiOR7H9OnT6e2bduWODZjY2PatGmTTNn8+fOLvFZx8war+eZR3N4OzZsDVasW/mrY0AjduzdG9+6N0bNnU7RtawsbG1GR5xTnJe/aoF5eXti6dSu/vWXLFn5NvBy+vr7Yvn071q1bh0ePHmHKlCkYPHgwzp8/zx+TkpICHx8f3Lp1C2fOnIGamhp++OEHSKVSmWtt27YNenp6uH79Ov78808sWLCArzUVx8OHD3HlyhV+bPzHjx9x8uRJjBs3Lt+DyipVquCnn35CYGAgiAgfP35EcHAwvL29oaenl+/aXzYB5HXx4kU4ODgUuG/v3r2oW7cu6tSpg8GDB2PLli2gEk6L8i2xAcDixYv5tQkLe71+/brAczMzM3H79m24urryZWpqanB1dcXVq1e/GntmZiZ27twJLy8vQZ9XyPs+jhw5gubNm6N///4wNzdHs2bNsHHjxkKPl0gkCAgIQEpKSr7Fex0dHXHjxg1+BkJ5sd4OeRS32SE6Gnj7tqgj1AAoz1SQgwcPxsyZM/Hq1SsAwOXLlxEQEIDQ0FAA3BwTixcvRkhICP8frUaNGrh06RLWr18PJycnAEC/fv1krrtlyxaYmZkhPDwcDRs25MsbN27MNxnUqlULa9aswZkzZ9CpU6dCYzx69Cj09fWRnZ2NjIwMqKmp8XMZP336FESEevXqFXhuvXr1EB8fj9jYWLx8+RJEJNfyS69evYKVlVWB+zZv3ozBgwcD4JpIEhMTcf78eXTo0KHY13/27JncsQHA2LFjMWDAgCKPKSz+uLg4SCQSfi3EHBYWFvj333+/eu9Dhw4hISFBIc8IFi9ejMWLF/PbaWlpuHbtGsaPH8+XhYeHF9gsJu/7ePHiBdauXQsfHx/MmjULN2/exMSJE6GlpQVPT0/+uAcPHqBVq1ZIT0+Hvr4+goKCUL9+fZlrWVlZITMzE9HR0bC1tS3x+8/Bkm8exU2+n2d+LHPy3tfMzAzdu3eHv78/iAjdu3fnJ/MBuKSQmpqaLzlmZmaiWbNm/PbTp08xZ84cXL9+HXFxcXyN9/Xr1/mSb16WlpZFtscBgLOzM9auXYuUlBSsWLECGhoa+ZJ9cWqaJa2N5pWWllbgWPyIiAjcuHEDQUFBALjFTN3d3bF58+YSJd9viQ3guihWrlz5m64hr82bN6Nr166FJveS+PKXyE8//YR+/fqhb9++fJki7pOXVCpF8+bN+aTfrFkzPHz4EOvWrZNJvnXq1MHdu3eRmJiI/fv3w9PTE+fPn5dJwDl/fX1rX32WfPMobvLN84AUAPhEZG5urvigFMTLy4uvWXz5cOTTp08AgGPHjsHa2lpmX96Vknv27AlbW1ts3LgRVlZWkEqlaNiwITIzM2XO0dTUlNkWiUT5mia+pKenh5o1awLgatRNmjThH37VrFkTIpEIjx8/xg8//JDv3MePH8PY2BhmZmb8XKrFqc19ydTUFPHx8fnKN2/ejOzsbJmEQEQQi8VYs2YNDA0NYWBgUOAT8ISEhM9TlnJ/BcgbG5C/xliQwmqMpqamUFdXR0xMjEx5TEwMP490YV69eoWQkBAcPHiw5EEX4MtfIjo6OjA3N+d//kWR931YWlrmq8HWq1cPBw4ckCnT0tLi43BwcMDNmzexatUqrF+/nj8m5+GumZnZV+MtCmvzzUOe3g5xcXF4+fIlXr9+/dXanZC6dOmCzMxMZGVlwc3NTWZf/fr1IRaL8fr1a9SsWVPmZWNjA4Dr/xoREYFff/0VHTt25P/ULw1qamqYNWsWfv31V6SlpcHExASdOnXC//73P5leGwDXbWvXrl1wd3eHSCRC5cqV4ebmBj8/P6SkpOS7dkJCQqH3bdasGcLDw2XKsrOzsX37dixbtgx3797lX/fu3YOVlRX27NkDgKsx3blzJ98179y5g9q1awPAN8UGcDXGvDEU9CqsxqilpQUHBweZ7mxSqZTvLVKUrVu3wtzcvMAudmVN3vfRpk2bfIvRPnny5KvNBlKpNF/b7sOHD1G1alWZvx7lUuLHfyomb2+Hy5dl933tqWVsbKxMr4ZXr16VQcTF92UvgsTEREpMTOS38/Z2mD17NpmYmJC/vz89e/aMbt++TX///Tf5+/sTEfdE2cTEhAYPHkxPnz6lM2fO0Pfff5/vCXdBT/3z3qc4cRIRZWVlkbW1NS1dupSIiJ48eUKmpqbUrl07On/+PL1+/ZpOnDhBDRs2pFq1atGHDx/4c58/f05VqlSh+vXr0/79++nJkycUHh5Oq1atorp16xYax/3790lDQ4M+fvzIlwUFBZGWlhYlJCTkO37atGnUvHlz/p7a2to0YcIEunfvHv3777+0bNky0tDQoBMnTnxzbIoQEBBAYrGY/P39KTw8nEaPHk1GRkYUHR1NRESrV68mFxcXmXMkEglVq1aNpk+fXuA1k5OTKSwsjMLCwggALV++nMLCwor8LCQnJ9O7d++KfOX0bCDK39vha++joPdy48YN0tDQoEWLFtHTp09p165dpKurK9MLaMaMGXT+/HmKjIyk+/fv04wZM0gkEtGpU6dk4vf09CQvL69C319xezuw5Jsn+d65I7uvqG9iQYlXiO5kRSkoqeWVNylKpVJauXIl1alThzQ1NcnMzIzc3Nzo/Pnz/PGnT5+mevXqkVgspsaNG1NoaGipJV8iIl9fXzIzM+O7Zb18+ZI8PT3JwsKCNDU1ycbGhiZMmEBxcXH5zv3vv//I29ubbG1tSUtLi6ytralXr1507ty5QuMg4roxrVu3jt/u0aMHdevWrcBjr1+/TgDo3r17RMR9wDt16kRmZmZkaGhILVq0KLDrlbyxKcLq1aupWrVqpKWlRY6OjnTt2jV+39y5c8nW1lbm+JMnTxIAioiIKPB6586d+/z5kX0V9fOeO3dugefkfUVGRvLHf5l8v/Y+Cnsv//zzDzVs2JDEYjHVrVuXNmzYILPfy8uL/5mYmZlRx44d8yXetLQ0MjQ0pKtXrxb6/oqbfNkyQnmWEXr82AB5H0Snp6cjMjIS1atXl3kQExsby/ccALgnrVWrVmVDhlXAsWPH8Msvv+Dhw4dsRRElIZFIEBYWhmbNmgk+mfratWsRFBSEU6dOFXpMYXnjS+yBWx7FafNliVe1de/eHU+fPsXbt2/59m6GyaGpqYnVq1cr5Fos+ebxteTLEm/FUNAcDQwDACNHjlTYtVjyzaOo5JudnY23eUZWVKlSBdbW1izxMgwjF9aolUdRw4s1NDRQq1YtqKurs8TLMMw3YzXfz9TUgC/GBvBynknq6emhQYMG0NTUZImXYZgCFbcPA6v5flZQk8PVq1dBRDLDCLW0tFjiZRimUDn54suRnl9iNd/PvmxyWLVqFSZPnoz169ejXbt2AABdXV2WeBmmjEkkEgBcFy6hu5oVJaei9v79exgZGX01VtbP93M/36pVExEVZQAAWLFiBXx8fABw8xKcPXs23yxKDMOUDalUiqioKNjY2JSLvtdGRkaoUqXKVytqrOb7WU7Nd/ny5fj555/58t9++w1OTk6QSqXIysoSKDqGqbg+ffqE7t2749atW9DX1xc6nCJpamoWu3bOku9n2trAsmXLMHXqVL5s3rx5/Ly06urqSv0nD8OoqszMTLx69QpaWlpFjhgrb5SyDl/SxfH27duHunXrQltbG40aNcLx48dLfM+EhHcyiXf+/Pkya4gxDMMoktIl35IujnflyhUMHDgQI0aMQFhYGPr06YM+ffrg4cOHJbrvmzdP+K8XLFiAOXPmfNP7YBiGKYrSPXBr0aIFvv/+e34JGalUChsbG0yYMAEzZszId7y7uztSUlJw9OhRvqxly5Zo2rQp1q1b99X75U6ssxfAAPz++++YPXu2ot4OwzDfKOczmpiYCAMDA6HDURilavPNWRxv5syZfNnXFse7evUq3zMhh5ubGw4dOlTg8RkZGTKTI+euPvARc+bMwYQJE5CUlPRN74NhGMXJ+TwqWT3xmylV8pVncbzo6OgCj4+Oji7weF9fX8yfP7+APWOxYAHX5MAwjPL58OEDvySTKlCq5FsWZs6cKVNTTkhIgK2tLV6/fl3ufrBJSUmwsbFBVFRUufpzrLzGDZTf2Mtr3AD312m1atUEWzy0tChV8pVncbwqVaqU6HixWCyzKGSOnEUQyyMDA4NyGXt5jRsov7GX17gBlIsBFiWhVO9GnsXxWrVqJXM8AJw+ffqriwIyDMMISalqvgDg4+MDT09PNG/eHI6Ojli5ciVSUlIwfPhwAMDQoUNhbW0NX19fAMCkSZPg5OSEZcuWoXv37ggICMCtW7ewYcMGId8GwzBMkZQu+bq7uyM2NhZz5sxBdHQ0mjZtiuDgYP6h2uvXr2X+/GjdujV2796NX3/9FbNmzUKtWrVw6NAhNGzYsFj3E4vFmDt3boFNEcquvMZeXuMGym/s5TVuoHzHXhSl6+fLMAxTEShVmy/DMExFwZIvwzCMAFjyZRiGEQBLvgzDMAKoEMlXiCkqFaUksW/cuBHt2rWDsbExjI2N4erq+tX3WlpK+j3PERAQAJFIhD59+pRugEUoaewJCQnw9vaGpaUlxGIxateuLcj/mZLGvXLlStSpUwc6OjqwsbHBlClTkJ6eXkbRci5cuICePXvCysoKIpGo0DlZ8goNDcV3330HsViMmjVrwt/fv9TjLBWk4gICAkhLS4u2bNlCjx49olGjRpGRkRHFxMQUePzly5dJXV2d/vzzTwoPD6dff/2VNDU16cGDB2UcecljHzRoEPn5+VFYWBg9fvyYhg0bRoaGhvTmzRuljjtHZGQkWVtbU7t27ah3795lE+wXShp7RkYGNW/enLp160aXLl2iyMhICg0Npbt37yp13Lt27SKxWEy7du2iyMhIOnnyJFlaWtKUKVPKNO7jx4/T7Nmz6eDBgwSAgoKCijz+xYsXpKurSz4+PhQeHk6rV68mdXV1Cg4OLpuAFUjlk6+joyN5e3vz2xKJhKysrMjX17fA4wcMGEDdu3eXKWvRogWNGTOmVOMsSElj/1J2djZVqlSJtm3bVlohFkieuLOzs6l169a0adMm8vT0FCz5ljT2tWvXUo0aNSgzM7OsQixQSeP29vYmFxcXmTIfHx9q06ZNqcZZlOIk32nTplGDBg1kytzd3cnNza0UIysdKt3skDNFpaurK19WnCkq8x4PcFNUFnZ8aZEn9i+lpqYiKyurTCckkTfuBQsWwNzcHCNGjCiLMAskT+xHjhxBq1at4O3tDQsLCzRs2BCLFy/mV9wtC/LE3bp1a9y+fZtvmnjx4gWOHz+Obt26lUnM8lKWz6ciKN0IN0UqiykqS4s8sX9p+vTpsLKyyveftTTJE/elS5ewefNm3L17twwiLJw8sb948QJnz57FTz/9hOPHj+PZs2cYN24csrKyymwZKnniHjRoEOLi4tC2bVsQEbKzszF27FjMmjWrLEKWW2Gfz6SkJKSlpUFHR0egyEpOpWu+FdmSJUsQEBCAoKAgpV50MDk5GUOGDMHGjRthamoqdDglJpVKYW5ujg0bNsDBwQHu7u6YPXt2sVZREVJoaCgWL16M//3vf7hz5w4OHjyIY8eOYeHChUKHVmGodM23LKaoLC3yxJ7jr7/+wpIlSxASEoLGjRuXZpj5lDTu58+f4+XLl+jZsydfJpVKAQAaGhqIiIiAvb196Qb9mTzfc0tLy3zLhderVw/R0dHIzMyElpZWqcYMyBf3b7/9hiFDhmDkyJEAgEaNGiElJQWjR4/G7NmzlXb6xsI+nwYGBuWq1guoeM23PE9RKU/sAPDnn39i4cKFCA4ORvPmzcsiVBkljbtu3bp48OAB7t69y7969eoFZ2dn3L17FzY2NkobOwC0adMGz549439hAMCTJ09gaWlZJokXkC/u1NTUfAk25xcIKfF0L8ry+VQIoZ/4lbaAgAASi8Xk7+9P4eHhNHr0aDIyMqLo6GgiIhoyZAjNmDGDP/7y5cukoaFBf/31Fz1+/Jjmzp0raFezksS+ZMkS0tLSov3799O7d+/4V3JyslLH/SUhezuUNPbXr19TpUqVaPz48RQREUFHjx4lc3Nz+v3335U67rlz51KlSpVoz5499OLFCzp16hTZ29vTgAEDyjTu5ORkCgsLo7CwMAJAy5cvp7CwMHr16hUREc2YMYOGDBnCH5/T1eyXX36hx48fk5+fH+tqpsxWr15N1apVIy0tLXJ0dKRr167x+5ycnMjT01Pm+L1791Lt2rVJS0uLGjRoQMeOHSvjiHOVJHZbW1sCkO81d+5cpY77S0ImX6KSx37lyhVq0aIFicViqlGjBi1atIiys7PLOOqSxZ2VlUXz5s0je3t70tbWJhsbGxo3bhzFx8eXacznzp0r8P9sTqyenp7k5OSU75ymTZuSlpYW1ahRg7Zu3VqmMSsKm1KSYRhGACrd5sswDKOsWPJlGIYRAEu+DMMwAmDJl2EYRgAs+TIMwwiAJV+GYRgBsOTLMAwjAJZ8GYZhBMCSbwUSGhoKkUiE0NBQoUMpVSKRCPPmzSvWsXZ2dhg2bFipxsMwBWHJtxzw9/eHSCQq8DVjxgyhwyvSl7Fra2ujdu3aGD9+fL7ZqUrLlStXMG/ePCQkJJTJ/YrDzs5O5vuip6cHR0dHbN++Xe5rHj9+vNi/dBjhqfSUkqpmwYIFqF69ukxZw4YNBYqmZHJiT09Px6VLl7B27VocP34cDx8+hK6urkLvlZaWBg2N3P/aV65cwfz58zFs2DAYGRnJHBsRESHY9IlNmzbFzz//DAB49+4dNm3aBE9PT2RkZGDUqFElvt7x48fh5+fHEnA5wZJvOdK1a1dBpolUhLyxjxw5EiYmJli+fDkOHz6MgQMHKvReJZk8XiwWK/TeJWFtbY3Bgwfz28OGDUONGjWwYsUKuZIvU76wZgcV8OrVK4wbN45fBtzExAT9+/fHy5cvv3ru06dP0a9fP1SpUgXa2tqoWrUqPDw8kJiYKHPczp074eDgAB0dHVSuXBkeHh6IioqSO2YXFxcAQGRkJAAgOzsbCxcuhL29PcRiMezs7DBr1ixkZGTInHfr1i24ubnB1NQUOjo6qF69Ory8vGSOydvmO2/ePPzyyy8AgOrVq/N/5ud8b/K2+d66dQsikQjbtm3LF+/JkychEolw9OhRvuzt27fw8vKChYUFxGIxGjRogC1btsj9PTEzM0PdunXx/PlzmfKLFy+if//+qFatGsRiMb/Me1paGn/MsGHD4Ofnx7//nFcOqVSKlStXokGDBtDW1oaFhQXGjBmD+Ph4ueNlvg2r+ZYjiYmJiIuLkykzNTXFzZs3ceXKFXh4eKBq1ap4+fIl1q5diw4dOiA8PLzQP+szMzPh5uaGjIwMTJgwAVWqVMHbt29x9OhRJCQkwNDQEACwaNEi/PbbbxgwYABGjhyJ2NhYrF69Gu3bt0dYWFi+P+WLIyfBmJiYAOBqw9u2bcOPP/6In3/+GdevX4evry8eP36MoKAgAMD79+/RuXNnmJmZYcaMGTAyMsLLly9x8ODBQu/Tt29fPHnyBHv27MGKFSv4pYrMzMzyHdu8eXPUqFEDe/fuhaenp8y+wMBAGBsbw83NDQC3ekLLli0hEokwfvx4mJmZ4cSJExgxYgSSkpIwefLkEn9PsrOz8ebNGxgbG8uU79u3D6mpqfi///s/mJiY4MaNG1i9ejXevHmDffv2AQDGjBmD//77D6dPn8aOHTvyXXvMmDHw9/fH8OHDMXHiRERGRmLNmjUICwvD5cuXoampWeJ4mW8k9JyWzNdt3bq1wDlPc358qamp+c65evUqAaDt27fzZTlzp547d46IiJ/Aet++fYXe++XLl6Surk6LFi2SKX/w4AFpaGjkKy8s9pCQEIqNjaWoqCgKCAggExMT0tHRoTdv3tDdu3cJAI0cOVLm3KlTpxIAOnv2LBERBQUFEQC6efNmkffEF3MYL126lABQZGRkvmNtbW1l5rmdOXMmaWpq0sePH/myjIwMMjIyIi8vL75sxIgRZGlpSXFxcTLX8/DwIENDwwJ/Jl/et3PnzhQbG0uxsbH04MEDGjJkCAGQWQKeqOCfr6+vL4lEIn7ScSJuOfiCPtIXL14kALRr1y6Z8uDg4ALLmbLBmh3KET8/P5w+fVrmBUBm7aqsrCx8+PABNWvWhJGREe7cuVPo9XJqtidPnkRqamqBxxw8eBBSqRQDBgxAXFwc/6pSpQpq1aqFc+fOFSt2V1dXmJmZwcbGBh4eHtDX10dQUBCsra1x/PhxAICPj4/MOTkPo44dOwYAfA376NGjyMrKKtZ9S8rd3R1ZWVkytelTp04hISEB7u7uALhldg4cOICePXuCiGS+L25ubkhMTCzy+573umZmZjAzM0OjRo2wY8cODB8+HEuXLpU5Lu/PNyUlBXFxcWjdujWICGFhYV+9z759+2BoaIhOnTrJxOrg4AB9ff1i/wwZxWLNDuWIo6NjgQ/c0tLS4Ovri61bt+Lt27cya3B92XabV/Xq1eHj44Ply5dj165daNeuHXr16oXBgwfzifnp06cgItSqVavAaxT3z1U/Pz/Url0bGhoasLCwQJ06dfheBq9evYKamhpq1qwpc06VKlVgZGSEV69eAQCcnJzQr18/zJ8/HytWrECHDh3Qp08fDBo0SGEPzpo0aYK6desiMDAQI0aMAMA1OZiamvLt1LGxsUhISMCGDRuwYcOGAq/z/v37r96rRYsW+P333yGRSPDw4UP8/vvviI+Pz7f22+vXrzFnzhwcOXIkXxttUT/fHE+fPkViYiLMzc3ljpVRPJZ8VcCECROwdetWTJ48Ga1atYKhoSFEIhE8PDxkFnYsyLJlyzBs2DAcPnwYp06dwsSJE+Hr64tr166hatWqkEqlEIlEOHHihMwKvTn09fWLFWNhvzjyyvuAqLD9+/fvx7Vr1/DPP//g5MmT8PLywrJly3Dt2rVix/I17u7uWLRoEeLi4lCpUiUcOXIEAwcO5Luv5XxPBw8enK9tOEdxVo02NTWFq6srAMDNzQ1169ZFjx49sGrVKv6vAIlEgk6dOuHjx4+YPn066tatCz09Pbx9+xbDhg376s83J15zc3Ps2rWrwP0FtX8zpY8lXxWwf/9+eHp6YtmyZXxZenp6sQcVNGrUCI0aNcKvv/6KK1euoE2bNli3bh1+//132Nvbg4hQvXp11K5du1Tit7W1hVQqxdOnT1GvXj2+PCYmBgkJCbC1tZU5vmXLlmjZsiUWLVqE3bt346effkJAQAC/DPqXvpbUv+Tu7o758+fjwIEDsLCwQFJSEjw8PPj9ZmZmqFSpEiQSCZ88FaF79+5wcnLC4sWLMWbMGOjp6eHBgwd48uQJtm3bhqFDh/LH5jQ55VXY+7S3t0dISAjatGlT7pZXV2WszVcFqKur51vue/Xq1ZBIJEWel5SUhOzsbJmyRo0aQU1Nje/i1bdvX6irq2P+/Pn57kFE+PDhwzfH361bNwDAypUrZcqXL18OgEtKABAfH58vhqZNmwJAvi5peenp6QFAsX8Z1atXD40aNUJgYCACAwNhaWmJ9u3b8/vV1dXRr18/HDhwAA8fPsx3fmxsbLHuU5Dp06fjw4cP2LhxI38vQHY5dyLCqlWr8p1b2PscMGAAJBIJFi5cmO+c7OxspRr5V5Gwmq8K6NGjB3bs2AFDQ0PUr18fV69eRUhICN+NqzBnz57F+PHj0b9/f9SuXRvZ2dnYsWMHn1wArtb0+++/Y+bMmXj58iX69OmDSpUqITIyEkFBQRg9ejSmTp36TfE3adIEnp6e2LBhAxISEuDk5IQbN25g27Zt6NOnD5ydnQEA27Ztw//+9z/88MMPsLe3R3JyMjZu3AgDAwM+gRfEwcEBADB79mx4eHhAU1MTPXv25JNVQdzd3TFnzhxoa2tjxIgR+UbBLVmyBOfOnUOLFi0watQo1K9fHx8/fsSdO3cQEhKCjx8/yvW96Nq1Kxo2bIjly5fD29sbdevWhb29PaZOnYq3b9/CwMAABw4cKLB/bs77nDhxItzc3KCurg4PDw84OTlhzJgx8PX1xd27d9G5c2doamri6dOn2LdvH1atWoUff/xRrniZbyBMJwumJHK6axXWxSo+Pp6GDx9OpqampK+vT25ubvTvv//m60b1ZVezFy9ekJeXF798eOXKlcnZ2ZlCQkLy3ePAgQPUtm1b0tPTIz09Papbty55e3tTRETEN8WeIysri+bPn0/Vq1cnTU1NsrGxoZkzZ1J6ejp/zJ07d2jgwIFUrVo1EovFZG5uTj169KBbt27JXAtfdDUjIlq4cCFZW1uTmpqaTLezL79HOZ4+fcp357t06VKBMcfExJC3tzfZ2NiQpqYmValShTp27EgbNmwo8r3m3Ld79+4F7vP39ycA/JLo4eHh5OrqSvr6+mRqakqjRo2ie/fuyRxDRJSdnU0TJkwgMzMzEolE+bqdbdiwgRwcHEhHR4cqVapEjRo1omnTptF///331XgZxWNLxzMMwwiAtfkyDMMIgCVfhmEYAbDkyzAMIwCWfBmGYQTAki/DMIwAWPJlGIYRAEu+DMMwAmDJl2EYRgAs+TIMwwiAJV+GYRgBsOTLMAwjAJZ8GYZhBPD/yWh28bI2FPsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# testing\n",
    "\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_testListFlat, y_probListFlat) # y_prob[:,1]\n",
    "#fpr, tpr, thresholds = roc_curve(y_test, y_prob) # way that usualy works\n",
    "\n",
    "# confidence intervals\n",
    "\n",
    "n_bootstraps = 1000\n",
    "rng_seed = 1  # control reproducibility\n",
    "bootstrapped_scores = []\n",
    "\n",
    "rng = np.random.RandomState(rng_seed)\n",
    "for i in range(n_bootstraps):\n",
    "    # bootstrap by sampling with replacement on the prediction indices\n",
    "    indices = rng.randint(0, len(y_predListFlat), len(y_predListFlat))\n",
    "    if len(np.unique(y_testListFlat)) < 2:\n",
    "        # We need at least one positive and one negative sample for ROC AUC\n",
    "        # to be defined: reject the sample\n",
    "        continue\n",
    "\n",
    "    score = roc_auc_score(y_testListFlat, y_probListFlat)\n",
    "    bootstrapped_scores.append(score)\n",
    "    print(\"Bootstrap #{} ROC area: {:0.3f}\".format(i + 1, score))\n",
    "\n",
    "# To get a confidence interval one can sort the samples:\n",
    "sorted_scores = np.array(bootstrapped_scores)\n",
    "sorted_scores.sort()\n",
    "\n",
    "# Computing the lower and upper bound of the 90% confidence interval\n",
    "# You can change the bounds percentiles to 0.025 and 0.975 to get\n",
    "# a 95% confidence interval instead.\n",
    "confidence_lower = sorted_scores[int(0.05 * len(sorted_scores))]\n",
    "confidence_upper = sorted_scores[int(0.95 * len(sorted_scores))]\n",
    "print(\"Confidence interval for the score: [{:0.3f} - {:0.3}]\".format(\n",
    "    confidence_lower, confidence_upper))\n",
    "\n",
    "#set up plotting area\n",
    "plt.figure(figsize=(3, 3))\n",
    "ax = plt.axes() # enables overlay\n",
    "\n",
    "plt.plot([0, 1], [0, 1],'--', color = 'black', lw = 2)\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "\n",
    "ax.plot(mean_fpr, mean_tpr, color='b',\n",
    "        label=r'Mean ROC (AUC = %0.2f $\\pm$ %0.2f)' % (mean_auc, confidence_lower),\n",
    "        lw=2, alpha=1)\n",
    "\n",
    "#std_tpr = np.std(tprs, axis=0)\n",
    "#tprs_upper = np.minimum(mean_tpr + std_tpr, 1)\n",
    "#tprs_lower = np.maximum(mean_tpr - std_tpr, 0)\n",
    "#ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.3,\n",
    "#                label=r'$\\pm$ 1 std. dev.')\n",
    "\n",
    "#ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],\n",
    "#       title=\"ROC curve\")\n",
    "\n",
    "\n",
    "#plt.figure(figsize=(4, 4))\n",
    "#plt.scatter(fpr, tpr, s=100, alpha=0.5, color=\"blue\", label=\"Scikit-learn\")\n",
    "plt.title(\"ROC Curve\", fontsize=12)\n",
    "plt.xlabel(\"False Positive Rate\", fontsize=12)\n",
    "plt.ylabel(\"True Positive Rate\", fontsize=12)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 947,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "object of too small depth for desired array",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[947], line 16\u001b[0m\n\u001b[1;32m     14\u001b[0m fpr           \u001b[38;5;241m=\u001b[39m fpr[i]\n\u001b[1;32m     15\u001b[0m tpr           \u001b[38;5;241m=\u001b[39m tpr[i]\n\u001b[0;32m---> 16\u001b[0m interp_tpr    \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minterp\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfpr_mean\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfpr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtpr\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     17\u001b[0m interp_tpr[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n\u001b[1;32m     18\u001b[0m interp_tprs\u001b[38;5;241m.\u001b[39mappend(interp_tpr)\n",
      "File \u001b[0;32m<__array_function__ internals>:180\u001b[0m, in \u001b[0;36minterp\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "File \u001b[0;32m~/Python/env_221209_3-10/lib/python3.10/site-packages/numpy/lib/function_base.py:1594\u001b[0m, in \u001b[0;36minterp\u001b[0;34m(x, xp, fp, left, right, period)\u001b[0m\n\u001b[1;32m   1591\u001b[0m     xp \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate((xp[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:]\u001b[38;5;241m-\u001b[39mperiod, xp, xp[\u001b[38;5;241m0\u001b[39m:\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m+\u001b[39mperiod))\n\u001b[1;32m   1592\u001b[0m     fp \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate((fp[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:], fp, fp[\u001b[38;5;241m0\u001b[39m:\u001b[38;5;241m1\u001b[39m]))\n\u001b[0;32m-> 1594\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minterp_func\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mxp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mleft\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mright\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mValueError\u001b[0m: object of too small depth for desired array"
     ]
    }
   ],
   "source": [
    " #testing\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_testListFlat, y_probListFlat) # y_prob[:,1]\n",
    "\n",
    "c_fill      = 'rgba(52, 152, 219, 0.2)'\n",
    "c_line      = 'rgba(52, 152, 219, 0.5)'\n",
    "c_line_main = 'rgba(41, 128, 185, 1.0)'\n",
    "c_grid      = 'rgba(189, 195, 199, 0.5)'\n",
    "c_annot     = 'rgba(149, 165, 166, 0.5)'\n",
    "c_highlight = 'rgba(192, 57, 43, 1.0)'\n",
    "fpr_mean    = np.linspace(0, 1, 100)\n",
    "interp_tprs = []\n",
    "for i in range(100):\n",
    "    fpr           = fpr[i]\n",
    "    tpr           = tpr[i]\n",
    "    interp_tpr    = np.interp(fpr_mean, fpr, tpr)\n",
    "    interp_tpr[0] = 0.0\n",
    "    interp_tprs.append(interp_tpr)\n",
    "tpr_mean     = np.mean(interp_tprs, axis=0)\n",
    "tpr_mean[-1] = 1.0\n",
    "tpr_std      = 2*np.std(interp_tprs, axis=0)\n",
    "tpr_upper    = np.clip(tpr_mean+tpr_std, 0, 1)\n",
    "tpr_lower    = tpr_mean-tpr_std\n",
    "auc          = np.mean(results[kind]['auc'])\n",
    "fig = go.Figure([\n",
    "    go.Scatter(\n",
    "        x          = fpr_mean,\n",
    "        y          = tpr_upper,\n",
    "        line       = dict(color=c_line, width=1),\n",
    "        hoverinfo  = \"skip\",\n",
    "        showlegend = False,\n",
    "        name       = 'upper'),\n",
    "    go.Scatter(\n",
    "        x          = fpr_mean,\n",
    "        y          = tpr_lower,\n",
    "        fill       = 'tonexty',\n",
    "        fillcolor  = c_fill,\n",
    "        line       = dict(color=c_line, width=1),\n",
    "        hoverinfo  = \"skip\",\n",
    "        showlegend = False,\n",
    "        name       = 'lower'),\n",
    "    go.Scatter(\n",
    "        x          = fpr_mean,\n",
    "        y          = tpr_mean,\n",
    "        line       = dict(color=c_line_main, width=2),\n",
    "        hoverinfo  = \"skip\",\n",
    "        showlegend = True,\n",
    "        name       = f'AUC: {auc:.3f}')\n",
    "])\n",
    "fig.add_shape(\n",
    "    type ='line', \n",
    "    line =dict(dash='dash'),\n",
    "    x0=0, x1=1, y0=0, y1=1\n",
    ")\n",
    "fig.update_layout(\n",
    "    template    = 'plotly_white', \n",
    "    title_x     = 0.5,\n",
    "    xaxis_title = \"1 - Specificity\",\n",
    "    yaxis_title = \"Sensitivity\",\n",
    "    width       = 800,\n",
    "    height      = 800,\n",
    "    legend      = dict(\n",
    "        yanchor=\"bottom\", \n",
    "        xanchor=\"right\", \n",
    "        x=0.95,\n",
    "        y=0.01,\n",
    "    )\n",
    ")\n",
    "fig.update_yaxes(\n",
    "    range       = [0, 1],\n",
    "    gridcolor   = c_grid,\n",
    "    scaleanchor = \"x\", \n",
    "    scaleratio  = 1,\n",
    "    linecolor   = 'black')\n",
    "fig.update_xaxes(\n",
    "    range       = [0, 1],\n",
    "    gridcolor   = c_grid,\n",
    "    constrain   = 'domain',\n",
    "    linecolor   = 'black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# non-customizable ROC plot using scikitplot: based on https://scikit-plot.readthedocs.io/en/stable/metrics.html\n",
    "# X_test and y_test are going to be the final ones from the train_test split, just FYI\n",
    "\n",
    "# plot the whole combined list of values and predictions\n",
    "plt_ = skplt.metrics.plot_roc(y_testListFlat, y_predProbAFormattedLikeSKLearn, # way that usually works\n",
    "#                             figsize=(3, 3),\n",
    "                             #plot_micro = False,\n",
    "                             #plot_macro = False,\n",
    "#                             cmap='Dark2', #Dark2, tab10\n",
    "                            ) \n",
    "\n",
    "#plt_.legend(bbox_to_anchor=(2.45, .55), loc='center right', borderaxespad=0)\n",
    "#plt_.figure.savefig('ROC_overlaid_scikitplot_TestSizes' + str(testSizes) + '.pdf',format='pdf', dpi=500, bbox_inches='tight')\n",
    "#plt_.figure.savefig('ROC_overlaid_scikitplot_TestSizes' + str(testSizes) + '.svg',format='svg', dpi=500, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(3, 3))\n",
    "ax = fig.add_subplot()\n",
    "\n",
    "# predict on the test data using the single-objective Auto-sklearn & plot perf\n",
    "single_objective_precision = sklearn.metrics.precision_score(y_test, y_pred)\n",
    "single_objective_recall = sklearn.metrics.recall_score(y_test, y_pred)\n",
    "\n",
    "ax.scatter(single_objective_precision, single_objective_recall, label= \"Pipeline\")\n",
    "\n",
    "# Label the plot\n",
    "_ = ax.set_xlabel(\"Precision / Positive Predictive Value\")\n",
    "_ = ax.set_ylabel(\"True Positive Rate / Recall\")\n",
    "_ = ax.set_xlim((0, 1))\n",
    "_ = ax.set_ylim((0, 1))\n",
    "_ = ax.legend(bbox_to_anchor=(1.75, .95), loc='center right', borderaxespad=0)\n",
    "_ = ax.set_title(\"Test performances of all models found by Auto-sklearn2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### save final lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 949,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save final classification lists to disk\n",
    "import pickle\n",
    "\n",
    "# save y_predList\n",
    "fileName = 'y_predList_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl'\n",
    "\n",
    "with open(fileName, \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(y_predList, fp)\n",
    "\n",
    "# reload y_predList\n",
    "with open(fileName, \"rb\") as fp:   # Unpickling\n",
    "    y_predList = pickle.load(fp)\n",
    "\n",
    "\n",
    "\n",
    "# save y_probList\n",
    "fileName = 'y_probList_target-' + target + '_quantType_' + quantType + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl'\n",
    "\n",
    "with open(fileName, \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(y_probList, fp)\n",
    "\n",
    "# reload y_probList\n",
    "with open(fileName, \"rb\") as fp:   # Unpickling\n",
    "    y_probList = pickle.load(fp)\n",
    "\n",
    "\n",
    "\n",
    "# save y_testList\n",
    "fileName = 'y_testList_target-' + target + '_quantType_' + quantType + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl'\n",
    "\n",
    "with open(fileName, \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(y_testList, fp)\n",
    "\n",
    "# reload y_probList\n",
    "with open(fileName, \"rb\") as fp:   # Unpickling\n",
    "    y_testList = pickle.load(fp)\n",
    "\n",
    "\n",
    "# save finalImportancesMeanList\n",
    "fileName = 'finalImportancesMeanList_target-' + target + '_quantType_' + quantType + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl'\n",
    "\n",
    "with open(fileName, \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(finalImportancesMeanList, fp)\n",
    "\n",
    "# reload finalImportancesMeanList\n",
    "with open(fileName, \"rb\") as fp:   # Unpickling\n",
    "    finalImportancesMeanList = pickle.load(fp)\n",
    "    \n",
    "\n",
    "# save finalFeaturesList\n",
    "fileName = 'finalFeaturesList_target-' + target + '_quantType_' + quantType + '_nSampleFilter' + str(nSamplesVal) + '_trainFrac-' + str(trainFrac) + '_seed-' + str(seed) + '.pkl'\n",
    "\n",
    "with open(fileName, \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(finalFeaturesList, fp)\n",
    "\n",
    "# reload finalFeaturesList\n",
    "with open(fileName, \"rb\") as fp:   # Unpickling\n",
    "    finalFeaturesList = pickle.load(fp)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}