notebooks/models_postprocessing/3_DMPNN_StructGNN_models_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import torch\n",
    "from scipy import stats\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from rdkit import Chem, DataStructs\n",
    "from rdkit.Chem import AllChem\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "import time\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from scipy.stats import shapiro\n",
    "from scipy.stats import normaltest\n",
    "from scipy.stats import ttest_ind\n",
    "from scipy.stats import mannwhitneyu\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_statistics(vals):\n",
    "    values = vals.copy()\n",
    "    mols = [Chem.MolFromSmiles(x) for x in values[SMILES_COLUMN]]\n",
    "    \n",
    "    atom_count = [mol.GetNumAtoms() for mol in mols]\n",
    "    values['atom_count'] = pd.Series(atom_count)\n",
    "    \n",
    "    values['ring_count'] = pd.Series([mol.GetRingInfo().NumRings() for mol in mols])\n",
    "    \n",
    "    mol_atoms = [mol.GetAtoms() for mol in mols]\n",
    "    non_simple_atom_count = [sum((num.GetAtomicNum() not in [1, 6, 8, 7]) for num in m_atoms) for m_atoms in mol_atoms]\n",
    "    values['non_organogens_count'] = pd.Series(non_simple_atom_count)\n",
    "    \n",
    "    values['non_organogens_percent'] = values.non_organogens_count / values.atom_count *100\n",
    "    \n",
    "    benzene_smiles = 'C1=CC=CC=C1'\n",
    "    pattern = Chem.MolFromSmiles(benzene_smiles)\n",
    "    values['aromatic_count'] = pd.Series([len(mol.GetSubstructMatches(pattern)) for mol in mols])\n",
    "    \n",
    "    values['aromatic_rings_percent'] = values.aromatic_count / values.ring_count *100\n",
    "    values['aromatic_rings_percent'] = values['aromatic_rings_percent'].fillna(0)\n",
    "    \n",
    "    values['bond_count'] =  [len(mol.GetBonds()) for mol in mols]\n",
    "    values['aromatic_bond_count'] = [sum(bond.GetIsAromatic() for bond in mol.GetBonds()) for mol in mols]\n",
    "    values['aromatic_bond_percent'] = values.aromatic_bond_count / values.bond_count * 100\n",
    "    \n",
    "    values_list = [pd.Series(Chem.CanonicalRankAtoms(mol, breakTies=False)).value_counts() for mol in mols]\n",
    "    values['is_symmetric'] =  [(len(values) - 1 <= len(values[values % 2 == 0])) or (len(values) - 1 <= len(values[(values % 2 == 1) & (values > 1)])) for values in values_list]\n",
    "    \n",
    "    return values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "FILENAME = \"test_predictions.csv\"\n",
    "\n",
    "SMILES_COLUMN = 'smiles'\n",
    "ERROR_COLUMN = 'Squared Error'\n",
    "\n",
    "FOLDS= 5\n",
    "\n",
    "N_VALUES = 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate best and worst molecules statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "NUM_EXPS = [364, 365, 351, 352, 359, 368]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [13:44<00:00, 137.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364\n",
      "DMPNN with rdkit 2d normalized wo MolLogP\n",
      "logP\n",
      "None\n",
      "\n",
      "logP_symmetric test RMSE  =  0.625 +/- 0.059\n",
      "logP_symmetric test R2  =  0.928 +/- 0.013\n",
      "logP_non-symmetric test RMSE  =  0.442 +/- 0.006\n",
      "logP_non-symmetric test R2  =  0.942 +/- 0.002\n",
      "logP_rmse  =  0.452 +/- 0.007\n",
      "logP_r2  =  0.941 +/- 0.001\n",
      "logP_atom_count  =  0.678 +/- 0.216\n",
      "logP_ring_count  =  0.562 +/- 0.233\n",
      "logP_non_organogens_count  =  0.458 +/- 0.278\n",
      "logP_non_organogens_percent  =  0.506 +/- 0.289\n",
      "logP_aromatic_count  =  0.489 +/- 0.325\n",
      "logP_aromatic_rings_percent  =  0.545 +/- 0.359\n",
      "logP_bond_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_bond_count  =  0.055 +/- 0.088\n",
      "logP_aromatic_bond_percent  =  0.037 +/- 0.06\n",
      "logP_is_symmetric  =  0.119 +/- 0.153\n",
      "\n",
      "===\n",
      "\n",
      "365\n",
      "DMPNN with rdkit 2d normalized wo MolLogP\n",
      "logD\n",
      "None\n",
      "\n",
      "logD_symmetric test RMSE  =  0.892 +/- 0.222\n",
      "logD_symmetric test R2  =  0.355 +/- 0.452\n",
      "logD_non-symmetric test RMSE  =  0.598 +/- 0.036\n",
      "logD_non-symmetric test R2  =  0.75 +/- 0.027\n",
      "logD_rmse  =  0.6 +/- 0.034\n",
      "logD_r2  =  0.748 +/- 0.027\n",
      "logD_atom_count  =  0.526 +/- 0.204\n",
      "logD_ring_count  =  0.599 +/- 0.301\n",
      "logD_non_organogens_count  =  0.56 +/- 0.346\n",
      "logD_non_organogens_percent  =  0.515 +/- 0.341\n",
      "logD_aromatic_count  =  0.52 +/- 0.224\n",
      "logD_aromatic_rings_percent  =  0.683 +/- 0.343\n",
      "logD_bond_count  =  0.224 +/- 0.108\n",
      "logD_aromatic_bond_count  =  0.483 +/- 0.341\n",
      "logD_aromatic_bond_percent  =  0.53 +/- 0.284\n",
      "logD_is_symmetric  =  0.432 +/- 0.146\n",
      "\n",
      "===\n",
      "\n",
      "351\n",
      "StructGNN with ext valence with rdkit wo fragments and counts\n",
      "logP\n",
      "None\n",
      "\n",
      "logP_symmetric test RMSE  =  0.624 +/- 0.111\n",
      "logP_symmetric test R2  =  0.926 +/- 0.027\n",
      "logP_non-symmetric test RMSE  =  0.427 +/- 0.008\n",
      "logP_non-symmetric test R2  =  0.946 +/- 0.002\n",
      "logP_rmse  =  0.439 +/- 0.008\n",
      "logP_r2  =  0.945 +/- 0.002\n",
      "logP_atom_count  =  0.464 +/- 0.281\n",
      "logP_ring_count  =  0.44 +/- 0.42\n",
      "logP_non_organogens_count  =  0.63 +/- 0.295\n",
      "logP_non_organogens_percent  =  0.6 +/- 0.267\n",
      "logP_aromatic_count  =  0.479 +/- 0.223\n",
      "logP_aromatic_rings_percent  =  0.551 +/- 0.299\n",
      "logP_bond_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_bond_count  =  0.195 +/- 0.257\n",
      "logP_aromatic_bond_percent  =  0.047 +/- 0.041\n",
      "logP_is_symmetric  =  0.215 +/- 0.393\n",
      "\n",
      "===\n",
      "\n",
      "352\n",
      "StructGNN with ext valence with rdkit wo fragments and counts\n",
      "logD\n",
      "None\n",
      "\n",
      "logD_symmetric test RMSE  =  0.918 +/- 0.181\n",
      "logD_symmetric test R2  =  0.356 +/- 0.406\n",
      "logD_non-symmetric test RMSE  =  0.554 +/- 0.018\n",
      "logD_non-symmetric test R2  =  0.786 +/- 0.01\n",
      "logD_rmse  =  0.557 +/- 0.016\n",
      "logD_r2  =  0.783 +/- 0.01\n",
      "logD_atom_count  =  0.654 +/- 0.312\n",
      "logD_ring_count  =  0.659 +/- 0.249\n",
      "logD_non_organogens_count  =  0.421 +/- 0.329\n",
      "logD_non_organogens_percent  =  0.487 +/- 0.366\n",
      "logD_aromatic_count  =  0.44 +/- 0.346\n",
      "logD_aromatic_rings_percent  =  0.471 +/- 0.337\n",
      "logD_bond_count  =  0.266 +/- 0.214\n",
      "logD_aromatic_bond_count  =  0.362 +/- 0.246\n",
      "logD_aromatic_bond_percent  =  0.609 +/- 0.232\n",
      "logD_is_symmetric  =  0.229 +/- 0.216\n",
      "\n",
      "===\n",
      "\n",
      "359\n",
      "StructGNN with ext valence with rdkit wo fragments and counts\n",
      "logP logD\n",
      "None\n",
      "\n",
      "logP_symmetric test RMSE  =  0.642 +/- 0.147\n",
      "logP_symmetric test R2  =  0.921 +/- 0.033\n",
      "logP_non-symmetric test RMSE  =  0.42 +/- 0.011\n",
      "logP_non-symmetric test R2  =  0.948 +/- 0.003\n",
      "logP_rmse  =  0.433 +/- 0.013\n",
      "logP_r2  =  0.946 +/- 0.002\n",
      "logP_atom_count  =  0.0 +/- 0.0\n",
      "logP_ring_count  =  0.0 +/- 0.0\n",
      "logP_non_organogens_count  =  0.0 +/- 0.0\n",
      "logP_non_organogens_percent  =  0.053 +/- 0.033\n",
      "logP_aromatic_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_rings_percent  =  0.283 +/- 0.187\n",
      "logP_bond_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_bond_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_bond_percent  =  0.0 +/- 0.0\n",
      "logP_is_symmetric  =  0.01 +/- 0.009\n",
      "logD_symmetric test RMSE  =  0.75 +/- 0.175\n",
      "logD_symmetric test R2  =  0.583 +/- 0.064\n",
      "logD_non-symmetric test RMSE  =  0.546 +/- 0.016\n",
      "logD_non-symmetric test R2  =  0.793 +/- 0.007\n",
      "logD_rmse  =  0.548 +/- 0.019\n",
      "logD_r2  =  0.791 +/- 0.008\n",
      "logD_atom_count  =  0.695 +/- 0.16\n",
      "logD_ring_count  =  0.446 +/- 0.291\n",
      "logD_non_organogens_count  =  0.001 +/- 0.002\n",
      "logD_non_organogens_percent  =  0.0 +/- 0.001\n",
      "logD_aromatic_count  =  0.245 +/- 0.274\n",
      "logD_aromatic_rings_percent  =  0.211 +/- 0.171\n",
      "logD_bond_count  =  0.0 +/- 0.0\n",
      "logD_aromatic_bond_count  =  0.0 +/- 0.0\n",
      "logD_aromatic_bond_percent  =  0.0 +/- 0.0\n",
      "logD_is_symmetric  =  0.003 +/- 0.006\n",
      "\n",
      "===\n",
      "\n",
      "368\n",
      "DMPNN with rdkit 2d normalized wo MolLogP\n",
      "logP logD\n",
      "None\n",
      "\n",
      "logP_symmetric test RMSE  =  0.619 +/- 0.142\n",
      "logP_symmetric test R2  =  0.927 +/- 0.031\n",
      "logP_non-symmetric test RMSE  =  0.449 +/- 0.008\n",
      "logP_non-symmetric test R2  =  0.94 +/- 0.003\n",
      "logP_rmse  =  0.459 +/- 0.012\n",
      "logP_r2  =  0.94 +/- 0.002\n",
      "logP_atom_count  =  0.0 +/- 0.0\n",
      "logP_ring_count  =  0.0 +/- 0.0\n",
      "logP_non_organogens_count  =  0.011 +/- 0.013\n",
      "logP_non_organogens_percent  =  0.363 +/- 0.24\n",
      "logP_aromatic_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_rings_percent  =  0.262 +/- 0.186\n",
      "logP_bond_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_bond_count  =  0.0 +/- 0.0\n",
      "logP_aromatic_bond_percent  =  0.0 +/- 0.0\n",
      "logP_is_symmetric  =  0.025 +/- 0.039\n",
      "logD_symmetric test RMSE  =  0.69 +/- 0.284\n",
      "logD_symmetric test R2  =  0.637 +/- 0.192\n",
      "logD_non-symmetric test RMSE  =  0.582 +/- 0.031\n",
      "logD_non-symmetric test R2  =  0.764 +/- 0.021\n",
      "logD_rmse  =  0.584 +/- 0.034\n",
      "logD_r2  =  0.763 +/- 0.022\n",
      "logD_atom_count  =  0.454 +/- 0.218\n",
      "logD_ring_count  =  0.19 +/- 0.134\n",
      "logD_non_organogens_count  =  0.0 +/- 0.0\n",
      "logD_non_organogens_percent  =  0.0 +/- 0.0\n",
      "logD_aromatic_count  =  0.403 +/- 0.377\n",
      "logD_aromatic_rings_percent  =  0.255 +/- 0.22\n",
      "logD_bond_count  =  0.0 +/- 0.0\n",
      "logD_aromatic_bond_count  =  0.0 +/- 0.0\n",
      "logD_aromatic_bond_percent  =  0.0 +/- 0.0\n",
      "logD_is_symmetric  =  0.0 +/- 0.001\n",
      "\n",
      "===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "kruskal_stistics = []\n",
    "for NUM_EXP in tqdm(NUM_EXPS):\n",
    "    kruskal_stats = {}\n",
    "    kruskal_stats_mean_std = {}\n",
    "    \n",
    "    DATA_PATH = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(NUM_EXP)+'/folds'\n",
    "    rmse = []\n",
    "    \n",
    "    preds = []\n",
    "    \n",
    "    for FOLD in range(FOLDS):\n",
    "        import json\n",
    "        with open(os.path.join(DATA_PATH,'fold_'+str(FOLD), 'args.json'), 'r') as f:\n",
    "            args = json.load(f)\n",
    "        VALUE_COLUMNS = args['target_columns']\n",
    "        PRED_COLUMNS = [value+'_pred' for value in args['target_columns']]\n",
    "        if args['additional_encoder']:\n",
    "            kruskal_stats_mean_std['Model'] = 'StructGNN with ext valence'\n",
    "        else:\n",
    "            kruskal_stats_mean_std['Model'] = 'DMPNN'\n",
    "        kruskal_stats_mean_std['Num_exp'] = NUM_EXP\n",
    "        kruskal_stats_mean_std['Data'] = ' '.join(args['target_columns'])\n",
    "        \n",
    "        kruskal_stats_mean_std['Dataset'] = args['separate_test_path']\n",
    "\n",
    "        kruskal_stats_mean_std['Model'] += ' with '+' '.join(args['features_generator'][0].split('_'))\n",
    "        \n",
    "        for VALUE_COLUMN in VALUE_COLUMNS:\n",
    "            if VALUE_COLUMN in kruskal_stats.keys():\n",
    "                pass\n",
    "            else:\n",
    "                kruskal_stats[VALUE_COLUMN] = {}\n",
    "                kruskal_stats[VALUE_COLUMN]['symmetric test RMSE'] = []\n",
    "                kruskal_stats[VALUE_COLUMN]['symmetric test R2'] = []\n",
    "                kruskal_stats[VALUE_COLUMN]['non-symmetric test RMSE'] = []\n",
    "                kruskal_stats[VALUE_COLUMN]['non-symmetric test R2'] = []\n",
    "                kruskal_stats[VALUE_COLUMN]['rmse']= []\n",
    "                kruskal_stats[VALUE_COLUMN]['r2'] = []\n",
    "            PRED_COLUMN = VALUE_COLUMN+'_pred'\n",
    "            test_prediction = pd.read_csv(os.path.join(DATA_PATH,'fold_'+str(FOLD), FILENAME))\n",
    "            kruskal_stats[VALUE_COLUMN]['rmse'].append(\\\n",
    "                                                       mean_squared_error(test_prediction[~test_prediction[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                          test_prediction[~test_prediction[VALUE_COLUMN].isna()][PRED_COLUMN])**0.5)\n",
    "            kruskal_stats[VALUE_COLUMN]['r2'].append(r2_score(\\\n",
    "                                                              test_prediction[~test_prediction[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                              test_prediction[~test_prediction[VALUE_COLUMN].isna()][PRED_COLUMN]))\n",
    "            rmse.append(mean_squared_error(test_prediction[~test_prediction[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                           test_prediction[~test_prediction[VALUE_COLUMN].isna()][PRED_COLUMN])**0.5)\n",
    "            \n",
    "\n",
    "            test_prediction[ERROR_COLUMN] = (test_prediction[VALUE_COLUMN] - test_prediction[PRED_COLUMN]) ** 2    \n",
    "            test_prediction_sorted = test_prediction.sort_values(by=[ERROR_COLUMN], ascending=False)    \n",
    "\n",
    "            compare_data_with_stats = create_statistics(test_prediction_sorted)\n",
    "\n",
    "            best_n_vals = compare_data_with_stats.iloc[-N_VALUES:]\n",
    "            worst_n_vals = compare_data_with_stats.iloc[:N_VALUES]\n",
    "\n",
    "\n",
    "            for stat in compare_data_with_stats.columns:\n",
    "                if stat not in [SMILES_COLUMN, ERROR_COLUMN] and stat not in VALUE_COLUMNS and stat not in PRED_COLUMNS:\n",
    "                    if stat not in kruskal_stats[VALUE_COLUMN].keys():\n",
    "                        kruskal_stats[VALUE_COLUMN][stat] = []\n",
    "                    kruskal_stats[VALUE_COLUMN][stat].append(stats.kruskal(best_n_vals[stat], worst_n_vals[stat])[1])\n",
    "\n",
    "            symmetric_rull = lambda values: (len(values) - 1 <= len(values[values % 2 == 0])) or\\\n",
    "            (len(values) - 1 <= len(values[(values % 2 == 1) & (values > 1)]))\n",
    "            values_list = [pd.Series(Chem.CanonicalRankAtoms(Chem.MolFromSmiles(smiles), breakTies=False)).value_counts()\\\n",
    "                           for smiles in\\\n",
    "                           compare_data_with_stats[~compare_data_with_stats[VALUE_COLUMN].isna()][SMILES_COLUMN]]\n",
    "            symmetric_indices = [symmetric_rull(values) for values in values_list]\n",
    "            not_symmetric_indices = [not e for e in symmetric_indices]\n",
    "            \n",
    "\n",
    "            symmetric_y_predicted = compare_data_with_stats[~compare_data_with_stats[VALUE_COLUMN].isna()][PRED_COLUMN][symmetric_indices]\n",
    "            symmetric_y_expected = compare_data_with_stats[~compare_data_with_stats[VALUE_COLUMN].isna()][VALUE_COLUMN][symmetric_indices]    \n",
    "            not_symmetric_y_predicted = compare_data_with_stats[~compare_data_with_stats[VALUE_COLUMN].isna()][PRED_COLUMN][not_symmetric_indices]    \n",
    "            not_symmetric_y_expected = compare_data_with_stats[~compare_data_with_stats[VALUE_COLUMN].isna()][VALUE_COLUMN][not_symmetric_indices]   \n",
    "            from sklearn.metrics import mean_squared_error    \n",
    "\n",
    "            kruskal_stats[VALUE_COLUMN]['symmetric test RMSE'].append(mean_squared_error(symmetric_y_expected, symmetric_y_predicted, squared=False))\n",
    "            kruskal_stats[VALUE_COLUMN]['symmetric test R2'].append(r2_score(symmetric_y_expected, symmetric_y_predicted))\n",
    "\n",
    "\n",
    "            kruskal_stats[VALUE_COLUMN]['non-symmetric test RMSE'].append(mean_squared_error(not_symmetric_y_expected, not_symmetric_y_predicted, squared=False))\n",
    "            kruskal_stats[VALUE_COLUMN]['non-symmetric test R2'].append(r2_score(not_symmetric_y_expected, not_symmetric_y_predicted))\n",
    "        \n",
    "    \n",
    "    for VALUE in kruskal_stats.keys():\n",
    "        for stat in kruskal_stats[VALUE].keys():\n",
    "            kruskal_stats_mean_std[VALUE+'_'+stat+'_mean'] = np.mean(kruskal_stats[VALUE][stat])\n",
    "            kruskal_stats_mean_std[VALUE+'_'+stat+'_std'] = np.std(kruskal_stats[VALUE][stat])\n",
    "    \n",
    "    kruskal_stistics.append(kruskal_stats_mean_std)\n",
    "    \n",
    "kruskal_stistics = pd.DataFrame(kruskal_stistics)\n",
    "\n",
    "import math\n",
    "for i in range(len(kruskal_stistics)):\n",
    "    stats_model = kruskal_stistics.iloc[i]\n",
    "    print(stats_model['Num_exp'])\n",
    "    print(stats_model['Model'])\n",
    "    print(stats_model['Data'])\n",
    "    print(stats_model['Dataset'])\n",
    "    print()\n",
    "    output = {}\n",
    "    for stat in stats_model.keys():\n",
    "        if stat=='Model' or stat=='Data' or stat=='Dataset' or stat=='Num_exp':\n",
    "            continue\n",
    "        stat_name = '_'.join(stat.split('_')[:-1])\n",
    "        stat_suf = stat.split('_')[-1]\n",
    "        if stat_name not in output.keys():\n",
    "            output[stat_name] = {}\n",
    "        output[stat_name][stat_suf] = stats_model[stat]\n",
    "    for stat in output.keys():\n",
    "        if math.isnan(output[stat]['mean']):\n",
    "            continue\n",
    "        print(stat,' = ', round(output[stat]['mean'],3), '+/-', round(output[stat]['std'], 3))\n",
    "    print()\n",
    "    print('===')\n",
    "    print()\n",
    "    \n",
    "values = {}\n",
    "for column in list(kruskal_stistics):\n",
    "    if column not in ['Model','Num_exp','Data','Dataset']:\n",
    "        property_name = ' '.join(column.split('_')[:-1])\n",
    "        value_name = column.split('_')[-1]\n",
    "        value = kruskal_stistics[column].values\n",
    "        if property_name not in values.keys():\n",
    "            values[property_name] = {}\n",
    "            values[property_name][value_name] = value\n",
    "        else:\n",
    "            values[property_name][value_name] = value\n",
    "            values[property_name] = [str(round(mean,3))+'+/-'+str(round(std, 3)) for mean, std in zip(values[property_name]['mean'], values[property_name]['std'])]\n",
    "stats_markdown_table = pd.DataFrame(columns = ['Model','Num_exp','Data','Dataset'])\n",
    "for column in list(stats_markdown_table):\n",
    "    stats_markdown_table[column] = kruskal_stistics[column]\n",
    "for column in values.keys():\n",
    "    stats_markdown_table[column] = values[column]\n",
    "stats_markdown_table = stats_markdown_table[['Model',\n",
    " 'Num_exp',\n",
    " 'Data',\n",
    " 'logD symmetric test RMSE',\n",
    " 'logP symmetric test RMSE',\n",
    " 'logD non-symmetric test RMSE',\n",
    " 'logP non-symmetric test RMSE',\n",
    " 'logD symmetric test R2',\n",
    " 'logP symmetric test R2',\n",
    " 'logD non-symmetric test R2',\n",
    " 'logP non-symmetric test R2',\n",
    " 'logD rmse',\n",
    " 'logP rmse',\n",
    " 'logD r2',\n",
    " 'logP r2',\n",
    " 'Dataset']]\n",
    "RAW_PATH = '../../data/raw/baselines/dmpnn/'\n",
    "with open(os.path.join(RAW_PATH, 'worst_best_stats.txt'), 'w') as f:\n",
    "    f.write(stats_markdown_table.set_index('Num_exp').to_markdown())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate p values for distribution of errors in models pairs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NUM_EXPS_PAIRS = [(313,314), (318, 320), (319, 321), (323, 316), (317, 324)]\n",
    "NUM_EXPS_PAIRS = [(364,351), (365, 352), (368, 359)]\n",
    "\n",
    "\n",
    "ALPHA = 0.05\n",
    "\n",
    "FILENAME = \"test_predictions.csv\"\n",
    "\n",
    "SMILES_COLUMN = 'smiles'\n",
    "ERROR_COLUMN = 'Squared Error'\n",
    "\n",
    "FOLDS= 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:00<00:00,  5.63it/s]\n"
     ]
    }
   ],
   "source": [
    "p_values = []\n",
    "for NUM_EXP_PAIR in tqdm(NUM_EXPS_PAIRS):\n",
    "    p_value = {} \n",
    "    p_value_mean_std = {}\n",
    "    \n",
    "    NUM_EXP_1, NUM_EXP_2 = NUM_EXP_PAIR\n",
    "    \n",
    "    DATA_PATH_1 = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(NUM_EXP_1)+'/folds'\n",
    "    DATA_PATH_2 = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(NUM_EXP_2)+'/folds'\n",
    "    rmse = []\n",
    "    \n",
    "    preds = []\n",
    "    \n",
    "    for FOLD in range(FOLDS):\n",
    "        import json\n",
    "        with open(os.path.join(DATA_PATH_1,'fold_'+str(FOLD), 'args.json'), 'r') as f:\n",
    "            args_1 = json.load(f)\n",
    "        with open(os.path.join(DATA_PATH_2,'fold_'+str(FOLD), 'args.json'), 'r') as f:\n",
    "            args_2 = json.load(f)\n",
    "        VALUE_COLUMNS = args_1['target_columns']\n",
    "        PRED_COLUMNS = [value+'_pred' for value in VALUE_COLUMNS]\n",
    "        if args_1['additional_encoder']:\n",
    "            p_value_mean_std['Model_1'] = 'StructGNN with ext valence'\n",
    "        else:\n",
    "            p_value_mean_std['Model_1'] = 'DMPNN'\n",
    "        if args_2['additional_encoder']:\n",
    "            p_value_mean_std['Model_2'] = 'StructGNN with ext valence'\n",
    "        else:\n",
    "            p_value_mean_std['Model_2'] = 'DMPNN'\n",
    "        p_value_mean_std['Num_exp_1'] = NUM_EXP_1\n",
    "        p_value_mean_std['Num_exp_2'] = NUM_EXP_2\n",
    "        p_value_mean_std['Data'] = ' '.join(args_1['target_columns'])\n",
    "        \n",
    "        p_value_mean_std['Dataset'] = args_1['separate_test_path']\n",
    "\n",
    "        p_value_mean_std['Model_1'] += ' with '+' '.join(args_1['features_generator'][0].split('_'))\n",
    "        p_value_mean_std['Model_2'] += ' with '+' '.join(args_2['features_generator'][0].split('_'))\n",
    "        \n",
    "        for VALUE_COLUMN in VALUE_COLUMNS:\n",
    "            if VALUE_COLUMN in p_value.keys():\n",
    "                pass\n",
    "            else:\n",
    "                p_value[VALUE_COLUMN] = {}\n",
    "                p_value[VALUE_COLUMN]['r2_1'] = []\n",
    "                p_value[VALUE_COLUMN]['rmse_1'] = []\n",
    "                p_value[VALUE_COLUMN]['r2_2'] = []\n",
    "                p_value[VALUE_COLUMN]['rmse_2'] = []\n",
    "                p_value[VALUE_COLUMN]['Shapiro_p_value_1'] = []\n",
    "                p_value[VALUE_COLUMN]['Shapiro_p_value_2'] = []\n",
    "                p_value[VALUE_COLUMN]['Normaltest_p_value_1'] = []\n",
    "                p_value[VALUE_COLUMN]['Normaltest_p_value_2'] = []\n",
    "                p_value[VALUE_COLUMN]['P_VALUE'] = []\n",
    "                \n",
    "            PRED_COLUMN = VALUE_COLUMN+'_pred'\n",
    "            test_prediction_1 = pd.read_csv(os.path.join(DATA_PATH_1,'fold_'+str(FOLD), FILENAME))\n",
    "            test_prediction_2 = pd.read_csv(os.path.join(DATA_PATH_2,'fold_'+str(FOLD), FILENAME))\n",
    "            \n",
    "            ERROR_COLUMN_1 = (test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][VALUE_COLUMN].values\\\n",
    "                                               - test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][PRED_COLUMN].values)**2\n",
    "            \n",
    "            ERROR_COLUMN_2 = (test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][VALUE_COLUMN].values\\\n",
    "                                               - test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][PRED_COLUMN].values)**2\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['rmse_1'].append(mean_squared_error(test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][PRED_COLUMN])**0.5)\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['rmse_2'].append(mean_squared_error(test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][PRED_COLUMN])**0.5)\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['r2_1'].append(r2_score(test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][PRED_COLUMN]))\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['r2_2'].append(r2_score(test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][PRED_COLUMN]))\n",
    "            \n",
    "            \n",
    "            stat, p = shapiro(ERROR_COLUMN_1)\n",
    "            p_value[VALUE_COLUMN]['Shapiro_p_value_1'].append(p)\n",
    "            stat, p = shapiro(ERROR_COLUMN_2)\n",
    "            p_value[VALUE_COLUMN]['Shapiro_p_value_2'].append(p)\n",
    "            \n",
    "            stat, p = normaltest(ERROR_COLUMN_1)\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_1'].append(p)\n",
    "            stat, p = normaltest(ERROR_COLUMN_2)\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_2'].append(p)\n",
    "            \n",
    "            if p_value[VALUE_COLUMN]['Shapiro_p_value_1'][-1]>ALPHA and \\\n",
    "            p_value[VALUE_COLUMN]['Shapiro_p_value_2'][-1]>ALPHA and \\\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_1'][-1]>ALPHA and \\\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_2'][-1]>ALPHA:\n",
    "                stat, p = ttest_ind(ERROR_COLUMN_1, ERROR_COLUMN_2)\n",
    "                p_value[VALUE_COLUMN]['P_VALUE'].append(p)\n",
    "            else:\n",
    "                if 'DMPNN' in p_value_mean_std['Model_1']:\n",
    "                    stat, p = mannwhitneyu(ERROR_COLUMN_1, ERROR_COLUMN_2, alternative = 'greater')\n",
    "                    p_value[VALUE_COLUMN]['P_VALUE'].append(p)\n",
    "                else:\n",
    "                    stat, p = mannwhitneyu(ERROR_COLUMN_1, ERROR_COLUMN_2, alternative = 'less')\n",
    "                    p_value[VALUE_COLUMN]['P_VALUE'].append(p)\n",
    "\n",
    "        \n",
    "    for VALUE in p_value.keys():\n",
    "        for stat in p_value[VALUE].keys():\n",
    "            p_value_mean_std[VALUE+'_'+stat+'_mean'] = np.mean(p_value[VALUE][stat])\n",
    "            p_value_mean_std[VALUE+'_'+stat+'_std'] = np.std(p_value[VALUE][stat])\n",
    "    \n",
    "    p_values.append(p_value_mean_std)\n",
    "    \n",
    "p_values = pd.DataFrame(p_values)\n",
    "\n",
    "values = {}\n",
    "for column in list(p_values):\n",
    "    if column not in ['Model_1','Num_exp_1','Model_2','Num_exp_2','Data','Dataset']:\n",
    "        property_name = ' '.join(column.split('_')[:-1])\n",
    "        value_name = column.split('_')[-1]\n",
    "        value = p_values[column].values\n",
    "        if property_name not in values.keys():\n",
    "            values[property_name] = {}\n",
    "            values[property_name][value_name] = value\n",
    "        else:\n",
    "            values[property_name][value_name] = value\n",
    "            values[property_name] = [str(round(mean,3))+'+/-'+str(round(std, 3)) for mean, std in zip(values[property_name]['mean'], values[property_name]['std'])]\n",
    "stats_markdown_table = pd.DataFrame(columns = ['Model_1',\n",
    " 'Model_2',\n",
    " 'Num_exp_1',\n",
    " 'Num_exp_2',\n",
    " 'Data',\n",
    " 'logP P VALUE',\n",
    " 'logD P VALUE',\n",
    " 'logP rmse 1',\n",
    " 'logP rmse 2',\n",
    " 'logP r2 1',\n",
    " 'logP r2 2',\n",
    " 'logD rmse 1',\n",
    " 'logD rmse 2',\n",
    " 'logD r2 1',\n",
    " 'logD r2 2',\n",
    " 'Dataset',\n",
    " 'logP Shapiro p value 1',\n",
    " 'logP Shapiro p value 2',\n",
    " 'logP Normaltest p value 1',\n",
    " 'logP Normaltest p value 2'\n",
    " 'logD Shapiro p value 1',\n",
    " 'logD Shapiro p value 2',\n",
    " 'logD Normaltest p value 1',\n",
    " 'logD Normaltest p value 2'])\n",
    "for column in p_values.keys():\n",
    "    stats_markdown_table[column] = p_values[column]\n",
    "for column in values.keys():\n",
    "    stats_markdown_table[column] = values[column]\n",
    "stats_markdown_table = stats_markdown_table[['Model_1',\n",
    " 'Model_2',\n",
    " 'Num_exp_1',\n",
    " 'Num_exp_2',\n",
    " 'Data',\n",
    " 'logP P VALUE',\n",
    " 'logD P VALUE',\n",
    " 'logP rmse 1',\n",
    " 'logP rmse 2',\n",
    " 'logP r2 1',\n",
    " 'logP r2 2',\n",
    " 'logD rmse 1',\n",
    " 'logD rmse 2',\n",
    " 'logD r2 1',\n",
    " 'logD r2 2',\n",
    " 'Dataset']]    \n",
    "RAW_PATH = '../../data/raw/baselines/dmpnn/'\n",
    "with open(os.path.join(RAW_PATH, 'DMPNN_StructDNN_p_value_comparison.txt'), 'w') as f:\n",
    "    f.write(stats_markdown_table.to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model_1</th>\n",
       "      <th>Model_2</th>\n",
       "      <th>Num_exp_1</th>\n",
       "      <th>Num_exp_2</th>\n",
       "      <th>Data</th>\n",
       "      <th>logP P VALUE</th>\n",
       "      <th>logD P VALUE</th>\n",
       "      <th>logP rmse 1</th>\n",
       "      <th>logP rmse 2</th>\n",
       "      <th>logP r2 1</th>\n",
       "      <th>logP r2 2</th>\n",
       "      <th>logD rmse 1</th>\n",
       "      <th>logD rmse 2</th>\n",
       "      <th>logD r2 1</th>\n",
       "      <th>logD r2 2</th>\n",
       "      <th>Dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DMPNN with rdkit 2d normalized wo MolLogP</td>\n",
       "      <td>StructGNN with ext valence with rdkit wo fragm...</td>\n",
       "      <td>364</td>\n",
       "      <td>351</td>\n",
       "      <td>logP</td>\n",
       "      <td>0.005+/-0.007</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>0.452+/-0.007</td>\n",
       "      <td>0.439+/-0.008</td>\n",
       "      <td>0.941+/-0.001</td>\n",
       "      <td>0.945+/-0.002</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DMPNN with rdkit 2d normalized wo MolLogP</td>\n",
       "      <td>StructGNN with ext valence with rdkit wo fragm...</td>\n",
       "      <td>365</td>\n",
       "      <td>352</td>\n",
       "      <td>logD</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>0.013+/-0.021</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>nan+/-nan</td>\n",
       "      <td>0.6+/-0.034</td>\n",
       "      <td>0.557+/-0.016</td>\n",
       "      <td>0.748+/-0.027</td>\n",
       "      <td>0.783+/-0.01</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DMPNN with rdkit 2d normalized wo MolLogP</td>\n",
       "      <td>StructGNN with ext valence with rdkit wo fragm...</td>\n",
       "      <td>368</td>\n",
       "      <td>359</td>\n",
       "      <td>logP logD</td>\n",
       "      <td>0.001+/-0.002</td>\n",
       "      <td>0.055+/-0.055</td>\n",
       "      <td>0.459+/-0.012</td>\n",
       "      <td>0.433+/-0.013</td>\n",
       "      <td>0.94+/-0.002</td>\n",
       "      <td>0.946+/-0.002</td>\n",
       "      <td>0.584+/-0.034</td>\n",
       "      <td>0.548+/-0.019</td>\n",
       "      <td>0.763+/-0.022</td>\n",
       "      <td>0.791+/-0.008</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Model_1  \\\n",
       "0  DMPNN with rdkit 2d normalized wo MolLogP   \n",
       "1  DMPNN with rdkit 2d normalized wo MolLogP   \n",
       "2  DMPNN with rdkit 2d normalized wo MolLogP   \n",
       "\n",
       "                                             Model_2  Num_exp_1  Num_exp_2  \\\n",
       "0  StructGNN with ext valence with rdkit wo fragm...        364        351   \n",
       "1  StructGNN with ext valence with rdkit wo fragm...        365        352   \n",
       "2  StructGNN with ext valence with rdkit wo fragm...        368        359   \n",
       "\n",
       "        Data   logP P VALUE   logD P VALUE    logP rmse 1    logP rmse 2  \\\n",
       "0       logP  0.005+/-0.007      nan+/-nan  0.452+/-0.007  0.439+/-0.008   \n",
       "1       logD      nan+/-nan  0.013+/-0.021      nan+/-nan      nan+/-nan   \n",
       "2  logP logD  0.001+/-0.002  0.055+/-0.055  0.459+/-0.012  0.433+/-0.013   \n",
       "\n",
       "       logP r2 1      logP r2 2    logD rmse 1    logD rmse 2      logD r2 1  \\\n",
       "0  0.941+/-0.001  0.945+/-0.002      nan+/-nan      nan+/-nan      nan+/-nan   \n",
       "1      nan+/-nan      nan+/-nan    0.6+/-0.034  0.557+/-0.016  0.748+/-0.027   \n",
       "2   0.94+/-0.002  0.946+/-0.002  0.584+/-0.034  0.548+/-0.019  0.763+/-0.022   \n",
       "\n",
       "       logD r2 2 Dataset  \n",
       "0      nan+/-nan    None  \n",
       "1   0.783+/-0.01    None  \n",
       "2  0.791+/-0.008    None  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_markdown_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_EXPS_PAIRS = [(364,351), (365, 352), (368, 359)]\n",
    "\n",
    "ALPHA = 0.05\n",
    "\n",
    "FILENAME = \"test_predictions.csv\"\n",
    "\n",
    "SMILES_COLUMN = 'smiles'\n",
    "ERROR_COLUMN = 'Squared Error'\n",
    "\n",
    "FOLDS= 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA01klEQVR4nO3deXxcZ33o/89XM9osWbIka9+872uiODQJIZCQOL1pEiCBAIUAKYbS3PZ3uW0J7W1KQ0tLaS9c2pTitmwpNKQhAQNOnEASlM2JNu+ybFletFq7LGufme/vjzlKJopkjazlzEjf9+t1Xj7znGW+j8ee7zzPc85zRFUxxhiz8MS4HYAxxhh3WAIwxpgFyhKAMcYsUJYAjDFmgbIEYIwxC5TX7QCmYunSpbps2TK3wzDGmKhSUVHRrqqZY8ujKgEsW7aM8vJyt8MwxpioIiJnxyu3LiBjjFmgLAEYY8wCZQnAGGMWqLASgIjsFJEaEakVkQfG2f55ETkmIodE5NciUhyy7V4ROeks94aUXykih51zflNEZGaqZIwxJhyTJgAR8QAPA7cCG4APi8iGMbtVASWqugV4HPh759h04C+Bq4EdwF+KSJpzzLeATwOrnWXntGtjjDEmbOG0AHYAtapap6rDwKPAHaE7qOrzqtrvvNwPFDjrtwDPqmqnqnYBzwI7RSQXSFHV/Rqcje4HwJ3Tr44xxphwhZMA8oH6kNcNTtlE7gOemuTYfGd90nOKyC4RKReR8ra2tjDCNcYYE44ZHQQWkd8FSoCvzdQ5VXW3qpaoaklm5tvuYzDGGHOZwkkAjUBhyOsCp+wtROQm4M+B21V1aJJjG3mzm2jCcxpjjJk94dwJXAasFpHlBL+k7wE+ErqDiGwHvg3sVNXWkE37gK+EDPzeDHxRVTtF5IKIvAN4Dfg48E/Tq4oxJiLs3j1++a5dcxuHmdSkCUBVfSJyP8Evcw/wHVU9KiIPAeWquodgl08y8N/O1ZznVPV254v+ywSTCMBDqtrprH8O+B6QSHDM4CmMMcbMmbDmAlLVvcDeMWUPhqzfdIljvwN8Z5zycmBT2JEaY6JC6dnSt7y+vvh6lyIxk7E7gY0xZoGyBGCMMQuUJQBjjFmgoup5AMaY6DM6JnC84s2yXVfaFUGRwFoAxhizQFkLwBhzSbsr3npd/1t+vY+95l+VnAO1FOyvpu6mK+hcdalZY4zbrAVgjJkZnZ3wzW+yZu9rxPf2s/6JF0k9d97tqMwlWAIwxkybtrTg+6sv4as9wS93ruDxP7iBodQkNv73b0hq6Zz8BMYVlgCMMZfNrwG+3VfKz5/8O4b8Q6z7jI/b3lHHx1Kf418/shZffCybH32O+At9bodqxmFjAMaYKRkdE7g4fJFHm/6Bju5mag5B9fXreWLtXZQ3V/ANTxl/nF7GwEc28Gf/eoycA6c46HLc5u2sBWCMmbK2vja+8uJXOCDn+cXrq/DEeNl86yfYElvAKk3jH33v4Vot4P/kHON8YTpLj59zO2QzDksAxpgpUVUeOfQI/SP9fKfznawvO41cey0sWfLGPvF4+WPf1cRqDD/f6CGpvYcldU3uBW3GZQnAGDMlrza8Sk1HDe9f/37e/WoLqMItt7xtvzQSuCFQxN9t6ESBFb+qePvJjKssARhjwtY71Mvjxx5nVfoqbkreQs6BWlo2LaP04lFKz5a+bSbQ9wfWUpfi53RhMit+XelS1GYilgCMMWF7qvYpBn2DfHTzR1n16yo8Pj8NO9ZPuP96zWBtIJ1HNvhIP9VE6pmWOYzWTMYSgDEmLP6An7KmMrZmbyVvcR7LXjhAf3oK/VlLJjxGEO4MrGH3xkEArv7mT1j3RGnwDuKJnhxm5owlAGNMWE50nODC0AWuyr+K+J4+8ipO0L62YNLjrgsU0LwYThYsYmm1XQ0UScJKACKyU0RqRKRWRB4YZ/v1IlIpIj4RuSuk/N0iciBkGRSRO51t3xOR0yHbts1UpYwxM6+sqYwEbwKbsjZR9OIhYvwBOtYWTnpcCvGs1DR+ttFDcls3iR0X5iBaE45JE4CIeICHgVuBDcCHRWTDmN3OAZ8AfhRaqKrPq+o2Vd0GvAfoB54J2eVPRrer6oHLrYQxZnaN+EeobK5kW8424jxxLHvhABezltCbmxHW8ds0i93rgncDp9U1z2aoZgrCaQHsAGpVtU5Vh4FHgTtCd1DVM6p6CAhc4jx3AU+pav9lR2uMccXRtqMM+Aa4Ku8qPIPDFL56lLPv2gYiYR2/LZDNySUBLibHk9LUPrvBmrCFkwDygfqQ1w1O2VTdA/zXmLK/EZFDIvJ1EYkf7yAR2SUi5SJS3tbWdhlva4yZrvKmcpLjklm/dD2Frx7FOzTC6XdvC/v4LZoFAjUFCSxutAQQKeZkEFhEcoHNwL6Q4i8C64CrgHTgC+Mdq6q7VbVEVUsyMzNnPVZjzFupKsfajrE5azOeGA/Lnj/AYMoimq9YE/Y5UolnZWAJpYUBErsvwgUbB4gE4SSARiB0pKfAKZuKDwJPqurIaIGqNmvQEPBdgl1NxpgIc77vPH0jfaxKX4X4/BS/eIhz79yCej1TOs9WzWZPgTMraF3dLERqpiqcBFAGrBaR5SISR7ArZ88U3+fDjOn+cVoFiIgAdwJHpnhOY8wcqO2sBWBV+ipyDtQS39vPmRu2Tfk82wNZvJoXwB8jlgAixKQJQFV9wP0Eu2+qgcdU9aiIPCQitwOIyFUi0gDcDXxbRI6OHi8iywi2IH4z5tQ/FJHDwGFgKfDXM1AfY8wMq+2sZXHcYrKTsskrryEQIzRetW7K59miWQzFwrmcREsAESKs5wGo6l5g75iyB0PWywh2DY137BnGGTRW1fdMJVBjjDtOdp5kZfpKRITcyhO0rytiJDlxyudZQgK5mkxlASyvOgN+/8wHa6bE7gQ2xkyoqbeJ9v52VqWvYsOPnyPn4CmGFi9i3ROlwSkdpmiVpvFcoQ9GRqBxqkOJZqZZAjDGTOjlcy8DsCptFYubOojxB+gpyrrs860KpPHLguC8QJw6NRMhmmmwBGCMmdBL514iNiaWotQilpw9j4rQU3j5CWC1pnF2CQynJMHp0zMXqLkslgCMMRN6qf4lVqStwBPjIfVcKxez0/AnxF32+VZpGgg0FKfZQHAEsARgjBlX/0g/B1oOsDJ9JZ6hEVIa26bV/QOQxSIWaxyVhV5oa4PW1hmK1lwOSwDGmHEdOn+IgAYoTi0m68hpYvwBuouzp3VOQVitaTyT74wD7N8/A5Gay2UJwBgzrsrm4CMcC1MKya08gcK0+v9HrdI0fpLZjopAhT0n2E2WAIwx46pqriI9MZ30xHRyK05Mu/9/1CpNozPOx1D2Uqi05wS7yRKAMWZcVS1VbM/ZjmfER/bhOnqm2f0zapWmAdBckGIJwGWWAIwxbzPiH+Fw62G252wn89hZvEMj0x4AHlWsqcTj5XCeF5qaoMUeFO8WSwDGmLepbq9m2D/M9tztZB8OXq55IX9mpmP3EsNGbx6/yXYGgquqZuS8ZuosARhj3qaqOfilHGwBnOFCXgYjSQkzdv6NsXn8MrM7+MIGgl1jCcAY8zZVLVUsil3Emow1ZB49Q9uGZTN6/o3eXGrievCvWmnjAC6yBGCMeZvK5kq2Zm/F09FJSlMHbRuXzej5N3hzAehav8wSgIssARhj3iKgAQ60HGB7znYoKwOY8RbABm8eAHUr0uDsWejomNHzm/BYAjDGvEXdt75C73Av20/2wre+hYqQdrJ+Rt9jmSeDRGKpzHEKbCDYFZYAjDFvcXCkAYBt3gI4c4b+jBQCcbEz+h4vn3uJgkAyP/AcDhbYQLArwkoAIrJTRGpEpFZEHhhn+/UiUikiPhG5a8w2v4gccJY9IeXLReQ155w/dp43bIxx2SFfAzEIG725cPYsvXkZs/I+yzSV44FWLuRl2DiASyZNACLiAR4GbgU2AB8WkQ1jdjsHfAL40TinGFDVbc5ye0j5V4Gvq+oqoAu47zLiN8bMsEMjjazxZJPY2Qu9vfTmzl4C6BrsonVNviUAl4TTAtgB1KpqnaoOA48Cd4TuoKpnVPUQEAjnTUVEgPcAjztF3wfuDDdoY8zsOehrYGtsQXBwFmatBbBclwBwavkSqK2Fnp5ZeR8zsXASQD4QOgLUwDgPeb+EBBEpF5H9InKnU5YBdKuq7zLPaYyZBReGLnDa386W2PzgE7u8XvqylszKexVrKgCH853xhQMHZuV9zMTmYhC4WFVLgI8A3xCRlVM5WER2OQmkvK2tbXYiNMYAcKT1CABbvE4LoLAQ9Xhm5b1ySSI2JpbXs0eCBTYQPOfCSQCNQGHI6wKnLCyq2uj8WQe8AGwHOoAlIuKd7JyqultVS1S1JDNzZuYiMcaM72DLQQAGmxvwn66jMX1mr/4J5SGGnOQcjsV0QG6utQBcEE4CKANWO1ftxAH3AHsmOQYAEUkTkXhnfSlwLXBMVRV4Hhi9Yuhe4GdTDd4YM7MOnT9EssZR3D6CZ8Q3a/3/o3KTc2nqbYLt2+1eABdMmgCcfvr7gX1ANfCYqh4VkYdE5HYAEblKRBqAu4Fvi8hR5/D1QLmIHCT4hf93qnrM2fYF4PMiUktwTOA/ZrJixpipO9R6iJW6hJSWLgAu5sxyAlicS9dgF8NbNkJ1NQwOzur7mbfyTr4LqOpeYO+YsgdD1ssIduOMPe4VYPME56wjeIWRMSYCBDTA4fOHea/mkdTahd/roT9j8ay+Z05y8Fbg+hVLWen3w5EjUFIyq+9p3mR3AhtjADjTfYbe4V5W6hKSznfRn5kKMbP7FTGaAA7nO79FrRtoTlkCMMYAwf5/gJWBVJJbu7mYlTbr75mVlEWMxFAR1wEpKZYA5pglAGMMELwCSBDW9sYTOzBE3xwkAG+Ml8xFmRzrPA7bttmVQHPMEoAxBoCD5w+yJmMNGed7AejLXjLr77nuiVJW98ZSffJV8HqD9wL4/bP+vibIEoAxBoADLQfYmrOV5FbnCqA5aAEAFGsKtb5WfIUFMDwMJ0/OyfsaSwDGGJwpILpPszV7K0nnuxlMTcKfMDcT9BZpKiP4achLChbYOMCcsQRgjHljAHhbzjaSW7u4mD03v/4h2AIAOJjpD3YD2TjAnLEEYIzhQMsBALamrCGxs3dOBoBHFTmTwlVrG+TlWQtgDlkCMMZwsOUgGYkZ5J3tRFS5OEszgI4niVjyY5ZQ7WuGwsJgAlCds/dfyCwBGGM4eP4g23K2IYeCXUF9c9gFBLDOm0O1ryWYANrboalpTt9/obIEYMwC5wv4ONx6mK3ZW+HgQXxxXgaXJM9pDOu9ORz3taAFzowy1g00JywBGLPA/U3p3zDoG6RzsJPml58OPgBGZE5jWO/NpVcHac5z5h46eHBO33+hsgRgzALXcKEBgILF+aSfbJzTAeBR67zBOYGOebtg+XI4fHjOY1iILAEYs8DVX6jHIx7W9MYTf3HAlQTQ0xJ8/vDPW1/iTGGKJYA5EtZ00MaY+av+Qj25i3PJqm0GmNN7AEalk0CyxnJWeojxxcHx4/DwwxAb8kSyXbvmPK75zloAxixgqkp9Tz2FKYWk1wafytqXmTrncQhCkaZyTi4ExyACAWhpmfM4FhpLAMYsYM0Xm+kd7qUotYj02kYu5C8lEDd7zwG+lGJN4az00JfptEAaw370uLlMlgCMWcBG7wAebQF0rsp3LZYiTaFTBmlNjw9OCWEJYNaFlQBEZKeI1IhIrYg8MM7260WkUkR8InJXSPk2EXlVRI6KyCER+VDItu+JyGkROeAs22akRsaYsFU1B6+3L47PJrW+1dUEUOxMCXHG2ws5OXYz2ByYNAGIiAd4GLgV2AB8WEQ2jNntHPAJ4EdjyvuBj6vqRmAn8A0RWRKy/U9UdZuzHLisGhhjLltVSxVZi7LIbegmxh+IiARwVi5Afr61AOZAOC2AHUCtqtap6jDwKHBH6A6qekZVDwGBMeUnVPWks94EtAKZMxK5MWbaqlqqKEx9cwDYzQSQSxJxGsM56QkmgK4u6OtzLZ6FIJwEkA/Uh7xucMqmRER2AHHAqZDiv3G6hr4uIvETHLdLRMpFpLytrW2qb2uMmUDPYA91XXVvJABfnJeewizX4vEQQ4GmvNkCAOsGmmVzMggsIrnAI8AnVXW0lfBFYB1wFZAOfGG8Y1V1t6qWqGpJZqY1HoyZKWMHgLuX56Jej6sxFWvKmy0AsG6gWRZOAmgECkNeFzhlYRGRFOCXwJ+r6v7RclVt1qAh4LsEu5qMMXOkqiU4ABwJVwCNKiKVZvoYTE2CRYssAcyycBJAGbBaRJaLSBxwD7AnnJM7+z8J/EBVHx+zLdf5U4A7gSNTiNsYM01VLVXkxKRw7eOvk9Teg3dgiHVPlLoaU7GmEBDlhL81+HAYSwCzatIEoKo+4H5gH1ANPKaqR0XkIRG5HUBErhKRBuBu4NsictQ5/IPA9cAnxrnc84cichg4DCwF/nomK2aMubSq5iq2xxaR1NYNELwD12WjVwJV+5qD3UBNTfZwmFkU1lxAqroX2Dum7MGQ9TKCXUNjj/tP4D8nOOd7phSpMWZG7K7YzYh/hKNtR9nmW09SazcAfZlLXI0LoFAXE6PCsdEEMDAQvBooPd3t0OYluxPYmAWosbeRgAZYHUgnqa2LkcR4hpMT3Q6LeLzkkvxmAgDrBppFlgCMWYDO9ZwDYI2mk9TaHfz1P8cPgZnIck21BDBHLAEYswDV99SzKHYRObqIpLaeiOj/H1WsqZzwnWc4IRaWLLFZQWeRJQBjFqBzF85RmFJIYncfnhFfRPT/j1quqfgIUOtrDc4J1NzsdkjzliUAYxYYf8BP44VGClMLI+oKoFGjVwId8zUHE0BLi10JNEssARizwLRcbGEkMEJRShFJ57tQIuMKoFFFmoIgHPU1Be8FGByE7m63w5qXLAEYs8CcuxAcAC5MLST5fBcD6SkE4iLn6bAJeFnhWfpmCwCsG2iWWAIwZoE513OO2JhYcpJzSD7f5cozgCezwZsbTAC5ucECGwieFZYAjFlg6nvqKUwtJOHiIAk9ffRFaAKo8Z1nJHlRcE4gawHMCksAxiwgAQ1QfyH4EPiMEw0AEdkC2OjNYwQ/pwLtwVaAJYBZYQnAmAWkrquOQd8gRalFZJwIPuYjEhPAhthg18+xkaZgArAuoFlhCcCYBaSiqQLgjQQwnJTASARMATFWR9MpROGX7a9yKr4fenuhvd3tsOYdSwDGLCCVzZV4Y7zkLc4jo6YhIn/9Q/BKoFySOS3d9C8N3hdAdbW7Qc1DlgCMWUAqWyrJW5xHnB/S6poiNgEArNAl1MV00780JVhgCWDGWQIwZoFQVSqbKylKLWLJ6WY8Pj99WZGdABro5UJKAsTFwbFjboc071gCMGaBONtzls6BTqf/37kCKCdy59lfrkvwi3I25kLwhjBrAcw4SwDGLBCVzZUAFKUUsbSmnpGEOAbSkl2OamIrA0sAqJPu4JVAlgBmXFgJQER2ikiNiNSKyAPjbL9eRCpFxCcid43Zdq+InHSWe0PKrxSRw845v+k8G9gYM0sqmyvxiIeClAIyTtTTuTofYiL3N2A+i4nTGE5Ld7AFUF8fvBrIzJhJP30R8QAPA7cCG4APi8iGMbudAz4B/GjMsenAXwJXAzuAvxSR0U7HbwGfBlY7y87LroUxZlKVzZVszNpIbIyXjBMNdKwpdDukS/ISQ7Gmcmq0BQBw/LirMc034aT/HUCtqtap6jDwKHBH6A6qekZVDwGBMcfeAjyrqp2q2gU8C+wUkVwgRVX3q6oCPwDunGZdjDETUFUqmiu4IvcKkls6ie/tj/gEAM6VQKEJwAaCZ1Q4CSAfqA953eCUhWOiY/Od9UnPKSK7RKRcRMrb2trCfFtjTKim3iZa+1q5MvdKMmqC/yXb10Z+AlipaXTIAO0ZiRAba+MAMyxyOwAdqrpbVUtUtSQzM9PtcIyJSqMDwNtztpN19DQBTwydq8L9HeeeFboEgMOBFli71loAMyycBNAIhP5UKHDKwjHRsY3O+uWc0xgzRWVNZXjEw/bc7WQdPk372kL8CXFuhzWpNxKArxE2bLAEMMPCSQBlwGoRWS4iccA9wJ4wz78PuFlE0pzB35uBfaraDFwQkXc4V/98HPjZZcRvjAlDWVMZG7M2sigmnqyjZ2jdtNztkMKSTgKpGs/hEScB1NXBwIDbYc0bkyYAVfUB9xP8Mq8GHlPVoyLykIjcDiAiV4lIA3A38G0ROeoc2wl8mWASKQMecsoAPgf8O1ALnAKemtGaGWOA4ABwWWMZV+VdBUePEjswROvmFW6HFRZBWKFLOOhrCCYAVaipcTuseSOs58Cp6l5g75iyB0PWy3hrl07oft8BvjNOeTmwaSrBGmOm7kz3GToGOijJK4H9+wE4HyUJAGCVpvHzkVP41q0JfmEdOwbbtrkc1fwQ8YPAxpjpKWsqAwi2APbvZ2BJMr35S12OKnxrNJ1BRqhJC4DHY+MAM8gSgDHzXFljGXGeODZnb4b9+2ndvByi6Mb7NYHgfEUVHYdh9WpLADPIEoAx81xZUxnbcrYRd6EPqqujqvsHoJDFJBJLVXOVXQk0wywBGDOP+QN+Kporgt0/r78OEDUDwKM8xLA1toDKlspgAqithaEht8OaFywBGDOP1XTUcHH44hv9/4jQtmGZ22FN2RWxRVQ1VxFYvw78fjh50u2Q5gVLAMbMY2WNzgBwvpMANm1iJCnB5aim7orYInqHe2kocJ4OZt1AMyKsy0CNMdHpuwe+S7wnntLTL7D6hedoX1fEuidK3Q5ryrbHFgFQltRDUUyMJYAZYi0AY+axuq46VqStIK2+ndjBYS5E0eWfoTZ6c4mNiaWs6wisWGEJYIZYAjBmnuod6qXhQgMr01aSfehUsCwvw+WoLk+8xLIpaxNVLc6VQEePuh3SvGAJwJh56vXG11GUlekryX/9OMNJCfQvTXU7rMt2Re4VVDZXouvXw4kTMDLidkhRzxKAMfPUK/WvIAjLU4opeK2aruU5UXUD2FhX5l5Je387HcuywecLXg5qpsUSgDHz1CsNr5C3OI+C0x0kdvXStTzX7ZCmZUf+DgCq0p17AGwcYNosARgzDwU0wKv1r7IybSUF+4NflN1RngA2Z28m3hPP8wnNwZbMkSNuhxT1LAEYMw8daztGz1APK9JXUPDaMTpWFzCcnOh2WNMS54njitwreKmjClauhEOH3A4p6lkCMGYeeqX+FQDWJRSSc+AUDe9Y73JEM2NH/g4qmisIbNkCBw+6HU7UswRgzDz0Sv0rZC7KZPPxLjwjPhqu3uB2SNO3ezc7ai7SP9LP+aEOOHUKenvdjiqqWQIwZh4qPVvKtUXXUvhaNb44Ly3bVrkd0oy4Ojb4KMtDOU6BjQNMS1gJQER2ikiNiNSKyAPjbI8XkR87218TkWVO+UdF5EDIEhCRbc62F5xzjm7LmsmKGbNQne0+y+nu07x72bvJf+0YLdtXR8UD4MOxwrOUDEni15kXgwXWDTQtk84FJCIe4GHgvUADUCYie1Q19Bqs+4AuVV0lIvcAXwU+pKo/BH7onGcz8FNVPRBy3EedR0MaY2bA7ordb/T/j5w9TXpdMyd+5xqXo5q+0rNvzl+0ypvK00nN/H1ioiWAaQqnBbADqFXVOlUdBh4F7hizzx3A9531x4EbRd52x8mHnWONMbOopqOG5LhkrjzcDkDD1fNjAHjUes3gqL8Zf36eJYBpCicB5AP1Ia8bnLJx91FVH9ADjJ105EPAf40p+67T/fMX4yQMAERkl4iUi0h5W1tbGOEas3CpKjXtNazJWMOqZ8rpKcikc3WB22HNqPWBDAIozXmLg5eCBgJuhxS15mQQWESuBvpVNXTE5qOquhl4p7N8bLxjVXW3qpaoaklmZuYcRGtM9Grvb6drsIurpZD8shpqd+6I6ukfxrNBgzOaHsoG+vrg9Gl3A4pi4SSARqAw5HWBUzbuPiLiBVKBjpDt9zDm17+qNjp/9gI/ItjVZIyZhpqOGgDuqBpAVIMJYJ5JIZ61nmx+ZQPB0xZOAigDVovIchGJI/hlvmfMPnuAe531u4DnVFUBRCQG+CAh/f8i4hWRpc56LHAbYNdzGTNNNe01pMSncMXzx2lbX0TPspzJD4pC18St5PH0ZjQmxhLANEyaAJw+/fuBfUA18JiqHhWRh0Tkdme3/wAyRKQW+DwQeqno9UC9qtaFlMUD+0TkEHCAYAvi36ZbGWMWMlWlpqOGm4eLyDx+jtqdV7sd0qz5rbgV1Hv7GFlRbAlgGsJ6JKSq7gX2jil7MGR9ELh7gmNfAN4xpqwPuHKKsRpjLuFI6xF6hnr47K+SUBHE74/Kxz+G47diVwDQtCKTZZYALpvdCWzMPPF07dOgcPWRHrqLs6N+8rdL2eDNJUUSOJCtcOYM9PS4HVJUsgRgzDzx9Kmn+Z2uTJK7+2jduMztcGbVS+deYq0/jSfjnJ7lw4fdDShKWQIwZh7oHerlxbMv8gdVsfhjPbSvLZz8oCi3UZfyXIpzsaF1A10WSwDGzAPPn3me1N4R3rP/POc3rZg3c/9cyqZAJg0pcDElESor3Q4nKlkCMGYeeLr2af7gQCyxI36aSta6Hc6cWK8ZiAg1y5Oh3KYUuxyWAIyJcqrKszV7ub/CQ8PV6+nPTHU7pDmRTBy5i3Mpy1U4ehT6+90OKepYAjAmytV01HDFq2dZ2jnIkXve43Y4c6qkexHPp3aB3w9/9Vewe3dwMWGxBGBMlPvp8Z/yh6+Bb3kx567d5HY4c2qjZvJSvj/44uxZd4OJQpYAjIly1U89wrX14P2j/wUxC+u/9OZAJk2LoW9xgiWAy7Cw/rUYM880fOur3PSLYwzFB2/qn693/k4kj2TSSOBEgSWAy2EJwJgo9uz5V/nQEei7pgQS5++dvxMRhE2ayW9yhqGlBQYH3Q4pqlgCMCaKeUtfwquQfuNtbofimk2BpTyb2w+qUF8/+QHmDZYAjIlSHR0N7Hytg+MbsmABPyxpk2ZSkeu8sG6gKbEEYEyUOv7NB8nsh5gbb3I7FFet0XS6F3vpTo23BDBFlgCMiUaq5H3nMY7leFi7/p1uR+OqODyUxBZzMM9jCWCKLAEYE4Xafv5jljf0ceJdm5AFdunneK6NW8mvcwfg/HkYGHA7nKgR1gNhjDGRpftrD6GLYLC4gNKzC+vSz/FcF7eKf857Jvji3Dl3g4kiYf10EJGdIlIjIrUi8sA42+NF5MfO9tdEZJlTvkxEBkTkgLP8a8gxV4rIYeeYb4qIzFitjJnH9PRpVr5czd5355PnXeJ2OBHhmriVbw4EnznjZihRZdIEICIe4GHgVmAD8GER2TBmt/uALlVdBXwd+GrItlOqus1ZPhtS/i3g08BqZ9l5+dUwZuFo/dqXCACez37O7VAiRkZMMlmpuTSnx8Lp026HEzXCaQHsAGpVtU5Vh4FHgTvG7HMH8H1n/XHgxkv9oheRXCBFVferqgI/AO6cavDGLDj9/SQ/8mP2rBf+x42fnXz/BeS62JW8lB9A6+rcDiVqhJMA8oHQuysanLJx91FVH9ADZDjblotIlYj8RkTeGbJ/wyTnBEBEdolIuYiUt7W1hRGuMfOQM8vl4bvfRdLFIV4syeLIX/+h21FFjNKzpSzt9VNa4Ed6eqChYfKDzKxfBdQMFKnqduDzwI9EJGUqJ1DV3apaoqolmQv4ZhdjUCWt4hiHs2B1wRa3o4k4mwOZ7C9wXuzf72os0SKcBNAIhD5gtMApG3cfEfECqUCHqg6pageAqlYAp4A1zv4FIcePd05jTAitraXgfD//fdUiNmI/hsbKI5n67HiGPWIJIEzhJIAyYLWILBeROOAeYM+YffYA9zrrdwHPqaqKSKYziIyIrCA42Funqs3ABRF5hzNW8HHgZzNQH2PmrbZnnqQrAWI2bkKwi+bGEoR1MVkcyrUEEK5JE4DTp38/sA+oBh5T1aMi8pCI3O7s9h9AhojUEuzqGb1U9HrgkIgcIDg4/FlV7XS2fQ74d6CWYMvgqZmpkjHzUFMTWYdO8e87PFznXeF2NBFrSyCL0oIAWlEOIyNuhxPxwroRTFX3AnvHlD0Ysj4I3D3OcT8BfjLBOcuBhfX4ImMuU+dTTxAfCw0la7kaj9vhRKwtmsUzBSD7h+DQIbjySrdDimh2D7kxke7MGVLLD/P9Kz3cnLDR7Wgi2kpdwuEC53etdQNNyhKAMRGu/aEH8AODN93AYuLcDieieYghbXEmrSleSwBhsARgTCQ7f57F//nf/Hirh0/m/w+3o4kKWzSbl/N8+Pa/4nYoEc8SgDERrPXB/03sSIDOm68jLSbJ7XCiwlbNZn8BeGvroKPD7XAimiUAYyJVdTVp//5Dvl/iZbUny2b9DNNaTefAsvjgi9deczeYCGcJwJhIpErnpz/GxTh46TO3kmx9/2HzEkPc1dfgiwFefdXtcCKaJQBjIpA+8QTpL1fwtzcncsWWW9wOJ+rsWP1uqnJg5DfPux1KRLMEYEyk6e+n/w9/n0NZ0Pqx9xPriXU7oqhzw7IbKC2GmNfLYGjI7XAiliUAYyJM4O67SGpq4x9uS+fjVQHWPWF9/1N1dcHVlK2IxzM0DOXlbocTsSwBGBNJjh5F9+3jkS3wvq0fxGv/RS9LnCcOrrsu+KLUEuhE7F+XMZEiEMD/e/fRE6/88LYi7ozf5nZE0Wv3bnb0p3EkEwb/83vB5ymYt7GHwhvjptAvpuefx7P/Nf7ofXCrrOHFcy+6F9c8cGP8OkqLYc3h0+D3ux1ORLIWgDGRoLMT/emTPLdCqNmUw3bNdjuiqFZ6tpSupjoqir3EDY3YE8ImYAnAmEjw2GP4/CP83m3KpwPb3I5mXohBuFgQTKR64oTL0UQmSwDGuO3YMaiq4ivXC0VLilirGZMfY8JSvLiAU2nQe/KI26FEJEsAxrjoxbrn6f/P79KSFsvfvcPPp3z2rN+ZVBLIobQYYk/Wgarb4UQcSwDGuCi/rIZFHRf4zE4f745ZwTJS3Q5pXskhmeqiRBL7h6G62u1wIk5YCUBEdopIjYjUisgD42yPF5EfO9tfE5FlTvl7RaRCRA47f74n5JgXnHMecJasGauVMdGguZmilw6zf3Ui+9bEcJ9/q9sRzUvDhfkADPz6aZcjiTyTJgDnoe4PA7cCG4APi8iGMbvdB3Sp6irg68BXnfJ24HdUdTPBh8Y/Mua4j6rqNmdpnUY9jIk+n/884vfzuzsH+GBgHZkscjuieWll6jLqU6Bzz2NuhxJxwmkB7ABqVbVOVYeBR4E7xuxzB/B9Z/1x4EYREVWtUtUmp/wokCgi8TMRuDFR7Re/gEcf5dvXJdKZnsBH/Paox9mykUx+vdpL+suVMDzsdjgRJZwEkA/Uh7xucMrG3UdVfUAPMPZShg8AlaoaOjPTd53un78QEZlS5MZEqwsX4Pd/n46Vefx/7+znPv9WFmETvs0WLzG0b1xG4sAIgdLfuB1ORJmTO4FFZCPBbqGbQ4o/qqqNIrIY+AnwMeAH4xy7C9gFUFRUNAfRGjNN4007sGvXm+tf/CLa2MgHPhPP1pgcbvOtnLvYFqiCjb/FoKeW7se+S85N73U7nIgRTgugESgMeV3glI27j4h4gVSgw3ldADwJfFxVT40eoKqNzp+9wI8IdjW9jaruVtUSVS3JzMwMp07GRK6XXoJ/+Rd+clMeVcVxfMH3DgRr/M6296Zs4zfLwfvUPrdDiSjhJIAyYLWILBeROOAeYM+YffYQHOQFuAt4TlVVRJYAvwQeUNWXR3cWEa+ILHXWY4HbALtTw8xvR4+i73sfrdnJfKKkkW/c8g2ysOf8zoWMmGROXL2apQ2d6MmTbocTMSZNAE6f/v3APqAaeExVj4rIQyJyu7PbfwAZIlILfB4YvVT0fmAV8OCYyz3jgX0icgg4QLAF8W8zWC9jXFd6tvSN5cdPfJm+d11Dt7+Paz54kS/c8hCf3P5Jt0NcUJbeHfyN2viofdWMCmsMQFX3AnvHlD0Ysj4I3D3OcX8N/PUEp70y/DCNiV4JXb3c8tmvMTDUxzvvDXDvBx7iL971F26HtaCUni2lefMVVC+Fgce+w97fXsWuK3dNfuA8Z9NBGzNbAgEWlR9k42+qGfAod+5K5i8//W/cs+ketyNbkJLjktm/NYOPvNBBVd+A2+FEBEsAxsyQ3RXBq3/WnS1FWlrI2/siq1qGeWo1/Ndnr+OD195tX/4uO39DCfG/3kfc86Vw/R+5HY7rbC4gY2aQ+Px4Xn6Vd3zv1yy6OMzXP5DPyF3v57p3fYwEb4Lb4S14i254L10JkPPMK26HEhGsBWDMDFnc2M5Vf/oNVtW08bNNXrpufhdXJuQA0OJybCYoIXEx+65eyp0vtzDc1kJcZo7bIbnKWgDGzIRjx/jte7/M0jNtfPH9KSTc/gFWJCzsL5dIs+6JUtY9Ucrg6uUk+ODIrjvdDsl11gIwZjp274aGBga//jX6GeR/fmopu9JuJMH+a0WsZVmrqSioIOPlyuAzAhbwLDTWAjBmOs6dY+Qfv0a7DLLr3jQ+Y1/+ES8G4di2Qpa1jVD70++6HY6r7F+qMZershL/1/8v52OH+MSnMvijxHcTP8l/qdErhUatm834zIQy1m+n+9mzdPy/v2XV+z7ldjiusRaAMZejshL/je+hKW6Q931qEbtX/S9SsZnOo0VybBIVV+ay7aVazp895nY4rrEWgDFTVV6O/7030SQXuf3eBB5Z+b9Z4c2kgYkfObjuidI5DNCEY9UN7yf+lYd5+Uuf4v3f3e92OK6wFoAx4fL74R/+gcC119Akvdx8Xyx/kHQDnU2nKD1rX/DRprh4C9VXFHLTf73GqcML8/OzBGBMOE6cYOSGd8Kf/Ak/XTHCPX+6kg/+9p+yRtPdjsxMQ9b3HifWDx2f/FDwiqAFxhKAMRN45Jl/4IUv3MPpjXmwdi0DZa9y753w87+9l2c+X0V+ytgH45lok7F5By//3i3sqGih4p/+zO1w5pyNAZiFK/TJXaoMtNTTWl3OSO1Jlpxu4mOdgwAcyYQvvjeGgXs+wJ/c9iCbsja5FLCZDdf+3/+m+pfZ5P+fr3L2dz5E8fJtboc0ZywBmAXJH/Dzo+ofw+nT5NeeZ9OZAbL6lGKgJQleKhJOXZFEy/JMlmYWk3f7B4j3xvNK/Su8Um/zyMwHpWdLOV4RXPc8+Enu/cy/UHbb9WSXniQhI9vd4OaIJQCzYJzrOkPlL/6N4V/8jBWvHuej9X5igLYkqFqxiIF1K0lYs5EVuev5HW8WL5976c2D97zmWtxm9vm3b+XIl/8nJX/xTzRtWUHasy+RsmG722HNOksAxhVjb4iaiYdzqCrt/e20fu9hWgO91Ps76ehsIq66hoK6Nq453s+d/RAQOLY8mReuT0NXriA2J58EEW4uvn7aMZjo9fotm9gXuIXf+8o+fFddScs/fYOcj38OvPP3a3L+1szMa219bbze+DqVzZUcOH+AV+pfoaO/g5ihEa6ph1tOwc5a2Ho+uH93agIHS4ppv+4KLr77WobSFr/t2ny7lNOk3fp+vp2fx50PfI8N9/0R3X/6Z8Tu+n2S7v09WLNm3s0bJBrGpU8ishP4f4AH+HdV/bsx2+OBHxB8zGMH8CFVPeNs+yJwH+AH/lBV94VzzvGUlJRoeXl52JUzkSOgAU53neZI6xGq26t56uRTXBy+yIBvABFh+ZLlpCakkp6QTlpiGumJ6STHJeMRD3710znQScvFFo63H+do21HO9ZwDhYIL8K7uVK5pT+TqumE2n+ghbsRPwBPD4MpiYjduIXbjZnjoIXZXvvVZsHZzlpnI/ps3kvjMc6x+7FfcfApiFPozUhi59rdY/M4bidm6DTZvhuzsqEgKIlKhqiVvK58sAYiIBzgBvBdoAMqAD6vqsZB9PgdsUdXPisg9wPtU9UMisgH4L2AHkAf8CljjHHbJc45nxhOAKoyMwNAQDA4Gl4GB4DI0FNw2MhLcz+MJNgVjYyEx8c0lPh4SEiAuLrg9Ev8xqILPF6zT0BD09weXvj64eDG49PUFy0brPzwcXEZGgjdABQLBJSYmuHg8wXonJqIJCfgS4hhK8NIXCz2BAc4Pd9I00Ep952ka2utoaDtFTN8AqUOQMgTp/jgyBiBlJAZPAOLw4FM/g4wwIH78EuyqARCFOD8k+2PIjFlM3lAcqd0DpHcPkTAw8kY1O1fm0bhjPeLz0VOUjT8+1qW/cDMfXF98PdUjzTzZ+CsuHKti4+k+rj8LxT1v7jOwKI6LuRkM5Wfjy8lCM9Lxp6URWJLCSFIiI8mLGE6Kx5cYj39RIixahCcxibikFOIXLSYxbhGLYheR4E0gwZtAnCcOj3iQGf4emSgBhNMFtAOoVdU650SPAncAoV/WdwBfctYfB/5ZgjW4A3hUVYeA0yJS65yPMM45cz7wAXj66eAXmGrwT59vdm788HiCi0jwi1LkrUlhphNEaB1U33yt+uYXt98/vfcYrc/o2wQCSCDwxmsBYp0lGcjmzSw/sWECnhj8cV40JgaVQPA9NA5ffCziD6C8WTePP0DA60G9MJIoDC/JoL0ggYH0FPqy0+jLXGJf+GbGrY/NZf2yj6HFv8sj537Bt2M6aBpoZVFrF7kdQ2S091PQ3UxxTTM5FZA+ALGByc87KgD4Y8AvoALDwLFM+K3f9xIjMcRIDIIgIlTuqmTt0rUzWr9wEkA+UB/yugG4eqJ9VNUnIj1AhlO+f8yxo3fPTHZOAERkFzA6QnhRRGomiHMp0H7JmswFv3/6X7hBkVEfmIn6jF8XfwAGht++98WIfmB35HwuM2M+1WeG6/LDmTvVpYxNGM3Ag7631WXdn09r7tji8QojfhBYVXcDuyfbT0TKx2viRKv5VB+rS+SaT/WxukxdOFNBNAKFIa8LnLJx9xERL5BKcDB4omPDOacxxphZFE4CKANWi8hyEYkD7gH2jNlnD3Cvs34X8JwGR5f3APeISLyILAdWA6+HeU5jjDGzaNIuIKdP/35gH8FLNr+jqkdF5CGgXFX3AP8BPOIM8nYS/ELH2e8xgoO7PuAPVNUPMN45p1mXSbuJosx8qo/VJXLNp/pYXaYorPsAjDHGzD82HbQxxixQlgCMMWaBivoEICJfE5HjInJIRJ4UkSVO+TIRGRCRA87yry6HOqmJ6uJs+6KI1IpIjYjc4mKYYRORu0XkqIgERKQkpDwaP5tx6+Jsi7rPZpSIfElEGkM+i992O6bLISI7nb//WhF5wO14pkNEzojIYefzmN25b1Q1qhfgZsDrrH8V+Kqzvgw44nZ8M1SXDcBBIB5YDpwCPG7HG0Z91gNrgReAkpDyaPxsJqpLVH42IfF/Cfhjt+OYZh08zt/7CiDO+Tw2uB3XNOpzBlg6F+8V9S0AVX1GVX3Oy/0E7ymISpeoyxtTaqjqaSB0So2IparVqjrRndtR5RJ1icrPZp55Y7oaVR0GRqeWMZOI+gQwxqeAp0JeLxeRKhH5jYi8062gLlNoXcabjiPaH0gbzZ9NqPnw2dzvdDt+R0TS3A7mMsyHzyCUAs+ISIUzFc6sifipIABE5FdAzjib/lxVf+bs8+cE7zUYncCjGShS1Q4RuRL4qYhsVNULcxL0BC6zLhErnPqMI2o/m2h0qXoB3wK+TPBL58vAPxL88WHcc52qNopIFvCsiBxX1VmZuzwqEoCq3nSp7SLyCeA24EZ1OtE0OAPpkLNeISKnCE5S6eoDBS6nLkTw1BmT1WeCY6Lys5lAxH42o8Ktl4j8G/CLWQ5nNkT8ZzAVqtro/NkqIk8S7OKalQQQ9V1AzoNl/hS4XVX7Q8oznWcZICIrCE5DUedOlOGZqC5MPKVGVIrGz+YSovqzEZHckJfvA464Fcs0zJupZUQkSUQWj64TvDBk1j6TqGgBTOKfCV6B8awE56zfr6qfBa4HHhKREYITrn5WVTvdCzMs49ZFLzGlRiQTkfcB/wRkAr8UkQOqegtR+NlMVJdo/WxC/L2IbCPYBXQG+Iyr0VwGnWC6GpfDulzZwJPO/38v8CNVfXq23symgjDGmAUq6ruAjDHGXB5LAMYYs0BZAjDGmAXKEoAxxixQlgCMMWaBsgRgjDELlCUAY4xZoP5/Zof68DHqOLkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 1/3 [00:02<00:04,  2.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DMPNN with rdkit 2d normalized wo MolLogP   364\n",
      "StructGNN with ext valence with rdkit wo fragments and counts   351\n",
      "logP\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7oUlEQVR4nO3deXyU1b348c93JnvISjYg7IR9LQEUhdalCtXrVm213lpre7ld7HK3X23v72dvbe3tcnttvbW1tNqqrVWvS8WK4taKVAQCYQsKhABZCEkgZF9n5vv745nAZAECJHkmyffNa16ZOed5nvnOkMx3zjnPc46oKsYYY4Yfj9sBGGOMcYclAGOMGaYsARhjzDBlCcAYY4YpSwDGGDNMRbgdwLlIS0vTCRMmuB2GMcYMKlu3bj2mquldywdVApgwYQJ5eXluh2GMMYOKiBzuqdy6gIwxZpiyBGCMMcOUJQBjjBmmLAEYY8wwZQnAGGOGKUsAxhgzTFkCMMaYYcoSgDHGDFOWAIwxZpgaVFcCG2PCxOrVZ65ftWpg4jAXxFoAxhgzTFkCMMaYYcoSgDHGDFO9SgAiskJE9opIoYjc00P9P4vIHhHZKSJvisj4kLrPiMj+4O0zIeULRWRX8JgPioj0zUsyxhjTG2dNACLiBR4CVgIzgdtEZGaXzfKBXFWdCzwL/Ci4byrwbWAJsBj4toikBPf5JfAPQE7wtuKCX40xxphe600LYDFQqKpFqtoGPAVcH7qBqv5FVZuCD98DsoP3rwZeV9VqVT0BvA6sEJFRQKKqvqeqCjwO3HDhL8cYY0xv9SYBjAFKQh6XBstO53PAK2fZd0zw/lmPKSKrRCRPRPKqqqp6Ea4xxpje6NPrAETk74Fc4MN9dUxVXQ2sBsjNzdW+Oq4x5vys3rqa6YfXn7Z++fjlAxiNuRC9aQGUAWNDHmcHyzoRkSuBfweuU9XWs+xbxqluotMe0xgTfqLqm4iubXA7DNMHetMC2ALkiMhEnA/pW4FPhW4gIguAXwErVLUypGod8P2Qgd+rgG+qarWI1InIRcAm4A7gfy7spRhj+lTo1b6NjfDYY1BYyJ2NjQAcWTCFAx/NRSO8LgVoLtRZE4Cq+kTkbpwPcy/wqKoWiMh9QJ6qrgF+DIwA/jd4Nmexql4X/KD/Lk4SAbhPVauD978E/A6IxRkzeAVjTPhpaICf/hTKy+GiiyiKaiSqvonsLXtJKK9mz03LaE0e4XaU5jyIcxLO4JCbm6t5eXluh2HM0HC2+XwA6uvhgQegogK++EWYPZv1wf7/kftKmPbSRvyREeStuhZ/TBQQHAOwuYDCiohsVdXcruV2JbAxpmeq8PDDUFkJd98Ns2d3qj4+dSy7brucqIZmxr+zy6UgzYWwBGCM6dmmTVBYCLfeCjNm9LhJ/eg0yhfkMCZvL/GVJwY4QHOhLAEYY7prbobnnoMJE2Dp0jNueugj8/DFRDHl1S1Oq8EMGrYegDGmk/WH1zPpja2Mqasj/+NLaSjZcMbtfbHRHLxsPlPXbiJj90GY0GeXAZl+Zi0AY0wncVU1jNmyl6Pzp9AwamSv9jk6bzINmSmM3bjHWgGDiCUAY0wnYzcW4I/0cvAj83q/kwhHFk4l/lgtHDjQf8GZPmUJwBhzyokTpO85zNF5k/HFxZzTrpUzx+OLioB33umn4ExfswRgjDnlr39FFI7kTjvnXQNRkVTOngh5eVBdffYdjOssARhjHK2tsH49x6Zm05KScF6HKF+QAz4fPP54Hwdn+oMlAGOMY+NGaGqibPH0HquVsw/uNmamwMSJ8Ktf2WDwIGAJwBgDgQC8+SZMmEBddnrnKpQnPLtZEfk0d0b8mZ94N/OW5zB+Aj0fa9ky+OAD2HDm00eN++w6AGMM7N/vTPlw110grSeLj9PM/RHvstVzlIsDo/GjvOk5xBrZzx3+2XzO38OZQgsXwpNPwne+A5/4RPd6mycobFgCMMY40z5ER8OCBVD+HgCHqOWfIt+ggXb+zbeEawKTEQQ/AX7gfY8/eApYGshmhna5ViAmBqZPhx074JZbwJkh2IQh6wIyZrhrb4etW50P/yhnRs9WfPxHxDsEUH7lW8G1gSkIzge5Fw9f9eeSSizfj3iXVnzdjzlvHhw7BkeODOQrMefIEoAxw92uXdDSAkuWnCz6uXcbBz21fMu3lEma3G2XBKK4x3cRxVLHau/27secO9f5uWNH/8Rs+oQlAGOGu02bIDHR6bYB3pZi1nj3c6t/Bkt09Gl3y9VR3OSfxrPevRTIsc6VycnORHI7d/Zf3OaC9SoBiMgKEdkrIoUick8P9ctFZJuI+ETk5pDyy0Rke8itRURuCNb9TkQOhtTN76sXZYzppcZG2L0bFi0Cj4difzU/itjE9MBIPt/TAG8Xq/zzidMIXvLs7145dy4cPAi1tf0QuOkLZ00AIuIFHgJWAjOB20RkZpfNioE7gSdDC1X1L6o6X1XnA5cDTcBrIZv8W0e9qm4/3xdhjDlP27Y5F24Fu3++UvsUPvzc67uESM6+1m8sEXw4MI63PcW0dB0LmBdMINYKCFu9aQEsBgpVtUhV24CngOtDN1DVQ6q6E053YjAANwOvqGrTeUdrjOlbmzdDVhaMG8eLLdtZ07qDO/1zGUPvrwS+KjCRJvHxN09p54oxY2DkSBsHCGO9SQBjgJKQx6XBsnN1K/DHLmX3i8hOEXlARKJ72klEVolInojkVVVVncfTGmN61NjorPi1YAGN2sZX655mdsRobgn0fCXw6czXTDI0jnWeg50rRJxWwAcfQFtbHwZu+sqADAKLyChgDrAupPibwHRgEZAKfKOnfVV1tarmqmpuenp6T5sYY85HQYFzBfC8edzX8GeK/dX8Mul2Is7xY8GD8NHARPKknOM0d66cPt05zfTgwZ53Nq7qzf90GTA25HF2sOxcfAJ4QVXbOwpUtVwdrcBvcbqajDEDZedOSEhg9+hI/rvxDe6KvYRLo6ac16Gu8k/EL85Vwp3k5DgtgX37Ljxe0+d6kwC2ADkiMlFEonC6ctac4/PcRpfun2CrABER4AZg9zke0xhzvtrboaAAnTObL9b/kSSJ5YeJN5334SaQxLRAKq917QaKi4PsbGeqCRN2zpoAVNUH3I3TffM+8IyqFojIfSJyHYCILBKRUuAW4FciUtCxv4hMwGlBvN3l0H8QkV3ALiAN+F4fvB5jTG+8+y40NfGXaVFsaC/kR4kfJ80z4oIOeVVgIvs9J9jd3qWDICcHioqcs41MWOnVXECquhZY26Xs3pD7W3C6hnra9xA9DBqr6uXnEqgxpm+s3rqaJb95ltleD3dkbmROIJ1JVT7Ws/6Cjnt5YDz/w1bWtO5gdmTIn3xODrz1Fhw6BFPOr4vJ9A+7EtiYYWj8OzvZMT6ao9Ht/JN/ER4ufMK2VGKZHEjmrda9nSumTnV+WjdQ2LEEYMwwk1hcQfLhCn47rZlbAtOZrCl9duwPaRZ/aztAy6nzPWDECBg92gaCw5AlAGOGmTHrnQuzNuXE8Bn/nD499oJAJi20815bUeeKnBw4cAD8/j59PnNhLAEYM8zEvP0Oe0fCzQkXEUdknx57nmbgQXirrUs3UE6Os+ZwcXGfPp+5MJYAjBlGPijfxfSCSnbNTuciPZ8L+s9sBFHkRo7nrdYPOlfYOEBYsgRgzDAR0AA/fejTJLRB1EdX9NvzXB41nU3tB2kItJwqTEqCzExLAGHGloQ0ZohZvXV1j+UbijeQvdHp/z9x0XyS39rWL89/efQ0ftD4KhvaClkRM/tUxcSJsGcPqNoykWHCWgDGDAONbY08//7zXFMay7GcbFqTL+yirzO5JGoKkXi7jwOMHw91dbZMZBixBGDMMPDi3hfxNzWy+FA7RxZN69fnipMoLo6axFttXcYBxo93fubl9evzm96zBGDMEFdcW8z6w+v5gm8+kW0+jiw6t+mez8flUdPY1l7CiUDjqcKxY8HjsQQQRiwBGDOEBTTAH3f/kRFRI/hkZQYBr4fyBTn9/ryXR09HUd5uCxn0jYpyLgizBBA2LAEYM4RtKdtC0YkibpxxIxO2HeDY9HG0j4jt9+ddHDmBKCLY2Hagc8W4cU4CUO33GMzZWQIwZogKaICX979MdmI2y1IXkLH7IGUD0P0DEC2RzI/MZlP7oc4V48fDsWN2QViYsARgzBCVdySPisYKrsm5hlG7DuHxBziS278DwKEWR04gr/0wfg1ZKrxjIHjr1gGLw5yeJQBjhqCABnil8BVGJ4xmftZ8srYXEvAIlXMmDVgMiyMn0qitvO8rP1WYnQ0RETYOECbsQjBjhqDtR7dzpP4I/893CTNf2MDk1/NozEhh8rrNAxbDkqiJAGxuP3RqfYDISJgzxxJAmOhVC0BEVojIXhEpFJF7eqhfLiLbRMQnIjd3qfOLyPbgbU1I+UQR2RQ85tPB5SaNMRdIVXl5/8tkxmdyWWAc4g+QUHaMuuz0AY1jijedZIljU3uXZSIXLrSB4DBx1gQgIl7gIWAlMBO4TURmdtmsGLgTeLKHQzSr6vzg7bqQ8h8CD6jqFOAE8LnziN8Y08XOyp2U1pWyMmclXjyMqKjG6/NTO3ZgE4BHPCyKHM/mtkOdK3Jz4cQJOHiwx/3MwOlNC2AxUKiqRaraBjwFXB+6gaoeUtWdQKCnA3QVXAj+cuDZYNFjOAvDG2Mu0NuH3iYlJoXFoxcDkFhSBTDgLQBwuoF2+cpo0rZThbm5zk/rBnJdbxLAGKAk5HEpPazxewYxIpInIu+JyA3BspFATXDB+TMeU0RWBffPq6qqOoenNWb4KaktYU/VHpaOXYrX4wUgsbSK5uQRtCXEDXg8iyMn4CdAfnvIaZ+zZzsDwTt2DHg8prOBOAtovKrmAp8Cfioik89lZ1Vdraq5qpqbnj7w32CMGUwe3/E4inJx9sVOgSpJJVWufPsHWBQ5AYBNbSHdPdHRzvoAu3a5EpM5pTcJoAwYG/I4O1jWK6paFvxZBPwVWAAcB5JFpOMspHM6pjGmu4AGeHT7o0wbOY30eOcDP+ZEPVFNLQPe/98hy5vEOG8qm7teEDZnDuze7UpM5pTeJIAtQE7wrJ0o4FZgzVn2AUBEUkQkOng/DbgE2KOqCvwF6Dhj6DPAi+cavDHmlPWH11N0oohLxl5ysizpZP9/hlthsSRyYvcEMHu2Mwjc0OBKTMZx1gQQ7Ke/G1gHvA88o6oFInKfiFwHICKLRKQUuAX4lYgUBHefAeSJyA6cD/wfqOqeYN03gH8WkUKcMYFH+vKFGTPcPJL/CEnRSSwYteBkWVJpFe2xUTSlJboW1+LICRz0H6PKX3+qcHZwoZg9e3reyQyIXl0IpqprgbVdyu4Nub8Fpxun637vAnNOc8winDOMjDEXqLallmf3PMtn53+WKO+pS2oSSyqd/n8XV+BaHBwH2Nx+iGu8wY+DjgSwaxcsto8Bt9hUEMYMAU/tfooWXwt3LbjrZFl0TQNx1fXUjXH35ImFkeMRhC2h3UCTJkFsrI0DuMwSgDFDwP/u+V+mp01n4aiFJ8sydzpTMddlp7kVFgDxnmimeTPJbw85m9zjgVmzLAG4zOYCMmaQq26u5q+H/sr/ueT/ICFdPZm7igh4hPpRI12MzrEgciwb2gqdB6uDi9ZHRsLmzaced1i1amCDG8asBWDMIPfS3pfwq58bp9/YqTxzRxGNmSkEIt3/nvehyHGUBE5wPBBy1s+YMc4i8fX1p9/R9Cv3fzOMMedk9dbO35h/seUXpMSksK18G/lH8wEQn5+MgoNUDOD0zx3WH17frcwrNRAJj5W+TK6OYvn45U4CADhyBKYN3DoF5hRrARgziLX6WtlTtYd5WfM6df+M3F9KRGu76/3/HXI0BYB9Un2qcPRo5+eRIy5EZMASgDGDWkFVAe2BdhZkLehUnrmjYwA4PKZPSSSaLI1nv5w4VZiUBHFxUGaTALjFEoAxg9j2o9uJj4wnJzWnU3nmziIaMpJpTYx3KbLupmgKhZ6QBCDidANZC8A1lgCMGaT8AT+7KncxN3PuyZk/O2TuPEDF3HOad7Hf5QRSKaGOJtpPFY4e7bQAbHEYV1gCMGaQ2nt8L03tTczPmt+pPK7yBAlHq6mYO/ADwGeSoymowIHQbqAxY6ClxVkgxgw4SwDGDFI7ju4gyhvFzPTOC/Rl7iwCCLsWwFRNBeg8DpCV5fw8etSFiIwlAGMGIVVld9Vupo2c1mnuH3C6f3xRERyfNvY0e7sjjViSNdoSQBixBGDMIFTZWMmxpmPMzpjdrS5rxwGqZk4IiwvAQglCjqayP/RU0MREiImBigr3AhvGLAEYMwjtrnTm0JmVPqtTubeljbQPijk6f4obYZ1VjqZwUGpp61gNVsRpBVgLwBWWAIwZhAqqCsiMzzy58leHjIJDePwBKuaFV/9/hxxNxScBCnwhp35mZVkLwCWWAIwZZNr8bew7vo9ZGbO61WVtdyZcC7cB4A5TA84VwdtCF4nPzHTOAmppcSmq4atXCUBEVojIXhEpFJF7eqhfLiLbRMQnIjeHlM8XkY0iUiAiO0XkkyF1vxORgyKyPXib3yevyJghbt/xfbQH2pmd3r3/P3PHAaonjaI1KXwuAAs1mgRiNYId7aWnCjsGgisr3QlqGDtrAhARL/AQsBKYCdwmIjO7bFYM3Ak82aW8CbhDVWcBK4CfikhySP2/qer84G37eb0CY4aZgsoCIj2R5IzsfPUvgYBzAViYdv8AeBAmazLbfSFrA2RmOj9tHGDA9eY0gcVAYXAJR0TkKeB64ORinqp6KFgXCN1RVfeF3D8iIpVAOlBzoYEbM1yd7vTPlKJyohuaOTovPAeAO0zRFN5qL0FVnQnsMjKcwWBLAAOuN11AY4CQdE1psOyciMhiIAo4EFJ8f7Br6AERiT7NfqtEJE9E8qqqqs71aY0ZUg5UH6CysfKM/f/hegZQh8maQp22cMh/3CmIjISRI20g2AUDMggsIqOAJ4DPqmpHK+GbwHRgEZAKfKOnfVV1tarmqmpuenp4zGxojFteLXwV6H76J0DWjkKaRiZSPyY8poA+nY6poXf4uowDWAtgwPUmAZQBoZcUZgfLekVEEoGXgX9X1fc6ylW1XB2twG9xupqMMWfw6oFXSYtLI3NEZre6zB0HnO6fkHUBwtFETcaDsL29yzhARQUEAqff0fS53iSALUCOiEwUkSjgVmBNbw4e3P4F4HFVfbZL3ajgTwFuAGx1aGPOoM3fxl8O/qXHb/9xlSdIPHKco/PDdwC4QwwRTPVmdk4AWVnQ3m6Twg2wsyYAVfUBdwPrgPeBZ1S1QETuE5HrAERkkYiUArcAvxKRguDunwCWA3f2cLrnH0RkF7ALSAO+15cvzJih5t2Sd2lsb+w2+RuEnP8f5gPAHeZFZnfvAgIbBxhgvZosRFXXAmu7lN0bcn8LTtdQ1/1+D/z+NMe8/JwiNWaYW1e4jghPBNNGdl8/d3TePtriYzgWZhPAnc78yLE83ZJHTaCJZE+cnQrqErsS2JhBYt2BdSwdu5TYyNhudaPz9lK+IAeN8PawZ/iZH+F8X9zZcUFYYiLExloCGGCWAIwZBCobK8k/ms/Vk6/uVhdXVUNycQVHcru3DMLVvEinpbK9oxtI5NRAsBkwlgCMGQReP/A6AFdNvqpb3ei8vQCDKgFkeRLJ8CR0nxLCEsCAsgRgzCCw7sA60uLS+NCoD3WrG523l5bEOKpzug3DhS0RYV5EduczgTIynLOAmprcC2yYCa8VI4wx3agqrx14jY9O+ige6fydbfrz6xn/9g4aslKZ9uIGlyI8P/Mjx/Jg41u0q59I8ToJAODAAZgzx93ghglrARgT5nZW7KSisaLH/v/o2gZiaxqoGd/9wrBwNy8ym1Z87PUFB347EsD+/e4FNcxYAjAmzK07sA7ouf8/+bDTZz4YE8D8CGcg+OT1AJYABpwlAGPC3KuFrzInYw6jEkZ1q0s+XEFbXDRN6ckDH9gFmhaRSTQRp8YBYmMhIcESwACyBGBMGKtvrWdD8QY+lvOx7pWqJB2uoHZcZtjP/9OTCPEyO3J05zOBMjIsAQwgSwDGhLE3D75Je6CdlVNWdqtLPnSUmLomaiZkuRBZ35gXMZbtvlJU1SnIyIDCQneDGkYsARgTxl7Z/wqJ0YksHbu0W924DbsAqJ4yeqDD6jPzI7OpCtRzNFDnFGRkwJEj0NjobmDDhCUAY8KUqrK2cC1XTrqSSG9kt/px7+yiISOZ1sTwXP+3N+Z3XBHcMQ7QMRBsrYABYdcBGBOmCqoKKK0r5dsf/na3uqj6JrJ2FFK6ZIYLkV249YfXA9BAG0TBC1XriQ9UE++vZiE44wDz5rka43BgLQBjwtQr+18BYMWUFd3qsjcW4PEHqJ5yzquzhpURRDFKR3BAagBoSUlwKmwgeEBYAjAmTL1S+ApzMuaQndh9iodxG3bTkhRP3eiRLkTWtyZrMoUeZyEYf3QkTSMTLQEMEEsAxoShuta6057+Kf4A4/62i5Kls8Ez+P+EcwIplFBHMz4AasfZqaADpVdjACKyAvgZ4AV+o6o/6FK/HPgpMBe4NXT5RxH5DPB/gw+/p6qPBcsXAr8DYnEWm/manjwXzJjha/XW1eSX59MeaMcX8LF66+pO9RkFB4mpbeTwsjlENTS7FGXfmaIpqMBBqWGmplE7NpNRmywBDISzfn0QES/wELASmAncJiJd16QrBu4EnuyybyrwbWAJzqLv3xaRlGD1L4F/AHKCt+4dncYMU7urdhMTEcPklO5r/I7bsIuA10Ppxd3XBh6MJqvzkVAoTjdQ7bgMZ1roujo3wxoWetN+XAwUqmqRqrYBTwHXh26gqodUdScQ6LLv1cDrqlqtqieA14EVwQXhE1X1veC3/sdxFoY3ZtgLaIBdFbuYmT4Tr6fLCl+qTPzLdo7Om0JbQpw7AfaxLOIZoZEnE0Dd2JBZQU2/6k0CGAOETNpNabCsN06375jg/bMeU0RWiUieiORVVVX18mmNGbyKa4upba1lXmb30yBT95eScrCcA1cvciGy/iEIkzWlcwsAbBxgAIT9CJKqrlbVXFXNTU9PdzscY/rdjoodeMTDnIzuc+JPeXUzAa+Hoiu6LwwzmE3RFIqkhgBKXXbw79wSQL/rTQIoA8aGPM4OlvXG6fYtC94/n2MaM6TtPLqTKSlTiI/qcoVvIMCUdVsouXgWrckj3Amun0zWFJrFxxHq8cVGw+jRlgAGQG8SwBYgR0QmikgUcCuwppfHXwdcJSIpwcHfq4B1qloO1InIRSIiwB3Ai+cRvzFDysETBymtL2Vu1txudVk7DjCi4gSFKxa7EFn/ygkOBO8PdgORk2PTQQyAsyYAVfUBd+N8mL8PPKOqBSJyn4hcByAii0SkFLgF+JWIFAT3rQa+i5NEtgD3BcsAvgT8BigEDgCv9OkrM2YQemnfSwDMzeieAKa8upn2mCgOL+9eN9iN1yS8KicvCGPKFGsBDIBeXQegqmtxztUPLbs35P4WOnfphG73KPBoD+V5wOxzCdaYoW7N3jWMGjGKzBGdV/jytPuY9MZWDn94Hr64GJei6z/ReBmvSRTKCZaB0wKorHROBU1MdDu8ISvsB4GNGS5qWmp4+/DbPZ79k/3eHmJqG4dk90+HHE3p3AUE1g3UzywBGBMmXi18FV/A12P///Q/baA5JYHSi7pegzl0TNEUjkszda11ThcQWDdQP7MEYEyYeHHvi2TEZzAxeWKn8vij1Yx7Zycf3HApgcihO4N7jqYCznUQTA5eAW0tgH5lCcCYMNDia+HP+/7MdVOvwyOd/yxnPO/Mnf/+TcvcCG3ATAmeCVRaVwrx8XYq6ACwBGBMGFhXuI6GtgZumXVLp3JPu4/pf9pA8aVzaRg1+Kd+PpOE4NoAxbXFToGdCtrvLAEYEwaeff9ZUmNTuWzCZZ3KJ76VT1x1PXtuXu5SZAMrR1MoqQvOHmOngva7oduhaMwg0eprZc3eNdw84+Zua//OfPav1I1JG9KDv6GmBFJY37iT+ocfJKG62jkV9Gc/g9jYUxutWuVegEOMtQCMcdnrRa9T11rHzTNv7lSeuq+EUfmF7Ln5w0Ni4Zfe6LgieIev9NQC8TYJZL+xFoAxA6zrAi+/2/47YiNiOVhzsFPdvCdeoy0umg9uuHSgQ3RNx5lA29tLuDQjeC1ARQWMG+diVEPX8PhaYUyY8gV87KjYwbyseUR4Tn0fG1F+nMmv5fHBjcuGzLz/vZFGLMkaTX57CXTM/mstgH5jLQBjXPTBsQ9oam/i+oPRTC9af7J80utbQZVdt13hYnQDTxCmaAr57cWQHA3Jyc44gOkX1gIwxkXbyrcRpxHk6qiTZRHNrYzaXkjVzPE0ZqW6GJ07cjSV3b4jtKnPGQewBNBvLAEY45J2fzv5R/O5JJBNFKeWfhy1bT/edt+wOfOnqxxNoR0/e3zlTjeQJYB+YwnAGJfsrtxNU3sTHw2cmvrB0+5jTN5eqieNojEjxcXo3JMTcFo9+e3FTgugvh6am12OamiyMQBjXLK5bDMJUQksbMs6WZa5s4ioxhZKLp4FwPTn159u9yErmwTixRkI/mzGVKewqsrOBOoH1gIwxgXN7c3srNxJ7uhcIjr+DAMBsje9T92YtFMLow9DHoT5EdlsbS+GzOC6CBUV7gY1RPUqAYjIChHZKyKFInJPD/XRIvJ0sH6TiEwIlt8uIttDbgERmR+s+2vwmB11w/c33gw7245uwxfwsWTMkpNlGXsOE1vTQPHFs0DExejctzByPNt9JfjTRjrvhSWAfnHWBCAiXuAhYCUwE7hNRLqOTn0OOKGqU4AHgB8CqOofVHW+qs4HPg0cVNXtIfvd3lGvqjbSY4aNzWWbSY9LZ0LyBKdAlbEbC2hMS6I6Z4yrsYWD3MjxNGkbH3iqITXVEkA/6U0LYDFQqKpFqtoGPAVc32Wb64HHgvefBa4ILvYe6rbgvsYMazUtNew9tpfFYxbT8WeSWlhGfFUtJUvt2z/Awkinv39r+2GnG+joUZcjGpp6kwDGACUhj0uDZT1uE1xEvhboOnftJ4E/din7bbD75//1kDAAEJFVIpInInlVdkWgGQLyjuShKIvHnFrecezGAlqS4qmcOd7FyMLHtIgs4iXaGQfouBZA1e2whpwBGQQWkSVAk6ruDim+XVXnAMuCt0/3tK+qrlbVXFXNTe+4NNyYQey90vcYlzSOrBHO2T+JJZUklR6jdMmMYTPp29l4xcP8iGzy2g9DVha0tDgLxJs+1ZvftjJgbMjj7GBZj9uISASQBBwPqb+VLt/+VbUs+LMeeBKnq8mYIS2/PJ+SuhKWZi89WTZ24x7aY6M5Om+yi5GFn5MDwRnBL342DtDnepMAtgA5IjJRRKJwPszXdNlmDfCZ4P2bgbdUnfaaiHiATxDS/y8iESKSFrwfCVwL7MaYIe6R/EeI8ESc7P5JKSxjZGEZZblTh/R6v+ejYyD4QFrwY8rGAfrcWX/jVNUnIncD6wAv8KiqFojIfUCeqq4BHgGeEJFCoBonSXRYDpSoalFIWTSwLvjh7wXeAH7dJ6/ImDDV4mvhD7v+wIKsBcRHxQMw9/ev44/0cmThNJejCz8dA8HvxdcwNTLSWgD9oFdfOVR1LbC2S9m9IfdbgFu67hes+ytwUZeyRmDhOcZqzKD2wvsvUNNSwyXjLgEg/mg1Oa9sonxBDr64aJejCz8dA8F5/mLuyMiwBNAPbMTJmAHySP4jTEyeyLSRzrf9OX98E4DSJdPdDCtsdQwEn7wi2BJAn7MEYMwAOHjiIG8efJPPzv8sHvEQVd/E9BfeoejKhbQmjXA7vLCVGxwIDmRkwLFj4Pe7HdKQYgnAmAHw2+2/RRDunH8nADOeW09UUys7Pn2Vu4GFuYXBgeAj6TEQCNjqYH3MEoAx/azd386j+Y9y1eSrGJs0Fk+7j9lPvUXp4hkcn24zXJ5Jx0DwzpHtToF1A/UpSwDG9LPn3n+OsvoyvrL4KwBMeXUz8cdq2XGHffs/m46B4LeTa50CSwB9yk48NqafPbjpQaakTmFlzkoIBJj7xOscm5pN2ZIZbocW9rziYWHkON7WMkhIsATQx6wFYEw/2lK2hY2lG/nK4q/gEQ+sXUtq0RF2fvoqm/Stl5ZETiS/vYRARrolgD5mLQBj+tHPNv2MBInhznyFHavhxz+mJSmeiMbmYbna1/lYEjmRNnwcSx9Bxp5DboczpFgLwJh+Ul5fzjMFz3BX7FISPbFQWAiFhZQutknfzsVFUc6ayXvTxZkQrrra5YiGDvstNKafPJz3ML6Aj6/EX+4UrFsH8fE26ds5GuNNYYwnmb+lBReGLyhwN6AhxLqAjOkHDW0N/CLvF1w79Vom16fDkSOwcydcey2BKPuzO5v1hzt3j02OSOC5pFLuAd5Z+zDLli1zJ7AhxloAxvSDn2/+OceajvF/l/9fp2DdOoiKgssuczewQWpmII28pCbaoyNIPXDE7XCGDEsAxvSxutY6fvzuj7km5xpn2ueqKti8GZYtgxE27cP5mKkjQaAyLY4USwB9xhKAMX3swU0PUt1czXc+8h2n4JVXwOuFq692N7BBbKqOxKvC/gwvqQfKbHnIPmIJwJg+VNNSw082/oTrp13PwtEL4eBB2LjR+faflOR2eINWLBFM1GS2ZviIqW101gg2F8wSgDF96NMvfJqalhrmZMxh9dbVvP8vdxAQeG9WEusPr+82uGl6b6aO5C8ZTc6D3baAYF/oVQIQkRUisldECkXknh7qo0Xk6WD9JhGZECyfICLNIrI9eHs4ZJ+FIrIruM+DInZZpBncyurKeP3A6yzIWsDYpLGMKD/O1Jc2Uj5/Cm0JcW6HN+jN0DS2ZAang7ZTQfvEWROAiHiBh4CVwEzgNhGZ2WWzzwEnVHUK8ADww5C6A6o6P3j7Qkj5L4F/AHKCtxXn/zKMcd8/v/bPBDTAx2d8HICFq/8MIpRe3PXPxZyPmYE0KuOhPiHaWgB9pDctgMVAoaoWqWobzuLu13fZ5nrgseD9Z4ErzvSNXkRGAYmq+l5w8fjHgRvONXhjwsVrB17jmYJnWJmzkvT4dFIKy8h5eSO7P3kZrYnxboc3JIwjkRFEUjg6xloAfaQ3CWAMUBLyuDRY1uM2quoDaoGRwbqJIpIvIm+LyLKQ7UvPckwARGSViOSJSF6VLQZhwlCLr4Uvr/0yU0dO5apJzhTPix/6E+1xMWy/0xq2fcWDMFPT2DGy3WkB2JlAF6y/B4HLgXGqugD4Z+BJEUk8lwOo6mpVzVXV3PT09H4J0pgL8cMNP6SwupCHPvYQkd5IsvL3M/6dnWy/cwWtyXbef1+aF8hgU0qTMydQWZnb4Qx6vUkAZcDYkMfZwbIetxGRCCAJOK6qrap6HEBVtwIHgKnB7bPPckxjwl5+eT73v3M/t86+lSsnXQmqLP6f52lMT2b3rZe7Hd6QM18zKej4HmjjABesNwlgC5AjIhNFJAq4FVjTZZs1wGeC928G3lJVFZH04CAyIjIJZ7C3SFXLgToRuSg4VnAH8GIfvB5jBkxjWyO3PXcb6fHp/HzlzwGYvG4LWTuL2LrqWvwxUS5HOPRM01T2ZQXnUrIEcMHOOiuVqvpE5G5gHeAFHlXVAhG5D8hT1TXAI8ATIlIIVOMkCYDlwH0i0g4EgC+oasdcrl8CfgfEAq8Eb8YMGl9/9evsO76PN+54g5FxI6GhgYt+9hxV08ex97pL3A5vSIrES/KYyVQlHSDdEsAF69W0hKq6FljbpezekPstwC097Pcc8NxpjpkHzD6XYI0JF8/ueZbf5P+Gey65h8snBrt67r+f+Koa3vjBKtRr11j2l5yROWxJ38vVeVvwuh3MIGe/pcaco6ITRfzDS//AotGLuO+y+5zC/fvhv/+bfR+7iAqb779fTU2dypYx4Hn/A2hsdDucQc0SgDHnoMXXws3P3AzAUzc/RaQ30jkd8Wtfg+hoNn31JpcjHPompkwkP9uLBAKwbZvb4QxqlgCMOQdfWfsV8o/m88SNTzApZZJT+L//68z4+Z3v0JxmE771tyhvFP6FH3IebNnibjCDnC1NZEwv/Tb/t/wm/zd869Jvce3Ua53Cmhrn2/+4cRATYwu9D5C5cz9KcdIWRm3aSKTbwQxilgCMCbF66+oey8vqyvjPDf/JtJHTGJs09uR2l37/D0yvrGD7nStoKP3bQIY6rC0fv5wto7/PyvfetQRwASwBGHMWbf42Hv/rz0gQL/9ZPouRL2wAILGkkpnPr6d08XQaslJdjnJ4WTp2KT8YLXz8zSNQXQ2p9v6fDxsDMOYsni54msNSy7d8SxlJLADi8zN17SZaEuM4tHyuyxEOPwnRCdTOneo8yMtzN5hBzBKAMWew5cgWNhRv4FOBWSzSUSfLx2/YRdzxOvZ/bAmBKOuEcEP25TcA0LjxbXcDGcQsARhzGtXN1fx+5++ZlDKJu/ynvuXHV1QzduMejs6ZxIlJo12McHi7fMFN7EuF6vWvuR3KoGUJwJgeqCqP73gcVeVzCz5HRPBPRfwBpv75Pdrjoim68kMuRzm85Y7OZde4aOK329oA58sSgDE9WH94Pe8fe5+bZ95MWlzayfKxGwtIqDhB4dWL8MVGuxih8YiHlg/NIbW6GX9Z6dl3MN1YAjCmi6rGKp57/zlmpM1g2bhlJ8vjK08wbsNuKmeM59j0cS5GaDpkfeTvANj/6h9cjmRwsgRgTIiABnhsx2OICHfMu4OOlU3FH2DqSxvxxURReHWuy1GaDh+69h9ojoDqtc+7HcqgZAnAmBDrD69nf/V+bpl5C6mxp84tH/uu0/Wzf+VifHExLkZoQqWkjGLXtGQyN+50O5RByRKAMUEltSU8//7zzEibwSVjT83nn777IOM37KJi1gSOTxt7hiMYNzR8+GIml7dw9AO7HuBcWQIwBuesny++/EUU5fY5t5/s+olsbOGKf/8NrYlxFF69yOUoTU+yP/5ZAPY9/QuXIxl8epUARGSFiOwVkUIRuaeH+mgReTpYv0lEJgTLPyoiW0VkV/Dn5SH7/DV4zO3BW0afvSpjztFTu5/i5f0vc92060iPTz9ZvvTHTzGi/DgfXLfUlngMUzkfvonKBA+89rrboQw6Z00AwTV9HwJWAjOB20RkZpfNPgecUNUpwAPAD4Plx4C/U9U5OGsGP9Flv9tVdX7wVnkBr8OY81bVWMXXXv0ai0Yv4oqJV5wsn/zqZqb9eSP5d32MurH2/SRciddLcW4O03eUUt14zO1wBpXetAAWA4WqWqSqbcBTwPVdtrkeeCx4/1ngChERVc1X1SPB8gIgVkTs5GkTNlSVL639EjUtNTxy3SN4xPmTSN1XwvLvPcHRuZPZ9vlrXI7SnE36DbeT0Qhv/Om/3Q5lUOnNbKBjgJKQx6XAktNtE1xEvhYYidMC6PBxYJuqtoaU/VZE/DjrBn9PVbXrk4vIKmAVwLhxdu616VtPFzzNs3ue5fuXf585mXPYWLqR6JoGrvrXX9KWEMfrP/pHNMJWng03Xaftjs2J4tNA0dMPs3r6BFYtXOVOYIPMgAwCi8gsnG6hfwwpvj3YNbQsePt0T/uq6mpVzVXV3PT09J42Mea8HG04ypfXfpnFYxbzb5f8G+DM8nnlN1cTd6yW1378BVvha5Bozkjh8NhEFhac4GjDUbfDGTR6kwDKgNBz37KDZT1uIyIRQBJwPPg4G3gBuENVD3TsoKplwZ/1wJM4XU3GDAhV5R///I80tTfx2A2PEeGJAJ+Py+79LWO27OWdb/09VbMnuh2mOQdVS+ex7DBsL9zgdiiDRm8SwBYgR0QmikgUcCuwpss2a3AGeQFuBt5SVRWRZOBl4B5VPblckohEiEha8H4kcC2w+4JeiTHn4MFND7Jm7xruv/x+pqdNB78fPvMZpry2hU1fuYn9117sdojmHB25cgkxfsha9zcCGnA7nEHhrGMAwT79u4F1gBd4VFULROQ+IE9V1wCPAE+ISCFQjZMkAO4GpgD3isi9wbKrgEZgXfDD3wu8Afy6D1+XMaf19qG3+ZfX/oXrp13P1y/6OrS0wOc/D08+yeYv38COz1ztdojmDE677rIqFaMT+eTmOv5y8C9cMemKnrczJ0kP465hKzc3V/Ns9R9zAUrrSlm4eiHJMcls/vxmksqOwSc+Adu2wf33s/rqtB73s8XeB4eImjqW/uIl/vXn1/FfX37R7XDChohsVdVuk1jZmsBm2Ghqb+LmZ26mqb2Jv9z+Okl/fB6+/nXweuFLX4K0NPugH+SK/u5SlvzyJdKefomiTxUxKWWS2yGFNZsKwgwLDW0NXPPkNWwr3sSb3s8x8yO3wF13waxZkJ8P8+a5HaLpA03pyRy8eAZ3bFe+99Z/uB1O2LMEYIY2VeoK9/CfX1vI5/77bep/Gsfib/wMoqPh+edhwwYYP97tKE0fKrrxw4yuh6oXfs/eY3vdDiesWReQGTpUYd8+yMuDXbtg1y78W7eQWFHF/UBr0ggOfXgOB6/4EMWXzAZPFeT/BoDph63rZ6govnQOgbSRfGFrDfetv48/3GSLxZyOJQAzuKzufAUoNTWwY4fzgV9UBI2NAKjXS3VmAuuy6shb6OWmaTdw6Y+e4u0djw58zGZABSIj8Hz1a1xz773815onKbj0W8zKmOV2WGHJEoAZVNYfXo/4/KR/UMyobftJKq0CoDl5BLVTssicczE7xkbyjbh3ec2/l8uipvFI0h1MjEiDCPt1Hzb+9V/x/+bX/M+6Mv79ym/wwu0vnZzi25xifxFm8KipYfz6nYzauo+o5laaUhM4tHwux6aNpSIthne8Zbwes55tvmKSArH8MvFTrIpbdnKCN1avtq6e4SI2Fu8DP2X2xz/OmD++zENTH+LuxXe7HVXYsQRgwl9tLTzwAPz0p4yvreV4zhgO5U7h3YmR7PRUscmzmQI5RkCUmYzil4mf4u9jlzDCY0s3Dms33ohecQU/eHs9U5//J3JH53JR9kVuRxVWLAGY8NXaCg8/jH73u8jx45RcsYgHJ1Xy8qgG9sl6/OJcxDgtkMrfB2axJDCaL2bdZE194xBBfvYzRsyfzzN/iuDvR97Ce1/OJy2u54v9hiNLACa8rF6N+v2UbXqdhJdeI6m6kQ2TIvj6x2Hb6C1Eq5dpmsqtgRnMCqQzS9NI5tQ3/XeK33ExeBN2Zs1Cfv1rln32s/zkt2V8IvUmnvvUi6TEprgdWViwBGDCQnl9Oa/uW8vxDT9m5WtFzKoIkJ8FP/n0CPwzp/PZ6Cn8OmoyNUeKiLDLV8y5uPNOqK/n+q9+lcYHN3BJ/WKeu30NM9JnuB2Z6ywBmAETuoiHqlLeUM628m28X7adi/9Wwjf+BrOqoCw1gidvnIR3+kxWSQLUA/XQwCH78Dfn5ytfgYYGPvWtbzHvR4f4/L75zPnYZ1mQtaDHLsPhsqCMJQAzoI7UH2HLkS1sK81j3P5K7tgBjxV4SGmCo5MyeH/pFKpmjCfbYx/0po9985swaxbTvrCK9Q9X8Mu8X/HUirHMX/pxpqdNH5ZjR5YATM+6XnAVatU5fDtSZd/hfN549/e899bjpJUc564S+MNhD8lN0B4VweGPLODd65ZStmQG01+wPnzTj44eJeIb36T0sZ/zxa0H+NKWEl7J+SlvzBvBieuuZvbki0iMTnQ7ygFjCcCcniocPw7FxVBeDpWVUFUF//VfUF3tnJ4ZCDjbeTwQGQlRUajHg0/9+PztRDa1MDUAU4EvBQ9bm5nM0Stmsm3RNA4vn0f7iFg3X6UZbmJjKVq5hNJL55CRv49l2/dxzf4Gmv/0HK9Nfo78eRk8fP1ecj98Kx8avfDUdSRDkK0HMJSFfItv1XYO+6s56D9Gsb+a0nmTKK0r5VjzMWpaamhoqGbE4aNMrfAxp6ydhYdamVXuJ7n51O9HfVIsDelJtCfEo/Fx+GOiaJMA7QRoCLRQ46unrr2B44FG/AE/iJCUls2kCQuYOXM570RXUDMhi4lvboVh2Nw2A2f5+OVnrF8fekFgIEBSSRXNzXVMensnGcdbAChOhC0TI6mal0PMxcuYuOw65k1eSnJMcj9G3j8uaD0AEVkB/Axn9a7fqOoPutRHA48DC3HWAv6kqh4K1n0T+BzgB76qqut6c0zTC4EAVFRAcTH+I0eoK95HXekBGo6X01RTRXPpIeoDzTQGWmjUNgLBz9xoP0wPwKX+aLKavaTX+0k/0YY34HzY+zxQlBHBm9OjyB8lbBodYFN6G/XRzUDzacPJ8iQyI2IU8xas5IpJV7B8/PJOzemKjkFg+/A34cTjoXZ8Jh/cdAs7v6kkllSy8nAU+srzXLZ1N6k79sDjewjwKwpTYduoOJrGj8YzaQpJU2aROW0ho6cuJC57AkRFuf1qzslZWwAi4gX2AR8FSnHWCL5NVfeEbPMlYK6qfkFEbgVuVNVPishM4I84C76Pxln6cWpwtzMesyd93QIIaABfwEebv402fxutvlaa2ptoam+ioa2B+rZ66lrrqG+tp7a1lrrWOupa62hsa6ShvYGm9iZ8AR++gI/DNYfxiAevePF6vERHRBPljSLGG0NsZCxXTrqSxOhEEqISSIhOID4yntjIWGIiYoj2ROENKF5fAGltI9DYgL+hjkDNCdqOVeA/VkWgohw9cgTv0QpijlQy4mg1SccaiPR3//9r9UJDFLR7AY8HD4IXD14VIhDwevEljsAXG01zSgItKSOoH51GzYQsaiZkcWLSaHJe3tj5vUJpoI1G2mmSdprxARBDBDHqJYkYEhhcv/zGnJUq0XVNeI9W0FpZTmzVCZKqmxh1op249u6b18V5aUiKpSUlAV9qCjoyFUlLIyI9k4iR6USOzCAqNZ2I5BQik1KJHJGEN34Entg4J3lERjpzVvXxl6QLaQEsBgpVtSh4oKeA64HQD+vrgf8I3n8W+Lk4Q+rXA0+paitwMLhm8OLgdmc7Zp+56embeLXwVQIaQNGTH/znIzYilhFRIxgRNYLYyFgiPZFEeiNpbGskQAB/wH8yqbT6W2nxtRDQAC988MLJY4xohcofgzcAHoWIXvbCtXmgPAGKk6ByVBS1c0fSMGokgewxRI+dyPveaiIyR5GclElqbCpzX3zvtMf64KYzN5G78iAkEk0i0TB4eg2NuTAitCbFQ9IkmDaJBqABmDxuGc11xyk5XkTF8WIaaqpojfJARQXRJ+qIq6kkdV85I5tgZLPT6j4XAQG8XjwerzO+JuIsXDRtWp++vN4kgDFAScjjUmDJ6bYJLiJfC4wMlr/XZd8xwftnOyYAIrIK6DjtpEFEXF3hoTn4r4qq8z5GAxDXuSgNOHbWHQNAbfBW3Bbc5RhwHm/J98NujvTevQdDm70Hg+Y96Ne/nzSUY/j8OD3nQdOnX8gxe1z1KOzPAlLV1cAZzkkc/EQkr6fm2XBi74G9B2DvAQzse9Cb85vKgLEhj7ODZT1uIyIRQBLOYPDp9u3NMY0xxvSj3iSALUCOiEwUkSjgVmBNl23WAJ8J3r8ZeEud0eU1wK0iEi0iE4EcYHMvj2mMMaYfnbULKNinfzewDueUzUdVtUBE7gPyVHUN8AjwRHCQtxrnA53gds/gDO76gC+rqh+gp2P2/csbNIZ0F1cv2Xtg7wHYewAD+B4MqgvBjDHG9J2he42zMcaYM7IEYIwxw5QlAJeIyC0iUiAiARHJDSmfICLNIrI9eHvYzTj70+neg2DdN0WkUET2isjVbsU40ETkP0SkLOT//2NuxzQQRGRF8P+6UETucTseN4jIIRHZFfx/H5BJz8L+OoAhbDdwE/CrHuoOqOr8gQ3HFT2+B8EpRG4FZhGcQkREpnacQDAMPKCq/+V2EAMlON3MQ4RMDSMia842NcwQdZmqDtiFcNYCcImqvq+qrl7V7LYzvAcnpxBR1YNA6BQiZug5Od2MqrYBHVPDmH5mCSA8TRSRfBF5W0SWuR2MC3qafmTMabYdiu4WkZ0i8qiIDIfVy4f7/3cHBV4Tka3BKXD6nXUB9SMReQPI6qHq31X1xdPsVg6MU9XjIrIQ+JOIzFLVun4LtB+d53swpJ3pPQF+CXwX58Pgu8BPgLsGLjrjoktVtUxEMoDXReQDVV1/1r0ugCWAfqSqV57HPq1Aa/D+VhE5gDOF9qBcCed83gOG+FQhvX1PROTXwJ/7OZxwMKT/v3tLVcuCPytF5AWcrrF+TQDWBRRmRCQ9OCiGiEzCmT6jyN2oBtzpphAZ8kRkVMjDG3EGyoe6YT81jIjEi0hCx33gKgbg/95aAC4RkRuB/wHSgZdFZLuqXg0sB+4TkXacCaC/oKrVLobab073HpxpCpFh4EciMh+nC+gQ8I+uRjMATjfdjMthDbRM4AVnGRUigCdV9dX+flKbCsIYY4Yp6wIyxphhyhKAMcYMU5YAjDFmmLIEYIwxw5QlAGOMGaYsARhjzDBlCcAYY4ap/w/1ogyo2GPOcAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████▋   | 2/3 [00:03<00:01,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DMPNN with rdkit 2d normalized wo MolLogP   365\n",
      "StructGNN with ext valence with rdkit wo fragments and counts   352\n",
      "logD\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxtklEQVR4nO3dd3xU153//9dnZqRRB4EaqAMSCBC9GRswuDuJMV4XnGI7jsPmt+tNvsnmkbKbtnbiTdndxBs7hdhe14AbBmwDNs2A6YgiqkACJCSKJAQC1dHMnN8fM2ABwgiQdGdGn+fjMQ/N3HPvnQ8j6a3DufeeK8YYlFJKhS6b1QUopZTqXBr0SikV4jTolVIqxGnQK6VUiNOgV0qpEOewuoCLJSQkmKysLKvLUEqpoFJQUFBtjElsqy3ggj4rK4stW7ZYXYZSSgUVESm9XJsO3SilVIjToFdKqRCnQa+UUiFOg14ppUKcBr1SSoU4DXqllApxGvRKKRXiNOiVUirEadArpVSIC7grY5VSnWD27LaXz5rVtXUoS2iPXimlQpwGvVJKhTgNeqWUCnEa9EopFeI06JVSKsRp0CulVIjToFdKqRCnQa+UUiFOg14ppUKcBr1SSoU4DXqllApxOteNUiFgdkHbc9nMGq1z2Sjt0SulVMjToFdKqRCnQa+UUiFOg14ppUKcBr1SSoU4DXqllApx7Qp6EblTRIpEpFhEftRG+/dEZI+IFIrIchHJbNX2qIgc8D8e7cjilVJKXdkVg15E7MDzwF3AYOBhERl80WrbgDHGmGHAO8Bv/dv2An4OjAfGAT8XkfiOK18ppdSVtKdHPw4oNsYcNMa4gLnA9NYrGGNWGmMa/C83AGn+53cAS40xNcaYU8BS4M6OKV0ppVR7tCfoU4EjrV6X+5ddzjeAxde4rVJKqQ7WoVMgiMhXgTHAlKvcbhYwCyAjI6MjS1JKAZw5A+HhEBFhdSXKAu3p0VcA6a1ep/mXXUBEbgX+HbjHGNN8NdsaY2YbY8YYY8YkJia2t3al1JXU1cFPfgI//jE89RSUlVldkbJAe4J+M5AjItkiEg7MBBa2XkFERgJ/xRfyla2aPgJuF5F4/0HY2/3LlFKdLPOT7dRnp8GvfkVlTl+aWhrx/ObXsH691aWpLnbFoRtjjFtEnsQX0HbgJWPMbhF5CthijFkI/A6IAd4WEYAyY8w9xpgaEXka3x8LgKeMMTWd8i9Rqjua7Zu1clDp6gsWR1WdZtRLi6lP6MG2R+/gbGoCYfVN5M3/lJ4vv+zr6d92mwUFKyu0a4zeGLMIWHTRsp+1en7r52z7EvDStRaolGq/JtyIx8vI99fjdoaxa+Y0WqJ94/It0REUPjyNyQt3wJIlcPPN1haruozOR69UkFvUtJNZYQupoZFGcfPLlTZuO+5l932Tzof8eTabryf/7LOwbZs1Basup0GvVJBpfZOR1NJlfC3sfWII44veAQw76uEHaw6wID+c2EFpbf+CDxoEiYmwevX5oZ8LzNKblYQanetGqSD2gn07tTTzM/dNPOkexXc+PEljdDiP3e3ibdu+tjey2WDSJDhwAI4e7dqClSU06JUKUodPH2aB7QAzvLnkml703l9O7PEajt48iqHONF6yF1LOmbY3njgRHA5fr16FPA16pYKQ13h5Y+cb9CKSb3iGgzFkri6koVcslUOy+a57LOHY+K1jI17MpTuIjYVRo2DDBmhuvrRdhRQNeqWC0Kdln1JWW8Y/e0YRTRgJ+8qIqTpN6aR8sNlIIIpveUaxw1bJVjne9k6mTIHGRti8ue12FTI06JUKQmvK1pDRI4Np3kzweslcs5P63nFU5Z2fIZzbvVlEGQcrbKVt76R/f0hK0rNvugENeqWCzNGzRymrLWNC6gQEIXFvGdHVtZROGuY70OrnxMFN3nRW2Y7gwnPpjkR8Z+AUF4OnjXYVMjTolQoymyo2YRMbY1PHgjFkrN1FfWIPqvMunRDwFm8mdeJisxxre2e5udDUBEeOtN2uQoIGvVJBxGu8bKrYRF5CHnHOOOJLjhJdXcuRG4b4eugXGWP60MM4WW473PYOc3N9X/fv77yileU06JUKImvL1nKy8STjU8cDkL5xL82xkReMzbfmwMYUbzprbeU04r50hR49IDlZgz7EadArFUReL3wdp93JiJQR9N5XRs/SE1SMHYSxX/5XeZo3iybxsN5W3vYKOTm+i6e83k6qWllNg16pINHsbuatPW8xPGU4ToeTYa8vxR3u4NiIAZ+73TCTSG8TyfLLnX0zcKBvnL78Mn8IVNDToFcqSCw6sIjTTacZnzqe6OM19F+6heMjBuCJCP/c7ezYmObNZKMc5SyuS1fIyfF9LSrqhKpVINCgVypIvLn7TZKik8hLyCN/znIAKsYOate2N3szaBEvm2xtzG0TH+87n/7AgY4sVwUQnb1SqQDVepZKj9fDgqIFjO4zmojGFgbN/5SDt4ymuUd0u/Y1yPQmxoRRcLmrZHNyfBdO6Th9SNIevVJBoLimmCZ3E8OSh5H7wXrC65vY+eVb2r29AxsjTDIFtuMY08bcN7m50NAAFZfc0lmFAA16pYLAjhM7cNgcDOqVy5A3V1I5JIuqodlXtY8x3hSOSz0HPdWXNp47n17H6UOSBr1SQWBn5U4G9h5IzuaD9Cw7wc6H29+bP2e0tw8Ay117L23s1Qt694ZDh663VBWANOiVCnAn6k5QWV9JfnI+Q+cupz6hB4duGXXV+0knlkQTxbLmy9yQJCMDysqus1oViDTolQpwhScKAbi5MYX09XvYc/8UvGFXfx6FIIz2prC8eR9e08ZB1/R0qKyEM5e5WYkKWhr0SgW4wspCUmNTuWnhNjxhDvbeN+ma9zXGpFBj6tnubuPiqAz/pGjbt1/z/lVg0qBXKoA1tDRQXFPM+Ng8cj7cQMntY2jqFXfN+xvlTQFgWXMb4/Tngl7npw85GvRKBbDdlbvxGi9f2wHhDc3sfnDqde2vN5EMdfRtO+h79PA9tm69rvdQgUeDXqkAVlhZSExYNFM+3OU7pXJI1nXv81ZnHmtcxTSZlksbMzI06EOQBr1SAcrj9bC7cjePncwg/vDx6+7Nn3NreB5NtLDOVXJpY0YG7Nnju3hKhQydAkGpQDPbN/WBp+w96sPq+acPTtDYM4aDt47ukN1PCh+ADeET136mOS+aKycjwzcNws6dMH58h7yfsp726JUKUOttFWSdhkEHatg34yY8zrAO2W+cLZJRYRmscrVxs5FzB2R1+CakaNArFaDWSQU/3RwFCHv+YUqH7ntKeC4bXYcuHaePj/ddIatBH1I06JUKQAfdVVR4a7l/u4uaAX2pT+nVofufEp5DM242ui6a8kAERo7UoA8xGvRKBaAPm3fypf0QV+/m2MicDt//pPAcBGl7+GbUKNi1C1xt3KREBSUNeqUC0AdNO/l2gZ3m2Chq+vXp8P33tEUx3JF2+aB3uXxn36iQoGfdKBVgznqbOHyiiCklHg5P6g82G4Pmre7w95kSnsPshjW4jJtwaRUFo/wTpm3dCiNGdPj7qq6nPXqlAswy114e2erBK3B8+Off+Pt6THHm0kgLm1sOX9jQvz/Exuo4fQjRoFcqwCyu38E3tsHJAam44qI67X0mhfvG/i8ZvrHZID/fdy69Cgka9EoFELfXTXPhNlLq4MSIzuvNAyTYYhjq6Muq5jZuCj5sGBQWQlu3HVRBR4NeqQCy6vAqHihoorFHNDX9+3b6+00Jz2VtSwlu47mwYdgwOH0aytuYzlgFnXYFvYjcKSJFIlIsIj9qo32yiGwVEbeI3H9Rm0dEtvsfCzuqcKVC0UdrX+XOEnBMmOgbQulkU8JzqTfNbG256M5Sw4f7vhYWdnoNqvNd8SdJROzA88BdwGDgYREZfNFqZcBjwN/b2EWjMWaE/3HPddarVMjyeD0435mHwwthEyZ2yXtO9o/Tr3RddFPwoUN9XzXoQ0J7ugzjgGJjzEFjjAuYC0xvvYIx5rAxphBo4/5kSqn2WFO2hhmb6qhJT4S+nT9sA5Bsj2OIo++lQR8XB9nZGvQhoj1BnwocafW63L+svSJEZIuIbBCRe6+mOKW6k3WL/sqo4xB9w+Qufd+p4bl86irBZdwXNgwbBjt2dGktqnN0xcHYTGPMGODLwB9EpP/FK4jILP8fgy1VVVVdUJJSgcXj9dDj7fdx2wXnuBu69L2nOQdRb5ovPZ9+2DAoKoKmpi6tR3W89lwZWwGkt3qd5l/WLsaYCv/XgyLyCTASKLlondnAbIAxY8bo+Vyq21l3aDUzttRzfNJo0mJjO/W9VpdeeJVtGM1IGKxoLuLG8AHn58PnxAnf3PS//KXvoYJWe3r0m4EcEckWkXBgJtCus2dEJF5EnP7nCcCNgE6godRFds15lr510Psf/1+Xv3ccTnJML1a49l3YkJbm+6qnWAa9Kwa9McYNPAl8BOwF3jLG7BaRp0TkHgARGSsi5cADwF9FZLd/8zxgi4jsAFYCvzbGaNAr1Yrb6yZh/sfURYcROeMBS2oYZZJZ5zpIo2k1Y2ViIoSFadCHgHZNamaMWQQsumjZz1o934xvSOfi7dYB+ddZo1IhbcXeRdyxs5GqL04jxum0pIaR3mTm2vey3nXws9sL2myQmgoV7R6pVQFKr4xVymKFr/6OOBekfuO7ltUwzCRhx3bp8E1qqq9Hr1MhBDUNeqUsVPuXZ0lbvJa6KAfhpUc+OxDaxaIIY1xYFiuaLzqfPi0N6up8B2ZV0NL56JWy0LwzG7m/yHBkcDJV5WstrWWacyC/rvuIs94mYm0RvoXnDsgWFkJKinXFqeuiPXqlLHRgx0piXeDKG2h1KUwLH4QHL5+6ij9bmOq/NlIvnApqGvRKWaSkpoT8wuM0RIdTm5lsdTncEN4PJw6WulqdGBcdDfHxGvRBToNeKYvM2fgiXyoC78iRXTJT5ZVESjg3O3NZ3LT7woa0NA36IGf9T5dS3ZDXeDn61gvEtEDM2K6ZqbI97nQOYZ/nOIfd1Z8tTEuDvXt1KoQgpkGvlAU+OfwJkzdX0RQTAbm5Vpdz3l1O3/TES5pb9erT08HjgT16rWOw0qBXygKvbvobdxeDY3hgDNuck2tPJtuewOLWQX/uzJvt2y2pSV0/Pb1SqS40u2A2DS0NnPzgbeKaYWeqk1MXTTJmJRHhTucQXm3cQLNpwSlhvqkQoqN1nD6IBU5XQqluYnPFZr60x0NzZDinswLv3PS7nEOoN82sdfknmbXZdG76IKc9eqU6yeyCz65yHTTP12sfBPzBtphni4TarBSMw25RdZc3NXwg4ThY3Lzrs3lvhg+HOXN8UyGIWFugumrao1eqC5XIKXpVnCKh3nAyN/3KG1ggxhbBpPABF47TjxgBtbVQVnbZ7VTg0qBXqgstspVw3z7w2G3UDLiaO3J2rbucQ9ntPsoRT41vwfDhvq96QDYoadAr1UVa8LBUDvHQPge1WSl4nGFWl3RZdzmHAK1Os8zP9w3Z6Dh9UNKgV6qLbJCjZFS6SD3lpjr3kts3BJQ8Rx8y7b35sGmnb0F0NOTkaI8+SOnBWKW6yFL7IR7e58Dg5mRO4AX9xfeSHWXvzRLPHhqNi0jwjdNv2WJFaeo6aY9eqS5wlmbWSQUP7bNzJi2BlphIq0u6ohu9aTTSwvJm/81Ihg+HgwfhzBlrC1NXTYNeqS6w0lZG0hkv/Y43B2Rvvi0jTBKxEsH7zYW+BecOyBYWWleUuiYa9Ep1gY9th3isyHczj2AJ+jDs3OkcwvtNhXiN1zd0AzpOH4Q06JXqZNUN1ey0VTGzKIzG+Fgae8dZXVK75dSFc8xby09W/ITZxz6goVcsRR/PsbosdZU06JXqZBvLNxLTDHmH631n2wTRlaUTvKnYjLDjxA4QoXpQJgl7S60uS10lDXqlOpExho0VG/nH4h7YPV5qcgL3Iqm29MBJvkmk8IRvXL5qcCbxB49CQ4PFlamroUGvVCcqrS3lRP0JHt4XRktkOLVpiVaXdNUmelMpP1POyYaTVOdlYvMaHacPMhr0SnWiLUe3EO61kV98hpr+qQE193x73ej1HTwuPFFI1eBM30I9nz6oBN9PnVJBwmu8FBwr4MtnswhvdAXN2TYXSyeO5OhkCisLaUjsSX1CDw36IKNBr1QnOXTqEDWNNTx0MBKv3capfn2sLuma5Sfns//kfprcTVTnZUJBgdUlqaugQa9UJ9lydAsOm4Px2yo5nZkc0JOYXcmwpGG4vW72Ve/zDd/s3Qt1dVaXpdpJg16pTuDxeig4VsAXvAOIL6sK2mGbcwb0GkCEI8I3Tp+X6bsBybZtVpel2kmDXqlOsKZsDbXNtXz1sO/iqGA7rfJidpudIYlD2FW5i8qB/j9aOk4fNDToleoEb+56k3B7OBO3VVM9MJ3muGirS7pu+cn51DbXsj/sDKSladAHEQ16pTqY2+vm3b3vMikqj5SdhyidPMzqkjrE0MShCEJhZSGMGaNBH0Q06JXqYCsPraSqoYqvlvfC5jWUTh5udUkdItYZS1bPLHae2OkL+v37dcriIKFBr1QHm7trLrHhsdy0/RT1iT2pHpRhdUkdZljyMEprS6kZ0s+3YOtWawtS7aJBr1QHcnlczNs3jwf6fYnMDXt9wzZBNInZ5Qyat5pB81Zzz64WABbvmu9r2LTJuqJUu2nQK9WBlpYs5XTTaf7x7EDCGpspnRQa4/Pn9Dc9STRRzAs7AP37w4YNVpek2kGDXqkO9ObuN4mPiGd0QQUtkU6Ojh1kdUkdShDGefuwvHkf3gnjYf163zn1KqBp0CvVQZrcTczfN58ZA+/F/v6HlE8YHNRXw17OGNOHWtPI4bw+cPw4lOr89IGuXUEvIneKSJGIFIvIj9ponywiW0XELSL3X9T2qIgc8D8e7ajClQo0S4qXcNZ1lifWNkJFBa7oiPNj26FktDcFQViWXO9bsG6dtQWpK7pi0IuIHXgeuAsYDDwsIoMvWq0MeAz4+0Xb9gJ+DowHxgE/F5H46y9bqcDz5u43SYhKYPyeWhDh5IC+VpfUKXrgZHRYBm+YQoiO9g3fqIDWnh79OKDYGHPQGOMC5gLTW69gjDlsjCkEvBdtewew1BhTY4w5BSwF7uyAupUKKPWuehYWLeT+vPuxFe6E/v1xR0VYXVanuS08j7XHNuIePUqDPgi0J+hTgSOtXpf7l7VHu7YVkVkiskVEtlRVVbVz10oFjvf3v09DSwOP9pwC5eUwYoTVJXWq252D8RgPB8PrfJObPfcczJ7te6iAExAHY40xs40xY4wxYxITg+9Wa0q9Xvg6aXFpjNta6VswLLROq7yY+/hRIo2Dd3qfAK+XHRsXWF2S+hztCfoKIL3V6zT/sva4nm2VCgpV9VUsKV7CV/K/gm3hQujTB5KTrS6rU4VjZ7hJ4s30WgBiK6otrkh9nvYE/WYgR0SyRSQcmAksbOf+PwJuF5F4/0HY2/3LlAoZb+5+E4/x8EjmPbBqVcj35s8Z6+1DYUw9Z+OjidOgD2hXDHpjjBt4El9A7wXeMsbsFpGnROQeABEZKyLlwAPAX0Vkt3/bGuBpfH8sNgNP+ZcpFTJeL3yd4cnDGbylFNxuGB4ak5hdyRiv79aI+9MiiSuv1gunApijPSsZYxYBiy5a9rNWzzfjG5Zpa9uXgJeuo0alAtaBkwfYWLGR3932O/jDe74hm+xsq8vqEpnEkWgiWZXuYfTOJqiuBj3GFpAC4mCsUsHqjZ1vIAgPD5gBixfD9Olg6x6/VoIw2tuH99L9944tKbG2IHVZ3eMnUqlOYIzh9cLXmZY9jdTN+3w3y54xw+qyutQIk8S65BZcTocGfQDToFfqGm0o30DJqRK+Ouyr8N57EBsLU6daXVaXGulNxmuD4rQoOHDA6nLUZWjQK3WNXtz2IlESzn3rTsFbb8GgQfDKK1aX1aVSiKGPieHTDIFjx3z/q1EBp10HY5VSFzrbfJa5u+YyM2IsB9cvZsTZs+xJdVJdGloTmLXHCG8S72aVMQuguNjqclQbtEev1DV4a/db1LfU80TUjSTsP4LXbuNU/9CcxOxKRppkPkl143XYdfgmQGnQK3UNXtj2AnkJeUxwZJNQdIRT2SkhOfd8e4z0JuNywNGMeO3RBygNeqWu0q7KXWwo38ATo55AKiqIqK3nZG76lTcMUUlEk2piWJ9hg7IyHacPQBr0Sl2lF7e+SJgtjK8N+xps3YoR4WROm9cLdhsjvMnMTa8Fr1enLQ5AGvRKXYXnNz3P37b+jWHJw3hv7zwaNqzhdGYyLdGhO/d8e4w0ySxNbcaIwJo1VpejLqJBr9RV2H58O/Ut9dyYfiO9DpQTVXOWqrwMq8uy3EhvMmcjoDK1hwZ9ANKgV+oqrDy8ksSoRPIS8+i/tMA3bNONx+fPSSCKXHsy6zPtsGEDNDdbXZJqRYNeqXYqOFpAyakSbs66GRtCv2UFnM7SYZtzpjoH8k7aGWhqgoICq8tRrWjQK9VOf9z0R5x2Jzem30jvoiP0OFJJVV6m1WUFjCnhOXyU0eJ7sWqVtcWoC2jQK9UOlfWVzNk1hxvSbiAyLJJ+ywrw2m1U67DNeVPCc6mOhursFA36AKNBr1Q7zC6YjcvjYmr2VDCG/ku3UD4+D3eU0+rSAkZfe09y7ElsHBABn34KLS1Wl6T8NOiVuoIWTwt/2vwn7uh/BykxKSTsLSWuopqDt462urSAc3N4Lm8lVkJ9vY7TBxANeqWu4O09b3Os7hj/Mu5fAMhZtBFPmIPDN4+wtrAANMWZy+K+Db4Xn3xiaS3qMxr0Sn0Or/HyzJpnyEvI466cu7C1uMlZvJHDNw/HFRdtdXkBZXXpapyVp6iKgSNpcRxZ+BqzC2ZbXZZCg16pz/Xe3vfYXbWbn07+KTaxkbGmkIjaevZ/caLVpQWkJKJIikpiXb8wUraXIG6P1SUpdD56pdo0u2A2XuPlV6t/RXJ0MrXNtcwumM0d76+nPrEn5RMGW11iwMrpncP7fTfzUKOLxL2lMN7qipT26JW6jMIThZSfLefunLuxiY3Ik2dIX7eLA3ePx9j1V+dycnvn8nGaC4A+BfstrkaBBr1SbTLG8OGBD0mMSmRs37EADFi8EZvHS9GXdNjm8+T2zj0/Tt+3oMjqchQa9Eq1aWflTspqy7gr5y7sNjsYw8D313EiP5varBSrywtoExfvoq+JYXOKh75b9sOf/2x1Sd2ejtErdRGXx8U7e94hKSqJCakTAEjYW0qvkqMcuHMcg+Z1v/vCXq0R3mQWZJVy3xY3lJZaXU63pz16pS7yvxv/lxP1J3hwyIO+3jww5K1P8IQ5qBysc9u0x0iTzOIst+9FkQ7fWE2DXqlWjp09xn+s+g/yk/LJT84HIPLkGQZ8tJnjw/rhiQi3uMLgMMqbTFUMVCRGaNAHAB26Ud3G5S7emTV61vnnP1z2Q1weFw8OefD8srx3V2FvcXN0zMBOrzFUJBBFpoljTT83M7cW++and+q8QFbRHr1SfmvL1vJa4Wt8/4bvkxSdBIDN1cLgd1dTduNQGnvHWVxhcBnlTeGd7Cbf5GZ6H1lLadArBTS2NPLE+0+QFpfGv036t/PL+3+8haiTZ9j58C0WVhecRntTWJbp9d1HdsUKq8vp1jTolQJ+vPzH7Kvex0v3vER0uH8OG2PIn7OcU9l9qBifZ22BQWiESeJshFCR1kOD3mIa9KrbW35wOc9ufJYnxz7Jbf1vO7+8z9YDJBQdYdfMaSBiYYXBKRYnOSaeT7KBjRuhrs7qkrotDXrVrTW0NPDYgscY2Hsgv7ntNxe0jfrbBzT0jmP/FyZYVF3wG21SeD2zFtxuWLPG6nK6LQ161a3N3TWXY2eP8eqMV4kKizq/vE/BflK3FLH90Tv0lMrrMMqbwup0gzfMocM3FtKgV93WpopNbKzYyE8n/5RxqeMuaBs9+30aesex977JFlUXGvJNIp5wB4fyUjToLaRBr7ql6oZq3tj5Bv3j+/Pvk//9wsZPPqFvwX62P3an9uavUwQOJob34+NMN2zbBjU1VpfULWnQq27H4/Xw4rYXAXh85OM4bBddN/iLX1Cf0IO9MyZZUF3ouTU8j9cTj4MxsHKl1eV0S+0KehG5U0SKRKRYRH7URrtTRN70t28UkSz/8iwRaRSR7f7HXzq4fqWu2qIDizh46iBfyf8KCVEJFzYuWwarVmlvvgPdFTGUTangcobBs8/C7Nm+h+oyV5wCQUTswPPAbUA5sFlEFhpj9rRa7RvAKWPMABGZCfwGeMjfVmKMGdGxZSt1bQ6dOsSi4kWMTx1/ybg8Hg9873uQlcU+7c13mJGOdBLDerC9vzBuzx5fz15PV+1S7ZnrZhxQbIw5CCAic4HpQOugnw78wv/8HeA5Ef1OqsDS7G7mpe0v0TOiJw8Pffj88nNz4Ayat5rJO3eyZ8ZN5Hyol+x3FBHhbudQ5mZvYtyeFqishORkq8vqVtozdJMKHGn1uty/rM11jDFuoBbo7W/LFpFtIrJKRNrsJonILBHZIiJbqqqqruofoFR7vbv3Xarqq3hs+GNEhkUCvnAfNG81Q/6+jAl/eIfatESqB2VYXGnouds5lPn9W3wvdu+2tphuqLMPxh4DMowxI4HvAX8XkUtmhjLGzDbGjDHGjElMTOzkklR3tKR4CatKV3Frv1sZmHDpLJQZ63YT3tBMyW2jdVihE9zmHEx5LzvVvSNhz54rb6A6VHuCvgJIb/U6zb+szXVExAH0AE4aY5qNMScBjDEFQAmQe71FK3U1qhuqeXzB4/SN7cv0gdMvaY84dZbUzfs4kZ9NXZ/ebexBXa9YWwSTw3NYOkBg/37flbKqy7Qn6DcDOSKSLSLhwExg4UXrLAQe9T+/H1hhjDEikug/mIuI9ANygIMdU7pSV2aM4Zvvf5OTjSd5fOTjhNnDLl6BnEUb8dptHLp5hCU1dhdfcOYzN7vBNzd9SYnV5XQrVwx6/5j7k8BHwF7gLWPMbhF5SkTu8a/2ItBbRIrxDdGcOwVzMlAoItvxHaT9ljFGr5hQXebFbS8yf998npn2DOlx6Ze0JxceJL70BIemjcQVG9XGHlRH+UJEPiuzwGMTHb7pYu26w5QxZhGw6KJlP2v1vAl4oI3t3gXevc4albqsz7tr1IGTB/jOku9wS/YtfPeG7/LC1hcuWCeyupZ+y7dSm57IsZE5XVFut5ZjTyIpOpG9GfUM1QOyXUqvjFUhqdndzFfmfQWn3cnL976MTS79UZ/4329ib3Gz/+4JegC2C4gIX3Dm83a/JjhyxHeapeoSGvQqJH33o++y+ehmXrznRdLi0i5p7/fxZvovLaD0pny9RWAXujdiBB8M8PpeLFlibTHdiN4cXIWc9eXreXn7y/xg4g+YkTfjkva4shNM/uVrHB/Wn/IJgy2osHtZXbq61SsvpSlOTsZ56b1gATzyiGV1dSfao1ch5UjtEd4ofIOpWVP51S2/uqTd3uTi1h/NxhvmYPkzT2Ds+ivQlezYmGKyeCfXg1myBBobrS6pW9CfchUyzjSf4S8FfyE6LJo5/zDn0lkpgRv+520S9pez8j++Tn1KLwuqVNO8Gcwb5EUaGmD5cqvL6RY06FVIcHlc/Gnzn6htquVbY75Fcsylc6kMmbuCwfNWs/3ROzhyU74FVSqAISaR4n49qY90wIIFVpfTLWjQq6DnNV5e3Poih08f5olRT5Adn33pSm+8wY3/9SaHpo5g8/936dWxquvYEGbEjuXD/h68Cxf4Zg1VnUqDXgU1Ywxv736b7Se28+CQBxmRMuLSlT78EB57jIqxA1nxyycwDnuX16kuNDNiDO8NNNgqq2DjRqvLCXka9CqoLSpexIrDK7gl+xamZU+7dIUPPoD774cRI/j4v/4JjzPs0nVUlxsdlsmesVm02EWHb7qABr0KWn/c+EcWFi3kDk82PylKPj/l8HkvvAD33gtDh8LixbRER1hWq7qQiPCFMQ/zSZbBPU8vnu9seh69CkqvbH+Fby/5Njd50/iBZwI2Wl3Zagw89RT8/Odwxx3wzjsQE8Og36++/A5Vl/vasK/xx4H/yW2LSmDvXsjLs7qkkKVBr4LOW7vf4vGFj3NL9i38sCgRR6v/mIrbw/Sv/wZ2HeJEfjb7R/TC/Oe3LKxWXU5eYh7Hb5uAZ/EGbK+9hjzzjNUlhSwdulFB5d097/Lld7/MxPSJzJ85HyefHVgNq2tk+BvLSN51iMOTh1H0xRv0gqgA9+Ct32HJAGj6v7/p2TedSH8LVNBYsG8BM9+dybjUcSz68iJiwmPOt0VVnmbky0uIrjzFnvsmUXZTvk5UFgRmDJrBvPFxRB6vhhUrrC4nZGnQq6Dw7p53eeDtBxjdZzRLvrqEWGfs+baeh44x4rWPEa9hx9du13u+BhGnw0mfh79JTSQ0/O1PVpcTsnSMXgW8V7a/wuMLHyerZxYzh85k7q6559umFJaQs2gjjb3j2PXQVJrjoi2sVLXX6tLV7CvwPY+MiWfOUHhi/kI4fRp69rSytJCkPXoV0J7f9DyPLXiMqVlT+c747xAV5r8LlDGM+L/FDPxgA7UZyWz/2u0a8kEqMTqRVVOzcbZ48cydY3U5IUmDXgUkYww/W/kznlz8JNMHTueDL39AhMN/HrzXyw3/8zbjnp9P5ZAsdj10M56IcGsLVtclYdId7EqEU3/5vdWlhCQNehVwXB4Xj85/lKdXP83XR3ydtx94+3zI21rcTP35/5E/Zzk7H57GvnsmYuw6pUGwG5YynHnjY0nYcQDvrp1WlxNyNOhVQDnddJq737ib1wpf46mbn+LFe14kzO6btiD8TD13/cv/krN4E5v+aTrrv/egnlkTImxi4+QDX6Q+DCr+7V+sLifk6MFYFTCKa4r50pwvUVxTzCv3vsIjw1vdfejwYaZ/43fEHalkxVNfp/juCdYVqjrFoLxJzLlpEV//YBVm/34kN9fqkkKGBr2y1OyC2QAUVRfx14K/ArDsa8uYkjXls5WWLoWvfpWYM6fZ9dBUHE2uC+e0USHBbrMT+aOf4lrzfWp+/CSp735sdUkhQ4NeWW5N6Rr+vuvvJEcn889j/5mik0UUnSxC3B7G/GUhI19eQk2/vpTcezuNCT2sLld1kLb+WE/ImMhrN8Tw6PylmJISpH9/CyoLPTpGryzj9rqZs2sOr+98nbyEPH544w9JjE4EIGFvKdO/8VtGvryEvTMm8d6rP9aQ7wbCxYHthz/CLTpW35G0R68sUdNYw4NvP8gnhz/htn63cV/efdjERsSps4z903wGzV9LY3wMS389i0O3jra6XNWFHr7jX3l74m+Z+c5iXPt2Ez5oiNUlBT0NetUlzo3FAxw5c4S/bPkLp5tO8+jwR5mYPpHwM/Xk/305+XOW42hysfPLt1DwzS/SEhNpYdWqq60uXc2+HVD6zfuo3/Qyx+6dQt7uE6Cn0F4XDXrVpTaWb+S1wteIDo/mX2/4V+5ZVELq5udI3bwPR3MLVYMyWPn045zO7mN1qcpCmYNv4LcPL+eZl49Q+8zP6fHTX1pdUlDToFddosndxPIP/8hCezHDvUn8qmYs+T//gL5binC43FQNTKdsUj71SfEa8gqA6Mdm8d7Gn/HFp/4TZsz03SlMXRMNetXpPi37lKdXP81JWzXfqMvlJ+vCSNvyEXaXm6q8DMpuHEpDUvz59fXUSQWQGJPEO9++lRv/dSnOh+6lR8EuiNDbQV4LDXrVaY6dPcYvV/+SP2/5MwNMPB8uz2bU5kM4mluozMug7KZ8GhJ7Wl2mCjCt/9Bn04tfzojnf98ooX7GF4lesAjCdV6jq6VBrzrc8brj/Pe6/+a5zc+RUNvCRyXDmbJ4D+GNNVTnpnN4cv4FPXilLseJg29P/B7fq3+G/5m/HPfMB3G89Q44NLquhn5aqm2zZ1+6bNasNlc1xnDw1EEWHVjEu3vfZd3BVXzxgLC2JI2RW48ippADt4+hNi2RhqSenVu3CjkDHEnc/rt5fLfxLn7/3gLMI48gL7+sPfuroEGvPtPcDCdOQG0tFBdDS8tnbXY7bNzIKZooriuj5MxhPixZwqlTR/GcOEZsbTNDK+FXR8MZWx5GeFMLpLTA978PTzzBytoVOvaurtmdK8rYdvu9/NAzn9/MmYNZuxZZtgxycqwuLSiIMcbqGi4wZswYs2XLFqvLCG2NjbB9+2ePvXvh0CGoqIDr+Hnw2oSTuekk3jbdt5/Bg8+f/7y6VENeXZvJmZMB3/8c/9+ZtygrWMGrC+3E2COR556DRx7RWUwBESkwxoxpq0179N1BTQ2sWQOrV8PatbB162e99fh432lrt9yCycxkQd0WDnirOVy1n71hZ2gRL2Ig1RtFtiuGrJZo+ngiSfRG0MOEU3HDUBp7xdEUH8vZPr1wR0Uwa/Sstod+lLoOIsKzPR7ir+P7MLzP33nrfQ9jHnsMnnsOnnoK7rxTA/8ytEcfBFpfVXrOrNFtj5cDUFn5WbCvXg07dvh62GFhkJkJ/ftDv340paewJ7aZT1tKWO06wBrXASq9ZwHo5+3JaJPCCG8Sw00SsTjbfKt9901uc7kO06iOcq5H39onzUU8WP8yX9hQw3+uiySlqhHGj4fHH4f774devSyo1Fqf16PXoA9AXuPlRN0JymrLOHLmCG/vfpvTTaepa6mjqaWJRncjCVEJ2LyGlOomsipd5BxvYdDhOnJKTpF0og4AV5iN8sx4Svv35mD/3uxOc1Jtb6LSe4b97koOe05i8H3/M+29mRQ+gPSzNsZ6+9AbnXpABYa2gh7g9CMP8vym53l+7e+5Z+1JfrA5jH6VLXgcdhpvvomI2+7EMWUqjBrl6+SEuOsOehG5E3gWsAMvGGN+fVG7E3gVGA2cBB4yxhz2t/0Y+AbgAb5tjPno894rqIO+ocF3MLOqCqqrfY/aWqitxdTW4tqykeaWRlwtzTSbFpr8j8ZesZz1NnHWXc9ZdwO17jpceBHAZnyPaLcQ53HQw2Ujqd6QXC/E17oI83z2/avoFUZBX9iUCmtSPazv66Wl1eBcnEQQb4umt0ST60hmkCOZPEcfbgjvR7rd1wPSsXQVLCb/6nUA6l31vLLjFRYfWET12qX8wzYX04sgp8a3XrNDOJEURW2feDwpKUQlp9I7OZteP/oPJC7Own9Bx7quoBcRO7AfuA0oBzYDDxtj9rRa55+AYcaYb4nITGCGMeYhERkMzAHGAX2BZUCuMcZzuffrrKA3xtDibcHtddPsbqbR3UiTu4l6Vz1nXWc523yWM021nK2voa7+FE1nT9F85hT7jmwlvK6JqAYXUfUu4s62kOWOpscZFz1rXfSqbSa+up6edW6im72Xff8mOzSGQbMdmh3gbTWUaDcQhh2HFxxGcBhwuA0igiAggglz4HU48IQ7aImOwBUVgSsmgsZecTT2iqWhdxzuqEuvGjTn++xgQ8cvVWjz4KVETnNQTlNXV0PvIydJP1pHRpWLgZVe0s9cuP7pKBvVidHUJcXjTojHm5SILSERW1wP7HE9scfEYXOE+f5H4LD7fpdEMDYBmx3sNrDZsTmd2J2R2CMicURE4YiIIiwimjBnFGGR0YQ5I3GEObHbHEgnHUe43oOx44BiY8xB/87mAtOBPa3WmQ78wv/8HeA58f1rpgNzjTHNwCERKfbvb/21/EM+T3VDNVl/yPIFmzHnv3qMB6/x4jW+EN40GwZXgdOAAOLvMdv9j/aqjbRRHWujKtZOWTJUDgijvkckp6IdnIlx0BQdgSs6girqsTmdRNqdxJhw4nDSwzjpgZOeJoJ4IojC4Qv0TiCdtmelAo8dG7mmF7mmF0T1g4H4HsAe3Kx2ncZ1qgpOnSaxZ18cR44Sc+wkPSuOkrSvjKQ6cHTiaLZHfJ28cw/j/+U0wIvjHLz3zZtY+ejKDn/f9gR9KnCk1etyYPzl1jHGuEWkFujtX77hom1TL34DEZkFnDu6WCciRe2q/hqM66gdNXp9j0r3uSUJ0FzdUbsPcgmAfhb6ObQWgJ/Fwa5/S0MC5jKfw1o3rP0Eeeyau2aZl2sIiNMrjTGzgaA+H09Etlzuv03djX4WPvo5fEY/Cx+rPof23EqwAkhv9TrNv6zNdUTEAfTAd1C2PdsqpZTqRO0J+s1Ajohki0g4MBNYeNE6C4FH/c/vB1YY31HehcBMEXGKSDaQA2zqmNKVUkq1xxWHbvxj7k8CH+E7vfIlY8xuEXkK2GKMWQi8CLzmP9hag++PAf713sJ34NYN/PPnnXET5IJ66KmD6Wfho5/DZ/Sz8LHkcwi4C6aUUkp1rPYM3SillApiGvRKKRXiNOivk4g8ICK7RcQrImNaLc8SkUYR2e5//MXKOjvb5T4Hf9uPRaRYRIpE5A6rarSCiPxCRCpa/RzcbXVNXUlE7vR/34tF5EdW12MlETksIjv9PwddOs9LQJxHH+R2AfcBf22jrcQYM6Jry7FMm5+DfxqMmcAQ/NNgiMjnToMRgn5vjPkvq4voav7pU56n1fQpIrKw9fQp3dBUY0yXXzimPfrrZIzZa4zptCt5g8XnfA7np8EwxhwCzk2DoULf+elTjDEu4Nz0KaqLadB3rmwR2SYiq0RkktXFWKStKTQumQYjxD0pIoUi8pKIdKe7ouv3/kIG+FhECvzTvnQZHbppBxFZBqS00fTvxpgFl9nsGJBhjDkpIqOB+SIyxBhz5jLrB7xr/BxC3ud9LsCfgafx/ZI/Dfw38HjXVacCyE3GmAoRSQKWisg+Y0yXzAuuQd8Oxphbr2GbZqDZ/7xAREqAXCBIJ9u/ts+BbjANRns/FxH5G/BBJ5cTSEL+e381jDEV/q+VIvIevqGtLgl6HbrpJCKS6D8YhYj0wzf9gwXT5VmuW0+DISJ9Wr2cge+gdXfRnulTugURiRaR2HPPgdvpwp8F7dFfJxGZAfwRSAQ+FJHtxpg7gMnAUyLSAniBbxljaiwstVNd7nPoZtNgtOW3IjIC39DNYeAfLa2mC11u+hSLy7JKMvCe/6YjDuDvxpglXfXmOgWCUkqFOB26UUqpEKdBr5RSIU6DXimlQpwGvVJKhTgNeqWUCnEa9EopFeI06JVSKsT9/+KwPm1X5RCYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxeUlEQVR4nO3deXxU9b3/8ddnZrKH7CFsIWENBGQLmyi41AXbCmq11S5X22u51nq73ba/9vZevdrN3nt7u1lb02rt5i4oVlBcQVSWhB1CSICEJED2fZvMzPf3xxkkhAABkpyZyef5eMyDmbPMfA5J3vnme77ne8QYg1JKqdDlsLsApZRSA0uDXimlQpwGvVJKhTgNeqWUCnEa9EopFeJcdhfQU0pKisnMzLS7DKWUCir5+fk1xpjU3tYFXNBnZmaSl5dndxlKKRVURKT0TOu060YppUKcBr1SSoU4DXqllApxGvRKKRXiNOiVUirEadArpVSI06BXSqkQp0GvlFIhToNeKaVCXMBdGauUCgK5uWdet2LF4NWh+kRb9EopFeI06JVSKsRp0CulVIjToFdKqRCnQa+UUiFOg14ppUKcBr1SSoU4DXqllApxGvRKKRXiNOiVUirEadArpVSI06BXSqkQp5OaKaXOS25+LlNKN/S6bknGkkGuRvWFtuiVUirE9SnoRWSpiBSKSLGIfK+X9d8SkX0isktE3hKRjG7r7hSRIv/jzv4sXiml1LmdM+hFxAn8FrgByAbuEJHsHpttB+YaY2YALwD/7d83CXgAWADMBx4QkcT+K18ppdS59KVFPx8oNsYcMsa4gWeA5d03MMa8Y4xp87/cBIzxP78eeMMYU2eMqQfeAJb2T+lKKaX6oi9BPxoo6/a63L/sTP4ZWHuB+yqllOpn/TrqRkQ+D8wFrjjP/VYAKwDGjh3bnyUppdSQ15cWfQWQ3u31GP+yU4jINcAPgGXGmM7z2dcYk2uMmWuMmZuamtrX2pVSSvVBX4J+KzBJRMaJSDhwO7C6+wYiMht4DCvkq7qteh24TkQS/Sdhr/MvU0opNUjO2XVjjPGIyH1YAe0EnjDG7BWRh4A8Y8xq4H+AWOB5EQE4YoxZZoypE5EfYv2yAHjIGFM3IEeilFKqV33qozfGrAHW9Fh2f7fn15xl3yeAJy60QKVUgPP5iGhupzMu2nqdm3v27VesGPia1Cl0CgSl1HkTj5f0D/eScKSK2GO1uNweGsekwoopkJZmd3mqB50CQSl1XhxdHrJXbiDzvd043V1UXjKew1fMJLqmER56CF57Dbxeu8tU3WiLXinVdx4PV//gcZKLj3Lghvkcnz3po1WVMyewcMMhWLUK2tvh5pttLFR1p0GvlOpdz752nw/+9CfGb9nGwWtyTgl5AHdsFNxzD/z5z7BuHSxcCCNHDmLB6ky060Yp1TcffghbtnD4iplUzJ/S+zYicMstEBkJTz8NxgxujapXGvRKqXNzu+GVVyAzk7JF086+bVwcLF8OhYWQlzc49amz0qBXSp3bO+9Afb3VWreulTm7JUtg7Fh4/nno6Bj4+tRZadArpc6utdUaSTN9OmRl9W0fhwM++1lobIS33hrY+tQ5adArpc7u9detUTQ33XR++40bB5Mnw+bN2ldvMw16pdSZ1dfD22/DggWQnn7u7XuaNw8qK6Gs7NzbqgGjQa+UOrONG8HjgU9+EgBjDGU0scpRyC+dWymQmrPvP2eO1Y2zZcsgFKvORMfRK6V65/NZQyqnTIHUVI57G/lE3SNsCz8CgMs4WOU8wKW+UXzZO4sJppe7hMbGwrRp1uibW26xQl8NOv1fV0r1rrAQamth0SKOexu5qvb/KPRW8jXPXJ5yL+OVrlu52zOTPVLDfa51lNHU+/vMm2d1AR08OLj1q49o0CulTpGbn0tufi5Vb7xEV2Q4q5NrWFj5I0o9NfzUvZhP+bIYzTCiCeMLvuk83vVxwnDyn64NtPo6T3/DmTMhLAy2bj19nRoUGvRKqdOEN7WSsv8IVdMyeChyM5W08jPPlcw0p89MmUYM/+m5jBJp5J6mv2N6jrCJjLTCPj9fJzuziQa9Uuo0E9bl4fD6+MesaPIdx7nHO6fXkD9hnhnJF70z+Fv7Zv7avqmXDeZBSwsUFAxg1epMNOiVUqfJWv0+TcMT+MHoQrJ8SSzzTTznPl/wTWdeWCYPtLxCl+nRcp82zWrZ79gxMAWrs9KgV0qdIrG4guH7Snlhdjh10sE3vfNx9iEqHAj3x36CEm8tf2/ffOrKsDCYOBGKiweoanU2GvRKqVOMfzMfn0P4/iVVLPNNYqpJ7vO+n4i4hNmudH7Sshav8Z26csIEOHbMGsmjBpWOo1dKnSJz/U52ToylOaadf+6aeV77igj/EftxPtXwGM925PHZqPknV070d//cf791crY3ej/ZAaEteqXUSYcPk1xUzt/GtfBJ30TiiTjvt7gpchbTXaP4UcsafN1b9ZmZ4HRq940NNOiVUie9/DIAq6fAp71nuLnIOTjEwfdil1LgOcY77sKTK8LDISNDL5yygQa9Uuoj7lUvsGc4pF6ykOHEXPD73BI5mziJ5K89T8pOnAglJdaNTNSg0aBXSllqa3Ft/ICXsuD6Cddf1FtFSTi3RubwYsc22ky3UJ840bpoqrT0IotV50ODXikFQNfql3D4DAWLJjFy2MXf1PsLUQtoMZ283LHj5MIJE6x/tZ9+UGnQK6UAOP73x6gYBsOX3NAv77ckfBLpjsRTu29iY2HkSA36QaZBr5SC9nZSNubzzow4soZP7Ze3dIiDz0UtYF3nPiq93Wa2nDjROiHr8515Z9WvNOiVUpSv+gtRnT7Cbv4UDum/WPh81AK8+Himo9vMlRMnWrcmPHas3z5HnZ1eMKWUouTZ35MYBlfe9V+8XPLaBb/PhtINpy2b5Eokt+EtZteEsSRjyckLp4qKYPToC/4s1XfaoldqiOvwdDDig10UTh9BWvLYfn//y33pFEgt9XRYC5KTIT4eDh/u989SvdOgV2qIe/2tXCbW+Ij5xM0D8v4LzSiMwBbHUWuBiHWj8fLyAfk8dToNeqWGuIPP/R6ASbffOyDvP9kkkWQi2SRHTy4cPdrqo9cbkQwK7aNXaojJzc/96Hl9ez3jNhVQkxjJyvb3If+Dfv88B8J83yg2OsrxGC8ucVpB7/XC8ePaTz8ItEWv1BCWV7aZjx2GigVTrS6VAXKpbzQt4mZTl79ffswY69+KigH7THWSBr1SQ5QxhpZNG0huh5rL5wzoZ801I3AaYU3HbmtBWpo1k6UG/aDQrhulhijnSy8zZ491E5Do6gamrDx9aGR/iSWcS0wqr3bu4SfcDC4XjBihJ2QHibbolRqiXncc4rqD0Dgiga6YyAH/vIW+0ezylFPurbcWjB6tLfpBokGv1BDk9XnZ1HWYRWXQOG7UoHzmQmN9ztrOPdaCMWOgvh5aWwfl84eyPgW9iCwVkUIRKRaR7/WyfomIbBMRj4jc2mOdV0R2+B+r+6twpdSF21e9j5mlbsJ80DDu4meq7ItME0+aI453Ow9YC06MttFW/YA7Z9CLiBP4LXADkA3cISLZPTY7AtwFPNXLW7QbY2b5H8susl6lVD/YcnQLNxxy4HU5aRyTOiifKQhXhE9ivfsAxhgN+kHUlxb9fKDYGHPIGOMGngGWd9/AGFNijNkF6HR0SgW4Tk8nO4/v5IYSF01jUjEu56B99pXhWVT4GjjkrYGEBIiJ0ROyg6AvQT8aKOv2uty/rK8iRSRPRDaJyE3nU5xSqv/tqtxFbFMn4yvdNGSkDepnXxE+CYD17gPWuH09ITsoBuNkbIYxZi7wWeCXIjKh5wYissL/yyCvurp6EEpSaujacnQLHz8aDUDjIAf9VNdIUh3DrKAH64Ts0aM6N/0A60vQVwDp3V6P8S/rE2NMhf/fQ8C7wOxetsk1xsw1xsxNTR2c/kKlhqLatlr2VO3hU5XJeMNcNI9MHtTPFxGWhE9ivbvIWjB6NHR2Qm3toNYx1PQl6LcCk0RknIiEA7cDfRo9IyKJIhLhf54CXAbsu9BilVIX58WCF/EZHwuL2mlMT8U4B3+E9RXhkyj11lLiqTl5Qlb76QfUOa+MNcZ4ROQ+4HXACTxhjNkrIg8BecaY1SIyD1gFJAI3isiDxphpwFTgMRHxYf1SedgYo0Gv1GDJzT3l5VO1PyenOZa00hoOXTXLlpKuCJ8MwHp3EZmj5lh99RUVMPu0P/ZVP+nTFAjGmDXAmh7L7u/2fCtWl07P/T4ALrnIGpVS/aDCW88GdxGPHhoDtNCQMcKWOqa7RpEkMax3H+DOhEshNVVPyA4wvTJWqSHi+Y58DIaPHRY8EWG0jEgc9Bo2lG5g45GNZHsTeb1tFxtKN1AbF0ZL2aFeb0Oo+ocGvVJDxDPtecx2pTOmtJ7G9OHgsO/Hf6ZvOEelhWraaEuOI6q+GYyxrZ5Qp0Gv1BBw2FPD5q7D3N05jaj65kEfP9/TdGONriuQGtqT4nB6vEQ06pw3A0WDXqkh4NmOPABuK40FsD3oJ5pEwoyDvY4a2pOHARBd12RrTaFMg16pIeCZ9q0sDBtHavFRuiLDaU0b/P757sJxMskksVdqaEuKAyCqVoN+oGjQKxXiCrqOsdNTzu1R8+DAARrHDh/Q2wb21TSTQqHU0R4TjicijGgN+gGjQa9UiHu2Iw9BuL11PNTU0DDW3m6bE7JNCm7xUuxooC0pjqi6ZrtLCll6K0GlQkxu/smLpLJK1/Nk2AZmmlTq89eTxuDPb3Mm03wpAOwTq58+obTK5opCl7bolQphh6SBUmniSl8GCaWVdEWF0zo8we6yABhONCkmygr6pDgimtv0blMDRINeqRD2ruMIDiNc4UsnobSSxvS0gOifB+tGJNkmhb2OGtqSrROyHDhgb1EhSoNeqRBljOFtRymzzHBGNniJbGylIWO43WWdItuXwlFpoTLZf3NyDfoBoUGvVIgqbyqnXJq5ypdBwhGr/9vu8fM9ZRurnz4vqQMDUFhoaz2hSoNeqRCVdzQPpxGW+NKJP1JJV1QEbakJdpd1iiyThNMIOyPq6YyP0aAfIBr0SoUgYwx5x/KYbdJIIJKE0koaAmT8fHeRuJhoEik4ceGUBv2A0KBXKgQdaTxCTVsNV/kyiGhoIbKxlcYAGT/fU5ZJ5oDU0ZY8zAp6ndys32nQKxWC8o/l4xAHi33pJBypBAKvf/6EKSaJFuniWFI4tLTAsWN2lxRyNOiVCjHGGLYd20ZWchbxRJBQUok7KoK21Hi7S+tVlrHuW7s3xX+DcO2+6Xca9EqFmLKmMqrbqskZmQPGkFBynIbMwBk/31OmiSfcONmc0mkt0KDvdxr0SoWYE902s0fOJqq2iYiWdhoy7bltYF+4cDDJJPJBfCNERWnQDwANeqVCiDGGbUetbpvY8FgSSo4DBHTQA0wxyRxw1GMmT9agHwAa9EqFkF2Vu6hqq2LOyDkAJJZU0hEXTUdCrM2VnV2WSaJDvDRlpEFxsd3lhBwNeqVCyPP7nkcQZo2YhXh9xB+ptFrzAdo/f0KWzzohW5oSBocPg8djc0WhRYNeqRDyYsGLTE6eTFxEHMkHygjrcAd8tw3AWOKIMi52DmuzQr601O6SQooGvVIhYl/1PvbX7P+o22b0lv1A4PfPAzgQskwSG8OtcwoUFdlbUIjRoFcqRLy470UAZo2YBcCorftpTYnHHRtlY1V9l2WSeU0OWi806PuVBr1SIeLFghdZlL6IhMgEHO4uRm4vCorW/AlTTBJHotx4Y6L1hGw/06BXKgQcrDvIzsqdfGrqpwBI230YV2eXdaFUkMjyJYNA/ehkbdH3Mw16pULAyoKVANw85WbA6rbxOSRgbgTeF6OIJT4intLUMA36fqZBr1QIeLHgReaMnMO4xHEApH+4l+rsTLyR4TZX1neCMGfkHHbGtUFJCXR12V1SyNCgVyrIlTeVs7li80fdNpH1zaTuK6Vs0TSbKzt/OSNz+DCyWodY9jMNeqWC3KqCVQDcMvUWAMZs2ocYQ9mi6XaWdUFyRuVQkOC1XugJ2X6jQa9UkFu5fyXZqdlMSZkCQPoHe2hPiKU6O8Pmys5fzsgcipL9L7Sfvt9o0CsVxGraathQuoFbpliteXw+xny4j/JLs8ERfD/eE5Im0J4YS0eUnpDtT8H3naCU+sjqwtX4jI+bp1qjbcjLI6qhJSi7bQAc4mDOqBxKUsO066YfadArFcRW7V9FRnwGs0fMthasXYsRoezS4DsRe0LOyBz2DOvAaIu+32jQKxWkmjubWXdwHbdMvQU5MTvl2rVUTcukM8CnJT6bnFE5FCb5rFksdYhlv9CgVypIrS1ei9vr/ugiKWprYcuWoBxW2V3OyByKkkC8Xh1i2U806JUKUisLVjI8ZjiL0hdZC9atgyAdVtndpORJHE3zT8Sm3Tf9QoNeqSDU4elgTdEalmctx+lwWgvXrIGUlKAcVtmdQxxETZ1hvdCg7xd9CnoRWSoihSJSLCLf62X9EhHZJiIeEbm1x7o7RaTI/7izvwpXaqjKzc/l2+u+TbO7mShXFLn5ufxx06N0vrySwksnB+Wwyp7GZy2kKQJ8RQfsLiUknPM7QkScwG+BG4Bs4A4Rye6x2RHgLuCpHvsmAQ8AC4D5wAMiknjxZSs1tG0/vp0oVxRZKVkAjMorJKK5jcNXz7a5sv6RM3ouxYnQum+H3aWEhL786p8PFBtjDhlj3MAzwPLuGxhjSowxuwBfj32vB94wxtQZY+qBN4Cl/VC3UkOW1+dl5/GdzEibgcvhAmDc29txR0dQMX+qzdX1jzkj53AgGR1i2U/6EvSjgbJur8v9y/qiT/uKyAoRyRORvOrq6j6+tVJD04G6A7R2tX50y0Dx+shYv4Oyyy7BGxFmc3X9Iys5i5LhYcRWVIPbbXc5QS8gOvOMMbnGmLnGmLmpqal2l6NUQNt+bDvhznCyU60e1LSdxUTXNYdMtw2A0+HEPSETh8/AoUN2lxP0+hL0FUB6t9dj/Mv64mL2VUr14DM+dhzfwfTU6YQ7rbnmx72zHU+4K+iHVfYUPW0WAN79++0tJAT0Jei3ApNEZJyIhAO3A6v7+P6vA9eJSKL/JOx1/mVKqQuwqXwTjZ2NzB7pb70bw7i3t1OxIJuumEh7i+tno+dcCUDNjvftLSQEnDPojTEe4D6sgC4AnjPG7BWRh0RkGYCIzBORcuA24DER2evftw74IdYvi63AQ/5lSqkLsLJgJS6Hi0uGXwJASkEpsZX1HPrYHJsr63+XTFlCVTQ07cmzu5Sg5+rLRsaYNcCaHsvu7/Z8K1a3TG/7PgE8cRE1KqUAYwwv7HuBKSlTiAqzrhwd9/Z2fE4HRxbPsLm6/jclZQpbUhwMP6CzWF6sgDgZq5Q6t7yjeZQ2lpIzMsdaYAwT3sjj6NwsOuNj7C1uALgcLurSk0koq7K7lKCnQa9UkHh+3/OEOcKYmTYTgLSdB4mrqKHohgU2VzZwvBMnkNLgxtfUaHcpQU2DXqkgYIzh+X3Pc834a4gJt1rvk9ZuxhMRRslVoTOssqdh062/Xsry3ra5kuDWpz56pZS98o/lU9JQwv1L7qfL14XD3cX4N/I4fNXs0Bptk5t7ysvR9dZ89OV//jUZxdWwYoUdVQU9DXqlgsDze5/H5XCxfMpyXtj3Aukf7CWyqY3iEOu22VC64ZTXvi43k4CK0j1sGLOBJWjQXwjtulEqwHXvtkmKSgJg0prNtCUNo3xBaMxtcyaOsHCOxTuIqWuxu5SgpkGvVIDbdmwbhxsOc1v2bQCEN7WS8d4uDl4/D+Ny2lzdwKtOjiStthMfxu5SgpYGvVIB7pk9zxDmCOOmKTcBMP6tbTi7PCE92qa7tqRhTKw1VJgmu0sJWhr0SgUwn/Hx9J6nWTpx6clum1c3UZ85gpqpwX0nqb5yJCWT0AllHcftLiVo6clYpQLYe6XvUdFcwf9e97/WgsJCRu4oZvN9N4MIU1ZuOPsbhIDIRGtG29Y6vXDqQmmLXqkA9tTup4gJi+HGyTdaCx5/HJ/TwYFPXmpvYYPInRQPgKOu3uZKgpcGvVIByu1180LBCyyfsty6SKqrC/78Z0oXz6A9Jd7u8gZNR3wMXQ4YVteC1+e1u5ygpEGvVIBad3Adde11fHb6Z60F//gHVFWx/6bL7S1ssDkc1CdFMb7WcKBWbxZ+IbSPXqkA9dTup0iOSua6d47Au7nwm99AQgIxx2uHRN98dx1JcWTVtpN3NI+pqaF97cBA0Ba9UgGoubOZlwtf5tbsWwkTJ9TXw969cOml4Bh6P7a+5CQm18KOI1vsLiUoDb3vGKWCwAv7XqCtq407Z95pLfjgAzAGLrvM3sJs0pGaSJgPqvRuUxdEg16pAPTkzieZnDyZhWMWgs8H778PWVmQmmp3abZoTfWffN63F4/PY28xQUiDXqkA8+MNP2ZD6QayU7P5w7Y/sPvd56C2loKpKadN+jVUtCXH4xOYeMxNQXWB3eUEHQ16pQLMpvJNCMLC0QsBGLm9CHd0BDVZ6TZXZh/jctKSGMO0KutOW+r8aNArFUB8xseH5R8yJWUKiVGJRFc3kFxUQeWMCRhn6E9gdjadKYlcUuNg69GtdpcSdDTolQogG0o3UNtey6L0RQBkvfw+YgzHZk20uTL7tafEM7HWsL1kk92lBB0NeqUCyJ92/IlIVySzRsxCvD6mrnqP+swRdCQNs7s027WmJuD0GToKdtHh6bC7nKCiQa9UgKhvr+e5vc+xYPQCwp3hpH+wh9jKeo7NnmR3aQGhzT/yJuu4lx3Hd9hbTJDRoFcqQPxt19/o8HSweOxiAKau3EBbchy1k8fYXFlgaEuKwzidTKuGrRXaT38+NOiVCgDGGHK35TJv1DzS49OJPVZL+vt7KFx2GcapP6ZgjbyRiRPJqYtgy1G9QvZ86HeQUgFgU/km9lTtYUWOdfPrqf65bPZ9aomdZQWeadOYWetiS4UG/fnQoFcqADyW/xix4bHcPv12HO4uprz0PkcWz6B1RJLdpQWW6dMZWdlG6fEDNHQ02F1N0NCgV8pmDR0NPLf3OT53yeeIDY9l3NvbiapvZu9tV9pdWuCZNg2Hz5BVqxdOnQ+dplipQZabn3vK6zcPvUm7p52U6BRy83NZ9vy7NKYPp2L+FJsqDFzPs4/bgOlV8MiWRzhUf+iU9Se6vtSptEWvlI18xsf6kvVMSJzA2PixJB0oY8TOg+y79YohOR3xuTSOHY7P6WB+QzQlDSV2lxM09DtJKRvtq95HVVsVV2ZeCUD2CxvwRIRReOPQuSfs+fCFuWjISGN2bQSHGw5jjLG7pKCgXTdK2eidkndIMpHcsbmVqPY3yHrlfaqzMxn/Zr7dpQWs+vGjyNq9n6bOVura60iOTra7pICnLXqlbFLVWsXeqr0s800iDCcjtxfh7PJSPi/L7tICWs2UsaRVtpLQzml99Kp3GvRK2eTdkncREZZ5JyFeL6PyDlA3biRtwxPtLi2gVWdnAHDpcRcH6w/aXE1w0KBXygbtXe28X/Y+OSNzSCaK4XtLiWhpp2KBjrQ5l5qpVtBfUxuvLfo+0qBXygbvl71Ph6eDa8ZfA8YwenMBranx1I8baXdpAc89LJqGscNZeMxJWVMZbq/b7pICnga9UoPM6/Py1uG3mJQ0icyETBJKjhNb3UD5/KkgYnd5QaE6O5Oppa34jI/ShlK7ywl4GvRKDbJtx7ZR117HteOvBWDM5gI6YyKpmpZpb2FBpDo7g8SaVtKa9YRsX/Qp6EVkqYgUikixiHyvl/URIvKsf/1mEcn0L88UkXYR2eF//L6f61cqqBhjWHdoHWkxaVySdgnDdx8i6dAxjs7LwriG9q0Cz0d1diYA12o/fZ+cM+hFxAn8FrgByAbuEJHsHpv9M1BvjJkI/AL4Wbd1B40xs/yPe/qpbqWC0vrS9RxpPMI146/BgTD/N6twR0dSMVeHVJ6P2qx0fA7hqppYDtYf1AunzqEvLfr5QLEx5pAxxg08Ayzvsc1y4M/+5y8AHxPRzkalevrpxp8yLHwYC8csZMyHexm17QBHLp+OLzzM7tKCiicqgvrxo8ip8NHsbqamrcbukgJaX4J+NFDW7XW5f1mv2xhjPEAjcOJytXEisl1E1ovI4t4+QERWiEieiORVV1ef1wEoFSw2l29m3cF1XDv+WsLFxfxHXqJpdArHZuuNvy9EdXYmEw82gNF++nMZ6JOxx4CxxpjZwLeAp0QkrudGxphcY8xcY8zc1NTUAS5JKXv8cMMPSY5K5orMK5iwLo+UA2Xk3bMM49S++QtRnZ1BTFM7k5vD9cKpc+hL0FcA6d1ej/Ev63UbEXEB8UCtMabTGFMLYIzJBw4Cky+2aKWCTf7RfF4tepVvLvwmMR4H8373MrWTxlB8/Ty7SwtaJ66Q/XjjcIrqimyuJrD1Jei3ApNEZJyIhAO3A6t7bLMauNP//FbgbWOMEZFU/8lcRGQ8MAnQv7HUkPOj935EQmQC982/jwW/WUlcRQ2bvnGrTkV8EeomjcEb5mJJVSRHm4/S4m6xu6SAdc7ZK40xHhG5D3gdcAJPGGP2ishDQJ4xZjXwOPBXESkG6rB+GQAsAR4SkS7AB9xjjKkbiANRKlBtO7aNl/a/xANXPED8Ox8Q/+w77L7jaioWTLW7tKDmC3NRO2kMM450Qg4cqD1gd0kBq0/TFBtj1gBreiy7v9vzDuC2XvZ7EXjxImtUKmgZY/jOG98hOSqZb43/PMy/nLoJo9hy3y12lxYSqqdlMunVD4kyLu2+OQv9u1GpAbS2eC1vH36b+xf/B3Ff/SY0NPDWj+/GG6HDKfvD0blZhLd1clPjSIpqNejPRG88olRf5eaeed2K0+9V6vF5+O4b32Vq3AS++qsP4R//gF//mvqJEQNY5NBSMS8Ln0O48UgUzyQUUd9eT2KUTvPckwa9Uv3sxM2/Nx7ZyKGKvWx+PR1n/nNs/tdb2LlIQ74/ueNiqM7O5NKCZswMw8YjG7kx60a7ywo42nWj1ABodbeyd8OLbHwqgmnby1n/H19g553X211WSKpYMJWxhcdJ7nSyvnS93eUEJA16pfpZdHUD4773MJt/3ca0SsNbP11B4U2X211WyCpfMBWHz/CZquEa9GegXTdK9YfWVlizBp59lk+/8hJ4vPzj2gyavv2vdCQOY8rKDXZXGLKqLhlPV1QEHy8N4/djt9HU2URcxGkX4A9p2qJX6kJ1dcGOHfDHP8Lw4fDpT2Pe38hf5oZx+XeTqXjoO3QkDrO7ypDnC3NxNGcy8/c14TM+1pdoq74nDXqlzld7O7zyCnznO/C738G+ffBP/wTvvMO9v1/Gl6/rYMlVdxHm1CGUg6V8YTapRxuY2hLFa8Wv2V1OwNGuG6X6yuuFd96BtWuhpQVmzYLFi2HqVPjKV/jLzr/w+/V/4LoJ1zE5Wad0GkwnrjK+t2ES/1e8FmMMOlP6SRr0SvVFdTX84hdQVGQF+003QWbmR6t3/fZ+7ql5mCvDJ/P/CpJxFWif/GBqyBxBy/AElh528a9jDlNUV6S/bLvRrhulzmXXLpg3D0pK4Etfgm9845SQr/G18Kn6x0hwRPNMwpdx6Y/V4BOhYsFUxuUfxOWFtUVr7a4ooGiLXqkTervytaDA6oePioJvf/uUgAdo8rWztO5XlHvreTP5G6Q54ygcnGqHpLONXjp81WyyXvmQuytHs7Z4LV9f+PVBrCywadArdSYlJVbIp6TA178O8fGnrG43bpbVP8rOrnJeSryXy8L1TlF2Kls0HVJT+Zd90Vw6dj3tXe1EhUXZXVZA0L8xlepNVRU88gjExsLXvnZayLf4Orip7ndscBfxl4Qv8onIS2wqVJ1gXE747GeZsbmEqKYO3i151+6SAoYGvVI9NTXBr34FPp8V8gkJp6yu9jZzdd0veNNdwOPxX+COqPn21KlOd+edOLq6+Pz+MB1m2Y123SjVndcLjz2Gt6GeXZ+7hubOA1B68oYWFTTzQNQmyrz1rEr8CssiZ9pYrDrNrFkwfTpf2VfOJ4v+wS+X/lKHWaIteqVOtWoVFBdT9PEFNI9OOWXVVjnGv4S9Ro2vhTeSv6EhH4hE4M47mVrcgKvoEFuPbrW7ooCgQa/UCdu3wxtvwBVXUDV93EeLDYZnHQV81/UOKSaKrSn/zuV64jVwfe5zGIeDu3Y7eHr303ZXExA06JUCKC6GJ5+0hk/edvKumM24+U/Xezzq2sblZgyPeq5ngivVtjJVH4wciVx3HXfvieD53c/g9Xntrsh2GvRKdXRY4e5wWHeKCrPmqCmQWr4ctpYPpJx7PXN4yLOYaHT+mqCwYgWpte1c9eFxnboYDXqlrCtdd+yAL34RkpPxGh9/dezhq67X8eLj155r+YxvKoKe1Asay5fjmzmDB9cLz23/u93V2E6DXg1tTz8Njz0G3/0uzJhBsaeKK2t/zh9dO7nCN5Ynuj7OdKNdNUHH4cDxox8zvs4Q/ven6fR02l2RrXR4pRq6CgutrprLLqPrwQf4+aOf58HmfxAuTn7gWcS1vsxeW/EbSnXCsqDwiU9QPzOLb79ZyLp9r3DjjFvtrsg22qJXQ1NDA9x0EyYigrd/uoI5f1rA95tX8YnISyhIfZDrfOO0qybYiTDs4V8wtgmO/Pw/7a7GVhr0aujxeChbughP8QE+/7koPvb2nVS2VPLjriV8rWkqxeW77K5Q9RPX9Us5MnMct6zcz66ijXaXYxvtulFDSou7hbLPL2Pq5gK+uBxeGdHIHZPv4PKxlzP9pQ/sLk9dhCkrN0D+6csTr1tG9P/+iqIV/wTvHBr8wgKABr0KSbn5J6cc7vR0UlBTQN7RPK5cuY3/fs3Lo1fG4L3zFn4yej7hznAbK1UDbdjEbN64ehzXvnWYqid/y/C7vmp3SYNOg16FpOrWavZW72V31W4Kawrp8nVx/wdhPLjOy87Fk3E9/A0udzntLlMNkqk33c2W/T9g+r/+G3xsGaSn213SoNKgVwGpe4u8NytyVpzyutPTyXtH3uPVA6+ypngNB2qtichSolNYPPZyvre2hWvXbaXohvlseeAu0JAfUsaEp/Cbf7+Z7G+uovPzdxDx9npwDp3vAQ16FbQaOxp5tehVXtr/EmuL19LibiHCGcGVmVcye8RspqVOY5TEcdnPn2fK6q0U3LyY977/WesKWDXkfPn2/+bf3nyFx1a9j7n7buTxx4fM94IGvQoqbq+bHcd3sKZoDWuL1+L2ukmLSeOO6Xdw4+QbuXrc1cSEx5Cbn8vwXQe56v7fEFdRw7Yv3UDeV5ZbsxuqIWli0kQm/ttPeLDuuzzw5JMwbJh134Eh8D2hQa8CnjGGg/UH+aDsA/KP5dPh6WDUsFHcO/dePj3t0ywYswCHdGuZHT3KpT9/lmnPvkNrWhKvPPYtjs+ZDJz9nqMq9H3r0m+x6HPPk+rbzb2/+Q3ExMBPfhLyYa9BrwJWq7uVTeWbeO/IexxrOUaEM4I5I+dw6ZhL+Z/r/ufUcAfrRt6//CU8+STTPF0ULr+MTV+/la5YvW+osjgdTv5005PMPj6LcWFjueHhh6GyEh59FCIj7S5vwIgxxu4aTjF37lyTl5dndxnKJsYYth7dyjdf/yZbK7bS5esiMyGTJWOXkDMqh0hXJFNWbmBJxhLweODIEdi1y5qU7NgxcLlg0SK2TEugI3GY3YejAsySjCUA/K51PV9teIpnNqbx6bcrYe5cWLkyqEfjiEi+MWZub+u0Ra8CQmVLJU/veZontj/B7qrdRDgjuHTMpSzJWEJ63BiiapuI33WExMPHGfdmHtRtgtJSK+xFYPJkWLIEcnIgPp4OnY9GncVXYq7Ag5fPLHmWolEZ/PvLhUhOjtVnf/vtIdeVoy16dUZnG+LYc3jj+fI99nsKPMd5q6OADVVbOF5TwqgmWNSayFXtI4gIj2ZYXQuxlfXEVNbhcns+2tfrctIyIpGm0ak0j06hMX04XTGh+2e36j8nWvQnPNr6Ll9tepqlnnE8+1IYcXsOwBVXwCOPwPTpNlV5YbRFr86L2+umrLGMgpoC6trqaHI30dzZTFtXG12+LjxeD68Vv0aEK4JIVyTDwoeREJlAfEQ8cRFxxEfGE+mKJNwZjlOctHW10dZch2fvbny7dxF54CDxB46QUedjRT187ZQbANVDWAtNw+NpS46nZspYSq6cRcuIJBrHDqchcwRjPtgTci0uZY97Y65kvDOFr/Iqibcc4NFFOXzpue24Zs1CbrvNulfBggV2l3nRNOhD2Jla5F6fl6bOJpZkLKFs5Z847K3hsLeWQ55qir3VlHpr8XHqX3oRzgiiw6IJd4bjcrioaqvC4/PQ5e2i09tJe1c7BkO4BzIbILvaekyrgpmVkFUDLv9bup1QlRqNd1QqHbPSiUzLgKQkSEyEhASIjuaZuWcJcg151Y+WRk5nzxd/xsMbH+br7/+Mf1/RyUOborjr5ReIeeYZjk7P5OAnF1Fy5Sy+cP137C73gvQp6EVkKfArwAn80RjzcI/1EcBfgBygFviMMabEv+77wD8DXuBrxpjX+616dYq2rjaONh+loq6U2uOH2LXjJXwN9dDYiKO5BWlpxdHSiqu9k0gPvNUF4V4I80GOCeNqiSTOEUW8M4NhzmjaO1vpnDSOsIhoHK4wfE5rlIvD48Xh9eLqcBPR1EZEUyuRdc3EHq0htqYJ6dYdWJcWR+XEND5cOoqGrAzaJo+jaWwaX945dK5KVIEvKiyKB696kJToFLYf385vx27iPxYf4Av5cO/WEhY/XMLih5+iLPtXNC6aS8RlS0i9+kYSxk6yu/Q+OWcfvYg4gQPAtUA5sBW4wxizr9s29wIzjDH3iMjtwM3GmM+ISDbwNDAfGAW8CUw2xpzxbr0D0kfv80FXl/Vwu617hLa1QXs7tLRYj+Zm69HUdPJ5a6v16Og4uf/hw1aLUsS6qs7ptEZ6OJ0wY4Z1v1GX62Sr0xjrhGFXlzU6xOOxHl6v9fD5rH/T0zEiGMDndOBzOfA6BY8YujC48eD2duHuasftbsfT3oavvRXT3oazqYWIDg8xnYb4Thjm7tt/i3E4ME4H4nQijm7B6/OBMfg8XTi8vjPu73U58USG44kKpysqko6EGDriY+iMj6U1JZ62lDh84b3fY7VnX2lPenMPZbdOPOyTWnZTSaPLQ86mI1y9u5U5x07+dXp8mFCRFk3NmCTcI4fDiBE4R44mMm0UUUlpRCWPICY+hciYeCJjE4iMGkZYWAThrojThwdfpIvto58PFBtjDvnf7BlgObCv2zbLgf/yP38BeERExL/8GWNMJ3BYRIr97/fhhRzIWVVXQ0aGFazGWGF1IkQvRFiYdTFFTIw1vjYszHrU11vrT7z3iYfHA3v3WoHu8Zz+Xi6XtdzppMnpoVya6XKAxwFeB5jGfYgBAVw+6xHmBacBp8/61wAOgXABXII3IgxfRDieqEg8SZG0R0bRGBVLZHQc0dHxVHbW4oqIouSaubhjIumKjqQrJtIK58hw7l7wFeQPfzjjf8HG0g3sv3kx4jOIz4f4Q9/ncmKcDqaseu/C/m+VCgIRuJht0phNGvtvXAI3woyZ/0RRxV5q3nsd3+YPCS88SNyRSnK2HSOlqazP7+0VcAv4HNbP9Y6RwtX/EsGC0Qt49653+/1Y+tKivxVYaoy52//6C8ACY8x93bbZ49+m3P/6ILAAK/w3GWP+5l/+OLDWGPNCj89YAZwYxpEFFF7g8aQANRe4b7AaiscMQ/O4h+Ixgx53X2UY0/sNjgPiZKwxJhc4+3SFfSAieWf60yVUDcVjhqF53EPxmEGPuz/eqy+dRBVA98vFxviX9bqNiLiAeKyTsn3ZVyml1ADqS9BvBSaJyDgRCQduB1b32GY1cKf/+a3A28bqE1oN3C4iESIyDpgEbOmf0pVSSvXFObtujDEeEbkPeB1reOUTxpi9IvIQkGeMWQ08DvzVf7K1DuuXAf7tnsM6cesBvnq2ETf94KK7f4LQUDxmGJrHPRSPGfS4L1rATYGglFKqfw2N26sopdQQpkGvlFIhLuiDXkT+R0T2i8guEVklIgnd1n1fRIpFpFBErrexzH4nIreJyF4R8YnI3G7LM0WkXUR2+B+/t7PO/nSmY/avC9mvdXci8l8iUtHt6/txu2saKCKy1P/1LBaR79ldz2ARkRIR2e3/+vbLNAEBMY7+Ir0BfN9/0vhnwPeB/+effuF2YBr+6RdE5KzTLwSZPcAtwGO9rDtojJk1uOUMil6PeQh8rXv6hTHmf+0uYiD5p175Ld2mXhGR1d2nXglxVxlj+u0isaBv0Rtj1hljTsw5sAlrrD50m37BGHMYODH9QkgwxhQYYy70CuKgdJZjDumv9RD10dQrxhg3cGLqFXUBgj7oe/gSsNb/fDTQffKJcv+yoWCciGwXkfUistjuYgbBUPta3+fvqnxCRBLtLmaADLWvaXcGWCci+f7pYS5aUHTdiMibwIheVv3AGPOyf5sfYI3V//tg1jaQ+nLcvTgGjDXG1IpIDvCSiEwzxjQNWKH96AKPOaSc7f8A+B3wQ6ww+CHwc6wGjgodlxtjKkRkOPCGiOw3xlzUdK5BEfTGmGvOtl5E7gI+CXzMnLwwIOinXzjXcZ9hn06g0/883z/B3GQgKO7PeCHHTAh8rbvr6/+BiPwB+McAl2OXkPqang9jTIX/3yoRWYXVjXVRQR/0XTf+m6J8F1hmjGnrtmpITr8gIqn+E1mIyHis4z5kb1UDbsh8rUVkZLeXN2OdoA5FfZl6JeSISIyIDDvxHLiOfvgaB0WL/hweASKw/sQBa1rke2yYfmFQicjNwG+AVOBVEdlhjLkeWAI8JCJdgA+4xxhTZ2Op/eZMxxzqX+se/ltEZmF13ZQA/2JrNQPkTFOv2FzWYEgDVvmzzAU8ZYx57WLfVKdAUEqpEBf0XTdKKaXOToNeKaVCnAa9UkqFOA16pZQKcRr0SikV4jTolVIqxGnQK6VUiPv/dhqYbzYYqaMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:06<00:00,  2.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DMPNN with rdkit 2d normalized wo MolLogP   368\n",
      "StructGNN with ext valence with rdkit wo fragments and counts   359\n",
      "logP logD\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "p_values = []\n",
    "for NUM_EXP_PAIR in tqdm(NUM_EXPS_PAIRS):\n",
    "    p_value = {} \n",
    "    p_value_mean_std = {}\n",
    "    \n",
    "    NUM_EXP_1, NUM_EXP_2 = NUM_EXP_PAIR\n",
    "    \n",
    "    DATA_PATH_1 = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(NUM_EXP_1)+'/folds'\n",
    "    DATA_PATH_2 = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(NUM_EXP_2)+'/folds'\n",
    "    rmse = []\n",
    "    \n",
    "    preds = []\n",
    "    \n",
    "    for FOLD in range(FOLDS):\n",
    "        import json\n",
    "        with open(os.path.join(DATA_PATH_1,'fold_'+str(FOLD), 'args.json'), 'r') as f:\n",
    "            args_1 = json.load(f)\n",
    "        with open(os.path.join(DATA_PATH_2,'fold_'+str(FOLD), 'args.json'), 'r') as f:\n",
    "            args_2 = json.load(f)\n",
    "        VALUE_COLUMNS = args_1['target_columns']\n",
    "        PRED_COLUMNS = [value+'_pred' for value in VALUE_COLUMNS]\n",
    "        if args_1['additional_encoder']:\n",
    "            p_value_mean_std['Model_1'] = 'StructGNN with ext valence'\n",
    "        else:\n",
    "            p_value_mean_std['Model_1'] = 'DMPNN'\n",
    "        if args_2['additional_encoder']:\n",
    "            p_value_mean_std['Model_2'] = 'StructGNN with ext valence'\n",
    "        else:\n",
    "            p_value_mean_std['Model_2'] = 'DMPNN'\n",
    "        p_value_mean_std['Num_exp_1'] = NUM_EXP_1\n",
    "        p_value_mean_std['Num_exp_2'] = NUM_EXP_2\n",
    "        p_value_mean_std['Data'] = ' '.join(args_1['target_columns'])\n",
    "        \n",
    "        p_value_mean_std['Dataset'] = args_1['separate_test_path']\n",
    "\n",
    "        p_value_mean_std['Model_1'] += ' with '+' '.join(args_1['features_generator'][0].split('_'))\n",
    "        p_value_mean_std['Model_2'] += ' with '+' '.join(args_2['features_generator'][0].split('_'))\n",
    "        \n",
    "        for VALUE_COLUMN in VALUE_COLUMNS:\n",
    "            if VALUE_COLUMN in p_value.keys():\n",
    "                pass\n",
    "            else:\n",
    "                p_value[VALUE_COLUMN] = {}\n",
    "                p_value[VALUE_COLUMN]['r2_1'] = []\n",
    "                p_value[VALUE_COLUMN]['rmse_1'] = []\n",
    "                p_value[VALUE_COLUMN]['r2_2'] = []\n",
    "                p_value[VALUE_COLUMN]['rmse_2'] = []\n",
    "                p_value[VALUE_COLUMN]['Shapiro_p_value_1'] = []\n",
    "                p_value[VALUE_COLUMN]['Shapiro_p_value_2'] = []\n",
    "                p_value[VALUE_COLUMN]['Normaltest_p_value_1'] = []\n",
    "                p_value[VALUE_COLUMN]['Normaltest_p_value_2'] = []\n",
    "                p_value[VALUE_COLUMN]['P_VALUE'] = []\n",
    "                \n",
    "            PRED_COLUMN = VALUE_COLUMN+'_pred'\n",
    "            test_prediction_1 = pd.read_csv(os.path.join(DATA_PATH_1,'fold_'+str(FOLD), FILENAME))\n",
    "            test_prediction_2 = pd.read_csv(os.path.join(DATA_PATH_2,'fold_'+str(FOLD), FILENAME))\n",
    "            \n",
    "            ERROR_COLUMN_1 = (test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][VALUE_COLUMN].values\\\n",
    "                                               - test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][PRED_COLUMN].values)**2\n",
    "            \n",
    "            ERROR_COLUMN_2 = (test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][VALUE_COLUMN].values\\\n",
    "                                               - test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][PRED_COLUMN].values)**2\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['rmse_1'].append(mean_squared_error(test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][PRED_COLUMN])**0.5)\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['rmse_2'].append(mean_squared_error(test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][PRED_COLUMN])**0.5)\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['r2_1'].append(r2_score(test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_1[~test_prediction_1[VALUE_COLUMN].isna()][PRED_COLUMN]))\n",
    "            \n",
    "            p_value[VALUE_COLUMN]['r2_2'].append(r2_score(test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][VALUE_COLUMN],\\\n",
    "                                                                     test_prediction_2[~test_prediction_2[VALUE_COLUMN].isna()][PRED_COLUMN]))\n",
    "            \n",
    "            \n",
    "            stat, p = shapiro(ERROR_COLUMN_1)\n",
    "            p_value[VALUE_COLUMN]['Shapiro_p_value_1'].append(p)\n",
    "            stat, p = shapiro(ERROR_COLUMN_2)\n",
    "            p_value[VALUE_COLUMN]['Shapiro_p_value_2'].append(p)\n",
    "            \n",
    "            stat, p = normaltest(ERROR_COLUMN_1)\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_1'].append(p)\n",
    "            stat, p = normaltest(ERROR_COLUMN_2)\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_2'].append(p)\n",
    "            \n",
    "            if p_value[VALUE_COLUMN]['Shapiro_p_value_1'][-1]>ALPHA and \\\n",
    "            p_value[VALUE_COLUMN]['Shapiro_p_value_2'][-1]>ALPHA and \\\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_1'][-1]>ALPHA and \\\n",
    "            p_value[VALUE_COLUMN]['Normaltest_p_value_2'][-1]>ALPHA:\n",
    "                stat, p = ttest_ind(ERROR_COLUMN_1, ERROR_COLUMN_2)\n",
    "                p_value[VALUE_COLUMN]['P_VALUE'].append(p)\n",
    "            else:\n",
    "                if 'DMPNN' in p_value_mean_std['Model_1']:\n",
    "                    stat, p = mannwhitneyu(ERROR_COLUMN_1, ERROR_COLUMN_2, alternative = 'greater')\n",
    "                    p_value[VALUE_COLUMN]['P_VALUE'].append(p)\n",
    "                else:\n",
    "                    stat, p = mannwhitneyu(ERROR_COLUMN_1, ERROR_COLUMN_2, alternative = 'less')\n",
    "                    p_value[VALUE_COLUMN]['P_VALUE'].append(p) \n",
    "                    \n",
    "            if 'DMPNN' in p_value_mean_std['Model_1']:\n",
    "                sns.distplot(np.log(ERROR_COLUMN_2), color='g')\n",
    "                sns.distplot(np.log(ERROR_COLUMN_1), color='r')\n",
    "            else:\n",
    "                sns.distplot(np.log(ERROR_COLUMN_1), color='g')\n",
    "                sns.distplot(np.log(ERROR_COLUMN_2), color='r')\n",
    "            plt.show()\n",
    "\n",
    "    \n",
    "    for VALUE in p_value.keys():\n",
    "        for stat in p_value[VALUE].keys():\n",
    "            p_value_mean_std[VALUE+'_'+stat+'_mean'] = np.mean(p_value[VALUE][stat])\n",
    "            p_value_mean_std[VALUE+'_'+stat+'_std'] = np.std(p_value[VALUE][stat])\n",
    "            \n",
    "    print(p_value_mean_std['Model_1'], ' ', str(p_value_mean_std['Num_exp_1']))\n",
    "    print(p_value_mean_std['Model_2'], ' ', str(p_value_mean_std['Num_exp_2']))\n",
    "    print(p_value_mean_std['Data'])\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    p_values.append(p_value_mean_std)\n",
    "    \n",
    "p_values = pd.DataFrame(p_values)\n",
    "\n",
    "values = {}\n",
    "for column in list(p_values):\n",
    "    if column not in ['Model_1','Num_exp_1','Model_2','Num_exp_2','Data','Dataset']:\n",
    "        property_name = ' '.join(column.split('_')[:-1])\n",
    "        value_name = column.split('_')[-1]\n",
    "        value = p_values[column].values\n",
    "        if property_name not in values.keys():\n",
    "            values[property_name] = {}\n",
    "            values[property_name][value_name] = value\n",
    "        else:\n",
    "            values[property_name][value_name] = value\n",
    "            values[property_name] = [str(round(mean,3))+'+/-'+str(round(std, 3)) for mean, std in zip(values[property_name]['mean'], values[property_name]['std'])]\n",
    "stats_markdown_table = pd.DataFrame(columns = ['Model_1',\n",
    " 'Model_2',\n",
    " 'Num_exp_1',\n",
    " 'Num_exp_2',\n",
    " 'Data',\n",
    " 'logP P VALUE',\n",
    " 'logD P VALUE',\n",
    " 'logP rmse 1',\n",
    " 'logP rmse 2',\n",
    " 'logP r2 1',\n",
    " 'logP r2 2',\n",
    " 'logD rmse 1',\n",
    " 'logD rmse 2',\n",
    " 'logD r2 1',\n",
    " 'logD r2 2',\n",
    " 'Dataset',\n",
    " 'logP Shapiro p value 1',\n",
    " 'logP Shapiro p value 2',\n",
    " 'logP Normaltest p value 1',\n",
    " 'logP Normaltest p value 2'\n",
    " 'logD Shapiro p value 1',\n",
    " 'logD Shapiro p value 2',\n",
    " 'logD Normaltest p value 1',\n",
    " 'logD Normaltest p value 2'])\n",
    "for column in p_values.keys():\n",
    "    stats_markdown_table[column] = p_values[column]\n",
    "for column in values.keys():\n",
    "    stats_markdown_table[column] = values[column]\n",
    "stats_markdown_table = stats_markdown_table[['Model_1',\n",
    " 'Model_2',\n",
    " 'Num_exp_1',\n",
    " 'Num_exp_2',\n",
    " 'Data',\n",
    " 'logP P VALUE',\n",
    " 'logD P VALUE',\n",
    " 'logP rmse 1',\n",
    " 'logP rmse 2',\n",
    " 'logP r2 1',\n",
    " 'logP r2 2',\n",
    " 'logD rmse 1',\n",
    " 'logD rmse 2',\n",
    " 'logD r2 1',\n",
    " 'logD r2 2',\n",
    " 'Dataset']]    \n",
    "RAW_PATH = '../../data/raw/baselines/dmpnn/'\n",
    "with open(os.path.join(RAW_PATH, 'DMPNN_StructDNN_p_value_comparison.txt'), 'w') as f:\n",
    "    f.write(stats_markdown_table.to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "FILENAME = \"test_predictions.csv\"\n",
    "\n",
    "SMILES_COLUMN = 'smiles'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "VALUE_COLUMN = 'logP'\n",
    "PRED_COLUMN = 'logP_pred'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    " DATA_PATH = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(331)+'/folds'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(331)+'/folds'\n",
    "test_prediction_MULTITASK = pd.read_csv(os.path.join(DATA_PATH,'fold_'+str(0), FILENAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(326)+'/folds'\n",
    "test_prediction_logP_DMPNN = pd.read_csv(os.path.join(DATA_PATH,'fold_'+str(0), FILENAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "Preds_MULTITASK  = test_prediction_MULTITASK[~test_prediction_MULTITASK[VALUE_COLUMN].isna()]\n",
    "Preds_logP_DMPNN  = test_prediction_logP_DMPNN[~test_prediction_logP_DMPNN[VALUE_COLUMN].isna()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2095, 5)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Preds_MULTITASK.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2067, 3)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Preds_logP_DMPNN.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "ERROR_MULTITASK = ERROR_MULTITASK[np.where(ERROR_MULTITASK<2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multitask error =  0.4157461049247995\n",
      "DMPNN error =  0.4503919784984623\n"
     ]
    }
   ],
   "source": [
    "ERROR_MULTITASK = (Preds_MULTITASK[VALUE_COLUMN].values - Preds_MULTITASK[PRED_COLUMN].values)**2\n",
    "print('Multitask error = ', np.sqrt(np.mean(ERROR_MULTITASK)))\n",
    "ERROR_DMPNN = (Preds_logP_DMPNN[VALUE_COLUMN].values-Preds_logP_DMPNN[PRED_COLUMN].values)**2\n",
    "print('DMPNN error = ', np.sqrt(np.mean(ERROR_DMPNN)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_MULTITASK = np.quantile(ERROR_MULTITASK,[0.25,0.75])\n",
    "ERROR_MULTITASK = ERROR_MULTITASK[np.where(ERROR_MULTITASK>Q_MULTITASK[0])]\n",
    "ERROR_MULTITASK = ERROR_MULTITASK[np.where(ERROR_MULTITASK<Q_MULTITASK[1])]\n",
    "\n",
    "\n",
    "Q_DMPNN = np.quantile(ERROR_DMPNN,[0.25,0.75])\n",
    "ERROR_DMPNN = ERROR_DMPNN[np.where(ERROR_DMPNN>Q_DMPNN[0])]\n",
    "ERROR_DMPNN = ERROR_DMPNN[np.where(ERROR_DMPNN<Q_DMPNN[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuqklEQVR4nO3dd3xW5f3/8deVO3f2JnuQCWEEAiGEjQIyRAVUaqUWRVFqVazVn36to9Vaq61a62i1OKpWReumgFuGjCBDVlhhJxCSkISELDLu6/fHfYMRE7Lu5OSQz/PxuB85OfN9n9z55OQ65zpHaa0RQghhPi5GBxBCCNE2UsCFEMKkpIALIYRJSQEXQgiTkgIuhBAm5dqZGwsODtZxcXGduUkhhDC9jRs3Htdah5w9vlMLeFxcHBs2bOjMTQohhOkppQ41Nl6aUIQQwqSkgAshhElJARdCCJOSAi6EECYlBVwIIUxKCrgQQpiUFHAhhDApKeBCCGFSUsCFEMKkOrUnZrexYEH7lp83zzk5hBDnNTkCF0IIk5ICLoQQJiUFXAghTEoKuBBCmJQUcCGEMCkp4EIIYVJSwIUQwqSkgAshhElJR54OsPLQyjYvOzZ2rBOTCCHOZ3IELoQQJiUFXAghTEoKuBBCmJQUcCGEMCkp4EIIYVJSwIUQwqSkgAshhElJARdCCJOSAi6EECYlBVwIIUxKCrgQQpiUFHAhhDCpZgu4UipGKbVMKbVDKZWllPqNY/xDSqkjSqnNjtfUjo8rhBDitJbcjbAOuEtrvUkp5QtsVEp96Zj2tNb6yY6LJ4QQoinNFnCtdR6Q5xg+qZTaCUR1dDAhhBDn1qo2cKVUHDAYWOcYdZtSaqtS6lWlVGATy8xTSm1QSm0oLCxsX1ohhBBntLiAK6V8gA+AO7TWZcALQCIwCPsR+lONLae1XqC1Ttdap4eEhLQ/sRBCCKCFBVwpZcVevN/SWn8IoLXO11rXa61twEtARsfFFEIIcbaWXIWigFeAnVrrvzUYH9FgtsuB7c6PJ4QQoiktuQplFDAb2KaU2uwYdx8wSyk1CNDAQeBXHZBPCCFEE1pyFcoqQDUyaanz4wghhGgp6YkphBAmJQVcCCFMSgq4EEKYlBRwIYQwKSngQghhUlLAhRDCpKSACyGESUkBF0IIk5ICLoQQJiUFXAghTEoKuBBCmJQUcCGEMCkp4EIIYVJSwIUQwqSkgAshhElJARdCCJOSAi6EECYlBVwIIUxKCrgQQpiUFHAhhDApKeBCCGFSUsCFEMKkpIALIYRJSQEXQgiTkgIuhBAmJQVcCCFMSgq4EEKYVLMFXCkVo5RappTaoZTKUkr9xjE+SCn1pVIq2/E1sOPjCiGEOK0lR+B1wF1a637AcOBWpVQ/4F7ga611L+Brx/dCCCE6SbMFXGudp7Xe5Bg+CewEooDpwOuO2V4HZnRQRiGEEI1oVRu4UioOGAysA8K01nmOSceAsCaWmaeU2qCU2lBYWNierEIIIRpocQFXSvkAHwB3aK3LGk7TWmtAN7ac1nqB1jpda50eEhLSrrBCCCF+0KICrpSyYi/eb2mtP3SMzldKRTimRwAFHRNRCCFEY1pyFYoCXgF2aq3/1mDSIuA6x/B1wCfOjyeEEKIpri2YZxQwG9imlNrsGHcf8DjwX6XUXOAQcFWHJBRCCNGoZgu41noVoJqYPMG5cYQQQrSU9MQUQgiTkgIuhBAmJQVcCCFMSgq4EEKYVEuuQhGdRWvIyoL//Q98fMDfH1JTwWIxOpkQoguSAt5FWCuqSf7fGtifB88++8OEUaPgjTcgIcG4cEKILkmaULoAv5wC0l5dSsChfPj5z2H9eli2DP7xD9i2zX4U/uqr9iN0IYRwkAJuML/DBaS++RU2VwvLrruAv2bUkrphLmm77+TvaTUUrVsO6ekwdy7cc4/RcYUQXYg0oRhI1dvo9dl3nPLz4qYbw1jotRzbSc0I/xHU2er47ee/5W4XV+bffStP9e2LevJJ6NUL5s0zOroQoguQI3ADRa3fhffxUv49OYy3vPdxiS2RPSF/ZM3cNXx303ds+/U25qTO4en1zzB3Qjl6yhS45Rb46iujowshugA5AjeIW1klsd9uY29SD27tu5/x9bHcVZ9B3pHdLNu44Mx8Q6OGkl+Rz7+3/4eDFw/hvewwfC6fxsev3UtpXPiP1jlviByZC9GdyBG4QRK/2ojWNmZMLSVRB3JP/XBUE7ecubT3pVze53KWlWzkpnnh2KyuTLjvZVRdfSenFkJ0JVLADeB/OJ+QXYf552gPcgMtPFo3Fs9m/hmakjSFi5Mu5qOqTbx68zCC9+Qw6LXPOimxEKIrkgJugIhN2VR7uPJ/oyqZWz+QcHxatNxlvS8jPiCeu71Xk3VRKmkvLyEoO7eD0wohuiop4J3MtfIUwbtz+GSgG16uHkyxJbZ4WYuLhbmD52LTNm4Yf5JTfl5c8PDr0pQiRDclBbyThW4/gEu9jUfTKplZ3wd3WtdNPsQ7hFkDZvFdzX6em9OXkF2HSX3j8w5KK4ToyqSAdyatidiyl52RbuwPszLd1qtNqxkeNZy08DQeDPieXRf0J+2VpXgfK3ZyWCFEVyeXEXYi36NFeBeW8vdLYbqtHz64NTpfnw9XNruu39GT2dYt/D61mIWrbUz6fy/AkSDp5CNENyJH4J0ofMteqqyK91MUM+v7tGtdoXgzuz6F94LyyBzZk5Bdh2H3biclFUKYgRTwTmI5VUtI1iHe668YYo2lB57tXudVtr5EaV/mjC6kyt8L3nkH6uqckFYIYQZSwDtJ8O4cXGvreDHNxjhbrFPW6YaF2+uGsNetnNcnhsLRo/DCC05ZtxCi65MC3kmCsnM57mtha7QrQ3WE09Y7XEcxwhbF/+uXS01yL3joISgpcdr6hRBdl5zE7ASqvp7AA3m8kaIZqWNwa+Wlg825uW4w11uX8MzFAdz9zF77PcVnzmzbyuQkqBCmIUfgncD/cAGuNXV83Nt5zScNxeHPVFsi9wdsomzYIPvDII4fd/p2hBBdixTwThC09yg1FkVmvCvpTmw+aej6+gFYlYV7x9WDUvDxxx2yHSFE1yEFvBME7TvCingY5Brd6p6XLRWMF3d5T+QF960cGT/U/li2gwc7ZFtCiK5BCngH8ywqw6v4JB/31h3SfNLQ3d6TCHXx5abhBWhfX3j/fXmOphDnsWYLuFLqVaVUgVJqe4NxDymljiilNjteUzs2pnkF7TsCwLJeHdd8cpqviwcP+lzCp2ovu6YMgexs2L69+QWFEKbUkiPw14ApjYx/Wms9yPFa6txY54+g7CPsDFGE+UV2WPNJQ/O8xhBn6cGclH3okBD48EOw2Tp8u0KIztdsAddarwTkTkltYDlVi39OAYt66w4/+j7NTbnyiM80vtM5rL1koL1zz7p1nbJtIUTnak8b+G1Kqa2OJpZApyU6jwTuz8PFplnSC9Jt4c0v4CSzPDMY4BrFnISt2GJj4ZNPoLa207YvhOgcbS3gLwCJwCAgD3iqqRmVUvOUUhuUUhsKCwvbuDlzCjiYR4Wb4lCMNxEtfOqOM1iUC3/2nUG2rZAlU5PsPTOXL++07QshOkebCrjWOl9rXa+1tgEvARnnmHeB1jpda50eEhLS1pym5H+4gFU9YRCRnb7tS9wHMMqayK8iNlDfry8sXQqVlZ2eQwjRcdpUwJVSDRt0LwfkUoezWMur8C4q45s43anNJ6cppXjc7wrybKW8PiXcXrw/lyf3CHE+acllhAuBtUCyUipXKTUX+KtSaptSaiswDvhtB+c0Hf+cAgBWxkKaDjMkw2i3JC51H8Cd/pmcGpoGX38NJ04YkkUI4XwtuQplltY6Qmtt1VpHa61f0VrP1loP0FoP1FpP01rndUZYM/E/XEClFcrDg/DF3bAcf/a9nDJdzdMTvOyXEy5ebFgWIYRzSU/MDuJ3OJ/VMTDIpXMuH2zKAGsUv/QcxsMe6zg5ZhisXg3HjhmaSQjhHFLAO4BrZTW+haUsj4MhBrR/n+2PPpdhQ/PgmFqwWu2XFQohTE8KeAfwz7FfLpkZ60KKNv7KmzjXYG71voDnLBvInzAcNm2CAweMjiWEaCcp4B3A/3A+Va5QGRHi9Ic3tNX9PlPxVR7cmlEAvr72LvZyoyshTE2eyNMBfA/nsyYG+rqEQSfehmTloZXnnD7LpQ8vun7PNyN7M/7zPWz75h1KkqIAGBs7tjMiCiGcSI7Ana2kBL/8E6yIhYFdoPmkoStsyYRpL+anF1IZ6EP8ss1yoyshTEwKuLOtWoUCVsVCXx1sdJofccfC3PpUdlhL+N+4SHwKTxC2/aDRsYQQbSQF3NlWrqTGAkVRQXh2wRaqibZ4etkC+U1KDqURgcSu3IKqqzc6lhCiDaSAO5lt9Wo2REKyJdToKI1yQXFb/RDyXap4aUIQHmWVRK3fZXQsIUQbSAF3ppoa2LSRNdEw0Na12r8bGqTDGGuL4feJh8hLCqPnmiwoLzc6lhCilaSAO9P33+Nyqoa1MXSJ67/P5ea6wdRh46GLrFhq6mDJEqMjCSFaSQq4M61ZA8ChaG+C8DQ4zLlF4cuVtmReCstld2o0rFgB3ex+7UKYnRRwJ9Jr13A40IVQb2PuPthas+tT8MedW8ZVoC0W+OgjoyMJIVpBCrgT1a1exeooGwN01zyBeTYf3JhXP4hl/sVsG9cPNm6EtWuNjiWEaCEp4M6Sk4P16DHWdvETmGe72JZIH1sPrkjfj83fD+68U7rYC2ESUsCdxXHkuqtXANH4Ghym5VxQ3FGfzj5rGe9PjoHMTHjnHaNjCSFaQAq4s6xdS5VV4ZMxGoUyOk2r9NXB3OA5itl9dlI9oB/cey9UVRkdSwjRDCngTlK7+lvWR2gy4kYZHaVNHvOdgafFnfsv8YDDh+Hpp42OJIRohhRwZ6iuxrJ5C2tjYHj0cKPTtEmoxY/HfC/nbx6byBmfDo89BnnypDwhujIp4M6wcSMutXWsi1EMjRxqdJo2m+c1hqGRQ/nZ0APomhp7U4oQosuSAu4MjhOYpWn98HbzNjhM21mUCy9e+iLrvUr4YnoKvPGGXFYoRBcmBdwJ9Jo1HAhUJPcz/0MR0iLSmJ8xn5lJm6gJD4H586Fe7lYoRFfU9e53akJ1matZE61N2/4NPzzNZ9dGiA+Ix9UvgDsn1PP8WxtZ+Yc57Lp8TJPLzhsyr7NiCiEakCPw9srNxZpXwLooGBE9wug0TuFp9WTWgFn8I6mYrD49GPqPj3ErqzA6lhDiLFLA22vdOgB2JvmRFJRkcBjnGRQ+iCGRQ7h2/AncyyrIeP5joyMJIc4iBby9MjOpcVV4p49EKXN14GnO1SlXsyvKndfG+tHvw5WEbt1ndCQhRANSwNupbu1qNoZr0uPN2YHnXPzc/biq/1X8ZkQpRT08GfPY2/L4NSG6ECng7VFbi9q4iczo86f9+2zDo4YTF53CzRNr6JGdS8o73xgdSQjh0GwBV0q9qpQqUEptbzAuSCn1pVIq2/E1sGNjdlHbtmGpPsV3UTA0yrwdeM5FKcXs1NksTXHjq/5epL+4CJ+8IqNjCSFo2RH4a8CUs8bdC3ytte4FfO34vvtxnMAsHtQbP3c/g8N0nACPAGYN+AVzJ1ZSp+sZ8+ibcstZIbqAZgu41nolUHzW6OnA647h14EZzo1lDjpzLQU+itiB5u/A05yhkUMJ7pvGPRNsxGTuIHnRGqMjCdHttbUNPExrffpOR8eAJp8hppSap5TaoJTaUHiePXOxZs0q1kZphsecn+3fDSmluGbANfxnlA9rEtwY/vR7eBWUGB1LiG6t3ScxtdYaaPL/aa31Aq11utY6PSTEPE+qaVZxMe57D5zXJzDP5uPmw5y0G7j2khp0zSnG/PktaUoRwkBtLeD5SqkIAMfXAudFMonvvgMgK96b5OBkg8N0nn4h/UjKmMy942zErtpG78VysyshjNLWAr4IuM4xfB3wiXPimMi6ddgUWDKG46K619WY05Ons3hyHN/GuTDirwthn3TwEcIILbmMcCGwFkhWSuUqpeYCjwMTlVLZwEWO77uVujWryAqB1F6jjY7S6SwuFm5Iv4m5M61U61ps1/wC6uqMjiVEt9Ps3Qi11rOamDTByVnMw2ZDZ2ayNsm8T+BpSp8PV7Z4XuU+nHmXruDd979Dz5iBmjYN5smdCYXoLN3rf39n2bULa1k5a2JgWNQwo9MYZpSOxrPvAF5PBb10Kezda3QkIboVKeBt4XhKTf7AeAI9u2cn1NPm1A9g8Yx+7A/QnHrpBSjofuezhTCKFPA20KtXU+yliEi7wOgohnNB8VL4Tdw1KwhbeTkVP5shT/ARopNIAW+D2tUrWdNNOvC0RICLF0/3v5PfXeqB98q1lN93t9GRhOgWpIC3VkkJbnv2sTam+3TgaYkE1xCumfBbXh9iweevT1P98QdGRxLivCcFvLUyMwH4Pt6DfiH9DA7TtQx1iyPk1XfZGAH1s35O1eYNRkcS4rwmBby11q6lXoFl2HAsLhaj03Q5UwdeyaHXn6HMtZ4Tk8ZQefSQ0ZGEOG9JAW+l2tXfsiUc0ntdaHSULuuKibez+cWHCCipZt+4QVSWy02vhOgIUsBbo74etW4da6NhdM/u1wOzNS6+5g+sf/QWBuw5wcqJvSk8mW90JCHOO1LAWyMrC9eKKjJ7KoZFd98OPC019p5/kHX7LKZkHueT6cnsKdpjdCQhzitSwFvD0YGnbHB/fNx8DA5jDv3//hbH5szkxmWlvHXtIL7e/7XRkYQ4b0gBbwXb6lXke0N82nijo5iHUoS/8i4nL7+Eh5dW8dadF/HgNw9SZ5ObXwnRXlLAW6Fm5TLWxMAoaf9uHRcXfN/5kLqLJvDyIjj4/J8Y//p4Dp44aHQyIUyt2bsRCofcXDwOHWHFZLin5yij03RdCxY0Ocl1+gzIyeWNj3dzk1pL/5xkHvK5lDu8L8KqLHInQyFaSY7AW2ql/Tar+wZGEekbaXAYk3Jzg9tuQ/VO5qWP6nkkK5x7Tn5I+vE/s+zUbqPTCWE6UsBbSC9fTqmHIijjQqOjmJubG9x6K6pXL+58L4fN28dQYqtgfPHfmPzmZDblbTI6oRCmIQW8hWqXfcW3MZpR8WONjmJ+7u4wfz4MGEDq+9+y77sRPOVzJRuObmDIgiFMWziN1YdXG51SiC5PCnhL5OfjtvcAK+KkA4/TuLnBzTfDiBFYFy/lzv8dZ/8te3j4wodZk7OG0f8ezehXR/PBjg/kihUhmiAFvCUc7d/f9/alT3Afg8OcRywWuPZamDQJVqzA/8pf8PvU2zl0xyGenfIsR04eYeZ7M0l4JoHHvn2Mggp5WIQQDclVKC2xYgUVborAEeO73RPoW2LloZY/R/NsY2PHwpVXQlgYLFwIw4fjvXgx84fN55aht7B4z2KeX/88931zHw+teIiZ/Wby6/RfMypmFEopJ74LIcxHaa07bWPp6el6wwbz3WK0pl8yy2r2sPft57k149Zm5195/y87IdX5pzQ2nIn3vIiqt7HiD9dxcNzgM9PyTuax8tBK1uaupaquimjfaB4Y+wDXDLxGesWK855SaqPWOv3s8XI42ZyiItx27mFFLIyPlx6YHSlvSG8+euM+SnuGMunuFxn5xDu41NQCEOEbwc9Tfs5fLvoLswfORinFzUtuJupvUdz5+Z3sL9lvcHohOp8U8OZ8+y0AWX17SPt3JzgZFcyiV+5h26wJpLy7jBlz/kLQnpwz091d3RndczT3j7mfNTes4ZJel/Dcd8+R9GwSl797OasOr6Iz/6sUwkjSBt4MvXw51a4QNHaStLl2EpvVlbV3XcXR9GTGPPofrpj9ZzbPmcKmuVOxuVkBUEqxrWAbF8ZdSGpYKisOreDLfV/y8a6PiQ+IZ3LiZFLDU5s8ZzFviPT6FOYnBbwZ1V9/ztpouKD3JKOjdDuHLkjlWGoiI55+j7RXlhL/9SYy75hJzqgUaPDHNNAzkBl9ZjC111RW56zmq/1f8eLGF4nwiWBy0mQyIjPk6UnivCRNKOeSn4/n9l18lSDt30Y5FeDD8oevZ+mz83Gpt3HxHc8z9dZnftSscpqbxY1xceN4ZNwj3Jh2IxZl4bXNr/HgsgdZcXAFtfW1BrwDITqOHIGfy+efA5A1JIae/j0NDtO95Y5M4b3/9qHvBysZ8tJiZv7iTxwcm8qW6yaTn5r4o3ldlAtDI4eSHpHO9oLtLMlewtvb32ZJ9hImJk5kTM8xBr0LIZyrXQVcKXUQOAnUA3WNXeZiZrZPP6XQRxE5+mKjowjsbeNZV48ne+owBiz8hv7/XUbc3C0cG5jIzivGcGBCGnWe7mfmV0oxIGwAKaEp7C7azdLspby/430+zf6UE9UnuC3jNoI8gwx8R0K0T7uuA3cU8HSt9fGWzG+q68Dr66kNDuLt2DI833qXq/pf1eJF5Trwttl1RevuM+NadYrkT1aT8s43+OcWUuPlzoEJQ9g/IY2j6cnUe7j9ZJl9Jfv4bO9nbM3firfVmxsG38Bvhv2GxKDERrYgRNfQ1HXgUsCbsm4dDB/O1VfCc68XEOId0uJFpYB3Mq3xyynEo6yShK824FZ5iloPN45k9OVIRh/yhvSmODESXH445TMiegRPrHmCd7a/Q52tjmnJ07g5/WYmJkyUE56iy2mqgLe3DVwDXyilNPAvrfVP7uavlJoHzAPo2dNE7ciffopNQcHIga0q3sIASlHWM5QyoLBPDAGH8+mxJ5fwLXuJW7kFgFoPN05G9uBkRA/KI4Lon1bEG0GjeDw4hX9WLudfOav5ZPcn9PTvyfWDrufqlKvlun/R5bX3CDxKa31EKRUKfAnM11o3eWMMMx2B12aks/HoRj5/8yH+cOEfWrWsHIF3He6l5fgfLsA/pwDfo8V4F55Anf7Me3lBVBRERVH7syv51reEZyu/4ZNjywFICU3hyr5XMilxEkMjh2K1WI17I6Jb65AjcK31EcfXAqXUR0AG0PY7G3UVRUW4btjEpxfAjORpRqcR7XDK34eCAT4UDEgAwKW2Du/8EgbXBUNurv2VmYl1+XLGA+OBupgocnoGsCqglCXfPMw7YQ9zNNyLkXFjGBIxhMERgxkYNpC4gDjcLD9tZxeis7S5gCulvAEXrfVJx/Ak4I9OS2akL75Aac33qaE8FD7I6DTCiWxWV05Gh0BsgxOmWkNxMRw5AkeP4nrkCPEHjxCfl8dsm32WU9ZqdoYvY03453waAY9GwI4Q8Ld4Eaa9sERE4ePmg7fVGw9XD9xd3XGzuOFmccPqYsVqseJmceNn/X6Gt5s3fu5+Z16uLnI1r2ib9nxywoCPHN3LXYG3tdafOSWVweqWLuGEF8ROuFK6z3cHSkGPHvbXwIE/jK+thWPHIDcX98OHGZSTQ+q2HNT6avtki+JAaD1bIytYl3CIdeF1fBtYRZVLfZObejrz6Z+M87Z64+Pmg5+7H0GeQQR5BtHDswdhPmGEeYfh5+6HUkq6/4ufaHMB11rvB1KdmKVrqKujfulivkiAy/rNMDqN6CCtuod5JBAZAcMjQA/Fo6Qc32NF+BwrJjSvmBnbi5m5sQqAeqsrRYk9yU+O4mhSGEd7hZMbG0iVFWrqa6ipr+FU3Smq66uprq2msraSkzUnKa8pp/RUKXuL91JSXYJN285s3svqRbRfNLuO7yItIo3h0cNJDEyUgwshPTF/YsUK3ItLWTrFg1diLzA6jehqlKI6yJfqIF8K+8XZx2nN0Yy+hOw4RPCuwwTvOkTvb7Yw4JNKAGwuitLYcIqToijqHU1xUhTFSXGURwf96J4up9m0jeKqYvLL88mvyOfoyaPklOXwwoYXqK6zH/0HewUzMmYkF8RecOaGXnL5Y/cjBfwseuFCKtwVTJ2Ku6t78wsIoRRlMaGUxYSyb/JQ+zit8ckrInh3Dj1259AjO5eQrAMkfvnDVVg13h4UJ0ZSkhBJSUKE/WtiJJXB/gR7BRPsFUx/+p+Z/4bBN7CjcAeZuZmszV3LqsOrWLR7EQABHgGMixvHxISJTEqcJB2Tugl5Ik9DNTXUhYawMLYMl/+8yTUDr2nTauQywu6npb1IreVVBO07StDeIwTtPULgfvuwR2nFmXlO+XpREh9BSaK9qBcnRlKcFEV1kN9P1ldSVUJ2cTa7ju9i5/GdFFcVAxDqFUr/0P6khKbQu0dvbsu4zTlvVBiiozrynF++/BLX0jLeG+DCa73k/iei5fp82LqrZ7WCol5RFPWKgknpWCuq8TpeitfxUryPl+JVWErSZ7lYq06dWabGy52K0EAqQgMoDwukPCyQymB/AmdeSEZUBlprCioK2FG4g+2F21l1eBXLDi7D6mJlafZSpvaaytReU0kITHD22xcGkQLegG3hQso8FXriRXKTI9F5lKLWx5NSH09K48J/GK811opqvAtL8So8gXfhCXwKSojYlI2lzn6lS72rheQlmRT2jaUgJR7/lHjC4i5kXPw4autr2VO0h+2F28kuzmb+p/OZ/+l8+gT34eKki7k46WLGxo6VpkITkyaU06qqqAvpwb+Tq/B/o3U3rzqbNKGIDmWz4VV0Ep/8YnyOFeNSV0/IzsNnjtar/b3JH5jIsdREjg1KorB/HDarK/nl+WQVZrGtYBt7ivZQZ6vDzeJGco9k+of0p39of0K9Q3+yObl80XjShNKcpUtxrahicZo370rvS9GVubhQGeJPZYg/BSnxAOydlI7X8VL8jhbhl1tISNYBYr/dCtiP0suigintGcaJuDBORg6k0jKAzSqfdfVHWVV+jG0F2yDLfnVL/5D+9AvpR3KPZDytnka+U9EMKeAONW/9h2IfiLlsNh6uHkbHEaJ1XFyoDA2kMjSQY4OSALBWVOOXW4j/4XwCDhcQ++1W4r6FequF0phQesWFMzE2gY03Tye/+jhZBVlkFWaRmZvJikMrcFEuJAYmUlhRyJSkKQyOGNzkM0aFMaQJBaCoiLqoCF5IrWXER+tJj2zfcymkCUV0Ra5Vp/A/XEDAoXwCDh7D+3gpANV+XuQN6c3R9D4cTU+mMC6EfSX72VG4g6zCLHLK7I+vC/MOY3LSZKYmTWVy0mQCPAIMfDfdizShnMuCBbiequWbSUncFjHE6DRCdIg6T3eKkmMoSo4BwK28ioCDx1AaIjfsJn7ZZgCqAnw4NrgXeYOTyB94DSOvu4XPc5bx6d5PWbxnMW9seQNXF1fG9BzD9OTpXN73cnnkoEHkCLy2ltqe0Sz3KmDrf57krpF3tXuVcgQuzOT0Ney+R44TuX4X4Zv3ErFpD35Hi+wzuLtDWhoMGYItNZWtES58wA4+OvQZWYVZAKRHpvOzfj/j6pSrpZh3gA55Ik9rdckC/vbbcM01XHaNCy+/eJQwn7B2r1IKuDCTpjoheeeXELptP6FZBwjddoAee3Jwq7Rf6aKV4mRkD/JjAtkdYGOjZwnrPYo5EAiu8QkMTBjJ0Kih3D7s9nNvfMFPngHTcvO6z9Ux0oTSGK2pfeoJ9gcrAmZc7ZTiLYTZNNcJqTQmlNKYULIvzsCj5CQ++SWOTkdlBBZXMXlLIZc26HAE+znuuZ+DQW/xdeQfsST2QvXqbb9dQHwEtT4/XNnS5xw3FRsb27pnpHZH3buAr12LddNmnrkE7rvgAaPTCNG1KUV1kN9Pu/RrjbXyFB4lJ/EorcC9tJyq0iJUWRE99xURu7EIN1vmmdmr/b0piwp2XNoYSkVoYKM39RLN69YFvObJv1DhAeVXX0HfkL5GxxHCnJSi1tuDWm8P+8MyGiikln+yjw0ndxFYWEFGgZVL8qz0zSkgdMchAE75eFKcFElRr2hKEiLRFrlUsaW6bwHPysL1k/+xYATcPfEho9MIcV7ywsrl9GG6bzLf+R1lYe8dPOhSgL92Z15xX2Yf8CF8bz4hOw8TsXkfNV7uFKTEc2xgAsQanb7r654FXGvqbv01J91hx3VT+b+wAUYnEuK85oJiuI5ieF0U21Uhb1q280SPnSwIcmfWoH5cUZtBxP5CwrbuJ3LDHqK/2wWr9sPEidC/vzSxNKF7FvD338d1xbc8MBV+c+kjRqcRoltJ0SE8XjeOneo4r1q28qLr97xn2cWc5AFM7TUaj8oawrfsI+H7g/DccxAZCZdeCoMHg4s0rzTU/fZGRQU1v72drWFQet3PSYtIMzqREN1SXx3ME3Xjea52IhHam6dcv+N66xKWeReSM6IfPPoozJljf+j0ggX27zdvtn8vgG54BF7/50dxO3KMB38dwCuXPm90HCG6vYE6lOfrJrFa5fIv1808YF3JAFsIL9l6MmzECBg2DNavh8WL4YUXIDERrrjC6NhdQvc6At+8Gf3kE7ydAtfNf4Vgr2CjEwkhAIVitI7h37WXcFddBrnqJMOLHufqkpfYbyuyF/GHHoLZs+H4cXjiCZgxA3bvNjq6obpPAT92jJpLLybPo44Vt0/jir7yF1yIrsYVF6bZevF27TQe8JnKouot9Cn8A3eW/ZciVQWjR8Mjj8D06fDNN/YTnLfcAvn5Rkc3RPco4NXVnJo2lfqCfG64Pog//eJloxMJIc7BCyuP+E4nO/QRrvUczjMV35BQcD+PnFzCSauGqVNh7164+WZ46SVISrIfoZeVGR29U53/BVxrqm6Yjfv67/nVVZ48ce/XhHiHNL+cEMJwUZZAXg64lq3Bv2e8Wx9+X76IhML7ebL8C8oDvOD55yErC6ZMgYcfhoQEePJJqKw0OnqnOL8LeEUFp666As+F7/PHCVZufuxLBoUPMjqVEKKV+lsj+Sjo16zrcS+DXWO4++QHxP49lj+u+CPFMcHw3nuwYQMMHQp33w2xsfarVk6cMDp6hzp/C/ihQ5RnDMb6wcfcN8nCyH8tYWTMSKNTCSHaIcMtni963EFmj3sZFTOKPyz/A9F/i+amRTexNdoKn34Kq1bZT3o+8AD07Al33AG7dhkdvUOcfwW8ro66lxdQkdqPuv3Z3DgvnBmvruGixIlGJxNCOMkwt3gWzVrE1pu38suBv+StbW+R+mIqw14exnOumyh491X4/nt7B6B//hP69oVx4+CNN6C01Oj4TnP+FPC6OirffI3ihEhcb/oV230qeeJvM3n22WwyojKMTieE6AADwgaw4LIF5N6Zy1OTnqKmvobbP7udyKciGbfltzxx62B2ff8V+s9/hkOH4LrrIDQUpk2zn/w8cMDot9Au7Xqgg1JqCvAMYAFe1lo/fq75nf5Ah9JSTny1mONvLiDs63X4njzF9hB4Z1YKY+c/yaSkyc7bVivIAx2EaL9m7wfexAMdthdsZ+G2hSzOXszW/K0AhHiFMCJ6OFeURTP2u3xivsjENfeofYH4ePvliWlp9ldKCgQFOfOttJvTn8ijlLIAe4CJQC6wHpiltd7R1DJtLeCFG1ZybN3X1B46gO1ILm4HcwjJPkJEYRUApe7wRT93jk4ewehb/sKQGGOPuKWAC9F+bS3gDeWW5fLZ3s9YdXgVa3PXsqdoj32ChiGlXlyZF8CF++rpu7+cgOKKM8vV+vtQHRdDfc8YCA9DhUfgEh6Oi38QlqAgLP6BuHj74OLtg/LyAjc3+8tqBVdXsFicet+WjngiTwawV2u937GBd4DpQJMFvK2OPnYfqR+uBqDaAocDFVtj/Vk5qTd6yBB6XX4jV8YOw0WdPy1CQoj2i/aL5sa0G7kx7UYAiquK2V6wnR2FO9hRuIO1Jw7ybulhDpcexq2wgrQ86HscEovLSSreSc81OwkvB/9TzWyoKS4uP7wWLYLJzm0VaM8R+Exgitb6Rsf3s4FhWuvbzppvHnD6T2Uy0LDvazBwvE0BOp+ZsoK58popK5grr5myguRtSqzW+icdWDr8ZlZa6wVAo08uVUptaOzfgq7ITFnBXHnNlBXMlddMWUHytlZ72hyOADENvo92jBNCCNEJ2lPA1wO9lFLxSik34GpgkXNiCSGEaE6bm1C01nVKqduAz7FfRviq1jqrlatptGmlizJTVjBXXjNlBXPlNVNWkLyt0q7rwIUQQhhHrrsTQgiTkgIuhBAm5fQCrpQKUkp9qZTKdnwNbGK+6xzzZCulrnOM81JKLVFK7VJKZSmlHm8w/xylVKFSarPjdWM7c05RSu1WSu1VSt3byHR3pdS7junrlFJxDab9zjF+t1JqckvX2dlZlVITlVIblVLbHF/HN1hmuWOdp/dnaBfIG6eUqmqQ6cUGywxxvI+9SqlnlVLK4KzXNMi5WSllU0oNckwzct+OVUptUkrVOfpqNJz2k985x3ij9m2jWZVSg5RSax01YKtS6ucNpr2mlDrQYN8OckbW9uR1TKtvkGlRg/Hxjs/NXsfnyM1ZeQHQWjv1BfwVuNcxfC/wl0bmCQL2O74GOoYDAS9gnGMeN+Bb4GLH93OA552U0QLsAxIc29kC9DtrnluAFx3DVwPvOob7OeZ3B+Id67G0ZJ0GZB0MRDqGU4AjDZZZDqR3wM+/PXnjgO1NrPc7YDiggE9Pfy6MynrWPAOAfV1k38YBA4E3gJkNxjf6O2fwvm0qa2+gl2M4EsgDAhzfv9Zw3q6wbx3TyptY73+Bqx3DLwK/dmbujmhCmQ687hh+HZjRyDyTgS+11sVa6xLgS+y9Oiu11ssAtNY1wCbs15c725nbADi2c/o2AA01fB/vAxMcRybTgXe01qe01geAvY71tWSdnZpVa/291tpxxx6yAE+llLsTMnVI3qZWqJSKAPy01pna/pvwBo1/rozKOsuxbEdrNq/W+qDWeitgO2vZRn/njNy3TWXVWu/RWmc7ho8CBUBHP0arPfu2UY7PyXjsnxtouh62WUcU8DCtdZ5j+BgQ1sg8UUBOg+9zHePOUEoFAJcBXzcYfaXjX6r3lVINOxG1VrPbbziP1roOKAV6nGPZlqyzs7M2dCWwSWvd8K4O/3b8y/egs/5tdkLeeKXU90qpFUqpMQ3mz21mnUZkPe3nwMKzxhm1b1u7rJH7tllKqQzsR8T7Gox+1FEHnnbiAUl783oopTYopTKVUjMc43oAJxyfm7ass1ltKuBKqa+UUtsbeZ39F0sDrb5OUSnliv0X4lntuFkW8D8gTms9EPvRw+tNLS9+TCnVH/gL8KsGo6/RWg8Axjhes43IdpY8oKfWejBwJ/C2UsrP4EznpJQaBlRqrbc3GN0V963pOP47+A9wvdb69FHv74A+wFDszUH/Z1C8s8Vqe5f6XwB/V0oldsZG21TAtdYXaa1TGnl9AuQ7dvzpH0BBI6torhv+AiBba/33BtssanD0+DIwpC3ZW7j9H83j+IPiDxSdY9mOurVAe7KilIoGPgKu1VqfOYrRWh9xfD0JvI39X0hnaHNeR7NUkSPXRuxHXb0d8zdsSusS+9bhas46+jZ437Z2WSP3bZMcf7iXAPdrrTNPj9da52m7U8C/6Rr7tuHPfD/2cyCDsX9OAhyfm1avs6UbdvbJgCf48UnMvzYyTxBwAPvJlEDHcJBj2p+ADwCXs5aJaDB8OZDZjoyu2E/ixPPDCYv+Z81zKz8+efVfx3B/fnwScz/2EyDNrtOArAGO+a9oZJ3BjmEr9ja6m530829P3hDA4hhOwP5hP/25OPtE21Qjszq+d3FkTOgq+7bBvK/x05OYTf3OGbJvz5HVDXvT6R2NzBvh+KqAvwOPd4F9Gwi4O4aDgWwcJ0CB9/jxScxbnJH3zLaduTJHyB6OnZ8NfNXgQ5KO/ak9p+e7AfsJwL3Y/0UC+18oDewENjteNzqmPYb9RNwWYBnQp505p2J/IMU+7H/lAf4ITHMMezh2/l7HB7zhL+n9juV20+CMfWPrdNI+bVNW4AGgosG+3AyEAt7ARmCrY58+g6NwGpz3SkeezdhPYF/WYJ3pwHbHOp/H0YvYqKyOaRdy1oFEF9i3Q7G3tVZgPwLMOtfvnMH7ttGswC+B2rM+t4Mc074Btjnyvgn4GL1vgZGOTFscX+c2WGeC43Oz1/E5cndWXq21dKUXQgizkp6YQghhUlLAhRDCpKSACyGESUkBF0IIk5ICLoQQJiUFXAghTEoKuBBCmNT/B9Vc068TDlhEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(ERROR_MULTITASK, color='g')\n",
    "sns.distplot(ERROR_DMPNN, color='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MultiTask Normaltest pvalue =  1.5944149045314972e-27\n",
      "DMPNN Normaltest pvalue =  1.0063785025909551e-28\n"
     ]
    }
   ],
   "source": [
    "stat, p = shapiro(ERROR_MULTITASK)\n",
    "print('MultiTask Normaltest pvalue = ', p)\n",
    "stat, p = shapiro(ERROR_DMPNN)\n",
    "print('DMPNN Normaltest pvalue = ', p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Errors p-value  0.7756001772628621\n"
     ]
    }
   ],
   "source": [
    "stat, p = mannwhitneyu(ERROR_MULTITASK, ERROR_DMPNN, alternative = 'less')\n",
    "print('Errors p-value ', p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "stat, p = mannwhitneyu(ERROR_MULTITASK, np.ones((100000, )), alternative = 'greater')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate correlation between logD and logP predictions in multitask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_EXPS= [317]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 20.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson's r  correlation =  0.802 +/- 0.014\n",
      "Spearman's rho  correlation =  0.859 +/- 0.004\n",
      "Kendall's tau  correlation =  0.688 +/- 0.006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "correlation_mean_std = {}\n",
    "correlation_mean_std['Pearson\\'s r'] = {}\n",
    "correlation_mean_std['Spearman\\'s rho'] = {}\n",
    "correlation_mean_std['Kendall\\'s tau'] = {}\n",
    "for NUM_EXP in tqdm(NUM_EXPS):\n",
    "    DATA_PATH = \"../../data/raw/baselines/dmpnn/logs/exp_\"+str(NUM_EXP)+'/folds'\n",
    "    rmse = []\n",
    "    \n",
    "    correlation = {}\n",
    "    correlation['Pearson\\'s r'] = []\n",
    "    correlation['Spearman\\'s rho'] = []\n",
    "    correlation['Kendall\\'s tau'] = []\n",
    "\n",
    "    for FOLD in range(FOLDS):\n",
    "        import json\n",
    "        with open(os.path.join(DATA_PATH,'fold_'+str(FOLD), 'args.json'), 'r') as f:\n",
    "            args = json.load(f)\n",
    "        VALUE_COLUMNS = args['target_columns']\n",
    "        PRED_COLUMNS = [value+'_pred' for value in args['target_columns']]\n",
    "        \n",
    "        test_prediction = pd.read_csv(os.path.join(DATA_PATH,'fold_'+str(FOLD), FILENAME))\n",
    "        preds_0 = test_prediction[PRED_COLUMNS[0]]\n",
    "        preds_1 = test_prediction[PRED_COLUMNS[1]]\n",
    "        smiles = test_prediction[~test_prediction[VALUE_COLUMN].isna()][SMILES_COLUMN]\n",
    "        values = test_prediction[~test_prediction[VALUE_COLUMN].isna()][VALUE_COLUMN]\n",
    "\n",
    "\n",
    "        correlation['Pearson\\'s r'].append(preds_0.corr(preds_1))\n",
    "        correlation['Spearman\\'s rho'].append(preds_0.corr(preds_1, method='spearman'))\n",
    "        correlation['Kendall\\'s tau'].append(preds_0.corr(preds_1, method='kendall'))\n",
    "        \n",
    "for method in correlation.keys():\n",
    "    correlation_mean_std[method]['mean'] = np.mean(correlation[method])\n",
    "    correlation_mean_std[method]['std'] = np.std(correlation[method])\n",
    "    \n",
    "for method in correlation_mean_std.keys():\n",
    "    print(method, ' correlation = ', \\\n",
    "          round(correlation_mean_std[method]['mean'], 3)\\\n",
    "          ,'+/-',\\\n",
    "          round(correlation_mean_std[method]['std'], 3))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}