collate_final_results_finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import scanpy as sc\n",
    "import anndata as ad\n",
    "import glob\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "from torch.nn.parallel import DistributedDataParallel as DDP\n",
    "import torch.distributed as dist\n",
    "import torch.multiprocessing as mp\n",
    "from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, precision_recall_fscore_support, classification_report\n",
    "import scipy\n",
    "\n",
    "from performer_pytorch import PerformerLM\n",
    "from utils import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "torch.cuda.is_available()\n",
    "world_size = torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ckpt_dir = \"ckpts\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#finetune_model_list = ['finetune_1GPU_batchsize32_gradacc1_seed1000', 'finetune_1GPU_batchsize32_gradacc1_seed2000', 'finetune_1GPU_batchsize32_gradacc1_seed3000', 'finetune_1GPU_batchsize32_gradacc1_seed4000',\n",
    "                       # 'finetune_1GPU_batchsize32_gradacc1_seed2021', 'finetune_only1GPU_batchsize32_gradacc1_seed2022', 'finetune_only1GPU_batchsize32_gradacc1_seed2023']\n",
    "#epoch 5-10 for some models:\n",
    "#finetune_redo_list = ['finetune_1GPU_batchsize32_gradacc1_seed1000', 'finetune_1GPU_batchsize32_gradacc1_seed2000', 'finetune_1GPU_batchsize32_gradacc1_seed3000', \n",
    "#                      'finetune_1GPU_batchsize32_gradacc1_seed4000', 'finetune_1GPU_batchsize32_gradacc1_seed2021']\n",
    "finetune_model_list = ['finetune_shuffledataseedwise_seed'+str(i) for i in [1000, 2000, 3000, 4000, 2021, 2022, 2023]]\n",
    "#the finetuning runs that came from sample efficiency experiments with fraction data = 1.0:\n",
    "sampleff_frac1_list = ['finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_0', 'finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_1', 'finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nog2v_model_list = ['finetune_nog2v_shuffledataseedwise_seed'+str(i) for i in [1000, 2000, 3000, 4000, 0, 5000, 10000, 2021, 2022, 2023]]\n",
    "#['finetune_nog2v_seed0', 'finetune_nog2v_seed1000', 'finetune_nog2v_seed10000',\n",
    "#                    'finetune_nog2v_seed2000', 'finetune_nog2v_seed3000', 'finetune_nog2v_seed4000', 'finetune_nog2v_seed5000',\n",
    "#                    'finetune_nog2v_seed2021', 'finetune_nog2v_seed2022', 'finetune_nog2v_seed2023']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "no_pretraining_model_list = ['finetune_nopretraining_shuffledataseedwise_seed'+str(i) for i in [1000, 2000, 3000, 4000, 0, 5000, 10000, 2021, 2022, 2023]]\n",
    "#['finetune_nopretraining_seed0', 'finetune_nopretraining_seed1000', 'finetune_nopretraining_seed10000', 'finetune_nopretraining_seed2000',\n",
    " #                               'finetune_nopretraining_seed3000', 'finetune_nopretraining_seed4000', 'finetune_nopretraining_seed5000',\n",
    "  #                               'finetune_nopretraining_seed2021', 'finetune_nopretraining_seed2022', 'finetune_nopretraining_seed2023']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class SCDataset(Dataset):\n",
    "    def __init__(self, data, label, use_continuous):\n",
    "        super().__init__()\n",
    "        self.data = data\n",
    "        self.label = label\n",
    "        self.use_continuous = use_continuous\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        #rand_start = random.randint(0, self.data.shape[0]-1)\n",
    "        full_seq = self.data[index].toarray()[0]\n",
    "        full_seq[full_seq > (CLASS - 2)] = CLASS - 2\n",
    "        full_seq = torch.from_numpy(full_seq).long() #long() converts to int64\n",
    "        if(not self.use_continuous):\n",
    "            full_seq = full_seq.long() #long() converts to int64\n",
    "        full_seq = torch.cat((full_seq, torch.tensor([0]))).to(device) #this is the CLS token ?\n",
    "        seq_label = self.label[index]\n",
    "        return full_seq, seq_label\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.data.shape[0]\n",
    "\n",
    "class Identity(torch.nn.Module):\n",
    "    def __init__(self, dropout = 0., h_dim = 100, out_dim = 10):\n",
    "        super(Identity, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 1, (1, 200))\n",
    "        self.act = nn.ReLU()\n",
    "        self.fc1 = nn.Linear(in_features=SEQ_LEN, out_features=512, bias=True)\n",
    "        self.act1 = nn.ReLU()\n",
    "        self.dropout1 = nn.Dropout(dropout)\n",
    "        self.fc2 = nn.Linear(in_features=512, out_features=h_dim, bias=True)\n",
    "        self.act2 = nn.ReLU()\n",
    "        self.dropout2 = nn.Dropout(dropout)\n",
    "        self.fc3 = nn.Linear(in_features=h_dim, out_features=out_dim, bias=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x[:,None,:,:]\n",
    "        x = self.conv1(x)\n",
    "        x = self.act(x)\n",
    "        x = x.view(x.shape[0],-1)\n",
    "        x = self.fc1(x)\n",
    "        x = self.act1(x)\n",
    "        x = self.dropout1(x)\n",
    "        x = self.fc2(x)\n",
    "        x = self.act2(x)\n",
    "        x = self.dropout2(x)\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "CLASS = 7 \n",
    "USE_CONTINUOUS = False\n",
    "SIN_EMB_WAVELENGTH = None\n",
    "DATA_PATH = '/data/rna_rep_learning/scBERT/Zheng68K.h5ad'\n",
    "G2V_FILE = '/data/rna_rep_learning/scBERT/gene2vec_16906.npy'\n",
    "device = 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size of training data: 54760\n"
     ]
    }
   ],
   "source": [
    "data = sc.read_h5ad(DATA_PATH)\n",
    "\n",
    "label_dict, label = np.unique(np.array(data.obs['celltype']), return_inverse=True)  # Convert strings categorical to integrate categorical, and label_dict[label] can be restored\n",
    "class_num = np.unique(label, return_counts=True)[1].tolist() \n",
    "#class_weight = torch.tensor([(1 - (x / sum(class_num))) ** 2 for x in class_num]) #doesn't get used anywhere\n",
    "class_weight = torch.tensor([1/x for x in class_num])  #use this simpler weighting\n",
    "label = torch.from_numpy(label)\n",
    "data = data.X\n",
    "SEQ_LEN = data.shape[1] + 1 # num_genes + 1\n",
    "\n",
    "sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=2022) #update to hardcode to reduce sources of randomness/isolate performance differences between MODELS\n",
    "for index_train, index_val in sss.split(data, label):\n",
    "    data_train, label_train = data[index_train], label[index_train]\n",
    "    data_val, label_val = data[index_val], label[index_val]\n",
    "    train_dataset = SCDataset(data_train, label_train, USE_CONTINUOUS)\n",
    "    val_dataset = SCDataset(data_val, label_val, USE_CONTINUOUS)\n",
    "print(\"size of training data: {}\".format(len(train_dataset)))\n",
    "#train_sampler = DistributedSampler(train_dataset, shuffle=True)\n",
    "#val_sampler = DistributedSampler(val_dataset, shuffle=True)\n",
    "\n",
    "# attempt to seed dataloader\n",
    "def seed_worker(worker_id):\n",
    "    worker_seed = torch.initial_seed() % 2**32\n",
    "    numpy.random.seed(worker_seed)\n",
    "    random.seed(worker_seed)\n",
    "\n",
    "g = torch.Generator()\n",
    "g.manual_seed(0)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, worker_init_fn=seed_worker, generator=g)\n",
    "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True, worker_init_fn=seed_worker, generator=g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_model_paths = [glob.glob('ckpts/{}/*/*best.pth'.format(x))[0] for x in finetune_model_list]\n",
    "all_model_paths += [glob.glob('ckpts/{}/*best.pth'.format(x))[0] for x in sampleff_frac1_list]\n",
    "#all_model_paths += [glob.glob('ckpts/{}/*/*best.pth'.format(x))[0] for x in nog2v_model_list]\n",
    "#all_model_paths += [glob.glob('ckpts/{}/*/*best.pth'.format(x))[0] for x in no_pretraining_model_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model: ckpts/finetune_shuffledataseedwise_seed1000/2023-Sep-06-11:53:39/finetune_shuffledataseedwise_seed1000_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7732651570489408 || Val F1: 0.6792136768260693\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune_shuffledataseedwise_seed2000/2023-Sep-06-16:20:42/finetune_shuffledataseedwise_seed2000_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7716581446311176 || Val F1: 0.672881168001843\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune_shuffledataseedwise_seed3000/2023-Sep-06-20:47:39/finetune_shuffledataseedwise_seed3000_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7385682980277575 || Val F1: 0.6641635728794179\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune_shuffledataseedwise_seed4000/2023-Sep-07-01:14:57/finetune_shuffledataseedwise_seed4000_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7738495252008766 || Val F1: 0.6731558172693294\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune_shuffledataseedwise_seed2021/2023-Sep-07-05:41:52/finetune_shuffledataseedwise_seed2021_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7309715120525931 || Val F1: 0.6465639848820897\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune_shuffledataseedwise_seed2022/2023-Sep-07-10:08:54/finetune_shuffledataseedwise_seed2022_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7802775748721694 || Val F1: 0.6790077087603601\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune_shuffledataseedwise_seed2023/2023-Sep-07-14:35:40/finetune_shuffledataseedwise_seed2023_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7673484295105917 || Val F1: 0.6614976914515405\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_0/finetune_sampleeff_1.0_0_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7888239590942293 || Val F1: 0.7040736525446557\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_1/finetune_sampleeff_1.0_1_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7593133674214755 || Val F1: 0.6739047406003574\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_2/finetune_sampleeff_1.0_2_best.pth\n",
      "Experiment: finetune\n",
      "Val accuracy: 0.7808619430241052 || Val F1: 0.6990191089133753\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#reweight for class balance\n",
    "class_weight = torch.tensor([1/x for x in class_num])  #use this simpler weighting\n",
    "loss_fn = nn.CrossEntropyLoss(weight=class_weight).to(device)\n",
    "\n",
    "# load model\n",
    "accs = []\n",
    "accs_hard = []\n",
    "f1s = []\n",
    "f1s_hard = []\n",
    "experiment = []\n",
    "for ckpt_path in all_model_paths:\n",
    "    if 'nog2v' in ckpt_path:\n",
    "        POS_EMBED_USING = False\n",
    "        exp = 'no_g2v'\n",
    "    else:\n",
    "        POS_EMBED_USING = True\n",
    "        if 'pretraining' in ckpt_path:\n",
    "            exp = 'no_pretraining'\n",
    "        else:\n",
    "            exp = 'finetune'\n",
    "    experiment.append(exp)\n",
    "    print(\"Loading model: {}\".format(ckpt_path))\n",
    "    print(\"Experiment: {}\".format(exp))\n",
    "    model = PerformerLM(\n",
    "        num_tokens = CLASS,\n",
    "        dim = 200,\n",
    "        depth = 6,\n",
    "        max_seq_len = SEQ_LEN,\n",
    "        heads = 10,\n",
    "        local_attn_heads = 0,\n",
    "        g2v_position_emb = POS_EMBED_USING,\n",
    "        g2v_file = G2V_FILE,\n",
    "        pred_continuous = USE_CONTINUOUS,\n",
    "        sin_emb_wavelength = SIN_EMB_WAVELENGTH,\n",
    "    )\n",
    "    model = model.to(device)\n",
    "\n",
    "    # Load checkpoint\n",
    "    checkpoint = torch.load(ckpt_path, map_location=device)\n",
    "    consume_prefix_in_state_dict_if_present(checkpoint['model_state_dict'], \"module.\")\n",
    "    #print(\"Loaded finetuned ckpt...\")\n",
    "    model.to_out = Identity(dropout=0., h_dim=128, out_dim=label_dict.shape[0])\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "    model = model.to(device)    \n",
    "    cur_epoch = checkpoint['epoch']\n",
    "\n",
    "    # model eval\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    predictions = []\n",
    "    truths = []\n",
    "    # hard to distinguish cell types: CD8+ cytotoxic T cells and CD8+/CD45RA+ T cells (F1-score = 0.788 versus 0.617, P-value = 9.025 × 10 −5; accuracy = 0.801 versus 0.724, P-value = 2.265 × 10−5)\n",
    "    predictions_hard = []\n",
    "    truths_hard = []\n",
    "    with torch.no_grad():\n",
    "        for index, (data_v, labels_v) in enumerate(val_loader):\n",
    "            index += 1\n",
    "            data_v, labels_v = data_v.to(device), labels_v.to(device)\n",
    "            logits = model(data_v)\n",
    "            loss = loss_fn(logits, labels_v)\n",
    "            running_loss += loss.item()\n",
    "            softmax = nn.Softmax(dim=-1)\n",
    "            final_prob = softmax(logits)\n",
    "            final = final_prob.argmax(dim=-1)\n",
    "            #final[np.amax(np.array(final_prob.cpu()), axis=-1) < UNASSIGN_THRES] = -1\n",
    "            predictions.append(final)\n",
    "            truths.append(labels_v)\n",
    "            predictions_hard.append(final[(labels_v == np.where(label_dict=='CD8+ Cytotoxic T')[0][0]) | (labels_v == np.where(label_dict=='CD8+/CD45RA+ Naive Cytotoxic')[0][0])])\n",
    "            truths_hard.append(labels_v[(labels_v == np.where(label_dict=='CD8+ Cytotoxic T')[0][0]) | (labels_v == np.where(label_dict=='CD8+/CD45RA+ Naive Cytotoxic')[0][0])])\n",
    "        del data_v, labels_v, logits, final_prob, final\n",
    "        # gather\n",
    "        predictions = torch.cat(predictions, dim=0)\n",
    "        predictions_hard = torch.cat(predictions_hard, dim=0).cpu()\n",
    "        truths = torch.cat(truths, dim=0)\n",
    "        truths_hard = torch.cat(truths_hard, dim=0).cpu()\n",
    "        no_drop = predictions != -1\n",
    "        predictions = np.array((predictions[no_drop]).cpu())\n",
    "        truths = np.array((truths[no_drop]).cpu())\n",
    "        cur_acc = accuracy_score(truths, predictions)\n",
    "        f1 = f1_score(truths, predictions, average='macro')\n",
    "        val_loss = running_loss / index\n",
    "        accs.append(cur_acc)\n",
    "        accs_hard.append(accuracy_score(truths_hard, predictions_hard))\n",
    "        f1s.append(f1)\n",
    "        f1s_hard.append(f1_score(truths, predictions, average='macro', labels=[np.where(label_dict=='CD8+ Cytotoxic T')[0][0], np.where(label_dict=='CD8+/CD45RA+ Naive Cytotoxic')[0][0]]))\n",
    "        print(\"Val accuracy: {} || Val F1: {}\".format(cur_acc, f1))\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>acc_hard</th>\n",
       "      <th>f1_hard</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.773265</td>\n",
       "      <td>0.679214</td>\n",
       "      <td>0.783155</td>\n",
       "      <td>0.789308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.771658</td>\n",
       "      <td>0.672881</td>\n",
       "      <td>0.779679</td>\n",
       "      <td>0.787832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.738568</td>\n",
       "      <td>0.664164</td>\n",
       "      <td>0.698529</td>\n",
       "      <td>0.751786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.773850</td>\n",
       "      <td>0.673156</td>\n",
       "      <td>0.774733</td>\n",
       "      <td>0.788734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.730972</td>\n",
       "      <td>0.646564</td>\n",
       "      <td>0.682086</td>\n",
       "      <td>0.745264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.780278</td>\n",
       "      <td>0.679008</td>\n",
       "      <td>0.815374</td>\n",
       "      <td>0.795863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.767348</td>\n",
       "      <td>0.661498</td>\n",
       "      <td>0.776337</td>\n",
       "      <td>0.784371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.788824</td>\n",
       "      <td>0.704074</td>\n",
       "      <td>0.813102</td>\n",
       "      <td>0.806227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.759313</td>\n",
       "      <td>0.673905</td>\n",
       "      <td>0.749733</td>\n",
       "      <td>0.773306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.780862</td>\n",
       "      <td>0.699019</td>\n",
       "      <td>0.774866</td>\n",
       "      <td>0.794109</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  experiment  accuracy        f1  acc_hard   f1_hard\n",
       "0   finetune  0.773265  0.679214  0.783155  0.789308\n",
       "1   finetune  0.771658  0.672881  0.779679  0.787832\n",
       "2   finetune  0.738568  0.664164  0.698529  0.751786\n",
       "3   finetune  0.773850  0.673156  0.774733  0.788734\n",
       "4   finetune  0.730972  0.646564  0.682086  0.745264\n",
       "5   finetune  0.780278  0.679008  0.815374  0.795863\n",
       "6   finetune  0.767348  0.661498  0.776337  0.784371\n",
       "7   finetune  0.788824  0.704074  0.813102  0.806227\n",
       "8   finetune  0.759313  0.673905  0.749733  0.773306\n",
       "9   finetune  0.780862  0.699019  0.774866  0.794109"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'experiment':experiment, 'accuracy':accs, 'f1':f1s, 'acc_hard':accs_hard, 'f1_hard':f1s_hard})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame({'experiment':experiment, 'accuracy':accs, 'f1':f1s, 'acc_hard':accs_hard, 'f1_hard':f1s_hard}).to_csv(\"outputs/finetuning_n10_shuffledata_final_metrics.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ran nog2v and nopretrain in separate notebooks so that would be quicker -- collate results\n",
    "res_ft = pd.read_csv(\"outputs/finetuning_n10_shuffledata_final_metrics.csv\")\n",
    "res_nog2v = pd.read_csv(\"outputs/finetuning_nog2v_n10_shuffledata_final_metrics.csv\")\n",
    "res_nopretraining = pd.read_csv(\"outputs/finetuning_nopretrain_n10_shuffledata_final_metrics.csv\")\n",
    "\n",
    "#concatenate\n",
    "res = pd.concat([res_ft, res_nog2v, res_nopretraining])\n",
    "res.drop(columns=['Unnamed: 0'], inplace=True)\n",
    "res['model'] = finetune_model_list + sampleff_frac1_list + nog2v_model_list + no_pretraining_model_list\n",
    "res = res.reset_index().drop(columns=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>acc_hard</th>\n",
       "      <th>f1_hard</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.773265</td>\n",
       "      <td>0.679214</td>\n",
       "      <td>0.783155</td>\n",
       "      <td>0.789308</td>\n",
       "      <td>finetune_shuffledataseedwise_seed1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.771658</td>\n",
       "      <td>0.672881</td>\n",
       "      <td>0.779679</td>\n",
       "      <td>0.787832</td>\n",
       "      <td>finetune_shuffledataseedwise_seed2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.738568</td>\n",
       "      <td>0.664164</td>\n",
       "      <td>0.698529</td>\n",
       "      <td>0.751786</td>\n",
       "      <td>finetune_shuffledataseedwise_seed3000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.773850</td>\n",
       "      <td>0.673156</td>\n",
       "      <td>0.774733</td>\n",
       "      <td>0.788734</td>\n",
       "      <td>finetune_shuffledataseedwise_seed4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.730972</td>\n",
       "      <td>0.646564</td>\n",
       "      <td>0.682086</td>\n",
       "      <td>0.745264</td>\n",
       "      <td>finetune_shuffledataseedwise_seed2021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.780278</td>\n",
       "      <td>0.679008</td>\n",
       "      <td>0.815374</td>\n",
       "      <td>0.795863</td>\n",
       "      <td>finetune_shuffledataseedwise_seed2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.767348</td>\n",
       "      <td>0.661498</td>\n",
       "      <td>0.776337</td>\n",
       "      <td>0.784371</td>\n",
       "      <td>finetune_shuffledataseedwise_seed2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.788824</td>\n",
       "      <td>0.704074</td>\n",
       "      <td>0.813102</td>\n",
       "      <td>0.806227</td>\n",
       "      <td>finetune-sampleeff-2023-Aug-31-23:37:02/finetu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.759313</td>\n",
       "      <td>0.673905</td>\n",
       "      <td>0.749733</td>\n",
       "      <td>0.773306</td>\n",
       "      <td>finetune-sampleeff-2023-Aug-31-23:37:02/finetu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>finetune</td>\n",
       "      <td>0.780862</td>\n",
       "      <td>0.699019</td>\n",
       "      <td>0.774866</td>\n",
       "      <td>0.794109</td>\n",
       "      <td>finetune-sampleeff-2023-Aug-31-23:37:02/finetu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.627977</td>\n",
       "      <td>0.518880</td>\n",
       "      <td>0.628209</td>\n",
       "      <td>0.626215</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.603944</td>\n",
       "      <td>0.525283</td>\n",
       "      <td>0.585294</td>\n",
       "      <td>0.598451</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.669248</td>\n",
       "      <td>0.548884</td>\n",
       "      <td>0.668850</td>\n",
       "      <td>0.661015</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed3000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.640614</td>\n",
       "      <td>0.503765</td>\n",
       "      <td>0.687032</td>\n",
       "      <td>0.647867</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.748210</td>\n",
       "      <td>0.649530</td>\n",
       "      <td>0.758957</td>\n",
       "      <td>0.762759</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.739299</td>\n",
       "      <td>0.631552</td>\n",
       "      <td>0.770187</td>\n",
       "      <td>0.761351</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.742732</td>\n",
       "      <td>0.643201</td>\n",
       "      <td>0.744385</td>\n",
       "      <td>0.757676</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.746238</td>\n",
       "      <td>0.634095</td>\n",
       "      <td>0.758021</td>\n",
       "      <td>0.757234</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed2021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.750621</td>\n",
       "      <td>0.648442</td>\n",
       "      <td>0.770722</td>\n",
       "      <td>0.764623</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>no_g2v</td>\n",
       "      <td>0.743608</td>\n",
       "      <td>0.642075</td>\n",
       "      <td>0.766043</td>\n",
       "      <td>0.753522</td>\n",
       "      <td>finetune_nog2v_shuffledataseedwise_seed2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.774872</td>\n",
       "      <td>0.694254</td>\n",
       "      <td>0.791578</td>\n",
       "      <td>0.791863</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.772243</td>\n",
       "      <td>0.680846</td>\n",
       "      <td>0.777005</td>\n",
       "      <td>0.788806</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.735427</td>\n",
       "      <td>0.665822</td>\n",
       "      <td>0.695722</td>\n",
       "      <td>0.750597</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.737546</td>\n",
       "      <td>0.665863</td>\n",
       "      <td>0.704278</td>\n",
       "      <td>0.749148</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.770051</td>\n",
       "      <td>0.666483</td>\n",
       "      <td>0.792647</td>\n",
       "      <td>0.790266</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_seed0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.739737</td>\n",
       "      <td>0.664911</td>\n",
       "      <td>0.728743</td>\n",
       "      <td>0.749744</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.727538</td>\n",
       "      <td>0.668378</td>\n",
       "      <td>0.663503</td>\n",
       "      <td>0.734399</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.777648</td>\n",
       "      <td>0.670851</td>\n",
       "      <td>0.803075</td>\n",
       "      <td>0.793973</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.771731</td>\n",
       "      <td>0.675579</td>\n",
       "      <td>0.774599</td>\n",
       "      <td>0.787474</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>no_pretraining</td>\n",
       "      <td>0.771950</td>\n",
       "      <td>0.666242</td>\n",
       "      <td>0.806818</td>\n",
       "      <td>0.785737</td>\n",
       "      <td>finetune_nopretraining_shuffledataseedwise_see...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        experiment  accuracy        f1  acc_hard   f1_hard  \\\n",
       "0         finetune  0.773265  0.679214  0.783155  0.789308   \n",
       "1         finetune  0.771658  0.672881  0.779679  0.787832   \n",
       "2         finetune  0.738568  0.664164  0.698529  0.751786   \n",
       "3         finetune  0.773850  0.673156  0.774733  0.788734   \n",
       "4         finetune  0.730972  0.646564  0.682086  0.745264   \n",
       "5         finetune  0.780278  0.679008  0.815374  0.795863   \n",
       "6         finetune  0.767348  0.661498  0.776337  0.784371   \n",
       "7         finetune  0.788824  0.704074  0.813102  0.806227   \n",
       "8         finetune  0.759313  0.673905  0.749733  0.773306   \n",
       "9         finetune  0.780862  0.699019  0.774866  0.794109   \n",
       "10          no_g2v  0.627977  0.518880  0.628209  0.626215   \n",
       "11          no_g2v  0.603944  0.525283  0.585294  0.598451   \n",
       "12          no_g2v  0.669248  0.548884  0.668850  0.661015   \n",
       "13          no_g2v  0.640614  0.503765  0.687032  0.647867   \n",
       "14          no_g2v  0.748210  0.649530  0.758957  0.762759   \n",
       "15          no_g2v  0.739299  0.631552  0.770187  0.761351   \n",
       "16          no_g2v  0.742732  0.643201  0.744385  0.757676   \n",
       "17          no_g2v  0.746238  0.634095  0.758021  0.757234   \n",
       "18          no_g2v  0.750621  0.648442  0.770722  0.764623   \n",
       "19          no_g2v  0.743608  0.642075  0.766043  0.753522   \n",
       "20  no_pretraining  0.774872  0.694254  0.791578  0.791863   \n",
       "21  no_pretraining  0.772243  0.680846  0.777005  0.788806   \n",
       "22  no_pretraining  0.735427  0.665822  0.695722  0.750597   \n",
       "23  no_pretraining  0.737546  0.665863  0.704278  0.749148   \n",
       "24  no_pretraining  0.770051  0.666483  0.792647  0.790266   \n",
       "25  no_pretraining  0.739737  0.664911  0.728743  0.749744   \n",
       "26  no_pretraining  0.727538  0.668378  0.663503  0.734399   \n",
       "27  no_pretraining  0.777648  0.670851  0.803075  0.793973   \n",
       "28  no_pretraining  0.771731  0.675579  0.774599  0.787474   \n",
       "29  no_pretraining  0.771950  0.666242  0.806818  0.785737   \n",
       "\n",
       "                                                model  \n",
       "0               finetune_shuffledataseedwise_seed1000  \n",
       "1               finetune_shuffledataseedwise_seed2000  \n",
       "2               finetune_shuffledataseedwise_seed3000  \n",
       "3               finetune_shuffledataseedwise_seed4000  \n",
       "4               finetune_shuffledataseedwise_seed2021  \n",
       "5               finetune_shuffledataseedwise_seed2022  \n",
       "6               finetune_shuffledataseedwise_seed2023  \n",
       "7   finetune-sampleeff-2023-Aug-31-23:37:02/finetu...  \n",
       "8   finetune-sampleeff-2023-Aug-31-23:37:02/finetu...  \n",
       "9   finetune-sampleeff-2023-Aug-31-23:37:02/finetu...  \n",
       "10        finetune_nog2v_shuffledataseedwise_seed1000  \n",
       "11        finetune_nog2v_shuffledataseedwise_seed2000  \n",
       "12        finetune_nog2v_shuffledataseedwise_seed3000  \n",
       "13        finetune_nog2v_shuffledataseedwise_seed4000  \n",
       "14           finetune_nog2v_shuffledataseedwise_seed0  \n",
       "15        finetune_nog2v_shuffledataseedwise_seed5000  \n",
       "16       finetune_nog2v_shuffledataseedwise_seed10000  \n",
       "17        finetune_nog2v_shuffledataseedwise_seed2021  \n",
       "18        finetune_nog2v_shuffledataseedwise_seed2022  \n",
       "19        finetune_nog2v_shuffledataseedwise_seed2023  \n",
       "20  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "21  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "22  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "23  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "24   finetune_nopretraining_shuffledataseedwise_seed0  \n",
       "25  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "26  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "27  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "28  finetune_nopretraining_shuffledataseedwise_see...  \n",
       "29  finetune_nopretraining_shuffledataseedwise_see...  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#reference for calculating confidence intervals with small dataset https://sphweb.bumc.bu.edu/otlt/MPH-Modules/PH717-QuantCore/PH717-Module6-RandomError/PH717-Module6-RandomError11.html#:~:text=However%2C%20when%20you%20want%20to,heavily%20on%20the%20t%2Ddistribution.\n",
    "def calc_95_ci_10samples(values):\n",
    "    return np.std(values) / np.sqrt(len(values)) * 2.262"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.7664937910883856\n",
      "95% CI accuracy 0.012626045570591682\n",
      "f1 0.6753481122129038\n",
      "95% CI f1 0.011472237256470699\n"
     ]
    }
   ],
   "source": [
    "# scBERT reproduced metrics\n",
    "print(\"accuracy\", res[res.experiment == 'finetune'].accuracy.mean())\n",
    "print(\"95% CI accuracy\", calc_95_ci_10samples(res[res.experiment == 'finetune'].accuracy))\n",
    "print(\"f1\", res[res.experiment == 'finetune'].f1.mean())\n",
    "print(\"95% CI f1\", calc_95_ci_10samples(res[res.experiment == 'finetune'].f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.7012490869247626\n",
      "95% CI accuracy 0.03993417892856087\n",
      "f1 0.5945706463907828\n",
      "95% CI f1 0.04191533795749718\n"
     ]
    }
   ],
   "source": [
    "# scBERT no g2v metrics\n",
    "print(\"accuracy\", res[res.experiment == 'no_g2v'].accuracy.mean())\n",
    "print(\"95% CI accuracy\", calc_95_ci_10samples(res[res.experiment == 'no_g2v'].accuracy))\n",
    "print(\"f1\", res[res.experiment == 'no_g2v'].f1.mean())\n",
    "print(\"95% CI f1\", calc_95_ci_10samples(res[res.experiment == 'no_g2v'].f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.7578743608473338\n",
      "95% CI accuracy 0.013555295123394498\n",
      "f1 0.6719229468422391\n",
      "95% CI f1 0.006359236659658663\n"
     ]
    }
   ],
   "source": [
    "# scBERT no pretraining metrics\n",
    "print(\"accuracy\", res[res.experiment == 'no_pretraining'].accuracy.mean())\n",
    "print(\"95% CI accuracy\", calc_95_ci_10samples(res[res.experiment == 'no_pretraining'].accuracy))\n",
    "print(\"f1\", res[res.experiment == 'no_pretraining'].f1.mean())\n",
    "print(\"95% CI f1\", calc_95_ci_10samples(res[res.experiment == 'no_pretraining'].f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t-test for difference in accuracy between finetune and no_g2v\n",
      "Ttest_indResult(statistic=3.3363061867867634, pvalue=0.0036744992642910456)\n",
      "t-test for difference in f1 between finetune and no_g2v\n",
      "Ttest_indResult(statistic=3.9832414019364695, pvalue=0.0008717702199745173)\n",
      "\n",
      "\n",
      "t-test for difference in accuracy between finetune and no_pretraining\n",
      "Ttest_indResult(statistic=0.9854250975229971, pvalue=0.3374768950301721)\n",
      "t-test for difference in f1 between finetune and no_pretraining\n",
      "Ttest_indResult(statistic=0.5476564849638882, pvalue=0.5906548499112718)\n"
     ]
    }
   ],
   "source": [
    "# t-tests\n",
    "print(\"t-test for difference in accuracy between finetune and no_g2v\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].accuracy, res[res.experiment == 'no_g2v'].accuracy))\n",
    "\n",
    "print(\"t-test for difference in f1 between finetune and no_g2v\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].f1, res[res.experiment == 'no_g2v'].f1))\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "print(\"t-test for difference in accuracy between finetune and no_pretraining\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].accuracy, res[res.experiment == 'no_pretraining'].accuracy))\n",
    "\n",
    "print(\"t-test for difference in f1 between finetune and no_pretraining\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].f1, res[res.experiment == 'no_pretraining'].f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='experiment', ylabel='accuracy'>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEHCAYAAAC5u6FsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAn5ElEQVR4nO3deZwV1Z338c+XZl+UrVFkV1HcEsQWHR0XjBg0iWLCo6DGLRGdxGXMxKhPMpqYMWPizPiYCTHRRI3GiJEExYXgrnENjaACiiC4AMomu6zN7/mjquH2Qve92LdvL9/363VffevUqXN/dRv6V3VO1SlFBGZmZtlqUegAzMyscXHiMDOznDhxmJlZTpw4zMwsJ04cZmaWk5aFDqA+dO/ePfr371/oMMzMGpVp06Ytj4jiyuXNInH079+f0tLSQodhZtaoSPqgunJ3VZmZWU6cOMzMLCdOHGZmlhMnDjMzy4kTh5mZ5cSJo4latGoDqz/bUugwzKwJahaX4zYn6zdt5ZI/TuPvc5fTuqgF3xm2D/964n6FDsvMmhCfcTQxd7/8Pn+fuxyAzWXb+H9PzeXdJWsLHJWZNSVOHE3Me8vWVSmbX02ZmdmucuJoYk48YI8Kyx3btOTIvbsVKBoza4o8xtHEnHJIT352+iE8MPVDOrdvzRUnDqRz+9aFDsvMmhAnjiborCP6ctYRfQsdhpk1Ue6qMjOznDhxmJlZTtxV1cBN/3AlN0+Zw5I1Gzn90F58d9i+SNpp/Yiocb2Z2eflxNGArd+0lfPvmsrqDckd4P/1xLt0bt+ac47sV6Xu028v4cePzOKT1Rv56hf24menH0K71kX1HbKZNQPuqmrApn+4anvSKPfcnGVV6q3esIXL7p/OR59uYEtZMHH6Im57bl59hWlmzYwTRwO2b4+OFLWo2O20/54dq9Sb88laPttcVqFs+ker8hmamVWyqWwTz374LFM/mUpEFDqcvHJXVQO25+5t+cmpB3HT5HdYt2krxwzszsXH7VOl3gE9O9GxTUvWbdq6vaykX9f6DNWsWVu+YTnffPybLFy3EICj9jqK2068jRZqmsfmThwN3DlH9mPUYb1Zv2kr3Tq2qbZOp7atuO2cIfz00dksXrWRr32xJ5ccv3c9R2rWfD0458HtSQPg5cUv8+rHr3LUXkcVMKr8yWvikDQCuBUoAn4XETdVWn8LMCxdbA/0iIjOkoYBt2RUHQSMjoiHJN0NHAesTtedHxEz8rcXhde2VRFtW9U80H3MwGKeuPK4eorIzDKt2bym2rKlny3l1Y9fZZ/d9+Gg7gcVILL8yFvikFQEjAOGAwuBqZImRcTs8joRcWVG/cuAQ9PyZ4HBaXlXYB7wREbzV0XEhHzF3hjNXbKWP5d+RNtWRZx1RF967t6u0CGZNRun7nMqD8x5gC3bkotZerTrQYeWHTjlr6ewqWwTAN8+5NtcMeSKQoZZZ/J5xjEUmBcR8wEkjQdOA2bvpP4Y4PpqykcBkyPis7xE2QS8t2wdp417afsA+QNTP+IPFw7l18+9x4Ll6xh+wJ58d9g+tCxqmv2tZoV2QLcDuPeUe5k4dyIdW3Vk9KDR/OjFH21PGgB3z7qb8w86n93b7F7ASOtGPhNHL+CjjOWFwBHVVZTUDxgAPFPN6tHA/1Qqu1HSdcDTwDURsanyRpLGAmMB+vZt2vM2TXx9UYWrqpau3cS5d77GsrWbAZi5aA0tBJd9aWChQjRrlMq2lVHUomo3cdm2Mt5a/hbd2nWjT6c+rNm8hqkfT6VDqw6csvcp7NlhT9Ztqfg4g63btrK5bHN9hZ5XDeUQdDQwISIqXFMqqSdwCDAlo/hakjGPw4GuwNXVNRgRt0dESUSUFBcX5yfqBqJDm6r5vzxplHvkzcX1FY5Zo/fJ+k+4cMqFDL53MKMmjeKdT9/Zvm7pZ0sZ+fBIvjn5m3zlr1/h5qk3c97k8/jvaf/NnTPvZPSjo5m5fCZn7n9mhTaH9RlGcfum8bcon4ljEdAnY7l3Wlad0cD91ZSfAUyMiO13wUXEx5HYBNxF0iXWrJ15eB/6dm2/ffmLfTrTrmXF+z/eW7ae95evr+/QzBqNiODtFW+zZP0Sbnz1RqZ+MhWAOSvncPULO45P75p5F++veT/ZhuCe2fcwb9WOG263bNvChHcncPrA0/n1l37NmfufyTVDr+Hm426u1/3Jp3x2VU0FBkoaQJIwRgNnVa4kaRDQBXilmjbGkJxhZNbvGREfK5mQaSQws47jbnS6dmjNE1ceyzPvLKVtqxYcO7CYa/7yJhNe35Gny7YFD89YzBUnurvKrLLlG5Zz0RMXMW/VPFqoBW2KKl76Pn/1fNZvWU+HVh1Y8tmSWttr27ItAMf0PoZjeh+Tl5gLKW9nHBGxFbiUpJvpbeDPETFL0g2STs2oOhoYH5VutZTUn+SM5flKTd8n6S3gLaA78B952oVGpW2rIk45pCcnDNqDlkUtOHb/HlXq7N7Ot+2YVefOmXduP2vYFtvYsHVDhfWDug6iQ6sOAJw84OQK6/ZsvydH7Llj+LZzm86cNajKMXKTkte/JBHxOPB4pbLrKi3/eCfbvk8ywF65/IS6i7DhWrl+M+8uWcvBvXavdgyjNl8+aA+G9O3M6x+uAmC/PTry9cN613GUZk3D4nVVxwAHFw9mzso5HNjtQK7/px0XfA7vN5ybj7uZx957jO7tu/Otg79Fzw49eXHRi3y68VOG9RlG57ad6zH6+qemPqcKQElJSZSWlhY6jKw98sZivv/gG2zauo1ObVvy+/MOZ+iA3KcQKdsWvPzecraUbeOYgcW08uW4ZtWavGAyP3jhB9uXe7TvweSvT6Z1UfN+7LKkaRFRUrncfRcNzLZtwU8emc2mrdsAWLtxKzc+/jYPf/fonNsqaiGOGdg0ruIwy6eTB5zMlm1bePS9RyluX8xFh1zU7JNGTZw4GphNW7exYn3F21I+XrVhJ7XNrK6cus+pnLrPqbVXtAZzH4el2rUuYvgBe1QoO/WLexUoGjOzqnzG0QDdcuZgfv3cPN5atIaj9unGt/95QKFDMjPbzomjAerQpiVXfXlQocMwM6uWu6rMzCwnThxmZpYTd1U1MOs3beWHE9/iydlLGFDcgRtOO5ghfbsUOiwzs+18xtHA/NcTc3hoxmLWby5j5qI1XHLvNLaUbSt0WGZm2zlxNDBT3/+0wvLStZv4YIVntTWzhsOJo4E5tE/FbqluHVrTJ2PKdDOzQnPiaGC+/+X9GX7gHrQQ7N29A/971qG0aVn1CWRmZoXiwfEGZvd2rbjj3BK2bQtatFDtG5iZ1TOfcTRQThpm1lA5cZiZWU6cOMzMLCd5TRySRkiaI2mepGuqWX+LpBnp611JqzLWlWWsm5RRPkDSa2mbD0jypPnWuCycBq/dDp+8VehIzHZJ3gbHJRUB44DhwEJgqqRJETG7vE5EXJlR/zLg0IwmNkTE4Gqa/jlwS0SMl/Qb4FvAbXnYBbO69/Kv4IkfpguC034Fh55T0JDMcpXPM46hwLyImB8Rm4HxwGk11B8D3F9Tg5IEnABMSIv+AIz8/KGa1YMIeOEXmQXw/M8LFo7Zrspn4ugFfJSxvDAtq0JSP2AA8ExGcVtJpZJelTQyLesGrIqIrbW1adbgREDZloplWzdXX3fRNLhnJPz6KHjp1mRbswaiodzHMRqYEBFlGWX9ImKRpL2BZyS9BazOtkFJY4GxAH379q3TYM12SYsWMPSiJBGUO/KSqvU2rYN7T4eN6T/3J6+Ddl1gyLn1E6dZLfKZOBYBfTKWe6dl1RkNfDezICIWpT/nS3qOZPzjL0BnSS3Ts46dthkRtwO3A5SUlPhwzRqGE38Cew1Jzij6HQ37j6ha56VbdySNcu9OceKwBiOfXVVTgYHpVVCtSZLDpMqVJA0CugCvZJR1kdQmfd8dOBqYHREBPAuMSqueBzycx30wq1sSHDQSTvpp9Ulj3lOVxkFSPQ7Ie2hm2cpb4kjPCC4FpgBvA3+OiFmSbpB0akbV0cD4NCmUOwAolfQGSaK4KeNqrKuB70maRzLm8ft87YNZvZs1sWpZj4PgqMvqPxazncjrGEdEPA48XqnsukrLP65mu5eBQ3bS5nySK7bMmp7delct27Ie3nsGDjq9/uMxq4bvHDeraxtWwQPnwE+L4baj4aOp2W97xMWw5xcqlq18HyZ8C1a8V5dRmu0yJw6zuvbU9fD2I1C2GZbMhD+fC2Vba98OoH1XuPgFOLDS2UWUwQcv1X2sZrvAicOsrn30j4rLaxfD6g+z316CgcOrlvcc/LnCMqsrThxmda1PpSG4TnvB7jneS/TF0VByIbRoBW12g8MugK0b6y5GqxMvLHyBi5+8mO8+/V2mfpJDl2Qjp2gGd6SWlJREaWlpocOoU5u2lvGPBZ/So1Nb9t+zU6HDsUwbVsKky2DO36B4f/jqLVWTSbbWfAz3nAbL5yTLA78MY8YnNxNaQc1aMYuzHjuLbbENgNYtWvPQyIfo06lPLVs2HpKmRURJ5fKGcue45WDRqg2c8ZtXWLRqAwBnH9GXG0+v9iI0K4R2XeDMP9ZNW2/cvyNpAMydAvOfgX1PrJv2bZc98+Ez25MGwOZtm3lh4QucfcDZBYyqfviwpRG644X525MGwH2vfcjcJWsLGJHlzWcrqpatr6bM6t3msqrzjDWls42aOHE0QsvWbcqqzJqAL5yRjHOUa98d9jupcPHYdq8veb1K2aCugwoQSf1z4miEvjGk4oTAfbu25/D+XQsUjeVVzy/CBZOTZ3YMvRi+/WTSFWYFs2bzGs6bfB5vLn+zyrrMrqumzGMcjdAJg/bgrvMP56/TF9GjUxsuOmZvWhX5GKDJ6nN48rIG4Y+z/8jrS6uebQzvN5w9O+xZgIjqnxNHIzVsUA+GDepR6DDMmp0P1nxQpezsA87m30r+rQDRFIYPU83McjCs77AKy51ad+LSwZfSKnMsqonzGYeZWQ5G9B/Bus3reGjeQ3Rt25VLvngJHVt3LHRY9cqJoxF6avYS/jp9Id06tGHssXvTp2v7Qodk1qyM2m8Uo/YbVXvFJsqJo5F55p0lfPueHXfBP/32Ep696njatCwqYFRm1px4jKOReWj64grLi1dv5B8LPi1QNGbWHPmMo5Hp0alNNWVtCxCJmQF8uvFT7njzDhasWcBxvY9j9P6jkVTosPLKiaORuejYvXnq7SW8v+IzAM45sq8nOTQroMufuZw3lr0BwEuLXmLD1g1cePCFBY4qv/KaOCSNAG4FioDfRcRNldbfApRf29Ye6BERnSUNBm4DdgPKgBsj4oF0m7uB44DV6XbnR8SMfO5HQ7LHbm156nvHUfrBSrp3bM2+PZw0zApl8brF25NGuckLJjtx7CpJRcA4YDiwEJgqaVJEzC6vExFXZtS/DDg0XfwMODci5kraC5gmaUpErErXXxURE/IVe0PXsqgFR+7drdBhmDV7u7fZnXYt27Fh645JR/ds3/TvHs/n4PhQYF5EzI+IzcB44LQa6o8B7geIiHcjYm76fjGwFCjOY6xmZjnr0KoDVx52JS1bJMfg3dt159JDLy1wVPmXz66qXsBHGcsLgSOqqyipHzAAeKaadUOB1sB7GcU3SroOeBq4JiKqTA0raSwwFqBv3xyfvmZmlqUxg8YwvN9wPlzzIQd3P5jWRa0LHVLeNZTLcUcDEyKiLLNQUk/gXuCCiO3TTl4LDAIOB7oCV1fXYETcHhElEVFSXOyTFTPLn+7tujNkjyHNImlAfhPHIiDzqSa907LqjCbtpionaTfgMeCHEfFqeXlEfByJTcBdJF1iZmZWT/KZOKYCAyUNkNSaJDlMqlxJ0iCgC/BKRllrYCJwT+VB8PQsBCUXSo8EZuZrB8zMrKq8jXFExFZJlwJTSC7HvTMiZkm6ASiNiPIkMhoYHxGRsfkZwLFAN0nnp2Xll93eJ6kYEDADuCRf+2BmZlWp4t/rpqmkpCRKS0trr2hmZttJmhYRJZXLG8rguJmZNRJOHGZmlhMnDjMzy4kTh5mZ5cSJw8zMcpJV4pD0V0lfkeREY2bWzGWbCH4NnAXMlXSTpP3zGJOZmTVgWSWOiHgqIs4GhgDvA09JelnSBZJa5TNAMzNrWLLuepLUDTgf+DYwneQBTUOAJ/MSmZmZNUhZTTkiaSKwP8lMtV+LiI/TVQ9I8i3ZZmbNSLZzVf0yIp6tbkV1t6ObmVnTlW1X1YGSOpcvSOoi6Tv5CcnMzBqybBPHRRnP+yYiVgIX5SUiMzNr0LJNHEXp8y8AkFRE8jhXMzNrZrId4/gbyUD4b9Pli9MyMzNrZrJNHFeTJIt/SZefBH6Xl4jMzKxByypxRMQ24Lb0ZWZmzVi293EMBP4TOBBoW14eEXvnKS4zM2ugsh0cv4vkbGMrMAy4B/hjbRtJGiFpjqR5kq6pZv0tkmakr3clrcpYd56kuenrvIzywyS9lbb5y8xBezMzy79sE0e7iHia5BnlH0TEj4Gv1LRBeuXVOOBkkjOVMZIOzKwTEVdGxOCIGAz8L/DXdNuuwPXAEcBQ4HpJXdLNbiO5FHhg+hqR5T6YmVkdyDZxbEqnVJ8r6VJJpwMda9lmKDAvIuZHxGZgPHBaDfXHAPen778MPBkRn6b3jDwJjJDUE9gtIl6NiCA58xmZ5T6YmVkdyDZxXAG0By4HDgPOAc6rcQvoBXyUsbwwLatCUj9gAPBMLdv2St9n0+ZYSaWSSpctW1ZLqGZmlq1aE0fa5XRmRKyLiIURcUFEfCMiXq3DOEYDEyKirK4ajIjbI6IkIkqKi4vrqlkzs2av1sSR/jH/511oexHQJ2O5d1pWndHs6KaqadtF6fts2jQzszzItqtquqRJkr4p6evlr1q2mQoMlDRAUmuS5DCpciVJg4AuwCsZxVOAk9LJFLsAJwFT0unc10g6Mr2a6lzg4Sz3wczM6kC2d463BVYAJ2SUBelVUNWJiK2SLiVJAkXAnRExS9INQGlElCeR0cD4dLC7fNtPJf2UJPkA3BARn6bvvwPcDbQDJqcvMzOrJ8r4e91klZSURGmpnzdlZpYLSdOqe+ZStneO30VyhlFBRFxYB7GZmVkjkm1X1aMZ79sCpwOL6z4cMzNr6LKd5PAvmcuS7gdezEtEZmbWoGV7VVVlA4EedRmImZk1DtmOcayl4hjHJyTP6DAzs2Ym266qTvkOxMzMGoesuqoknS5p94zlzpJG5i0qMzNrsLId47g+IlaXL0TEKpJpz83MrJnJNnFUVy/bS3nNzKwJyTZxlEr6H0n7pK//AablMzAzM2uYsk0clwGbgQdIHsi0EfhuvoIyM7OGK9urqtYDVZ4ZbmZmzU+2V1U9KalzxnIXSVPyFpWZmTVY2XZVdU+vpAIgfQ647xw3M2uGsk0c2yT1LV+Q1J9qZss1M7OmL9tLan8IvCjpeUDAMcDYvEVlZmYNVraD43+TVEKSLKYDDwEb8hiXmZk1UNlOcvht4AqgNzADOJLkGeEn1LCZmZk1QdmOcVwBHA58EBHDgEOBVbVtJGmEpDmS5kmq9nJeSWdImi1plqQ/pWXDJM3IeG0snxtL0t2SFmSsG5zlPpiZWR3IdoxjY0RslISkNhHxjqT9a9pAUhEwDhgOLASmSpoUEbMz6gwErgWOjoiVknoARMSzwOC0TldgHvBERvNXRcSELGM3M7M6lG3iWJjex/EQ8KSklcAHtWwzFJgXEfMBJI0HTgNmZ9S5CBiXXt5LRCytpp1RwOSI+CzLWM3MLI+y6qqKiNMjYlVE/Bj4d+D3wMhaNusFfJSxvDAty7QfsJ+klyS9KmlENe2MBu6vVHajpDcl3SKpTXUfLmmspFJJpcuWLaslVDMzy1bOj46NiOcjYlJEbK6Dz29J8hja44ExwB2V7lDvCRwCZN6lfi0wiGTMpSs7eRJhRNweESURUVJcXFwHoZqZGez6M8ezsQjok7HcOy3LtBCYFBFbImIB8C5JIil3BjAxIraUF0TEx5HYBNxF0iVmZmb1JJ+JYyowUNIASa1JupwmVarzEMnZBpK6k3Rdzc9YP4ZK3VTpWQiSRNJdNrPuQzerByvegzcfhFUfFjoSs5zk7WFMEbFV0qUk3UxFwJ0RMUvSDUBpRExK150kaTZQRnK11ArYPq1JH+D5Sk3fJ6mY5A72GcAl+doHs7yZ9gd45AogQEUw6vdw0OmFjsosK4po+lNOlZSURGlpaaHDMEtEwM37wmfLd5R13Qcuf71wMZlVQ9K0iCipXJ7Priozq04EbFpbsWzj6sLEYrYLnDjM6luLFjDkmxXLDjuvMLGY7YK8jXGYWQ1G/Bz2OAgWTYN+R8MXxxQ6IrOsOXGYFUJRSyi5MHmZNTLuqjIzs5w4cZiZWU6cOMzMLCdOHGZmlhMnDjMzy4kTh5mZ5cSJw8zMcuLEYWZmOXHiMDOznDhxmJlZTpw4zMwsJ04cZmaWEycOMzPLSV4Th6QRkuZImifpmp3UOUPSbEmzJP0po7xM0oz0NSmjfICk19I2H0ifZ25mZvUkb4lDUhEwDjgZOBAYI+nASnUGAtcCR0fEQcC/ZqzeEBGD09epGeU/B26JiH2BlcC38rUPZmZWVT7POIYC8yJifkRsBsYDp1WqcxEwLiJWAkTE0poalCTgBGBCWvQHYGRdBm1mZjXLZ+LoBXyUsbwwLcu0H7CfpJckvSppRMa6tpJK0/KRaVk3YFVEbK2hTTMzy6NCPwGwJTAQOB7oDbwg6ZCIWAX0i4hFkvYGnpH0FrA624YljQXGAvTt27eu4zYza7byecaxCOiTsdw7Lcu0EJgUEVsiYgHwLkkiISIWpT/nA88BhwIrgM6SWtbQJul2t0dESUSUFBcX180emZlZXhPHVGBgehVUa2A0MKlSnYdIzjaQ1J2k62q+pC6S2mSUHw3MjogAngVGpdufBzycx30wM7NK8pY40nGIS4EpwNvAnyNilqQbJJVfJTUFWCFpNklCuCoiVgAHAKWS3kjLb4qI2ek2VwPfkzSPZMzj9/naBzMzq0rJQXzTVlJSEqWlpYUOw8ysUZE0LSJKKpf7znEzM8uJE4eZmeWk0JfjmjVf0++DtydBlwFwzPegY49CR2SWFScOs0IovRMevXLH8vsvwr+8WLh4zHLgriqzQnjzwYrLS96CpW8XJhazHDlxmBXCbj0rLrdoBR18o6o1Dk4cZoVw/LXQaa/kvVrAsP8LHboXNiazLHmMw6wQug+EK96AhVOhc1/o3Kf2bcwaCCcOs0Jp2Rr6H13oKMxy5q4qMzPLiROHmZnlxInDzMxy4sRhZmY5ceIwM7OcOHGYmVlOnDjM6tqnC2DlB4WOwixvfB+HWV0p2wITLoC3H0mWDx4FX78dWhQVNi6zOuYzDrO6MuuhHUkDYOYEmDO5YOGY5UteE4ekEZLmSJon6Zqd1DlD0mxJsyT9KS0bLOmVtOxNSWdm1L9b0gJJM9LX4Hzug1nWVi6oWvbp/PqPwyzP8tZVJakIGAcMBxYCUyVNiojZGXUGAtcCR0fESknlT7L5DDg3IuZK2guYJmlKRKxK118VERPyFbvZLtn/ZHjuJoiyZLlFq6TMrInJ5xjHUGBeRMwHkDQeOA2YnVHnImBcRKwEiIil6c93yytExGJJS4FiYFUe4zX7fPY8BM7+M7zy62TG26MuSyYzNGti8pk4egEfZSwvBI6oVGc/AEkvAUXAjyPib5kVJA0FWgPvZRTfKOk64GngmojYVPnDJY0FxgL07dv38+2JWbb2PTF5mTVhhR4cbwkMBI4HxgB3SOpcvlJST+Be4IKI2JYWXwsMAg4HugJXV9dwRNweESURUVJc7AfkmJnVlXwmjkVA5kMGeqdlmRYCkyJiS0QsAN4lSSRI2g14DPhhRLxavkFEfByJTcBdJF1iZmZWT/KZOKYCAyUNkNQaGA1MqlTnIZKzDSR1J+m6mp/WnwjcU3kQPD0LQZKAkcDM/O2CmZlVlrcxjojYKulSYArJ+MWdETFL0g1AaURMStedJGk2UEZytdQKSecAxwLdJJ2fNnl+RMwA7pNUDAiYAVySr30wM7OqFBGFjiHvSkpKorS0tNBhmJk1KpKmRURJ5fJCD46bmVkj48RhZmY5ceIwM7OcOHGYmVlOnDjMzCwnThxmZpYTJw4zM8uJE4eZmeXEicPMzHLixGFmZjlx4jAzs5w4cZiZWU6cOMzMLCdOHGZmlpN8PnPczHKxdTPMfhjWfgwHngpd+hc6IrNqOXGYNRT3jYIFzyfvn/0ZXPA49BpS2JjMquGuKrOGYNHrO5IGwNYN8NpvCxePWQ2cOMwahOqexNn0n85pjVNeE4ekEZLmSJon6Zqd1DlD0mxJsyT9KaP8PElz09d5GeWHSXorbfOXkpTPfTCrF70Og/7H7Fhu2RaGXly4eMxqkLcxDklFwDhgOLAQmCppUkTMzqgzELgWODoiVkrqkZZ3Ba4HSkgOu6al264EbgMuAl4DHgdGAJPztR9m9ebsCTBrIqxdDAeOhG77FDois2rlc3B8KDAvIuYDSBoPnAbMzqhzETAuTQhExNK0/MvAkxHxabrtk8AISc8Bu0XEq2n5PcBInDisKWjVFgaPKXQUZrXKZ1dVL+CjjOWFaVmm/YD9JL0k6VVJI2rZtlf6vqY2AZA0VlKppNJly5Z9jt0wM7NMhR4cbwkMBI4HxgB3SOpcFw1HxO0RURIRJcXFxXXRpJmZkd/EsQjok7HcOy3LtBCYFBFbImIB8C5JItnZtovS9zW1aWZmeZTPxDEVGChpgKTWwGhgUqU6D5GcbSCpO0nX1XxgCnCSpC6SugAnAVMi4mNgjaQj06upzgUezuM+mJlZJXkbHI+IrZIuJUkCRcCdETFL0g1AaURMYkeCmA2UAVdFxAoAST8lST4AN5QPlAPfAe4G2pEMintg3MysHimi6d9kVFJSEqWlpYUOw8ysUZE0LSJKqpQ3h8QhaRnwQaHjyKPuwPJCB2G7xL+7xq2p//76RUSVq4uaReJo6iSVVndUYA2ff3eNW3P9/RX6clwzM2tknDjMzCwnThxNw+2FDsB2mX93jVuz/P15jMPMzHLiMw4zM8uJE4eZmeXEiaMAJF0u6W1JK3f2gKss2ugs6Tt1HZuZWW2cOArjO8DwiOgSETftYhud03asCZI0XNK09GmX0ySdUOiYLCHpeElH7cJ2JZJ+mUW9l3ctsvrjxFHPJP0G2BuYLOlKSb9Ky+9OH4X7sqT5kkZlbHOVpKmS3pT0k7T4JmAfSTMk3Zz+Y340Y5tfSTo/ff++pJ9Iej39QzQoLe8g6U5J/5A0XdJp9fQ1WO2WA1+LiEOA84B7CxxPsyKppnn8jgeqTRw1bRcRpRFxeW2fHRE5J6X65sRRzyLiEmAxMAxYWWl1T+Cfga+SJAYknUQy1fxQYDBwmKRjgWuA9yJicERclcVHL4+IISSP3v1+WvZD4JmIGJrGc7OkDp9j95oVSf3TLsc7JM2S9ISkdpIGpw8me1PSxHSG5521cXhar/wAYCZAREyPiMVptVlAO0ltJF0i6eaM7c8vP/hoDuroO39O0q3pdz5T0tC0/MeS7pX0EnCvpGJJf0kP2qZKOlpSf+AS4Mp0+2PSg77fSHoN+IWkoZJeSQ/GXpa0f9r+9oO79LPuTGOZL+nyjPjWZdR/TtIESe9Iuk+S0nWnpGXT0gPOR6lHThwNy0MRsS19LvseadlJ6Ws68DowiCSR5Oqv6c9pQP+Mtq+RNAN4DmgL9N2VwJuxgSSPPz4IWAV8A7gHuDoivgC8BVxfw/Z3ARdHxGCSGaKr8w3g9YjYBPwFOD1j3ZnA+M+zA43Q5/3OAdqn3/l3gDszyg8EToyIMcCtwC0RcXj6Gb+LiPeB36TlgyPi7+l2vYGjIuJ7wDvAMRFxKHAd8LOdxDCI5DHZQ4HrJbWqps6hwL+mce0NHC2pLfBb4OSIOAyo9yfV5fOZ45a7TRnvlfHzPyPit5kV0yOfTFupeCDQdidtl7Hj9y7gGxExZ1cDNhZExIz0/TRgH6BzRDyflv0BeLC6DZU87bJTRLySFv2J5Gwzs85BwM9JkjwRsSw9Qj0SmEvyx+elOtubxmGXv/MM9wNExAuSdtOOJ49OiogN6fsTgQPTg3yA3SR13El7D0ZEeeLfHfiDpIFAANUlBIDH0oOBTZKWkhwsLqxU5x8RsRAgPcDrD6wD5qcPvyvfl7G17G+d8hlHwzcFuLD8H6ykXpJ6AGuBThn1PiD5R94m/U/wpSzbvizj9PfQOo28echM9mUkFy3UCUm9gYnAuRHxXsaq8cAZJEfBE6P53cVbF9955e+sfHl9RlkL4Mj0zGJwRPSKiHU7aS9zu58Cz0bEwcDXqHoQV67yflR3IJ9NnXrnxNHARcQTJEeir0h6C5hAcpS6Angp7aO9OSI+Av4MzEx/Ts+i+Z+SHA29KWlWumyfz2pgpaRj0uVvAs9XVzEiVgFrJR2RFo0uX5cm/8eAayKi8hnFROA0YAzNr5uqOll/5xnOBJD0z8DqiFhdTZ0ngMvKFyQNTt9WPmirbHd2PNL6/Fri2BVzgL0zeh3OzMNn1KhBZK/mJiL6p2/vTl9ExPmV6nTMeH8rSX9r5XbOqrT8A+AHNXweEVFK+rje9JT84tz3wGpxHvAbSe1JHoV8QQ11vwXcIWkbyR+78j9glwL7AtdJui4tOykilkbESklvAwdGxD/yswuNTi7fOcBGSdNJDpwu3Emdy4Fxkt4k+Vv5AsnA+CPAhPQqxMuq2e4XJF1VPyJJ/nUqIjYouYfrb5LWs+NJqfXGc1WZFZCkjuXdH0puBu0ZEVcUOKwmTdJzwPfTg6hGqfzfTdrNPA6YGxG31Nfnu6vKrLC+Un5ZKHAM8B+FDsgahYvSwfJZJF1jv625et3yGYdZPZA0Dji6UvGtEXFXIeJpDvyd548Th5mZ5cRdVWZmlhMnDjMzy4kTh1kBSbpB0ol5/ozzJe2Vz8+w5sVjHGYFIqkoY5qKfH7OczTyy0+tYfEZh1klks5RMtX8DEm/lXREOutqWyVT0c+SdHA6e+kLkh6TNCedIbVF2sZJ6Qypr0t6MGPKmPcl/VzS68D/SWdWHZWx7j/Tzy2VNETSFEnvSbokI74q0+xr57PGjgJKgPvSdtvV+xdqTY4Th1kGSQeQTOFwdMaMtfsDk0jusfgF8MeImJluMpTk7uEDSSbb+7qk7sCPSGZZHQKUAt/L+JgVETEkIqqbLuTD9HP/TjKrwCjgSKA8Qexsmn2oZtbYiJiQfv7Z6XxLGzD7nDzliFlFXwIOA6amcz+2A5YCN5BM7bCRZCqKcv+IiPkAku4neZ7KRpJE8lLaRmvglYxtHqjh8yelP98COkbEWpL5rDal81dlTrMP0JEkYXxI1Vlj+2e/22bZc+Iwq0jAHyLi2gqFUk+SP9KtSGY7LZ8NtbpZVgU8mT7ToTrrd1IOO2ZD3UbFmVG3kfx/rWma/cozqbpbyvLCXVVmFT0NjEqnrkdSV0n9SKZ0+HfgPpLnY5QbKmlAOrZxJvAi8CrJA3f2TdvoIGm/OopvZ9Ps16S22VzNcuIzDrMMETE7ndX0iTQZbAEeBrZExJ8kFQEvSzqB5CxgKvArkplsnyV5PsY2Jc97v19Sm7TpHwHv1kF8T6TjMK+k3WDrgHPY+dMDIRkr+Y2kDcA/eZzDPi9fjmu2iyQdT3KZ61drqWrWpLiryszMcuIzDjMzy4nPOMzMLCdOHGZmlhMnDjMzy4kTh5mZ5cSJw8zMcvL/AXyzdmzXVtZMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.stripplot(data=res, x=\"experiment\", y=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### results on \"hard to predict\" cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='experiment', ylabel='acc_hard'>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEHCAYAAABbZ7oVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAhz0lEQVR4nO3deZhU5Zn+8e9N0+zIIuACsqgooCaADRrRiQuiZlMT45KY0UmiMS6ZMYsh1+SXGDK/0cQZoxOJ0SSaxYVEE4UxiVsiahAjjbgAboiCDVFAAQVZennmjzoN1UtBN1T1qe66P9fVV9d5z3uq7u6Ges4576n3KCIwMzNrTqe0A5iZWfFykTAzs5xcJMzMLCcXCTMzy8lFwszMcuqcdoB8GjBgQAwfPjztGGZm7cr8+fPXRMTA5tZ1qCIxfPhwKisr045hZtauSFqWa51PN5mZWU4uEmZmlpOLhJmZ5eQiYWZmOblImJlZTi4S7cDbG7bw1rub045hZiWoQ10C2xF9d+ZCbvv7cmrrgo8ctjfXnTWOLp1d282sbfjdpojU1NYxZ8kanl6+FoAnlqzhV3OXUVuXmc79T8+/yb0LVqQZ0cxKjI8kisS697fy6Z/O5ZVVGwA4YdQg/umgAU36vbp6Q1tHM7MS5iOJInHHU8u3FQiAv7y4ip5dOlNepgb9Thi9V1tHM7MS5iOJIvHOhq1N2srKxC3nT+DG2a+ypaaO844azsQR/VNIZ2alykWiSJw2bjC/mvs61bWZ8Yf+Pbtw/Ki96NO9nGNGNjvvlplZwblIFIlDB/dhxoVHcudTb9CzSxn/MmkEfbqXpx3LrKQsXbeULbVbGL3n6LSjFA0XiSJy+LD+HD7Mp5PM2lpd1HHFY1fwwOsPADB+0HhunHwjPcp7pJwsfR64NrOSN2fFnG0FAuDpVU8z89WZKSYqHi4SZlby3nz/zaZtG5u2lSIXCTMraRu2buCRZY80aCtTGVOGT0kpUXFxkTCzknbjszfy+MrHty13VmeuO+46DtnzkBRTFY+CFwlJJ0t6SdISSVObWT9U0iOSFkh6TtJHstZ9K9nuJUknFTqrmZWe51Y/12C5JmoY1GNQSmmKT0GLhKQyYDpwCjAGOEfSmEbdvg38LiLGAWcDP0m2HZMsHwKcDPwkeT4zs7yp2LuiwXK/rv04oO8BKaUpPoU+kpgILImIpRGxFZgBnNqoTwB7JI/7ACuTx6cCMyJiS0S8BixJns/MLG++9IEvcfqBp9O7vDej+4/muuOuo2tZ17RjFY1Cf05iMPBG1nIVcESjPlcCD0q6DOgJTM7a9slG2w5u/AKSLgQuBBg6dGheQptZ6ejWuRvTJk1j2qRpaUcpSsUwcH0O8MuIGAJ8BPiNpBbnioibI6IiIioGDvT0FWZm+VToI4kVwH5Zy0OStmxfIDPmQETMldQNGNDCbc3MrIAKfSQxDxgpaYSkLmQGomc16rMcOAFA0migG7A66Xe2pK6SRgAjgacKnNfMrIG6qGNrbdNZmktFQYtERNQAlwIPAC+QuYppkaRpkj6RdPsacIGkZ4E7gfMjYxHwO2AxcD9wSUTUFjKvmVm2+1+/nxPuOoEJt0/g649+nU01m3L23VK7hVXvr2rDdG1DEZF2hrypqKiIysrKtGOYWQewdvNaJt81ma11248ivvzBL3Px2Iub9P3fV/+Xq/5+Fe9Vv8cHB36Q6467jgHdm95ZslhJmh8RFc2tK4aBazOzovPK2lcaFAiAhWsWNun37tZ3mTZ3Gu9VvwfAs6uf5cZnbmyTjG3BRcLMrBlj9hxDz/KeDdom7t30o1pV71WxuXZzg7Yl65YUNFtbcpEwM2tGry69uP646xndfzT9u/Xnc2M+x7ljzm3Sb2S/kezVo+G9548ZckxbxSw4j0mYme2mV9a+wo/m/4iqDVWcOOxEvvzBL9O5U/u5p9uOxiTaz09hZlakRvYbyU8m/yTtGAXh001FqGrt+9y/8B+senfzzjubmRWQjySKzN3zq7ji7mepCygvE9M/M54ph+yddiwzK1E+kigiEcHVf36BumSYqLo2+MH9L6YbysxKmotEEakLWL+pukHb2verc/Q2Mys8F4kisGFLDfcsqOKBRW9y2tiGs6GfWbFfjq3MzArPYxIpW7NhC6feMIcV6zJzwhyy7x78+0dGsWjluxyx/56c5SJhZilykUjZXZVV2woEwKKV7/LlYw/ggn/KffvE19ZsZGtNHQfv3bstIppZMzbXbGbmkpkse28ZJww9gcP3OjztSAXhIpGyTdVNJ7bdXF3XbN+6uuDffvsMs57N3OF10oF78ovzJtCt3Lf+Nmtrl8++nL+t+BsAty2+jf8+9r85cdiJKafKP49JpOyM8UPo3XV7rd6nTzemHJL5iP/MZ1Yw9ffPcduTy6iprePRl1dvKxAAc5a8zRW/f67NM5uVsuraan658JfbCgRAEMx4cUaKqQrHRxIpG7pnD+77ytHcPb+Krp07ceaE/dijWzk3/PUV/uvBlzOd5r3B81XrOXRInybbz3pmJR89bB9O8mcpzNrEJX+5hLn/mNukvVvnbimkKTwXiSIwbM+efG3KwQ3a7vj78gbLf1hQxUXH7k95maiubTjf1l9fWOUiYdYGXnznxeYLRFk3vnDoF1JIVHg+3VSkencrb7DcvbyM/fr14JozPtik78i9erVVLCuEmq2w5hWorUk7ie2EUJO2Q/c8lPtOv4/xe41PIVHhuUgUqa9OOYjOnbb/g/zalIPpXNaJ08YN5vLJB9Glc+ZPN3n0IM49clhaMW13vfYYXDsabqiAqwbDvRfD5vVpp7IcDu5/MEcPPnrbcteyrkw9Yip79dxrB1u1b54qvIitWLeJea+9w6GD9+DAQQ0vd313czWbq2sZ1LtjngctGdePhbWvNWzb/1j455lppLEWqK6r5uFlD/PWxrc4YegJ7LdH+/8sk6cKb6cG9+3O4HGDm123R7dy9mh0SsramdoaWPt60/als+G9t6B3x907bc/KO5VzyohT0o7RZny6ySwtZZ3hwMlN27v0gq4eZ7Li4CJhlpaXH4SVTzdq7ASTr4QuPZvbwqzN+XSTWRpqa2DmJfD+29vbhkyAT/8S+gxJLZZZYz6SMEvDpndg46qGbRtXu0BY0XGRMEtDr0Gwz9iGbSNPSiWK2Y64SJil5azb4JBPwoCD4chL4MTvpZ3IrAmPSZiloa4Oeu8Dn7417SRmO+QjCbO2Nu8XcM0B8P/3gbvOh7qm08WbFQsXCbO2tPpl+OPXMgPXdVth0T1w/Qfh/XfSTmbWLBcJs7a08mmg0VQ469+Ap25OJY7ZzrhImLWloUfS7H+7dcubtpkVARcJs7bUbzicegOo0X+9MaelkcZsp3x1k1lbG/dZGDweHv/vzFjE+M/BQVPSTmXWLBcJszQMGg2f+nnaKcx2quCnmySdLOklSUskTW1m/Y8kPZN8vSxpXda62qx1swqdtRgtWbWBv7zwFu9trk47ipmVoIIeSUgqA6YDJwJVwDxJsyJicX2fiLg8q/9lwLisp9gUEWMLmbGYXffwy1z38CsA9Olezu1fPIJDB/dJOZWZlZJCH0lMBJZExNKI2ArMAE7dQf9zgDsLnKldeGfjVqY/smTb8vpN1Vz/l1dSTGRmpajQRWIw8EbWclXS1oSkYcAI4K9Zzd0kVUp6UtJpOba7MOlTuXr16jzFTt+7m6qprm14Pf07G7emlMbMSlUxXQJ7NnB3RGTPUTAsue/qZ4DrJB3QeKOIuDkiKiKiYuDAgW2VteCGD+jJhOH9GrSdcbinkTaztlXoq5tWANl3CR+StDXnbOCS7IaIWJF8XyppNpnxilfzH7M4/fy8Cdzyt9d4bc1GTjpkbz76gX3SjmS7onozLL43c7+IMadC36FpJzJrsUIXiXnASEkjyBSHs8kcFTQgaRTQD5ib1dYPeD8itkgaAEwCfljgvEWlT/dyLj/xoLRj2O6IgN+cDsufyCw/chV84QHY+7B0c5m1UEFPN0VEDXAp8ADwAvC7iFgkaZqkT2R1PRuYERHZJ+FHA5WSngUeAa7OvirKrF1Y/uT2AgFQvRGe+ll6ecxaqeAfpouIPwF/atT2nUbLVzaz3ROAd7esnYtmmuraPobZLiqmgWuzjmfoh2C/I7Yvl/eAiRekl8eslTwth1khSfDPM2Hh75OB69Og/4i0U5m1mIuEWaGVd4dx56adwmyX+HSTmZnl5CJhZmY5uUiYmVlOLhJmZpaTi4SZmeXkImFmZjm5SJhZyWs4I5Blc5Ews5IVEVz/9PUceceRHD3jaG5/4fa0IxUdFwkzK1kPL3+Ynz//c96veZ/1W9Zz9VNXs2jNorRjFRUXCTMrWc+tfq5p25qmbaXMRcLMStb4QeNb1FbKXCTMrGQdN/Q4Lht3Gf279Wfvnntz5Yeu5OD+B6cdq6ioI43qV1RURGVlZdoxzMzaFUnzI6KiuXU+kjAzs5x2OlW4pP47Wh8R7+QvjpmZFZOW3E9iPpl7MAoYCqxNHvcFlgO+g4qZWQe109NNETEiIvYHHgY+HhEDImJP4GPAg4UOaGZm6WnNmMSREfGn+oWI+DNwVP4jmZlZsWjN7UtXSvo2cFuy/FlgZf4jmZlZsWjNkcQ5wEDgHuAPyeNzChHKzMyKQ4uOJCSVAT+OiM8WOI+ZmRWRFh1JREQtMExSlwLnMTOzItKaMYmlwBxJs4CN9Y0RcW3eU5mZWVFoTZF4NfnqBPQuTBwzMysmLS4SEfG9QgYxM7Pi0+IiIWkgcAVwCNCtvj0iji9ALjMzKwKtuQT2duBFMtNwfA94HZhXgExmZlYkWlMk9oyIXwDVEfFoRHwe8FGEmVkH1poiUZ18/4ekj0oaB+xwhlgzs1LweNXjfHLWJ/nwbz/MtZXXUltXm3akvGnN1U3/IakP8DXgx8AewOUFSWVm1k6s3byWy2dfzpbaLQDcuuhW9um1D+eM6hgTUrT4SCIi7ouI9RGxMCKOi4jDI2JWIcOZmRW759c8v61A1Jv3ZscZrm3t1U0XAMOzt0vGJna03cnA9UAZ8POIuLrR+h8BxyWLPYBBEdE3WXce8O1k3X9ExK9amtfMrC2M6j+KzupMTdRsazt0wKEpJsqv1pxumgk8Tua+Ei064ZbM+TQdOBGoAuZJmhURi+v7RMTlWf0vA8Ylj/sD3wUqyNz0aH6y7dpWZDYzK6hBPQYxbdI0/qvyv3h3y7ucMuIUzh19btqx8qY1RaJHRHyzlc8/EVgSEUsBJM0ATgUW5+h/DpnCAHAS8FD97VElPQScDNzZygztSnVtHTOfWcnrazYyecxejN2vb9qRzGwnPn7Ax/nY/h+jJmoo71Sedpy8as3VTfdJ+kgrn38w8EbWclXS1oSkYWQ+g/HX1mwr6UJJlZIqV69e3cp4xecrdy7g63c9yw2PLOGTP5nDA4veTDuSmbWApA5XIKAFRULSe5LeBf6VTKHYJOndrPZ8ORu4O5lxtsUi4uaIqIiIioEDB+YxTttbsW4Tf164vSjUBdw657UUE5mVjuraau56+S5+8NQPeGLlE2nHKRo7Pd0UES2azE/SIRGxqFHzCmC/rOUhSVtzzgYuabTtsY22nd2SLO1V505CgojtbeVlrTnYM7NdNfXxqTy47EEAbnvhNr4/6fucduBp6YYqAvl8B/pNM23zgJGSRiT3ojgbaHLZrKRRQD9gblbzA8AUSf0k9QOmJG0d1l57dOPMw7fX1C5lnbjowwekmMisNLy96W0eWvZQg7Y7X+zQw58t1pqB651R44aIqJF0KZk39zLglohYJGkaUJn1OYuzgRkR2/ehI+IdSd9n+/xQ0+oHsTuyqz91GCcftjevr9nIcQcPYviAnmlHMuvwysvKKetURk3d9stYu3funmKi4qHIPrexO08kPR0R4/PyZLuooqIiKisr04xgZu3UtfOv5daFtwJQ3qmcG46/gaMGH5VyqrYhaX5EVDS3Lp9HEmZm7dZXD/8qx+93PK+ue5UP7fsh9u21b9qRikI+i8TWPD6XmVmbGztoLGMHjU07RlFp8cC1pNOTCf7ql/tKOq1+OSKOzHM2s46trg42rkk7hdkOtebqpu9GxPr6hYhYx/ZPR5tZa1RVwv+MhWsOgOlHwKoX005k1qzWFInm+npMw2xX3HsxrFuWebz6RfjjV9PNY5ZDa4pEpaRrJR2QfF0LzC9UMLMOq2YrrHmpYdubC9PJYrYTrSkSl5EZnP4tMAPYTMNPSJtZS3TuAsOPadh2oO8EbMWpxaeLImIjMLWAWWw31NTW0dlTeLQfn7wZ/vxNWLkAhh8NJ/1n2onMmtWamw49BHw6GbAmmSpjRkScVKBs1gJ/X/o2U//wPK+/vZEPHzSQa88cS/+eXdKOZTuzx75wVnMz2ZgVl9bseg6oLxAAyc1/BuU9kbVYdW0dl965gNfWbCQCZr+0mqv+9ELascysA2lNkaiTNLR+QdJwMneMs5T8Y91mVr/X8N66z1atSyeMmXVIrbmE9d+Bv0l6lMxkfscAFxYklbXI4H7dGdy3OyvWbdrWdsSIPVNMZGYdTYuPJCLifjL3m36JzC1EvwZs2uFGVlBlncRNnzuccUP70rtbZ04buy9XnHxw2rHMrANpzcD1F8ncnW4I8AxwJJn7P/javRQdOrgP91w8Ke0YZtZBtWZM4l+BCcCyiDgOGAesK0QoMzMrDq0pEpsjYjOApK4R8SLgcxtmZh1YawauqyT1Be4FHpK0FlhWiFBmZlYcWvOJ69OTh1dKegToA9xfkFRmZlYUdmkW14h4NN9BzDq8TevgxfugczcY9VEo9z2Urfh5qm+ztrBhFdx8LLy7IrO812HwxYehvFuqscx2xjPCmbWFp3+9vUAAvPU8vPSn9PKYtZCLhFlbqK1ups23hbfi5yJh1hbGfga69tm+3GdoZlzCrMh5TMKsLfQbBhc9Ds/OyIxDjP0sdO2ddirLo5fXvsyPF/yYtza+xSkjTuH8Q85HUtqxdpuLhFlb6TcMjv1m2imsALbWbuVLD32JNZvWAPDCOy/Qo3MPzhp1VsrJdp9PN5mZ7aaFaxZuKxD1Hql6JKU0+eUiYWa2m/brvR+d1fDEzP599k8pTX65SJiZ7aaBPQbyjQnfoHvnzAckPzDwA3zxsC+mnCo/PCZhZpYHnxn9GT5xwCdYv3U9g3sNTjtO3rhImJnlSa8uvejVpVfaMfLKp5vMzCwnFwkzM8vJRcLMzHJykTAzs5wKXiQknSzpJUlLJE3N0edMSYslLZJ0R1Z7raRnkq9Zhc5qZmYNFfTqJkllwHTgRKAKmCdpVkQszuozEvgWMCki1koalPUUmyJibCEzmplZboU+kpgILImIpRGxFZgBnNqozwXA9IhYCxARqwqcyczMWqjQRWIw8EbWclXSlu0g4CBJcyQ9KenkrHXdJFUm7ac19wKSLkz6VK5evTqv4c3MSl0xfJiuMzASOBYYAjwm6bCIWAcMi4gVkvYH/irp+Yh4NXvjiLgZuBmgoqIi2jS5mVkHV+gjiRXAflnLQ5K2bFXArIiojojXgJfJFA0iYkXyfSkwGxhX4LxmZpal0EViHjBS0ghJXYCzgcZXKd1L5igCSQPInH5aKqmfpK5Z7ZOAxZiZWZsp6OmmiKiRdCnwAFAG3BIRiyRNAyojYlayboqkxUAt8I2IeFvSUcBNkurIFLOrs6+KMjOzwlNExzmNX1FREZWVlWnHMDNrVyTNj4iK5tb5E9dmZpaTi4SZmeXkImFmZjm5SJiZWU4uEmZmlpOLhJmZ5eQiYWZmOblImJlZTi4SZmaWk4uEmZnl5CJhlrY3n4e/3wRvzEs7iVkTxXA/CbPSteB2mHkJkMyhNvl7cPS/pZnIrAEfSZil6bEfsq1AADx+LdTVphbHrDEXCbM01VY3Wt4KHWhmZmv/XCTM0nTklxsuT/wilPkssBUP/2s0S9NRl8HAUfD647DveBhzatqJzBpwkTBL28gTM19mRcinm8zMLCcXCTMzy8lFwszMcnKRMDOznFwkzMwsJxcJMzPLyUXCzMxycpEwM7OcXCTMzCwnFwkzM8vJRcLMzHJykTAzs5xcJMzMLCcXCTMzy8lFwszMcnKRMDOznFwkzMwsp4IXCUknS3pJ0hJJU3P0OVPSYkmLJN2R1X6epFeSr/MKndXMzBoq6O1LJZUB04ETgSpgnqRZEbE4q89I4FvApIhYK2lQ0t4f+C5QAQQwP9l2bSEzm5nZdoU+kpgILImIpRGxFZgBNL7T+wXA9Po3/4hYlbSfBDwUEe8k6x4CTi5wXjMzy1LoIjEYeCNruSppy3YQcJCkOZKelHRyK7ZF0oWSKiVVrl69Oo/RzcysGAauOwMjgWOBc4CfSerb0o0j4uaIqIiIioEDBxYmoZlZiSp0kVgB7Je1PCRpy1YFzIqI6oh4DXiZTNFoybZmZlZAhS4S84CRkkZI6gKcDcxq1OdeMkcRSBpA5vTTUuABYIqkfpL6AVOSNjMzayMFvbopImokXUrmzb0MuCUiFkmaBlRGxCy2F4PFQC3wjYh4G0DS98kUGoBpEfFOIfOamVlDioi0M+RNRUVFVFZWph3DzKxdkTQ/IiqaW1cMA9dmZlakXCTMzCwnFwkzM8vJRcLMzHIq6NVNZiWjZgssuhc2roYxn4C+Q9NOZJYXLhJmuysCfn0qLJ+bWZ59FXz+Adj70HRzmeWBTzeZ7a7lc7cXCICtG2Dez9LLY5ZHLhJmu6u5zxpFXdvnMCsAFwmz3TX0QzBk4vbl8h4w4YL08pjlkcckzHZXp05w3ixY+Ptk4Po06D8i7VRmeeEiYZYP5d1h3LlppzDLO59uMjOznFwkzMwsJxcJMzPLyUXCzMxycpEwM7OcXCTMzCynDnVnOkmrgWVp5yigAcCatEPYLvPfr/3q6H+7YRExsLkVHapIdHSSKnPdYtCKn/9+7Vcp/+18usnMzHJykTAzs5xcJNqXm9MOYLvFf7/2q2T/dh6TMDOznHwkYWZmOblImJlZTi4SBSbpK5JekLRW0tRdfI6+ki7OdzYzs51xkSi8i4ETI6JfRFy9i8/RN3ke64AknShpvqTnk+/Hp53JMiQdK+moXdiuQtL/tKDfE7uWrO24SBSQpJ8C+wN/lnS5pBuS9l9K+h9JT0haKumMrG2+IWmepOckfS9pvho4QNIzkq5J/uHel7XNDZLOTx6/Lul7kp5O3nRGJe09Jd0i6SlJCySd2ka/Btu5NcDHI+Iw4DzgNynnKSmSdnTztWOBZovEjraLiMqI+MrOXjsiWl2A2pqLRAFFxEXASuA4YG2j1fsARwMfI1MEkDQFGAlMBMYCh0v6J2Aq8GpEjI2Ib7TgpddExHjgRuDrSdu/A3+NiIlJnmsk9dyNH6+kSBqenDb8maRFkh6U1F3SWElPJkX9Hkn9dvAcE5J+9cV+IUBELIiIlUm3RUB3SV0lXSTpmqztz6/f0SgFefqdz5Z0ffI7XyhpYtJ+paTfSJoD/EbSQEm/T3bQ5kmaJGk4cBFwebL9MckO3k8l/R34oaSJkuYmO15PSDo4ef5tO3LJa92SZFkq6StZ+TZk9Z8t6W5JL0q6XZKSdR9J2uYnO5f30YZcJNJzb0TURcRiYK+kbUrytQB4GhhFpmi01h+S7/OB4VnPPVXSM8BsoBswdFeCl7CRwPSIOARYB3wK+DXwzYj4APA88N0dbH8r8KWIGAvU5ujzKeDpiNgC/B44PWvdWcCM3fkB2qHd/Z0D9Eh+5xcDt2S1jwEmR8Q5wPXAjyJiQvIaP4+I14GfJu1jI+LxZLshwFER8VXgReCYiBgHfAf4zxwZRgEnkdkB/K6k8mb6jAP+Lcm1PzBJUjfgJuCUiDgcaHZ+pULyPa7TsyXrsbK+XxURN2V3TPZostXQsMB3y/HctWz/Gwv4VES8tKuBjdci4pnk8XzgAKBvRDyatP0KuKu5DSX1BXpHxNyk6Q4yR5HZfQ4BfkCmoBMRq5M9zyOBV8i80czJ20/TPuzy7zzLnQAR8ZikPZK/BcCsiNiUPJ4MjEl23gH2kNQrx/PdFRH1Rb4P8CtJI4EAmnvzB/hjUvi3SFpFZsewqlGfpyKiCiDZmRsObACWRsRrWT/LhTv5efPKRxLF5QHg8/X/OCUNljQIeA/ondVvGZl/0F2Tf/AntPC5L8s6hB2X1+SlIbuw15K5oCAvJA0B7gH+OSJezVo1AziTzN7tPVF6n37Nx++88e+sfnljVlsn4MjkiGFsRAyOiA05ni97u+8Dj0TEocDHabrDVq/xz9HcDnpL+rQ5F4kiEhEPktnDnCvpeeBuMnufbwNzknOq10TEG8DvgIXJ9wUtePrvk9nLeU7SomTZds96YK2kY5LlzwGPNtcxItYB70k6Imk6u35dUuj/CEyNiMZHCvcApwLnUHqnmprT4t95lrMAJB0NrI+I9c30eRC4rH5B0tjkYeMdtMb6ACuSx+fvJMeueAnYP+tswlkFeI0dKopK1ZFFxPDk4S+TLyLi/EZ9emU9vp7M+dHGz/OZRstXAFfs4PWIiEoyV2eQHFZ/qfU/ge3EecBPJfUAlgL/soO+XwB+JqmOzBtb/ZvVpcCBwHckfSdpmxIRqyJiraQXgDER8VRhfoR2pzW/c4DNkhaQ2Un6fI4+XwGmS3qOzPviY2QGrf8XuDu5GvCyZrb7IZnTTd8mU+jzKiI2KfMZqfslbQTm5fs1dsZzN5m1EUm96k9hKPPByn0i4l9TjtWhSZoNfD3ZYWqX6v/dJKeKpwOvRMSP2ur1fbrJrO18tP5STOAY4D/SDmTtwgXJQPYiMqe3btpx9/zykYRZnkmaDkxq1Hx9RNyaRp5S4N954bhImJlZTj7dZGZmOblImJlZTi4SZm1E0jRJkwv8GudL2reQr2GlxWMSZm1AUlnWVA6FfJ3ZtPNLPq24+EjCSpqkc5WZPv0ZSTdJOiKZXbSbMtOrL5J0aDJL52OS/ijppWQm0E7Jc0xJZgJ9WtJdWdOqvC7pB5KeBj6dzCB6Rta6q5LXrZQ0XtIDkl6VdFFWviZTxyv37KhnABXA7cnzdm/zX6h1OC4SVrIkjSYzzcGkrJlZDwZmkfkMww+B2yJiYbLJRDKfuh1DZqK5T0oaAHybzGyi44FK4KtZL/N2RIyPiOam1FievO7jZD6NfwZwJFBfDHJNHQ/NzI4aEXcnr//ZZP6hTZjtJk/LYaXsBOBwYF4y72F3YBUwjcz0B5vJTNdQ76mIWAog6U4y9wPZTKZozEmeowswN2ub3+7g9Wcl358HekXEe2Tmd9qSzOeUPXU8QC8yxWE5TWdHHd7yH9us5VwkrJQJ+FVEfKtBo7QPmTfkcjKzetbP+tncbKICHkruSdCcjTnaYfusn3U0nAG0jsz/zR1NHd94xlCfWrKC8OkmK2V/Ac5IpmNHUn9Jw8hMe/D/gNvJ3N+h3kRJI5KxiLOAvwFPkrk5zIHJc/SUdFCe8uWaOn5HdjZrqVmr+EjCSlZELE5m73wweeOvBmYC1RFxh6Qy4AlJx5PZu58H3EBmxtZHyNzfoU6Z+4vfKalr8tTfBl7OQ74Hk3GTucmprA3AueS+qx1kxjZ+KmkT8CGPS9ju8iWwZi0g6Vgyl5Z+bCddzToUn24yM7OcfCRhZmY5+UjCzMxycpEwM7OcXCTMzCwnFwkzM8vJRcLMzHL6PxcnGmAEm/HTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.stripplot(data=res, x=\"experiment\", y=\"acc_hard\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.7647593582887702\n",
      "95% CI accuracy 0.029702628612152265\n",
      "f1 0.7816800762766779\n",
      "95% CI f1 0.01319169765115049\n"
     ]
    }
   ],
   "source": [
    "# scBERT reproduced metrics\n",
    "print(\"accuracy\", res[res.experiment == 'finetune'].acc_hard.mean())\n",
    "print(\"95% CI accuracy\", calc_95_ci_10samples(res[res.experiment == 'finetune'].acc_hard))\n",
    "print(\"f1\", res[res.experiment == 'finetune'].f1_hard.mean())\n",
    "print(\"95% CI f1\", calc_95_ci_10samples(res[res.experiment == 'finetune'].f1_hard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.7137700534759357\n",
      "95% CI accuracy 0.04560930436139858\n",
      "f1 0.7090712417439384\n",
      "95% CI f1 0.045531364747484204\n"
     ]
    }
   ],
   "source": [
    "# scBERT no g2v metrics\n",
    "print(\"accuracy\", res[res.experiment == 'no_g2v'].acc_hard.mean())\n",
    "print(\"95% CI accuracy\", calc_95_ci_10samples(res[res.experiment == 'no_g2v'].acc_hard))\n",
    "print(\"f1\", res[res.experiment == 'no_g2v'].f1_hard.mean())\n",
    "print(\"95% CI f1\", calc_95_ci_10samples(res[res.experiment == 'no_g2v'].f1_hard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.7537967914438503\n",
      "95% CI accuracy 0.03486210114378854\n",
      "f1 0.7722006588576715\n",
      "95% CI f1 0.015689108962724744\n"
     ]
    }
   ],
   "source": [
    "# scBERT no pretraining metrics\n",
    "print(\"accuracy\", res[res.experiment == 'no_pretraining'].acc_hard.mean())\n",
    "print(\"95% CI accuracy\", calc_95_ci_10samples(res[res.experiment == 'no_pretraining'].acc_hard))\n",
    "print(\"f1\", res[res.experiment == 'no_pretraining'].f1_hard.mean())\n",
    "print(\"95% CI f1\", calc_95_ci_10samples(res[res.experiment == 'no_pretraining'].f1_hard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t-test for difference in accuracy between finetune and no_g2v\n",
      "Ttest_indResult(statistic=2.010328735354663, pvalue=0.059624286138472676)\n",
      "t-test for difference in f1 between finetune and no_g2v\n",
      "Ttest_indResult(statistic=3.2869235477372825, pvalue=0.004097870729776195)\n",
      "\n",
      "\n",
      "t-test for difference in accuracy between finetune and no_pretraining\n",
      "Ttest_indResult(statistic=0.5136455208473122, pvalue=0.6137473808085099)\n",
      "t-test for difference in f1 between finetune and no_pretraining\n",
      "Ttest_indResult(statistic=0.9923925272382635, pvalue=0.33416019300929245)\n"
     ]
    }
   ],
   "source": [
    "# t-tests\n",
    "print(\"t-test for difference in accuracy between finetune and no_g2v\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].acc_hard, res[res.experiment == 'no_g2v'].acc_hard))\n",
    "\n",
    "print(\"t-test for difference in f1 between finetune and no_g2v\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].f1_hard, res[res.experiment == 'no_g2v'].f1_hard))\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "print(\"t-test for difference in accuracy between finetune and no_pretraining\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].acc_hard, res[res.experiment == 'no_pretraining'].acc_hard))\n",
    "\n",
    "print(\"t-test for difference in f1 between finetune and no_pretraining\")\n",
    "print(scipy.stats.ttest_ind(res[res.experiment == 'finetune'].f1_hard, res[res.experiment == 'no_pretraining'].f1_hard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sample efficiency results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2\"\n",
    "torch.cuda.is_available()\n",
    "world_size = torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sampleeff_model_paths = glob.glob('ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/*/*best.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.5_1/finetune_sampleeff_0.5_1_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.25_2/finetune_sampleeff_0.25_2_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_0/finetune_sampleeff_1.0_0_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.75_1/finetune_sampleeff_0.75_1_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.5_2/finetune_sampleeff_0.5_2_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.1_0/finetune_sampleeff_0.1_0_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.25_1/finetune_sampleeff_0.25_1_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.75_2/finetune_sampleeff_0.75_2_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.75_0/finetune_sampleeff_0.75_0_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_1/finetune_sampleeff_1.0_1_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.5_0/finetune_sampleeff_0.5_0_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.1_2/finetune_sampleeff_0.1_2_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_2/finetune_sampleeff_1.0_2_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.25_0/finetune_sampleeff_0.25_0_best.pth',\n",
       " 'ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.1_1/finetune_sampleeff_0.1_1_best.pth']"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampleeff_model_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.5_1/finetune_sampleeff_0.5_1_best.pth\n",
      "Experiment: finetune_sampleeff_0.5_1\n",
      "Run: 1\n",
      "Fraction: 0.5\n",
      "Val accuracy: 0.735354273192111 || Val F1: 0.6299132294645715\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.25_2/finetune_sampleeff_0.25_2_best.pth\n",
      "Experiment: finetune_sampleeff_0.25_2\n",
      "Run: 2\n",
      "Fraction: 0.25\n",
      "Val accuracy: 0.6778670562454346 || Val F1: 0.55810039454442\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_0/finetune_sampleeff_1.0_0_best.pth\n",
      "Experiment: finetune_sampleeff_1.0_0\n",
      "Run: 0\n",
      "Fraction: 1.0\n",
      "Val accuracy: 0.7892622352081812 || Val F1: 0.7046072236415316\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.75_1/finetune_sampleeff_0.75_1_best.pth\n",
      "Experiment: finetune_sampleeff_0.75_1\n",
      "Run: 1\n",
      "Fraction: 0.75\n",
      "Val accuracy: 0.756245434623813 || Val F1: 0.6650647201377757\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.5_2/finetune_sampleeff_0.5_2_best.pth\n",
      "Experiment: finetune_sampleeff_0.5_2\n",
      "Run: 2\n",
      "Fraction: 0.5\n",
      "Val accuracy: 0.7179693206720233 || Val F1: 0.6199298416377507\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.1_0/finetune_sampleeff_0.1_0_best.pth\n",
      "Experiment: finetune_sampleeff_0.1_0\n",
      "Run: 0\n",
      "Fraction: 0.1\n",
      "Val accuracy: 0.5347699050401753 || Val F1: 0.39725468949716\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.25_1/finetune_sampleeff_0.25_1_best.pth\n",
      "Experiment: finetune_sampleeff_0.25_1\n",
      "Run: 1\n",
      "Fraction: 0.25\n",
      "Val accuracy: 0.6963476990504017 || Val F1: 0.587389257459403\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.75_2/finetune_sampleeff_0.75_2_best.pth\n",
      "Experiment: finetune_sampleeff_0.75_2\n",
      "Run: 2\n",
      "Fraction: 0.75\n",
      "Val accuracy: 0.7593133674214755 || Val F1: 0.6569421372168878\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.75_0/finetune_sampleeff_0.75_0_best.pth\n",
      "Experiment: finetune_sampleeff_0.75_0\n",
      "Run: 0\n",
      "Fraction: 0.75\n",
      "Val accuracy: 0.7412710007304602 || Val F1: 0.6552819089363892\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_1/finetune_sampleeff_1.0_1_best.pth\n",
      "Experiment: finetune_sampleeff_1.0_1\n",
      "Run: 1\n",
      "Fraction: 1.0\n",
      "Val accuracy: 0.7594594594594595 || Val F1: 0.6747570582492036\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.5_0/finetune_sampleeff_0.5_0_best.pth\n",
      "Experiment: finetune_sampleeff_0.5_0\n",
      "Run: 0\n",
      "Fraction: 0.5\n",
      "Val accuracy: 0.7255661066471877 || Val F1: 0.6318919780761131\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.1_2/finetune_sampleeff_0.1_2_best.pth\n",
      "Experiment: finetune_sampleeff_0.1_2\n",
      "Run: 2\n",
      "Fraction: 0.1\n",
      "Val accuracy: 0.6794740686632579 || Val F1: 0.5462329679119718\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_1.0_2/finetune_sampleeff_1.0_2_best.pth\n",
      "Experiment: finetune_sampleeff_1.0_2\n",
      "Run: 2\n",
      "Fraction: 1.0\n",
      "Val accuracy: 0.7807158509861213 || Val F1: 0.6989091196832841\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.25_0/finetune_sampleeff_0.25_0_best.pth\n",
      "Experiment: finetune_sampleeff_0.25_0\n",
      "Run: 0\n",
      "Fraction: 0.25\n",
      "Val accuracy: 0.7017531044558072 || Val F1: 0.5841780876335424\n",
      "\n",
      "\n",
      "Loading model: ckpts/finetune-sampleeff-2023-Aug-31-23:37:02/finetune_sampleeff_0.1_1/finetune_sampleeff_0.1_1_best.pth\n",
      "Experiment: finetune_sampleeff_0.1_1\n",
      "Run: 1\n",
      "Fraction: 0.1\n",
      "Val accuracy: 0.683418553688824 || Val F1: 0.5453318438151805\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#### reweight for class balance\n",
    "class_weight = torch.tensor([1/x for x in class_num])  #use this simpler weighting\n",
    "loss_fn = nn.CrossEntropyLoss(weight=class_weight).to(device)\n",
    "POS_EMBED_USING = True\n",
    "\n",
    "# load model\n",
    "sampeff_accs = []\n",
    "sampeff_f1s = []\n",
    "sampeff_experiment = []\n",
    "sampeff_frac = []\n",
    "sampeff_run = []\n",
    "for ckpt_path in sampleeff_model_paths:\n",
    "    exp = ckpt_path.split(\"/\")[2]\n",
    "    sampeff_experiment.append(exp)\n",
    "    sampeff_frac.append(exp.split(\"_\")[-2])\n",
    "    sampeff_run.append(exp.split(\"_\")[-1])\n",
    "    print(\"Loading model: {}\".format(ckpt_path))\n",
    "    print(\"Experiment: {}\".format(exp))\n",
    "    print(\"Run: {}\".format(exp.split(\"_\")[-1]))\n",
    "    print(\"Fraction: {}\".format(exp.split(\"_\")[-2]))\n",
    "    model = PerformerLM(\n",
    "        num_tokens = CLASS,\n",
    "        dim = 200,\n",
    "        depth = 6,\n",
    "        max_seq_len = SEQ_LEN,\n",
    "        heads = 10,\n",
    "        local_attn_heads = 0,\n",
    "        g2v_position_emb = POS_EMBED_USING,\n",
    "        g2v_file = G2V_FILE,\n",
    "        pred_continuous = USE_CONTINUOUS,\n",
    "        sin_emb_wavelength = SIN_EMB_WAVELENGTH,\n",
    "    )\n",
    "    model = model.to(device)\n",
    "\n",
    "    # Load checkpoint\n",
    "    checkpoint = torch.load(ckpt_path, map_location=device)\n",
    "    consume_prefix_in_state_dict_if_present(checkpoint['model_state_dict'], \"module.\")\n",
    "    #print(\"Loaded finetuned ckpt...\")\n",
    "    model.to_out = Identity(dropout=0., h_dim=128, out_dim=label_dict.shape[0])\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "    model = model.to(device)    \n",
    "    cur_epoch = checkpoint['epoch']\n",
    "\n",
    "    # model eval\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    predictions = []\n",
    "    truths = []\n",
    "    with torch.no_grad():\n",
    "        for index, (data_v, labels_v) in enumerate(val_loader):\n",
    "            index += 1\n",
    "            data_v, labels_v = data_v.to(device), labels_v.to(device)\n",
    "            logits = model(data_v)\n",
    "            loss = loss_fn(logits, labels_v)\n",
    "            running_loss += loss.item()\n",
    "            softmax = nn.Softmax(dim=-1)\n",
    "            final_prob = softmax(logits)\n",
    "            final = final_prob.argmax(dim=-1)\n",
    "            #final[np.amax(np.array(final_prob.cpu()), axis=-1) < UNASSIGN_THRES] = -1\n",
    "            predictions.append(final)\n",
    "            truths.append(labels_v)\n",
    "        del data_v, labels_v, logits, final_prob, final\n",
    "        # gather\n",
    "        predictions = torch.cat(predictions, dim=0)\n",
    "        truths = torch.cat(truths, dim=0)\n",
    "        no_drop = predictions != -1\n",
    "        predictions = np.array((predictions[no_drop]).cpu())\n",
    "        truths = np.array((truths[no_drop]).cpu())\n",
    "        cur_acc = accuracy_score(truths, predictions)\n",
    "        f1 = f1_score(truths, predictions, average='macro')\n",
    "        val_loss = running_loss / index\n",
    "        sampeff_accs.append(cur_acc)\n",
    "        sampeff_f1s.append(f1)\n",
    "        print(\"Val accuracy: {} || Val F1: {}\".format(cur_acc, f1))\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_df = pd.DataFrame({'experiment':sampeff_experiment, 'accuracy':sampeff_accs, 'f1':sampeff_f1s,\n",
    "             'frac':sampeff_frac, 'run':sampeff_run})\n",
    "res_df.to_csv(\"sampleeff_metrics.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_df.frac = res_df.frac.astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_lr = pd.read_csv(\"sampleeff_LR_metrics.csv\")\n",
    "res_lr.frac_training_data = res_lr.frac_training_data.astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='frac', ylabel='accuracy'>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA6p0lEQVR4nO3deXjkV3Xg/e+pfdO+tHpVd7vVi3djYfbEBAMGYhuSCTFMEmCSeN4MkAxJeAPZyGsyGbKHJE6C4SVkmAQHMglPD5h4HPbFJpYxtnHb7W733nZLaq1Vqv1XZ/64v5Kq22q3WqpSaTmf59Ej1fKTrqrV99S5y7miqhhjjDHnCzS7AcYYY1YmCxDGGGPmZQHCGGPMvCxAGGOMmZcFCGOMMfMKNbsB9dLd3a3bt29vdjOMMWZVefjhh8+qas98j62ZALF9+3aGhoaa3QxjjFlVROT4hR6zISZjjDHzsgBhjDFmXhYgjDHGzMsChDHGmHlZgDDGGDMvCxDGGGPmZQHCGGPMvNbMPghjjFk3illIPwfTz7rPoRhcfmvdf4wFCGOMWSkqHsyM+p3/c5B+1n2ePj0XDNJnoDB97nXdu2HfLSBS1+ZYgDDGmOVQyJz7rn/qtOv4q0EgcwYyo6DeuddJAOIdEO+EZBd07YJkNyS6INkLrZuhd1/dgwNYgDDGmKWpeJAZcR3/1Cn3kT491+mnz0BmGIozz782HHcdf7zDZQHbXgaJHkh1Q6oPOnZARz9EUhAMQzDiPgLLM31sAcIYYy4kNznX6Z8zzOMP9WRGIDcOWjn3OglCrA3i7a7z79rlv+PvgVQvtPdD2zaXCYRjEAjNdf7BldMtr5yWGGPMcqh4UMq5zn76VM0Yv9/xZ4ZhZgRmzkI5//zrwwl/yKfdDe0kuuaGfFo2uiGfts3uXX847t75B6rv/sMNGQpqlIYGCBG5GfgoEAQ+oaofOe/xbcDfAe3+cz6gqvf6j30Q+FnAA35RVe9rZFuNMauUquv0KyXwSpAdn3u3P3WyZnx/xE0A58YhP/X8d/2B4NxwT+tm6LvadfrxTkh0uvH+ts3u63ASIom5Tj8YcUFgmYZ+lkvDAoSIBIG7gNcCp4CHRGS/qh6oedpvAp9V1b8WkcuBe4Ht/te3A1cAm4B/E5HdqufP3hhj1iRV19lXyq7jr3huaef0ab/TPz03tp8ZhdyYCwzZcfAKz/9+0Za5zr5ju+vkq8Eg0ekygNQG/11/AiJJCEXPG/cPLvvL0GyNzCBuAA6r6hEAEbkHuA2oDRAKtPpftwHP+l/fBtyjqgXgqIgc9r/fAw1srzGm0SoVN2zjFf3OvwzFnHtnXx3uST/nbmfHIDfh3vFn/Xf96LnfLxB2K3sS/uqeLf7QT7yrJgi0u/mAcNx1/OHE3NDPbOcfWlVDP8ulkQFiM3Cy5vYp4CXnPed3gP8jIu8FksBNNdc+eN61mxvTTGNM3XklFwjKBbd6Z/oUDD8J44dh6lm/8/c7/ty4Cxjni7VBott9dO927/Zj7f67/i73Odri3umH4/7H+hj6WS7NnqR+G/ApVf1jEXkZ8GkRuXKhF4vIHcAdANu2bWtQE40x81J1HXsp5wJBfhomjsLoUzBxzF/nf9J9zk/OXRcMu9U8iW7o3Ts31BNvh1i182/3J3SDfsefmMsAbOhn2TQyQJwGttbc3uLfV+tngZsBVPUBEYkB3Qu8FlW9G7gbYHBwUM9/3BhTB9VhoepHdhzOHoKzT7v5gKnTLkOYOgWl7Nx1kaQb79/2MmjdBK0bIbXRBYdAABBXIiKS8Mf9E+52becfDDfrtzY0NkA8BAyIyA5c53478PbznnMCeA3wKRHZB8SAUWA/8A8i8ie4SeoB4N8b2FZjzPnDQtkxGD0IY4eevxegdkgo3uk2cw28Ftq2Qkuf2+QVa3Pj+hKCWAqi7RBvc0EgFHUBwMb9V7SGBQhVLYvIe4D7cEtYP6mqT4jIncCQqu4HfgX4uIi8Dzf79E5VVeAJEfksbkK7DLzbVjAZUweqLgBUA0Eh7VYCjTzphoWqq4SmTrulobNLQcV1/O39sPUGPxBscpu+wglmJ48jSYj6G8Sq8wKhmAWCVUpcf7z6DQ4O6tDQULObYczKUPHmgkAp54q7TZ6Eswfd5+nTc1lB9uzcdRKEti0uI2jvh/Ztbmgo2etqAiGAunf/sTY3aRxNuSAQjtt8wCokIg+r6uB8jzV7ktoYsxReaW6SuDjjSkNMHPNXC52amx+YPu0vE/WFYq7z33Stmydo73eBINHlsgxVQF3AiKZcIIi1ubIQoTiEIs34bc0yswBhzEp3/rBQfsrtDxg7DJPHXRCobh6bPu0CRlW0xXX+21/pZwT9LkOItfkb0KpDSDo3PBRr8yeObXhovbMAYcxKUTssVMxCMQ2Zs25YaHaC+NRcmehKee7aRDd0bHPlITr8oaG2ba6j94ou06gdHoq22PDQKlepKDPFMulcmVzZY2d3ErHzIIxZ5crFc1cL5acgPQzjh+Y6/2pAyAyfO1HcutFlAdte5gJBNRgEo3M7lKvzihJwK4haum14aA2oVJRsySOTL3E2U2RipoinWg377OhK1j3ZswBhTCOo1uwd8DeR5adcxz953A8Ap9yu4ulTbklpVSDkVgn17IGB17kA0NHv7guEXK2hcnEugyjlQcUtN421+eUkbHhotVNVZooeM35AGJ8p4lUUBOLhIK3xMAH/33dsZp76U3VgAcKYpaiWji4X3Cax/LTbNTxxvGY4qOZzIT13bTjuOv/N19esGvIniyXosoFywc0VKO7aQMgFgdRGN0xUzQpW0BkCZnFUlWzRZQhjM0XGZopUKoriAkJLLEwwsLwB3/6qjFmIc4aFMnMTxRPHa4aETs2VmC7XvKOLtbmOf8eNbp6gumoo2ePe4XslPysouECQm3TDQ5GUqzBaLTQXjrsNZmZNUFVyJY9MvszZmQLjmSLlihsejIWCtESXPyCczwKEMedTdXMDhbTbIzB1GiZPzNUWmn52bkdx7URxssd1/huvcQGgmhXE293jFe/c4aHsOKAuA4i3u0njSNLfaRyzAnNrzGxAKJQZyxQZyxQpexUQiIaCpFZAQDifBQhjagNCZsTtKh5+3H0+e9AFBmomfls3PX/paLu/Yqj6/arDQ14RZsbc9TY8tK6oKvlSxQ8IBcZmXEBQqgEhtOICwvnsL9OsP5UKlGagkHGHzjz7fRj+AYw+6SqRVieMIynouxJ23TSXEbRtcctEZ79X2R96yvn7D9SGh9axXNFlCBPZIqPpAiXPrUCLBoMkIys/IJzPAoRZ+2YDQtplA6ce8gPCU+6jOOOel+x2w0N9V8PGq91wkfjDPBXPL23tT0SDSyrCcRcE2tr8swlseGg9yftDRuMzRcYyBYplRVGiwSCJcJBQrLHVaCdmihwayTCZK/JDAz11//4WIMzaUxsQxo/BiQdh5Acw8pSrTFqtRNq2FXbe6AJC39WuGJ3I3BLV/JS/01j9iqQtriZRrHVuc5mVo15X8iWPGT8gnM0UKJTcHEIkGCAeDpKKNu6NwXSuxOGRDIdGMxweSXNoOMPYjPtb3tIR53037a77z7QAYVa/SsWtLCqk3RkFx78DIwf8CqVH3UYzCbgjKffd6rKDvqvcITUwV88oOwaICxKxNletNNbqqpWGoranYB2qBoSJbJGz6SKFsucSx0CAeMRNLDfCTKHMM6MZDo9keHrEBYTh6bmVcZvaYly5uY1dvSkGelO0JxrTDgsQZvWpBoTcFAw/BscfcAFh9Em3sgjcPEHv5XDtf3QBofcKN4msFbexrJybmzwOxd18Qbxj7sxiGyJalwplj5mCx/hMgbOZIvmSO2WgGhCS0fp3mfmSx5GzMxwaTrsMYSTD6cm5elq9LVEGelPcfMVGBjakuKwnReq8dthGObN+Vby5SqWn/t0fMnrCDRnlxt1zIimXFey9xQWE7t1u+Kda5K6UdR8SgkQHtG/1VxIlrfTEOlYNCBP+HMJM0UMEQoEAiYibWK6nYrnCsbEZDo1kZgPCyYks/vYHOpMRBnpTvHpPD7t6W9jVm6It3rxhTAsQZuWpeC5DmBmD49+Ck991B96fPW9CedO1504oq380ZskvbYG6wNGyyc8O/KMtbaho3SqWK8wUykzmipxN+wEBCPoBoTtVvy6x7FU4MZ51wcAfJjo+lp3dDNcaCzGwoYWXXtbF7l6XGXSlVtZKNwsQpvmqAWH6OTjyNbfKaOSAK2d9oQnl1AaoFN1wUcVzm86CYVePqLPbHypK2h6Dda7k+QEhW+JspkC6UEZwGUI8HKQrWZ+/D6+inJrIcngkMztMdORshpLngkEyEmRXb4o3X7t5dt6gpyVa9+qr9Wb/e8zyqwaE8aPwzFfh9JALCBPHLjyhHG2Zq3mEQH4Coq1uf0L1/AIrTrfuVQPCVLbEaKZApuB2ugdFiEeCdCeX/g69osqZqTxP+0NEh0czPDOaIV9yex5i4QCX9aR401Ub2dXbwkBvio1tsRUfDOZjAcI0nld2K4xGDrgM4fTDboVR+rwJ5et+ygWDnn1ukriUd0tOK577Ot4FiU6XHUSSdn6BoeRVyBY8pnJFRtNFpgslAkDADwhdSwwIqspIujCbFRwaSfPMSIaZopu8jgQD7OhOctO+DQz0ptjV28Lm9viq2xB3IRYgTP15ZXcG8qmH4djX/YAwz4Tyvlvm5g+qG9HALTsNt0HrFpc5RJK2C9kAblx/puAxnS8xki6QzpcAPyCEg3QlIkt6pz6WKXB4NMOh4bl5g+m8y0JCAWF7V5JXDfSwqzfF7g0ptnYkCAXX7oo3CxBm6bwy5Mbg+INw7Btw+pELTyhvuMJtSPNKNeceByDV7bKDcMKWmZpZZa/CTNFjOufmEKbzrvS5+AGhcwkBYSpX4tBIem7eYDjDeNa9SQkIbOtMcMOOTgb81UQ7upOE13AwmI8FCHPpvDKkn4Nj34Rj33K1jObbobzhKujd5zab1e5IjrZComsuO7DdyMbnVZRMoUw67wLCVK6EqptDiIWDdMQXFxAyhTLP1AwTHR7JMJJ2ewcE2NwR5+qtbQz0phjobWFHd5JY2IYwLUCYi/NKbgL5yNddQDjz6HkTygNw+W0uO+gecMdfAq6Occqdlxxv97ODuE0km1ne7LnKLiBMZssoSkCEWGhxASFX9GZ3IVeHiZ6dys8+3tcaY/eGFt501UYGelNc1psiUef9DmuFvSrm+cpFN6F89Btw/Nvw3GPPn1C+9j9Cz17o3OEHBHWPJTpddmDLTM08Sl7FnYmQKzOaKTCZLZ0bEBLhSwoIhbLH0bMzs0NEh0YznBrPVouz051yu5Bfs2/D7PLSlgYX0FsuFVWK5QplTy/+5EWy/73GLR09NeQCwokH4Mzj8+xQfiN073HDR8HQXL2ieDfE2/zsINbc38OsKGWvQrbkkSuUmcq7AnfV0hUCxMJB2hNz5ypfTMmrcHwse868wfHxrDunGWiPh9nVm+JVu7rZ1ZtiV0+KjuTq3yVf8ir+h1KuVGbvDwaERCREVyJMWyxEoAErpyxArEeFjAsER78JJx+A4SfmJpQTfsnr3n1uL0LbVhcMQnFIdrmNaJGEyw5sItn4yn5mkCt5TGZLTM4UyfrBANx5CNFwYMGlK7yKcnI8O1u99NBwmqNnZ2Z3IaeiIQZ6U/zYdZvdvMGGFrqSS1vB1EzVbKDkVShXFFX3eyoukKaiIVqiIRLRENFwgGgoQCQYaPjvawFiPZgZd8tNj37L1TIafercCeXtPwQ9e9z8QbLXnXwWb6+ZSE5ZvSIzy6u4ozOzRbchbTJbnN0XAG5vQCwcpGuBwaCiyrOTuZq9BhmOjGYolN275XjY7UK+5ZpNs5PIG1pX/i7k+VSzgWK5gqdzQ0PBgJCMhmhPRElFQ8TCQT8QBJu6p6KhAUJEbgY+CgSBT6jqR857/E+BV/s3E0Cvqrb7j3nA4/5jJ1T11ka2dU2ZOu02pB37pitbMX6kZkL5MtjzBjdc1LMHom0QTfoTyVavyJyrGgxyRbcZbTJbmt2dLEA46DqxhZasUFWG04XZQnXVncjZ6sazkNuF/Por+mbnDDa1xxc8DLUS1GYD1RPlYC4baImGaGkLEY8sbzawGA0LECISBO4CXgucAh4Skf2qeqD6HFV9X83z3wtcV/Mtcqp6baPat+bkp+CBu+D7/wBTJ919wYgLBFf+Bzeh3D3gMoK4P5EcTdkyUzPLqyj5kkfWDwZT2TLpQmn28XDAZQaXsvcgWyxzaCTDwTNp9zGcZirnvmcoIOzoTvLDu3vY7e812NqZWDW7kBeSDbTEQkRDKyMbWIxGZhA3AIdV9QiAiNwD3AYcuMDz3wZ8qIHtWZvSw/DtP4Pv/Q9X32jDFbDzp125is4dLhAkuyHWbvWKzKxKNTMouU1oEzPVzEBnD8SJhgOXFAwqqpyeyHHwTJqnhtMcPDPNifG5Utab2+Nc39/B3r4WBnpb6O9KrPiNZ/NlA6qAnJsNJKIhIqGVnQ0sRiMDxGbgZM3tU8BL5nuiiPQDO4Cv1NwdE5EhoAx8RFU/P891dwB3AGzbtq0+rV4txo/CN/4IHv+cm0/Y+hK3F2Hz9dC+zeoVmVmVipIvV4eJSkzmSmTyZSr+u95QwHVsl7rENJ0v8fRwhoNnpjk47LKDmYIbKkpGguzpa+FlO7vY09fK7g0re3npfNmAKoSC52YDsXDQDwSrLxtYjJUySX078E+q6tXc16+qp0VkJ/AVEXlcVZ+pvUhV7wbuBhgcHGzcYuCV5LnH4Ru/D0990WUCO34Y9t4KG6+Czp1uHmGNvHsxl051bs4gnXdHZabzZVRdZlANBu3xSwsGXkU5MT7DU2fSPOUPF1VPPauWpXjlrh72bmhhT18LmztW3ryBV9GaJaMXnhtIRENE/SAQCa3sDKfRGhkgTgNba25v8e+bz+3Au2vvUNXT/ucjIvI13PzEM8+/dJ049m34+h/A0a+5jWl73gh7f3Rus1q8vdktNMtMVcmX3PLSdK7ERK5IOucyg9pg0BZf+F6DqolskaeH0zz1nMsMDo2kZ8tZt8XD7NnQwo/s7WVvn5s7WEk7kRc6NxALB2eDwHrIBhajkf+qDwEDIrIDFxhuB95+/pNEZC/QATxQc18HkFXVgoh0A68A/qCBbV2ZVOHgvfDNP3YVUSMpuOqtsPtmtxqpc4fbrGbWPFWlUK6QLXpk8iUmsiWmc6XZYFCtVdS6iGBQ8iocPTszmxkcHJ5meNrVKQoGhJ1+Oes9G1rY29e6IpaYXigbACEaDlg2UCcNCxCqWhaR9wD34Za5flJVnxCRO4EhVd3vP/V24B5VrR0i2gd8TEQqQAA3B3Ghye21p+LBY5+Fb/0pnD3oyldc/y7Y+Wo3jNS53a1GMmtSNRjkih6ZgtuBPJ0vUam4KeSgCNHQ4oKBqnI2U3RzBmemOXgmzeHRuZPPulMR9vh1ivb0tXJZT5JoqDnzWKpK2Q8E52cDoYCQsGyg4eTcfnn1Ghwc1KGhoWY3Y2nKBRj6JDzwlzB1Clo3weVvgf5XuMDQvtUtTTVrxvnBYDJXYipbpOwpIq6sdWwJHV+h7HHYX2b6lL/MdHzGbZKMBANc1pvyMwM3d9DdhDORXygbiIcDJKMhWmKWDTSKiDysqoPzPbZyBg7Xs/w0PPjX8O8fg+wYdF4Gr/pV2PJid5hO+za3RNWsevmSR77kMVMoM5EtMZkt4fn1dUSEaChAKhpeVDBQVc5M5+eGis6kOTo2M1urqK81xlWb21ww2NDC9mU+36C6ZLRQrpxTUygUEFKxEB3Jml3EoSDRUKAh9YXMwlmAaKbMCHzrz+B7f+f2MPRdDS97tztHoWMHtG9x5bHNqlQoe+SLFTIFFwgmcyVKXgUBxB8rT0VDix4SyRbLHBrOzO45OHhm7vSzeDjIwAZXq2hvXwu7N7TQnli+cilepRoMvNmhoaAILfEQPS1xUjHLBlYDCxDNMH4MvvlH8Phn3bDStpfB5be6Xc8d26F1s1VGXWWK/jDRTKHMZL7IxExNMPAzg2Rk8cGgosqpidxsIHjqTJoTNWWtt3bEuWFHJ3s2tLKnr4Vty7gj2asohbJHoVzxi8wJgQC0xsL0tUVJxcLEw0Fi4bWzgWy9sACxnM78wC1VfeoL7vZlr4a9t7i5hc7LoHWjnb28ChTLbmlptlBmIldkKluaLSwXECESDJAIBwktYWNYOl/yJ5JdMHh6OD1brygZDbJnQyuv2NXNHj87SEWX57/ybDAoVfw9126lU1siwqb2MKloiHjEDQ9ZMFj9LEAsh+PfcYHhyNdc3aO9P+qWqrZudJPPLRutWuoKVT3xLFsou2GibIlc2aM6KBINBYmFgqSiiw8GXkU5Pla7zPTcTWj9Xa5e0R5/E9pyFa8re26+oOhVZjfahYNuk92WjjDJ2fkCCwZrlQWIRlGFg1/y9zAMubIX174NLrvJ1Ufq2gUtG6xQ3gqU9+sTjaYLjM0UqagiQMSfOE0u8d36xEzRnzdwcweHRuZKW7fHw+zpa+E1+3rZu6GFXb0txCONX2ZaqgaDsofi/nyjoQDtyTDtsYgLBpFA05a8muawAFFvlQo89o+ugN7oU670xYt/zp25EO9wG9xSG+wozhXEqyiZfJnxbIGR6QK5oueKsYWCi9qFXKvkVTgyOsPB4bm5g5G024QWCgg7e5K87vIN7OlrZW9fC70tjd+EVg0GhbIbslIgFgrQlgjTEU+QiLhhIps8NtZL1Uu5AA9/Cr7zF67cdstGeMUvwdaXuk1t3QOQ7LHieStErugxnSsymikyPlNE1Z2LnIyGSKQW999CVRnNFOb2HJxJ88xoZvYUtO5UlL19LdxyzSb2bmhhZ0+q4Z1wyatQKFUoeHNlzmJhd/ZzeyJBPBIiHrZgYOZnAWKp8mn47t/Av/8NzJx1cwo3ftAd2xltdUNJyR47nrPJyl5ldlfySLowezZyPLz4LCFf8jehDc/tOxjP+pvQQgEGelPces0m9vj7DroauAlNVSl5bgK5WLPZLBEO0tUSoS0WIh51wWCll9g2K4cFiMWaOevvYfgUFNLQdxW8/Behey/EWqB7tzuYxwJDU1SrmqZzJYbTBSayRVe+ORAgEQku+Gzk2u/33FR+djfywTPTHD07M3vWwca2GFdvbfOrmbayvStBqEEdsapS9DODUs2Gs1Q0RE9LlLaEW1YaDwcb1gazPliAuFQTx90ehsc+C+U8bHupO7Gtvd8Vzuva5Won2aqOZVfNEsYyRUbSeYrlCorLEjril3agvapyfCzL0PEJnnh2ioPDadI1m9B2b0jxE9dvnV1m2hZvzGKDaimO6u5j9xsIqWiIDW1R2uIR4hEXDKwGkak3CxALNXzA38Pwv935zpf9CFzxZkj1zQUGO4thWakq2aJHOl9ieDrPZK50TpZwqUtPc0WPR09NMnR8goePj3M244aLtnYmeOnOrtmaRVs6GrMJrbYURcmruFpMuDIUG9tis5lBzIKBWSYWIC7m+APwjT+EZ77i72F4E+y7BaJt7ijPzh0uMJhlUfIqZPJlxmYKjKYLFEoVkMVnCacnc35AmOAHp6coV5R4OMh129p5+w0dvGhbR0PmDp5Xl0gh4Nck2pSK0RqfGyayekSmWSxAzEcVDt3njvQ89RCEE3DN29zmtnAcUr12FsMyqWYJU9kSI5k8k9nqgfcBkovIEgpljx+cnmbo+DgPH5/guak84LKEW67ZxIv7O9i7sbWuE7nn1yUS3I7ral2ilphbVhoLWTAwK4sFiFqVijvj+dt/BiMHINYON9wBO2902UOqz9VKirU2t51rXLFcnUsoMJIuzJaAToRDdCYuLUsAGJ7O8/DxCR46Ns5jp6colitEQgGu2dLGm6/dzPX9HWxorU/tK6tLZNYSCxAA5aLbw/DAX8DkCRcIXvFfof/lIAF3LkN7v53F0CCqykzRYyrrlqBO5UoIc3MJl1rTqORVePK5aYaOTzB0fIKT41nAlbt+3eUbGOzv5KrNbUte+291icxaZwFi/Ch86k0wfdqV2H71b8DG69xj7VvdRyTZ3DauQcVyhXS+xNhMkZHpwuyZCInI4rKEsUyB752Y4KFjE3z/5CS5kkcoIFy5uY3X+0FhU3ts0R11bV2iil++Ohx05z1v7gjXnGNgwcCsHRYg2re5g3l6ftp9Rl220L7VzmKoI1UlUygzlSsxPF1gOu+yhEhwcWcieBXl6eG0nyWMc2R0BnBHZv7Q7h4G+zu4Zkv7kuoY5Yoe2ZJb2lotUteecEXqXGZgu+LN2mYBIhCEWz4Kzz7igoWdxVA3hbJHJl/mbMatOCp7SiAgxMNBupOXvjJoKlfiET9LeOTEBOlCmYDAvo2tvONl2xns76C/K7Gkd/D5ksdMsYwqtCfC7OtqoS0RIRa2YGDWHwsQ4FYjbX+lVVZdoopfGnsyW2JkukC64FYcuSzh0o/RrKhyZHSGoePjDB2b4OnhNIqrePriHZ0M9ndw3dYOUrGl/RnnSx7ZYhkFWuNh9vVZUDAGLEA4IhYcFqlQ9kjny4ymC5zNFPAqSlCEWDhI1yKyhJlCme+fnJxdhjqRdUNRAxtSvO2GbQz2d3BZb2rJ5yFUMwUUUvEQuze00JG0oGBMLQsQ5pJUKkqmWGbSL3qXzpcRgWgwSMsisgRV5cS4K2kxdGycJ8+k8SpKMhrk+m0dXN/fyYu2tdflPOVC2R0JqgqpWIjdvS4oLMd5C8asRhYgzEXlS36WkMkzli7iqcsS4pEg3YvYZZwveTzml7QYOj7BqH8+wo7uJD92nduXsLevtS7lJNyeihIKJCNBdvWm6ExGLSgYswAWIMzzeBV/xVG2yPB0npmih6o7R6AldulZAsCzk7nZuYTHa0paXLu1nZ8c3Mpgf/1KWpT8on2VihKPBNnZk6IrFSFxiRVcjVnv7H+MAWqyhHSesxl3zGZAhGQkRFfy0v9MiuUKP3h2anYHc7WkxZaOOD969UYGt3dyeR1LWpS8CjOFMp4qsXCQHd1JOpORJR8Pasx6Zv971qnqMZsT2SIj03lyJc8/enLxB+iMpF1Ji6FjEzx6apJCuUIkGODqLW3cds0mru/vpK+tfkuIS16FmWKZslchHg7R35WgMxUlGQnaZjVj6sACxDqS80tjj6QLjGWKKG4uIREJ0bmILKHsVXjyTJqH/aGj435Ji96WKDft28BgfwdXbm6r68qg6pkP5YoSCwfY2pGgKxUhFQ1ZUDCmzhoaIETkZuCjQBD4hKp+5LzH/xR4tX8zAfSqarv/2DuA3/Qf+11V/btGtnUtqmYJ49kCw9MF8kUPxGUJ7YnFZQkTM0WXJRwf55GTk2SLrqTFFZtauWnfDq7f3sGW9nhdO+tqUPBUCQcDbOmM052KWlAwpsEaFiBEJAjcBbwWOAU8JCL7VfVA9Tmq+r6a578XuM7/uhP4EDAIKPCwf+1Eo9q7FtQeszmSKTIx4+YSgiIkoyGSqUv/5/YqyqERV9Li4WMTHB7NANCZjPDKXd2upMXW9rpPAFcnysuVCqFAgE3tcXpao7RYUDBm2Szof7WI/DPw/wNfUtXKxZ7vuwE4rKpH/O9xD3AbcOACz38bLigAvB64X1XH/WvvB24GPrPAn71u5EsemUKZ8ZmiX85i7pjNxc4lTOdKfO+EO0Tn4RMTpPOupMWevlZ+5qX9DG7vYHtXsu4d9blBQdjYFqe7xQUFOyfBmOW30Ld9fwW8C/hzEfkc8LeqevAi12wGTtbcPgW8ZL4nikg/sAP4ygtcu3me6+4A7gDYtm3bxX+LNaBa32giW5w9Ua06uZwIX3ppbHCZx5GzM36WMM7B4TQVhdZYiMH+Dgb7O7luWzsti/jeF+NVlJlCmZIfFPraYvSkYrTELCgY02wLChCq+m/Av4lIG+6d/r+JyEng48D/VNXSEttxO/BPqupdykWqejdwN8Dg4KAusQ0rUrHslm9OZIucTReYKXqIQDgQIL6IE9WqssUyj5yYdFnC8QnGs+785V29KbcvYXsnl/WkGnL2sVdRssUyBa9COCD0tsbobYnSGgtbUDBmBVnwwLGIdAE/Bfw08Ajw98ArgXcAN85zyWlga83tLf5987kdePd519Z+zy3A1xba1tWs7FWYKXhMZouMZgqkC+XZw3Pi4SDdi5hHAJclnJzIMXTM1Th64rlpV9IiEuS6bR0M9nfwov4OOupQ0mI+FXWZQtGrEAwIvS1RNrTGFr3xzhjTeAudg/gXYA/waeAWVX3Of+gfRWToApc9BAyIyA5ch3878PZ5vvdeoAN4oObu+4DfE5EO//brgA8upK2rTXXcfTpXYjRTYDrnkrHZUhaLKHhX5UpaTM0WvhvxS1ps70rw5ms3M9jfwd6+FkJ1PH+5VkWVbMGj4HkEROhpidLXGqM1bkHBmNVgoW9H/1xVvzrfA6o6eIH7yyLyHlxnHwQ+qapPiMidwJCq7vefejtwj6pqzbXjIvJhXJABuLM6Yb3aVUtip3NlRjMFJrOl2f0IsXBwUaep1XpuKsfQMVfj6PHTk5Q8t1/gmi3t/MT1W7m+v4OelvqUtJhPRZVs0aNQdkNhvS0xNrS20BoLNSwQGWMaQ2r65Qs/SeTdwN+r6qR/uwN4m6r+VWObt3CDg4M6NHShZKZ51O8w0/kSZzNFxv2lpwLEw6G6HV7vVZRPfecon//+swBsbo9zfX/H7Ga1epW0mE/1d8z7QaEn5YaP2uJhCwrGrHAi8vCF3ugvNIP4eVW9q3pDVSdE5Odxq5tMjepehEy+zNkZt2O5UtElLz19IdlimT+87yBDxyd441Ubue2aTWxqb+xxqdXfM1fyEKArFWVXW4q2eLihwcgYs3wWGiCCIiLVYSB/E1xjZjNXoepehLFMgbOZIqVyBQSiocWdkXApzkzn+fAXDnB6Msd/ufEy3nDlxob9rGpQyPorqTqTEXb2uKAQCVlQMGatWWiA+FfchPTH/Nv/2b9vXWrEXoTFeOLZKX7v3iepKPx/t17BNVva6/4zajMFBToTYXZ0J2lPRCwoGLPGLTRA/BouKPyCf/t+4BMNadEK1Ki9CEvx5SeH+cuvHmZDa4zfetPlbO6o75BSruiRLbnT1zqTLii0JcJEQ3bQjjHrxUI3ylWAv/Y/1rwX2ouQiCx+L0I9eBXl0w8e43997zTXbGnjAzfvIxWrT3vyJY9ssYwCbfEw+7paaEvYOc3GrFcL3QcxAPx34HJgtqC/qu5sULuWlaoynT93L4IAgTrsRainXNHjj+8/yHePjvOGK/u441U7l7xKKF/ymCmWAWiJhdnT10K7BQVjDAsfYvpbXCG9annudwFrZgB6Ol/m4ePjhAOBuuxFaISRtJuMPjGe5T//0E5+9OpNi/5ehbKbVEchFQuxu7eFzpQFBWPMuRYaIOKq+mV/JdNx4HdE5GHgtxvYtuWjbudye4PKTCzVU89N89/ufZKSV+FDt1zBi7Z1XPyi8xTKHjMFN3yUjATZ3dtCRzJCPGJBwRgzv4UGiIKIBIBD/u7o00Cqcc0yVV87OMKff+UQ3akov/emq9jamVjwtcWyO5KzUlGS0SCX9aboTEbqfnaDMWZtWmhP8Uu4E99+EfgwbpjpHY1qlHElK/7+uyf47NBJrtzUygffsI/W+MVXS9UeyZmIBNnRnaQzGSEZtaBgjLk0F+01/E1xP6mqvwpkcPMPpoHyJY8/uf9pHjgyxusu38D/88OXLXh38kSuxM7uBD0tMRKR4IqbSzHGrB4XDRCq6onIK5ejMQbOZgr87hcPcPTsDD/7yh3cds2mBXfy2WKZzkSY7d02+meMWbqFjjs8IiL7gc8BM9U7VfWfG9Kqderp4TT/7YtPkit5/OabLufF2zsv6fqZosfevtYGtc4Ys94sNEDEgDHgR2ruU8ACRJ1889Aof/Zvh+hIhrnztqvp70pe0vXZYpmORJjWuM01GGPqY6E7qW3eoUFUlXseOsk//PsJLt/Yyq+/cR9tC5iMPl+26LGnr8XmHIwxdbPQndR/i8sYzqGq/6nuLVpHCmWPj375EN88dJbX7O3l3a/etahS2dlimbZEeFGBxRhjLmSh4xFfqPk6BrwFeLb+zVk/xmeK/O4XD3B4JMO7Xr6dt1y3edHv/rNFj+ssezDG1NlCh5j+V+1tEfkM8K2GtGgdeGY0w4e/cICZYpnfeNM+XrKja9Hfy7IHY0yjLHZGcwDorWdD1ovvPHOWP7n/aVrjYf7gx69hR/elTUafL1ssc11fh2UPxpi6W+gcRJpz5yDO4M6IMAukqnzu4VN8+sHj7O1r4dffuI+OJdZ+yhbLtMQtezDGNMZCh5haGt2QtaxYrvAXXznE154e5cbdPbz3RwbqchpbrlTmmg2WPRhjGmNBvZSIvEVE2mput4vImxvWqjVkIlvk1//lcb729Cg//dJ+fvm1u+sSHLLFMqlYmPaEZQ/GmMZYaE/1IVWdqt5Q1Unc+RDmBRw9m+GXP/sox8Zm+OAb9vLWwa11e7efK5W5rDtl2YMxpmEWOkk9XyCxLbsv4LtHx/ij/3OQZCTER37sanb11q8+Uq7oWfZgjGm4hXbyQyLyJ8Bd/u13Aw83pkmrm6ryz4+c5u++c4xdvSl+802X05ms70FEM8US12y1uQdjTGMtNEC8F/gt4B9xq5nuxwUJU6PkVbjrq4f58lMjvGqgm196zQDRUH1PbMsVPVpiYTosezDGNNhCVzHNAB+41G8uIjcDHwWCwCdU9SPzPOetwO/gAs+jqvp2/34PeNx/2glVvfVSf/5ymsqV+L17n+TAc9O8/YZt3P7i+s031Joplrlma7tlD8aYhlvoPoj7gZ/wJ6cRkQ7gHlV9/QtcE8QNSb0WOAU8JCL7VfVAzXMGgA8Cr1DVCRGp3XyXU9VrL/H3aYrjYzPc+YUDTGZL/L+v38OrBnoa8nNyRY+WaMiyB2PMsljoKqbuanAAUNUJLr6T+gbgsKoeUdUicA9w23nP+XngLv/7oaojC2zPijF0bJz3/9NjlD3lv//YVQ0LDuCyh529tnLJGLM8FhogKiKyrXpDRLYzT3XX82wGTtbcPuXfV2s3sFtEvi0iD/pDUlUxERny73/zAtu5bFSVz3//NB/+4gE2tcf447dew+4NjdtPmC9Z9mCMWV4LnaT+DeBbIvJ1QIBXAXfU6ecPADcCW4BviMhVfrbSr6qnRWQn8BUReVxVn6m9WETuqLZj27ZtLJeSV+FjX3+G+w4M8/LLunjfTbuJhes7GX2+TKHM1VvaLHswxiybBWUQqvqvwCBwEPgM8CtA7iKXnQa21tze4t9X6xSwX1VLqnoUeBoXMFDV0/7nI8DXgOvmadfdqjqoqoM9PY0b2qk1nSvxof1PcN+BYX5ycCu/dvPehgeHfMkjFQ3WfbmsMca8kIWW2vg54Mu4wPCrwKdxK49eyEPAgIjsEJEIcDuw/7znfB6XPSAi3bghpyMi0iEi0Zr7XwEcoMlOTmT51X96lKfOTPMrr93NT720n8AyvKPPFMvs7LG5B2PM8lroHMQvAS8Gjqvqq3Hv5idf6AJVLQPvAe4DngQ+q6pPiMidIlJdsnofMCYiB4CvAu9X1TFgH25z3qP+/R+pXf3UDN87McH7P/couZLH773lKm7cszzVzvMlj1TEsgdjzPJb6BxEXlXzIoKIRFX1KRHZc7GLVPVe4N7z7vvtmq8V+GX/o/Y53wGuWmDbGu4Ljz3Lx795hP6uJL/5pn30tsSW7WdnimWu3mxzD8aY5bfQAHFKRNpxQ0L3i8gEcLxRjVopyl6Fj3/rKPc+/hwv2dHJr7x2D/FIY+cbauVLHslIcMnnRhhjzGIsdCf1W/wvf0dEvgq0Af/asFatAJl8md+/7ym+f3KSH3/RFn7mZcsz31ArXShz1eZWAgHLHowxy++SK7Kq6tcb0ZCV5NnJHHd+4QDD03n+62sGeM2+Dcvehmr20JWMLvvPNsYYsJLdz/PoqUk+8qWnCAj87puv5IpNbRe/qAEyhTJXWvZgjGkiCxA1vvSD5/ibrz/Dlo4Ev/Wjl9PXunyT0bXyJY+EZQ/GmCazAAGUKxX+54PHuf/JEQb7O3j/6/eQiDTvpbHswRizEqz7ADGdL/HezzzCd54Z483XbuKdL99BsIkdc6Fs2YMxZmVY9wEiX/Q4PpblXa/Yzo9dt6XZzSGdL3PFJssejDHNt+4DRG9rjH/5Ly/nyeemm90UCmWPeCRId8qyB2NM8y201MaaVu9jQRcrXSixsztp2YMxZkWwALFCFMoe8XDIsgdjzIphAWKFsOzBGLPSWIBYAYrlCvGQZQ/GmJXFAsQKkC6U2Nlj2YMxZmWxANFkxXKFaChg2YMxZsWxANFk6UKJy3pSlj0YY1YcCxBNVM0euix7MMasQBYgmmg671YuNbO0hzHGXIgFiCYplitEwwG6l/H4UmOMuRQWIJokXShxmWUPxpgVzAJEE5S8CpGQZQ/GmJXNAkQTTOdL7Oyy7MEYs7JZgFhm1eyhp0mn1RljzEJZgFhm0/kSOyx7MMasAhYgllHJqxAOBui17MEYswpYgFhGtu/BGLOaWIBYJtXsoafFdk0bY1aHhgYIEblZRA6KyGER+cAFnvNWETkgIk+IyD/U3P8OETnkf7yjke1cDlM5lz2EghaTjTGrQ8POpBaRIHAX8FrgFPCQiOxX1QM1zxkAPgi8QlUnRKTXv78T+BAwCCjwsH/tRKPa20izK5csezDGrCKNfDt7A3BYVY+oahG4B7jtvOf8PHBXteNX1RH//tcD96vquP/Y/cDNDWxrQ03l3Molyx6MMatJI3uszcDJmtun/Ptq7QZ2i8i3ReRBEbn5Eq5FRO4QkSERGRodHa1j0+un5FUIh4TeVssejDGrS7Pf0oaAAeBG4G3Ax0WkfaEXq+rdqjqoqoM9PT2NaeESuV3TKcsejDGrTiN7rdPA1prbW/z7ap0C9qtqSVWPAk/jAsZCrl3xSl6FUMCyB2PM6tTIAPEQMCAiO0QkAtwO7D/vOZ/HZQ+ISDduyOkIcB/wOhHpEJEO4HX+fauK2/dg2YMxZnVq2ComVS2LyHtwHXsQ+KSqPiEidwJDqrqfuUBwAPCA96vqGICIfBgXZADuVNXxRrW1ESx7MMasdqKqzW5DXQwODurQ0NCirp3Klvj+yQk6k/XrzMdmCuzqTbGlI1G372mMMfUmIg+r6uB8j9nYRwOU/eyhz2ouGWNWMQsQDTCVL7Hddk0bY1Y568HqrJo9bLDswRizylmAqLNq9hC27MEYs8pZL1ZHlj0YY9YSCxB1NJUv0d9l2YMxZm2wnqxOyl6FYEDoa7PswRizNliAqJPpfIntlj0YY9YQ683qoOxVCFj2YIxZYyxA1IFlD8aYtch6tCXyKkrAVi4ZY9YgCxBLNJUr0t+ZIBKyl9IYs7ZYr7YEXkUREfra4s1uijHG1J0FiCWYzBXZ3mXZgzFmbbKebZG8ihK07MEYs4ZZgFikyVyRbTb3YIxZw6x3WwSvogQENrZb9mCMWbssQCyCW7mUtOzBGLOmWQ93idzKJehrt30Pxpi1zQLEJapmD9FQsNlNMcaYhrIAcQksezDGrCcWIC7BVK7I1s6EZQ/GmHXBAsQCeRUFYJOtXDLGrBMWIBZoOl9iW5dlD8aY9cMCxAJ4FUVVLXswxqwrFiAWwLIHY8x6ZAHiIqrZw0aruWSMWWcaGiBE5GYROSgih0XkA/M8/k4RGRWR7/sfP1fzmFdz//5GtvOFTOdLbO1MEAtb9mCMWV9CjfrGIhIE7gJeC5wCHhKR/ap64Lyn/qOqvmeeb5FT1Wsb1b6F8CpKxeYejDHrVCMziBuAw6p6RFWLwD3AbQ38eXWXzpfY2mHZgzFmfWpkgNgMnKy5fcq/73w/LiKPicg/icjWmvtjIjIkIg+KyJvn+wEicof/nKHR0dH6tRyXPXiqbO6w7MEYsz41e5L6fwPbVfVq4H7g72oe61fVQeDtwJ+JyGXnX6yqd6vqoKoO9vT01LVhlj0YY9a7RgaI00BtRrDFv2+Wqo6pasG/+Qng+prHTvufjwBfA65rYFvPUVHLHowxppEB4iFgQER2iEgEuB04ZzWSiGysuXkr8KR/f4eIRP2vu4FXAOdPbjfMdK7EFssejDHrXMNWMalqWUTeA9wHBIFPquoTInInMKSq+4FfFJFbgTIwDrzTv3wf8DERqeCC2EfmWf3UEBVVyqpssezBGLPONSxAAKjqvcC959332zVffxD44DzXfQe4qpFtu5Bpm3swxhig+ZPUK0pFFa9i2YMxxoAFiHNM50ts6Yhb9mCMMTR4iGk1qQBaUbZ0JJrdFGOMWREsg/DlCmU2t1v2YIwxVRYgfKlY2LIHY4ypYQECCAWFHV1J4hHLHowxpsrmIIBkNEQyai+FMcbUsgzCGGPMvCxAGGOMmZcFCGOMMfOyAGGMMWZeFiCMMcbMywKEMcaYeVmAMMYYMy8LEMYYY+YlqtrsNtSFiIwCx5vdjiXqBs42uxEriL0e57LXY469FudayuvRr6o98z2wZgLEWiAiQ6o62Ox2rBT2epzLXo859lqcq1Gvhw0xGWOMmZcFCGOMMfOyALGy3N3sBqww9nqcy16POfZanKshr4fNQRhjjJmXZRDGGGPmZQHCGGPMvCxANIGI3CwiB0XksIh8YJ7Hf1lEDojIYyLyZRHpb0Y7l8vFXo+a5/24iKiIrNnljQt5LUTkrf7fxxMi8g/L3cbltID/K9tE5Ksi8oj//+WNzWjnchCRT4rIiIj84AKPi4j8uf9aPSYiL1ryD1VV+1jGDyAIPAPsBCLAo8Dl5z3n1UDC//oXgH9sdrub+Xr4z2sBvgE8CAw2u91N/NsYAB4BOvzbvc1ud5Nfj7uBX/C/vhw41ux2N/D1+CHgRcAPLvD4G4EvAQK8FPjuUn+mZRDL7wbgsKoeUdUicA9wW+0TVPWrqpr1bz4IbFnmNi6ni74evg8Dvw/kl7Nxy2whr8XPA3ep6gSAqo4scxuX00JeDwVa/a/bgGeXsX3LSlW/AYy/wFNuA/6HOg8C7SKycSk/0wLE8tsMnKy5fcq/70J+FveuYK266Ovhp8pbVfWLy9mwJljI38ZuYLeIfFtEHhSRm5etdctvIa/H7wA/JSKngHuB9y5P01akS+1bLiq0pOaYhhKRnwIGgR9udluaRUQCwJ8A72xyU1aKEG6Y6UZcZvkNEblKVSeb2agmehvwKVX9YxF5GfBpEblSVSvNbthaYBnE8jsNbK25vcW/7xwichPwG8CtqlpYprY1w8VejxbgSuBrInIMN7a6f41OVC/kb+MUsF9VS6p6FHgaFzDWooW8Hj8LfBZAVR8AYrjCdevRgvqWS2EBYvk9BAyIyA4RiQC3A/trnyAi1wEfwwWHtTzGDBd5PVR1SlW7VXW7qm7HzcncqqpDzWluQ130bwP4PC57QES6cUNOR5axjctpIa/HCeA1ACKyDxcgRpe1lSvHfuBn/NVMLwWmVPW5pXxDG2JaZqpaFpH3APfhVml8UlWfEJE7gSFV3Q/8IZACPiciACdU9damNbqBFvh6rAsLfC3uA14nIgcAD3i/qo41r9WNs8DX41eAj4vI+3AT1u9Uf0nPWiMin8G9Oej251w+BIQBVPVvcHMwbwQOA1ngXUv+mWv0tTTGGLNENsRkjDFmXhYgjDHGzMsChDHGmHlZgDDGGDMvCxDGGGPmZQHCmCUSkV8UkSdF5O+b3RZj6smWuRqzRCLyFHCTqp6quS+kquUmNsuYJbMMwpglEJG/wZWj/pKITInIp0Xk27iaQNtF5Jsi8j3/4+U11/2aiDwuIo+KyEea9gsY8wIsgzBmifwaUYPAe4BbgFeqak5EEkBFVfMiMgB8RlUHReQNwG/hso6siHSq6guVcTamKazUhjH1tV9Vc/7XYeAvReRaXFmM3f79NwF/Wz3zw4KDWaksQBhTXzM1X78PGAauwQ3nruXDjswaZHMQxjROG/CcfzbBT+MKzgHcD7zLH4JCRDqb1D5jXpAFCGMa56+Ad4jIo8Be/OxCVf8VV5p5SES+D/xq01pozAuwSWpjjDHzsgzCGGPMvCxAGGOMmZcFCGOMMfOyAGGMMWZeFiCMMcbMywKEMcaYeVmAMMYYM6//CxSkBFduGQLZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.lineplot(data=res_df, x=\"frac\", y=\"accuracy\")\n",
    "sns.lineplot(data=res_lr, x=\"frac_training_data\", y=\"test_acc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_df['model'] = \"scBERT (reproduced)\"\n",
    "res_lr['model'] = \"Logistic Regression\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_df = res_df.rename(columns={'frac':'fraction training data'})\n",
    "res_lr = res_lr.rename(columns={'frac_training_data':'fraction training data', 'test_acc':'accuracy', 'test_f1':'f1'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_res = pd.concat([res_df, res_lr], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA48AAAFICAYAAADwG8sqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAB5XElEQVR4nO3deVhU1f8H8PcwDPu+C5gLyiirQEWI+5ZpVJprmBJaSkmbW/Uti6xcciHEpcKFzMo1Kc1K0VzRJAlFcQcFFARE9mUY7u8Pf06OgzLgwLC8X8/D8zhnzj3zOTPCZ869554jEgRBABEREREREdFD6Gg7ACIiIiIiImr+OHgkIiIiIiKiOnHwSERERERERHXi4JGIiIiIiIjqxMEjERERERER1YmDx3tUV1cjMzMT1dXV2g6FiIio2WB+JCIigINHJdnZ2Rg4cCCys7O1HQoREVGzwfxIREQAB49ERERERESkBg4eiYiIiIiIqE4cPBIREREREVGdOHgkIiIiIiKiOnHwSERERERERHXi4JGIiIiIiIjqxMEjERERERER1YmDRyIialHkZUWoqarUdhhERETNilxWieqyokZ9Dd1GbZ2IiEhDynOuoaYkH/LCXOgYGENsagMDZ1eIRCJth0ZERKRV5ddSIcvPQk1FKSRWDhBb2MPAvqPGX4eDRyIiavaEGjlqbt9A7s4VqKkoBQAYe/QBRIChs1TL0REREWlPeeYF5O9dj6oblxRlNsOmQc/aETq6ehp9LU5bJSKiZq/y5jUUHNykGDgCQGnKQVTfztFiVERERNpXfStLaeAIAAUHf0JVbobGX4tXHomIqPmrqUbVzauqxWXFWgiGiIjuJ1TLUFVwA0JVJXQt7KFrbKbtkNqMmqoKlTJ5aREgr9b4a3HwSEREzZ6OkQX023dHZUaqUrnYwk5LERER0V3y8hIU/r0Tt49sA4QaSGwfg/0L70DP7jFth9YmSKwdAR0xUCNXlBlJ/SEytdH4a3HaKhERNXt6Fraw6jseupbt7hTo6MKy73hI7DpoNzANi4+PR1BQEDw8PDBkyBBs3br1ofWPHz8OqVRa68/u3bsV9aqqqjB//nwEBATAy8sLISEhuHz5cmN3h4jaiMrsy7h9eAsg1AAAZLnXUHB4M2qqq7QcWdtQZmAHuxHvQmLtBJGuHkw8+8L8ySDcrjbQ+GvxyiMREbUIhh3cYT/mPciL8iDSM4SutRMkhibaDktjkpOTER4ejrCwMAwbNgwJCQmYO3cuLCwsMGjQoFqP8fHxweHDh5XKNmzYgA0bNqBPnz6KsoULF2L37t1YsGAB7O3tERUVhdDQUPz2228wNjZu1H4RUesnu3VDpazs8r+oKSuGjpm1FiJqW0pgjGOZRhg0YgYEeTWqRPrYkVyM0YMNNf5aHDwSEVGLoW/jDNg4azuMRrF+/Xr4+fkhPDwcAODi4oLk5GTExMQ8cPCop6cHW1tbpbI//vgDzz77rGJQWFJSgk2bNmHevHno27cvAGDRokUIDAzErl27MGbMmEbsFRG1BbpmtiplBk6u0DHgyamm4GRrjCsWFpiy8gwKS6rg1cUGrwS5Q1es+UmmnLZKRETUDCQlJaFXr15KZb1790ZKSgpkMplabRw/fhzp6elKA8LTp09DJpMhMDBQUWZiYgJfX1+cPHlSpY2ioiJkZmYq/WRnZzewV0TUFui3c4GJV3/FYx0jM1j1D4aOnuanTZKqtOtFiPwpCYUld6YJn7qUhx//PI/KKnkdR9YfrzwSEalJEATIbl1H9e0c6BiaQc/GmYmRNCYvLw/W1srTu2xtbSGTyVBQUAA7u7oXB9q8eTO6d+8OT09PpXZFIpFK2zY2NsjNzVVpIzY2FtHR0Q3sBRG1RbomFrAe/ArMfIegpqocEitHSMxVr0ZS47ieWwJBUC77+0w2bhVVoJ2NZq/+NvngMT4+HpGRkUhLS4OjoyNee+01jBo16qHHZGZmYtGiRThx4gTKy8vRqVMnvPrqqxg2bJiijiAIWL16NX766SfcunUL3bp1wwcffAAfH5/G7hIRtRHlaaeQs2UBhP9fAMAi8EWYB7wAsb6RliMjAgoKCvDnn3/ivffee6R2Jk2ahBEjRiiVZWdnIzg4+JHaJaLWTWxgDLGTq7bDaJNMjfVUyuytDGGor/mhXpNOW727GMCQIUMQFxeHiRMnYu7cudi7d+9Dj3v99ddRUFCAb7/9Fr/++isGDRqEd999F8nJyYo669atwzfffIPZs2fj559/Rrdu3TB58mROtSEijaguKUDuzhXQMTCGkdQfeu1ccPvINlTdvKbt0KiVsLGxQX5+vlJZXl4edHV1YWlpWefxcXFxEIvFeO6551TaFQRBpe38/HyV+yUBwMzMDM7Ozko/Dg4ODegREbUllfk3UHoxEcUph1Bx/SKEavWm29Oj6+xkDn/3//5O6+iIEPaiNyxM9TX+Wk165bEhiwGUlpbi/PnzWLVqFTw8PAAAb7zxBr777jukpKTA29sbgiBg7dq1mDx5MoYPHw4A+PTTT3H48GH8+OOPeOedd1TaLSoqQlFRkVIZB5pE9CA15SUwce8F6Oig7EIi9Gzbw6zHIMjLbms7NGolfHx8cOTIEUydOlVRdujQIXh6ekIikdR5/JYtWzB06FCYmpoqld89/ujRo3jhhRcA3MmtJ0+exAcffKDRPhBR21SVm4Giv3ei+N94AAJ0LR1g88xUGHXy0nZobYKlqQHCx/TA8OuFKCmTwcnWGB3amTfKazXplceGLAZgbGyMbt264ddff0VxcTFqamqwa9cuVFRU4KmnngJwZ1prbm6u0mIAIpEIgYGBtS4GANy5p2PgwIFKP5ySQ0QPomNkhuqSWyg8+jNkeRkoTT2KW/s3Qmxc9xUhInWEhIQgMTER0dHRuHLlCjZu3IidO3diypQpijp79uzB0KFDkZOTo3RsYmIiLl26hNGjR6u0a2JigrFjx2Lx4sU4ePAgzp07h9mzZ8Pc3FxxwpWI6FFU3byK4n/3Arhz4111QTYKj8WhuuS2VuNqS8xN9OHjaofePZzQ2ckCYh1Ro7xOk155bOhiAGvWrMFbb72Fxx9/HLq6ujAwMMDy5cvh4uICAIob/m1sbJSOs7GxwYkTJ2ptk/d0EFF9yEsKUHrmiFJZTUUJqm/nAM5SLUVFrYm3tzeioqIQGRmJ1atXw8HBAREREUozc4qLi5GWlqZywnXLli1wcXGBn59frW3PmTMHYrEYs2fPRllZGXx8fLB27Vru8UhEGiErVF18qyLzPGoqSgATi6YPiBpNs19tVRAEfPrppxCLxdiwYQNMTU3x559/4p133sH333+P7t27N6hdMzMzmJmZaThaImqthBo5RGJdxWI5/z0h1H4AUQMMGjTogbdxAMDIkSMxcuRIlfKFCxc+tF09PT188MEHnKZKRI1CYql6X7S+kyvEZja11KaWrEmnrTZkMYBjx47hjz/+QGRkJJ588kl0794db731Fjw8PBAbGwsAihv+8/LylI7Nz89Xa2lzIqK6iM1tYOo7RKlMYu0EiZWjliIiIiJqHvTbd4OJ90DFY7GpFaz6joOY21m1Ok165bEhiwGUl5cDAHR0lMe5YrEYwv+f8Xd2doatrS2OHDmi2JpDEAQcOXIEQUFBjdEVImpjJEbmMOkeCLGROWR5GRCbWkPfqSv0bNtrOzQiIiKtkphYwnrIKzDrMRA1VWXQs24PXXNedWyNmvTKY0MWA/Dx8YGVlRVmz56NM2fO4OrVq/j2229x9OhRDB48GMCdxXFCQ0OxZs0a/Pbbb7h06RI+/vhj3L59G+PGjWvKLhJRK2bg7AoTj94we3wYTDz7wriLH3R4VpWIiAhiPUMYOEth1NmHA8dWrEmvPDZkMQBLS0usXbsWy5Ytw+TJk1FZWYnHHnsM8+fPVzrulVdeQWVlJRYsWIBbt26hW7duiImJQbt27Zqyi0TUyknMbSExV90bj4iIiKi1EwkCV3u4KzMzEwMHDkR8fDycnZ21HQ4REd2juKwKKZfzcPTUDTjbm+Ap93bo0I4LnzUF5kciIgJawGqrREREALA/MQPfxqUoHu88lIYF03vBydZEi1ERUXMiVFdDEOTQkehrOxSiVomDRyIiavbybpfjhz/OKZXdLqlE2vVCDh6JCEKNHBUZqbidEIeasiKYPTkcRp19IDYy1XZoRK1Kky6YQ0RE1BCCIKC6RvUui5payoio7am8fhk3Nkag/PJJVN64hNy4r1B2MVHbYRG1Ohw8ErUwssKbKM9IRVV+FgShRtvhEDUJGwtDjOrfVanMyEAXnRx5zyMRAeXXUoD7cuLtYzsgryjVUkRErROnrRK1IOXpp5GzfQlqyosh0tWDzbBpMHHrCZG49n1SiVoLkUiEpwM6wMbCAH8cv4qO7czwtH9HtLfn4JGIAB1d1XscRRIDQMTrJNR2yOU1kFXXwEC/8YZ4HDwStRCyonzc3BGJmvJiAIBQXYXcX6OhZ9cB+vYdtRscUROwNDXAoCc7oL9fe+joiCASibQdEhE1EwYd3CHSM4RQVa4os+w9GmJ9Qy1GRdR0Ll4rQNzBy7iaXYwh/h3Q06sdrM01//+fg0eiFkJeUgB56W3lQqEG1UX5HDxSm1FeIUNeYQX09cSwszTSdjhE1Ezo23eE48ufouxyEmrKi2HU5XHoO7lqOyyiJnEtpxj/W30U5ZXVAIBvdpxGflEFXn6mO8Q6mj3RysEjUQshNjaDjqEJairLoWtqDXnpbQjVMuiaWGo7NKImkZFTjK9/Po3ki7kwNZLgtRc80dPLEXoSsbZDI6JmQN+hM/QdOms7DKImd/VGoWLgeNcvBy/jmac6wN7aWKOvxcEjUQshMbeD3YiZqMw8B1l+FnTNbaDn6AqJDTfsptavqkqODb+dRfLFXABAcZkMS344CQdrY3TraKXl6IiIiLRHV6x6b6+erg50dDR/zy8Hj0QtRE21DGWXElH0906IJAYQZJXQb98dBs5S6Ej0tB0eUaMqKK7AsTPZKuVZeSUcPBIRAKCmqhyy/BuokVdBYtUOukbm2g6JqEl0cjSHnaUhbhb8d8/vhGe6w9aS9zwStVmygmzI8q/Dst9LqCkrgo6BCWS3cyDLy4CuMRMktW6GBrpwtDFGVq7ysvvmxqorLBJR21NdXICCAz+iODkeACCxfQz2I96Fnm17LUdG1PgcrI0R8WoAks7fRGZuKfy62cGtU+OcWOXgkailEARILOxQ8NcPiiLDjl5chpzaBDNjfUwb6YWImGOolgsAgKc8HODizBMnRARUZJ5TDBwBQJZ7DYWJv8Hm6SkQ6fC+aGr9nO1N4Wxv2uivw8EjUUsh1KAoaa9SUXn6KZj3fEE78RA1Ma8utlj2Tj9k5RbDxEAPHR3NYG7CK49EBFTeuKRSVn75X9RUlEFs1PhfqAkQBAHZ+WWokslhZ2kIQwPuQd0acfBI1FIINUBNtWp5TU3Tx0KkBTo6InRsZ4aO7cy0HQoRNTMSC3uVMn0nV0CHX3WbQlllNfaduIbYXWdRUSVHD1dbTB3hCWc7DtxbG853I2ohdM3toO8kVSrTMTSFxKqdliIiIiJqJiT6MOr6uOKhrmU7GDi7AoJci0G1HZcybuPrn0+jourO+/3vhVxs23cJ1dU8wd3acPBI1EKIDU1g++zrMPUeAB1DUxh28UW78R9BYumg7dCIiIi0SmJmA0FeDcs+Y2HRazQMO3oAEEFsaKLt0NqEzJxilbKjp6/jdkmlFqKhxsRr+UQtiJ6NM2yeeQ2W/V6Cjr4RdCS834uIiEjfsSvMfIfg1v6NqKkohfmTz8K4W4C2w2ozrMwNVMo6tjODkSGHGq0NP1GiFkYklkDXxFLbYRARETUbOhI9GEv9YfCYOwR5NXRNLLQdUpvSpb0F/LrZ4Z9zNwEABnpihAx3g5E+F81pbTh4JCIiIqJWgdNUtcPG3BDvjPfFtZxiyGRy2FkZcbGcVor3PBIRERER0SMpLKlCelYhTl3KQ15hOSoqa1khnlo8Dh6JiIiIiKjBMm8WY/Pe8yitqIauWAdJ53Nx8vxNbYdFjYDTVomIiIiIqMFu5JeirLIaG/84BwAwM9aDg5URikoqYWbCxf1aE155JCIiIiKiBisqqcKJszn/PS6twl8nM1Ep49TV1oaDRyIiIiIiarDScplKWdr1IohEHGq0NvxEiYiImon4+HgEBQXBw8MDQ4YMwdatW9U6LiEhAS+99BJ69OgBX19fjB8/HoWFhYrnBwwYAKlUqvQzc+bMxuoGEbUxjzmorqzq180OZiZ6WoiGGhPveSQiqodKWTVuF1fBSF8XpsZMik2tpKwKuQXlMNAXw8HaGCKRSNshaUxycjLCw8MRFhaGYcOGISEhAXPnzoWFhQUGDRr0wOP27duHd955B2FhYYiIiICuri4uXLgAsVisVG/atGmYMGGC4rGBgeqm3kREDdHF2QLBT3fDT3vOQ14joLOTOcYPkUJPV1z3wdSicPBIRKSmjJxibPw9FQmnb8DJ1gRTR3rBq4tNqxrANGdXbxThq00ncTGjEAZ6Ykx+zgP9fJ1hoN86Utn69evh5+eH8PBwAICLiwuSk5MRExPzwMGjXC7HvHnzEBISgmnTpinKO3XqpFLX2NgYtra2jRM8EbVpJkZ6GDWwK3p6O6KyqhoOVsY8wdpKcdoqEZEayitk+GbHaRw5dQM1ApBxswSffHsMV7OLtR1am1BRVY11O8/gYkbh/z+WY8XWZKRdL6zjyJYjKSkJvXr1Uirr3bs3UlJSIJOp3k8EAGfOnMH169dha2uL8ePHIyAgAC+99BISEhJU6sbGxsLf3x9BQUFYvHgxSktLa22zqKgImZmZSj/Z2dmP3kEiatV0xTp4zN4UXdtbcuDYirWO07VERI0st7AC/17IVSqrltcgK7cYHduZaSmqtuN2cSX+Oae6Z9j1/FJ072SthYg0Ly8vD9bWyn2xtbWFTCZDQUEB7OzsVI7JyMgAACxfvhyzZs2Cm5sbdu3ahcmTJ2P79u3o1q0bAGDChAno3r07rKyskJqaiqVLlyI1NRVr1qxRaTM2NhbR0dGN0EMiImrpOHgkIlKDgZ4YJoYSlNy3opyhvkRLEbUtxoYSPOZgimv3Xem1NG3b9+0JggAAGDt2LEaNGgUAcHNzw/Hjx/Hjjz8iIiICABAaGqo4RiqVwtnZGcHBwTh79izc3NyU2pw0aRJGjBihVJadnY3g4ODG7EqrIggCp7MTUavEaatERGrQl+hgZL8uSmVeXWxgbMBzcE3B1EgPYSO9oC/5b/GF/r7OcHE012JUmmVjY4P8/Hylsry8POjq6sLS0rLWY+7ew+ji4qJU7uLighs3bjzwtby9vSESiZCenq7ynJmZGZydnZV+HBwc6tmbtkeokaP86hnk/LwMOVsWouxyEmpkFdoOi4hIo/ith+pFXlmGquw0VBflQdfMBnr2HSE2MNZ2WESNrrCkCkdOXcfEYd1RUSWHnq4OsnJLkHu7HNIO2o6ubfBwsUHkO32RlVcCE0MJOjiYwcSo9dxX4+PjgyNHjmDq1KmKskOHDsHT0xMSSe1XuN3d3aGvr4+0tDSl8vT0dJUrivdKTU2FIAhcQEeDKrIu4MbGTwChBgBQdvEE7Mf+D8ZdfLUbGBGRBnHwSGqrqZah6MRvKDjwo6LMotdoWASOhI5u6/kCR1QbcxN9VFRV47vfUpXKhwd21lJEbZOzvSmc7VX3E2sNQkJCMH78eERHRyu26ti5cyeioqIUdfbs2YMlS5YgNjYW9vb2MDExQXBwMDZs2ACpVAo3Nzfs3LkTZ86cwaeffgrgzkI8//77L/z9/WFmZoZz585hwYIF8PT0hJ+fn7a62+qUnjsGE8++kJjbQoCAmvISFJ7YBaNOXhCJ+XWLiFqHJv9rFh8fj8jISKSlpcHR0RGvvfaa4j6N2hw/fhwTJ06s9bnIyEg888wzAO5sgJyVlaX0/N0V5UgzZPnXUZF5Hpb9XoIgq4RIVw+V1y9ClpcFfQfVZeGJWhNzE328NdYHX6w/gdslldAVixAa5IEO7VrnQIaanre3N6KiohAZGYnVq1fDwcEBERERStt0FBcXIy0tTWn11RkzZkAikeDzzz9HaWkpXF1dERMTA6lUCgDQ09PD77//jpUrV6KiogKOjo4YPHgwwsLCoKPDu1c0Rd++MwpP7IIsLxMisQTVpQUw939e22EREWmUSLh7t30TSE5Oxvjx45U2QP7iiy8QFRX1wD2sqqqqUFiovBT7hg0bsGHDBhw+fBjGxnemTA4YMABBQUEqGyCbmqr/xS4zMxMDBw5EfHw8nJ2dG9DD1q38WipKTu9H8b/xijITz74w8RoAo44eWoyMqOnkFpThZkEZTI304GhrAl0xv3xT68f8WLfCpHgIVWUoT0uGUC2DUeceEEQiWDz1PBfPIaJWo0mvPDZkA2Q9PT2VezL++OMPPPvss4qB413cALmR1VSj+N99SkUlpw/AxLOvlgIianq2lkawtTTSdhhE1MyIdHSQvzcWwJ1z8hVXU2A9OBSCXAYRb+0golaiSU+ZN2QD5PsdP34c6enpGDNmjMpz6m6ADHAT5IaoqZbhblK8l1Ct3mdHRETUWlVmXcD9ObL0wt8QiTg7gYhajya98tiQDZDvt3nzZnTv3h2enp5K5fXZABngJsgNoWf7GHTNbFFd9N9G6WITS0hs22sxKiIiIu0T6emrlom5DywRtS4tavmvgoIC/Pnnn3jvvfdUnqvPBsgAN0FuCIm5DexenIlb+75HRUYq9J2lsB7wMvQs7LUdGhERkVaZdO+JohO7gZrq/y8RwfypIK60SkStSpP+RWvIBsj3iouLg1gsxnPPPVdn3Xs3QK5t8GhmZgYzMzP1gycAgIFjFziMeQ/y8mLoGJhArG+o7ZCIiIi0Tt+xCxxfnofS1COokVXBxL0XDJxctR1Wm3Ip8zaOp9xAYUkVenq1Q/eOVtDX4+CdSJOa9DeqIRsg32vLli0YOnSoWiuocgPkxqOjZwAdPQNth0FERNRsiEQ6MHB2hYEzB4zacCWrEO+vOIyKKjkAYHdCOj4M9Ye/u4OWIyNqXZr0Lu6QkBAkJiYiOjoaV65cwcaNG7Fz505MmTJFUWfPnj0YOnQocnJylI5NTEzEpUuXMHr0aJV2k5KSsG7dOpw9exaZmZnYu3cv3n33XW6ATEQaJ68RUFBcgYqq6rorExFRk0i+mKsYON7105/nUV7JRf2INKlJrzw2dANk4M5VRxcXl1oHg9wAmYiawo28Uuw6moaDJzPRoZ0Zgp/uhm4drbQdFhFRm1dRqXpCr0omR01Nk21nTtQmiARB4G/V/+MmyET0IJVVciz78R8cOXVDUWagJ8bSt/uivX3dU+mJWjLmR2ruDiZlYukPJyG/Z7A4YWg3PNOzE8yMuc8mkabwLmIiIjXk3i7D0dM3lMoqquTIyCnm4JGISMvEIhFeedYdSRduorRCBr9u9qiW18BQX6zt0IhaFc7pJCJSg65YB/oS1S8hBlzJj4hI67o8ZoFDyVkoKZNBX6KL346kwd+jHSS6HDwSaRK/9RARqcHeyggTh7vhm59PK8q6dbRCx3bc8oeISNvsrYzxQciTuHqjCFXVNWhvbwJHGxNth0XU6nDwSNSC3C6uwNm0Wziblo9OjubwdLGBnZWRtsNqE0QiEQY+3h7t7UxxOfM27KyMIO1gCStzbltDRNQcWJkZwMqMf5OJGhMHj0QthKxajm37L2HHgcuKMg8Xa7w38QmYm+hrMbK2w8hAgh6utujhyv1jiYiaG1m1HDfySiGrroGDtTGMDeveQ5yI6oeDR6IW4kZeKX45eFmpLOVyPq5lF8OzCwePRETUdhWWVOLnvy7h5wOXUVMjwLOLDd4Y5Q0nW05dJdIkLphD1EJUy2tQ23ZVsuqapg+GiIioGTl39Ra27b+k2Nfx9KU8/J6Qzn0eiTSMg0eiFsLB2hi+UuXpktbmBnC251lVIiJq2y5cLVApSzh9AyXlMi1EQ9R6cdoqUQthZCDB1BFe2HviGo6eugG3zlYI6tUZdpZcMIeIiNq2xxxUV77u1tEKRtznkUijOHgkakEcbU3w8jPd8WL/rjDQE0Ms5uQBIiIit05W6NHVFv9ezAUAWJjqY2S/LtDlPo9EGsXBI1ELIxKJuIIcERHRPWwtjTBzgh8ycopRJauBs50Jt7IiagQcPBK1MNXyGhSXVcHIQBf6Ev4KExERAYC5iT63riJqZPzmSdSCZOYUI+7gZRw7kw3pY5YYN0SKLs4W2g6LiIiIiNoADh6JWoiS8ios3/IvzqbdAgAcP5ON1PRbWPJWHzhYG2s5OiKiB5MV5qEi8xxkeZnQd+wCAydXiI1UFzghIqLmjYNHohYiJ79MMXC8q6i0Clm5JRw8ElGzJS8rQt7u1Si/nKQoM+85ElZ9xkAk5v3bREQtCZdqJGohJBIxxDoilXJ9Pa4kR0TNV1VuhtLAEQAKj8VBditbSxEREVFDcfCoYYIgaDsEaqUcbYwxemBXpbIn3R3wmL2pliIiIqqbUF2lWlgjhyDn5u1ERC0Np61qiLyyDBXpKSg6+SfExmYw830a+k6uEIlUrxQRNYSuWAfP9XFBt45WuJJVCCcbE0g7WMLMmCvLEVHzJbF2gtjYHPLSQkWZQQd36FrYazEqIiJqCA4eNaTsQiJyf/lK8bjk7BE4vvwZDJy6PuQoovoxNdJDZydzWJnqw9hQAmsLQ22HRET0UBILOziM+xDFKYdQU1UBsYklTN17QWzAe7WJiFoaDh41QF5ZhtsJP99XWI3ytH85eCSNOpd+C19+n4ibBeUwNZLgzbE+eMLNodZ7IYmImg2RDmoqSlCZcQ5GUn+As3KIiFok3vOoATVVlRDpqL6VQk2NFqKh1upWYTkWbbgzcASA4jIZFn53Alk3i7UcGRFpSnx8PIKCguDh4YEhQ4Zg69atah2XkJCAl156CT169ICvry/Gjx+PwsL/polWVVVh/vz5CAgIgJeXF0JCQnD58uXG6oYSWWEesjd9gZLkfZDduo7ChJ+R/8cayCvLmuT1iYhIczh41ACxviFMvPorlYl09aBn11E7AVGrlF9Ugdzb5Upl1XIBObf4BYyoNUhOTkZ4eDiGDBmCuLg4TJw4EXPnzsXevXsfety+ffswbdo09OnTB1u2bMG2bdsQEhICsfi/lZgXLlyIX3/9FQsWLMDmzZthZGSE0NBQlJaWNna3IMvPhLw4X6ms/EoSqgtyGv21iYhIszhtVQMEoQY6xhawGvAyKjLPQUffCPoOLhCJ+faS5pga6cHIQBdlFdVK5RamXDCHqDVYv349/Pz8EB4eDgBwcXFBcnIyYmJiMGjQoFqPkcvlmDdvHkJCQjBt2jRFeadOnRT/LikpwaZNmzBv3jz07dsXALBo0SIEBgZi165dGDNmjFKbRUVFKCoqUirLzn6EbTVqW4VcpAOuTU5E1PLwyqNGiFBy+hAgCDB4zB169h0hLy+GrIB7WJHmOFgbY/roHtC55/7Gl5/phvbcqoOoVUhKSkKvXr2Uynr37o2UlBTIZLVva3HmzBlcv34dtra2GD9+PAICAvDSSy8hISFBUef06dOQyWQIDAxUlJmYmMDX1xcnT55UaTM2NhYDBw5U+gkODm5wv0S6+tB3VL7/38Szb623exARUfPGS2MaINY3hFmP/ri57UuIdPUg1NRApKMDx4mfazs0amUCPNvhq3f6IudWGSzNDPCYvSkM9PhrTNQa5OXlwdraWqnM1tYWMpkMBQUFsLOzUzkmIyMDALB8+XLMmjULbm5u2LVrFyZPnozt27ejW7duyMvLg0gkUmnbxsYGubm5Km1OmjQJI0aMUCrLzs5+hAGkAH3HLjDs5IXqojxILB1QlZcJkYiDRyKilobfOjXEqHMPOIz7ECUpByA2Moexey/ot+us7bColdEV66Cjozk6OpprOxQiagaE/58SOnbsWIwaNQoA4ObmhuPHj+PHH39EREREvds0MzODmZmZxmLUs22P2wk/o+LqGegYmaEk5RDMe46AxMpBY69BRERNg4NHDdHRM4CRiw+MXHy0HQoREbVANjY2yM9XXlgmLy8Purq6sLS0rPUYW1tbAHfuj7yXi4sLbty4oWhXEATk5+crXb28/3FjERuZwWbYNFRmnkdVXhb023WGgZMrRGJJo782ERFpFueMEBERNQM+Pj44cuSIUtmhQ4fg6ekJiaT2gZa7uzv09fWRlpamVJ6eng4nJycAUBx/9OhRxfOlpaU4efIkfH19NdyL2knMbGDiFgirPmNg3PVxiI00d2WTiIiajtqDx6qqqsaMg4iIqE0LCQlBYmIioqOjceXKFWzcuBE7d+7ElClTFHX27NmDoUOHIifnzjYXJiYmCA4OxoYNG7B7925cvXoVK1aswJkzZzBu3DhFnbFjx2Lx4sU4ePAgzp07h9mzZ8Pc3BzDhw/XSl+JiKhlUnvaaq9evfD8889jzJgx6Nq1a90HEBERkdq8vb0RFRWFyMhIrF69Gg4ODoiIiFDapqO4uBhpaWlKq6/OmDEDEokEn3/+OUpLS+Hq6oqYmBhIpVJFnTlz5kAsFmP27NkoKyuDj48P1q5dC2Nj4ybtIxERtWwiQahtAyZVX331FbZv346bN2/Cy8sLY8aMwbBhw2BoaNjYMTaZzMxMDBw4EPHx8XB2dtZ2OERERM0C86N6SstlyMotgbymBo62JjA35j68RNS6qD1t9a233sL+/fuxcuVKWFlZYe7cuejVqxfmzp2LlJSUxoyRiIiIqFm7WVCGZT/+gxlfHcTs5YcxL+Y4snJLtB0WEZFG1WvBHB0dHfTv3x+rVq3C/v378eqrr+Lo0aMYPXo0RowYgR9//BFlZWWNFSsRERFRs5R8MRfHz+TA1sIQDtZGOH+tAH/9k6ntsIiINKrBq63K5XLIZDLIZDKIRCLF/RYDBw5UWtGNWpfcgnLEn7iGyJ9OYs/fV3GzgCcLiIiILmfexivPuqGHqy26d7RCaJA7rt4oQpVMru3QiIg0pl77PMrlcuzbtw+bN2/G0aNHYWVlhVGjRmHMmDFo164dbt68iY8++ggRERH4448/am0jPj4ekZGRSEtLg6OjI1577TXFxsa1OX78OCZOnFjrc5GRkXjmmWcA3FkNdsmSJfjll19QWloKX19ffPTRRyp7X1HDlVXIEPPLaRw9dWfvsPgTGXiiuz3eDfaFiaGelqMjIiLSnh6udlj4XSKq5TUAgL9OZuLd8b7Qk4i1HBkRkeaoPXhcvHgxduzYgfz8fPTs2RORkZEYMGAAxOL//ija2dnh1VdfxYQJE2ptIzk5GeHh4QgLC8OwYcOQkJCAuXPnwsLCQmk1uXv5+Pjg8OHDSmUbNmzAhg0b0KdPH0XZwoULsXv3bixYsAD29vaIiopCaGgofvvtN64mpyFZuSWKgeNdJ1JzkHWzBNIOVlqKioiISPtS024pBo4AIAjAgaQs9O7hBLGY22oTUeug9uBx+/btGDlyJMaOHYv27ds/sF7nzp0xf/78Wp9bv349/Pz8EB4eDgBwcXFBcnIyYmJiHjh41NPTg62trVLZH3/8gWeffVYxKCwpKcGmTZswb9489O3bFwCwaNEiBAYGYteuXRgzZoy63aSHkMtrX5j3QeVERERtRVW16vTUyqpqMEMSUWui9qmwAwcOYObMmQ8dOAKAlZUVRowYUetzSUlJ6NWrl1JZ7969kZKSorRn1cMcP34c6enpSgPC06dPQyaTITAwUFFmYmICX19fnDx5stZ2ioqKkJmZqfSTnZ2tVgxtlaOtCaQdLJXKOjuZw8nOREsRERERNQ+BXo4QiZTLnu/jAl1edSSiVkTtK4+XL19GTk6O4srevQ4cOAB7e3t069btoW3k5eXB2tpaqczW1hYymQwFBQWws7OrM47Nmzeje/fu8PT0VGpXJBKptG1jY4Pc3Nxa24mNjUV0dHSdr0f/MTPWw2sveOLAyUycuZKP7p2s0N+3PcxNuI8VERG1bdIOlvh0ak/E/XUJlTI5nu/jAq+utnUfSETUgqg9eFywYAF69OhR6+AxOTkZSUlJWLdunUaDu19BQQH+/PNPvPfee4/c1qRJk1SukGZnZyM4OPiR226tCksqEb35X5RUyPCYvSlOnM1B8sVcfB4WCEtTA22HR0REpDUSXTF6dLWFR2drAAJ0xVwoh4haH7UHj6mpqXjllVdqfc7HxwcbN26ssw0bGxvk5+crleXl5UFXVxeWlpYPOOo/cXFxEIvFeO6551TaFQQB+fn5Slcv7398LzMzM5iZmdX5mvSf63klSLtRBODOlh13ZeWWcPBIREQEcJoqEbVqav+Fq6ysRE1NTa3PVVdXo6Kios42fHx8cOTIEaWyQ4cOwdPTExKJpM7jt2zZgqFDh8LU1FSp/O7x9+4vWVpaipMnT8LX17fOdkk9kgecRdXT5dlVIiIiIqLWTu3BY5cuXbB79+5an/v999/V2k8xJCQEiYmJiI6OxpUrV7Bx40bs3LkTU6ZMUdTZs2cPhg4dipycHKVjExMTcenSJYwePVqlXRMTE4wdOxaLFy/GwYMHce7cOcyePRvm5uYYPny4ul2kOjjZmWDA485KZb29neBkywVziIiIiIhaO7WnrYaEhGDmzJkQiUQYPXo07O3tkZOTg61bt+KXX37BwoUL62zD29sbUVFRiIyMxOrVq+Hg4ICIiAilbTqKi4uRlpamsvrqli1b4OLiAj8/v1rbnjNnDsRiMWbPno2ysjL4+Phg7dq13ONRgwz1dTFxmBuecHPAhWsF6OJsAbdO1jA2rPuqMRERERERtWwiQRDU3oJo7dq1WL58uWKKqiAIMDQ0xFtvvYWQkJDGirHJZGZmYuDAgYiPj4ezs3PdBxAREbUBzI9ERATU48ojAISGhmLMmDFISkrC7du3YWlpiR49esDEhNMWiYiIiIiIWrN6DR6BO/cX9u7duzFiISIiIiIiomaq3oPH1NRUpKWloaqqSuW5F154QRMxERERERERUTOj9uCxsLAQr776Kk6dOgWRSIS7t0qKRCJFHQ4eiYiIiIiIWie1t+pYsmQJSktLsWnTJgiCgFWrVuH777/Hiy++CGdnZ2zdurUx42wRbt4qwz/ncnD6ch4KSyq1HQ4REREREZHGqH3l8ciRI3jrrbfg4eEBALCxsYGnpycef/xxfPbZZ4iJicGyZcsaLdDm7sr1Qnz8TQJuF98ZNPboaos3x/aAraWRliMjIiIiIiJ6dGpfeczLy4OjoyPEYjEMDQ1x+/ZtxXMDBgzAoUOHGiO+FqG6Wo7t+y8qBo4A8O/FXJy5kq/FqIiIiIiIiDRH7cGjvb09CgsLAQDOzs44evSo4rnU1FRIJG13o/jSymqcSy9QKc/IKdZCNERERERERJqn9rRVf39/nDhxAgMHDsTYsWPx2Wef4dKlS5BIJDhw4ABefPHFxoyzWTM11MNTHu0Qd/CyUnnXxyy1FBERERFpQ0lZFWoEAWbG+toOhYhI49QePL799tuKqaoTJkyAXC7H77//joqKCoSGhuKNN95orBibPR0dEXxcbXExowBn025BR0eEQU+0h6WZgbZD07hKWTWu3SjGzdvlsLUwxGMOpjDQq/eOL0RERK1KRWU1ElNz8MOf51Alq8HogV0R4NmOg0gialXU+tYvl8uRm5sLOzs7RdmkSZMwadKkRgusJSktl2HNr2fgZGuMcYNdIQjAidQcOFzMhbQVXX2sltdg798ZWL39lKJsyvMeGNazIyS6Yi1GRkREpF2p6bewcEOi4nH0lmQY6Omir6+zFqMiItIste551NHRwahRo5CamtrY8bRIOjqAgZ4Yx1Ky8dOeC9i09wKuZBVCT9K6BlTXc0sQE3daqWztr2eQdbNESxERERE1D0dOXVcp++3IFciq5VqIhoiocag1eBSJRHB0dERpaWljx9MiGepLMG6w9L4yXXi62GgposZRVFqFarmgVFZTI6CotEpLERERNW+VlZW4fl11UEGtj1Utt6pYmxtCR0fttQmJiJo9tf+iTZo0Cd9++y1KSniVqTberjb4fFpPBPXqhOCnu2H+64Ho7GSu7bA0ys7SCGbGekplxga6sLXiXpZERLX566+/MHDgQG2HQU3gKY92MDL4724gXbEOgnp3hlhHpMWoiIg0S+2VTlJTU5GdnY3+/fvDz88PNjY2EIn++4MoEonw6aefNkqQLYG+RBdeXW3h1dVW26E0GjsrI3wQ8gSW/XgSObfuLJjzzngftLM21nZoREStQnx8PCIjI5GWlgZHR0e89tprGDVq1EOPGTBgALKyspTKgoKCsHjx4nrVoUfT2ckcC6f3Qmp6AeTVNZB2tEQXZwtth0VEpFFqDx6PHj0KiUQCiUSCCxcu4MKFC0rP3zuQpNbLvbMNvnyzDwqLK2Fmol/rNB0iotYuOjparXqXL1+uu9L/S05ORnh4OMLCwjBs2DAkJCRg7ty5sLCwwKBBgx567LRp0zBhwgTFYwMD1b/N6tShR9OxnTk6tmtds46IiO6l9uBx3759jRkHtSCWpgawNOWXDiJqu6KjoyEW170omiAIdda5a/369fDz80N4eDgAwMXFBcnJyYiJialz8GhsbAxb24fPfFGnDhER0cNwgz4iIqJ6srOzw4ABA/DJJ588tN7vv/+Od955R602k5KSMH78eKWy3r1744MPPoBMJoNEInngsbGxsVizZg3s7OzQt29fhIWFwdjYuN51AKCoqAhFRUVKZdnZ2Wr1gYiIWje1B4+//vprnXWCgoIeKRgiIqKWwMvLC6dPn66zXn1u6cjLy4O1tbVSma2tLWQyGQoKCpT2Wr7XhAkT0L17d1hZWSE1NRVLly5Famoq1qxZU686d8XGxqo9LVddRaWVyMgpRkWVHCaGEnRwMIOBPs9fExG1NGr/5Z41a1at5fcmRg4eiYioLRg4cCB27txZZ70uXbrgjTfeaNRYQkNDFf+WSqVwdnZGcHAwzp49Czc3N7Xr3DVp0iSMGDFCqSw7OxvBwcENiq+0XIZ/UnNw/tptZNwsRldnC9wqroS/mwN0uBIpEVGLovbgMT4+XqXs9u3b2L9/P3799Vd8+eWXGg2MiIioufruu++waNEiAHfufxw9ejTs7e1V6rm4uGD69OlqtWljY4P8/Hylsry8POjq6sLS0lLt2Ly9vSESiZCenq4yMFSnjpmZGczMzNR+vbpk5RZjc/xFZN68s9XXqYt5eMrDAY/Zm8LJ1kRjr0NERI1P7X0enZycVH7c3d0xffp0BAUFISYmpjHjJCIiajYuXLiAsrIyAMCKFSuQk5PzyG36+PjgyJEjSmWHDh2Cp6fnQ+93vF9qaioEQXjo4jjq1NGUmwXlioHjXcfPZKOgqKLRX5uIiDRLIzcc+Pn5Ye3atZpoioiIqNmzt7fH3r17YW1tDUEQkJubi+vXrz+wvqOjY51thoSEYPz48YiOjlZs1bFz505ERUUp6uzZswdLlixBbGws7O3tkZSUhH///Rf+/v4wMzPDuXPnsGDBAnh6esLPzw8A1KrTmPR0VVelFQRAX1L3arVERNS8aGTwmJycDHNz7mtERERtw4QJE7Bo0SLExMRAJBLVOTU1NTW1zja9vb0RFRWFyMhIrF69Gg4ODoiIiFDapqO4uBhpaWmQyWQAAD09Pfz+++9YuXIlKioq4OjoiMGDByMsLAw6Ojpq12lMLs7maG9vgoyc/64+9vFxQnsH00Z/bSIi0iyRoOYmVLWtvCaTyXDx4kUcOHAA06ZNU+xN1VJlZmZi4MCBiI+Ph7Ozs7bDISKiZuz8+fO4fPky3n33XUyfPh2PPfbYA+s+99xzTRiZ5j1qfsy8WYzD/15HypV8BHg64Ak3B9hZGjVCpERE1JjUvvK4fft2lTJ9fX04OjoiIiICI0eO1GhgREREzZlUKoVUKsWhQ4fw/PPPo3379toOqdlytjPFuCFSbYdBRESPSO3B4759+xozDiIiohZp/vz52g6BiIioSTT+zQ5ERERERETU4qk9eNywYQMWL15c63OLFy/Gxo0bNRYUERERERERNS9qDx5//PHHB97P0aFDB/z4448aC4qIiIiIiIiaF7UHj1lZWejYsWOtzz322GPIzMzUVExERERERETUzKg9eDQ0NMSNGzdqfe769evQ19fXWFBERERERETUvKg9ePT398fq1atRWFioVF5UVIRvvvkG/v7+Gg+OiIiIiIiImge1t+p45513MHr0aAwePBiDBg2Cvb09cnJyEB8fr3heHfHx8YiMjERaWhocHR3x2muvYdSoUXUel5CQgOXLl+Ps2bPQ0dGBVCrF6tWrYW5uDgAYMGAAsrKylI4JCgp64CI/REREREREpD61B48dO3bEtm3bEBUVhYMHD+L27duwsLBA3759ER4ertbmyMnJyQgPD0dYWBiGDRuGhIQEzJ07FxYWFhg0aNADj9u3bx/eeecdhIWFISIiArq6urhw4QLEYrFSvWnTpmHChAmKxwYGBup2j4iIiIiIiB5C7cEjcGdhnEe5krd+/Xr4+fkhPDwcAODi4oLk5GTExMQ8cPAol8sxb948hISEYNq0aYryTp06qdQ1NjaGra1tg+MjIiIiIiKi2ql9z2NJSQlu3rxZ63M3b95EaWlpnW0kJSWhV69eSmW9e/dGSkoKZDJZrcecOXMG169fh62tLcaPH4+AgAC89NJLSEhIUKkbGxsLf39/xXTVh8VUVFSEzMxMpZ/s7Ow6+0BERERERNQWqX3lce7cuTAwMMAXX3yh8lxUVBTKy8uxZMmSh7aRl5cHa2trpTJbW1vIZDIUFBTAzs5O5ZiMjAwAwPLlyzFr1iy4ublh165dmDx5MrZv345u3boBACZMmIDu3bvDysoKqampWLp0KVJTU7FmzZpaY4mNjUV0dLRafSciIiIiImrr1B48/v3335g7d26tz/Xt2xfz5s3TWFD3EgQBADB27FjFwjpubm44fvw4fvzxR0RERAAAQkNDFcdIpVI4OzsjODgYZ8+ehZubm0q7kyZNwogRI5TKsrOzERwc3Cj9ICIiIiIiasnUHjwWFhbC1NS01ueMjY1RUFBQZxs2NjbIz89XKsvLy4Ouri4sLS1rPebuPYwuLi5K5S4uLg/cdxIAvL29IRKJkJ6eXuvg0czMDGZmZnXGTERERERERPW459HR0RGJiYm1PvfPP//AwcGhzjZ8fHxw5MgRpbJDhw7B09MTEomk1mPc3d2hr6+PtLQ0pfL09HQ4OTk98LVSU1MhCAIX0CEiIiIiItIAtQePQUFB+Pbbb7Ft2zbU1NQAAGpqarB9+3bExMQgKCiozjZCQkKQmJiI6OhoXLlyBRs3bsTOnTsxZcoURZ09e/Zg6NChyMnJAQCYmJggODgYGzZswO7du3H16lWsWLECZ86cwbhx4wDcWYhn3bp1OHv2LDIzM7F37168++678PT0hJ+fX73eECIiIiIiIlKl9rTVqVOnIiUlBf/73//wySefwNraGvn5+ZDJZOjfvz/CwsLqbMPb2xtRUVGIjIzE6tWr4eDggIiICKVtOoqLi5GWlqa0+uqMGTMgkUjw+eefo7S0FK6uroiJiYFUKgUA6Onp4ffff8fKlStRUVEBR0dHDB48GGFhYdDRUXt8TERERERERA8gEu6uSKOmo0eP4ujRoygsLISlpSUCAgIQEBDQWPE1qczMTAwcOBDx8fFwdnbWdjhERETNAvMjEREB9bjyeFfPnj0hlUpRWVmpKLt+/TqAO/dFEhERERERUeuj9uBRLpdj2bJl2Lx5M4qLi2utk5qaqrHAiIiIiIiIqPlQ+4bAtWvXYtOmTXj11VchCAJef/11TJ8+HR07dkT79u3xxRdfNGacREREREREpEVqDx537NiB6dOnIzQ0FAAwYMAAvPHGG9i1axfat2+PK1euNFqQREREREREpF1qDx6zsrLg5uYGsVgMXV1dxdRVHR0dTJgwATt27GisGImIiIiIiEjL1B48mpmZoby8HABgb2+P8+fPK54rLS1FaWmp5qMjIiIiIiKiZkHtBXO8vb1x7tw59OnTB4MHD8aKFSsgk8mgq6uLNWvWwMfHpzHjJCIiIiIiIi1Se/D46quvIisrCwDwxhtvICMjA8uWLYNcLoe3tzciIiIaLUgiIiIiIiLSLrUHj15eXvDy8gIAmJqaYsWKFaiqqkJVVRVMTEwaLUAiIiIiIiLSPrUHj7XR09ODnp6epmIhIiIiIiKiZkrtBXOIiIioccXHxyMoKAgeHh4YMmQItm7dWucxAwYMgFQqVfqZOXOmUp2qqirMnz8fAQEB8PLyQkhICC5fvtxY3SAiolbqka48EhERkWYkJycjPDwcYWFhGDZsGBISEjB37lxYWFhg0KBBDz122rRpmDBhguKxgYGB0vMLFy7E7t27sWDBAtjb2yMqKgqhoaH47bffYGxs3Cj9ISKi1oeDRyIiomZg/fr18PPzQ3h4OADAxcUFycnJiImJqXPwaGxsDFtb21qfKykpwaZNmzBv3jz07dsXALBo0SIEBgZi165dGDNmjGY7QkRErRanrRIRETUDSUlJ6NWrl1JZ7969kZKSAplM9tBjY2Nj4e/vj6CgICxevFhp7+XTp09DJpMhMDBQUWZiYgJfX1+cPHlSpa2ioiJkZmYq/WRnZz9i74iIqDXglUciIqJmIC8vD9bW1kpltra2kMlkKCgogJ2dXa3HTZgwAd27d4eVlRVSU1OxdOlSpKamYs2aNYp2RSKRSts2NjbIzc1VaS82NhbR0dEa6hUREbUmHDwSERG1YKGhoYp/S6VSODs7Izg4GGfPnoWbm1u925s0aRJGjBihVJadnY3g4OBHjpWIiFo2Dh6JiIiaARsbG+Tn5yuV5eXlQVdXF5aWlmq34+3tDZFIhPT0dLi5ucHGxgaCICA/P1/p6uX9j+8yMzODmZlZwztCREStFu95JCIiagZ8fHxw5MgRpbJDhw7B09MTEolE7XZSU1MhCIJiAZ27xx89elRRp7S0FCdPnoSvr69mgiciojaBg0ciIqJmICQkBImJiYiOjsaVK1ewceNG7Ny5E1OmTFHU2bNnD4YOHYqcnBwAdxbZWbduHc6ePYvMzEzs3bsX7777Ljw9PeHn5wfgzuI4Y8eOxeLFi3Hw4EGcO3cOs2fPhrm5OYYPH66VvhIRUcvEaatERETNgLe3N6KiohAZGYnVq1fDwcEBERERStt0FBcXIy0tTbH6qp6eHn7//XesXLkSFRUVcHR0xODBgxEWFgYdnf/OD8+ZMwdisRizZ89GWVkZfHx8sHbtWu7xSERE9SISBEHQdhDNRWZmJgYOHIj4+Hg4OztrOxwiIqJmgfmRiIgATlslIiIiIiIiNXDwSERERERERHXi4JGIiIiIiIjqxMEjERERERER1YmDRyIiIiIiIqoTB49ERERERERUJw4eiYiIiIiIqE4cPBIREREREVGdOHgkIiIiIiKiOnHwSERERERERHXi4JGIiIiIiIjqpKvtAIjo0RUVFeHmzZuQyWTaDoWoWZBIJLCzs4OZmZm2QyEiLWJ+JFJlbGwMZ2dn6OjU/zoiB49ELVxRURFycnLg5OQEQ0NDiEQibYdEpFWCIKC8vBxZWVkAwAEkURvF/EikqqamBllZWcjLy4OdnV29j2/yaavx8fEICgqCh4cHhgwZgq1bt6p1XEJCAl566SX06NEDvr6+GD9+PAoLCxXPV1VVYf78+QgICICXlxdCQkJw+fLlxuoGUbNx8+ZNODk5wcjIiImRCIBIJIKRkRGcnJxw8+ZNbYdDRFrC/EikSkdHB/b29krjqHodr+F4Hio5ORnh4eEYMmQI4uLiMHHiRMydOxd79+596HH79u3DtGnT0KdPH2zZsgXbtm1DSEgIxGKxos7ChQvx66+/YsGCBdi8eTOMjIwQGhqK0tLSxu4WkVbJZDIYGhpqOwyiZsfQ0JBT1YjaMOZHotpJJBJUV1c36Ngmnba6fv16+Pn5ITw8HADg4uKC5ORkxMTEYNCgQbUeI5fLMW/ePISEhGDatGmK8k6dOin+XVJSgk2bNmHevHno27cvAGDRokUIDAzErl27MGbMmEbsFZH28YwqkSr+XhAR/w4QqXqU34smvfKYlJSEXr16KZX17t0bKSkpDzw7fObMGVy/fh22trYYP348AgIC8NJLLyEhIUFR5/Tp05DJZAgMDFSUmZiYwNfXFydPnqy13aKiImRmZir9ZGdna6CXRERERERErU+TXnnMy8uDtbW1UpmtrS1kMhkKCgpqvWkzIyMDALB8+XLMmjULbm5u2LVrFyZPnozt27ejW7duyMvLg0gkUmnbxsYGubm5tcYSGxuL6OhoDfWMiIiIiIiodWv2+zwKggAAGDt2LEaNGgU3NzfFIPLHH39scLuTJk1CfHy80s/GjRs1FTYRkZLExERIpVIcP368XsdlZmZCKpUiLi6ukSIjIiLSHubHlqVJrzza2NggPz9fqSwvLw+6urqwtLSs9RhbW1sAd+6PvJeLiwtu3LihaFcQBOTn5ytdvbz/8b3MzMy4fDsREREREZGamvTKo4+PD44cOaJUdujQIXh6ekIikdR6jLu7O/T19ZGWlqZUnp6eDicnJwBQHH/06FHF86WlpTh58iR8fX013AsiIiIiIqK2p0kHjyEhIUhMTER0dDSuXLmCjRs3YufOnZgyZYqizp49ezB06FDk5OQAuLPwTXBwMDZs2IDdu3fj6tWrWLFiBc6cOYNx48Yp6owdOxaLFy/GwYMHce7cOcyePRvm5uYYPnx4U3aRiFqA9957D4MHD0ZiYiJGjhwJLy8vPP/88zh16pRiz9innnoKAQEBWLhwIeRyueLYkydP4uWXX4a3tzd8fX0xbdo0XLlyRal9mUyGBQsWwN/fHz4+Pnj77bdx+/btWmP5/vvvMXz4cHh4eKBXr15YsGABqqqqGrP7REREtWJ+pLo06bRVb29vREVFITIyEqtXr4aDgwMiIiKUtukoLi5GWlqa0uqrM2bMgEQiweeff47S0lK4uroiJiYGUqlUUWfOnDkQi8WYPXs2ysrK4OPjg7Vr18LY2Lgpu0hELcTt27fxySef4NVXX4W5uTkWL16M119/HT179oSRkREWL16MpKQkREdHo3Pnzhg9ejTOnj2LSZMmwd3dHUuWLEFlZSWioqLw0ksv4ZdfflFMk1+2bBm+++47hIWFwcfHB/v27cOnn36qEsPChQuxYcMGTJ48GU8++STS09OxbNky5OTkYNmyZU39lhARETE/0sMJpJCRkSG4uroKGRkZ2g6FSG1nz57Vdggtzpw5cwRXV1fh9OnTirJ9+/YJrq6uwtSpU5XqjhgxQggNDRUEQRCmT58uPPXUU0Jpaani+czMTMHd3V348ssvBUEQhIKCAsHLy0tYuHChUjtvvPGG4OrqKhw7dkwQBEG4du2a0K1bN2HNmjVK9eLi4gRXV1fh4sWLgiD893dpx44dGup928LfD81gfqSWiL//9cf82HY09Pej2a+2SkTUGCwsLODh4aF43LFjRwBQ2Yu2U6dOimn0J06cwMCBA2FkZKR43snJCT4+PkhMTAQAXLhwARUVFRg8eLBSO08//bTS44SEBNTU1GD48OGorq5W/Nx9/X/++UczHSUiIqoH5kd6mCadtkpE1Fzcv9ry3UW7TE1NVcorKysBAEVFRbCxsVFpy8bGBufOnQMAxd6yte07e6+7K0/36dOn1vgedA8IERFRY2J+pIfh4JGISE1mZmbIy8tTKc/Ly4O5uTmA/7YXys/Px2OPPaZU515362/YsAGGhoYqbdrb22ssbiIiosbE/Nh2cPBIRKSmJ554Avv27UNFRQUMDAwAADdu3EBSUhJCQkIAAK6urjAwMMCePXvg4+OjOPaPP/5QaiswMBA6Ojq4efMmnn322SbrAxERkaYxP7YdHDwSEakpLCwMY8eORWhoKEJDQ1FVVYXly5fDxMQEEydOBHDnXpHg4GCsX78eRkZG6NGjB/bv34/Tp08rtdWhQweEhobio48+wsWLF+Hn5wexWIysrCzs378fH374oWIvWyIiouaM+bHt4OCRiEhNbm5uiI2NxdKlSzFjxgyIxWI8+eSTWLFihWIZcgB45513IJfLsWHDBqxZswZ9+/bF3Llz8frrryu1N2vWLHTq1Ak//PAD1q1bB4lEAicnJ/Tu3RsWFhZN3DtqDuLj4xEZGYm0tDQ4Ojritddew6hRo9Q6tqamBqGhoUhISMDSpUuV9jkeMGAAsrKylOoHBQVh8eLFGo2fiNom5se2QyQIgqDtIJqLzMxMDBw4EPHx8XB2dtZ2OERqSU1NRffu3bUdBlGz1JJ+P5KTkzF+/HiEhYVh2LBhSEhIwBdffIGoqCil/ZAfZNWqVUhOTsb+/ftrHTwGBQVhwoQJijIDAwOVBTAehPmRWqKW9PtP1NQa+vvBK49ERETNwPr16+Hn54fw8HAAgIuLC5KTkxETE1Pn4DExMRE//fQTfv75ZwQEBNRax9jYWLFgBRERUUNwn0ciIqJmICkpSWUftd69eyMlJQUymeyBx92+fRuzZs3CF198ASsrqwfWi42Nhb+/v2K6amlpaa31ioqKkJmZqfSTnZ3dsE4REVGrwiuPREREzUBeXp7K/me2traQyWQoKChQum/oXu+//z6GDh2KwMDAB7Y9YcIEdO/eHVZWVkhNTcXSpUuRmpqKNWvWqNSNjY1FdHT0o3WGiIhaJQ4eiYiIWqjvv/8eOTk5+Oqrrx5aLzQ0VPFvqVQKZ2dnBAcH4+zZs3Bzc1OqO2nSJIwYMUKpLDs7G8HBwZoLnIiIWiQOHomIiJoBGxsb5OfnK5Xl5eVBV1cXlpaWtR5z9OhRpKamokePHkrlM2fOxDfffIO4uLhaj/P29oZIJEJ6errK4NHMzAxmZmYN7wgREbVaHDwSERE1Az4+Pjhy5AimTp2qKDt06BA8PT0hkUhqPebDDz/E22+/rVQWFBSEmTNnPnSRndTUVAiCwAV0iIioXjh4JCIiagZCQkIwfvx4REdHK7bq2LlzJ6KiohR19uzZgyVLliA2Nhb29vZwdHSstS0HBwd06NABwJ2FeP7991/4+/vDzMwM586dw4IFC+Dp6Qk/P78m6RsREbUOHDwSERE1A97e3oiKikJkZCRWr14NBwcHREREKF1BLC4uRlpa2kNXX72fnp4efv/9d6xcuRIVFRVwdHTE4MGDERYWBh0dLrpORETq4+CRiIiomRg0aNBDp5uOHDkSI0eOfGgb58+fV3rs7u6OTZs2aSQ+IiJq23jKkYiokcjlcgQFBWH//v3aDqXe/v33X0ilUmRmZmo7FHzzzTcYMGCA4nFMTAxef/11LUZERESPijlSM5o6R/LKIxG1OMePH8fEiRMVj/X09ODo6IgxY8Zg8uTJivLly5c/cL+6M2fOQFdXF++99x5+/vlnAICOjg5sbW3x1FNPYcaMGZDJZBg4cOBDYxkxYgQWLFhQ63Pbt2+Hvr4++vfvX98u0kNMmDABa9euRWJiIh5//HFth0NE1KwwR7ZtjZ0jOXgkohbrl19+gZWVFSorK5GQkIBPPvkEdnZ2CAoKUtTp0KEDNm7cqHKsru5/f/78/f2xZMkS1NTU4OrVq5g3bx7eeustbNy4EYcPH1bU++2337B48WLs27dPUWZgYPDA+GJjY5USeG2qqqqgp6enVn/rqzHb1iYDAwMEBQUhNjaWg0ciogdgjnw45siG4bRVItKKEydOYOzYsfDx8YGfnx9GjhyJ06dPK55PT09HeHg4nnjiCXh7e+OFF17AsWPHlNqwsrKCra0tnJ2dMXr0aEilUpw5c0apzt0zpff/3EsikcDW1hb29vZ48sknMWbMGCQlJaG8vFzpGFNTUwCotex+qampuHjxospZWalUiu+//x5vvfUWfHx8MHfuXADAqVOnMGnSJPTo0QOBgYGYOXOm0p5/7733HiZPnoyYmBj06tULPXr0wFtvvYWioiKVOuvWrUO/fv3g5eUFAMjJycGbb74JPz8/9OjRA5MnT8bly5eV4vr1118xcOBAeHl54eWXX0ZGRobS89u3b4enp6dSWWZmJqRSKf7991+1P7e6+llTU4PFixfD398fvr6+eP/991FeXq7y/g4aNAj79u1DSUlJre8/EVFLxhzJHNlccyQHj0QEAPjrnwyEfvYnnpsRh9DP/sRf/2TUfVADVVdX4/XXX4evry927NiBrVu3YsqUKYq97G7evInx48ejvLwc33zzDX755ReEhYU9sD1BEJCYmIgrV67Ax8fnkWLLycnBH3/8AbFY/EgrUf79999wcnKCtbW1ynPLly/Hk08+ibi4OLz++uu4cOECJk2ahKeeegrbt2/Ht99+i4KCArz++usQBEFxXFJSEpKSkrBmzRqsXr0aZ86cwUcffaTU9smTJ5GUlITVq1djx44dEAQBYWFhyMrKwpo1a/Djjz8CAKZMmYLKykoAwOnTpzFr1iw899xziIuLw+jRo7Fw4cJ697muz02dfq5fvx4bN27E//73P2zbtg0ODg6IjY1VeS0PDw/U1NQgKSmp3nESEdVHU+ZHgDmSObJ550hOWyUi/PVPBqK3JKNSJgcA5BaUI3pLMgCgn197jb9eSUkJioqK0L9/f8VedJ06dVI8v3HjRujq6iI6Olox5eVuvXsNHjwYIpEIMpkMMpkMr7zyCp5++mmlOunp6SrJ0s3NTWmaTkJCAnx8fFBTU4OKigoAwMSJE2FkZNTgPmZmZsLOzq7W555++mkEBwcrHs+ZMweDBg1SSiJffvklAgICcPr0acXZUQBYuHAhTExMAAAfffQRpk6diuvXryv2+5NIJFi4cCEMDQ0BAEePHsXZs2fx+++/o2PHjgCAJUuWoH///vjll18wevRorF+/Ho8//jjeeustAHc+i7S0NKxcubJefa7rc1uzZk2d/Vy3bh1eeeUVPPfccwCAt956C3///Tdu3Lih9FqGhoYwNTVFVlZWvWIkIqqPps6PAHMkc2TzzpEcPBIRvtudqkiMd1XK5Phud2qjJEcLCwuMHDkSkydPRkBAAPz9/fH000/D2dkZAHD27Fn4+fk99F4J4M4ZOEtLS1RXV+PcuXP47LPPYGRkhDfffFNRx9nZGWvWrFE6Tl9fX+mxj48PvvjiC1RWVuK3337DkSNHMGPGjEfqY2Vlpcrr3HX/1JaUlBRcvXoVe/fuVal77do1RWJ0dXVVJEUA8PX1hSAIuHLliiIxdunSRZEUAeDSpUuwsbFRJEXgzvvftWtXxbScy5cvo3fv3kqv6+vrW4/e3lHX51ZXPzt16oSbN2+qfJHx9fXFrl27VI7R19dXfJEhImoMTZ0fAeZI5sjmnSM5eCQi5BWozpd/WLkmzJ8/H5MmTcKhQ4fw119/ITIyEsuWLXvoHnf3c3JyUtyb4eLigszMTCxfvhzTpk1T3ASvq6tb6xnZexkYGCjquLq64urVq4iIiMD8+fMb2DvA0tISp06dqvW5+8/W1tTUKL4o3K+2KT0Pc29S1CQdHR2l6UHAnalV9VFXP+9vvy6FhYWwsrKq1zFERPWhjfwIMEfeizmyeeVI3vNIRLCxrP2P6YPKNaVbt2549dVXsWHDBvTu3Rvbtm0DcGfKzMmTJ+t9xkwsFqO6uhpVVVWPFNf06dOxY8cOpcUJ6svd3R3p6elqJQ93d3dcuHABHTp0UPm59yzqxYsXUVpaqnh88uRJiEQidO7c+YFtd+nSBXl5eUhPT1eU3b59GxcvXkSXLl0A3PlScf99Efc/trKygkwmQ0FBgaLs7NmzSnXq+tzq6qepqSns7OzqjAW4cxa2srIS7u7uD+w7EdGj0lZ+BJgj763LHNl8ciQHj0SEic90h75ErFSmLxFj4jPdG+X1rl69iiVLluDkyZO4fv06/v77b6SmpsLFxQUA8NJLL6GyshLTp09HUlISMjIysHfvXpWV5G7duoXc3FzcuHEDf/31F2JjY/HUU08pJZOamhrk5uaq/MjlytOQ7uXi4oJ+/fohMjKywX309/eHXC5XK7lOnToV58+fx/vvv4+UlBRkZGTgyJEj+PDDD5VWShMEAe+99x4uXLiA48eP47PPPsOQIUMU03FqExAQADc3N8yYMQPJyclITU3FjBkzYGFhoViufdKkSUhMTMTy5cuRlpaGX3/9FVu2bFFqx8vLC8bGxli8eDGuXr2KAwcOYNWqVUp16vrc1OlnSEgI1q1bh19//RVpaWlYvny5SgIG7qxE6OTkpPg/Q0TUGJo6PwLMkfdjjmxeOZLTVolIcd/Gd7tTkVdQDhtLQ0x8pnuj3c9haGiIK1euYMeOHSgoKIC1tTWGDBmC8PBwAIC9vT1++OEHfPnll5g8eTJqamrQuXNnzJkzR6mduzeMi8Vi2NvbY8CAAYob2u+6evUqevXqpRLDn3/++dCpOlOmTMFLL72E48ePw9/fv959NDc3x7BhwxAXF1fn6nZdu3bFDz/8gMjISEyaNAnV1dVo164devXqpbQHlY+PDzw9PfHKK6+gpKQEffv2xaeffvrQtkUiEVatWoXPP/8cr7zyCmpqauDn54eYmBjF/SZeXl5YuHAhvvrqK3zzzTfw9vbGrFmzMGvWLEU7FhYWWLp0KRYuXIigoCC4u7tj5syZeO211xR16vrc1OlnSEgI8vPzMW/ePMhkMgwZMgSTJk1CXFycUr9++eUXjBkzRo1Pgoio4Zo6PwLMkfdjjmxeOVIk1HcCbSuWmZmJgQMHIj4+XnFTMlFzl5qaiu7dG+8MKDVcWloaxo4di927d9f7voz7vffee8jNzVVZ2KAtOnPmDF599VX8+eefSmfQa8PfD81gfqSWiL//zRtzZONQN0c29PeD01aJiBpJp06d8PHHHyMzM1PbobQqubm5WLRoUZ0DRyIiar6YIxtHY+dITlslImpEw4cP13YIrU6/fv20HQIREWkAc6TmNXaO5OCRiKgFWLBggbZDICIiapaYI5sOp60SERERERFRnTh4JCIiIiIiojpx8EhERERERER1avJ7HuPj4xEZGYm0tDQ4Ojritddew6hRox56zIABA5CVlaVUFhQUhMWLF9erDhERERERETVMkw4ek5OTER4ejrCwMAwbNgwJCQmYO3cuLCwsMGjQoIceO23aNEyYMEHx2MDAoEF1iIiIiIiIqP6adPC4fv16+Pn5ITw8HADg4uKC5ORkxMTE1Dl4NDY2hq2t7SPXuauoqAhFRUVKZdnZ2WodS0RERERE1NY06T2PSUlJ6NWrl1JZ7969kZKSAplM9tBjY2Nj4e/vr5iKWlpa2qA699YdOHCg0k9wcHDDOkZE9fbee+9h8uTJGm9XKpVi165dj9TGgAED8M0332gootbh5Zdfxty5c7UdBhFRm8Ac2bK0pRzZpFce8/LyYG1trVRma2sLmUyGgoIC2NnZ1XrchAkT0L17d1hZWSE1NRVLly5Famoq1qxZU68695o0aRJGjBihVJadnc0BJFELd/jwYZiZmalVd+7cuUhLS8OGDRuUyrdu3cpp7/dZvnw5dHW5NTARUUvGHNk42lKObBG9DA0NVfxbKpXC2dkZwcHBOHv2LNzc3NSucy8zMzO1f3mIqOVQd+r6w1hZWWkgkv9UVVVBT09Po23eSyaTQSKRNFr7AGBhYdGo7RMRUeNjjmwcbSlHNum0VRsbG+Tn5yuV5eXlQVdXF5aWlmq34+3tDZFIhPT09EeqQ0TNR1VVFRYuXIhevXrBw8MDzz//PPbv369U5/Tp03jxxRfh6emJYcOG4fDhw/D09MT27dsVde6fkrNixQoMGDAAHh4eCAwMxJtvvgngzlnCTZs24e+//4ZUKoVUKlW0c/+UnJKSEnz22Wfo06cPPDw8MGjQIHz33XcP7Mvd6Ubr1q1Dv3794OXlpWgnIiICvXr1go+PD0aPHo2jR48qHXvo0CEMGzYMnp6eGDlyJA4cOACpVIrjx48DAI4fPw6pVIoDBw5g7Nix8PDwwK+//goAiIuLQ1BQEDw9PTFo0CB89dVXqKqqUrS9Z88ePPfcc/Dy8sITTzyB8ePHK1apLi4uxuzZsxEQEABPT0+V9+D+KTnFxcX44IMP4O/vD09PT4wbNw5JSUmK5+/GmZCQgPHjx8PLywvPPvusSn+JiKhuzJF3MEdqX5NeefTx8cGRI0cwdepURdmhQ4fg6elZrzMCqampEAThoWdP1KlDRP8pTjmIgv0bUV2UD10za1j2D4apR58me/0lS5bgl19+waeffgoXFxds3boVb7zxBn7++WdIpVKUlZVh6tSp8PPzwxdffIGioiLMnz8f1dXVD2zzjz/+wJo1a7B06VK4urri1q1bSExMBHBntkJmZiauXr2K5cuXAwBMTU1V2hAEAa+99hry8/MRERGBLl26IDMzs84Ftk6ePAljY2OsXr1a0c60adOgq6uL5cuXw9raGvHx8Xjttdewbds2SKVS3LhxA6+//jpefPFFLF++HJmZmZg/f36t7S9cuBCzZ8+Gi4sL9PX1sXXrVnz55Zf46KOP4O3tjYyMDHz88ceoqqrCrFmzkJubi3feeQfvvvsuhgwZgvLycpw6dQoikQgAEBkZifPnz+Prr7+GtbU1srKykJOT88D+vf/++zh//jwiIyNhY2ODb7/9FpMnT8aff/4JGxsbpThnzpwJZ2dnrFy5Em+//Tb27dsHExOTh75/RETNhbbzI8AcyRzZfDTp4DEkJATjx49HdHS0YquOnTt3IioqSlFnz549WLJkCWJjY2Fvb4+kpCT8+++/8Pf3h5mZGc6dO4cFCxbA09MTfn5+AKBWHSJ6sOKUg8jbtRpCdSUAoLooD3m77vxBb4oEWVZWho0bN+Ljjz/G4MGDAQCzZ8/GyZMnERMTgy+//BK//vorKisrsWDBAhgbGwMA5syZg4kTJz6w3evXr8PW1haBgYGQSCRwdHSEh4cHgDurM+vr60MikTz0JFNCQgL++ecfxMXFoVu3bgCA9u3b19kniUSChQsXwtDQEABw7NgxnD59GkeOHFEkhVdeeQUJCQn46aef8PHHH+PHH39Eu3btMHfuXOjo6MDFxQX5+fl4//33Vdp//fXX0a9fP8XjFStWYObMmXj22WcVMb777rv44IMPMHPmTNy8eRMymQxPP/00nJycAABdu3ZVHJ+VlQU3NzfFGeC7dWqTnp6OPXv2YN26dQgICAAAfP755zhx4gS+//57vP3224q64eHhioXSZsyYgbi4OJw9exZPPvlkne8hEZG2aTs/AsyRzJHNS5MOHr29vREVFYXIyEisXr0aDg4OiIiIUNqmo7i4GGlpaYrVV/X09PD7779j5cqVqKiogKOjIwYPHoywsDDo6OioXYeIHqxg/0ZFYrxLqK5Ewf6NTZIcr127BplMpnKy5/HHH1dM4bh8+TK6du2qSIoA0KNHD8VZwdoMHToU3333HQYOHIjAwED06tULgwYNgr6+vtqxnTlzBtbW1oqkqK4uXbookiIApKSkoLKyEr1791aqV1VVhaeeegrAnT56enoq/d3q0aNHre3fTWAAcOvWLVy/fh2fffYZvvjiC0V5TU0NKioqkJubi27duqFnz5549tlnERgYiICAADz99NOKM6Djx4/HW2+9hZSUFPTs2RP9+vVTJL37Xb58GSKRSOnzkkgk6NGjBy5fvqxUt3v37op/310ULS8vr9Z2iYiaG23nR4A5kjmyeWnyBXMGDRr00D0dR44ciZEjRyoeu7u7Y9OmTQ9tU506RPRg1UX59SpvKdq1a4fff/8dCQkJSEhIwOLFixEVFYWtW7fWOv1Gk+5NisCdJGVhYVHr36qGrFp3b/s1NTUAgI8++ghPPPGESl0rKyuIxWKsXbsWycnJOHz4MLZv344lS5Zg/fr18PLyQt++fbFv3z4cOnQIx44dU5y1XbZsWb1ju9e9tyTc/RIjCMIjtUlE1FRaa34EmCPvYo6sH16WIyLomlnXq1zTOnToAIlEgn/++UepPDExEV26dAEAuLi44OLFi0r7tyYnJ9f5R1ZfXx/9+vXD+++/jx07duDq1as4duwYgDuzFuRy+UOPd3d3R35+Ps6dO9eQril4eHigoKAAcrkcHTp0UPqxt7dX9PH06dNKfUpOTq6zbRsbGzg4OODatWsqbXfo0EGxfLhIJEKPHj0wffp0bNu2DY899hh++eUXRTtWVlZ4/vnnMX/+fHz55Zf47bffUFhYqPJ6Xbp0gSAISp+XTCbDv//+q/i8qGHi4+MRFBQEDw8PDBkyBFu3blX72JqaGoSEhNS6j1tVVRXmz5+PgIAAeHl5ISQkROUMOBGp0nZ+BJgjmSObFw4eiQiW/YMh0lWepiLS1Ydl/6bZ99TQ0BATJkzA0qVLsXfvXly5cgWLFi3CqVOnFJskBwUFQV9fH++//z4uXLiAxMRELFq0CDo6Og+clrNlyxZs2bIF58+fR2ZmJrZv3w6RSISOHTsCAJydnXHlyhVcvHgRt27dUlp17a6AgAD4+Pjgrbfewv79+5GRkYETJ05gx44d9epjQEAA/P398cYbb2Dfvn3IyMjAqVOnEBMTgz///BPAnWkxN27cwLx583D58mUcOHBAsZrbw6YeAcDbb7+NdevW4ZtvvsHly5dx+fJl7N69G4sWLQJwZ3GCVatW4dSpU7h+/Tr++usvZGZmwsXFBQCwbNky7N27F+np6bh8+TL++OMP2Nra1rqlUYcOHTBkyBB8/PHHSEhIwKVLl/C///0PhYWF3Cv3ESQnJyM8PBxDhgxBXFwcJk6ciLlz52Lv3r1qHf/1118/8Az9woUL8euvv2LBggXYvHkzjIyMEBoaqvRFk4hUaTs/AsyRzJHNS4vY57ElqZbLIRLpQKzz8P/ERM3J3fs2tLma3LvvvguRSISPP/4YhYWFcHFxwYoVKyCVSgEARkZGWL16NSIiIjBy5Eg89thjmDNnDqZNm/bA+zPMzc2xZs0aLFy4ENXV1ejUqROWLVumuAl+5MiRSEhIwLhx41BSUoL58+crTZsH7iSkb7/9FosXL8aHH36IwsJCtGvXDpMmTapX/0QiEb7++mtERUVh3rx5yM3NhYWFBby8vBT3c7Rr1w4rVqzA/PnzsXnzZri6uuLtt9/Gm2++Wec9KCNGjICRkRG+/fZbREdHQ09PDx06dMCIESMA3Fkl759//sGGDRtQVFQEe3t7hISEYNy4cQDuTJ1ZtmwZsrKyIJFI4OnpiZiYmAcm5C+++ALz58/H22+/jbKyMri7u2PNmjVKq8hR/axfvx5+fn4IDw8HcOcse3JyMmJiYh56uwdw5wrETz/9hJ9//lnlPpySkhJs2rQJ8+bNQ9++fQEAixYtQmBgIHbt2oUxY8Y0TofuIwgCZNU10JOIm+T1iDShOeRHgDkSYI5sNgRSyMjIEFxdXYWMjIx6H1tcWikc+jdTmLP8oPDF+r+FlMt5glxe0whREik7e/astkPQmuTkZMHV1VVISUnRdiiN5rfffhOkUqlw69YtbYfSIrWk34++ffsKq1evViqLi4sT3N3dhaqqqgceV1BQIPTr1084fPiwIAiC4OrqKuzcuVPx/NGjRwVXV1chJydH6biQkBBhzpw5Ku0VFhYKGRkZSj8nTpxocH4UBEG4eqNQWPPLaeGdZX8JW+MvCDfyShrUDlF9tKTf/8bAHEkP09DfD1551JATqTlY+sNJxePjKTewcHpvSDtYajEqotYlLi4ODg4OcHJywtWrVzF//ny4ubnB3d1d26FpzA8//IDu3bvDxsYG58+fx4IFCzBo0CBYWvJvSWuXl5cHa2vl+6hsbW0hk8lQUFCgWInvfu+//z6GDh2KwMDAB7YrEolU2raxsUFubq5K/djYWERHRzewF7W8/u1yzFt7HNn5ZQCAixm3ceFaAd4Z7wsDfX4NIdIU5khqCvyrrQGl5TJsjb+oVCavEXD6ci4Hj0QaVFBQgKioKNy8eROWlpZ46qmnMGfOHG2HpVFZWVn4+uuvkZ+fDzs7OwwePBjvvPOOtsOiZur7779HTk4OvvrqK421OWnSJMVUrruys7MbfK9ORk6xYuB419HTNzBuSCk6OZo3OE4iUsYcSU2Bg0cN0NEBJLqqaw/pirkeEZEmhYSEICQkRNthNKpZs2Zh1qxZ2g6DtMDGxgb5+crL/+fl5UFXV/eBZ9WPHj2K1NRUlb3OZs6ciW+++QZxcXGwsbGBIAiKL1t33f/4LjMzs1oXgWio2tYAEIkAnToWuCCi+mGOpKbA0Y0GGOpLMG6IVKnMQE8Mry6t4KZYIiJqEj4+Pjhy5IhS2aFDh+Dp6am0F9i9PvzwQ8TFxWHHjh2KH+DO4DEqKgoAFMff3UwcAEpLS3Hy5En4+vo2Tmfu0d7BFF3bWyiVDX2qAxxsjBr9tYmISLN45VFDfFxt8dm0njicnAVzY3085dEOnZ0stB0WtRGCINS5TDVRWyM0842W7xcSEoLx48cjOjoaw4YNQ0JCAnbu3KkYBALAnj17sGTJEsTGxsLe3h6Ojo61tuXg4IAOHToAAExMTDB27FgsXrwYVlZWsLOzw/Lly2Fubo7hw4c3er8sTQ0wa8LjOHkuB+evFcBHagevLjbQl/ArCDU+5kciVY+SH/mXW0P09XTh3dUW3l1ttR0KtTESiQTl5eUwMuJZfKJ7lZeXP/CKXXPk7e2NqKgoREZGYvXq1XBwcEBERITSNh3FxcVIS0uDTCarV9tz5syBWCzG7NmzUVZWBh8fH6xduxbGxsaa7kat2tkYY3ivzmj8oSrRf5gfiWonk8mgq9uwYaBIaGmnZhtRZmYmBg4ciPj4eDg7O2s7HCK1FBUVIScnB05OTjA0NOQZVmrzBEFAeXk5srKyYG9vr9H799oq5kdqiZgfiVTV1NQgKysL+vr6D1zF+2F45ZGohbv7xfj69ev1vhpB1FpJJBIOHInaOOZHotoZGxvDxqZha7Nw8EjUCmh6dUQiIqLWgPmRSLO42ioRERERERHViYNHIiIiIiIiqhMHj0RERERERFQnDh6JiIiIiIioTlww5x5yuRwAkJ2dreVIiIioKTg4ODR4r6u2hPmRiKjtqS1HMmPeIzc3FwAQHBys5UiIiKgpcN9C9TA/EhG1PbXlSJEgCIKW4ml2KioqkJKSAltbW4jF4ga1kZ2djeDgYGzcuBEODg4ajrD5aCv9bK74/msX33/t0uT7zyuP6mF+VF9b6Wdzxfdfu/j+a5em339eeayDgYEBHn/8cY205eDg0CbOZreVfjZXfP+1i++/dvH9bzrMj/XXVvrZXPH91y6+/9rVmO8/F8whIiIiIiKiOnHwSERERERERHXi4JGIiIiIiIjqxMGjhpmZmWH69OkwMzPTdiiNqq30s7ni+69dfP+1i+9/y9RWPre20s/miu+/dvH9166meP+52ioRERERERHViVceiYiIiIiIqE4cPBIREREREVGdOHgkIiIiIiKiOnHwSERERERERHXi4LEe4uPjERQUBA8PDwwZMgRbt26t85hVq1Zh3Lhx8Pb2hqenZxNEWX/17df169fx0UcfYfDgwfDy8kL//v0xb948FBYWKtUbMGAApFKp0s/MmTMbsyutQkP+n/G9brj6vt/Hjx9Xea/v/uzevVtRj59Jw5w4cQLTpk1Dr169IJVKsWvXrjqPqaqqwvz58xEQEAAvLy+EhITg8uXLTRAt3as15kjmx+aHObLpMD82L80lP+o+0tFtSHJyMsLDwxEWFoZhw4YhISEBc+fOhYWFBQYNGvTA42QyGYYOHYoePXpg48aNTRixehrSr7S0NFRUVODDDz9Ep06dkJWVhU8++QTp6elYs2aNUt1p06ZhwoQJiscGBgaN2p+WrqH/zwC+1w3RkPfbx8cHhw8fVirbsGEDNmzYgD59+iiV8zOpv7KyMkilUrz44ouYPn26WscsXLgQu3fvxoIFC2Bvb4+oqCiEhobit99+g7GxcSNHTEDrzJHMj80Pc2TTYX5sfppNfhRILW+//bYwYcIEpbKZM2cKY8eOVev4bdu2CR4eHo0R2iN51H7dtXfvXkEqlQrFxcWKsv79+wtff/21RuJsKxr6efC9bhhN/f8fMmSI8OGHHyqV8TN5dK6ursLOnTsfWqe4uFhwd3cXtm/frlTm5eUlbNq0qbFDpP/XGnMk82PzwxzZdJgfmzdt5kdOW1VTUlISevXqpVTWu3dvpKSkQCaTaSmqR6epfhUXF0NPT0/lzFFsbCz8/f0RFBSExYsXo7S0VCNxt1aP8nnwva4/Tfz/P378ONLT0zFmzBiV5/iZNL7Tp09DJpMhMDBQUWZiYgJfX1+cPHlSi5G1La0xRzI/Nj/MkU2H+bHla6z8yGmrasrLy4O1tbVSma2tLWQyGQoKCmBnZ6elyB6NJvp169YtREVFYcyYMdDV/e+/1IQJE9C9e3dYWVkhNTUVS5cuRWpqqsrUHfpPQz8PvtcNo4n//5s3b0b37t1V7tfiZ9I08vLyIBKJVD5HGxsb5Obmaimqtqc15kjmx+aHObLpMD+2fI2VHzl4pEdSVFSEV199FR06dMDs2bOVngsNDVX8WyqVwtnZGcHBwTh79izc3NyaOtRWje+1dhQUFODPP//Ee++9p/IcPxOito35sfng+930mB9bL05bVZONjQ3y8/OVyvLy8qCrqwtLS0stRfXoHqVfBQUFCAkJgbm5OVatWgU9Pb2H1vf29oZIJEJ6evqjht1qaer/Gd9r9Tzq+x0XFwexWIznnnuuzrr8TBqHjY0NBEFQ+Rzz8/Nha2urpajantaYI5kfmx/myKbD/NjyNVZ+5OBRTT4+Pjhy5IhS2aFDh+Dp6QmJRKKlqB5dQ/uVl5eHiRMnwtLSEqtWrVJrlazU1FQIgsAvdA+hqf9nfK/V86jv95YtWzB06FCYmprWWZefSeO4+1kdPXpUUVZaWoqTJ0/C19dXi5G1La0xRzI/Nj/MkU2H+bHla6z8yMGjmkJCQpCYmIjo6GhcuXIFGzduxM6dOzFlyhRFnT179mDo0KHIyclRlF2/fh2pqam4fv06BEFAamoqUlNTcfv2bS30QlVD+pWTk4MJEybAyMgIn376KYqKipCbm4vc3FxUVVUBuHOj9bp163D27FlkZmZi7969ePfdd+Hp6Qk/Pz+t9LUlaMjnwfe64Rr6ew0AiYmJuHTpEkaPHq3SLj+ThistLVX8nQSArKwspKam4urVqwBUPw8TExOMHTsWixcvxsGDB3Hu3DnMnj0b5ubmGD58uNb60da0xhzJ/Nj8MEc2HebH5qfZ5McGr9PaBu3Zs0cYPny44O7uLgwcOFDYvHmz0vPbtm0TXF1dhYyMDEXZnDlzBFdXV5Wfbdu2NXX4D1Tfft19XNvPsWPHBEEQhJSUFGHMmDHC448/Lnh4eAhDhgwRFixYIBQWFjZ5/1qa+n4efK8fTUN+rwVBEGbPni0888wztbbJz6Thjh07VuvflrtLxtf2eVRWVgqff/654O/vL3h6egoTJ04ULl26pK0utFmtMUcyPzY/zJFNh/mxeWku+VEkCILwyENhIiIiIiIiatU4bZWIiIiIiIjqxMEjERERERER1YmDRyIiIiIiIqoTB49ERERERERUJw4eiYiIiIiIqE4cPBIREREREVGdOHikFm/t2rXo378/3Nzc8PzzzzfJax4/fhzR0dG1lkulUiQmJjZJHACQmpqK5cuXo6SkRKPtZmZmQiqVIi4urt7HvvzyywgJCdFoPA3l5uaG5cuX1+uYzMxMLF++HNevX2+kqIiIGh/zI/PjwzA/UkPoajsAokdx7tw5LFy4EJMnT8agQYNgbGzcJK/7999/Y9WqVZg+fbpSubu7OzZt2oQuXbo0SRzAneQYHR2N0aNHw8TERGPt2tnZYdOmTXjsscfqfezHH38MkUiksViaWlZWFqKjoxEQEABHR0dth0NEVG/Mj8yPjYH5kTh4pBbtypUrAIDx48ejffv2D6xXVVUFPT29Ro/HxMQEPXr0aPTXaaj6vA96enoN7ktTfjkgIiJVzI/1w/xIpB5OW6UW67333sM777wDABg0aBCkUimWL1+umE6yefNmzJs3DwEBAejZsycAIDk5GdOnT0fv3r3h7e2NYcOG4ZtvvkF1dbVS21VVVYiOjsbTTz8NDw8P9OrVC2+//TaKi4uxfPlyREdHQy6XQyqVQiqVYsCAAQBqn5YjCAK+/fZbDB48GB4eHujXrx+WLl2KqqoqRZ27xx04cAAffvghHn/8cQQGBuLTTz9FZWXlA9+D7du34/333wcA9O3bVxHPvW3Gx8dj5syZeOKJJzB69GgAwMGDBzFlyhQEBgaiR48eeO6557BlyxaltmublnN3us1ff/2FoKAgeHt7Y+TIkSrTkO6flrN9+3ZIpVKcPn0ab775Jnx8fNCvXz8sX74cNTU1Ssfu3bsXzzzzDDw9PfHCCy/g2LFjGDx4MN57770Hvg93bd26FQMHDoSXlxfGjRuH1NRUlTp19f348eOYOHEiACA4OFjxnmZmZgIAli1bhhdeeAE+Pj7o2bMnXn31VVy8eLHO2IiImgrzI/Pj/ZgfSVN45ZFarNdffx3Ozs6KZGVrawsHBwdFoouKioK/vz8WLFgAmUwG4M50Cw8PD4waNQqGhoaK+yFu376N2bNnK9p+8803cfjwYUyZMgVPPPEEiouLceDAAZSVlWH06NHIzs7Gzz//jB9++AEAHnq2cvHixVizZg1eeeUVBAYG4tSpU1i5ciWysrKwZMkSpbqffvophgwZgq+++gopKSmIjIyEpaUlwsPDa227X79+CAsLw6pVq7Bq1SpYWVmp1Pnkk0/w9NNPIzIyEnK5HACQkZGBXr16YeLEiZBIJPjnn38QERGBqqoqBAcHP/R9v3TpEpYuXYpp06bBxMQEUVFRmDZtGvbt2wczM7OHHjtz5ky88MILGDduHA4ePIjo6Gg4OTlh5MiRAICzZ8/izTffRM+ePTFnzhzk5eXhww8/RGFh4UPbBYC//voL//vf//Dss8/i+eefx5UrVxAeHg5BEJTq1dV3d3d3zJ07F59++ik+/fRTxZcNOzs7AEBubi5CQ0Nhb2+P4uJibNq0CePGjcPu3bsVdYiItIn5kfnxXsyPpFECUQu2Y8cOwdXVVcjIyFCUZWRkCK6urkJwcPBDj62pqRFkMpnw9ddfC35+foJcLhcEQRCOHDkiuLq6Cps3b37gsVFRUUL37t1Vyo8dOya4uroKJ06cEARBEG7duiW4u7sLn3zyiVK91atXC66ursKlS5eUjvv444+V6k2dOlV45plnHtqPbdu2Ca6ursKNGzdqjWXWrFkPPV4ulwsymUz4+OOPhWeffVZRfvd93LFjh6JswoQJgoeHh5CVlaUoS0lJEVxdXYVdu3Yp1Zs0aZJKjN98843Saz/77LPClClTFI/ffPNNoU+fPkJVVZWi7MCBA4Krq6swZ86ch/Zj9OjRwqhRo5TKYmNjBVdXVyEqKqpefb//c3yQ6upqoaKiQvDz8xPWrl370LpERE2J+ZH58S7mR9IkXnmkVqt///4qZUVFRVi5ciX+/PNP5OTkKE3Hyc/Ph62tLY4ePQqJRIIRI0Y8cgzJycmQyWR49tlnlcqHDx+OpUuXIjExES4uLoryPn36KNXr2rUrjh8//kgx3J0ydK+cnBxERUXh8OHDyM3NVZxxVed+j65duyrdJN+1a1cAwI0bN+o8trb+XbhwQfH49OnT6Nu3LyQSiaKsd+/e0NfXf2i7crkcZ86cwZtvvqlU/vTTT+Pzzz9XKnuUvgPA4cOHsWrVKly8eFHpjG9aWppaxxMRaRvz4x3Mj8yPVH8cPFKrZW1trVL2/vvv4/jx43jjjTfQrVs3mJiYYP/+/VixYoXi3onbt2/DysoKurqP/utx94+nra2tUvndx/dPN7l/Wouent5D7+lQh42NjdLjmpoahIWFIT8/H1OnTkXnzp1hbGyMzZs3Y/PmzXW2V1uMANSK09zcXOXYe4/Lzc1VmVokEolgaWn50HZv3bqF6upqlWPv/z/wqH0/ffo0pk6disDAQHz22WewsbGBRCJBWFiY0j06RETNGfPjHcyP/2F+JHVx8Eit1v1LYVdWVmL//v14++238corryjKDx48qFTP0tJS8cf2URPk3WSQm5urtKR3bm4uAMDCwuKR2m+Ia9eu4cyZM1i6dCmGDx+uKP/xxx+bPJb72dra4tatW0plgiCgoKDgocfd/TJz/7H5+flKjx+173v37oWenh5Wrlyp+L8hl8tx+/ZttY4nImoOmB9rx/zI/Eh142qr1GZUVVVBLpcrTfmQy+XYuXOnUr2ePXtCJpNhx44dD2xLT08PcrlcsdDAg3h7e0MikeC3335TKr/72M/Pr569qD0WQL0zmwBQUVEBAErvQ0lJCeLj4x85lkfl6emJAwcOKL2vBw8erLNvYrEY7u7u2Lt3r1L5H3/8ofRY3b7ffU/vP1taXl4OXV1dpS9eu3btqvP/ARFRc8b8eAfzI/Mj1Y1XHqnNMDU1hbe3N9asWQMbGxuYmJjg+++/V/nDFhAQgP79+yMiIgLXr1+Hn58fysrK8Ndff+HNN9+Evb294j6MdevW4amnnoK+vr5i1bF7WVpaYtKkSVizZg0MDAzQs2dPnD59GtHR0QgKClK6n6Oh7raxceNGDBs2DGKxGJ6eng+s37lzZ7Rr1w5LliyBjo4O5HI5vv76a5iammr9DOHUqVMxatQohIWFYcKECcjPz8fKlSthZmZW56bKr7/+OqZOnYoZM2YoVpP7/vvvoaPz3zkydfvesWNHiMVibN26FQYGBpBIJJBKpQgMDERsbCw+/PBDPPfcczh//jzWrl2rMt2IiKglYX68g/mR+ZHqxiuP1KYsWbIErq6umDt3Lj766CN069YNYWFhKvWioqIwefJkxMXFYerUqYiIiEBZWRmMjY0B3FlsYOLEifjuu+8wduzYWtu4a+bMmZgxYwb++OMPTJ06FT/99BNCQ0PxxRdfaKRP3bt3x9tvv409e/bgpZdewqhRox5aX09PDytWrIClpSVmzJiBBQsWYNiwYYrlwLXJzc0NUVFRyMrKwvTp07F+/XrMmzcPurq6MDU1feix/fr1w2effYakpCS88cYb+P333xEdHa2UVNXtu6WlJSIiIpCSkoKJEydi1KhRuHnzJvr27Yv3338fx44dw9SpUxWvweRIRC0d8yPzI/MjqUMkCPdt8kJE1IycP38ezz33HBYsWKCRFf6IiIhaA+ZH0gYOHomoWYmIiMATTzwBa2trXLt2DatWrUJNTQ1+++03GBkZaTs8IiIirWB+pOaA9zwSUbNSWlqK+fPno6CgAEZGRvD398esWbOYGImIqE1jfqTmgFceiYiIiIiIqE5cMIeIiIiIiIjqxMEjERERERER1YmDRyIiIiIiIqoTB49ERERERERUJw4eiYiIiIiIqE7/B+2vUiAwbkPfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.set(font_scale=1.3, style=\"ticks\")\n",
    "fig, axes = plt.subplots(1,2, figsize=[15,5])\n",
    "sns.scatterplot(data=all_res, x=\"fraction training data\", y=\"accuracy\", hue=\"model\", ax=axes[0])\n",
    "sns.scatterplot(data=all_res, x=\"fraction training data\", y=\"f1\", hue=\"model\", ax=axes[1])\n",
    "sns.despine()\n",
    "plt.setp(axes, xticks=[0.1, 0.25, 0.5, 0.75, 1.0], xticklabels = [0.1, 0.25, 0.5, 0.75, 1.0]);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>fraction training data</th>\n",
       "      <th>k</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>logistic regression</th>\n",
       "      <td>0.716435</td>\n",
       "      <td>0.599507</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scBERT (reproduced)</th>\n",
       "      <td>0.534770</td>\n",
       "      <td>0.397255</td>\n",
       "      <td>0.1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     accuracy        f1  fraction training data    k\n",
       "model                                                               \n",
       "logistic regression  0.716435  0.599507                     0.1  0.0\n",
       "scBERT (reproduced)  0.534770  0.397255                     0.1  NaN"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_res[all_res['fraction training data']==0.1].groupby('model').min('accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>fraction training data</th>\n",
       "      <th>k</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>logistic regression</th>\n",
       "      <td>0.719795</td>\n",
       "      <td>0.609187</td>\n",
       "      <td>0.1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scBERT (reproduced)</th>\n",
       "      <td>0.683419</td>\n",
       "      <td>0.546233</td>\n",
       "      <td>0.1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     accuracy        f1  fraction training data    k\n",
       "model                                                               \n",
       "logistic regression  0.719795  0.609187                     0.1  2.0\n",
       "scBERT (reproduced)  0.683419  0.546233                     0.1  NaN"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_res[all_res['fraction training data']==0.1].groupby('model').max('accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rna-pretrain",
   "language": "python",
   "name": "rna-pretrain"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "ea531a0049080d5658e3e61cf7db54ce403b028279b105adc411fb7eae04af57"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}