training/plot_roc_curves_2class.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.nn as nn\n",
    "import torchnet\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "from munch import Munch\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from torch.autograd import Variable\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pytorch 1.0.0 and torchvision 0.1.9\n",
    "import torchvision\n",
    "assert '0.1.9' in torchvision.__file__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://discuss.pytorch.org/t/inception3-runtimeerror-the-expanded-size-of-the-tensor-3-must-match-the-existing-size-864-at-non-singleton-dimension-3/32090\n",
    "from inception import inception_v3\n",
    "models.__dict__['inception_v3'] = inception_v3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pretrained alexnet_2class\n",
    "#config_path = 'pretrained/alexnet_2class/config.yml'\n",
    "#epoch = 45\n",
    "\n",
    "# pretrained vgg16_2class\n",
    "#config_path = 'pretrained/vgg16_2class/config.yml'\n",
    "#epoch = 12\n",
    "\n",
    "# pretrained inception_v3_2class\n",
    "#config_path = 'pretrained/inception_v3_2class/config.yml'\n",
    "#epoch = 7\n",
    "\n",
    "# pretrained resnet152_2class\n",
    "config_path = 'pretrained/resnet152_2class/config.yml'\n",
    "epoch = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#config_path = 'logs/your_log_dir/config.yml'\n",
    "#epoch = your_best_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(config_path, 'r') as f:\n",
    "    cfg = Munch.fromYAML(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> creating model 'resnet152'\n"
     ]
    }
   ],
   "source": [
    "model_names = sorted(name for name in models.__dict__\n",
    "    if name.islower() and not name.startswith(\"__\")\n",
    "    and callable(models.__dict__[name]))\n",
    "\n",
    "print(\"=> creating model '{}'\".format(cfg.arch.model))\n",
    "model = models.__dict__[cfg.arch.model]()\n",
    "\n",
    "if cfg.arch.model.startswith('alexnet') or cfg.arch.model.startswith('vgg'):\n",
    "    model.classifier._modules['6'] = nn.Linear(4096, cfg.arch.num_classes)\n",
    "elif cfg.arch.model == 'inception_v3':\n",
    "    model = models.inception_v3(transform_input=True)  # if pretrained is False then transform_input defaults to False\n",
    "    model.aux_logits = False\n",
    "    model.fc = nn.Linear(2048, cfg.arch.num_classes)\n",
    "elif cfg.arch.model == 'resnet152':\n",
    "    model.fc = nn.Linear(2048, cfg.arch.num_classes)\n",
    "else:\n",
    "    raise Exception\n",
    "\n",
    "if cfg.arch.model.startswith('alexnet') or cfg.arch.model.startswith('vgg'):\n",
    "    model.features = torch.nn.DataParallel(model.features)\n",
    "    model.cuda()\n",
    "else:\n",
    "    model = torch.nn.DataParallel(model).cuda()\n",
    "cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> loading checkpoint 'pretrained/resnet152_2class/checkpoint_00000005.pth.tar'\n",
      "=> loaded checkpoint 'pretrained/resnet152_2class/checkpoint_00000005.pth.tar' (epoch 5)\n"
     ]
    }
   ],
   "source": [
    "resume_path = cfg.training.resume.replace(cfg.training.resume[-16:-8], '{:08}'.format(epoch))\n",
    "if os.path.isfile(resume_path):\n",
    "    print(\"=> loading checkpoint '{}'\".format(resume_path))\n",
    "    checkpoint = torch.load(resume_path)\n",
    "    start_epoch = checkpoint['epoch']\n",
    "    model.load_state_dict(checkpoint['state_dict'])\n",
    "    model.eval()\n",
    "    print(\"=> loaded checkpoint '{}' (epoch {})\".format(resume_path, checkpoint['epoch']))\n",
    "else:\n",
    "    print(\"=> no checkpoint found at '{}'\".format(resume_path))\n",
    "    print('')\n",
    "    raise Exception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "val_transforms = []\n",
    "if cfg.arch.model == 'inception_v3':\n",
    "    val_transforms.append(transforms.Scale(299))\n",
    "val_dataset = datasets.DDSM(cfg.data.root, 'val', transforms.Compose(val_transforms + [\n",
    "    transforms.ToTensor(),\n",
    "    normalize,\n",
    "]))\n",
    "val_loader = torch.utils.data.DataLoader(\n",
    "    val_dataset, batch_size=cfg.data.batch_size, shuffle=False,\n",
    "    num_workers=cfg.data.workers, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0814ac983f42427f85d2e7921fd56faf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1395), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "class 0\n",
      "torchnet.meter.AUCMeter: 0.8757150899236701\n",
      "sklearn.metrics.roc_auc_score: 0.8757150468388264\n",
      "\n",
      "class 1\n",
      "torchnet.meter.AUCMeter: 0.8757151545509355\n",
      "sklearn.metrics.roc_auc_score: 0.8757151437797249\n",
      "\n"
     ]
    }
   ],
   "source": [
    "targets = [[] for _ in range(cfg.arch.num_classes)]\n",
    "probs = [[] for _ in range(cfg.arch.num_classes)]\n",
    "aucs = [torchnet.meter.AUCMeter() for _ in range(cfg.arch.num_classes)]\n",
    "\n",
    "with torch.no_grad():\n",
    "    for input, target in tqdm(val_loader):\n",
    "        output = model(input)\n",
    "        prob = nn.Softmax(dim=1)(output)\n",
    "        for i in range(cfg.arch.num_classes):\n",
    "            aucs[i].add(prob[:, i], target == i)\n",
    "            targets[i].extend(target.numpy() == i)\n",
    "            probs[i].extend(prob[:, i].cpu().numpy())\n",
    "\n",
    "for i in range(cfg.arch.num_classes):\n",
    "    print('class {}'.format(i))\n",
    "    print('torchnet.meter.AUCMeter: {}'.format(aucs[i].value()[0]))\n",
    "    print('sklearn.metrics.roc_auc_score: {}'.format(roc_auc_score(targets[i], probs[i])))\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_roc_curve(y_true, y_score):\n",
    "    plt.figure(figsize=(8, 8))\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.ylim([0.0, 1.0])\n",
    "    plt.gca().set_aspect('equal')\n",
    "    plt.xlabel('False positive rate')\n",
    "    plt.ylabel('True positive rate')\n",
    "    plt.title('ROC Curve')\n",
    "    fpr, tpr, _ = roc_curve(y_true, y_score)\n",
    "    plt.plot(fpr, tpr, color='black')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class 0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe4AAAHwCAYAAABgy4y9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XuYXvO9///nOzM5kYREQpxyIkhQp3GoY2yl1LGlxLHNj6p2F9+2u3XaSrurira03XaxsbXq3CqpzUZp61BBIk4JItKWCCEkRE4zmfn8/rjvjMlhkjsxa9ase56P65or91r3uu/1mnWRV9a61/35REoJSZJUDF3yDiBJkipncUuSVCAWtyRJBWJxS5JUIBa3JEkFYnFLklQgFrckSQVicUsdUET8IyIWRMRHEfF2RNwQEb2W2Wb3iHg4IuZGxAcR8ceIGLnMNn0i4oqIeL38Xq+Vl/u3st+IiDMi4sWImBcR0yPijojYNsvfV1LlLG6p4zo0pdQL2B7YAThnyRMR8WngAeBuYCNgKPAc8HhEDCtv0w14CNgaOBDoA3waeA/YpZV9/hw4EzgD6AdsAdwFHLy64SOidnVfI2nVwpHTpI4nIv4BnJJS+lN5+VJg65TSweXlR4EXUkpfX+Z19wHvppROiohTgIuAzVJKH1Wwz+HAy8CnU0pPtbLNX4DfppSuLS9/uZxzz/JyAr4B/D+gFvg/YF5K6d9avMfdwF9TSj+LiI2AXwJ7Ax8Bl6eUflHBIZI6Lc+4pQ4uIjYBDgKmlpfXAnYH7ljB5rcD+5cffwb4v0pKu2w/YHprpb0ajgB2BUYCtwDHREQARERf4ADg1ojoAvyR0pWCjcv7/38R8dlPuH+pqlncUsd1V0TMBd4A3gEuKK/vR+n/3bdW8Jq3gCWfX6/XyjatWd3tW3NxSun9lNIC4FEgAXuVnzsKeCKlNAPYGRiQUvpBSqk+pTQN+G9gdBtkkKqWxS11XEeklHoDo4Ct+LiQZwNNwIYreM2GwKzy4/da2aY1q7t9a95Y8iCVPou7FTi2vOo44Kby48HARhExZ8kPcC6wQRtkkKqWxS11cCmlvwI3AD8pL88DngC+uILNj6Z0QxrAn4DPRsTaFe7qIWCTiKhbyTbzgLVaLA9cUeRllm8BjoqIwZQuof++vP4N4O8ppXVb/PROKX2uwrxSp2RxS8VwBbB/RGxXXj4b+FL5q1u9I6JvRPyQ0l3j3y9vcyOlcvx9RGwVEV0iYr2IODcilivHlNKrwH8Bt0TEqIjoFhE9ImJ0RJxd3uxZ4AsRsVZEbA6cvKrgKaWJlK4CXAvcn1KaU37qKWBuRJwVET0joiYitomIndfkAEmdhcUtFUBK6V3gN8D3ysuPAZ8FvkDpc+l/UvrK2J7lAialtIjSDWovAw8CH1Iqy/7Ak63s6gzgP4ErgTnAa8DnKd1EBnA5UA/MBH7Nx5e9V+XmcpabW/xOjcAhlL7u9nc+Lvd1KnxPqVPy62CSJBWIZ9ySJBVIZsUdEddHxDsR8WIrz0dE/CIipkbE8xGxY1ZZJEmqFlmecd9AaZjF1hwEDC//nAr8KsMskiRVhcyKO6X0CPD+SjY5HPhNKhkHrBsRbfEdUkmSqlaen3FvTIuBGoDp5XWSJKkVhZi9JyJOpXQ5nbXXXnunrbbaKudEkpSfxYsXU19fT0NDA01NTSxatIiIaH5cU1MDQEqJJd8cSimxYMECamtrWbBgAY2NjTQ1NbV79vKw9cv92djYSG1t7XLrV/W4qamJlBJdu3ZtdV9tuS6lRENDA926dVvxL1ihOXPmzEopDViT1+ZZ3G8Cm7ZY3qS8bjkppWuAawDq6urS+PHjs08nSW2kqamJWbNm8cEHHzBr1iwWLlzI4sWLaWhoYPr06c1FvHDhQqZNm0afPn2YMGEC6667LvX19bz44ouss846vP/++7z55gr/mlxO3759qampoaamhtraWmpqaujduzezZ89mjz32YP78+ay//vrssMMOdOvWje7duzN79mwGDRrU/LolP/Pnz2fTTTela9euyz1XU1NDRLDOOutQW1tLbW1t83a9e/du3veKirAzi4h/rulr8yzuscA3IuJWSsMgfpBSaosJDiQpU4sXL+add95h/vz5vPzyy3zwwQfMmDGD6dOn8/TTT7PWWmsxdepUUkq8/vrra7SPfv36sWDBArbeems23nhj3n//ffbee2/q6+uJCPbee2+23nprevTowcCBA1lvvfXo2bMnXbt2tSSrXGbFHRG3UJocoX9ETKc0s1FXgJTSVcC9wOcoTVU4HxiTVRZJWpnXX3+diRMnMm/ePKZNm8a0adP48MMPee+996ipqaG+vp633nqLadOm0a1bNxYuXLjS96utreVf/uVfmDt3LgcddBD19fVsuummbLTRRvTvX5orZuDAgc1nqOussw79+/ene/fudO/enS5dHGJDrcusuFNKx67i+QT8a1b7l9S5ffjhh8yfP5+33nqLKVOm0NDQwMSJE3n99dfp3bs3EydOZPLkyc2fWa5Inz59GD58OH369GGzzTZj4MCBbLnllmywwQY0Njay5H6bnXbaiV69etG/f3969erlGa8yVYib0yRpRVJKfPjhh7z00ktceOGFDBgwgL///e88/vjjq3zt0KFDWW+99dhjjz0YNmwYG2+8MQcccABrrbUW/fr1o1evXu3wG0irz+KWVBhvvPEGDz74IPfffz8PPPAAc+bMWW6bESNGMHjwYAYPHszo0aNZuHAhQ4cOZauttmK99dZjwIA1upFX6jAsbkkdSkqJV199lWeffZbnn3+eW265hWnTpq1w20MPPZTNNtuMQYMGMXLkSA444AAvU6vqWdyS2lVKiXfffZe3336bd955h8cff5xnnnmG2bNn8+ijj67wNeuttx6HHXYYAwcOZPjw4ey///6sv/76n/i7tFIRWdySMpFS4oMPPuDRRx/lhz/8IRHBhAkTWLx4cauvGTFiBAMGDGCvvfZil112YejQoYwcObJ5QBFJFrekNjJnzhz+9Kc/cfXVVzNu3Dg++uijpZ7v378/hx9+OCklBg8ezOabb85mm21G9+7d2WOPPVY48pWk5Vnckj6Rv/71r3zlK1/h1VdfXWr9/vvvz/Dhw9lss83Ybbfd2H333XNKKFUXi1vSaqmvr+ftt9/m6aef5vTTT+ettz4e8PBHP/oRhx56KFtvvbU3iUkZsbglrVJDQwMXXHABl1xyyQonprj33ns56KCDckgmdT4Wt6SVmjFjBhtv/PGMu5tuuiknnXQSW2yxBXvttRdDhgzx7FpqRxa3pOUsWrSIcePG8Yc//IGf//znAAwaNIiHHnqIzTffPOd0UudmcUuioaGBBx54gOeff55zzz13ueeHDx/O5MmTqa31rwwpb/5fKHVSs2bN4u9//zsPPvgg55133lLPrbvuunz729/ms5/9LDvvvHNOCSWtiMUtdSIPP/ww3/3ud5kxY8ZSd4MD7Lnnnlx55ZVLTT0pqeOxuKUqlVJi8eLFvPrqqzz99NN8+ctfbn5u7bXXZtddd+XQQw9lt912Y9iwYQwdOjS/sJIqZnFLVWTx4sV89atf5frrr291m6eeesrL31KBWdxSlTjvvPP40Y9+1Lw8YsQIPvOZz7Deeuux2267MXz4cIYNG5ZjQkltweKWCmzBggW8+OKL7LfffsydOxeAk046iSuvvJJevXrlnE5SFixuqWAaGhq4+uqr+fnPf87UqVOXem7cuHHsuuuuOSWT1B4sbqkAUkrceeedXHfdddx3333N62tra/nOd77DNttsw7HHHusIZlInYHFLHdTbb7/NWWedxW9/+9vlxgc/5JBDuPHGG1l33XVzSicpLxa31IGklPjWt77FzTffzDvvvNO8ftttt2XgwIH813/9l0OOSp2cxS11APPnz+fUU0/lpptual7Xr18/vvrVr/L973+frl275phOUkdicUs5O+OMM/jlL3/ZvHzQQQfx29/+ln79+uWYSlJHZXFLOfnf//1fDjnkkOblM844g3PPPZcNNtggx1SSOjqLW2pH8+fP5/bbb2fMmDFLrX/55ZfZcsstc0olqUgsbiljs2fPZu+99+b9999nxowZSz03YcIEdtxxx5ySSSoii1tqY3/961+54YYbGDt2LB988AGNjY3Nzx155JFssMEGnH/++QwcODDHlJKKyuKW2sjYsWM5/vjj+eijj5rX9e7dm3/5l39h5513Xm7Oa0laExa39Am88cYbHHfccbz66qvMnDmzef2tt97KMccck2MySdWqS94BpCK6/vrriQgGDRrEY489xsyZM6mrq2Ps2LGklCxtSZnxjFtaTSeffHLzfNd77bUXY8aM4fjjj6dbt245J5PUGVjcUgUWLlzIL3/5S+677z7+/Oc/AzBp0iRGjhyZczJJnY3FLa1ESokrr7yS008/fan1v/vd7yxtSbmwuKVWPPjggxxwwAHNyyeeeCLXXnutl8Ql5crillbgiCOO4O677wagV69evPjiiwwePDjnVJLkXeXSUhobG9l1112bS/sPf/gDc+fOtbQldRgWt1R24YUXUltby1NPPQXAK6+8whFHHJFzKklamsUtURr17Pvf/z4Axx13HG+++SZbbLFFzqkkaXkWtzqllBIvvPACRx99NBHB4YcfDsCdd97JTTfdxEYbbZRzQklaMW9OU6fz7LPPssMOOyy1btSoUXz3u9/loIMOyimVJFXG4lanc/HFFwOw5ZZbctVVVzFq1Kh8A0nSarC41WlMnTqVI488kueff561116bl19+Oe9IkrTaLG51CvPmzWPHHXdk7ty5AJx55pk5J5KkNWNxq6rNmzeP0aNHc8899wCwxRZb8Morr+ScSpLWnHeVq6pdfvnlzaV90kknMW7cuJwTSdIn4xm3qta4ceM4//zzgdLsXt27d885kSR9cp5xqyq99NJLfPrTnwZKA6pY2pKqhcWtqnPCCSc0T7k5ZswYbrrpppwTSVLbsbhVFWbMmMF+++1HRDQX9QUXXMD111+fczJJalt+xq2qsO+++zJlyhQA9ttvP6699lqGDBmSbyhJyoDFrcK75JJLmkt78eLF1NTU5JxIkrLjpXIV2n//939z9tlnA/DCCy9Y2pKqnsWtwrrllls49dRTAbjmmmvYZpttck4kSdnzUrkKZ9GiRZx00kncfvvtQKnAR48enXMqSWofFrcKZd68efTq1at5+bbbbuPoo4/OMZEktS+LW4Xxz3/+c6k7xWfNmsV6662XXyBJyoGfcaswdt11VwAOPfRQmpqaLG1JnZLFrUL46KOPmDlzJgB33303EZFzIknKh8WtQvjCF74AwEUXXWRpS+rU/IxbHdpLL73Efvvtx1tvvQXAWWedlXMiScqXZ9zqsN555x1GjhzJW2+9xbBhw3j88ccdYEVSp+cZtzqsT33qU81/PvfcczmnkaSOwTNudUivv/56881oTz75ZM5pJKnjsLjV4TQ2NrLnnnsC8Jvf/IYePXrknEiSOg6LWx1KfX09o0aN4o033gDguOOOyzmRJHUsFrc6jO985zt0796dxx57DIApU6Z4M5okLcPiVq4aGxt5/PHH2W677fjJT34CwIknnkhDQwPDhw/POZ0kdTzeVa5cpJS49dZbl7sU/vDDD7PvvvvmlEqSOj6LW+3uww8/pF+/fjQ2NgKwzz77cMEFFzBq1ChHRZOkVbC41a6ampoYPXp0c2m/9tprDBs2LOdUklQcFrfaTX19PX369GHRokUALFq0iG7duuWcSpKKxZvT1C7OOeccunfvzqJFixg4cCCzZs2ytCVpDVjcytwbb7zBj3/8YwAOP/xwJk+e7FzakrSGLG5lqqmpiUGDBgFw+eWXc9ddd9G3b9+cU0lScVncysxjjz3WPIDKoEGDOO2003JOJEnFZ3ErE1OmTGGvvfYCYL311mPSpEmOOS5JbcDiVpt76qmn2HLLLQH42te+xqxZs+jVq1fOqSSpOmRa3BFxYES8EhFTI+LsFTw/KCL+HBETI+L5iPhclnmUnUWLFvH73/+evfbai1133bV5/RVXXJFjKkmqPpl9jzsiaoArgf2B6cDTETE2pTS5xWb/DtyeUvpVRIwE7gWGZJVJ2Xj66afZZZddllp3xx13cNRRR+WUSJKqV5Zn3LsAU1NK01JK9cCtwOHLbJOAPuXH6wAzMsyjDEyZMqW5tA855BCeffZZUkqWtiRlJMvi3hh4o8Xy9PK6li4EToiI6ZTOtk/PMI/a2I9//OPmz7IPPPBAxo4dy3bbbZdzKkmqbnnfnHYscENKaRPgc8CNEbFcpog4NSLGR8T4d999t91DanlNTU2cc845APz85z/nvvvuc4IQSWoHWRb3m8CmLZY3Ka9r6WTgdoCU0hNAD6D/sm+UUrompVSXUqobMGBARnFVqQkTJjR/P3uPPfbgjDPOyDmRJHUeWRb308DwiBgaEd2A0cDYZbZ5HdgPICJGUCpuT6k7qFdffZW+fftSV1fXvO6ee+7JMZEkdT6ZFXdKaTHwDeB+4CVKd49PiogfRMRh5c2+DXwlIp4DbgG+nFJKWWXSmpszZw5bbLEFc+bMAeC2226jqamJddddN+dkktS5ZDqtZ0rpXko3nbVc970WjycDe2SZQW3ja1/7GgBHHnkkd9xxh59nS1JO8r45TQVw4403cuuttwKlM21LW5LyY3FrlX74wx8C8N3vfrf5pjRJUj4sbq3URx99xJQpUzjmmGO45JJL8o4jSZ2exa1WNTQ0NI+K5iQhktQxZHpzmoqrsbGRtddem4aGBgAuuuiinBNJksDiViseeeSR5tJetGgR3bp1yzmRJAm8VK5WzJ8/H4BHH33U0pakDsTi1gpde+21APTo0SPnJJKklrxUrqWklLj44ou56667APjUpz6VcyJJUksWt5rddtttjB49unn59NNP9zK5JHUwXioXAFdccUVzae+///68/fbb/OIXv8g5lSRpWRa3+P3vf883v/lNoDS39gMPPMAGG2yQcypJ0opY3J3cK6+8wlFHHQXAZZdd5tzaktTBWdyd2Msvv8xWW20FwMUXX8y//du/5ZxIkrQqFncnNXv2bEaMGAHAnnvuydlnn51zIklSJSzuTui6666jX79+AGy00UY8+uijOSeSJFXK4u5kJk6cyCmnnALAbrvtxptvvplzIknS6rC4O5HHH3+cHXfcEYD777+fJ554IudEkqTVZXF3El/84hfZc889Afjc5z7HAQcckHMiSdKacOS0Kjdz5kwGDhzYvHzLLbcsNTqaJKlYLO4qt6S0u3fvzowZM5pvSpMkFZPFXaVSSkt9L3v+/Pl06eInI5JUdBZ3FXrkkUfYZ599mpenTJliaUtSlfBv8yrz3nvvNZf20KFD+cc//sHw4cNzTiVJaisWd5VZcnn8sMMOY9q0aQwePDjnRJKkthQppbwzrJa6uro0fvz4vGN0SPX19XTv3p2amhoWLlxIba2fhEhSRxQRE1JKdWvyWs+4q8SkSZPYb7/9ADj00EMtbUmqUv7tXiVOPvlknnzySbp168YFF1yQdxxJUkYs7iowY8YMnnzySdZff31mzpyZdxxJUoa8VF5wM2bMYOONNwbgX//1X3NOI0nKmsVdcMcccwwAI0aMWGrAFUlSdfJSeYHNmjWLxx57jK5du/Lcc8/RtWvXvCNJkjLmGXdBLV68mCuvvBKAs846y9KWpE7CM+6C+uY3v8l//ud/ArDvvvvmnEaS1F4s7gI699xzm0t78uTJjBgxIudEkqT24qXygnnkkUe4+OKLAbjjjjssbUnqZCzuAvnxj3/cPIHI3XffzVFHHZVzIklSe/NSeUGcddZZXHrppQCcccYZHHbYYTknkiTlweLu4BobG+nbty9z584F4LHHHmOPPfbIOZUkKS8Wdwd3zjnnNJf2q6++yuabb55zIklSnvyMuwNLKXHZZZcBpbvHLW1JksXdQaWUGDJkCADHH3+8d49LkgCLu8OaMWMGr7/+OkDzTWmSJFncHdRnP/tZAG644QY22mijnNNIkjoKi7uDevnllwE44ogjck4iSepIvKu8g2lsbGTs2LE0NjZy2mmnsc466+QdSZLUgVjcHcj48ePZeeedm5ePO+64HNNIkjoiL5V3EJMmTWou7Z122olrrrmGvfbaK+dUkqSOxjPuDmLJ97XHjBnD9ddfn3MaSVJHFSmlvDOslrq6ujR+/Pi8Y7SpqVOnMnz4cNZaay3mzZuXdxxJUsYiYkJKqW5NXuul8pw1NjYyfPhwAA499NCc00iSOjqLO2cnnHACAD179uTWW2/NOY0kqaOzuHM0efLk5rKeOXNmzmkkSUVgcefoxBNPBOCnP/0pvXv3zjmNJKkILO6cTJw4kWeeeQaAM888M+c0kqSisLhzcvzxxwNw5513UlNTk3MaSVJRWNztaNGiRVxwwQVEBC+99BIAhxxySM6pJElFYnG3o8suu4wf/OAHAHTp0oUXXniBrl275pxKklQkjpzWTh544AHOP/98AGbPns26666bcyJJUhF5xt0OjjjiiOb5tbfaaitLW5K0xizujC1YsIC7774bgHHjxjV/ti1J0pqwuDP205/+FIAf/ehH7LrrrjmnkSQVncWdsRtuuAGA0047Ld8gkqSqYHFnaO7cubz22mvssMMO9O3bN+84kqQqYHFn5I9//CN9+vQBYPvtt885jSSpWljcGRkzZgwAp556KpdffnnOaSRJ1cLvcWfgxRdf5L333mPkyJFcffXVeceRJFURz7jbWFNTE9tuuy3w8VzbkiS1FYu7jf3+978HYJNNNuGcc87JOY0kqdpY3G1s6tSpADz00EM5J5EkVSOLu40tuRFt8ODBOSeRJFUji7sNTZ8+nXfffZeuXbvSvXv3vONIkqqQxd2Gvv71rwM0T90pSVJbi5RS3hlWS11dXRo/fnzeMZazcOFCevbsCUB9fb3zbEuSWhURE1JKdWvyWs+428ill14KwDbbbGNpS5IyY3G3kQ8//BCAZ555JuckkqRqZnG3gfr6+ubpOz3bliRlyeJuA/fffz/gZCKSpOxlWtwRcWBEvBIRUyPi7Fa2OToiJkfEpIi4Ocs8WZk4cSIAv/71r3NOIkmqdplNMhIRNcCVwP7AdODpiBibUprcYpvhwDnAHiml2RGxflZ5snLHHXdwwQUXADBgwICc00iSql2WZ9y7AFNTStNSSvXArcDhy2zzFeDKlNJsgJTSOxnmycTRRx8NwLnnnsuGG26YcxpJUrXLsrg3Bt5osTy9vK6lLYAtIuLxiBgXEQdmmCczJ554IhdddFHeMSRJnUDe83HXAsOBUcAmwCMRsW1KaU7LjSLiVOBUgEGDBrV3xlb95S9/AaB///75BpEkdRqrPOOOiJ4RcU5EXFVe3jwiDqrgvd8ENm2xvEl5XUvTgbEppYaU0t+BKZSKfCkppWtSSnUppbqO8jlyfX09++67LwBHHnlkzmkkSZ1FJZfKrwcC2LO8PAP4UQWvexoYHhFDI6IbMBoYu8w2d1E62yYi+lO6dD6tgvfO3YUXXghAz5492X333fMNI0nqNCop7uEppR8BDQAppfmUinylUkqLgW8A9wMvAbenlCZFxA8i4rDyZvcD70XEZODPwHdSSu+twe/R7i6++GIAnnjiCSJWeTgkSWoTlXzGXR8RPYAEEBFDgfpK3jyldC9w7zLrvtficQK+Vf4pjPfeK/3b4vTTT2e77bbLOY0kqTOp5Iz7P4D/AzaJiF9TOjM+N9NUHdgbb7zRfDNaR7pRTpLUOazyjDuldF9EjAd2p3SJ/DtF/L51W5k+fToARx11FN/4xjdyTiNJ6mwquav8gZTSuymlu1NKd6WU3omIB9ojXEd2yimn0KNHj7xjSJI6mVbPuMt3gvcANoiI3nx8Q1ofwGvEkiTlYGWXyv+V0k1j6wOT+Li4PwSuyjhXhzV27LLfaJMkqf20WtwppcuByyPi/6WUrmjHTB3S/Pnz6d27N01NTQBsueWWOSeSJHVGldycdkVEbAWMpHTpfMn6Qk7BuaaOPfbY5tJ+6qmnGDJkSL6BJEmd0iqLOyL+HTgA2IrSgCmfBR4DOlVxP/jggwA0NDRQW5v3EO+SpM6qku9xHwPsC7yVUjoR2A5YO9NUHczs2bNZsGABG264oaUtScpVJcW9IKXUCCwu313+NjA421gdS2NjIwDnnHNOzkkkSZ1dJaePEyNiXUqTjYyndFf5U5mm6qAck1ySlLeVFneUmurC8vzYV0bE/UCflNIz7ZKug3jooYfyjiBJErCK4k4ppYh4ENimvDy1XVJ1MA88UBoo7sADD8w5iSSps6vkM+5nI2KHzJN0YDffXLqBftiwYTknkSR1dpV8xr0D8HREvAbMozSCWkop7Zhpsg5knXXWYc8996RLl0r+nSNJUnYqKe7DMk/RwdXU1DjgiiSpQ6hk5LTX2iOIJElaNa/9rsKkSZOYMWMGixcvzjuKJEkW96pMmzYNgH322SfnJJIkVVjcEbFJROxbftw9IjrNkKeHHVb6iH/77bfPOYkkSRUUd0T8f8BY4NryqsHA3VmG6ihmzJjR/HjbbbfNMYkkSSWVnHGfAexGaahTUkpTgPWzDNVR/OlPfwLg2muvpaamJuc0kiRVVtwLU0r1SxYioobSd7mr3te//nUARo0alW8QSZLKKinuxyPiu0CP8ufctwH3ZBur4+jTpw+bbbZZ3jEkSQIqK+7vAnOBl4EzgYeA87IM1VF07dqVL3/5y3nHkCSpWSUjpx0MXJtS+lXWYSRJ0spVcsb9RWBqRPxPRBxY/oy76s2ePZs5c+bkHUOSpKWssrhTSicCWwB/BMYA0yLiqqyD5e3BBx8EYIMNNsg5iSRJH6vkUjkppUURcTewAKgBjgZOyzJY3pqamgD4whe+kHMSSZI+VskALPtHxLXAa8DxwG+AgVkHy9uECRPyjiBJ0nIqOeM+ldJXwE5PKS3IOE+HMH36dH7yk58Apbm4JUnqKCqZ1vOL7RGkI1lyU9qFF17IhhtumHMaSZI+1mpxR8RfU0r7RMRsILV8CkgppX58LA+mAAAXpUlEQVSZp8vJVVeV7r3bYYcdck4iSdLSIqW04iciuqSUmlr7+ldKqTHTZK2oq6tL48ePz3Qf3bp1o6Ghgfr6erp27ZrpviRJnU9ETEgp1a3Ja1u9OS2l1FR+eF1KqbHlD3DdmuysKBoaGujfv7+lLUnqcCoZgOVTLRfKZ+A7ZxOnY+jWrRunnHJK3jEkSVpOq8UdEWeVP9/+VES8X/6ZDbwL3NtuCSVJUrOVnXFfCgwALi//OQDon1Lql1L6TnuEkyRJS1vZ18E2Tym9GhE3AlsvWRlRmoo7pfR8xtly8cwzz1BfX7/qDSVJysHKivts4GTgyhU8l4C9M0mUo1mzZrHTTjsBsMUWW+ScRpKk5bVa3Cmlk8t/7tV+cfL1z3/+E4DjjjuOMWPG5JxGkqTlVTJW+Rcionf58dkRcXtEbJd9tPa3aNEiAEaPHp1zEkmSVqySr4NdmFKaGxG7A58DbgKuzjZWPpaMmLb22mvnnESSpBWrpLiXjJB2CHB1SuluoHt2kfLTvXvp19pnn31yTiJJ0opVMjvYWxFxJXAQsFNEdKOywi+cRx55hA022ICamhWO8ipJUu4qKe6jKV0i/2VKaXZEbETpjvOq0tjYyJQpU5q/7iZJUke0yjPnlNJHwCRgVEScBvRNKd2XebJ2Nn36dADOO++8nJNIktS6Su4q/wZwBzCo/HN7RHw962Dt7Wc/+xkAW2+99Sq2lCQpP61O69m8QcTzwO7lM28iohfwt5TSp1b6woxkNa1n3759mTNnDh999JF3lUuSMpXJtJ4t3x9oOQZoQ3ld1fif//kf5syZwx577GFpS5I6tEpuTrsReDIifk+psI8Afp1pqnb2pz/9CYBf/epXOSeRJGnlVlncKaVLI+IvwJ6Uxig/LaX0dNbB2susWbO4+eabGTJkCNtuu23ecSRJWqlKv4+9EFjU4s+qMHPmTAYMGADQPLmIJEkdWSV3lZ8H3AJsCGwC3BwR52QdrD08++yzAAwdOpTf/e53OaeRJGnVKvmM+yRgh5TSfICIuAiYCFycZbD2cOuttwJwyy235JxEkqTKVHKp/C2WLvja8rrCmzNnDgDbb799zkkkSapMJWfc7wOTIuJ+SjenHQA8HRE/A0gpfSvDfJl54IEHuOuuu9h2222bJxeRJKmjq6S4/7f8s8S4jLK0q7/97W8AfPvb3845iSRJlavk62DXtUeQvHzpS1/KO4IkSRWryuk5KzFt2rS8I0iStNo6ZXHPmDGDG2+8Me8YkiSttoqLOyKq5g6uJ598EoCLLroo5ySSJK2eSgZg2SUiXgBeLS9vFxG/zDxZOzj44IPzjiBJ0mqp5Iz7F8AhwHsAKaXngH2zDJW1Bx98MO8IkiStkUqKu0tK6Z/LrGvMIkx7WLRoUfMsYIMGDco5jSRJq6eS73G/ERG7ACkiaoDTgSnZxsrOm2++CZQuk/ft2zfnNJIkrZ5Kzri/BnwLGATMBHYrryukWbNmAXD00UfnnESSpNVXyQAs7wCj2yFLu1gyscj666+fcxJJklbfKos7Iv6b0hjlS0kpnZpJoozV1pZ+5QMPPDDnJJIkrb5KPuP+U4vHPYDPA29kEydbjY2NXHbZZXnHkCRpjVVyqfy2lssRcSPwWGaJMnTPPfcA3k0uSSquNRnydCiwQVsHydqCBQs44ogjgI8LXJKkoqnkM+7ZfPwZdxdK83OfnWWoLHzrW6Vpw7t06cKIESNyTiNJ0ppZaXFHRADbAW+WVzWllJa7Ua0IunQpXVxYsGBB8w1qkiQVzUovlZdL+t6UUmP5p5ClvUT//v3p1q1b3jEkSVpjlXzG/WxE7JB5EkmStEqtXjOOiNqU0mJgB+DpiHgNmAcEpZPxHdspY5vwhjRJUjVY2Ye9TwE7Aoe1U5ZMTZ8+nW233TbvGJIkfSIru1QeACml11b0U8mbR8SBEfFKREyNiFbvRI+IIyMiRUTdauavWFNTE4ceemhWby9JUrtY2Rn3gIj4VmtPppR+trI3Ls8kdiWwPzCd0uX2sSmlycts1xs4E3iy4tSr6b777luyr6x2IUlSu1jZGXcN0Avo3crPquwCTE0pTUsp1QO3AoevYLv/AC4BFq5G7tUydepUAI499tisdiFJUrtY2Rn3WymlH3yC996Ypcc0nw7s2nKDiNgR2DSl9L8R8Z1PsK9WzZs3jzPOOAOAAQMGZLELSZLazSo/485KRHQBfgZ8u4JtT42I8REx/t13312t/Vx33XUAbL755vTv339NokqS1GGsrLj3+4Tv/SawaYvlTfh4BDYoXW7fBvhLRPwD2A0Yu6Ib1FJK16SU6lJKdatz1pxS4swzzwTg4YcfXu1fQJKkjqbV4k4pvf8J3/tpYHhEDI2IbsBoYGyL9/8gpdQ/pTQkpTQEGAccllIa/wn326y+vh4ozQa26aabrmJrSZI6vjWZHawi5cFbvgHcD7wE3J5SmhQRP4iIdvlu+FtvvQXAaaed1h67kyQpc5nOtpFSuhe4d5l132tl21Ftvf+7774bgOHDh7f1W0uSlIvMzrg7giVzonzmM5/JOYkkSW2jqov7ySczG9NFkqRcVHVxP/HEEwD07Nkz5ySSJLWNqi7ubt26cfjhh9O9e/e8o0iS1Caquri7dOlCjx498o4hSVKbqeriliSp2ljckiQViMUtSVKBWNySJBWIxS1JUoFY3JIkFYjFLUlSgVjckiQViMUtSVKBVHVxL1iwIO8IkiS1qaot7pdeeonXX3+d+vr6vKNIktRmqra43377bQAOPfTQnJNIktR2qra4H3roIQCGDRuWcxJJktpO1Rb3lClTANhiiy1yTiJJUtupyuJevHgxd9xxByNGjGDDDTfMO44kSW2mKot77ty5AIwYMSLnJJIkta2qLO4l9t5777wjSJLUpqq6uCVJqjYWtyRJBVKVxd3Y2Jh3BEmSMlGVxX3DDTcA0KNHj3yDSJLUxqqyuF944QUAjjvuuJyTSJLUtqquuJ944gl+85vfANCzZ8+c00iS1Laqrrjff/99AG6++WZqa2tzTiNJUtuquuJeYvPNN887giRJba5qi1uSpGpUdcXd0NCQdwRJkjJTdcV9+eWXA34VTJJUnaquuPv06QPANttsk3MSSZLaXlUVd1NTE/fccw877LADEZF3HEmS2lxVFfeMGTMAqKmpyTmJJEnZqKriXuKrX/1q3hEkScpEVRa3JEnVyuKWJKlALG5JkgrE4pYkqUAsbkmSCsTiliSpQCxuSZIKxOKWJKlALG5JkgrE4pYkqUAsbkmSCsTiliSpQCxuSZIKxOKWJKlALG5JkgrE4pYkqUAsbkmSCsTiliSpQKqquJ955pm8I0iSlKmqKu5///d/B2D33XfPOYkkSdmoquKeNm0aACNHjsw5iSRJ2ajNO0Bb6t69O2PGjMk7hiRJmamqM25JkqqdxS1JUoFUTXFPmzaN999/P+8YkiRlqmqKe8KECQDU1dXlnESSpOxUTXH/6le/AmDnnXfOOYkkSdmpmuKOCAC23HLLnJNIkpSdqilugD333JOampq8Y0iSlJmqKm5JkqqdxS1JUoFY3JIkFYjFLUlSgVRFcX/00Uc8/PDDNDU15R1FkqRMVUVxz5gxA4AhQ4bkG0SSpIxVRXEvcfDBB+cdQZKkTFVVcUuSVO0sbkmSCsTiliSpQKqiuJ977rm8I0iS1C6qorifeOIJAHbaaaeck0iSlK1MizsiDoyIVyJiakScvYLnvxURkyPi+Yh4KCIGr+m+evfu7cxgkqSql1lxR0QNcCVwEDASODYiRi6z2USgLqX0KeB3wKVrsq8777zzk0SVJKkwsjzj3gWYmlKallKqB24FDm+5QUrpzyml+eXFccAmq7uTOXPm8M9//pNNN930EweWJKmjy7K4NwbeaLE8vbyuNScD963uTn73u98BcMIJJ6zuSyVJKpzavAMARMQJQB2wTyvPnwqcCjBo0KClnrvkkksA+NKXvpRpRkmSOoIsz7jfBFpev96kvG4pEfEZ4DzgsJTSohW9UUrpmpRSXUqpbsCAAc3rH3jgAaZOncq6667LRhtt1LbpJUnqgLIs7qeB4RExNCK6AaOBsS03iIgdgKsplfY7q7uDJTelXXHFFZ88rSRJBZBZcaeUFgPfAO4HXgJuTylNiogfRMRh5c0uA3oBd0TEsxExtpW3a9UGG2zgZXJJUqeR6WfcKaV7gXuXWfe9Fo8/k+X+JUmqNlUxcpokSZ2FxS1JUoFY3JIkFYjFLUlSgVjckiQViMUtSVKBWNySJBWIxS1JUoFY3JIkFYjFLUlSgVjckiQVSKGL+49//COLFy/OO4YkSe0m00lGspRSYsaMGdTU1OQdRZKkdlPoM26A888/P+8IkiS1m8IXtyRJnYnFLUlSgVjckiQViMUtSVKBWNySJBWIxS1JUoFY3JIkFYjFLUlSgRS2uF988cW8I0iS1O4KW9wXXHABAHvssUfOSSRJaj+FLe6pU6cCMGrUqHyDSJLUjgpb3ACf//znqa0t7DwpkiSttkIXtyRJnY3FLUlSgVjckiQViMUtSVKBWNySJBWIxS1JUoFY3JIkFUghi7upqYkpU6bkHUOSpHZXyOJ+5plnWLRoEV26FDK+JElrrJDNV19fD8BXvvKVnJNIktS+ClncM2fOBCAick4iSVL7KmRx33TTTQD069cv5ySSJLWvQhZ3z5496dWrF3V1dXlHkSSpXRWyuKdOncr666+fdwxJktpd4Yo7pcS4ceNoamrKO4okSe2ukMUNcMopp+ScRJKk9le44p43bx4AXbt2zTmJJEntr3DFPX/+fAD22muvnJNIktT+ClfcDQ0NAGyzzTY5J5Ekqf0Vrrhnz54NQLdu3XJOIklS+ytccQMcfPDBdO/ePe8YkiS1u8IVd319PX369Mk7hiRJuShccQMceOCBeUeQJCkXhSzuYcOG5R1BkqRcFLK4JUnqrCxuSZIKxOKWJKlALG5JkgrE4pYkqUAsbkmSCqSQxe3MYJKkziqWzG9dFBGRGhoaqK2tzTuKJElrJCImpJTq1uS1hTvj7tGjh6UtSeq0ClfckiR1Zha3JEkFYnFLklQgFrckSQVicUuSVCAWtyRJBWJxS5JUIBa3JEkFYnFLklQgFrckSQVicUuSVCAWtyRJBWJxS5JUIBa3JEkFYnFLklQgFrckSQVicUuSVCAWtyRJBWJxS5JUIJkWd0QcGBGvRMTUiDh7Bc93j4jbys8/GRFDsswjSVLRZVbcEVEDXAkcBIwEjo2IkctsdjIwO6W0OXA5cElWeSRJqgZZnnHvAkxNKU1LKdUDtwKHL7PN4cCvy49/B+wXEZFhJkmSCi3L4t4YeKPF8vTyuhVuk1JaDHwArJdhJkmSCq027wCViIhTgVPLi4si4sU881Sx/sCsvENUKY9tdjy22fHYZmfLNX1hlsX9JrBpi+VNyutWtM30iKgF1gHeW/aNUkrXANcARMT4lFJdJok7OY9tdjy22fHYZsdjm52IGL+mr83yUvnTwPCIGBoR3YDRwNhlthkLfKn8+Cjg4ZRSyjCTJEmFltkZd0ppcUR8A7gfqAGuTylNiogfAONTSmOB64AbI2Iq8D6lcpckSa3I9DPulNK9wL3LrPtei8cLgS+u5tte0wbRtGIe2+x4bLPjsc2OxzY7a3xswyvTkiQVh0OeSpJUIB22uB0uNTsVHNtvRcTkiHg+Ih6KiMF55CyiVR3bFtsdGREpIrxjt0KVHNuIOLr83+6kiLi5vTMWVQV/JwyKiD9HxMTy3wufyyNn0UTE9RHxTmtfYY6SX5SP+/MRsWNFb5xS6nA/lG5mew0YBnQDngNGLrPN14Gryo9HA7flnbsIPxUe232BtcqPv+axbbtjW96uN/AIMA6oyzt3EX4q/O92ODAR6FteXj/v3EX4qfDYXgN8rfx4JPCPvHMX4QfYG9gReLGV5z8H3AcEsBvwZCXv21HPuB0uNTurPLYppT+nlOaXF8dR+g6+Vq2S/24B/oPSuPwL2zNcwVVybL8CXJlSmg2QUnqnnTMWVSXHNgF9yo/XAWa0Y77CSik9QukbU605HPhNKhkHrBsRG67qfTtqcTtcanYqObYtnUzpX4RatVUe2/KlsE1TSv/bnsGqQCX/3W4BbBERj0fEuIg4sN3SFVslx/ZC4ISImE7pm0Knt0+0qre6fx8DBRnyVPmIiBOAOmCfvLNUg4joAvwM+HLOUapVLaXL5aMoXSV6JCK2TSnNyTVVdTgWuCGl9NOI+DSl8Te2SSk15R2sM+qoZ9yrM1wqKxsuVcup5NgSEZ8BzgMOSyktaqdsRbeqY9sb2Ab4S0T8g9JnWmO9Qa0ilfx3Ox0Ym1JqSCn9HZhCqci1cpUc25OB2wFSSk8APSiNY65PpqK/j5fVUYvb4VKzs8pjGxE7AFdTKm0/J6zcSo9tSumDlFL/lNKQlNIQSvcPHJZSWuMxizuRSv5OuIvS2TYR0Z/SpfNp7RmyoCo5tq8D+wFExAhKxf1uu6asTmOBk8p3l+8GfJBSemtVL+qQl8qTw6VmpsJjexnQC7ijfL/f6ymlw3ILXRAVHlutgQqP7f3AARExGWgEvpNS8ircKlR4bL8N/HdEfJPSjWpf9kRp1SLiFkr/mOxfvj/gAqArQErpKkr3C3wOmArMB8ZU9L4ee0mSiqOjXiqXJEkrYHFLklQgFrckSQVicUuSVCAWtyRJBWJxSxmLiMaIeLbFz5CVbDuktZmE2ltE1EXEL8qPR0XE7i2eOy0iTmrHLNs7I5VU0iG/xy1VmQUppe3zDrG6ygPDLBkcZhTwEfC38nNXtfX+IqK2PO/AimxPafjde9t6v1LReMYt5aB8Zv1oRDxT/tl9BdtsHRFPlc/Sn4+I4eX1J7RYf3VE1Kzgtf+IiEsj4oXytpu32O/D8fFc64PK678YES9GxHMR8Uh53aiIuKd8heA04Jvlfe4VERdGxL9FxFYR8dQyv9cL5cc7RcRfI2JCRNy/olmPIuKGiLgqIp4ELo2IXSLiiSjN+/y3iNiyPJrXD4Bjyvs/JiLWjtJcx0+Vt13RLGxSVbK4pez1bHGZ/A/lde8A+6eUdgSOAX6xgtedBvy8fLZeB0wvDzd5DLBHeX0jcHwr+/0gpbQt8J/AFeV1vwR+nVL6FHBTi/1+D/hsSmk7YKlR8lJK/wCuAi5PKW2fUnq0xXMvA90iYmh51THAbRHRtbyvo1JKOwHXAxe1knMTYPeU0reAl4G9Uko7lDP9qDzV5PcozQu/fUrpNkrj6D+cUtqF0vzxl0XE2q28v1RVvFQuZW9Fl8q7Av8ZEUvKd4sVvO4J4LyI2AS4M6X0akTsB+wEPF0ejrYnpX8ErMgtLf68vPz408AXyo9vBC4tP34cuCEibgfuXJ1fjtLkE8cAPy7/eQywJaUJVR4s56wBWhuD+Y6UUmP58TrAr8tXFxLl4SFX4ADgsIj4t/JyD2AQ8NJqZpcKx+KW8vFNYCawHaUrXwuX3SCldHP5EvLBwL0R8VUgKJ0xn1PBPlIrj5ffMKXTImLX8r4mRMROlf0aANxGaVz7O0tvlV6NiG2BSSmlT1fw+nktHv8H8OeU0ufLl+j/0sprAjgypfTKauSUqoKXyqV8rAO8VZ7P+ERKZ6RLiYhhwLSU0i+Au4FPAQ8BR0XE+uVt+kXE4Fb2cUyLP58oP/4bH0/IczzwaPl9NkspPZlS+h6lWZ9aTjUIMJfStKTLSSm9RumqwfmUShzgFWBAlOZuJiK6RsTWreRsaR0+ntbwyyvZ//3A6VE+nY/SjHZSp2BxS/n4L+BLEfEcsBVLn3UucTTwYkQ8S+my829SSpOBfwceiIjngQeB5W76Kutb3uZMSmf4AKcDY8rrTyw/B6XPiF8ofxXtb8Bzy7zXH4HPL7k5bQX7ug04gY/nbK6nNN3uJeXf8VlguRvwVuBS4OKImMjSVwT/DIxccnMapTPzrsDzETGpvCx1Cs4OJlWhiPgHUJdSmpV3FkltyzNuSZIKxDNuSZIKxDNuSZIKxOKWJKlALG5JkgrE4pYkqUAsbkmSCsTiliSpQP5/PzWptQw3p3MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class 1\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe4AAAHwCAYAAABgy4y9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xm4VWXd//H3l1EUFBzAARDFCVBBJQccMi3NCc1UtHrK1B+ajzmQlo2aZT1piVmaOWA+PirOiYloqWmpiBqIAiIIDjjkABjIfM79+2NvjofhwAbO3uuss9+v6zoXe629ztofdubHew33ipQSkiQpH1pkHUCSJJXO4pYkKUcsbkmScsTiliQpRyxuSZJyxOKWJClHLG5JknLE4paaoIh4PSLmR8TciHgvIv4UEe2X22ZARDwWEXMi4uOIeCAiei+3zYYRcWVEvFnc12vF5U0b+NyIiLMj4uWI+CQiZkTEXRGxSzn/vpJKZ3FLTddRKaX2QD9gN+D7S9+IiH2AR4D7gS2BbYAXgaciYtviNm2AR4E+wBeBDYF9gI+APRv4zN8C5wBnAxsDOwB/Bo5Y0/AR0WpNf0fS6oUzp0lNT0S8DpyWUvpbcfkyoE9K6Yji8j+Al1JKZy73ew8BH6SUvh4RpwGXAj1TSnNL+MztgVeAfVJKYxrY5u/A/6WUbigun1zMuV9xOQFnAecCrYBRwCcppfPr7eN+4ImU0hURsSXwO+AAYC4wNKV0VQlfkVS1HHFLTVxEdAUOA6YWl9cHBgB3rWTzO4EvFF9/HhhVSmkXHQzMaKi018AxwF5Ab+B2YFBEBEBEdAIOAYZHRAvgAQpHCrYqfv65EXHoOn6+1KxZ3FLT9eeImAO8BbwPXFRcvzGF/+++u5LfeRdYev56kwa2aciabt+QX6aUZqaU5gP/ABKwf/G944BnUkrvAJ8BNkspXZJSWpRSmgZcD5zYCBmkZsvilpquY1JKHYADgZ34tJBnAbXAFiv5nS2AD4uvP2pgm4as6fYNeWvpi1Q4FzccOKm46ivArcXXWwNbRsTspT/AD4AujZBBarYsbqmJSyk9AfwJ+HVx+RPgGeD4lWx+AoUL0gD+BhwaERuU+FGPAl0jov8qtvkEWL/e8uYri7zc8u3AcRGxNYVD6PcU178FTE8pdaz30yGldHiJeaWqZHFL+XAl8IWI6FtcvhD4RvHWrQ4R0Skifk7hqvGfFre5hUI53hMRO0VEi4jYJCJ+EBErlGNKaQpwDXB7RBwYEW0iYr2IODEiLixuNg44NiLWj4jtgFNXFzylNJbCUYAbgIdTSrOLb40B5kTE9yKiXUS0jIidI+Iza/MFSdXC4pZyIKX0AfC/wE+Ky/8EDgWOpXBe+g0Kt4ztVyxgUkoLKVyg9grwV+A/FMpyU+DZBj7qbOD3wNXAbOA14EsULiIDGAosAv4N3Mynh71X57Ziltvq/Z1qgCMp3O42nU/LfaMS9ylVJW8HkyQpRxxxS5KUI2Ur7ogYFhHvR8TLDbwfEXFVREyNiPERsXu5skiS1FyUc8T9JwrTLDbkMGD74s9g4A9lzCJJUrNQtuJOKT0JzFzFJkcD/5sKRgMdI6Ix7iGVJKnZyvIc91bUm6gBmFFcJ0mSGpCLp/dExGAKh9PZYIMN9thpp50yTiRJaooWLlxITU0Nc+fOZf78+dTW1jJz5qoO/q5cx44dy5DuU7Nnz/4wpbTZ2vxulsX9NtCt3nLX4roVpJSuA64D6N+/f3r++efLn06S1CTV1tZy33338fHHHzNnzhx++9vfsskmm/DOO+/wzjvvLLNtly5d6NSpEwMGDGCvvfbiwAMPpFevXqvc/yabbELxuThlExFvrO3vZlncI4CzImI4hWkQP04pNcYDDiRJTcT06dNZsGDBarcbN24cP/3pT5k8eTKtWrWiRYuGz+QuWrRohXULFiygX79+9OzZk6997WtsscUW9OnTh2233Xad8jdFZSvuiLidwsMRNo2IGRSebNQaIKV0LTASOJzCowrnAd8sVxZJ0tq5//77+dWvfkWbNm3W+HefeeaZlZbs6gwZMmSVxQ2QUuIb3/gGG2ywAeuttx6dO3de48/Jq7IVd0rppNW8n4D/LtfnS5LWXEqJCRMmcMstt3DZZZfVrd9vv/1o2bLlGu1rn332YebMmVxwwQUlFX+vXr3Ydddd1zhztcnFxWmSpHX33nvv8dxzz62wfvz48Tz22GO0bduWhx56aJn3evbsyU9/+lO++tWvViqmVsPilqQc+vjjj1m8eHHd8iuvvMJjjz3G9ddfz+LFi1c6Ol7+wq3l9e3bl/79+7NkyRIuvvhiBg4cWPaLtLTmLG5JKpMFCxYwevRoampqGm2f//rXv/if//mf1d7idNppp610fffu3Tn88BUfed6lSxe6du3aKBlVXha3JK2D+++/n9dee22Zdb/4xS/YaKONmDZtWtk+d7PNNuPCCy9c5tzxrrvuyr777rvG56KVLxa3JK3CnDlzePnlZZ+VlFLiD3/4A/PmzePee+9d6e8tWLCAr3zlK7Ro0YLBgwc3aqZu3brRo0ePRt2n8sPilqR6nnrqKV566SWeeOIJxo0bxyuvvLLK7bfbbjuuvfZaPvOZz9Sta9GiBe3bty93VFUpi1tS7i1ZsoRJkybxzjvvcOedd9KmTRsmT57M448/vs77PuaYY+jUqRODBg1aZn3r1q3Zd999adu27Tp/hrQmLG5JufXRRx9x8cUX8/vf/36Z9W3atKF9+/ZEBKeddhpbbrllyfusra3l4IMPZscdd2SjjTaiXbt2jR1bWicWt6RcqK2tZfr06dTW1nL//ffz7rvvcsUVV9S9f9BBB3HWWWfRuXNn9t133wyTSuVlcUtqcubMmcPAgQOXuTr60UcfXWG7li1bMmDAAB544AE22mijSkaUMmNxS2oyFi9ezK9//Wt+8IMf1K1bOnoeMGAAixcv5pxzziEi+PznP19V81NLS1nckjIxa9YsnnnmGebMmcMf/vAHNtxwQx544IG699u1a8fcuXNX+7AJqdpY3JIq4tVXX+Xuu+8mInjyyScZNWrUMu+3bduWfv360blzZ+655x5vp5IaYHFLKos77riDe++9l4hg5syZ/PWvf11hm/POO48TTzyR9u3b06tXL+fFlkpgcUtqFG+99Rb33nsvt9xyC3PmzOHVV18FYMcdd6S2tpatttqKs88+m3POOQeAVq1aOTWntBYsbknr7J133qF79+7LrDv++OM55JBDGnzYhaS1Y3FLWmtXXHEFN9xwA5MmTQIK03+OGTOGjTbayIvKpDKxuCWV7I033mDy5Mlce+213HfffXXrDz74YHbeeWeGDh3qeWqpzCxuSSt4/fXXefrpp0kpcdlll9GhQwemTJnC+++/v8x2AwcO5NJLL2XnnXfOKKlUfSxuqYrV1tbyxhtvLLPutdde4wtf+MIK2x500EHMnj2bwYMH06dPH/bbb79KxZRUj8UtVZkXXniBN998k6FDh/KPf/yjwe0GDRrEJZdcQqtWrdhmm208BC41ERa31MyklJY5pD1z5kzuuOMOIoIPPviAq6++epnt99prL771rW8ts27LLbdc6ahbUvYsbqkZqK2t5emnn+aBBx7gsssuW+32P/7xjznhhBPo3bu3V39LOWNxSzmyaNEiFi5cWLf8zDPP8OKLL/Ld7353me369u3L6aefXrfcpUsXvvSlL9Ute9hbyi+LW2qCamtrGTt2LPPnzwfg4YcfZvz48YwYMWKVv/f444/Tq1cvunTpUomYkjJgcUtNyNy5c9l7772ZMGHCSt/fYost+NznPsfuu+8OFM5nH3DAAfTp04cNNtigklElZcTilpqQrbfempkzZwJwyCGHcPbZZ9O2bVsAdt55ZzbffPMs40lqAixuKSMvv/wyzz33XN3yU089VVfatbW1noeWtFIWt1Rh7733Hh9++CG77LLLSt//y1/+YmlLapD3gUgVMnfuXL7zne+wxRZb1JX2UUcdxeuvv173M3v2bI444oiMk0pqyhxxS2U0e/Zs5syZw6hRoxg8eHDd+gsuuIA999yTww8/nPXXXz/DhJLyxuKWyuRvf/vbCrOP9ejRg2eeecaLzCStNYtbakTz58/nd7/7HdOmTeOPf/wjAN/4xjfYf//9OeSQQ+jWrVvGCSXlncUtraOamhq+8pWvUFtby913373Me9/+9re56qqrMkomqTmyuKW1kFLi5Zdf5vjjj2fy5Ml167fffnv69evHsGHDaNeuHS1btswwpaTmyOKWSjR//nx+9atf8frrr3PzzTcv897xxx/PsGHDaN++fUbpJFULi1sqwcSJE+nTp0/dcrdu3Wjbti2/+c1vOOqoo7zvWlLFWNzSKvz1r3/lkEMOqVvu1asXjz32mFeFS8qME7BIDfjggw/qSvvAAw/k9ttvZ+LEiZa2pExZ3FI906dP57TTTiMi6Ny5MwBf/vKXefzxxznxxBMzTidJHiqX6hxxxBGMHDmybvmggw7i0EMP5dxzz80wlSQty+KWgEmTJtWV9vDhw/nyl79Mq1b+30NS0+OhclW12tpazjzzTHr37g3ArbfeyqBBgyxtSU2Wxa2qdscdd/CHP/wBgM985jOcdNJJGSeSpFVzWKGq9pWvfAWAGTNmsNVWW2WcRpJWzxG3qtKoUaPo3r173bKlLSkvHHGrqjz55JOccsopvPbaa3Xrxo4dm2EiSVozFreqQk1NDQMHDlzmdq+nn36avffe2+lKJeWKh8rV7L3yyiu0atWqrrSHDRtGSol99tnH0paUOxa3mq2XX36Zvn370qtXL6DwyM05c+bwzW9+M+NkkrT2PFSuZmn8+PH07dsXKDzJ69hjj+XKK6/MOJUkrTuLW83S0tI+6aSTuO222zJOI0mNx+JWs/Lhhx/y4IMPAtC5c2dLW1KzY3Er95YsWcLTTz/NkUceyZw5c+rW//rXv84wlSSVh8WtXFuwYAHbbrst7777bt26a665hiOPPJJu3bplmEySysPiVi598sknnH766dx6661165588kn22GMP1l9//QyTSVJ5WdzKpQ033JDa2loALr74Yr71rW/RuXPnjFNJUvlZ3MqVlBLnnXdeXWlPnTqVnj17ZpxKkirHCViUK7///e/57W9/C8CECRMsbUlVx+JWbowYMYKzzz4bgH/84x/07t0740SSVHkWt3Lhoosu4uijjwbglFNOYb/99ss4kSRlw3PcatJSSpx66qncdNNNQOE52oceemjGqSQpOxa3mqwJEyZw77331pX2HXfcYWlLqnoWt5qcmTNnss8++/Dqq6/WrXvmmWfYe++9M0wlSU2D57jVpLzyyitssskmvPrqq3Tq1Ilhw4YxadIkS1uSihxxq0mYPn06Rx55JBMnTgSgV69ePP/8886CJknLccStzL3//vsccMABTJw4kW222YYbbriBsWPHWtqStBKOuJWZ+fPnc8wxx/DII4/UrZsyZQotW7bMMJUkNW2OuJWJF154gfXXX59HHnmEjh07MmTIEKZOnWppS9JqOOJWJvr37w/A0Ucfze233067du0yTiRJ+WBxq6JSSgwZMqRu+c9//nOGaSQpfyxuVczbb79N165d65afffbZDNNIUj5Z3KqYHj161L3+4IMP2HTTTbMLI0k5ZXGrIk4//XSWLFnCxhtvzIcffkhEZB1JknLJq8pVdmPGjOG6664DCofHLW1JWnsWt8ompcT999/PXnvtBcDll1/Odtttl3EqSco3i1tl8+c//5ljjjkGKNz2df7552ecSJLyz3PcKotPPvmEY489FoBHH32Ugw46KONEktQ8lHXEHRFfjIjJETE1Ii5cyfvdI+LxiBgbEeMj4vBy5lFlTJ48mfbt2wOw7bbbWtqS1IjKVtwR0RK4GjgM6A2cFBG9l9vsR8CdKaXdgBOBa8qVR5UxZswYdtppJ6Bw+9fkyZMzTiRJzUs5R9x7AlNTStNSSouA4cDRy22TgA2LrzcC3iljHpXZI488Unch2mGHHcb06dNp1cqzMZLUmMpZ3FsBb9VbnlFcV9/FwNciYgYwEvh2GfOojH75y19y6KGHAvCzn/2MkSNHZpxIkpqnrK8qPwn4U0qpK3A4cEtErJApIgZHxPMR8fwHH3xQ8ZBatXvuuYcf/OAHANx000386Ec/yjiRJDVf5Szut4Fu9Za7FtfVdypwJ0BK6RlgPWCFeTBTStellPqnlPpvttlmZYqrtXXccccBcNVVV3HyySdnG0aSmrlyFvdzwPYRsU1EtKFw8dmI5bZ5EzgYICJ6UShuh9Q5MmJE4X/SHj168O1ve6ZDksqtbFcOpZSWRMRZwMNAS2BYSmlCRFwCPJ9SGgF8B7g+Is6jcKHaySmlVK5MWncpJW666SZmzZrFjTfeyKRJkwC49dZbM04mSdWhrJf8ppRGUrjorP66n9R7PRHYt5wZ1HiWLFlC69atV1h/4YUXss8++2SQSJKqj/fqqCSzZ8/ms5/9bN3y22+/TYcOHWjfvr0PDZGkCrK4tUq33HILX//615dZN3/+fNZbb72MEklSdbO4tVKzZs3iwAMPZPz48QD88Ic/ZNNNN+X000+3tCUpQxa3VvDhhx9S/7a7iy++mIsuuijDRJKkpSxurWDAgAEAHHDAAfz973/3HLYkNSFZz5ymJmbGjBlMmTIFKMw9bmlLUtNicavOwoUL6datMNndb37zG9q2bZtxIknS8ixuAXDppZcuc9HZ2WefnWEaSVJDLG4xefLkugeDnHLKKcybN8/HcUpSE2VxV7mnn36anXbaCYBzzjmHG2+8kXbt2mWcSpLUEIu7yu27b2HG2UMPPZShQ4dmnEaStDoeD61iEyZMAGDXXXdl1KhRGaeRJJXCEXcV+853vgPABRdckHESSVKpLO4qNW3aNB5++GEAjjvuuIzTSJJKZXFXoTFjxtCzZ08Ahg4d6tzjkpQjFneV+ec//8lee+0FQKtWrTjnnHMyTiRJWhMWdxVZsGAB+++/P1B42tfixYud0lSScsbirhIvvvhi3f3ZEcHPf/7zjBNJktaGxV0Fhg4dSr9+/QDYeeedmTVrVsaJJElry+Ju5j766COGDBkCwG233cYLL7zARhttlHEqSdLacgKWZiylVPe0r4MOOoiTTjop40SSpHXliLuZmjdvHvvvvz/z58+nY8eO3HfffVlHkiQ1AkfczVS/fv2YMmUKULgwbcMNN8w4kSSpMVjczdBf/vKXutJeuHAhbdq0yTiRJKmxeKi8GTrqqKMAeOGFFyxtSWpmLO5m5uOPPwZg2223Zffdd884jSSpsVnczczAgQMBOP/88zNOIkkqB4u7Gfn5z3/Ok08+CcDJJ5+cbRhJUllY3M3EkiVL+PGPfwzAk08+WTe9qSSpebG4m4GUEscccwwAe+65Z92DRCRJzY/FnWMpJUaPHk2XLl148MEHAer+lCQ1T97HnVNvvPEGvXr1Yv78+XXrJk+ezKabbpphKklSuTnizqEJEybQo0ePutIeNWoUtbW17LDDDhknkySVm8WdQ6eeeioAhx12GDU1NRx66KFERMapJEmVYHHnyKuvvsrmm2/Os88+y4YbbsjIkSNp0cL/CSWpmniOOwcmT57MTjvttMy6hx56KKM0kqQsOVxr4ubNm7dMad91110sWbKEAQMGZJhKkpQVR9xNWG1tLRtssAEAhxxyCKNGjfJctiRVOUfcTdi5555b9/qhhx6ytCVJFndTddNNN/G73/0OKFyU5kVokiSwuJukYcOGccoppwBwzz33sP3222ecSJLUVFjcTcxPf/rTuvu0r7nmGo499tiME0mSmhIvTmsiZs2aRc+ePZk1axZQGHV/85vfzDiVJKmpsbibiO9973t1pT1u3Dj69u2bcSJJUlNkcTcR119/PQAffvghm2yyScZpJElNlee4m4Clt33169fP0pYkrZLFnbEpU6bw29/+FoD77rsv4zSSpKbO4s7Qiy++WPcozrPPPpsePXpkG0iS1ORZ3BlYvHgxZ5xxBv369QPg+OOP58orr8w4lSQpD7w4LQODBw/mT3/6EwCDBg1i+PDh2QaSJOWGxV1htbW1daU9a9YsOnbsmG0gSVKueKi8gmbMmEG7du0AOOiggyxtSdIas7gr6IILLmDRokV07dqVm2++Oes4kqQc8lB5BbzwwgucdNJJTJkyBYA333zTR3RKktaKxV0B/fv3B6Br164MHjzY0pYkrTWLu8zef/99ALbcckveeuutjNNIkvLOc9xltnRWtAsvvDDjJJKk5iBSSllnWCP9+/dPzz//fNYxSrb0sPjMmTPp1KlTxmkkSU1BRLyQUuq/Nr/riLuMHnzwQQB22GEHS1uS1Cgs7jIZOnQoRx55JAAXXXRRxmkkSc2FF6eVyZAhQwB44oknOOCAAzJOI0lqLhxxl8Fee+0FwN57721pS5IalcXdyI4//njGjBkDwF133ZVxGklSc2NxN6JXXnmFu+++G4CXXnqJrl27ZpxIktTcWNyNaJdddgHg6quvZuedd844jSSpObK4G8n999/PkiVLADjzzDMzTiNJaq4s7kawZMkSjjnmGAD+9a9/ZZxGktScWdyNYPjw4QAccMAB7LbbbhmnkSQ1ZxZ3I/j73/8OwK233pptEElSs+cELOvo5JNP5uabbwYKTwCTJKmcHHGvg+HDh9eV9siRI2nRwq9TklRejrjXwUknnQTAiy++yK677ppxGklSNXCIuJZqamoA2H777S1tSVLFWNxr6ZRTTgHgS1/6UsZJJEnVxOJeS5MmTQLg+9//fsZJJEnVxOJeC6NHj+a5556jd+/edOzYMes4kqQqYnGvhUGDBgHw4x//OOMkkqRqY3GvocmTJ/Pmm28CcOKJJ2acRpJUbcpa3BHxxYiYHBFTI+LCBrY5ISImRsSEiLitnHkaw9JnbF966aUZJ5EkVaOy3ccdES2Bq4EvADOA5yJiREppYr1ttge+D+ybUpoVEZ3LlaexPPbYYwCce+65GSeRJFWjco649wSmppSmpZQWAcOBo5fb5v8BV6eUZgGklN4vY55G8Z///AeA9ddfP+MkkqRqVM7i3gp4q97yjOK6+nYAdoiIpyJidER8sYx51tns2bN54YUX2GOPPbKOIkmqUllPedoK2B44EOgKPBkRu6SUZtffKCIGA4MBunfvXumMdZY+c3uXXXbJLIMkqbqtdsQdEe0i4vsRcW1xebuIOKyEfb8NdKu33LW4rr4ZwIiU0uKU0nTgVQpFvoyU0nUppf4ppf6bbbZZCR9dHksPkw8bNiyzDJKk6lbKofJhQAD7FZffAX5Rwu89B2wfEdtERBvgRGDEctv8mcJom4jYlMKh82kl7LviampqGDt2LL169SIiso4jSapSpRT39imlXwCLAVJK8ygU+SqllJYAZwEPA5OAO1NKEyLikogYWNzsYeCjiJgIPA5ckFL6aC3+HmU3btw4AB8oIknKVCnnuBdFxHpAAoiIbYBFpew8pTQSGLncup/Ue52AIcWfJq1///4AnHnmmRknkSRVs1KK+2fAKKBrRNwMfBY4raypmpjx48fXvd5///0zTCJJqnarLe6U0kMR8TwwgMIh8gvycL91Y+rbty8Aw4cP9/y2JClTpVxV/khK6YOU0v0ppT+nlN6PiEcqEa4p+O53v1v3+oQTTsgwiSRJqxhxF68EXw/oEhEd+PSCtA2B7G6mrrDLL78cKBwud7QtScraqg6V/zeFi8Y6AxP4tLj/A1xb5lxNwtLSPuKII5x0RZLUJEThwu5VbBBxbkrpygrlWa3+/fun559/vuyfk1KiRYvCmYTXX3+drbfeuuyfKUmqDhHxQkqp/9r8bikXp10ZETsBvSkcOl+6vsk/gnNd3HjjjQBsvvnmlrYkqclYbXFHxI+AQ4CdKEyYcijwT6BZF/f1118PwCOPVM11eJKkHChl5rRBwOeAd1NK/wX0BTYoa6qMzZ49mzFjxgA+UESS1LSUUtzzU0o1wJLi1eXvAc362PH//d//AXDUUUdlnESSpGWVMnPa2IjoSOFhI89TuKp8TFlTZez73/8+ADfccEPGSSRJWtYqizsKNy5fXHw+9tUR8TCwYUrpXxVJl5FPPvkEgM6dO2ecRJKkZa2yuFNKKSL+CuxcXJ5akVQZmjBhAiklBgwYkHUUSZJWUMo57nERsVvZkzQRS68mP//88zNOIknSikqZgGUCsCPwGvAJhRnUUkpp9/LHW1E5J2Cpra2lZcuWAPznP/+hQ4cOZfkcSVJ1K+sELMDAtdlxHj366KMA7LHHHpa2JKlJKmXmtNcqEaQpuOSSSwCvJpckNV2lnOOuCikl/vnPfwKfPn9bkqSmxuIuWnoL2CGHHOLjOyVJTVZJxR0RXSPic8XXbSOi2U152r9/4RqBAw88MNsgkiStwmqLOyJOAUYAS0/8bg3cX85QlXbllVcyefJkAIYMGZJxGkmSGlbKiPtsYG8KU52SUnoVaDZTii1atIjzzjsPgL/97W+0bds240SSJDWslOJekFJatHQhIlpSuJe7WfjlL38JwGGHHcbBBx+ccRpJklatlOJ+KiK+C6xXPM99B/CX8saqnIsvvhiA4cOHZxtEkqQSlFLc3wXmAK8A5wCPAj8sZ6hK2m23wmyuG264YcZJJElavVJmTjsCuCGl9Idyh8nC2LFjOfHEE7OOIUlSSUoZcR8PTI2ImyLii8Vz3M3CokWFU/erm69dkqSmYrXFnVL6L2AH4AHgm8C0iLi23MEqYdy4cQDsvffeGSeRJKk0pRwqJ6W0MCLuB+YDLYETgDPKGawS9tprLwB23HHHjJNIklSaUiZg+UJE3EDhsZ5fBf4X2Lzcwcpt4sSJda8PPfTQDJNIklS6UkbcgyncAvbtlNL8MuepmAsuuACAUaNG0aKFU7ZLkvKhlMd6Hl+JIJW29KEijrYlSXnSYHFHxBMppc9GxCyg/mXXAaSU0sZlT1cmNTU1PPHEE/Tp0yfrKJIkrZFVjbg/V/xz00oEqaSlo+0BAwZknESSpDXT4MndlFJt8eWNKaWa+j/AjZWJVx533303AP3OpC4ZAAAXoklEQVT69cs4iSRJa6aUq7J2rb9QnIDlM+WJUxmnnnoq4P3bkqT8abC4I+J7xfPbu0bEzOLPLOADYGTFEjayd955B4AuXbqw++67Z5xGkqQ1s6oR92XAZsDQ4p+bAZumlDZOKV1QiXDlMHbsWAB+9rOfZZxEkqQ1t6qL07ZLKU2JiFuAusuvIwqP4k4pjS9ztrI45ZRTAA+TS5LyaVXFfSFwKnD1St5LwAFlSVRmH3zwARHBLrvsknUUSZLWWIPFnVI6tfjn/pWLU17vvfceKaW6UbckSXlTylzlx0ZEh+LrCyPizojoW/5oje+BBx4AYN999804iSRJa6eU28EuTinNiYgBwOHArcAfyxur8U2dOpXBgwcDcOyxx2acRpKktVNKcdcU/zwS+GNK6X6gbfkilcf2228PQLdu3ejYsWPGaSRJWjulPB3s3Yi4GjgM2CMi2lBa4TcZw4YNA2DzzTfnjTfeyDiNJElrr5QCPgF4Ajg8pTSLwtzlF5Y1VSO76KKLAPjLX/5SdzubJEl5tNriTinNBSYAB0bEGUCnlNJDZU/WiObMmUPHjh3ZY489so4iSdI6KeWq8rOAu4DuxZ87I+LMcgdrLLW1tXz88cccccQRWUeRJGmdlXKOezCwZ3HkTUT8AngauKacwRrL+PGFCd66dOmScRJJktZdKee4A1hUb3lxcV0ufO973wPggANyOdGbJEnLKGXEfQvwbETcQ6GwjwFuLmuqRjJu3DgeeeQRAA4//PCM00iStO5WW9wppcsi4u/AfhTmKD8jpfRcuYM1hrPOOguAa665htatW2ecRpKkdVfKiBtgAbAQqC3+2eSllHjqqacAOOOMMzJOI0lS4yjlqvIfArcDWwBdgdsi4vvlDraupkyZAsB///d/e++2JKnZKGXE/XVgt5TSPICIuBQYC/yynMHW1eWXXw7Afvvtl3ESSZIaTylXlb/LsgXfqriuSXv00UcBOO644zJOIklS4yllxD0TmBARD1O4OO0Q4LmIuAIgpTSkjPnW2ttvv80WW2xBq1alnsaXJKnpK6XVHiz+LDW6TFkaVW1tLbvuumvWMSRJalSl3A52YyWCNKYpU6awZMkS9tlnn6yjSJLUqHL1eM5SLb1/u1+/fhknkSSpcTXL4m7RovDXOvroozNOIklS4yq5uCOibTmDNKb333/f0bYkqVkqZQKWPSPiJWBKcblvRPyu7MnWwb/+9S9SSlnHkCSp0ZUy4r4KOBL4CCCl9CLwuXKGWhdLC3vjjTfOOIkkSY2vlOJukVJ6Y7l1NeUI0xgWLFgAwOc+12T/20KSpLVWyn3cb0XEnkCKiJbAt4FXyxtr7X300UdA4T5uSZKam1JG3N8ChgDdgX8DexfXNUnjx48HYOutt844iSRJja+UCVjeB06sQJZG8cQTTwA4+YokqVlabXFHxPUU5ihfRkppcFkSraPXXnsNgB122CHjJJIkNb5SznH/rd7r9YAvAW+VJ866Gzt2LK1bt/YZ3JKkZqmUQ+V31F+OiFuAf5Yt0Tp6++232XHHHbOOIUlSWazNlKfbAF0aO0hjmDt3LgsXLmTzzTfPOookSWVRyjnuWXx6jrsFhedzX1jOUGvrvffeA+DAAw/MNogkSWWyyuKOwonivsDbxVW1qQnPJfrSSy8BsO2222acRJKk8ljlofJiSY9MKdUUf5psaQMMHToUgL322ivjJJIklUcp57jHRcRuZU/SCGbOnAk44pYkNV8NHiqPiFYppSXAbsBzEfEa8AkQFAbju1coY8lmzpzJwQcfnHUMSZLKZlXnuMcAuwMDK5RlnSxatIh3332X3XbLxcEBSZLWyqoOlQdASum1lf2UsvOI+GJETI6IqRHR4JXoEfHliEgR0X8N89dZemFar1691nYXkiQ1easacW8WEUMaejOldMWqdlx8ktjVwBeAGRQOt49IKU1cbrsOwDnAsyWnXonbb78dgMMPP3xddiNJUpO2qhF3S6A90KGBn9XZE5iaUpqWUloEDAeOXsl2PwN+BSxYg9wrGD16NICHyiVJzdqqRtzvppQuWYd9b8Wyc5rPAJa5Tysidge6pZQejIgL1uGzmDZtGnvssQedOnVal91IktSkrfYcd7lERAvgCuA7JWw7OCKej4jnP/jgg5Vu8+6777LVVls1ckpJkpqWVRX3ut5X9TbQrd5yVz6dgQ0Kh9t3Bv4eEa8DewMjVnaBWkrpupRS/5RS/80222yFD5ozZw4AXbo0ySnUJUlqNA0Wd0pp5jru+zlg+4jYJiLaACcCI+rt/+OU0qYppR4ppR7AaGBgSun5Nf2gK64oXCfnw0UkSc3d2jwdrCTFyVvOAh4GJgF3ppQmRMQlEdGo94a3alU4Vf/d7363MXcrSVKTs9qng62LlNJIYORy637SwLYHru3nLL2Hu3379mu7C0mScqFsI+5KevDBB7OOIElSRTSL4p47dy7du3fPOoYkSWXXLIobYMCAAVlHkCSp7HJf3IsXLwagT58+GSeRJKn8cl/c06dPB2DhwoUZJ5EkqfxyX9zvvfceALvv3uQeDy5JUqPLfXHPmDEDgHbt2mWcRJKk8st9cc+ePRuAnj17ZpxEkqTyy31xLz233blz54yTSJJUfrkv7ocffhiA9ddfP+MkkiSVX+6Le9q0aQC0bt064ySSJJVf7ov7ww8/ZN999806hiRJFZHr4l6wYAGzZs2iU6dOWUeRJKkicl3ckyZNAmCXXXbJOIkkSZWR6+JesGAB4OQrkqTqkevinjdvHuCtYJKk6pHr4h4/fjzgrGmSpOqR6+JeOt3pjjvumHESSZIqI9fFvfSRnh06dMg4iSRJlZHr4q6pqWGTTTYhIrKOIklSReS6uBctWkTbtm2zjiFJUsXkurhfffVVpzqVJFWVXBf33Llz+eijj7KOIUlSxeS6uFNKzlMuSaoquS7usWPHeg+3JKmq5Lq411tvPVJKWceQJKliclvcKSUWLFhAr169so4iSVLF5La4X3vtNaBwgZokSdUit8U9e/ZswEd6SpKqS26L+9///jcAPXv2zDiJJEmVk9viXrhwIYAzp0mSqkpui/uRRx4BoGvXrhknkSSpcnJb3G+88QYAW2+9dcZJJEmqnNwWd4cOHWjbtq1PBpMkVZXcFvf06dPp06dP1jEkSaqoVlkHWFtz586lpqYm6xiSJFVUbkfc//73v9l5552zjiFJUkXltrhnzZrF4sWLs44hSVJF5bK4lz5YxHPckqRqk8viXjo/+YIFCzJOIklSZeWyuBctWgQ43akkqfrksriXTnfapk2bjJNIklRZuSzuefPmARa3JKn65LK458+fD3iOW5JUfXJZ3EsLu3v37hknkSSpsnJZ3G+++SYA6623XsZJJEmqrFwW99KJVzbYYIOMk0iSVFm5LO6l93FvuummGSeRJKmyclnc7777LgDt2rXLOIkkSZWVy+Ju3bo1ABtvvHHGSSRJqqxcFveMGTMAL06TJFWfXBZ3RCzzpyRJ1SKXxT1jxgy6dOmSdQxJkioul8VdU1PDJ598knUMSZIqLpfFvWTJEnbcccesY0iSVHG5LO7Ro0fToUOHrGNIklRxuSzuWbNmsWTJkqxjSJJUcbksboC+fftmHUGSpIrLXXEvHWlvvvnmGSeRJKnyclfcSx8wsskmm2ScRJKkysttcXfs2DHjJJIkVV7uirumpgaAnj17ZpxEkqTKy21x+2QwSVI1yl1xL7XhhhtmHUGSpIrLbXG3bNky6wiSJFVc7oo7pQRY3JKk6mRxS5KUI7kr7qUsbklSNcpdcS+9j9viliRVo9wV91Lrr79+1hEkSaq4XBZ369atadOmTdYxJEmquNwV9/z582nRInexJUlqFLlrQM9tS5KqWe6K+5NPPnGecklS1cpdcdfU1BARWceQJCkTuSvuVq1ascEGG2QdQ5KkTOSuuFNKbLfddlnHkCQpE7ks7latWmUdQ5KkTJS1uCPiixExOSKmRsSFK3l/SERMjIjxEfFoRGy9un1a3JKkala24o6IlsDVwGFAb+CkiOi93GZjgf4ppV2Bu4HLVrdfi1uSVM3KOeLeE5iaUpqWUloEDAeOrr9BSunxlNK84uJooGspO7a4JUnVqpzFvRXwVr3lGcV1DTkVeGh1O12yZImTsEiSqlaTGLpGxNeA/sBnG3h/MDB46fLMmTMrlEySpKalnMX9NtCt3nLX4rplRMTngR8Cn00pLVzZjlJK1wHXFbdP22yzTeOnlSQpB8p5qPw5YPuI2CYi2gAnAiPqbxARuwF/BAamlN4vdcetW7du1KCSJOVF2Yo7pbQEOAt4GJgE3JlSmhARl0TEwOJmlwPtgbsiYlxEjGhgd8uwuCVJ1aqs57hTSiOBkcut+0m9159fm/16VbkkqVrlbuY0gMWLF2cdQZKkTOSyuNdbb72sI0iSlIlcFnenTp2yjiBJUiZyWdxenCZJqlYWtyRJOWJxS5KUI7ks7gULFmQdQZKkTOSyuLt06ZJ1BEmSMpHL4vZQuSSpWlnckiTlSC6L2ylPJUnVKpfFLUlStcplcbdv3z7rCJIkZSKXxd2yZcusI0iSlIlcFneLFrmMLUnSOstlA0ZE1hEkScpELovbEbckqVrlsgEtbklStcplA1rckqRqlcsGtLglSdUqlw3oxWmSpGqVy+J2xC1Jqla5bECLW5JUrXLZgBa3JKla5bIBLW5JUrXKZQNa3JKkapXLBmzdunXWESRJykQui9sRtySpWuWyAS1uSVK1ymUDWtySpGqVywa0uCVJ1SqXDWhxS5KqVS4b0LnKJUnVKpfFLUlStbK4JUnKkdwVt4fJJUnVLHfFLUlSNctdcTviliRVs9wVd0op6wiSJGXG4pYkKUdyV9wtW7bMOoIkSZnJXXFLklTNLG5JknLE4pYkKUdyV9zeDiZJqma5K25JkqqZxS1JUo5Y3JIk5YjFLUlSjljckiTliMUtSVKOWNySJOVI7orb+7glSdUsd8UtSVI1s7glScoRi1uSpByxuCVJyhGLW5KkHLG4JUnKkdwVt7eDSZKqWe6KW5KkamZxS5KUIxa3JEk5YnFLkpQjFrckSTlicUuSlCMWtyRJOZK74vY+bklSNctdcUuSVM0sbkmScsTiliQpRyxuSZJyxOKWJClHLG5JknLE4pYkKUdyV9zexy1Jqma5K25JkqqZxS1JUo6Utbgj4osRMTkipkbEhSt5v21E3FF8/9mI6FHOPJIk5V3ZijsiWgJXA4cBvYGTIqL3cpudCsxKKW0HDAV+Va48kiQ1B+Ucce8JTE0pTUspLQKGA0cvt83RwM3F13cDB4dXn0mS1KByFvdWwFv1lmcU1610m5TSEuBjYJMyZpIkKddaZR2gFBExGBhcXFwYES9nmacZ2xT4MOsQzZTfbfn43ZaP32357Li2v1jO4n4b6FZvuWtx3cq2mRERrYCNgI+W31FK6TrgOoCIeD6l1L8siauc3235+N2Wj99t+fjdlk9EPL+2v1vOQ+XPAdtHxDYR0QY4ERix3DYjgG8UXx8HPJZSSmXMJElSrpVtxJ1SWhIRZwEPAy2BYSmlCRFxCfB8SmkEcCNwS0RMBWZSKHdJktSAsp7jTimNBEYut+4n9V4vAI5fw91e1wjRtHJ+t+Xjd1s+frfl43dbPmv93YZHpiVJyg+nPJUkKUeabHE7XWr5lPDdDomIiRExPiIejYits8iZR6v7butt9+WISBHhFbslKuW7jYgTiv/sToiI2yqdMa9K+HdC94h4PCLGFv+9cHgWOfMmIoZFxPsN3cIcBVcVv/fxEbF7STtOKTW5HwoXs70GbAu0AV4Eei+3zZnAtcXXJwJ3ZJ07Dz8lfrefA9Yvvv6W323jfbfF7ToATwKjgf5Z587DT4n/3G4PjAU6FZc7Z507Dz8lfrfXAd8qvu4NvJ517jz8AAcAuwMvN/D+4cBDQAB7A8+Wst+mOuJ2utTyWe13m1J6PKU0r7g4msI9+Fq9Uv65BfgZhXn5F1QyXM6V8t3+P+DqlNIsgJTS+xXOmFelfLcJ2LD4eiPgnQrmy62U0pMU7phqyNHA/6aC0UDHiNhidfttqsXtdKnlU8p3W9+pFP6LUKu32u+2eCisW0rpwUoGawZK+ed2B2CHiHgqIkZHxBcrli7fSvluLwa+FhEzKNwp9O3KRGv21vTfx0BOpjxVNiLia0B/4LNZZ2kOIqIFcAVwcsZRmqtWFA6XH0jhKNGTEbFLSml2pqmah5OAP6WUfhMR+1CYf2PnlFJt1sGqUVMdca/JdKmsarpUraCU75aI+DzwQ2BgSmlhhbLl3eq+2w7AzsDfI+J1Cue0RniBWklK+ed2BjAipbQ4pTQdeJVCkWvVSvluTwXuBEgpPQOsR2Eec62bkv59vLymWtxOl1o+q/1uI2I34I8UStvzhKVb5XebUvo4pbRpSqlHSqkHhesHBqaU1nrO4ipSyr8T/kxhtE1EbErh0Pm0SobMqVK+2zeBgwEioheF4v6goimbpxHA14tXl+8NfJxSend1v9QkD5Unp0stmxK/28uB9sBdxev93kwpDcwsdE6U+N1qLZT43T4MHBIRE4Ea4IKUkkfhVqPE7/Y7wPURcR6FC9VOdqC0ehFxO4X/mNy0eH3ARUBrgJTStRSuFzgcmArMA75Z0n797iVJyo+meqhckiSthMUtSVKOWNySJOWIxS1JUo5Y3JIk5YjFLZVZRNRExLh6Pz1WsW2Php4kVGkR0T8iriq+PjAiBtR774yI+HoFs/TziVRSQZO8j1tqZuanlPplHWJNFSeGWTo5zIHAXODp4nvXNvbnRUSr4nMHVqYfhel3Rzb250p544hbykBxZP2PiPhX8WfASrbpExFjiqP08RGxfXH91+qt/2NEtFzJ774eEZdFxEvFbber97mPxafPWu9eXH98RLwcES9GxJPFdQdGxF+KRwjOAM4rfub+EXFxRJwfETtFxJjl/l4vFV/vERFPRMQLEfHwyp56FBF/iohrI+JZ4LKI2DMinonCc5+fjogdi7N5XQIMKn7+oIjYIArPOh5T3HZlT2GTmiWLWyq/dvUOk99XXPc+8IWU0u7AIOCqlfzeGcBvi6P1/sCM4nSTg4B9i+trgK828Lkfp5R2AX4PXFlc9zvg5pTSrsCt9T73J8ChKaW+wDKz5KWUXgeuBYamlPqllP5R771XgDYRsU1x1SDgjohoXfys41JKewDDgEsbyNkVGJBSGgK8AuyfUtqtmOkXxUdN/oTCc+H7pZTuoDCP/mMppT0pPD/+8ojYoIH9S82Kh8ql8lvZofLWwO8jYmn57rCS33sG+GFEdAXuTSlNiYiDgT2A54rT0baj8B8BK3N7vT+HFl/vAxxbfH0LcFnx9VPAnyLiTuDeNfnLUXj4xCDgf4p/DgJ2pPBAlb8Wc7YEGpqD+a6UUk3x9UbAzcWjC4ni9JArcQgwMCLOLy6vB3QHJq1hdil3LG4pG+cB/wb6UjjytWD5DVJKtxUPIR8BjIyI04GgMGL+fgmfkRp4veKGKZ0REXsVP+uFiNijtL8GAHdQmNf+3sKu0pSI2AWYkFLap4Tf/6Te658Bj6eUvlQ8RP/3Bn4ngC+nlCavQU6pWfBQuZSNjYB3i88z/i8KI9JlRMS2wLSU0lXA/cCuwKPAcRHRubjNxhGxdQOfMajen88UXz/Npw/k+Srwj+J+eqaUnk0p/YTCU5/qP2oQYA6Fx5KuIKX0GoWjBj+mUOIAk4HNovDsZiKidUT0aSBnfRvx6WMNT17F5z8MfDuKw/koPNFOqgoWt5SNa4BvRMSLwE4sO+pc6gTg5YgYR+Gw8/+mlCYCPwIeiYjxwF+BFS76KupU3OYcCiN8gG8D3yyu/6/ie1A4R/xS8Va0p4EXl9vXA8CXll6ctpLPugP4Gp8+s3kRhcft/qr4dxwHrHAB3kpcBvwyIsay7BHBx4HeSy9OozAybw2Mj4gJxWWpKvh0MKkZiojXgf4ppQ+zziKpcTniliQpRxxxS5KUI464JUnKEYtbkqQcsbglScoRi1uSpByxuCVJyhGLW5KkHPn/3cdzAeUOVlsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(cfg.arch.num_classes):\n",
    "    print('class {}'.format(i))\n",
    "    plot_roc_curve(targets[i], probs[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}