examples/DeepCross二分类.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e6b971c0",
   "metadata": {},
   "source": [
    "# DeepCross 广告点击率预测"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7496cf63-bdb5-4eaf-9bcf-856b53bd18ed",
   "metadata": {},
   "source": [
    "谷歌的DeepCross系列模型(DCN-vector, DCN-matrix, DCN-mix) 围绕解决的核心问题是三高问题。\n",
    "\n",
    "如何实现 高效率，高表达能力 的 高阶显式特征交叉？\n",
    "\n",
    "本范例演示使用 torchkeras.tabular.DeepCrossModel 模型 进行广告点击率预测。\n",
    "\n",
    "有关DeepCross模型的详细理论介绍和代码讲解，参考如下eat_pytorch_in_20_days中的讲解教程：\n",
    "\n",
    "https://github.com/lyhue1991/eat_pytorch_in_20_days/blob/master/7-6%2CDeepCross%E6%A8%A1%E5%9E%8B.ipynb\n",
    "\n",
    "\n",
    "公众号**算法美食屋**后台回复关键词：torchkeras，获取本文notebook源码和所用criteo_small数据集下载链接。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "81b4a7a1-a2d8-4d4c-a0ad-224ab0e9a9fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa19852f",
   "metadata": {},
   "source": [
    "## 一，准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "85cbdc51",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0f799967",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "dfdata = pd.read_csv('criteo_small.zip',sep='\\t',header=None)\n",
    "dfdata.columns = [\"label\"] + [\"I\"+str(x) for x in range(1,14)] + [\n",
    "    \"C\"+str(x) for x in range(14,40)]\n",
    "\n",
    "target_col = 'label'\n",
    "cat_cols = [x for x in dfdata.columns if x.startswith('C')]\n",
    "num_cols = [x for x in dfdata.columns if x.startswith('I')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6a809254",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dftrain_val,dftest_raw = train_test_split(dfdata,test_size=0.2,random_state=42)\n",
    "dftrain_raw,dfval_raw = train_test_split(dftrain_val,test_size=0.2,random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2fde517f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "352f414f95594d1b96f959f58c925b87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from torchkeras.tabular import TabularPreprocessor\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "\n",
    "#特征工程\n",
    "pipe = TabularPreprocessor(cat_features = cat_cols, onehot_max_cat_num=3)\n",
    "encoder = OrdinalEncoder()\n",
    "\n",
    "dftrain = pipe.fit_transform(dftrain_raw.drop(target_col,axis=1))\n",
    "dftrain[target_col] = encoder.fit_transform(\n",
    "    dftrain_raw[target_col].values.reshape(-1,1)).astype(np.int32)\n",
    "\n",
    "dfval = pipe.transform(dfval_raw.drop(target_col,axis=1))\n",
    "dfval[target_col] = encoder.transform(\n",
    "    dfval_raw[target_col].values.reshape(-1,1)).astype(np.int32)\n",
    "\n",
    "dftest = pipe.transform(dftest_raw.drop(target_col,axis=1))\n",
    "dftest[target_col] = encoder.transform(\n",
    "    dftest_raw[target_col].values.reshape(-1,1)).astype(np.int32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b219ae5c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torchkeras.tabular import TabularDataset\n",
    "from torch.utils.data import Dataset,DataLoader \n",
    "\n",
    "def get_dataset(dfdata):\n",
    "    return TabularDataset(\n",
    "                data = dfdata,\n",
    "                task = 'binary', # regression,binary,multiclass\n",
    "                target = [target_col],\n",
    "                continuous_cols = pipe.get_numeric_features(),\n",
    "                categorical_cols = pipe.get_embedding_features()\n",
    "        )\n",
    "\n",
    "def get_dataloader(ds,batch_size=512,num_workers=0,shuffle=False):\n",
    "    dl = DataLoader(\n",
    "            ds,\n",
    "            batch_size=batch_size,\n",
    "            shuffle=shuffle,\n",
    "            num_workers=num_workers,\n",
    "            pin_memory=False,\n",
    "        )\n",
    "    return dl \n",
    "    \n",
    "ds_train = get_dataset(dftrain)\n",
    "ds_val = get_dataset(dfval)\n",
    "ds_test = get_dataset(dftest)\n",
    "\n",
    "dl_train = get_dataloader(ds_train,batch_size=2048,num_workers=4,shuffle=True)\n",
    "dl_val = get_dataloader(ds_val,batch_size=2048,shuffle=False)\n",
    "dl_test = get_dataloader(ds_test,batch_size=2048,shuffle=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5eef69c5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for batch in dl_train:\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f35e558f",
   "metadata": {},
   "source": [
    "## 二，定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "816ad379-8854-4239-9c45-5eb1cabbdbdb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_embed_max =  32\n",
      "\n",
      " categorical_cardinality =  [1000, 529, 1000, 1000, 246, 15, 1000, 502, 1000, 1000, 1000, 1000, 26, 1000, 1000, 10, 1000, 1000, 1000, 16, 15, 1000, 63, 1000]\n"
     ]
    }
   ],
   "source": [
    "from torchkeras.tabular.models import DeepCrossConfig,DeepCrossModel\n",
    "\n",
    "model_config = DeepCrossConfig(\n",
    "    task=\"binary\",\n",
    "    input_embed_max = 32, \n",
    "    cross_type = \"mix\",\n",
    "    cross_order = 2,\n",
    "    experts_num=8,\n",
    "    low_rank=16,\n",
    "    mlp_layers = \"64-32\",\n",
    "    mlp_dropout=0.10,\n",
    "    stacked = False\n",
    ")\n",
    "config = model_config.merge_dataset_config(ds_train)\n",
    "\n",
    "print('input_embed_max = ', config.input_embed_max)\n",
    "print('\\n categorical_cardinality = ',config.categorical_cardinality)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ef0e5773-4f8c-4d84-b64d-cd98e67d1dbb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "net = DeepCrossModel(config = config)\n",
    "\n",
    "#初始化参数\n",
    "net.reset_weights()\n",
    "net.data_aware_initialization(dl_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "db727a39",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.6929, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "output = net.forward(batch)\n",
    "loss = net.compute_loss(output,batch['target'])\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2844c86-6a33-4da8-a8ed-0574ec2fad49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2d339957",
   "metadata": {},
   "source": [
    "## 三，训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c4b24884",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torchkeras import KerasModel \n",
    "from torchkeras.tabular import StepRunner \n",
    "KerasModel.StepRunner = StepRunner \n",
    "\n",
    "import torch \n",
    "from torchkeras.metrics import AUC \n",
    "\n",
    "optimizer = torch.optim.AdamW(net.parameters(),lr = 1e-3)\n",
    "#scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01, \n",
    "#                                                steps_per_epoch=len(dl_train), epochs=20)\n",
    "keras_model = KerasModel(net,\n",
    "                   loss_fn=None,\n",
    "                   optimizer = optimizer,\n",
    "                   metrics_dict = {'auc':AUC()}\n",
    "                   )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "99d5e7d0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;31m<<<<<< 🚀 mps is used >>>>>>\u001b[0m\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAGJCAYAAABcsOOZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABqt0lEQVR4nO3dd1zU9R8H8NdxwLFBBRmCgmKYiiMHubUsSzOV3Jrb8peaipojtZykpeFGS7PhVszKsszU1Fy5yoULBQ0ZDo497j6/P75ycnDAMb+M1/PxuAd3n+/n+733se59n6kQQggQERERlTITuQMgIiKiyolJCBEREcmCSQgRERHJgkkIERERyYJJCBEREcmCSQgRERHJgkkIERERyYJJCBEREcmCSQgRERHJgkkIlTkff/wxFAoFYmNj5Q6l1Ny5cwcKhQKbNm2SOxQqQUuWLEG9evWg1WrlDqXUlNbf85UrV2BqaopLly6V6PNQ8WISQvTUokWL8P3338sdRoX25MkTvPPOO3BycoK1tTU6deqEc+fOGXWuQqHI9fbKK6/o6mW+6eV2O378uMHrp6eno379+lAoFPjss8/0jv33338YPHgwfHx8YGtrCwcHB7Rs2RJff/01jN35Qq1WY/HixZg2bRpMTPL/15uSkoL09HSjrl2R5fbztLCw0KtXv359dOvWDXPmzJEpUioMU7kDICorFi1ahN69e6Nnz55yh1IhabVadOvWDRcvXsTUqVPh6OiINWvWoGPHjjh79izq1q2b5/nffvttjrK///4by5cvx6uvvqor8/f3h7e3d466M2fOREJCAlq0aGHw+itXrkR4eLjBY7Gxsbh37x569+6NmjVrIj09HQcOHMCwYcMQGhqKRYsW5Rk7AGzcuBEZGRkYMGBArnXOnDmDFStW4LfffkN0dDQUCgVq1KiBXr164f333zf4uiqLtWvXwsbGRvdYqVTmqDNmzBh07doVt27dQp06dUozPCosQVTGfPTRRwKAiImJKdXntba2FkOHDi3V58wUFhYmAIivvvpKlucvDdu3bxcAxM6dO3Vl0dHRwsHBQQwYMKBQ1xw5cqRQKBQiIiIiz3rh4eFCoVCI0aNHGzweFRUl7O3txbx58wQA8emnnxr1/G+88YawtrYWGRkZ+dZt1KiRGDx4sMFj6enpYuzYsUKhUIh27dqJzz77TPz4449i9+7dYtGiRaJJkybCwsJCrFq1yqi4ypKi/j0X5Py0tDRRpUoVMXv27EI9F5U+dsdQmRUbG4u+ffvCzs4O1apVw4QJE5CSkpKj3nfffYdmzZrB0tISVatWRf/+/REREaFX58aNG3jrrbfg4uICCwsLuLu7o3///oiLiwMgNfUnJibi66+/1jX3Dhs2zGBcUVFRMDU1xdy5c3McCw0NhUKhwKpVqwAAjx49wpQpU+Dr6wsbGxvY2dnh9ddfx8WLF4v43QHS0tIwZ84cNGvWDPb29rC2tka7du1w6NAhvXqHDx+GQqHA4cOH9cpzG4dy7do19O3bF05OTrC0tISPjw8+/PDDIse7a9cuODs7w9/fX1fm5OSEvn37Yu/evUhNTS3Q9VJTU7F792506NAB7u7uedbdunUrhBAYNGiQwePTp0+Hj48PBg8eXKAYPD09kZSUhLS0tDzrhYWF4Z9//kHnzp0NHh8+fDi2bNmCn3/+GX/++ScmT56MN954A/7+/pgxYwbOnz+P4OBgTJkyBcHBwTnOv3//PkaMGAFnZ2eoVCo0aNAAGzdu1KuT+Xuwfft2zJw5Ey4uLrC2tsabb76Z4+8FAHbu3Kn7u3J0dMTgwYNx//79HPWM/X158uQJhg0bBgcHB9jb22P48OFISkrK8/uWlRACarU6z+4vMzMzdOzYEXv37jX6uiQvdsdQmdW3b194enoiMDAQJ0+exIoVK/D48WN88803ujoLFy7E7Nmz0bdvX4waNQoxMTFYuXIl2rdvj/Pnz8PBwQFpaWno0qULUlNTMX78eLi4uOD+/fv46aef8OTJE9jb2+Pbb7/FqFGj0LJlS7zzzjsAkGtzrrOzMzp06IAdO3bgo48+0ju2fft2KJVK9OnTBwBw+/ZtfP/99+jTpw+8vLwQFRWFdevWoUOHDrhy5Qrc3NwK/f1Rq9X48ssvMWDAAIwePRrx8fHYsGEDunTpgtOnT6NJkyYFvuY///yDdu3awczMDO+88w48PT1x69Yt/Pjjj1i4cCEAaexEZvKWn6pVq+rGP5w/fx4vvPBCjvEQLVu2xPr163H9+nX4+voaHevPP/+MJ0+e5JpYZLV582Z4eHigffv2OY6dPn0aX3/9NY4dOwaFQpHndZKTk5GYmIiEhAQcOXIEX331FVq1agVLS8s8z/vrr78AAC+88EKOY99++y327NmDU6dOoUGDBgCkN9zExERd90NsbCzefvttODo6ok+fPnj99ddRq1YtAFJS/OKLL0KhUGDcuHFwcnLCL7/8gpEjR0KtVmPixIl6z7dw4UIoFApMmzYN0dHRCAoKQufOnXHhwgXd69i0aROGDx+OFi1aIDAwEFFRUVi+fDmOHz+u+7sCjPt9ydS3b194eXkhMDAQ586dw5dffonq1atj8eLFeX7vMtWuXRsJCQmwtrZGz549sXTpUjg7O+eo16xZM+zduxdqtRp2dnZGXZtkJG9DDFFOmc2vb775pl75e++9JwCIixcvCiGEuHPnjlAqlWLhwoV69f79919hamqqKz9//nyObgBDCtIds27dOgFA/Pvvv3rl9evXFy+99JLucUpKitBoNHp1wsLChEqlEvPmzdMrQwG7YzIyMkRqaqpe2ePHj4Wzs7MYMWKEruzQoUMCgDh06FCOOLI/Z/v27YWtra24e/euXl2tVpvjesbcwsLCdOdZW1vrxZVp3759AoDYv3+/0a9dCCHeeustoVKpxOPHj/Osd+nSJQFAfPDBBzmOabVa0bJlS113UOb3JLfumMDAQL3X9/LLL4vw8PB8Y501a5YAIOLj43M8v5eXlwgKCtKV7d27V7i5uQkAombNmuLXX3/V+1726tVLzJw5U1d/5MiRwtXVVcTGxupdu3///sLe3l4kJSUJIZ793GrUqCHUarWu3o4dOwQAsXz5ciGE1KVRvXp10bBhQ5GcnKyr99NPPwkAYs6cOboyY35fMv+es//se/XqJapVq5bv9y4oKEiMGzdObN68WezatUtMmDBBmJqairp164q4uLgc9bds2SIAiFOnTuV7bZIfu2OozBo7dqze4/HjxwOQPgEDQEhICLRaLfr27YvY2FjdzcXFBXXr1tV1S9jb2wMAfv311wI1/+bF398fpqam2L59u67s0qVLuHLlCvr166crU6lUuk/+Go0GDx8+hI2NDXx8fIyeFZIbpVIJc3NzANKgz0ePHiEjIwPNmzcv1LVjYmLw559/YsSIEahZs6besawtBI0bN8aBAweMurm4uOjOS05OhkqlyvG8mbMckpOTjY5VrVZj37596Nq1q+5TeW42b94MAAZbTDZt2oR///3X6E/jAwYMwIEDB7BlyxYMHDjQ6LgfPnwIU1NTvYGVAHD27FlER0dj5MiRAKRulQEDBqBly5bYvXs3Jk2ahBEjRuid07NnT13XmhACu3fvRvfu3SGE0Ps76NKlC+Li4nL8LgwZMgS2tra6x71794arq6vu7+rvv/9GdHQ03nvvPb0ZKN26dUO9evWwb98+AMb/vmQaM2aM3uN27drh4cOHUKvVeX7vJkyYgJUrV2LgwIF46623EBQUhK+//ho3btzAmjVrctSvUqUKAFSqKf7lGbtjqMzKPluiTp06MDExwZ07dwBI4zyEELnOqjAzMwMAeHl5ISAgAMuWLcPmzZvRrl07vPnmmxg8eLAuQSkoR0dHvPzyy9ixYwfmz58PQOqKMTU11RvzoNVqsXz5cqxZswZhYWHQaDS6Y9WqVSvUc2f19ddfY+nSpbh27ZredE4vL68CX+v27dsAgIYNG+ZZr0qVKrmObciLpaWlwXEfmeN88uvSyGr37t1ISUnJtytGCIEtW7agYcOGaNSokd4xtVqNGTNmYOrUqfDw8DDqeWvVqqXrBhkwYADeeecddO7cGaGhoQWKP9PZs2fRvHlzXXKyefNm1KhRA7t27dLN/nBwcMDw4cN15zg7OyMmJgaAlAg8efIE69evx/r16w0+R3R0tN7j7H8vCoUC3t7eur+ru3fvAgB8fHxyXKtevXo4duwYAON/XzJlT1Qyk4XHjx8XuNtk4MCBmDx5Mn7//XdMnz5d75h4OmYkv641KhuYhFC5kf2filarhUKhwC+//GJwul7WT51Lly7FsGHDsHfvXvz22294//33dWNN8hvUmJv+/ftj+PDhuHDhApo0aYIdO3bg5ZdfhqOjo67OokWLMHv2bIwYMQLz58/XjZGYOHFikRes+u677zBs2DD07NkTU6dORfXq1aFUKhEYGIhbt27p6uX2zzhrQlQQaWlpePTokVF1nZycdD8bV1dXREZG5qiTWVaQ8TGbN2+Gvb093njjjTzrHT9+HHfv3kVgYGCOY5999hnS0tLQr18/3RvwvXv3AEhvjHfu3IGbm5uutcmQ3r1744svvsCff/6JLl265FqvWrVqyMjIQHx8vF4rxMOHD/Ve9507d9C0aVO93+eWLVvqXSsiIkKXwGb+Dg0ePBhDhw41+NzZky+5GPobBWD0OivZeXh4GPw9fPz4MQDo/R1S2cUkhMqsGzdu6H2iv3nzJrRaLTw9PQFILSNCCHh5eeG5557L93q+vr7w9fXFrFmz8Ndff6FNmzYIDg7GggULABT8k1PPnj3x7rvv6rpkrl+/jhkzZujV2bVrFzp16oQNGzbolT958qTI/yR37dqF2rVrIyQkRC/27INlMz9xPnnyRK888xNvptq1awNAvitO/vXXX+jUqZNRMYaFhel+Xk2aNMHRo0eh1Wr1BqeeOnUKVlZWRv0MASlpOXToEIYNG2aweyerzZs3Q6FQ6LpOsgoPD8fjx491g0GzWrRoERYtWoTz58/nOcA3sysmv4G69erVAyB9P7ImBXZ2dnrnuri44PTp03rnZrY4ANIb9oYNG3QtUU5OTrC1tYVGozG6derGjRt6j4UQuHnzpi6uzJae0NBQvPTSS3p1Q0NDdceN/X0pCUIIXcKWXVhYGExMTIz+fSJ5cUwIlVmrV6/We7xy5UoAwOuvvw5AGpehVCoxd+7cHJ+mhBB4+PAhAKnZPSMjQ++4r68vTExM9LoHrK2tc7xR58XBwQFdunTBjh07sG3bNpibm+dY6EypVOaIbefOnQanOhZU5ifLrNc/deoUTpw4oVevVq1aUCqV+PPPP/XKs/enOzk5oX379ti4cWOORbuyPkdhx4T07t0bUVFRCAkJ0ZXFxsZi586d6N69u15CcevWLb3WnKy2bdsGrVabb1dMeno6du7cibZt2+boCgCA999/H3v27NG7rVu3DgAwbNgw7NmzR5cEZ3Z/ZLdhwwYoFAqDs16yatWqFQBpvEVWzz//PM6cOaNr0ejRowfOnz+POXPm4Pbt2zh69CimTp0KQJpd9NZbb+HevXuYMGECAOl34K233sLu3bsNJgOG4v7mm28QHx+ve7xr1y5ERkbq/q6aN2+O6tWrIzg4WO/v45dffsHVq1fRrVs3AMb/vhSVodewdu1axMTE4LXXXstx7OzZs2jQoEGhu1qplJX+WFiivGWOpvf19RXdu3cXq1evFoMHDxYAxMCBA/XqZs5WaN26tViyZIlYu3at+OCDD0TdunV1Mxz27NkjatSoISZOnCjWrFkjVqxYIVq0aCHMzMzEiRMndNfq2rWrsLa2FkuXLhVbt24VJ0+ezDfW7777TgAQtra2onv37jmOz5kzRwAQw4YNE+vXrxfjx48XVatWFbVr1xYdOnTQ1SvM7JiNGzfqZhGtW7dOTJ8+XTg4OIgGDRqIWrVq6dXt37+/MDU1FQEBAWL16tXi9ddfF82aNcvxnBcuXBA2NjaiWrVqYsaMGWL9+vVi5syZonHjxkbHlZuMjAzx4osvChsbGzF37lyxevVq0aBBA2FrayuuXbumV7dWrVo5XkOmZs2aCTc3txyzjrL78ccfBQARHBxsdIy5zY6ZMGGCaN68uZg1a5ZYv369+OSTT0SLFi0EADF+/Hijrt2wYcMci7KlpKQIe3t7sWfPHl3ZokWLhImJiQAgTE1NxfLly3WzcV599VVx+/ZtvWs8ePBA1KpVS1hZWYkJEyaIdevWicDAQNGnTx9RpUoVXb3M2TG+vr6iUaNG4vPPPxfTp08XFhYWwtvbWyQmJurqfvXVVwKA8PPzE0FBQWLGjBnCyspKeHp66s1GMub3JbfFxjKfI+sMKkMsLS3FsGHDxNKlS8Xq1avFgAEDhEKhEE2aNNGLWQhpZk/VqlXFrFmz8rwmlR1MQqjMyfyndeXKFdG7d29ha2srqlSpIsaNG6c3ZTDT7t27Rdu2bYW1tbWwtrYW9erVE2PHjhWhoaFCCCFu374tRowYIerUqSMsLCxE1apVRadOncTvv/+ud51r166J9u3bC0tLSwHAqOm6arVaV/+7777LcTwlJUVMnjxZuLq6CktLS9GmTRtx4sQJ0aFDhyInIVqtVixatEjUqlVLqFQq0bRpU/HTTz+JoUOH5ngDj4mJEW+99ZawsrISVapUEe+++65u6mr257x06ZLo1auXcHBwEBYWFsLHx6fYVqB89OiRGDlypKhWrZqwsrISHTp0EGfOnMlRL7ck5Nq1awKACAgIyPe5+vfvL8zMzMTDhw+Nji+3JOS3334Tb7zxhnBzcxNmZmbC1tZWtGnTRnz11Vd601HzsmzZMmFjY6ObMpvpo48+ErVr1xaPHj3Sld2/f1/8+eef4sGDB0IIIY4dOyaio6NzvXZUVJQYO3as8PDwEGZmZsLFxUW8/PLLYv369bo6mUnI1q1bxYwZM0T16tWFpaWl6NatW44ptkJIK9w2bdpUqFQqUbVqVTFo0CBx7969HPXy+30pahIyatQoUb9+fWFrayvMzMyEt7e3mDZtmt4040y//PKLACBu3LiR5zWp7FAIUYztZkREZFBcXBxq166NJUuW6KbkAtLsoDZt2kCpVGLv3r1wdXU1eP6uXbvQq1evXAd45ufw4cPo1KkTdu7cid69exfqGmVdz549oVAosGfPHrlDISNxTAgRUSmwt7fHBx98gE8//VRvZpSFhQV+/vlnKBQK+Pj4YNq0afjzzz9x9+5dXLt2Dd988w1atWqFoUOHFnltmYrs6tWr+Omnn3RT5ql8YEsIURljzBRYe3v7Qq1LQWVXWloaVq1ahVWrViEsLExXbmFhgV69emHu3Ln57jScl8rQEkLlD6foEpUxxkyB/eqrr3LdYI/KJ3NzcwQEBCAgIAB37tzB/fv3YWFhgeeffx5WVlZyh0dUItgSQlTGPH78GGfPns2zToMGDXIdO0BEVF4wCSEiIiJZcGAqERERyYJjQgzQarX477//YGtry02QiIiICkAIgfj4eLi5uelt0WAIkxAD/vvvP6N31SQiIqKcIiIi8t0glEmIAZm7XEZERBR4i2kiIqLKTK1Ww8PDQ2/H6NwwCTEgswvGzs6OSQgREVEhGDOcgQNTiYiISBZMQoiIiEgWTEKIiIhIFhwTUkhCCGRkZECj0cgdChWCmZlZoXcjJSKi4sEkpBDS0tIQGRmJpKQkuUOhQlIoFHB3d4eNjY3coRARVVpMQgpIq9UiLCwMSqUSbm5uMDc354Jm5YwQAjExMbh37x7q1q3LFhEiIpkwCSmgtLQ0aLVaeHh4cGfLcszJyQl37txBeno6kxAiqpQ0GuDoUSAyEnB1Bdq1A0r73yGTkELKbylaKtvYekVElVlICDBhAnDv3rMyd3dg+XLA37/04uA7KRERUSUSEgL07q2fgADA/ftSeUhI6cXCJISIiKiS0GikFhAhch7LLJs4UapXGpiEyESjAQ4fBrZulb6Wt5m+np6eCAoKkjsMIiIqgKNHc7aAZCUEEBEh1SsNHBMiA7n64jp27IgmTZoUS/Jw5swZWFtbFz0oIiIqNTdvGlcvMrJk48jElpBSVpb64rLLXIDNGE5OTpwdRERUTqSnA6tWAQEBxtV3dS3ZeDIxCSlGiYm531JSjOuLmzABSEjI/7oFNWzYMBw5cgTLly+HQqGAQqHApk2boFAo8Msvv6BZs2ZQqVQ4duwYbt26hR49esDZ2Rk2NjZo0aIFfv/9d73rZe+OUSgU+PLLL9GrVy9YWVmhbt26+OGHH4yKTaPRYOTIkfDy8oKlpSV8fHywfPlyvTodO3bExIkT9cp69uyJYcOG6R6npqZi2rRp8PDwgEqlgre3NzZs2FCg7xMRUUXzyy9Ao0bA+PFAfDxgagrkNkFQoQA8PKTpuqWBSUgxsrHJ/fbWW8b1xd27B7Rtq1/u6ZnzegW1fPlytGrVCqNHj0ZkZCQiIyPh4eEBAJg+fTo++eQTXL16FY0aNUJCQgK6du2KgwcP4vz583jttdfQvXt3hIeH5/kcc+fORd++ffHPP/+ga9euGDRoEB49epRvbFqtFu7u7ti5cyeuXLmCOXPmYObMmdixY0eBXuOQIUOwdetWrFixAlevXsW6deu4IioRVVqXLwOvvQZ07QpcuwY4OgJr1gBbtkjHsycimY+DgkpvvRCOCSlFxvaxpaUV/3Pb29vD3NwcVlZWcHFxAQBcu3YNADBv3jy88sorurpVq1ZF48aNdY/nz5+PPXv24IcffsC4ceNyfY5hw4ZhwIABAIBFixZhxYoVOH36NF577bU8YzMzM8PcuXN1j728vHDixAns2LEDffv2Ner1Xb9+HTt27MCBAwfQuXNnAEDt2rWNOpeIqKK5fx9o2lTqhjEzk1rZP/wQcHCQjiuVhscmBgWV7johTEKKUfZulKyUSuDkSeOu8/nn+o/v3Cl0SEZp3ry53uOEhAR8/PHH2LdvHyIjI5GRkYHk5OR8W0IaNWqku29tbQ07OztER0cbFcPq1auxceNGhIeHIzk5GWlpaWjSpInRr+HChQtQKpXo0KGD0ecQEVUkQjxrzahRAxg4EFCrgSVLAG9v/br+/kCPHlwxtULJb7JIu3ZSpnn/vuFxIQqFdPzpB3mjr1tU2We5TJkyBQcOHMBnn30Gb29vWFpaonfv3kjLp4nGzMxM77FCoYBWq833+bdt24YpU6Zg6dKlaNWqFWxtbfHpp5/i1KlTujomJiYQ2b5p6enpuvuWlpb5Pg8RUUUkBPD998CsWdLXunWl8i++kFpBcqNUAh07lkKAeeCYkFKkVErTcAF5+uLMzc2hMWJBkuPHj2PYsGHo1asXfH194eLigjsl2Bxz/PhxtG7dGu+99x6aNm0Kb29v3Lp1S6+Ok5MTIrP0Z2k0Gly6dEn32NfXF1qtFkeOHCmxOImIyprz54GXXpJaNq5cAQIDnx3LKwEpK5iElDJ/f2DXLqmpLCt3d6m8JPviPD09cerUKdy5cwexsbG5tlLUrVsXISEhuHDhAi5evIiBAwca1aJRWHXr1sXff/+NX3/9FdevX8fs2bNx5swZvTovvfQS9u3bh3379uHatWv43//+hydPnui9tqFDh2LEiBH4/vvvERYWhsOHDxd4cCsRUXkQGQmMHAk0ayYteGlhIY35yDaxsMxjEiIDf39pnMehQ9Io5UOHgLCwkh8MNGXKFCiVStSvXx9OTk65jvFYtmwZqlSpgtatW6N79+7o0qULXnjhhRKL691334W/vz/69esHPz8/PHz4EO+9955enREjRmDo0KEYMmQIOnTogNq1a6NTp056ddauXYvevXvjvffeQ7169TB69GgkFmY+MxFRGfb551KXy8aNUldM//7S7JcFCwBbW7mjKxiFyN7RTlCr1bC3t0dcXBzs7Oz0jqWkpCAsLAxeXl6wsLCQKUIqKv4ciai8WrhQGv/h5yclJK1ayR2RvrzeQ7PjwFQiIqIyLHOMvp+f9DUgQGoJ6d0bMCnn/RnlPHwqD8aMGQMbGxuDtzFjxsgdHhFRmRQRAQwaBLz4IvDOO882OrW0BPr2Lf8JCMCWECoF8+bNw5QpUwwey6+pjoiosklIkNb2+PRTacsPhQJ44QUgKan8jfnID5MQKnHVq1dH9erV5Q6DiKhM02qBb74BZs58tsJ2+/bSuI8SnBsgKyYhREREZcD+/cDw4dL92rWl1hB//9w3m6sImIQQERHJJCVFWuMDAF5/XdpsrmNH4P33AZVK1tBKBZMQIiKiUhYXJ0213boVuHQJsLeXWjx++qlit3xkVwHG1hIREZUPGRlAcLA0xfbTT6VdbLdvf3a8MiUgAFtCiIiISsWBA9IaH5nbXvn4AEuXSl0wlRVbQmSiEQKHHz/G1qgoHH78GJpysHCtp6cngoKC5A6DiKhcycgA3nwTePVVKQGpUgVYsQL491+gW7fK1/qRFVtCZBASE4MJN2/iXmqqrsxdpcJyb2/4OznJGBkRERWERgMcPSpNqXV1Bdq1y7kTuqmplHiYmgJjxwJz5gBVq8oTb1nDJKSUhcTEoPfly8je7nE/NRW9L1/GrgYNmIgQEZUDISHAhAnSuI5M7u5SF8uDB1IrR506Uvknn0jrf/j4yBNrWcXumGIghECiRpPvTZ2Rgfdv3MiRgADQlU24eRPqjAyjrleQvQfXr18PNzc3aLVavfIePXpgxIgRuHXrFnr06AFnZ2fY2NigRYsW+P333wv9PVm2bBl8fX1hbW0NDw8PvPfee0hISNAd//jjj9GkSRO9c4KCguDp6alXtnHjRjRo0AAqlQqurq4YN25coWMiIiouISHS3i1ZExBAetyvn5ScfPDBs3JXVyYghsiehKxevRqenp6wsLCAn58fTp8+nWf9oKAg+Pj4wNLSEh4eHpg0aRJSUlIM1v3kk0+gUCgwceLEEoj8mSStFjZHj+Z7sz92DPfT0nK9jgBwLzUV9seOGXW9pGwJRV769OmDhw8f4tChQ7qyR48eYf/+/Rg0aBASEhLQtWtXHDx4EOfPn8drr72G7t27Izw8vFDfExMTE6xYsQKXL1/G119/jT/++AMfZP2LNMLatWsxduxYvPPOO/j333/xww8/wNvbu1DxEBEVF41GSjLy+hxoYgK88kredUjm7pjt27cjICAAwcHB8PPzQ1BQELp06YLQ0FCDy3xv2bIF06dPx8aNG9G6dWtcv34dw4YNg0KhwLJly/TqnjlzBuvWrUOjRo1K6+WUaVWqVMHrr7+OLVu24OWXXwYA7Nq1C46OjujUqRNMTEzQuHFjXf358+djz549+OGHHwrV+pA18fP09MSCBQswZswYrFmzxuhrLFiwAJMnT8aECRN0ZS1atChwLERExeno0ZwtINlptUC9epV70KkxZE1Cli1bhtGjR2P403Vqg4ODsW/fPmzcuBHTp0/PUf+vv/5CmzZtMHDgQADSm9uAAQNwKnOf46cSEhIwaNAgfPHFF1iwYEGJvw4rExMktGuXb70/nzxB13//zbfez76+aO/gYNTzFsSgQYMwevRorFmzBiqVCps3b0b//v1hYmKChIQEfPzxx9i3bx8iIyORkZGB5OTkQreE/P777wgMDMS1a9egVquRkZGBlJQUJCUlwcrKKt/zo6Oj8d9//+kSJiKisiJzX5fiqleZydYdk5aWhrNnz6Jz587PgjExQefOnXHixAmD57Ru3Rpnz57Vddncvn0bP//8M7pmm2Q9duxYdOvWTe/aeUlNTYVarda7FYRCoYC1Upnv7dWqVeGuUiG3xFgBwEOlwqtVqxp1PUUBU+zu3btDCIF9+/YhIiICR48exaBBgwAAU6ZMwZ49e7Bo0SIcPXoUFy5cgK+vL9Ly6D7KzZ07d/DGG2+gUaNG2L17N86ePYvVq1cDgO56JiYmOca0pKen6+5bWloW+HmJiEqDsZt/u7qWbBwVgWwtIbGxsdBoNHB2dtYrd3Z2xrVr1wyeM3DgQMTGxqJt27YQQiAjIwNjxozBzJkzdXW2bduGc+fO4cyZM0bHEhgYiLlz5xbuhRSAUqHAcm9v9L58GQpAb4BqZjoR5O0NZQm131lYWMDf3x+bN2/GzZs34ePjgxeebs14/PhxDBs2DL169QIgtSbduXOnUM9z9uxZaLVaLF26FCZPW2t27NihV8fJyQkPHjyAEEKXTF24cEF33NbWFp6enjh48CA6depUqDiIiIrTzZvSYmPh4dIsmPv3DY/5UCik40Y0kFd6sg9MLYjDhw9j0aJFWLNmDc6dO4eQkBDs27cP8+fPBwBERERgwoQJ2Lx5MywydwQywowZMxAXF6e7RURElNRLgL+TE3Y1aIAa2XYmclepSmV67qBBg3RdXpmtIABQt25dhISE4MKFC7h48SIGDhyYYyaNsby9vZGeno6VK1fi9u3b+PbbbxEcHKxXp2PHjoiJicGSJUtw69YtrF69Gr/88otenY8//hhLly7FihUrcOPGDZw7dw4rV64sVExERIUVHw9MmwbUrw/8+CNw+TLw3nvSseyfGTMfBwXlXC+EDBAySU1NFUqlUuzZs0evfMiQIeLNN980eE7btm3FlClT9Mq+/fZbYWlpKTQajdizZ48AIJRKpe4GQCgUCqFUKkVGRoZRscXFxQkAIi4uLsex5ORkceXKFZGcnGzcC81FhlYrDj16JLY8eCAOPXokMrTaIl3PWBqNRri6ugoA4tatW7rysLAw0alTJ2FpaSk8PDzEqlWrRIcOHcSECRN0dWrVqiU+//xzo55n2bJlwtXVVVhaWoouXbqIb775RgAQjx8/1tVZu3at8PDwENbW1mLIkCFi4cKFolatWnrXCQ4OFj4+PsLMzEy4urqK8ePHF+HVP1NcP0ciqrg0GiE2bRLCxUUIqc1DiC5dhLhyRTq+e7cQ7u7PjgFCeHhI5ZVZXu+h2SmEkG8CkZ+fH1q2bKn7dKvValGzZk2MGzfO4MDUZs2aoXPnzli8eLGubOvWrRg5ciTi4+ORlJSEu3fv6p0zfPhw1KtXD9OmTUPDhg2NikutVsPe3h5xcXGwy9b5l5KSgrCwMHh5eRWotYXKFv4ciSgvsbHSYmOZq0Z4ewOff55zmXVjVkytbPJ6D81O1tkxAQEBGDp0KJo3b46WLVsiKCgIiYmJutkyQ4YMQY0aNRAYGAhAGli5bNkyNG3aFH5+frh58yZmz56N7t27Q6lUwtbWNkeiYW1tjWrVqhmdgBAREVWrJiUTtrbA7NnA++8D2XrRAUh1OnYs9fAqDFmTkH79+iEmJgZz5szBgwcP0KRJE+zfv183WDU8PFw3sBEAZs2aBYVCgVmzZuH+/ftwcnJC9+7dsXDhQrleQqW0efNmvPvuuwaP1apVC5cvXy7liIiIiiYlBVizBhg9Wko8FApg0yZpJoyLi9zRVVyydseUVeyOyVt8fDyioqIMHjMzM0OtWrVKOaKC48+RiABpJMfevcDkycDt29IA1E8+kTuq8q3cdMdQ+WRrawtbW1u5wyAiKpIrV4CJE4EDB6THrq5AloWjqRSUqym6ZQkbkMo3/vyIKq/Hj6W9Xxo1khIQc3Nph9vr14EBA+SOrnJhS0gBmZmZAQCSkpK4qmc5lrlyq7KyD2MnqoSmTwfWr5fu9+wJLF0K1K4ta0iVFpOQAlIqlXBwcEB0dDQAwMrKqsDLp5O8tFotYmJiYGVlBVNT/gkQVQbp6cDTz5CYPRu4eBFYsAAwcncPKiH8D1wILk+HSmcmIlT+mJiYoGbNmkwgiSq4u3eBqVOlAag7d0pl7u7AyZPyxkUSJiGFoFAo4OrqiurVq+ttukblh7m5ud70byKqWJKSgMWLgSVLpOm3JibS3i/e3nJHRlkxCSkCpVLJMQVERGWIEMD27VLrx717UlnHjsDy5UxAyiImIUREVCGEhwODBgHHjkmPa9WSBp36++fcaI7KBiYhRERUIVSrBty5A1haAjNmAFOmSPep7GISQkRE5VJ6OrB1KzB4sDTmw9oa2LYNqFkT8PCQOzoyBpMQIiIqd379VVrt9No1ICMDGDFCKm/TRtawqIA4PYCIiMqNGzeA7t2B116TEhAnJ3a5lGdMQoiIqMxTq4EPPgAaNAB++gkwNQUCArjUennH7hgiIirzBg2Skg8AeP114PPPAR8feWOiomMSQkREstJogKNHgchIaSfbdu0ApVJa8yNzau2sWVJXzLJlQNeu8sZLxYdJCBERySYkRNrRNnNhMUBKROrWBdq3B+bPl8r8/IDLl6XkhCoOheCe5jmo1WrY29sjLi4OdnZ2codDRFQhhYQAvXtLLR6GmJsD//0nrf9B5UdB3kM5MJWIiEqdRiO1gOT1MbhKFcDBodRCIhkwCSEiolJ39Kh+F4whUVFSPaq4mIQQEVGpi4ws3npUPnFgKhERlZrwcOD774FGjYyr7+paouGQzNgSQkREJe7hQ2lDueeek8aCmJgA7u65726rUEj7v7RrV7pxUuliEkJERCUmKQkIDATq1AGWLgVSU4GOHaUBp8uXS3WyJyKZj4OCOCW3omMSQkRExS4jA/jiC2m9j5kzgbg4oHFj4JdfgD/+kLpj/P2BXbuAGjX0z3V3l8r9/eWJnUoP1wkxgOuEEBEVTUqK1PUSEQF4egILFkh7vJgY+Oib24qpVD4V5D2UA1OJiKhY/PWXtLKpUglYWACffQY8eAC8+y6gUuV+nlIpddFQ5cPuGCIiKpJ//gG6dQPatAG+/fZZed++wPvv552AUOXGJISIiArlzh1gyBCgSRPg558BU1Pg7l25o6LyhN0xRERUILGxwMKFwJo1QFqaVNa3rzTuo25deWOj8oVJCBERFciQIdIsFwB46SVg8WKgeXN5Y6LyiUkIERHlKT1dullZSY9nzZIGnH7yCfDKK7kvOEaUH44JISIig4SQ1uto0ACYN+9ZeevWwNmzwKuvMgGhomESQkREORw6JE237dMHuHED2Lbt2fgPgMkHFQ8mIUREpHPhAvDaa9JYjzNnABsb4OOPgX//BczN5Y6OKhqOCSEiIgDAl18Co0dL983MgDFjpPEf1avLGxdVXExCiIgIgNQCYmkJ9OwJzJ8vbTpHVJJk745ZvXo1PD09YWFhAT8/P5w+fTrP+kFBQfDx8YGlpSU8PDwwadIkpKSk6I4HBgaiRYsWsLW1RfXq1dGzZ0+EhoaW9MsgIipXEhKkwaZDhz4rc3cHwsKALVuYgFDpkDUJ2b59OwICAvDRRx/h3LlzaNy4Mbp06YLo6GiD9bds2YLp06fjo48+wtWrV7FhwwZs374dM2fO1NU5cuQIxo4di5MnT+LAgQNIT0/Hq6++isTExNJ6WUREZVZaGrB6tZRkfPQR8M030kyXTM7O8sVGlY+su+j6+fmhRYsWWLVqFQBAq9XCw8MD48ePx/Tp03PUHzduHK5evYqDBw/qyiZPnoxTp07h2LFjBp8jJiYG1atXx5EjR9C+fXuj4uIuukRU0Wi1wM6dwIcfArduSWV16kgrn/bpY3h3W6LCKMh7qGy/dmlpaTh79iw6d+78LBgTE3Tu3BknTpwweE7r1q1x9uxZXZfN7du38fPPP6Nr1665Pk9cXBwAoGrVqrnWSU1NhVqt1rsREZUnGg1w+DCwdav0VaN5duz2baBlS6B/fykBqV5dag25cgXo148JCMlHtoGpsbGx0Gg0cM7W9ufs7Ixr164ZPGfgwIGIjY1F27ZtIYRARkYGxowZo9cdk5VWq8XEiRPRpk0bNGzYMNdYAgMDMXfu3MK/GCIiGYWEABMmAPfuPStzdweWLwf8/QFXV2mFUxsbYOpUICBAuk8kt3KV/x4+fBiLFi3CmjVrcO7cOYSEhGDfvn2YP3++wfpjx47FpUuXsG3btjyvO2PGDMTFxeluERERJRE+EVGxCwkBevfWT0AA6XHv3tJxS0upK+bWLWDOHCYgVHbI1hLi6OgIpVKJqKgovfKoqCi4uLgYPGf27Nl4++23MWrUKACAr68vEhMT8c477+DDDz+ESZY2xXHjxuGnn37Cn3/+CXd39zxjUalUUKlURXxFRESlS6ORWkByG9knBDBxItCjB9CqVamGRmQU2VpCzM3N0axZM71BplqtFgcPHkSrXP5akpKS9BINAFAqlQCAzPG1QgiMGzcOe/bswR9//AEvL68SegVERPI6ejRnC0h2ERFSPaKySNbFygICAjB06FA0b94cLVu2RFBQEBITEzF8+HAAwJAhQ1CjRg0EBgYCALp3745ly5ahadOm8PPzw82bNzF79mx0795dl4yMHTsWW7Zswd69e2Fra4sHDx4AAOzt7WFpaSnPCyUiKgHG9hxHRpZsHESFJWsS0q9fP8TExGDOnDl48OABmjRpgv379+sGq4aHh+u1fMyaNQsKhQKzZs3C/fv34eTkhO7du2PhwoW6OmvXrgUAdOzYUe+5vvrqKwwbNqzEXxMRUWnJp6dZx9W1ZOMgKixZ1wkpq7hOCBGVRWo1sHIl8L//AVWrSmNC3NyAXNZ3hELxbBXUp43FRCWuXKwTQkRExomPBxYtAjw9pQ3lli2TypVKYO1aKdlQKPTPyXwcFMQEhMouJiFERGVUYiKwZAng5SWtdPr4MeDjA7zwwrM6/v7Arl1AjRr657q7S+X+/qUbM1FBcBddIqIyaMUKaUn1zK6WunWlvV7698/ZsuHvL03DPXpUGoTq6gq0a8cWECr7mIQQEZVB//wjJSC1a0sLjA0aBJjm8R9bqQSyjccnKvOYhBARySw1FfjySymJaNBAKps9W1pgbMgQwMxM1vCISgzHhBARySQtDQgOBry9gXHjpO6WTLVqASNHMgGhio0tIUREpSw9Hfj6a2DBAuDuXamsRg3g5Zelpdazz3QhqqiYhBARlaKdO4Fp06S1OwDAxQWYORMYPRqwsJA3NqLSxiSEiKgU3b0rJSDVqwPTpwNjxki73BJVRkxCiIhKiEYD7NgBODoCr7wilb33njTLZfRowNpa3viI5MYkhIiomGm1wO7dwMcfA1euAA0bAhcvAiYmgJUVMHGi3BESlQ2cHUNEVEyEAPbsAZo0Afr2lRIQBwegXz9pMCoR6WNLCBFRMThyBJg0CTh/XnpsZyc9njhRSkSIKCcmIURExSA+XkpAbGyACROAgABpp1siyh2TECKiAhIC+O03IDZWWk4dALp1k3asHTRIGohKRPnjmBAiIiMJARw8CLRtC7z2GjB+PKBWS8cUCqkFhAkIkfHYEkJE9JRGk/tOtEeOSBvJ/fmn9NjCAhg6VDqHiAqHSQgREYCQEKkl4969Z2Xu7sDkycCPPwJ//CGVmZsD774rLTTm5iZPrEQVBZMQIqr0QkKA3r2l7pas7t+XBpgC0kZyo0ZJS6y7u5d+jEQVEZMQIqrUNBqpBSR7AgI820yuShXgzBmgdu3Sj4+oIuPAVCKq1I4e1e+CyU4I4NEjIDy89GIiqiyYhBBRpSUE8MsvxtWNjCzZWIgqI3bHEFGldO2atKbHuXPG1Xd1Ldl4iCojtoQQUaXk5gbcvi1tKGdjI439MEShADw8pOm6RFS8mIQQUYWXmgqsW6c/A8bOTtrpNjwc+PprqSx7IpL5OCjo2XohRFR8mIQQUYWVmAh8/rk0q2XMGCnp+PXXZ8dfegmoVg3w9wd27QJq1NA/391dKvf3L924iSoLjgkhogonLg5YtUpqwYiNlcpq1ACmTgXatzd8jr8/0KNH7iumElHxYxJCRBVKaCjg5yclIgBQp460uunbbwMqVd7nKpVAx44lHiIRPcUkhIjKvdTUZwlG3bpSN4q7u7S6ad++gCn/0xGVSfzTJKJy6/ZtYPFi4OefpRYQKyvAxAT47TfAxUW6T0RlF/9EiajcuXJF6l557jlg/XppxdMffnh23M2NCQhRecA/UyIqN86dA956C2jYEPjuO2nfl9deA/78E+jfX+7oiKig2B1DROVCeDjQvPmzdT78/aUxH82ayRsXERUekxAiKpOEAC5fllo9AKBmTaBPH8DcHJgxA6hfX974iKjomIQQUZmi1UrjOxYuBM6fB27eBDw9pWNbt3KsB1FFwj9nIioTMjKALVuARo2AXr2Av/+WWj3+/vtZHSYgRBWL7H/Sq1evhqenJywsLODn54fTp0/nWT8oKAg+Pj6wtLSEh4cHJk2ahJSUlCJdk4jkk5YGfPklUK+etKvt5cvSvi4zZwJ370r7vRBRxSRrErJ9+3YEBATgo48+wrlz59C4cWN06dIF0dHRButv2bIF06dPx0cffYSrV69iw4YN2L59O2bOnFnoaxKRvJKSgMmTgVu3pH1cFiyQko+FCwEnJ7mjI6KSpBAic6x56fPz80OLFi2watUqAIBWq4WHhwfGjx+P6dOn56g/btw4XL16FQcPHtSVTZ48GadOncKxY8cKdU1D1Go17O3tERcXBzs7u6K+TKJKRaPJe/8VtVraFG748Ge71C5bJt1/5x3A2lqeuImoeBTkPVS2lpC0tDScPXsWnTt3fhaMiQk6d+6MEydOGDyndevWOHv2rK575fbt2/j555/RtWvXQl8TAFJTU6FWq/VuRFRwISHSINJOnYCBA6Wvnp5S+cOHwJw50iyXkSOBQ4eenRcQAEyaxASEqLKRbXZMbGwsNBoNnJ2d9cqdnZ1x7do1g+cMHDgQsbGxaNu2LYQQyMjIwJgxY3TdMYW5JgAEBgZi7ty5RXxFRJVbSIg0fiN72+r9+9ICYyqVtMcLADz/vNRiQkSVm+wDUwvi8OHDWLRoEdasWYNz584hJCQE+/btw/z584t03RkzZiAuLk53i4iIKKaIiSoHjQaYMCFnAgI8K0tNBZo0kbpiLl0CXnmlVEMkojJItpYQR0dHKJVKREVF6ZVHRUXBxcXF4DmzZ8/G22+/jVGjRgEAfH19kZiYiHfeeQcffvhhoa4JACqVCqr89vgmolwdPSrt35KfZcukLhoiIkDGlhBzc3M0a9ZMb5CpVqvFwYMH0apVK4PnJCUlwSTbQgHKpyPehBCFuiYRFd1//xlX78GDko2DiMoXWVdMDQgIwNChQ9G8eXO0bNkSQUFBSExMxPDhwwEAQ4YMQY0aNRAYGAgA6N69O5YtW4amTZvCz88PN2/exOzZs9G9e3ddMpLfNYmo+Gg0wJ49wLx5xtV3dS3ZeIiofJE1CenXrx9iYmIwZ84cPHjwAE2aNMH+/ft1A0vDw8P1Wj5mzZoFhUKBWbNm4f79+3ByckL37t2xcOFCo69JREWXnAxs2gQsXSqt75EfhQJwd5em6xIRZZJ1nZCyiuuEEOXuyRPAxwfIXP+valVg3DhpKu7IkVJZ1v8qmWuB7Nol7XxLRBVbQd5DuYEdEeXr0SMp2QAABwegRQtpefWAAGDEiGfre9jbS7Nksg5SdXcHgoKYgBBRToVqCYmLi4NGo0HVzP9KTz169AimpqblvvWALSFEknPngE8/Bb7/Hrh+HfDwkMqjo6WkxNTAx5j8VkwlooqtxFdM7d+/P7Zt25ajfMeOHejfv39hLklEZYQQwIED0joezZoB27YBKSnATz89q1O9uuEEBJASjo4dgQEDpK9MQIgoN4VKQk6dOoVOBib7d+zYEadOnSpyUERU+jIygK1bgRdeAF59Ffj9dymBGDhQahH53//kjpCIKppCjQlJTU1FRkZGjvL09HQkJycXOSgiKn3x8cDo0UBiImBlJd2fNAmoVUvuyIiooipUS0jLli2xfv36HOXBwcFo1qxZkYMiopIXHQ2sWfNsJkuVKsD06cCCBUBEhDSYlAkIEZWkQrWELFiwAJ07d8bFixfx8ssvAwAOHjyIM2fO4LfffivWAImoeN24Ia3vsWmTtJ+Lr++z9TtmzZI1NCKqZArVEtKmTRucOHECHh4e2LFjB3788Ud4e3vjn3/+QTuuRkRUJp0+Le1y6+MDrFsnJSAtWz5bx4OIqLRxsTIDOEWXKpIHD4D+/YEjR56VdesGfPCB1ALCJISIilOJL1YWHh6e5/GaNWsW5rJEVAKcnKTFw8zMgEGDgClTgAYN5I6KiKiQSYinpycUeXx80mg0hQ6IiApPrQbWrwd27JAWDFOppGm2334rLTTm7i53hEREzxQqCTl//rze4/T0dJw/fx7Lli3T20yOiErHf/8BK1YAa9dKiQgAbN8ODBki3W/VSr7YiIhyU6gkpHHjxjnKmjdvDjc3N3z66afw5yYRRKXi6lXgs8+klo70dKns+eelLpd+/eSNjYgoP8W6gZ2Pjw/OnDlTnJckqpSM2X8lLEwa25E5tLxtW2mwabdugEmh5r0REZWuQiUh6sz23qeEEIiMjMTHH3+MunXrFktgRJVVSIjhnWg//xyoUwdo2lQq8/KSlle3sgKmTmWXCxGVP4VKQhwcHHIMTBVCwMPDw+DGdkRknJAQaS2P7BPn790D+vSRNo0LD5daRwDgxx+lWS9EROVRoZKQQ4cO6T02MTGBk5MTvL29YZrb1ppElCeNRmoByWvlHo0GOH/+WRLCBISIyrNCZQwdOnQAAFy5cgXh4eFIS0vD48ePcf36dQDAm2++WXwRElUSR4/qd8EYIoTU/UJEVBEUKgm5ffs2/P398c8//0ChUCBz0dXMLhquE0JUcJGRxVuPiKisK9QY+gkTJsDT0xPR0dGwsrLCpUuX8Oeff6J58+Y4fPhwMYdIVLFdugQsXPisiyU/xtYjIirrCpWEnDhxAvPmzYOjoyNMTEygVCrRtm1bBAYG4v333y/uGIkqpJMngR49pF1sZ82Sxne4u+e+l4tCIa16yj0iiaiiKFQSotFoYGtrCwBwdHTEf//9BwCoVasWQkNDiy86ogpGCODAAeCll6QptT/8ICUXvXsD1aoBy5dL9bInIpmPg4JyrhdCRFReFWpMSMOGDXHx4kV4eXnBz88PS5Ysgbm5OdavX4/atWsXd4xEFUJ4OPDWW8Dff0uPTU2Bt98Gpk0DfHyksnr1gF27DK8TEhQEcDFiIqpIFELkNSHQsF9//RWJiYnw9/fHzZs38cYbb+D69euoVq0atm/fjpdeeqkkYi01BdmGmMhY6elA3bpAdDQwejQweTKQ24bTxqyYSkRUFhXkPbRQSYghjx49QpUqVfLcXbe8YBJCRZWUBGzcKO1me/Dgs/U8Tp6UVj11cpI3PiKiklKQ99BiW1msatWqxXUponLryRNgzRqp6yQmRirbvh0YPFi6/+KLckVGRFT2cHlTomIQFSUlHmvWAJlbK3l6ShvK9e4tZ2RERGUXkxCiIoqIAJ57DkhJkR43aADMmAH06ycNPiUiIsP4L5KoEGJjAUdH6b6HhzTdNilJSj66dwdMCjX5nYiocmESQlQAZ84AgYHAr78CYWFA9epS+Z49gJ1d7guNERFRTvy8RpQPIYBDh4BXXgFatpQSjqQk4LffntWxt2cCQkRUUExCiHKh1QJ790pdLS+9BPz+u7RWx5AhwOXLz2a8EBFR4bA7higXMTHS4NLUVMDCAhg5EpgyRZr1QkRERcckhOiplBSpi+XNN6XHzs7S8ukmJsDEidJjIiIqPkxCqNKLjweCg4Fly4AHD6S9XZo1k44tXixvbEREFRmTEKqw8tt/JTZW2rV21SpppVNAmm4bHS1LuERElY7sA1NXr14NT09PWFhYwM/PD6dPn861bseOHaFQKHLcunXrpquTkJCAcePGwd3dHZaWlqhfvz6Cg4NL46VQGRISIo3d6NQJGDhQ+urpKZXHxQGTJgG1agELFkgJiI8P8NVXwM2bwOuvyxw8EVElIWtLyPbt2xEQEIDg4GD4+fkhKCgIXbp0QWhoKKpnLsCQRUhICNLS0nSPHz58iMaNG6NPnz66soCAAPzxxx/47rvv4Onpid9++w3vvfce3Nzc8GZmZz9VaCEh0lLp2bdmvH9fKt+6Fdi5U5pm+8ILwMyZQM+e3KWWiKi0FdsuuoXh5+eHFi1aYNWqVQAArVYLDw8PjB8/HtOnT8/3/KCgIMyZMweRkZGwtrYGADRs2BD9+vXD7NmzdfWaNWuG119/HQsWLDAqLu6iW35pNFKLx717ho8rFIC7O7B0qbS2xyuvcH0PIqLiVJD3UNm6Y9LS0nD27Fl07tz5WTAmJujcuTNOnDhh1DU2bNiA/v376xIQAGjdujV++OEH3L9/H0IIHDp0CNevX8err76a63VSU1OhVqv1blQ+HT2aewICSK0jERGAkxPw6qtMQIiI5CRbEhIbGwuNRgPnbPMenZ2d8eDBg3zPP336NC5duoRRo0bpla9cuRL169eHu7s7zM3N8dprr2H16tVo3759rtcKDAyEvb297ubh4VG4F0Wyi4ws3npERFRyZB+YWlgbNmyAr68vWrZsqVe+cuVKnDx5Ej/88APOnj2LpUuXYuzYsfj9999zvdaMGTMQFxenu0VERJR0+FRCsgwZypOra8nGQURE+ZNtYKqjoyOUSiWioqL0yqOiouDi4pLnuYmJidi2bRvmzZunV56cnIyZM2diz549uhkzjRo1woULF/DZZ5/pdf1kpVKpoFKpivBqqCwQAli3Lu86mWNC2rUrnZiIiCh3srWEmJubo1mzZjh48KCuTKvV4uDBg2jVqlWe5+7cuROpqakYnG3zjvT0dKSnp8Mk2z7qSqUSWq22+IKnMiVzaLVCAaxZAzRvLt3PPt4j83FQEGfClLgLF6S5zhcuyB0JEZVhsnbHBAQE4IsvvsDXX3+Nq1ev4n//+x8SExMxfPhwAMCQIUMwY8aMHOdt2LABPXv2RLVq1fTK7ezs0KFDB0ydOhWHDx9GWFgYNm3ahG+++Qa9evUqlddEpSchQVpO/eOPn5U1aQKcOQPs2gXUqKFf391dKvf3L8UgK6vdu4H9+6X50kREuZB1nZB+/fohJiYGc+bMwYMHD9CkSRPs379fN1g1PDw8R6tGaGgojh07ht+y7qOexbZt2zBjxgwMGjQIjx49Qq1atbBw4UKMGTOmxF8PlZ79+4F33wXCwwEzM2DUKGm100z+/kCPHnmvmEol6Mcfn33N1m1KRJRJ1nVCyiquE1J2xcZKq51+9530uFYtaRxIly7yxkVZREUBWcd1RUUBBhYfJKKKqVysE0JUEEIAW7YAzz8vJSAKhdQVc+kSE5Ay59df835MRPQUkxAqFx48AEaPllpCGjYETpwAPv8csLGROzLKTrtvH7RP+720SiW0+/bJHBERlVXcRZfKLCGezWhxdQU+/RR49Aj44APA3Fze2Cq1+/elLhYD/nj0CC1++gm2Gg0AwESjgfqnn/D377/jpapVDV/P2TnnKGIiqhQ4JsQAjgmR3+XLUsvHwoXSDriVjUYIHH3yBJFpaXA1N0c7Bwcoy8oa8y+/DPzxR66HtQoFTLL8W8n+2OD18lhMkIjKl4K8h7IlhMqU1FQgMBBYtAhITwemTAH+/rty7fESEhODCTdv4l5qqq7MXaXCcm9v+Ds5yRjZU2PGAOfOAU+eGDycPeHIMwFxcJCmORFRpcSWEAPYEiKPv/6SptpevSo97t5dWnzM3b1w1yvTrQm5CImJQe/Ll5H9jzIz6l0NGhQpERFCIEWrRZJWiySNBslZv2YrS9Jq9Y9nKTONjsaoefPQ5o8/8m/pyB6DQgGFEECvXkBwMGfOEFUwBXkPZRJiAJOQ0hUfD8yYISUcQkjvSStXAn36FL4FpMy3JmQjhECiRgOf06fxXx4b4FQxNcXsWrV0iURytmQha1nW5CJrveLU59AhBC9bBrukJJgace0MExOorazw4bRpuP7GG3jOygo+lpbSVysr1FKpYGrC8fJE5RmTkCJiElK6du4E+vaV7g8fDnz2GZDbGEZjlFRrguZpoqC7abVI1GiQlOV+9mPGPk7SanPEW9LMFApYmZjASqmEZdav2cqsTEz0j2crc3j4EN7vvQevw4eRV84oABzw88PgadMQU6VKrjHVsbTUJSbPWVrCx8oKz1lZobqZGRRlvCWLiDgmhMoBrRbI/MDbuzfwv/9Jq5zmsseg0TRCYMLNmwbf0DPLRoWG4kZSElJySSpye5xaRvL1VnZ2eN7KSj8hePo1ayKRV5mliUnxtThUrw5tmzbQHD0K06ezYgzRKJXo/MoruPbGG7ielITryckIzfL1RnIyUrRaXEtKwrWkJODhQ73z7ZXKHInJc0+TFetiWgq3PHbhEZVnbAkxgC0hJUcI4NtvgcWLpSXVi9LikZ1GCGyMjMQ7168X30UNUACwViphbWIifX16s8r6ONsxYx6fi49Ht0uX8n3+Q40bo2MuLQmyadIE4uLFfFtCFE2aAOfPGzyuFQL3UlN1icn1pCTd/TspKXm2FNUwN9dLTHyefvW0sDA62SpvXXhEZRW7Y4qISUjJuHNHmgiRue3P7NlF21YkPiMDp9RqHFercSwuDifVaiTk8Uk8q7Z2dmhoY1OoxMHCxKREugU0QsDz5EncT001+IargPSmGPbii2Xr0/mDB9JCLllkDlY1OGj1wQNpbZACSNFocCslRS8xuZ6UhNDkZMSmp+d6Xmb3TtbEJHP8SdbunZIeEExUmbA7hsoUjUYaaPrhh0BSEqBSSTvfTp5csOtEpKTgeFwcjqvVOB4Xh4sJCcg+FNLSxMSowZfzvbzKXGuCUqHAcm9v9L58GQpA7w0x880wyNu7bCUgQI5l2YVSCY2NDS6PHIl6GzZAkZAARdbk8NdfgSFDCvQUFkolGlhbo4G1dY5jj9LTcSOza+dpYnL9afdOch7dO3ZPu3fqWlhg36NHuXbhKQBMvHkTPRwdy973nqicY0uIAWwJKT7//itNuz19Wnrcvj3wxRfAc8/lfZ5GCPyTkKCXdERkaSbPVEulQht7e93teSsr1Dl1qvy1JmRhqFvAQ6VCUFntFujXD9i1S+pryz71NjpaWldkzx5pqpNCIU172ratxMPK7N4xNP4kv+4dQ8pkNxhRGcTumCJiElJ8Ro0CNmwA7OykZddHjXo2IDWrrF0rx592rcRn61pRAmhsY/Ms6bCzg7uFRY5rZTatA4ZbE8pD03q5GSCZkQFUqwao1dLCY+vWPZvqlNWOHVJf3JMn0i/Do0dAMQ0mLYwUjQa3U1IQmpSEHTEx2BYdne859a2s4O/khNZ2dnjRzg5VzMxKIVKi8odJSBExCSmajAzA9GlH36NH0qqnCxYAbm7P6hjTtWKrVKKVnZ0u6fCztYWNqXE9iOWuNaG8io+Xmre8vPJfeCyzVeTOHeDIEcDWttTCzMvhx4/R6eLFAp/3vJUVWtvZobW9PVrb2eE5KyuYlMVEkaiUMQkpIiYhhaNWA9OnAxERwA8/PFtoTCME/k1I0CUcx4zsWmlobV2kT//lpjWhvNNoCtaqUdD6JcyYAcHO5ub4uFYtnIyPx19xcbienJyjXlVTU7R6mpS0srNDSzu7Yps6TFSeMAkpIiYhzxj7Rv7jj9JaH/fvA7DMwPIDajyuUfSuFaLSUNAuvJi0NJxUq/GXWo2/4uJwOj4eKdkGRGf+jme2lLS2t0dNlYoLrlGFxySkiJiESIxZNyEqChg1IwU/3VEDDeNg3iwOGTUToM32f7YoXStEpaEoXXjpWi0uJiTokpK/1GqDrX1u5ua61pLWdnZoamsLFZeppwqGSUgRMQnJf92EES6uuHRTgzPJcdA65fxnW1OlQtti7FohKg3F2YUXkZKCE1mSkvMJCcjI9u9WpVCgua2trguntb09nM3Ni+OlEMmGSUgRVfYkJLOP/F5KKvJcAjOTFqhnaoNXXNm1QpSbJI0GZ+Pj9VpLDC20VtvCQq8Lx9gEnmOgqKzgYmVUJEefPJGapPP5/9UFzqh5zQVLhtvCQcVfJaK8WCmVaOfggHYODgCknZNvJifrJSWXExNxOyUFt1NS8F1UFADARqmE39PWkszpwQ7ZpgdzyXkqr9gSYkBlbwn58n4kRt8Izbfedz7PY5BrwZbfJqLcxT1dLyczKTE0qFsBac2SzKQkXqMxuGljeVoXhyoWdscUUWVNQtK1WnwZGYmpN24jEfnvwfI5GmNiR64gSVRSNELgcmIi/oqLk8aXqNW4aWB6cG7KwwrBVPGwO4YKRAiBHx8+xLTbt6U9NgAgA9IcQ0P/t7QAYlRw0jqUWoxElZFSoUAjGxs0srHBmBo1AADRaWm6Aa8/P3qES4mJuZ4vAESkpuKXhw/xhqNjKUVNZDzODavk/lar0enCBfS4dAnXkpLgaGaG91EXWFBfqpB9GVMtpMRktTdquPKTFVFpq25ujh6Ojlhcpw5m1qxp1DndL11C07//xvs3bmBndDQeGJg+TCQHtoRUUndTUjDz9m1sebpnhoWJCSa5u2NazZqwUZgi5BZw7yMFMO4mUD3LP6wYFbDGGx53nNCunUzBExEAwLUA03kvJCTgQkICVt6/DwDwtrREO3t7tLO3R3sHB9S2sOBCalTqmIRUMk/S0xEYHo7l9+4h9elwoCHOzljg5QWPLNNqlywBBg50Ao47Ar5PgGppwENz4JIDFFoFgnaVqZW3iSqldg4OcFep8t01+njTpjihVuNoXByOPnmCfxITcTM5GTeTk/HVgwcApIQmMylp5+AAX2tr7oVDJY4DUw2oiANT07RaBP/3H+bduYOHGRkAgJccHPBpnTp4wcBGYkIA06YBK1YAWVtuPTyAoCDA37+UAieiPBVm1+gn6ek4rlbj6JMnOBoXhzPx8UjP9lbgYGqKNnZ20rRie3s0t7WFOVd3JSNwdkwRVaQkRAiBkNhYTL99Wzeqvr6VFT6tUwevV62ab/OrRgMcPQpERgKurkC7dmwBISprirprdLJGg9Px8fjzaVLyV1wcErPthWNhYgI/W1u0f5qUtLKz49YLZBCTkCKqKEnIybg4TL51C3+p1QAAZzMzzPPywggXF5jm8YkmIQGwsSmtKImoOBTniqkZWi0uJCRI3TdPb9lXd1UCaGprq+vCaWtvDycuOU9gElJk5T0JuZWcjBm3b2NnTAwAwNLEBFM9PDDFwwO2+Xxy+fVXYNAg4JNPgFGjSiNaIirrhBC4lpT0LCl58gR3Dcywed7KSm9cSS0jt2/gkvMVC9cJqaQepadjwd27WHX/PtKFgALAcBcXzPPyQg2VKt/z09OBiROBhw+Bp13MRERQKBR43toaz1tb4x03NwDSBn1H4+J0XThXkpJw9eltfWQkAKlLqP3ThKSdvT2et7LK0QXMJecrN7aEGFDeWkJStVqsun8fC+7exZOng067VKmCJXXqoFEB+lU+/xwICACcnIDr14GnW1wQEeUrNi1Nb7Dr2fj4HOsuVzM1RdunU4Lb2dvjTkoK+l25wiXnKxh2xxRReUlChBDYHh2NmWFhCEtJAQA0srbGp3Xq4NWqVQt0rago4LnnALUa+PJLYOTIkoiYiCqLhIwMnMycFhwXh5NqNZKzDXZVAAanFmce45Lz5RO7YyqBo0+eYMqtWzgdHw8AcDM3xwIvLwxxcSnUH+yHH0oJSLNmwPDhxR0tEVU2Nqam6Fy1Kjo//UCUptXiXHy81IUTF4dDjx/nmIGTVeaS80eePMFLVbhHVUXFlhADynJLyPWkJEy/fRt7YmMBANYmJphWsyYCPDxgXci5s3//DbRsKa0Ncvw40Lp1cUZMRJTT5qgoDL56Nd96lgoF2jo44EU7O/jZ2cHP1haOnIVTphXkPVT2lWdWr14NT09PWFhYwM/PD6dPn861bseOHaFQKHLcunXrplfv6tWrePPNN2Fvbw9ra2u0aNEC4eHhJf1SSlRMWhrG37iBBmfOYE9sLEwAvOvqipt+fpjt6VnoBAQAjhyRvr79NhMQIiodNYxMJJKFwIHHjzH/7l288e+/cPrrL9Q5eRKDrlzBinv3cEqtRmoeLSpUtsnaHbN9+3YEBAQgODgYfn5+CAoKQpcuXRAaGorq1avnqB8SEoK0tDTd44cPH6Jx48bo06ePruzWrVto27YtRo4ciblz58LOzg6XL1+GhZFTxcqaZI0Gy+/dQ2B4ONQaaZjXG9WqYXHt2qhvbV0szzF5MtCxI/B00DsRUYkzZsn5GioV9jRogDPx8TilVuNUfDyuJSXhdkoKbqek6Pa+Mlco0NTGRmopeXrjXjjlg6zdMX5+fmjRogVWrVoFANBqtfDw8MD48eMxffr0fM8PCgrCnDlzEBkZCeunb8j9+/eHmZkZvv3220LHVRa6Y7RCYEtUFGaGhSHi6dS1pjY2+KxOHfaPElGFUNgl509nJiVPE5PsC6kBgKOZGfxsbXVJSUtbWziYmZXQK6GsysXsmLS0NFhZWWHXrl3o2bOnrnzo0KF48uQJ9u7dm+81fH190apVK6xfvx6AlMTY29vjgw8+wLFjx3D+/Hl4eXlhxowZes+RXWpqKlKzzFFXq9Xw8PCQLQk59Pgxpty6hXMJCQCkufYLvbwwyNm5WDeU+uYb4MUXpVkxRERyKOqS80II3E5JeZaUqNU4n5CANANvbfWsrOBna6sbX+JrbZ3n6tFUOOVidkxsbCw0Gg2cnZ31yp2dnXHt2rV8zz99+jQuXbqEDRs26Mqio6ORkJCATz75BAsWLMDixYuxf/9++Pv749ChQ+jQoYPBawUGBmLu3LlFe0HF4GpiIj64fRs/PXwIALBVKjGzZk1McHeHZTFv2HLjxrMVUf/5B6hXr1gvT0RkFH8nJ/RwdCz0iqkKhQJ1LC1Rx9ISA5++n6Q+XXY+Myk5qVbjdkoKriUl4VpSEr6OigIgrSbdLDMpedpq4q5SFagbh6u9Fk25naK7YcMG+Pr6omXLlroy7dPBST169MCkSZMAAE2aNMFff/2F4ODgXJOQGTNmICAgQPc4syWktESlpeGjsDB8GRkJDaQ9Gca4ueEjT88S24shIEBaIfW11wAfnxJ5CiIioygVCnQsxm5mlYmJrhsmU0xamq4b56RajdNqNeI0GhyLi8OxuDhdPVdzc72ZOM1tbXPdqI+rvRadbEmIo6MjlEolop5mpJmioqLg4uKS57mJiYnYtm0b5s2bl+OapqamqF+/vl75888/j2PHjuV6PZVKBZURy5oXVm6ZcpJGg2UREVgcEYGEp4NOezo64pPateFjZVVi8fzyC/DTT4CpqbRKKpN2IqronMzN0a1aNXSrVg2ANO7uelISTsXH4+TTFpN/EhIQmZaGPbGxumUQTAA0tLZ+NujV1hbPW1tjb2wsel++nGNQ7f3UVPS+fJmrvRpJtiTE3NwczZo1w8GDB3XjNbRaLQ4ePIhx48blee7OnTuRmpqKwYMH57hmixYtEBoaqld+/fp11KpVq1jjN5bBTNncHD0cHbEnNhb/PZ3t08LWFkvr1EG7El4rPS1N2h8GACZMYDcMEVVOJgoF6llbo561NYY+/eCbpNHgbJaZOKfUakSkpuKfxET8k5iIL57uiWNjYoJ0IQzO6hGQBtZOvHkTPRwd2TWTD1m7YwICAjB06FA0b94cLVu2RFBQEBITEzH86ZKdQ4YMQY0aNRAYGKh33oYNG9CzZ09Ue5rRZjV16lT069cP7du3R6dOnbB//378+OOPOHz4cGm8JD2ZI7+z/6LeS0vD6v/+AwB4Wlgg0MsLfatXL9ZBp7lZsULaF6Z6dWD27BJ/OiKicsNKqZQ228vyYfC/1FS9mThn1Gok5LMuSeZqr4efPMHLnM2YJ1mTkH79+iEmJgZz5szBgwcP0KRJE+zfv183WDU8PBwm2UYuh4aG4tixY/jtt98MXrNXr14IDg5GYGAg3n//ffj4+GD37t1o27Ztib+erDRCYMLNm7nuiwAADkolLrdoAatiHnSamwcPgMwerE8+AeztS+VpiYjKLTeVCr2cnNDradeKRgh8Fh6O6WFh+Z7b9Z9/0MTGBo0zb9bWaGRjA7tcxphURly23YDiWCfk8OPH6HTxYr71DjVuXKwDsvKSkiKNATlwAPj9d4Az04iICs7Y/++58bKw0CUlmQmKp4VFqbSGl4ZyMUW3oovMsrJrcdQrDhYWwIwZwPTpHIxKRFRYxq72+mujRriUmIiLCQnSLTER91JTEZaSgrCUFHz/dPArIC3J0ChLUtLYxga+1tal1lIuFyYhJcTVyKm1xtYrCq1WumW2ADIBISIqPKVCgeXe3uh9+TIUMLza63Jvb9S3tkZ9a2v0zbINycP0dPyTJSm5mJCAy4mJiNdocFytxnG1Wu9adS0t9bpzGtvYFHgtk7KM3TEGFEd3jEYIeJ48mWem7K5SIezFF0t89PTXXwOffQasXCntEUNEREVX1NVeM6VrtQhNStIlJZm3KAPL0QNAVVNTNMrWndPA2hqqAvaxl9RCa+Vi2fayrLj2jinMvgjFTa2WFiN78ABYsgSYOrVEn46IqFIpyRVTo9LS9JKSi4mJuJqYCI2BukpIy9I3zjYQ1iWXNbBKcqE1JiFFVJwb2BVXplxYH3wAfPopULcucOkSUAq9P0REVEJStVpcyWwxydJy8igjw2D96mZmObpzriYlof+VKzla6YvrAzKTkCIq7l105dpb4Pp1oGFDaXn2ffuArl1L/CmJiKiUCSFwPzU1R3fO9eTkPJeJMKQ4hgpwdkwZU9z7Ihhr0iQpAenalQkIEVFFpVAo4G5hAXcLC92y9IC0Amz22Tln4+ORnMdia5kLrR198qRU3reYhFRQ+/YBP/8MmJlJa4MQEVHlYqVUoqWdHVpmaY3YEhWFQVev5ntuaS0fweWqKqhdu6SvEycCzz0nayhERFRGuJWh5SMAtoRUWBs3Sl0wXbrIHQkREZUVxiy05q5SlfhmqpnYElJBKRRAnz5AMYyrJSKiCiJzoTXg2WyYTJmPg7y9S233XyYhFcyWLcCTJ3JHQUREZZW/kxN2NWiAGtnWEHFXqUpl/aqs2B1TgZw8CQwaBDg7A9euAaXUmkZEROWMv5MTejg6yrJ8RFZMQioIrRYYP166360bExAiIsqbXMtHZMXumApi0ybg77+lMSCLFskdDRERUf6YhFQAcXHAjBnS/Y8+krpjiIiIyjomIRXAvHlAdLS0Ud24cXJHQ0REZBwmIeXctWvAihXS/aAgblBHRETlBwemlnNVqgBvvw08egS89prc0RARERmPSUg55+wsrY6ayy7OREREZRa7Y8qp7JsgmjKdJCKicoZJSDm1eDHQowdw65bckRARERUOPz+XQ/fvAwsXAomJ0v4wderIHREREVHBsSWkHJo2TUpAWrWSlmknIiIqj5iElDPHjwObN0u75K5cKX0lIiIqj5iElCMaDfD++9L9kSOBZs3kjYeIiKgomISUI199BZw7B9jbS2NCiIiIyjMmIeWEEMAXX0j3P/4YqF5d1nCIiIiKjLNjygmFAjh8GPjyS2DMGLmjISIiKjomIeWIpSUwfrzcURARERUPdseUcUIA338vDUolIiKqSJiElHF79wK9egFt2jARISKiioVJSBmWkgIEBEj3X34ZUCrljYeIiKg4MQkpw5YuBcLCgBo1gBkz5I6GiIioeDEJKaPu3QMWLZLuL1kC2NjIGw8REVFxYxJSRn3wAZCUJI0FGTBA7miIiIiKX5lIQlavXg1PT09YWFjAz88Pp0+fzrVux44doVAocty6detmsP6YMWOgUCgQFBRUQtEXv6NHga1buT8MERFVbLKvE7J9+3YEBAQgODgYfn5+CAoKQpcuXRAaGorqBpYFDQkJQVpamu7xw4cP0bhxY/Tp0ydH3T179uDkyZNwc3Mr0ddQ3OztgdatgYYNgaZN5Y6GiIioZMjeErJs2TKMHj0aw4cPR/369REcHAwrKyts3LjRYP2qVavCxcVFdztw4ACsrKxyJCH379/H+PHjsXnzZpiZmZXGSyk2jRoBx44B5ajxhoiIqMBkTULS0tJw9uxZdO7cWVdmYmKCzp0748SJE0ZdY8OGDejfvz+sra11ZVqtFm+//TamTp2KBg0a5HuN1NRUqNVqvZvcFApphVQiIqKKStYkJDY2FhqNBs7Oznrlzs7OePDgQb7nnz59GpcuXcKoUaP0yhcvXgxTU1O8n7nvfT4CAwNhb2+vu3l4eBj/IorR1KnSgNQykAMRERGVONm7Y4piw4YN8PX1RcuWLXVlZ8+exfLly7Fp0yYojBzROWPGDMTFxeluERERJRVyri5dAj7/HPj0U+Dvv0v96YmIiEqdrEmIo6MjlEoloqKi9MqjoqLg4uKS57mJiYnYtm0bRo4cqVd+9OhRREdHo2bNmjA1NYWpqSnu3r2LyZMnw9PT0+C1VCoV7Ozs9G6lSQhgwgRpWXZ/f+Cll0r16YmIiGQhaxJibm6OZs2a4eDBg7oyrVaLgwcPolWrVnmeu3PnTqSmpmLw4MF65W+//Tb++ecfXLhwQXdzc3PD1KlT8euvv5bI6yiqPXuAP/4AVCrgs8/kjoaIiKh0yD5FNyAgAEOHDkXz5s3RsmVLBAUFITExEcOHDwcADBkyBDVq1EBgYKDeeRs2bEDPnj1RrVo1vfJq1arlKDMzM4OLiwt8fHxK9sUUQnLys/1hPvgA8PKSNx4iIqLSInsS0q9fP8TExGDOnDl48OABmjRpgv379+sGq4aHh8PERL/BJjQ0FMeOHcNvv/0mR8jF6rPPgLt3AXd3YNo0uaMhIiIqPQohhJA7iLJGrVbD3t4ecXFxJTo+JDkZqFULiIkBtm0D+vUrsaciIiIqFQV5D5W9JaQys7QEzp8HvvoK6NtX7miIiIhKF5MQmdWoAcyaJXcUREREpa9crxNSXmVkSMuyExERVWZMQmTwxRdAu3ZAtoVeiYiIKhUmIaXs0aNn3S/cIZeIiCozjgkpBRoNcPQoEBkJ7NghJSK+vsC778odGRERkXyYhJSwkBBpSfZ79/TL33oLMOV3n4iIKjF2x5SgkBCgd++cCQgAzJ0rHSciIqqsmISUEI1GagHJaym4iROlekRERJURk5AScvSo4RaQTEIAERFSPSIiosqISUgJiYws3npEREQVDZOQEuLqWrz1iIiIKhomISWkXTtpZ1yFwvBxhQLw8JDqERERVUZMQkqIUgksXy7dz56IZD4OCpLqERERVUZMQkqQvz+wa5e0SV1W7u5Sub+/PHERERGVBVwuq4T5+wM9ejxbMdXVVeqCYQsIERFVdkxCSoFSCXTsKHcUREREZQu7Y4iIiEgWTEKIiIhIFkxCiIiISBZMQoiIiEgWTEKIiIhIFkxCiIiISBacomuAEAIAoFarZY6EiIiofMl878x8L80LkxAD4uPjAQAeHh4yR0JERFQ+xcfHw97ePs86CmFMqlLJaLVa/Pfff7C1tYUitx3oKhG1Wg0PDw9ERETAzs5O7nAqDX7f5cHvuzz4fZdHSXzfhRCIj4+Hm5sbTEzyHvXBlhADTExM4O7uLncYZY6dnR3/OciA33d58PsuD37f5VHc3/f8WkAycWAqERERyYJJCBEREcmCSQjlS6VS4aOPPoJKpZI7lEqF33d58PsuD37f5SH3950DU4mIiEgWbAkhIiIiWTAJISIiIlkwCSEiIiJZMAkhIiIiWTAJoVwFBgaiRYsWsLW1RfXq1dGzZ0+EhobKHVal8sknn0ChUGDixIlyh1Ip3L9/H4MHD0a1atVgaWkJX19f/P3333KHVaFpNBrMnj0bXl5esLS0RJ06dTB//nyj9h0h4/3555/o3r073NzcoFAo8P333+sdF0Jgzpw5cHV1haWlJTp37owbN26UeFxMQihXR44cwdixY3Hy5EkcOHAA6enpePXVV5GYmCh3aJXCmTNnsG7dOjRq1EjuUCqFx48fo02bNjAzM8Mvv/yCK1euYOnSpahSpYrcoVVoixcvxtq1a7Fq1SpcvXoVixcvxpIlS7By5Uq5Q6tQEhMT0bhxY6xevdrg8SVLlmDFihUIDg7GqVOnYG1tjS5duiAlJaVE4+IUXTJaTEwMqlevjiNHjqB9+/Zyh1OhJSQk4IUXXsCaNWuwYMECNGnSBEFBQXKHVaFNnz4dx48fx9GjR+UOpVJ544034OzsjA0bNujK3nrrLVhaWuK7776TMbKKS6FQYM+ePejZsycAqRXEzc0NkydPxpQpUwAAcXFxcHZ2xqZNm9C/f/8Si4UtIWS0uLg4AEDVqlVljqTiGzt2LLp164bOnTvLHUql8cMPP6B58+bo06cPqlevjqZNm+KLL76QO6wKr3Xr1jh48CCuX78OALh48SKOHTuG119/XebIKo+wsDA8ePBA7/+Nvb09/Pz8cOLEiRJ9bm5gR0bRarWYOHEi2rRpg4YNG8odToW2bds2nDt3DmfOnJE7lErl9u3bWLt2LQICAjBz5kycOXMG77//PszNzTF06FC5w6uwpk+fDrVajXr16kGpVEKj0WDhwoUYNGiQ3KFVGg8ePAAAODs765U7OzvrjpUUJiFklLFjx+LSpUs4duyY3KFUaBEREZgwYQIOHDgACwsLucOpVLRaLZo3b45FixYBAJo2bYpLly4hODiYSUgJ2rFjBzZv3owtW7agQYMGuHDhAiZOnAg3Nzd+3ysBdsdQvsaNG4effvoJhw4dgru7u9zhVGhnz55FdHQ0XnjhBZiamsLU1BRHjhzBihUrYGpqCo1GI3eIFZarqyvq16+vV/b8888jPDxcpogqh6lTp2L69Ono378/fH198fbbb2PSpEkIDAyUO7RKw8XFBQAQFRWlVx4VFaU7VlKYhFCuhBAYN24c9uzZgz/++ANeXl5yh1Thvfzyy/j3339x4cIF3a158+YYNGgQLly4AKVSKXeIFVabNm1yTEG/fv06atWqJVNElUNSUhJMTPTfipRKJbRarUwRVT5eXl5wcXHBwYMHdWVqtRqnTp1Cq1atSvS52R1DuRo7diy2bNmCvXv3wtbWVtc3aG9vD0tLS5mjq5hsbW1zjLmxtrZGtWrVOBanhE2aNAmtW7fGokWL0LdvX5w+fRrr16/H+vXr5Q6tQuvevTsWLlyImjVrokGDBjh//jyWLVuGESNGyB1ahZKQkICbN2/qHoeFheHChQuoWrUqatasiYkTJ2LBggWoW7cuvLy8MHv2bLi5uelm0JQYQZQLAAZvX331ldyhVSodOnQQEyZMkDuMSuHHH38UDRs2FCqVStSrV0+sX79e7pAqPLVaLSZMmCBq1qwpLCwsRO3atcWHH34oUlNT5Q6tQjl06JDB/+dDhw4VQgih1WrF7NmzhbOzs1CpVOLll18WoaGhJR4X1wkhIiIiWXBMCBEREcmCSQgRERHJgkkIERERyYJJCBEREcmCSQgRERHJgkkIERERyYJJCBEREcmCSQgRERHJgkkIEVUKhw8fhkKhwJMnT+QOhYieYhJCREREsmASQkRERLJgEkJEpUKr1SIwMBBeXl6wtLRE48aNsWvXLgDPukr27duHRo0awcLCAi+++CIuXbqkd43du3ejQYMGUKlU8PT0xNKlS/WOp6amYtq0afDw8IBKpYK3tzc2bNigV+fs2bNo3rw5rKys0Lp1a4SGhpbsCyeiXDEJIaJSERgYiG+++QbBwcG4fPkyJk2ahMGDB+PIkSO6OlOnTsXSpUtx5swZODk5oXv37khPTwcgJQ99+/ZF//798e+//+Ljjz/G7NmzsWnTJt35Q4YMwdatW7FixQpcvXoV69atg42NjV4cH374IZYuXYq///4bpqam3DKeSE4lvk8vEVV6KSkpwsrKSvz111965SNHjhQDBgzQbTO+bds23bGHDx8KS0tLsX37diGEEAMHDhSvvPKK3vlTp04V9evXF0IIERoaKgCIAwcOGIwh8zl+//13Xdm+ffsEAJGcnFwsr5OICoYtIURU4m7evImkpCS88sorsLGx0d2++eYb3Lp1S1evVatWuvtVq1aFj48Prl69CgC4evUq2rRpo3fdNm3a4MaNG9BoNLhw4QKUSiU6dOiQZyyNGjXS3Xd1dQUAREdHF/k1ElHBmcodABFVfAkJCQCAffv2oUaNGnrHVCqVXiJSWJaWlkbVMzMz091XKBQApPEqRFT62BJCRCWufv36UKlUCA8Ph7e3t97Nw8NDV+/kyZO6+48fP8b169fx/PPPAwCef/55HD9+XO+6x48fx3PPPQelUglfX19otVq9MSZEVLaxJYSISpytrS2mTJmCSZMmQavVom3btoiLi8Px48dhZ2eHWrVqAQDmzZuHatWqwdnZGR9++CEcHR3Rs2dPAMDkyZPRokULzJ8/H/369cOJEyewatUqrFmzBgDg6emJoUOHYsSIEVixYgUaN26Mu3fvIjo6Gn379pXrpRNRHpiEEFGpmD9/PpycnBAYGIjbt2/DwcEBL7zwAmbOnKnrDvnkk08wYcIE3LhxA02aNMGPP/4Ic3NzAMALL7yAHTt2YM6cOZg/fz5cXV0xb948DBs2TPcca9euxcyZM/Hee+/h4cOHqFmzJmbOnCnHyyUiIyiEEELuIIiocjt8+DA6deqEx48fw8HBQe5wiKiUcEwIERERyYJJCBEREcmC3TFEREQkC7aEEBERkSyYhBAREZEsmIQQERGRLJiEEBERkSyYhBAREZEsmIQQERGRLJiEEBERkSyYhBAREZEs/g8X3D88MdW6ZgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* background: */\n",
       "    progress::-webkit-progress-bar {background-color: #CDCDCD; width: 100%;}\n",
       "    progress {background-color: #CDCDCD;}\n",
       "\n",
       "    /* value: */\n",
       "    progress::-webkit-progress-value {background-color: #00BFFF  !important;}\n",
       "    progress::-moz-progress-bar {background-color: #00BFFF  !important;}\n",
       "    progress {color: #00BFFF ;}\n",
       "\n",
       "    /* optional */\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #000000;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='10' class='progress-bar-interrupted' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      33.33% [10/30] [9:30:51<19:01:42]\n",
       "      <br>\n",
       "      ████████████████████100.00% [79/79] [val_loss=0.5115, val_auc=0.7541]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;31m<<<<<< val_auc without improvement in 5 epoch,early stopping >>>>>> \n",
      "\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_auc</th>\n",
       "      <th>lr</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.498843</td>\n",
       "      <td>0.732795</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.483020</td>\n",
       "      <td>0.757059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.476000</td>\n",
       "      <td>0.766988</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.475342</td>\n",
       "      <td>0.768114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.466130</td>\n",
       "      <td>0.779811</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.471723</td>\n",
       "      <td>0.772724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.458502</td>\n",
       "      <td>0.789287</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.471544</td>\n",
       "      <td>0.773577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.450772</td>\n",
       "      <td>0.798471</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.473855</td>\n",
       "      <td>0.774328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.442359</td>\n",
       "      <td>0.808066</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.475346</td>\n",
       "      <td>0.771335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.432851</td>\n",
       "      <td>0.818195</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.481294</td>\n",
       "      <td>0.766819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0.422288</td>\n",
       "      <td>0.828934</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.488921</td>\n",
       "      <td>0.762048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0.410224</td>\n",
       "      <td>0.840528</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.498291</td>\n",
       "      <td>0.757735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0.397602</td>\n",
       "      <td>0.851858</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.511527</td>\n",
       "      <td>0.754068</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  train_loss  train_auc     lr  val_loss   val_auc\n",
       "0      1    0.498843   0.732795  0.001  0.483020  0.757059\n",
       "1      2    0.476000   0.766988  0.001  0.475342  0.768114\n",
       "2      3    0.466130   0.779811  0.001  0.471723  0.772724\n",
       "3      4    0.458502   0.789287  0.001  0.471544  0.773577\n",
       "4      5    0.450772   0.798471  0.001  0.473855  0.774328\n",
       "5      6    0.442359   0.808066  0.001  0.475346  0.771335\n",
       "6      7    0.432851   0.818195  0.001  0.481294  0.766819\n",
       "7      8    0.422288   0.828934  0.001  0.488921  0.762048\n",
       "8      9    0.410224   0.840528  0.001  0.498291  0.757735\n",
       "9     10    0.397602   0.851858  0.001  0.511527  0.754068"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras_model.fit(\n",
    "    train_data = dl_train,\n",
    "    val_data= dl_val,\n",
    "    ckpt_path='checkpoint',\n",
    "    epochs=30,\n",
    "    patience=5,\n",
    "    monitor=\"val_auc\", \n",
    "    mode=\"max\",\n",
    "    plot = True,\n",
    "    wandb = False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26147264-b2c2-4da6-986a-b721d1373c5d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bec9f969",
   "metadata": {},
   "source": [
    "## 四，评估模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60a0d622",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_model.evaluate(dl_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6d68615",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_model.evaluate(dl_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6de6ee16",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_model.evaluate(dl_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e785f2fa",
   "metadata": {},
   "source": [
    "## 五，使用模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e55e8fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm \n",
    "\n",
    "net,dl_test = keras_model.accelerator.prepare(net,dl_test)\n",
    "net.eval()\n",
    "preds = []\n",
    "with torch.no_grad():\n",
    "    for batch in tqdm(dl_test):\n",
    "        preds.append(net.predict(batch))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c30b8f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "yhat_list = [yd.sigmoid().reshape(-1).tolist() for yd in preds]\n",
    "yhat = []\n",
    "for yd in yhat_list:\n",
    "    yhat.extend(yd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "168cb211",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dftest_raw = dftest_raw.rename(columns = {target_col: 'y'})\n",
    "dftest_raw['yhat'] = yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d5e7ab7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(dftest_raw['y'],dftest_raw['yhat'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e8f8552",
   "metadata": {},
   "source": [
    "## 六，保存模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9eeae65b",
   "metadata": {},
   "source": [
    "最佳模型权重已经保存在ckpt_path = 'checkpoint'位置了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eb2899d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "net.load_state_dict(torch.load('checkpoint'))"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 5446668,
     "sourceId": 9035782,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30747,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}