04_SimpleFineTuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c2445d81-8b47-4887-85ce-0a7b58ac4c7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Onsiter\\.conda\\envs\\nlp\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b83f250-eca5-40e3-88ee-cd105d0bbe0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_projector.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.weight', 'pre_classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_name = 'distilbert-base-uncased'\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21658bf6-82b6-48b8-b062-b16c06b118e1",
   "metadata": {},
   "source": [
    "## 1. Data Processing"
   ]
  },
  {
   "attachments": {
    "529a6f3c-2533-454c-8ab9-d461213332ff.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAABBUAAAFZCAYAAADHIxe3AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAEKoSURBVHhe7d1fbBtndvD/Q4VK6rbZFsgaJlMJu8TrqwLuhdALUYCxiYAgBTYIYBIyYLI3gfemry4WVXyRVJTjyCSSC0do2voFCtQwikJ0YZVyf9gEbeFWycKt6AKtiq7RAi3ccrNWK7putkACpNiVEv7mPPMMOaT4X5TEP99PMPHMcDh/KIric+Y85wmUHAIAAAAAANChwA9/+EOCCgAAAAAAoGPlTIW9vT2zIhgMmn8baXc79fjxYzl16pRdqq/Xx+X86uP86uP8GuP89uP86uP86uP8GuP89uP86uP86uP8GuP89uP86uP86utkf54x+y8AAACAAfHo0SM7BwDHi6ACAAAAMGB+9KMf2TkAOF4EFQAAAAAAQFeqaiqMj4+blQAAAAAAYDR1MkjkvqBCJ08GAAAAAADDIxAIdBQXoPsDAAAAAADoCkEFAAAAAADQFYIKAAAAAACgK4FisWg6S3z11Vfy/PPPU1MBAAAAAIARpTUVisWiXRI5deqUnauPQo0AAAAAAMCgUCMAAAAAADgSBBUAAAAAAEBXCCoAAAAAAICuEFQAAAAAAABdIagAAAAAAAC6QlABAAAAAAB0haACAAAAAADoytje3p54EwAAAAAAGG2dxAnIVAAAAAAAAF0ZCwaD4k39In81IIFAzXQ+K0X7eN/azkrcOdf4rd6e6YFfj/uZzs/JXkvtcQ96bcVb8X379KbMfWeDBsetd73l1+Vq3q6xan4O3nZm/5a7LiPuM/OS8Y5TNcUlu93sce/5AAAAADA8OokT9HGmQlo2SyUp6ZRPi6wlZb7HjfWem0hIzjnf3IWQXdFLvtejtClp5/UItxNY0AZ2NGUXOhfL7thjutNBry10IVe5Bl0xtyo7dt+L02YTo+q4j1Yltu/nn5eNy3b28kZbjfvUSovXa3mzckwz5SQxYR9TVY/r+adkpjagAQAAAAAjZDC6P0xflNU5kfU7G06jsCjZ8wEJnM9IRv9tcDe5+o66fY73uL8heD9TWV9z57kqQ8DfgK+5m16+A151h9w7z6xky/tptP+4ZK7qHXzvWlqJyqINtNywx67OAPC9JpNJWXfm1hPh8nVXXVfXd9sbvd6Nfj4HMBGRKeef9YcFd1nd33Ca9CLpZQ1NpGTDl4XQUE8DUxGJOO9JeVCovC8AAAAAYMQMbk2FtS2JrHh3k7WBOyMp706y0+DWRrTX2M9fDUtS7B1xbYxfnqmk2kdT9q74jqzOpWTGCx7cz8jMZafRmneeU3Wn3Gk0LzgNdXuHfXPZadJGmzTM13Iir9l9OI3fa16j1u7fPfZ1iTzQpn8HnIZ2zPlnq+Dsz7mO+cSUzWTQO+jrklzQ64jKojmuPc5S1D3uA/ta2HPq/G5789fbqPr52HXd2i7IlvNP7HTEXXbk75qQgswuzZqMh9TdVtcQk5gGphI3ugyi1CpIYc3550xEDiMvBQAAAAAGwWAEFe7fkKTTgIudm6004ObiMus1Vr271i85jWZlMxvcdHc3Tb783OlF0xDWVPvivZzT/I5J/Kw+EpLZc07ze81pLOp2VioakMx2nW4NtvtBZEkbzotO872RKYnoedbcbXcbxTXH7pbpdrEoEZOt4DT27eq69PpvJ6Sg2Qo2i6EZk+FQm9XQ9PW2/D+fLlQd1zlPye74Xn/b9WF51nndozK77My37AIxJZcW3KyGcmCn1uUZ37U6U233kqrHbVBFAzUAAAAAMKL6OKiQkhmvAac1AZwGXKP+/MWCuY/tNt5r2bvc9RQemo4Bkpx0jxNO6PKWFDRd3zaUlQYW9HH3TnxIEqZx6tDAgp5fbePTby4ilfvrtWzA4aBsF47wnbjsmIwLu74Or5vEjGiWga1r0ER1TQU3eNL09e6RSvaIuzwV8f3sa4Ia0ZfcYEHLLhDTiyazZD0xL9ce2HV+VTUTnOl2ohLEUvZx3YfyZ04AAAAAwCgajEKNOjW5IxyKmBwANxhQq15/fCty2nQMkNVHvuOU0/VDkrjtrtvJulkE5W4ONtuh3Cjvuq++DWB0Y7tgsgy0se11BdisbQTvU5SNO86ztOvGAe6wN329e8r5Gay43TcqXUyKkl1xczG8YI8JOulyq0KMjuhrur91WdeuC12KLrk/dw1OHLheBAAAAAAMsMGtqeA3XdOv3naXSC9oI7smPd5XTDF0Nm4amLl72hStFFY0S+aOvltkMHThunvH3GQd2AKFpg6BLZjoqLqT3gb37rp37LzcMFkS7bIN67lVuTjtBUdsgMJee30hiZxx/rFdPIq3rpk7/h1r+nr32ERCLpnMANttYXtDcnp9NVkFJntgLScbrRr55f0dRFQumkDTuiRv2tcAAAAAAEbQ2N7ennjT4HIa95o14PV5j6ZMgUVviMLoki3CqI9pHQGvK4XWIrBFBgOBsNMwrtzt16EPN5e9rhH6WExWV9wghSl+6DuWNnD9wyG2pZyKr8e+5uzDzYZozNcdRM9HC0+Wz/WS08i35xrdMgUJy7UhJmYl7iyb42gNCHOn3t1XOCHuth2PYND89e41f2bA//v/tA5GpeuDpzpI05y3v31qayo4U1XxSZ9yoMl5TqNtAAAAAGAQdRInCOzu7pZ0Rjc+ceKEueuLI6B1EHyNcR3mceaydvloVvQRAAAAAIDDozdXd3d37ZJIMBi0c/WN6QbehCPkjZhg6wK4w1cSUAD6ji2E6makVHeTak63bTLcLAAAANCnOokTDEdNhYFUKQTpTYfVfQBAr9jf25ZFUTX7SLtN2QUAAABgSBFUAIAa3tCrJkPhrl1p1MlUsJkM7uQWd9Xna/aRWwvFXQcAAAAMI4IKAOC3nZV5HY3FjjAy22yMFB1NxtZGKZW0IOy6JBeyIqbQq26gdVK8YWoBAACA4UNQAQB8ivd0hJGYrL7mVjhxRxapz902LbOm61L7XSMAAACAYUFQAQC6VHioA5wCAAAAo4ugAgDssy6FNuogRE7HnP9vtbUtAAAAMIwIKgCAT+hsXDRUkLrrDgaZv9u4poK77brk7rllG/NXtVgjw0gCAABgdIzt7e2JNwHAyJtISC6fFrk8Y0Z02JDGNRW8bdcTYbPtzOWYrD5aFK3G4NZi0NEfApK5b7YGAAAABkIncYLA7u5uSWd04xMnTphq5wAAAAAAYPTozbLd3V27JBIMBu1cfYGSjSJoUGF8fJygAgAAAAAAI0qDCp3EBaipAAAAAAAAukJQAQAAAAAAdIWgAgAAAAAA6ApBBQAAAAAA0BWCCgAAAAAAoCsEFQAAAAAAQFcIKgAAAAAAgK4QVAAAAAAAAF0JFIvFks589dVX8vzzz0upZBYBYKS8+OKL8vHHH9slYDB94xvfkOvXr8u3v/1tuwYAAKAzgUBAisWiXRI5deqUnasvULJRhL29PRkfHyeoAGAk6Ycnn38YdPo+Vh988AGBBQAA0JVOvxfT/QEAgCGytrYmr7zyinz44Yd2DQAAwOEhqAAAwBB5+eWXCSwAAIAjQ1ABAIAhQ2ABAAAcFYIKAAAMIQILAADgKBBUAABgSBFYAAAAh42gAgAAQ4zAAgAAOEwEFQAAGHIEFgAAwGEZ29vbE28CAADDicACAABoVydxAjIVAAAYEQQWAABAr40Fg0HxJgAAMNwILAAAgFY6iRMESg6d0bSG8fFxsYsAMFICgQCffxh4L7zwgnz/+9+3S6194xvfkB/+8Id2CQAAoPPvxQQVAMBBUAGjiPc9AACo1en3A2oqAAAAAACArhBUAABgRL311lt2DgAAoDsEFQAAAAAAQFcIKgAAAAAAgK4QVAAAAAAAAF0Z01EfvAkAAAAAAIy2TuIEZCoAAAAAAICujAWDQfEmAAAAAAAw2jqJE5CpAAAAAAAAukJQAQAAAAAAdIWgAgAAAAAA6ApBBQAAAAAA0BWCCgDQre2sxAMBCVzN2xUAAADAaCGoAABdKt7LiWR3pLQUtWsAAACA0UJQAQAAAAAAdGVsb29PvAkAAAAAAIy2TuIEZCoAQFeKsnFnXaYiIbsMAAAAjJ6xYDAo3gQAaEdRsufnRVZKsjhtVwEAAABDopM4AZkKANCxkCRuXxdZCEjmvrsGAAAAGEUEFQCgKyGZPReTrULRLgMAAACjh6ACAAAAAADoCkEFAAAAAADQFYIKANCl0Nm4SCIsgat5uwYAAAAYLQQVAKBbEwnJlUpSWoraFQAAAMBoIagAAAAAAAC6QlABAAAAAAB0Zezx48ei05MnT+wqAMBRK96K29oMRcmeD0jmvru+t/KSCcQlu20XG9nOSjyQcbbez5zn+awUm2xTn3tdgUBA4rcYhhMAAKCfeXECnVoZO3XqlOh08uRJuwoAcHxCkrhdksVpu3gcTK2IRTmMShHpfElyF0J2CQAAAP3IixPo1ArdHwCgD4QiUxI7HXHmfJkKNhsgc9W9w69TOYPhfsZkDGTLj3lZA7WZDl52gq6fkZSsS3KyRbZCbRaCHsscIy43Htp1ExGZmouInnG1NrMhAAAAMBQIKgBAP5hebHAHPyVbp3ekVCrJ5rKztJKVcueBtaTkyo+lZKbp0JaaAbEpaYnJ6qOcJCbs6lY0wBBNmQyDUum6RB6s2weisng74ey1lrO+1MH+AQAAMNAIKgBAX4tJ/KzbdI++lDb/VqTlkg1ERF9bldjljQ5qHLRpuyDrc6ty0XTHCEliofYcAAAAMMoIKgBAX5uSSKO7/nW7H/RWsbBl5wAAAID9CCoAwKBaK0jBzpqMAjtbxVl/kLCA1noAAAAAGiGoAAADKyUbtiBj/m5KZHm2PGJD6q4t23gvVz/Y0K7pWUmvJeWGOU5RsivOcQAAAACLoAIADKq5tMiKO/rDzINV2VnSkEJIEitaX2HGrJ+XuFSqIEQkMtfG6A9VorKYT0sqqscJS+Gcs2/7SH2M/gAAADBKAru7uyWd2dvbkxMnTpgq4gAwarQBPlCffzrM40pEduqOwNCPdEjLsBQWSrJoij6iH1y5csVMAAAAHv1evLu7a5dEgsGgnauPTAUAGDnawHczHPZPh5tloBkP8VvlQTEBAAAw4AIle2tOMxXGx8fJVAAwkrRBzecfRg2ZCgAAoFan34vJVAAAAAAAAF0hqAAAAAAAALpCUAEAcPS00KSp4ZARd/BLazsr8dp1fuZxr/5Dk+3K+3enjB168+Aq9Sh6t08AAIDBRVABAHA85lZlp7QoOhCmoQGDyaSs28X9nAb9Qk7ij0qmn99OdktmzmedtbXyknF2uulso9uVzJCYTQIQHQlJ4nZJNpftIgAAwIgjqAAAOH6aWTCZk3h+VWJ21X7aoM9JYsIunY1LbK0gBXfRJyqL/mDF9KykJSUbZBYAAAD0HEEFAMDxm16UUqkSMGhH8V5O1uciErHLDW0XZEtiEulg3wAAAGgPQQUAwIDJSyYQkHBiXdILCQnZtfVpl4mkSPZ6RwELAAAAtGdsb29PvAkAgP6n3Ru0XsKmM9usYKIWVQxL8sym5C40Dz0AAACgopM4AZkKAIABFZHInMhWYX+pRjebISy5cztSWipXVwAAAECPjQWDQfEmAAD6lwYK4pLdtovbG5Jbi0n8bG0WgmYozMhWdocMBQAAgC50EicgUwEA0Mf8gYSoLD6KS24yIIGAM00mZSrvFXf0bWeCDSLribC7nZ3cbhLVgYn81YDEb9lMBx3SMuANPVkTwGio3e0AAACGE0EFAED/mEhIzj8cpDN3MTtl5x3mca2n4E6L03a9f7uabaq3rd5fdKlUyWaoOrbWbWhnNIqa8wMAABgxBBUAAMdjLSnhcmZAI0XZeBiR2ZaN+15v14h2rQjIzGW7eOD9AQAADLZASW/fOLSq4/j4uLmbAwCjRtPj+fzDqLly5YqZAAAAPJ1+LyZTAQAAAAAAdIWgAgAAAAAA6ApBBQDA0bufsaMy2JoKZuQFb6SGJnUWys/zj+jQRNWIDl0qn9sB9wMAADCExrSWgjcBAHBk5lZlx4y2UJTsQk7ij9xRGnayWzJzPuusrZWXTFRk09lGtyvl05KKNmnoazBgMinrdrFrZlSITUnbRQAAgGHXSZyATAUcOv9dRabhnICDCUnidmX4xtDZuMTWClJwF310mEffcJPTs05DPyUb9bIVNKNhMifx/KrE7CoAAAD03lgwGBRvAg6LuavINJQT0GvFezlZn4tIxC43tF2QLYlJpN5wjtOLzvuzEqgAAABA+zqJE5CpAADoE3nJBAISTqxLeiEhIbu2Pu0ykRTJXidwAAAAcIwIKgAA+oR2b9AMmE1ntlkRxqJkz4cleWZTcheahx4AAABwuAgqAAD6TEQicyJbhf2lGt1shrDkzu1IaalcXQEAAADHhKACAOCYaaAgLtltu7i9Ibm1mMTP1mYhaIbCjGxld8hQAAAA6BMEFQAAxywqi4/ikpu0I4pMJmUq7xVZ9AUcTLBBZD0Rrhp9xO0mUROYaKh6u/zVgMRv2YwIHYIy4A1R2WR/VdsBAACMNoIKAIDjN5GQnG9UkcVpu16icjE75c7WbFO9rW87P/Mc3zCUNdtFl0qVrIeqbbW+Q4PRI5ztLi3beQAAgBFHUAEAcDzWkhJuece/KBsPIzLbcoSHXm/nYzITZiRlFzWLYUNmfYEKAACA0RUo6W0ex97enoyPj5u7PkAvaXoy76vhNSw/X96nGEVXrlwxEwAAgKfT78Vjjx8/Fp2ePHliVw2+F1980bwQRzV985vflA8//NAeHQAAAACAweXFCXRqZezUqVOi08mTJ+2qwffxxx+byMpRTZ988om88sorBBYAAAAAAAPPixPo1Ao1FXpkbW2NwAIAAAAAYKQQVOiRl19+mcACAAAAAGCkEFToIQILAAAAAIBRQlChxwgsAAAAAABGBUGFQ0BgAQAAAAAwCggqHBICCwAAAACAYUdQ4RARWAAAAAAADLOxvb098Sb0HoEFAAAAAMAg6SROQKbCESCwAAC18pIJxCW7bZeuBiR+q+gubGclHsg4WwAAAKDfjQWDQfEmHB4CCwDgF5XFUk4SE3ZpqSS5CyF3YSIhudKiswUAoOc0cHv1iMO2JljsCx53ouvzLUr2fEAC57PO3DE4jtcZ6KFO4gRkKvTAt771Lfna177WcpqbmzPbz8/Pm38BAACAo5OXzGRS1u1S/zvI+YYkcbskpdsJZ+6oDdrrDBwMQYUe+Pjjj6VUcj602pw++eQT+0wAAADgKOid+xlJ6ezlGd8dfO2OFpCAnSrZBLV3+u123t33+xmzfeaq+6+73j7H7qu8refhDZOxUPexfeqcr5fx4BzT7Meem3ahKx+z3H2u5vy9873l7kPn282cqNp/+fVQ1a9d5r6ua/Q6A8OLoAIAAAAw9PTO/aakdXZ5097B10ax0wDWZb35lU/LeiJsG8ftScms+9ylqNP4DktSVmXH7ksb1f59rT+IyHXnsZ1szHnsWrmuTn31ztfl7cesu5+RmQf2mI9WJeac0UyTgEXqYURyzraby85+EjdsAKIJ3f9lkXTe7n8tKfMmGGGDB3PusfWaUlGtFdT4vIFhRVABAAAAGEX3N8wd9fRLtorN9EVZnXMa3ivt310vP9dpnm84je/YuVm3ET29aIINi9PmQdeZiHksFJly/r8uhaZBhcbKx1B6HKfhXtBsgja6HMROR8y/kdMx5/9bbZ9DKhqQzLbW/LE1gLY3JLdWOZfQ2bjEDnBNwCAjqIADe++99+Tzzz+3S33Mpr3VmxpG5E2aXa+q0LtpeJ1E/9E7A/M+HRFeKmlXRbsAAD1RLGw5/49JxBbNPZDtgtNEb85r0PdS8Vbc/D2ZEc22sBkCTUxFyuGI9thAi9LAQvl7o3O9GsDQzA7zfdIGNLYK/F3D6CGogANLpVLy3HPPyZtvvtnfjTYbMddJU95MSppdroqiHxq3YNDRHAu1BuZ9OiJ0tAeT/goAODYHzRioMhERs7eHBXf5SBRl447TlNcuCEuHNWaQLfjofF/0/m6lohnJO9erS7HsTvn7pE7lkYyAEUJQAQf27rvvylNPPSUrKyty8uTJAW202WI+Gmn2ItB1mLurXsEdWyzIfY4vm0EzIs5nJFPenzcWfyVTwYuq+6fqMfq99bX7jVcVJkL7huN9CgBAD03Pmjv7qbv228b9G5JcE0kvaB2AkETOOOvWCmLCBLarRGNRmdWbNpc33O8uXlHFQ81Iqz7H4q1rLc6xc+53Nve7XOjCdTdrYS4ikYlZiTvz63c2zHcy/3bAqBnb29sTbwK68d3vfld+7ud+Tn7605/KT37yk4FstNUWFnIL7dgHLQ0oXDu9UylsNJmUKS3a4zxnczklM/6G/przJ21BH9tx/visS/JmdQeK0IWcL6qtqXoxiZ/VyHar/YrEHzmPUfSnY8PwPgUA4GC8hv+MvXERlUX9HmKWAxKIpkxBQi+rMvqaLXyoj92Vll0Lokv6vcdur90BljcPeOe+9nz3859jOCES00b/g0LPbr7od7bNZee73KTe7HG+L67FZHXFDbqYgoxrSQmbY687r11OEqYrSevzBvpdJ3GCwO7ubklndOMTJ06Yhsyg0w/Ffr4OPb9RMD4+Lq+++qrkcm4Dup9ogMD0vTOpcppBEJbCgvdH1F3OnduR3NkNiU8WJJ7dkuTDS3Z7h2YNOLObpUXnz4ZDo/HOdpd0ueYxjVyHzXMjNcdRbtVl8f6Ad7DffjHo7+d+fp8OO/3dmJfrpIoeoytXrsjbb79tl47OCy+8IB999JFdAgAA/US/3+/u7tolkWAwaOfqG9MNvAlHRxsvwzRpX3XP008/Lc8884y8/vrrcvPmTbu2nxWksGZnDZtKV5aSnMQl7aXzldlIvE6mOI+vgrCmxdnZxjR4MSNb2Z2aOgsH3e/Rq/ee6MdpsN+nwOGo97ty2NPHH39sjw5gtFV3P62ejqgrQVW305qJ7qYYYZ3ECaipgAN7//335Ysvvig30hYWFuTJkyfyzjvvyLPPPmu36mdOQ91W9XUVpfDAzhppuXQhIRezW9XjHttxiStflL2Ut/ZolwuTDVF7l/aA+0V9g/8+BQBg2FSKIO6fjuj7z4Q7TGTdc6C7KdAWggo4sDfeeMN0nxncRpqbmVAek9kUKfJqHFSELlyS9OVrbtRcCxutJeWGLejoFudpv8+c1/1iX0DhgPtFY4P/PgUAAAD6D0EFHFg6nZZPP/10oBtpprCQuIV23CJF9aLjUVnMT0lyQYMPzvyjVdmy4xVrYaDVR+3WOrCZEF5RJG8yWRAH2S+aGYb36TDTQNvhVggHAADAYQiUNLfHoXfwtGCZXRxo2hjr5+vo9/PrtVG73lEzLD9f3qdHr6pQ43ZWMvdmZZGijUdKCzXqdNT4fQMAoH91+neaTAUAwJHTzAQdfstTvFeQSE2XIwAAAPQ/ggoAgCMXXXKLYHl1RUIXFilICgAAMIAIKgAAAAAAgK6MaS0FbwIAAAAAAKOtkzgBmQoA0AfM8KFmBJBm8pIJxN1hTXXpwCMm+PdXlOz5gGTscKYD435GAuftcLA+9V8bvV53ZJWBu04AAIA+NRYMBsWbAACjKiSJ2yVZnLaLQysmq49G4ToBAAC610mcgEwFAOgDociUxE5H7FLljrqZTAaDZhLMSErWJTkZl/93PS4zl0XWE+HKHXm9a+89p3z33mYgXPU9Vmd/2e2aTIXtrMS97QMZ54wskxmQkYyzrftYJXOiIbOvjGQ1G6P8HN81+jM0/NfgTJWMAvf8vPX1MzTsNs7+IqdjMhXRIpDV2R0AAADoLYIKANAPphftSAhuY38ru2NGRyg9WpXY5WtOo1gzCTYlbe605+T/zudkc1kk5mxnnqcN9+iWuQtfKu3IqiQl7Guspx5EZEf3l0+L1Nlf9cgLTkN8Miliz2FzOSUz/i4GaymRBXucuXVJ3vQFBRpKSU6u2/1pIOOaRPRcy9fnbFJ1DSXZycYkteIet3hrXpJnNs36UmlTphI3KoEOQ1+3sBT0vJaiErqQs9kIUVks1V4fAAAAeiXgfEEr6YwWYBgfHzdf2Aad3sXq5+vo9/PrtVG73lEzLD/f/r0OvdOuDXBtGPvnnUeuBuTaaTeooDUZwnfisnM7IWaQRr3jvxJxlmdlw2ls587Z4EPD/VUa5YsTTuN+siCXSotOk9yhjf3JnMR1u21nv87KTfuYOe7DS6Yh35D/+c55Vz+n+pqqlK8hIdLoOHab1TNJydnXorEmxxpRV65cMdNR4+8CcHBvv/22nQNw3D799FP5nd/5Hbs0+Dr9O02mAgD0m6ouANpFoU1rSQl7z4s6z1orSME+5HYFaNN2QdbtrDERkSk7a8xFxOuo0b4pibTRkNdASfna9RoszTzYlJnyY1WFFp3rLpxeFbmzUcmmAIAhdxwBQQD1/e7v/q4kEgm7NHrIVDgG/X5+vTZq1ztqhuXn2z/XoXfTZ5x/vGKCjTIL3AZ4VaZC3YwBXwZC0/11kKlgswc0TNF+pkJlfw0zFWqyIPyZCtUhkZrn2G0KvtejvurXD2QqAIOM3yOgf+jv4/nz5+Wpp56SbDZr1w6uTj9fyFQAgD5WvHWtrUyF0Nl4pTaBw9zxrzPUYltMZkJKrtliiPmbSVmfi8vskTbEi5JdqVz5/iEi92c+RF9bFdlXawEAAODw/cEf/IF8+eWXI5mxQFABAPpKVC5qgcKom+Y/L5dMMcSCCRZEJKKFEc1oDc7S6Vhl9IeJhOTyU85j7vNmLqdlc98d/lrV+6uIyuIjbaCHO9hXD0xfdK41JTPOMQOBeZGFVYnZLhzRJS3O6J6PdgmRvM1m8HNeg+vZLZnxjyZhshMY/QEAABy+UQ0s0P3hGPT7+fXaqF3vqBmWny/v01FA94dadH8ABhe/R0D/0N/Hzz77zC6JfOc73xnorhCdfr6MPX78WHR68uSJXTX4XnjhBfNC9Ouk5wcAw0PrMdT/vAv0XZaAZmbUFHoEAADooWHIWPDiBDq1MpSZCugv2rDgfTW8huXny/sUo4hMBWBw8XsE9A/9ffRnKngGNWOh088XaioAAAAAANBjo1JjgaACAAAAAACHYBQCCwQVAABH737GpNYFApnqISC3sxKvXVdPr7drxuyjzrkCAAC0YdgDCwQVAADHY25Vdkq+oSG18T6ZlHW72FCvt2tFh+ssbUraLgIAAHRqmAMLBBUAAMdPMxcmcxLPr0rMrqqr19sBAAD0wDvvvNNy+j//5//ID37wA/mVX/kV+6zhQFABAHD8phelVMpJYsIuN9Lr7QAAAA5IR1J65pln2prm5ubkwYMH9pnDgSElcegY8mi4DcvPl/fpEdNMgpWI7NxOSMiuMkyXhYJc8neLqKfX27WUl0xgQ2YPvJ/+wpCSwODi9wgYXP3++9vp+Y1pMMGbAAAAAADAaOskTkD3BxwJjXYxDecEAAAAYHSNBYNB8abhoCmqAYnfKtplVZTseacBdLX5YGD5q/sbTO7U2TBiup/q4482TZ1hGu4JAAAAwPDoJE4whJkKUVl8tCqSmJfstrumeGtekrIqO0vNe8JGl7xGkjt0WDrvLXfWh1b3k7tQ1UsYANAVDRTHy5/njXW3XVUQWOsvlIPITfZXtR0AHI+33nrLzgHA8RrO7g8TCbmeFUkuZKXofPmbT4isrtQUA+uUFhU7H3e+SAacf539OquKt+K+bIY6X1LtF8+MLwMic9/dBgBQh/P5nasK5EblYnbKzvv0aLuqIHDVtlFZbDR6hLPdpWU7DwAAMOKGtqZC6MJ1WZWkhCeTItnrvRlWbE0k/qgkJa1Wfj8j4cSUbNr0783ldUnerHffKiVbp3fsNs7SihuQAICRt+Z8Rre841+UjYcRmW35Gd7r7XxMgHjG+TT35GVDZn2BCgAAgNE1xIUaQzJ7Lub8G5P42V51RZiSiPdF1IyBXrn7FTmtx6qncvzoS9qpAgDgfoa2070sJImldjLNer2dj8lg8J9rVBZbdKcDAAAYFcMbVLDdHtLLthuEXX0gcxGJ2Fm922WKP9puDeHEul1fyxeIAAAAAABgiAxpUMFp8C+43R4Wl9xuEPM9Ho2hXPzRdn/YyTbKVAAAAAAOidb9Mje5KCAL4HgMZVAhfzVsGvzXTfGtkCRWqkeD6L283GiYqQAA2KfRl+B2R1bo9XbNmH3whR1AH5vTG12tupMBwOEY29vbE28aBjoiw8zlWPVoDxN2NIjJ3n0hDF24JGlTZEy/aG7IbD4t8qDQm24WADAKar8Ea+N9MiktQ7S93q4VU1PBHWoYAABgFHQSJxi6TIXQhZyU6gwD5q5vN4KrQ4mVZHHaLiotKqajPthFb5tyoTHf4+UhyqqGJ3Ps2wcAwNDMhcmcxPOr0rQzWa+3AwAAwIGMBYNB8abhV11csXqKH2L3CABAU2Y0iP0B4X16vR0AAAD26SROMMRDStYTksRtL7ugduLLJwAAAAAAnRixoAIAAAAAAOgVggoAAAAAAKArBBUAAAAAAEBXCCoAAPpYXjJtFdLtbrv81YDEb9nBgHUIyoA39HCT/VVtBwAAMNoIKgAA+kftULzO3MXslJ336dF25SGAVdW2OmxwgwK+znaXlu08APSDtaSECXYCOCYEFQAAx6OtL8FF2XgYkdmWo/P0ejsfk5kwIym7qFkMGzLrC1QAwDEyQ+jqSGb+ACoAHJ3A7u5uSWf29vbkxIkT5kMJAEZNIBDg8w8j58qVK2Y6avy+AQd3XL+/AA6u3/8O6vnt7u7aJZFgMGjn6iNTAQAAAAAAdGVMow7eBAAAAGBIme5cAQlcpfoCgOY6iROQqQAAfUBHIdBUs/JIBAAA9FjxXk4kuyOlJaovAOgdggoA0Ad0FIKdbMwuAQAAAIOBoAIAAAAAAOgKQQUAAABg6BVl4866TEVCdhkAeoOgAgAAADDUipI9Py+yUpLFabsKAHqEoAIAAAAw1EKSuH1dZCEgmfvuGgDoFYIKAAAAwNALyey5mGwVGGUIQG8RVAAAAAAAAF0Ze/z4sej05MkTuwoAAGD4BAIBpiGfAAC94cUJdGpl7NSpU6LTyZMn7SoAAIDhVCqVmIZ0Qmuhs3GRRFgCV/N2DQDU58UJdGqF7g8A0AfyVwMSTqzbJQAADsFEQnIahFmK2hUAcHAEFQCgD0SX3DttuQuMHw4AAIDBQVABAAAAAAB0haACAAAAgJa0q178VlFkOyvxQEYOpTJDu/u+n5HA+azsHyAzL5lAQDL3RYq34h3XjyhfYx/QcylfhxYjrbneVtdXfp5OTbbT45S36+XPVX9G/vPWZep5DCWCCgDQBfcPcFyy23YFAACjwtRmWJRjrcwwvSil2wkZ2k6DTgN8RjZlcdouL29WXa8GDJrWYtrOyvyduOyYQqY7svpgpn6wxB7HK3i6uZySmV41/PVn9GhVYnZRlzedo2mgBMOFoAIAdEFrIOxkRXL3+uNuBgAAhy1yOiZTEadZ688mMBkDGcmc9+50ewH3omSddZlbuq37WLlRW5uNYPahd7PzkplMyro4DdtWd8zLz1Husczxr26YNSoUmZLY6Yhdqmh0h1/Xz1wWWU+EK+eqx7HnX3u8zFXfY779Vd359x/HXLf3WO31x93HzDGc/a9syepr9cM2uv/wnbhsZsvN9f008FMOQoRk9lxM1h8WzFIVbfj7CndGX0qLXN5o/tofQPS1Vdla8V5HDAuCCgAAAABaCl3IVe6c+62lRBbsHfG5dUnerDRJU4mCXNK74I9WRRLzLTL8orJo7mynZbODTIj81bAkZdW9K/+Sc0y7XhvM9Qog63XUGwFD128ui8SyO+7zNAgQdRr3j+y1OUcJ+4IEqQcR95h5bYhfc6/tfkZmHthzMRkCdr0NmIiz73JGQDlI4VgTietxNBCwvSE5icvshH2shinu7Gy3P1zSSFE27qzXDbDUKha2ROYiHey7QxOzzpXlZINMz6Eytre3J94EAAAAAJ1Jy6wJNrh3xP1i2YtucGAiIZeW1w8hw68ohQfOGSzYu/LTF2V1zjxwYMV7OVmf8xr3IUkspEUeFMqBgNi5WXvMWecVqMd5zu2cJPT52wXZcra6ZIMcesc+tuZvXE9JxAsiONuun4m4+z4ok2kRluRa5dgNaZeJhMjqymF2KwlJ5My6FAgq9L1O4gRkKgBAlzStcv3OBil8AIDR1uTOtukucagKUlizs4dhLSlhr8tCNOUsO8ezD9W9tulF2TmXKz+n0uWjIFUVECYiMmVnDd9raLIFekW7N5hsCmnepUSzMiaTMpW3QRCgA2PBYFC8CQDQAf1DvSIy709fBAAAZVsF7y+km1FQz8Ea0U5jvEeZCXVpgUTTlcGbWnfLMN0rzLY7Er8TdgsTTkQqBQuVyVyoT29a9Jw5/lb9DAHNZpjMme4Xdbu3YCR1EicgUwEAuqV/hBdErg9z9WkAAA6gnNGndQLWYhI/6/3FTMmGGQXA7e/fPbfLRcor/nf/hiR7lLkQOhuXmFcrwWEKMLa4kbC/CGTM7dZgMhNScs1mLuRvJn1dK2poAMDXzaIrpvhj5Vyru3L4lOtGHFWGggaX7GuCoUFQAQC6pHdWyv0pAQDAPukzBbcrgD+1fiIh17MxSUW1i8C8yDlfRQLb+G45+oNP6MJ1t4iiHmclIqvL9oEGGo3+oHSEi/LoDzqCQn5KkpNuV4aZy2nZbHEjwZzLgxm3u0QgLLlz121j3S1CKc6+W+6r22KGZnQJ+7rVdMMIJ6Yqx/NtZ4INzn/eNbqT3Yd/f87/M76htDXA0nA0j2b8RSg7eR76WqCkuTkOLcAwPj5uUnUAYNToH9FOP//0S8m8XK9bWRoYBFeuXDHTUevm960Xjuu4OBqj9vM9rt/f9umwi2EpLJBS3xXNNLg7a0ap0O8b4YeX6o5YUSt/NSOy1LqLRq+3a0gDB76sTg1GbLxUeU8ceP8Dqt8/rzo9PzIVAAAAAPQXcxfbf/fcN41CLaPpRdmUGbceg7o808Z152VDZttooPd6uwZMrYZkpUClszzjXFUlyHTA/aNvkKkAAA79ktLJ559G2mcux46wDyLQe2QqYJiM2s+3/zMVADTS759XnZ4fmQoA0IXoklZ1JqAAAACA0UZQAQAAAAAAdGVMuz14EwAAAADUU1Xxv5EmIwZ0xb+/gRwtQAtWBiq1ETxtXktbr/kR0XPR6zCjZwT217ZoNqqGX6+3a0bPWc/VfQ31Z3HA9+MI6SROQKYCAAAAgP6nQzyWRm+kgL5QW2RxeVNKviExzQgViXJJxoZ6vV0r2l11JxuzSyFJrMQltzAChT6P2FgwGBRvAgAAAIB6IqdjMhWxzUit7K93q+3k3onPS8ZU+0/JTOBNefv8jDO3LslJ7+6we9e++jkOe9c+Y+8q199fRvK1d/f951C+a24zA676Hmvnbrfuy9lHtnwO3vHcZX+2gLmD7u3bn4lR9Zo0yEIw2zjPkYhMzUWc/zsaZC3ocWYui6wnwpXjd3HN3t362vXucb3Hal7X83H3MXMMZ/8rW7L6Wv1wju4/fCcum+XGe3293q4rEwm5dCYpN2ozR7BPJ3ECMhUAAAAAtBS6kHPvVGtjNOo0Mh9p0WL3TnBqRRufUVl8tCoxSctm6R156/amM1cZKSl/NSzJM5vmOaV8WlJRfyp6SrZO75jHNpedpX37q8lQMOeQknRez2FHViUpYV+DOfUgIjv2OHL5Wnsp72tJKbxk9zeXkpnJglyy+1hP3HAb3U6DO5yYcs5Ht9NzXZfkTX0kL5mq12RLrtV2W9DG+oqel74ezrV5d/obZGDo662vRSy7I7kLzpZVr3ub1+wcc+bBqrten/PAey3cgI04+3avw7lef3eGNZG4HkfPcXtDchKX2QbFqU3xamc7EyBpotfbdSv6kvPeu1t53XBwBBUAAAAAtM80gisjIIUiU+5MU3nZuCySfsk2nacvOg33dSmUG/sxiZ91syC00dfSdkHW51bloknHD0liQRvSG+W77bFzs26DfXpW2tiblZZZu7/IGW3MX3Qb+hMR5+ys6UWnEV4JAGj2Rj0aEDCBAE9BAwIim14goQvFeznnmr3Gvb3mB4VyIKD1NTvPuW1/bs7rt+VsdcmeY/S1VYmt5WSj/POYkogXRNDX+kyk6/PuO/rz9L1uODiCCgAAAAA6UpVSH03Zta2lot7zwpJcE9kqeE07XyO2DcXClp2z/A1/R7mbRie87ghNVXfhqPT716yKuOQm7WP+u/6O1B2R+HJKNg6adr+WlLA9tnnd1wpSsA/VvebpRdk5lys/p9yNQgMF7pzLef2qQkO+12Lfaw3UGMqgwosvvlj+RT+K6Zvf/KZ8+OGH9ugAAADAENOU+svaJUFT6p1J0+3bol0h7HPsVHU3vwP7siNqG8mHpHhrXpLidSfwFwF0mAwOu14b8r6uCemFhCReW5WtaINaC+3SAon2GO7UunClZk242+5I/E7YrVdRE4RxMxfqay8TBaNsKIMKH3/8se8X7fCnTz75RF555RUCCwCAgfL222/bueHw3nvvyeeff26XjkP1HcyqqeauZedsIba6dzl12L79x6y/7X56x/lwhqzT82L4tm796q/+qvzRH/2RXepnWsSvnUyFqMyW6w84bJHAdt+n+2ijeM0ruGfPYXm2ZQO7t/Jyw8tUMNdTHTCIna7Je9Aigcup/bUW2hQ6G5eYrz6EyRZp8dliikr6izNKzM0IMZkJlXPJ30z6ulbU0Nd6mLoLDFt3jj5A94ceWVtbI7AAAMAxSqVS8txzz8mbb755TMEF7a9sbzqY4nK+u7IH6EfdLrdgXeX4W9EDNNhw7D799FP5jd/4DfmlX/ol+f3f/327tk+Yegg6IoMGsOZFFrQ/vk3Dt41VM1qDRCQyVxn9IbqkhQJn3MCXLRJYHqKwkar9+WhWgCn2qOcQdrMHlg4/pBC6cEnS5S4IGzKrWRra4HbO53p2y74m7ugF1+tkYUSXNmUqMV8dbKsTkPBozYby6A/mmqec19M9hskWafHZErpwvfKaO69T7tx1WwvDLYIpzr5b7mtiVuLir7fQpibXVaWr7aqDllXB0Rb7y99NlWt7HF5QdcQ4f3iM3d3dkm9xoB31dejxPvvss9La2pqZ/+CDD+wjAAbFsHz+AZ04rvf9YR33t3/7t0s/8zM/U3r66adLzzzzTOmNN94wf589R3q9j1ZLMYmVVh/ZZZVPm3Mw09xqaces3CmtzkkpNhcz69N5s7K0uWy3K69zt0tndb/u+ljW3YOzdclp1pSf69nJOvssH8cu2+eKPTf/Om9/9bYz/OcvaeeoHvfcvMf857tvH4dIjzVMvve975V+/ud/3lzXz/7sz5a+/vWvl1ZWVkpffvmlefytt94y/wJHQn//l93fevMZYedb2Vz2f1Y01uvt/PR8y5+X+tns+1zU5XT5s/To9PvnVafnN7a3tyfehIN5+eWXyVgAAOCYfPe73xWn8SU//elP5Sc/+Yk4DTA5efLkMWYu+OidsybD362fuaTf4MxdWzM2vdh+0ybjoHLHTYu9Xdf1Zoi7mrudNUyqtHf3uMEwePuGrOtiuLz6wwRq1kb1cILDTL9H/+///q95n/3P//yPPHnyRHZ2duRHP/qR/Pu//7v867/+q/zzP/+z/OAHP5C///u/l7/927+Vv/mbv5Hvf//78ld/9Vfy53/+5/LBBx/In/7pn8qf/MmfyB//8R+b/fzCL/yC2f8XX3wh//3f/y1LS0vyi7/4i7KwsGDWowPm7rV7h3/fdODuSSNgetH5VJqpZD9dnmnjdcvLhrTTJaXX21VoJkKlmGZRsgs5ia9UMjKK9woSsaOOoFoncYKAzVAwG584ccL8QRh0+uFwlNehx/vss8/skshf/MVfyNzcnPnj8O1vf9uuBdDPtMCr1mMBRskLL7wgH330kV06Okf9+zY+Pi6vvvqq5HJusbIjoQ2YSefLq9eg9san99KLddn5ZrxZuiiF85qSbBv1+qXXWS4suAGGCnd9ZTtN/b0mEbN/nZ9xVtU8x5yDjrO/v5CbBi7CDy9JaSlqvnRfO+3tt1plO/Edzz5o1B7bf/7+czQbHyr9PnYc9Pfor//6ryUYDJr3mv7rn2rXdbLNxsaG/Md//Ic9ksgzzzwjznd3+fVf/3WJRCJy5coV+wiAQXLU7dVO6fnpZ41HP4+aGfN/gKE3yFgABo82rMxdNiamEZqOI6CgDvP3TWsqeJ5++mnTCHv99dfl5s2bdu3xaH/4u4IU1uxsHR0Nk1dVDV8b+5U7s5U7d7UabddiuLyGwwQerXrvicOe9P2sX741U0FvMv34xz+W//qv/5L//M//NJkK//Zv/yb/8i//Iv/0T/8k//iP/yh/93d/J/fv3zeBCA2u/eVf/qX82Z/9mXzve9+TO3fumO+Q2WzWBAweP35srkvfx2NjY/Kd73zHBBn+8A//0KwHgMPSSZyAQo2HhMACAABH6/333zdp4l4wQVPENQ39nXfekWeffdZudTzaH/5OC9vZ2QMygQw71nzTYfB8uhsur3fDBKLi3XffNf/q+zmZTMrDhw/l937v9yQU4rUF0F8IKhwiAgsAABydN954w3Tn7KdgQplmJrQ1/F1IImdEUndtg930A+9iWEbnefOJdTM2/v4mqG8YvKbaGS6vx8MEwtB6DFpn4dd+7dfkH/7hH+TGjRumu8NxK1fKr/N+2M/Neun4vaBdgzQTpq1j+M6pD+i5eNdrhnK0GT+V10BfE4ZZxfAhqHDICCwAAHA00um0GYavr4IJng6Gv6sddm8q315NgkoXBO95lRoLDYfBcx7zD1nXzXB5jYcJrB5OEO0Lh8Ny9+5d0yXil3/5l+1a9LX7GVNg1bz3/QVPTbFV73cgJImVuOQWKAyJIVOyGFKye3o8HbKq2bTGcJMAAByrYfmeg/pG7ed7HENK6tB8ZshQM2RqZWg/M8Sg8/q7kzeEaLNhUB1mH95zfMME6tCFZsi/zVLaDv1n9l9nCEP/cb19+4djrRzPnsuyb1hU3/78z6k6TtNzjLmPmXPU/VeGTq0+X/fY/mvX49UOAYvRou+pftbp+ZGpcETIWAAAAMAg0yFIq0YYUf678s5UGYbUlUroCCTuHXspD4Oal4zNpnGfk5KZfcMTRmXRjpaix9WRSmrVDolqhmN9YGuCmOOFq7pfpB5E3Mc0A+fyNfdcNMPAe44O9/rArrfnqJk3dc9xTSSutUT0HLc3JCdxmbUZRYWH67Z7kHK7NK0/NIO7GtGX0pUuTsAQIKjQA9/61rfka1/7WstJh5lU8/Pz5l8AAABgoE0vOo3uyrCl2p3GL5a96D42kZBLy+uSu+c0y+9vSErSMmsDFNHXViW2VpBKs7sbRdm44zTmz826dUTM8apHIik/Nj3rHL2ekCRu2+5G2wXZcra65HXzMeeYk41yV54piXjdkrTw6pmIu2/nPAoPzExjWmPFdj8ChgFBhR7Q4YA0gtnu9Mknn9hnAgAAAIOs+XCljYdBTZVrdGgtjnWnCV8oN9i7p/VBvHOZuVydIVD3XKYX3dFM7HPKRR9rR2iZiEjVGC52ZBVVPWSsm5kAjBKCCgAAAAC60mq40kqmQM0d/LnKc9ypvYKkraRtl4ry1KAgqp/pXmG235H4HdtlQrMJ3IddJnOhvtohY03x03Iww73uSncIYPiMPX78WHTSoZcAAAAAoDv7hytdv7Phpvlr3YG1mMTPhtzuB+UhVjUwocMvth4+srmQzJ6LSWrFq3uQl4w/86ABc+yr/iPH3G4NJjMhJdfs8/M3k7I+V6mbUKWmO4MJMlzecK/Hf92equ4SQH/y4gQ6tTJ26tQp0enkyZN2FQAAwOB577335PPPP7dLAI5Cs+FKVfqcyLw+ZgozetkIUVk0Qy26XQ7CCZHVR5W6DLX2N/wrqodEzcnmGe9cZiS1vGkKODYTunC9MiRqICy5c9erzlGLPbpdKdKyaQtH7jMxK3Hx1VvQLhXeEKxV1+3K301J+qXWGRTAcfLiBDq1Eihpro9jb29PxsfHTerPoNNf/H6+jn4/PwAABtGJEyfkyy+/lNdff11+67d+S5599ln7iIu/v8Nt1H6+V65cMRP6xP2MBO7Otu5usZ2V+ILI9UYBCoyEYWuvUlMBAAAMhXfffVeeeuopWVlZMRmYb775JpkLAI7G9KJsykzVEJb7FSW7kJP4CgEFDBcyFY4Bd0oAADgczz33nPz4xz82808//bT5m/ubv/mbJnNBh3fm7+/wGrXvV2QqAINr2NqrBBWOgZ4fAAA4Gvr95tVXX5Vczq3wjuFEUAHAoCCoMACG7YcEAADa8/Wvf10+/fRTM0+mwmgZte9XBBWAwTVs7VVqKgAAgKHw/vvvyxdffGGCCc8884wsLCyYIbPfeeedfUUbgVGTv6qjG8Ql641QAAA9QlABAAAMhTfeeMNkXhJMAPaLLpVkJyuSu+cN9ggAvTGmf3y9CQAAYFCl02nT9YFgAgAAB9NJnIBMBQAAMBRef/11ggkAAByxsWAwKN4EAAAAYDiFIlOyfmdD6AABoJVO4gRkKgAAAACjYHpRSisi8+ezBBYA9AxBBQAAAGAU3M9IYEHk+u2EhOwqADgoggoAAADACCgWtiR2bpaAAoCeIqgAAAAAjJy8ZAJxyW7bRQDoEkEFAAAAYORE5WJ2ys4DQPcIKgAAAABDLn81IOGESPys1/mhKBsPIzI7YRcBoEsEFQAAAIAhF10qSamUk0Q5iBCSxBIFGwEc3Nje3p54EwAAAAAAGG2dxAnIVAAAACMjEAgwDekEADgeY8FgULwJAABgWJVKmv7NNMwTAKA3OokTkKkAAAAAAAC6QlABAAAAAAB0haACAAAAAADoylAGFV544YW6BXz6ZdLzAwAAAABg0A1lUOGjjz6qW7ynXyY9PwAAAAAABh3dHwAAAAAAQFcIKgAAAAAAgK6M7e3tiTcBAAAAGEZ5yQTikt22S1cDEr9VdBe2sxIPZJwtAMDVSZwgsLu7W9IZ3fjEiROmzz8AAACA/nXlyhUzARg8Wry/n9vden67u7t2SSQYDNq5+sZ0A28CAAAAAACjrZM4ATUVAAAAAABAVwgqAAAAAACArhBUAAAAAAAAXSGoAAAAAAwoHcVBi6qVR3IAgCNGUAEAAAAYUNGlkuxkY3YJAI4eQQUAAAAAANAVggoAAAAAAKArBBUAAAAAAEBXxh4/fiw6PXnyxK4CAAAAAACjyosT6NTK2KlTp0SnkydP2lWjyaucu3/KSN5u0w7dD9V3AQAAAACDyosT6NQK3R8srZxbKum0KWlnOZ33lhcl6m7SFt1P7kLILgEAAAAAMLyGM6hwPyOB81kp5wtsZyXuX+6U2V9c4pq5YPdTvBX3ZTPEJbvtblrOVNBjBjKS8WVAZO672wAAAAAAMAyGM6gwPSvptZxs2IZ+8V5O5NysHCh/YE0k/qgkpdsJCd3PSDgxJZsmk6Ekm8vrkrxZr5NESrZO79htnKWVAwQ2AAAAAADoM0Pa/SEqs05Dv2CCCkXZuCMSP3vQLglTEpmws9OLVd0iIqdjdq5WrHzc6EvaqQIAAADoHc2SDSfW7RIAHL2hramgjfjU3bzI9obkJC6zXkCgW3MRidhZDVRkz1e6NTT+IPcFIgAAAIAe8+qCUdMLwHEZ3kKN2gXi8oZke9H1oUbx1rwkZVV2bPeHnWyjTAUAAAAAAIbX8AYVTBeIlCQTvej60ExebpByBgAAgCP09ttv2zkAOF5je3t74k3DxtQxmOtB14caoQuXJL2WlLDp/rAhs3nnOA8KFGEEAAAAAAy8TuIEgd3d3ZLO6MYnTpww6fzDQgvXXDu9Qx8zAAAADBWt6zVM39uBUdLvv796fru7u3ZJJBgM2rn6xnQDbxoebiHFmQercr0cUKgurlg9xSVrh58EAAAAAGCUdRInCJRsiEQzFcbHx4l4AgAAAH1Ob4zxvR0YTP3++9vp+Q1xoUYAAAAAAHCYCCoAAAAAAICuEFQAAAAAAABdIagAAAAAAAC6QlABAAAAAAB0haACAAAAAADoypgOJelNAAAAAABgtHUSJyBTAQAAAAAAdCVQcuiMRiDGx8fFLgIAAADoU4FAgO/twIDq99/fTs+PTAUAAAAAANAVggoAAAAAAKArdH8AAAAABoymJ1+5csUuARgk+rtL9wcAAAAAx4aAAjC4hu33l0wFAAAAAABgkKkAAAAAAACOBEEFAAAAAADQlTHt9uBNAAAAAABgtHUSJyBTAQAAAAAAdIVCjQAAAAAAwKBQIwAAAAAAOBIEFQAAAAAAQFcIKgAAAAAAgK4QVAAAAAAAAF0hqAAAAAAAALpCUAEAAAAAAHSFoAIAAAAAAOgKQQUAAAAAANCVQLFYLOnMV199Jc8//7yUSmYRAAAAAACMmEAgIMVi0S6JnDp1ys7VFyjZKMLe3p6Mj48TVAAAAAAAYERpUKGTuADdHwAAAAAAQFcIKgAAAAAAgK4QVAAAAAAAAF0hqAAAAAAAALpCUAEAAAAAAHSFoAIAAAAAAOgKQQUAAAAAANCVsb29PfEmAAAAAAAw2jqJEwR2d3dLOqMbnzhxwqwEAAAAAACjaXd3186JBINBO1dfoOTQGS8C0eoJ7W6nHj9+LKdOnbJL9fX6uJxffZxffZxfY5zffpxffZxffZxfY5zffpxffZxffbq/R48eSSQSsWvqO87zU/38+inObz/Obz/OrzVqKgAAAAADZnJy0s4BwPEiqAAAAAAAALog8v8DZpObDBYsp1cAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "id": "558dca1f-a702-44a4-977c-5b9892688f18",
   "metadata": {},
   "source": [
    "![image.png](attachment:529a6f3c-2533-454c-8ab9-d461213332ff.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a5146fcd-591b-4202-9299-a353f85d616b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data():\n",
    "    \"\"\"\n",
    "    Load spam, split random data to Train, Test\n",
    "    \"\"\"\n",
    "    df_raw = pd.read_csv(\"data/spam.csv\", encoding='cp1252')[[\"v1\", \"v2\"]]\n",
    "    df_raw.rename(columns={\"v1\": \"label\", \"v2\": \"sentences\"}, inplace=True)\n",
    "\n",
    "    # Encode label\n",
    "    df_raw['label_encoded'] = df_raw.label.apply(lambda x: int(x=='spam'))\n",
    "\n",
    "    # Split train and test\n",
    "    X_train, X_test, y_train, y_test = train_test_split(df_raw[\"sentences\"], df_raw[\"label_encoded\"], test_size=0.2) \n",
    "    \n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1)\n",
    "    \n",
    "    return df_raw, X_train.tolist(), X_val.tolist(), X_test.tolist(), y_train.tolist(), y_val.tolist(), y_test.tolist()\n",
    "\n",
    "\n",
    "class TorchDataSet(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "        \n",
    "        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]).to(self.device) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx]).to(self.device)\n",
    "        return item\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "569d4c0a-368e-448b-88b0-2f64fc31bd9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load and split data set\n",
    "df_raw, X_train, X_val, X_test, y_train, y_val, y_test = load_data()\n",
    "\n",
    "# Tokenizer X\n",
    "tokenize_train = tokenizer(X_train, truncation=True, padding=True)\n",
    "tokenize_val = tokenizer(X_val, truncation=True, padding=True)\n",
    "tokenize_test = tokenizer(X_test, truncation=True, padding=True)\n",
    "\n",
    "# Create Data set\n",
    "torch_train_set = TorchDataSet(tokenize_train, y_train)\n",
    "torch_val_set = TorchDataSet(tokenize_val, y_val)\n",
    "torch_test_set = TorchDataSet(tokenize_test, y_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fd0d5cf-54ab-4051-9cd7-cffa784dbeb5",
   "metadata": {},
   "source": [
    "## 2. Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "923dcec2-3c9a-4d7c-aca2-1559128afb04",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"results\", \n",
    "    evaluation_strategy=\"epoch\",\n",
    "    per_device_train_batch_size=3,  # batch size per device during training\n",
    "    per_device_eval_batch_size=3,   # batch size for evaluation\n",
    "    dataloader_pin_memory=False     # Got ERROR relate to cannot pin 'torch.cuda.FloatTensor' only dense CPU tensors can be pinned. Removed this setting if you are not got the same error\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c495f24-8bf0-4939-9e5f-79a0d5a71541",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Onsiter\\.conda\\envs\\nlp\\lib\\site-packages\\transformers\\optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n",
      "***** Running training *****\n",
      "  Num examples = 4011\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 3\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 4011\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1312' max='4011' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1312/4011 01:16 < 02:38, 17.07 it/s, Epoch 0.98/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to results\\checkpoint-500\n",
      "Configuration saved in results\\checkpoint-500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attempted to log scalar metric loss:\n",
      "0.1024\n",
      "Attempted to log scalar metric learning_rate:\n",
      "4.3767140363999005e-05\n",
      "Attempted to log scalar metric epoch:\n",
      "0.37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results\\checkpoint-500\\pytorch_model.bin\n",
      "Saving model checkpoint to results\\checkpoint-1000\n",
      "Configuration saved in results\\checkpoint-1000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attempted to log scalar metric loss:\n",
      "0.1026\n",
      "Attempted to log scalar metric learning_rate:\n",
      "3.753428072799801e-05\n",
      "Attempted to log scalar metric epoch:\n",
      "0.75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in results\\checkpoint-1000\\pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=torch_train_set,\n",
    "    eval_dataset=torch_val_set,\n",
    "    # logging_dir=\"./log\"\n",
    ")\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06a30e22-ec2a-4930-88cf-5efed558719a",
   "metadata": {},
   "source": [
    "## 3. Predict and evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb52111a-9097-46a0-9d28-f4ed8e43cdef",
   "metadata": {},
   "outputs": [],
   "source": [
    "out = trainer.predict(torch_test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1bb2b774-5b86-4ac1-af7b-bf0e59eef13e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
       "       0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,\n",
       "       1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.label_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e951e546-ea36-4048-856d-db7f04db3dbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.6345205, -8.359074 ],\n",
       "       [ 7.6147013, -8.346968 ],\n",
       "       [ 7.5546374, -8.267267 ],\n",
       "       [ 7.604041 , -8.367223 ],\n",
       "       [ 5.0721536, -5.6135902],\n",
       "       [-5.250854 ,  5.858702 ],\n",
       "       [ 7.660727 , -8.369206 ],\n",
       "       [ 7.427806 , -8.186454 ],\n",
       "       [ 7.616889 , -8.372325 ],\n",
       "       [ 7.5471196, -8.203858 ],\n",
       "       [-5.2502174,  5.855975 ],\n",
       "       [ 7.578126 , -8.3405285],\n",
       "       [ 7.5185275, -8.284544 ],\n",
       "       [ 7.7437067, -8.477866 ],\n",
       "       [ 7.663892 , -8.414421 ],\n",
       "       [-5.2498918,  5.8562274],\n",
       "       [ 7.589204 , -8.326045 ],\n",
       "       [ 7.194022 , -7.922475 ],\n",
       "       [-5.2517033,  5.8587317],\n",
       "       [ 7.5378   , -8.297657 ],\n",
       "       [-5.252551 ,  5.859781 ],\n",
       "       [ 7.3942204, -8.111136 ],\n",
       "       [ 7.6039515, -8.355018 ],\n",
       "       [-5.223846 ,  5.8220634],\n",
       "       [-5.2486873,  5.854101 ],\n",
       "       [ 7.7012453, -8.430389 ],\n",
       "       [ 7.5400114, -8.287841 ],\n",
       "       [ 7.6136665, -8.361971 ],\n",
       "       [ 7.603157 , -8.390098 ],\n",
       "       [ 7.554998 , -8.306894 ],\n",
       "       [ 7.582926 , -8.353763 ],\n",
       "       [ 7.4783697, -8.235782 ],\n",
       "       [-5.251097 ,  5.8575764],\n",
       "       [-5.250824 ,  5.8578534],\n",
       "       [-5.245877 ,  5.8555593],\n",
       "       [ 7.614287 , -8.346133 ],\n",
       "       [-5.252001 ,  5.859633 ],\n",
       "       [ 7.6339283, -8.357838 ],\n",
       "       [ 7.6177983, -8.358019 ],\n",
       "       [-5.246552 ,  5.851259 ],\n",
       "       [ 7.6254916, -8.376148 ],\n",
       "       [-5.2514815,  5.859473 ],\n",
       "       [ 7.5938597, -8.376367 ],\n",
       "       [ 7.5490446, -8.304238 ],\n",
       "       [ 7.5757494, -8.335074 ],\n",
       "       [ 7.577084 , -8.349301 ],\n",
       "       [ 7.6022444, -8.301832 ],\n",
       "       [ 7.5279236, -8.28189  ],\n",
       "       [ 7.549303 , -8.302657 ],\n",
       "       [ 7.568212 , -8.322427 ],\n",
       "       [-5.2472343,  5.8520117],\n",
       "       [ 7.5155287, -8.288524 ],\n",
       "       [ 7.547925 , -8.331525 ],\n",
       "       [ 7.6026726, -8.324295 ],\n",
       "       [ 7.478077 , -8.224049 ],\n",
       "       [ 7.601641 , -8.368843 ],\n",
       "       [ 7.672229 , -8.401021 ],\n",
       "       [ 7.6755915, -8.413891 ],\n",
       "       [ 7.6501307, -8.3868   ],\n",
       "       [ 7.67888  , -8.400555 ],\n",
       "       [ 7.6565166, -8.40848  ],\n",
       "       [ 7.4015193, -8.110532 ],\n",
       "       [ 7.5704775, -8.278926 ],\n",
       "       [ 7.553138 , -8.320663 ],\n",
       "       [ 7.412381 , -8.136753 ],\n",
       "       [ 7.639923 , -8.4096155],\n",
       "       [ 7.6277475, -8.339644 ],\n",
       "       [ 4.270929 , -4.748739 ],\n",
       "       [ 7.6151977, -8.331593 ],\n",
       "       [ 7.5574164, -8.328786 ],\n",
       "       [ 7.6210856, -8.3749485],\n",
       "       [ 7.5792646, -8.314985 ],\n",
       "       [ 7.5708566, -8.310897 ],\n",
       "       [ 7.589541 , -8.368303 ],\n",
       "       [ 7.5357723, -8.27347  ],\n",
       "       [ 7.6454034, -8.35673  ],\n",
       "       [ 7.6069345, -8.333805 ],\n",
       "       [ 7.6272225, -8.389696 ],\n",
       "       [ 7.453946 , -8.207724 ],\n",
       "       [ 7.701745 , -8.432688 ],\n",
       "       [ 7.588123 , -8.347449 ],\n",
       "       [ 7.6423335, -8.367168 ],\n",
       "       [ 7.6539392, -8.372921 ],\n",
       "       [-5.2462516,  5.8543553],\n",
       "       [ 7.673957 , -8.39631  ],\n",
       "       [-5.2524867,  5.8597674],\n",
       "       [ 7.557351 , -8.3274555],\n",
       "       [ 7.6878967, -8.412318 ],\n",
       "       [-5.252281 ,  5.859276 ],\n",
       "       [ 7.4462333, -8.18482  ],\n",
       "       [-5.249316 ,  5.8556376],\n",
       "       [-5.252458 ,  5.8591456],\n",
       "       [ 7.595747 , -8.340628 ],\n",
       "       [ 7.639106 , -8.3645525],\n",
       "       [ 7.589934 , -8.329053 ],\n",
       "       [ 7.4919915, -8.242226 ],\n",
       "       [ 6.2014112, -6.7948494],\n",
       "       [ 7.5413303, -8.237333 ],\n",
       "       [ 7.6289654, -8.363042 ],\n",
       "       [ 7.6255703, -8.372314 ],\n",
       "       [ 7.617696 , -8.3687525],\n",
       "       [ 5.8916407, -6.4759293],\n",
       "       [ 7.572042 , -8.326908 ],\n",
       "       [ 7.6638207, -8.41213  ],\n",
       "       [ 7.539739 , -8.282247 ],\n",
       "       [ 7.526288 , -8.289141 ],\n",
       "       [ 7.644304 , -8.389502 ],\n",
       "       [ 7.5705004, -8.338072 ],\n",
       "       [-5.2483535,  5.855911 ],\n",
       "       [ 7.628574 , -8.400289 ],\n",
       "       [ 7.669401 , -8.394154 ],\n",
       "       [ 7.4901657, -8.246081 ],\n",
       "       [ 7.66262  , -8.3891535],\n",
       "       [-5.254275 ,  5.863248 ],\n",
       "       [-5.252285 ,  5.8598456],\n",
       "       [ 7.5905924, -8.332008 ],\n",
       "       [ 7.529918 , -8.284044 ],\n",
       "       [ 7.605851 , -8.349022 ],\n",
       "       [ 7.625098 , -8.370308 ],\n",
       "       [ 7.658048 , -8.36436  ],\n",
       "       [-5.252453 ,  5.8601527],\n",
       "       [ 7.5253043, -8.266187 ],\n",
       "       [ 7.558261 , -8.281338 ],\n",
       "       [ 7.5960135, -8.329704 ],\n",
       "       [ 7.626105 , -8.357273 ],\n",
       "       [ 5.885728 , -6.4563484],\n",
       "       [ 7.629835 , -8.406803 ],\n",
       "       [ 7.5614376, -8.298093 ],\n",
       "       [ 7.4023666, -8.14549  ],\n",
       "       [ 7.508242 , -8.239576 ],\n",
       "       [ 7.5881186, -8.352122 ],\n",
       "       [ 7.6084986, -8.325965 ],\n",
       "       [ 7.5463996, -8.289492 ],\n",
       "       [ 7.635685 , -8.379074 ],\n",
       "       [ 7.4311485, -8.16515  ],\n",
       "       [ 7.481827 , -8.24251  ],\n",
       "       [-5.249967 ,  5.8574343],\n",
       "       [ 7.4706106, -8.195532 ],\n",
       "       [ 1.0409594, -1.1928949],\n",
       "       [ 7.5337152, -8.280959 ],\n",
       "       [ 7.5124555, -8.243763 ],\n",
       "       [ 7.56546  , -8.340524 ],\n",
       "       [ 7.640131 , -8.394612 ],\n",
       "       [ 7.563384 , -8.332599 ],\n",
       "       [-5.2517033,  5.8587317],\n",
       "       [ 7.590725 , -8.332363 ],\n",
       "       [-5.251106 ,  5.8585134],\n",
       "       [ 7.487912 , -8.223494 ],\n",
       "       [ 7.6140065, -8.358908 ],\n",
       "       [ 7.531092 , -8.2901   ],\n",
       "       [-5.2526374,  5.8625045],\n",
       "       [ 7.745039 , -8.474318 ],\n",
       "       [ 7.746896 , -8.476096 ],\n",
       "       [ 7.549434 , -8.320211 ],\n",
       "       [ 7.664445 , -8.406099 ],\n",
       "       [ 7.659256 , -8.412489 ],\n",
       "       [ 7.6299777, -8.347211 ],\n",
       "       [ 7.605404 , -8.348748 ],\n",
       "       [ 7.603956 , -8.364334 ],\n",
       "       [ 7.548388 , -8.319914 ],\n",
       "       [ 7.625041 , -8.351381 ],\n",
       "       [-4.465702 ,  5.0163383],\n",
       "       [ 7.47343  , -8.196982 ],\n",
       "       [ 7.5298433, -8.276372 ],\n",
       "       [ 7.6997643, -8.454237 ],\n",
       "       [ 7.2050896, -7.914278 ],\n",
       "       [ 7.5352736, -8.286174 ],\n",
       "       [ 7.547489 , -8.285317 ],\n",
       "       [ 7.558297 , -8.286085 ],\n",
       "       [ 7.5814376, -8.329992 ],\n",
       "       [ 7.657982 , -8.427413 ],\n",
       "       [ 7.537094 , -8.253075 ],\n",
       "       [ 7.5178022, -8.292228 ],\n",
       "       [ 7.5540795, -8.259023 ],\n",
       "       [ 7.613701 , -8.333399 ],\n",
       "       [ 7.6095605, -8.357131 ],\n",
       "       [ 7.555289 , -8.282349 ],\n",
       "       [ 7.631008 , -8.344823 ],\n",
       "       [ 7.7117724, -8.471964 ],\n",
       "       [ 7.635723 , -8.389645 ],\n",
       "       [ 7.4516964, -8.158386 ],\n",
       "       [ 7.454723 , -8.198783 ],\n",
       "       [ 7.670522 , -8.38702  ],\n",
       "       [ 7.549332 , -8.315275 ],\n",
       "       [ 7.565007 , -8.319033 ],\n",
       "       [ 7.6273403, -8.370992 ],\n",
       "       [ 7.6086206, -8.356557 ],\n",
       "       [ 7.522815 , -8.291328 ],\n",
       "       [ 7.2922945, -7.9706035],\n",
       "       [ 7.598181 , -8.3787155],\n",
       "       [ 7.5431166, -8.239541 ],\n",
       "       [ 7.607487 , -8.373061 ],\n",
       "       [ 7.509705 , -8.2973175],\n",
       "       [ 7.623105 , -8.360053 ],\n",
       "       [ 7.6250772, -8.397939 ],\n",
       "       [ 7.604933 , -8.371687 ],\n",
       "       [ 7.5566044, -8.292242 ],\n",
       "       [ 7.5689063, -8.32243  ],\n",
       "       [ 7.6204677, -8.362482 ],\n",
       "       [ 7.560477 , -8.288062 ],\n",
       "       [ 7.5191836, -8.25636  ],\n",
       "       [ 7.7082705, -8.440409 ],\n",
       "       [ 7.5237646, -8.268455 ],\n",
       "       [ 7.637865 , -8.386903 ],\n",
       "       [ 7.385026 , -8.104917 ],\n",
       "       [ 7.6601443, -8.408875 ],\n",
       "       [ 7.6262536, -8.373192 ],\n",
       "       [ 7.546232 , -8.28442  ],\n",
       "       [-5.2472243,  5.851523 ],\n",
       "       [ 7.4835615, -8.22624  ],\n",
       "       [ 7.5353575, -8.284545 ],\n",
       "       [ 5.545005 , -6.0864754],\n",
       "       [ 7.5819063, -8.374094 ],\n",
       "       [ 7.590769 , -8.35683  ],\n",
       "       [-5.2525344,  5.8608794],\n",
       "       [ 7.5978107, -8.383392 ],\n",
       "       [ 7.531478 , -8.254238 ],\n",
       "       [ 7.6020327, -8.379122 ],\n",
       "       [ 7.626261 , -8.342614 ],\n",
       "       [ 7.6432805, -8.410303 ],\n",
       "       [ 7.6448717, -8.399418 ],\n",
       "       [-5.251709 ,  5.8593135],\n",
       "       [ 7.552971 , -8.282474 ],\n",
       "       [ 7.662974 , -8.383566 ],\n",
       "       [ 7.2747946, -7.970381 ],\n",
       "       [ 7.601077 , -8.312073 ],\n",
       "       [ 7.641613 , -8.420462 ],\n",
       "       [ 7.6136036, -8.359554 ],\n",
       "       [ 4.560238 , -5.099602 ],\n",
       "       [ 7.543438 , -8.310806 ],\n",
       "       [ 7.6875076, -8.409359 ],\n",
       "       [ 7.082871 , -7.772393 ],\n",
       "       [ 7.507592 , -8.239612 ],\n",
       "       [ 7.5804877, -8.351732 ],\n",
       "       [-5.252204 ,  5.859301 ],\n",
       "       [ 7.6854773, -8.412088 ],\n",
       "       [ 7.521722 , -8.298942 ],\n",
       "       [-4.6146917,  5.1608114],\n",
       "       [ 7.5735655, -8.288543 ],\n",
       "       [ 7.5637817, -8.311525 ],\n",
       "       [ 7.6572976, -8.380521 ],\n",
       "       [ 7.425477 , -8.15442  ],\n",
       "       [-5.203301 ,  5.8034883],\n",
       "       [ 7.509531 , -8.2407465],\n",
       "       [-5.247689 ,  5.8534265],\n",
       "       [-5.2519546,  5.859321 ],\n",
       "       [ 7.4881907, -8.220454 ],\n",
       "       [ 7.5791636, -8.362831 ],\n",
       "       [ 7.443017 , -8.18698  ],\n",
       "       [ 7.5457087, -8.301891 ],\n",
       "       [ 7.6064353, -8.342539 ],\n",
       "       [ 7.3549175, -8.0545   ],\n",
       "       [ 7.602238 , -8.328213 ],\n",
       "       [ 7.7284884, -8.456503 ],\n",
       "       [ 7.5438433, -8.289815 ],\n",
       "       [ 7.547149 , -8.282713 ],\n",
       "       [ 7.5563593, -8.312904 ],\n",
       "       [ 7.6265707, -8.379855 ],\n",
       "       [ 7.5725546, -8.310825 ],\n",
       "       [ 7.4840918, -8.235626 ],\n",
       "       [ 7.5631757, -8.328766 ],\n",
       "       [ 7.6403112, -8.33424  ],\n",
       "       [ 7.5600643, -8.294328 ],\n",
       "       [ 7.6555347, -8.380944 ],\n",
       "       [ 7.6083117, -8.363953 ],\n",
       "       [ 7.5621815, -8.269183 ],\n",
       "       [-5.239398 ,  5.8438606],\n",
       "       [ 7.6852913, -8.444149 ],\n",
       "       [-5.2422867,  5.844893 ],\n",
       "       [ 7.587573 , -8.301436 ],\n",
       "       [-4.6343455,  5.184994 ],\n",
       "       [ 7.612002 , -8.373689 ],\n",
       "       [ 7.663657 , -8.412585 ],\n",
       "       [ 7.298623 , -8.028792 ],\n",
       "       [ 7.540496 , -8.249382 ],\n",
       "       [ 7.6675777, -8.388839 ],\n",
       "       [ 7.5658774, -8.309678 ],\n",
       "       [ 7.720261 , -8.4481325],\n",
       "       [ 4.2845535, -4.7974997],\n",
       "       [-5.2510705,  5.85665  ],\n",
       "       [ 7.6281147, -8.392454 ],\n",
       "       [ 7.5953236, -8.3417015],\n",
       "       [ 7.3754416, -8.068024 ],\n",
       "       [-5.249533 ,  5.8549647],\n",
       "       [ 7.6240726, -8.37813  ],\n",
       "       [ 7.6039515, -8.355018 ],\n",
       "       [ 7.531377 , -8.283064 ],\n",
       "       [ 7.6821017, -8.386851 ],\n",
       "       [ 7.6912613, -8.437476 ],\n",
       "       [ 7.579322 , -8.345947 ],\n",
       "       [ 7.589231 , -8.314208 ],\n",
       "       [ 7.55397  , -8.298797 ],\n",
       "       [ 7.554852 , -8.290119 ],\n",
       "       [ 7.2729487, -7.952231 ],\n",
       "       [ 7.5558233, -8.275735 ],\n",
       "       [ 7.6237097, -8.399854 ],\n",
       "       [ 7.5428257, -8.27793  ],\n",
       "       [ 7.6425447, -8.375258 ],\n",
       "       [ 7.670947 , -8.401555 ],\n",
       "       [ 7.6005135, -8.384587 ],\n",
       "       [-5.2384367,  5.84055  ],\n",
       "       [ 7.5477014, -8.292054 ],\n",
       "       [ 7.451629 , -8.228495 ],\n",
       "       [ 7.4565315, -8.200366 ],\n",
       "       [ 7.4693966, -8.255749 ],\n",
       "       [ 7.6402607, -8.410301 ],\n",
       "       [ 7.5930147, -8.32333  ],\n",
       "       [-5.250742 ,  5.8578157],\n",
       "       [ 7.7223325, -8.442632 ],\n",
       "       [ 7.5814543, -8.306546 ],\n",
       "       [ 7.53473  , -8.25247  ],\n",
       "       [-5.2487993,  5.854325 ],\n",
       "       [ 7.702195 , -8.445192 ],\n",
       "       [ 7.542034 , -8.3113575],\n",
       "       [ 7.5303445, -8.292042 ],\n",
       "       [ 7.6476936, -8.379395 ],\n",
       "       [ 7.558485 , -8.293279 ],\n",
       "       [ 7.565018 , -8.334775 ],\n",
       "       [ 7.570138 , -8.290829 ],\n",
       "       [ 7.582498 , -8.343649 ],\n",
       "       [ 7.5428677, -8.2745905],\n",
       "       [ 7.5416703, -8.260521 ],\n",
       "       [ 7.6810703, -8.417286 ],\n",
       "       [ 7.5174255, -8.270167 ],\n",
       "       [-5.2503567,  5.8564234],\n",
       "       [ 7.592873 , -8.34271  ],\n",
       "       [ 7.6438475, -8.382373 ],\n",
       "       [ 7.627519 , -8.380461 ],\n",
       "       [-5.251341 ,  5.85825  ],\n",
       "       [ 7.613352 , -8.37071  ],\n",
       "       [ 7.6228294, -8.344584 ],\n",
       "       [-5.251873 ,  5.860893 ],\n",
       "       [ 7.643807 , -8.375763 ],\n",
       "       [ 7.6511846, -8.36335  ],\n",
       "       [-5.250536 ,  5.8567395],\n",
       "       [ 7.532034 , -8.256321 ],\n",
       "       [ 7.684739 , -8.441362 ],\n",
       "       [ 7.4772835, -8.214681 ],\n",
       "       [ 7.587618 , -8.322034 ],\n",
       "       [ 7.6039443, -8.362972 ],\n",
       "       [ 7.558876 , -8.331547 ],\n",
       "       [ 7.6931553, -8.442081 ],\n",
       "       [ 7.653843 , -8.415816 ],\n",
       "       [-5.2483373,  5.853567 ],\n",
       "       [ 7.6750026, -8.439881 ],\n",
       "       [ 7.5513206, -8.274494 ],\n",
       "       [-5.2486005,  5.854054 ],\n",
       "       [-3.4749506,  3.8909445],\n",
       "       [ 7.6644177, -8.416852 ],\n",
       "       [ 7.598858 , -8.345535 ],\n",
       "       [ 6.1289263, -6.728544 ],\n",
       "       [ 7.5979576, -8.334131 ],\n",
       "       [-5.2510705,  5.85665  ],\n",
       "       [ 7.637555 , -8.393072 ],\n",
       "       [ 7.352777 , -8.031239 ],\n",
       "       [ 7.611336 , -8.365016 ],\n",
       "       [ 7.4592752, -8.20841  ],\n",
       "       [ 7.5100317, -8.2679205],\n",
       "       [ 7.587479 , -8.3372555],\n",
       "       [ 7.5176888, -8.233915 ],\n",
       "       [ 7.6171503, -8.363439 ],\n",
       "       [ 7.4843483, -8.225303 ],\n",
       "       [ 7.4539194, -8.187805 ],\n",
       "       [ 7.596415 , -8.360098 ],\n",
       "       [ 7.6088004, -8.359897 ],\n",
       "       [ 7.6154814, -8.374072 ],\n",
       "       [ 7.4980507, -8.228984 ],\n",
       "       [ 7.573188 , -8.301871 ],\n",
       "       [ 7.6115007, -8.313132 ],\n",
       "       [ 7.541816 , -8.26197  ],\n",
       "       [ 7.6668873, -8.434727 ],\n",
       "       [-5.2511086,  5.8586044],\n",
       "       [ 7.569207 , -8.335344 ],\n",
       "       [ 7.5327835, -8.283948 ],\n",
       "       [ 7.560976 , -8.32766  ],\n",
       "       [-5.2534304,  5.8635435],\n",
       "       [ 7.5432625, -8.288326 ],\n",
       "       [ 7.5190334, -8.261041 ],\n",
       "       [ 7.7166   , -8.4441805],\n",
       "       [ 7.5388823, -8.288562 ],\n",
       "       [-5.2472224,  5.853087 ],\n",
       "       [ 7.5907784, -8.319691 ],\n",
       "       [ 7.670382 , -8.379931 ],\n",
       "       [ 7.570998 , -8.3253765],\n",
       "       [ 7.5259485, -8.295523 ],\n",
       "       [ 7.5327644, -8.279808 ],\n",
       "       [ 7.6289654, -8.363042 ],\n",
       "       [-5.252779 ,  5.8618865],\n",
       "       [ 7.630412 , -8.394071 ],\n",
       "       [-5.24995  ,  5.859267 ],\n",
       "       [-5.2488027,  5.855585 ],\n",
       "       [ 7.4244328, -8.146019 ],\n",
       "       [ 7.5376797, -8.298314 ],\n",
       "       [ 7.6282697, -8.383434 ],\n",
       "       [ 7.6133366, -8.398637 ],\n",
       "       [ 7.6525292, -8.378881 ],\n",
       "       [ 7.5931654, -8.313433 ],\n",
       "       [ 7.680616 , -8.448978 ],\n",
       "       [ 7.584288 , -8.356336 ],\n",
       "       [ 7.642079 , -8.373949 ],\n",
       "       [ 7.5659   , -8.308207 ],\n",
       "       [-5.24567  ,  5.8506064],\n",
       "       [ 7.5855303, -8.3561   ],\n",
       "       [-5.2508607,  5.8573318],\n",
       "       [ 7.7157264, -8.439567 ],\n",
       "       [ 7.4989023, -8.257257 ],\n",
       "       [ 7.6036625, -8.373937 ],\n",
       "       [ 7.654283 , -8.399307 ],\n",
       "       [ 7.6044226, -8.343374 ],\n",
       "       [ 7.605862 , -8.381064 ],\n",
       "       [ 7.6160316, -8.351792 ],\n",
       "       [ 7.6308236, -8.39065  ],\n",
       "       [ 7.605895 , -8.365038 ],\n",
       "       [ 7.551901 , -8.319857 ],\n",
       "       [ 7.6177926, -8.348686 ],\n",
       "       [ 7.6252174, -8.347843 ],\n",
       "       [ 7.5619416, -8.298776 ],\n",
       "       [ 7.6083493, -8.375766 ],\n",
       "       [ 7.665314 , -8.411704 ],\n",
       "       [ 7.594079 , -8.36704  ],\n",
       "       [-5.2510595,  5.8582954],\n",
       "       [ 7.6857567, -8.401493 ],\n",
       "       [ 7.5431976, -8.26217  ],\n",
       "       [ 7.53364  , -8.288118 ],\n",
       "       [ 7.596803 , -8.339126 ],\n",
       "       [ 7.6209826, -8.373034 ],\n",
       "       [ 7.596384 , -8.331917 ],\n",
       "       [ 7.589958 , -8.360411 ],\n",
       "       [ 7.6776724, -8.393989 ],\n",
       "       [ 7.6097965, -8.370576 ],\n",
       "       [ 7.6551466, -8.428358 ],\n",
       "       [ 7.648414 , -8.423066 ],\n",
       "       [ 7.5833483, -8.364419 ],\n",
       "       [-5.2338853,  5.8405404],\n",
       "       [ 7.50492  , -8.246146 ],\n",
       "       [ 7.6657405, -8.388137 ],\n",
       "       [ 7.608806 , -8.367283 ],\n",
       "       [-5.2513423,  5.858045 ],\n",
       "       [ 7.5539074, -8.285994 ],\n",
       "       [ 7.598975 , -8.303747 ],\n",
       "       [ 7.5615864, -8.261818 ],\n",
       "       [ 7.516762 , -8.264385 ],\n",
       "       [-5.2482767,  5.853627 ],\n",
       "       [ 7.6280036, -8.386332 ],\n",
       "       [ 7.623302 , -8.3894615],\n",
       "       [ 7.6599035, -8.371429 ]], dtype=float32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86fb9825-bf35-4de4-9daa-b094789be140",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}