sentiment_analysis_BiLSTM_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from gensim import models\n",
    "import numpy as np\n",
    "import gensim\n",
    "import gensim.downloader\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import os\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('data/sentiment_treebank/train.csv')\n",
    "df_eval = pd.read_csv('data/sentiment_treebank/eval.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentences = df_train['sentence'].to_list()\n",
    "train_sentences = [sentence.lower().split() for sentence in train_sentences]\n",
    "train_labels = df_train['label'].to_list()\n",
    "\n",
    "eval_sentences = df_train['sentence'].to_list()\n",
    "eval_sentences = [sentence.lower().split() for sentence in eval_sentences]\n",
    "eval_labels = df_train['label'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = -np.inf\n",
    "for sentence in train_sentences:\n",
    "    if len(sentence) > max_length:\n",
    "        max_length = len(sentence)\n",
    "\n",
    "for sentence in eval_sentences:\n",
    "    if len(sentence) > max_length:\n",
    "        max_length = len(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = 'logs/sentiment_analysis'\n",
    "os.makedirs(root_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_size = 50\n",
    "window_size = 5\n",
    "negative_size = 15\n",
    "sentence_size = 60\n",
    "\n",
    "wv_model_file = root_dir + '/' + 'wv_bilstm.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wv_model = gensim.downloader.load('glove-wiki-gigaword-50')\n",
    "# wv_model = models.Word2Vec(sentences=sentences, vector_size=vector_size, window=window_size, negative=negative_size).wv\n",
    "# wv_model = models.Word2Vec(corpus_file='data/corpus.txt', vector_size=vector_size, window=window_size, negative=negative_size).wv\n",
    "\n",
    "# wv_model.save(wv_model_file)\n",
    "# del wv_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_sentences(sentences, wv, sentence_size):\n",
    "    voc = wv.key_to_index.keys()\n",
    "    unk = wv['<unk>']\n",
    "    eos = wv['<eos>']\n",
    "    lengths = []\n",
    "    for i, sentence in enumerate(sentences):\n",
    "        lengths.append(len(sentence))\n",
    "        \n",
    "        for i, token in enumerate(sentence):\n",
    "            if token in voc:\n",
    "                sentence[i] = wv[token]\n",
    "            else:\n",
    "                sentence[i] = unk\n",
    "        \n",
    "        \n",
    "        while len(sentence) < sentence_size:\n",
    "            sentence.append(eos)\n",
    "        \n",
    "        \n",
    "    return sentences, lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv = models.KeyedVectors.load(wv_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv.add_vectors(\n",
    "    ['<unk>', '<eos>'],\n",
    "    [np.zeros(wv.vector_size), np.ones(wv.vector_size)]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(model, file_name):\n",
    "    torch.save(model.state_dict(), file_name)\n",
    "def load_model(model, file_name):\n",
    "    return model.load_state_dict(torch.load(file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 256\n",
    "num_layers = 2\n",
    "bidirectional = True\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "lr = 0.001\n",
    "num_epochs = 50\n",
    "\n",
    "\n",
    "model_file = root_dir + '/' + 'lstm_model_v2.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentences, train_lengths = vectorize_sentences(train_sentences, wv, sentence_size)\n",
    "train_sentences = np.array(train_sentences)\n",
    "train_sentences = torch.tensor(train_sentences, dtype=torch.float)\n",
    "\n",
    "train_lengths = torch.tensor(train_lengths, dtype=torch.int)\n",
    "train_labels = torch.tensor(train_labels, dtype=torch.long)\n",
    "\n",
    "\n",
    "eval_sentences, eval_lengths = vectorize_sentences(eval_sentences, wv, sentence_size)\n",
    "eval_sentences = np.array(eval_sentences)\n",
    "eval_sentences = torch.tensor(eval_sentences, dtype=torch.float)\n",
    "\n",
    "eval_lengths = torch.tensor(eval_lengths, dtype=torch.int)\n",
    "eval_labels = torch.tensor(eval_labels, dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(list(zip(train_sentences, train_lengths, train_labels)), batch_size, shuffle=True)\n",
    "\n",
    "eval_loader = torch.utils.data.DataLoader(list(zip(eval_sentences, eval_lengths, eval_labels)), batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Classifier, self).__init__()\n",
    "        lstm_dim = hidden_size * 2 * (2 if bidirectional else 1)\n",
    "        \n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=vector_size,\n",
    "                              hidden_size=hidden_size,\n",
    "                              num_layers=num_layers,\n",
    "                              bidirectional=bidirectional,\n",
    "                              batch_first=True\n",
    "                             )\n",
    "        self.dropout = nn.Dropout(p=0.5)\n",
    "\n",
    "        \n",
    "        self.fcnn_1 = nn.Linear(in_features=lstm_dim, out_features=64)\n",
    "        \n",
    "        self.fcnn_2 = nn.Linear(in_features=64, out_features=2)\n",
    "        \n",
    "        \n",
    "    def forward(self, sentences, lengths):\n",
    "        sentences = pack_padded_sequence(sentences, lengths.cpu(), batch_first=True, enforce_sorted=False)\n",
    "        h_lstm, _ = self.lstm(sentences)\n",
    "        output, _ = pad_packed_sequence(h_lstm, batch_first=True)\n",
    "        \n",
    "        avg_pool = torch.mean(output, 1)\n",
    "        max_pool, _ = torch.max(output, 1)\n",
    "        \n",
    "        \n",
    "        output = torch.cat([avg_pool, max_pool], 1)\n",
    "        output = self.dropout(output)\n",
    "        \n",
    "        output = self.fcnn_1(output)\n",
    "        output = torch.relu(output)\n",
    "        \n",
    "        output = self.fcnn_2(output)\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = Classifier().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weights = 1/(train_y.sum(dim=0) / train_y.sum())\n",
    "# weights = weights.detach().to(device)\n",
    "# weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(classifier.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 0 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.66it/s, Epoch=0, Train loss=0.643]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.46it/s, Epoch=0, Eval loss=0.556, Eval score=0.729]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 1 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.63it/s, Epoch=1, Train loss=0.541]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.03it/s, Epoch=1, Eval loss=0.507, Eval score=0.766]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 2 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 24.75it/s, Epoch=2, Train loss=0.498]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.43it/s, Epoch=2, Eval loss=0.484, Eval score=0.769]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 3 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.83it/s, Epoch=3, Train loss=0.476]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 80.16it/s, Epoch=3, Eval loss=0.448, Eval score=0.786]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 4 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.85it/s, Epoch=4, Train loss=0.455]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.02it/s, Epoch=4, Eval loss=0.424, Eval score=0.801]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 5 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:01<00:00, 26.18it/s, Epoch=5, Train loss=0.435]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 88.95it/s, Epoch=5, Eval loss=0.422, Eval score=0.802]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 6 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.44it/s, Epoch=6, Train loss=0.411]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.32it/s, Epoch=6, Eval loss=0.372, Eval score=0.836]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 7 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:01<00:00, 26.15it/s, Epoch=7, Train loss=0.389]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.06it/s, Epoch=7, Eval loss=0.355, Eval score=0.845]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 8 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:01<00:00, 26.01it/s, Epoch=8, Train loss=0.367]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 81.05it/s, Epoch=8, Eval loss=0.351, Eval score=0.842]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 9 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.95it/s, Epoch=9, Train loss=0.338]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.16it/s, Epoch=9, Eval loss=0.335, Eval score=0.851]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 10 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.98it/s, Epoch=10, Train loss=0.32] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.86it/s, Epoch=10, Eval loss=0.251, Eval score=0.899]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 11 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.69it/s, Epoch=11, Train loss=0.272]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.97it/s, Epoch=11, Eval loss=0.232, Eval score=0.903]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 12 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.52it/s, Epoch=12, Train loss=0.248]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.71it/s, Epoch=12, Eval loss=0.177, Eval score=0.935]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 13 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.77it/s, Epoch=13, Train loss=0.208]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.44it/s, Epoch=13, Eval loss=0.145, Eval score=0.951]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 14 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.66it/s, Epoch=14, Train loss=0.17] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.14it/s, Epoch=14, Eval loss=0.139, Eval score=0.944]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 15 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.91it/s, Epoch=15, Train loss=0.152]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.23it/s, Epoch=15, Eval loss=0.152, Eval score=0.937]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 16 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.98it/s, Epoch=16, Train loss=0.127]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.23it/s, Epoch=16, Eval loss=0.0865, Eval score=0.966]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 17 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.52it/s, Epoch=17, Train loss=0.1]   \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.51it/s, Epoch=17, Eval loss=0.0588, Eval score=0.981]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 18 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:01<00:00, 26.65it/s, Epoch=18, Train loss=0.0838]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 76.37it/s, Epoch=18, Eval loss=0.0569, Eval score=0.982]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 19 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.48it/s, Epoch=19, Train loss=0.0752]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.13it/s, Epoch=19, Eval loss=0.06, Eval score=0.982]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 20 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.74it/s, Epoch=20, Train loss=0.0754]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.39it/s, Epoch=20, Eval loss=0.0382, Eval score=0.989]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 21 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.61it/s, Epoch=21, Train loss=0.0573]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.40it/s, Epoch=21, Eval loss=0.0295, Eval score=0.99] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 22 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.57it/s, Epoch=22, Train loss=0.058] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.48it/s, Epoch=22, Eval loss=0.0302, Eval score=0.989]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 23 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.46it/s, Epoch=23, Train loss=0.049] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.19it/s, Epoch=23, Eval loss=0.024, Eval score=0.994] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 24 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.05it/s, Epoch=24, Train loss=0.0414]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 75.97it/s, Epoch=24, Eval loss=0.0164, Eval score=0.995]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 25 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.56it/s, Epoch=25, Train loss=0.041] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 76.66it/s, Epoch=25, Eval loss=0.0268, Eval score=0.99] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 26 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.59it/s, Epoch=26, Train loss=0.0374]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.35it/s, Epoch=26, Eval loss=0.0344, Eval score=0.988]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 27 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.77it/s, Epoch=27, Train loss=0.0337]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.04it/s, Epoch=27, Eval loss=0.0133, Eval score=0.997]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 28 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.52it/s, Epoch=28, Train loss=0.0209]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 76.97it/s, Epoch=28, Eval loss=0.0092, Eval score=0.997] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 29 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.47it/s, Epoch=29, Train loss=0.0185]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.66it/s, Epoch=29, Eval loss=0.00576, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 30 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.73it/s, Epoch=30, Train loss=0.0436]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.49it/s, Epoch=30, Eval loss=0.0232, Eval score=0.993]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 31 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.34it/s, Epoch=31, Train loss=0.027] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.53it/s, Epoch=31, Eval loss=0.0134, Eval score=0.997]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 32 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.53it/s, Epoch=32, Train loss=0.0125]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 76.94it/s, Epoch=32, Eval loss=0.0045, Eval score=0.999] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 33 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:01<00:00, 26.20it/s, Epoch=33, Train loss=0.00865]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.81it/s, Epoch=33, Eval loss=0.00544, Eval score=0.998]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 34 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:01<00:00, 26.25it/s, Epoch=34, Train loss=0.0226] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 85.80it/s, Epoch=34, Eval loss=0.0149, Eval score=0.996]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 35 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.82it/s, Epoch=35, Train loss=0.0195]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.69it/s, Epoch=35, Eval loss=0.00377, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 36 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.49it/s, Epoch=36, Train loss=0.00911]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.93it/s, Epoch=36, Eval loss=0.00446, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 37 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.69it/s, Epoch=37, Train loss=0.0182]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.39it/s, Epoch=37, Eval loss=0.0314, Eval score=0.991]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 38 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.72it/s, Epoch=38, Train loss=0.0481]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.37it/s, Epoch=38, Eval loss=0.0285, Eval score=0.991]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 39 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.75it/s, Epoch=39, Train loss=0.0244]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.63it/s, Epoch=39, Eval loss=0.00735, Eval score=0.998]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 40 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.80it/s, Epoch=40, Train loss=0.0131]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.30it/s, Epoch=40, Eval loss=0.00513, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 41 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.79it/s, Epoch=41, Train loss=0.00816]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.92it/s, Epoch=41, Eval loss=0.00248, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 42 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.40it/s, Epoch=42, Train loss=0.0078] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 79.69it/s, Epoch=42, Eval loss=0.00323, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 43 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.48it/s, Epoch=43, Train loss=0.00822]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.99it/s, Epoch=43, Eval loss=0.00702, Eval score=0.998]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 44 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.32it/s, Epoch=44, Train loss=0.0105] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.85it/s, Epoch=44, Eval loss=0.00283, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 45 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.90it/s, Epoch=45, Train loss=0.0104] \n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.07it/s, Epoch=45, Eval loss=0.0091, Eval score=0.997] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 46 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.85it/s, Epoch=46, Train loss=0.0266]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 78.33it/s, Epoch=46, Eval loss=0.0306, Eval score=0.988]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 47 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.63it/s, Epoch=47, Train loss=0.0216]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 76.67it/s, Epoch=47, Eval loss=0.00731, Eval score=0.998]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 48 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.45it/s, Epoch=48, Train loss=0.00969]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 77.66it/s, Epoch=48, Eval loss=0.00361, Eval score=0.999]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 49 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 52/52 [00:02<00:00, 25.74it/s, Epoch=49, Train loss=0.00792]\n",
      "100%|\u001b[32m██████████\u001b[0m| 52/52 [00:00<00:00, 82.02it/s, Epoch=49, Eval loss=0.00603, Eval score=0.998]\n"
     ]
    }
   ],
   "source": [
    "min_loss = np.inf\n",
    "\n",
    "global_train_losses = []\n",
    "global_eval_losses = []\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    print(f'---> Epoch {i} <---')\n",
    "    time.sleep(0.5)\n",
    "    \n",
    "    classifier.train()\n",
    "    loader = tqdm(train_loader, postfix={'Epoch': i})\n",
    "    train_losses = []\n",
    "    \n",
    "    for sentences, lengths, targets in loader:\n",
    "        sentences = sentences.to(device)\n",
    "        targets = targets.to(device)\n",
    "        lengths = lengths.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        outputs = classifier(sentences, lengths)\n",
    "        \n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            train_losses.append(loss.item())\n",
    "            loader.set_postfix({\n",
    "                'Epoch': i,\n",
    "                'Train loss': np.mean(train_losses)\n",
    "            }, refresh=True)\n",
    "    \n",
    "    global_train_losses.append(np.mean(train_losses))\n",
    "    \n",
    "    time.sleep(0.5)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        classifier.eval()\n",
    "        loader = tqdm(eval_loader, postfix={'Epoch': i,}, colour='green')\n",
    "        eval_losses = []\n",
    "        eval_scores = []\n",
    "\n",
    "        for sentences, lengths, targets in loader:\n",
    "            sentences = sentences.to(device)\n",
    "            targets = targets.to(device)\n",
    "            lengths = lengths.to(device)\n",
    "\n",
    "            outputs = classifier(sentences, lengths)\n",
    "            loss = criterion(outputs, targets)\n",
    "            \n",
    "            score = (outputs.argmax(dim=1) == targets).detach().cpu().numpy()\n",
    "            eval_scores.append(score)\n",
    "            eval_losses.append(loss.item())\n",
    "            loader.set_postfix({\n",
    "                'Epoch': i,\n",
    "                'Eval loss': np.mean(eval_losses),\n",
    "                'Eval score': np.concatenate(eval_scores).mean()\n",
    "            }, refresh=True)\n",
    "\n",
    "\n",
    "        eval_loss = np.mean(eval_losses)\n",
    "        if eval_loss <= min_loss:\n",
    "            min_loss = eval_loss\n",
    "            save_model(classifier, model_file)\n",
    "            loader.write('*** save ***')\n",
    "        \n",
    "    global_eval_losses.append(np.mean(eval_losses))\n",
    "    \n",
    "    time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyA0lEQVR4nO3de5zMZfvA8c9l11qnKG3IyqHI2coiZUVyKHKoFD0lqTxSUT2ddFCJ0unpqF+kojOVHKJ0eJJDKWuRnLXIIi0hZK3dvX5/3LNrsYexO7uzM3O9X695zcz3e898r+9Y19xzf++DqCrGGGMCXyl/B2CMMcY3LKEbY0yQsIRujDFBwhK6McYECUvoxhgTJML9deDTTz9da9eu7a/DG2NMQFq6dOkuVY3KaZ/fEnrt2rWJj4/31+GNMSYgiciW3PZZk4sxxgQJS+jGGBMkLKEbY0yQ8FsbujEm8B05coSkpCRSUlL8HUrQiYyMJDo6mtKlS3v9GkvoxpgCS0pKomLFitSuXRsR8Xc4QUNV2b17N0lJSdSpU8fr11mTizGmwFJSUqhSpYolcx8TEapUqXLSv3wsoRtjCsWSedEoyOcacAl90SIYMQJs1l9jjDlWwCX0hAQYOxa2bfN3JMYYf9u9ezcxMTHExMRQrVo1atSokfU8NTU1z9fGx8czbNiwAh974MCBfPLJJwV+fVEIuIuiLVu6+4QEiI72byzGGP+qUqUKy5cvB+Cxxx6jQoUK3HPPPVn709LSCA/POc3FxsYSGxtbHGEWm4CroTdvDqVKwdKl/o7EGFMSDRw4kCFDhtCmTRvuu+8+fv75Z9q2bUuLFi244IILWLduHQDz5s2jR48egPsyGDRoEB06dKBu3bq8/PLLBTp2SkoKN954I02bNqVFixZ89913AKxatYrWrVsTExNDs2bN2LBhAwcPHqR79+40b96cJk2aMGXKlEKfe8DV0MuXhwYNLKEbU9LceSd4Kss+ExMDL7548q9LSkrihx9+ICwsjL///psFCxYQHh7ON998w4MPPsinn356wmvWrl3Ld999x/79+zn33HO59dZbT6oPOMC4ceMQEVauXMnatWvp0qUL69ev5/XXX2f48OH861//IjU1lfT0dObMmcOZZ57J7NmzAdi3b9/Jn+hxAi6hg2t2+eYbf0dhjCmp+vbtS1hYGOAS5Q033MCGDRsQEY4cOZLja7p3706ZMmUoU6YMZ5xxBjt37iT6JNt1Fy5cyB133AFAgwYNqFWrFuvXr6dt27aMGTOGpKQkrrjiCurVq0fTpk35z3/+w/3330+PHj2Ii4sr3EkTwAn93Xdhxw6oXt3f0RhjoGA16aJSvnz5rMePPPIIHTt25LPPPmPz5s106NAhx9eUKVMm63FYWBhpaWk+i+faa6+lTZs2zJ49m8suu4zx48dz8cUXk5CQwJw5c3j44Yfp1KkTI0eOLNRxAq4NHY5eGLVmF2NMfvbt20eNGjUAmDRpUpEeKy4ujvfffx+A9evX8/vvv3PuueeSmJhI3bp1GTZsGL169eKXX35h+/btlCtXjuuuu457772XhISEQh8/IBN6TAyIWEI3xuTvvvvuY8SIEbRo0aJAte7LLruM7du357jv3//+N9HR0URHR9O2bVuGDh1KRkYGTZs25ZprrmHSpEmUKVOGqVOn0qRJE2JiYvj1118ZMGAAK1euzLpQ+vjjj/Pwww8X9lQR9dMIndjYWC3MAhcNG0L9+jBjhg+DMsaclDVr1tCwYUN/hxG0cvp8RWSpqubY3zIga+jgml2shm6MMUd5ldBFpJuIrBORjSLyQC5lrhaR1SKySkQ+8G2YJzrvPDdadOfOoj6SMcYEhnwTuoiEAeOAS4FGQH8RaXRcmXrACOBCVW0M3On7UI+VfcSoMcYY72rorYGNqpqoqqnAR0Cv48rcAoxT1T0Aqvqnb8M8UYsW7t6aXYwxxvEmodcAtmZ7nuTZll19oL6ILBKRxSLSLac3EpHBIhIvIvHJyckFi9jjlFPcRVFL6MYY4/jqomg4UA/oAPQH3hCRyscXUtUJqhqrqrFRUVGFPuh551lCN8aYTN4k9G1AzWzPoz3bsksCZqrqEVXdBKzHJfgi1bIlbN0KhazsG2MCWFhYWNaUuTExMYwdO7ZA79OhQwfy60r92GOP8dxzzxXo/YuDN0P/lwD1RKQOLpH3A649rsx0XM38bRE5HdcEk+jDOHOU/cJo165FfTRjTElUtmzZrCl0Q12+NXRVTQNuB+YCa4CpqrpKREaJSE9PsbnAbhFZDXwH3Kuqu4sq6Ex2YdQYk5Mvv/ySvn37Zj3PPlXurbfeSmxsLI0bN+bRRx8t9LFUlXvvvZcmTZrQtGnTrGlwd+zYQfv27YmJiaFJkyYsWLCA9PR0Bg4cmFX2hRdeKPTxs/Nqci5VnQPMOW7byGyPFbjbcys2lSvD2Wdb10VjSgQ/zZ976NAhYmJisp6PGDGCK6+8ksGDB3Pw4EHKly/PlClT6NevHwBjxozhtNNOIz09nU6dOvHLL7/QrFmzAoc4bdo0li9fzooVK9i1axetWrWiffv2fPDBB3Tt2pWHHnqI9PR0/vnnH5YvX862bdv49ddfAdi7d2+Bj5uTgJxtMbuWLeHnn/0dhTHGX3JrcunWrRuzZs3iqquuYvbs2TzzzDMATJ06lQkTJpCWlsaOHTtYvXp1oRL6woUL6d+/P2FhYVStWpWLLrqIJUuW0KpVKwYNGsSRI0fo3bs3MTEx1K1bl8TERO644w66d+9Oly5dCnzcnARFQp86FXbvhipV/B2NMSGsJM2fC/Tr149XX32V0047jdjYWCpWrMimTZt47rnnWLJkCaeeeioDBw4kJSWlSI7fvn175s+fz+zZsxk4cCB33303AwYMYMWKFcydO5fXX3+dqVOn8tZbb/nsmAE7l0smGzFqjMnJRRddREJCAm+88UZWc8vff/9N+fLlqVSpEjt37uSLL74o9HHi4uKYMmUK6enpJCcnM3/+fFq3bs2WLVuoWrUqt9xyCzfffDMJCQns2rWLjIwMrrzySkaPHu2TKXOzC/gaeuaF0YQE6NzZv7EYY4rf8W3o3bp1Y+zYsYSFhdGjRw8mTZrE5MmTAWjevDktWrSgQYMG1KxZkwsvvDDH97z55psZMmRIjotIjx49mhez/RrZunUrP/74I82bN0dEeOaZZ6hWrRqTJ0/m2WefpXTp0lSoUIF33nmHbdu2ceONN5KRkQHAU0895bsPggCePje7unUhNtY1vRhjio9Nn1u0Qmb63OxsKl1jjAmShH7eeZCYCHv2+DsSY4zxn8BL6FOmQFwcZFtKKvPC6LJlforJmBDmr2bbYFeQzzXwEnrp0rBwIWS7Om2LRhvjH5GRkezevduSuo+pKrt37yYyMvKkXhd4vVwuvxyqV4fXX3ePcf3Pa9WyhG5McYuOjiYpKYnCTodtThQZGUl0dPRJvSbwEnrp0nDzzTB6NGzeDLVrA64d3fqiG1O8SpcuTZ06dfwdhvEIvCYXcAldBN54I2tTy5awYQPs2+fHuIwxxo8CM6GfdRZ07w5vvgmpqYDrhw7w/fd+jMsYY/woMBM6wJAhsHMnzJgBQMeOEB0Nzz/v57iMMcZPAjehd+3qroS+/joAERFw990wfz4sXuzn2Iwxxg8CN6GHhcHgwfC//8G6dQDccguceio8/bSfYzPGGD8I3IQOMGgQhIfDhAkAVKgAt98O06fDmjX+Dc0YY4pbYCf0atWgTx+YNAkOHQLgjjugbFl49ln/hmaMMcUtsBM6uIujf/0Fn3wCQFQU3HQTvPceJCX5OTZjjClGgZ/QO3aE+vWzLo4C/Oc/kJEBPl5/1RhjSrTAT+gi8O9/ww8/wC+/AG7w6DXXuKb1v/7yb3jGGFNcAj+hA9xwA5QpA+PHZ226/344cABee82PcRljTDHyKqGLSDcRWSciG0XkgRz2DxSRZBFZ7rnd7PtQ81ClClx9Nbz7LuzfD0CzZnDppfDyy1nXS40xJqjlm9BFJAwYB1wKNAL6i0ijHIpOUdUYz22ij+PM3+23uyr5dddlzZV+//2QnAxvv13s0RhjTLHzpobeGtioqomqmgp8BPQq2rAKoHVreOUVmDnT9XxRpX17OP9814Ux23oYxhgTlLxJ6DWArdmeJ3m2He9KEflFRD4RkZo5vZGIDBaReBGJL5L5k2+7DR55xE3a9fDDiLha+ubNx0zMaIwxQclXF0VnAbVVtRnwNTA5p0KqOkFVY1U1NioqykeHPs7jj7spAZ58El5+mZ49oVMnN+Bo1qyiOaQxxpQE3iT0bUD2Gne0Z1sWVd2tqoc9TycCLX0TXgGIuK4tffrA8OGUmvoRn33mFsDo2xfmzfNbZMYYU6S8SehLgHoiUkdEIoB+wMzsBUSkeranPQH/zqQSFgYffADt28OAAVRc/DVffAFnn+1WrYuP92t0xhhTJPJN6KqaBtwOzMUl6qmqukpERolIT0+xYSKySkRWAMOAgUUVsNciI91c6Q0bQp8+VNm6nK++gtNPh27dbPIuY0zwEX+t1h0bG6vxxVFV3rHDtbfUrg0//MDG34R27dwkjQsXZi1JaowxAUFElqpqbE77gmOkaF6qV4cxY9yqFx9/zDnnwFdfwcGD0Lkz/PGHvwM0xhjfCP6EDm5qgGbN4IEH4PBhmjWDOXNg+3a38NGePf4O0BhjCi80EnpYGDz3HGzaBOPGAdC2rVsIY+1at970wYP+DdEYYworNBI6uPaVbt3giSdg9+6sTR9+CD/95Ho5Hj6cz3sYY0wJFjoJHdwcAH//DaNHZ2264go3sPTrr+Haa22KAGNM4AqthN6kiVvOaNw42Lgxa/PAgfDiizBtmltoOiPDbxEaY0yBhVZCBxg1CiIi3AXSbIYPh8cec8uT3n03+Kk3pzHGFFjoJfRq1dyMXZ9+CosWHbNr5Ei480546SWX3I0xJpCEXkIHt+jomWe6+2xVcRF4/nm48UZXkR850mrqxpjAEZoJvVw5N9jop59gypRjdpUq5abaHTTIdYgZMcKSujEmMIRmQge4/no3JcDQobBhwzG7wsJcUh8yBJ5++oSKvDHGlEihm9DDwuDjj919jx4nDBctVcrNwjtsGLzwgptP3Xq/GGNKstBN6AB167q+ips2uUWmj+uELuK6M95zj+vp+O9/W1I3xpRcoZ3QAeLi4PXX4Ztv4K67TtgtAs88Aw8+CBMnurZ1S+rGmJIo3N8BlAiDBsHq1a6LS8OGrl09GxE3uLR0abfCXXT0MYNNjTGmRLCEnunpp91MXcOGQf36cMklx+wWgUcfhaQk10GmUSM3VYAxxpQU1uSSKXPZugYN3OKj69efUCRzudK4OFep//lnP8RpjDG5sBp6dqecArNmQevWrgoeFXXCLeLcc/n0k9to3Ubo3RuWLIEaNfwduDHGWEI/UZ06MG8evP8+JCcfvSUkwM6d8PffRDVuzMyZHbngAujdG77/3o1VMsYYfwr+NUV96dAhVx3v0gU++oiZM11Cv/pqN6+6iL8DNMYEu9BeU9SXypZ1y9lNmwbJyfTsCU895WYPGDPG38EZY0KdJfSTdcstcOQITJ4MwH33uVkEHnkEPvrIz7EZY0KaVwldRLqJyDoR2SgiD+RR7koRURHJ8edAUGjUCNq1gwkTQBUR97B9exgwwK18ZIwx/pBvQheRMGAccCnQCOgvIo1yKFcRGA785OsgS5zBg92EXt9/D0BkJMyY4cYk9enjer4YY0xx86aG3hrYqKqJqpoKfAT0yqHcE8DTQIoP4yuZrroKKld2VXOPypXhyy9d78bLLoN16/wWnTEmRHmT0GsAW7M9T/JsyyIi5wE1VXV2Xm8kIoNFJF5E4pOTk0862BKjbFnXvvLpp7BrV9bm6tXhq69cb5euXWH7dj/GaIwJOYW+KCoipYD/Av/Jr6yqTlDVWFWNjYqKKuyh/WvwYEhNzbo4mqlePfjiC9i92yX142blNcaYIuNNQt8G1Mz2PNqzLVNFoAkwT0Q2A+cDM4P6wihA48Zw4YVZF0eza9kSpk93swf07Om6rxtjTFHzJqEvAeqJSB0RiQD6ATMzd6rqPlU9XVVrq2ptYDHQU1UDbNRQAQwe7LL2/Pkn7OrUCd57z61D3a/fCVOtG2OMz+Wb0FU1DbgdmAusAaaq6ioRGSUiPYs6wBKtb98TLo4ev/uVV2DmTLjtNlvGzhhTtLyay0VV5wBzjts2MpeyHQofVoAoW9aNKho/Hl5+GapUOaHIbbe5i6NPPukumj72WPGHaYwJDTZStLBuucVdHH3nnVyLjB4NN97oFscYP74YYzPGhBRL6IXVtCm0bZvjxdFMmaNJu3d3iyFNn168IRpjQoMldF8YPNitdpRH9Ts83E3i1aqVu0i6cGExxmeMCQmW0H2hf3/o3BluvRXuuQfS03MsVr48fP451KoFl18Ov/5azHEaY4KaJXRfKFMG5syB2293C0336gV//51j0dNPh7lz3fXUXr1g//5ijtUYE7QsoftKeLjro/jaa25SlwsugMTEHIvWru2aXzZvdmtSG2OML1hC97Vbb3UTumzf7tYmzWHQEbiFph98ECZNgo8/Lt4QjTHByRJ6Ubj4YvjpJ9e+csklrs/im2+6C6fZesKMHOly/uDBsHVrHu9njDFesIReVOrVg8WLXZeWWbPg5pvdhOlRUW6Cl2eeofSWjbz/vlsAacCAXK+lGmOMVyyhF6XKld2Ao+RkWLMGJk50yXztWrj/fmjenHMWv8crr8C8ee56qjHGFJSonyYYiY2N1fj44J+/K1e//+6mDZg/Hx38b67980U+nR3J4sVw3nn+Ds4YU1KJyFJVzXE2W6uh+8tZZ8G338L99yMTxvPupna0PG0T114L//zj7+CMMYHIEro/hYfD2LEwYwbhmzcy/+B51Fv3OcOH28yMxpiTZwm9JOjZExISKF2vDrO4nKoTRzN4sM2hbow5OZbQS4q6deGHH9A+fRgljzJl4t/07g0HD/o7MGNMoLCEXpJERiJDh1JKM/jgjh/54gvo2NF1kjHGmPxYQi9pzj8fwsLoccoCpk2DlSvdLAK//ebvwIwxJZ0l9JKmQgVo0QIWLKBXL/jf/2DPHjfl+pIl/g7OGFOSWUIvieLi3NQBhw/Ttq1baLp8edf88vvv/g7OGFNSWUIvieLi4PBh8Ay8Ovdc12X90CF4/XU/x2aMKbEsoZdE7dq5+wULsjbVresWxZg40eV6Y4w5niX0kigqCho0OGGduqFDXY+XTz/1U1zGmBLNq4QuIt1EZJ2IbBSRB3LYP0REVorIchFZKCKNfB9qiImLc43nGRlZmy65BM45x62hYYwxx8s3oYtIGDAOuBRoBPTPIWF/oKpNVTUGeAb4r68DDTlxcbB37zELj5Yq5dbPWLQIVqzwX2jGmJLJmxp6a2CjqiaqairwEdArewFVzb6AZnnAZiIprBza0QEGDoTISPi//yv+kIwxJZs3Cb0GkH09nSTPtmOIyG0i8huuhp7jSpkiMlhE4kUkPtmGP+atdm2oUeOEhH7aadC/P7z3Huzb55/QjDElk88uiqrqOFU9G7gfeDiXMhNUNVZVY6Oionx16OAk4ppdFiw4YerFoUPdHC/vvuun2IwxJZI3CX0bUDPb82jPttx8BPQuREwmU1ycW2x606ZjNsfGQqtW7uKoTbNrjMnkTUJfAtQTkToiEgH0A2ZmLyAi9bI97Q5s8F2IISwuzt0f130RXC19zRr4/vtijskYU2Llm9BVNQ24HZgLrAGmquoqERklIj09xW4XkVUishy4G7ihqAIOKY0bw6mnntCODnDNNW6XdWE0xmQK96aQqs4B5hy3bWS2x8N9HJcB10/xwgtzTOhly8KgQfDSS65V5swz/RCfMaZEsZGiJV27drBuHfz55wm7hgxxqxpNnOiHuIwxJY4l9JIuj3b0c86Brl1h/Hg4cqSY4zLGlDiW0Eu62Fg3kiiHZhdwF0e3b7daujHGEnrJFxEBbdrkmtC7d4cuXWDYMDfFrjEmdFlCDwRxcbBsGezff8KusDCYOtXNmX7llbB6tR/iM8aUCJbQA0FcnJt1cfHiHHdXqgSzZ7uWme7dYefOYo7PGFMiWEIPBG3bui6MuTS7ANSqBbNmuWTeq5db3cgYE1osoQeCihWzFo7OS6tW8P778PPPMGDAMVOpG2NCgCX0QNGunWtyyWf9uT594Lnn4JNP4MEHiyk2Y0yJYAk9UFx2GaSkwLXX5pvU77rLDTp6+mk3za4xJjRYQg8UXbrAiy/CtGlutegDB3ItKgKvvALnnw8jRtii0saECkvogWT4cHj7bdfhvHNn2LMn16Lh4TBqFCQluZcYY4KfJfRAM3CgayBPSICLLoIdO3ItesklroPMU09BamrxhWiM8Q9L6IGoTx/X8Twx0fVRP24BjEwiMHIk/P47vPNOMcdojCl2ltAD1SWXuKaXv/5yPWA2b86xWNeurjvjk0/aBF7GBDtL6IGsTRuYPx927XITo+cgs5a+aZP1eDEm2FlCD3RNmrheLx98kGsVvHt3OO88GDPGzZ9ujAlOltCDwYABbgGMr77KcXdmLf233+DDD4s5NmNMsbGEHgy6dYPTT8/zymfPntC8OYweDenpxRibMabYWEIPBhER0L8/zJgBe/fmWEQEHnkE1q+HKVOKNzxjTPGwhB4sBgxwQ0KnTs21SJ8+0Lix1dKNCVaW0INFy5bQsGGezS6lSrla+po18OmnxRibMaZYeJXQRaSbiKwTkY0i8kAO++8WkdUi8ouIfCsitXwfqsmTCNxwAyxa5K5+5uKqq1zef+wx6/FiTLDJN6GLSBgwDrgUaAT0F5FGxxVbBsSqajPgE+AZXwdqvPCvf7nE/u67uRYJC4MnnnC19MmTizE2Y0yR86aG3hrYqKqJqpoKfAT0yl5AVb9T1X88TxcD0b4N03glOho6dXLNLqq5FrviCjcm6dFH4Z9/ci1mjAkw3iT0GsDWbM+TPNtycxPwRU47RGSwiMSLSHxycrL3URrvDRjghoUuWpRrERF45hnYtg1efrkYYzPGFCmfXhQVkeuAWODZnPar6gRVjVXV2KioKF8e2mTq0wfKl893Nq727aFHDxg7FnbvLqbYjDFFypuEvg2ome15tGfbMUTkEuAhoKeq2pIK/lKhgrvyOXVqvitFjx0L+/e7KQGMMYHPm4S+BKgnInVEJALoB8zMXkBEWgDjccn8T9+HaU7KgAGwbx/MnJlnscaN3fTq48blOlmjMSaA5JvQVTUNuB2YC6wBpqrqKhEZJSI9PcWeBSoAH4vIchHJO5OYotWhA9Ss6dUk6I8/7vqnP/xw0YdljClaonn0hihKsbGxGh8f75djh4QHH3RXPpOSoFq1PIuOGOGaXxISoEWLYorPGFMgIrJUVWNz2mcjRYPV9de78f159EnPdP/9cNpp7t4YE7gsoQerhg3dqkZjxuS57ihA5crw0EPw9dfuZowJTJbQg9m4cZCSAnfemW/R226DWrXgnnvgwIGiD80Y43uW0INZ/fpuNq6pU+Hzz/MsWqaMG2T066/QubNbqtQYE1gsoQe7e+91/ROHDs236t2zJ3z8sbs4etFF+bbUGGNKGEvowS4iAt54w/V2eeSRfItfcQXMmeNmD2jXDhITiyFGY4xPWEIPBW3bwq23ujaVJUvyLd6pE3z7rVv8qF07WLmy6EM0xhSeJfRQ8eSTrj/6LbfAkSP5Fm/TBubPdxN5XXQRLF5cDDEaYwrFEnqoqFQJXn0VVqyAF1/06iWNG8Oibw7xdMa9nN02ipWfri/aGI0xhWIJPZT06QO9e7uJ0L1pHP/hB2r3juGWfc8RxS7mDprCzp1FHqUxpoAsoYeaV16B8HDXN3HUKFi16sTFMP75B+6+2zWgHz4M33zDwabnc/H+GfTt61WLjTHGDyyhh5roaNc38cwz3cKiTZq4UaUPPeT6Ky5YAM2bwwsvwJAh7opop06Uv7YX5+lSEhckcddd/j4JY0xOLKGHoq5dXeLetg1ee80l+aefhpYt3coXaWmum8trr0HFiu41Pd3Emi91msW4cfDWW36M3xiTI5tt0Ti7dsGMGe7+ttvcQhnZqUL9+mScfQ5d0r5gwQLXC6ZNG/+Ea0yoymu2xfDiDsaUUKefDjfdlPt+EejZk1KvvsqUdfuJ7ViRK66ApUvznZ3XGFNMrMnFeK9XL0hNpUr8XKZPhz173Gp3qan+DswYA5bQzcm44AI3cfrMmTRvDm+/DYsWuQ4xxhj/s4RuvBceDj16wOzZkJbGNdfAXXe5WXqnTPF3cMYYS+jm5PTs6ebWXbQIcJ1jLrgAbr4Z1q71c2zGhDhL6ObkdOniZnCcMQOA0qVd7Twy0rWnHzzo5/iMCWGW0M3JqVjRTcc4c2bWCNPoaPjgA1i92k3q6KeesMaEPEvo5uT17Am//eYyuEfnzm6KmHffhYkT/RibMSHMq4QuIt1EZJ2IbBSRB3LY315EEkQkTUSu8n2YpkS5/HJ3P3PmMZsfftgl9jvugGXL/BCXMSEu34QuImHAOOBSoBHQX0QaHVfsd2Ag8IGvAzQlUI0aEBub1Y6eKSwM3n/fjVHq29ctkGGMKT7e1NBbAxtVNVFVU4GPgF7ZC6jqZlX9BcgoghhNSdSrF/z0E/zxxzGbo6LcmtRbtriK/K5dforPmBDkTUKvAWzN9jzJs+2kichgEYkXkfjk5OSCvIUpKTyTdfH55yfsuuACeO89t9pdq1a2hJ0xxaVYL4qq6gRVjVXV2KioqOI8tPG1pk2hdu0Tml0yXXONm7zr8GGX4HMpZozxIW8S+jagZrbn0Z5tJpR5Juvim29y7XzeurWrpTdo4BZLeuop69JoTFHyJqEvAeqJSB0RiQD6ATPzeY0JBb16QUpKjs0umWrUcDX1fv3gwQfhuuvg0KFijNGYEJJvQlfVNOB2YC6wBpiqqqtEZJSI9AQQkVYikgT0BcaLyKqiDNqUEHFxrvp9112QxzWRsmVd75cnn3QDkFq0gPHjbVSpMb5mC1yYwlm+3K1y0bkzzJrlmmLyMGcOPPKIW+2uUiUYNAiGDoVzzimecI0JdHktcGEJ3RTeyy/D8OHw0kswbFi+xVVh8WK3XvXHH7sV7y69FK6/HsqVg4wMd1N19+XKuVXzSpcuhnMxpoSzhG6Klqq7QPrVV65vekyM1y/dsQMmTIDXXz+hS/sxLrvM9W8vX77w4RoTyCyhm6K3axc0awannOLWpTvJzJua6qaGUYVSpdxNxN1/+y3ceafr0/75524kqjGhytYUNUXv9NPdaKJLLnHNLm++eVIvj4jIvWLfqBHUrAn9+8OFF8Lcua4LvDHmWDbbovGdiy+GESPgrbfgo498+ta9e8PXX8Off0LbtrBihU/f3pigYE0uxreOHIGLLoJVq+CLL+Dcc906pDn1fklNhY0bYd06dytd2tXu87j6uWoVdOsGf/8N06dDx45FdyrGlETW5GKKT+nSrrN5TIxrH8ncVrUqVK8O1apBerpL4Js2uW4s2a1YAZMmucbzHDRuDD/84JJ6t26uT/s//7jBStlvjRu77vF9+1rvGBM6rIZuisaWLa7Hy44drvtK9ntwNffst/r1XT/GRx5xV0D/+988+7Tv2eMS9h9/uIFL2W8REa6/+7p1bjWl4cPhlltcv3djAp31cjGBQdVl6ZdegjFj3FwBBZSR4ZL688/DvHlu5bybb3YtOnZB1QSyvBK6XRQ1JYeIq5lffz089JDrnF5ApUpBjx7w3XeuF+Xll7vxT3Xruo44771nUw8Uh2nT3Foomzb5O5LQYAndlCylSrkujz16uDkBpk4t9Fued56bS2bTJnjsMUhMdN8Z1arBTTfBggUlbxbItDQYNw42b/Z3JAW3fbv7fJcudf+c+/b5O6LgZ00upmQ6dMiN91+82I0m6tLFZ2+dkQELF8Lkye774sABd8321FNd+3tEhLuQGhEBkZFQr54bM9W0KTRpAhUq+CyUXN19N7zwAlSp4mq57dsX/TF9SdX9Kvrf/1wL2tChrkfS7Nl2kbqwrA3dBKa9e6FDB9iwAe67D4YMcZnXhw4edAnzm2/cTMCpqe525Ii7P3gQ1q51ST9T3bouwTdv7m4xMa5dPp95ybz23nvuF8T118PPP8Nvv8Frr7kLu4Fi0iS48caj0/u8+aa7hjFkiDsXX31WoSivhI6q+uXWsmVLNSZff/yh2r27KqiWLq16/fWqP/9crCGkp6smJqpOn676xOPpOrLDPH2/8q3alS/V1UVVTzlFNS5O9Y47VL/+uuDHWrpUNTJS9aKLVFNTVffsUe3a1R1j2DDVI0d8dVZFZ+tW93m0b+8+u0z33efO44UX/BZaUADiNZe8ajV0Exg2bIBXX4W334b9++H88+GOO9y8uykpJ3ZEr1XLtVP46vf95s3wzjuu6um5wqcREax99nMWRHZm+XLXhX7FCler79cPXnzx5H5QJCe7C4iqEB8PZ5zhtqeluR8oL7zgZimeMsU1D5Uo774LH3yA1qnL69/V5+tN9XhxTn3Oal8bwt1wl4wMNy7gs8/ckoSXX+7fkE/KokXuZ1uHDv6OxGroJojs26f68suq9eppVvU4t1vlyqrXXqs6ZYp73clKTVV9913Vjh3d+4modurktm3dqtqsmWrZsqrz52e9JCVF9fHHVSMiVE89VfXNN1UzMrw7VIcOrnYeH59zmTffdD9S6tdXXbPm5E+nyCxYoBoWphodrSllKx37bxAertqqlermzaqqevCgasuWquXLqy5b5teovffVV5oREaEZ4eGa/vW3/o4mzxq6JXQTmNLTVb//XvXzz1W/+UZ10SLVhASX6TZtUv3sM9WBA1WrVHF/5hERqt26qU6YoLp3b97vnZam+s47qnXruteefbbqE09kJaUsO3eqnnuuasWKJzQDrV7tmmDAJep16/I+5PDhruw772TbuGGDy/TZLFigGhWlWq6c6sSJ3n1Z5CV7k0iB7NqlGh2tWreubvllr1askKF92v2p6QsWqb79tuqIEaqVKrnPcPt2VVXdts29pEYN18RUoi1apEfKlNOVYc30VxrpXqmkt3Vcpf/9r4s9La34Q7KEbkLXkSMu8d99t0sq4KrB117rGruzZ7T0dNWPP1Zt2NCVi4lRnTUr76yZlKRap46rjq9Yccyu9HT3/VGpkmqZMqp33aX61lvu+2f9etVDh1y5yZPd4YYP97wwI0N11Ci3sVYt1f/7P1f1z3bIiy92u6+6SvWvv07+Yzl82OXaiAjV6tXdl87gwarPP+++Izds8OLLIiNDtUcP1YgITf85Xi++WLVCBfd9eowffnBV8saNVZOTVdXVzqOi3I+eG25w51TSZCxbrociK+k66mmHhn/oh2M3656y1XRreC2tyg4F92979dXu37O4WEI3RtUloJ9+Ur31VtccA6pnnaX6yCOuWaZFC7etYUOX2L2tviYmuurmGWeorl17wu7t21WvucYlr+NbhapWdUm1Y0dPZfzwYffLAlSvvFK1TRv3uEYN19T0zz+q6kJ75hnXohEdrTpvnvcfw8qV7rsKXDIaOFC1bdujP2Yyb23auO+8XBP7c8+pgh546mW98073mvHjcyn77bfuW61ly6xfSHv3ugulERHuF8ejj6oeOOD9eRSlg8vW6Z4yZ+gWaurQHluOxrVkiWq5cnq4Wax+OPGA3nKL+4EWEaH64IPFE78ldGOOd+iQ6ocfqnbpcjTT1qnjqssF+R29dq1L6DVq5JjUVV2u/u031e++c4cZNUr15ptVBw1S/fNPdV1aMqvejz3mMmlGhupXXx1tv6la1SXS/ftV1bW316vnTuGhh05ooTlGerqrgZcpoxp1eob+OHKO65azeHFWmV27XOvVSy+p1qypWU1GixYd92Y//qgZ4eGaUPcKLRuZoaA6YEA+tfrPP3ffQBdeeEzmS0x0XyygeuaZqpMm+bc3z+YFW3R76Zq6kyh94561J57TjBnuA+/VSzUtTbdvd52vwH25TplS+KawvFhCNyYvW7eqzpmTdzb0xooVrukl8392796qo0erzp3rMmVeNm9WbdTIXfWcNCnnMvPmHU34lSu76u3vv+v+/ao33XT0sFdc4Zr8Z89W3bHj6Nt36KAaxhF9tuWHmtq4uWZd6M2sqv/22zGHO3TIJfaqVV2RSy917caLv/hLd5arpYnU1qjSe/TGG12t3ytTp6qWKqXaufPRNiePhQvd9VNwtd5LL1UdO1b1xx8L/0+j6pLszp3uS3DmTNVp09yllhkz3PNZs1Q/fPEP3VCqvu6lki56NSH3N3vpJRfonXceE3/mL5+OHVV/+aVoEnteCd26LRrjS4mJrk9efLy7rV9/dF+tWm64aePGbshpkybQoAH8+qvrw3fokBvldPHFeR/jxx/dnDfTprkROlddBXfdxfQdbfjwQ1i2zPXyzFStGqQdSKH/kcmMPuUZTklOdMe9/363FuyLL8Jzz7n+kbffDg8/7Oaw9zh40PUYffpp2LNH+Yw+XMYc3hy4kN5PtqZ69ZP8jCZPhoED3Tnfe69bjurMMyEigowMmDULvvzSTaq2dq17SfnybjbmevVcT9TwcHfLfCziehVmDgg7cgTK/J1MZPJW4g80YOP2cmzdCocP5xSQ0oJl9GY61/Mu1WQnf330NTWuvjDv87jzTjdy6okn3PnUqEF6hjBhgpuKaM8eN9r4jDNc99XM2xlnwBV9lNZtCja6qtAjRUWkG/ASEAZMVNWxx+0vA7wDtAR2A9eo6ua83tMSugkJe/dCQoJL7gkJboWOdetcxgE3d01YmEtoc+a49fa8tXmzy7RvvOFW/Dj/fLj0Ujh0iNS/DvDX1gPs336AQ8kHOOuvZVRO2QmtW8MDD0CvXsfOOb9tG4wc6fr5V6rkZro877yj/fpTUvjnr0Osm7KCFj++xuGn/kuZB+4q+Ocybpz78siualU333HNmu7Lr25d9pxal5+T6/Dlujp880M5duxwH11amuf+SAaRpFCdHbRgGS1LLaeFLKNZxnLO1O0ApBFGUuUm7KzZigONWlGqTWsqxp5L5dU/UOm76VSaN4OInVvRUqU40LwdZcY+TkSXDvmfQ3q6+zKdPt09r1Ila/jw/rrN+WJjPVK27CR82xbKJv/OKXu3EHVwC9WPbGH1jc9x0Vs3FOijK1RCF5EwYD3QGUgClgD9VXV1tjJDgWaqOkRE+gF9VPWavN7XEroJWUeOuCr0r7+62759bum+atUK9n7797sBTy+95OYJCA938wVXqHD0Fh3tBmJ16JD3uPuVK90opi+/zL3M1Ve7JQYLO35/0yb3OSQludvWrUfvN28+cTrMatXcuWSuaPLPPydWucPCoGFDt/JJTIw7719+gSVL3G3PnmPLR0a6OYN694bu3SEq6uTOIT3drbiSOapsxQr3GaakHFuuXDn3JVWrFlrzLNKvvY7wDnEndyyPwib0tsBjqtrV83wEgKo+la3MXE+ZH0UkHPgDiNI83twSujE+puq+LCIiCv9ey5a5L4qyZV3Sy1w9JDIy9yUFfUnVDZ3dtMk1Y2XeUlJcHOXKHY2pXDlXO46Jcc1ZkZG5v2diopsgZ/VqaNnSDb0tX963saeluS+qxET3JVSrlovPR59ZYZegqwFszfY8CWiTWxlVTRORfUAVYNdxgQwGBgOcddZZXgVvjPGSiG+SObgarj+JuMbmM86ANsenm0K859lnu1tRCg93vxIaNiza4+SgWOdDV9UJqhqrqrFRJ/vTxhhjTJ68SejbgJrZnkd7tuVYxtPkUgl3cdQYY0wx8SahLwHqiUgdEYkA+gEzjyszE8i8ZHsV8L+82s+NMcb4Xr5t6J428duBubhui2+p6ioRGYXr4D4TeBN4V0Q2An/hkr4xxphi5M1FUVR1DjDnuG0jsz1OAfr6NjRjjDEnwxaJNsaYIGEJ3RhjgoQldGOMCRJ+m5xLRJKBLQV8+ekcN2gpRITqeUPonrudd2jx5rxrqWqOA3n8ltALQ0Ticxv6GsxC9bwhdM/dzju0FPa8rcnFGGOChCV0Y4wJEoGa0Cf4OwA/CdXzhtA9dzvv0FKo8w7INnRjjDEnCtQaujHGmONYQjfGmCARcAldRLqJyDoR2SgiD/g7nqIiIm+JyJ8i8mu2baeJyNcissFzf6o/YywKIlJTRL4TkdUiskpEhnu2B/W5i0ikiPwsIis85/24Z3sdEfnJ8/c+xTPjadARkTARWSYin3ueB/15i8hmEVkpIstFJN6zrVB/5wGV0D3rm44DLgUaAf1F5CRW1Q0ok4Bux217APhWVesB33qeB5s04D+q2gg4H7jN828c7Od+GLhYVZsDMUA3ETkfeBp4QVXPAfYAN/kvxCI1HFiT7XmonHdHVY3J1ve8UH/nAZXQgdbARlVNVNVU4COgl59jKhKqOh83FXF2vYDJnseTgd7FGVNxUNUdqprgebwf95+8BkF+7uoc8Dwt7bkpcDHwiWd70J03gIhEA92BiZ7nQgicdy4K9XceaAk9p/VNa/gpFn+oqqo7PI//AKr6M5iiJiK1gRbAT4TAuXuaHZYDfwJfA78Be1U1zVMkWP/eXwTuAzI8z6sQGuetwFcistSz3jIU8u/cq/nQTcmjqioiQdvnVEQqAJ8Cd6rq35JtxfRgPXdVTQdiRKQy8BnQwL8RFT0R6QH8qapLRaSDn8Mpbu1UdZuInAF8LSJrs+8syN95oNXQvVnfNJjtFJHqAJ77P/0cT5EQkdK4ZP6+qk7zbA6JcwdQ1b3Ad0BboLJnnV4Izr/3C4GeIrIZ14R6MfASwX/eqOo2z/2fuC/w1hTy7zzQEro365sGs+xrt94AzPBjLEXC0376JrBGVf+bbVdQn7uIRHlq5ohIWaAz7vrBd7h1eiEIz1tVR6hqtKrWxv1//p+q/osgP28RKS8iFTMfA12AXynk33nAjRQVkctwbW6Z65uO8W9ERUNEPgQ64KbT3Ak8CkwHpgJn4aYevlpVj79wGtBEpB2wAFjJ0TbVB3Ht6EF77iLSDHcRLAxX0ZqqqqNEpC6u5noasAy4TlUP+y/SouNpcrlHVXsE+3l7zu8zz9Nw4ANVHSMiVSjE33nAJXRjjDE5C7QmF2OMMbmwhG6MMUHCEroxxgQJS+jGGBMkLKEbY0yQsIRujDFBwhK6McYEif8H6ZcvWo1233sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(num_epochs), global_train_losses, label='Train. Loss', color='blue')\n",
    "plt.plot(range(num_epochs), global_eval_losses, label='Eval. Loss', color='red')\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}