qqp_BiLSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from gensim import models\n",
    "import numpy as np\n",
    "import gensim\n",
    "import gensim.downloader\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import math\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64665</td>\n",
       "      <td>92664</td>\n",
       "      <td>What is the total number of MBBS seats availab...</td>\n",
       "      <td>What is the total number of MBBS seats (govt a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>514255</td>\n",
       "      <td>177254</td>\n",
       "      <td>What are the difference between the hard drive...</td>\n",
       "      <td>Which is the best external hard drive, Seagate...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>250407</td>\n",
       "      <td>468939</td>\n",
       "      <td>Does IQ increase with age?</td>\n",
       "      <td>I got glasses around age 14 which is about -1 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9610</td>\n",
       "      <td>9611</td>\n",
       "      <td>How did caterpillars evolve to digest themselv...</td>\n",
       "      <td>What is the evolutionary advantage of metamorp...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>297688</td>\n",
       "      <td>442567</td>\n",
       "      <td>My brother had hemorrhage on his right brain a...</td>\n",
       "      <td>A teenaged cousin brother, who is sharp and in...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404282</th>\n",
       "      <td>397063</td>\n",
       "      <td>397064</td>\n",
       "      <td>What's the best thing to do in Goa?</td>\n",
       "      <td>What is the best thing we can do in Goa?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404283</th>\n",
       "      <td>210926</td>\n",
       "      <td>210927</td>\n",
       "      <td>Can we write in our own words in IPCC theory s...</td>\n",
       "      <td>What's the saddest story you can write in six ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404284</th>\n",
       "      <td>67643</td>\n",
       "      <td>67644</td>\n",
       "      <td>What is an aristocracy?</td>\n",
       "      <td>What is aristocracy?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404285</th>\n",
       "      <td>143972</td>\n",
       "      <td>143973</td>\n",
       "      <td>How does Stripes compare to Spring MVC?</td>\n",
       "      <td>Which is better, Play Framework or Spring MVC?...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404286</th>\n",
       "      <td>38552</td>\n",
       "      <td>142767</td>\n",
       "      <td>What is the best answer for why should we hire...</td>\n",
       "      <td>What is the best answer when we are asked by i...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>404287 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          qid1    qid2                                          question1  \\\n",
       "0        64665   92664  What is the total number of MBBS seats availab...   \n",
       "1       514255  177254  What are the difference between the hard drive...   \n",
       "2       250407  468939                         Does IQ increase with age?   \n",
       "3         9610    9611  How did caterpillars evolve to digest themselv...   \n",
       "4       297688  442567  My brother had hemorrhage on his right brain a...   \n",
       "...        ...     ...                                                ...   \n",
       "404282  397063  397064                What's the best thing to do in Goa?   \n",
       "404283  210926  210927  Can we write in our own words in IPCC theory s...   \n",
       "404284   67643   67644                            What is an aristocracy?   \n",
       "404285  143972  143973            How does Stripes compare to Spring MVC?   \n",
       "404286   38552  142767  What is the best answer for why should we hire...   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "0       What is the total number of MBBS seats (govt a...             0  \n",
       "1       Which is the best external hard drive, Seagate...             0  \n",
       "2       I got glasses around age 14 which is about -1 ...             0  \n",
       "3       What is the evolutionary advantage of metamorp...             0  \n",
       "4       A teenaged cousin brother, who is sharp and in...             0  \n",
       "...                                                   ...           ...  \n",
       "404282           What is the best thing we can do in Goa?             1  \n",
       "404283  What's the saddest story you can write in six ...             0  \n",
       "404284                               What is aristocracy?             1  \n",
       "404285  Which is better, Play Framework or Spring MVC?...             0  \n",
       "404286  What is the best answer when we are asked by i...             1  \n",
       "\n",
       "[404287 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/qqp/train_v1.csv', header=0, index_col=0)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149263"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.is_duplicate == 1].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "255024"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.is_duplicate == 0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "question1_list = df['question1'].to_list()\n",
    "question2_list = df['question2'].to_list()\n",
    "\n",
    "question1_list = [question.lower().split() for question in question1_list]\n",
    "question2_list = [question.lower().split() for question in question2_list]\n",
    "\n",
    "labels = df['is_duplicate'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = -np.inf\n",
    "for sentence in question1_list:\n",
    "    if len(sentence) > max_length:\n",
    "        max_length = len(sentence)\n",
    "        \n",
    "for sentence in question2_list:\n",
    "    if len(sentence) > max_length:\n",
    "        max_length = len(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "237"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = 'logs/qqp'\n",
    "os.makedirs(root_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_size = 50\n",
    "window_size = 5\n",
    "negative_size = 15\n",
    "\n",
    "wv_model_file = root_dir + '/' + 'wv_bilstm.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wv_model = gensim.downloader.load('glove-wiki-gigaword-50')\n",
    "# wv_model = models.Word2Vec(sentences=sentences, vector_size=vector_size, window=window_size, negative=negative_size).wv\n",
    "# wv_model = models.Word2Vec(corpus_file='data/corpus.txt', vector_size=vector_size, window=window_size, negative=negative_size).wv\n",
    "\n",
    "# wv_model.save(wv_model_file)\n",
    "# del wv_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_sentences(sentences, wv, sentence_size):\n",
    "    voc = wv.key_to_index.keys()\n",
    "    unk = wv['<unk>']\n",
    "    eos = wv['<eos>']\n",
    "    lengths = []\n",
    "    for i, sentence in enumerate(sentences):\n",
    "        lengths.append(len(sentence))\n",
    "        \n",
    "        for i, token in enumerate(sentence):\n",
    "            if token in voc:\n",
    "                sentence[i] = wv[token]\n",
    "            else:\n",
    "                sentence[i] = unk\n",
    "        \n",
    "        \n",
    "        while len(sentence) < sentence_size:\n",
    "            sentence.append(eos)\n",
    "        \n",
    "        \n",
    "    return sentences, lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv = models.KeyedVectors.load(wv_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv.add_vectors(\n",
    "    ['<unk>', '<eos>'],\n",
    "    [np.zeros(wv.vector_size), np.ones(wv.vector_size)]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(model, file_name):\n",
    "    torch.save(model.state_dict(), file_name)\n",
    "def load_model(model, file_name):\n",
    "    return model.load_state_dict(torch.load(file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 50\n",
    "sentence_size = 256\n",
    "num_layers = 2\n",
    "bidirectional = True\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "lr = 0.001\n",
    "num_epochs = 20\n",
    "\n",
    "\n",
    "eval_rate = 0.1\n",
    "\n",
    "model_file = 'logs/qqp/lstm_qqp_v1.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "question1_vec, question1_lengths = vectorize_sentences(question1_list, wv, sentence_size)\n",
    "question2_vec, question2_lengths = vectorize_sentences(question2_list, wv, sentence_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# question1_vec = np.array(question1_vec)\n",
    "# question2_vec = np.array(question2_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "question1_lengths = np.array(question1_lengths)\n",
    "question2_lengths = np.array(question2_lengths)\n",
    "\n",
    "labels = np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vec_labels = np.zeros((labels.size, labels.max()+1))\n",
    "# vec_labels[np.arange(labels.size), labels] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vectors = vec_sentences.reshape((-1, 50))\n",
    "\n",
    "# mu = vectors.mean(axis=0)\n",
    "# sigma = np.sqrt(((vectors - mu) ** 2).mean(axis=0))\n",
    "\n",
    "# vec_sentences = (vec_sentences - mu) / sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_index = int(len(question1_vec) * eval_rate)\n",
    "\n",
    "question1_train = question1_vec[eval_index:]\n",
    "question1_eval = question1_vec[:eval_index]\n",
    "\n",
    "question2_train = question2_vec[eval_index:]\n",
    "question2_eval = question2_vec[:eval_index]\n",
    "\n",
    "question1_len_train = question1_lengths[eval_index:]\n",
    "question1_len_eval = question1_lengths[:eval_index]\n",
    "\n",
    "question2_len_train = question2_lengths[eval_index:]\n",
    "question2_len_eval = question2_lengths[:eval_index]\n",
    "\n",
    "\n",
    "label_train = labels[eval_index:]\n",
    "label_eval = labels[:eval_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# question1_train = torch.tensor(question1_train, dtype=torch.float)\n",
    "# question1_eval = torch.tensor(question1_eval, dtype=torch.float)\n",
    "\n",
    "# question2_train = torch.tensor(question2_train, dtype=torch.float)\n",
    "# question2_eval = torch.tensor(question2_eval, dtype=torch.float)\n",
    "\n",
    "question1_len_train = torch.tensor(question1_len_train, dtype=torch.int)\n",
    "question1_len_eval = torch.tensor(question1_len_eval, dtype=torch.int)\n",
    "\n",
    "question2_len_train = torch.tensor(question2_len_train, dtype=torch.int)\n",
    "question2_len_eval = torch.tensor(question2_len_eval, dtype=torch.int)\n",
    "\n",
    "label_train = torch.tensor(label_train, dtype=torch.long)\n",
    "label_eval = torch.tensor(label_eval, dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Classifier, self).__init__()\n",
    "        lstm_dim = hidden_size * 2 * (2 if bidirectional else 1)\n",
    "        \n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=vector_size,\n",
    "                              hidden_size=hidden_size,\n",
    "                              num_layers=num_layers,\n",
    "                              bidirectional=bidirectional,\n",
    "                              batch_first=True\n",
    "                             )\n",
    "\n",
    "        \n",
    "        self.fcnn_1 = nn.Linear(in_features=lstm_dim * 2, out_features=64)\n",
    "        \n",
    "        self.fcnn_2 = nn.Linear(in_features=64, out_features=2)\n",
    "        \n",
    "    def forward(self, question1, question2, question1_len, question2_len):\n",
    "        question1 = pack_padded_sequence(question1, question1_len.cpu(), batch_first=True, enforce_sorted=False)\n",
    "        question1, _ = self.lstm(question1)\n",
    "        question1, _ = pad_packed_sequence(question1, batch_first=True)\n",
    "        \n",
    "        question2 = pack_padded_sequence(question2, question2_len.cpu(), batch_first=True, enforce_sorted=False)\n",
    "        question2, _ = self.lstm(question2)\n",
    "        question2, _ = pad_packed_sequence(question2, batch_first=True)\n",
    "        \n",
    "        \n",
    "        \n",
    "        avg_pool = torch.mean(question1, 1)\n",
    "        max_pool, _ = torch.max(question1, 1)\n",
    "        output1 = torch.cat([avg_pool, max_pool], dim=1)\n",
    "        \n",
    "        avg_pool = torch.mean(question2, 1)\n",
    "        max_pool, _ = torch.max(question2, 1)\n",
    "        output2 = torch.cat([avg_pool, max_pool], dim=1)\n",
    "        \n",
    "        \n",
    "        output = torch.cat([output1, output2], dim=1)\n",
    "        \n",
    "        output = self.fcnn_1(output)\n",
    "        output = torch.relu(output)\n",
    "        \n",
    "        output = self.fcnn_2(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weights = torch.log(1/(train_y.sum(dim=0) / train_y.sum()))\n",
    "# weights = weights.detach().to(device)\n",
    "# weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_train_batch = int(len(question1_train) / batch_size)\n",
    "num_eval_batch = int(len(question1_eval) / batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = Classifier().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(classifier.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 0 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:30<00:00, 18.89it/s, Epoch=0, Train loss=0.544]\n",
      "  0%|\u001b[32m          \u001b[0m| 0/315 [00:00<?, ?it/s, Epoch=0]<ipython-input-29-aef89975aa7f>:60: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  question1 = torch.tensor(question1, dtype=torch.float, device=device)\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:16<00:00,  4.11it/s, Epoch=0, Eval loss=0.508, Eval score=0.742]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 1 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:29<00:00, 19.07it/s, Epoch=1, Train loss=0.488]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:16<00:00,  4.11it/s, Epoch=1, Eval loss=0.478, Eval score=0.76] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 2 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:23<00:00, 19.83it/s, Epoch=2, Train loss=0.455]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:15<00:00,  4.18it/s, Epoch=2, Eval loss=0.458, Eval score=0.773]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 3 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:23<00:00, 19.80it/s, Epoch=3, Train loss=0.43] \n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:16<00:00,  4.12it/s, Epoch=3, Eval loss=0.447, Eval score=0.78] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 4 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:26<00:00, 19.40it/s, Epoch=4, Train loss=0.409]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:19<00:00,  3.94it/s, Epoch=4, Eval loss=0.441, Eval score=0.785]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 5 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:33<00:00, 18.51it/s, Epoch=5, Train loss=0.391]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:22<00:00,  3.81it/s, Epoch=5, Eval loss=0.439, Eval score=0.79] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 6 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:35<00:00, 18.26it/s, Epoch=6, Train loss=0.375]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:23<00:00,  3.76it/s, Epoch=6, Eval loss=0.437, Eval score=0.791]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 7 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:33<00:00, 18.47it/s, Epoch=7, Train loss=0.361]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:16<00:00,  4.13it/s, Epoch=7, Eval loss=0.437, Eval score=0.794]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** save ***\n",
      "---> Epoch 8 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:24<00:00, 19.69it/s, Epoch=8, Train loss=0.347]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:20<00:00,  3.92it/s, Epoch=8, Eval loss=0.439, Eval score=0.796]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 9 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:37<00:00, 18.04it/s, Epoch=9, Train loss=0.334]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:24<00:00,  3.73it/s, Epoch=9, Eval loss=0.447, Eval score=0.794]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 10 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:28<00:00, 19.17it/s, Epoch=10, Train loss=0.322]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:24<00:00,  3.74it/s, Epoch=10, Eval loss=0.454, Eval score=0.797]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 11 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:24<00:00, 19.61it/s, Epoch=11, Train loss=0.312]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:15<00:00,  4.16it/s, Epoch=11, Eval loss=0.46, Eval score=0.794] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 12 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:26<00:00, 19.39it/s, Epoch=12, Train loss=0.302]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:19<00:00,  3.96it/s, Epoch=12, Eval loss=0.473, Eval score=0.796]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 13 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:26<00:00, 19.44it/s, Epoch=13, Train loss=0.293]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:19<00:00,  3.98it/s, Epoch=13, Eval loss=0.477, Eval score=0.796]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 14 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:26<00:00, 19.34it/s, Epoch=14, Train loss=0.284]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:17<00:00,  4.05it/s, Epoch=14, Eval loss=0.483, Eval score=0.794]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 15 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:25<00:00, 19.50it/s, Epoch=15, Train loss=0.276]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:18<00:00,  4.00it/s, Epoch=15, Eval loss=0.495, Eval score=0.796]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 16 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2842/2842 [02:30<00:00, 18.88it/s, Epoch=16, Train loss=0.269]\n",
      "100%|\u001b[32m██████████\u001b[0m| 315/315 [01:20<00:00,  3.91it/s, Epoch=16, Eval loss=0.509, Eval score=0.798]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Epoch 17 <---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 2781/2842 [02:28<00:03, 18.72it/s, Epoch=17, Train loss=0.263]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-aef89975aa7f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquestion2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquestion1_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquestion2_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1103\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-24-95f25ca1e9ec>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, question1, question2, question1_len, question2_len)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mquestion2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpack_padded_sequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquestion2_len\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_first\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menforce_sorted\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0mquestion2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0mquestion2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpad_packed_sequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_first\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1103\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    692\u001b[0m                               self.dropout, self.training, self.bidirectional, self.batch_first)\n\u001b[1;32m    693\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 694\u001b[0;31m             result = _VF.lstm(input, batch_sizes, hx, self._flat_weights, self.bias,\n\u001b[0m\u001b[1;32m    695\u001b[0m                               self.num_layers, self.dropout, self.training, self.bidirectional)\n\u001b[1;32m    696\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "min_loss = np.inf\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    print(f'---> Epoch {i} <---')\n",
    "    time.sleep(0.5)\n",
    "    \n",
    "    classifier.train()\n",
    "    loader = tqdm(range(num_train_batch), postfix={'Epoch': i})\n",
    "    train_losses = []\n",
    "    \n",
    "    for i_batch in loader:\n",
    "        question1, question2, question1_len, question2_len, targets = (\n",
    "            question1_train[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "            question2_train[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "            question1_len_train[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "            question2_len_train[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "            label_train[i_batch*batch_size:(i_batch+1)*batch_size]\n",
    "        )\n",
    "        \n",
    "        question1 = torch.tensor(np.array(question1), dtype=torch.float, device=device)\n",
    "        question2 = torch.tensor(np.array(question2), dtype=torch.float, device=device)\n",
    "        \n",
    "        \n",
    "        targets = targets.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        outputs = classifier(question1, question2, question1_len, question2_len)\n",
    "        \n",
    "        loss = criterion(outputs, targets)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            train_losses.append(loss.item())\n",
    "            loader.set_postfix({\n",
    "                'Epoch': i,\n",
    "                'Train loss': np.mean(train_losses)\n",
    "            }, refresh=True)\n",
    "    \n",
    "    \n",
    "    time.sleep(0.5)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        classifier.eval()\n",
    "        loader = tqdm(range(num_eval_batch), postfix={'Epoch': i,}, colour='green')\n",
    "        eval_losses = []\n",
    "        eval_scores = []\n",
    "\n",
    "        for i_batch in loader:\n",
    "            question1, question2, question1_len, question2_len, targets = (\n",
    "                question1_eval[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "                question2_eval[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "                question1_len_eval[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "                question2_len_eval[i_batch*batch_size:(i_batch+1)*batch_size],\n",
    "                label_eval[i_batch*batch_size:(i_batch+1)*batch_size]\n",
    "            )\n",
    "            \n",
    "            question1 = torch.tensor(question1, dtype=torch.float, device=device)\n",
    "            question2 = torch.tensor(question2, dtype=torch.float, device=device)\n",
    "            targets = targets.to(device)\n",
    "            \n",
    "\n",
    "            outputs = classifier(question1, question2, question1_len, question2_len)\n",
    "            \n",
    "            loss = criterion(outputs, targets)\n",
    "            \n",
    "            score = (outputs.argmax(dim=1) == targets).detach().cpu().numpy()\n",
    "            eval_scores.append(score)\n",
    "            eval_losses.append(loss.item())\n",
    "            loader.set_postfix({\n",
    "                'Epoch': i,\n",
    "                'Eval loss': np.mean(eval_losses),\n",
    "                'Eval score': np.concatenate(eval_scores).mean()\n",
    "            }, refresh=True)\n",
    "\n",
    "\n",
    "        eval_loss = np.mean(eval_losses)\n",
    "        if eval_loss <= min_loss:\n",
    "            min_loss = eval_loss\n",
    "            save_model(classifier, model_file)\n",
    "            loader.write('*** save ***')\n",
    "        \n",
    "    time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('logs/qqp/1/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}