autoencoder/pytorch_lstm_getting_started_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch lstm getting started classification",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experiments/blob/master/autoencoder/pytorch_lstm_getting_started_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VBIIB0L72eCM",
        "colab_type": "text"
      },
      "source": [
        "Simple LSTM for text classification using torchtext"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GvX2EFF8CFS2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NOz_mT7NqLap",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "1089872d-66c6-4af1-8695-7c1450bf1dfd"
      },
      "source": [
        "from torchtext import data\n",
        "from torchtext import datasets\n",
        "from torchtext.vocab import Vectors, GloVe\n",
        "\n",
        "def normalize(comment):\n",
        "    comment = comment.lower()\n",
        "    lines = comment.split()\n",
        "    lines = [x.strip(' ') for x in lines]\n",
        "    lines = [x.replace('\"', '') for x in lines]\n",
        "    lines = [x.replace('\\\\\"', '') for x in lines]\n",
        "    lines = [x.replace(u'\\xa0', u'') for x in lines]\n",
        "\n",
        "    return lines\n",
        "\n",
        "tokenize = lambda x: x.split()\n",
        "TEXT = data.Field(sequential=True, tokenize=normalize, lower=True, include_lengths=True, batch_first=True, fix_length=200)\n",
        "LABEL = data.LabelField()\n",
        "\n",
        "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)\n",
        "\n",
        "TEXT.build_vocab(train_data, vectors=GloVe(name='6B', dim=300))\n",
        "LABEL.build_vocab(train_data)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "aclImdb_v1.tar.gz: 100%|██████████| 84.1M/84.1M [00:07<00:00, 11.3MB/s]\n",
            ".vector_cache/glove.6B.zip: 862MB [06:30, 2.21MB/s]                          \n",
            "100%|█████████▉| 399880/400000 [00:50<00:00, 12046.96it/s]"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WZpPckSJ5OeV",
        "colab_type": "text"
      },
      "source": [
        "Above, we are tokenizing the text and label. \n",
        "\n",
        "Also we genering word embedding using glove vectors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CXG5g1d7xovr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "450c845e-5fa0-455d-d682-d2c48d350782"
      },
      "source": [
        "word_embeddings = TEXT.vocab.vectors\n",
        "print (\"Length of Text Vocabulary: \" + str(len(TEXT.vocab)))\n",
        "print (\"Vector size of Text Vocabulary: \", TEXT.vocab.vectors.size())\n",
        "print (\"Label Length: \" + str(len(LABEL.vocab)))\n",
        "\n",
        "train_data, valid_data = train_data.split() # Further splitting of training_data to create new training_data & validation_data\n",
        "train_iter, valid_iter, test_iter = data.BucketIterator.splits((train_data, valid_data, test_data), batch_size=32, sort_key=lambda x: len(x.text), repeat=False, shuffle=True)\n",
        "\n",
        "'''Alternatively we can also use the default configurations'''\n",
        "# train_iter, test_iter = datasets.IMDB.iters(batch_size=32)\n",
        "\n",
        "vocab_size = len(TEXT.vocab)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Length of Text Vocabulary: 233023\n",
            "Vector size of Text Vocabulary:  torch.Size([233023, 300])\n",
            "Label Length: 2\n",
            "Length of Text Vocabulary: 233023\n",
            "Vector size of Text Vocabulary:  torch.Size([233023, 300])\n",
            "Label Length: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAyEQv4TKFoX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a3391724-bf99-45d2-9937-a0c8c13d59b8"
      },
      "source": [
        "import torch\n",
        "import torch.autograd as autograd\n",
        "import torch.nn as nn\n",
        "import torch.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
        "\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)\n",
        "\n",
        "class LSTMClassifier(nn.Module):\n",
        "  def __init__(self, vocab_size, embedding_dim, hidden_dim, output_size, weights):\n",
        "    super(LSTMClassifier, self).__init__()\n",
        "    self.embedding_dim = embedding_dim\n",
        "    self.hidden_dim = hidden_dim\n",
        "    self.vocab_size = vocab_size\n",
        "\n",
        "    self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "    self.embedding.weight = nn.Parameter(weights, requires_grad=False)\n",
        "    self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers=1)\n",
        "\n",
        "    self.hidden2out = nn.Linear(hidden_dim, output_size)\n",
        "    self.softmax = nn.LogSoftmax()\n",
        "\n",
        "    self.dropout_layer = nn.Dropout(p=0.2)\n",
        "\n",
        "\n",
        "  def init_hidden(self, batch_size):\n",
        "    h0 = torch.zeros(1, batch_size, self.hidden_dim).to(device)\n",
        "    # Initialize cell state\n",
        "    c0 = torch.zeros(1, batch_size, self.hidden_dim).to(device)\n",
        "    return(h0,c0)\n",
        "\n",
        "\n",
        "  def forward(self, batch):\n",
        "    \n",
        "\n",
        "    # print(batch.shape, \"batch shape\")\n",
        "    self.hidden = self.init_hidden(batch.size(0))\n",
        "\n",
        "    embeds = self.embedding(batch)\n",
        "    embeds = embeds.permute(1, 0, 2)\n",
        "    # print(embeds.shape, \"embeds\")\n",
        "    # packed_input = pack_padded_sequence(embeds, lengths)\n",
        "    outputs, (ht, ct) = self.lstm(embeds, self.hidden)\n",
        "\n",
        "    # ht is the last hidden state of the sequences\n",
        "    # ht = (1 x batch_size x hidden_dim)\n",
        "    # ht[-1] = (batch_size x hidden_dim)\n",
        "    output = self.dropout_layer(ht[-1])\n",
        "    # print(output.shape, \"output\")\n",
        "    output = self.hidden2out(output)\n",
        "    # print(output.shape, \"output\")\n",
        "    output = self.softmax(output)\n",
        "    # print(output.shape, \"output\")\n",
        "\n",
        "    return output\n",
        "\n",
        "import torch.nn.functional as F\n",
        "loss_fn = F.cross_entropy\n",
        "\n",
        "batch_size = 32\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def clip_gradient(model, clip_value):\n",
        "    params = list(filter(lambda p: p.grad is not None, model.parameters()))\n",
        "    for p in params:\n",
        "        p.grad.data.clamp_(-clip_value, clip_value)\n",
        "\n",
        "\n",
        "def train_model(model, train_iter, epoch):\n",
        "    total_epoch_loss = 0\n",
        "    total_epoch_acc = 0\n",
        "    model.to(device)\n",
        "    optim = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-5)\n",
        "    steps = 0\n",
        "    model.train()\n",
        "    for idx, batch in enumerate(train_iter):\n",
        "        # print(idx)\n",
        "        text = batch.text[0].to(device) # this is a tuple for some reason\n",
        "        target = batch.label\n",
        "        target = target.to(device)\n",
        "        if (text.size()[0] is not 32):# One of the batch returned by BucketIterator has length different than 32.\n",
        "            continue\n",
        "        optim.zero_grad()\n",
        "        prediction = model(text)\n",
        "        loss = loss_fn(prediction, target)\n",
        "        num_corrects = (torch.max(prediction, 1)[1].view(target.size()).data == target.data).float().sum()\n",
        "        acc = 100.0 * num_corrects/len(batch)\n",
        "        loss.backward()\n",
        "        # clip_gradient(model, 1e-1)\n",
        "        optim.step()\n",
        "        steps += 1\n",
        "        \n",
        "        if steps % 100 == 0:\n",
        "            print (f'Epoch: {epoch+1}, Idx: {idx+1}, Training Loss: {loss.item():.4f}, Training Accuracy: {acc.item(): .2f}%')\n",
        "        \n",
        "        total_epoch_loss += loss.item()\n",
        "        total_epoch_acc += acc.item()\n",
        "        \n",
        "    return total_epoch_loss/len(train_iter), total_epoch_acc/len(train_iter)\n",
        "\n",
        "def eval_model(model, val_iter):\n",
        "    total_epoch_loss = 0\n",
        "    total_epoch_acc = 0\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for idx, batch in enumerate(val_iter):\n",
        "            text = batch.text[0]\n",
        "            if (text.size()[0] is not 32):\n",
        "                continue\n",
        "            target = batch.label\n",
        "            target = torch.autograd.Variable(target).long()\n",
        "            if torch.cuda.is_available():\n",
        "                text = text.cuda()\n",
        "                target = target.cuda()\n",
        "            prediction = model(text)\n",
        "            loss = loss_fn(prediction, target)\n",
        "            num_corrects = (torch.max(prediction, 1)[1].view(target.size()).data == target.data).sum()\n",
        "            acc = 100.0 * num_corrects/len(batch)\n",
        "            total_epoch_loss += loss.item()\n",
        "            total_epoch_acc += acc.item()\n",
        "\n",
        "    return total_epoch_loss/len(val_iter), total_epoch_acc/len(val_iter)\n",
        "\n",
        "output_size = 2\n",
        "hidden_size = 256\n",
        "embedding_length = 300\n",
        "\n",
        "model = LSTMClassifier(vocab_size, embedding_length,  hidden_size, output_size, word_embeddings)\n"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-xDnb9d846Y",
        "colab_type": "code",
        "outputId": "b6ac7253-f26f-4bc6-a756-0ceddd96dda9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\n",
        "for epoch in range(10):\n",
        "  train_loss, train_acc = train_model(model, train_iter, epoch)\n",
        "  val_loss, val_acc = eval_model(model, valid_iter)\n",
        "    \n",
        "  print(f'Epoch: {epoch+1:02}, Train Loss: {train_loss:.3f}, Train Acc: {train_acc:.2f}%, Val. Loss: {val_loss:3f}, Val. Acc: {val_acc:.2f}%')\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:56: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1, Idx: 100, Training Loss: 0.6917, Training Accuracy:  56.25%\n",
            "Epoch: 1, Idx: 200, Training Loss: 0.6950, Training Accuracy:  46.88%\n",
            "Epoch: 1, Idx: 300, Training Loss: 0.6929, Training Accuracy:  50.00%\n",
            "Epoch: 1, Idx: 400, Training Loss: 0.6895, Training Accuracy:  59.38%\n",
            "Epoch: 1, Idx: 500, Training Loss: 0.6615, Training Accuracy:  59.38%\n",
            "Epoch: 01, Train Loss: 0.690, Train Acc: 51.74%, Val. Loss: 0.679772, Val. Acc: 54.30%\n",
            "Epoch: 2, Idx: 100, Training Loss: 0.7088, Training Accuracy:  59.38%\n",
            "Epoch: 2, Idx: 200, Training Loss: 0.6871, Training Accuracy:  56.25%\n",
            "Epoch: 2, Idx: 300, Training Loss: 0.7076, Training Accuracy:  68.75%\n",
            "Epoch: 2, Idx: 400, Training Loss: 0.6397, Training Accuracy:  65.62%\n",
            "Epoch: 2, Idx: 500, Training Loss: 0.6778, Training Accuracy:  43.75%\n",
            "Epoch: 02, Train Loss: 0.671, Train Acc: 57.25%, Val. Loss: 0.645017, Val. Acc: 61.32%\n",
            "Epoch: 3, Idx: 100, Training Loss: 0.6255, Training Accuracy:  65.62%\n",
            "Epoch: 3, Idx: 200, Training Loss: 0.6423, Training Accuracy:  59.38%\n",
            "Epoch: 3, Idx: 300, Training Loss: 0.7456, Training Accuracy:  37.50%\n",
            "Epoch: 3, Idx: 400, Training Loss: 0.6564, Training Accuracy:  56.25%\n",
            "Epoch: 3, Idx: 500, Training Loss: 0.7173, Training Accuracy:  53.12%\n",
            "Epoch: 03, Train Loss: 0.664, Train Acc: 61.37%, Val. Loss: 0.650595, Val. Acc: 62.53%\n",
            "Epoch: 4, Idx: 100, Training Loss: 0.6382, Training Accuracy:  62.50%\n",
            "Epoch: 4, Idx: 200, Training Loss: 0.5931, Training Accuracy:  68.75%\n",
            "Epoch: 4, Idx: 300, Training Loss: 0.6705, Training Accuracy:  65.62%\n",
            "Epoch: 4, Idx: 400, Training Loss: 0.6975, Training Accuracy:  46.88%\n",
            "Epoch: 4, Idx: 500, Training Loss: 0.7012, Training Accuracy:  46.88%\n",
            "Epoch: 04, Train Loss: 0.670, Train Acc: 57.88%, Val. Loss: 0.687981, Val. Acc: 52.06%\n",
            "Epoch: 5, Idx: 100, Training Loss: 0.6748, Training Accuracy:  59.38%\n",
            "Epoch: 5, Idx: 200, Training Loss: 0.6850, Training Accuracy:  46.88%\n",
            "Epoch: 5, Idx: 300, Training Loss: 0.5592, Training Accuracy:  81.25%\n",
            "Epoch: 5, Idx: 400, Training Loss: 0.6279, Training Accuracy:  71.88%\n",
            "Epoch: 5, Idx: 500, Training Loss: 0.5739, Training Accuracy:  75.00%\n",
            "Epoch: 05, Train Loss: 0.650, Train Acc: 62.71%, Val. Loss: 0.704587, Val. Acc: 65.92%\n",
            "Epoch: 6, Idx: 100, Training Loss: 0.6546, Training Accuracy:  65.62%\n",
            "Epoch: 6, Idx: 200, Training Loss: 0.6730, Training Accuracy:  59.38%\n",
            "Epoch: 6, Idx: 300, Training Loss: 0.5273, Training Accuracy:  84.38%\n",
            "Epoch: 6, Idx: 400, Training Loss: 0.7245, Training Accuracy:  53.12%\n",
            "Epoch: 6, Idx: 500, Training Loss: 0.5092, Training Accuracy:  84.38%\n",
            "Epoch: 06, Train Loss: 0.615, Train Acc: 68.86%, Val. Loss: 0.716640, Val. Acc: 61.06%\n",
            "Epoch: 7, Idx: 100, Training Loss: 0.7283, Training Accuracy:  59.38%\n",
            "Epoch: 7, Idx: 200, Training Loss: 0.4041, Training Accuracy:  87.50%\n",
            "Epoch: 7, Idx: 300, Training Loss: 0.6616, Training Accuracy:  65.62%\n",
            "Epoch: 7, Idx: 400, Training Loss: 0.5683, Training Accuracy:  75.00%\n",
            "Epoch: 7, Idx: 500, Training Loss: 0.6286, Training Accuracy:  71.88%\n",
            "Epoch: 07, Train Loss: 0.592, Train Acc: 70.68%, Val. Loss: 0.598513, Val. Acc: 68.91%\n",
            "Epoch: 8, Idx: 100, Training Loss: 0.6824, Training Accuracy:  62.50%\n",
            "Epoch: 8, Idx: 200, Training Loss: 0.6079, Training Accuracy:  71.88%\n",
            "Epoch: 8, Idx: 300, Training Loss: 0.6734, Training Accuracy:  56.25%\n",
            "Epoch: 8, Idx: 400, Training Loss: 0.6048, Training Accuracy:  71.88%\n",
            "Epoch: 8, Idx: 500, Training Loss: 0.6346, Training Accuracy:  71.88%\n",
            "Epoch: 08, Train Loss: 0.615, Train Acc: 67.84%, Val. Loss: 0.557828, Val. Acc: 73.56%\n",
            "Epoch: 9, Idx: 100, Training Loss: 0.6759, Training Accuracy:  68.75%\n",
            "Epoch: 9, Idx: 200, Training Loss: 0.5453, Training Accuracy:  75.00%\n",
            "Epoch: 9, Idx: 300, Training Loss: 0.7042, Training Accuracy:  53.12%\n",
            "Epoch: 9, Idx: 400, Training Loss: 0.5954, Training Accuracy:  68.75%\n",
            "Epoch: 9, Idx: 500, Training Loss: 0.6900, Training Accuracy:  50.00%\n",
            "Epoch: 09, Train Loss: 0.600, Train Acc: 69.65%, Val. Loss: 0.666736, Val. Acc: 59.20%\n",
            "Epoch: 10, Idx: 100, Training Loss: 0.6294, Training Accuracy:  65.62%\n",
            "Epoch: 10, Idx: 200, Training Loss: 0.5458, Training Accuracy:  75.00%\n",
            "Epoch: 10, Idx: 300, Training Loss: 0.5101, Training Accuracy:  81.25%\n",
            "Epoch: 10, Idx: 400, Training Loss: 0.4978, Training Accuracy:  75.00%\n",
            "Epoch: 10, Idx: 500, Training Loss: 0.5471, Training Accuracy:  75.00%\n",
            "Epoch: 10, Train Loss: 0.576, Train Acc: 72.33%, Val. Loss: 0.551396, Val. Acc: 74.75%\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}