RNN_prj2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RNN_prj2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nhathoang0110/project2/blob/master/RNN_prj2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "feXhhWMA-_Fz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch  \n",
        "import torch.nn as nn\n",
        "from torch.autograd import Variable\n",
        "import numpy as np\n",
        "import time\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rE8za8a0_Lmp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "d4e83ba8-68a4-46ac-8083-b957f23edf81"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJQilbljIInf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "outputId": "9b2b49e0-0fab-4a34-c8b5-e5d98bcc19ef"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Jun 22 12:41:11 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.36.06    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P0    28W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFYLAYwQICKN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data import TensorDataset\n",
        "from time import time\n",
        "\n",
        "\n",
        "def load_data(data_path):\n",
        "  with open(data_path, encoding = 'latin1') as f:\n",
        "    d_lines = f.read().splitlines()\n",
        "  data, labels, sentence_lengths = [], [], []\n",
        "  for line in d_lines:\n",
        "    features = line.split('<fff>')\n",
        "    label, doc_id, sentence_len = int(features[0]), int(features[1]), int(features[2])\n",
        "    vector = [int(ID) for ID in features[3].split()]\n",
        "    data.append(vector)\n",
        "    labels.append(label)\n",
        "    sentence_lengths.append(sentence_len)\n",
        "  return torch.tensor(data), torch.tensor(labels), torch.tensor(sentence_lengths)\n",
        "  \n",
        "train_data, train_labels, train_sentence_lengths = load_data(\n",
        "    data_path='/content/drive/My Drive/20news-train-encoded.txt'\n",
        ")\n",
        "test_data, test_labels, test_sentence_lengths = load_data(\n",
        "    data_path='/content/drive/My Drive/20news-test-encoded.txt'\n",
        ")\n",
        "with open('/content/drive/My Drive/vocab-raw.txt', encoding = 'latin1') as f:\n",
        "  vocab_size = len(f.read().splitlines())\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MO-jN6o_Nvb2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "d770833c-48a5-4de3-b489-f499d935959b"
      },
      "source": [
        "device = torch.device(\"cuda\")\n",
        "print(device)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3o22pkaxI2lC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data import TensorDataset\n",
        "import numpy as np\n",
        "\n",
        "MAX_DOC_LENGTH = 500\n",
        "NUM_CLASSES = 20\n",
        "class RNN(nn.Module):\n",
        "  def __init__(self, vocab_size, embedding_size, lstm_size, batch_size):\n",
        "    super().__init__()\n",
        "    self._vocab_size = vocab_size\n",
        "    self._embedding_size = embedding_size\n",
        "    self._lstm_size = lstm_size\n",
        "    self._batch_size = batch_size\n",
        "    self.build_graph()\n",
        "\n",
        "  def build_graph(self):\n",
        "    self._embedding_layer = nn.Embedding(self._vocab_size+2, self._embedding_size)\n",
        "    self._LSTM_layer = nn.LSTM(self._embedding_size, self._lstm_size, batch_first=True)\n",
        "    self._full_connected_layer = nn.Linear(self._lstm_size, NUM_CLASSES)\n",
        "    self._dropout = nn.Dropout(p=0.5)\n",
        "\n",
        "  def forward(self, data, sentence_lengths):\n",
        "    h = torch.zeros(1, data.size(0), self._lstm_size).to(device)\n",
        "    c = torch.zeros(1, data.size(0), self._lstm_size).to(device)\n",
        "    embeddings = self._embedding_layer(data)  \n",
        "    lstm_outputs, (hidden, cell)  = self._LSTM_layer(embeddings, (h, c))\n",
        "    lstm_outputs = [lstm_outputs[i][:sentence_lengths[i]].mean(0) for i in range(data.size(0))]\n",
        "    lstm_outputs = torch.stack(lstm_outputs)\n",
        "    outputs = self._dropout(lstm_outputs)\n",
        "    outputs = self._full_connected_layer(lstm_outputs)\n",
        "    return outputs\n"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ezWMTT3dDOVC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 131
        },
        "outputId": "c7b5c4cb-b87f-4872-8fa7-960f8bdb4831"
      },
      "source": [
        "rnn = RNN(\n",
        "    vocab_size=vocab_size, \n",
        "    embedding_size=475, \n",
        "    lstm_size=90, \n",
        "    batch_size=10,\n",
        ")\n",
        "rnn.to(device)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RNN(\n",
              "  (_embedding_layer): Embedding(18988, 475)\n",
              "  (_LSTM_layer): LSTM(475, 90, batch_first=True)\n",
              "  (_full_connected_layer): Linear(in_features=90, out_features=20, bias=True)\n",
              "  (_dropout): Dropout(p=0.5, inplace=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dV0uRZ3qSruC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "9450b907-3a5f-4da4-b4b1-ada72d2d47a7"
      },
      "source": [
        "batch_size=10\n",
        "max_epochs=50\n",
        "learning_rate=0.01\n",
        "def test(model, test_data, test_labels, test_sentence_lengths):\n",
        "  data_set = TensorDataset(test_data, test_labels, test_sentence_lengths)\n",
        "  data_loader = DataLoader(data_set, batch_size = batch_size, shuffle = False)\n",
        "  num_true_predict = 0\n",
        "  for data, labels, sentence_lengths in data_loader:\n",
        "    data = data.to(device)\n",
        "    labels = labels.to(device)\n",
        "    predicted = torch.argmax(model.forward(data, sentence_lengths), dim = 1)\n",
        "    num_true_predict += sum((predicted == labels).float())\n",
        "  return num_true_predict*100./len(test_data)\n",
        "\n",
        "data_set = TensorDataset(train_data, train_labels, train_sentence_lengths)\n",
        "data_loader = DataLoader(data_set, batch_size = batch_size, shuffle = True)\n",
        "opt = torch.optim.Adam(rnn.parameters(), lr = learning_rate)\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "rnn.train()\n",
        "last_loss = 0\n",
        "for epoch in range(max_epochs):\n",
        "  new_loss = 0\n",
        "  for data, labels, sentence_lengths in data_loader:\n",
        "    data = data.to(device)\n",
        "    labels = labels.to(device)\n",
        "    opt.zero_grad()\n",
        "    prediced = rnn.forward(data, sentence_lengths)\n",
        "    loss = loss_function(prediced, labels)\n",
        "    new_loss += loss\n",
        "    loss.backward()\n",
        "    opt.step()\n",
        "  new_loss = new_loss / len(data_loader)\n",
        "  print('epoch: {}, loss: {}'.format(epoch, new_loss))\n",
        "  print('test accuracy:',test(rnn,test_data, test_labels, test_sentence_lengths))\n",
        "  print('train accuracy:', test(rnn,train_data, train_labels, train_sentence_lengths))\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch: 0, loss: 0.0052651516161859035\n",
            "test accuracy: tensor(75.4647, device='cuda:0')\n",
            "train accuracy: tensor(99.9381, device='cuda:0')\n",
            "epoch: 1, loss: 0.004627746995538473\n",
            "test accuracy: tensor(75.3186, device='cuda:0')\n",
            "train accuracy: tensor(99.9470, device='cuda:0')\n",
            "epoch: 2, loss: 0.002904835855588317\n",
            "test accuracy: tensor(75.4381, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 3, loss: 0.0019345226464793086\n",
            "test accuracy: tensor(75.9825, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 4, loss: 0.0018781314138323069\n",
            "test accuracy: tensor(75.4780, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 5, loss: 0.001613859785720706\n",
            "test accuracy: tensor(76.0621, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 6, loss: 0.0013219816610217094\n",
            "test accuracy: tensor(75.7966, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 7, loss: 0.0013298067497089505\n",
            "test accuracy: tensor(76.1152, device='cuda:0')\n",
            "train accuracy: tensor(99.9646, device='cuda:0')\n",
            "epoch: 8, loss: 0.0015114675043150783\n",
            "test accuracy: tensor(75.6771, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 9, loss: 0.0016727575566619635\n",
            "test accuracy: tensor(76.1683, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 10, loss: 0.0022723679430782795\n",
            "test accuracy: tensor(75.7700, device='cuda:0')\n",
            "train accuracy: tensor(99.9470, device='cuda:0')\n",
            "epoch: 11, loss: 0.0029581659473478794\n",
            "test accuracy: tensor(75.1726, device='cuda:0')\n",
            "train accuracy: tensor(99.9912, device='cuda:0')\n",
            "epoch: 12, loss: 0.00205149594694376\n",
            "test accuracy: tensor(75.7700, device='cuda:0')\n",
            "train accuracy: tensor(99.9912, device='cuda:0')\n",
            "epoch: 13, loss: 0.0014026141725480556\n",
            "test accuracy: tensor(75.7302, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 14, loss: 0.0013832777040079236\n",
            "test accuracy: tensor(76.0489, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 15, loss: 0.00138088408857584\n",
            "test accuracy: tensor(76.4737, device='cuda:0')\n",
            "train accuracy: tensor(99.9912, device='cuda:0')\n",
            "epoch: 16, loss: 0.0006623002118431032\n",
            "test accuracy: tensor(76.3011, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 17, loss: 0.0010081222280859947\n",
            "test accuracy: tensor(75.8497, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 18, loss: 0.0014377146726474166\n",
            "test accuracy: tensor(75.6771, device='cuda:0')\n",
            "train accuracy: tensor(99.9646, device='cuda:0')\n",
            "epoch: 19, loss: 0.0015117210568860173\n",
            "test accuracy: tensor(75.2921, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 20, loss: 0.0010816038120537996\n",
            "test accuracy: tensor(76.4073, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 21, loss: 0.001108189346268773\n",
            "test accuracy: tensor(76.1285, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 22, loss: 0.0009135518921539187\n",
            "test accuracy: tensor(76.3941, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 23, loss: 0.0007583455299027264\n",
            "test accuracy: tensor(76.1020, device='cuda:0')\n",
            "train accuracy: tensor(99.9646, device='cuda:0')\n",
            "epoch: 24, loss: 0.0012418469414114952\n",
            "test accuracy: tensor(75.7700, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 25, loss: 0.0010969055583700538\n",
            "test accuracy: tensor(76.2746, device='cuda:0')\n",
            "train accuracy: tensor(99.9912, device='cuda:0')\n",
            "epoch: 26, loss: 0.0012690718285739422\n",
            "test accuracy: tensor(75.7700, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n",
            "epoch: 27, loss: 0.001291319145821035\n",
            "test accuracy: tensor(75.7037, device='cuda:0')\n",
            "train accuracy: tensor(99.9558, device='cuda:0')\n",
            "epoch: 28, loss: 0.0018091374076902866\n",
            "test accuracy: tensor(75.3717, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 29, loss: 0.0008207631763070822\n",
            "test accuracy: tensor(75.8232, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 30, loss: 0.0007951329462230206\n",
            "test accuracy: tensor(75.9958, device='cuda:0')\n",
            "train accuracy: tensor(99.9912, device='cuda:0')\n",
            "epoch: 31, loss: 0.0006377504323609173\n",
            "test accuracy: tensor(76.0090, device='cuda:0')\n",
            "train accuracy: tensor(99.9823, device='cuda:0')\n",
            "epoch: 32, loss: 0.0006764736608602107\n",
            "test accuracy: tensor(76.0223, device='cuda:0')\n",
            "train accuracy: tensor(99.9646, device='cuda:0')\n",
            "epoch: 33, loss: 0.0009672417654655874\n",
            "test accuracy: tensor(75.6506, device='cuda:0')\n",
            "train accuracy: tensor(99.9735, device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OaPe9XPiC_WQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}