tutorial-text-classification-ulmfit-fastai-library.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kumartr/NLP/blob/master/tutorial-text-classification-ulmfit-fastai-library.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "pndN0sIokoD_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 169
        },
        "outputId": "4d5ab001-e32c-478a-b4a7-1f7b4a3d28cb"
      },
      "cell_type": "code",
      "source": [
        "!pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cu92/torch_nightly.html"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Looking in links: https://download.pytorch.org/whl/nightly/cu92/torch_nightly.html\n",
            "Collecting torch_nightly\n",
            "\u001b[?25l  Downloading https://download.pytorch.org/whl/nightly/cu92/torch_nightly-1.0.0.dev20181129-cp36-cp36m-linux_x86_64.whl (576.0MB)\n",
            "\u001b[K    100% |████████████████████████████████| 576.0MB 24kB/s \n",
            "tcmalloc: large alloc 1073750016 bytes == 0x62e84000 @  0x7f9a34b472a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n",
            "\u001b[?25hInstalling collected packages: torch-nightly\n",
            "Successfully installed torch-nightly-1.0.0.dev20181129\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "4mAceDVRk14X",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1074
        },
        "outputId": "1d638397-ac1f-4a4d-e516-c813dc19d80a"
      },
      "cell_type": "code",
      "source": [
        "!pip install fastai"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fastai\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1c/67/8dd2051bb7dbde20ecca36215bd3e02ed436b976643078e3cdd41bd0256f/fastai-1.0.30-py3-none-any.whl (132kB)\n",
            "\u001b[K    100% |████████████████████████████████| 133kB 2.5MB/s \n",
            "\u001b[?25hCollecting numexpr (from fastai)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/db/ea/efd9e16283637eb5b6c0042b6cc3521f1b9a5b47767ac463c88bbd37670c/numexpr-2.6.8-cp36-cp36m-manylinux1_x86_64.whl (162kB)\n",
            "\u001b[K    100% |████████████████████████████████| 163kB 3.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from fastai) (4.0.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from fastai) (2.18.4)\n",
            "Collecting fastprogress>=0.1.16 (from fastai)\n",
            "  Downloading https://files.pythonhosted.org/packages/ea/93/b35cabbab4d25a2fdc5cd196114fbe1160451df5cf1459a80781893f3b0f/fastprogress-0.1.16-py3-none-any.whl\n",
            "Requirement already satisfied: numpy>=1.12 in /usr/local/lib/python3.6/dist-packages (from fastai) (1.14.6)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from fastai) (2.1.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from fastai) (0.22.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from fastai) (2018.1.10)\n",
            "Collecting bottleneck (from fastai)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/05/ae/cedf5323f398ab4e4ff92d6c431a3e1c6a186f9b41ab3e8258dff786a290/Bottleneck-1.2.1.tar.gz (105kB)\n",
            "\u001b[K    100% |████████████████████████████████| 112kB 4.0MB/s \n",
            "\u001b[?25hCollecting dataclasses (from fastai)\n",
            "  Downloading https://files.pythonhosted.org/packages/26/2f/1095cdc2868052dd1e64520f7c0d5c8c550ad297e944e641dbf1ffbb9a5d/dataclasses-0.6-py3-none-any.whl\n",
            "Requirement already satisfied: thinc==6.12.0 in /usr/local/lib/python3.6/dist-packages (from fastai) (6.12.0)\n",
            "Requirement already satisfied: cymem==2.0.2 in /usr/local/lib/python3.6/dist-packages (from fastai) (2.0.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from fastai) (3.13)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from fastai) (1.1.0)\n",
            "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from fastai) (3.6.6)\n",
            "Collecting torchvision-nightly (from fastai)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ca/bd/d0f9a33c81c79710eb7ee428b66869b49a8be16c7f1e446c211a7fbfb7be/torchvision_nightly-0.2.1-py2.py3-none-any.whl (54kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 6.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy==2.0.16 in /usr/local/lib/python3.6/dist-packages (from fastai) (2.0.16)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow->fastai) (0.46)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (2.6)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (2018.10.15)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (1.22)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai) (2.5.3)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai) (1.11.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai) (2018.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai) (2.3.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (4.28.1)\n",
            "Requirement already satisfied: wrapt<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (1.10.11)\n",
            "Requirement already satisfied: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (0.9.0.1)\n",
            "Requirement already satisfied: dill<0.3.0,>=0.2.7 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (0.2.8.2)\n",
            "Requirement already satisfied: preshed<3.0.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (2.0.1)\n",
            "Requirement already satisfied: msgpack-numpy<0.4.4.0 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (0.4.3.2)\n",
            "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (0.9.6)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (1.0.1)\n",
            "Requirement already satisfied: msgpack<1.0.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from thinc==6.12.0->fastai) (0.5.6)\n",
            "Requirement already satisfied: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.16->fastai) (1.35)\n",
            "Requirement already satisfied: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc==6.12.0->fastai) (0.9.0)\n",
            "Building wheels for collected packages: bottleneck\n",
            "  Running setup.py bdist_wheel for bottleneck ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/f2/bf/ec/e0f39aa27001525ad455139ee57ec7d0776fe074dfd78c97e4\n",
            "Successfully built bottleneck\n",
            "\u001b[31mtorchvision-nightly 0.2.1 has requirement pillow>=4.1.1, but you'll have pillow 4.0.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: numexpr, fastprogress, bottleneck, dataclasses, torchvision-nightly, fastai\n",
            "Successfully installed bottleneck-1.2.1 dataclasses-0.6 fastai-1.0.30 fastprogress-0.1.16 numexpr-2.6.8 torchvision-nightly-0.2.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "2Q9b16kulRUr",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# import libraries\n",
        "import fastai\n",
        "from fastai import *\n",
        "from fastai.text import * \n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from functools import partial\n",
        "import io\n",
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Cg-6ESAileLN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "087dd8e9-ec0b-43c0-a1ca-ff1ede100445"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.datasets import fetch_20newsgroups\n",
        "dataset = fetch_20newsgroups(shuffle=True, random_state=1, remove=('headers', 'footers', 'quotes'))\n",
        "documents = dataset.data"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading 20news dataset. This may take a few minutes.\n",
            "Downloading dataset from https://ndownloader.figshare.com/files/5975967 (14 MB)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "lQVO7WdnmB9W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "d55c6c59-fc27-470f-f8d4-0cf2502e18eb"
      },
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame({'label':dataset.target, 'text':dataset.data})\n",
        "\n",
        "df.shape"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(11314, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "YOmyHvzwmQUk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df[df['label'].isin([1,10])]\n",
        "df = df.reset_index(drop = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hsg0EaXLmdFj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "39b07fcf-f15a-4370-8999-34357aa72513"
      },
      "cell_type": "code",
      "source": [
        "df['label'].value_counts()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10    600\n",
              "1     584\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "Tye1qyynmh5A",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df['text'] = df['text'].str.replace(\"[^a-zA-Z]\", \" \")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DztX-xo8oOBn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "4432cffe-b314-4fad-dcf8-38eb554c508c"
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "\n",
        "from nltk.corpus import stopwords \n",
        "stop_words = stopwords.words('english')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "lEXI-Cr9oSiI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# tokenization \n",
        "tokenized_doc = df['text'].apply(lambda x: x.split())\n",
        "\n",
        "# remove stop-words \n",
        "tokenized_doc = tokenized_doc.apply(lambda x: [item for item in x if item not in stop_words])\n",
        "\n",
        "# de-tokenization \n",
        "detokenized_doc = [] \n",
        "for i in range(len(df)): \n",
        "    t = ' '.join(tokenized_doc[i]) \n",
        "    detokenized_doc.append(t) \n",
        "\n",
        "df['text'] = detokenized_doc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hS-iuHP-obXX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "34bb8f24-bb7e-40b5-a400-fcfe8cad91f9"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# split data into training and validation set\n",
        "df_trn, df_val = train_test_split(df, stratify = df['label'], test_size = 0.4, random_state = 12)\n",
        "\n",
        "df_trn.shape, df_val.shape"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((710, 2), (474, 2))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "metadata": {
        "id": "MuDrhnEmog1q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Language model data\n",
        "data_lm = TextLMDataBunch.from_df(train_df = df_trn, valid_df = df_val, path = \"\")\n",
        "\n",
        "# Classifier model data\n",
        "data_clas = TextClasDataBunch.from_df(path = \"\", train_df = df_trn, valid_df = df_val, vocab=data_lm.train_ds.vocab, bs=32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "l7A9LLX4omYa",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "learn = language_model_learner(data_lm, pretrained_model=URLs.WT103, drop_mult=0.7)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FtaPgyA0otQb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        },
        "outputId": "d025bac5-7d4e-47dc-8def-899238849f17"
      },
      "cell_type": "code",
      "source": [
        "# train the learner object with learning rate = 1e-2\n",
        "learn.fit_one_cycle(1, 1e-2)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Total time: 00:10 <p><table style='width:300px; margin-bottom:10px'>\n",
              "  <tr>\n",
              "    <th>epoch</th>\n",
              "    <th>train_loss</th>\n",
              "    <th>valid_loss</th>\n",
              "    <th>accuracy</th>\n",
              "  </tr>\n",
              "  <tr>\n",
              "    <th>1</th>\n",
              "    <th>7.794309</th>\n",
              "    <th>6.316229</th>\n",
              "    <th>0.138552</th>\n",
              "  </tr>\n",
              "  <tr>\n",
              "\n",
              "  </tr>\n",
              "</table>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "dTxT6lcCo0qB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "learn.save_encoder('ft_enc')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MTD_hlkny1FL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "learn = text_classifier_learner(data_clas, drop_mult=0.7)\n",
        "learn.load_encoder('ft_enc')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uKM3Mp9Zy4Ma",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        },
        "outputId": "81b55673-b2ec-4511-f9b7-1826c2c87c6c"
      },
      "cell_type": "code",
      "source": [
        "learn.fit_one_cycle(1, 1e-2)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Total time: 00:37 <p><table style='width:300px; margin-bottom:10px'>\n",
              "  <tr>\n",
              "    <th>epoch</th>\n",
              "    <th>train_loss</th>\n",
              "    <th>valid_loss</th>\n",
              "    <th>accuracy</th>\n",
              "  </tr>\n",
              "  <tr>\n",
              "    <th>1</th>\n",
              "    <th>0.546250</th>\n",
              "    <th>0.381360</th>\n",
              "    <th>0.862869</th>\n",
              "  </tr>\n",
              "  <tr>\n",
              "\n",
              "  </tr>\n",
              "</table>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "GaQk4f_Vy_k1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "outputId": "55b78243-72df-4b7d-bef6-bb009c88b796"
      },
      "cell_type": "code",
      "source": [
        "# get predictions\n",
        "preds, targets = learn.get_preds()\n",
        "\n",
        "predictions = np.argmax(preds, axis = 1)\n",
        "pd.crosstab(predictions, targets)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>216</td>\n",
              "      <td>47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>18</td>\n",
              "      <td>193</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0    0    1\n",
              "row_0          \n",
              "0      216   47\n",
              "1       18  193"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "metadata": {
        "id": "e68p66BdzWb-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}