Us_Twitter_Airline_Sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMZH12dMdPUVeu3Jae4XZzJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sudhan22z265/Machine_learning/blob/master/NLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "##**Project Twitter US Airline Sentiment Analysis**\n"
      ],
      "metadata": {
        "id": "Y-akuZyXXUfr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Reading Dataset"
      ],
      "metadata": {
        "id": "uJyYE8Xtu1R2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "YoBypwiwXfax"
      },
      "execution_count": 88,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('training_twitter_x_y_train.csv')"
      ],
      "metadata": {
        "id": "6lpTOI_LXi5p"
      },
      "execution_count": 89,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = data.copy()"
      ],
      "metadata": {
        "id": "Z4jdCDv9YJfZ"
      },
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 486
        },
        "id": "XOclYumQYKva",
        "outputId": "024c853a-c6a9-4b33-dd20-6fd2e1e648d4"
      },
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             tweet_id airline_sentiment    airline airline_sentiment_gold  \\\n",
              "0  567900433542488064          negative  Southwest                    NaN   \n",
              "1  569989168903819264          positive  Southwest                    NaN   \n",
              "2  568089179520954368          positive     United                    NaN   \n",
              "3  568928195581513728          negative  Southwest                    NaN   \n",
              "4  568594180014014464          negative     United                    NaN   \n",
              "\n",
              "            name negativereason_gold  retweet_count  \\\n",
              "0  ColeyGirouard                 NaN              0   \n",
              "1  WalterFaddoul                 NaN              0   \n",
              "2      LocalKyle                 NaN              0   \n",
              "3    amccarthy19                 NaN              0   \n",
              "4        J_Okayy                 NaN              0   \n",
              "\n",
              "                                                text tweet_coord  \\\n",
              "0  @SouthwestAir I am scheduled for the morning, ...         NaN   \n",
              "1  @SouthwestAir seeing your workers time in and ...         NaN   \n",
              "2  @united Flew ORD to Miami and back and  had gr...         NaN   \n",
              "3     @SouthwestAir @dultch97 that's horse radish 😤🐴         NaN   \n",
              "4  @united so our flight into ORD was delayed bec...         NaN   \n",
              "\n",
              "               tweet_created              tweet_location  \\\n",
              "0  2015-02-17 20:16:29 -0800             Washington D.C.   \n",
              "1  2015-02-23 14:36:22 -0800  Indianapolis, Indiana; USA   \n",
              "2  2015-02-18 08:46:29 -0800                    Illinois   \n",
              "3  2015-02-20 16:20:26 -0800                         NaN   \n",
              "4  2015-02-19 18:13:11 -0800                         NaN   \n",
              "\n",
              "                user_timezone  \n",
              "0      Atlantic Time (Canada)  \n",
              "1  Central Time (US & Canada)  \n",
              "2  Central Time (US & Canada)  \n",
              "3      Atlantic Time (Canada)  \n",
              "4  Eastern Time (US & Canada)  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2a276fa0-a239-40e8-891c-924ba1b07109\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline_sentiment</th>\n",
              "      <th>airline</th>\n",
              "      <th>airline_sentiment_gold</th>\n",
              "      <th>name</th>\n",
              "      <th>negativereason_gold</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_coord</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>tweet_location</th>\n",
              "      <th>user_timezone</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>567900433542488064</td>\n",
              "      <td>negative</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>NaN</td>\n",
              "      <td>ColeyGirouard</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@SouthwestAir I am scheduled for the morning, ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-17 20:16:29 -0800</td>\n",
              "      <td>Washington D.C.</td>\n",
              "      <td>Atlantic Time (Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>569989168903819264</td>\n",
              "      <td>positive</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>NaN</td>\n",
              "      <td>WalterFaddoul</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@SouthwestAir seeing your workers time in and ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-23 14:36:22 -0800</td>\n",
              "      <td>Indianapolis, Indiana; USA</td>\n",
              "      <td>Central Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>568089179520954368</td>\n",
              "      <td>positive</td>\n",
              "      <td>United</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LocalKyle</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@united Flew ORD to Miami and back and  had gr...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-18 08:46:29 -0800</td>\n",
              "      <td>Illinois</td>\n",
              "      <td>Central Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>568928195581513728</td>\n",
              "      <td>negative</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>NaN</td>\n",
              "      <td>amccarthy19</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@SouthwestAir @dultch97 that's horse radish 😤🐴</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-20 16:20:26 -0800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Atlantic Time (Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>568594180014014464</td>\n",
              "      <td>negative</td>\n",
              "      <td>United</td>\n",
              "      <td>NaN</td>\n",
              "      <td>J_Okayy</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@united so our flight into ORD was delayed bec...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-19 18:13:11 -0800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Eastern Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2a276fa0-a239-40e8-891c-924ba1b07109')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2a276fa0-a239-40e8-891c-924ba1b07109 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2a276fa0-a239-40e8-891c-924ba1b07109');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ad602a7b-5521-4314-9dda-8ef31e69a7a0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ad602a7b-5521-4314-9dda-8ef31e69a7a0')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ad602a7b-5521-4314-9dda-8ef31e69a7a0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Storing airline sentiment in df_cat\n",
        "\n",
        "*   cat -> category\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "nqUgaxhQu4wh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_cat = df['airline_sentiment']"
      ],
      "metadata": {
        "id": "pabduE2nYYiW"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Storing airline twitter text in df_msg\n"
      ],
      "metadata": {
        "id": "ZAbL4MmnvGBq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_msg = df['text']"
      ],
      "metadata": {
        "id": "kHqjytcpYezr"
      },
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_cat.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Idi62lvYmZk",
        "outputId": "099b21ce-37aa-4bff-ac48-a08003a8fd9f"
      },
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    negative\n",
              "1    positive\n",
              "2    positive\n",
              "3    negative\n",
              "4    negative\n",
              "Name: airline_sentiment, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_msg.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UkUPIcZLYpkR",
        "outputId": "2f6946c4-28ca-4a9d-fb63-9f8d457833cd"
      },
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    @SouthwestAir I am scheduled for the morning, ...\n",
              "1    @SouthwestAir seeing your workers time in and ...\n",
              "2    @united Flew ORD to Miami and back and  had gr...\n",
              "3       @SouthwestAir @dultch97 that's horse radish 😤🐴\n",
              "4    @united so our flight into ORD was delayed bec...\n",
              "Name: text, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Converting dataFrame to list\n",
        "\n",
        "\n",
        "*   storing df_cat as cat(list)\n",
        "*   storing df_msg as msg(list)\n",
        "\n"
      ],
      "metadata": {
        "id": "gIB-XSiZvYti"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cat = list(df_cat)"
      ],
      "metadata": {
        "id": "iRrTxcmUYtEH"
      },
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "msg = list(df_msg)\n",
        "msg"
      ],
      "metadata": {
        "id": "MGpoeU5-Y7H1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "msg[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "duNSruaCZIae",
        "outputId": "3ba6a8ff-17f9-4c3b-c948-3712229a50ad"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'@SouthwestAir I am scheduled for the morning, 2 days after the fact, yes..not sure why my evening flight was the only one Cancelled Flightled'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Tokenizing words and stored it as array of tuples\n",
        "#Each tuple contains list of tokenized words and its sentiment"
      ],
      "metadata": {
        "id": "LLdoCfyMvpGr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "\n",
        "documents=[]\n",
        "for i in range(len(msg)):\n",
        "  new_msg=()\n",
        "  tokened_word = word_tokenize(msg[i])\n",
        "  new_msg += (tokened_word,)\n",
        "  new_msg+=(cat[i],)\n",
        "  documents.append(new_msg)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "EpJs3cx4ZdLY"
      },
      "execution_count": 99,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "documents[:5]"
      ],
      "metadata": {
        "id": "rFzQRYYYaW8G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Shuffling the datas"
      ],
      "metadata": {
        "id": "PWjMLsffv3WQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "random.shuffle(documents)"
      ],
      "metadata": {
        "id": "xjICgX59bTgU"
      },
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "documents[:5]"
      ],
      "metadata": {
        "id": "Si3zzhnQd5EH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "lemmatizer = WordNetLemmatizer()"
      ],
      "metadata": {
        "id": "b032FN2ce85c"
      },
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "\n",
        "*   **Removing Stopwords(i.e, most commonly used words)**\n",
        "*   **Removing punctuations**\n",
        "\n",
        "*   **Making all words as Root words using Lemmatization**\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "vzV9h6y8v6j2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "import string\n",
        "\n",
        "stopwords = stopwords.words('english')\n",
        "\n",
        "stopwords = stopwords + list(string.punctuation)\n",
        "stopwords"
      ],
      "metadata": {
        "id": "L-n0sytmfHPv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.corpus import wordnet\n",
        "def get_simple_pos(tag):\n",
        "  if tag.startswith('J'):\n",
        "    return wordnet.ADJ\n",
        "  elif tag.startswith('V'):\n",
        "    return wordnet.VERB\n",
        "  elif tag.startswith('N'):\n",
        "    return wordnet.NOUN\n",
        "  elif tag.startswith('R'):\n",
        "    return wordnet.ADV\n",
        "  else:\n",
        "    return wordnet.NOUN"
      ],
      "metadata": {
        "id": "l0Y_nT06gq6a"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk import pos_tag\n",
        "def clean_words(words):\n",
        "  output_words = []\n",
        "  for w in words:\n",
        "    if w.lower() not in stopwords:\n",
        "      pos = pos_tag([w])\n",
        "      clean_word = lemmatizer.lemmatize(w,pos =get_simple_pos(pos[0][1]))\n",
        "      output_words.append(clean_word.lower())\n",
        "  return output_words"
      ],
      "metadata": {
        "id": "u6AgvOG1eAKb"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Cleaning Words"
      ],
      "metadata": {
        "id": "SJPWYQdmwvr2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')\n",
        "documents = [(clean_words(i),j) for i,j in documents]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ejoIyDYRiD_z",
        "outputId": "3a974ab8-59f9-4004-94c4-37cd19fa96b1"
      },
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(documents)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oBUPAOWAiQK_",
        "outputId": "c2af92e9-69ab-4a37-90ad-dbc701118dd9"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10980"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Splitting training and testing data"
      ],
      "metadata": {
        "id": "TokFeQNMwyuA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "documents_training = documents[0:9000]\n",
        "documents_testing = documents[9000:]"
      ],
      "metadata": {
        "id": "CoeNJvTrikVJ"
      },
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "allwords = []\n",
        "for doc in documents:\n",
        "  allwords+=doc[0]"
      ],
      "metadata": {
        "id": "oYGMrGo-jNkr"
      },
      "execution_count": 110,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Getting Frequency Distribution"
      ],
      "metadata": {
        "id": "xDhZxvDHw2j6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "freq = nltk.FreqDist(allwords)"
      ],
      "metadata": {
        "id": "2H23FgW3jhqY"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "common = freq.most_common(3000)\n",
        "features =  [i[0] for i in common]"
      ],
      "metadata": {
        "id": "ovjrxobbjorx"
      },
      "execution_count": 112,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Choosing features as most common words"
      ],
      "metadata": {
        "id": "SAqRKji2w9ja"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "features[0:20]"
      ],
      "metadata": {
        "id": "VawjJqCBj2EL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_feature(words):\n",
        "  current_features = {}\n",
        "  words_set = set(words)\n",
        "  for w in features:\n",
        "    current_features[w] = w in words_set\n",
        "  return current_features"
      ],
      "metadata": {
        "id": "MbkhQkTrkYPX"
      },
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = [(get_feature(i),j) for i,j in documents_training]\n",
        "test_data = [(get_feature(i),j) for i,j in documents_testing]"
      ],
      "metadata": {
        "id": "RJvG68F6lVCa"
      },
      "execution_count": 115,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data[0:2]"
      ],
      "metadata": {
        "id": "nwngIEiStEH6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Training NaiveBayes Classifier"
      ],
      "metadata": {
        "id": "ZLqMy-BlxCf8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk import NaiveBayesClassifier\n",
        "classifier = NaiveBayesClassifier.train(train_data)\n"
      ],
      "metadata": {
        "id": "AW0du2p3ljrW"
      },
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Checking accuracy for testing data"
      ],
      "metadata": {
        "id": "Dbz8xEaWxGr4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.classify.accuracy(classifier,test_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yXPsPENAlxZo",
        "outputId": "0b586b01-d5f9-4c30-a2ac-02c7269b81b6"
      },
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7722222222222223"
            ]
          },
          "metadata": {},
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Getting most informative features"
      ],
      "metadata": {
        "id": "w_K9kRo6xOWb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "classifier.show_most_informative_features(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wGh--yHMnq27",
        "outputId": "f43ababc-8957-4406-c06c-c1f1dbf705dc"
      },
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Most Informative Features\n",
            "                   kudos = True           positi : negati =     44.4 : 1.0\n",
            "               fantastic = True           positi : negati =     29.2 : 1.0\n",
            "                favorite = True           positi : negati =     29.2 : 1.0\n",
            "             outstanding = True           positi : negati =     29.2 : 1.0\n",
            "                   march = True           neutra : negati =     24.7 : 1.0\n",
            "                 awesome = True           positi : negati =     23.5 : 1.0\n",
            "                   thank = True           positi : negati =     22.7 : 1.0\n",
            "                    hold = True           negati : positi =     21.3 : 1.0\n",
            "                   great = True           positi : neutra =     19.8 : 1.0\n",
            "                   sweet = True           positi : negati =     19.0 : 1.0\n"
          ]
        }
      ]
    }
  ]
}