data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pandas_profiling'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-15d7c6d8911c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mpandas_profiling\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'pandas_profiling'"
     ]
    }
   ],
   "source": [
    "import pandas_profiling\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sb\n",
    "from nltk.corpus import stopwords\n",
    "import warnings \n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import unidecode\n",
    "from wordcloud import WordCloud\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "nltk.download('wordnet')\n",
    "from nltk.stem import PorterStemmer\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize import word_tokenize \n",
    "import matplotlib.animation as animation\n",
    "import operator\n",
    "import plotly.express as px\n",
    "from collections import Counter\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df = pd.read_csv('mydataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(202)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pandas_profiling.ProfileReport(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df.drop_duplicates(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df['text'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['value'].isna().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code to remove @\n",
    "df['clean_tweet'] = df['text'].apply(lambda x : ' '.join([text for text in x.split()if not text.startswith(\"@\")]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(126)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#Removing all the greek characters using unidecode library\n",
    "\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([unidecode.unidecode(word) for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "  df[\"clean_tweet\"] = df['clean_tweet'].str.replace('[^\\w\\s]','')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing the word 'hmm' and it's variants\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([word for word in x.split() if not word == 'h(m)+' ]))\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([word for word in x.split() if not word == 'ky(a)+' ]))\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([word for word in x.split() if not word == 'mod(i)+' ]))\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([word for word in x.split() if not word == 'k(i)+' ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import string\n",
    "df['clean_tweet'] = df['clean_tweet'].str.replace('\\d+', '')\n",
    "df['clean_tweet'] = df['clean_tweet'].str.replace('_', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df['clean_tweet'][202]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code for removing slang words\n",
    "d = {'luv':'love','wud':'would','wnna':'want','lyk':'like','uh':'you','wateva':'whatever','ttyl':'talk to you later','kul':'cool','fyn':'fine','omg':'oh my god!','fam':'family','bruh':'brother','F.O.':'fuck off',\n",
    "'cud':'could','fud':'food'} ## Need a huge dictionary\n",
    "words = \"i luv uh\"\n",
    "words = words.split()\n",
    "reformed = [d[word] if word in d else word for word in words]\n",
    "reformed = \" \".join(reformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "reformed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join(d[word] if word in d else word for word in x.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_tweet'][135]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing stopwords\n",
    "STOP_WORDS = nltk.corpus.stopwords.words('english')\n",
    "newstop = [\n",
    "\"के\",\n",
    "\"का\",\n",
    "\"एक\",\n",
    "\"में\",\n",
    "\"की\",\n",
    "\"है\",\n",
    "\"यह\",\n",
    "\"और\",\n",
    "\"से\",\n",
    "\"हैं\",\n",
    "\"को\",\n",
    "\"पर\",\n",
    "\"इस\",\n",
    "\"होता\",\n",
    "\"कि\",\n",
    "\"जो\",\n",
    "\"कर\",\n",
    "\"मे\",\n",
    "\"गया\",\n",
    "\"करने\",\n",
    "\"किया\",\n",
    "\"लिये\",\n",
    "\"अपने\",\n",
    "\"ने\",\n",
    "\"बनी\",\n",
    "\"नहीं\",\n",
    "\"तो\",\n",
    "\"ही\",\n",
    "\"या\",\n",
    "\"एवं\",\n",
    "\"दिया\",\n",
    "\"हो\",\n",
    "\"इसका\",\n",
    "\"था\",\n",
    "\"द्वारा\",\n",
    "\"हुआ\",\n",
    "\"तक\",\n",
    "\"साथ\",\n",
    "\"करना\",\n",
    "\"वाले\",\n",
    "\"बाद\",\n",
    "\"लिए\",\n",
    "\"आप\",\n",
    "\"कुछ\",\n",
    "\"सकते\",\n",
    "\"किसी\",\n",
    "\"ये\",\n",
    "\"इसके\",\n",
    "\"सबसे\",\n",
    "\"इसमें\",\n",
    "\"थे\",\n",
    "\"दो\",\n",
    "\"होने\",\n",
    "\"वह\",\n",
    "\"वे\",\n",
    "\"करते\",\n",
    "\"बहुत\",\n",
    "\"कहा\",\n",
    "\"वर्ग\",\n",
    "\"कई\",\n",
    "\"करें\",\n",
    "\"होती\",\n",
    "\"अपनी\",\n",
    "\"उनके\",\n",
    "\"थी\",\n",
    "\"यदि\",\n",
    "\"हुई\",\n",
    "\"जा\",\n",
    "\"ना\",\n",
    "\"इसे\",\n",
    "\"कहते\",\n",
    "\"जब\",\n",
    "\"होते\",\n",
    "\"कोई\",\n",
    "\"हुए\",\n",
    "\"व\",\n",
    "\"न\",\n",
    "\"अभी\",\n",
    "\"जैसे\",\n",
    "\"सभी\",\n",
    "\"करता\",\n",
    "\"उनकी\",\n",
    "\"तरह\",\n",
    "\"उस\",\n",
    "\"आदि\",\n",
    "\"कुल\",\n",
    "\"एस\",\n",
    "\"रहा\",\n",
    "\"इसकी\",\n",
    "\"सकता\",\n",
    "\"रहे\",\n",
    "\"उनका\",\n",
    "\"इसी\",\n",
    "\"रखें\",\n",
    "\"अपना\",\n",
    "\"पे\",\n",
    " \"क्या\",\n",
    "  \"आये\",\n",
    "\"उसके\",\n",
    "\"कौन\",\n",
    "\"देने \",\n",
    "\"कभी\",\n",
    "\"आपने\",\n",
    "\"क्यों\",\n",
    "\"मिला\",\n",
    "\"बोलो\",\n",
    "\"चाहिए\",\n",
    "\"भी\",\n",
    "\"डालने\",\n",
    "\"रखती\",\n",
    "\"देखा\",\n",
    "\"साल\",\n",
    "\"हु\",\n",
    "\"नही\",\n",
    "\"लगा\",\n",
    "\"वो\",\n",
    "\"आपको\",\n",
    "\"बजे\",\n",
    "\"उसका\",\n",
    "\"आया\",\n",
    "\"फिर\",\n",
    " \"मेने\",\n",
    " \"इनके\",\n",
    "  \"सामने\",\n",
    "   \"झुक\",\n",
    "   \"मनाई\",\n",
    "   \"वैसे\",\n",
    "   \"उठाया\",\n",
    "   \"बोला\",\n",
    "   \"रख\",\"सोने\", \"दे\", \"काट\",\n",
    "   \"बोला\",\n",
    "   \"रख\",\n",
    "   \"अब\", \n",
    "   \"गये\",\n",
    "   \"सबका\",\n",
    "   \"इसको\",\n",
    "      \"ये\",\n",
    "    \"समझ\",\n",
    "     \"सका\",\n",
    "     \"कैसे\",\n",
    "      \"अगर\",\n",
    "      \"तू\",\n",
    "      \"क्यू\",\n",
    "       \"चला\",\n",
    "        \"जाता\",\n",
    "\"तेरे\",\n",
    "\"किसने\",\n",
    "\"जी\",\n",
    "\"बोले\",\n",
    "\"होगा\",\n",
    "\"वही\",\n",
    " \"जाती\",\n",
    "  \"शुरू\",\n",
    "  \"आज\",\n",
    "   \"पूरे\",\n",
    "   \"मारेगा\" ,\n",
    "   \"आए\",\n",
    "   \"अबे\",\n",
    "   \"होतीं\",\n",
    "   \"अबकी\",\n",
    "   \"बार\",\n",
    "   \"वाला\",\n",
    "    \"बताने\",\n",
    "      \"मै\",\n",
    "      \"तुमको\",\n",
    "      \"हूँ\",\n",
    "      \"कह\",\n",
    "      \"सब\",\n",
    "     \"मना\",\n",
    "     \"आ\",\n",
    "      \"रही\",\n",
    "    \"वरना\",\n",
    "     \"कल\",\n",
    "     \"सबके\",\n",
    "     \"सब\",\n",
    "     \" कह\",\n",
    "     \"हर\",\n",
    "     \"तुम\",\n",
    "     \"बोल\",\n",
    "\"आता\",\n",
    " \"इधर\",\n",
    " \"लगता\",\"लेकिन\", \"तनिक\",\n",
    " \"अरे\",\n",
    "\"रा\",\n",
    " \"है\",\n",
    " \"।\",\n",
    " \"देखो\",\n",
    "  \"कितने\",\n",
    "  \"निकलेगा\",\n",
    "  \"है।\",\n",
    "  \"तुने\",\n",
    "  \"आपकी\",\n",
    "  \"हे\" ,\n",
    "  \"रे\",\n",
    "  \"तूँ।\",\n",
    "  \"तेरी\",\n",
    "  \"जायेगी\",\n",
    "  \"बनकर\",\n",
    "  \"जाने\",\n",
    "  \"तेरी\",\n",
    "  \"बनाने\",\n",
    "  \"गए\",\n",
    "  \"जरा\",\n",
    "\"करलो\",\n",
    " \"इतना\",\n",
    "  \"ज्यादा\",\n",
    "   \"कि\",\n",
    "   \"करे\",\n",
    "   \"देने\",\n",
    "   \"यही\",\n",
    "   \"हैं।\",\n",
    "   \"तभी\",\n",
    "  \"तुमने\",\n",
    "   \"वालो\",\n",
    "   \"हमने\",\n",
    "   \",\",\n",
    "   \".\",\n",
    "   \"/\",\n",
    "   \"-\",\n",
    "   \"=\",\n",
    "   \"#\",\n",
    "   \"$\",\n",
    "   \"!\",\n",
    "   \";\",\":\",\"?\",\"(\",\")\",\"&\",\"^\",\"{\",\"}\",\"[\",\"]\",\n",
    "   \"*\",\n",
    "   \"करके\",\n",
    "    \"लिया\",\n",
    "     \"गया।\",\"यही\",\n",
    "  \"ही\",\"जाए\",\"ली\", \"सारे\",\n",
    "  \"उठाए\",\n",
    "  \"र\",\"तने\",\"प\", \"जादां\", \"नू\",\n",
    "  \"तुम्हारी\", \"चुकी\",\n",
    "  \"तुझे\", \"हॆ\",\"आयी\", \"तब\", \"तु\",\n",
    "  \"ले\",\"इनको\" ,\"देकर\",\"करेगा\",\n",
    "  \"इतनी\", \"उतनी\",\"पहुंचा\",\"इन्होंने\",\n",
    "  \"बोलते\", \"बोलता\",\"लगवाए\",\"जैसा\", \"पहले\",\"हीं\",\n",
    "  \"said\",\"come\",\"would\",\"use\",\"get\",\"like\",\"want\",\n",
    "  \"say\",\"much\",\"let\",\"this\",\"ever\",\"a\",\"kii\",\"kaa\",\"bhii\",\"isii\",\"kii\",\"isee\",\"aadi\",\"thaa\",\"aaj\",\"aap\",\"aapne\",\"aata\",\"aati\",\"aaya\",\"aaye\",\"ab\",\"abbe\",\"abbey\",\"abe\",\"abhi\",\"able\",\"about\",\"above\",\"accha\",\"acha\",\"achcha\",\"across\",\"actually\",\"after\",\"afterwards\",\"again\",\"against\",\n",
    "\"agar\",\n",
    "\"ain\",\n",
    "\"aisa\",\n",
    "\"aise\",\n",
    "\"aisi\",\n",
    "\"alag\",\n",
    "\"andar\",\n",
    "\"ap\",\n",
    "\"apan\",\n",
    "\"apart\",\n",
    "\"apna\",\n",
    "\"apnaa\",\n",
    "\"apne\",\n",
    "\"apni\",\n",
    "\"arre\"\n",
    "\"aur\"\n",
    "\"avum\"\n",
    "\"aya\",\n",
    "\"aye\",\n",
    "\"baad\",\n",
    "\"baar\",\n",
    "\"bahut\",\n",
    "\"bana\",\n",
    "\"banae\",\n",
    "\"banai\",\n",
    "\"banao\",\n",
    "\"banaya\",\n",
    "\"banaye\",\n",
    "\"banayi\",\n",
    "\"banda\",\n",
    "\"bande\",\n",
    "\"bandi\",\n",
    "\"bane\",\n",
    "\"bani\",\n",
    "\"bas\",\n",
    "\"bata\",\n",
    "\"batao\",\n",
    "\"be\",\n",
    "\"bhai\",\n",
    "\"bheetar\",\n",
    "\"bhi\",\n",
    "\"bhitar\",\n",
    "\"bht\",\n",
    "\"bilkul\",\n",
    "\"bohot\",\n",
    "\"bol\",\n",
    "\"bola\",\n",
    "\"bole\",\n",
    "\"boli\",\n",
    "\"bolo\",\n",
    "\"bolta\",\n",
    "\"bolte\",\n",
    "\"bolti\",\n",
    "\"chahiye\",\n",
    "\"chaiye\",\n",
    "\"chal\",\n",
    "\"chalega\",\n",
    "\"chhaiye\",\n",
    "\"com\"\n",
    "\"d\",\n",
    "\"de\",\n",
    "\"dede\",\n",
    "\"dega\",\n",
    "\"degi\",\n",
    "\"dekh\",\n",
    "\"dekha\",\n",
    "\"dekhe\",\n",
    "\"dekhi\",\n",
    "\"dekho\",\n",
    "\"denge\",\n",
    "\"dhang\",\n",
    "\"di\",\n",
    "\"dijiye\",\n",
    "\"diya\",\n",
    "\"diyaa\",\n",
    "\"diye\",\n",
    "\"diyo\",\n",
    "\"do\",\n",
    "\"dono\",\n",
    "\"doosra\",\n",
    "\"doosre\",\n",
    "\"dunga\",\n",
    "\"dungi\",\n",
    "\"dusra\",\n",
    "\"dusre\",\n",
    "\"dusri\",\n",
    "\"dvaara\",\n",
    "\"dvara\",\n",
    "\"dwaara\",\n",
    "\"dwara\",\n",
    "\"ek\",\n",
    "\"fir\",\n",
    "\"gaya\",\n",
    "\"gaye\",\n",
    "\"gayi\",\n",
    "\"ghar\",\n",
    "\"haan\",\n",
    "\"hai\",\n",
    "\"hain\",\n",
    "\"hamara\",\n",
    "\"hamare\",\n",
    "\"hamari\",\n",
    "\"hamne\",\n",
    "\"han\",\n",
    "\"har\",\n",
    "\"hmm\",\n",
    "\"ho\",\n",
    "\"hoga\",\n",
    "\"hoge\",\n",
    "\"hogi\",\n",
    "\"hona\",\n",
    "\"honaa\",\n",
    "\"hone\",\n",
    "\"honge\",\n",
    "\"hongi\",\n",
    "\"honi\",\n",
    "\"hota\",\n",
    "\"hotaa\",\n",
    "\"hote\",\n",
    "\"hoti\",\n",
    "\"hoyenge\",\n",
    "\"hoyengi\",\n",
    "\"hu\",\n",
    "\"hua\",\n",
    "\"hue\",\n",
    "\"huh\",\n",
    "\"hui\",\n",
    "\"hum\",\n",
    "\"humein\",\n",
    "\"humne\",\n",
    "\"hun\",\n",
    "\"huye\",\n",
    "\"huyi\",\n",
    "\"idk\",\n",
    "\"inhe\",\n",
    "\"inhi\",\n",
    "\"inho\",\n",
    "\"inka\",\n",
    "\"inkaa\",\n",
    "\"inke\",\n",
    "\"inki\",\n",
    "\"inn\",\n",
    "\"inse\",\n",
    "\"ise\",\n",
    "\"isi\",\n",
    "\"iska\",\n",
    "\"iskaa\",\n",
    "\"iske\",\n",
    "\"iski\",\n",
    "\"isme\",\n",
    "\"isn\",\n",
    "\"isne\",\n",
    "\"iss\",\n",
    "\"isse\",\n",
    "\"issi\",\n",
    "\"isski\",\n",
    "\"itna\",\n",
    "\"itne\",\n",
    "\"itni\",\n",
    "\"itno\",\n",
    "\"ityaadi\",\n",
    "\"ityadi\",\n",
    "\"ja\",\n",
    "\"jaa\",\n",
    "\"jab\",\n",
    "\"jabh\",\n",
    "\"jaha\",\n",
    "\"jahaan\",\n",
    "\"jahan\",\n",
    "\"jaisa\",\n",
    "\"jaise\",\n",
    "\"jaisi\",\n",
    "\"jata\",\n",
    "\"jayega\",\n",
    "\"jidhar\",\n",
    "\"jin\",\n",
    "\"jinhe\",\n",
    "\"jinhi\",\n",
    "\"jinho\",\n",
    "\"jinhone\",\n",
    "\"jinka\",\n",
    "\"jinke\",\n",
    "\"jinki\",\n",
    "\"jinn\",\n",
    "\"jis\",\n",
    "\"jise\",\n",
    "\"jiska\",\n",
    "\"jiske\",\n",
    "\"jiski\",\n",
    "\"jisme\",\n",
    "\"jiss\",\n",
    "\"jisse\",\n",
    "\"jitna\",\n",
    "\"jitne\",\n",
    "\"jitni\",\n",
    "\"jo\",\n",
    "\"jyaada\",\n",
    "\"jyada\",\n",
    "\"k\",\n",
    "\"ka\",\n",
    "\"kaafi\",\n",
    "\"kab\",\n",
    "\"kabhi\",\n",
    "\"kafi\",\n",
    "\"kaha\",\n",
    "\"kahaa\",\n",
    "\"kahaan\",\n",
    "\"kahan\",\n",
    "\"kahi\",\n",
    "\"kahin\",\n",
    "\"kahte\",\n",
    "\"kaisa\",\n",
    "\"kaise\",\n",
    "\"kaisi\",\n",
    "\"kal\",\n",
    "\"kam\",\n",
    "\"kar\",\n",
    "\"kara\",\n",
    "\"kare\",\n",
    "\"karega\",\n",
    "\"karegi\",\n",
    "\"karen\",\n",
    "\"karenge\",\n",
    "\"kari\",\n",
    "\"karke\",\n",
    "\"karna\",\n",
    "\"karne\",\n",
    "\"karni\",\n",
    "\"karo\",\n",
    "\"karta\",\n",
    "\"karte\",\n",
    "\"karti\",\n",
    "\"karu\",\n",
    "\"karun\",\n",
    "\"karunga\",\n",
    "\"karungi\",\n",
    "\"kaun\",\n",
    "\"kaunsa\",\n",
    "\"kayi\",\n",
    "\"kch\",\n",
    "\"ke\",\n",
    "\"keh\",\n",
    "\"kehte\",\n",
    "\"khud\",\n",
    "\"ki\",\n",
    "\"kin\",\n",
    "\"kine\",\n",
    "\"kinhe\",\n",
    "\"kinho\",\n",
    "\"kinka\",\n",
    "\"kinke\",\n",
    "\"kinki\",\n",
    "\"kinko\",\n",
    "\"kinn\",\n",
    "\"kino\",\n",
    "\"kis\",\n",
    "\"kise\",\n",
    "\"kisi\",\n",
    "\"kiska\",\n",
    "\"kiske\",\n",
    "\"kiski\",\n",
    "\"kisko\",\n",
    "\"kisliye\",\n",
    "\"kisne\",\n",
    "\"kitna\",\n",
    "\"kitne\",\n",
    "\"kitni\",\n",
    "\"kitno\",\n",
    "\"kiya\",\n",
    "\"kiye\",\n",
    "\"ko\",\n",
    "\"koi\",\n",
    "\"kon\",\n",
    "\"konsa\",\n",
    "\"koyi\",\n",
    "\"krna\",\n",
    "\"krne\",\n",
    "\"kuch\",\n",
    "\"kuchch\",\n",
    "\"kuchh\",\n",
    "\"kul\",\n",
    "\"kull\",\n",
    "\"kya\",\n",
    "\"kyaa\",\n",
    "\"kyu\",\n",
    "\"kyuki\",\n",
    "\"kyun\",\n",
    "\"kyunki\",\n",
    "\"lagta\",\n",
    "\"lagte\",\n",
    "\"lagti\",\n",
    "\"lekar\",\n",
    "\"lekin\",\n",
    "\"li\",\n",
    "\"liya\",\n",
    "\"liye\",\n",
    "\"ll\",\n",
    "\"lo\",\n",
    "\"log\",\n",
    "\"logon\",\n",
    "\"lol\",\n",
    "\"ltd\",\n",
    "\"lunga\",\n",
    "\"m\",\n",
    "\"maan\",\n",
    "\"maana\",\n",
    "\"maane\",\n",
    "\"maani\",\n",
    "\"maano\",\n",
    "\"magar\",\n",
    "\"mai\",\n",
    "\"main\",\n",
    "\"maine\",\n",
    "\"mainly\",\n",
    "\"mana\",\n",
    "\"mane\",\n",
    "\"mani\",\n",
    "\"mano\",\n",
    "\"many\",\n",
    "\"mat\",\n",
    "\"mein\",\n",
    "\"mera\",\n",
    "\"mere\",\n",
    "\"merely\",\n",
    "\"liye\",\n",
    "\"meri\",\n",
    "\"mil\",\n",
    "\"mjhe\",\n",
    "\"mujhe\",\n",
    "\"na\",\n",
    "\"naa\",\n",
    "\"naah\",\n",
    "\"nahi\",\n",
    "\"nahin\",\n",
    "\"nai\",\n",
    "\"name\",\n",
    "\"ne\",\n",
    "\"neeche\",\n",
    "\"nhi\",\n",
    "\"nine\",\n",
    "\"no\",\n",
    "\"non\",\n",
    "\"none\",\n",
    "\"nope\",\n",
    "\"o\",\n",
    "\"of\",\n",
    "\"off\",\n",
    "\"often\",\n",
    "\"par\",\n",
    "\"pata\",\n",
    "\"pe\",\n",
    "\"pehla\",\n",
    "\"pehle\",\n",
    "\"pehli\",\n",
    "\"per\",\n",
    "\"phla\",\n",
    "\"phle\",\n",
    "\"phli\",\n",
    "\"poora\",\n",
    "\"poori\",\n",
    "\"pura\",\n",
    "\"puri\",\n",
    "\"q\",\n",
    "\"que\",\n",
    "\"raha\",\n",
    "\"rahaa\",\n",
    "\"rahe\",\n",
    "\"rahi\",\n",
    "\"rakh\",\n",
    "\"rakha\",\n",
    "\"rakhe\",\n",
    "\"rakhen\",\n",
    "\"rakhi\",\n",
    "\"rakho\",\n",
    "\"rather\",\n",
    "\"re\",\n",
    "\"rehte\",\n",
    "\"rha\",\n",
    "\"rhaa\",\n",
    "\"rhe\",\n",
    "\"rhi\",\n",
    "\"ri\",\n",
    "\"s\",\n",
    "\"sa\",\n",
    "\"saara\",\n",
    "\"saare\",\n",
    "\"saath\",\n",
    "\"sab\",\n",
    "\"sabhi\",\n",
    "\"sabse\",\n",
    "\"sahi\",\n",
    "\"sakta\",\n",
    "\"saktaa\",\n",
    "\"sakte\",\n",
    "\"sakti\",\n",
    "\"sara\",\n",
    "\"sath\",\n",
    "\"se\",\n",
    "\"shant\",\n",
    "\"si\",\n",
    "\"so\",\n",
    "\"soch\",\n",
    "\"sub\",\n",
    "\"such\",\n",
    "\"sup\",\n",
    "\"sure\",\n",
    "\"tab\",\n",
    "\"tabh\",\n",
    "\"tak\",\n",
    "\"tarah\",\n",
    "\"teen\",\n",
    "\"teeno\",\n",
    "\"teesra\",\n",
    "\"teesre\",\n",
    "\"teesri\",\n",
    "\"tera\",\n",
    "\"के\",\n",
    "\"का\",\n",
    "\"एक\",\n",
    "\"में\",\n",
    "\"की\",\n",
    "\"है\",\n",
    "\"यह\",\n",
    "\"और\",\n",
    "\"से\",\n",
    "\"हैं\",\n",
    "\"को\",\n",
    "\"पर\",\n",
    "\"इस\",\n",
    "\n",
    "\"tere\",\"teri\",\"th\",\"tha\",\"the\",\"theek\",\"thi\",\"thik\",\"thoda\",\".\",\"thodi\",\"thru\",\"thus\",\"tjhe\",\"to\",\"toh\",\"too\",\"tu\",\"tujhe\",\"tum\",\"tumhara\",\"tumhare\",\"tumhari\",\"um\",\"umm\",\"un\",\"unhi\",\"unho\",\"unhone\",\"unka\",\"unkaa\",\"unke\",\"unki\",\"unko\",\"unn\",\"unse\",\"up\",\"upar\",\"us\",\"use\",\"usi\",\"uska\",\"uske\",\"usne\",\"uss\",\"usse\",\"ussi\",\"vaala\",\"vaale\",\"vaali\",\"vahaan\",\"vahan\",\"vahi\",\"vahin\",\"vaisa\",\"vaise\",\"vaisi\",\"vala\",\"vale\",\"vali\",\"ve\",\"viz\",\"vo\",\"waala\",\"waale\",\"waali\",\"wagaira\",\"wagairah\",\"wagerah\",\"waha\",\"wahaan\",\"wahan\",\"wahi\",\"wahin\",\"waisa\",\"waise\",\"waisi\",\"wala\",\"wale\",\"wali\",\"wo\",\"woh\",\"wohi\",\"won\",\"y\",\"ya\",\"yadi\",\"yah\",\"yaha\",\"yahaan\",\"yahan\",\"yahi\",\"yahin\",\"ye\",\"yeh\",\"yehi\",\"yes\",\"yet\",\"ka\",\"ki\",\"aur\",\"tmne\",\"tumne\",\"mjhe\",\"meri\",\"teri\",\"tjhe\",\"is\",\"wo\",\"us\",\"use\",\"kon\",\"kya\",\"kaise\",\"kyu\",\"kyun\",\"kise\",\"kyaa\",\"aaye\",\"jo\",\"pr\",\"bhi\",\"bhii\",\"aisi\",\"aisii\",\"ki\",\"nhii\",\"/\",\"thaa\",\"aa\",\"sb\",\".\",\"/\",\"-\",\"'\",\"dene\",\"diya\",\"hai\",\"kaun\",\"ke\",\"wo\",\"to\",\"pr\",\"aa\",\"ab\",\"ae\",\"aee\",\"koii\",\"rkhtii\",\"vhii\",\"smjh\",\"sktii\",\"liyaa\",\"gyaa\",\"jaisaa\",\"milaa\",\"hii\",\"din\",\"rhii\",\"uskaa\",\"kbhii\",\"diyaa\",\"diiye\",\"hogii\",\"diiyaa\",\"diiyaa\",\"hotiin\",\"hain\",\"diiyaa\",\"caahie\",\"n\",\"huaa\",\"ginaa\",\"ddaalne\",\"jaanne\",\"lie\",\"kr\",\"phir\",\"kren\",\"mnaaii\",\"bhut\",\"dekhaa\",\"kiyaa\",\"lgaa\",\"httane\",\"paa\",\"nhiin\",\"bnaa\",\"kyo\",\"kitnaa\",\"jaago\",\"skte\",\"agle\",\"saal\",\"aapko\",\"aapnne\",\"aage\",\"naam\",\"rhiye\",\"aayaa\",\"mene\",\"utthaayaa\",\"bolaa\",\"rkh\",\"sone\",\"tkriibn\",\"saamne\",\"jhuk\",\"khaa\",\"nii\",\"cle\",\"ge\",\"vaalaa\",\"jaataa\",\"hisaab\",\"gye\",\"sbkaa\",\"taaki\",\"h\",\"isko\",\"nyii\",\"ptaa\",\"hogaa\",\"jb\",\"suntaa\",\"tb\",\"detaa\",\"ptaa\",\"skaa\",\"nhiin\",\"ismen\",\"agr\",\"unpe\",\"bcca\",\"lete\",\"dekhtaa\",\"huun\",\"rokegaa\",\"tuu\",\"kyuu\",\"rokaa\",\"gaaye\",\"paas\",\"apnii\",\"le\",\"hmaare\",\"jii\",\"shrii\",\"sdaa\",\"yuun\",\"rhen\",\"mst\",\"aapke\",\"krtaa\",\"huun\",\"shrii\",\"shri\",\"shree\",\"bndii\",\"kyon\",\"prduussnn\",\"pttaakh\",\"ever\",\"hiin\",\"voh\",\"nott\",\"kisii\",\"jaatii\",\"shuru\",\"htyaa\",\"dete\",\"munh\",\"dhii\",\"jmaaye\",\"baitthe\",\"puure\",\"dm\",\"aae\",\"men\",\"khtm\",\"jaati\",\"v\",\"yh\",\"piir\",\"bnte\",\"btaane\",\"jruurt\",\"thii\",\"tumko\",\"maanna\",\"jy\",\"cnd\",\"bhaag\",\"bcci\",\"abkii\",\"ronaa\",\"shuruu\",\"mnaa\",\"kaam\",\"kh\",\"yaad\",\"jaaduu\",\"vrnaa\",\"kl\",\"tk\",\"khel\",\"sbke\",\"baat\",\"spnaa\",\"puuraa\",\"ucit\",\"mupht\",\"yaatraa\",\"acche\",\"skuul\",\"aisaa\",\"dikhaao\",\"dikhaoo\",\"himmt\",\"lge\",\"rhte\",\"hr\",\"jgh\",\"aataa\",\"idhr\",\"aakr\",\"ruktaa\",\"baakii\",\"kevl\",\"saarii\",\"hvaa\",\"lgtaa\",\"abhii\",\"der\",\"niklegaa\",\"phntaa\",\"duusraa\",\"eese\",\"huii\",\"krte\",\"shii\",\"rhtaa\",\"kro\",\"rhtii\",\"pheko\",\"aur\",\"jaate\",\"yhaan\",\"log\",\"nikaal\",\"sun\",\"tune\",\"lekr\",\"bec\",\"haatho\",\"aapkii\",\"trh\",\"aate\",\"dilaa\",\"duun\",\"kyuun\",\"terii\",\"pkkii\",\"jaayegii\",\"bnkr\",\"bnaao\",\"ruup\",\"smy\",\"nyaa\",\"jaane\",\"phuce\",\"acaank\",\"terii\",\"mnaataa\",\"bnaane\",\"plaan\",\"chotte\",\"kone\",\"phuncaa\",\"kie\",\"jaaenge\",\"bolenge\",\"hmne\",\"inkii\",\"hm\",\"prshn\",\"jyaadaa\",\"let\",\"need\",\"mt\",\"utthaake\",\"dauraan\",\"terii\",\"tthiik\",\"hme\",\"jinko\",\"likhaa\",\"jraa\",\"dillii\",\"krlo\",\"itnaa\",\"jyaadaa\",\"udhr\",\"denii\",\"ekdm\",\"kre\",\"krnaa\",\"yhii\",\"lene\",\"unhen\",\"yaa\",\"donon\",\"kaatt\",\"oh\",\"u\",\"new\",\"paidaa\",\"sirph\",\"tbhii\",\"jaaegaa\",\"viklp\",\"soc\",\"laayk\",\"vaalo\",\"like\",\"ccaa\",\"jruur\",\"gaate\",\"hmne\",\"itte\",\"ittaa\",\"kmlesh\",\"tivaarii\",\"link\",\"phon\",\"smy\",\"krnevaale\",\"aate\",\"krke\",\"sbse\",\"ismen\",\"bhkton\",\"bhgvaan\",\"aadmii\",\"laaluu\",\"ghr\",\"wnna\",\"go\",\"dont\",\"hlaalaa\",\"merkel\",\"german\",\"pte\",\"milte\",\"sir\",\"sunke\",\"sunkr\",\"khii\",\"lii\",\"utthaae\",\"bnnaa\",\"tne\",\"p\",\"claa\",\"nuu\",\"lgaayaa\",\"bolegaa\",\"puujaa\",\"ii\",\"hv\",\"smjht\",\"huvaa\",\"kraa\",\"gyl\",\"rkhe\",\"vaalii\",\"legii\",\"tumhaarii\",\"ammii\",\"phcaan\",\"phek\",\"bnaakr\",\"rkhegaa\",\"teraa\",\"dho\",\"bs\",\"hinduu\",\"bhn\",\"jntaa\",\"dhone\",\"lgegaa\",\"aayii\",\"huaaa\",\"jrie\",\"bnaayaa\",\"mr\",\"inko\",\"dekr\",\"inhonne\",\"khuub\",\"tujh\",\"lgaaen\",\"aakhir\",\"km\",\"jaae\",\"kregaa\",\"itnii\",\"utnii\",\"boltaa\",\"aajkl\",\"acchaa\",\"jaante\",\"dikhaayaa\",\"tumlogo\",\"aaygi\",\"nikl\",\"bhr\",\"sari\",\"bna\",\"rkha\",\"ni\",\"get\",\"make\",\"https\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "   \n",
    "\n",
    "]\n",
    "STOP_WORDS.extend(newstop)\n",
    "\n",
    "df['clean_tweet'] = df['clean_tweet'].str.lower().apply(lambda x : ' '.join([word for word in x.split() if not word in STOP_WORDS]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_tweet'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop_duplicates(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_tweet'][202]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Lemmitization\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([lemmatizer.lemmatize(word) for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stemming\n",
    "ps = PorterStemmer()\n",
    "adwait = df\n",
    "#adwait.head()\n",
    "df['clean_tweet'] = df['clean_tweet'].apply(lambda x : ' '.join([ps.stem(word) for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tokenization\n",
    "corpus = []\n",
    "for i in range(0,200):\n",
    "    tweet = df['clean_tweet'][i]\n",
    "    tweet = tweet.lower()\n",
    "    tweet = tweet.split()\n",
    "    tweet = [ps.stem(word) for word in tweet if not       word in set(stopwords.words('english'))]\n",
    "    tweet = ' '.join(tweet)\n",
    "    corpus.append(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normal_words = ' '.join([word for word in df['clean_tweet'][df['value'] == 0]])\n",
    "wordcloud = WordCloud(width = 800, height = 500, max_font_size = 110,max_words = 100).generate(normal_words)\n",
    "print('Normal words')\n",
    "plt.figure(figsize= (12,8))\n",
    "plt.imshow(wordcloud, interpolation = 'bilinear',cmap='viridis')\n",
    "plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "normal_words = ' '.join([word for word in df['clean_tweet'][df['value'] == 1]])\n",
    "wordcloud = WordCloud(width = 800, height = 500, max_font_size = 110,max_words = 100).generate(normal_words)\n",
    "print('Normal words')\n",
    "plt.figure(figsize= (12,8))\n",
    "plt.imshow(wordcloud, interpolation = 'bilinear')\n",
    "plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#TF-IDF approach\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.90, min_df=2)\n",
    "# TF-IDF feature matrix\n",
    "X1 = tfidf_vectorizer.fit_transform(corpus)\n",
    "Y1 = df.loc[:,'value'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X1_train, X1_test, Y1_train, Y1_test = train_test_split(df['clean_tweet'], df['value'], test_size = 0.3, random_state=0, shuffle = True, stratify=df['value'])\n",
    "vectorizer = tfidf_vectorizer\n",
    "X1_train_vect = vectorizer.fit_transform(X1_train)\n",
    "Y1 = df.loc[:,'value'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest using pipelines\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = Pipeline([('tfidf', TfidfVectorizer()), ('rf', RandomForestClassifier())])\n",
    "rf.fit(X1_train, Y1_train)\n",
    "y_pred = rf.predict(X1_test)\n",
    "print(pd.crosstab(Y1_test,y_pred,rownames=['Actual'],colnames=['Predicted']))\n",
    "print(classification_report(Y1_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df =  pd.read_csv(\"clean.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train, test = train_test_split(df,test_size = 0.2, stratify = df['value'], random_state=21)\n",
    "\n",
    "# get the shape of train and test split.\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(lowercase= True, max_features=1000, stop_words=STOP_WORDS)\n",
    "\n",
    "# fit the object with the training data tweets\n",
    "tfidf_vectorizer.fit(train.clean_tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idf = tfidf_vectorizer.transform(train.clean_tweet)\n",
    "test_idf  = tfidf_vectorizer.transform(test.clean_tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "model_LR = LogisticRegression()\n",
    "\n",
    "# fit the model with the training data\n",
    "model_LR.fit(train_idf, train.value)\n",
    "\n",
    "# predict the label on the traning data\n",
    "predict_train = model_LR.predict(train_idf)\n",
    "\n",
    "# predict the model on the test data\n",
    "predict_test = model_LR.predict(test_idf)\n",
    "\n",
    "# f1 score on train data\n",
    "f1_score(y_true= train.value, y_pred= predict_train)\n",
    "## >> 0.4888178913738019\n",
    "\n",
    "f1_score(y_true= test.value, y_pred= predict_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "pipeline = Pipeline(steps= [('tfidf', TfidfVectorizer(lowercase=True,\n",
    "                                                      max_features=1000,\n",
    "                                                      stop_words= STOP_WORDS)),\n",
    "                            ('model', LogisticRegression())])\n",
    "\n",
    "# fit the pipeline model with the training data                            \n",
    "pipeline.fit(train.clean_tweet, train.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = [\"Virat Kohli, AB de Villiers set to auction their 'Green Day' kits from 2016 IPL match to raise funds\"]\n",
    "\n",
    "# predict the label using the pipeline\n",
    "pipeline.predict(text)\n",
    "## >> array([0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import dump\n",
    "\n",
    "# dump the pipeline model\n",
    "dump(pipeline, filename=\"text_classification.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import load\n",
    "\n",
    "# sample tweet text\n",
    "text = [\"Virat Kohli, AB de Villiers set to auction their 'Green Day' kits from 2016 IPL match to raise funds\"]\n",
    "\n",
    "# load the saved pipleine model\n",
    "pipeline = load(\"text_classification.joblib\")\n",
    "\n",
    "# predict on the sample tweet text\n",
    "pipeline.predict(text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "2db524e06e9f5f4ffedc911c917cb75e12dbc923643829bf417064a77eb14d37"
    }
   },
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}